@ai-sdk/openai 4.0.14 → 4.0.16

package/src/responses/openai-responses-language-model.ts

@@ -38,6 +38,7 @@ import type {
   codeInterpreterInputSchema,
   codeInterpreterOutputSchema,
 } from '../tool/code-interpreter';
+import type { computerInputSchema } from '../tool/computer';
 import type { fileSearchOutputSchema } from '../tool/file-search';
 import type { imageGenerationOutputSchema } from '../tool/image-generation';
 import type { localShellInputSchema } from '../tool/local-shell';
@@ -64,6 +65,7 @@ import {
   type OpenAIResponsesWebSearchAction,
   type OpenAIResponsesApplyPatchOperationDiffDeltaChunk,
   type OpenAIResponsesApplyPatchOperationDiffDoneChunk,
+  type OpenAIResponsesComputerAction,
 } from './openai-responses-api';
 import {
   openaiLanguageModelResponsesOptionsSchema,
@@ -104,6 +106,87 @@ function extractApprovalRequestIdToToolCallIdMapping(
   return mapping;
 }
 
+function mapComputerAction(
+  action: OpenAIResponsesComputerAction,
+): InferSchema<typeof computerInputSchema>['actions'][number] {
+  switch (action.type) {
+    case 'click':
+      return {
+        type: 'click',
+        button: action.button,
+        x: action.x,
+        y: action.y,
+        ...(action.keys != null && { keys: action.keys }),
+      };
+    case 'double_click':
+      return {
+        type: 'double_click',
+        x: action.x,
+        y: action.y,
+        ...(action.keys != null && { keys: action.keys }),
+      };
+    case 'drag':
+      return {
+        type: 'drag',
+        path: action.path,
+        ...(action.keys != null && { keys: action.keys }),
+      };
+    case 'keypress':
+      return action;
+    case 'move':
+      return {
+        type: 'move',
+        x: action.x,
+        y: action.y,
+        ...(action.keys != null && { keys: action.keys }),
+      };
+    case 'screenshot':
+      return action;
+    case 'scroll':
+      return {
+        type: 'scroll',
+        x: action.x,
+        y: action.y,
+        scrollX: action.scroll_x,
+        scrollY: action.scroll_y,
+        ...(action.keys != null && { keys: action.keys }),
+      };
+    case 'type':
+      return action;
+    case 'wait':
+      return action;
+  }
+}
+
+function mapComputerCallInput({
+  action,
+  actions,
+  pending_safety_checks,
+  status,
+}: {
+  action?: OpenAIResponsesComputerAction | null;
+  actions?: OpenAIResponsesComputerAction[] | null;
+  pending_safety_checks?: Array<{
+    id: string;
+    code?: string | null;
+    message?: string | null;
+  }> | null;
+  status: 'in_progress' | 'completed' | 'incomplete';
+}): InferSchema<typeof computerInputSchema> {
+  return {
+    actions: (actions ?? (action != null ? [action] : [])).map(
+      mapComputerAction,
+    ),
+    pendingSafetyChecks:
+      pending_safety_checks?.map(safetyCheck => ({
+        id: safetyCheck.id,
+        ...(safetyCheck.code != null && { code: safetyCheck.code }),
+        ...(safetyCheck.message != null && { message: safetyCheck.message }),
+      })) ?? [],
+    status,
+  };
+}
+
 export class OpenAIResponsesLanguageModel implements LanguageModelV4 {
   readonly specificationVersion = 'v4';
 
@@ -220,6 +303,7 @@ export class OpenAIResponsesLanguageModel implements LanguageModelV4 {
       tools,
       providerToolNames: {
         'openai.code_interpreter': 'code_interpreter',
+        'openai.computer': 'computer',
         'openai.file_search': 'file_search',
         'openai.image_generation': 'image_generation',
         'openai.local_shell': 'local_shell',
@@ -264,6 +348,7 @@ export class OpenAIResponsesLanguageModel implements LanguageModelV4 {
         hasLocalShellTool: hasOpenAITool('openai.local_shell'),
         hasShellTool: hasOpenAITool('openai.shell'),
         hasApplyPatchTool: hasOpenAITool('openai.apply_patch'),
+        hasComputerTool: hasOpenAITool('openai.computer'),
         customProviderToolNames:
           customProviderToolNames.size > 0
             ? customProviderToolNames
@@ -569,6 +654,21 @@ export class OpenAIResponsesLanguageModel implements LanguageModelV4 {
       });
     }
 
+    if (response.output == null) {
+      const detail = response.incomplete_details?.reason;
+      throw new APICallError({
+        message: detail
+          ? `Responses API returned no output (${detail})`
+          : 'Responses API returned no output',
+        url,
+        requestBodyValues: body,
+        statusCode: 500,
+        responseHeaders,
+        responseBody: rawResponse as string,
+        isRetryable: false,
+      });
+    }
+
     const content: Array<LanguageModelV4Content> = [];
     const logprobs: Array<OpenAIResponsesLogprobs> = [];
 
@@ -576,8 +676,8 @@ export class OpenAIResponsesLanguageModel implements LanguageModelV4 {
     let hasFunctionCall = false;
     const hostedToolSearchCallIds: string[] = [];
 
-    // map response content to content array (defined when there is no error)
-    for (const part of response.output!) {
+    // map response content to content array
+    for (const part of response.output) {
       switch (part.type) {
         case 'reasoning': {
           // when there are no summary parts, we need to add an empty reasoning part:
@@ -957,21 +1057,39 @@ export class OpenAIResponsesLanguageModel implements LanguageModelV4 {
         }
 
         case 'computer_call': {
-          content.push({
-            type: 'tool-call',
-            toolCallId: part.id,
-            toolName: toolNameMapping.toCustomToolName('computer_use'),
-            input: '',
-            providerExecuted: true,
-          });
+          if (part.call_id == null) {
+            content.push({
+              type: 'tool-call',
+              toolCallId: part.id,
+              toolName: toolNameMapping.toCustomToolName('computer_use'),
+              input: '',
+              providerExecuted: true,
+            });
+
+            content.push({
+              type: 'tool-result',
+              toolCallId: part.id,
+              toolName: toolNameMapping.toCustomToolName('computer_use'),
+              result: {
+                type: 'computer_use_tool_result',
+                status: part.status,
+              },
+            });
+            break;
+          }
+
+          hasFunctionCall = true;
+          const toolName = toolNameMapping.toCustomToolName('computer');
 
           content.push({
-            type: 'tool-result',
-            toolCallId: part.id,
-            toolName: toolNameMapping.toCustomToolName('computer_use'),
-            result: {
-              type: 'computer_use_tool_result',
-              status: part.status || 'completed',
+            type: 'tool-call',
+            toolCallId: part.call_id,
+            toolName,
+            input: JSON.stringify(mapComputerCallInput(part)),
+            providerMetadata: {
+              [providerOptionsName]: {
+                itemId: part.id,
+              },
             },
           });
           break;
@@ -1303,16 +1421,16 @@ export class OpenAIResponsesLanguageModel implements LanguageModelV4 {
                   providerExecuted: true,
                 });
               } else if (value.item.type === 'computer_call') {
+                const toolCallId = value.item.call_id ?? value.item.id;
                 ongoingToolCalls[value.output_index] = {
-                  toolName: toolNameMapping.toCustomToolName('computer_use'),
-                  toolCallId: value.item.id,
+                  toolName: toolNameMapping.toCustomToolName('computer'),
+                  toolCallId,
                 };
 
                 controller.enqueue({
                   type: 'tool-input-start',
-                  id: value.item.id,
-                  toolName: toolNameMapping.toCustomToolName('computer_use'),
-                  providerExecuted: true,
+                  id: toolCallId,
+                  toolName: toolNameMapping.toCustomToolName('computer'),
                 });
               } else if (value.item.type === 'code_interpreter_call') {
                 ongoingToolCalls[value.output_index] = {
@@ -1553,26 +1671,54 @@ export class OpenAIResponsesLanguageModel implements LanguageModelV4 {
               } else if (value.item.type === 'computer_call') {
                 ongoingToolCalls[value.output_index] = undefined;
 
+                if (value.item.call_id == null) {
+                  controller.enqueue({
+                    type: 'tool-input-end',
+                    id: value.item.id,
+                  });
+                  controller.enqueue({
+                    type: 'tool-call',
+                    toolCallId: value.item.id,
+                    toolName: toolNameMapping.toCustomToolName('computer_use'),
+                    input: '',
+                    providerExecuted: true,
+                  });
+                  controller.enqueue({
+                    type: 'tool-result',
+                    toolCallId: value.item.id,
+                    toolName: toolNameMapping.toCustomToolName('computer_use'),
+                    result: {
+                      type: 'computer_use_tool_result',
+                      status: value.item.status,
+                    },
+                  });
+                  return;
+                }
+
+                hasFunctionCall = true;
+                const toolName = toolNameMapping.toCustomToolName('computer');
+                const input = JSON.stringify(mapComputerCallInput(value.item));
+
                 controller.enqueue({
-                  type: 'tool-input-end',
-                  id: value.item.id,
+                  type: 'tool-input-delta',
+                  id: value.item.call_id,
+                  delta: input,
                 });
 
                 controller.enqueue({
-                  type: 'tool-call',
-                  toolCallId: value.item.id,
-                  toolName: toolNameMapping.toCustomToolName('computer_use'),
-                  input: '',
-                  providerExecuted: true,
+                  type: 'tool-input-end',
+                  id: value.item.call_id,
                 });
 
                 controller.enqueue({
-                  type: 'tool-result',
-                  toolCallId: value.item.id,
-                  toolName: toolNameMapping.toCustomToolName('computer_use'),
-                  result: {
-                    type: 'computer_use_tool_result',
-                    status: value.item.status || 'completed',
+                  type: 'tool-call',
+                  toolCallId: value.item.call_id,
+                  toolName,
+                  input,
+                  providerMetadata: {
+                    [providerOptionsName]: {
+                      itemId: value.item.id,
+                    },
                   },
                 });
               } else if (value.item.type === 'file_search_call') {

package/src/responses/openai-responses-prepare-tools.ts

@@ -62,6 +62,7 @@ export async function prepareResponsesTools({
     | { type: 'mcp' }
     | { type: 'image_generation' }
     | { type: 'apply_patch' }
+    | { type: 'computer' }
     | {
         type: 'allowed_tools';
         mode: 'auto' | 'required';
@@ -171,6 +172,12 @@ export async function prepareResponsesTools({
             });
             break;
           }
+          case 'openai.computer': {
+            openaiTools.push({
+              type: 'computer',
+            });
+            break;
+          }
           case 'openai.web_search_preview': {
             const args = await validateTypes({
               value: tool.args,
@@ -374,7 +381,8 @@ export async function prepareResponsesTools({
           resolvedToolName === 'web_search_preview' ||
           resolvedToolName === 'web_search' ||
           resolvedToolName === 'mcp' ||
-          resolvedToolName === 'apply_patch'
+          resolvedToolName === 'apply_patch' ||
+          resolvedToolName === 'computer'
             ? { type: resolvedToolName }
             : resolvedCustomProviderToolNames.has(resolvedToolName)
               ? { type: 'custom', name: resolvedToolName }

package/src/tool/computer.ts

@@ -0,0 +1,147 @@
+import {
+  createProviderDefinedToolFactoryWithOutputSchema,
+  lazySchema,
+  zodSchema,
+} from '@ai-sdk/provider-utils';
+import { z } from 'zod/v4';
+
+const safetyCheckSchema = z.object({
+  id: z.string(),
+  code: z.string().optional(),
+  message: z.string().optional(),
+});
+
+const computerActionSchema = z.discriminatedUnion('type', [
+  z.object({
+    type: z.literal('click'),
+    button: z.enum(['left', 'right', 'wheel', 'back', 'forward']),
+    x: z.number(),
+    y: z.number(),
+    keys: z.array(z.string()).optional(),
+  }),
+  z.object({
+    type: z.literal('double_click'),
+    x: z.number(),
+    y: z.number(),
+    keys: z.array(z.string()).optional(),
+  }),
+  z.object({
+    type: z.literal('drag'),
+    path: z.array(z.object({ x: z.number(), y: z.number() })),
+    keys: z.array(z.string()).optional(),
+  }),
+  z.object({
+    type: z.literal('keypress'),
+    keys: z.array(z.string()),
+  }),
+  z.object({
+    type: z.literal('move'),
+    x: z.number(),
+    y: z.number(),
+    keys: z.array(z.string()).optional(),
+  }),
+  z.object({
+    type: z.literal('screenshot'),
+  }),
+  z.object({
+    type: z.literal('scroll'),
+    x: z.number(),
+    y: z.number(),
+    scrollX: z.number(),
+    scrollY: z.number(),
+    keys: z.array(z.string()).optional(),
+  }),
+  z.object({
+    type: z.literal('type'),
+    text: z.string(),
+  }),
+  z.object({
+    type: z.literal('wait'),
+  }),
+]);
+
+export const computerInputSchema = lazySchema(() =>
+  zodSchema(
+    z.object({
+      actions: z.array(computerActionSchema),
+      pendingSafetyChecks: z.array(safetyCheckSchema),
+      status: z.enum(['in_progress', 'completed', 'incomplete']),
+    }),
+  ),
+);
+
+export const computerOutputSchema = lazySchema(() =>
+  zodSchema(
+    z.object({
+      output: z.union([
+        z.object({
+          type: z.literal('computer_screenshot'),
+          imageUrl: z.string(),
+          fileId: z.string().optional(),
+          detail: z.enum(['auto', 'low', 'high', 'original']).optional(),
+        }),
+        z.object({
+          type: z.literal('computer_screenshot'),
+          fileId: z.string(),
+          imageUrl: z.string().optional(),
+          detail: z.enum(['auto', 'low', 'high', 'original']).optional(),
+        }),
+      ]),
+      acknowledgedSafetyChecks: z.array(safetyCheckSchema).optional(),
+    }),
+  ),
+);
+
+export type OpenAIComputerAction = z.infer<typeof computerActionSchema>;
+export type OpenAIComputerSafetyCheck = z.infer<typeof safetyCheckSchema>;
+
+const computerToolFactory = createProviderDefinedToolFactoryWithOutputSchema<
+  {
+    /**
+     * Ordered UI actions to execute.
+     */
+    actions: OpenAIComputerAction[];
+
+    /**
+     * Safety checks that must be acknowledged before continuing.
+     */
+    pendingSafetyChecks: OpenAIComputerSafetyCheck[];
+
+    /**
+     * Status of the computer call.
+     */
+    status: 'in_progress' | 'completed' | 'incomplete';
+  },
+  {
+    /**
+     * The screenshot captured after executing all actions.
+     */
+    output:
+      | {
+          type: 'computer_screenshot';
+          imageUrl: string;
+          fileId?: string;
+          detail?: 'auto' | 'low' | 'high' | 'original';
+        }
+      | {
+          type: 'computer_screenshot';
+          fileId: string;
+          imageUrl?: string;
+          detail?: 'auto' | 'low' | 'high' | 'original';
+        };
+
+    /**
+     * Safety checks that the application has reviewed and acknowledged.
+     */
+    acknowledgedSafetyChecks?: OpenAIComputerSafetyCheck[];
+  },
+  {}
+>({
+  id: 'openai.computer',
+  inputSchema: computerInputSchema,
+  outputSchema: computerOutputSchema,
+});
+
+export const computer = (
+  options: Parameters<typeof computerToolFactory>[0] = {},
+) => computerToolFactory(options);