@ai-sdk/openai 4.0.14 → 4.0.15

package/docs/03-openai.mdx

@@ -769,6 +769,95 @@ The MCP tool can be configured with:
   to AI SDK tools first.
 </Note>
 
+#### Computer Tool
+
+The OpenAI Responses API supports the GA computer tool through
+`openai.tools.computer`. The model returns an ordered batch of browser or
+desktop actions for your application to execute, and your application returns
+an updated screenshot.
+
+```ts
+import { openai } from '@ai-sdk/openai';
+import { generateText, isStepCount } from 'ai';
+import { captureScreenshot, executeComputerAction } from '@/utils/computer-use';
+
+const result = await generateText({
+  model: openai.responses('gpt-5.4'),
+  tools: {
+    computer: openai.tools.computer({
+      needsApproval: ({ pendingSafetyChecks }) =>
+        pendingSafetyChecks.length > 0,
+
+      execute: async ({ actions, pendingSafetyChecks }) => {
+        for (const action of actions) {
+          await executeComputerAction(action);
+        }
+
+        return {
+          output: {
+            type: 'computer_screenshot',
+            imageUrl: `data:image/png;base64,${await captureScreenshot()}`,
+            detail: 'original',
+          },
+          acknowledgedSafetyChecks: pendingSafetyChecks,
+        };
+      },
+    }),
+  },
+  prompt: 'Open the settings page and enable dark mode.',
+  stopWhen: isStepCount(10),
+});
+```
+
+The execute callback receives:
+
+- **actions** - An ordered array of `click`, `double_click`, `drag`,
+  `keypress`, `move`, `screenshot`, `scroll`, `type`, and `wait` actions.
+  Execute every action in order before capturing the next screenshot.
+- **pendingSafetyChecks** - Safety checks returned by OpenAI. Review these
+  before continuing and include accepted checks in
+  `acknowledgedSafetyChecks`.
+- **status** - The computer call status.
+
+The action union is also exported as `OpenAIComputerAction`. Action-specific
+fields are:
+
+| Action         | Fields                                               |
+| -------------- | ---------------------------------------------------- |
+| `click`        | `button`, `x`, `y`, optional `keys`                  |
+| `double_click` | `x`, `y`, optional `keys`                            |
+| `drag`         | `path` containing `{ x, y }` points, optional `keys` |
+| `keypress`     | `keys`                                               |
+| `move`         | `x`, `y`, optional `keys`                            |
+| `screenshot`   | No additional fields                                 |
+| `scroll`       | `x`, `y`, `scrollX`, `scrollY`, optional `keys`      |
+| `type`         | `text`                                               |
+| `wait`         | No additional fields                                 |
+
+Safety checks use the exported `OpenAIComputerSafetyCheck` type.
+
+Return a `computer_screenshot` with either:
+
+- **imageUrl** - A fully qualified URL or image data URL.
+- **fileId** - The ID of an uploaded screenshot file.
+
+`detail` can be `auto`, `low`, `high`, or `original`. `original` preserves the
+captured resolution and is recommended when accurate coordinates are
+important.
+
+Computer calls and `computer_call_output` screenshot results are carried
+through multi-step generations when using `store`, `previousResponseId`, or
+OpenAI conversations. You can also set `store: false`; the provider will
+serialize the complete call and screenshot result into each follow-up request.
+
+<Note type="warning">
+  Run computer use in an isolated browser or VM. Restrict reachable domains,
+  accounts, credentials, environment variables, and file-system access. Treat
+  on-screen content as untrusted prompt-injection input, require confirmation
+  immediately before consequential actions, and avoid exposing sensitive
+  information in screenshots.
+</Note>
+
 #### Local Shell Tool
 
 The OpenAI responses API support the local shell tool for Codex models through the `openai.tools.localShell` tool.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ai-sdk/openai",
-  "version": "4.0.14",
+  "version": "4.0.15",
   "type": "module",
   "license": "Apache-2.0",
   "sideEffects": false,

package/src/index.ts

@@ -22,6 +22,10 @@ export type { OpenAIEmbeddingModelOptions } from './embedding/openai-embedding-m
 export type { OpenAISpeechModelOptions } from './speech/openai-speech-model-options';
 export type { OpenAITranscriptionModelOptions } from './transcription/openai-transcription-model-options';
 export type { OpenAIFilesOptions } from './files/openai-files-options';
+export type {
+  OpenAIComputerAction,
+  OpenAIComputerSafetyCheck,
+} from './tool/computer';
 export type {
   OpenaiResponsesCompactionProviderMetadata,
   OpenaiResponsesProviderMetadata,

package/src/openai-tools.ts

@@ -1,5 +1,6 @@
 import { applyPatch } from './tool/apply-patch';
 import { codeInterpreter } from './tool/code-interpreter';
+import { computer } from './tool/computer';
 import { customTool } from './tool/custom';
 import { fileSearch } from './tool/file-search';
 import { imageGeneration } from './tool/image-generation';
@@ -39,6 +40,16 @@ export const openaiTools = {
    */
   codeInterpreter,
 
+  /**
+   * The computer tool allows models to operate a browser or desktop through
+   * batched UI actions. Your application executes the actions and returns an
+   * updated screenshot.
+   *
+   * WARNING: Run computer use in an isolated environment, treat on-screen
+   * content as untrusted, and require confirmation for consequential actions.
+   */
+  computer,
+
   /**
    * File search is a tool available in the Responses API. It enables models to
    * retrieve information in a knowledge base of previously uploaded files through

package/src/responses/convert-to-openai-responses-input.ts

@@ -21,6 +21,7 @@ import {
   applyPatchInputSchema,
   applyPatchOutputSchema,
 } from '../tool/apply-patch';
+import { computerInputSchema, computerOutputSchema } from '../tool/computer';
 import {
   localShellInputSchema,
   localShellOutputSchema,
@@ -77,6 +78,7 @@ export async function convertToOpenAIResponsesInput({
   hasLocalShellTool = false,
   hasShellTool = false,
   hasApplyPatchTool = false,
+  hasComputerTool = false,
   customProviderToolNames,
 }: {
   prompt: LanguageModelV4Prompt;
@@ -92,6 +94,7 @@ export async function convertToOpenAIResponsesInput({
   hasLocalShellTool?: boolean;
   hasShellTool?: boolean;
   hasApplyPatchTool?: boolean;
+  hasComputerTool?: boolean;
   customProviderToolNames?: Set<string>;
 }): Promise<{
   input: OpenAIResponsesInput;
@@ -401,7 +404,7 @@ export async function convertToOpenAIResponsesInput({
               }
 
               // Provider-defined tool calls (local_shell, shell, apply_patch,
-              // and custom tools) are stored by the API and can be sent as an
+              // computer, and custom tools) are stored by the API and can be sent as an
               // `item_reference` to reduce payload size. Plain client-executed
               // function calls must NOT be: the matching `function_call_output`
               // can only reference the call by `call_id` (`call_...`), which
@@ -414,6 +417,7 @@ export async function convertToOpenAIResponsesInput({
                 (hasLocalShellTool && resolvedToolName === 'local_shell') ||
                 (hasShellTool && resolvedToolName === 'shell') ||
                 (hasApplyPatchTool && resolvedToolName === 'apply_patch') ||
+                (hasComputerTool && resolvedToolName === 'computer') ||
                 (customProviderToolNames?.has(resolvedToolName) ?? false);
 
               if (store && id != null && isProviderDefinedToolCall) {
@@ -479,6 +483,55 @@ export async function convertToOpenAIResponsesInput({
                 break;
               }
 
+              if (hasComputerTool && resolvedToolName === 'computer') {
+                const parsedInput = await validateTypes({
+                  value: part.input,
+                  schema: computerInputSchema,
+                });
+                input.push({
+                  type: 'computer_call',
+                  call_id: part.toolCallId,
+                  id: id!,
+                  status: parsedInput.status,
+                  actions: parsedInput.actions.map(action => {
+                    switch (action.type) {
+                      case 'click':
+                      case 'double_click':
+                      case 'move':
+                        return {
+                          ...action,
+                          keys: action.keys,
+                        };
+                      case 'drag':
+                        return {
+                          ...action,
+                          keys: action.keys,
+                        };
+                      case 'scroll':
+                        return {
+                          type: 'scroll' as const,
+                          x: action.x,
+                          y: action.y,
+                          scroll_x: action.scrollX,
+                          scroll_y: action.scrollY,
+                          keys: action.keys,
+                        };
+                      default:
+                        return action;
+                    }
+                  }),
+                  pending_safety_checks: parsedInput.pendingSafetyChecks.map(
+                    safetyCheck => ({
+                      id: safetyCheck.id,
+                      code: safetyCheck.code,
+                      message: safetyCheck.message,
+                    }),
+                  ),
+                });
+
+                break;
+              }
+
               if (customProviderToolNames?.has(resolvedToolName)) {
                 input.push({
                   type: 'custom_tool_call',
@@ -869,6 +922,35 @@ export async function convertToOpenAIResponsesInput({
             continue;
           }
 
+          if (
+            hasComputerTool &&
+            resolvedToolName === 'computer' &&
+            output.type === 'json'
+          ) {
+            const parsedOutput = await validateTypes({
+              value: output.value,
+              schema: computerOutputSchema,
+            });
+
+            input.push({
+              type: 'computer_call_output',
+              call_id: part.toolCallId,
+              output: {
+                type: 'computer_screenshot',
+                image_url: parsedOutput.output.imageUrl,
+                file_id: parsedOutput.output.fileId,
+                detail: parsedOutput.output.detail,
+              },
+              acknowledged_safety_checks:
+                parsedOutput.acknowledgedSafetyChecks?.map(safetyCheck => ({
+                  id: safetyCheck.id,
+                  code: safetyCheck.code,
+                  message: safetyCheck.message,
+                })),
+            });
+            continue;
+          }
+
           if (customProviderToolNames?.has(resolvedToolName)) {
             let outputValue: OpenAIResponsesCustomToolCallOutput['output'];
             switch (output.type) {

package/src/responses/openai-responses-api.ts

@@ -17,6 +17,73 @@ const jsonValueSchema: z.ZodType<JSONValue> = z.lazy(() =>
   ]),
 );
 
+const openaiResponsesComputerSafetyCheckSchema = z.object({
+  id: z.string(),
+  code: z.string().nullish(),
+  message: z.string().nullish(),
+});
+
+const openaiResponsesComputerActionSchema = z.discriminatedUnion('type', [
+  z.object({
+    type: z.literal('click'),
+    button: z.enum(['left', 'right', 'wheel', 'back', 'forward']),
+    x: z.number(),
+    y: z.number(),
+    keys: z.array(z.string()).nullish(),
+  }),
+  z.object({
+    type: z.literal('double_click'),
+    x: z.number(),
+    y: z.number(),
+    keys: z.array(z.string()).nullish(),
+  }),
+  z.object({
+    type: z.literal('drag'),
+    path: z.array(z.object({ x: z.number(), y: z.number() })),
+    keys: z.array(z.string()).nullish(),
+  }),
+  z.object({
+    type: z.literal('keypress'),
+    keys: z.array(z.string()),
+  }),
+  z.object({
+    type: z.literal('move'),
+    x: z.number(),
+    y: z.number(),
+    keys: z.array(z.string()).nullish(),
+  }),
+  z.object({
+    type: z.literal('screenshot'),
+  }),
+  z.object({
+    type: z.literal('scroll'),
+    x: z.number(),
+    y: z.number(),
+    scroll_x: z.number(),
+    scroll_y: z.number(),
+    keys: z.array(z.string()).nullish(),
+  }),
+  z.object({
+    type: z.literal('type'),
+    text: z.string(),
+  }),
+  z.object({
+    type: z.literal('wait'),
+  }),
+]);
+
+const openaiResponsesComputerCallSchema = z.object({
+  type: z.literal('computer_call'),
+  id: z.string(),
+  call_id: z.string().nullish(),
+  status: z.enum(['in_progress', 'completed', 'incomplete']),
+  action: openaiResponsesComputerActionSchema.nullish(),
+  actions: z.array(openaiResponsesComputerActionSchema).nullish(),
+  pending_safety_checks: z
+    .array(openaiResponsesComputerSafetyCheckSchema)
+    .nullish(),
+});
+
 export type OpenAIResponsesInput = Array<OpenAIResponsesInputItem>;
 
 export type OpenAIResponsesInputItem =
@@ -29,6 +96,7 @@ export type OpenAIResponsesInputItem =
   | OpenAIResponsesCustomToolCallOutput
   | OpenAIResponsesMcpApprovalResponse
   | OpenAIResponsesComputerCall
+  | OpenAIResponsesComputerCallOutput
   | OpenAIResponsesLocalShellCall
   | OpenAIResponsesLocalShellCallOutput
   | OpenAIResponsesShellCall
@@ -185,10 +253,28 @@ export type OpenAIResponsesMcpApprovalResponse = {
   approve: boolean;
 };
 
-export type OpenAIResponsesComputerCall = {
-  type: 'computer_call';
-  id: string;
-  status?: string;
+export type OpenAIResponsesComputerAction = InferSchema<
+  typeof openaiResponsesComputerActionSchema
+>;
+
+export type OpenAIResponsesComputerCall = InferSchema<
+  typeof openaiResponsesComputerCallSchema
+>;
+
+export type OpenAIResponsesComputerCallOutput = {
+  type: 'computer_call_output';
+  call_id: string;
+  output: {
+    type: 'computer_screenshot';
+    image_url?: string;
+    file_id?: string;
+    detail?: 'auto' | 'low' | 'high' | 'original';
+  };
+  acknowledged_safety_checks?: Array<{
+    id: string;
+    code?: string;
+    message?: string;
+  }>;
 };
 
 export type OpenAIResponsesLocalShellCall = {
@@ -355,6 +441,9 @@ export type OpenAIResponsesTool =
   | {
       type: 'apply_patch';
     }
+  | {
+      type: 'computer';
+    }
   | {
       type: 'web_search';
       external_web_access: boolean | undefined;
@@ -678,11 +767,7 @@ export const openaiResponsesChunkSchema = lazySchema(() =>
             id: z.string(),
             status: z.string(),
           }),
-          z.object({
-            type: z.literal('computer_call'),
-            id: z.string(),
-            status: z.string(),
-          }),
+          openaiResponsesComputerCallSchema,
           z.object({
             type: z.literal('file_search_call'),
             id: z.string(),
@@ -913,11 +998,7 @@ export const openaiResponsesChunkSchema = lazySchema(() =>
               env: z.record(z.string(), z.string()).optional(),
             }),
           }),
-          z.object({
-            type: z.literal('computer_call'),
-            id: z.string(),
-            status: z.literal('completed'),
-          }),
+          openaiResponsesComputerCallSchema,
           z.object({
             type: z.literal('mcp_call'),
             id: z.string(),
@@ -1340,11 +1421,7 @@ export const openaiResponsesResponseSchema = lazySchema(() =>
               input: z.string(),
               id: z.string(),
             }),
-            z.object({
-              type: z.literal('computer_call'),
-              id: z.string(),
-              status: z.string().optional(),
-            }),
+            openaiResponsesComputerCallSchema,
             z.object({
               type: z.literal('reasoning'),
               id: z.string(),

package/src/responses/openai-responses-language-model.ts

@@ -38,6 +38,7 @@ import type {
   codeInterpreterInputSchema,
   codeInterpreterOutputSchema,
 } from '../tool/code-interpreter';
+import type { computerInputSchema } from '../tool/computer';
 import type { fileSearchOutputSchema } from '../tool/file-search';
 import type { imageGenerationOutputSchema } from '../tool/image-generation';
 import type { localShellInputSchema } from '../tool/local-shell';
@@ -64,6 +65,7 @@ import {
   type OpenAIResponsesWebSearchAction,
   type OpenAIResponsesApplyPatchOperationDiffDeltaChunk,
   type OpenAIResponsesApplyPatchOperationDiffDoneChunk,
+  type OpenAIResponsesComputerAction,
 } from './openai-responses-api';
 import {
   openaiLanguageModelResponsesOptionsSchema,
@@ -104,6 +106,87 @@ function extractApprovalRequestIdToToolCallIdMapping(
   return mapping;
 }
 
+function mapComputerAction(
+  action: OpenAIResponsesComputerAction,
+): InferSchema<typeof computerInputSchema>['actions'][number] {
+  switch (action.type) {
+    case 'click':
+      return {
+        type: 'click',
+        button: action.button,
+        x: action.x,
+        y: action.y,
+        ...(action.keys != null && { keys: action.keys }),
+      };
+    case 'double_click':
+      return {
+        type: 'double_click',
+        x: action.x,
+        y: action.y,
+        ...(action.keys != null && { keys: action.keys }),
+      };
+    case 'drag':
+      return {
+        type: 'drag',
+        path: action.path,
+        ...(action.keys != null && { keys: action.keys }),
+      };
+    case 'keypress':
+      return action;
+    case 'move':
+      return {
+        type: 'move',
+        x: action.x,
+        y: action.y,
+        ...(action.keys != null && { keys: action.keys }),
+      };
+    case 'screenshot':
+      return action;
+    case 'scroll':
+      return {
+        type: 'scroll',
+        x: action.x,
+        y: action.y,
+        scrollX: action.scroll_x,
+        scrollY: action.scroll_y,
+        ...(action.keys != null && { keys: action.keys }),
+      };
+    case 'type':
+      return action;
+    case 'wait':
+      return action;
+  }
+}
+
+function mapComputerCallInput({
+  action,
+  actions,
+  pending_safety_checks,
+  status,
+}: {
+  action?: OpenAIResponsesComputerAction | null;
+  actions?: OpenAIResponsesComputerAction[] | null;
+  pending_safety_checks?: Array<{
+    id: string;
+    code?: string | null;
+    message?: string | null;
+  }> | null;
+  status: 'in_progress' | 'completed' | 'incomplete';
+}): InferSchema<typeof computerInputSchema> {
+  return {
+    actions: (actions ?? (action != null ? [action] : [])).map(
+      mapComputerAction,
+    ),
+    pendingSafetyChecks:
+      pending_safety_checks?.map(safetyCheck => ({
+        id: safetyCheck.id,
+        ...(safetyCheck.code != null && { code: safetyCheck.code }),
+        ...(safetyCheck.message != null && { message: safetyCheck.message }),
+      })) ?? [],
+    status,
+  };
+}
+
 export class OpenAIResponsesLanguageModel implements LanguageModelV4 {
   readonly specificationVersion = 'v4';
 
@@ -220,6 +303,7 @@ export class OpenAIResponsesLanguageModel implements LanguageModelV4 {
       tools,
       providerToolNames: {
         'openai.code_interpreter': 'code_interpreter',
+        'openai.computer': 'computer',
         'openai.file_search': 'file_search',
         'openai.image_generation': 'image_generation',
         'openai.local_shell': 'local_shell',
@@ -264,6 +348,7 @@ export class OpenAIResponsesLanguageModel implements LanguageModelV4 {
         hasLocalShellTool: hasOpenAITool('openai.local_shell'),
         hasShellTool: hasOpenAITool('openai.shell'),
         hasApplyPatchTool: hasOpenAITool('openai.apply_patch'),
+        hasComputerTool: hasOpenAITool('openai.computer'),
         customProviderToolNames:
           customProviderToolNames.size > 0
             ? customProviderToolNames
@@ -957,21 +1042,39 @@ export class OpenAIResponsesLanguageModel implements LanguageModelV4 {
         }
 
         case 'computer_call': {
-          content.push({
-            type: 'tool-call',
-            toolCallId: part.id,
-            toolName: toolNameMapping.toCustomToolName('computer_use'),
-            input: '',
-            providerExecuted: true,
-          });
+          if (part.call_id == null) {
+            content.push({
+              type: 'tool-call',
+              toolCallId: part.id,
+              toolName: toolNameMapping.toCustomToolName('computer_use'),
+              input: '',
+              providerExecuted: true,
+            });
+
+            content.push({
+              type: 'tool-result',
+              toolCallId: part.id,
+              toolName: toolNameMapping.toCustomToolName('computer_use'),
+              result: {
+                type: 'computer_use_tool_result',
+                status: part.status,
+              },
+            });
+            break;
+          }
+
+          hasFunctionCall = true;
+          const toolName = toolNameMapping.toCustomToolName('computer');
 
           content.push({
-            type: 'tool-result',
-            toolCallId: part.id,
-            toolName: toolNameMapping.toCustomToolName('computer_use'),
-            result: {
-              type: 'computer_use_tool_result',
-              status: part.status || 'completed',
+            type: 'tool-call',
+            toolCallId: part.call_id,
+            toolName,
+            input: JSON.stringify(mapComputerCallInput(part)),
+            providerMetadata: {
+              [providerOptionsName]: {
+                itemId: part.id,
+              },
             },
           });
           break;
@@ -1303,16 +1406,16 @@ export class OpenAIResponsesLanguageModel implements LanguageModelV4 {
                   providerExecuted: true,
                 });
               } else if (value.item.type === 'computer_call') {
+                const toolCallId = value.item.call_id ?? value.item.id;
                 ongoingToolCalls[value.output_index] = {
-                  toolName: toolNameMapping.toCustomToolName('computer_use'),
-                  toolCallId: value.item.id,
+                  toolName: toolNameMapping.toCustomToolName('computer'),
+                  toolCallId,
                 };
 
                 controller.enqueue({
                   type: 'tool-input-start',
-                  id: value.item.id,
-                  toolName: toolNameMapping.toCustomToolName('computer_use'),
-                  providerExecuted: true,
+                  id: toolCallId,
+                  toolName: toolNameMapping.toCustomToolName('computer'),
                 });
               } else if (value.item.type === 'code_interpreter_call') {
                 ongoingToolCalls[value.output_index] = {
@@ -1553,26 +1656,54 @@ export class OpenAIResponsesLanguageModel implements LanguageModelV4 {
               } else if (value.item.type === 'computer_call') {
                 ongoingToolCalls[value.output_index] = undefined;
 
+                if (value.item.call_id == null) {
+                  controller.enqueue({
+                    type: 'tool-input-end',
+                    id: value.item.id,
+                  });
+                  controller.enqueue({
+                    type: 'tool-call',
+                    toolCallId: value.item.id,
+                    toolName: toolNameMapping.toCustomToolName('computer_use'),
+                    input: '',
+                    providerExecuted: true,
+                  });
+                  controller.enqueue({
+                    type: 'tool-result',
+                    toolCallId: value.item.id,
+                    toolName: toolNameMapping.toCustomToolName('computer_use'),
+                    result: {
+                      type: 'computer_use_tool_result',
+                      status: value.item.status,
+                    },
+                  });
+                  return;
+                }
+
+                hasFunctionCall = true;
+                const toolName = toolNameMapping.toCustomToolName('computer');
+                const input = JSON.stringify(mapComputerCallInput(value.item));
+
                 controller.enqueue({
-                  type: 'tool-input-end',
-                  id: value.item.id,
+                  type: 'tool-input-delta',
+                  id: value.item.call_id,
+                  delta: input,
                 });
 
                 controller.enqueue({
-                  type: 'tool-call',
-                  toolCallId: value.item.id,
-                  toolName: toolNameMapping.toCustomToolName('computer_use'),
-                  input: '',
-                  providerExecuted: true,
+                  type: 'tool-input-end',
+                  id: value.item.call_id,
                 });
 
                 controller.enqueue({
-                  type: 'tool-result',
-                  toolCallId: value.item.id,
-                  toolName: toolNameMapping.toCustomToolName('computer_use'),
-                  result: {
-                    type: 'computer_use_tool_result',
-                    status: value.item.status || 'completed',
+                  type: 'tool-call',
+                  toolCallId: value.item.call_id,
+                  toolName,
+                  input,
+                  providerMetadata: {
+                    [providerOptionsName]: {
+                      itemId: value.item.id,
+                    },
                   },
                 });
               } else if (value.item.type === 'file_search_call') {