@ai-sdk/openai 4.0.13 → 4.0.15

package/docs/03-openai.mdx

@@ -769,6 +769,95 @@ The MCP tool can be configured with:
   to AI SDK tools first.
 </Note>
 
+#### Computer Tool
+
+The OpenAI Responses API supports the GA computer tool through
+`openai.tools.computer`. The model returns an ordered batch of browser or
+desktop actions for your application to execute, and your application returns
+an updated screenshot.
+
+```ts
+import { openai } from '@ai-sdk/openai';
+import { generateText, isStepCount } from 'ai';
+import { captureScreenshot, executeComputerAction } from '@/utils/computer-use';
+
+const result = await generateText({
+  model: openai.responses('gpt-5.4'),
+  tools: {
+    computer: openai.tools.computer({
+      needsApproval: ({ pendingSafetyChecks }) =>
+        pendingSafetyChecks.length > 0,
+
+      execute: async ({ actions, pendingSafetyChecks }) => {
+        for (const action of actions) {
+          await executeComputerAction(action);
+        }
+
+        return {
+          output: {
+            type: 'computer_screenshot',
+            imageUrl: `data:image/png;base64,${await captureScreenshot()}`,
+            detail: 'original',
+          },
+          acknowledgedSafetyChecks: pendingSafetyChecks,
+        };
+      },
+    }),
+  },
+  prompt: 'Open the settings page and enable dark mode.',
+  stopWhen: isStepCount(10),
+});
+```
+
+The execute callback receives:
+
+- **actions** - An ordered array of `click`, `double_click`, `drag`,
+  `keypress`, `move`, `screenshot`, `scroll`, `type`, and `wait` actions.
+  Execute every action in order before capturing the next screenshot.
+- **pendingSafetyChecks** - Safety checks returned by OpenAI. Review these
+  before continuing and include accepted checks in
+  `acknowledgedSafetyChecks`.
+- **status** - The computer call status.
+
+The action union is also exported as `OpenAIComputerAction`. Action-specific
+fields are:
+
+| Action         | Fields                                               |
+| -------------- | ---------------------------------------------------- |
+| `click`        | `button`, `x`, `y`, optional `keys`                  |
+| `double_click` | `x`, `y`, optional `keys`                            |
+| `drag`         | `path` containing `{ x, y }` points, optional `keys` |
+| `keypress`     | `keys`                                               |
+| `move`         | `x`, `y`, optional `keys`                            |
+| `screenshot`   | No additional fields                                 |
+| `scroll`       | `x`, `y`, `scrollX`, `scrollY`, optional `keys`      |
+| `type`         | `text`                                               |
+| `wait`         | No additional fields                                 |
+
+Safety checks use the exported `OpenAIComputerSafetyCheck` type.
+
+Return a `computer_screenshot` with either:
+
+- **imageUrl** - A fully qualified URL or image data URL.
+- **fileId** - The ID of an uploaded screenshot file.
+
+`detail` can be `auto`, `low`, `high`, or `original`. `original` preserves the
+captured resolution and is recommended when accurate coordinates are
+important.
+
+Computer calls and `computer_call_output` screenshot results are carried
+through multi-step generations when using `store`, `previousResponseId`, or
+OpenAI conversations. You can also set `store: false`; the provider will
+serialize the complete call and screenshot result into each follow-up request.
+
+<Note type="warning">
+  Run computer use in an isolated browser or VM. Restrict reachable domains,
+  accounts, credentials, environment variables, and file-system access. Treat
+  on-screen content as untrusted prompt-injection input, require confirmation
+  immediately before consequential actions, and avoid exposing sensitive
+  information in screenshots.
+</Note>
+
 #### Local Shell Tool
 
 The OpenAI responses API support the local shell tool for Codex models through the `openai.tools.localShell` tool.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ai-sdk/openai",
-  "version": "4.0.13",
+  "version": "4.0.15",
   "type": "module",
   "license": "Apache-2.0",
   "sideEffects": false,
@@ -36,15 +36,15 @@
   },
   "dependencies": {
     "@ai-sdk/provider": "4.0.3",
-    "@ai-sdk/provider-utils": "5.0.9"
+    "@ai-sdk/provider-utils": "5.0.10"
   },
   "devDependencies": {
     "@types/node": "22.19.19",
     "tsup": "^8.5.1",
     "typescript": "5.8.3",
     "zod": "3.25.76",
-    "@vercel/ai-tsconfig": "0.0.0",
-    "@ai-sdk/test-server": "2.0.0"
+    "@ai-sdk/test-server": "2.0.0",
+    "@vercel/ai-tsconfig": "0.0.0"
   },
   "peerDependencies": {
     "zod": "^3.25.76 || ^4.1.8"

package/src/index.ts

@@ -22,6 +22,10 @@ export type { OpenAIEmbeddingModelOptions } from './embedding/openai-embedding-m
 export type { OpenAISpeechModelOptions } from './speech/openai-speech-model-options';
 export type { OpenAITranscriptionModelOptions } from './transcription/openai-transcription-model-options';
 export type { OpenAIFilesOptions } from './files/openai-files-options';
+export type {
+  OpenAIComputerAction,
+  OpenAIComputerSafetyCheck,
+} from './tool/computer';
 export type {
   OpenaiResponsesCompactionProviderMetadata,
   OpenaiResponsesProviderMetadata,

package/src/openai-tools.ts

@@ -1,5 +1,6 @@
 import { applyPatch } from './tool/apply-patch';
 import { codeInterpreter } from './tool/code-interpreter';
+import { computer } from './tool/computer';
 import { customTool } from './tool/custom';
 import { fileSearch } from './tool/file-search';
 import { imageGeneration } from './tool/image-generation';
@@ -39,6 +40,16 @@ export const openaiTools = {
    */
   codeInterpreter,
 
+  /**
+   * The computer tool allows models to operate a browser or desktop through
+   * batched UI actions. Your application executes the actions and returns an
+   * updated screenshot.
+   *
+   * WARNING: Run computer use in an isolated environment, treat on-screen
+   * content as untrusted, and require confirmation for consequential actions.
+   */
+  computer,
+
   /**
    * File search is a tool available in the Responses API. It enables models to
    * retrieve information in a knowledge base of previously uploaded files through

package/src/responses/convert-to-openai-responses-input.ts

@@ -21,6 +21,7 @@ import {
   applyPatchInputSchema,
   applyPatchOutputSchema,
 } from '../tool/apply-patch';
+import { computerInputSchema, computerOutputSchema } from '../tool/computer';
 import {
   localShellInputSchema,
   localShellOutputSchema,
@@ -77,6 +78,7 @@ export async function convertToOpenAIResponsesInput({
   hasLocalShellTool = false,
   hasShellTool = false,
   hasApplyPatchTool = false,
+  hasComputerTool = false,
   customProviderToolNames,
 }: {
   prompt: LanguageModelV4Prompt;
@@ -92,6 +94,7 @@ export async function convertToOpenAIResponsesInput({
   hasLocalShellTool?: boolean;
   hasShellTool?: boolean;
   hasApplyPatchTool?: boolean;
+  hasComputerTool?: boolean;
   customProviderToolNames?: Set<string>;
 }): Promise<{
   input: OpenAIResponsesInput;
@@ -401,7 +404,7 @@ export async function convertToOpenAIResponsesInput({
               }
 
               // Provider-defined tool calls (local_shell, shell, apply_patch,
-              // and custom tools) are stored by the API and can be sent as an
+              // computer, and custom tools) are stored by the API and can be sent as an
               // `item_reference` to reduce payload size. Plain client-executed
               // function calls must NOT be: the matching `function_call_output`
               // can only reference the call by `call_id` (`call_...`), which
@@ -414,6 +417,7 @@ export async function convertToOpenAIResponsesInput({
                 (hasLocalShellTool && resolvedToolName === 'local_shell') ||
                 (hasShellTool && resolvedToolName === 'shell') ||
                 (hasApplyPatchTool && resolvedToolName === 'apply_patch') ||
+                (hasComputerTool && resolvedToolName === 'computer') ||
                 (customProviderToolNames?.has(resolvedToolName) ?? false);
 
               if (store && id != null && isProviderDefinedToolCall) {
@@ -479,6 +483,55 @@ export async function convertToOpenAIResponsesInput({
                 break;
               }
 
+              if (hasComputerTool && resolvedToolName === 'computer') {
+                const parsedInput = await validateTypes({
+                  value: part.input,
+                  schema: computerInputSchema,
+                });
+                input.push({
+                  type: 'computer_call',
+                  call_id: part.toolCallId,
+                  id: id!,
+                  status: parsedInput.status,
+                  actions: parsedInput.actions.map(action => {
+                    switch (action.type) {
+                      case 'click':
+                      case 'double_click':
+                      case 'move':
+                        return {
+                          ...action,
+                          keys: action.keys,
+                        };
+                      case 'drag':
+                        return {
+                          ...action,
+                          keys: action.keys,
+                        };
+                      case 'scroll':
+                        return {
+                          type: 'scroll' as const,
+                          x: action.x,
+                          y: action.y,
+                          scroll_x: action.scrollX,
+                          scroll_y: action.scrollY,
+                          keys: action.keys,
+                        };
+                      default:
+                        return action;
+                    }
+                  }),
+                  pending_safety_checks: parsedInput.pendingSafetyChecks.map(
+                    safetyCheck => ({
+                      id: safetyCheck.id,
+                      code: safetyCheck.code,
+                      message: safetyCheck.message,
+                    }),
+                  ),
+                });
+
+                break;
+              }
+
               if (customProviderToolNames?.has(resolvedToolName)) {
                 input.push({
                   type: 'custom_tool_call',
@@ -869,6 +922,35 @@ export async function convertToOpenAIResponsesInput({
             continue;
           }
 
+          if (
+            hasComputerTool &&
+            resolvedToolName === 'computer' &&
+            output.type === 'json'
+          ) {
+            const parsedOutput = await validateTypes({
+              value: output.value,
+              schema: computerOutputSchema,
+            });
+
+            input.push({
+              type: 'computer_call_output',
+              call_id: part.toolCallId,
+              output: {
+                type: 'computer_screenshot',
+                image_url: parsedOutput.output.imageUrl,
+                file_id: parsedOutput.output.fileId,
+                detail: parsedOutput.output.detail,
+              },
+              acknowledged_safety_checks:
+                parsedOutput.acknowledgedSafetyChecks?.map(safetyCheck => ({
+                  id: safetyCheck.id,
+                  code: safetyCheck.code,
+                  message: safetyCheck.message,
+                })),
+            });
+            continue;
+          }
+
           if (customProviderToolNames?.has(resolvedToolName)) {
             let outputValue: OpenAIResponsesCustomToolCallOutput['output'];
             switch (output.type) {

package/src/responses/openai-responses-api.ts

@@ -17,6 +17,73 @@ const jsonValueSchema: z.ZodType<JSONValue> = z.lazy(() =>
   ]),
 );
 
+const openaiResponsesComputerSafetyCheckSchema = z.object({
+  id: z.string(),
+  code: z.string().nullish(),
+  message: z.string().nullish(),
+});
+
+const openaiResponsesComputerActionSchema = z.discriminatedUnion('type', [
+  z.object({
+    type: z.literal('click'),
+    button: z.enum(['left', 'right', 'wheel', 'back', 'forward']),
+    x: z.number(),
+    y: z.number(),
+    keys: z.array(z.string()).nullish(),
+  }),
+  z.object({
+    type: z.literal('double_click'),
+    x: z.number(),
+    y: z.number(),
+    keys: z.array(z.string()).nullish(),
+  }),
+  z.object({
+    type: z.literal('drag'),
+    path: z.array(z.object({ x: z.number(), y: z.number() })),
+    keys: z.array(z.string()).nullish(),
+  }),
+  z.object({
+    type: z.literal('keypress'),
+    keys: z.array(z.string()),
+  }),
+  z.object({
+    type: z.literal('move'),
+    x: z.number(),
+    y: z.number(),
+    keys: z.array(z.string()).nullish(),
+  }),
+  z.object({
+    type: z.literal('screenshot'),
+  }),
+  z.object({
+    type: z.literal('scroll'),
+    x: z.number(),
+    y: z.number(),
+    scroll_x: z.number(),
+    scroll_y: z.number(),
+    keys: z.array(z.string()).nullish(),
+  }),
+  z.object({
+    type: z.literal('type'),
+    text: z.string(),
+  }),
+  z.object({
+    type: z.literal('wait'),
+  }),
+]);
+
+const openaiResponsesComputerCallSchema = z.object({
+  type: z.literal('computer_call'),
+  id: z.string(),
+  call_id: z.string().nullish(),
+  status: z.enum(['in_progress', 'completed', 'incomplete']),
+  action: openaiResponsesComputerActionSchema.nullish(),
+  actions: z.array(openaiResponsesComputerActionSchema).nullish(),
+  pending_safety_checks: z
+    .array(openaiResponsesComputerSafetyCheckSchema)
+    .nullish(),
+});
+
 export type OpenAIResponsesInput = Array<OpenAIResponsesInputItem>;
 
 export type OpenAIResponsesInputItem =
@@ -29,6 +96,7 @@ export type OpenAIResponsesInputItem =
   | OpenAIResponsesCustomToolCallOutput
   | OpenAIResponsesMcpApprovalResponse
   | OpenAIResponsesComputerCall
+  | OpenAIResponsesComputerCallOutput
   | OpenAIResponsesLocalShellCall
   | OpenAIResponsesLocalShellCallOutput
   | OpenAIResponsesShellCall
@@ -185,10 +253,28 @@ export type OpenAIResponsesMcpApprovalResponse = {
   approve: boolean;
 };
 
-export type OpenAIResponsesComputerCall = {
-  type: 'computer_call';
-  id: string;
-  status?: string;
+export type OpenAIResponsesComputerAction = InferSchema<
+  typeof openaiResponsesComputerActionSchema
+>;
+
+export type OpenAIResponsesComputerCall = InferSchema<
+  typeof openaiResponsesComputerCallSchema
+>;
+
+export type OpenAIResponsesComputerCallOutput = {
+  type: 'computer_call_output';
+  call_id: string;
+  output: {
+    type: 'computer_screenshot';
+    image_url?: string;
+    file_id?: string;
+    detail?: 'auto' | 'low' | 'high' | 'original';
+  };
+  acknowledged_safety_checks?: Array<{
+    id: string;
+    code?: string;
+    message?: string;
+  }>;
 };
 
 export type OpenAIResponsesLocalShellCall = {
@@ -355,6 +441,9 @@ export type OpenAIResponsesTool =
   | {
       type: 'apply_patch';
     }
+  | {
+      type: 'computer';
+    }
   | {
       type: 'web_search';
       external_web_access: boolean | undefined;
@@ -678,11 +767,7 @@ export const openaiResponsesChunkSchema = lazySchema(() =>
             id: z.string(),
             status: z.string(),
           }),
-          z.object({
-            type: z.literal('computer_call'),
-            id: z.string(),
-            status: z.string(),
-          }),
+          openaiResponsesComputerCallSchema,
           z.object({
             type: z.literal('file_search_call'),
             id: z.string(),
@@ -913,11 +998,7 @@ export const openaiResponsesChunkSchema = lazySchema(() =>
               env: z.record(z.string(), z.string()).optional(),
             }),
           }),
-          z.object({
-            type: z.literal('computer_call'),
-            id: z.string(),
-            status: z.literal('completed'),
-          }),
+          openaiResponsesComputerCallSchema,
           z.object({
             type: z.literal('mcp_call'),
             id: z.string(),
@@ -1340,11 +1421,7 @@ export const openaiResponsesResponseSchema = lazySchema(() =>
               input: z.string(),
               id: z.string(),
             }),
-            z.object({
-              type: z.literal('computer_call'),
-              id: z.string(),
-              status: z.string().optional(),
-            }),
+            openaiResponsesComputerCallSchema,
             z.object({
               type: z.literal('reasoning'),
               id: z.string(),