@langchain/openai 1.1.3 → 1.2.0-dev-1765432861398

package/dist/tools/computerUse.d.ts

@@ -0,0 +1,234 @@
+import { OpenAI as OpenAI$1 } from "openai";
+import { ToolMessage } from "@langchain/core/messages";
+import { DynamicStructuredTool, ToolRuntime } from "@langchain/core/tools";
+import { z } from "zod/v4";
+
+//#region src/tools/computerUse.d.ts
+
+/**
+ * The type of computer environment to control.
+ */
+type ComputerUseEnvironment = "browser" | "mac" | "windows" | "linux" | "ubuntu";
+/**
+ * Re-export action types from OpenAI SDK for convenience.
+ */
+type ComputerUseClickAction = OpenAI$1.Responses.ResponseComputerToolCall.Click;
+type ComputerUseDoubleClickAction = OpenAI$1.Responses.ResponseComputerToolCall.DoubleClick;
+type ComputerUseDragAction = OpenAI$1.Responses.ResponseComputerToolCall.Drag;
+type ComputerUseKeypressAction = OpenAI$1.Responses.ResponseComputerToolCall.Keypress;
+type ComputerUseMoveAction = OpenAI$1.Responses.ResponseComputerToolCall.Move;
+type ComputerUseScreenshotAction = OpenAI$1.Responses.ResponseComputerToolCall.Screenshot;
+type ComputerUseScrollAction = OpenAI$1.Responses.ResponseComputerToolCall.Scroll;
+type ComputerUseTypeAction = OpenAI$1.Responses.ResponseComputerToolCall.Type;
+type ComputerUseWaitAction = OpenAI$1.Responses.ResponseComputerToolCall.Wait;
+/**
+ * Union type of all computer use actions from OpenAI SDK.
+ */
+type ComputerUseAction = OpenAI$1.Responses.ResponseComputerToolCall["action"];
+/**
+ * Input structure for the Computer Use tool.
+ * The action is wrapped in an `action` property.
+ */
+interface ComputerUseInput {
+  action: ComputerUseAction;
+}
+type ComputerUseReturnType = string | Promise<string> | ToolMessage<any> | Promise<ToolMessage<any>>;
+/**
+ * Options for the Computer Use tool.
+ */
+interface ComputerUseOptions {
+  /**
+   * The width of the computer display in pixels.
+   */
+  displayWidth: number;
+  /**
+   * The height of the computer display in pixels.
+   */
+  displayHeight: number;
+  /**
+   * The type of computer environment to control.
+   * - `browser`: Browser automation (recommended for most use cases)
+   * - `mac`: macOS environment
+   * - `windows`: Windows environment
+   * - `linux`: Linux environment
+   * - `ubuntu`: Ubuntu environment
+   */
+  environment: ComputerUseEnvironment;
+  /**
+   * Execute function that handles computer action execution.
+   * This function receives the action input and should return a base64-encoded
+   * screenshot of the result.
+   */
+  execute: (action: ComputerUseAction, runtime: ToolRuntime<any, any>) => ComputerUseReturnType;
+}
+/**
+ * OpenAI Computer Use tool type for the Responses API.
+ */
+type ComputerUseTool = OpenAI$1.Responses.ComputerTool;
+/**
+ * Creates a Computer Use tool that allows models to control computer interfaces
+ * and perform tasks by simulating mouse clicks, keyboard input, scrolling, and more.
+ *
+ * **Computer Use** is a practical application of OpenAI's Computer-Using Agent (CUA)
+ * model (`computer-use-preview`), which combines vision capabilities with advanced
+ * reasoning to simulate controlling computer interfaces.
+ *
+ * **How it works**:
+ * The tool operates in a continuous loop:
+ * 1. Model sends computer actions (click, type, scroll, etc.)
+ * 2. Your code executes these actions in a controlled environment
+ * 3. You capture a screenshot of the result
+ * 4. Send the screenshot back to the model
+ * 5. Repeat until the task is complete
+ *
+ * **Important**: Computer use is in beta and requires careful consideration:
+ * - Use in sandboxed environments only
+ * - Do not use for high-stakes or authenticated tasks
+ * - Always implement human-in-the-loop for important decisions
+ * - Handle safety checks appropriately
+ *
+ * @see {@link https://platform.openai.com/docs/guides/tools-computer-use | OpenAI Computer Use Documentation}
+ *
+ * @param options - Configuration options for the Computer Use tool
+ * @returns A Computer Use tool that can be passed to `bindTools`
+ *
+ * @example
+ * ```typescript
+ * import { ChatOpenAI, tools } from "@langchain/openai";
+ *
+ * const model = new ChatOpenAI({ model: "computer-use-preview" });
+ *
+ * // With execute callback for automatic action handling
+ * const computer = tools.computerUse({
+ *   displayWidth: 1024,
+ *   displayHeight: 768,
+ *   environment: "browser",
+ *   execute: async (action) => {
+ *     if (action.type === "screenshot") {
+ *       return captureScreenshot();
+ *     }
+ *     if (action.type === "click") {
+ *       await page.mouse.click(action.x, action.y, { button: action.button });
+ *       return captureScreenshot();
+ *     }
+ *     if (action.type === "type") {
+ *       await page.keyboard.type(action.text);
+ *       return captureScreenshot();
+ *     }
+ *     // Handle other actions...
+ *     return captureScreenshot();
+ *   },
+ * });
+ *
+ * const llmWithComputer = model.bindTools([computer]);
+ * const response = await llmWithComputer.invoke(
+ *   "Check the latest news on bing.com"
+ * );
+ * ```
+ *
+ * @example
+ * ```typescript
+ * // Without execute callback (manual action handling)
+ * const computer = tools.computerUse({
+ *   displayWidth: 1024,
+ *   displayHeight: 768,
+ *   environment: "browser",
+ * });
+ *
+ * const response = await model.invoke("Check the news", {
+ *   tools: [computer],
+ * });
+ *
+ * // Access the computer call from the response
+ * const computerCall = response.additional_kwargs.tool_outputs?.find(
+ *   (output) => output.type === "computer_call"
+ * );
+ * if (computerCall) {
+ *   console.log("Action to execute:", computerCall.action);
+ *   // Execute the action manually, then send back a screenshot
+ * }
+ * ```
+ *
+ * @example
+ * ```typescript
+ * // For macOS desktop automation with Docker
+ * const computer = tools.computerUse({
+ *   displayWidth: 1920,
+ *   displayHeight: 1080,
+ *   environment: "mac",
+ *   execute: async (action) => {
+ *     if (action.type === "click") {
+ *       await dockerExec(
+ *         `DISPLAY=:99 xdotool mousemove ${action.x} ${action.y} click 1`,
+ *         containerName
+ *       );
+ *     }
+ *     // Capture screenshot from container
+ *     return await getDockerScreenshot(containerName);
+ *   },
+ * });
+ * ```
+ *
+ * @remarks
+ * - Only available through the Responses API (not Chat Completions)
+ * - Requires `computer-use-preview` model
+ * - Actions include: click, double_click, drag, keypress, move, screenshot, scroll, type, wait
+ * - Safety checks may be returned that require acknowledgment before proceeding
+ * - Use `truncation: "auto"` parameter when making requests
+ * - Recommended to use with `reasoning.summary` for debugging
+ */
+declare function computerUse(options: ComputerUseOptions): DynamicStructuredTool<z.ZodObject<{
+  action: z.ZodDiscriminatedUnion<[z.ZodObject<{
+    type: z.ZodLiteral<"screenshot">;
+  }, z.core.$strip>, z.ZodObject<{
+    type: z.ZodLiteral<"click">;
+    x: z.ZodNumber;
+    y: z.ZodNumber;
+    button: z.ZodDefault<z.ZodEnum<{
+      back: "back";
+      forward: "forward";
+      left: "left";
+      right: "right";
+      wheel: "wheel";
+    }>>;
+  }, z.core.$strip>, z.ZodObject<{
+    type: z.ZodLiteral<"double_click">;
+    x: z.ZodNumber;
+    y: z.ZodNumber;
+    button: z.ZodDefault<z.ZodEnum<{
+      back: "back";
+      forward: "forward";
+      left: "left";
+      right: "right";
+      wheel: "wheel";
+    }>>;
+  }, z.core.$strip>, z.ZodObject<{
+    type: z.ZodLiteral<"drag">;
+    path: z.ZodArray<z.ZodObject<{
+      x: z.ZodNumber;
+      y: z.ZodNumber;
+    }, z.core.$strip>>;
+  }, z.core.$strip>, z.ZodObject<{
+    type: z.ZodLiteral<"keypress">;
+    keys: z.ZodArray<z.ZodString>;
+  }, z.core.$strip>, z.ZodObject<{
+    type: z.ZodLiteral<"move">;
+    x: z.ZodNumber;
+    y: z.ZodNumber;
+  }, z.core.$strip>, z.ZodObject<{
+    type: z.ZodLiteral<"scroll">;
+    x: z.ZodNumber;
+    y: z.ZodNumber;
+    scroll_x: z.ZodNumber;
+    scroll_y: z.ZodNumber;
+  }, z.core.$strip>, z.ZodObject<{
+    type: z.ZodLiteral<"type">;
+    text: z.ZodString;
+  }, z.core.$strip>, z.ZodObject<{
+    type: z.ZodLiteral<"wait">;
+    duration: z.ZodOptional<z.ZodNumber>;
+  }, z.core.$strip>]>;
+}, z.core.$strip>, ComputerUseInput, unknown, ComputerUseReturnType>;
+//#endregion
+export { ComputerUseAction, ComputerUseClickAction, ComputerUseDoubleClickAction, ComputerUseDragAction, ComputerUseEnvironment, ComputerUseInput, ComputerUseKeypressAction, ComputerUseMoveAction, ComputerUseOptions, ComputerUseScreenshotAction, ComputerUseScrollAction, ComputerUseTool, ComputerUseTypeAction, ComputerUseWaitAction, computerUse };
+//# sourceMappingURL=computerUse.d.ts.map

package/dist/tools/computerUse.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"computerUse.d.ts","names":["z","OpenAI","OpenAIClient","DynamicStructuredTool","ToolRuntime","ToolMessage","ComputerUseEnvironment","ComputerUseClickAction","Responses","ResponseComputerToolCall","Click","ComputerUseDoubleClickAction","DoubleClick","ComputerUseDragAction","Drag","ComputerUseKeypressAction","Keypress","ComputerUseMoveAction","Move","ComputerUseScreenshotAction","Screenshot","ComputerUseScrollAction","Scroll","ComputerUseTypeAction","Type","ComputerUseWaitAction","Wait","ComputerUseAction","ComputerUseScreenshotActionSchema","ZodLiteral","core","$strip","ZodObject","ComputerUseClickActionSchema","ZodNumber","ZodEnum","ZodDefault","ComputerUseDoubleClickActionSchema","ComputerUseDragActionSchema","ZodArray","ComputerUseKeypressActionSchema","ZodString","ComputerUseMoveActionSchema","ComputerUseScrollActionSchema","ComputerUseTypeActionSchema","ComputerUseWaitActionSchema","ZodOptional","ComputerUseActionSchema","ZodDiscriminatedUnion","ComputerUseScreenshotActionType","infer","ComputerUseClickActionType","ComputerUseDoubleClickActionType","ComputerUseDragActionType","ComputerUseKeypressActionType","ComputerUseMoveActionType","ComputerUseScrollActionType","ComputerUseTypeActionType","ComputerUseWaitActionType","ComputerUseInput","ComputerUseReturnType","Promise","ComputerUseOptions","ComputerUseTool","ComputerTool","computerUse"],"sources":["../../src/tools/computerUse.d.ts"],"sourcesContent":["import { z } from \"zod/v4\";\nimport { OpenAI as OpenAIClient } from \"openai\";\nimport { type DynamicStructuredTool } from \"@langchain/core/tools\";\nimport { type ToolRuntime } from \"@langchain/core/tools\";\nimport { ToolMessage } from \"@langchain/core/messages\";\n/**\n * The type of computer environment to control.\n */\nexport type ComputerUseEnvironment = \"browser\" | \"mac\" | \"windows\" | \"linux\" | \"ubuntu\";\n/**\n * Re-export action types from OpenAI SDK for convenience.\n */\nexport type ComputerUseClickAction = OpenAIClient.Responses.ResponseComputerToolCall.Click;\nexport type ComputerUseDoubleClickAction = OpenAIClient.Responses.ResponseComputerToolCall.DoubleClick;\nexport type ComputerUseDragAction = OpenAIClient.Responses.ResponseComputerToolCall.Drag;\nexport type ComputerUseKeypressAction = OpenAIClient.Responses.ResponseComputerToolCall.Keypress;\nexport type ComputerUseMoveAction = OpenAIClient.Responses.ResponseComputerToolCall.Move;\nexport type ComputerUseScreenshotAction = OpenAIClient.Responses.ResponseComputerToolCall.Screenshot;\nexport type ComputerUseScrollAction = OpenAIClient.Responses.ResponseComputerToolCall.Scroll;\nexport type ComputerUseTypeAction = OpenAIClient.Responses.ResponseComputerToolCall.Type;\nexport type ComputerUseWaitAction = OpenAIClient.Responses.ResponseComputerToolCall.Wait;\n/**\n * Union type of all computer use actions from OpenAI SDK.\n */\nexport type ComputerUseAction = OpenAIClient.Responses.ResponseComputerToolCall[\"action\"];\ndeclare const ComputerUseScreenshotActionSchema: z.ZodObject<{\n    type: z.ZodLiteral<\"screenshot\">;\n}, z.core.$strip>;\ndeclare const ComputerUseClickActionSchema: z.ZodObject<{\n    type: z.ZodLiteral<\"click\">;\n    x: z.ZodNumber;\n    y: z.ZodNumber;\n    button: z.ZodDefault<z.ZodEnum<{\n        back: \"back\";\n        forward: \"forward\";\n        left: \"left\";\n        right: \"right\";\n        wheel: \"wheel\";\n    }>>;\n}, z.core.$strip>;\ndeclare const ComputerUseDoubleClickActionSchema: z.ZodObject<{\n    type: z.ZodLiteral<\"double_click\">;\n    x: z.ZodNumber;\n    y: z.ZodNumber;\n    button: z.ZodDefault<z.ZodEnum<{\n        back: \"back\";\n        forward: \"forward\";\n        left: \"left\";\n        right: \"right\";\n        wheel: \"wheel\";\n    }>>;\n}, z.core.$strip>;\ndeclare const ComputerUseDragActionSchema: z.ZodObject<{\n    type: z.ZodLiteral<\"drag\">;\n    path: z.ZodArray<z.ZodObject<{\n        x: z.ZodNumber;\n        y: z.ZodNumber;\n    }, z.core.$strip>>;\n}, z.core.$strip>;\ndeclare const ComputerUseKeypressActionSchema: z.ZodObject<{\n    type: z.ZodLiteral<\"keypress\">;\n    keys: z.ZodArray<z.ZodString>;\n}, z.core.$strip>;\ndeclare const ComputerUseMoveActionSchema: z.ZodObject<{\n    type: z.ZodLiteral<\"move\">;\n    x: z.ZodNumber;\n    y: z.ZodNumber;\n}, z.core.$strip>;\ndeclare const ComputerUseScrollActionSchema: z.ZodObject<{\n    type: z.ZodLiteral<\"scroll\">;\n    x: z.ZodNumber;\n    y: z.ZodNumber;\n    scroll_x: z.ZodNumber;\n    scroll_y: z.ZodNumber;\n}, z.core.$strip>;\ndeclare const ComputerUseTypeActionSchema: z.ZodObject<{\n    type: z.ZodLiteral<\"type\">;\n    text: z.ZodString;\n}, z.core.$strip>;\ndeclare const ComputerUseWaitActionSchema: z.ZodObject<{\n    type: z.ZodLiteral<\"wait\">;\n    duration: z.ZodOptional<z.ZodNumber>;\n}, z.core.$strip>;\nexport declare const ComputerUseActionSchema: z.ZodObject<{\n    action: z.ZodDiscriminatedUnion<[z.ZodObject<{\n        type: z.ZodLiteral<\"screenshot\">;\n    }, z.core.$strip>, z.ZodObject<{\n        type: z.ZodLiteral<\"click\">;\n        x: z.ZodNumber;\n        y: z.ZodNumber;\n        button: z.ZodDefault<z.ZodEnum<{\n            back: \"back\";\n            forward: \"forward\";\n            left: \"left\";\n            right: \"right\";\n            wheel: \"wheel\";\n        }>>;\n    }, z.core.$strip>, z.ZodObject<{\n        type: z.ZodLiteral<\"double_click\">;\n        x: z.ZodNumber;\n        y: z.ZodNumber;\n        button: z.ZodDefault<z.ZodEnum<{\n            back: \"back\";\n            forward: \"forward\";\n            left: \"left\";\n            right: \"right\";\n            wheel: \"wheel\";\n        }>>;\n    }, z.core.$strip>, z.ZodObject<{\n        type: z.ZodLiteral<\"drag\">;\n        path: z.ZodArray<z.ZodObject<{\n            x: z.ZodNumber;\n            y: z.ZodNumber;\n        }, z.core.$strip>>;\n    }, z.core.$strip>, z.ZodObject<{\n        type: z.ZodLiteral<\"keypress\">;\n        keys: z.ZodArray<z.ZodString>;\n    }, z.core.$strip>, z.ZodObject<{\n        type: z.ZodLiteral<\"move\">;\n        x: z.ZodNumber;\n        y: z.ZodNumber;\n    }, z.core.$strip>, z.ZodObject<{\n        type: z.ZodLiteral<\"scroll\">;\n        x: z.ZodNumber;\n        y: z.ZodNumber;\n        scroll_x: z.ZodNumber;\n        scroll_y: z.ZodNumber;\n    }, z.core.$strip>, z.ZodObject<{\n        type: z.ZodLiteral<\"type\">;\n        text: z.ZodString;\n    }, z.core.$strip>, z.ZodObject<{\n        type: z.ZodLiteral<\"wait\">;\n        duration: z.ZodOptional<z.ZodNumber>;\n    }, z.core.$strip>]>;\n}, z.core.$strip>;\nexport type ComputerUseScreenshotActionType = z.infer<typeof ComputerUseScreenshotActionSchema>;\nexport type ComputerUseClickActionType = z.infer<typeof ComputerUseClickActionSchema>;\nexport type ComputerUseDoubleClickActionType = z.infer<typeof ComputerUseDoubleClickActionSchema>;\nexport type ComputerUseDragActionType = z.infer<typeof ComputerUseDragActionSchema>;\nexport type ComputerUseKeypressActionType = z.infer<typeof ComputerUseKeypressActionSchema>;\nexport type ComputerUseMoveActionType = z.infer<typeof ComputerUseMoveActionSchema>;\nexport type ComputerUseScrollActionType = z.infer<typeof ComputerUseScrollActionSchema>;\nexport type ComputerUseTypeActionType = z.infer<typeof ComputerUseTypeActionSchema>;\nexport type ComputerUseWaitActionType = z.infer<typeof ComputerUseWaitActionSchema>;\n/**\n * Input structure for the Computer Use tool.\n * The action is wrapped in an `action` property.\n */\nexport interface ComputerUseInput {\n    action: ComputerUseAction;\n}\nexport type ComputerUseReturnType = string | Promise<string> | ToolMessage<any> | Promise<ToolMessage<any>>;\n/**\n * Options for the Computer Use tool.\n */\nexport interface ComputerUseOptions {\n    /**\n     * The width of the computer display in pixels.\n     */\n    displayWidth: number;\n    /**\n     * The height of the computer display in pixels.\n     */\n    displayHeight: number;\n    /**\n     * The type of computer environment to control.\n     * - `browser`: Browser automation (recommended for most use cases)\n     * - `mac`: macOS environment\n     * - `windows`: Windows environment\n     * - `linux`: Linux environment\n     * - `ubuntu`: Ubuntu environment\n     */\n    environment: ComputerUseEnvironment;\n    /**\n     * Execute function that handles computer action execution.\n     * This function receives the action input and should return a base64-encoded\n     * screenshot of the result.\n     */\n    execute: (action: ComputerUseAction, runtime: ToolRuntime<any, any>) => ComputerUseReturnType;\n}\n/**\n * OpenAI Computer Use tool type for the Responses API.\n */\nexport type ComputerUseTool = OpenAIClient.Responses.ComputerTool;\n/**\n * Creates a Computer Use tool that allows models to control computer interfaces\n * and perform tasks by simulating mouse clicks, keyboard input, scrolling, and more.\n *\n * **Computer Use** is a practical application of OpenAI's Computer-Using Agent (CUA)\n * model (`computer-use-preview`), which combines vision capabilities with advanced\n * reasoning to simulate controlling computer interfaces.\n *\n * **How it works**:\n * The tool operates in a continuous loop:\n * 1. Model sends computer actions (click, type, scroll, etc.)\n * 2. Your code executes these actions in a controlled environment\n * 3. You capture a screenshot of the result\n * 4. Send the screenshot back to the model\n * 5. Repeat until the task is complete\n *\n * **Important**: Computer use is in beta and requires careful consideration:\n * - Use in sandboxed environments only\n * - Do not use for high-stakes or authenticated tasks\n * - Always implement human-in-the-loop for important decisions\n * - Handle safety checks appropriately\n *\n * @see {@link https://platform.openai.com/docs/guides/tools-computer-use | OpenAI Computer Use Documentation}\n *\n * @param options - Configuration options for the Computer Use tool\n * @returns A Computer Use tool that can be passed to `bindTools`\n *\n * @example\n * ```typescript\n * import { ChatOpenAI, tools } from \"@langchain/openai\";\n *\n * const model = new ChatOpenAI({ model: \"computer-use-preview\" });\n *\n * // With execute callback for automatic action handling\n * const computer = tools.computerUse({\n *   displayWidth: 1024,\n *   displayHeight: 768,\n *   environment: \"browser\",\n *   execute: async (action) => {\n *     if (action.type === \"screenshot\") {\n *       return captureScreenshot();\n *     }\n *     if (action.type === \"click\") {\n *       await page.mouse.click(action.x, action.y, { button: action.button });\n *       return captureScreenshot();\n *     }\n *     if (action.type === \"type\") {\n *       await page.keyboard.type(action.text);\n *       return captureScreenshot();\n *     }\n *     // Handle other actions...\n *     return captureScreenshot();\n *   },\n * });\n *\n * const llmWithComputer = model.bindTools([computer]);\n * const response = await llmWithComputer.invoke(\n *   \"Check the latest news on bing.com\"\n * );\n * ```\n *\n * @example\n * ```typescript\n * // Without execute callback (manual action handling)\n * const computer = tools.computerUse({\n *   displayWidth: 1024,\n *   displayHeight: 768,\n *   environment: \"browser\",\n * });\n *\n * const response = await model.invoke(\"Check the news\", {\n *   tools: [computer],\n * });\n *\n * // Access the computer call from the response\n * const computerCall = response.additional_kwargs.tool_outputs?.find(\n *   (output) => output.type === \"computer_call\"\n * );\n * if (computerCall) {\n *   console.log(\"Action to execute:\", computerCall.action);\n *   // Execute the action manually, then send back a screenshot\n * }\n * ```\n *\n * @example\n * ```typescript\n * // For macOS desktop automation with Docker\n * const computer = tools.computerUse({\n *   displayWidth: 1920,\n *   displayHeight: 1080,\n *   environment: \"mac\",\n *   execute: async (action) => {\n *     if (action.type === \"click\") {\n *       await dockerExec(\n *         `DISPLAY=:99 xdotool mousemove ${action.x} ${action.y} click 1`,\n *         containerName\n *       );\n *     }\n *     // Capture screenshot from container\n *     return await getDockerScreenshot(containerName);\n *   },\n * });\n * ```\n *\n * @remarks\n * - Only available through the Responses API (not Chat Completions)\n * - Requires `computer-use-preview` model\n * - Actions include: click, double_click, drag, keypress, move, screenshot, scroll, type, wait\n * - Safety checks may be returned that require acknowledgment before proceeding\n * - Use `truncation: \"auto\"` parameter when making requests\n * - Recommended to use with `reasoning.summary` for debugging\n */\nexport declare function computerUse(options: ComputerUseOptions): DynamicStructuredTool<z.ZodObject<{\n    action: z.ZodDiscriminatedUnion<[z.ZodObject<{\n        type: z.ZodLiteral<\"screenshot\">;\n    }, z.core.$strip>, z.ZodObject<{\n        type: z.ZodLiteral<\"click\">;\n        x: z.ZodNumber;\n        y: z.ZodNumber;\n        button: z.ZodDefault<z.ZodEnum<{\n            back: \"back\";\n            forward: \"forward\";\n            left: \"left\";\n            right: \"right\";\n            wheel: \"wheel\";\n        }>>;\n    }, z.core.$strip>, z.ZodObject<{\n        type: z.ZodLiteral<\"double_click\">;\n        x: z.ZodNumber;\n        y: z.ZodNumber;\n        button: z.ZodDefault<z.ZodEnum<{\n            back: \"back\";\n            forward: \"forward\";\n            left: \"left\";\n            right: \"right\";\n            wheel: \"wheel\";\n        }>>;\n    }, z.core.$strip>, z.ZodObject<{\n        type: z.ZodLiteral<\"drag\">;\n        path: z.ZodArray<z.ZodObject<{\n            x: z.ZodNumber;\n            y: z.ZodNumber;\n        }, z.core.$strip>>;\n    }, z.core.$strip>, z.ZodObject<{\n        type: z.ZodLiteral<\"keypress\">;\n        keys: z.ZodArray<z.ZodString>;\n    }, z.core.$strip>, z.ZodObject<{\n        type: z.ZodLiteral<\"move\">;\n        x: z.ZodNumber;\n        y: z.ZodNumber;\n    }, z.core.$strip>, z.ZodObject<{\n        type: z.ZodLiteral<\"scroll\">;\n        x: z.ZodNumber;\n        y: z.ZodNumber;\n        scroll_x: z.ZodNumber;\n        scroll_y: z.ZodNumber;\n    }, z.core.$strip>, z.ZodObject<{\n        type: z.ZodLiteral<\"type\">;\n        text: z.ZodString;\n    }, z.core.$strip>, z.ZodObject<{\n        type: z.ZodLiteral<\"wait\">;\n        duration: z.ZodOptional<z.ZodNumber>;\n    }, z.core.$strip>]>;\n}, z.core.$strip>, ComputerUseInput, unknown, ComputerUseReturnType>;\nexport {};\n//# sourceMappingURL=computerUse.d.ts.map"],"mappings":";;;;;;;;AAQA;AAIA;AACYW,KALAL,sBAAAA,GAK4B,SAAA,GAAGJ,KAAAA,GAAaM,SAAAA,GAAUC,OAAAA,GAAAA,QAAAA;AAClE;AACA;AACA;AACYU,KALAZ,sBAAAA,GAAyBL,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBC,KAKpBD;AACrDY,KALAV,4BAAAA,GAA+BT,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBG,WAKC;AAChFW,KALAV,qBAAAA,GAAwBX,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBK,IAKAU;AACxEC,KALAV,yBAAAA,GAA4Bb,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBO,QAKA;AAI5EW,KARAV,qBAAAA,GAAwBf,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBS,IAQL;AA4H9DyC,KAnILxC,2BAAAA,GAA8BjB,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBW,UAoI7D;AAEjBwC,KArIAvC,uBAAAA,GAA0BnB,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBa,MAqIrD;AAAYuC,KApIjCtC,qBAAAA,GAAwBrB,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBe,IAoIvCqC;AAAkBxD,KAnInDoB,qBAAAA,GAAwBvB,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBiB,IAmIrBrB;;;AAA0B;AAIxEyD,KAnILnC,iBAAAA,GAAoBzB,QAAAA,CAAaM,SAAAA,CAAUC,wBAmIpB,CAAA,QAAA,CAAA;;;;;UAPlBkD,gBAAAA;UACLhC;;KAEAiC,qBAAAA,YAAiCC,kBAAkBxD,mBAAmBwD,QAAQxD;;;;UAIzEyD,kBAAAA;;;;;;;;;;;;;;;;;eAiBAxD;;;;;;oBAMKqB,4BAA4BvB,0BAA0BwD;;;;;KAKhEG,eAAAA,GAAkB7D,QAAAA,CAAaM,SAAAA,CAAUwD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAiH7BC,WAAAA,UAAqBH,qBAAqB3D,sBAAsBH,CAAAA,CAAEgC;UAC9EhC,CAAAA,CAAEgD,uBAAuBhD,CAAAA,CAAEgC;UACzBhC,CAAAA,CAAE6B;KACT7B,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS/B,CAAAA,CAAEgC;UACXhC,CAAAA,CAAE6B;OACL7B,CAAAA,CAAEkC;OACFlC,CAAAA,CAAEkC;YACGlC,CAAAA,CAAEoC,WAAWpC,CAAAA,CAAEmC;;;;;;;KAOxBnC,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS/B,CAAAA,CAAEgC;UACXhC,CAAAA,CAAE6B;OACL7B,CAAAA,CAAEkC;OACFlC,CAAAA,CAAEkC;YACGlC,CAAAA,CAAEoC,WAAWpC,CAAAA,CAAEmC;;;;;;;KAOxBnC,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS/B,CAAAA,CAAEgC;UACXhC,CAAAA,CAAE6B;UACF7B,CAAAA,CAAEuC,SAASvC,CAAAA,CAAEgC;SACZhC,CAAAA,CAAEkC;SACFlC,CAAAA,CAAEkC;OACNlC,CAAAA,CAAE8B,IAAAA,CAAKC;KACX/B,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS/B,CAAAA,CAAEgC;UACXhC,CAAAA,CAAE6B;UACF7B,CAAAA,CAAEuC,SAASvC,CAAAA,CAAEyC;KACpBzC,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS/B,CAAAA,CAAEgC;UACXhC,CAAAA,CAAE6B;OACL7B,CAAAA,CAAEkC;OACFlC,CAAAA,CAAEkC;KACNlC,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS/B,CAAAA,CAAEgC;UACXhC,CAAAA,CAAE6B;OACL7B,CAAAA,CAAEkC;OACFlC,CAAAA,CAAEkC;cACKlC,CAAAA,CAAEkC;cACFlC,CAAAA,CAAEkC;KACblC,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS/B,CAAAA,CAAEgC;UACXhC,CAAAA,CAAE6B;UACF7B,CAAAA,CAAEyC;KACTzC,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS/B,CAAAA,CAAEgC;UACXhC,CAAAA,CAAE6B;cACE7B,CAAAA,CAAE8C,YAAY9C,CAAAA,CAAEkC;KAC3BlC,CAAAA,CAAE8B,IAAAA,CAAKC;GACX/B,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS4B,2BAA2BC"}

package/dist/tools/computerUse.js

@@ -0,0 +1,239 @@
+import { ToolMessage } from "@langchain/core/messages";
+import { tool } from "@langchain/core/tools";
+import { z } from "zod/v4";
+
+//#region src/tools/computerUse.ts
+const ComputerUseScreenshotActionSchema = z.object({ type: z.literal("screenshot") });
+const ComputerUseClickActionSchema = z.object({
+	type: z.literal("click"),
+	x: z.number(),
+	y: z.number(),
+	button: z.enum([
+		"left",
+		"right",
+		"wheel",
+		"back",
+		"forward"
+	]).default("left")
+});
+const ComputerUseDoubleClickActionSchema = z.object({
+	type: z.literal("double_click"),
+	x: z.number(),
+	y: z.number(),
+	button: z.enum([
+		"left",
+		"right",
+		"wheel",
+		"back",
+		"forward"
+	]).default("left")
+});
+const ComputerUseDragActionSchema = z.object({
+	type: z.literal("drag"),
+	path: z.array(z.object({
+		x: z.number(),
+		y: z.number()
+	}))
+});
+const ComputerUseKeypressActionSchema = z.object({
+	type: z.literal("keypress"),
+	keys: z.array(z.string())
+});
+const ComputerUseMoveActionSchema = z.object({
+	type: z.literal("move"),
+	x: z.number(),
+	y: z.number()
+});
+const ComputerUseScrollActionSchema = z.object({
+	type: z.literal("scroll"),
+	x: z.number(),
+	y: z.number(),
+	scroll_x: z.number(),
+	scroll_y: z.number()
+});
+const ComputerUseTypeActionSchema = z.object({
+	type: z.literal("type"),
+	text: z.string()
+});
+const ComputerUseWaitActionSchema = z.object({
+	type: z.literal("wait"),
+	duration: z.number().optional()
+});
+const ComputerUseActionUnionSchema = z.discriminatedUnion("type", [
+	ComputerUseScreenshotActionSchema,
+	ComputerUseClickActionSchema,
+	ComputerUseDoubleClickActionSchema,
+	ComputerUseDragActionSchema,
+	ComputerUseKeypressActionSchema,
+	ComputerUseMoveActionSchema,
+	ComputerUseScrollActionSchema,
+	ComputerUseTypeActionSchema,
+	ComputerUseWaitActionSchema
+]);
+const ComputerUseActionSchema = z.object({ action: ComputerUseActionUnionSchema });
+const TOOL_NAME = "computer_use";
+/**
+* Creates a Computer Use tool that allows models to control computer interfaces
+* and perform tasks by simulating mouse clicks, keyboard input, scrolling, and more.
+*
+* **Computer Use** is a practical application of OpenAI's Computer-Using Agent (CUA)
+* model (`computer-use-preview`), which combines vision capabilities with advanced
+* reasoning to simulate controlling computer interfaces.
+*
+* **How it works**:
+* The tool operates in a continuous loop:
+* 1. Model sends computer actions (click, type, scroll, etc.)
+* 2. Your code executes these actions in a controlled environment
+* 3. You capture a screenshot of the result
+* 4. Send the screenshot back to the model
+* 5. Repeat until the task is complete
+*
+* **Important**: Computer use is in beta and requires careful consideration:
+* - Use in sandboxed environments only
+* - Do not use for high-stakes or authenticated tasks
+* - Always implement human-in-the-loop for important decisions
+* - Handle safety checks appropriately
+*
+* @see {@link https://platform.openai.com/docs/guides/tools-computer-use | OpenAI Computer Use Documentation}
+*
+* @param options - Configuration options for the Computer Use tool
+* @returns A Computer Use tool that can be passed to `bindTools`
+*
+* @example
+* ```typescript
+* import { ChatOpenAI, tools } from "@langchain/openai";
+*
+* const model = new ChatOpenAI({ model: "computer-use-preview" });
+*
+* // With execute callback for automatic action handling
+* const computer = tools.computerUse({
+*   displayWidth: 1024,
+*   displayHeight: 768,
+*   environment: "browser",
+*   execute: async (action) => {
+*     if (action.type === "screenshot") {
+*       return captureScreenshot();
+*     }
+*     if (action.type === "click") {
+*       await page.mouse.click(action.x, action.y, { button: action.button });
+*       return captureScreenshot();
+*     }
+*     if (action.type === "type") {
+*       await page.keyboard.type(action.text);
+*       return captureScreenshot();
+*     }
+*     // Handle other actions...
+*     return captureScreenshot();
+*   },
+* });
+*
+* const llmWithComputer = model.bindTools([computer]);
+* const response = await llmWithComputer.invoke(
+*   "Check the latest news on bing.com"
+* );
+* ```
+*
+* @example
+* ```typescript
+* // Without execute callback (manual action handling)
+* const computer = tools.computerUse({
+*   displayWidth: 1024,
+*   displayHeight: 768,
+*   environment: "browser",
+* });
+*
+* const response = await model.invoke("Check the news", {
+*   tools: [computer],
+* });
+*
+* // Access the computer call from the response
+* const computerCall = response.additional_kwargs.tool_outputs?.find(
+*   (output) => output.type === "computer_call"
+* );
+* if (computerCall) {
+*   console.log("Action to execute:", computerCall.action);
+*   // Execute the action manually, then send back a screenshot
+* }
+* ```
+*
+* @example
+* ```typescript
+* // For macOS desktop automation with Docker
+* const computer = tools.computerUse({
+*   displayWidth: 1920,
+*   displayHeight: 1080,
+*   environment: "mac",
+*   execute: async (action) => {
+*     if (action.type === "click") {
+*       await dockerExec(
+*         `DISPLAY=:99 xdotool mousemove ${action.x} ${action.y} click 1`,
+*         containerName
+*       );
+*     }
+*     // Capture screenshot from container
+*     return await getDockerScreenshot(containerName);
+*   },
+* });
+* ```
+*
+* @remarks
+* - Only available through the Responses API (not Chat Completions)
+* - Requires `computer-use-preview` model
+* - Actions include: click, double_click, drag, keypress, move, screenshot, scroll, type, wait
+* - Safety checks may be returned that require acknowledgment before proceeding
+* - Use `truncation: "auto"` parameter when making requests
+* - Recommended to use with `reasoning.summary` for debugging
+*/
+function computerUse(options) {
+	const computerTool = tool(async (input, runtime) => {
+		/**
+		* get computer_use call id from runtime
+		*/
+		const aiMessage = runtime.state?.messages.at(-1);
+		const computerToolCall = aiMessage?.tool_calls?.find((tc) => tc.name === "computer_use");
+		const computerToolCallId = computerToolCall?.id;
+		if (!computerToolCallId) throw new Error("Computer use call id not found");
+		const result = await options.execute(input.action, runtime);
+		/**
+		* make sure {@link ToolMessage} is returned with the correct additional kwargs
+		*/
+		if (typeof result === "string") return new ToolMessage({
+			content: result,
+			tool_call_id: computerToolCallId,
+			additional_kwargs: { type: "computer_call_output" }
+		});
+		/**
+		* make sure {@link ToolMessage} is returned with the correct additional kwargs
+		*/
+		return new ToolMessage({
+			...result,
+			tool_call_id: computerToolCallId,
+			additional_kwargs: {
+				type: "computer_call_output",
+				...result.additional_kwargs
+			}
+		});
+	}, {
+		name: TOOL_NAME,
+		description: "Control a computer interface by executing mouse clicks, keyboard input, scrolling, and other actions.",
+		schema: ComputerUseActionSchema
+	});
+	computerTool.extras = {
+		...computerTool.extras ?? {},
+		providerToolDefinition: {
+			type: "computer_use_preview",
+			display_width: options.displayWidth,
+			display_height: options.displayHeight,
+			environment: options.environment
+		}
+	};
+	/**
+	* return as typed {@link DynamicStructuredTool} so we don't get any type
+	* errors like "can't export tool without reference"
+	*/
+	return computerTool;
+}
+
+//#endregion
+export { computerUse };
+//# sourceMappingURL=computerUse.js.map

package/dist/tools/computerUse.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"computerUse.js","names":["options: ComputerUseOptions","input: ComputerUseInput","runtime: ToolRuntime<{ messages: BaseMessage[] }>"],"sources":["../../src/tools/computerUse.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\nimport { z } from \"zod/v4\";\nimport { OpenAI as OpenAIClient } from \"openai\";\nimport { tool, type DynamicStructuredTool } from \"@langchain/core/tools\";\nimport { type ToolRuntime } from \"@langchain/core/tools\";\nimport {\n  ToolMessage,\n  type AIMessage,\n  type BaseMessage,\n} from \"@langchain/core/messages\";\n\n/**\n * The type of computer environment to control.\n */\nexport type ComputerUseEnvironment =\n  | \"browser\"\n  | \"mac\"\n  | \"windows\"\n  | \"linux\"\n  | \"ubuntu\";\n\n/**\n * Re-export action types from OpenAI SDK for convenience.\n */\nexport type ComputerUseClickAction =\n  OpenAIClient.Responses.ResponseComputerToolCall.Click;\nexport type ComputerUseDoubleClickAction =\n  OpenAIClient.Responses.ResponseComputerToolCall.DoubleClick;\nexport type ComputerUseDragAction =\n  OpenAIClient.Responses.ResponseComputerToolCall.Drag;\nexport type ComputerUseKeypressAction =\n  OpenAIClient.Responses.ResponseComputerToolCall.Keypress;\nexport type ComputerUseMoveAction =\n  OpenAIClient.Responses.ResponseComputerToolCall.Move;\nexport type ComputerUseScreenshotAction =\n  OpenAIClient.Responses.ResponseComputerToolCall.Screenshot;\nexport type ComputerUseScrollAction =\n  OpenAIClient.Responses.ResponseComputerToolCall.Scroll;\nexport type ComputerUseTypeAction =\n  OpenAIClient.Responses.ResponseComputerToolCall.Type;\nexport type ComputerUseWaitAction =\n  OpenAIClient.Responses.ResponseComputerToolCall.Wait;\n\n/**\n * Union type of all computer use actions from OpenAI SDK.\n */\nexport type ComputerUseAction =\n  OpenAIClient.Responses.ResponseComputerToolCall[\"action\"];\n\n// Zod schemas for computer use actions\nconst ComputerUseScreenshotActionSchema = z.object({\n  type: z.literal(\"screenshot\"),\n});\n\nconst ComputerUseClickActionSchema = z.object({\n  type: z.literal(\"click\"),\n  x: z.number(),\n  y: z.number(),\n  button: z.enum([\"left\", \"right\", \"wheel\", \"back\", \"forward\"]).default(\"left\"),\n});\n\nconst ComputerUseDoubleClickActionSchema = z.object({\n  type: z.literal(\"double_click\"),\n  x: z.number(),\n  y: z.number(),\n  button: z.enum([\"left\", \"right\", \"wheel\", \"back\", \"forward\"]).default(\"left\"),\n});\n\nconst ComputerUseDragActionSchema = z.object({\n  type: z.literal(\"drag\"),\n  path: z.array(z.object({ x: z.number(), y: z.number() })),\n});\n\nconst ComputerUseKeypressActionSchema = z.object({\n  type: z.literal(\"keypress\"),\n  keys: z.array(z.string()),\n});\n\nconst ComputerUseMoveActionSchema = z.object({\n  type: z.literal(\"move\"),\n  x: z.number(),\n  y: z.number(),\n});\n\nconst ComputerUseScrollActionSchema = z.object({\n  type: z.literal(\"scroll\"),\n  x: z.number(),\n  y: z.number(),\n  scroll_x: z.number(),\n  scroll_y: z.number(),\n});\n\nconst ComputerUseTypeActionSchema = z.object({\n  type: z.literal(\"type\"),\n  text: z.string(),\n});\n\nconst ComputerUseWaitActionSchema = z.object({\n  type: z.literal(\"wait\"),\n  duration: z.number().optional(),\n});\n\n// Discriminated union schema for individual action types\nconst ComputerUseActionUnionSchema = z.discriminatedUnion(\"type\", [\n  ComputerUseScreenshotActionSchema,\n  ComputerUseClickActionSchema,\n  ComputerUseDoubleClickActionSchema,\n  ComputerUseDragActionSchema,\n  ComputerUseKeypressActionSchema,\n  ComputerUseMoveActionSchema,\n  ComputerUseScrollActionSchema,\n  ComputerUseTypeActionSchema,\n  ComputerUseWaitActionSchema,\n]);\n\n// Schema for the input structure received from parseComputerCall\n// The action is wrapped in an `action` property: { action: { type: 'screenshot' } }\nexport const ComputerUseActionSchema = z.object({\n  action: ComputerUseActionUnionSchema,\n});\n\n// TypeScript types derived from Zod schemas\nexport type ComputerUseScreenshotActionType = z.infer<\n  typeof ComputerUseScreenshotActionSchema\n>;\nexport type ComputerUseClickActionType = z.infer<\n  typeof ComputerUseClickActionSchema\n>;\nexport type ComputerUseDoubleClickActionType = z.infer<\n  typeof ComputerUseDoubleClickActionSchema\n>;\nexport type ComputerUseDragActionType = z.infer<\n  typeof ComputerUseDragActionSchema\n>;\nexport type ComputerUseKeypressActionType = z.infer<\n  typeof ComputerUseKeypressActionSchema\n>;\nexport type ComputerUseMoveActionType = z.infer<\n  typeof ComputerUseMoveActionSchema\n>;\nexport type ComputerUseScrollActionType = z.infer<\n  typeof ComputerUseScrollActionSchema\n>;\nexport type ComputerUseTypeActionType = z.infer<\n  typeof ComputerUseTypeActionSchema\n>;\nexport type ComputerUseWaitActionType = z.infer<\n  typeof ComputerUseWaitActionSchema\n>;\n\n/**\n * Input structure for the Computer Use tool.\n * The action is wrapped in an `action` property.\n */\nexport interface ComputerUseInput {\n  action: ComputerUseAction;\n}\n\nexport type ComputerUseReturnType =\n  | string\n  | Promise<string>\n  | ToolMessage<any>\n  | Promise<ToolMessage<any>>;\n\n/**\n * Options for the Computer Use tool.\n */\nexport interface ComputerUseOptions {\n  /**\n   * The width of the computer display in pixels.\n   */\n  displayWidth: number;\n\n  /**\n   * The height of the computer display in pixels.\n   */\n  displayHeight: number;\n\n  /**\n   * The type of computer environment to control.\n   * - `browser`: Browser automation (recommended for most use cases)\n   * - `mac`: macOS environment\n   * - `windows`: Windows environment\n   * - `linux`: Linux environment\n   * - `ubuntu`: Ubuntu environment\n   */\n  environment: ComputerUseEnvironment;\n\n  /**\n   * Execute function that handles computer action execution.\n   * This function receives the action input and should return a base64-encoded\n   * screenshot of the result.\n   */\n  execute: (\n    action: ComputerUseAction,\n    runtime: ToolRuntime<any, any>\n  ) => ComputerUseReturnType;\n}\n\n/**\n * OpenAI Computer Use tool type for the Responses API.\n */\nexport type ComputerUseTool = OpenAIClient.Responses.ComputerTool;\n\nconst TOOL_NAME = \"computer_use\";\n\n/**\n * Creates a Computer Use tool that allows models to control computer interfaces\n * and perform tasks by simulating mouse clicks, keyboard input, scrolling, and more.\n *\n * **Computer Use** is a practical application of OpenAI's Computer-Using Agent (CUA)\n * model (`computer-use-preview`), which combines vision capabilities with advanced\n * reasoning to simulate controlling computer interfaces.\n *\n * **How it works**:\n * The tool operates in a continuous loop:\n * 1. Model sends computer actions (click, type, scroll, etc.)\n * 2. Your code executes these actions in a controlled environment\n * 3. You capture a screenshot of the result\n * 4. Send the screenshot back to the model\n * 5. Repeat until the task is complete\n *\n * **Important**: Computer use is in beta and requires careful consideration:\n * - Use in sandboxed environments only\n * - Do not use for high-stakes or authenticated tasks\n * - Always implement human-in-the-loop for important decisions\n * - Handle safety checks appropriately\n *\n * @see {@link https://platform.openai.com/docs/guides/tools-computer-use | OpenAI Computer Use Documentation}\n *\n * @param options - Configuration options for the Computer Use tool\n * @returns A Computer Use tool that can be passed to `bindTools`\n *\n * @example\n * ```typescript\n * import { ChatOpenAI, tools } from \"@langchain/openai\";\n *\n * const model = new ChatOpenAI({ model: \"computer-use-preview\" });\n *\n * // With execute callback for automatic action handling\n * const computer = tools.computerUse({\n *   displayWidth: 1024,\n *   displayHeight: 768,\n *   environment: \"browser\",\n *   execute: async (action) => {\n *     if (action.type === \"screenshot\") {\n *       return captureScreenshot();\n *     }\n *     if (action.type === \"click\") {\n *       await page.mouse.click(action.x, action.y, { button: action.button });\n *       return captureScreenshot();\n *     }\n *     if (action.type === \"type\") {\n *       await page.keyboard.type(action.text);\n *       return captureScreenshot();\n *     }\n *     // Handle other actions...\n *     return captureScreenshot();\n *   },\n * });\n *\n * const llmWithComputer = model.bindTools([computer]);\n * const response = await llmWithComputer.invoke(\n *   \"Check the latest news on bing.com\"\n * );\n * ```\n *\n * @example\n * ```typescript\n * // Without execute callback (manual action handling)\n * const computer = tools.computerUse({\n *   displayWidth: 1024,\n *   displayHeight: 768,\n *   environment: \"browser\",\n * });\n *\n * const response = await model.invoke(\"Check the news\", {\n *   tools: [computer],\n * });\n *\n * // Access the computer call from the response\n * const computerCall = response.additional_kwargs.tool_outputs?.find(\n *   (output) => output.type === \"computer_call\"\n * );\n * if (computerCall) {\n *   console.log(\"Action to execute:\", computerCall.action);\n *   // Execute the action manually, then send back a screenshot\n * }\n * ```\n *\n * @example\n * ```typescript\n * // For macOS desktop automation with Docker\n * const computer = tools.computerUse({\n *   displayWidth: 1920,\n *   displayHeight: 1080,\n *   environment: \"mac\",\n *   execute: async (action) => {\n *     if (action.type === \"click\") {\n *       await dockerExec(\n *         `DISPLAY=:99 xdotool mousemove ${action.x} ${action.y} click 1`,\n *         containerName\n *       );\n *     }\n *     // Capture screenshot from container\n *     return await getDockerScreenshot(containerName);\n *   },\n * });\n * ```\n *\n * @remarks\n * - Only available through the Responses API (not Chat Completions)\n * - Requires `computer-use-preview` model\n * - Actions include: click, double_click, drag, keypress, move, screenshot, scroll, type, wait\n * - Safety checks may be returned that require acknowledgment before proceeding\n * - Use `truncation: \"auto\"` parameter when making requests\n * - Recommended to use with `reasoning.summary` for debugging\n */\nexport function computerUse(options: ComputerUseOptions) {\n  const computerTool = tool(\n    async (\n      input: ComputerUseInput,\n      runtime: ToolRuntime<{ messages: BaseMessage[] }>\n    ) => {\n      /**\n       * get computer_use call id from runtime\n       */\n      const aiMessage = runtime.state?.messages.at(-1) as AIMessage | undefined;\n      const computerToolCall = aiMessage?.tool_calls?.find(\n        (tc) => tc.name === \"computer_use\"\n      );\n      const computerToolCallId = computerToolCall?.id;\n      if (!computerToolCallId) {\n        throw new Error(\"Computer use call id not found\");\n      }\n\n      const result = await options.execute(input.action, runtime);\n\n      /**\n       * make sure {@link ToolMessage} is returned with the correct additional kwargs\n       */\n      if (typeof result === \"string\") {\n        return new ToolMessage({\n          content: result,\n          tool_call_id: computerToolCallId,\n          additional_kwargs: {\n            type: \"computer_call_output\",\n          },\n        });\n      }\n\n      /**\n       * make sure {@link ToolMessage} is returned with the correct additional kwargs\n       */\n      return new ToolMessage({\n        ...result,\n        tool_call_id: computerToolCallId,\n        additional_kwargs: {\n          type: \"computer_call_output\",\n          ...result.additional_kwargs,\n        },\n      });\n    },\n    {\n      name: TOOL_NAME,\n      description:\n        \"Control a computer interface by executing mouse clicks, keyboard input, scrolling, and other actions.\",\n      schema: ComputerUseActionSchema,\n    }\n  );\n\n  computerTool.extras = {\n    ...(computerTool.extras ?? {}),\n    providerToolDefinition: {\n      type: \"computer_use_preview\",\n      display_width: options.displayWidth,\n      display_height: options.displayHeight,\n      environment: options.environment,\n    } satisfies ComputerUseTool,\n  };\n\n  /**\n   * return as typed {@link DynamicStructuredTool} so we don't get any type\n   * errors like \"can't export tool without reference\"\n   */\n  return computerTool as DynamicStructuredTool<\n    typeof ComputerUseActionSchema,\n    ComputerUseInput,\n    unknown,\n    ComputerUseReturnType\n  >;\n}\n"],"mappings":";;;;;AAkDA,MAAM,oCAAoC,EAAE,OAAO,EACjD,MAAM,EAAE,QAAQ,aAAa,CAC9B,EAAC;AAEF,MAAM,+BAA+B,EAAE,OAAO;CAC5C,MAAM,EAAE,QAAQ,QAAQ;CACxB,GAAG,EAAE,QAAQ;CACb,GAAG,EAAE,QAAQ;CACb,QAAQ,EAAE,KAAK;EAAC;EAAQ;EAAS;EAAS;EAAQ;CAAU,EAAC,CAAC,QAAQ,OAAO;AAC9E,EAAC;AAEF,MAAM,qCAAqC,EAAE,OAAO;CAClD,MAAM,EAAE,QAAQ,eAAe;CAC/B,GAAG,EAAE,QAAQ;CACb,GAAG,EAAE,QAAQ;CACb,QAAQ,EAAE,KAAK;EAAC;EAAQ;EAAS;EAAS;EAAQ;CAAU,EAAC,CAAC,QAAQ,OAAO;AAC9E,EAAC;AAEF,MAAM,8BAA8B,EAAE,OAAO;CAC3C,MAAM,EAAE,QAAQ,OAAO;CACvB,MAAM,EAAE,MAAM,EAAE,OAAO;EAAE,GAAG,EAAE,QAAQ;EAAE,GAAG,EAAE,QAAQ;CAAE,EAAC,CAAC;AAC1D,EAAC;AAEF,MAAM,kCAAkC,EAAE,OAAO;CAC/C,MAAM,EAAE,QAAQ,WAAW;CAC3B,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC;AAC1B,EAAC;AAEF,MAAM,8BAA8B,EAAE,OAAO;CAC3C,MAAM,EAAE,QAAQ,OAAO;CACvB,GAAG,EAAE,QAAQ;CACb,GAAG,EAAE,QAAQ;AACd,EAAC;AAEF,MAAM,gCAAgC,EAAE,OAAO;CAC7C,MAAM,EAAE,QAAQ,SAAS;CACzB,GAAG,EAAE,QAAQ;CACb,GAAG,EAAE,QAAQ;CACb,UAAU,EAAE,QAAQ;CACpB,UAAU,EAAE,QAAQ;AACrB,EAAC;AAEF,MAAM,8BAA8B,EAAE,OAAO;CAC3C,MAAM,EAAE,QAAQ,OAAO;CACvB,MAAM,EAAE,QAAQ;AACjB,EAAC;AAEF,MAAM,8BAA8B,EAAE,OAAO;CAC3C,MAAM,EAAE,QAAQ,OAAO;CACvB,UAAU,EAAE,QAAQ,CAAC,UAAU;AAChC,EAAC;AAGF,MAAM,+BAA+B,EAAE,mBAAmB,QAAQ;CAChE;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACD,EAAC;AAIF,MAAa,0BAA0B,EAAE,OAAO,EAC9C,QAAQ,6BACT,EAAC;AAqFF,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkHlB,SAAgB,YAAYA,SAA6B;CACvD,MAAM,eAAe,KACnB,OACEC,OACAC,YACG;;;;EAIH,MAAM,YAAY,QAAQ,OAAO,SAAS,GAAG,GAAG;EAChD,MAAM,mBAAmB,WAAW,YAAY,KAC9C,CAAC,OAAO,GAAG,SAAS,eACrB;EACD,MAAM,qBAAqB,kBAAkB;AAC7C,MAAI,CAAC,mBACH,OAAM,IAAI,MAAM;EAGlB,MAAM,SAAS,MAAM,QAAQ,QAAQ,MAAM,QAAQ,QAAQ;;;;AAK3D,MAAI,OAAO,WAAW,SACpB,QAAO,IAAI,YAAY;GACrB,SAAS;GACT,cAAc;GACd,mBAAmB,EACjB,MAAM,uBACP;EACF;;;;AAMH,SAAO,IAAI,YAAY;GACrB,GAAG;GACH,cAAc;GACd,mBAAmB;IACjB,MAAM;IACN,GAAG,OAAO;GACX;EACF;CACF,GACD;EACE,MAAM;EACN,aACE;EACF,QAAQ;CACT,EACF;CAED,aAAa,SAAS;EACpB,GAAI,aAAa,UAAU,CAAE;EAC7B,wBAAwB;GACtB,MAAM;GACN,eAAe,QAAQ;GACvB,gBAAgB,QAAQ;GACxB,aAAa,QAAQ;EACtB;CACF;;;;;AAMD,QAAO;AAMR"}

package/dist/tools/custom.d.cts

@@ -1,6 +1,6 @@
 import OpenAI from "openai";
-import { RunnableFunc } from "@langchain/core/runnables";
 import { DynamicTool, ToolRunnableConfig } from "@langchain/core/tools";
+import { RunnableFunc } from "@langchain/core/runnables";
 
 //#region src/tools/custom.d.ts
 type CustomToolFields = Omit<OpenAI.Responses.CustomTool, "type">;

package/dist/tools/custom.d.cts.map

@@ -0,0 +1 @@
+{"version":3,"file":"custom.d.cts","names":["RunnableFunc","DynamicTool","ToolRunnableConfig","OpenAI","CustomToolFields","Responses","CustomTool","Omit","customTool"],"sources":["../../src/tools/custom.d.ts"],"sourcesContent":["import { RunnableFunc } from \"@langchain/core/runnables\";\nimport { DynamicTool, ToolRunnableConfig } from \"@langchain/core/tools\";\nimport OpenAI from \"openai\";\nexport type CustomToolFields = Omit<OpenAI.Responses.CustomTool, \"type\">;\nexport declare function customTool(func: RunnableFunc<string, string, ToolRunnableConfig>, fields: CustomToolFields): DynamicTool<string>;\n//# sourceMappingURL=custom.d.ts.map"],"mappings":";;;;;KAGYI,gBAAAA,GAAmBG,KAAKJ,MAAAA,CAAOE,SAAAA,CAAUC;iBAC7BE,UAAAA,OAAiBR,6BAA6BE,6BAA6BE,mBAAmBH"}

package/dist/tools/custom.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"custom.d.ts","names":["RunnableFunc","DynamicTool","ToolRunnableConfig","OpenAI","CustomToolFields","Responses","CustomTool","Omit","customTool"],"sources":["../../src/tools/custom.d.ts"],"sourcesContent":["import { RunnableFunc } from \"@langchain/core/runnables\";\nimport { DynamicTool, ToolRunnableConfig } from \"@langchain/core/tools\";\nimport OpenAI from \"openai\";\nexport type CustomToolFields = Omit<OpenAI.Responses.CustomTool, \"type\">;\nexport declare function customTool(func: RunnableFunc<string, string, ToolRunnableConfig>, fields: CustomToolFields): DynamicTool<string>;\n//# sourceMappingURL=custom.d.ts.map"],"mappings":";;;;;KAGYI,gBAAAA,GAAmBG,KAAKJ,MAAAA,CAAOE,SAAAA,CAAUC;iBAC7BE,UAAAA,OAAiBR,6BAA6BE,6BAA6BE,mBAAmBH"}

package/dist/tools/dalle.d.cts

@@ -71,7 +71,6 @@ interface DallEAPIWrapperParams extends ToolParams {
   /**
    * @deprecated Use dallEResponseFormat instead for the Dall-E response type.
    */
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
   responseFormat?: any;
   /**
    * A unique identifier representing your end-user, which will help
@@ -95,7 +94,7 @@ declare class DallEAPIWrapper extends Tool {
   name: string;
   description: string;
   protected client: OpenAI$1;
-  static readonly toolName: string;
+  static readonly toolName = "dalle_api_wrapper";
   private model;
   private style;
   private quality;

package/dist/tools/dalle.d.cts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dalle.d.cts","names":["OpenAI","OpenAIClient","Tool","ToolParams","MessageContentComplex","OpenAIImageModelId","ImageModel","NonNullable","DallEAPIWrapperParams","DallEAPIWrapper","Promise"],"sources":["../../src/tools/dalle.d.ts"],"sourcesContent":["import { OpenAI as OpenAIClient } from \"openai\";\nimport { Tool, ToolParams } from \"@langchain/core/tools\";\nimport { MessageContentComplex } from \"@langchain/core/messages\";\n/**\n * @see https://platform.openai.com/docs/api-reference/images/create\n */\nexport type OpenAIImageModelId = OpenAIClient.ImageModel | (string & NonNullable<unknown>);\n/**\n * An interface for the Dall-E API Wrapper.\n */\nexport interface DallEAPIWrapperParams extends ToolParams {\n    /**\n     * The OpenAI API key\n     * Alias for `apiKey`\n     */\n    openAIApiKey?: string;\n    /**\n     * The OpenAI API key\n     */\n    apiKey?: string;\n    /**\n     * The model to use.\n     * Alias for `model`\n     * @params \"dall-e-2\" | \"dall-e-3\"\n     * @default \"dall-e-3\"\n     * @deprecated Use `model` instead.\n     */\n    modelName?: OpenAIImageModelId;\n    /**\n     * The model to use.\n     * @params \"dall-e-2\" | \"dall-e-3\"\n     * @default \"dall-e-3\"\n     */\n    model?: OpenAIImageModelId;\n    /**\n     * The style of the generated images. Must be one of vivid or natural.\n     * Vivid causes the model to lean towards generating hyper-real and dramatic images.\n     * Natural causes the model to produce more natural, less hyper-real looking images.\n     * @default \"vivid\"\n     */\n    style?: \"natural\" | \"vivid\";\n    /**\n     * The quality of the image that will be generated. ‘hd’ creates images with finer\n     * details and greater consistency across the image.\n     * @default \"standard\"\n     */\n    quality?: \"standard\" | \"hd\";\n    /**\n     * The number of images to generate.\n     * Must be between 1 and 10.\n     * For dall-e-3, only `n: 1` is supported.\n     * @default 1\n     */\n    n?: number;\n    /**\n     * The size of the generated images.\n     * Must be one of 256x256, 512x512, or 1024x1024 for DALL·E-2 models.\n     * Must be one of 1024x1024, 1792x1024, or 1024x1792 for DALL·E-3 models.\n     * @default \"1024x1024\"\n     */\n    size?: \"256x256\" | \"512x512\" | \"1024x1024\" | \"1792x1024\" | \"1024x1792\";\n    /**\n     * The format in which the generated images are returned.\n     * Must be one of \"url\" or \"b64_json\".\n     * @default \"url\"\n     */\n    dallEResponseFormat?: \"url\" | \"b64_json\";\n    /**\n     * @deprecated Use dallEResponseFormat instead for the Dall-E response type.\n     */\n    responseFormat?: any;\n    /**\n     * A unique identifier representing your end-user, which will help\n     * OpenAI to monitor and detect abuse.\n     */\n    user?: string;\n    /**\n     * The organization to use\n     */\n    organization?: string;\n    /**\n     * The base URL of the OpenAI API.\n     */\n    baseUrl?: string;\n}\n/**\n * A tool for generating images with Open AIs Dall-E 2 or 3 API.\n */\nexport declare class DallEAPIWrapper extends Tool {\n    static lc_name(): string;\n    name: string;\n    description: string;\n    protected client: OpenAIClient;\n    static readonly toolName = \"dalle_api_wrapper\";\n    private model;\n    private style;\n    private quality;\n    private n;\n    private size;\n    private dallEResponseFormat;\n    private user?;\n    constructor(fields?: DallEAPIWrapperParams);\n    /**\n     * Processes the API response if multiple images are generated.\n     * Returns a list of MessageContentImageUrl objects. If the response\n     * format is `url`, then the `image_url` field will contain the URL.\n     * If it is `b64_json`, then the `image_url` field will contain an object\n     * with a `url` field with the base64 encoded image.\n     *\n     * @param {OpenAIClient.Images.ImagesResponse[]} response The API response\n     * @returns {MessageContentImageUrl[]}\n     */\n    private processMultipleGeneratedUrls;\n    /** @ignore */\n    _call(input: string): Promise<string | MessageContentComplex[]>;\n}\n//# sourceMappingURL=dalle.d.ts.map"],"mappings":";;;;;;;;AAMA;AAIiBQ,KAJLH,kBAAAA,GAAqBJ,QAAAA,CAAaK,UAIR,GAAA,CAAA,MAAA,GAJ+BC,WAI/B,CAAA,OAAA,CAAA,CAAA;;;;AAAmB,UAAxCC,qBAAAA,SAA8BL,UAAU,CAAA;EA8EpCM;;;;EA0BKC,YAAAA,CAAAA,EAAAA,MAAAA;EA1BmBR;AAAI;;;;;;;;;;cA7DjCG;;;;;;UAMJA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAuDSI,eAAAA,SAAwBP,IAAAA;;;;oBAIvBD;;;;;;;;;uBASGO;;;;;;;;;;;;;wBAaCE,iBAAiBN"}

package/dist/tools/dalle.d.ts

@@ -71,7 +71,6 @@ interface DallEAPIWrapperParams extends ToolParams {
   /**
    * @deprecated Use dallEResponseFormat instead for the Dall-E response type.
    */
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
   responseFormat?: any;
   /**
    * A unique identifier representing your end-user, which will help
@@ -95,7 +94,7 @@ declare class DallEAPIWrapper extends Tool {
   name: string;
   description: string;
   protected client: OpenAI$1;
-  static readonly toolName: string;
+  static readonly toolName = "dalle_api_wrapper";
   private model;
   private style;
   private quality;

package/dist/tools/dalle.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dalle.d.ts","names":["OpenAI","OpenAIClient","Tool","ToolParams","MessageContentComplex","OpenAIImageModelId","ImageModel","NonNullable","DallEAPIWrapperParams","DallEAPIWrapper","Promise"],"sources":["../../src/tools/dalle.d.ts"],"sourcesContent":["import { OpenAI as OpenAIClient } from \"openai\";\nimport { Tool, ToolParams } from \"@langchain/core/tools\";\nimport { MessageContentComplex } from \"@langchain/core/messages\";\n/**\n * @see https://platform.openai.com/docs/api-reference/images/create\n */\nexport type OpenAIImageModelId = OpenAIClient.ImageModel | (string & NonNullable<unknown>);\n/**\n * An interface for the Dall-E API Wrapper.\n */\nexport interface DallEAPIWrapperParams extends ToolParams {\n    /**\n     * The OpenAI API key\n     * Alias for `apiKey`\n     */\n    openAIApiKey?: string;\n    /**\n     * The OpenAI API key\n     */\n    apiKey?: string;\n    /**\n     * The model to use.\n     * Alias for `model`\n     * @params \"dall-e-2\" | \"dall-e-3\"\n     * @default \"dall-e-3\"\n     * @deprecated Use `model` instead.\n     */\n    modelName?: OpenAIImageModelId;\n    /**\n     * The model to use.\n     * @params \"dall-e-2\" | \"dall-e-3\"\n     * @default \"dall-e-3\"\n     */\n    model?: OpenAIImageModelId;\n    /**\n     * The style of the generated images. Must be one of vivid or natural.\n     * Vivid causes the model to lean towards generating hyper-real and dramatic images.\n     * Natural causes the model to produce more natural, less hyper-real looking images.\n     * @default \"vivid\"\n     */\n    style?: \"natural\" | \"vivid\";\n    /**\n     * The quality of the image that will be generated. ‘hd’ creates images with finer\n     * details and greater consistency across the image.\n     * @default \"standard\"\n     */\n    quality?: \"standard\" | \"hd\";\n    /**\n     * The number of images to generate.\n     * Must be between 1 and 10.\n     * For dall-e-3, only `n: 1` is supported.\n     * @default 1\n     */\n    n?: number;\n    /**\n     * The size of the generated images.\n     * Must be one of 256x256, 512x512, or 1024x1024 for DALL·E-2 models.\n     * Must be one of 1024x1024, 1792x1024, or 1024x1792 for DALL·E-3 models.\n     * @default \"1024x1024\"\n     */\n    size?: \"256x256\" | \"512x512\" | \"1024x1024\" | \"1792x1024\" | \"1024x1792\";\n    /**\n     * The format in which the generated images are returned.\n     * Must be one of \"url\" or \"b64_json\".\n     * @default \"url\"\n     */\n    dallEResponseFormat?: \"url\" | \"b64_json\";\n    /**\n     * @deprecated Use dallEResponseFormat instead for the Dall-E response type.\n     */\n    responseFormat?: any;\n    /**\n     * A unique identifier representing your end-user, which will help\n     * OpenAI to monitor and detect abuse.\n     */\n    user?: string;\n    /**\n     * The organization to use\n     */\n    organization?: string;\n    /**\n     * The base URL of the OpenAI API.\n     */\n    baseUrl?: string;\n}\n/**\n * A tool for generating images with Open AIs Dall-E 2 or 3 API.\n */\nexport declare class DallEAPIWrapper extends Tool {\n    static lc_name(): string;\n    name: string;\n    description: string;\n    protected client: OpenAIClient;\n    static readonly toolName = \"dalle_api_wrapper\";\n    private model;\n    private style;\n    private quality;\n    private n;\n    private size;\n    private dallEResponseFormat;\n    private user?;\n    constructor(fields?: DallEAPIWrapperParams);\n    /**\n     * Processes the API response if multiple images are generated.\n     * Returns a list of MessageContentImageUrl objects. If the response\n     * format is `url`, then the `image_url` field will contain the URL.\n     * If it is `b64_json`, then the `image_url` field will contain an object\n     * with a `url` field with the base64 encoded image.\n     *\n     * @param {OpenAIClient.Images.ImagesResponse[]} response The API response\n     * @returns {MessageContentImageUrl[]}\n     */\n    private processMultipleGeneratedUrls;\n    /** @ignore */\n    _call(input: string): Promise<string | MessageContentComplex[]>;\n}\n//# sourceMappingURL=dalle.d.ts.map"],"mappings":";;;;;;;;AAMA;AAIiBQ,KAJLH,kBAAAA,GAAqBJ,QAAAA,CAAaK,UAIR,GAAA,CAAA,MAAA,GAJ+BC,WAI/B,CAAA,OAAA,CAAA,CAAA;;;;AAAmB,UAAxCC,qBAAAA,SAA8BL,UAAU,CAAA;EA8EpCM;;;;EA0BKC,YAAAA,CAAAA,EAAAA,MAAAA;EA1BmBR;AAAI;;;;;;;;;;cA7DjCG;;;;;;UAMJA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAuDSI,eAAAA,SAAwBP,IAAAA;;;;oBAIvBD;;;;;;;;;uBASGO;;;;;;;;;;;;;wBAaCE,iBAAiBN"}