@langchain/openai 1.2.7 → 1.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/azure/chat_models/common.cjs +9 -10
- package/dist/azure/chat_models/common.cjs.map +1 -1
- package/dist/azure/chat_models/common.d.cts +0 -1
- package/dist/azure/chat_models/common.d.cts.map +1 -1
- package/dist/azure/chat_models/common.d.ts +0 -1
- package/dist/azure/chat_models/common.d.ts.map +1 -1
- package/dist/azure/chat_models/common.js +1 -1
- package/dist/azure/chat_models/common.js.map +1 -1
- package/dist/azure/chat_models/completions.cjs.map +1 -1
- package/dist/azure/chat_models/completions.d.cts.map +1 -1
- package/dist/azure/chat_models/completions.d.ts.map +1 -1
- package/dist/azure/chat_models/completions.js.map +1 -1
- package/dist/azure/chat_models/index.cjs.map +1 -1
- package/dist/azure/chat_models/index.d.cts +0 -1
- package/dist/azure/chat_models/index.d.cts.map +1 -1
- package/dist/azure/chat_models/index.d.ts +0 -1
- package/dist/azure/chat_models/index.d.ts.map +1 -1
- package/dist/azure/chat_models/index.js.map +1 -1
- package/dist/azure/chat_models/responses.cjs.map +1 -1
- package/dist/azure/chat_models/responses.d.cts.map +1 -1
- package/dist/azure/chat_models/responses.d.ts.map +1 -1
- package/dist/azure/chat_models/responses.js.map +1 -1
- package/dist/azure/embeddings.cjs +10 -13
- package/dist/azure/embeddings.cjs.map +1 -1
- package/dist/azure/embeddings.d.cts +1 -2
- package/dist/azure/embeddings.d.cts.map +1 -1
- package/dist/azure/embeddings.d.ts +1 -2
- package/dist/azure/embeddings.d.ts.map +1 -1
- package/dist/azure/embeddings.js +3 -5
- package/dist/azure/embeddings.js.map +1 -1
- package/dist/azure/llms.cjs +9 -10
- package/dist/azure/llms.cjs.map +1 -1
- package/dist/azure/llms.d.cts +2 -2
- package/dist/azure/llms.d.cts.map +1 -1
- package/dist/azure/llms.d.ts +2 -2
- package/dist/azure/llms.d.ts.map +1 -1
- package/dist/azure/llms.js +3 -3
- package/dist/azure/llms.js.map +1 -1
- package/dist/chat_models/base.cjs +38 -45
- package/dist/chat_models/base.cjs.map +1 -1
- package/dist/chat_models/base.d.cts.map +1 -1
- package/dist/chat_models/base.d.ts.map +1 -1
- package/dist/chat_models/base.js +10 -16
- package/dist/chat_models/base.js.map +1 -1
- package/dist/chat_models/completions.cjs +9 -13
- package/dist/chat_models/completions.cjs.map +1 -1
- package/dist/chat_models/completions.d.cts.map +1 -1
- package/dist/chat_models/completions.d.ts.map +1 -1
- package/dist/chat_models/completions.js +3 -6
- package/dist/chat_models/completions.js.map +1 -1
- package/dist/chat_models/index.cjs.map +1 -1
- package/dist/chat_models/index.d.cts.map +1 -1
- package/dist/chat_models/index.d.ts.map +1 -1
- package/dist/chat_models/index.js.map +1 -1
- package/dist/chat_models/profiles.cjs +291 -94
- package/dist/chat_models/profiles.cjs.map +1 -1
- package/dist/chat_models/profiles.js +291 -94
- package/dist/chat_models/profiles.js.map +1 -1
- package/dist/chat_models/responses.cjs +4 -7
- package/dist/chat_models/responses.cjs.map +1 -1
- package/dist/chat_models/responses.d.cts.map +1 -1
- package/dist/chat_models/responses.d.ts.map +1 -1
- package/dist/chat_models/responses.js +2 -4
- package/dist/chat_models/responses.js.map +1 -1
- package/dist/converters/completions.cjs +36 -45
- package/dist/converters/completions.cjs.map +1 -1
- package/dist/converters/completions.d.cts +0 -1
- package/dist/converters/completions.d.cts.map +1 -1
- package/dist/converters/completions.d.ts +0 -1
- package/dist/converters/completions.d.ts.map +1 -1
- package/dist/converters/completions.js +14 -22
- package/dist/converters/completions.js.map +1 -1
- package/dist/converters/index.js +2 -0
- package/dist/converters/responses.cjs +39 -54
- package/dist/converters/responses.cjs.map +1 -1
- package/dist/converters/responses.d.cts.map +1 -1
- package/dist/converters/responses.d.ts.map +1 -1
- package/dist/converters/responses.js +31 -45
- package/dist/converters/responses.js.map +1 -1
- package/dist/embeddings.cjs +11 -15
- package/dist/embeddings.cjs.map +1 -1
- package/dist/embeddings.d.cts +1 -3
- package/dist/embeddings.d.cts.map +1 -1
- package/dist/embeddings.d.ts +1 -3
- package/dist/embeddings.d.ts.map +1 -1
- package/dist/embeddings.js +3 -6
- package/dist/embeddings.js.map +1 -1
- package/dist/index.cjs +2 -2
- package/dist/llms.cjs +27 -33
- package/dist/llms.cjs.map +1 -1
- package/dist/llms.d.cts +0 -1
- package/dist/llms.d.cts.map +1 -1
- package/dist/llms.d.ts +0 -1
- package/dist/llms.d.ts.map +1 -1
- package/dist/llms.js +15 -20
- package/dist/llms.js.map +1 -1
- package/dist/tools/applyPatch.cjs +3 -4
- package/dist/tools/applyPatch.cjs.map +1 -1
- package/dist/tools/applyPatch.d.cts +0 -1
- package/dist/tools/applyPatch.d.cts.map +1 -1
- package/dist/tools/applyPatch.d.ts +0 -1
- package/dist/tools/applyPatch.d.ts.map +1 -1
- package/dist/tools/applyPatch.js.map +1 -1
- package/dist/tools/codeInterpreter.cjs.map +1 -1
- package/dist/tools/codeInterpreter.d.cts +0 -1
- package/dist/tools/codeInterpreter.d.cts.map +1 -1
- package/dist/tools/codeInterpreter.d.ts +0 -1
- package/dist/tools/codeInterpreter.d.ts.map +1 -1
- package/dist/tools/codeInterpreter.js.map +1 -1
- package/dist/tools/computerUse.cjs +7 -13
- package/dist/tools/computerUse.cjs.map +1 -1
- package/dist/tools/computerUse.d.cts +0 -1
- package/dist/tools/computerUse.d.cts.map +1 -1
- package/dist/tools/computerUse.d.ts +0 -1
- package/dist/tools/computerUse.d.ts.map +1 -1
- package/dist/tools/computerUse.js +1 -6
- package/dist/tools/computerUse.js.map +1 -1
- package/dist/tools/custom.cjs +6 -7
- package/dist/tools/custom.cjs.map +1 -1
- package/dist/tools/custom.d.cts.map +1 -1
- package/dist/tools/custom.d.ts.map +1 -1
- package/dist/tools/custom.js.map +1 -1
- package/dist/tools/dalle.cjs +10 -16
- package/dist/tools/dalle.cjs.map +1 -1
- package/dist/tools/dalle.d.cts +0 -1
- package/dist/tools/dalle.d.cts.map +1 -1
- package/dist/tools/dalle.d.ts +0 -1
- package/dist/tools/dalle.d.ts.map +1 -1
- package/dist/tools/dalle.js +6 -11
- package/dist/tools/dalle.js.map +1 -1
- package/dist/tools/fileSearch.cjs.map +1 -1
- package/dist/tools/fileSearch.d.cts +0 -1
- package/dist/tools/fileSearch.d.cts.map +1 -1
- package/dist/tools/fileSearch.d.ts +0 -1
- package/dist/tools/fileSearch.d.ts.map +1 -1
- package/dist/tools/fileSearch.js.map +1 -1
- package/dist/tools/imageGeneration.cjs.map +1 -1
- package/dist/tools/imageGeneration.d.cts +0 -1
- package/dist/tools/imageGeneration.d.cts.map +1 -1
- package/dist/tools/imageGeneration.d.ts +0 -1
- package/dist/tools/imageGeneration.d.ts.map +1 -1
- package/dist/tools/imageGeneration.js.map +1 -1
- package/dist/tools/index.cjs.map +1 -1
- package/dist/tools/index.d.cts +0 -1
- package/dist/tools/index.d.cts.map +1 -1
- package/dist/tools/index.d.ts +0 -1
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/localShell.cjs +3 -4
- package/dist/tools/localShell.cjs.map +1 -1
- package/dist/tools/localShell.d.cts +0 -1
- package/dist/tools/localShell.d.cts.map +1 -1
- package/dist/tools/localShell.d.ts +0 -1
- package/dist/tools/localShell.d.ts.map +1 -1
- package/dist/tools/localShell.js.map +1 -1
- package/dist/tools/mcp.cjs.map +1 -1
- package/dist/tools/mcp.d.cts +0 -1
- package/dist/tools/mcp.d.cts.map +1 -1
- package/dist/tools/mcp.d.ts +0 -1
- package/dist/tools/mcp.d.ts.map +1 -1
- package/dist/tools/mcp.js.map +1 -1
- package/dist/tools/shell.cjs +3 -4
- package/dist/tools/shell.cjs.map +1 -1
- package/dist/tools/shell.d.cts +0 -1
- package/dist/tools/shell.d.cts.map +1 -1
- package/dist/tools/shell.d.ts +0 -1
- package/dist/tools/shell.d.ts.map +1 -1
- package/dist/tools/shell.js.map +1 -1
- package/dist/tools/webSearch.cjs.map +1 -1
- package/dist/tools/webSearch.d.cts +0 -1
- package/dist/tools/webSearch.d.cts.map +1 -1
- package/dist/tools/webSearch.d.ts +0 -1
- package/dist/tools/webSearch.d.ts.map +1 -1
- package/dist/tools/webSearch.js.map +1 -1
- package/dist/types.d.cts +0 -1
- package/dist/types.d.cts.map +1 -1
- package/dist/types.d.ts +0 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/utils/azure.cjs +2 -3
- package/dist/utils/azure.cjs.map +1 -1
- package/dist/utils/azure.d.cts.map +1 -1
- package/dist/utils/azure.d.ts.map +1 -1
- package/dist/utils/azure.js.map +1 -1
- package/dist/utils/client.cjs +9 -3
- package/dist/utils/client.cjs.map +1 -1
- package/dist/utils/client.d.cts.map +1 -1
- package/dist/utils/client.d.ts.map +1 -1
- package/dist/utils/client.js +8 -1
- package/dist/utils/client.js.map +1 -1
- package/dist/utils/errors.cjs.map +1 -1
- package/dist/utils/errors.js.map +1 -1
- package/dist/utils/misc.cjs +2 -3
- package/dist/utils/misc.cjs.map +1 -1
- package/dist/utils/misc.d.cts +0 -1
- package/dist/utils/misc.d.cts.map +1 -1
- package/dist/utils/misc.d.ts +0 -1
- package/dist/utils/misc.d.ts.map +1 -1
- package/dist/utils/misc.js.map +1 -1
- package/dist/utils/output.cjs +7 -8
- package/dist/utils/output.cjs.map +1 -1
- package/dist/utils/output.js.map +1 -1
- package/dist/utils/prompts.cjs +1 -2
- package/dist/utils/prompts.cjs.map +1 -1
- package/dist/utils/prompts.d.cts +0 -1
- package/dist/utils/prompts.d.cts.map +1 -1
- package/dist/utils/prompts.d.ts +0 -1
- package/dist/utils/prompts.d.ts.map +1 -1
- package/dist/utils/prompts.js +1 -2
- package/dist/utils/prompts.js.map +1 -1
- package/dist/utils/tools.cjs +9 -12
- package/dist/utils/tools.cjs.map +1 -1
- package/dist/utils/tools.d.cts +0 -1
- package/dist/utils/tools.d.cts.map +1 -1
- package/dist/utils/tools.d.ts +0 -1
- package/dist/utils/tools.d.ts.map +1 -1
- package/dist/utils/tools.js +5 -7
- package/dist/utils/tools.js.map +1 -1
- package/package.json +4 -4
- package/dist/_virtual/rolldown_runtime.cjs +0 -25
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"applyPatch.cjs","names":["z"
|
|
1
|
+
{"version":3,"file":"applyPatch.cjs","names":["z"],"sources":["../../src/tools/applyPatch.ts"],"sourcesContent":["import { z } from \"zod/v4\";\nimport { OpenAI as OpenAIClient } from \"openai\";\nimport { tool, type DynamicStructuredTool } from \"@langchain/core/tools\";\n\n/**\n * Re-export operation types from OpenAI SDK for convenience.\n */\nexport type ApplyPatchCreateFileOperation =\n OpenAIClient.Responses.ResponseApplyPatchToolCall.CreateFile;\nexport type ApplyPatchUpdateFileOperation =\n OpenAIClient.Responses.ResponseApplyPatchToolCall.UpdateFile;\nexport type ApplyPatchDeleteFileOperation =\n OpenAIClient.Responses.ResponseApplyPatchToolCall.DeleteFile;\n\n/**\n * Union type of all apply patch operations from OpenAI SDK.\n */\nexport type ApplyPatchOperation = NonNullable<\n OpenAIClient.Responses.ResponseApplyPatchToolCall[\"operation\"]\n>;\n\n// Zod schemas for apply patch operations\nexport const ApplyPatchCreateFileOperationSchema = z.object({\n type: z.literal(\"create_file\"),\n path: z.string(),\n diff: z.string(),\n});\n\nexport const ApplyPatchUpdateFileOperationSchema = z.object({\n type: z.literal(\"update_file\"),\n path: z.string(),\n diff: z.string(),\n});\n\nexport const ApplyPatchDeleteFileOperationSchema = z.object({\n type: z.literal(\"delete_file\"),\n path: z.string(),\n});\n\n// Union schema for all apply patch operations\nexport const ApplyPatchOperationSchema = z.union([\n ApplyPatchCreateFileOperationSchema,\n ApplyPatchUpdateFileOperationSchema,\n ApplyPatchDeleteFileOperationSchema,\n]);\n\n/**\n * Options for the Apply Patch tool.\n */\nexport interface ApplyPatchOptions {\n /**\n * Execute function that handles patch operations.\n * This function receives the operation input and should return a string\n * describing the result (success or failure message).\n *\n * The operation types are:\n * - `create_file`: Create a new file at the specified path with the diff content\n * - `update_file`: Modify an existing file using V4A diff format\n * - `delete_file`: Remove a file at the specified path\n *\n * @example\n * ```typescript\n * execute: async (operation) => {\n * if (operation.type === \"create_file\") {\n * const content = applyDiff(\"\", operation.diff, \"create\");\n * await fs.writeFile(operation.path, content);\n * return `Created ${operation.path}`;\n * }\n * if (operation.type === \"update_file\") {\n * const current = await fs.readFile(operation.path, \"utf-8\");\n * const newContent = applyDiff(current, operation.diff);\n * await fs.writeFile(operation.path, newContent);\n * return `Updated ${operation.path}`;\n * }\n * if (operation.type === \"delete_file\") {\n * await fs.unlink(operation.path);\n * return `Deleted ${operation.path}`;\n * }\n * return \"Unknown operation type\";\n * }\n * ```\n */\n execute: (operation: ApplyPatchOperation) => string | Promise<string>;\n}\n\n/**\n * OpenAI Apply Patch tool type for the Responses API.\n */\nexport type ApplyPatchTool = OpenAIClient.Responses.ApplyPatchTool;\n\nconst TOOL_NAME = \"apply_patch\";\n\n/**\n * Creates an Apply Patch tool that allows models to propose structured diffs\n * that your integration applies. This enables iterative, multi-step code\n * editing workflows.\n *\n * **Apply Patch** lets GPT-5.1 create, update, and delete files in your codebase\n * using structured diffs. Instead of just suggesting edits, the model emits\n * patch operations that your application applies and then reports back on.\n *\n * **When to use**:\n * - **Multi-file refactors** – Rename symbols, extract helpers, or reorganize modules\n * - **Bug fixes** – Have the model both diagnose issues and emit precise patches\n * - **Tests & docs generation** – Create new test files, fixtures, and documentation\n * - **Migrations & mechanical edits** – Apply repetitive, structured updates\n *\n * **How it works**:\n * The tool operates in a continuous loop:\n * 1. Model sends patch operations (`apply_patch_call` with operation type)\n * 2. Your code applies the patch to your working directory or repo\n * 3. You return success/failure status and optional output\n * 4. Repeat until the task is complete\n *\n * **Security Warning**: Applying patches can modify files in your codebase.\n * Always validate paths, implement backups, and consider sandboxing.\n *\n * @see {@link https://platform.openai.com/docs/guides/tools-apply-patch | OpenAI Apply Patch Documentation}\n *\n * @param options - Configuration options for the Apply Patch tool\n * @returns An Apply Patch tool that can be passed to `bindTools`\n *\n * @example\n * ```typescript\n * import { ChatOpenAI, tools } from \"@langchain/openai\";\n * import { applyDiff } from \"@openai/agents\";\n * import * as fs from \"fs/promises\";\n *\n * const model = new ChatOpenAI({ model: \"gpt-5.1\" });\n *\n * // With execute callback for automatic patch handling\n * const patchTool = tools.applyPatch({\n * execute: async (operation) => {\n * if (operation.type === \"create_file\") {\n * const content = applyDiff(\"\", operation.diff, \"create\");\n * await fs.writeFile(operation.path, content);\n * return `Created ${operation.path}`;\n * }\n * if (operation.type === \"update_file\") {\n * const current = await fs.readFile(operation.path, \"utf-8\");\n * const newContent = applyDiff(current, operation.diff);\n * await fs.writeFile(operation.path, newContent);\n * return `Updated ${operation.path}`;\n * }\n * if (operation.type === \"delete_file\") {\n * await fs.unlink(operation.path);\n * return `Deleted ${operation.path}`;\n * }\n * return \"Unknown operation type\";\n * },\n * });\n *\n * const llmWithPatch = model.bindTools([patchTool]);\n * const response = await llmWithPatch.invoke(\n * \"Rename the fib() function to fibonacci() in lib/fib.py\"\n * );\n * ```\n *\n * @remarks\n * - Only available through the Responses API (not Chat Completions)\n * - Designed for use with `gpt-5.1` model\n * - Operations include: `create_file`, `update_file`, `delete_file`\n * - Patches use V4A diff format for updates\n * - Always validate paths to prevent directory traversal attacks\n * - Consider backing up files before applying patches\n * - Implement \"all-or-nothing\" semantics if atomicity is required\n */\nexport function applyPatch(options: ApplyPatchOptions) {\n const patchTool = tool(options.execute, {\n name: TOOL_NAME,\n description:\n \"Apply structured diffs to create, update, or delete files in the codebase.\",\n schema: ApplyPatchOperationSchema,\n });\n\n patchTool.extras = {\n ...(patchTool.extras ?? {}),\n providerToolDefinition: {\n type: \"apply_patch\",\n } satisfies ApplyPatchTool,\n };\n\n return patchTool as DynamicStructuredTool<\n typeof ApplyPatchOperationSchema,\n ApplyPatchOperation,\n unknown,\n string\n >;\n}\n"],"mappings":";;;;AAsBA,MAAa,sCAAsCA,SAAE,OAAO;CAC1D,MAAMA,SAAE,QAAQ,cAAc;CAC9B,MAAMA,SAAE,QAAQ;CAChB,MAAMA,SAAE,QAAQ;CACjB,CAAC;AAEF,MAAa,sCAAsCA,SAAE,OAAO;CAC1D,MAAMA,SAAE,QAAQ,cAAc;CAC9B,MAAMA,SAAE,QAAQ;CAChB,MAAMA,SAAE,QAAQ;CACjB,CAAC;AAEF,MAAa,sCAAsCA,SAAE,OAAO;CAC1D,MAAMA,SAAE,QAAQ,cAAc;CAC9B,MAAMA,SAAE,QAAQ;CACjB,CAAC;AAGF,MAAa,4BAA4BA,SAAE,MAAM;CAC/C;CACA;CACA;CACD,CAAC;AA8CF,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6ElB,SAAgB,WAAW,SAA4B;CACrD,MAAM,4CAAiB,QAAQ,SAAS;EACtC,MAAM;EACN,aACE;EACF,QAAQ;EACT,CAAC;AAEF,WAAU,SAAS;EACjB,GAAI,UAAU,UAAU,EAAE;EAC1B,wBAAwB,EACtB,MAAM,eACP;EACF;AAED,QAAO"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"applyPatch.d.cts","names":[
|
|
1
|
+
{"version":3,"file":"applyPatch.d.cts","names":[],"sources":["../../src/tools/applyPatch.ts"],"mappings":";;;;;;;AAOA;KAAY,6BAAA,GACV,QAAA,CAAa,SAAA,CAAU,0BAAA,CAA2B,UAAA;AAAA,KACxC,6BAAA,GACV,QAAA,CAAa,SAAA,CAAU,0BAAA,CAA2B,UAAA;AAAA,KACxC,6BAAA,GACV,QAAA,CAAa,SAAA,CAAU,0BAAA,CAA2B,UAAA;;;;KAKxC,mBAAA,GAAsB,WAAA,CAChC,QAAA,CAAa,SAAA,CAAU,0BAAA;;;;UA+BR,iBAAA;EA/BkC;;AA+BnD;;;;;;;;;;AAuCA;;;;;;;;;AA+EA;;;;;;;;;;;EArFE,OAAA,GAAU,SAAA,EAAW,mBAAA,cAAiC,OAAA;AAAA;;;;KAM5C,cAAA,GAAiB,QAAA,CAAa,SAAA,CAAU,cAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA+EpC,UAAA,CAAW,OAAA,EAAS,iBAAA,GAAiB,qBAAA,CAAA,CAAA,CAAA,QAAA,WAAA,CAAA,CAAA,SAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"applyPatch.d.ts","names":[
|
|
1
|
+
{"version":3,"file":"applyPatch.d.ts","names":[],"sources":["../../src/tools/applyPatch.ts"],"mappings":";;;;;;;AAOA;KAAY,6BAAA,GACV,QAAA,CAAa,SAAA,CAAU,0BAAA,CAA2B,UAAA;AAAA,KACxC,6BAAA,GACV,QAAA,CAAa,SAAA,CAAU,0BAAA,CAA2B,UAAA;AAAA,KACxC,6BAAA,GACV,QAAA,CAAa,SAAA,CAAU,0BAAA,CAA2B,UAAA;;;;KAKxC,mBAAA,GAAsB,WAAA,CAChC,QAAA,CAAa,SAAA,CAAU,0BAAA;;;;UA+BR,iBAAA;EA/BkC;;AA+BnD;;;;;;;;;;AAuCA;;;;;;;;;AA+EA;;;;;;;;;;;EArFE,OAAA,GAAU,SAAA,EAAW,mBAAA,cAAiC,OAAA;AAAA;;;;KAM5C,cAAA,GAAiB,QAAA,CAAa,SAAA,CAAU,cAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA+EpC,UAAA,CAAW,OAAA,EAAS,iBAAA,GAAiB,qBAAA,CAAA,CAAA,CAAA,QAAA,WAAA,CAAA,CAAA,SAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"applyPatch.js","names":[
|
|
1
|
+
{"version":3,"file":"applyPatch.js","names":[],"sources":["../../src/tools/applyPatch.ts"],"sourcesContent":["import { z } from \"zod/v4\";\nimport { OpenAI as OpenAIClient } from \"openai\";\nimport { tool, type DynamicStructuredTool } from \"@langchain/core/tools\";\n\n/**\n * Re-export operation types from OpenAI SDK for convenience.\n */\nexport type ApplyPatchCreateFileOperation =\n OpenAIClient.Responses.ResponseApplyPatchToolCall.CreateFile;\nexport type ApplyPatchUpdateFileOperation =\n OpenAIClient.Responses.ResponseApplyPatchToolCall.UpdateFile;\nexport type ApplyPatchDeleteFileOperation =\n OpenAIClient.Responses.ResponseApplyPatchToolCall.DeleteFile;\n\n/**\n * Union type of all apply patch operations from OpenAI SDK.\n */\nexport type ApplyPatchOperation = NonNullable<\n OpenAIClient.Responses.ResponseApplyPatchToolCall[\"operation\"]\n>;\n\n// Zod schemas for apply patch operations\nexport const ApplyPatchCreateFileOperationSchema = z.object({\n type: z.literal(\"create_file\"),\n path: z.string(),\n diff: z.string(),\n});\n\nexport const ApplyPatchUpdateFileOperationSchema = z.object({\n type: z.literal(\"update_file\"),\n path: z.string(),\n diff: z.string(),\n});\n\nexport const ApplyPatchDeleteFileOperationSchema = z.object({\n type: z.literal(\"delete_file\"),\n path: z.string(),\n});\n\n// Union schema for all apply patch operations\nexport const ApplyPatchOperationSchema = z.union([\n ApplyPatchCreateFileOperationSchema,\n ApplyPatchUpdateFileOperationSchema,\n ApplyPatchDeleteFileOperationSchema,\n]);\n\n/**\n * Options for the Apply Patch tool.\n */\nexport interface ApplyPatchOptions {\n /**\n * Execute function that handles patch operations.\n * This function receives the operation input and should return a string\n * describing the result (success or failure message).\n *\n * The operation types are:\n * - `create_file`: Create a new file at the specified path with the diff content\n * - `update_file`: Modify an existing file using V4A diff format\n * - `delete_file`: Remove a file at the specified path\n *\n * @example\n * ```typescript\n * execute: async (operation) => {\n * if (operation.type === \"create_file\") {\n * const content = applyDiff(\"\", operation.diff, \"create\");\n * await fs.writeFile(operation.path, content);\n * return `Created ${operation.path}`;\n * }\n * if (operation.type === \"update_file\") {\n * const current = await fs.readFile(operation.path, \"utf-8\");\n * const newContent = applyDiff(current, operation.diff);\n * await fs.writeFile(operation.path, newContent);\n * return `Updated ${operation.path}`;\n * }\n * if (operation.type === \"delete_file\") {\n * await fs.unlink(operation.path);\n * return `Deleted ${operation.path}`;\n * }\n * return \"Unknown operation type\";\n * }\n * ```\n */\n execute: (operation: ApplyPatchOperation) => string | Promise<string>;\n}\n\n/**\n * OpenAI Apply Patch tool type for the Responses API.\n */\nexport type ApplyPatchTool = OpenAIClient.Responses.ApplyPatchTool;\n\nconst TOOL_NAME = \"apply_patch\";\n\n/**\n * Creates an Apply Patch tool that allows models to propose structured diffs\n * that your integration applies. This enables iterative, multi-step code\n * editing workflows.\n *\n * **Apply Patch** lets GPT-5.1 create, update, and delete files in your codebase\n * using structured diffs. Instead of just suggesting edits, the model emits\n * patch operations that your application applies and then reports back on.\n *\n * **When to use**:\n * - **Multi-file refactors** – Rename symbols, extract helpers, or reorganize modules\n * - **Bug fixes** – Have the model both diagnose issues and emit precise patches\n * - **Tests & docs generation** – Create new test files, fixtures, and documentation\n * - **Migrations & mechanical edits** – Apply repetitive, structured updates\n *\n * **How it works**:\n * The tool operates in a continuous loop:\n * 1. Model sends patch operations (`apply_patch_call` with operation type)\n * 2. Your code applies the patch to your working directory or repo\n * 3. You return success/failure status and optional output\n * 4. Repeat until the task is complete\n *\n * **Security Warning**: Applying patches can modify files in your codebase.\n * Always validate paths, implement backups, and consider sandboxing.\n *\n * @see {@link https://platform.openai.com/docs/guides/tools-apply-patch | OpenAI Apply Patch Documentation}\n *\n * @param options - Configuration options for the Apply Patch tool\n * @returns An Apply Patch tool that can be passed to `bindTools`\n *\n * @example\n * ```typescript\n * import { ChatOpenAI, tools } from \"@langchain/openai\";\n * import { applyDiff } from \"@openai/agents\";\n * import * as fs from \"fs/promises\";\n *\n * const model = new ChatOpenAI({ model: \"gpt-5.1\" });\n *\n * // With execute callback for automatic patch handling\n * const patchTool = tools.applyPatch({\n * execute: async (operation) => {\n * if (operation.type === \"create_file\") {\n * const content = applyDiff(\"\", operation.diff, \"create\");\n * await fs.writeFile(operation.path, content);\n * return `Created ${operation.path}`;\n * }\n * if (operation.type === \"update_file\") {\n * const current = await fs.readFile(operation.path, \"utf-8\");\n * const newContent = applyDiff(current, operation.diff);\n * await fs.writeFile(operation.path, newContent);\n * return `Updated ${operation.path}`;\n * }\n * if (operation.type === \"delete_file\") {\n * await fs.unlink(operation.path);\n * return `Deleted ${operation.path}`;\n * }\n * return \"Unknown operation type\";\n * },\n * });\n *\n * const llmWithPatch = model.bindTools([patchTool]);\n * const response = await llmWithPatch.invoke(\n * \"Rename the fib() function to fibonacci() in lib/fib.py\"\n * );\n * ```\n *\n * @remarks\n * - Only available through the Responses API (not Chat Completions)\n * - Designed for use with `gpt-5.1` model\n * - Operations include: `create_file`, `update_file`, `delete_file`\n * - Patches use V4A diff format for updates\n * - Always validate paths to prevent directory traversal attacks\n * - Consider backing up files before applying patches\n * - Implement \"all-or-nothing\" semantics if atomicity is required\n */\nexport function applyPatch(options: ApplyPatchOptions) {\n const patchTool = tool(options.execute, {\n name: TOOL_NAME,\n description:\n \"Apply structured diffs to create, update, or delete files in the codebase.\",\n schema: ApplyPatchOperationSchema,\n });\n\n patchTool.extras = {\n ...(patchTool.extras ?? {}),\n providerToolDefinition: {\n type: \"apply_patch\",\n } satisfies ApplyPatchTool,\n };\n\n return patchTool as DynamicStructuredTool<\n typeof ApplyPatchOperationSchema,\n ApplyPatchOperation,\n unknown,\n string\n >;\n}\n"],"mappings":";;;;AAsBA,MAAa,sCAAsC,EAAE,OAAO;CAC1D,MAAM,EAAE,QAAQ,cAAc;CAC9B,MAAM,EAAE,QAAQ;CAChB,MAAM,EAAE,QAAQ;CACjB,CAAC;AAEF,MAAa,sCAAsC,EAAE,OAAO;CAC1D,MAAM,EAAE,QAAQ,cAAc;CAC9B,MAAM,EAAE,QAAQ;CAChB,MAAM,EAAE,QAAQ;CACjB,CAAC;AAEF,MAAa,sCAAsC,EAAE,OAAO;CAC1D,MAAM,EAAE,QAAQ,cAAc;CAC9B,MAAM,EAAE,QAAQ;CACjB,CAAC;AAGF,MAAa,4BAA4B,EAAE,MAAM;CAC/C;CACA;CACA;CACD,CAAC;AA8CF,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6ElB,SAAgB,WAAW,SAA4B;CACrD,MAAM,YAAY,KAAK,QAAQ,SAAS;EACtC,MAAM;EACN,aACE;EACF,QAAQ;EACT,CAAC;AAEF,WAAU,SAAS;EACjB,GAAI,UAAU,UAAU,EAAE;EAC1B,wBAAwB,EACtB,MAAM,eACP;EACF;AAED,QAAO"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"codeInterpreter.cjs","names":[
|
|
1
|
+
{"version":3,"file":"codeInterpreter.cjs","names":[],"sources":["../../src/tools/codeInterpreter.ts"],"sourcesContent":["import { OpenAI as OpenAIClient } from \"openai\";\nimport type { ServerTool } from \"@langchain/core/tools\";\n\n/**\n * Memory limit options for the Code Interpreter container.\n * Higher tiers offer more RAM and are billed at different rates.\n */\nexport type CodeInterpreterMemoryLimit = \"1g\" | \"4g\" | \"16g\" | \"64g\";\n\n/**\n * Auto container configuration for Code Interpreter.\n * Creates a new container automatically or reuses an active one.\n */\nexport interface CodeInterpreterAutoContainer {\n /**\n * Memory limit for the container.\n * - `\"1g\"` (default): 1 GB RAM\n * - `\"4g\"`: 4 GB RAM\n * - `\"16g\"`: 16 GB RAM\n * - `\"64g\"`: 64 GB RAM\n */\n memoryLimit?: CodeInterpreterMemoryLimit;\n /**\n * Optional list of uploaded file IDs to make available to the code.\n * Files in the model input are automatically uploaded, so this is only\n * needed for additional files.\n */\n fileIds?: string[];\n}\n\n/**\n * Options for the Code Interpreter tool.\n */\nexport interface CodeInterpreterOptions {\n /**\n * The container configuration for the Code Interpreter.\n *\n * Can be either:\n * - A string container ID for explicit mode (created via `/v1/containers` endpoint)\n * - An auto configuration object that creates/reuses containers automatically\n *\n * If not provided, defaults to auto mode with default settings.\n */\n container?: string | CodeInterpreterAutoContainer;\n}\n\n/**\n * OpenAI Code Interpreter tool type for the Responses API.\n */\nexport type CodeInterpreterTool = OpenAIClient.Responses.Tool.CodeInterpreter;\n\n/**\n * Converts container options to the API format.\n */\nfunction convertContainer(\n container: string | CodeInterpreterAutoContainer | undefined\n):\n | string\n | OpenAIClient.Responses.Tool.CodeInterpreter.CodeInterpreterToolAuto {\n // If a string container ID is provided, use it directly\n if (typeof container === \"string\") {\n return container;\n }\n\n // Auto mode configuration\n return {\n type: \"auto\",\n file_ids: container?.fileIds,\n memory_limit: container?.memoryLimit,\n };\n}\n\n/**\n * Creates a Code Interpreter tool that allows models to write and run Python code\n * in a sandboxed environment to solve complex problems.\n *\n * Use Code Interpreter for:\n * - **Data analysis**: Processing files with diverse data and formatting\n * - **File generation**: Creating files with data and images of graphs\n * - **Iterative coding**: Writing and running code iteratively to solve problems\n * - **Visual intelligence**: Cropping, zooming, rotating, and transforming images\n *\n * The tool runs in a container, which is a fully sandboxed virtual machine.\n * Containers can be created automatically (auto mode) or explicitly via the\n * `/v1/containers` endpoint.\n *\n * @see {@link https://platform.openai.com/docs/guides/tools-code-interpreter | OpenAI Code Interpreter Documentation}\n *\n * @param options - Configuration options for the Code Interpreter tool\n * @returns A Code Interpreter tool definition to be passed to the OpenAI Responses API\n *\n * @example\n * ```typescript\n * import { ChatOpenAI, tools } from \"@langchain/openai\";\n *\n * const model = new ChatOpenAI({ model: \"gpt-4.1\" });\n *\n * // Basic usage with auto container (default 1GB memory)\n * const response = await model.invoke(\n * \"Solve the equation 3x + 11 = 14\",\n * { tools: [tools.codeInterpreter()] }\n * );\n *\n * // With increased memory limit for larger computations\n * const response = await model.invoke(\n * \"Analyze this large dataset and create visualizations\",\n * {\n * tools: [tools.codeInterpreter({\n * container: { memoryLimit: \"4g\" }\n * })]\n * }\n * );\n *\n * // With specific files available to the code\n * const response = await model.invoke(\n * \"Process the uploaded CSV file\",\n * {\n * tools: [tools.codeInterpreter({\n * container: {\n * memoryLimit: \"4g\",\n * fileIds: [\"file-abc123\", \"file-def456\"]\n * }\n * })]\n * }\n * );\n *\n * // Using an explicit container ID (created via /v1/containers)\n * const response = await model.invoke(\n * \"Continue working with the data\",\n * {\n * tools: [tools.codeInterpreter({\n * container: \"cntr_abc123\"\n * })]\n * }\n * );\n * ```\n *\n * @remarks\n * - Containers expire after 20 minutes of inactivity\n * - While called \"Code Interpreter\", the model knows it as the \"python tool\"\n * - For explicit prompting, ask for \"the python tool\" in your prompts\n * - Files in model input are automatically uploaded to the container\n * - Generated files are returned as `container_file_citation` annotations\n */\nexport function codeInterpreter(options?: CodeInterpreterOptions): ServerTool {\n return {\n type: \"code_interpreter\",\n container: convertContainer(options?.container),\n } satisfies CodeInterpreterTool;\n}\n"],"mappings":";;;;;AAsDA,SAAS,iBACP,WAGsE;AAEtE,KAAI,OAAO,cAAc,SACvB,QAAO;AAIT,QAAO;EACL,MAAM;EACN,UAAU,WAAW;EACrB,cAAc,WAAW;EAC1B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2EH,SAAgB,gBAAgB,SAA8C;AAC5E,QAAO;EACL,MAAM;EACN,WAAW,iBAAiB,SAAS,UAAU;EAChD"}
|
|
@@ -2,7 +2,6 @@ import { OpenAI as OpenAI$1 } from "openai";
|
|
|
2
2
|
import { ServerTool } from "@langchain/core/tools";
|
|
3
3
|
|
|
4
4
|
//#region src/tools/codeInterpreter.d.ts
|
|
5
|
-
|
|
6
5
|
/**
|
|
7
6
|
* Memory limit options for the Code Interpreter container.
|
|
8
7
|
* Higher tiers offer more RAM and are billed at different rates.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"codeInterpreter.d.cts","names":[
|
|
1
|
+
{"version":3,"file":"codeInterpreter.d.cts","names":[],"sources":["../../src/tools/codeInterpreter.ts"],"mappings":";;;;;;AAOA;;KAAY,0BAAA;;;AAMZ;;UAAiB,4BAAA;EAQyB;;;;;;AAY1C;EAZE,WAAA,GAAc,0BAAA;;;;AA4BhB;;EAtBE,OAAA;AAAA;;;;UAMe,sBAAA;EAgB4D;;AA+F7E;;;;;;;EArGE,SAAA,YAAqB,4BAAA;AAAA;;;;KAMX,mBAAA,GAAsB,QAAA,CAAa,SAAA,CAAU,IAAA,CAAK,eAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA+F9C,eAAA,CAAgB,OAAA,GAAU,sBAAA,GAAyB,UAAA"}
|
|
@@ -2,7 +2,6 @@ import { OpenAI as OpenAI$1 } from "openai";
|
|
|
2
2
|
import { ServerTool } from "@langchain/core/tools";
|
|
3
3
|
|
|
4
4
|
//#region src/tools/codeInterpreter.d.ts
|
|
5
|
-
|
|
6
5
|
/**
|
|
7
6
|
* Memory limit options for the Code Interpreter container.
|
|
8
7
|
* Higher tiers offer more RAM and are billed at different rates.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"codeInterpreter.d.ts","names":[
|
|
1
|
+
{"version":3,"file":"codeInterpreter.d.ts","names":[],"sources":["../../src/tools/codeInterpreter.ts"],"mappings":";;;;;;AAOA;;KAAY,0BAAA;;;AAMZ;;UAAiB,4BAAA;EAQyB;;;;;;AAY1C;EAZE,WAAA,GAAc,0BAAA;;;;AA4BhB;;EAtBE,OAAA;AAAA;;;;UAMe,sBAAA;EAgB4D;;AA+F7E;;;;;;;EArGE,SAAA,YAAqB,4BAAA;AAAA;;;;KAMX,mBAAA,GAAsB,QAAA,CAAa,SAAA,CAAU,IAAA,CAAK,eAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA+F9C,eAAA,CAAgB,OAAA,GAAU,sBAAA,GAAyB,UAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"codeInterpreter.js","names":[
|
|
1
|
+
{"version":3,"file":"codeInterpreter.js","names":[],"sources":["../../src/tools/codeInterpreter.ts"],"sourcesContent":["import { OpenAI as OpenAIClient } from \"openai\";\nimport type { ServerTool } from \"@langchain/core/tools\";\n\n/**\n * Memory limit options for the Code Interpreter container.\n * Higher tiers offer more RAM and are billed at different rates.\n */\nexport type CodeInterpreterMemoryLimit = \"1g\" | \"4g\" | \"16g\" | \"64g\";\n\n/**\n * Auto container configuration for Code Interpreter.\n * Creates a new container automatically or reuses an active one.\n */\nexport interface CodeInterpreterAutoContainer {\n /**\n * Memory limit for the container.\n * - `\"1g\"` (default): 1 GB RAM\n * - `\"4g\"`: 4 GB RAM\n * - `\"16g\"`: 16 GB RAM\n * - `\"64g\"`: 64 GB RAM\n */\n memoryLimit?: CodeInterpreterMemoryLimit;\n /**\n * Optional list of uploaded file IDs to make available to the code.\n * Files in the model input are automatically uploaded, so this is only\n * needed for additional files.\n */\n fileIds?: string[];\n}\n\n/**\n * Options for the Code Interpreter tool.\n */\nexport interface CodeInterpreterOptions {\n /**\n * The container configuration for the Code Interpreter.\n *\n * Can be either:\n * - A string container ID for explicit mode (created via `/v1/containers` endpoint)\n * - An auto configuration object that creates/reuses containers automatically\n *\n * If not provided, defaults to auto mode with default settings.\n */\n container?: string | CodeInterpreterAutoContainer;\n}\n\n/**\n * OpenAI Code Interpreter tool type for the Responses API.\n */\nexport type CodeInterpreterTool = OpenAIClient.Responses.Tool.CodeInterpreter;\n\n/**\n * Converts container options to the API format.\n */\nfunction convertContainer(\n container: string | CodeInterpreterAutoContainer | undefined\n):\n | string\n | OpenAIClient.Responses.Tool.CodeInterpreter.CodeInterpreterToolAuto {\n // If a string container ID is provided, use it directly\n if (typeof container === \"string\") {\n return container;\n }\n\n // Auto mode configuration\n return {\n type: \"auto\",\n file_ids: container?.fileIds,\n memory_limit: container?.memoryLimit,\n };\n}\n\n/**\n * Creates a Code Interpreter tool that allows models to write and run Python code\n * in a sandboxed environment to solve complex problems.\n *\n * Use Code Interpreter for:\n * - **Data analysis**: Processing files with diverse data and formatting\n * - **File generation**: Creating files with data and images of graphs\n * - **Iterative coding**: Writing and running code iteratively to solve problems\n * - **Visual intelligence**: Cropping, zooming, rotating, and transforming images\n *\n * The tool runs in a container, which is a fully sandboxed virtual machine.\n * Containers can be created automatically (auto mode) or explicitly via the\n * `/v1/containers` endpoint.\n *\n * @see {@link https://platform.openai.com/docs/guides/tools-code-interpreter | OpenAI Code Interpreter Documentation}\n *\n * @param options - Configuration options for the Code Interpreter tool\n * @returns A Code Interpreter tool definition to be passed to the OpenAI Responses API\n *\n * @example\n * ```typescript\n * import { ChatOpenAI, tools } from \"@langchain/openai\";\n *\n * const model = new ChatOpenAI({ model: \"gpt-4.1\" });\n *\n * // Basic usage with auto container (default 1GB memory)\n * const response = await model.invoke(\n * \"Solve the equation 3x + 11 = 14\",\n * { tools: [tools.codeInterpreter()] }\n * );\n *\n * // With increased memory limit for larger computations\n * const response = await model.invoke(\n * \"Analyze this large dataset and create visualizations\",\n * {\n * tools: [tools.codeInterpreter({\n * container: { memoryLimit: \"4g\" }\n * })]\n * }\n * );\n *\n * // With specific files available to the code\n * const response = await model.invoke(\n * \"Process the uploaded CSV file\",\n * {\n * tools: [tools.codeInterpreter({\n * container: {\n * memoryLimit: \"4g\",\n * fileIds: [\"file-abc123\", \"file-def456\"]\n * }\n * })]\n * }\n * );\n *\n * // Using an explicit container ID (created via /v1/containers)\n * const response = await model.invoke(\n * \"Continue working with the data\",\n * {\n * tools: [tools.codeInterpreter({\n * container: \"cntr_abc123\"\n * })]\n * }\n * );\n * ```\n *\n * @remarks\n * - Containers expire after 20 minutes of inactivity\n * - While called \"Code Interpreter\", the model knows it as the \"python tool\"\n * - For explicit prompting, ask for \"the python tool\" in your prompts\n * - Files in model input are automatically uploaded to the container\n * - Generated files are returned as `container_file_citation` annotations\n */\nexport function codeInterpreter(options?: CodeInterpreterOptions): ServerTool {\n return {\n type: \"code_interpreter\",\n container: convertContainer(options?.container),\n } satisfies CodeInterpreterTool;\n}\n"],"mappings":";;;;AAsDA,SAAS,iBACP,WAGsE;AAEtE,KAAI,OAAO,cAAc,SACvB,QAAO;AAIT,QAAO;EACL,MAAM;EACN,UAAU,WAAW;EACrB,cAAc,WAAW;EAC1B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2EH,SAAgB,gBAAgB,SAA8C;AAC5E,QAAO;EACL,MAAM;EACN,WAAW,iBAAiB,SAAS,UAAU;EAChD"}
|
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
const zod_v4 = require_rolldown_runtime.__toESM(require("zod/v4"));
|
|
1
|
+
let _langchain_core_messages = require("@langchain/core/messages");
|
|
2
|
+
let _langchain_core_tools = require("@langchain/core/tools");
|
|
3
|
+
let zod_v4 = require("zod/v4");
|
|
5
4
|
|
|
6
5
|
//#region src/tools/computerUse.ts
|
|
7
6
|
const ComputerUseScreenshotActionSchema = zod_v4.z.object({ type: zod_v4.z.literal("screenshot") });
|
|
@@ -186,19 +185,14 @@ const TOOL_NAME = "computer_use";
|
|
|
186
185
|
* - Recommended to use with `reasoning.summary` for debugging
|
|
187
186
|
*/
|
|
188
187
|
function computerUse(options) {
|
|
189
|
-
const computerTool = (0,
|
|
190
|
-
|
|
191
|
-
* get computer_use call id from runtime
|
|
192
|
-
*/
|
|
193
|
-
const aiMessage = runtime.state?.messages.at(-1);
|
|
194
|
-
const computerToolCall = aiMessage?.tool_calls?.find((tc) => tc.name === "computer_use");
|
|
195
|
-
const computerToolCallId = computerToolCall?.id;
|
|
188
|
+
const computerTool = (0, _langchain_core_tools.tool)(async (input, runtime) => {
|
|
189
|
+
const computerToolCallId = ((runtime.state?.messages.at(-1))?.tool_calls?.find((tc) => tc.name === "computer_use"))?.id;
|
|
196
190
|
if (!computerToolCallId) throw new Error("Computer use call id not found");
|
|
197
191
|
const result = await options.execute(input.action, runtime);
|
|
198
192
|
/**
|
|
199
193
|
* make sure {@link ToolMessage} is returned with the correct additional kwargs
|
|
200
194
|
*/
|
|
201
|
-
if (typeof result === "string") return new
|
|
195
|
+
if (typeof result === "string") return new _langchain_core_messages.ToolMessage({
|
|
202
196
|
content: result,
|
|
203
197
|
tool_call_id: computerToolCallId,
|
|
204
198
|
additional_kwargs: { type: "computer_call_output" }
|
|
@@ -206,7 +200,7 @@ function computerUse(options) {
|
|
|
206
200
|
/**
|
|
207
201
|
* make sure {@link ToolMessage} is returned with the correct additional kwargs
|
|
208
202
|
*/
|
|
209
|
-
return new
|
|
203
|
+
return new _langchain_core_messages.ToolMessage({
|
|
210
204
|
...result,
|
|
211
205
|
tool_call_id: computerToolCallId,
|
|
212
206
|
additional_kwargs: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"computerUse.cjs","names":["z","options: ComputerUseOptions","input: ComputerUseInput","runtime: ToolRuntime<{ messages: BaseMessage[] }>","ToolMessage"],"sources":["../../src/tools/computerUse.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\nimport { z } from \"zod/v4\";\nimport { OpenAI as OpenAIClient } from \"openai\";\nimport { tool, type DynamicStructuredTool } from \"@langchain/core/tools\";\nimport { type ToolRuntime } from \"@langchain/core/tools\";\nimport {\n ToolMessage,\n type AIMessage,\n type BaseMessage,\n} from \"@langchain/core/messages\";\n\n/**\n * The type of computer environment to control.\n */\nexport type ComputerUseEnvironment =\n | \"browser\"\n | \"mac\"\n | \"windows\"\n | \"linux\"\n | \"ubuntu\";\n\n/**\n * Re-export action types from OpenAI SDK for convenience.\n */\nexport type ComputerUseClickAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Click;\nexport type ComputerUseDoubleClickAction =\n OpenAIClient.Responses.ResponseComputerToolCall.DoubleClick;\nexport type ComputerUseDragAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Drag;\nexport type ComputerUseKeypressAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Keypress;\nexport type ComputerUseMoveAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Move;\nexport type ComputerUseScreenshotAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Screenshot;\nexport type ComputerUseScrollAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Scroll;\nexport type ComputerUseTypeAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Type;\nexport type ComputerUseWaitAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Wait;\n\n/**\n * Union type of all computer use actions from OpenAI SDK.\n */\nexport type ComputerUseAction =\n OpenAIClient.Responses.ResponseComputerToolCall[\"action\"];\n\n// Zod schemas for computer use actions\nconst ComputerUseScreenshotActionSchema = z.object({\n type: z.literal(\"screenshot\"),\n});\n\nconst ComputerUseClickActionSchema = z.object({\n type: z.literal(\"click\"),\n x: z.number(),\n y: z.number(),\n button: z.enum([\"left\", \"right\", \"wheel\", \"back\", \"forward\"]).default(\"left\"),\n});\n\nconst ComputerUseDoubleClickActionSchema = z.object({\n type: z.literal(\"double_click\"),\n x: z.number(),\n y: z.number(),\n button: z.enum([\"left\", \"right\", \"wheel\", \"back\", \"forward\"]).default(\"left\"),\n});\n\nconst ComputerUseDragActionSchema = z.object({\n type: z.literal(\"drag\"),\n path: z.array(z.object({ x: z.number(), y: z.number() })),\n});\n\nconst ComputerUseKeypressActionSchema = z.object({\n type: z.literal(\"keypress\"),\n keys: z.array(z.string()),\n});\n\nconst ComputerUseMoveActionSchema = z.object({\n type: z.literal(\"move\"),\n x: z.number(),\n y: z.number(),\n});\n\nconst ComputerUseScrollActionSchema = z.object({\n type: z.literal(\"scroll\"),\n x: z.number(),\n y: z.number(),\n scroll_x: z.number(),\n scroll_y: z.number(),\n});\n\nconst ComputerUseTypeActionSchema = z.object({\n type: z.literal(\"type\"),\n text: z.string(),\n});\n\nconst ComputerUseWaitActionSchema = z.object({\n type: z.literal(\"wait\"),\n duration: z.number().optional(),\n});\n\n// Union schema for individual action types\nconst ComputerUseActionUnionSchema = z.union([\n ComputerUseScreenshotActionSchema,\n ComputerUseClickActionSchema,\n ComputerUseDoubleClickActionSchema,\n ComputerUseDragActionSchema,\n ComputerUseKeypressActionSchema,\n ComputerUseMoveActionSchema,\n ComputerUseScrollActionSchema,\n ComputerUseTypeActionSchema,\n ComputerUseWaitActionSchema,\n]);\n\n// Schema for the input structure received from parseComputerCall\n// The action is wrapped in an `action` property: { action: { type: 'screenshot' } }\nexport const ComputerUseActionSchema = z.object({\n action: ComputerUseActionUnionSchema,\n});\n\n// TypeScript types derived from Zod schemas\nexport type ComputerUseScreenshotActionType = z.infer<\n typeof ComputerUseScreenshotActionSchema\n>;\nexport type ComputerUseClickActionType = z.infer<\n typeof ComputerUseClickActionSchema\n>;\nexport type ComputerUseDoubleClickActionType = z.infer<\n typeof ComputerUseDoubleClickActionSchema\n>;\nexport type ComputerUseDragActionType = z.infer<\n typeof ComputerUseDragActionSchema\n>;\nexport type ComputerUseKeypressActionType = z.infer<\n typeof ComputerUseKeypressActionSchema\n>;\nexport type ComputerUseMoveActionType = z.infer<\n typeof ComputerUseMoveActionSchema\n>;\nexport type ComputerUseScrollActionType = z.infer<\n typeof ComputerUseScrollActionSchema\n>;\nexport type ComputerUseTypeActionType = z.infer<\n typeof ComputerUseTypeActionSchema\n>;\nexport type ComputerUseWaitActionType = z.infer<\n typeof ComputerUseWaitActionSchema\n>;\n\n/**\n * Input structure for the Computer Use tool.\n * The action is wrapped in an `action` property.\n */\nexport interface ComputerUseInput {\n action: ComputerUseAction;\n}\n\nexport type ComputerUseReturnType =\n | string\n | Promise<string>\n | ToolMessage<any>\n | Promise<ToolMessage<any>>;\n\n/**\n * Options for the Computer Use tool.\n */\nexport interface ComputerUseOptions {\n /**\n * The width of the computer display in pixels.\n */\n displayWidth: number;\n\n /**\n * The height of the computer display in pixels.\n */\n displayHeight: number;\n\n /**\n * The type of computer environment to control.\n * - `browser`: Browser automation (recommended for most use cases)\n * - `mac`: macOS environment\n * - `windows`: Windows environment\n * - `linux`: Linux environment\n * - `ubuntu`: Ubuntu environment\n */\n environment: ComputerUseEnvironment;\n\n /**\n * Execute function that handles computer action execution.\n * This function receives the action input and should return a base64-encoded\n * screenshot of the result.\n */\n execute: (\n action: ComputerUseAction,\n runtime: ToolRuntime<any, any>\n ) => ComputerUseReturnType;\n}\n\n/**\n * OpenAI Computer Use tool type for the Responses API.\n */\nexport type ComputerUseTool = OpenAIClient.Responses.ComputerTool;\n\nconst TOOL_NAME = \"computer_use\";\n\n/**\n * Creates a Computer Use tool that allows models to control computer interfaces\n * and perform tasks by simulating mouse clicks, keyboard input, scrolling, and more.\n *\n * **Computer Use** is a practical application of OpenAI's Computer-Using Agent (CUA)\n * model (`computer-use-preview`), which combines vision capabilities with advanced\n * reasoning to simulate controlling computer interfaces.\n *\n * **How it works**:\n * The tool operates in a continuous loop:\n * 1. Model sends computer actions (click, type, scroll, etc.)\n * 2. Your code executes these actions in a controlled environment\n * 3. You capture a screenshot of the result\n * 4. Send the screenshot back to the model\n * 5. Repeat until the task is complete\n *\n * **Important**: Computer use is in beta and requires careful consideration:\n * - Use in sandboxed environments only\n * - Do not use for high-stakes or authenticated tasks\n * - Always implement human-in-the-loop for important decisions\n * - Handle safety checks appropriately\n *\n * @see {@link https://platform.openai.com/docs/guides/tools-computer-use | OpenAI Computer Use Documentation}\n *\n * @param options - Configuration options for the Computer Use tool\n * @returns A Computer Use tool that can be passed to `bindTools`\n *\n * @example\n * ```typescript\n * import { ChatOpenAI, tools } from \"@langchain/openai\";\n *\n * const model = new ChatOpenAI({ model: \"computer-use-preview\" });\n *\n * // With execute callback for automatic action handling\n * const computer = tools.computerUse({\n * displayWidth: 1024,\n * displayHeight: 768,\n * environment: \"browser\",\n * execute: async (action) => {\n * if (action.type === \"screenshot\") {\n * return captureScreenshot();\n * }\n * if (action.type === \"click\") {\n * await page.mouse.click(action.x, action.y, { button: action.button });\n * return captureScreenshot();\n * }\n * if (action.type === \"type\") {\n * await page.keyboard.type(action.text);\n * return captureScreenshot();\n * }\n * // Handle other actions...\n * return captureScreenshot();\n * },\n * });\n *\n * const llmWithComputer = model.bindTools([computer]);\n * const response = await llmWithComputer.invoke(\n * \"Check the latest news on bing.com\"\n * );\n * ```\n *\n * @example\n * ```typescript\n * // Without execute callback (manual action handling)\n * const computer = tools.computerUse({\n * displayWidth: 1024,\n * displayHeight: 768,\n * environment: \"browser\",\n * });\n *\n * const response = await model.invoke(\"Check the news\", {\n * tools: [computer],\n * });\n *\n * // Access the computer call from the response\n * const computerCall = response.additional_kwargs.tool_outputs?.find(\n * (output) => output.type === \"computer_call\"\n * );\n * if (computerCall) {\n * console.log(\"Action to execute:\", computerCall.action);\n * // Execute the action manually, then send back a screenshot\n * }\n * ```\n *\n * @example\n * ```typescript\n * // For macOS desktop automation with Docker\n * const computer = tools.computerUse({\n * displayWidth: 1920,\n * displayHeight: 1080,\n * environment: \"mac\",\n * execute: async (action) => {\n * if (action.type === \"click\") {\n * await dockerExec(\n * `DISPLAY=:99 xdotool mousemove ${action.x} ${action.y} click 1`,\n * containerName\n * );\n * }\n * // Capture screenshot from container\n * return await getDockerScreenshot(containerName);\n * },\n * });\n * ```\n *\n * @remarks\n * - Only available through the Responses API (not Chat Completions)\n * - Requires `computer-use-preview` model\n * - Actions include: click, double_click, drag, keypress, move, screenshot, scroll, type, wait\n * - Safety checks may be returned that require acknowledgment before proceeding\n * - Use `truncation: \"auto\"` parameter when making requests\n * - Recommended to use with `reasoning.summary` for debugging\n */\nexport function computerUse(options: ComputerUseOptions) {\n const computerTool = tool(\n async (\n input: ComputerUseInput,\n runtime: ToolRuntime<{ messages: BaseMessage[] }>\n ) => {\n /**\n * get computer_use call id from runtime\n */\n const aiMessage = runtime.state?.messages.at(-1) as AIMessage | undefined;\n const computerToolCall = aiMessage?.tool_calls?.find(\n (tc) => tc.name === \"computer_use\"\n );\n const computerToolCallId = computerToolCall?.id;\n if (!computerToolCallId) {\n throw new Error(\"Computer use call id not found\");\n }\n\n const result = await options.execute(input.action, runtime);\n\n /**\n * make sure {@link ToolMessage} is returned with the correct additional kwargs\n */\n if (typeof result === \"string\") {\n return new ToolMessage({\n content: result,\n tool_call_id: computerToolCallId,\n additional_kwargs: {\n type: \"computer_call_output\",\n },\n });\n }\n\n /**\n * make sure {@link ToolMessage} is returned with the correct additional kwargs\n */\n return new ToolMessage({\n ...result,\n tool_call_id: computerToolCallId,\n additional_kwargs: {\n type: \"computer_call_output\",\n ...result.additional_kwargs,\n },\n });\n },\n {\n name: TOOL_NAME,\n description:\n \"Control a computer interface by executing mouse clicks, keyboard input, scrolling, and other actions.\",\n schema: ComputerUseActionSchema,\n }\n );\n\n computerTool.extras = {\n ...(computerTool.extras ?? {}),\n providerToolDefinition: {\n type: \"computer_use_preview\",\n display_width: options.displayWidth,\n display_height: options.displayHeight,\n environment: options.environment,\n } satisfies ComputerUseTool,\n };\n\n /**\n * return as typed {@link DynamicStructuredTool} so we don't get any type\n * errors like \"can't export tool without reference\"\n */\n return computerTool as DynamicStructuredTool<\n typeof ComputerUseActionSchema,\n ComputerUseInput,\n unknown,\n ComputerUseReturnType\n >;\n}\n"],"mappings":";;;;;;AAkDA,MAAM,oCAAoCA,SAAE,OAAO,EACjD,MAAMA,SAAE,QAAQ,aAAa,CAC9B,EAAC;AAEF,MAAM,+BAA+BA,SAAE,OAAO;CAC5C,MAAMA,SAAE,QAAQ,QAAQ;CACxB,GAAGA,SAAE,QAAQ;CACb,GAAGA,SAAE,QAAQ;CACb,QAAQA,SAAE,KAAK;EAAC;EAAQ;EAAS;EAAS;EAAQ;CAAU,EAAC,CAAC,QAAQ,OAAO;AAC9E,EAAC;AAEF,MAAM,qCAAqCA,SAAE,OAAO;CAClD,MAAMA,SAAE,QAAQ,eAAe;CAC/B,GAAGA,SAAE,QAAQ;CACb,GAAGA,SAAE,QAAQ;CACb,QAAQA,SAAE,KAAK;EAAC;EAAQ;EAAS;EAAS;EAAQ;CAAU,EAAC,CAAC,QAAQ,OAAO;AAC9E,EAAC;AAEF,MAAM,8BAA8BA,SAAE,OAAO;CAC3C,MAAMA,SAAE,QAAQ,OAAO;CACvB,MAAMA,SAAE,MAAMA,SAAE,OAAO;EAAE,GAAGA,SAAE,QAAQ;EAAE,GAAGA,SAAE,QAAQ;CAAE,EAAC,CAAC;AAC1D,EAAC;AAEF,MAAM,kCAAkCA,SAAE,OAAO;CAC/C,MAAMA,SAAE,QAAQ,WAAW;CAC3B,MAAMA,SAAE,MAAMA,SAAE,QAAQ,CAAC;AAC1B,EAAC;AAEF,MAAM,8BAA8BA,SAAE,OAAO;CAC3C,MAAMA,SAAE,QAAQ,OAAO;CACvB,GAAGA,SAAE,QAAQ;CACb,GAAGA,SAAE,QAAQ;AACd,EAAC;AAEF,MAAM,gCAAgCA,SAAE,OAAO;CAC7C,MAAMA,SAAE,QAAQ,SAAS;CACzB,GAAGA,SAAE,QAAQ;CACb,GAAGA,SAAE,QAAQ;CACb,UAAUA,SAAE,QAAQ;CACpB,UAAUA,SAAE,QAAQ;AACrB,EAAC;AAEF,MAAM,8BAA8BA,SAAE,OAAO;CAC3C,MAAMA,SAAE,QAAQ,OAAO;CACvB,MAAMA,SAAE,QAAQ;AACjB,EAAC;AAEF,MAAM,8BAA8BA,SAAE,OAAO;CAC3C,MAAMA,SAAE,QAAQ,OAAO;CACvB,UAAUA,SAAE,QAAQ,CAAC,UAAU;AAChC,EAAC;AAGF,MAAM,+BAA+BA,SAAE,MAAM;CAC3C;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACD,EAAC;AAIF,MAAa,0BAA0BA,SAAE,OAAO,EAC9C,QAAQ,6BACT,EAAC;AAqFF,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkHlB,SAAgB,YAAYC,SAA6B;CACvD,MAAM,gDACJ,OACEC,OACAC,YACG;;;;EAIH,MAAM,YAAY,QAAQ,OAAO,SAAS,GAAG,GAAG;EAChD,MAAM,mBAAmB,WAAW,YAAY,KAC9C,CAAC,OAAO,GAAG,SAAS,eACrB;EACD,MAAM,qBAAqB,kBAAkB;AAC7C,MAAI,CAAC,mBACH,OAAM,IAAI,MAAM;EAGlB,MAAM,SAAS,MAAM,QAAQ,QAAQ,MAAM,QAAQ,QAAQ;;;;AAK3D,MAAI,OAAO,WAAW,SACpB,QAAO,IAAIC,sCAAY;GACrB,SAAS;GACT,cAAc;GACd,mBAAmB,EACjB,MAAM,uBACP;EACF;;;;AAMH,SAAO,IAAIA,sCAAY;GACrB,GAAG;GACH,cAAc;GACd,mBAAmB;IACjB,MAAM;IACN,GAAG,OAAO;GACX;EACF;CACF,GACD;EACE,MAAM;EACN,aACE;EACF,QAAQ;CACT,EACF;CAED,aAAa,SAAS;EACpB,GAAI,aAAa,UAAU,CAAE;EAC7B,wBAAwB;GACtB,MAAM;GACN,eAAe,QAAQ;GACvB,gBAAgB,QAAQ;GACxB,aAAa,QAAQ;EACtB;CACF;;;;;AAMD,QAAO;AAMR"}
|
|
1
|
+
{"version":3,"file":"computerUse.cjs","names":["z","ToolMessage"],"sources":["../../src/tools/computerUse.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\nimport { z } from \"zod/v4\";\nimport { OpenAI as OpenAIClient } from \"openai\";\nimport { tool, type DynamicStructuredTool } from \"@langchain/core/tools\";\nimport { type ToolRuntime } from \"@langchain/core/tools\";\nimport {\n ToolMessage,\n type AIMessage,\n type BaseMessage,\n} from \"@langchain/core/messages\";\n\n/**\n * The type of computer environment to control.\n */\nexport type ComputerUseEnvironment =\n | \"browser\"\n | \"mac\"\n | \"windows\"\n | \"linux\"\n | \"ubuntu\";\n\n/**\n * Re-export action types from OpenAI SDK for convenience.\n */\nexport type ComputerUseClickAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Click;\nexport type ComputerUseDoubleClickAction =\n OpenAIClient.Responses.ResponseComputerToolCall.DoubleClick;\nexport type ComputerUseDragAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Drag;\nexport type ComputerUseKeypressAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Keypress;\nexport type ComputerUseMoveAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Move;\nexport type ComputerUseScreenshotAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Screenshot;\nexport type ComputerUseScrollAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Scroll;\nexport type ComputerUseTypeAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Type;\nexport type ComputerUseWaitAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Wait;\n\n/**\n * Union type of all computer use actions from OpenAI SDK.\n */\nexport type ComputerUseAction =\n OpenAIClient.Responses.ResponseComputerToolCall[\"action\"];\n\n// Zod schemas for computer use actions\nconst ComputerUseScreenshotActionSchema = z.object({\n type: z.literal(\"screenshot\"),\n});\n\nconst ComputerUseClickActionSchema = z.object({\n type: z.literal(\"click\"),\n x: z.number(),\n y: z.number(),\n button: z.enum([\"left\", \"right\", \"wheel\", \"back\", \"forward\"]).default(\"left\"),\n});\n\nconst ComputerUseDoubleClickActionSchema = z.object({\n type: z.literal(\"double_click\"),\n x: z.number(),\n y: z.number(),\n button: z.enum([\"left\", \"right\", \"wheel\", \"back\", \"forward\"]).default(\"left\"),\n});\n\nconst ComputerUseDragActionSchema = z.object({\n type: z.literal(\"drag\"),\n path: z.array(z.object({ x: z.number(), y: z.number() })),\n});\n\nconst ComputerUseKeypressActionSchema = z.object({\n type: z.literal(\"keypress\"),\n keys: z.array(z.string()),\n});\n\nconst ComputerUseMoveActionSchema = z.object({\n type: z.literal(\"move\"),\n x: z.number(),\n y: z.number(),\n});\n\nconst ComputerUseScrollActionSchema = z.object({\n type: z.literal(\"scroll\"),\n x: z.number(),\n y: z.number(),\n scroll_x: z.number(),\n scroll_y: z.number(),\n});\n\nconst ComputerUseTypeActionSchema = z.object({\n type: z.literal(\"type\"),\n text: z.string(),\n});\n\nconst ComputerUseWaitActionSchema = z.object({\n type: z.literal(\"wait\"),\n duration: z.number().optional(),\n});\n\n// Union schema for individual action types\nconst ComputerUseActionUnionSchema = z.union([\n ComputerUseScreenshotActionSchema,\n ComputerUseClickActionSchema,\n ComputerUseDoubleClickActionSchema,\n ComputerUseDragActionSchema,\n ComputerUseKeypressActionSchema,\n ComputerUseMoveActionSchema,\n ComputerUseScrollActionSchema,\n ComputerUseTypeActionSchema,\n ComputerUseWaitActionSchema,\n]);\n\n// Schema for the input structure received from parseComputerCall\n// The action is wrapped in an `action` property: { action: { type: 'screenshot' } }\nexport const ComputerUseActionSchema = z.object({\n action: ComputerUseActionUnionSchema,\n});\n\n// TypeScript types derived from Zod schemas\nexport type ComputerUseScreenshotActionType = z.infer<\n typeof ComputerUseScreenshotActionSchema\n>;\nexport type ComputerUseClickActionType = z.infer<\n typeof ComputerUseClickActionSchema\n>;\nexport type ComputerUseDoubleClickActionType = z.infer<\n typeof ComputerUseDoubleClickActionSchema\n>;\nexport type ComputerUseDragActionType = z.infer<\n typeof ComputerUseDragActionSchema\n>;\nexport type ComputerUseKeypressActionType = z.infer<\n typeof ComputerUseKeypressActionSchema\n>;\nexport type ComputerUseMoveActionType = z.infer<\n typeof ComputerUseMoveActionSchema\n>;\nexport type ComputerUseScrollActionType = z.infer<\n typeof ComputerUseScrollActionSchema\n>;\nexport type ComputerUseTypeActionType = z.infer<\n typeof ComputerUseTypeActionSchema\n>;\nexport type ComputerUseWaitActionType = z.infer<\n typeof ComputerUseWaitActionSchema\n>;\n\n/**\n * Input structure for the Computer Use tool.\n * The action is wrapped in an `action` property.\n */\nexport interface ComputerUseInput {\n action: ComputerUseAction;\n}\n\nexport type ComputerUseReturnType =\n | string\n | Promise<string>\n | ToolMessage<any>\n | Promise<ToolMessage<any>>;\n\n/**\n * Options for the Computer Use tool.\n */\nexport interface ComputerUseOptions {\n /**\n * The width of the computer display in pixels.\n */\n displayWidth: number;\n\n /**\n * The height of the computer display in pixels.\n */\n displayHeight: number;\n\n /**\n * The type of computer environment to control.\n * - `browser`: Browser automation (recommended for most use cases)\n * - `mac`: macOS environment\n * - `windows`: Windows environment\n * - `linux`: Linux environment\n * - `ubuntu`: Ubuntu environment\n */\n environment: ComputerUseEnvironment;\n\n /**\n * Execute function that handles computer action execution.\n * This function receives the action input and should return a base64-encoded\n * screenshot of the result.\n */\n execute: (\n action: ComputerUseAction,\n runtime: ToolRuntime<any, any>\n ) => ComputerUseReturnType;\n}\n\n/**\n * OpenAI Computer Use tool type for the Responses API.\n */\nexport type ComputerUseTool = OpenAIClient.Responses.ComputerTool;\n\nconst TOOL_NAME = \"computer_use\";\n\n/**\n * Creates a Computer Use tool that allows models to control computer interfaces\n * and perform tasks by simulating mouse clicks, keyboard input, scrolling, and more.\n *\n * **Computer Use** is a practical application of OpenAI's Computer-Using Agent (CUA)\n * model (`computer-use-preview`), which combines vision capabilities with advanced\n * reasoning to simulate controlling computer interfaces.\n *\n * **How it works**:\n * The tool operates in a continuous loop:\n * 1. Model sends computer actions (click, type, scroll, etc.)\n * 2. Your code executes these actions in a controlled environment\n * 3. You capture a screenshot of the result\n * 4. Send the screenshot back to the model\n * 5. Repeat until the task is complete\n *\n * **Important**: Computer use is in beta and requires careful consideration:\n * - Use in sandboxed environments only\n * - Do not use for high-stakes or authenticated tasks\n * - Always implement human-in-the-loop for important decisions\n * - Handle safety checks appropriately\n *\n * @see {@link https://platform.openai.com/docs/guides/tools-computer-use | OpenAI Computer Use Documentation}\n *\n * @param options - Configuration options for the Computer Use tool\n * @returns A Computer Use tool that can be passed to `bindTools`\n *\n * @example\n * ```typescript\n * import { ChatOpenAI, tools } from \"@langchain/openai\";\n *\n * const model = new ChatOpenAI({ model: \"computer-use-preview\" });\n *\n * // With execute callback for automatic action handling\n * const computer = tools.computerUse({\n * displayWidth: 1024,\n * displayHeight: 768,\n * environment: \"browser\",\n * execute: async (action) => {\n * if (action.type === \"screenshot\") {\n * return captureScreenshot();\n * }\n * if (action.type === \"click\") {\n * await page.mouse.click(action.x, action.y, { button: action.button });\n * return captureScreenshot();\n * }\n * if (action.type === \"type\") {\n * await page.keyboard.type(action.text);\n * return captureScreenshot();\n * }\n * // Handle other actions...\n * return captureScreenshot();\n * },\n * });\n *\n * const llmWithComputer = model.bindTools([computer]);\n * const response = await llmWithComputer.invoke(\n * \"Check the latest news on bing.com\"\n * );\n * ```\n *\n * @example\n * ```typescript\n * // Without execute callback (manual action handling)\n * const computer = tools.computerUse({\n * displayWidth: 1024,\n * displayHeight: 768,\n * environment: \"browser\",\n * });\n *\n * const response = await model.invoke(\"Check the news\", {\n * tools: [computer],\n * });\n *\n * // Access the computer call from the response\n * const computerCall = response.additional_kwargs.tool_outputs?.find(\n * (output) => output.type === \"computer_call\"\n * );\n * if (computerCall) {\n * console.log(\"Action to execute:\", computerCall.action);\n * // Execute the action manually, then send back a screenshot\n * }\n * ```\n *\n * @example\n * ```typescript\n * // For macOS desktop automation with Docker\n * const computer = tools.computerUse({\n * displayWidth: 1920,\n * displayHeight: 1080,\n * environment: \"mac\",\n * execute: async (action) => {\n * if (action.type === \"click\") {\n * await dockerExec(\n * `DISPLAY=:99 xdotool mousemove ${action.x} ${action.y} click 1`,\n * containerName\n * );\n * }\n * // Capture screenshot from container\n * return await getDockerScreenshot(containerName);\n * },\n * });\n * ```\n *\n * @remarks\n * - Only available through the Responses API (not Chat Completions)\n * - Requires `computer-use-preview` model\n * - Actions include: click, double_click, drag, keypress, move, screenshot, scroll, type, wait\n * - Safety checks may be returned that require acknowledgment before proceeding\n * - Use `truncation: \"auto\"` parameter when making requests\n * - Recommended to use with `reasoning.summary` for debugging\n */\nexport function computerUse(options: ComputerUseOptions) {\n const computerTool = tool(\n async (\n input: ComputerUseInput,\n runtime: ToolRuntime<{ messages: BaseMessage[] }>\n ) => {\n /**\n * get computer_use call id from runtime\n */\n const aiMessage = runtime.state?.messages.at(-1) as AIMessage | undefined;\n const computerToolCall = aiMessage?.tool_calls?.find(\n (tc) => tc.name === \"computer_use\"\n );\n const computerToolCallId = computerToolCall?.id;\n if (!computerToolCallId) {\n throw new Error(\"Computer use call id not found\");\n }\n\n const result = await options.execute(input.action, runtime);\n\n /**\n * make sure {@link ToolMessage} is returned with the correct additional kwargs\n */\n if (typeof result === \"string\") {\n return new ToolMessage({\n content: result,\n tool_call_id: computerToolCallId,\n additional_kwargs: {\n type: \"computer_call_output\",\n },\n });\n }\n\n /**\n * make sure {@link ToolMessage} is returned with the correct additional kwargs\n */\n return new ToolMessage({\n ...result,\n tool_call_id: computerToolCallId,\n additional_kwargs: {\n type: \"computer_call_output\",\n ...result.additional_kwargs,\n },\n });\n },\n {\n name: TOOL_NAME,\n description:\n \"Control a computer interface by executing mouse clicks, keyboard input, scrolling, and other actions.\",\n schema: ComputerUseActionSchema,\n }\n );\n\n computerTool.extras = {\n ...(computerTool.extras ?? {}),\n providerToolDefinition: {\n type: \"computer_use_preview\",\n display_width: options.displayWidth,\n display_height: options.displayHeight,\n environment: options.environment,\n } satisfies ComputerUseTool,\n };\n\n /**\n * return as typed {@link DynamicStructuredTool} so we don't get any type\n * errors like \"can't export tool without reference\"\n */\n return computerTool as DynamicStructuredTool<\n typeof ComputerUseActionSchema,\n ComputerUseInput,\n unknown,\n ComputerUseReturnType\n >;\n}\n"],"mappings":";;;;;AAkDA,MAAM,oCAAoCA,SAAE,OAAO,EACjD,MAAMA,SAAE,QAAQ,aAAa,EAC9B,CAAC;AAEF,MAAM,+BAA+BA,SAAE,OAAO;CAC5C,MAAMA,SAAE,QAAQ,QAAQ;CACxB,GAAGA,SAAE,QAAQ;CACb,GAAGA,SAAE,QAAQ;CACb,QAAQA,SAAE,KAAK;EAAC;EAAQ;EAAS;EAAS;EAAQ;EAAU,CAAC,CAAC,QAAQ,OAAO;CAC9E,CAAC;AAEF,MAAM,qCAAqCA,SAAE,OAAO;CAClD,MAAMA,SAAE,QAAQ,eAAe;CAC/B,GAAGA,SAAE,QAAQ;CACb,GAAGA,SAAE,QAAQ;CACb,QAAQA,SAAE,KAAK;EAAC;EAAQ;EAAS;EAAS;EAAQ;EAAU,CAAC,CAAC,QAAQ,OAAO;CAC9E,CAAC;AAEF,MAAM,8BAA8BA,SAAE,OAAO;CAC3C,MAAMA,SAAE,QAAQ,OAAO;CACvB,MAAMA,SAAE,MAAMA,SAAE,OAAO;EAAE,GAAGA,SAAE,QAAQ;EAAE,GAAGA,SAAE,QAAQ;EAAE,CAAC,CAAC;CAC1D,CAAC;AAEF,MAAM,kCAAkCA,SAAE,OAAO;CAC/C,MAAMA,SAAE,QAAQ,WAAW;CAC3B,MAAMA,SAAE,MAAMA,SAAE,QAAQ,CAAC;CAC1B,CAAC;AAEF,MAAM,8BAA8BA,SAAE,OAAO;CAC3C,MAAMA,SAAE,QAAQ,OAAO;CACvB,GAAGA,SAAE,QAAQ;CACb,GAAGA,SAAE,QAAQ;CACd,CAAC;AAEF,MAAM,gCAAgCA,SAAE,OAAO;CAC7C,MAAMA,SAAE,QAAQ,SAAS;CACzB,GAAGA,SAAE,QAAQ;CACb,GAAGA,SAAE,QAAQ;CACb,UAAUA,SAAE,QAAQ;CACpB,UAAUA,SAAE,QAAQ;CACrB,CAAC;AAEF,MAAM,8BAA8BA,SAAE,OAAO;CAC3C,MAAMA,SAAE,QAAQ,OAAO;CACvB,MAAMA,SAAE,QAAQ;CACjB,CAAC;AAEF,MAAM,8BAA8BA,SAAE,OAAO;CAC3C,MAAMA,SAAE,QAAQ,OAAO;CACvB,UAAUA,SAAE,QAAQ,CAAC,UAAU;CAChC,CAAC;AAGF,MAAM,+BAA+BA,SAAE,MAAM;CAC3C;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACD,CAAC;AAIF,MAAa,0BAA0BA,SAAE,OAAO,EAC9C,QAAQ,8BACT,CAAC;AAqFF,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkHlB,SAAgB,YAAY,SAA6B;CACvD,MAAM,+CACJ,OACE,OACA,YACG;EAQH,MAAM,uBAJY,QAAQ,OAAO,SAAS,GAAG,GAAG,GACZ,YAAY,MAC7C,OAAO,GAAG,SAAS,eACrB,GAC4C;AAC7C,MAAI,CAAC,mBACH,OAAM,IAAI,MAAM,iCAAiC;EAGnD,MAAM,SAAS,MAAM,QAAQ,QAAQ,MAAM,QAAQ,QAAQ;;;;AAK3D,MAAI,OAAO,WAAW,SACpB,QAAO,IAAIC,qCAAY;GACrB,SAAS;GACT,cAAc;GACd,mBAAmB,EACjB,MAAM,wBACP;GACF,CAAC;;;;AAMJ,SAAO,IAAIA,qCAAY;GACrB,GAAG;GACH,cAAc;GACd,mBAAmB;IACjB,MAAM;IACN,GAAG,OAAO;IACX;GACF,CAAC;IAEJ;EACE,MAAM;EACN,aACE;EACF,QAAQ;EACT,CACF;AAED,cAAa,SAAS;EACpB,GAAI,aAAa,UAAU,EAAE;EAC7B,wBAAwB;GACtB,MAAM;GACN,eAAe,QAAQ;GACvB,gBAAgB,QAAQ;GACxB,aAAa,QAAQ;GACtB;EACF;;;;;AAMD,QAAO"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"computerUse.d.cts","names":["z","OpenAI","OpenAIClient","DynamicStructuredTool","ToolRuntime","ToolMessage","ComputerUseEnvironment","ComputerUseClickAction","Responses","ResponseComputerToolCall","Click","ComputerUseDoubleClickAction","DoubleClick","ComputerUseDragAction","Drag","ComputerUseKeypressAction","Keypress","ComputerUseMoveAction","Move","ComputerUseScreenshotAction","Screenshot","ComputerUseScrollAction","Scroll","ComputerUseTypeAction","Type","ComputerUseWaitAction","Wait","ComputerUseAction","ComputerUseScreenshotActionSchema","ZodLiteral","core","$strip","ZodObject","ComputerUseClickActionSchema","ZodNumber","ZodEnum","ZodDefault","ComputerUseDoubleClickActionSchema","ComputerUseDragActionSchema","ZodArray","ComputerUseKeypressActionSchema","ZodString","ComputerUseMoveActionSchema","ComputerUseScrollActionSchema","ComputerUseTypeActionSchema","ComputerUseWaitActionSchema","ZodOptional","ComputerUseActionSchema","ZodUnion","ComputerUseScreenshotActionType","infer","ComputerUseClickActionType","ComputerUseDoubleClickActionType","ComputerUseDragActionType","ComputerUseKeypressActionType","ComputerUseMoveActionType","ComputerUseScrollActionType","ComputerUseTypeActionType","ComputerUseWaitActionType","ComputerUseInput","ComputerUseReturnType","Promise","ComputerUseOptions","ComputerUseTool","ComputerTool","computerUse"],"sources":["../../src/tools/computerUse.d.ts"],"sourcesContent":["import { z } from \"zod/v4\";\nimport { OpenAI as OpenAIClient } from \"openai\";\nimport { type DynamicStructuredTool } from \"@langchain/core/tools\";\nimport { type ToolRuntime } from \"@langchain/core/tools\";\nimport { ToolMessage } from \"@langchain/core/messages\";\n/**\n * The type of computer environment to control.\n */\nexport type ComputerUseEnvironment = \"browser\" | \"mac\" | \"windows\" | \"linux\" | \"ubuntu\";\n/**\n * Re-export action types from OpenAI SDK for convenience.\n */\nexport type ComputerUseClickAction = OpenAIClient.Responses.ResponseComputerToolCall.Click;\nexport type ComputerUseDoubleClickAction = OpenAIClient.Responses.ResponseComputerToolCall.DoubleClick;\nexport type ComputerUseDragAction = OpenAIClient.Responses.ResponseComputerToolCall.Drag;\nexport type ComputerUseKeypressAction = OpenAIClient.Responses.ResponseComputerToolCall.Keypress;\nexport type ComputerUseMoveAction = OpenAIClient.Responses.ResponseComputerToolCall.Move;\nexport type ComputerUseScreenshotAction = OpenAIClient.Responses.ResponseComputerToolCall.Screenshot;\nexport type ComputerUseScrollAction = OpenAIClient.Responses.ResponseComputerToolCall.Scroll;\nexport type ComputerUseTypeAction = OpenAIClient.Responses.ResponseComputerToolCall.Type;\nexport type ComputerUseWaitAction = OpenAIClient.Responses.ResponseComputerToolCall.Wait;\n/**\n * Union type of all computer use actions from OpenAI SDK.\n */\nexport type ComputerUseAction = OpenAIClient.Responses.ResponseComputerToolCall[\"action\"];\ndeclare const ComputerUseScreenshotActionSchema: z.ZodObject<{\n type: z.ZodLiteral<\"screenshot\">;\n}, z.core.$strip>;\ndeclare const ComputerUseClickActionSchema: z.ZodObject<{\n type: z.ZodLiteral<\"click\">;\n x: z.ZodNumber;\n y: z.ZodNumber;\n button: z.ZodDefault<z.ZodEnum<{\n back: \"back\";\n forward: \"forward\";\n left: \"left\";\n right: \"right\";\n wheel: \"wheel\";\n }>>;\n}, z.core.$strip>;\ndeclare const ComputerUseDoubleClickActionSchema: z.ZodObject<{\n type: z.ZodLiteral<\"double_click\">;\n x: z.ZodNumber;\n y: z.ZodNumber;\n button: z.ZodDefault<z.ZodEnum<{\n back: \"back\";\n forward: \"forward\";\n left: \"left\";\n right: \"right\";\n wheel: \"wheel\";\n }>>;\n}, z.core.$strip>;\ndeclare const ComputerUseDragActionSchema: z.ZodObject<{\n type: z.ZodLiteral<\"drag\">;\n path: z.ZodArray<z.ZodObject<{\n x: z.ZodNumber;\n y: z.ZodNumber;\n }, z.core.$strip>>;\n}, z.core.$strip>;\ndeclare const ComputerUseKeypressActionSchema: z.ZodObject<{\n type: z.ZodLiteral<\"keypress\">;\n keys: z.ZodArray<z.ZodString>;\n}, z.core.$strip>;\ndeclare const ComputerUseMoveActionSchema: z.ZodObject<{\n type: z.ZodLiteral<\"move\">;\n x: z.ZodNumber;\n y: z.ZodNumber;\n}, z.core.$strip>;\ndeclare const ComputerUseScrollActionSchema: z.ZodObject<{\n type: z.ZodLiteral<\"scroll\">;\n x: z.ZodNumber;\n y: z.ZodNumber;\n scroll_x: z.ZodNumber;\n scroll_y: z.ZodNumber;\n}, z.core.$strip>;\ndeclare const ComputerUseTypeActionSchema: z.ZodObject<{\n type: z.ZodLiteral<\"type\">;\n text: z.ZodString;\n}, z.core.$strip>;\ndeclare const ComputerUseWaitActionSchema: z.ZodObject<{\n type: z.ZodLiteral<\"wait\">;\n duration: z.ZodOptional<z.ZodNumber>;\n}, z.core.$strip>;\nexport declare const ComputerUseActionSchema: z.ZodObject<{\n action: z.ZodUnion<readonly [z.ZodObject<{\n type: z.ZodLiteral<\"screenshot\">;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"click\">;\n x: z.ZodNumber;\n y: z.ZodNumber;\n button: z.ZodDefault<z.ZodEnum<{\n back: \"back\";\n forward: \"forward\";\n left: \"left\";\n right: \"right\";\n wheel: \"wheel\";\n }>>;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"double_click\">;\n x: z.ZodNumber;\n y: z.ZodNumber;\n button: z.ZodDefault<z.ZodEnum<{\n back: \"back\";\n forward: \"forward\";\n left: \"left\";\n right: \"right\";\n wheel: \"wheel\";\n }>>;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"drag\">;\n path: z.ZodArray<z.ZodObject<{\n x: z.ZodNumber;\n y: z.ZodNumber;\n }, z.core.$strip>>;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"keypress\">;\n keys: z.ZodArray<z.ZodString>;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"move\">;\n x: z.ZodNumber;\n y: z.ZodNumber;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"scroll\">;\n x: z.ZodNumber;\n y: z.ZodNumber;\n scroll_x: z.ZodNumber;\n scroll_y: z.ZodNumber;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"type\">;\n text: z.ZodString;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"wait\">;\n duration: z.ZodOptional<z.ZodNumber>;\n }, z.core.$strip>]>;\n}, z.core.$strip>;\nexport type ComputerUseScreenshotActionType = z.infer<typeof ComputerUseScreenshotActionSchema>;\nexport type ComputerUseClickActionType = z.infer<typeof ComputerUseClickActionSchema>;\nexport type ComputerUseDoubleClickActionType = z.infer<typeof ComputerUseDoubleClickActionSchema>;\nexport type ComputerUseDragActionType = z.infer<typeof ComputerUseDragActionSchema>;\nexport type ComputerUseKeypressActionType = z.infer<typeof ComputerUseKeypressActionSchema>;\nexport type ComputerUseMoveActionType = z.infer<typeof ComputerUseMoveActionSchema>;\nexport type ComputerUseScrollActionType = z.infer<typeof ComputerUseScrollActionSchema>;\nexport type ComputerUseTypeActionType = z.infer<typeof ComputerUseTypeActionSchema>;\nexport type ComputerUseWaitActionType = z.infer<typeof ComputerUseWaitActionSchema>;\n/**\n * Input structure for the Computer Use tool.\n * The action is wrapped in an `action` property.\n */\nexport interface ComputerUseInput {\n action: ComputerUseAction;\n}\nexport type ComputerUseReturnType = string | Promise<string> | ToolMessage<any> | Promise<ToolMessage<any>>;\n/**\n * Options for the Computer Use tool.\n */\nexport interface ComputerUseOptions {\n /**\n * The width of the computer display in pixels.\n */\n displayWidth: number;\n /**\n * The height of the computer display in pixels.\n */\n displayHeight: number;\n /**\n * The type of computer environment to control.\n * - `browser`: Browser automation (recommended for most use cases)\n * - `mac`: macOS environment\n * - `windows`: Windows environment\n * - `linux`: Linux environment\n * - `ubuntu`: Ubuntu environment\n */\n environment: ComputerUseEnvironment;\n /**\n * Execute function that handles computer action execution.\n * This function receives the action input and should return a base64-encoded\n * screenshot of the result.\n */\n execute: (action: ComputerUseAction, runtime: ToolRuntime<any, any>) => ComputerUseReturnType;\n}\n/**\n * OpenAI Computer Use tool type for the Responses API.\n */\nexport type ComputerUseTool = OpenAIClient.Responses.ComputerTool;\n/**\n * Creates a Computer Use tool that allows models to control computer interfaces\n * and perform tasks by simulating mouse clicks, keyboard input, scrolling, and more.\n *\n * **Computer Use** is a practical application of OpenAI's Computer-Using Agent (CUA)\n * model (`computer-use-preview`), which combines vision capabilities with advanced\n * reasoning to simulate controlling computer interfaces.\n *\n * **How it works**:\n * The tool operates in a continuous loop:\n * 1. Model sends computer actions (click, type, scroll, etc.)\n * 2. Your code executes these actions in a controlled environment\n * 3. You capture a screenshot of the result\n * 4. Send the screenshot back to the model\n * 5. Repeat until the task is complete\n *\n * **Important**: Computer use is in beta and requires careful consideration:\n * - Use in sandboxed environments only\n * - Do not use for high-stakes or authenticated tasks\n * - Always implement human-in-the-loop for important decisions\n * - Handle safety checks appropriately\n *\n * @see {@link https://platform.openai.com/docs/guides/tools-computer-use | OpenAI Computer Use Documentation}\n *\n * @param options - Configuration options for the Computer Use tool\n * @returns A Computer Use tool that can be passed to `bindTools`\n *\n * @example\n * ```typescript\n * import { ChatOpenAI, tools } from \"@langchain/openai\";\n *\n * const model = new ChatOpenAI({ model: \"computer-use-preview\" });\n *\n * // With execute callback for automatic action handling\n * const computer = tools.computerUse({\n * displayWidth: 1024,\n * displayHeight: 768,\n * environment: \"browser\",\n * execute: async (action) => {\n * if (action.type === \"screenshot\") {\n * return captureScreenshot();\n * }\n * if (action.type === \"click\") {\n * await page.mouse.click(action.x, action.y, { button: action.button });\n * return captureScreenshot();\n * }\n * if (action.type === \"type\") {\n * await page.keyboard.type(action.text);\n * return captureScreenshot();\n * }\n * // Handle other actions...\n * return captureScreenshot();\n * },\n * });\n *\n * const llmWithComputer = model.bindTools([computer]);\n * const response = await llmWithComputer.invoke(\n * \"Check the latest news on bing.com\"\n * );\n * ```\n *\n * @example\n * ```typescript\n * // Without execute callback (manual action handling)\n * const computer = tools.computerUse({\n * displayWidth: 1024,\n * displayHeight: 768,\n * environment: \"browser\",\n * });\n *\n * const response = await model.invoke(\"Check the news\", {\n * tools: [computer],\n * });\n *\n * // Access the computer call from the response\n * const computerCall = response.additional_kwargs.tool_outputs?.find(\n * (output) => output.type === \"computer_call\"\n * );\n * if (computerCall) {\n * console.log(\"Action to execute:\", computerCall.action);\n * // Execute the action manually, then send back a screenshot\n * }\n * ```\n *\n * @example\n * ```typescript\n * // For macOS desktop automation with Docker\n * const computer = tools.computerUse({\n * displayWidth: 1920,\n * displayHeight: 1080,\n * environment: \"mac\",\n * execute: async (action) => {\n * if (action.type === \"click\") {\n * await dockerExec(\n * `DISPLAY=:99 xdotool mousemove ${action.x} ${action.y} click 1`,\n * containerName\n * );\n * }\n * // Capture screenshot from container\n * return await getDockerScreenshot(containerName);\n * },\n * });\n * ```\n *\n * @remarks\n * - Only available through the Responses API (not Chat Completions)\n * - Requires `computer-use-preview` model\n * - Actions include: click, double_click, drag, keypress, move, screenshot, scroll, type, wait\n * - Safety checks may be returned that require acknowledgment before proceeding\n * - Use `truncation: \"auto\"` parameter when making requests\n * - Recommended to use with `reasoning.summary` for debugging\n */\nexport declare function computerUse(options: ComputerUseOptions): DynamicStructuredTool<z.ZodObject<{\n action: z.ZodUnion<readonly [z.ZodObject<{\n type: z.ZodLiteral<\"screenshot\">;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"click\">;\n x: z.ZodNumber;\n y: z.ZodNumber;\n button: z.ZodDefault<z.ZodEnum<{\n back: \"back\";\n forward: \"forward\";\n left: \"left\";\n right: \"right\";\n wheel: \"wheel\";\n }>>;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"double_click\">;\n x: z.ZodNumber;\n y: z.ZodNumber;\n button: z.ZodDefault<z.ZodEnum<{\n back: \"back\";\n forward: \"forward\";\n left: \"left\";\n right: \"right\";\n wheel: \"wheel\";\n }>>;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"drag\">;\n path: z.ZodArray<z.ZodObject<{\n x: z.ZodNumber;\n y: z.ZodNumber;\n }, z.core.$strip>>;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"keypress\">;\n keys: z.ZodArray<z.ZodString>;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"move\">;\n x: z.ZodNumber;\n y: z.ZodNumber;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"scroll\">;\n x: z.ZodNumber;\n y: z.ZodNumber;\n scroll_x: z.ZodNumber;\n scroll_y: z.ZodNumber;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"type\">;\n text: z.ZodString;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"wait\">;\n duration: z.ZodOptional<z.ZodNumber>;\n }, z.core.$strip>]>;\n}, z.core.$strip>, ComputerUseInput, unknown, ComputerUseReturnType, string>;\nexport {};\n//# sourceMappingURL=computerUse.d.ts.map"],"mappings":";;;;;;;;AAQA;AAIA;AACYW,KALAL,sBAAAA,GAK4B,SAAA,GAAGJ,KAAAA,GAAaM,SAAAA,GAAUC,OAAAA,GAAAA,QAAAA;AAClE;AACA;AACA;AACYU,KALAZ,sBAAAA,GAAyBL,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBC,KAKpBD;AACrDY,KALAV,4BAAAA,GAA+BT,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBG,WAKC;AAChFW,KALAV,qBAAAA,GAAwBX,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBK,IAKAU;AACxEC,KALAV,yBAAAA,GAA4Bb,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBO,QAKA;AAI5EW,KARAV,qBAAAA,GAAwBf,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBS,IAQL;AA4H9DyC,KAnILxC,2BAAAA,GAA8BjB,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBW,UAoI7D;AAEjBwC,KArIAvC,uBAAAA,GAA0BnB,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBa,MAqIrD;AAAYuC,KApIjCtC,qBAAAA,GAAwBrB,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBe,IAoIvCqC;AAAkBxD,KAnInDoB,qBAAAA,GAAwBvB,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBiB,IAmIrBrB;;;AAA0B;AAIxEyD,KAnILnC,iBAAAA,GAAoBzB,QAAAA,CAAaM,SAAAA,CAAUC,wBAmIpB,CAAA,QAAA,CAAA;;;;;UAPlBkD,gBAAAA;UACLhC;;KAEAiC,qBAAAA,YAAiCC,kBAAkBxD,mBAAmBwD,QAAQxD;;;;UAIzEyD,kBAAAA;;;;;;;;;;;;;;;;;eAiBAxD;;;;;;oBAMKqB,4BAA4BvB,0BAA0BwD;;;;;KAKhEG,eAAAA,GAAkB7D,QAAAA,CAAaM,SAAAA,CAAUwD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAiH7BC,WAAAA,UAAqBH,qBAAqB3D,sBAAsBH,CAAAA,CAAEgC;UAC9EhC,CAAAA,CAAEgD,mBAAmBhD,CAAAA,CAAEgC;UACrBhC,CAAAA,CAAE6B;KACT7B,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS/B,CAAAA,CAAEgC;UACXhC,CAAAA,CAAE6B;OACL7B,CAAAA,CAAEkC;OACFlC,CAAAA,CAAEkC;YACGlC,CAAAA,CAAEoC,WAAWpC,CAAAA,CAAEmC;;;;;;;KAOxBnC,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS/B,CAAAA,CAAEgC;UACXhC,CAAAA,CAAE6B;OACL7B,CAAAA,CAAEkC;OACFlC,CAAAA,CAAEkC;YACGlC,CAAAA,CAAEoC,WAAWpC,CAAAA,CAAEmC;;;;;;;KAOxBnC,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS/B,CAAAA,CAAEgC;UACXhC,CAAAA,CAAE6B;UACF7B,CAAAA,CAAEuC,SAASvC,CAAAA,CAAEgC;SACZhC,CAAAA,CAAEkC;SACFlC,CAAAA,CAAEkC;OACNlC,CAAAA,CAAE8B,IAAAA,CAAKC;KACX/B,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS/B,CAAAA,CAAEgC;UACXhC,CAAAA,CAAE6B;UACF7B,CAAAA,CAAEuC,SAASvC,CAAAA,CAAEyC;KACpBzC,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS/B,CAAAA,CAAEgC;UACXhC,CAAAA,CAAE6B;OACL7B,CAAAA,CAAEkC;OACFlC,CAAAA,CAAEkC;KACNlC,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS/B,CAAAA,CAAEgC;UACXhC,CAAAA,CAAE6B;OACL7B,CAAAA,CAAEkC;OACFlC,CAAAA,CAAEkC;cACKlC,CAAAA,CAAEkC;cACFlC,CAAAA,CAAEkC;KACblC,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS/B,CAAAA,CAAEgC;UACXhC,CAAAA,CAAE6B;UACF7B,CAAAA,CAAEyC;KACTzC,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS/B,CAAAA,CAAEgC;UACXhC,CAAAA,CAAE6B;cACE7B,CAAAA,CAAE8C,YAAY9C,CAAAA,CAAEkC;KAC3BlC,CAAAA,CAAE8B,IAAAA,CAAKC;GACX/B,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS4B,2BAA2BC"}
|
|
1
|
+
{"version":3,"file":"computerUse.d.cts","names":[],"sources":["../../src/tools/computerUse.ts"],"mappings":";;;;;;;AAcA;;KAAY,sBAAA;;;AAUZ;KAAY,sBAAA,GACV,QAAA,CAAa,SAAA,CAAU,wBAAA,CAAyB,KAAA;AAAA,KACtC,4BAAA,GACV,QAAA,CAAa,SAAA,CAAU,wBAAA,CAAyB,WAAA;AAAA,KACtC,qBAAA,GACV,QAAA,CAAa,SAAA,CAAU,wBAAA,CAAyB,IAAA;AAAA,KACtC,yBAAA,GACV,QAAA,CAAa,SAAA,CAAU,wBAAA,CAAyB,QAAA;AAAA,KACtC,qBAAA,GACV,QAAA,CAAa,SAAA,CAAU,wBAAA,CAAyB,IAAA;AAAA,KACtC,2BAAA,GACV,QAAA,CAAa,SAAA,CAAU,wBAAA,CAAyB,UAAA;AAAA,KACtC,uBAAA,GACV,QAAA,CAAa,SAAA,CAAU,wBAAA,CAAyB,MAAA;AAAA,KACtC,qBAAA,GACV,QAAA,CAAa,SAAA,CAAU,wBAAA,CAAyB,IAAA;AAAA,KACtC,qBAAA,GACV,QAAA,CAAa,SAAA,CAAU,wBAAA,CAAyB,IAAA;AAflD;;;AAAA,KAoBY,iBAAA,GACV,QAAA,CAAa,SAAA,CAAU,wBAAA;;;;;UA2GR,gBAAA;EACf,MAAA,EAAQ,iBAAA;AAAA;AAAA,KAGE,qBAAA,YAER,OAAA,WACA,WAAA,QACA,OAAA,CAAQ,WAAA;;;;UAKK,kBAAA;EAmC0B;;;EA/BzC,YAAA;EAmJc;;;EA9Id,aAAA;;;;;;;;;EAUA,WAAA,EAAa,sBAAA;;;;;;EAOb,OAAA,GACE,MAAA,EAAQ,iBAAA,EACR,OAAA,EAAS,WAAA,eACN,qBAAA;AAAA;;;;KAMK,eAAA,GAAkB,QAAA,CAAa,SAAA,CAAU,YAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAoHrC,WAAA,CAAY,OAAA,EAAS,kBAAA,GAAkB,qBAAA,CAAA,CAAA,CAAA,SAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"computerUse.d.ts","names":["z","OpenAI","OpenAIClient","DynamicStructuredTool","ToolRuntime","ToolMessage","ComputerUseEnvironment","ComputerUseClickAction","Responses","ResponseComputerToolCall","Click","ComputerUseDoubleClickAction","DoubleClick","ComputerUseDragAction","Drag","ComputerUseKeypressAction","Keypress","ComputerUseMoveAction","Move","ComputerUseScreenshotAction","Screenshot","ComputerUseScrollAction","Scroll","ComputerUseTypeAction","Type","ComputerUseWaitAction","Wait","ComputerUseAction","ComputerUseScreenshotActionSchema","ZodLiteral","core","$strip","ZodObject","ComputerUseClickActionSchema","ZodNumber","ZodEnum","ZodDefault","ComputerUseDoubleClickActionSchema","ComputerUseDragActionSchema","ZodArray","ComputerUseKeypressActionSchema","ZodString","ComputerUseMoveActionSchema","ComputerUseScrollActionSchema","ComputerUseTypeActionSchema","ComputerUseWaitActionSchema","ZodOptional","ComputerUseActionSchema","ZodUnion","ComputerUseScreenshotActionType","infer","ComputerUseClickActionType","ComputerUseDoubleClickActionType","ComputerUseDragActionType","ComputerUseKeypressActionType","ComputerUseMoveActionType","ComputerUseScrollActionType","ComputerUseTypeActionType","ComputerUseWaitActionType","ComputerUseInput","ComputerUseReturnType","Promise","ComputerUseOptions","ComputerUseTool","ComputerTool","computerUse"],"sources":["../../src/tools/computerUse.d.ts"],"sourcesContent":["import { z } from \"zod/v4\";\nimport { OpenAI as OpenAIClient } from \"openai\";\nimport { type DynamicStructuredTool } from \"@langchain/core/tools\";\nimport { type ToolRuntime } from \"@langchain/core/tools\";\nimport { ToolMessage } from \"@langchain/core/messages\";\n/**\n * The type of computer environment to control.\n */\nexport type ComputerUseEnvironment = \"browser\" | \"mac\" | \"windows\" | \"linux\" | \"ubuntu\";\n/**\n * Re-export action types from OpenAI SDK for convenience.\n */\nexport type ComputerUseClickAction = OpenAIClient.Responses.ResponseComputerToolCall.Click;\nexport type ComputerUseDoubleClickAction = OpenAIClient.Responses.ResponseComputerToolCall.DoubleClick;\nexport type ComputerUseDragAction = OpenAIClient.Responses.ResponseComputerToolCall.Drag;\nexport type ComputerUseKeypressAction = OpenAIClient.Responses.ResponseComputerToolCall.Keypress;\nexport type ComputerUseMoveAction = OpenAIClient.Responses.ResponseComputerToolCall.Move;\nexport type ComputerUseScreenshotAction = OpenAIClient.Responses.ResponseComputerToolCall.Screenshot;\nexport type ComputerUseScrollAction = OpenAIClient.Responses.ResponseComputerToolCall.Scroll;\nexport type ComputerUseTypeAction = OpenAIClient.Responses.ResponseComputerToolCall.Type;\nexport type ComputerUseWaitAction = OpenAIClient.Responses.ResponseComputerToolCall.Wait;\n/**\n * Union type of all computer use actions from OpenAI SDK.\n */\nexport type ComputerUseAction = OpenAIClient.Responses.ResponseComputerToolCall[\"action\"];\ndeclare const ComputerUseScreenshotActionSchema: z.ZodObject<{\n type: z.ZodLiteral<\"screenshot\">;\n}, z.core.$strip>;\ndeclare const ComputerUseClickActionSchema: z.ZodObject<{\n type: z.ZodLiteral<\"click\">;\n x: z.ZodNumber;\n y: z.ZodNumber;\n button: z.ZodDefault<z.ZodEnum<{\n back: \"back\";\n forward: \"forward\";\n left: \"left\";\n right: \"right\";\n wheel: \"wheel\";\n }>>;\n}, z.core.$strip>;\ndeclare const ComputerUseDoubleClickActionSchema: z.ZodObject<{\n type: z.ZodLiteral<\"double_click\">;\n x: z.ZodNumber;\n y: z.ZodNumber;\n button: z.ZodDefault<z.ZodEnum<{\n back: \"back\";\n forward: \"forward\";\n left: \"left\";\n right: \"right\";\n wheel: \"wheel\";\n }>>;\n}, z.core.$strip>;\ndeclare const ComputerUseDragActionSchema: z.ZodObject<{\n type: z.ZodLiteral<\"drag\">;\n path: z.ZodArray<z.ZodObject<{\n x: z.ZodNumber;\n y: z.ZodNumber;\n }, z.core.$strip>>;\n}, z.core.$strip>;\ndeclare const ComputerUseKeypressActionSchema: z.ZodObject<{\n type: z.ZodLiteral<\"keypress\">;\n keys: z.ZodArray<z.ZodString>;\n}, z.core.$strip>;\ndeclare const ComputerUseMoveActionSchema: z.ZodObject<{\n type: z.ZodLiteral<\"move\">;\n x: z.ZodNumber;\n y: z.ZodNumber;\n}, z.core.$strip>;\ndeclare const ComputerUseScrollActionSchema: z.ZodObject<{\n type: z.ZodLiteral<\"scroll\">;\n x: z.ZodNumber;\n y: z.ZodNumber;\n scroll_x: z.ZodNumber;\n scroll_y: z.ZodNumber;\n}, z.core.$strip>;\ndeclare const ComputerUseTypeActionSchema: z.ZodObject<{\n type: z.ZodLiteral<\"type\">;\n text: z.ZodString;\n}, z.core.$strip>;\ndeclare const ComputerUseWaitActionSchema: z.ZodObject<{\n type: z.ZodLiteral<\"wait\">;\n duration: z.ZodOptional<z.ZodNumber>;\n}, z.core.$strip>;\nexport declare const ComputerUseActionSchema: z.ZodObject<{\n action: z.ZodUnion<readonly [z.ZodObject<{\n type: z.ZodLiteral<\"screenshot\">;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"click\">;\n x: z.ZodNumber;\n y: z.ZodNumber;\n button: z.ZodDefault<z.ZodEnum<{\n back: \"back\";\n forward: \"forward\";\n left: \"left\";\n right: \"right\";\n wheel: \"wheel\";\n }>>;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"double_click\">;\n x: z.ZodNumber;\n y: z.ZodNumber;\n button: z.ZodDefault<z.ZodEnum<{\n back: \"back\";\n forward: \"forward\";\n left: \"left\";\n right: \"right\";\n wheel: \"wheel\";\n }>>;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"drag\">;\n path: z.ZodArray<z.ZodObject<{\n x: z.ZodNumber;\n y: z.ZodNumber;\n }, z.core.$strip>>;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"keypress\">;\n keys: z.ZodArray<z.ZodString>;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"move\">;\n x: z.ZodNumber;\n y: z.ZodNumber;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"scroll\">;\n x: z.ZodNumber;\n y: z.ZodNumber;\n scroll_x: z.ZodNumber;\n scroll_y: z.ZodNumber;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"type\">;\n text: z.ZodString;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"wait\">;\n duration: z.ZodOptional<z.ZodNumber>;\n }, z.core.$strip>]>;\n}, z.core.$strip>;\nexport type ComputerUseScreenshotActionType = z.infer<typeof ComputerUseScreenshotActionSchema>;\nexport type ComputerUseClickActionType = z.infer<typeof ComputerUseClickActionSchema>;\nexport type ComputerUseDoubleClickActionType = z.infer<typeof ComputerUseDoubleClickActionSchema>;\nexport type ComputerUseDragActionType = z.infer<typeof ComputerUseDragActionSchema>;\nexport type ComputerUseKeypressActionType = z.infer<typeof ComputerUseKeypressActionSchema>;\nexport type ComputerUseMoveActionType = z.infer<typeof ComputerUseMoveActionSchema>;\nexport type ComputerUseScrollActionType = z.infer<typeof ComputerUseScrollActionSchema>;\nexport type ComputerUseTypeActionType = z.infer<typeof ComputerUseTypeActionSchema>;\nexport type ComputerUseWaitActionType = z.infer<typeof ComputerUseWaitActionSchema>;\n/**\n * Input structure for the Computer Use tool.\n * The action is wrapped in an `action` property.\n */\nexport interface ComputerUseInput {\n action: ComputerUseAction;\n}\nexport type ComputerUseReturnType = string | Promise<string> | ToolMessage<any> | Promise<ToolMessage<any>>;\n/**\n * Options for the Computer Use tool.\n */\nexport interface ComputerUseOptions {\n /**\n * The width of the computer display in pixels.\n */\n displayWidth: number;\n /**\n * The height of the computer display in pixels.\n */\n displayHeight: number;\n /**\n * The type of computer environment to control.\n * - `browser`: Browser automation (recommended for most use cases)\n * - `mac`: macOS environment\n * - `windows`: Windows environment\n * - `linux`: Linux environment\n * - `ubuntu`: Ubuntu environment\n */\n environment: ComputerUseEnvironment;\n /**\n * Execute function that handles computer action execution.\n * This function receives the action input and should return a base64-encoded\n * screenshot of the result.\n */\n execute: (action: ComputerUseAction, runtime: ToolRuntime<any, any>) => ComputerUseReturnType;\n}\n/**\n * OpenAI Computer Use tool type for the Responses API.\n */\nexport type ComputerUseTool = OpenAIClient.Responses.ComputerTool;\n/**\n * Creates a Computer Use tool that allows models to control computer interfaces\n * and perform tasks by simulating mouse clicks, keyboard input, scrolling, and more.\n *\n * **Computer Use** is a practical application of OpenAI's Computer-Using Agent (CUA)\n * model (`computer-use-preview`), which combines vision capabilities with advanced\n * reasoning to simulate controlling computer interfaces.\n *\n * **How it works**:\n * The tool operates in a continuous loop:\n * 1. Model sends computer actions (click, type, scroll, etc.)\n * 2. Your code executes these actions in a controlled environment\n * 3. You capture a screenshot of the result\n * 4. Send the screenshot back to the model\n * 5. Repeat until the task is complete\n *\n * **Important**: Computer use is in beta and requires careful consideration:\n * - Use in sandboxed environments only\n * - Do not use for high-stakes or authenticated tasks\n * - Always implement human-in-the-loop for important decisions\n * - Handle safety checks appropriately\n *\n * @see {@link https://platform.openai.com/docs/guides/tools-computer-use | OpenAI Computer Use Documentation}\n *\n * @param options - Configuration options for the Computer Use tool\n * @returns A Computer Use tool that can be passed to `bindTools`\n *\n * @example\n * ```typescript\n * import { ChatOpenAI, tools } from \"@langchain/openai\";\n *\n * const model = new ChatOpenAI({ model: \"computer-use-preview\" });\n *\n * // With execute callback for automatic action handling\n * const computer = tools.computerUse({\n * displayWidth: 1024,\n * displayHeight: 768,\n * environment: \"browser\",\n * execute: async (action) => {\n * if (action.type === \"screenshot\") {\n * return captureScreenshot();\n * }\n * if (action.type === \"click\") {\n * await page.mouse.click(action.x, action.y, { button: action.button });\n * return captureScreenshot();\n * }\n * if (action.type === \"type\") {\n * await page.keyboard.type(action.text);\n * return captureScreenshot();\n * }\n * // Handle other actions...\n * return captureScreenshot();\n * },\n * });\n *\n * const llmWithComputer = model.bindTools([computer]);\n * const response = await llmWithComputer.invoke(\n * \"Check the latest news on bing.com\"\n * );\n * ```\n *\n * @example\n * ```typescript\n * // Without execute callback (manual action handling)\n * const computer = tools.computerUse({\n * displayWidth: 1024,\n * displayHeight: 768,\n * environment: \"browser\",\n * });\n *\n * const response = await model.invoke(\"Check the news\", {\n * tools: [computer],\n * });\n *\n * // Access the computer call from the response\n * const computerCall = response.additional_kwargs.tool_outputs?.find(\n * (output) => output.type === \"computer_call\"\n * );\n * if (computerCall) {\n * console.log(\"Action to execute:\", computerCall.action);\n * // Execute the action manually, then send back a screenshot\n * }\n * ```\n *\n * @example\n * ```typescript\n * // For macOS desktop automation with Docker\n * const computer = tools.computerUse({\n * displayWidth: 1920,\n * displayHeight: 1080,\n * environment: \"mac\",\n * execute: async (action) => {\n * if (action.type === \"click\") {\n * await dockerExec(\n * `DISPLAY=:99 xdotool mousemove ${action.x} ${action.y} click 1`,\n * containerName\n * );\n * }\n * // Capture screenshot from container\n * return await getDockerScreenshot(containerName);\n * },\n * });\n * ```\n *\n * @remarks\n * - Only available through the Responses API (not Chat Completions)\n * - Requires `computer-use-preview` model\n * - Actions include: click, double_click, drag, keypress, move, screenshot, scroll, type, wait\n * - Safety checks may be returned that require acknowledgment before proceeding\n * - Use `truncation: \"auto\"` parameter when making requests\n * - Recommended to use with `reasoning.summary` for debugging\n */\nexport declare function computerUse(options: ComputerUseOptions): DynamicStructuredTool<z.ZodObject<{\n action: z.ZodUnion<readonly [z.ZodObject<{\n type: z.ZodLiteral<\"screenshot\">;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"click\">;\n x: z.ZodNumber;\n y: z.ZodNumber;\n button: z.ZodDefault<z.ZodEnum<{\n back: \"back\";\n forward: \"forward\";\n left: \"left\";\n right: \"right\";\n wheel: \"wheel\";\n }>>;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"double_click\">;\n x: z.ZodNumber;\n y: z.ZodNumber;\n button: z.ZodDefault<z.ZodEnum<{\n back: \"back\";\n forward: \"forward\";\n left: \"left\";\n right: \"right\";\n wheel: \"wheel\";\n }>>;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"drag\">;\n path: z.ZodArray<z.ZodObject<{\n x: z.ZodNumber;\n y: z.ZodNumber;\n }, z.core.$strip>>;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"keypress\">;\n keys: z.ZodArray<z.ZodString>;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"move\">;\n x: z.ZodNumber;\n y: z.ZodNumber;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"scroll\">;\n x: z.ZodNumber;\n y: z.ZodNumber;\n scroll_x: z.ZodNumber;\n scroll_y: z.ZodNumber;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"type\">;\n text: z.ZodString;\n }, z.core.$strip>, z.ZodObject<{\n type: z.ZodLiteral<\"wait\">;\n duration: z.ZodOptional<z.ZodNumber>;\n }, z.core.$strip>]>;\n}, z.core.$strip>, ComputerUseInput, unknown, ComputerUseReturnType, string>;\nexport {};\n//# sourceMappingURL=computerUse.d.ts.map"],"mappings":";;;;;;;;AAQA;AAIA;AACYW,KALAL,sBAAAA,GAK4B,SAAA,GAAGJ,KAAAA,GAAaM,SAAAA,GAAUC,OAAAA,GAAAA,QAAAA;AAClE;AACA;AACA;AACYU,KALAZ,sBAAAA,GAAyBL,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBC,KAKpBD;AACrDY,KALAV,4BAAAA,GAA+BT,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBG,WAKC;AAChFW,KALAV,qBAAAA,GAAwBX,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBK,IAKAU;AACxEC,KALAV,yBAAAA,GAA4Bb,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBO,QAKA;AAI5EW,KARAV,qBAAAA,GAAwBf,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBS,IAQL;AA4H9DyC,KAnILxC,2BAAAA,GAA8BjB,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBW,UAoI7D;AAEjBwC,KArIAvC,uBAAAA,GAA0BnB,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBa,MAqIrD;AAAYuC,KApIjCtC,qBAAAA,GAAwBrB,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBe,IAoIvCqC;AAAkBxD,KAnInDoB,qBAAAA,GAAwBvB,QAAAA,CAAaM,SAAAA,CAAUC,wBAAAA,CAAyBiB,IAmIrBrB;;;AAA0B;AAIxEyD,KAnILnC,iBAAAA,GAAoBzB,QAAAA,CAAaM,SAAAA,CAAUC,wBAmIpB,CAAA,QAAA,CAAA;;;;;UAPlBkD,gBAAAA;UACLhC;;KAEAiC,qBAAAA,YAAiCC,kBAAkBxD,mBAAmBwD,QAAQxD;;;;UAIzEyD,kBAAAA;;;;;;;;;;;;;;;;;eAiBAxD;;;;;;oBAMKqB,4BAA4BvB,0BAA0BwD;;;;;KAKhEG,eAAAA,GAAkB7D,QAAAA,CAAaM,SAAAA,CAAUwD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAiH7BC,WAAAA,UAAqBH,qBAAqB3D,sBAAsBH,CAAAA,CAAEgC;UAC9EhC,CAAAA,CAAEgD,mBAAmBhD,CAAAA,CAAEgC;UACrBhC,CAAAA,CAAE6B;KACT7B,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS/B,CAAAA,CAAEgC;UACXhC,CAAAA,CAAE6B;OACL7B,CAAAA,CAAEkC;OACFlC,CAAAA,CAAEkC;YACGlC,CAAAA,CAAEoC,WAAWpC,CAAAA,CAAEmC;;;;;;;KAOxBnC,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS/B,CAAAA,CAAEgC;UACXhC,CAAAA,CAAE6B;OACL7B,CAAAA,CAAEkC;OACFlC,CAAAA,CAAEkC;YACGlC,CAAAA,CAAEoC,WAAWpC,CAAAA,CAAEmC;;;;;;;KAOxBnC,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS/B,CAAAA,CAAEgC;UACXhC,CAAAA,CAAE6B;UACF7B,CAAAA,CAAEuC,SAASvC,CAAAA,CAAEgC;SACZhC,CAAAA,CAAEkC;SACFlC,CAAAA,CAAEkC;OACNlC,CAAAA,CAAE8B,IAAAA,CAAKC;KACX/B,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS/B,CAAAA,CAAEgC;UACXhC,CAAAA,CAAE6B;UACF7B,CAAAA,CAAEuC,SAASvC,CAAAA,CAAEyC;KACpBzC,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS/B,CAAAA,CAAEgC;UACXhC,CAAAA,CAAE6B;OACL7B,CAAAA,CAAEkC;OACFlC,CAAAA,CAAEkC;KACNlC,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS/B,CAAAA,CAAEgC;UACXhC,CAAAA,CAAE6B;OACL7B,CAAAA,CAAEkC;OACFlC,CAAAA,CAAEkC;cACKlC,CAAAA,CAAEkC;cACFlC,CAAAA,CAAEkC;KACblC,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS/B,CAAAA,CAAEgC;UACXhC,CAAAA,CAAE6B;UACF7B,CAAAA,CAAEyC;KACTzC,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS/B,CAAAA,CAAEgC;UACXhC,CAAAA,CAAE6B;cACE7B,CAAAA,CAAE8C,YAAY9C,CAAAA,CAAEkC;KAC3BlC,CAAAA,CAAE8B,IAAAA,CAAKC;GACX/B,CAAAA,CAAE8B,IAAAA,CAAKC,SAAS4B,2BAA2BC"}
|
|
1
|
+
{"version":3,"file":"computerUse.d.ts","names":[],"sources":["../../src/tools/computerUse.ts"],"mappings":";;;;;;;AAcA;;KAAY,sBAAA;;;AAUZ;KAAY,sBAAA,GACV,QAAA,CAAa,SAAA,CAAU,wBAAA,CAAyB,KAAA;AAAA,KACtC,4BAAA,GACV,QAAA,CAAa,SAAA,CAAU,wBAAA,CAAyB,WAAA;AAAA,KACtC,qBAAA,GACV,QAAA,CAAa,SAAA,CAAU,wBAAA,CAAyB,IAAA;AAAA,KACtC,yBAAA,GACV,QAAA,CAAa,SAAA,CAAU,wBAAA,CAAyB,QAAA;AAAA,KACtC,qBAAA,GACV,QAAA,CAAa,SAAA,CAAU,wBAAA,CAAyB,IAAA;AAAA,KACtC,2BAAA,GACV,QAAA,CAAa,SAAA,CAAU,wBAAA,CAAyB,UAAA;AAAA,KACtC,uBAAA,GACV,QAAA,CAAa,SAAA,CAAU,wBAAA,CAAyB,MAAA;AAAA,KACtC,qBAAA,GACV,QAAA,CAAa,SAAA,CAAU,wBAAA,CAAyB,IAAA;AAAA,KACtC,qBAAA,GACV,QAAA,CAAa,SAAA,CAAU,wBAAA,CAAyB,IAAA;AAflD;;;AAAA,KAoBY,iBAAA,GACV,QAAA,CAAa,SAAA,CAAU,wBAAA;;;;;UA2GR,gBAAA;EACf,MAAA,EAAQ,iBAAA;AAAA;AAAA,KAGE,qBAAA,YAER,OAAA,WACA,WAAA,QACA,OAAA,CAAQ,WAAA;;;;UAKK,kBAAA;EAmC0B;;;EA/BzC,YAAA;EAmJc;;;EA9Id,aAAA;;;;;;;;;EAUA,WAAA,EAAa,sBAAA;;;;;;EAOb,OAAA,GACE,MAAA,EAAQ,iBAAA,EACR,OAAA,EAAS,WAAA,eACN,qBAAA;AAAA;;;;KAMK,eAAA,GAAkB,QAAA,CAAa,SAAA,CAAU,YAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAoHrC,WAAA,CAAY,OAAA,EAAS,kBAAA,GAAkB,qBAAA,CAAA,CAAA,CAAA,SAAA"}
|
|
@@ -186,12 +186,7 @@ const TOOL_NAME = "computer_use";
|
|
|
186
186
|
*/
|
|
187
187
|
function computerUse(options) {
|
|
188
188
|
const computerTool = tool(async (input, runtime) => {
|
|
189
|
-
|
|
190
|
-
* get computer_use call id from runtime
|
|
191
|
-
*/
|
|
192
|
-
const aiMessage = runtime.state?.messages.at(-1);
|
|
193
|
-
const computerToolCall = aiMessage?.tool_calls?.find((tc) => tc.name === "computer_use");
|
|
194
|
-
const computerToolCallId = computerToolCall?.id;
|
|
189
|
+
const computerToolCallId = ((runtime.state?.messages.at(-1))?.tool_calls?.find((tc) => tc.name === "computer_use"))?.id;
|
|
195
190
|
if (!computerToolCallId) throw new Error("Computer use call id not found");
|
|
196
191
|
const result = await options.execute(input.action, runtime);
|
|
197
192
|
/**
|