@langchain/openai 1.5.5 → 1.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/azure/chat_models/common.cjs.map +1 -1
- package/dist/azure/chat_models/common.d.cts +1 -1
- package/dist/azure/chat_models/common.d.cts.map +1 -1
- package/dist/azure/chat_models/common.d.ts +1 -1
- package/dist/azure/chat_models/common.d.ts.map +1 -1
- package/dist/azure/chat_models/common.js.map +1 -1
- package/dist/azure/chat_models/completions.cjs.map +1 -1
- package/dist/azure/chat_models/completions.d.cts +0 -1
- package/dist/azure/chat_models/completions.d.cts.map +1 -1
- package/dist/azure/chat_models/completions.d.ts +0 -1
- package/dist/azure/chat_models/completions.d.ts.map +1 -1
- package/dist/azure/chat_models/completions.js.map +1 -1
- package/dist/azure/chat_models/index.cjs.map +1 -1
- package/dist/azure/chat_models/index.d.cts +1 -2
- package/dist/azure/chat_models/index.d.cts.map +1 -1
- package/dist/azure/chat_models/index.d.ts +0 -1
- package/dist/azure/chat_models/index.d.ts.map +1 -1
- package/dist/azure/chat_models/index.js.map +1 -1
- package/dist/azure/chat_models/responses.cjs.map +1 -1
- package/dist/azure/chat_models/responses.d.cts +0 -1
- package/dist/azure/chat_models/responses.d.cts.map +1 -1
- package/dist/azure/chat_models/responses.d.ts +0 -1
- package/dist/azure/chat_models/responses.d.ts.map +1 -1
- package/dist/azure/chat_models/responses.js.map +1 -1
- package/dist/azure/embeddings.cjs.map +1 -1
- package/dist/azure/embeddings.d.cts +2 -2
- package/dist/azure/embeddings.d.cts.map +1 -1
- package/dist/azure/embeddings.d.ts +2 -2
- package/dist/azure/embeddings.d.ts.map +1 -1
- package/dist/azure/embeddings.js.map +1 -1
- package/dist/azure/llms.cjs.map +1 -1
- package/dist/azure/llms.d.cts +0 -1
- package/dist/azure/llms.d.cts.map +1 -1
- package/dist/azure/llms.d.ts +0 -1
- package/dist/azure/llms.d.ts.map +1 -1
- package/dist/azure/llms.js.map +1 -1
- package/dist/chat_models/base.cjs +37 -3
- package/dist/chat_models/base.cjs.map +1 -1
- package/dist/chat_models/base.d.cts +6 -7
- package/dist/chat_models/base.d.cts.map +1 -1
- package/dist/chat_models/base.d.ts +5 -6
- package/dist/chat_models/base.d.ts.map +1 -1
- package/dist/chat_models/base.js +37 -3
- package/dist/chat_models/base.js.map +1 -1
- package/dist/chat_models/completions.cjs.map +1 -1
- package/dist/chat_models/completions.d.cts +0 -1
- package/dist/chat_models/completions.d.cts.map +1 -1
- package/dist/chat_models/completions.d.ts +0 -1
- package/dist/chat_models/completions.d.ts.map +1 -1
- package/dist/chat_models/completions.js.map +1 -1
- package/dist/chat_models/index.cjs.map +1 -1
- package/dist/chat_models/index.d.cts +14 -17
- package/dist/chat_models/index.d.cts.map +1 -1
- package/dist/chat_models/index.d.ts +14 -17
- package/dist/chat_models/index.d.ts.map +1 -1
- package/dist/chat_models/index.js.map +1 -1
- package/dist/chat_models/profiles.cjs.map +1 -1
- package/dist/chat_models/profiles.js.map +1 -1
- package/dist/chat_models/responses.cjs.map +1 -1
- package/dist/chat_models/responses.d.cts +0 -1
- package/dist/chat_models/responses.d.cts.map +1 -1
- package/dist/chat_models/responses.d.ts +0 -1
- package/dist/chat_models/responses.d.ts.map +1 -1
- package/dist/chat_models/responses.js.map +1 -1
- package/dist/converters/completions.cjs +2 -4
- package/dist/converters/completions.cjs.map +1 -1
- package/dist/converters/completions.d.cts +0 -1
- package/dist/converters/completions.d.cts.map +1 -1
- package/dist/converters/completions.d.ts +0 -1
- package/dist/converters/completions.d.ts.map +1 -1
- package/dist/converters/completions.js +2 -4
- package/dist/converters/completions.js.map +1 -1
- package/dist/converters/index.cjs +14 -2
- package/dist/converters/index.d.cts +2 -1
- package/dist/converters/index.d.ts +2 -1
- package/dist/converters/index.js +3 -3
- package/dist/converters/responses.cjs.map +1 -1
- package/dist/converters/responses.d.cts +0 -1
- package/dist/converters/responses.d.cts.map +1 -1
- package/dist/converters/responses.d.ts +0 -1
- package/dist/converters/responses.d.ts.map +1 -1
- package/dist/converters/responses.js.map +1 -1
- package/dist/embeddings.cjs.map +1 -1
- package/dist/embeddings.d.cts +2 -2
- package/dist/embeddings.d.cts.map +1 -1
- package/dist/embeddings.d.ts +2 -2
- package/dist/embeddings.d.ts.map +1 -1
- package/dist/embeddings.js.map +1 -1
- package/dist/index.d.cts +2 -1
- package/dist/index.d.ts +2 -1
- package/dist/llms.cjs +1 -1
- package/dist/llms.cjs.map +1 -1
- package/dist/llms.d.cts +1 -2
- package/dist/llms.d.cts.map +1 -1
- package/dist/llms.d.ts +1 -2
- package/dist/llms.d.ts.map +1 -1
- package/dist/llms.js +1 -1
- package/dist/llms.js.map +1 -1
- package/dist/tools/applyPatch.cjs +4 -0
- package/dist/tools/applyPatch.cjs.map +1 -1
- package/dist/tools/applyPatch.d.cts +1 -2
- package/dist/tools/applyPatch.d.cts.map +1 -1
- package/dist/tools/applyPatch.d.ts +1 -2
- package/dist/tools/applyPatch.d.ts.map +1 -1
- package/dist/tools/applyPatch.js +1 -1
- package/dist/tools/applyPatch.js.map +1 -1
- package/dist/tools/codeInterpreter.cjs.map +1 -1
- package/dist/tools/codeInterpreter.d.cts +0 -1
- package/dist/tools/codeInterpreter.d.cts.map +1 -1
- package/dist/tools/codeInterpreter.d.ts +0 -1
- package/dist/tools/codeInterpreter.d.ts.map +1 -1
- package/dist/tools/codeInterpreter.js.map +1 -1
- package/dist/tools/computerUse.cjs +1 -0
- package/dist/tools/computerUse.cjs.map +1 -1
- package/dist/tools/computerUse.d.cts +1 -2
- package/dist/tools/computerUse.d.cts.map +1 -1
- package/dist/tools/computerUse.d.ts +1 -2
- package/dist/tools/computerUse.d.ts.map +1 -1
- package/dist/tools/computerUse.js +1 -1
- package/dist/tools/computerUse.js.map +1 -1
- package/dist/tools/custom.cjs.map +1 -1
- package/dist/tools/custom.d.cts +1 -2
- package/dist/tools/custom.d.cts.map +1 -1
- package/dist/tools/custom.d.ts +1 -2
- package/dist/tools/custom.d.ts.map +1 -1
- package/dist/tools/custom.js.map +1 -1
- package/dist/tools/dalle.cjs.map +1 -1
- package/dist/tools/dalle.d.cts +0 -1
- package/dist/tools/dalle.d.cts.map +1 -1
- package/dist/tools/dalle.d.ts +0 -1
- package/dist/tools/dalle.d.ts.map +1 -1
- package/dist/tools/dalle.js.map +1 -1
- package/dist/tools/fileSearch.cjs.map +1 -1
- package/dist/tools/fileSearch.d.cts +0 -1
- package/dist/tools/fileSearch.d.cts.map +1 -1
- package/dist/tools/fileSearch.d.ts +0 -1
- package/dist/tools/fileSearch.d.ts.map +1 -1
- package/dist/tools/fileSearch.js.map +1 -1
- package/dist/tools/imageGeneration.cjs.map +1 -1
- package/dist/tools/imageGeneration.d.cts +0 -1
- package/dist/tools/imageGeneration.d.cts.map +1 -1
- package/dist/tools/imageGeneration.d.ts +0 -1
- package/dist/tools/imageGeneration.d.ts.map +1 -1
- package/dist/tools/imageGeneration.js.map +1 -1
- package/dist/tools/index.cjs +2 -1
- package/dist/tools/index.cjs.map +1 -1
- package/dist/tools/index.d.cts +1 -2
- package/dist/tools/index.d.cts.map +1 -1
- package/dist/tools/index.d.ts +1 -2
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +2 -2
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/localShell.cjs +2 -0
- package/dist/tools/localShell.cjs.map +1 -1
- package/dist/tools/localShell.d.cts +1 -2
- package/dist/tools/localShell.d.cts.map +1 -1
- package/dist/tools/localShell.d.ts +1 -2
- package/dist/tools/localShell.d.ts.map +1 -1
- package/dist/tools/localShell.js +1 -1
- package/dist/tools/localShell.js.map +1 -1
- package/dist/tools/mcp.cjs.map +1 -1
- package/dist/tools/mcp.d.cts +0 -1
- package/dist/tools/mcp.d.cts.map +1 -1
- package/dist/tools/mcp.d.ts +0 -1
- package/dist/tools/mcp.d.ts.map +1 -1
- package/dist/tools/mcp.js.map +1 -1
- package/dist/tools/shell.cjs +1 -0
- package/dist/tools/shell.cjs.map +1 -1
- package/dist/tools/shell.d.cts +1 -2
- package/dist/tools/shell.d.cts.map +1 -1
- package/dist/tools/shell.d.ts +1 -2
- package/dist/tools/shell.d.ts.map +1 -1
- package/dist/tools/shell.js +1 -1
- package/dist/tools/shell.js.map +1 -1
- package/dist/tools/toolSearch.cjs.map +1 -1
- package/dist/tools/toolSearch.d.cts +0 -1
- package/dist/tools/toolSearch.d.cts.map +1 -1
- package/dist/tools/toolSearch.d.ts +0 -1
- package/dist/tools/toolSearch.d.ts.map +1 -1
- package/dist/tools/toolSearch.js.map +1 -1
- package/dist/tools/webSearch.cjs.map +1 -1
- package/dist/tools/webSearch.d.cts +1 -2
- package/dist/tools/webSearch.d.cts.map +1 -1
- package/dist/tools/webSearch.d.ts +1 -2
- package/dist/tools/webSearch.d.ts.map +1 -1
- package/dist/tools/webSearch.js.map +1 -1
- package/dist/types.d.cts +0 -1
- package/dist/types.d.cts.map +1 -1
- package/dist/types.d.ts +0 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/utils/azure.cjs.map +1 -1
- package/dist/utils/azure.d.cts.map +1 -1
- package/dist/utils/azure.d.ts.map +1 -1
- package/dist/utils/azure.js.map +1 -1
- package/dist/utils/client.cjs.map +1 -1
- package/dist/utils/client.d.cts.map +1 -1
- package/dist/utils/client.d.ts.map +1 -1
- package/dist/utils/client.js.map +1 -1
- package/dist/utils/errors.cjs.map +1 -1
- package/dist/utils/errors.js.map +1 -1
- package/dist/utils/misc.cjs +2 -0
- package/dist/utils/misc.cjs.map +1 -1
- package/dist/utils/misc.d.cts +0 -1
- package/dist/utils/misc.d.cts.map +1 -1
- package/dist/utils/misc.d.ts +0 -1
- package/dist/utils/misc.d.ts.map +1 -1
- package/dist/utils/misc.js +1 -1
- package/dist/utils/misc.js.map +1 -1
- package/dist/utils/output.cjs.map +1 -1
- package/dist/utils/output.js.map +1 -1
- package/dist/utils/prompts.cjs.map +1 -1
- package/dist/utils/prompts.d.cts +0 -1
- package/dist/utils/prompts.d.cts.map +1 -1
- package/dist/utils/prompts.d.ts +0 -1
- package/dist/utils/prompts.d.ts.map +1 -1
- package/dist/utils/prompts.js.map +1 -1
- package/dist/utils/responses_stream_events.cjs.map +1 -1
- package/dist/utils/responses_stream_events.d.cts +0 -1
- package/dist/utils/responses_stream_events.d.cts.map +1 -1
- package/dist/utils/responses_stream_events.d.ts +0 -1
- package/dist/utils/responses_stream_events.d.ts.map +1 -1
- package/dist/utils/responses_stream_events.js.map +1 -1
- package/dist/utils/tools.cjs.map +1 -1
- package/dist/utils/tools.d.cts +2 -0
- package/dist/utils/tools.d.cts.map +1 -1
- package/dist/utils/tools.d.ts +2 -0
- package/dist/utils/tools.d.ts.map +1 -1
- package/dist/utils/tools.js.map +1 -1
- package/package.json +6 -6
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"computerUse.cjs","names":["z","ToolMessage"],"sources":["../../src/tools/computerUse.ts"],"sourcesContent":["/* oxlint-disable @typescript-eslint/no-explicit-any */\nimport { z } from \"zod/v4\";\nimport { OpenAI as OpenAIClient } from \"openai\";\nimport { tool, type DynamicStructuredTool } from \"@langchain/core/tools\";\nimport { type ToolRuntime } from \"@langchain/core/tools\";\nimport {\n ToolMessage,\n type AIMessage,\n type BaseMessage,\n} from \"@langchain/core/messages\";\n\n/**\n * The type of computer environment to control.\n */\nexport type ComputerUseEnvironment =\n | \"browser\"\n | \"mac\"\n | \"windows\"\n | \"linux\"\n | \"ubuntu\";\n\n/**\n * Re-export action types from OpenAI SDK for convenience.\n */\nexport type ComputerUseClickAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Click;\nexport type ComputerUseDoubleClickAction =\n OpenAIClient.Responses.ResponseComputerToolCall.DoubleClick;\nexport type ComputerUseDragAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Drag;\nexport type ComputerUseKeypressAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Keypress;\nexport type ComputerUseMoveAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Move;\nexport type ComputerUseScreenshotAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Screenshot;\nexport type ComputerUseScrollAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Scroll;\nexport type ComputerUseTypeAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Type;\nexport type ComputerUseWaitAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Wait;\n\n/**\n * Union type of all computer use actions from OpenAI SDK.\n */\nexport type ComputerUseAction =\n OpenAIClient.Responses.ResponseComputerToolCall[\"action\"];\n\n// Zod schemas for computer use actions\nconst ComputerUseScreenshotActionSchema = z.object({\n type: z.literal(\"screenshot\"),\n});\n\nconst ComputerUseClickActionSchema = z.object({\n type: z.literal(\"click\"),\n x: z.number(),\n y: z.number(),\n button: z.enum([\"left\", \"right\", \"wheel\", \"back\", \"forward\"]).default(\"left\"),\n});\n\nconst ComputerUseDoubleClickActionSchema = z.object({\n type: z.literal(\"double_click\"),\n x: z.number(),\n y: z.number(),\n button: z.enum([\"left\", \"right\", \"wheel\", \"back\", \"forward\"]).default(\"left\"),\n});\n\nconst ComputerUseDragActionSchema = z.object({\n type: z.literal(\"drag\"),\n path: z.array(z.object({ x: z.number(), y: z.number() })),\n});\n\nconst ComputerUseKeypressActionSchema = z.object({\n type: z.literal(\"keypress\"),\n keys: z.array(z.string()),\n});\n\nconst ComputerUseMoveActionSchema = z.object({\n type: z.literal(\"move\"),\n x: z.number(),\n y: z.number(),\n});\n\nconst ComputerUseScrollActionSchema = z.object({\n type: z.literal(\"scroll\"),\n x: z.number(),\n y: z.number(),\n scroll_x: z.number(),\n scroll_y: z.number(),\n});\n\nconst ComputerUseTypeActionSchema = z.object({\n type: z.literal(\"type\"),\n text: z.string(),\n});\n\nconst ComputerUseWaitActionSchema = z.object({\n type: z.literal(\"wait\"),\n duration: z.number().optional(),\n});\n\n// Union schema for individual action types\nconst ComputerUseActionUnionSchema = z.union([\n ComputerUseScreenshotActionSchema,\n ComputerUseClickActionSchema,\n ComputerUseDoubleClickActionSchema,\n ComputerUseDragActionSchema,\n ComputerUseKeypressActionSchema,\n ComputerUseMoveActionSchema,\n ComputerUseScrollActionSchema,\n ComputerUseTypeActionSchema,\n ComputerUseWaitActionSchema,\n]);\n\n// Schema for the input structure received from parseComputerCall\n// The action is wrapped in an `action` property: { action: { type: 'screenshot' } }\nexport const ComputerUseActionSchema = z.object({\n action: ComputerUseActionUnionSchema,\n});\n\n// TypeScript types derived from Zod schemas\nexport type ComputerUseScreenshotActionType = z.infer<\n typeof ComputerUseScreenshotActionSchema\n>;\nexport type ComputerUseClickActionType = z.infer<\n typeof ComputerUseClickActionSchema\n>;\nexport type ComputerUseDoubleClickActionType = z.infer<\n typeof ComputerUseDoubleClickActionSchema\n>;\nexport type ComputerUseDragActionType = z.infer<\n typeof ComputerUseDragActionSchema\n>;\nexport type ComputerUseKeypressActionType = z.infer<\n typeof ComputerUseKeypressActionSchema\n>;\nexport type ComputerUseMoveActionType = z.infer<\n typeof ComputerUseMoveActionSchema\n>;\nexport type ComputerUseScrollActionType = z.infer<\n typeof ComputerUseScrollActionSchema\n>;\nexport type ComputerUseTypeActionType = z.infer<\n typeof ComputerUseTypeActionSchema\n>;\nexport type ComputerUseWaitActionType = z.infer<\n typeof ComputerUseWaitActionSchema\n>;\n\n/**\n * Input structure for the Computer Use tool.\n * The action is wrapped in an `action` property.\n */\nexport interface ComputerUseInput {\n action: ComputerUseAction;\n}\n\nexport type ComputerUseReturnType =\n | string\n | Promise<string>\n | ToolMessage<any>\n | Promise<ToolMessage<any>>;\n\n/**\n * Options for the Computer Use tool.\n */\nexport interface ComputerUseOptions {\n /**\n * The width of the computer display in pixels.\n */\n displayWidth: number;\n\n /**\n * The height of the computer display in pixels.\n */\n displayHeight: number;\n\n /**\n * The type of computer environment to control.\n * - `browser`: Browser automation (recommended for most use cases)\n * - `mac`: macOS environment\n * - `windows`: Windows environment\n * - `linux`: Linux environment\n * - `ubuntu`: Ubuntu environment\n */\n environment: ComputerUseEnvironment;\n\n /**\n * Execute function that handles computer action execution.\n * This function receives the action input and should return a base64-encoded\n * screenshot of the result.\n */\n execute: (\n action: ComputerUseAction,\n runtime: ToolRuntime<any, any>\n ) => ComputerUseReturnType;\n}\n\n/**\n * OpenAI Computer Use tool type for the Responses API.\n */\nexport type ComputerUseTool = OpenAIClient.Responses.ComputerUsePreviewTool;\n\nconst TOOL_NAME = \"computer_use\";\n\n/**\n * Creates a Computer Use tool that allows models to control computer interfaces\n * and perform tasks by simulating mouse clicks, keyboard input, scrolling, and more.\n *\n * **Computer Use** is a practical application of OpenAI's Computer-Using Agent (CUA)\n * model (`computer-use-preview`), which combines vision capabilities with advanced\n * reasoning to simulate controlling computer interfaces.\n *\n * **How it works**:\n * The tool operates in a continuous loop:\n * 1. Model sends computer actions (click, type, scroll, etc.)\n * 2. Your code executes these actions in a controlled environment\n * 3. You capture a screenshot of the result\n * 4. Send the screenshot back to the model\n * 5. Repeat until the task is complete\n *\n * **Important**: Computer use is in beta and requires careful consideration:\n * - Use in sandboxed environments only\n * - Do not use for high-stakes or authenticated tasks\n * - Always implement human-in-the-loop for important decisions\n * - Handle safety checks appropriately\n *\n * @see {@link https://platform.openai.com/docs/guides/tools-computer-use | OpenAI Computer Use Documentation}\n *\n * @param options - Configuration options for the Computer Use tool\n * @returns A Computer Use tool that can be passed to `bindTools`\n *\n * @example\n * ```typescript\n * import { ChatOpenAI, tools } from \"@langchain/openai\";\n *\n * const model = new ChatOpenAI({ model: \"computer-use-preview\" });\n *\n * // With execute callback for automatic action handling\n * const computer = tools.computerUse({\n * displayWidth: 1024,\n * displayHeight: 768,\n * environment: \"browser\",\n * execute: async (action) => {\n * if (action.type === \"screenshot\") {\n * return captureScreenshot();\n * }\n * if (action.type === \"click\") {\n * await page.mouse.click(action.x, action.y, { button: action.button });\n * return captureScreenshot();\n * }\n * if (action.type === \"type\") {\n * await page.keyboard.type(action.text);\n * return captureScreenshot();\n * }\n * // Handle other actions...\n * return captureScreenshot();\n * },\n * });\n *\n * const llmWithComputer = model.bindTools([computer]);\n * const response = await llmWithComputer.invoke(\n * \"Check the latest news on bing.com\"\n * );\n * ```\n *\n * @example\n * ```typescript\n * // Without execute callback (manual action handling)\n * const computer = tools.computerUse({\n * displayWidth: 1024,\n * displayHeight: 768,\n * environment: \"browser\",\n * });\n *\n * const response = await model.invoke(\"Check the news\", {\n * tools: [computer],\n * });\n *\n * // Access the computer call from the response\n * const computerCall = response.additional_kwargs.tool_outputs?.find(\n * (output) => output.type === \"computer_call\"\n * );\n * if (computerCall) {\n * console.log(\"Action to execute:\", computerCall.action);\n * // Execute the action manually, then send back a screenshot\n * }\n * ```\n *\n * @example\n * ```typescript\n * // For macOS desktop automation with Docker\n * const computer = tools.computerUse({\n * displayWidth: 1920,\n * displayHeight: 1080,\n * environment: \"mac\",\n * execute: async (action) => {\n * if (action.type === \"click\") {\n * await dockerExec(\n * `DISPLAY=:99 xdotool mousemove ${action.x} ${action.y} click 1`,\n * containerName\n * );\n * }\n * // Capture screenshot from container\n * return await getDockerScreenshot(containerName);\n * },\n * });\n * ```\n *\n * @remarks\n * - Only available through the Responses API (not Chat Completions)\n * - Requires `computer-use-preview` model\n * - Actions include: click, double_click, drag, keypress, move, screenshot, scroll, type, wait\n * - Safety checks may be returned that require acknowledgment before proceeding\n * - Use `truncation: \"auto\"` parameter when making requests\n * - Recommended to use with `reasoning.summary` for debugging\n */\nexport function computerUse(options: ComputerUseOptions) {\n const computerTool = tool(\n async (\n input: ComputerUseInput,\n runtime: ToolRuntime<{ messages: BaseMessage[] }>\n ) => {\n /**\n * get computer_use call id from runtime\n */\n const aiMessage = runtime.state?.messages.at(-1) as AIMessage | undefined;\n const computerToolCall = aiMessage?.tool_calls?.find(\n (tc) => tc.name === \"computer_use\"\n );\n const computerToolCallId = computerToolCall?.id;\n if (!computerToolCallId) {\n throw new Error(\"Computer use call id not found\");\n }\n\n const result = await options.execute(input.action, runtime);\n\n /**\n * make sure {@link ToolMessage} is returned with the correct additional kwargs\n */\n if (typeof result === \"string\") {\n return new ToolMessage({\n content: result,\n tool_call_id: computerToolCallId,\n additional_kwargs: {\n type: \"computer_call_output\",\n },\n });\n }\n\n /**\n * make sure {@link ToolMessage} is returned with the correct additional kwargs\n */\n return new ToolMessage({\n ...result,\n tool_call_id: computerToolCallId,\n additional_kwargs: {\n type: \"computer_call_output\",\n ...result.additional_kwargs,\n },\n });\n },\n {\n name: TOOL_NAME,\n description:\n \"Control a computer interface by executing mouse clicks, keyboard input, scrolling, and other actions.\",\n schema: ComputerUseActionSchema,\n }\n );\n\n computerTool.extras = {\n ...(computerTool.extras ?? {}),\n providerToolDefinition: {\n type: \"computer_use_preview\",\n display_width: options.displayWidth,\n display_height: options.displayHeight,\n environment: options.environment,\n } satisfies ComputerUseTool,\n };\n\n /**\n * return as typed {@link DynamicStructuredTool} so we don't get any type\n * errors like \"can't export tool without reference\"\n */\n return computerTool as DynamicStructuredTool<\n typeof ComputerUseActionSchema,\n ComputerUseInput,\n unknown,\n ComputerUseReturnType\n >;\n}\n"],"mappings":";;;;AAkDA,MAAM,oCAAoCA,OAAAA,EAAE,OAAO,EACjD,MAAMA,OAAAA,EAAE,QAAQ,aAAa,EAC9B,CAAC;AAEF,MAAM,+BAA+BA,OAAAA,EAAE,OAAO;CAC5C,MAAMA,OAAAA,EAAE,QAAQ,QAAQ;CACxB,GAAGA,OAAAA,EAAE,QAAQ;CACb,GAAGA,OAAAA,EAAE,QAAQ;CACb,QAAQA,OAAAA,EAAE,KAAK;EAAC;EAAQ;EAAS;EAAS;EAAQ;EAAU,CAAC,CAAC,QAAQ,OAAO;CAC9E,CAAC;AAEF,MAAM,qCAAqCA,OAAAA,EAAE,OAAO;CAClD,MAAMA,OAAAA,EAAE,QAAQ,eAAe;CAC/B,GAAGA,OAAAA,EAAE,QAAQ;CACb,GAAGA,OAAAA,EAAE,QAAQ;CACb,QAAQA,OAAAA,EAAE,KAAK;EAAC;EAAQ;EAAS;EAAS;EAAQ;EAAU,CAAC,CAAC,QAAQ,OAAO;CAC9E,CAAC;AAEF,MAAM,8BAA8BA,OAAAA,EAAE,OAAO;CAC3C,MAAMA,OAAAA,EAAE,QAAQ,OAAO;CACvB,MAAMA,OAAAA,EAAE,MAAMA,OAAAA,EAAE,OAAO;EAAE,GAAGA,OAAAA,EAAE,QAAQ;EAAE,GAAGA,OAAAA,EAAE,QAAQ;EAAE,CAAC,CAAC;CAC1D,CAAC;AAEF,MAAM,kCAAkCA,OAAAA,EAAE,OAAO;CAC/C,MAAMA,OAAAA,EAAE,QAAQ,WAAW;CAC3B,MAAMA,OAAAA,EAAE,MAAMA,OAAAA,EAAE,QAAQ,CAAC;CAC1B,CAAC;AAEF,MAAM,8BAA8BA,OAAAA,EAAE,OAAO;CAC3C,MAAMA,OAAAA,EAAE,QAAQ,OAAO;CACvB,GAAGA,OAAAA,EAAE,QAAQ;CACb,GAAGA,OAAAA,EAAE,QAAQ;CACd,CAAC;AAEF,MAAM,gCAAgCA,OAAAA,EAAE,OAAO;CAC7C,MAAMA,OAAAA,EAAE,QAAQ,SAAS;CACzB,GAAGA,OAAAA,EAAE,QAAQ;CACb,GAAGA,OAAAA,EAAE,QAAQ;CACb,UAAUA,OAAAA,EAAE,QAAQ;CACpB,UAAUA,OAAAA,EAAE,QAAQ;CACrB,CAAC;AAEF,MAAM,8BAA8BA,OAAAA,EAAE,OAAO;CAC3C,MAAMA,OAAAA,EAAE,QAAQ,OAAO;CACvB,MAAMA,OAAAA,EAAE,QAAQ;CACjB,CAAC;AAEF,MAAM,8BAA8BA,OAAAA,EAAE,OAAO;CAC3C,MAAMA,OAAAA,EAAE,QAAQ,OAAO;CACvB,UAAUA,OAAAA,EAAE,QAAQ,CAAC,UAAU;CAChC,CAAC;AAGF,MAAM,+BAA+BA,OAAAA,EAAE,MAAM;CAC3C;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACD,CAAC;AAIF,MAAa,0BAA0BA,OAAAA,EAAE,OAAO,EAC9C,QAAQ,8BACT,CAAC;AAqFF,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkHlB,SAAgB,YAAY,SAA6B;CACvD,MAAM,gBAAA,GAAA,sBAAA,MACJ,OACE,OACA,YACG;EAQH,MAAM,uBAJY,QAAQ,OAAO,SAAS,GAAG,GAAG,GACZ,YAAY,MAC7C,OAAO,GAAG,SAAS,eACrB,GAC4C;AAC7C,MAAI,CAAC,mBACH,OAAM,IAAI,MAAM,iCAAiC;EAGnD,MAAM,SAAS,MAAM,QAAQ,QAAQ,MAAM,QAAQ,QAAQ;;;;AAK3D,MAAI,OAAO,WAAW,SACpB,QAAO,IAAIC,yBAAAA,YAAY;GACrB,SAAS;GACT,cAAc;GACd,mBAAmB,EACjB,MAAM,wBACP;GACF,CAAC;;;;AAMJ,SAAO,IAAIA,yBAAAA,YAAY;GACrB,GAAG;GACH,cAAc;GACd,mBAAmB;IACjB,MAAM;IACN,GAAG,OAAO;IACX;GACF,CAAC;IAEJ;EACE,MAAM;EACN,aACE;EACF,QAAQ;EACT,CACF;AAED,cAAa,SAAS;EACpB,GAAI,aAAa,UAAU,EAAE;EAC7B,wBAAwB;GACtB,MAAM;GACN,eAAe,QAAQ;GACvB,gBAAgB,QAAQ;GACxB,aAAa,QAAQ;GACtB;EACF;;;;;AAMD,QAAO"}
|
|
1
|
+
{"version":3,"file":"computerUse.cjs","names":["z","ToolMessage"],"sources":["../../src/tools/computerUse.ts"],"sourcesContent":["/* oxlint-disable @typescript-eslint/no-explicit-any */\nimport { z } from \"zod/v4\";\nimport { OpenAI as OpenAIClient } from \"openai\";\nimport { tool, type DynamicStructuredTool } from \"@langchain/core/tools\";\nimport { type ToolRuntime } from \"@langchain/core/tools\";\nimport {\n ToolMessage,\n type AIMessage,\n type BaseMessage,\n} from \"@langchain/core/messages\";\n\n/**\n * The type of computer environment to control.\n */\nexport type ComputerUseEnvironment =\n | \"browser\"\n | \"mac\"\n | \"windows\"\n | \"linux\"\n | \"ubuntu\";\n\n/**\n * Re-export action types from OpenAI SDK for convenience.\n */\nexport type ComputerUseClickAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Click;\nexport type ComputerUseDoubleClickAction =\n OpenAIClient.Responses.ResponseComputerToolCall.DoubleClick;\nexport type ComputerUseDragAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Drag;\nexport type ComputerUseKeypressAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Keypress;\nexport type ComputerUseMoveAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Move;\nexport type ComputerUseScreenshotAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Screenshot;\nexport type ComputerUseScrollAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Scroll;\nexport type ComputerUseTypeAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Type;\nexport type ComputerUseWaitAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Wait;\n\n/**\n * Union type of all computer use actions from OpenAI SDK.\n */\nexport type ComputerUseAction =\n OpenAIClient.Responses.ResponseComputerToolCall[\"action\"];\n\n// Zod schemas for computer use actions\nconst ComputerUseScreenshotActionSchema = z.object({\n type: z.literal(\"screenshot\"),\n});\n\nconst ComputerUseClickActionSchema = z.object({\n type: z.literal(\"click\"),\n x: z.number(),\n y: z.number(),\n button: z.enum([\"left\", \"right\", \"wheel\", \"back\", \"forward\"]).default(\"left\"),\n});\n\nconst ComputerUseDoubleClickActionSchema = z.object({\n type: z.literal(\"double_click\"),\n x: z.number(),\n y: z.number(),\n button: z.enum([\"left\", \"right\", \"wheel\", \"back\", \"forward\"]).default(\"left\"),\n});\n\nconst ComputerUseDragActionSchema = z.object({\n type: z.literal(\"drag\"),\n path: z.array(z.object({ x: z.number(), y: z.number() })),\n});\n\nconst ComputerUseKeypressActionSchema = z.object({\n type: z.literal(\"keypress\"),\n keys: z.array(z.string()),\n});\n\nconst ComputerUseMoveActionSchema = z.object({\n type: z.literal(\"move\"),\n x: z.number(),\n y: z.number(),\n});\n\nconst ComputerUseScrollActionSchema = z.object({\n type: z.literal(\"scroll\"),\n x: z.number(),\n y: z.number(),\n scroll_x: z.number(),\n scroll_y: z.number(),\n});\n\nconst ComputerUseTypeActionSchema = z.object({\n type: z.literal(\"type\"),\n text: z.string(),\n});\n\nconst ComputerUseWaitActionSchema = z.object({\n type: z.literal(\"wait\"),\n duration: z.number().optional(),\n});\n\n// Union schema for individual action types\nconst ComputerUseActionUnionSchema = z.union([\n ComputerUseScreenshotActionSchema,\n ComputerUseClickActionSchema,\n ComputerUseDoubleClickActionSchema,\n ComputerUseDragActionSchema,\n ComputerUseKeypressActionSchema,\n ComputerUseMoveActionSchema,\n ComputerUseScrollActionSchema,\n ComputerUseTypeActionSchema,\n ComputerUseWaitActionSchema,\n]);\n\n// Schema for the input structure received from parseComputerCall\n// The action is wrapped in an `action` property: { action: { type: 'screenshot' } }\nexport const ComputerUseActionSchema = z.object({\n action: ComputerUseActionUnionSchema,\n});\n\n// TypeScript types derived from Zod schemas\nexport type ComputerUseScreenshotActionType = z.infer<\n typeof ComputerUseScreenshotActionSchema\n>;\nexport type ComputerUseClickActionType = z.infer<\n typeof ComputerUseClickActionSchema\n>;\nexport type ComputerUseDoubleClickActionType = z.infer<\n typeof ComputerUseDoubleClickActionSchema\n>;\nexport type ComputerUseDragActionType = z.infer<\n typeof ComputerUseDragActionSchema\n>;\nexport type ComputerUseKeypressActionType = z.infer<\n typeof ComputerUseKeypressActionSchema\n>;\nexport type ComputerUseMoveActionType = z.infer<\n typeof ComputerUseMoveActionSchema\n>;\nexport type ComputerUseScrollActionType = z.infer<\n typeof ComputerUseScrollActionSchema\n>;\nexport type ComputerUseTypeActionType = z.infer<\n typeof ComputerUseTypeActionSchema\n>;\nexport type ComputerUseWaitActionType = z.infer<\n typeof ComputerUseWaitActionSchema\n>;\n\n/**\n * Input structure for the Computer Use tool.\n * The action is wrapped in an `action` property.\n */\nexport interface ComputerUseInput {\n action: ComputerUseAction;\n}\n\nexport type ComputerUseReturnType =\n | string\n | Promise<string>\n | ToolMessage<any>\n | Promise<ToolMessage<any>>;\n\n/**\n * Options for the Computer Use tool.\n */\nexport interface ComputerUseOptions {\n /**\n * The width of the computer display in pixels.\n */\n displayWidth: number;\n\n /**\n * The height of the computer display in pixels.\n */\n displayHeight: number;\n\n /**\n * The type of computer environment to control.\n * - `browser`: Browser automation (recommended for most use cases)\n * - `mac`: macOS environment\n * - `windows`: Windows environment\n * - `linux`: Linux environment\n * - `ubuntu`: Ubuntu environment\n */\n environment: ComputerUseEnvironment;\n\n /**\n * Execute function that handles computer action execution.\n * This function receives the action input and should return a base64-encoded\n * screenshot of the result.\n */\n execute: (\n action: ComputerUseAction,\n runtime: ToolRuntime<any, any>\n ) => ComputerUseReturnType;\n}\n\n/**\n * OpenAI Computer Use tool type for the Responses API.\n */\nexport type ComputerUseTool = OpenAIClient.Responses.ComputerUsePreviewTool;\n\nconst TOOL_NAME = \"computer_use\";\n\n/**\n * Creates a Computer Use tool that allows models to control computer interfaces\n * and perform tasks by simulating mouse clicks, keyboard input, scrolling, and more.\n *\n * **Computer Use** is a practical application of OpenAI's Computer-Using Agent (CUA)\n * model (`computer-use-preview`), which combines vision capabilities with advanced\n * reasoning to simulate controlling computer interfaces.\n *\n * **How it works**:\n * The tool operates in a continuous loop:\n * 1. Model sends computer actions (click, type, scroll, etc.)\n * 2. Your code executes these actions in a controlled environment\n * 3. You capture a screenshot of the result\n * 4. Send the screenshot back to the model\n * 5. Repeat until the task is complete\n *\n * **Important**: Computer use is in beta and requires careful consideration:\n * - Use in sandboxed environments only\n * - Do not use for high-stakes or authenticated tasks\n * - Always implement human-in-the-loop for important decisions\n * - Handle safety checks appropriately\n *\n * @see {@link https://platform.openai.com/docs/guides/tools-computer-use | OpenAI Computer Use Documentation}\n *\n * @param options - Configuration options for the Computer Use tool\n * @returns A Computer Use tool that can be passed to `bindTools`\n *\n * @example\n * ```typescript\n * import { ChatOpenAI, tools } from \"@langchain/openai\";\n *\n * const model = new ChatOpenAI({ model: \"computer-use-preview\" });\n *\n * // With execute callback for automatic action handling\n * const computer = tools.computerUse({\n * displayWidth: 1024,\n * displayHeight: 768,\n * environment: \"browser\",\n * execute: async (action) => {\n * if (action.type === \"screenshot\") {\n * return captureScreenshot();\n * }\n * if (action.type === \"click\") {\n * await page.mouse.click(action.x, action.y, { button: action.button });\n * return captureScreenshot();\n * }\n * if (action.type === \"type\") {\n * await page.keyboard.type(action.text);\n * return captureScreenshot();\n * }\n * // Handle other actions...\n * return captureScreenshot();\n * },\n * });\n *\n * const llmWithComputer = model.bindTools([computer]);\n * const response = await llmWithComputer.invoke(\n * \"Check the latest news on bing.com\"\n * );\n * ```\n *\n * @example\n * ```typescript\n * // Without execute callback (manual action handling)\n * const computer = tools.computerUse({\n * displayWidth: 1024,\n * displayHeight: 768,\n * environment: \"browser\",\n * });\n *\n * const response = await model.invoke(\"Check the news\", {\n * tools: [computer],\n * });\n *\n * // Access the computer call from the response\n * const computerCall = response.additional_kwargs.tool_outputs?.find(\n * (output) => output.type === \"computer_call\"\n * );\n * if (computerCall) {\n * console.log(\"Action to execute:\", computerCall.action);\n * // Execute the action manually, then send back a screenshot\n * }\n * ```\n *\n * @example\n * ```typescript\n * // For macOS desktop automation with Docker\n * const computer = tools.computerUse({\n * displayWidth: 1920,\n * displayHeight: 1080,\n * environment: \"mac\",\n * execute: async (action) => {\n * if (action.type === \"click\") {\n * await dockerExec(\n * `DISPLAY=:99 xdotool mousemove ${action.x} ${action.y} click 1`,\n * containerName\n * );\n * }\n * // Capture screenshot from container\n * return await getDockerScreenshot(containerName);\n * },\n * });\n * ```\n *\n * @remarks\n * - Only available through the Responses API (not Chat Completions)\n * - Requires `computer-use-preview` model\n * - Actions include: click, double_click, drag, keypress, move, screenshot, scroll, type, wait\n * - Safety checks may be returned that require acknowledgment before proceeding\n * - Use `truncation: \"auto\"` parameter when making requests\n * - Recommended to use with `reasoning.summary` for debugging\n */\nexport function computerUse(options: ComputerUseOptions) {\n const computerTool = tool(\n async (\n input: ComputerUseInput,\n runtime: ToolRuntime<{ messages: BaseMessage[] }>\n ) => {\n /**\n * get computer_use call id from runtime\n */\n const aiMessage = runtime.state?.messages.at(-1) as AIMessage | undefined;\n const computerToolCall = aiMessage?.tool_calls?.find(\n (tc) => tc.name === \"computer_use\"\n );\n const computerToolCallId = computerToolCall?.id;\n if (!computerToolCallId) {\n throw new Error(\"Computer use call id not found\");\n }\n\n const result = await options.execute(input.action, runtime);\n\n /**\n * make sure {@link ToolMessage} is returned with the correct additional kwargs\n */\n if (typeof result === \"string\") {\n return new ToolMessage({\n content: result,\n tool_call_id: computerToolCallId,\n additional_kwargs: {\n type: \"computer_call_output\",\n },\n });\n }\n\n /**\n * make sure {@link ToolMessage} is returned with the correct additional kwargs\n */\n return new ToolMessage({\n ...result,\n tool_call_id: computerToolCallId,\n additional_kwargs: {\n type: \"computer_call_output\",\n ...result.additional_kwargs,\n },\n });\n },\n {\n name: TOOL_NAME,\n description:\n \"Control a computer interface by executing mouse clicks, keyboard input, scrolling, and other actions.\",\n schema: ComputerUseActionSchema,\n }\n );\n\n computerTool.extras = {\n ...(computerTool.extras ?? {}),\n providerToolDefinition: {\n type: \"computer_use_preview\",\n display_width: options.displayWidth,\n display_height: options.displayHeight,\n environment: options.environment,\n } satisfies ComputerUseTool,\n };\n\n /**\n * return as typed {@link DynamicStructuredTool} so we don't get any type\n * errors like \"can't export tool without reference\"\n */\n return computerTool as DynamicStructuredTool<\n typeof ComputerUseActionSchema,\n ComputerUseInput,\n unknown,\n ComputerUseReturnType\n >;\n}\n"],"mappings":";;;;AAkDA,MAAM,oCAAoCA,OAAAA,EAAE,OAAO,EACjD,MAAMA,OAAAA,EAAE,QAAQ,YAAY,EAC9B,CAAC;AAED,MAAM,+BAA+BA,OAAAA,EAAE,OAAO;CAC5C,MAAMA,OAAAA,EAAE,QAAQ,OAAO;CACvB,GAAGA,OAAAA,EAAE,OAAO;CACZ,GAAGA,OAAAA,EAAE,OAAO;CACZ,QAAQA,OAAAA,EAAE,KAAK;EAAC;EAAQ;EAAS;EAAS;EAAQ;CAAS,CAAC,CAAC,CAAC,QAAQ,MAAM;AAC9E,CAAC;AAED,MAAM,qCAAqCA,OAAAA,EAAE,OAAO;CAClD,MAAMA,OAAAA,EAAE,QAAQ,cAAc;CAC9B,GAAGA,OAAAA,EAAE,OAAO;CACZ,GAAGA,OAAAA,EAAE,OAAO;CACZ,QAAQA,OAAAA,EAAE,KAAK;EAAC;EAAQ;EAAS;EAAS;EAAQ;CAAS,CAAC,CAAC,CAAC,QAAQ,MAAM;AAC9E,CAAC;AAED,MAAM,8BAA8BA,OAAAA,EAAE,OAAO;CAC3C,MAAMA,OAAAA,EAAE,QAAQ,MAAM;CACtB,MAAMA,OAAAA,EAAE,MAAMA,OAAAA,EAAE,OAAO;EAAE,GAAGA,OAAAA,EAAE,OAAO;EAAG,GAAGA,OAAAA,EAAE,OAAO;CAAE,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,kCAAkCA,OAAAA,EAAE,OAAO;CAC/C,MAAMA,OAAAA,EAAE,QAAQ,UAAU;CAC1B,MAAMA,OAAAA,EAAE,MAAMA,OAAAA,EAAE,OAAO,CAAC;AAC1B,CAAC;AAED,MAAM,8BAA8BA,OAAAA,EAAE,OAAO;CAC3C,MAAMA,OAAAA,EAAE,QAAQ,MAAM;CACtB,GAAGA,OAAAA,EAAE,OAAO;CACZ,GAAGA,OAAAA,EAAE,OAAO;AACd,CAAC;AAED,MAAM,gCAAgCA,OAAAA,EAAE,OAAO;CAC7C,MAAMA,OAAAA,EAAE,QAAQ,QAAQ;CACxB,GAAGA,OAAAA,EAAE,OAAO;CACZ,GAAGA,OAAAA,EAAE,OAAO;CACZ,UAAUA,OAAAA,EAAE,OAAO;CACnB,UAAUA,OAAAA,EAAE,OAAO;AACrB,CAAC;AAED,MAAM,8BAA8BA,OAAAA,EAAE,OAAO;CAC3C,MAAMA,OAAAA,EAAE,QAAQ,MAAM;CACtB,MAAMA,OAAAA,EAAE,OAAO;AACjB,CAAC;AAED,MAAM,8BAA8BA,OAAAA,EAAE,OAAO;CAC3C,MAAMA,OAAAA,EAAE,QAAQ,MAAM;CACtB,UAAUA,OAAAA,EAAE,OAAO,CAAC,CAAC,SAAS;AAChC,CAAC;AAGD,MAAM,+BAA+BA,OAAAA,EAAE,MAAM;CAC3C;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;AAID,MAAa,0BAA0BA,OAAAA,EAAE,OAAO,EAC9C,QAAQ,6BACV,CAAC;AAqFD,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkHlB,SAAgB,YAAY,SAA6B;CACvD,MAAM,gBAAA,GAAA,sBAAA,KAAA,CACJ,OACE,OACA,YACG;EAQH,MAAM,uBAJY,QAAQ,OAAO,SAAS,GAAG,EAAE,EAAA,EACX,YAAY,MAC7C,OAAO,GAAG,SAAS,cACtB,EAAA,EAC6C;EAC7C,IAAI,CAAC,oBACH,MAAM,IAAI,MAAM,gCAAgC;EAGlD,MAAM,SAAS,MAAM,QAAQ,QAAQ,MAAM,QAAQ,OAAO;;;;EAK1D,IAAI,OAAO,WAAW,UACpB,OAAO,IAAIC,yBAAAA,YAAY;GACrB,SAAS;GACT,cAAc;GACd,mBAAmB,EACjB,MAAM,uBACR;EACF,CAAC;;;;EAMH,OAAO,IAAIA,yBAAAA,YAAY;GACrB,GAAG;GACH,cAAc;GACd,mBAAmB;IACjB,MAAM;IACN,GAAG,OAAO;GACZ;EACF,CAAC;CACH,GACA;EACE,MAAM;EACN,aACE;EACF,QAAQ;CACV,CACF;CAEA,aAAa,SAAS;EACpB,GAAI,aAAa,UAAU,CAAC;EAC5B,wBAAwB;GACtB,MAAM;GACN,eAAe,QAAQ;GACvB,gBAAgB,QAAQ;GACxB,aAAa,QAAQ;EACvB;CACF;;;;;CAMA,OAAO;AAMT"}
|
|
@@ -2,7 +2,6 @@ import { OpenAI } from "openai";
|
|
|
2
2
|
import { ToolMessage } from "@langchain/core/messages";
|
|
3
3
|
import { DynamicStructuredTool, ToolRuntime } from "@langchain/core/tools";
|
|
4
4
|
import { z } from "zod/v4";
|
|
5
|
-
|
|
6
5
|
//#region src/tools/computerUse.d.ts
|
|
7
6
|
/**
|
|
8
7
|
* The type of computer environment to control.
|
|
@@ -230,5 +229,5 @@ type ComputerUseTool = OpenAI.Responses.ComputerUsePreviewTool;
|
|
|
230
229
|
*/
|
|
231
230
|
declare function computerUse(options: ComputerUseOptions): DynamicStructuredTool<typeof ComputerUseActionSchema, ComputerUseInput, unknown, ComputerUseReturnType>;
|
|
232
231
|
//#endregion
|
|
233
|
-
export { ComputerUseAction, ComputerUseClickAction, ComputerUseDoubleClickAction, ComputerUseDragAction, ComputerUseEnvironment, ComputerUseInput, ComputerUseKeypressAction, ComputerUseMoveAction, ComputerUseOptions, ComputerUseScreenshotAction, ComputerUseScrollAction, ComputerUseTool, ComputerUseTypeAction, ComputerUseWaitAction, computerUse };
|
|
232
|
+
export { ComputerUseAction, ComputerUseActionSchema, ComputerUseClickAction, ComputerUseDoubleClickAction, ComputerUseDragAction, ComputerUseEnvironment, ComputerUseInput, ComputerUseKeypressAction, ComputerUseMoveAction, ComputerUseOptions, ComputerUseReturnType, ComputerUseScreenshotAction, ComputerUseScrollAction, ComputerUseTool, ComputerUseTypeAction, ComputerUseWaitAction, computerUse };
|
|
234
233
|
//# sourceMappingURL=computerUse.d.cts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"computerUse.d.cts","names":[],"sources":["../../src/tools/computerUse.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"computerUse.d.cts","names":[],"sources":["../../src/tools/computerUse.ts"],"mappings":";;;;;;;;KAcY;;;;KAUA,yBACV,OAAa,UAAU,yBAAyB;KACtC,+BACV,OAAa,UAAU,yBAAyB;KACtC,wBACV,OAAa,UAAU,yBAAyB;KACtC,4BACV,OAAa,UAAU,yBAAyB;KACtC,wBACV,OAAa,UAAU,yBAAyB;KACtC,8BACV,OAAa,UAAU,yBAAyB;KACtC,0BACV,OAAa,UAAU,yBAAyB;KACtC,wBACV,OAAa,UAAU,yBAAyB;KACtC,wBACV,OAAa,UAAU,yBAAyB;;;;KAKtC,oBACV,OAAa,UAAU;cAsEZ,yBAAuB,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAElC,EAAA,KAAA;;;;;UAmCe;EACf,QAAQ;;KAGE,iCAER,kBACA,mBACA,QAAQ;;;;UAKK;;;;EAIf;;;;EAKA;;;;;;;;;EAUA,aAAa;;;;;;EAOb,UACE,QAAQ,mBACR,SAAS,0BACN;;;;;KAMK,kBAAkB,OAAa,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAoHrC,YAAY,SAAS,qBAmEZ,6BACd,yBACP,2BAEA"}
|
|
@@ -2,7 +2,6 @@ import { OpenAI } from "openai";
|
|
|
2
2
|
import { ToolMessage } from "@langchain/core/messages";
|
|
3
3
|
import { DynamicStructuredTool, ToolRuntime } from "@langchain/core/tools";
|
|
4
4
|
import { z } from "zod/v4";
|
|
5
|
-
|
|
6
5
|
//#region src/tools/computerUse.d.ts
|
|
7
6
|
/**
|
|
8
7
|
* The type of computer environment to control.
|
|
@@ -230,5 +229,5 @@ type ComputerUseTool = OpenAI.Responses.ComputerUsePreviewTool;
|
|
|
230
229
|
*/
|
|
231
230
|
declare function computerUse(options: ComputerUseOptions): DynamicStructuredTool<typeof ComputerUseActionSchema, ComputerUseInput, unknown, ComputerUseReturnType>;
|
|
232
231
|
//#endregion
|
|
233
|
-
export { ComputerUseAction, ComputerUseClickAction, ComputerUseDoubleClickAction, ComputerUseDragAction, ComputerUseEnvironment, ComputerUseInput, ComputerUseKeypressAction, ComputerUseMoveAction, ComputerUseOptions, ComputerUseScreenshotAction, ComputerUseScrollAction, ComputerUseTool, ComputerUseTypeAction, ComputerUseWaitAction, computerUse };
|
|
232
|
+
export { ComputerUseAction, ComputerUseActionSchema, ComputerUseClickAction, ComputerUseDoubleClickAction, ComputerUseDragAction, ComputerUseEnvironment, ComputerUseInput, ComputerUseKeypressAction, ComputerUseMoveAction, ComputerUseOptions, ComputerUseReturnType, ComputerUseScreenshotAction, ComputerUseScrollAction, ComputerUseTool, ComputerUseTypeAction, ComputerUseWaitAction, computerUse };
|
|
234
233
|
//# sourceMappingURL=computerUse.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"computerUse.d.ts","names":[],"sources":["../../src/tools/computerUse.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"computerUse.d.ts","names":[],"sources":["../../src/tools/computerUse.ts"],"mappings":";;;;;;;;KAcY;;;;KAUA,yBACV,OAAa,UAAU,yBAAyB;KACtC,+BACV,OAAa,UAAU,yBAAyB;KACtC,wBACV,OAAa,UAAU,yBAAyB;KACtC,4BACV,OAAa,UAAU,yBAAyB;KACtC,wBACV,OAAa,UAAU,yBAAyB;KACtC,8BACV,OAAa,UAAU,yBAAyB;KACtC,0BACV,OAAa,UAAU,yBAAyB;KACtC,wBACV,OAAa,UAAU,yBAAyB;KACtC,wBACV,OAAa,UAAU,yBAAyB;;;;KAKtC,oBACV,OAAa,UAAU;cAsEZ,yBAAuB,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAElC,EAAA,KAAA;;;;;UAmCe;EACf,QAAQ;;KAGE,iCAER,kBACA,mBACA,QAAQ;;;;UAKK;;;;EAIf;;;;EAKA;;;;;;;;;EAUA,aAAa;;;;;;EAOb,UACE,QAAQ,mBACR,SAAS,0BACN;;;;;KAMK,kBAAkB,OAAa,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAoHrC,YAAY,SAAS,qBAmEZ,6BACd,yBACP,2BAEA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"computerUse.js","names":[],"sources":["../../src/tools/computerUse.ts"],"sourcesContent":["/* oxlint-disable @typescript-eslint/no-explicit-any */\nimport { z } from \"zod/v4\";\nimport { OpenAI as OpenAIClient } from \"openai\";\nimport { tool, type DynamicStructuredTool } from \"@langchain/core/tools\";\nimport { type ToolRuntime } from \"@langchain/core/tools\";\nimport {\n ToolMessage,\n type AIMessage,\n type BaseMessage,\n} from \"@langchain/core/messages\";\n\n/**\n * The type of computer environment to control.\n */\nexport type ComputerUseEnvironment =\n | \"browser\"\n | \"mac\"\n | \"windows\"\n | \"linux\"\n | \"ubuntu\";\n\n/**\n * Re-export action types from OpenAI SDK for convenience.\n */\nexport type ComputerUseClickAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Click;\nexport type ComputerUseDoubleClickAction =\n OpenAIClient.Responses.ResponseComputerToolCall.DoubleClick;\nexport type ComputerUseDragAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Drag;\nexport type ComputerUseKeypressAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Keypress;\nexport type ComputerUseMoveAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Move;\nexport type ComputerUseScreenshotAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Screenshot;\nexport type ComputerUseScrollAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Scroll;\nexport type ComputerUseTypeAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Type;\nexport type ComputerUseWaitAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Wait;\n\n/**\n * Union type of all computer use actions from OpenAI SDK.\n */\nexport type ComputerUseAction =\n OpenAIClient.Responses.ResponseComputerToolCall[\"action\"];\n\n// Zod schemas for computer use actions\nconst ComputerUseScreenshotActionSchema = z.object({\n type: z.literal(\"screenshot\"),\n});\n\nconst ComputerUseClickActionSchema = z.object({\n type: z.literal(\"click\"),\n x: z.number(),\n y: z.number(),\n button: z.enum([\"left\", \"right\", \"wheel\", \"back\", \"forward\"]).default(\"left\"),\n});\n\nconst ComputerUseDoubleClickActionSchema = z.object({\n type: z.literal(\"double_click\"),\n x: z.number(),\n y: z.number(),\n button: z.enum([\"left\", \"right\", \"wheel\", \"back\", \"forward\"]).default(\"left\"),\n});\n\nconst ComputerUseDragActionSchema = z.object({\n type: z.literal(\"drag\"),\n path: z.array(z.object({ x: z.number(), y: z.number() })),\n});\n\nconst ComputerUseKeypressActionSchema = z.object({\n type: z.literal(\"keypress\"),\n keys: z.array(z.string()),\n});\n\nconst ComputerUseMoveActionSchema = z.object({\n type: z.literal(\"move\"),\n x: z.number(),\n y: z.number(),\n});\n\nconst ComputerUseScrollActionSchema = z.object({\n type: z.literal(\"scroll\"),\n x: z.number(),\n y: z.number(),\n scroll_x: z.number(),\n scroll_y: z.number(),\n});\n\nconst ComputerUseTypeActionSchema = z.object({\n type: z.literal(\"type\"),\n text: z.string(),\n});\n\nconst ComputerUseWaitActionSchema = z.object({\n type: z.literal(\"wait\"),\n duration: z.number().optional(),\n});\n\n// Union schema for individual action types\nconst ComputerUseActionUnionSchema = z.union([\n ComputerUseScreenshotActionSchema,\n ComputerUseClickActionSchema,\n ComputerUseDoubleClickActionSchema,\n ComputerUseDragActionSchema,\n ComputerUseKeypressActionSchema,\n ComputerUseMoveActionSchema,\n ComputerUseScrollActionSchema,\n ComputerUseTypeActionSchema,\n ComputerUseWaitActionSchema,\n]);\n\n// Schema for the input structure received from parseComputerCall\n// The action is wrapped in an `action` property: { action: { type: 'screenshot' } }\nexport const ComputerUseActionSchema = z.object({\n action: ComputerUseActionUnionSchema,\n});\n\n// TypeScript types derived from Zod schemas\nexport type ComputerUseScreenshotActionType = z.infer<\n typeof ComputerUseScreenshotActionSchema\n>;\nexport type ComputerUseClickActionType = z.infer<\n typeof ComputerUseClickActionSchema\n>;\nexport type ComputerUseDoubleClickActionType = z.infer<\n typeof ComputerUseDoubleClickActionSchema\n>;\nexport type ComputerUseDragActionType = z.infer<\n typeof ComputerUseDragActionSchema\n>;\nexport type ComputerUseKeypressActionType = z.infer<\n typeof ComputerUseKeypressActionSchema\n>;\nexport type ComputerUseMoveActionType = z.infer<\n typeof ComputerUseMoveActionSchema\n>;\nexport type ComputerUseScrollActionType = z.infer<\n typeof ComputerUseScrollActionSchema\n>;\nexport type ComputerUseTypeActionType = z.infer<\n typeof ComputerUseTypeActionSchema\n>;\nexport type ComputerUseWaitActionType = z.infer<\n typeof ComputerUseWaitActionSchema\n>;\n\n/**\n * Input structure for the Computer Use tool.\n * The action is wrapped in an `action` property.\n */\nexport interface ComputerUseInput {\n action: ComputerUseAction;\n}\n\nexport type ComputerUseReturnType =\n | string\n | Promise<string>\n | ToolMessage<any>\n | Promise<ToolMessage<any>>;\n\n/**\n * Options for the Computer Use tool.\n */\nexport interface ComputerUseOptions {\n /**\n * The width of the computer display in pixels.\n */\n displayWidth: number;\n\n /**\n * The height of the computer display in pixels.\n */\n displayHeight: number;\n\n /**\n * The type of computer environment to control.\n * - `browser`: Browser automation (recommended for most use cases)\n * - `mac`: macOS environment\n * - `windows`: Windows environment\n * - `linux`: Linux environment\n * - `ubuntu`: Ubuntu environment\n */\n environment: ComputerUseEnvironment;\n\n /**\n * Execute function that handles computer action execution.\n * This function receives the action input and should return a base64-encoded\n * screenshot of the result.\n */\n execute: (\n action: ComputerUseAction,\n runtime: ToolRuntime<any, any>\n ) => ComputerUseReturnType;\n}\n\n/**\n * OpenAI Computer Use tool type for the Responses API.\n */\nexport type ComputerUseTool = OpenAIClient.Responses.ComputerUsePreviewTool;\n\nconst TOOL_NAME = \"computer_use\";\n\n/**\n * Creates a Computer Use tool that allows models to control computer interfaces\n * and perform tasks by simulating mouse clicks, keyboard input, scrolling, and more.\n *\n * **Computer Use** is a practical application of OpenAI's Computer-Using Agent (CUA)\n * model (`computer-use-preview`), which combines vision capabilities with advanced\n * reasoning to simulate controlling computer interfaces.\n *\n * **How it works**:\n * The tool operates in a continuous loop:\n * 1. Model sends computer actions (click, type, scroll, etc.)\n * 2. Your code executes these actions in a controlled environment\n * 3. You capture a screenshot of the result\n * 4. Send the screenshot back to the model\n * 5. Repeat until the task is complete\n *\n * **Important**: Computer use is in beta and requires careful consideration:\n * - Use in sandboxed environments only\n * - Do not use for high-stakes or authenticated tasks\n * - Always implement human-in-the-loop for important decisions\n * - Handle safety checks appropriately\n *\n * @see {@link https://platform.openai.com/docs/guides/tools-computer-use | OpenAI Computer Use Documentation}\n *\n * @param options - Configuration options for the Computer Use tool\n * @returns A Computer Use tool that can be passed to `bindTools`\n *\n * @example\n * ```typescript\n * import { ChatOpenAI, tools } from \"@langchain/openai\";\n *\n * const model = new ChatOpenAI({ model: \"computer-use-preview\" });\n *\n * // With execute callback for automatic action handling\n * const computer = tools.computerUse({\n * displayWidth: 1024,\n * displayHeight: 768,\n * environment: \"browser\",\n * execute: async (action) => {\n * if (action.type === \"screenshot\") {\n * return captureScreenshot();\n * }\n * if (action.type === \"click\") {\n * await page.mouse.click(action.x, action.y, { button: action.button });\n * return captureScreenshot();\n * }\n * if (action.type === \"type\") {\n * await page.keyboard.type(action.text);\n * return captureScreenshot();\n * }\n * // Handle other actions...\n * return captureScreenshot();\n * },\n * });\n *\n * const llmWithComputer = model.bindTools([computer]);\n * const response = await llmWithComputer.invoke(\n * \"Check the latest news on bing.com\"\n * );\n * ```\n *\n * @example\n * ```typescript\n * // Without execute callback (manual action handling)\n * const computer = tools.computerUse({\n * displayWidth: 1024,\n * displayHeight: 768,\n * environment: \"browser\",\n * });\n *\n * const response = await model.invoke(\"Check the news\", {\n * tools: [computer],\n * });\n *\n * // Access the computer call from the response\n * const computerCall = response.additional_kwargs.tool_outputs?.find(\n * (output) => output.type === \"computer_call\"\n * );\n * if (computerCall) {\n * console.log(\"Action to execute:\", computerCall.action);\n * // Execute the action manually, then send back a screenshot\n * }\n * ```\n *\n * @example\n * ```typescript\n * // For macOS desktop automation with Docker\n * const computer = tools.computerUse({\n * displayWidth: 1920,\n * displayHeight: 1080,\n * environment: \"mac\",\n * execute: async (action) => {\n * if (action.type === \"click\") {\n * await dockerExec(\n * `DISPLAY=:99 xdotool mousemove ${action.x} ${action.y} click 1`,\n * containerName\n * );\n * }\n * // Capture screenshot from container\n * return await getDockerScreenshot(containerName);\n * },\n * });\n * ```\n *\n * @remarks\n * - Only available through the Responses API (not Chat Completions)\n * - Requires `computer-use-preview` model\n * - Actions include: click, double_click, drag, keypress, move, screenshot, scroll, type, wait\n * - Safety checks may be returned that require acknowledgment before proceeding\n * - Use `truncation: \"auto\"` parameter when making requests\n * - Recommended to use with `reasoning.summary` for debugging\n */\nexport function computerUse(options: ComputerUseOptions) {\n const computerTool = tool(\n async (\n input: ComputerUseInput,\n runtime: ToolRuntime<{ messages: BaseMessage[] }>\n ) => {\n /**\n * get computer_use call id from runtime\n */\n const aiMessage = runtime.state?.messages.at(-1) as AIMessage | undefined;\n const computerToolCall = aiMessage?.tool_calls?.find(\n (tc) => tc.name === \"computer_use\"\n );\n const computerToolCallId = computerToolCall?.id;\n if (!computerToolCallId) {\n throw new Error(\"Computer use call id not found\");\n }\n\n const result = await options.execute(input.action, runtime);\n\n /**\n * make sure {@link ToolMessage} is returned with the correct additional kwargs\n */\n if (typeof result === \"string\") {\n return new ToolMessage({\n content: result,\n tool_call_id: computerToolCallId,\n additional_kwargs: {\n type: \"computer_call_output\",\n },\n });\n }\n\n /**\n * make sure {@link ToolMessage} is returned with the correct additional kwargs\n */\n return new ToolMessage({\n ...result,\n tool_call_id: computerToolCallId,\n additional_kwargs: {\n type: \"computer_call_output\",\n ...result.additional_kwargs,\n },\n });\n },\n {\n name: TOOL_NAME,\n description:\n \"Control a computer interface by executing mouse clicks, keyboard input, scrolling, and other actions.\",\n schema: ComputerUseActionSchema,\n }\n );\n\n computerTool.extras = {\n ...(computerTool.extras ?? {}),\n providerToolDefinition: {\n type: \"computer_use_preview\",\n display_width: options.displayWidth,\n display_height: options.displayHeight,\n environment: options.environment,\n } satisfies ComputerUseTool,\n };\n\n /**\n * return as typed {@link DynamicStructuredTool} so we don't get any type\n * errors like \"can't export tool without reference\"\n */\n return computerTool as DynamicStructuredTool<\n typeof ComputerUseActionSchema,\n ComputerUseInput,\n unknown,\n ComputerUseReturnType\n >;\n}\n"],"mappings":";;;;AAkDA,MAAM,oCAAoC,EAAE,OAAO,EACjD,MAAM,EAAE,QAAQ,aAAa,EAC9B,CAAC;AAEF,MAAM,+BAA+B,EAAE,OAAO;CAC5C,MAAM,EAAE,QAAQ,QAAQ;CACxB,GAAG,EAAE,QAAQ;CACb,GAAG,EAAE,QAAQ;CACb,QAAQ,EAAE,KAAK;EAAC;EAAQ;EAAS;EAAS;EAAQ;EAAU,CAAC,CAAC,QAAQ,OAAO;CAC9E,CAAC;AAEF,MAAM,qCAAqC,EAAE,OAAO;CAClD,MAAM,EAAE,QAAQ,eAAe;CAC/B,GAAG,EAAE,QAAQ;CACb,GAAG,EAAE,QAAQ;CACb,QAAQ,EAAE,KAAK;EAAC;EAAQ;EAAS;EAAS;EAAQ;EAAU,CAAC,CAAC,QAAQ,OAAO;CAC9E,CAAC;AAEF,MAAM,8BAA8B,EAAE,OAAO;CAC3C,MAAM,EAAE,QAAQ,OAAO;CACvB,MAAM,EAAE,MAAM,EAAE,OAAO;EAAE,GAAG,EAAE,QAAQ;EAAE,GAAG,EAAE,QAAQ;EAAE,CAAC,CAAC;CAC1D,CAAC;AAEF,MAAM,kCAAkC,EAAE,OAAO;CAC/C,MAAM,EAAE,QAAQ,WAAW;CAC3B,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC;CAC1B,CAAC;AAEF,MAAM,8BAA8B,EAAE,OAAO;CAC3C,MAAM,EAAE,QAAQ,OAAO;CACvB,GAAG,EAAE,QAAQ;CACb,GAAG,EAAE,QAAQ;CACd,CAAC;AAEF,MAAM,gCAAgC,EAAE,OAAO;CAC7C,MAAM,EAAE,QAAQ,SAAS;CACzB,GAAG,EAAE,QAAQ;CACb,GAAG,EAAE,QAAQ;CACb,UAAU,EAAE,QAAQ;CACpB,UAAU,EAAE,QAAQ;CACrB,CAAC;AAEF,MAAM,8BAA8B,EAAE,OAAO;CAC3C,MAAM,EAAE,QAAQ,OAAO;CACvB,MAAM,EAAE,QAAQ;CACjB,CAAC;AAEF,MAAM,8BAA8B,EAAE,OAAO;CAC3C,MAAM,EAAE,QAAQ,OAAO;CACvB,UAAU,EAAE,QAAQ,CAAC,UAAU;CAChC,CAAC;AAGF,MAAM,+BAA+B,EAAE,MAAM;CAC3C;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACD,CAAC;AAIF,MAAa,0BAA0B,EAAE,OAAO,EAC9C,QAAQ,8BACT,CAAC;AAqFF,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkHlB,SAAgB,YAAY,SAA6B;CACvD,MAAM,eAAe,KACnB,OACE,OACA,YACG;EAQH,MAAM,uBAJY,QAAQ,OAAO,SAAS,GAAG,GAAG,GACZ,YAAY,MAC7C,OAAO,GAAG,SAAS,eACrB,GAC4C;AAC7C,MAAI,CAAC,mBACH,OAAM,IAAI,MAAM,iCAAiC;EAGnD,MAAM,SAAS,MAAM,QAAQ,QAAQ,MAAM,QAAQ,QAAQ;;;;AAK3D,MAAI,OAAO,WAAW,SACpB,QAAO,IAAI,YAAY;GACrB,SAAS;GACT,cAAc;GACd,mBAAmB,EACjB,MAAM,wBACP;GACF,CAAC;;;;AAMJ,SAAO,IAAI,YAAY;GACrB,GAAG;GACH,cAAc;GACd,mBAAmB;IACjB,MAAM;IACN,GAAG,OAAO;IACX;GACF,CAAC;IAEJ;EACE,MAAM;EACN,aACE;EACF,QAAQ;EACT,CACF;AAED,cAAa,SAAS;EACpB,GAAI,aAAa,UAAU,EAAE;EAC7B,wBAAwB;GACtB,MAAM;GACN,eAAe,QAAQ;GACvB,gBAAgB,QAAQ;GACxB,aAAa,QAAQ;GACtB;EACF;;;;;AAMD,QAAO"}
|
|
1
|
+
{"version":3,"file":"computerUse.js","names":[],"sources":["../../src/tools/computerUse.ts"],"sourcesContent":["/* oxlint-disable @typescript-eslint/no-explicit-any */\nimport { z } from \"zod/v4\";\nimport { OpenAI as OpenAIClient } from \"openai\";\nimport { tool, type DynamicStructuredTool } from \"@langchain/core/tools\";\nimport { type ToolRuntime } from \"@langchain/core/tools\";\nimport {\n ToolMessage,\n type AIMessage,\n type BaseMessage,\n} from \"@langchain/core/messages\";\n\n/**\n * The type of computer environment to control.\n */\nexport type ComputerUseEnvironment =\n | \"browser\"\n | \"mac\"\n | \"windows\"\n | \"linux\"\n | \"ubuntu\";\n\n/**\n * Re-export action types from OpenAI SDK for convenience.\n */\nexport type ComputerUseClickAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Click;\nexport type ComputerUseDoubleClickAction =\n OpenAIClient.Responses.ResponseComputerToolCall.DoubleClick;\nexport type ComputerUseDragAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Drag;\nexport type ComputerUseKeypressAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Keypress;\nexport type ComputerUseMoveAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Move;\nexport type ComputerUseScreenshotAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Screenshot;\nexport type ComputerUseScrollAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Scroll;\nexport type ComputerUseTypeAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Type;\nexport type ComputerUseWaitAction =\n OpenAIClient.Responses.ResponseComputerToolCall.Wait;\n\n/**\n * Union type of all computer use actions from OpenAI SDK.\n */\nexport type ComputerUseAction =\n OpenAIClient.Responses.ResponseComputerToolCall[\"action\"];\n\n// Zod schemas for computer use actions\nconst ComputerUseScreenshotActionSchema = z.object({\n type: z.literal(\"screenshot\"),\n});\n\nconst ComputerUseClickActionSchema = z.object({\n type: z.literal(\"click\"),\n x: z.number(),\n y: z.number(),\n button: z.enum([\"left\", \"right\", \"wheel\", \"back\", \"forward\"]).default(\"left\"),\n});\n\nconst ComputerUseDoubleClickActionSchema = z.object({\n type: z.literal(\"double_click\"),\n x: z.number(),\n y: z.number(),\n button: z.enum([\"left\", \"right\", \"wheel\", \"back\", \"forward\"]).default(\"left\"),\n});\n\nconst ComputerUseDragActionSchema = z.object({\n type: z.literal(\"drag\"),\n path: z.array(z.object({ x: z.number(), y: z.number() })),\n});\n\nconst ComputerUseKeypressActionSchema = z.object({\n type: z.literal(\"keypress\"),\n keys: z.array(z.string()),\n});\n\nconst ComputerUseMoveActionSchema = z.object({\n type: z.literal(\"move\"),\n x: z.number(),\n y: z.number(),\n});\n\nconst ComputerUseScrollActionSchema = z.object({\n type: z.literal(\"scroll\"),\n x: z.number(),\n y: z.number(),\n scroll_x: z.number(),\n scroll_y: z.number(),\n});\n\nconst ComputerUseTypeActionSchema = z.object({\n type: z.literal(\"type\"),\n text: z.string(),\n});\n\nconst ComputerUseWaitActionSchema = z.object({\n type: z.literal(\"wait\"),\n duration: z.number().optional(),\n});\n\n// Union schema for individual action types\nconst ComputerUseActionUnionSchema = z.union([\n ComputerUseScreenshotActionSchema,\n ComputerUseClickActionSchema,\n ComputerUseDoubleClickActionSchema,\n ComputerUseDragActionSchema,\n ComputerUseKeypressActionSchema,\n ComputerUseMoveActionSchema,\n ComputerUseScrollActionSchema,\n ComputerUseTypeActionSchema,\n ComputerUseWaitActionSchema,\n]);\n\n// Schema for the input structure received from parseComputerCall\n// The action is wrapped in an `action` property: { action: { type: 'screenshot' } }\nexport const ComputerUseActionSchema = z.object({\n action: ComputerUseActionUnionSchema,\n});\n\n// TypeScript types derived from Zod schemas\nexport type ComputerUseScreenshotActionType = z.infer<\n typeof ComputerUseScreenshotActionSchema\n>;\nexport type ComputerUseClickActionType = z.infer<\n typeof ComputerUseClickActionSchema\n>;\nexport type ComputerUseDoubleClickActionType = z.infer<\n typeof ComputerUseDoubleClickActionSchema\n>;\nexport type ComputerUseDragActionType = z.infer<\n typeof ComputerUseDragActionSchema\n>;\nexport type ComputerUseKeypressActionType = z.infer<\n typeof ComputerUseKeypressActionSchema\n>;\nexport type ComputerUseMoveActionType = z.infer<\n typeof ComputerUseMoveActionSchema\n>;\nexport type ComputerUseScrollActionType = z.infer<\n typeof ComputerUseScrollActionSchema\n>;\nexport type ComputerUseTypeActionType = z.infer<\n typeof ComputerUseTypeActionSchema\n>;\nexport type ComputerUseWaitActionType = z.infer<\n typeof ComputerUseWaitActionSchema\n>;\n\n/**\n * Input structure for the Computer Use tool.\n * The action is wrapped in an `action` property.\n */\nexport interface ComputerUseInput {\n action: ComputerUseAction;\n}\n\nexport type ComputerUseReturnType =\n | string\n | Promise<string>\n | ToolMessage<any>\n | Promise<ToolMessage<any>>;\n\n/**\n * Options for the Computer Use tool.\n */\nexport interface ComputerUseOptions {\n /**\n * The width of the computer display in pixels.\n */\n displayWidth: number;\n\n /**\n * The height of the computer display in pixels.\n */\n displayHeight: number;\n\n /**\n * The type of computer environment to control.\n * - `browser`: Browser automation (recommended for most use cases)\n * - `mac`: macOS environment\n * - `windows`: Windows environment\n * - `linux`: Linux environment\n * - `ubuntu`: Ubuntu environment\n */\n environment: ComputerUseEnvironment;\n\n /**\n * Execute function that handles computer action execution.\n * This function receives the action input and should return a base64-encoded\n * screenshot of the result.\n */\n execute: (\n action: ComputerUseAction,\n runtime: ToolRuntime<any, any>\n ) => ComputerUseReturnType;\n}\n\n/**\n * OpenAI Computer Use tool type for the Responses API.\n */\nexport type ComputerUseTool = OpenAIClient.Responses.ComputerUsePreviewTool;\n\nconst TOOL_NAME = \"computer_use\";\n\n/**\n * Creates a Computer Use tool that allows models to control computer interfaces\n * and perform tasks by simulating mouse clicks, keyboard input, scrolling, and more.\n *\n * **Computer Use** is a practical application of OpenAI's Computer-Using Agent (CUA)\n * model (`computer-use-preview`), which combines vision capabilities with advanced\n * reasoning to simulate controlling computer interfaces.\n *\n * **How it works**:\n * The tool operates in a continuous loop:\n * 1. Model sends computer actions (click, type, scroll, etc.)\n * 2. Your code executes these actions in a controlled environment\n * 3. You capture a screenshot of the result\n * 4. Send the screenshot back to the model\n * 5. Repeat until the task is complete\n *\n * **Important**: Computer use is in beta and requires careful consideration:\n * - Use in sandboxed environments only\n * - Do not use for high-stakes or authenticated tasks\n * - Always implement human-in-the-loop for important decisions\n * - Handle safety checks appropriately\n *\n * @see {@link https://platform.openai.com/docs/guides/tools-computer-use | OpenAI Computer Use Documentation}\n *\n * @param options - Configuration options for the Computer Use tool\n * @returns A Computer Use tool that can be passed to `bindTools`\n *\n * @example\n * ```typescript\n * import { ChatOpenAI, tools } from \"@langchain/openai\";\n *\n * const model = new ChatOpenAI({ model: \"computer-use-preview\" });\n *\n * // With execute callback for automatic action handling\n * const computer = tools.computerUse({\n * displayWidth: 1024,\n * displayHeight: 768,\n * environment: \"browser\",\n * execute: async (action) => {\n * if (action.type === \"screenshot\") {\n * return captureScreenshot();\n * }\n * if (action.type === \"click\") {\n * await page.mouse.click(action.x, action.y, { button: action.button });\n * return captureScreenshot();\n * }\n * if (action.type === \"type\") {\n * await page.keyboard.type(action.text);\n * return captureScreenshot();\n * }\n * // Handle other actions...\n * return captureScreenshot();\n * },\n * });\n *\n * const llmWithComputer = model.bindTools([computer]);\n * const response = await llmWithComputer.invoke(\n * \"Check the latest news on bing.com\"\n * );\n * ```\n *\n * @example\n * ```typescript\n * // Without execute callback (manual action handling)\n * const computer = tools.computerUse({\n * displayWidth: 1024,\n * displayHeight: 768,\n * environment: \"browser\",\n * });\n *\n * const response = await model.invoke(\"Check the news\", {\n * tools: [computer],\n * });\n *\n * // Access the computer call from the response\n * const computerCall = response.additional_kwargs.tool_outputs?.find(\n * (output) => output.type === \"computer_call\"\n * );\n * if (computerCall) {\n * console.log(\"Action to execute:\", computerCall.action);\n * // Execute the action manually, then send back a screenshot\n * }\n * ```\n *\n * @example\n * ```typescript\n * // For macOS desktop automation with Docker\n * const computer = tools.computerUse({\n * displayWidth: 1920,\n * displayHeight: 1080,\n * environment: \"mac\",\n * execute: async (action) => {\n * if (action.type === \"click\") {\n * await dockerExec(\n * `DISPLAY=:99 xdotool mousemove ${action.x} ${action.y} click 1`,\n * containerName\n * );\n * }\n * // Capture screenshot from container\n * return await getDockerScreenshot(containerName);\n * },\n * });\n * ```\n *\n * @remarks\n * - Only available through the Responses API (not Chat Completions)\n * - Requires `computer-use-preview` model\n * - Actions include: click, double_click, drag, keypress, move, screenshot, scroll, type, wait\n * - Safety checks may be returned that require acknowledgment before proceeding\n * - Use `truncation: \"auto\"` parameter when making requests\n * - Recommended to use with `reasoning.summary` for debugging\n */\nexport function computerUse(options: ComputerUseOptions) {\n const computerTool = tool(\n async (\n input: ComputerUseInput,\n runtime: ToolRuntime<{ messages: BaseMessage[] }>\n ) => {\n /**\n * get computer_use call id from runtime\n */\n const aiMessage = runtime.state?.messages.at(-1) as AIMessage | undefined;\n const computerToolCall = aiMessage?.tool_calls?.find(\n (tc) => tc.name === \"computer_use\"\n );\n const computerToolCallId = computerToolCall?.id;\n if (!computerToolCallId) {\n throw new Error(\"Computer use call id not found\");\n }\n\n const result = await options.execute(input.action, runtime);\n\n /**\n * make sure {@link ToolMessage} is returned with the correct additional kwargs\n */\n if (typeof result === \"string\") {\n return new ToolMessage({\n content: result,\n tool_call_id: computerToolCallId,\n additional_kwargs: {\n type: \"computer_call_output\",\n },\n });\n }\n\n /**\n * make sure {@link ToolMessage} is returned with the correct additional kwargs\n */\n return new ToolMessage({\n ...result,\n tool_call_id: computerToolCallId,\n additional_kwargs: {\n type: \"computer_call_output\",\n ...result.additional_kwargs,\n },\n });\n },\n {\n name: TOOL_NAME,\n description:\n \"Control a computer interface by executing mouse clicks, keyboard input, scrolling, and other actions.\",\n schema: ComputerUseActionSchema,\n }\n );\n\n computerTool.extras = {\n ...(computerTool.extras ?? {}),\n providerToolDefinition: {\n type: \"computer_use_preview\",\n display_width: options.displayWidth,\n display_height: options.displayHeight,\n environment: options.environment,\n } satisfies ComputerUseTool,\n };\n\n /**\n * return as typed {@link DynamicStructuredTool} so we don't get any type\n * errors like \"can't export tool without reference\"\n */\n return computerTool as DynamicStructuredTool<\n typeof ComputerUseActionSchema,\n ComputerUseInput,\n unknown,\n ComputerUseReturnType\n >;\n}\n"],"mappings":";;;;AAkDA,MAAM,oCAAoC,EAAE,OAAO,EACjD,MAAM,EAAE,QAAQ,YAAY,EAC9B,CAAC;AAED,MAAM,+BAA+B,EAAE,OAAO;CAC5C,MAAM,EAAE,QAAQ,OAAO;CACvB,GAAG,EAAE,OAAO;CACZ,GAAG,EAAE,OAAO;CACZ,QAAQ,EAAE,KAAK;EAAC;EAAQ;EAAS;EAAS;EAAQ;CAAS,CAAC,CAAC,CAAC,QAAQ,MAAM;AAC9E,CAAC;AAED,MAAM,qCAAqC,EAAE,OAAO;CAClD,MAAM,EAAE,QAAQ,cAAc;CAC9B,GAAG,EAAE,OAAO;CACZ,GAAG,EAAE,OAAO;CACZ,QAAQ,EAAE,KAAK;EAAC;EAAQ;EAAS;EAAS;EAAQ;CAAS,CAAC,CAAC,CAAC,QAAQ,MAAM;AAC9E,CAAC;AAED,MAAM,8BAA8B,EAAE,OAAO;CAC3C,MAAM,EAAE,QAAQ,MAAM;CACtB,MAAM,EAAE,MAAM,EAAE,OAAO;EAAE,GAAG,EAAE,OAAO;EAAG,GAAG,EAAE,OAAO;CAAE,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,kCAAkC,EAAE,OAAO;CAC/C,MAAM,EAAE,QAAQ,UAAU;CAC1B,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC;AAC1B,CAAC;AAED,MAAM,8BAA8B,EAAE,OAAO;CAC3C,MAAM,EAAE,QAAQ,MAAM;CACtB,GAAG,EAAE,OAAO;CACZ,GAAG,EAAE,OAAO;AACd,CAAC;AAED,MAAM,gCAAgC,EAAE,OAAO;CAC7C,MAAM,EAAE,QAAQ,QAAQ;CACxB,GAAG,EAAE,OAAO;CACZ,GAAG,EAAE,OAAO;CACZ,UAAU,EAAE,OAAO;CACnB,UAAU,EAAE,OAAO;AACrB,CAAC;AAED,MAAM,8BAA8B,EAAE,OAAO;CAC3C,MAAM,EAAE,QAAQ,MAAM;CACtB,MAAM,EAAE,OAAO;AACjB,CAAC;AAED,MAAM,8BAA8B,EAAE,OAAO;CAC3C,MAAM,EAAE,QAAQ,MAAM;CACtB,UAAU,EAAE,OAAO,CAAC,CAAC,SAAS;AAChC,CAAC;AAGD,MAAM,+BAA+B,EAAE,MAAM;CAC3C;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;AAID,MAAa,0BAA0B,EAAE,OAAO,EAC9C,QAAQ,6BACV,CAAC;AAqFD,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkHlB,SAAgB,YAAY,SAA6B;CACvD,MAAM,eAAe,KACnB,OACE,OACA,YACG;EAQH,MAAM,uBAJY,QAAQ,OAAO,SAAS,GAAG,EAAE,EAAA,EACX,YAAY,MAC7C,OAAO,GAAG,SAAS,cACtB,EAAA,EAC6C;EAC7C,IAAI,CAAC,oBACH,MAAM,IAAI,MAAM,gCAAgC;EAGlD,MAAM,SAAS,MAAM,QAAQ,QAAQ,MAAM,QAAQ,OAAO;;;;EAK1D,IAAI,OAAO,WAAW,UACpB,OAAO,IAAI,YAAY;GACrB,SAAS;GACT,cAAc;GACd,mBAAmB,EACjB,MAAM,uBACR;EACF,CAAC;;;;EAMH,OAAO,IAAI,YAAY;GACrB,GAAG;GACH,cAAc;GACd,mBAAmB;IACjB,MAAM;IACN,GAAG,OAAO;GACZ;EACF,CAAC;CACH,GACA;EACE,MAAM;EACN,aACE;EACF,QAAQ;CACV,CACF;CAEA,aAAa,SAAS;EACpB,GAAI,aAAa,UAAU,CAAC;EAC5B,wBAAwB;GACtB,MAAM;GACN,eAAe,QAAQ;GACvB,gBAAgB,QAAQ;GACxB,aAAa,QAAQ;EACvB;CACF;;;;;CAMA,OAAO;AAMT"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"custom.cjs","names":["DynamicTool","AsyncLocalStorageProviderSingleton"],"sources":["../../src/tools/custom.ts"],"sourcesContent":["import {\n patchConfig,\n pickRunnableConfigKeys,\n RunnableFunc,\n} from \"@langchain/core/runnables\";\nimport { AsyncLocalStorageProviderSingleton } from \"@langchain/core/singletons\";\nimport { DynamicTool, ToolRunnableConfig } from \"@langchain/core/tools\";\nimport OpenAI from \"openai\";\n\nexport type CustomToolFields = Omit<OpenAI.Responses.CustomTool, \"type\">;\n\nexport function customTool(\n func: RunnableFunc<string, string, ToolRunnableConfig>,\n fields: CustomToolFields\n): DynamicTool<string> {\n return new DynamicTool({\n ...fields,\n description: \"\",\n metadata: {\n customTool: fields,\n },\n func: async (input, runManager, config) =>\n new Promise<string>((resolve, reject) => {\n const childConfig = patchConfig(config, {\n callbacks: runManager?.getChild(),\n });\n // oxlint-disable-next-line no-void\n void AsyncLocalStorageProviderSingleton.runWithConfig(\n pickRunnableConfigKeys(childConfig),\n async () => {\n try {\n resolve(func(input, childConfig));\n } catch (e) {\n reject(e);\n }\n }\n );\n }),\n });\n}\n"],"mappings":";;;;AAWA,SAAgB,WACd,MACA,QACqB;
|
|
1
|
+
{"version":3,"file":"custom.cjs","names":["DynamicTool","AsyncLocalStorageProviderSingleton"],"sources":["../../src/tools/custom.ts"],"sourcesContent":["import {\n patchConfig,\n pickRunnableConfigKeys,\n RunnableFunc,\n} from \"@langchain/core/runnables\";\nimport { AsyncLocalStorageProviderSingleton } from \"@langchain/core/singletons\";\nimport { DynamicTool, ToolRunnableConfig } from \"@langchain/core/tools\";\nimport OpenAI from \"openai\";\n\nexport type CustomToolFields = Omit<OpenAI.Responses.CustomTool, \"type\">;\n\nexport function customTool(\n func: RunnableFunc<string, string, ToolRunnableConfig>,\n fields: CustomToolFields\n): DynamicTool<string> {\n return new DynamicTool({\n ...fields,\n description: \"\",\n metadata: {\n customTool: fields,\n },\n func: async (input, runManager, config) =>\n new Promise<string>((resolve, reject) => {\n const childConfig = patchConfig(config, {\n callbacks: runManager?.getChild(),\n });\n // oxlint-disable-next-line no-void\n void AsyncLocalStorageProviderSingleton.runWithConfig(\n pickRunnableConfigKeys(childConfig),\n async () => {\n try {\n resolve(func(input, childConfig));\n } catch (e) {\n reject(e);\n }\n }\n );\n }),\n });\n}\n"],"mappings":";;;;AAWA,SAAgB,WACd,MACA,QACqB;CACrB,OAAO,IAAIA,sBAAAA,YAAY;EACrB,GAAG;EACH,aAAa;EACb,UAAU,EACR,YAAY,OACd;EACA,MAAM,OAAO,OAAO,YAAY,WAC9B,IAAI,SAAiB,SAAS,WAAW;GACvC,MAAM,eAAA,GAAA,0BAAA,YAAA,CAA0B,QAAQ,EACtC,WAAW,YAAY,SAAS,EAClC,CAAC;GAED,2BAAKC,mCAAmC,eAAA,GAAA,0BAAA,uBAAA,CACf,WAAW,GAClC,YAAY;IACV,IAAI;KACF,QAAQ,KAAK,OAAO,WAAW,CAAC;IAClC,SAAS,GAAG;KACV,OAAO,CAAC;IACV;GACF,CACF;EACF,CAAC;CACL,CAAC;AACH"}
|
package/dist/tools/custom.d.cts
CHANGED
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
import OpenAI$1 from "openai";
|
|
2
2
|
import { DynamicTool, ToolRunnableConfig } from "@langchain/core/tools";
|
|
3
3
|
import { RunnableFunc } from "@langchain/core/runnables";
|
|
4
|
-
|
|
5
4
|
//#region src/tools/custom.d.ts
|
|
6
5
|
type CustomToolFields = Omit<OpenAI$1.Responses.CustomTool, "type">;
|
|
7
6
|
declare function customTool(func: RunnableFunc<string, string, ToolRunnableConfig>, fields: CustomToolFields): DynamicTool<string>;
|
|
8
7
|
//#endregion
|
|
9
|
-
export { customTool };
|
|
8
|
+
export { CustomToolFields, customTool };
|
|
10
9
|
//# sourceMappingURL=custom.d.cts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"custom.d.cts","names":[],"sources":["../../src/tools/custom.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"custom.d.cts","names":[],"sources":["../../src/tools/custom.ts"],"mappings":";;;;KASY,mBAAmB,KAAK,SAAO,UAAU;iBAErC,WACd,MAAM,6BAA6B,qBACnC,QAAQ,mBACP"}
|
package/dist/tools/custom.d.ts
CHANGED
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
import OpenAI$1 from "openai";
|
|
2
2
|
import { RunnableFunc } from "@langchain/core/runnables";
|
|
3
3
|
import { DynamicTool, ToolRunnableConfig } from "@langchain/core/tools";
|
|
4
|
-
|
|
5
4
|
//#region src/tools/custom.d.ts
|
|
6
5
|
type CustomToolFields = Omit<OpenAI$1.Responses.CustomTool, "type">;
|
|
7
6
|
declare function customTool(func: RunnableFunc<string, string, ToolRunnableConfig>, fields: CustomToolFields): DynamicTool<string>;
|
|
8
7
|
//#endregion
|
|
9
|
-
export { customTool };
|
|
8
|
+
export { CustomToolFields, customTool };
|
|
10
9
|
//# sourceMappingURL=custom.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"custom.d.ts","names":[],"sources":["../../src/tools/custom.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"custom.d.ts","names":[],"sources":["../../src/tools/custom.ts"],"mappings":";;;;KASY,mBAAmB,KAAK,SAAO,UAAU;iBAErC,WACd,MAAM,6BAA6B,qBACnC,QAAQ,mBACP"}
|
package/dist/tools/custom.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"custom.js","names":[],"sources":["../../src/tools/custom.ts"],"sourcesContent":["import {\n patchConfig,\n pickRunnableConfigKeys,\n RunnableFunc,\n} from \"@langchain/core/runnables\";\nimport { AsyncLocalStorageProviderSingleton } from \"@langchain/core/singletons\";\nimport { DynamicTool, ToolRunnableConfig } from \"@langchain/core/tools\";\nimport OpenAI from \"openai\";\n\nexport type CustomToolFields = Omit<OpenAI.Responses.CustomTool, \"type\">;\n\nexport function customTool(\n func: RunnableFunc<string, string, ToolRunnableConfig>,\n fields: CustomToolFields\n): DynamicTool<string> {\n return new DynamicTool({\n ...fields,\n description: \"\",\n metadata: {\n customTool: fields,\n },\n func: async (input, runManager, config) =>\n new Promise<string>((resolve, reject) => {\n const childConfig = patchConfig(config, {\n callbacks: runManager?.getChild(),\n });\n // oxlint-disable-next-line no-void\n void AsyncLocalStorageProviderSingleton.runWithConfig(\n pickRunnableConfigKeys(childConfig),\n async () => {\n try {\n resolve(func(input, childConfig));\n } catch (e) {\n reject(e);\n }\n }\n );\n }),\n });\n}\n"],"mappings":";;;;AAWA,SAAgB,WACd,MACA,QACqB;
|
|
1
|
+
{"version":3,"file":"custom.js","names":[],"sources":["../../src/tools/custom.ts"],"sourcesContent":["import {\n patchConfig,\n pickRunnableConfigKeys,\n RunnableFunc,\n} from \"@langchain/core/runnables\";\nimport { AsyncLocalStorageProviderSingleton } from \"@langchain/core/singletons\";\nimport { DynamicTool, ToolRunnableConfig } from \"@langchain/core/tools\";\nimport OpenAI from \"openai\";\n\nexport type CustomToolFields = Omit<OpenAI.Responses.CustomTool, \"type\">;\n\nexport function customTool(\n func: RunnableFunc<string, string, ToolRunnableConfig>,\n fields: CustomToolFields\n): DynamicTool<string> {\n return new DynamicTool({\n ...fields,\n description: \"\",\n metadata: {\n customTool: fields,\n },\n func: async (input, runManager, config) =>\n new Promise<string>((resolve, reject) => {\n const childConfig = patchConfig(config, {\n callbacks: runManager?.getChild(),\n });\n // oxlint-disable-next-line no-void\n void AsyncLocalStorageProviderSingleton.runWithConfig(\n pickRunnableConfigKeys(childConfig),\n async () => {\n try {\n resolve(func(input, childConfig));\n } catch (e) {\n reject(e);\n }\n }\n );\n }),\n });\n}\n"],"mappings":";;;;AAWA,SAAgB,WACd,MACA,QACqB;CACrB,OAAO,IAAI,YAAY;EACrB,GAAG;EACH,aAAa;EACb,UAAU,EACR,YAAY,OACd;EACA,MAAM,OAAO,OAAO,YAAY,WAC9B,IAAI,SAAiB,SAAS,WAAW;GACvC,MAAM,cAAc,YAAY,QAAQ,EACtC,WAAW,YAAY,SAAS,EAClC,CAAC;GAED,mCAAwC,cACtC,uBAAuB,WAAW,GAClC,YAAY;IACV,IAAI;KACF,QAAQ,KAAK,OAAO,WAAW,CAAC;IAClC,SAAS,GAAG;KACV,OAAO,CAAC;IACV;GACF,CACF;EACF,CAAC;CACL,CAAC;AACH"}
|
package/dist/tools/dalle.cjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dalle.cjs","names":["Tool","OpenAIClient"],"sources":["../../src/tools/dalle.ts"],"sourcesContent":["import { getEnvironmentVariable } from \"@langchain/core/utils/env\";\nimport { OpenAI as OpenAIClient } from \"openai\";\nimport { Tool, ToolParams } from \"@langchain/core/tools\";\nimport {\n MessageContentComplex,\n MessageContentImageUrl,\n} from \"@langchain/core/messages\";\n\n/**\n * @see https://platform.openai.com/docs/api-reference/images/create\n */\nexport type OpenAIImageModelId =\n | OpenAIClient.ImageModel\n | (string & NonNullable<unknown>);\n\n/**\n * An interface for the Dall-E API Wrapper.\n */\nexport interface DallEAPIWrapperParams extends ToolParams {\n /**\n * The OpenAI API key\n * Alias for `apiKey`\n */\n openAIApiKey?: string;\n /**\n * The OpenAI API key\n */\n apiKey?: string;\n /**\n * The model to use.\n * Alias for `model`\n * @params \"dall-e-2\" | \"dall-e-3\"\n * @default \"dall-e-3\"\n * @deprecated Use `model` instead.\n */\n modelName?: OpenAIImageModelId;\n /**\n * The model to use.\n * @params \"dall-e-2\" | \"dall-e-3\"\n * @default \"dall-e-3\"\n */\n model?: OpenAIImageModelId;\n /**\n * The style of the generated images. Must be one of vivid or natural.\n * Vivid causes the model to lean towards generating hyper-real and dramatic images.\n * Natural causes the model to produce more natural, less hyper-real looking images.\n * @default \"vivid\"\n */\n style?: \"natural\" | \"vivid\";\n /**\n * The quality of the image that will be generated. ‘hd’ creates images with finer\n * details and greater consistency across the image.\n * @default \"standard\"\n */\n quality?: \"standard\" | \"hd\";\n /**\n * The number of images to generate.\n * Must be between 1 and 10.\n * For dall-e-3, only `n: 1` is supported.\n * @default 1\n */\n n?: number;\n /**\n * The size of the generated images.\n * Must be one of 256x256, 512x512, or 1024x1024 for DALL·E-2 models.\n * Must be one of 1024x1024, 1792x1024, or 1024x1792 for DALL·E-3 models.\n * @default \"1024x1024\"\n */\n size?: \"256x256\" | \"512x512\" | \"1024x1024\" | \"1792x1024\" | \"1024x1792\";\n /**\n * The format in which the generated images are returned.\n * Must be one of \"url\" or \"b64_json\".\n * @default \"url\"\n */\n dallEResponseFormat?: \"url\" | \"b64_json\";\n /**\n * @deprecated Use dallEResponseFormat instead for the Dall-E response type.\n */\n // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n responseFormat?: any;\n /**\n * A unique identifier representing your end-user, which will help\n * OpenAI to monitor and detect abuse.\n */\n user?: string;\n /**\n * The organization to use\n */\n organization?: string;\n /**\n * The base URL of the OpenAI API.\n */\n baseUrl?: string;\n}\n\n/**\n * A tool for generating images with Open AIs Dall-E 2 or 3 API.\n */\nexport class DallEAPIWrapper extends Tool {\n static lc_name() {\n return \"DallEAPIWrapper\";\n }\n\n name = \"dalle_api_wrapper\";\n\n description =\n \"A wrapper around OpenAI DALL-E API. Useful for when you need to generate images from a text description. Input should be an image description.\";\n\n protected client: OpenAIClient;\n\n static readonly toolName = \"dalle_api_wrapper\";\n\n private model = \"dall-e-3\";\n\n private style: \"natural\" | \"vivid\" = \"vivid\";\n\n private quality: \"standard\" | \"hd\" = \"standard\";\n\n private n = 1;\n\n private size:\n | \"256x256\"\n | \"512x512\"\n | \"1024x1024\"\n | \"1792x1024\"\n | \"1024x1792\" = \"1024x1024\";\n\n private dallEResponseFormat: \"url\" | \"b64_json\" = \"url\";\n\n private user?: string;\n\n constructor(fields?: DallEAPIWrapperParams) {\n // Shim for new base tool param name\n if (\n fields?.responseFormat !== undefined &&\n [\"url\", \"b64_json\"].includes(fields.responseFormat)\n ) {\n // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n fields.dallEResponseFormat = fields.responseFormat as any;\n fields.responseFormat = \"content\";\n }\n super(fields);\n const openAIApiKey =\n fields?.apiKey ??\n fields?.openAIApiKey ??\n getEnvironmentVariable(\"OPENAI_API_KEY\");\n\n const organization =\n fields?.organization ?? getEnvironmentVariable(\"OPENAI_ORGANIZATION\");\n\n const clientConfig = {\n apiKey: openAIApiKey,\n organization,\n dangerouslyAllowBrowser: true,\n baseURL: fields?.baseUrl,\n };\n this.client = new OpenAIClient(clientConfig);\n this.model = fields?.model ?? fields?.modelName ?? this.model;\n this.style = fields?.style ?? this.style;\n this.quality = fields?.quality ?? this.quality;\n this.n = fields?.n ?? this.n;\n this.size = fields?.size ?? this.size;\n this.dallEResponseFormat =\n fields?.dallEResponseFormat ?? this.dallEResponseFormat;\n this.user = fields?.user;\n }\n\n /**\n * Processes the API response if multiple images are generated.\n * Returns a list of MessageContentImageUrl objects. If the response\n * format is `url`, then the `image_url` field will contain the URL.\n * If it is `b64_json`, then the `image_url` field will contain an object\n * with a `url` field with the base64 encoded image.\n *\n * @param {OpenAIClient.Images.ImagesResponse[]} response The API response\n * @returns {MessageContentImageUrl[]}\n */\n private processMultipleGeneratedUrls(\n response: OpenAIClient.Images.ImagesResponse[]\n ): MessageContentImageUrl[] {\n if (this.dallEResponseFormat === \"url\") {\n return response.flatMap((res) => {\n const imageUrlContent =\n res.data\n ?.flatMap((item) => {\n if (!item.url) return [];\n return {\n type: \"image_url\" as const,\n image_url: item.url,\n };\n })\n .filter(\n (item) =>\n item !== undefined &&\n item.type === \"image_url\" &&\n typeof item.image_url === \"string\" &&\n item.image_url !== undefined\n ) ?? [];\n return imageUrlContent;\n });\n } else {\n return response.flatMap((res) => {\n const b64Content =\n res.data\n ?.flatMap((item) => {\n if (!item.b64_json) return [];\n return {\n type: \"image_url\" as const,\n image_url: {\n url: item.b64_json,\n },\n };\n })\n .filter(\n (item) =>\n item !== undefined &&\n item.type === \"image_url\" &&\n typeof item.image_url === \"object\" &&\n \"url\" in item.image_url &&\n typeof item.image_url.url === \"string\" &&\n item.image_url.url !== undefined\n ) ?? [];\n return b64Content;\n });\n }\n }\n\n /** @ignore */\n async _call(input: string): Promise<string | MessageContentComplex[]> {\n const generateImageFields = {\n model: this.model,\n prompt: input,\n n: 1,\n size: this.size,\n response_format: this.dallEResponseFormat,\n style: this.style,\n quality: this.quality,\n user: this.user,\n };\n\n if (this.n > 1) {\n const results = await Promise.all(\n Array.from({ length: this.n }).map(() =>\n this.client.images.generate(generateImageFields)\n )\n );\n\n return this.processMultipleGeneratedUrls(results);\n }\n\n const response = await this.client.images.generate(generateImageFields);\n\n let data = \"\";\n if (this.dallEResponseFormat === \"url\") {\n [data] =\n response.data\n ?.map((item) => item.url)\n .filter((url): url is string => url !== \"undefined\") ?? [];\n } else {\n [data] =\n response.data\n ?.map((item) => item.b64_json)\n .filter((b64_json): b64_json is string => b64_json !== \"undefined\") ??\n [];\n }\n return data;\n }\n}\n"],"mappings":";;;;;;;AAkGA,IAAa,kBAAb,cAAqCA,sBAAAA,KAAK;CACxC,OAAO,UAAU;AACf,SAAO;;CAGT,OAAO;CAEP,cACE;CAEF;CAEA,OAAgB,WAAW;CAE3B,QAAgB;CAEhB,QAAqC;CAErC,UAAqC;CAErC,IAAY;CAEZ,OAKkB;CAElB,sBAAkD;CAElD;CAEA,YAAY,QAAgC;AAE1C,MACE,QAAQ,mBAAmB,KAAA,KAC3B,CAAC,OAAO,WAAW,CAAC,SAAS,OAAO,eAAe,EACnD;AAEA,UAAO,sBAAsB,OAAO;AACpC,UAAO,iBAAiB;;AAE1B,QAAM,OAAO;EASb,MAAM,eAAe;GACnB,QARA,QAAQ,UACR,QAAQ,iBAAA,GAAA,0BAAA,wBACe,iBAAiB;GAOxC,cAJA,QAAQ,iBAAA,GAAA,0BAAA,wBAAuC,sBAAsB;GAKrE,yBAAyB;GACzB,SAAS,QAAQ;GAClB;AACD,OAAK,SAAS,IAAIC,OAAAA,OAAa,aAAa;AAC5C,OAAK,QAAQ,QAAQ,SAAS,QAAQ,aAAa,KAAK;AACxD,OAAK,QAAQ,QAAQ,SAAS,KAAK;AACnC,OAAK,UAAU,QAAQ,WAAW,KAAK;AACvC,OAAK,IAAI,QAAQ,KAAK,KAAK;AAC3B,OAAK,OAAO,QAAQ,QAAQ,KAAK;AACjC,OAAK,sBACH,QAAQ,uBAAuB,KAAK;AACtC,OAAK,OAAO,QAAQ;;;;;;;;;;;;CAatB,6BACE,UAC0B;AAC1B,MAAI,KAAK,wBAAwB,MAC/B,QAAO,SAAS,SAAS,QAAQ;AAiB/B,UAfE,IAAI,MACA,SAAS,SAAS;AAClB,QAAI,CAAC,KAAK,IAAK,QAAO,EAAE;AACxB,WAAO;KACL,MAAM;KACN,WAAW,KAAK;KACjB;KACD,CACD,QACE,SACC,SAAS,KAAA,KACT,KAAK,SAAS,eACd,OAAO,KAAK,cAAc,YAC1B,KAAK,cAAc,KAAA,EACtB,IAAI,EAAE;IAEX;MAEF,QAAO,SAAS,SAAS,QAAQ;AAqB/B,UAnBE,IAAI,MACA,SAAS,SAAS;AAClB,QAAI,CAAC,KAAK,SAAU,QAAO,EAAE;AAC7B,WAAO;KACL,MAAM;KACN,WAAW,EACT,KAAK,KAAK,UACX;KACF;KACD,CACD,QACE,SACC,SAAS,KAAA,KACT,KAAK,SAAS,eACd,OAAO,KAAK,cAAc,YAC1B,SAAS,KAAK,aACd,OAAO,KAAK,UAAU,QAAQ,YAC9B,KAAK,UAAU,QAAQ,KAAA,EAC1B,IAAI,EAAE;IAEX;;;CAKN,MAAM,MAAM,OAA0D;EACpE,MAAM,sBAAsB;GAC1B,OAAO,KAAK;GACZ,QAAQ;GACR,GAAG;GACH,MAAM,KAAK;GACX,iBAAiB,KAAK;GACtB,OAAO,KAAK;GACZ,SAAS,KAAK;GACd,MAAM,KAAK;GACZ;AAED,MAAI,KAAK,IAAI,GAAG;GACd,MAAM,UAAU,MAAM,QAAQ,IAC5B,MAAM,KAAK,EAAE,QAAQ,KAAK,GAAG,CAAC,CAAC,UAC7B,KAAK,OAAO,OAAO,SAAS,oBAAoB,CACjD,CACF;AAED,UAAO,KAAK,6BAA6B,QAAQ;;EAGnD,MAAM,WAAW,MAAM,KAAK,OAAO,OAAO,SAAS,oBAAoB;EAEvE,IAAI,OAAO;AACX,MAAI,KAAK,wBAAwB,MAC/B,EAAC,QACC,SAAS,MACL,KAAK,SAAS,KAAK,IAAI,CACxB,QAAQ,QAAuB,QAAQ,YAAY,IAAI,EAAE;MAE9D,EAAC,QACC,SAAS,MACL,KAAK,SAAS,KAAK,SAAS,CAC7B,QAAQ,aAAiC,aAAa,YAAY,IACrE,EAAE;AAEN,SAAO"}
|
|
1
|
+
{"version":3,"file":"dalle.cjs","names":["Tool","OpenAIClient"],"sources":["../../src/tools/dalle.ts"],"sourcesContent":["import { getEnvironmentVariable } from \"@langchain/core/utils/env\";\nimport { OpenAI as OpenAIClient } from \"openai\";\nimport { Tool, ToolParams } from \"@langchain/core/tools\";\nimport {\n MessageContentComplex,\n MessageContentImageUrl,\n} from \"@langchain/core/messages\";\n\n/**\n * @see https://platform.openai.com/docs/api-reference/images/create\n */\nexport type OpenAIImageModelId =\n | OpenAIClient.ImageModel\n | (string & NonNullable<unknown>);\n\n/**\n * An interface for the Dall-E API Wrapper.\n */\nexport interface DallEAPIWrapperParams extends ToolParams {\n /**\n * The OpenAI API key\n * Alias for `apiKey`\n */\n openAIApiKey?: string;\n /**\n * The OpenAI API key\n */\n apiKey?: string;\n /**\n * The model to use.\n * Alias for `model`\n * @params \"dall-e-2\" | \"dall-e-3\"\n * @default \"dall-e-3\"\n * @deprecated Use `model` instead.\n */\n modelName?: OpenAIImageModelId;\n /**\n * The model to use.\n * @params \"dall-e-2\" | \"dall-e-3\"\n * @default \"dall-e-3\"\n */\n model?: OpenAIImageModelId;\n /**\n * The style of the generated images. Must be one of vivid or natural.\n * Vivid causes the model to lean towards generating hyper-real and dramatic images.\n * Natural causes the model to produce more natural, less hyper-real looking images.\n * @default \"vivid\"\n */\n style?: \"natural\" | \"vivid\";\n /**\n * The quality of the image that will be generated. ‘hd’ creates images with finer\n * details and greater consistency across the image.\n * @default \"standard\"\n */\n quality?: \"standard\" | \"hd\";\n /**\n * The number of images to generate.\n * Must be between 1 and 10.\n * For dall-e-3, only `n: 1` is supported.\n * @default 1\n */\n n?: number;\n /**\n * The size of the generated images.\n * Must be one of 256x256, 512x512, or 1024x1024 for DALL·E-2 models.\n * Must be one of 1024x1024, 1792x1024, or 1024x1792 for DALL·E-3 models.\n * @default \"1024x1024\"\n */\n size?: \"256x256\" | \"512x512\" | \"1024x1024\" | \"1792x1024\" | \"1024x1792\";\n /**\n * The format in which the generated images are returned.\n * Must be one of \"url\" or \"b64_json\".\n * @default \"url\"\n */\n dallEResponseFormat?: \"url\" | \"b64_json\";\n /**\n * @deprecated Use dallEResponseFormat instead for the Dall-E response type.\n */\n // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n responseFormat?: any;\n /**\n * A unique identifier representing your end-user, which will help\n * OpenAI to monitor and detect abuse.\n */\n user?: string;\n /**\n * The organization to use\n */\n organization?: string;\n /**\n * The base URL of the OpenAI API.\n */\n baseUrl?: string;\n}\n\n/**\n * A tool for generating images with Open AIs Dall-E 2 or 3 API.\n */\nexport class DallEAPIWrapper extends Tool {\n static lc_name() {\n return \"DallEAPIWrapper\";\n }\n\n name = \"dalle_api_wrapper\";\n\n description =\n \"A wrapper around OpenAI DALL-E API. Useful for when you need to generate images from a text description. Input should be an image description.\";\n\n protected client: OpenAIClient;\n\n static readonly toolName = \"dalle_api_wrapper\";\n\n private model = \"dall-e-3\";\n\n private style: \"natural\" | \"vivid\" = \"vivid\";\n\n private quality: \"standard\" | \"hd\" = \"standard\";\n\n private n = 1;\n\n private size:\n | \"256x256\"\n | \"512x512\"\n | \"1024x1024\"\n | \"1792x1024\"\n | \"1024x1792\" = \"1024x1024\";\n\n private dallEResponseFormat: \"url\" | \"b64_json\" = \"url\";\n\n private user?: string;\n\n constructor(fields?: DallEAPIWrapperParams) {\n // Shim for new base tool param name\n if (\n fields?.responseFormat !== undefined &&\n [\"url\", \"b64_json\"].includes(fields.responseFormat)\n ) {\n // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n fields.dallEResponseFormat = fields.responseFormat as any;\n fields.responseFormat = \"content\";\n }\n super(fields);\n const openAIApiKey =\n fields?.apiKey ??\n fields?.openAIApiKey ??\n getEnvironmentVariable(\"OPENAI_API_KEY\");\n\n const organization =\n fields?.organization ?? getEnvironmentVariable(\"OPENAI_ORGANIZATION\");\n\n const clientConfig = {\n apiKey: openAIApiKey,\n organization,\n dangerouslyAllowBrowser: true,\n baseURL: fields?.baseUrl,\n };\n this.client = new OpenAIClient(clientConfig);\n this.model = fields?.model ?? fields?.modelName ?? this.model;\n this.style = fields?.style ?? this.style;\n this.quality = fields?.quality ?? this.quality;\n this.n = fields?.n ?? this.n;\n this.size = fields?.size ?? this.size;\n this.dallEResponseFormat =\n fields?.dallEResponseFormat ?? this.dallEResponseFormat;\n this.user = fields?.user;\n }\n\n /**\n * Processes the API response if multiple images are generated.\n * Returns a list of MessageContentImageUrl objects. If the response\n * format is `url`, then the `image_url` field will contain the URL.\n * If it is `b64_json`, then the `image_url` field will contain an object\n * with a `url` field with the base64 encoded image.\n *\n * @param {OpenAIClient.Images.ImagesResponse[]} response The API response\n * @returns {MessageContentImageUrl[]}\n */\n private processMultipleGeneratedUrls(\n response: OpenAIClient.Images.ImagesResponse[]\n ): MessageContentImageUrl[] {\n if (this.dallEResponseFormat === \"url\") {\n return response.flatMap((res) => {\n const imageUrlContent =\n res.data\n ?.flatMap((item) => {\n if (!item.url) return [];\n return {\n type: \"image_url\" as const,\n image_url: item.url,\n };\n })\n .filter(\n (item) =>\n item !== undefined &&\n item.type === \"image_url\" &&\n typeof item.image_url === \"string\" &&\n item.image_url !== undefined\n ) ?? [];\n return imageUrlContent;\n });\n } else {\n return response.flatMap((res) => {\n const b64Content =\n res.data\n ?.flatMap((item) => {\n if (!item.b64_json) return [];\n return {\n type: \"image_url\" as const,\n image_url: {\n url: item.b64_json,\n },\n };\n })\n .filter(\n (item) =>\n item !== undefined &&\n item.type === \"image_url\" &&\n typeof item.image_url === \"object\" &&\n \"url\" in item.image_url &&\n typeof item.image_url.url === \"string\" &&\n item.image_url.url !== undefined\n ) ?? [];\n return b64Content;\n });\n }\n }\n\n /** @ignore */\n async _call(input: string): Promise<string | MessageContentComplex[]> {\n const generateImageFields = {\n model: this.model,\n prompt: input,\n n: 1,\n size: this.size,\n response_format: this.dallEResponseFormat,\n style: this.style,\n quality: this.quality,\n user: this.user,\n };\n\n if (this.n > 1) {\n const results = await Promise.all(\n Array.from({ length: this.n }).map(() =>\n this.client.images.generate(generateImageFields)\n )\n );\n\n return this.processMultipleGeneratedUrls(results);\n }\n\n const response = await this.client.images.generate(generateImageFields);\n\n let data = \"\";\n if (this.dallEResponseFormat === \"url\") {\n [data] =\n response.data\n ?.map((item) => item.url)\n .filter((url): url is string => url !== \"undefined\") ?? [];\n } else {\n [data] =\n response.data\n ?.map((item) => item.b64_json)\n .filter((b64_json): b64_json is string => b64_json !== \"undefined\") ??\n [];\n }\n return data;\n }\n}\n"],"mappings":";;;;;;;AAkGA,IAAa,kBAAb,cAAqCA,sBAAAA,KAAK;CACxC,OAAO,UAAU;EACf,OAAO;CACT;CAEA,OAAO;CAEP,cACE;CAEF;CAEA,OAAgB,WAAW;CAE3B,QAAgB;CAEhB,QAAqC;CAErC,UAAqC;CAErC,IAAY;CAEZ,OAKkB;CAElB,sBAAkD;CAElD;CAEA,YAAY,QAAgC;EAE1C,IACE,QAAQ,mBAAmB,KAAA,KAC3B,CAAC,OAAO,UAAU,CAAC,CAAC,SAAS,OAAO,cAAc,GAClD;GAEA,OAAO,sBAAsB,OAAO;GACpC,OAAO,iBAAiB;EAC1B;EACA,MAAM,MAAM;EASZ,MAAM,eAAe;GACnB,QARA,QAAQ,UACR,QAAQ,iBAAA,GAAA,0BAAA,uBAAA,CACe,gBAAgB;GAOvC,cAJA,QAAQ,iBAAA,GAAA,0BAAA,uBAAA,CAAuC,qBAAqB;GAKpE,yBAAyB;GACzB,SAAS,QAAQ;EACnB;EACA,KAAK,SAAS,IAAIC,OAAAA,OAAa,YAAY;EAC3C,KAAK,QAAQ,QAAQ,SAAS,QAAQ,aAAa,KAAK;EACxD,KAAK,QAAQ,QAAQ,SAAS,KAAK;EACnC,KAAK,UAAU,QAAQ,WAAW,KAAK;EACvC,KAAK,IAAI,QAAQ,KAAK,KAAK;EAC3B,KAAK,OAAO,QAAQ,QAAQ,KAAK;EACjC,KAAK,sBACH,QAAQ,uBAAuB,KAAK;EACtC,KAAK,OAAO,QAAQ;CACtB;;;;;;;;;;;CAYA,6BACE,UAC0B;EAC1B,IAAI,KAAK,wBAAwB,OAC/B,OAAO,SAAS,SAAS,QAAQ;GAiB/B,OAfE,IAAI,MACA,SAAS,SAAS;IAClB,IAAI,CAAC,KAAK,KAAK,OAAO,CAAC;IACvB,OAAO;KACL,MAAM;KACN,WAAW,KAAK;IAClB;GACF,CAAC,CAAC,CACD,QACE,SACC,SAAS,KAAA,KACT,KAAK,SAAS,eACd,OAAO,KAAK,cAAc,YAC1B,KAAK,cAAc,KAAA,CACvB,KAAK,CAAC;EAEZ,CAAC;OAED,OAAO,SAAS,SAAS,QAAQ;GAqB/B,OAnBE,IAAI,MACA,SAAS,SAAS;IAClB,IAAI,CAAC,KAAK,UAAU,OAAO,CAAC;IAC5B,OAAO;KACL,MAAM;KACN,WAAW,EACT,KAAK,KAAK,SACZ;IACF;GACF,CAAC,CAAC,CACD,QACE,SACC,SAAS,KAAA,KACT,KAAK,SAAS,eACd,OAAO,KAAK,cAAc,YAC1B,SAAS,KAAK,aACd,OAAO,KAAK,UAAU,QAAQ,YAC9B,KAAK,UAAU,QAAQ,KAAA,CAC3B,KAAK,CAAC;EAEZ,CAAC;CAEL;;CAGA,MAAM,MAAM,OAA0D;EACpE,MAAM,sBAAsB;GAC1B,OAAO,KAAK;GACZ,QAAQ;GACR,GAAG;GACH,MAAM,KAAK;GACX,iBAAiB,KAAK;GACtB,OAAO,KAAK;GACZ,SAAS,KAAK;GACd,MAAM,KAAK;EACb;EAEA,IAAI,KAAK,IAAI,GAAG;GACd,MAAM,UAAU,MAAM,QAAQ,IAC5B,MAAM,KAAK,EAAE,QAAQ,KAAK,EAAE,CAAC,CAAC,CAAC,UAC7B,KAAK,OAAO,OAAO,SAAS,mBAAmB,CACjD,CACF;GAEA,OAAO,KAAK,6BAA6B,OAAO;EAClD;EAEA,MAAM,WAAW,MAAM,KAAK,OAAO,OAAO,SAAS,mBAAmB;EAEtE,IAAI,OAAO;EACX,IAAI,KAAK,wBAAwB,OAC/B,CAAC,QACC,SAAS,MACL,KAAK,SAAS,KAAK,GAAG,CAAC,CACxB,QAAQ,QAAuB,QAAQ,WAAW,KAAK,CAAC;OAE7D,CAAC,QACC,SAAS,MACL,KAAK,SAAS,KAAK,QAAQ,CAAC,CAC7B,QAAQ,aAAiC,aAAa,WAAW,KACpE,CAAC;EAEL,OAAO;CACT;AACF"}
|
package/dist/tools/dalle.d.cts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { OpenAI } from "openai";
|
|
2
2
|
import { MessageContentComplex } from "@langchain/core/messages";
|
|
3
3
|
import { Tool, ToolParams } from "@langchain/core/tools";
|
|
4
|
-
|
|
5
4
|
//#region src/tools/dalle.d.ts
|
|
6
5
|
/**
|
|
7
6
|
* @see https://platform.openai.com/docs/api-reference/images/create
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dalle.d.cts","names":[],"sources":["../../src/tools/dalle.ts"],"mappings":";;;;;;;
|
|
1
|
+
{"version":3,"file":"dalle.d.cts","names":[],"sources":["../../src/tools/dalle.ts"],"mappings":";;;;;;;KAWY,qBACR,OAAa,uBACH;;;;UAKG,8BAA8B;;;;;EAK7C;;;;EAIA;;;;;;;;EAQA,YAAY;;;;;;EAMZ,QAAQ;;;;;;;EAOR;;;;;;EAMA;;;;;;;EAOA;;;;;;;EAOA;;;;;;EAMA;;;;EAKA;;;;;EAKA;;;;EAIA;;;;EAIA;;;;;cAMW,wBAAwB;SAC5B;EAIP;EAEA;YAGU,QAAQ;kBAEF;UAER;UAEA;UAEA;UAEA;UAEA;UAOA;UAEA;EAER,YAAY,SAAS;;;;;;;;;;;UA8Cb;;EAmDF,MAAM,gBAAgB,iBAAiB"}
|
package/dist/tools/dalle.d.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { OpenAI } from "openai";
|
|
2
2
|
import { MessageContentComplex } from "@langchain/core/messages";
|
|
3
3
|
import { Tool, ToolParams } from "@langchain/core/tools";
|
|
4
|
-
|
|
5
4
|
//#region src/tools/dalle.d.ts
|
|
6
5
|
/**
|
|
7
6
|
* @see https://platform.openai.com/docs/api-reference/images/create
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dalle.d.ts","names":[],"sources":["../../src/tools/dalle.ts"],"mappings":";;;;;;;
|
|
1
|
+
{"version":3,"file":"dalle.d.ts","names":[],"sources":["../../src/tools/dalle.ts"],"mappings":";;;;;;;KAWY,qBACR,OAAa,uBACH;;;;UAKG,8BAA8B;;;;;EAK7C;;;;EAIA;;;;;;;;EAQA,YAAY;;;;;;EAMZ,QAAQ;;;;;;;EAOR;;;;;;EAMA;;;;;;;EAOA;;;;;;;EAOA;;;;;;EAMA;;;;EAKA;;;;;EAKA;;;;EAIA;;;;EAIA;;;;;cAMW,wBAAwB;SAC5B;EAIP;EAEA;YAGU,QAAQ;kBAEF;UAER;UAEA;UAEA;UAEA;UAEA;UAOA;UAEA;EAER,YAAY,SAAS;;;;;;;;;;;UA8Cb;;EAmDF,MAAM,gBAAgB,iBAAiB"}
|
package/dist/tools/dalle.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dalle.js","names":["OpenAIClient"],"sources":["../../src/tools/dalle.ts"],"sourcesContent":["import { getEnvironmentVariable } from \"@langchain/core/utils/env\";\nimport { OpenAI as OpenAIClient } from \"openai\";\nimport { Tool, ToolParams } from \"@langchain/core/tools\";\nimport {\n MessageContentComplex,\n MessageContentImageUrl,\n} from \"@langchain/core/messages\";\n\n/**\n * @see https://platform.openai.com/docs/api-reference/images/create\n */\nexport type OpenAIImageModelId =\n | OpenAIClient.ImageModel\n | (string & NonNullable<unknown>);\n\n/**\n * An interface for the Dall-E API Wrapper.\n */\nexport interface DallEAPIWrapperParams extends ToolParams {\n /**\n * The OpenAI API key\n * Alias for `apiKey`\n */\n openAIApiKey?: string;\n /**\n * The OpenAI API key\n */\n apiKey?: string;\n /**\n * The model to use.\n * Alias for `model`\n * @params \"dall-e-2\" | \"dall-e-3\"\n * @default \"dall-e-3\"\n * @deprecated Use `model` instead.\n */\n modelName?: OpenAIImageModelId;\n /**\n * The model to use.\n * @params \"dall-e-2\" | \"dall-e-3\"\n * @default \"dall-e-3\"\n */\n model?: OpenAIImageModelId;\n /**\n * The style of the generated images. Must be one of vivid or natural.\n * Vivid causes the model to lean towards generating hyper-real and dramatic images.\n * Natural causes the model to produce more natural, less hyper-real looking images.\n * @default \"vivid\"\n */\n style?: \"natural\" | \"vivid\";\n /**\n * The quality of the image that will be generated. ‘hd’ creates images with finer\n * details and greater consistency across the image.\n * @default \"standard\"\n */\n quality?: \"standard\" | \"hd\";\n /**\n * The number of images to generate.\n * Must be between 1 and 10.\n * For dall-e-3, only `n: 1` is supported.\n * @default 1\n */\n n?: number;\n /**\n * The size of the generated images.\n * Must be one of 256x256, 512x512, or 1024x1024 for DALL·E-2 models.\n * Must be one of 1024x1024, 1792x1024, or 1024x1792 for DALL·E-3 models.\n * @default \"1024x1024\"\n */\n size?: \"256x256\" | \"512x512\" | \"1024x1024\" | \"1792x1024\" | \"1024x1792\";\n /**\n * The format in which the generated images are returned.\n * Must be one of \"url\" or \"b64_json\".\n * @default \"url\"\n */\n dallEResponseFormat?: \"url\" | \"b64_json\";\n /**\n * @deprecated Use dallEResponseFormat instead for the Dall-E response type.\n */\n // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n responseFormat?: any;\n /**\n * A unique identifier representing your end-user, which will help\n * OpenAI to monitor and detect abuse.\n */\n user?: string;\n /**\n * The organization to use\n */\n organization?: string;\n /**\n * The base URL of the OpenAI API.\n */\n baseUrl?: string;\n}\n\n/**\n * A tool for generating images with Open AIs Dall-E 2 or 3 API.\n */\nexport class DallEAPIWrapper extends Tool {\n static lc_name() {\n return \"DallEAPIWrapper\";\n }\n\n name = \"dalle_api_wrapper\";\n\n description =\n \"A wrapper around OpenAI DALL-E API. Useful for when you need to generate images from a text description. Input should be an image description.\";\n\n protected client: OpenAIClient;\n\n static readonly toolName = \"dalle_api_wrapper\";\n\n private model = \"dall-e-3\";\n\n private style: \"natural\" | \"vivid\" = \"vivid\";\n\n private quality: \"standard\" | \"hd\" = \"standard\";\n\n private n = 1;\n\n private size:\n | \"256x256\"\n | \"512x512\"\n | \"1024x1024\"\n | \"1792x1024\"\n | \"1024x1792\" = \"1024x1024\";\n\n private dallEResponseFormat: \"url\" | \"b64_json\" = \"url\";\n\n private user?: string;\n\n constructor(fields?: DallEAPIWrapperParams) {\n // Shim for new base tool param name\n if (\n fields?.responseFormat !== undefined &&\n [\"url\", \"b64_json\"].includes(fields.responseFormat)\n ) {\n // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n fields.dallEResponseFormat = fields.responseFormat as any;\n fields.responseFormat = \"content\";\n }\n super(fields);\n const openAIApiKey =\n fields?.apiKey ??\n fields?.openAIApiKey ??\n getEnvironmentVariable(\"OPENAI_API_KEY\");\n\n const organization =\n fields?.organization ?? getEnvironmentVariable(\"OPENAI_ORGANIZATION\");\n\n const clientConfig = {\n apiKey: openAIApiKey,\n organization,\n dangerouslyAllowBrowser: true,\n baseURL: fields?.baseUrl,\n };\n this.client = new OpenAIClient(clientConfig);\n this.model = fields?.model ?? fields?.modelName ?? this.model;\n this.style = fields?.style ?? this.style;\n this.quality = fields?.quality ?? this.quality;\n this.n = fields?.n ?? this.n;\n this.size = fields?.size ?? this.size;\n this.dallEResponseFormat =\n fields?.dallEResponseFormat ?? this.dallEResponseFormat;\n this.user = fields?.user;\n }\n\n /**\n * Processes the API response if multiple images are generated.\n * Returns a list of MessageContentImageUrl objects. If the response\n * format is `url`, then the `image_url` field will contain the URL.\n * If it is `b64_json`, then the `image_url` field will contain an object\n * with a `url` field with the base64 encoded image.\n *\n * @param {OpenAIClient.Images.ImagesResponse[]} response The API response\n * @returns {MessageContentImageUrl[]}\n */\n private processMultipleGeneratedUrls(\n response: OpenAIClient.Images.ImagesResponse[]\n ): MessageContentImageUrl[] {\n if (this.dallEResponseFormat === \"url\") {\n return response.flatMap((res) => {\n const imageUrlContent =\n res.data\n ?.flatMap((item) => {\n if (!item.url) return [];\n return {\n type: \"image_url\" as const,\n image_url: item.url,\n };\n })\n .filter(\n (item) =>\n item !== undefined &&\n item.type === \"image_url\" &&\n typeof item.image_url === \"string\" &&\n item.image_url !== undefined\n ) ?? [];\n return imageUrlContent;\n });\n } else {\n return response.flatMap((res) => {\n const b64Content =\n res.data\n ?.flatMap((item) => {\n if (!item.b64_json) return [];\n return {\n type: \"image_url\" as const,\n image_url: {\n url: item.b64_json,\n },\n };\n })\n .filter(\n (item) =>\n item !== undefined &&\n item.type === \"image_url\" &&\n typeof item.image_url === \"object\" &&\n \"url\" in item.image_url &&\n typeof item.image_url.url === \"string\" &&\n item.image_url.url !== undefined\n ) ?? [];\n return b64Content;\n });\n }\n }\n\n /** @ignore */\n async _call(input: string): Promise<string | MessageContentComplex[]> {\n const generateImageFields = {\n model: this.model,\n prompt: input,\n n: 1,\n size: this.size,\n response_format: this.dallEResponseFormat,\n style: this.style,\n quality: this.quality,\n user: this.user,\n };\n\n if (this.n > 1) {\n const results = await Promise.all(\n Array.from({ length: this.n }).map(() =>\n this.client.images.generate(generateImageFields)\n )\n );\n\n return this.processMultipleGeneratedUrls(results);\n }\n\n const response = await this.client.images.generate(generateImageFields);\n\n let data = \"\";\n if (this.dallEResponseFormat === \"url\") {\n [data] =\n response.data\n ?.map((item) => item.url)\n .filter((url): url is string => url !== \"undefined\") ?? [];\n } else {\n [data] =\n response.data\n ?.map((item) => item.b64_json)\n .filter((b64_json): b64_json is string => b64_json !== \"undefined\") ??\n [];\n }\n return data;\n }\n}\n"],"mappings":";;;;;;;AAkGA,IAAa,kBAAb,cAAqC,KAAK;CACxC,OAAO,UAAU;AACf,SAAO;;CAGT,OAAO;CAEP,cACE;CAEF;CAEA,OAAgB,WAAW;CAE3B,QAAgB;CAEhB,QAAqC;CAErC,UAAqC;CAErC,IAAY;CAEZ,OAKkB;CAElB,sBAAkD;CAElD;CAEA,YAAY,QAAgC;AAE1C,MACE,QAAQ,mBAAmB,KAAA,KAC3B,CAAC,OAAO,WAAW,CAAC,SAAS,OAAO,eAAe,EACnD;AAEA,UAAO,sBAAsB,OAAO;AACpC,UAAO,iBAAiB;;AAE1B,QAAM,OAAO;EASb,MAAM,eAAe;GACnB,QARA,QAAQ,UACR,QAAQ,gBACR,uBAAuB,iBAAiB;GAOxC,cAJA,QAAQ,gBAAgB,uBAAuB,sBAAsB;GAKrE,yBAAyB;GACzB,SAAS,QAAQ;GAClB;AACD,OAAK,SAAS,IAAIA,OAAa,aAAa;AAC5C,OAAK,QAAQ,QAAQ,SAAS,QAAQ,aAAa,KAAK;AACxD,OAAK,QAAQ,QAAQ,SAAS,KAAK;AACnC,OAAK,UAAU,QAAQ,WAAW,KAAK;AACvC,OAAK,IAAI,QAAQ,KAAK,KAAK;AAC3B,OAAK,OAAO,QAAQ,QAAQ,KAAK;AACjC,OAAK,sBACH,QAAQ,uBAAuB,KAAK;AACtC,OAAK,OAAO,QAAQ;;;;;;;;;;;;CAatB,6BACE,UAC0B;AAC1B,MAAI,KAAK,wBAAwB,MAC/B,QAAO,SAAS,SAAS,QAAQ;AAiB/B,UAfE,IAAI,MACA,SAAS,SAAS;AAClB,QAAI,CAAC,KAAK,IAAK,QAAO,EAAE;AACxB,WAAO;KACL,MAAM;KACN,WAAW,KAAK;KACjB;KACD,CACD,QACE,SACC,SAAS,KAAA,KACT,KAAK,SAAS,eACd,OAAO,KAAK,cAAc,YAC1B,KAAK,cAAc,KAAA,EACtB,IAAI,EAAE;IAEX;MAEF,QAAO,SAAS,SAAS,QAAQ;AAqB/B,UAnBE,IAAI,MACA,SAAS,SAAS;AAClB,QAAI,CAAC,KAAK,SAAU,QAAO,EAAE;AAC7B,WAAO;KACL,MAAM;KACN,WAAW,EACT,KAAK,KAAK,UACX;KACF;KACD,CACD,QACE,SACC,SAAS,KAAA,KACT,KAAK,SAAS,eACd,OAAO,KAAK,cAAc,YAC1B,SAAS,KAAK,aACd,OAAO,KAAK,UAAU,QAAQ,YAC9B,KAAK,UAAU,QAAQ,KAAA,EAC1B,IAAI,EAAE;IAEX;;;CAKN,MAAM,MAAM,OAA0D;EACpE,MAAM,sBAAsB;GAC1B,OAAO,KAAK;GACZ,QAAQ;GACR,GAAG;GACH,MAAM,KAAK;GACX,iBAAiB,KAAK;GACtB,OAAO,KAAK;GACZ,SAAS,KAAK;GACd,MAAM,KAAK;GACZ;AAED,MAAI,KAAK,IAAI,GAAG;GACd,MAAM,UAAU,MAAM,QAAQ,IAC5B,MAAM,KAAK,EAAE,QAAQ,KAAK,GAAG,CAAC,CAAC,UAC7B,KAAK,OAAO,OAAO,SAAS,oBAAoB,CACjD,CACF;AAED,UAAO,KAAK,6BAA6B,QAAQ;;EAGnD,MAAM,WAAW,MAAM,KAAK,OAAO,OAAO,SAAS,oBAAoB;EAEvE,IAAI,OAAO;AACX,MAAI,KAAK,wBAAwB,MAC/B,EAAC,QACC,SAAS,MACL,KAAK,SAAS,KAAK,IAAI,CACxB,QAAQ,QAAuB,QAAQ,YAAY,IAAI,EAAE;MAE9D,EAAC,QACC,SAAS,MACL,KAAK,SAAS,KAAK,SAAS,CAC7B,QAAQ,aAAiC,aAAa,YAAY,IACrE,EAAE;AAEN,SAAO"}
|
|
1
|
+
{"version":3,"file":"dalle.js","names":["OpenAIClient"],"sources":["../../src/tools/dalle.ts"],"sourcesContent":["import { getEnvironmentVariable } from \"@langchain/core/utils/env\";\nimport { OpenAI as OpenAIClient } from \"openai\";\nimport { Tool, ToolParams } from \"@langchain/core/tools\";\nimport {\n MessageContentComplex,\n MessageContentImageUrl,\n} from \"@langchain/core/messages\";\n\n/**\n * @see https://platform.openai.com/docs/api-reference/images/create\n */\nexport type OpenAIImageModelId =\n | OpenAIClient.ImageModel\n | (string & NonNullable<unknown>);\n\n/**\n * An interface for the Dall-E API Wrapper.\n */\nexport interface DallEAPIWrapperParams extends ToolParams {\n /**\n * The OpenAI API key\n * Alias for `apiKey`\n */\n openAIApiKey?: string;\n /**\n * The OpenAI API key\n */\n apiKey?: string;\n /**\n * The model to use.\n * Alias for `model`\n * @params \"dall-e-2\" | \"dall-e-3\"\n * @default \"dall-e-3\"\n * @deprecated Use `model` instead.\n */\n modelName?: OpenAIImageModelId;\n /**\n * The model to use.\n * @params \"dall-e-2\" | \"dall-e-3\"\n * @default \"dall-e-3\"\n */\n model?: OpenAIImageModelId;\n /**\n * The style of the generated images. Must be one of vivid or natural.\n * Vivid causes the model to lean towards generating hyper-real and dramatic images.\n * Natural causes the model to produce more natural, less hyper-real looking images.\n * @default \"vivid\"\n */\n style?: \"natural\" | \"vivid\";\n /**\n * The quality of the image that will be generated. ‘hd’ creates images with finer\n * details and greater consistency across the image.\n * @default \"standard\"\n */\n quality?: \"standard\" | \"hd\";\n /**\n * The number of images to generate.\n * Must be between 1 and 10.\n * For dall-e-3, only `n: 1` is supported.\n * @default 1\n */\n n?: number;\n /**\n * The size of the generated images.\n * Must be one of 256x256, 512x512, or 1024x1024 for DALL·E-2 models.\n * Must be one of 1024x1024, 1792x1024, or 1024x1792 for DALL·E-3 models.\n * @default \"1024x1024\"\n */\n size?: \"256x256\" | \"512x512\" | \"1024x1024\" | \"1792x1024\" | \"1024x1792\";\n /**\n * The format in which the generated images are returned.\n * Must be one of \"url\" or \"b64_json\".\n * @default \"url\"\n */\n dallEResponseFormat?: \"url\" | \"b64_json\";\n /**\n * @deprecated Use dallEResponseFormat instead for the Dall-E response type.\n */\n // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n responseFormat?: any;\n /**\n * A unique identifier representing your end-user, which will help\n * OpenAI to monitor and detect abuse.\n */\n user?: string;\n /**\n * The organization to use\n */\n organization?: string;\n /**\n * The base URL of the OpenAI API.\n */\n baseUrl?: string;\n}\n\n/**\n * A tool for generating images with Open AIs Dall-E 2 or 3 API.\n */\nexport class DallEAPIWrapper extends Tool {\n static lc_name() {\n return \"DallEAPIWrapper\";\n }\n\n name = \"dalle_api_wrapper\";\n\n description =\n \"A wrapper around OpenAI DALL-E API. Useful for when you need to generate images from a text description. Input should be an image description.\";\n\n protected client: OpenAIClient;\n\n static readonly toolName = \"dalle_api_wrapper\";\n\n private model = \"dall-e-3\";\n\n private style: \"natural\" | \"vivid\" = \"vivid\";\n\n private quality: \"standard\" | \"hd\" = \"standard\";\n\n private n = 1;\n\n private size:\n | \"256x256\"\n | \"512x512\"\n | \"1024x1024\"\n | \"1792x1024\"\n | \"1024x1792\" = \"1024x1024\";\n\n private dallEResponseFormat: \"url\" | \"b64_json\" = \"url\";\n\n private user?: string;\n\n constructor(fields?: DallEAPIWrapperParams) {\n // Shim for new base tool param name\n if (\n fields?.responseFormat !== undefined &&\n [\"url\", \"b64_json\"].includes(fields.responseFormat)\n ) {\n // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n fields.dallEResponseFormat = fields.responseFormat as any;\n fields.responseFormat = \"content\";\n }\n super(fields);\n const openAIApiKey =\n fields?.apiKey ??\n fields?.openAIApiKey ??\n getEnvironmentVariable(\"OPENAI_API_KEY\");\n\n const organization =\n fields?.organization ?? getEnvironmentVariable(\"OPENAI_ORGANIZATION\");\n\n const clientConfig = {\n apiKey: openAIApiKey,\n organization,\n dangerouslyAllowBrowser: true,\n baseURL: fields?.baseUrl,\n };\n this.client = new OpenAIClient(clientConfig);\n this.model = fields?.model ?? fields?.modelName ?? this.model;\n this.style = fields?.style ?? this.style;\n this.quality = fields?.quality ?? this.quality;\n this.n = fields?.n ?? this.n;\n this.size = fields?.size ?? this.size;\n this.dallEResponseFormat =\n fields?.dallEResponseFormat ?? this.dallEResponseFormat;\n this.user = fields?.user;\n }\n\n /**\n * Processes the API response if multiple images are generated.\n * Returns a list of MessageContentImageUrl objects. If the response\n * format is `url`, then the `image_url` field will contain the URL.\n * If it is `b64_json`, then the `image_url` field will contain an object\n * with a `url` field with the base64 encoded image.\n *\n * @param {OpenAIClient.Images.ImagesResponse[]} response The API response\n * @returns {MessageContentImageUrl[]}\n */\n private processMultipleGeneratedUrls(\n response: OpenAIClient.Images.ImagesResponse[]\n ): MessageContentImageUrl[] {\n if (this.dallEResponseFormat === \"url\") {\n return response.flatMap((res) => {\n const imageUrlContent =\n res.data\n ?.flatMap((item) => {\n if (!item.url) return [];\n return {\n type: \"image_url\" as const,\n image_url: item.url,\n };\n })\n .filter(\n (item) =>\n item !== undefined &&\n item.type === \"image_url\" &&\n typeof item.image_url === \"string\" &&\n item.image_url !== undefined\n ) ?? [];\n return imageUrlContent;\n });\n } else {\n return response.flatMap((res) => {\n const b64Content =\n res.data\n ?.flatMap((item) => {\n if (!item.b64_json) return [];\n return {\n type: \"image_url\" as const,\n image_url: {\n url: item.b64_json,\n },\n };\n })\n .filter(\n (item) =>\n item !== undefined &&\n item.type === \"image_url\" &&\n typeof item.image_url === \"object\" &&\n \"url\" in item.image_url &&\n typeof item.image_url.url === \"string\" &&\n item.image_url.url !== undefined\n ) ?? [];\n return b64Content;\n });\n }\n }\n\n /** @ignore */\n async _call(input: string): Promise<string | MessageContentComplex[]> {\n const generateImageFields = {\n model: this.model,\n prompt: input,\n n: 1,\n size: this.size,\n response_format: this.dallEResponseFormat,\n style: this.style,\n quality: this.quality,\n user: this.user,\n };\n\n if (this.n > 1) {\n const results = await Promise.all(\n Array.from({ length: this.n }).map(() =>\n this.client.images.generate(generateImageFields)\n )\n );\n\n return this.processMultipleGeneratedUrls(results);\n }\n\n const response = await this.client.images.generate(generateImageFields);\n\n let data = \"\";\n if (this.dallEResponseFormat === \"url\") {\n [data] =\n response.data\n ?.map((item) => item.url)\n .filter((url): url is string => url !== \"undefined\") ?? [];\n } else {\n [data] =\n response.data\n ?.map((item) => item.b64_json)\n .filter((b64_json): b64_json is string => b64_json !== \"undefined\") ??\n [];\n }\n return data;\n }\n}\n"],"mappings":";;;;;;;AAkGA,IAAa,kBAAb,cAAqC,KAAK;CACxC,OAAO,UAAU;EACf,OAAO;CACT;CAEA,OAAO;CAEP,cACE;CAEF;CAEA,OAAgB,WAAW;CAE3B,QAAgB;CAEhB,QAAqC;CAErC,UAAqC;CAErC,IAAY;CAEZ,OAKkB;CAElB,sBAAkD;CAElD;CAEA,YAAY,QAAgC;EAE1C,IACE,QAAQ,mBAAmB,KAAA,KAC3B,CAAC,OAAO,UAAU,CAAC,CAAC,SAAS,OAAO,cAAc,GAClD;GAEA,OAAO,sBAAsB,OAAO;GACpC,OAAO,iBAAiB;EAC1B;EACA,MAAM,MAAM;EASZ,MAAM,eAAe;GACnB,QARA,QAAQ,UACR,QAAQ,gBACR,uBAAuB,gBAAgB;GAOvC,cAJA,QAAQ,gBAAgB,uBAAuB,qBAAqB;GAKpE,yBAAyB;GACzB,SAAS,QAAQ;EACnB;EACA,KAAK,SAAS,IAAIA,OAAa,YAAY;EAC3C,KAAK,QAAQ,QAAQ,SAAS,QAAQ,aAAa,KAAK;EACxD,KAAK,QAAQ,QAAQ,SAAS,KAAK;EACnC,KAAK,UAAU,QAAQ,WAAW,KAAK;EACvC,KAAK,IAAI,QAAQ,KAAK,KAAK;EAC3B,KAAK,OAAO,QAAQ,QAAQ,KAAK;EACjC,KAAK,sBACH,QAAQ,uBAAuB,KAAK;EACtC,KAAK,OAAO,QAAQ;CACtB;;;;;;;;;;;CAYA,6BACE,UAC0B;EAC1B,IAAI,KAAK,wBAAwB,OAC/B,OAAO,SAAS,SAAS,QAAQ;GAiB/B,OAfE,IAAI,MACA,SAAS,SAAS;IAClB,IAAI,CAAC,KAAK,KAAK,OAAO,CAAC;IACvB,OAAO;KACL,MAAM;KACN,WAAW,KAAK;IAClB;GACF,CAAC,CAAC,CACD,QACE,SACC,SAAS,KAAA,KACT,KAAK,SAAS,eACd,OAAO,KAAK,cAAc,YAC1B,KAAK,cAAc,KAAA,CACvB,KAAK,CAAC;EAEZ,CAAC;OAED,OAAO,SAAS,SAAS,QAAQ;GAqB/B,OAnBE,IAAI,MACA,SAAS,SAAS;IAClB,IAAI,CAAC,KAAK,UAAU,OAAO,CAAC;IAC5B,OAAO;KACL,MAAM;KACN,WAAW,EACT,KAAK,KAAK,SACZ;IACF;GACF,CAAC,CAAC,CACD,QACE,SACC,SAAS,KAAA,KACT,KAAK,SAAS,eACd,OAAO,KAAK,cAAc,YAC1B,SAAS,KAAK,aACd,OAAO,KAAK,UAAU,QAAQ,YAC9B,KAAK,UAAU,QAAQ,KAAA,CAC3B,KAAK,CAAC;EAEZ,CAAC;CAEL;;CAGA,MAAM,MAAM,OAA0D;EACpE,MAAM,sBAAsB;GAC1B,OAAO,KAAK;GACZ,QAAQ;GACR,GAAG;GACH,MAAM,KAAK;GACX,iBAAiB,KAAK;GACtB,OAAO,KAAK;GACZ,SAAS,KAAK;GACd,MAAM,KAAK;EACb;EAEA,IAAI,KAAK,IAAI,GAAG;GACd,MAAM,UAAU,MAAM,QAAQ,IAC5B,MAAM,KAAK,EAAE,QAAQ,KAAK,EAAE,CAAC,CAAC,CAAC,UAC7B,KAAK,OAAO,OAAO,SAAS,mBAAmB,CACjD,CACF;GAEA,OAAO,KAAK,6BAA6B,OAAO;EAClD;EAEA,MAAM,WAAW,MAAM,KAAK,OAAO,OAAO,SAAS,mBAAmB;EAEtE,IAAI,OAAO;EACX,IAAI,KAAK,wBAAwB,OAC/B,CAAC,QACC,SAAS,MACL,KAAK,SAAS,KAAK,GAAG,CAAC,CACxB,QAAQ,QAAuB,QAAQ,WAAW,KAAK,CAAC;OAE7D,CAAC,QACC,SAAS,MACL,KAAK,SAAS,KAAK,QAAQ,CAAC,CAC7B,QAAQ,aAAiC,aAAa,WAAW,KACpE,CAAC;EAEL,OAAO;CACT;AACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fileSearch.cjs","names":[],"sources":["../../src/tools/fileSearch.ts"],"sourcesContent":["import { OpenAI as OpenAIClient } from \"openai\";\nimport type { ServerTool } from \"@langchain/core/tools\";\n\n/**\n * Comparison operators for file attribute filtering.\n */\nexport type FileSearchComparisonType =\n | \"eq\"\n | \"ne\"\n | \"gt\"\n | \"gte\"\n | \"lt\"\n | \"lte\";\n\n/**\n * A filter used to compare a specified attribute key to a given value.\n */\nexport interface FileSearchComparisonFilter {\n /**\n * The comparison operator to use.\n * - `eq`: equals\n * - `ne`: not equal\n * - `gt`: greater than\n * - `gte`: greater than or equal\n * - `lt`: less than\n * - `lte`: less than or equal\n */\n type: FileSearchComparisonType;\n /**\n * The attribute key to compare against.\n */\n key: string;\n /**\n * The value to compare against the attribute key.\n * Supports string, number, boolean, or array of strings/numbers.\n */\n value: string | number | boolean | Array<string | number>;\n}\n\n/**\n * Combine multiple filters using `and` or `or`.\n */\nexport interface FileSearchCompoundFilter {\n /**\n * Type of operation: `and` or `or`.\n */\n type: \"and\" | \"or\";\n /**\n * Array of filters to combine.\n */\n filters: Array<FileSearchComparisonFilter | FileSearchCompoundFilter>;\n}\n\n/**\n * Filter type for file search - can be a comparison or compound filter.\n */\nexport type FileSearchFilter =\n | FileSearchComparisonFilter\n | FileSearchCompoundFilter;\n\n/**\n * Weights for hybrid search balancing semantic and keyword matches.\n */\nexport interface FileSearchHybridSearchWeights {\n /**\n * The weight of semantic embedding matches (0-1).\n */\n embeddingWeight: number;\n /**\n * The weight of keyword/text matches (0-1).\n */\n textWeight: number;\n}\n\n/**\n * Ranking options for file search results.\n */\nexport interface FileSearchRankingOptions {\n /**\n * The ranker to use for the file search.\n * - `auto`: Automatically select the best ranker\n * - `default-2024-11-15`: Default ranker as of November 2024\n */\n ranker?: \"auto\" | \"default-2024-11-15\";\n /**\n * The score threshold for results (0-1).\n * Numbers closer to 1 return only the most relevant results but may return fewer.\n */\n scoreThreshold?: number;\n /**\n * Weights that control how reciprocal rank fusion balances semantic\n * embedding matches versus sparse keyword matches when hybrid search is enabled.\n */\n hybridSearch?: FileSearchHybridSearchWeights;\n}\n\n/**\n * Options for the File Search tool.\n */\nexport interface FileSearchOptions {\n /**\n * The IDs of the vector stores to search.\n * You must have previously created vector stores and uploaded files to them.\n */\n vectorStoreIds: string[];\n /**\n * The maximum number of results to return.\n * Must be between 1 and 50 inclusive.\n */\n maxNumResults?: number;\n /**\n * A filter to apply based on file attributes/metadata.\n * Use this to narrow down search results to specific categories or file types.\n */\n filters?: FileSearchFilter;\n /**\n * Ranking options to customize how results are scored and ordered.\n */\n rankingOptions?: FileSearchRankingOptions;\n}\n\n/**\n * OpenAI File Search tool type for the Responses API.\n */\nexport type FileSearchTool = OpenAIClient.Responses.FileSearchTool;\n\n/**\n * Converts ranking options to the API format.\n */\nfunction convertRankingOptions(\n options: FileSearchRankingOptions | undefined\n): OpenAIClient.Responses.FileSearchTool.RankingOptions | undefined {\n if (!options) return undefined;\n return {\n ranker: options.ranker,\n score_threshold: options.scoreThreshold,\n hybrid_search: options.hybridSearch\n ? {\n embedding_weight: options.hybridSearch.embeddingWeight,\n text_weight: options.hybridSearch.textWeight,\n }\n : undefined,\n };\n}\n\n/**\n * Creates a File Search tool that allows models to search your files\n * for relevant information using semantic and keyword search.\n *\n * File Search enables models to retrieve information from a knowledge base\n * of previously uploaded files stored in vector stores. This is a hosted tool\n * managed by OpenAI - you don't need to implement the search execution yourself.\n *\n * **Prerequisites**: Before using File Search, you must:\n * 1. Upload files to the File API with `purpose: \"assistants\"`\n * 2. Create a vector store\n * 3. Add files to the vector store\n *\n * @see {@link https://platform.openai.com/docs/guides/tools-file-search | OpenAI File Search Documentation}\n *\n * @param options - Configuration options for the File Search tool\n * @returns A File Search tool definition to be passed to the OpenAI Responses API\n *\n * @example\n * ```typescript\n * import { ChatOpenAI, tools } from \"@langchain/openai\";\n *\n * const model = new ChatOpenAI({ model: \"gpt-4.1\" });\n *\n * // Basic usage with a vector store\n * const response = await model.invoke(\n * \"What is deep research by OpenAI?\",\n * {\n * tools: [tools.fileSearch({\n * vectorStoreIds: [\"vs_abc123\"]\n * })]\n * }\n * );\n *\n * // Limit the number of results for lower latency\n * const response = await model.invoke(\n * \"Find information about pricing\",\n * {\n * tools: [tools.fileSearch({\n * vectorStoreIds: [\"vs_abc123\"],\n * maxNumResults: 5\n * })]\n * }\n * );\n *\n * // With metadata filtering\n * const response = await model.invoke(\n * \"Find recent blog posts about AI\",\n * {\n * tools: [tools.fileSearch({\n * vectorStoreIds: [\"vs_abc123\"],\n * filters: {\n * type: \"eq\",\n * key: \"category\",\n * value: \"blog\"\n * }\n * })]\n * }\n * );\n *\n * // With compound filters (AND/OR)\n * const response = await model.invoke(\n * \"Find technical docs from 2024\",\n * {\n * tools: [tools.fileSearch({\n * vectorStoreIds: [\"vs_abc123\"],\n * filters: {\n * type: \"and\",\n * filters: [\n * { type: \"eq\", key: \"category\", value: \"technical\" },\n * { type: \"gte\", key: \"year\", value: 2024 }\n * ]\n * }\n * })]\n * }\n * );\n *\n * // With ranking options for more relevant results\n * const response = await model.invoke(\n * \"Find the most relevant information\",\n * {\n * tools: [tools.fileSearch({\n * vectorStoreIds: [\"vs_abc123\"],\n * rankingOptions: {\n * scoreThreshold: 0.8,\n * ranker: \"auto\"\n * }\n * })]\n * }\n * );\n *\n * // Search multiple vector stores\n * const response = await model.invoke(\n * \"Search across all knowledge bases\",\n * {\n * tools: [tools.fileSearch({\n * vectorStoreIds: [\"vs_abc123\", \"vs_def456\"]\n * })]\n * }\n * );\n * ```\n *\n * @remarks\n * - Vector stores must be created and populated before using this tool\n * - The tool returns file citations in the response annotations\n * - Use `include: [\"file_search_call.results\"]` in the API call to get search results\n * - Supported file types include PDF, DOCX, TXT, MD, and many code file formats\n */\nexport function fileSearch(options: FileSearchOptions): ServerTool {\n return {\n type: \"file_search\",\n vector_store_ids: options.vectorStoreIds,\n max_num_results: options.maxNumResults,\n filters: options.filters as FileSearchTool[\"filters\"],\n ranking_options: convertRankingOptions(options.rankingOptions),\n } satisfies FileSearchTool;\n}\n"],"mappings":";;;;AAiIA,SAAS,sBACP,SACkE;
|
|
1
|
+
{"version":3,"file":"fileSearch.cjs","names":[],"sources":["../../src/tools/fileSearch.ts"],"sourcesContent":["import { OpenAI as OpenAIClient } from \"openai\";\nimport type { ServerTool } from \"@langchain/core/tools\";\n\n/**\n * Comparison operators for file attribute filtering.\n */\nexport type FileSearchComparisonType =\n | \"eq\"\n | \"ne\"\n | \"gt\"\n | \"gte\"\n | \"lt\"\n | \"lte\";\n\n/**\n * A filter used to compare a specified attribute key to a given value.\n */\nexport interface FileSearchComparisonFilter {\n /**\n * The comparison operator to use.\n * - `eq`: equals\n * - `ne`: not equal\n * - `gt`: greater than\n * - `gte`: greater than or equal\n * - `lt`: less than\n * - `lte`: less than or equal\n */\n type: FileSearchComparisonType;\n /**\n * The attribute key to compare against.\n */\n key: string;\n /**\n * The value to compare against the attribute key.\n * Supports string, number, boolean, or array of strings/numbers.\n */\n value: string | number | boolean | Array<string | number>;\n}\n\n/**\n * Combine multiple filters using `and` or `or`.\n */\nexport interface FileSearchCompoundFilter {\n /**\n * Type of operation: `and` or `or`.\n */\n type: \"and\" | \"or\";\n /**\n * Array of filters to combine.\n */\n filters: Array<FileSearchComparisonFilter | FileSearchCompoundFilter>;\n}\n\n/**\n * Filter type for file search - can be a comparison or compound filter.\n */\nexport type FileSearchFilter =\n | FileSearchComparisonFilter\n | FileSearchCompoundFilter;\n\n/**\n * Weights for hybrid search balancing semantic and keyword matches.\n */\nexport interface FileSearchHybridSearchWeights {\n /**\n * The weight of semantic embedding matches (0-1).\n */\n embeddingWeight: number;\n /**\n * The weight of keyword/text matches (0-1).\n */\n textWeight: number;\n}\n\n/**\n * Ranking options for file search results.\n */\nexport interface FileSearchRankingOptions {\n /**\n * The ranker to use for the file search.\n * - `auto`: Automatically select the best ranker\n * - `default-2024-11-15`: Default ranker as of November 2024\n */\n ranker?: \"auto\" | \"default-2024-11-15\";\n /**\n * The score threshold for results (0-1).\n * Numbers closer to 1 return only the most relevant results but may return fewer.\n */\n scoreThreshold?: number;\n /**\n * Weights that control how reciprocal rank fusion balances semantic\n * embedding matches versus sparse keyword matches when hybrid search is enabled.\n */\n hybridSearch?: FileSearchHybridSearchWeights;\n}\n\n/**\n * Options for the File Search tool.\n */\nexport interface FileSearchOptions {\n /**\n * The IDs of the vector stores to search.\n * You must have previously created vector stores and uploaded files to them.\n */\n vectorStoreIds: string[];\n /**\n * The maximum number of results to return.\n * Must be between 1 and 50 inclusive.\n */\n maxNumResults?: number;\n /**\n * A filter to apply based on file attributes/metadata.\n * Use this to narrow down search results to specific categories or file types.\n */\n filters?: FileSearchFilter;\n /**\n * Ranking options to customize how results are scored and ordered.\n */\n rankingOptions?: FileSearchRankingOptions;\n}\n\n/**\n * OpenAI File Search tool type for the Responses API.\n */\nexport type FileSearchTool = OpenAIClient.Responses.FileSearchTool;\n\n/**\n * Converts ranking options to the API format.\n */\nfunction convertRankingOptions(\n options: FileSearchRankingOptions | undefined\n): OpenAIClient.Responses.FileSearchTool.RankingOptions | undefined {\n if (!options) return undefined;\n return {\n ranker: options.ranker,\n score_threshold: options.scoreThreshold,\n hybrid_search: options.hybridSearch\n ? {\n embedding_weight: options.hybridSearch.embeddingWeight,\n text_weight: options.hybridSearch.textWeight,\n }\n : undefined,\n };\n}\n\n/**\n * Creates a File Search tool that allows models to search your files\n * for relevant information using semantic and keyword search.\n *\n * File Search enables models to retrieve information from a knowledge base\n * of previously uploaded files stored in vector stores. This is a hosted tool\n * managed by OpenAI - you don't need to implement the search execution yourself.\n *\n * **Prerequisites**: Before using File Search, you must:\n * 1. Upload files to the File API with `purpose: \"assistants\"`\n * 2. Create a vector store\n * 3. Add files to the vector store\n *\n * @see {@link https://platform.openai.com/docs/guides/tools-file-search | OpenAI File Search Documentation}\n *\n * @param options - Configuration options for the File Search tool\n * @returns A File Search tool definition to be passed to the OpenAI Responses API\n *\n * @example\n * ```typescript\n * import { ChatOpenAI, tools } from \"@langchain/openai\";\n *\n * const model = new ChatOpenAI({ model: \"gpt-4.1\" });\n *\n * // Basic usage with a vector store\n * const response = await model.invoke(\n * \"What is deep research by OpenAI?\",\n * {\n * tools: [tools.fileSearch({\n * vectorStoreIds: [\"vs_abc123\"]\n * })]\n * }\n * );\n *\n * // Limit the number of results for lower latency\n * const response = await model.invoke(\n * \"Find information about pricing\",\n * {\n * tools: [tools.fileSearch({\n * vectorStoreIds: [\"vs_abc123\"],\n * maxNumResults: 5\n * })]\n * }\n * );\n *\n * // With metadata filtering\n * const response = await model.invoke(\n * \"Find recent blog posts about AI\",\n * {\n * tools: [tools.fileSearch({\n * vectorStoreIds: [\"vs_abc123\"],\n * filters: {\n * type: \"eq\",\n * key: \"category\",\n * value: \"blog\"\n * }\n * })]\n * }\n * );\n *\n * // With compound filters (AND/OR)\n * const response = await model.invoke(\n * \"Find technical docs from 2024\",\n * {\n * tools: [tools.fileSearch({\n * vectorStoreIds: [\"vs_abc123\"],\n * filters: {\n * type: \"and\",\n * filters: [\n * { type: \"eq\", key: \"category\", value: \"technical\" },\n * { type: \"gte\", key: \"year\", value: 2024 }\n * ]\n * }\n * })]\n * }\n * );\n *\n * // With ranking options for more relevant results\n * const response = await model.invoke(\n * \"Find the most relevant information\",\n * {\n * tools: [tools.fileSearch({\n * vectorStoreIds: [\"vs_abc123\"],\n * rankingOptions: {\n * scoreThreshold: 0.8,\n * ranker: \"auto\"\n * }\n * })]\n * }\n * );\n *\n * // Search multiple vector stores\n * const response = await model.invoke(\n * \"Search across all knowledge bases\",\n * {\n * tools: [tools.fileSearch({\n * vectorStoreIds: [\"vs_abc123\", \"vs_def456\"]\n * })]\n * }\n * );\n * ```\n *\n * @remarks\n * - Vector stores must be created and populated before using this tool\n * - The tool returns file citations in the response annotations\n * - Use `include: [\"file_search_call.results\"]` in the API call to get search results\n * - Supported file types include PDF, DOCX, TXT, MD, and many code file formats\n */\nexport function fileSearch(options: FileSearchOptions): ServerTool {\n return {\n type: \"file_search\",\n vector_store_ids: options.vectorStoreIds,\n max_num_results: options.maxNumResults,\n filters: options.filters as FileSearchTool[\"filters\"],\n ranking_options: convertRankingOptions(options.rankingOptions),\n } satisfies FileSearchTool;\n}\n"],"mappings":";;;;AAiIA,SAAS,sBACP,SACkE;CAClE,IAAI,CAAC,SAAS,OAAO,KAAA;CACrB,OAAO;EACL,QAAQ,QAAQ;EAChB,iBAAiB,QAAQ;EACzB,eAAe,QAAQ,eACnB;GACE,kBAAkB,QAAQ,aAAa;GACvC,aAAa,QAAQ,aAAa;EACpC,IACA,KAAA;CACN;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8GA,SAAgB,WAAW,SAAwC;CACjE,OAAO;EACL,MAAM;EACN,kBAAkB,QAAQ;EAC1B,iBAAiB,QAAQ;EACzB,SAAS,QAAQ;EACjB,iBAAiB,sBAAsB,QAAQ,cAAc;CAC/D;AACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fileSearch.d.cts","names":[],"sources":["../../src/tools/fileSearch.ts"],"mappings":";;;;;;
|
|
1
|
+
{"version":3,"file":"fileSearch.d.cts","names":[],"sources":["../../src/tools/fileSearch.ts"],"mappings":";;;;;;KAMY;;;;UAWK;;;;;;;;;;EAUf,MAAM;;;;EAIN;;;;;EAKA,mCAAmC;;;;;UAMpB;;;;EAIf;;;;EAIA,SAAS,MAAM,6BAA6B;;;;;KAMlC,mBACR,6BACA;;;;UAKa;;;;EAIf;;;;EAIA;;;;;UAMe;;;;;;EAMf;;;;;EAKA;;;;;EAKA,eAAe;;;;;UAMA;;;;;EAKf;;;;;EAKA;;;;;EAKA,UAAU;;;;EAIV,iBAAiB;;;;;KAMP,iBAAiB,OAAa,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAiIpC,WAAW,SAAS,oBAAoB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fileSearch.d.ts","names":[],"sources":["../../src/tools/fileSearch.ts"],"mappings":";;;;;;
|
|
1
|
+
{"version":3,"file":"fileSearch.d.ts","names":[],"sources":["../../src/tools/fileSearch.ts"],"mappings":";;;;;;KAMY;;;;UAWK;;;;;;;;;;EAUf,MAAM;;;;EAIN;;;;;EAKA,mCAAmC;;;;;UAMpB;;;;EAIf;;;;EAIA,SAAS,MAAM,6BAA6B;;;;;KAMlC,mBACR,6BACA;;;;UAKa;;;;EAIf;;;;EAIA;;;;;UAMe;;;;;;EAMf;;;;;EAKA;;;;;EAKA,eAAe;;;;;UAMA;;;;;EAKf;;;;;EAKA;;;;;EAKA,UAAU;;;;EAIV,iBAAiB;;;;;KAMP,iBAAiB,OAAa,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAiIpC,WAAW,SAAS,oBAAoB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fileSearch.js","names":[],"sources":["../../src/tools/fileSearch.ts"],"sourcesContent":["import { OpenAI as OpenAIClient } from \"openai\";\nimport type { ServerTool } from \"@langchain/core/tools\";\n\n/**\n * Comparison operators for file attribute filtering.\n */\nexport type FileSearchComparisonType =\n | \"eq\"\n | \"ne\"\n | \"gt\"\n | \"gte\"\n | \"lt\"\n | \"lte\";\n\n/**\n * A filter used to compare a specified attribute key to a given value.\n */\nexport interface FileSearchComparisonFilter {\n /**\n * The comparison operator to use.\n * - `eq`: equals\n * - `ne`: not equal\n * - `gt`: greater than\n * - `gte`: greater than or equal\n * - `lt`: less than\n * - `lte`: less than or equal\n */\n type: FileSearchComparisonType;\n /**\n * The attribute key to compare against.\n */\n key: string;\n /**\n * The value to compare against the attribute key.\n * Supports string, number, boolean, or array of strings/numbers.\n */\n value: string | number | boolean | Array<string | number>;\n}\n\n/**\n * Combine multiple filters using `and` or `or`.\n */\nexport interface FileSearchCompoundFilter {\n /**\n * Type of operation: `and` or `or`.\n */\n type: \"and\" | \"or\";\n /**\n * Array of filters to combine.\n */\n filters: Array<FileSearchComparisonFilter | FileSearchCompoundFilter>;\n}\n\n/**\n * Filter type for file search - can be a comparison or compound filter.\n */\nexport type FileSearchFilter =\n | FileSearchComparisonFilter\n | FileSearchCompoundFilter;\n\n/**\n * Weights for hybrid search balancing semantic and keyword matches.\n */\nexport interface FileSearchHybridSearchWeights {\n /**\n * The weight of semantic embedding matches (0-1).\n */\n embeddingWeight: number;\n /**\n * The weight of keyword/text matches (0-1).\n */\n textWeight: number;\n}\n\n/**\n * Ranking options for file search results.\n */\nexport interface FileSearchRankingOptions {\n /**\n * The ranker to use for the file search.\n * - `auto`: Automatically select the best ranker\n * - `default-2024-11-15`: Default ranker as of November 2024\n */\n ranker?: \"auto\" | \"default-2024-11-15\";\n /**\n * The score threshold for results (0-1).\n * Numbers closer to 1 return only the most relevant results but may return fewer.\n */\n scoreThreshold?: number;\n /**\n * Weights that control how reciprocal rank fusion balances semantic\n * embedding matches versus sparse keyword matches when hybrid search is enabled.\n */\n hybridSearch?: FileSearchHybridSearchWeights;\n}\n\n/**\n * Options for the File Search tool.\n */\nexport interface FileSearchOptions {\n /**\n * The IDs of the vector stores to search.\n * You must have previously created vector stores and uploaded files to them.\n */\n vectorStoreIds: string[];\n /**\n * The maximum number of results to return.\n * Must be between 1 and 50 inclusive.\n */\n maxNumResults?: number;\n /**\n * A filter to apply based on file attributes/metadata.\n * Use this to narrow down search results to specific categories or file types.\n */\n filters?: FileSearchFilter;\n /**\n * Ranking options to customize how results are scored and ordered.\n */\n rankingOptions?: FileSearchRankingOptions;\n}\n\n/**\n * OpenAI File Search tool type for the Responses API.\n */\nexport type FileSearchTool = OpenAIClient.Responses.FileSearchTool;\n\n/**\n * Converts ranking options to the API format.\n */\nfunction convertRankingOptions(\n options: FileSearchRankingOptions | undefined\n): OpenAIClient.Responses.FileSearchTool.RankingOptions | undefined {\n if (!options) return undefined;\n return {\n ranker: options.ranker,\n score_threshold: options.scoreThreshold,\n hybrid_search: options.hybridSearch\n ? {\n embedding_weight: options.hybridSearch.embeddingWeight,\n text_weight: options.hybridSearch.textWeight,\n }\n : undefined,\n };\n}\n\n/**\n * Creates a File Search tool that allows models to search your files\n * for relevant information using semantic and keyword search.\n *\n * File Search enables models to retrieve information from a knowledge base\n * of previously uploaded files stored in vector stores. This is a hosted tool\n * managed by OpenAI - you don't need to implement the search execution yourself.\n *\n * **Prerequisites**: Before using File Search, you must:\n * 1. Upload files to the File API with `purpose: \"assistants\"`\n * 2. Create a vector store\n * 3. Add files to the vector store\n *\n * @see {@link https://platform.openai.com/docs/guides/tools-file-search | OpenAI File Search Documentation}\n *\n * @param options - Configuration options for the File Search tool\n * @returns A File Search tool definition to be passed to the OpenAI Responses API\n *\n * @example\n * ```typescript\n * import { ChatOpenAI, tools } from \"@langchain/openai\";\n *\n * const model = new ChatOpenAI({ model: \"gpt-4.1\" });\n *\n * // Basic usage with a vector store\n * const response = await model.invoke(\n * \"What is deep research by OpenAI?\",\n * {\n * tools: [tools.fileSearch({\n * vectorStoreIds: [\"vs_abc123\"]\n * })]\n * }\n * );\n *\n * // Limit the number of results for lower latency\n * const response = await model.invoke(\n * \"Find information about pricing\",\n * {\n * tools: [tools.fileSearch({\n * vectorStoreIds: [\"vs_abc123\"],\n * maxNumResults: 5\n * })]\n * }\n * );\n *\n * // With metadata filtering\n * const response = await model.invoke(\n * \"Find recent blog posts about AI\",\n * {\n * tools: [tools.fileSearch({\n * vectorStoreIds: [\"vs_abc123\"],\n * filters: {\n * type: \"eq\",\n * key: \"category\",\n * value: \"blog\"\n * }\n * })]\n * }\n * );\n *\n * // With compound filters (AND/OR)\n * const response = await model.invoke(\n * \"Find technical docs from 2024\",\n * {\n * tools: [tools.fileSearch({\n * vectorStoreIds: [\"vs_abc123\"],\n * filters: {\n * type: \"and\",\n * filters: [\n * { type: \"eq\", key: \"category\", value: \"technical\" },\n * { type: \"gte\", key: \"year\", value: 2024 }\n * ]\n * }\n * })]\n * }\n * );\n *\n * // With ranking options for more relevant results\n * const response = await model.invoke(\n * \"Find the most relevant information\",\n * {\n * tools: [tools.fileSearch({\n * vectorStoreIds: [\"vs_abc123\"],\n * rankingOptions: {\n * scoreThreshold: 0.8,\n * ranker: \"auto\"\n * }\n * })]\n * }\n * );\n *\n * // Search multiple vector stores\n * const response = await model.invoke(\n * \"Search across all knowledge bases\",\n * {\n * tools: [tools.fileSearch({\n * vectorStoreIds: [\"vs_abc123\", \"vs_def456\"]\n * })]\n * }\n * );\n * ```\n *\n * @remarks\n * - Vector stores must be created and populated before using this tool\n * - The tool returns file citations in the response annotations\n * - Use `include: [\"file_search_call.results\"]` in the API call to get search results\n * - Supported file types include PDF, DOCX, TXT, MD, and many code file formats\n */\nexport function fileSearch(options: FileSearchOptions): ServerTool {\n return {\n type: \"file_search\",\n vector_store_ids: options.vectorStoreIds,\n max_num_results: options.maxNumResults,\n filters: options.filters as FileSearchTool[\"filters\"],\n ranking_options: convertRankingOptions(options.rankingOptions),\n } satisfies FileSearchTool;\n}\n"],"mappings":";;;;AAiIA,SAAS,sBACP,SACkE;
|
|
1
|
+
{"version":3,"file":"fileSearch.js","names":[],"sources":["../../src/tools/fileSearch.ts"],"sourcesContent":["import { OpenAI as OpenAIClient } from \"openai\";\nimport type { ServerTool } from \"@langchain/core/tools\";\n\n/**\n * Comparison operators for file attribute filtering.\n */\nexport type FileSearchComparisonType =\n | \"eq\"\n | \"ne\"\n | \"gt\"\n | \"gte\"\n | \"lt\"\n | \"lte\";\n\n/**\n * A filter used to compare a specified attribute key to a given value.\n */\nexport interface FileSearchComparisonFilter {\n /**\n * The comparison operator to use.\n * - `eq`: equals\n * - `ne`: not equal\n * - `gt`: greater than\n * - `gte`: greater than or equal\n * - `lt`: less than\n * - `lte`: less than or equal\n */\n type: FileSearchComparisonType;\n /**\n * The attribute key to compare against.\n */\n key: string;\n /**\n * The value to compare against the attribute key.\n * Supports string, number, boolean, or array of strings/numbers.\n */\n value: string | number | boolean | Array<string | number>;\n}\n\n/**\n * Combine multiple filters using `and` or `or`.\n */\nexport interface FileSearchCompoundFilter {\n /**\n * Type of operation: `and` or `or`.\n */\n type: \"and\" | \"or\";\n /**\n * Array of filters to combine.\n */\n filters: Array<FileSearchComparisonFilter | FileSearchCompoundFilter>;\n}\n\n/**\n * Filter type for file search - can be a comparison or compound filter.\n */\nexport type FileSearchFilter =\n | FileSearchComparisonFilter\n | FileSearchCompoundFilter;\n\n/**\n * Weights for hybrid search balancing semantic and keyword matches.\n */\nexport interface FileSearchHybridSearchWeights {\n /**\n * The weight of semantic embedding matches (0-1).\n */\n embeddingWeight: number;\n /**\n * The weight of keyword/text matches (0-1).\n */\n textWeight: number;\n}\n\n/**\n * Ranking options for file search results.\n */\nexport interface FileSearchRankingOptions {\n /**\n * The ranker to use for the file search.\n * - `auto`: Automatically select the best ranker\n * - `default-2024-11-15`: Default ranker as of November 2024\n */\n ranker?: \"auto\" | \"default-2024-11-15\";\n /**\n * The score threshold for results (0-1).\n * Numbers closer to 1 return only the most relevant results but may return fewer.\n */\n scoreThreshold?: number;\n /**\n * Weights that control how reciprocal rank fusion balances semantic\n * embedding matches versus sparse keyword matches when hybrid search is enabled.\n */\n hybridSearch?: FileSearchHybridSearchWeights;\n}\n\n/**\n * Options for the File Search tool.\n */\nexport interface FileSearchOptions {\n /**\n * The IDs of the vector stores to search.\n * You must have previously created vector stores and uploaded files to them.\n */\n vectorStoreIds: string[];\n /**\n * The maximum number of results to return.\n * Must be between 1 and 50 inclusive.\n */\n maxNumResults?: number;\n /**\n * A filter to apply based on file attributes/metadata.\n * Use this to narrow down search results to specific categories or file types.\n */\n filters?: FileSearchFilter;\n /**\n * Ranking options to customize how results are scored and ordered.\n */\n rankingOptions?: FileSearchRankingOptions;\n}\n\n/**\n * OpenAI File Search tool type for the Responses API.\n */\nexport type FileSearchTool = OpenAIClient.Responses.FileSearchTool;\n\n/**\n * Converts ranking options to the API format.\n */\nfunction convertRankingOptions(\n options: FileSearchRankingOptions | undefined\n): OpenAIClient.Responses.FileSearchTool.RankingOptions | undefined {\n if (!options) return undefined;\n return {\n ranker: options.ranker,\n score_threshold: options.scoreThreshold,\n hybrid_search: options.hybridSearch\n ? {\n embedding_weight: options.hybridSearch.embeddingWeight,\n text_weight: options.hybridSearch.textWeight,\n }\n : undefined,\n };\n}\n\n/**\n * Creates a File Search tool that allows models to search your files\n * for relevant information using semantic and keyword search.\n *\n * File Search enables models to retrieve information from a knowledge base\n * of previously uploaded files stored in vector stores. This is a hosted tool\n * managed by OpenAI - you don't need to implement the search execution yourself.\n *\n * **Prerequisites**: Before using File Search, you must:\n * 1. Upload files to the File API with `purpose: \"assistants\"`\n * 2. Create a vector store\n * 3. Add files to the vector store\n *\n * @see {@link https://platform.openai.com/docs/guides/tools-file-search | OpenAI File Search Documentation}\n *\n * @param options - Configuration options for the File Search tool\n * @returns A File Search tool definition to be passed to the OpenAI Responses API\n *\n * @example\n * ```typescript\n * import { ChatOpenAI, tools } from \"@langchain/openai\";\n *\n * const model = new ChatOpenAI({ model: \"gpt-4.1\" });\n *\n * // Basic usage with a vector store\n * const response = await model.invoke(\n * \"What is deep research by OpenAI?\",\n * {\n * tools: [tools.fileSearch({\n * vectorStoreIds: [\"vs_abc123\"]\n * })]\n * }\n * );\n *\n * // Limit the number of results for lower latency\n * const response = await model.invoke(\n * \"Find information about pricing\",\n * {\n * tools: [tools.fileSearch({\n * vectorStoreIds: [\"vs_abc123\"],\n * maxNumResults: 5\n * })]\n * }\n * );\n *\n * // With metadata filtering\n * const response = await model.invoke(\n * \"Find recent blog posts about AI\",\n * {\n * tools: [tools.fileSearch({\n * vectorStoreIds: [\"vs_abc123\"],\n * filters: {\n * type: \"eq\",\n * key: \"category\",\n * value: \"blog\"\n * }\n * })]\n * }\n * );\n *\n * // With compound filters (AND/OR)\n * const response = await model.invoke(\n * \"Find technical docs from 2024\",\n * {\n * tools: [tools.fileSearch({\n * vectorStoreIds: [\"vs_abc123\"],\n * filters: {\n * type: \"and\",\n * filters: [\n * { type: \"eq\", key: \"category\", value: \"technical\" },\n * { type: \"gte\", key: \"year\", value: 2024 }\n * ]\n * }\n * })]\n * }\n * );\n *\n * // With ranking options for more relevant results\n * const response = await model.invoke(\n * \"Find the most relevant information\",\n * {\n * tools: [tools.fileSearch({\n * vectorStoreIds: [\"vs_abc123\"],\n * rankingOptions: {\n * scoreThreshold: 0.8,\n * ranker: \"auto\"\n * }\n * })]\n * }\n * );\n *\n * // Search multiple vector stores\n * const response = await model.invoke(\n * \"Search across all knowledge bases\",\n * {\n * tools: [tools.fileSearch({\n * vectorStoreIds: [\"vs_abc123\", \"vs_def456\"]\n * })]\n * }\n * );\n * ```\n *\n * @remarks\n * - Vector stores must be created and populated before using this tool\n * - The tool returns file citations in the response annotations\n * - Use `include: [\"file_search_call.results\"]` in the API call to get search results\n * - Supported file types include PDF, DOCX, TXT, MD, and many code file formats\n */\nexport function fileSearch(options: FileSearchOptions): ServerTool {\n return {\n type: \"file_search\",\n vector_store_ids: options.vectorStoreIds,\n max_num_results: options.maxNumResults,\n filters: options.filters as FileSearchTool[\"filters\"],\n ranking_options: convertRankingOptions(options.rankingOptions),\n } satisfies FileSearchTool;\n}\n"],"mappings":";;;;AAiIA,SAAS,sBACP,SACkE;CAClE,IAAI,CAAC,SAAS,OAAO,KAAA;CACrB,OAAO;EACL,QAAQ,QAAQ;EAChB,iBAAiB,QAAQ;EACzB,eAAe,QAAQ,eACnB;GACE,kBAAkB,QAAQ,aAAa;GACvC,aAAa,QAAQ,aAAa;EACpC,IACA,KAAA;CACN;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8GA,SAAgB,WAAW,SAAwC;CACjE,OAAO;EACL,MAAM;EACN,kBAAkB,QAAQ;EAC1B,iBAAiB,QAAQ;EACzB,SAAS,QAAQ;EACjB,iBAAiB,sBAAsB,QAAQ,cAAc;CAC/D;AACF"}
|