@browserbasehq/orca 3.0.9-alpha-4 → 3.1.0-patch.1

package/dist/esm/lib/v3/agent/utils/googleCustomToolHandler.js

@@ -0,0 +1,145 @@
+import { Type } from "@google/genai";
+import { toJsonSchema } from "../../zodCompat.js";
+/**
+ * Execute a custom tool and format the response for Google's API
+ * This handles tool execution, result formatting, and error handling
+ * specific to Google's function response format
+ */
+export async function executeGoogleCustomTool(toolName, toolArgs, tools, functionCall, logger) {
+    try {
+        logger({
+            category: "agent",
+            message: `Executing custom tool: ${toolName} with args: ${JSON.stringify(toolArgs)}`,
+            level: 1,
+        });
+        const tool = tools[toolName];
+        const toolResult = await tool.execute(toolArgs, {
+            toolCallId: `tool_${Date.now()}`,
+            messages: [],
+        });
+        logger({
+            category: "agent",
+            message: `Tool ${toolName} completed successfully. Result: ${JSON.stringify(toolResult)}`,
+            level: 1,
+        });
+        // Create function response with the result
+        const functionResponsePart = {
+            functionResponse: {
+                name: toolName,
+                response: {
+                    result: JSON.stringify(toolResult),
+                },
+            },
+        };
+        return {
+            functionResponse: functionResponsePart,
+            success: true,
+        };
+    }
+    catch (toolError) {
+        const errorMessage = toolError instanceof Error ? toolError.message : String(toolError);
+        logger({
+            category: "agent",
+            message: `Error executing custom tool ${toolName}: ${errorMessage}`,
+            level: 0,
+        });
+        // Create error function response
+        const functionResponsePart = {
+            functionResponse: {
+                name: toolName,
+                response: {
+                    error: errorMessage,
+                },
+            },
+        };
+        return {
+            functionResponse: functionResponsePart,
+            success: false,
+        };
+    }
+}
+/**
+ * Check if a function call is a custom tool
+ */
+export function isCustomTool(functionCall, tools) {
+    return !!(tools && functionCall.name && functionCall.name in tools);
+}
+/**
+ * Convert ToolSet to Google's FunctionDeclaration array
+ * Handles the conversion of Zod schemas to Google's parameter format
+ */
+export function convertToolSetToFunctionDeclarations(tools) {
+    const functionDeclarations = [];
+    for (const [name, tool] of Object.entries(tools)) {
+        const functionDeclaration = convertToolToFunctionDeclaration(name, tool);
+        if (functionDeclaration) {
+            functionDeclarations.push(functionDeclaration);
+        }
+    }
+    return functionDeclarations;
+}
+/**
+ * Convert a single ToolSet tool to Google's FunctionDeclaration format
+ */
+function convertToolToFunctionDeclaration(name, tool) {
+    try {
+        // Convert Zod schema to JSON schema
+        const schema = tool.inputSchema;
+        const jsonSchema = toJsonSchema(schema);
+        const parameters = convertJsonSchemaToGoogleParameters(jsonSchema);
+        return {
+            name,
+            description: tool.description || `Execute ${name}`,
+            parameters,
+        };
+    }
+    catch (error) {
+        console.error(`Error converting tool ${name} to function declaration:`, error);
+        return null;
+    }
+}
+/**
+ * Convert JSON schema to Google's parameter format
+ */
+function convertJsonSchemaToGoogleParameters(schema) {
+    const properties = {};
+    if (schema.properties) {
+        for (const [key, value] of Object.entries(schema.properties)) {
+            const propSchema = value;
+            properties[key] = {
+                type: mapJsonTypeToGoogleType(propSchema.type || "string"),
+                ...(propSchema.description
+                    ? { description: propSchema.description }
+                    : {}),
+            };
+        }
+    }
+    return {
+        type: Type.OBJECT,
+        properties,
+        ...(schema.required && schema.required.length > 0
+            ? { required: schema.required }
+            : {}),
+    };
+}
+/**
+ * Map JSON schema types to Google's Type enum
+ */
+function mapJsonTypeToGoogleType(jsonType) {
+    switch (jsonType.toLowerCase()) {
+        case "string":
+            return Type.STRING;
+        case "number":
+        case "integer":
+            return Type.NUMBER;
+        case "boolean":
+            return Type.BOOLEAN;
+        case "array":
+            return Type.ARRAY;
+        case "object":
+            return Type.OBJECT;
+        default:
+            return Type.STRING;
+    }
+}
+//# sourceMappingURL=googleCustomToolHandler.js.map

package/dist/esm/lib/v3/agent/utils/googleCustomToolHandler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"googleCustomToolHandler.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/googleCustomToolHandler.ts"],"names":[],"mappings":"AAAA,OAAO,EAA2C,IAAI,EAAE,MAAM,eAAe,CAAC;AAG9E,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAWlD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,QAAgB,EAChB,QAAiC,EACjC,KAAc,EACd,YAA0B,EAC1B,MAAkC;IAElC,IAAI,CAAC;QACH,MAAM,CAAC;YACL,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,0BAA0B,QAAQ,eAAe,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE;YACpF,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC7B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE;YAC9C,UAAU,EAAE,QAAQ,IAAI,CAAC,GAAG,EAAE,EAAE;YAChC,QAAQ,EAAE,EAAE;SACb,CAAC,CAAC;QAEH,MAAM,CAAC;YACL,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,QAAQ,QAAQ,oCAAoC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE;YACzF,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QAEH,2CAA2C;QAC3C,MAAM,oBAAoB,GAAS;YACjC,gBAAgB,EAAE;gBAChB,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE;oBACR,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC;iBACnC;aACF;SACF,CAAC;QAEF,OAAO;YACL,gBAAgB,EAAE,oBAAoB;YACtC,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IAAC,OAAO,SAAS,EAAE,CAAC;QACnB,MAAM,YAAY,GAChB,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAErE,MAAM,CAAC;YACL,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,+BAA+B,QAAQ,KAAK,YAAY,EAAE;YACnE,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QAEH,iCAAiC;QACjC,MAAM,oBAAoB,GAAS;YACjC,gBAAgB,EAAE;gBAChB,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE;oBACR,KAAK,EAAE,YAAY;iBACpB;aACF;SACF,CAAC;QAEF,OAAO;YACL,gBAAgB,EAAE,oBAAoB;YACtC,OAAO,EAAE,KAAK;SACf,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAC1B,YAA0B,EAC1B,KAAe;IAEf,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,YAAY,CAAC,IAAI,IAAI,YAAY,CAAC,IAAI,IAAI,KAAK,CAAC,CAAC;AACtE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oCAAoC,CAClD,KAAc;IAEd,MAAM,oBAAoB,GAA0B,EAAE,CAAC;IAEvD,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACjD,MAAM,mBAAmB,GAAG,gCAAgC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACzE,IAAI,mBAAmB,EAAE,CAAC;YACxB,oBAAoB,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IAED,OAAO,oBAAoB,CAAC;AAC9B,CAAC;AAED;;GAEG;AACH,SAAS,gCAAgC,CACvC,IAAY,EACZ,IAAoD;IAEpD,IAAI,CAAC;QACH,oCAAoC;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,WAAiC,CAAC;QACtD,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,CAIrC,CAAC;QAEF,MAAM,UAAU,GAAG,mCAAmC,CAAC,UAAU,CAAC,CAAC;QAEnE,OAAO;YACL,IAAI;YACJ,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,WAAW,IAAI,EAAE;YAClD,UAAU;SACX,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CACX,yBAAyB,IAAI,2BAA2B,EACxD,KAAK,CACN,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,mCAAmC,CAAC,MAI5C;IAKC,MAAM,UAAU,GAAyD,EAAE,CAAC;IAE5E,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACtB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7D,MAAM,UAAU,GAAG,KAIlB,CAAC;YACF,UAAU,CAAC,GAAG,CAAC,GAAG;gBAChB,IAAI,EAAE,uBAAuB,CAAC,UAAU,CAAC,IAAI,IAAI,QAAQ,CAAC;gBAC1D,GAAG,CAAC,UAAU,CAAC,WAAW;oBACxB,CAAC,CAAC,EAAE,WAAW,EAAE,UAAU,CAAC,WAAW,EAAE;oBACzC,CAAC,CAAC,EAAE,CAAC;aACR,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO;QACL,IAAI,EAAE,IAAI,CAAC,MAAM;QACjB,UAAU;QACV,GAAG,CAAC,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC;YAC/C,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE;YAC/B,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,uBAAuB,CAAC,QAAgB;IAC/C,QAAQ,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC;QAC/B,KAAK,QAAQ;YACX,OAAO,IAAI,CAAC,MAAM,CAAC;QACrB,KAAK,QAAQ,CAAC;QACd,KAAK,SAAS;YACZ,OAAO,IAAI,CAAC,MAAM,CAAC;QACrB,KAAK,SAAS;YACZ,OAAO,IAAI,CAAC,OAAO,CAAC;QACtB,KAAK,OAAO;YACV,OAAO,IAAI,CAAC,KAAK,CAAC;QACpB,KAAK,QAAQ;YACX,OAAO,IAAI,CAAC,MAAM,CAAC;QACrB;YACE,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;AACH,CAAC","sourcesContent":["import { Part, FunctionCall, FunctionDeclaration, Type } from \"@google/genai\";\nimport { ToolSet } from \"ai\";\nimport { LogLine } from \"../../types/public/logs.js\";\nimport { toJsonSchema } from \"../../zodCompat.js\";\nimport type { StagehandZodSchema } from \"../../zodCompat.js\";\n\n/**\n * Result of executing a custom tool for Google CUA\n */\nexport interface CustomToolExecutionResult {\n  functionResponse: Part;\n  success: boolean;\n}\n\n/**\n * Execute a custom tool and format the response for Google's API\n * This handles tool execution, result formatting, and error handling\n * specific to Google's function response format\n */\nexport async function executeGoogleCustomTool(\n  toolName: string,\n  toolArgs: Record<string, unknown>,\n  tools: ToolSet,\n  functionCall: FunctionCall,\n  logger: (message: LogLine) => void,\n): Promise<CustomToolExecutionResult> {\n  try {\n    logger({\n      category: \"agent\",\n      message: `Executing custom tool: ${toolName} with args: ${JSON.stringify(toolArgs)}`,\n      level: 1,\n    });\n\n    const tool = tools[toolName];\n    const toolResult = await tool.execute(toolArgs, {\n      toolCallId: `tool_${Date.now()}`,\n      messages: [],\n    });\n\n    logger({\n      category: \"agent\",\n      message: `Tool ${toolName} completed successfully. Result: ${JSON.stringify(toolResult)}`,\n      level: 1,\n    });\n\n    // Create function response with the result\n    const functionResponsePart: Part = {\n      functionResponse: {\n        name: toolName,\n        response: {\n          result: JSON.stringify(toolResult),\n        },\n      },\n    };\n\n    return {\n      functionResponse: functionResponsePart,\n      success: true,\n    };\n  } catch (toolError) {\n    const errorMessage =\n      toolError instanceof Error ? toolError.message : String(toolError);\n\n    logger({\n      category: \"agent\",\n      message: `Error executing custom tool ${toolName}: ${errorMessage}`,\n      level: 0,\n    });\n\n    // Create error function response\n    const functionResponsePart: Part = {\n      functionResponse: {\n        name: toolName,\n        response: {\n          error: errorMessage,\n        },\n      },\n    };\n\n    return {\n      functionResponse: functionResponsePart,\n      success: false,\n    };\n  }\n}\n\n/**\n * Check if a function call is a custom tool\n */\nexport function isCustomTool(\n  functionCall: FunctionCall,\n  tools?: ToolSet,\n): boolean {\n  return !!(tools && functionCall.name && functionCall.name in tools);\n}\n\n/**\n * Convert ToolSet to Google's FunctionDeclaration array\n * Handles the conversion of Zod schemas to Google's parameter format\n */\nexport function convertToolSetToFunctionDeclarations(\n  tools: ToolSet,\n): FunctionDeclaration[] {\n  const functionDeclarations: FunctionDeclaration[] = [];\n\n  for (const [name, tool] of Object.entries(tools)) {\n    const functionDeclaration = convertToolToFunctionDeclaration(name, tool);\n    if (functionDeclaration) {\n      functionDeclarations.push(functionDeclaration);\n    }\n  }\n\n  return functionDeclarations;\n}\n\n/**\n * Convert a single ToolSet tool to Google's FunctionDeclaration format\n */\nfunction convertToolToFunctionDeclaration(\n  name: string,\n  tool: { description?: string; inputSchema: unknown },\n): FunctionDeclaration | null {\n  try {\n    // Convert Zod schema to JSON schema\n    const schema = tool.inputSchema as StagehandZodSchema;\n    const jsonSchema = toJsonSchema(schema) as {\n      properties?: Record<string, unknown>;\n      required?: string[];\n      type?: string;\n    };\n\n    const parameters = convertJsonSchemaToGoogleParameters(jsonSchema);\n\n    return {\n      name,\n      description: tool.description || `Execute ${name}`,\n      parameters,\n    };\n  } catch (error) {\n    console.error(\n      `Error converting tool ${name} to function declaration:`,\n      error,\n    );\n    return null;\n  }\n}\n\n/**\n * Convert JSON schema to Google's parameter format\n */\nfunction convertJsonSchemaToGoogleParameters(schema: {\n  properties?: Record<string, unknown>;\n  required?: string[];\n  type?: string;\n}): {\n  type: Type;\n  properties: Record<string, { type: Type; description?: string }>;\n  required?: string[];\n} {\n  const properties: Record<string, { type: Type; description?: string }> = {};\n\n  if (schema.properties) {\n    for (const [key, value] of Object.entries(schema.properties)) {\n      const propSchema = value as {\n        type?: string;\n        description?: string;\n        items?: { type?: string };\n      };\n      properties[key] = {\n        type: mapJsonTypeToGoogleType(propSchema.type || \"string\"),\n        ...(propSchema.description\n          ? { description: propSchema.description }\n          : {}),\n      };\n    }\n  }\n\n  return {\n    type: Type.OBJECT,\n    properties,\n    ...(schema.required && schema.required.length > 0\n      ? { required: schema.required }\n      : {}),\n  };\n}\n\n/**\n * Map JSON schema types to Google's Type enum\n */\nfunction mapJsonTypeToGoogleType(jsonType: string): Type {\n  switch (jsonType.toLowerCase()) {\n    case \"string\":\n      return Type.STRING;\n    case \"number\":\n    case \"integer\":\n      return Type.NUMBER;\n    case \"boolean\":\n      return Type.BOOLEAN;\n    case \"array\":\n      return Type.ARRAY;\n    case \"object\":\n      return Type.OBJECT;\n    default:\n      return Type.STRING;\n  }\n}\n"]}

package/dist/esm/lib/v3/agent/utils/handleDoneToolCall.d.ts

@@ -0,0 +1,22 @@
+import { ModelMessage, LanguageModel } from "ai";
+import { LogLine } from "../../types/public/logs.js";
+import { StagehandZodObject } from "../../zodCompat.js";
+interface DoneResult {
+    reasoning: string;
+    taskComplete: boolean;
+    messages: ModelMessage[];
+    output?: Record<string, unknown>;
+}
+/**
+ * Force a done tool call at the end of an agent run.
+ * This ensures we always get a structured final response,
+ * even if the main loop ended without calling done.
+ */
+export declare function handleDoneToolCall(options: {
+    model: LanguageModel;
+    inputMessages: ModelMessage[];
+    instruction: string;
+    outputSchema?: StagehandZodObject;
+    logger: (message: LogLine) => void;
+}): Promise<DoneResult>;
+export {};

package/dist/esm/lib/v3/agent/utils/handleDoneToolCall.js

@@ -0,0 +1,101 @@
+import { generateText } from "ai";
+import { z } from "zod";
+import { tool } from "ai";
+import { getZFactory } from "../../../utils.js";
+function buildBaseDoneSchema(factory) {
+    return factory.object({
+        reasoning: factory
+            .string()
+            .describe("Brief summary of what actions were taken and the outcome"),
+        taskComplete: factory
+            .boolean()
+            .describe("true if the task was fully completed, false otherwise"),
+    });
+}
+/**
+ * Force a done tool call at the end of an agent run.
+ * This ensures we always get a structured final response,
+ * even if the main loop ended without calling done.
+ */
+export async function handleDoneToolCall(options) {
+    const { model, inputMessages, instruction, outputSchema, logger } = options;
+    logger({
+        category: "agent",
+        message: "Agent calling tool: done",
+        level: 1,
+    });
+    // Use the same Zod version as the user's outputSchema to avoid v3/v4 mixing
+    const factory = outputSchema
+        ? getZFactory(outputSchema)
+        : z;
+    const baseDoneSchema = buildBaseDoneSchema(factory);
+    // Merge base done schema with user-provided output schema if present
+    const doneToolSchema = outputSchema
+        ? baseDoneSchema.extend({
+            output: outputSchema.describe("The specific data the user requested from this task"),
+        })
+        : baseDoneSchema;
+    const outputInstructions = outputSchema
+        ? `\n\nThe user also requested the following information from this task. Provide it in the "output" field:\n${JSON.stringify(Object.fromEntries(Object.entries(outputSchema.shape).map(([key, value]) => [
+            key,
+            value.description || "no description",
+        ])), null, 2)}`
+        : "";
+    const systemPrompt = `You are a web automation assistant that was tasked with completing a task.
+
+The task was:
+"${instruction}"
+
+Review what was accomplished and provide your final assessment in whether the task was completed successfully. you have been provided with the history of the actions taken so far, use this to determine if the task was completed successfully.${outputInstructions}
+
+Call the "done" tool with:
+1. A brief summary of what was done
+2. Whether the task was completed successfully${outputSchema ? "\n3. The requested output data based on what you found" : ""}`;
+    const doneTool = tool({
+        description: outputSchema
+            ? "Complete the task with your assessment and the requested output data."
+            : "Complete the task with your final assessment.",
+        inputSchema: doneToolSchema,
+        execute: async (params) => {
+            return { success: true, ...params };
+        },
+    });
+    const userPrompt = {
+        role: "user",
+        content: outputSchema
+            ? "Provide your final assessment and the requested output data."
+            : "Provide your final assessment.",
+    };
+    const result = await generateText({
+        model,
+        system: systemPrompt,
+        messages: [...inputMessages, userPrompt],
+        tools: { done: doneTool },
+        toolChoice: { type: "tool", toolName: "done" },
+    });
+    const doneToolCall = result.toolCalls.find((tc) => tc.toolName === "done");
+    const outputMessages = [
+        userPrompt,
+        ...(result.response?.messages || []),
+    ];
+    if (!doneToolCall) {
+        return {
+            reasoning: result.text || "Task execution completed",
+            taskComplete: false,
+            messages: outputMessages,
+        };
+    }
+    const input = doneToolCall.input;
+    logger({
+        category: "agent",
+        message: `Task completed`,
+        level: 1,
+    });
+    return {
+        reasoning: input.reasoning,
+        taskComplete: input.taskComplete,
+        messages: outputMessages,
+        output: input.output,
+    };
+}
+//# sourceMappingURL=handleDoneToolCall.js.map

package/dist/esm/lib/v3/agent/utils/handleDoneToolCall.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"handleDoneToolCall.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/handleDoneToolCall.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAwC,MAAM,IAAI,CAAC;AACxE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAG1B,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAUhD,SAAS,mBAAmB,CAAC,OAAiB;IAC5C,OAAO,OAAO,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,OAAO;aACf,MAAM,EAAE;aACR,QAAQ,CAAC,0DAA0D,CAAC;QACvE,YAAY,EAAE,OAAO;aAClB,OAAO,EAAE;aACT,QAAQ,CAAC,uDAAuD,CAAC;KACrE,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAMxC;IACC,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAE5E,MAAM,CAAC;QACL,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,0BAA0B;QACnC,KAAK,EAAE,CAAC;KACT,CAAC,CAAC;IACH,4EAA4E;IAC5E,MAAM,OAAO,GAAG,YAAY;QAC1B,CAAC,CAAC,WAAW,CAAC,YAAkC,CAAC;QACjD,CAAC,CAAC,CAAC,CAAC;IACN,MAAM,cAAc,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAEpD,qEAAqE;IACrE,MAAM,cAAc,GAAG,YAAY;QACjC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,YAAY,CAAC,QAAQ,CAC3B,qDAAqD,CACtD;SACF,CAAC;QACJ,CAAC,CAAC,cAAc,CAAC;IAEnB,MAAM,kBAAkB,GAAG,YAAY;QACrC,CAAC,CAAC,4GAA4G,IAAI,CAAC,SAAS,CACxH,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,GAAG,CACpC,CAAC,CAAC,GAAG,EAAE,KAAK,CAA+B,EAAE,EAAE,CAAC;YAC9C,GAAG;YACH,KAAK,CAAC,WAAW,IAAI,gBAAgB;SACtC,CACF,CACF,EACD,IAAI,EACJ,CAAC,CACF,EAAE;QACL,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,YAAY,GAAG;;;GAGpB,WAAW;;mPAEqO,kBAAkB;;;;gDAIrN,YAAY,CAAC,CAAC,CAAC,wDAAwD,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IAE7H,MAAM,QAAQ,GAAG,IAAI,CAAC;QACpB,WAAW,EAAE,YAAY;YACvB,CAAC,CAAC,uEAAuE;YACzE,CAAC,CAAC,+CAA+C;QACnD,WAAW,EAAE,cAAc;QAC3B,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YACxB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC;QACtC,CAAC;KACF,CAAC,CAAC;IAEH,MAAM,UAAU,GAAiB;QAC/B,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,YAAY;YACnB,CAAC,CAAC,8DAA8D;YAChE,CAAC,CAAC,gCAAgC;KACrC,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;QAChC,KAAK;QACL,MAAM,EAAE,YAAY;QACpB,QAAQ,EAAE,CAAC,GAAG,aAAa,EAAE,UAAU,CAAC;QACxC,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAa;QACpC,UAAU,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE;KAC/C,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC;IAC3E,MAAM,cAAc,GAAmB;QACrC,UAAU;QACV,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,IAAI,EAAE,CAAC;KACrC,CAAC;IAEF,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,OAAO;YACL,SAAS,EAAE,MAAM,CAAC,IAAI,IAAI,0BAA0B;YACpD,YAAY,EAAE,KAAK;YACnB,QAAQ,EAAE,cAAc;SACzB,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,YAAY,CAAC,KAI1B,CAAC;IACF,MAAM,CAAC;QACL,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,gBAAgB;QACzB,KAAK,EAAE,CAAC;KACT,CAAC,CAAC;IAEH,OAAO;QACL,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,QAAQ,EAAE,cAAc;QACxB,MAAM,EAAE,KAAK,CAAC,MAAM;KACrB,CAAC;AACJ,CAAC","sourcesContent":["import { generateText, ModelMessage, LanguageModel, ToolSet } from \"ai\";\nimport { z } from \"zod\";\nimport { tool } from \"ai\";\nimport { LogLine } from \"../../types/public/logs.js\";\nimport { StagehandZodObject } from \"../../zodCompat.js\";\nimport { getZFactory } from \"../../../utils.js\";\nimport type { StagehandZodSchema } from \"../../zodCompat.js\";\n\ninterface DoneResult {\n  reasoning: string;\n  taskComplete: boolean;\n  messages: ModelMessage[];\n  output?: Record<string, unknown>;\n}\n\nfunction buildBaseDoneSchema(factory: typeof z) {\n  return factory.object({\n    reasoning: factory\n      .string()\n      .describe(\"Brief summary of what actions were taken and the outcome\"),\n    taskComplete: factory\n      .boolean()\n      .describe(\"true if the task was fully completed, false otherwise\"),\n  });\n}\n\n/**\n * Force a done tool call at the end of an agent run.\n * This ensures we always get a structured final response,\n * even if the main loop ended without calling done.\n */\nexport async function handleDoneToolCall(options: {\n  model: LanguageModel;\n  inputMessages: ModelMessage[];\n  instruction: string;\n  outputSchema?: StagehandZodObject;\n  logger: (message: LogLine) => void;\n}): Promise<DoneResult> {\n  const { model, inputMessages, instruction, outputSchema, logger } = options;\n\n  logger({\n    category: \"agent\",\n    message: \"Agent calling tool: done\",\n    level: 1,\n  });\n  // Use the same Zod version as the user's outputSchema to avoid v3/v4 mixing\n  const factory = outputSchema\n    ? getZFactory(outputSchema as StagehandZodSchema)\n    : z;\n  const baseDoneSchema = buildBaseDoneSchema(factory);\n\n  // Merge base done schema with user-provided output schema if present\n  const doneToolSchema = outputSchema\n    ? baseDoneSchema.extend({\n        output: outputSchema.describe(\n          \"The specific data the user requested from this task\",\n        ),\n      })\n    : baseDoneSchema;\n\n  const outputInstructions = outputSchema\n    ? `\\n\\nThe user also requested the following information from this task. Provide it in the \"output\" field:\\n${JSON.stringify(\n        Object.fromEntries(\n          Object.entries(outputSchema.shape).map(\n            ([key, value]: [string, StagehandZodSchema]) => [\n              key,\n              value.description || \"no description\",\n            ],\n          ),\n        ),\n        null,\n        2,\n      )}`\n    : \"\";\n\n  const systemPrompt = `You are a web automation assistant that was tasked with completing a task.\n\nThe task was:\n\"${instruction}\"\n\nReview what was accomplished and provide your final assessment in whether the task was completed successfully. you have been provided with the history of the actions taken so far, use this to determine if the task was completed successfully.${outputInstructions}\n\nCall the \"done\" tool with:\n1. A brief summary of what was done\n2. Whether the task was completed successfully${outputSchema ? \"\\n3. The requested output data based on what you found\" : \"\"}`;\n\n  const doneTool = tool({\n    description: outputSchema\n      ? \"Complete the task with your assessment and the requested output data.\"\n      : \"Complete the task with your final assessment.\",\n    inputSchema: doneToolSchema,\n    execute: async (params) => {\n      return { success: true, ...params };\n    },\n  });\n\n  const userPrompt: ModelMessage = {\n    role: \"user\",\n    content: outputSchema\n      ? \"Provide your final assessment and the requested output data.\"\n      : \"Provide your final assessment.\",\n  };\n\n  const result = await generateText({\n    model,\n    system: systemPrompt,\n    messages: [...inputMessages, userPrompt],\n    tools: { done: doneTool } as ToolSet,\n    toolChoice: { type: \"tool\", toolName: \"done\" },\n  });\n\n  const doneToolCall = result.toolCalls.find((tc) => tc.toolName === \"done\");\n  const outputMessages: ModelMessage[] = [\n    userPrompt,\n    ...(result.response?.messages || []),\n  ];\n\n  if (!doneToolCall) {\n    return {\n      reasoning: result.text || \"Task execution completed\",\n      taskComplete: false,\n      messages: outputMessages,\n    };\n  }\n\n  const input = doneToolCall.input as {\n    reasoning: string;\n    taskComplete: boolean;\n    output?: Record<string, unknown>;\n  };\n  logger({\n    category: \"agent\",\n    message: `Task completed`,\n    level: 1,\n  });\n\n  return {\n    reasoning: input.reasoning,\n    taskComplete: input.taskComplete,\n    messages: outputMessages,\n    output: input.output,\n  };\n}\n"]}

package/dist/esm/lib/v3/agent/utils/imageCompression.d.ts

@@ -0,0 +1,53 @@
+import { AnthropicMessage, AnthropicToolResult, ResponseInputItem as OpenAIResponseInputItem } from "../../types/public/agent.js";
+import type { Content as GoogleContent } from "@google/genai";
+export type ResponseInputItem = AnthropicMessage | AnthropicToolResult;
+export type AnthropicResponseInputItem = AnthropicMessage | AnthropicToolResult;
+export type SupportedInputItem = AnthropicResponseInputItem | OpenAIResponseInputItem | GoogleContent;
+/**
+ * Finds all items in the conversation history that contain images
+ * @param items - Array of conversation items to check
+ * @returns Array of indices where images were found
+ */
+export declare function findItemsWithImages(items: ResponseInputItem[]): number[];
+/**
+ * Compresses conversation history by removing images from older items
+ * while keeping the most recent images intact
+ * @param items - Array of conversation items to process
+ * @param keepMostRecentCount - Number of most recent image-containing items to preserve (default: 2)
+ * @returns Object with processed items
+ */
+export declare function compressConversationImages(items: ResponseInputItem[], keepMostRecentCount?: number): {
+    items: ResponseInputItem[];
+};
+/**
+ * Finds all items in the conversation history that contain images (Google format)
+ * @param items - Array of conversation items to check
+ * @returns Array of indices where images were found
+ */
+export declare function findGoogleItemsWithImages(items: GoogleContent[]): number[];
+/**
+ * Finds all items in the conversation history that contain images (OpenAI format)
+ * @param items - Array of conversation items to check
+ * @returns Array of indices where images were found
+ */
+export declare function findOpenAIItemsWithImages(items: OpenAIResponseInputItem[]): number[];
+/**
+ * Compresses OpenAI conversation history by removing images from older items
+ * while keeping the most recent images intact
+ * @param items - Array of conversation items to process
+ * @param keepMostRecentCount - Number of most recent image-containing items to preserve (default: 2)
+ * @returns Object with processed items
+ */
+export declare function compressOpenAIConversationImages(items: OpenAIResponseInputItem[], keepMostRecentCount?: number): {
+    items: OpenAIResponseInputItem[];
+};
+/**
+ * Compresses Google conversation history by removing images from older items
+ * while keeping the most recent images intact
+ * @param items - Array of conversation items to process
+ * @param keepMostRecentCount - Number of most recent image-containing items to preserve (default: 2)
+ * @returns Object with processed items
+ */
+export declare function compressGoogleConversationImages(items: GoogleContent[], keepMostRecentCount?: number): {
+    items: GoogleContent[];
+};

package/dist/esm/lib/v3/agent/utils/imageCompression.js

@@ -0,0 +1,204 @@
+/**
+ * Finds all items in the conversation history that contain images
+ * @param items - Array of conversation items to check
+ * @returns Array of indices where images were found
+ */
+export function findItemsWithImages(items) {
+    const itemsWithImages = [];
+    items.forEach((item, index) => {
+        let hasImage = false;
+        if (Array.isArray(item.content)) {
+            hasImage = item.content.some((contentItem) => contentItem.type === "tool_result" &&
+                "content" in contentItem &&
+                Array.isArray(contentItem.content) &&
+                contentItem.content.some((nestedItem) => nestedItem.type === "image"));
+        }
+        if (hasImage) {
+            itemsWithImages.push(index);
+        }
+    });
+    return itemsWithImages;
+}
+/**
+ * Compresses conversation history by removing images from older items
+ * while keeping the most recent images intact
+ * @param items - Array of conversation items to process
+ * @param keepMostRecentCount - Number of most recent image-containing items to preserve (default: 2)
+ * @returns Object with processed items
+ */
+export function compressConversationImages(items, keepMostRecentCount = 2) {
+    const itemsWithImages = findItemsWithImages(items);
+    items.forEach((item, index) => {
+        const imageIndex = itemsWithImages.indexOf(index);
+        const shouldCompress = imageIndex >= 0 &&
+            imageIndex < itemsWithImages.length - keepMostRecentCount;
+        if (shouldCompress) {
+            if (Array.isArray(item.content)) {
+                item.content = item.content.map((contentItem) => {
+                    if (contentItem.type === "tool_result" &&
+                        "content" in contentItem &&
+                        Array.isArray(contentItem.content) &&
+                        contentItem.content.some((nestedItem) => nestedItem.type === "image")) {
+                        return {
+                            ...contentItem,
+                            content: "screenshot taken",
+                        };
+                    }
+                    return contentItem;
+                });
+            }
+        }
+    });
+    return {
+        items,
+    };
+}
+/**
+ * Finds all items in the conversation history that contain images (Google format)
+ * @param items - Array of conversation items to check
+ * @returns Array of indices where images were found
+ */
+export function findGoogleItemsWithImages(items) {
+    const itemsWithImages = [];
+    items.forEach((item, index) => {
+        let hasImage = false;
+        if (item.parts && Array.isArray(item.parts)) {
+            hasImage = item.parts.some((part) => {
+                // Check for functionResponse with data containing images
+                if (part.functionResponse?.response?.data) {
+                    const data = part.functionResponse.response
+                        .data;
+                    return data.some((dataItem) => dataItem.inlineData?.mimeType?.startsWith("image/"));
+                }
+                // Check for functionResponse with parts containing images
+                if (part.functionResponse?.parts) {
+                    return part.functionResponse.parts.some((responsePart) => responsePart.inlineData?.mimeType?.startsWith("image/"));
+                }
+                // Check for direct inline data
+                return part.inlineData?.mimeType?.startsWith("image/");
+            });
+        }
+        if (hasImage) {
+            itemsWithImages.push(index);
+        }
+    });
+    return itemsWithImages;
+}
+/**
+ * Finds all items in the conversation history that contain images (OpenAI format)
+ * @param items - Array of conversation items to check
+ * @returns Array of indices where images were found
+ */
+export function findOpenAIItemsWithImages(items) {
+    const itemsWithImages = [];
+    items.forEach((item, index) => {
+        let hasImage = false;
+        // Check for computer_call_output with image
+        if ("type" in item &&
+            item.type === "computer_call_output" &&
+            "output" in item) {
+            const output = item.output;
+            hasImage = output?.type === "input_image" && !!output?.image_url;
+        }
+        if (hasImage) {
+            itemsWithImages.push(index);
+        }
+    });
+    return itemsWithImages;
+}
+/**
+ * Compresses OpenAI conversation history by removing images from older items
+ * while keeping the most recent images intact
+ * @param items - Array of conversation items to process
+ * @param keepMostRecentCount - Number of most recent image-containing items to preserve (default: 2)
+ * @returns Object with processed items
+ */
+export function compressOpenAIConversationImages(items, keepMostRecentCount = 2) {
+    const itemsWithImages = findOpenAIItemsWithImages(items);
+    items.forEach((item, index) => {
+        const imageIndex = itemsWithImages.indexOf(index);
+        const shouldCompress = imageIndex >= 0 &&
+            imageIndex < itemsWithImages.length - keepMostRecentCount;
+        if (shouldCompress) {
+            // For computer_call_output with image, replace with text
+            if ("type" in item &&
+                item.type === "computer_call_output" &&
+                "output" in item) {
+                const output = item.output;
+                if (output?.type === "input_image") {
+                    // Replace the image with a text message
+                    item.output = "screenshot taken";
+                }
+            }
+        }
+    });
+    return {
+        items,
+    };
+}
+/**
+ * Compresses Google conversation history by removing images from older items
+ * while keeping the most recent images intact
+ * @param items - Array of conversation items to process
+ * @param keepMostRecentCount - Number of most recent image-containing items to preserve (default: 2)
+ * @returns Object with processed items
+ */
+export function compressGoogleConversationImages(items, keepMostRecentCount = 2) {
+    const itemsWithImages = findGoogleItemsWithImages(items);
+    items.forEach((item, index) => {
+        const imageIndex = itemsWithImages.indexOf(index);
+        const shouldCompress = imageIndex >= 0 &&
+            imageIndex < itemsWithImages.length - keepMostRecentCount;
+        if (shouldCompress && item.parts && Array.isArray(item.parts)) {
+            item.parts = item.parts.map((part) => {
+                // Replace functionResponse with data containing images
+                if (part.functionResponse?.response?.data) {
+                    const data = part.functionResponse.response
+                        .data;
+                    const hasImage = data.some((dataItem) => dataItem.inlineData?.mimeType?.startsWith("image/"));
+                    if (hasImage) {
+                        return {
+                            ...part,
+                            functionResponse: {
+                                ...part.functionResponse,
+                                data: [],
+                                response: {
+                                    ...part.functionResponse.response,
+                                    compressed: "screenshot taken",
+                                },
+                            },
+                        };
+                    }
+                }
+                // Replace functionResponse with parts containing images
+                if (part.functionResponse?.parts) {
+                    const hasImageInParts = part.functionResponse.parts.some((responsePart) => responsePart.inlineData?.mimeType?.startsWith("image/"));
+                    if (hasImageInParts) {
+                        return {
+                            ...part,
+                            functionResponse: {
+                                ...part.functionResponse,
+                                parts: part.functionResponse.parts.filter((responsePart) => !responsePart.inlineData?.mimeType?.startsWith("image/")),
+                                response: {
+                                    ...part.functionResponse.response,
+                                    compressed: "screenshot taken",
+                                },
+                            },
+                        };
+                    }
+                }
+                // Replace direct inline data images
+                if (part.inlineData?.mimeType?.startsWith("image/")) {
+                    return {
+                        text: "screenshot taken",
+                    };
+                }
+                return part;
+            });
+        }
+    });
+    return {
+        items,
+    };
+}
+//# sourceMappingURL=imageCompression.js.map

package/dist/esm/lib/v3/agent/utils/imageCompression.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"imageCompression.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/imageCompression.ts"],"names":[],"mappings":"AAyBA;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAA0B;IAC5D,MAAM,eAAe,GAAa,EAAE,CAAC;IAErC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC5B,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAChC,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAC1B,CAAC,WAAkC,EAAE,EAAE,CACrC,WAAW,CAAC,IAAI,KAAK,aAAa;gBAClC,SAAS,IAAI,WAAW;gBACxB,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC;gBACjC,WAAW,CAAC,OAAmC,CAAC,IAAI,CACnD,CAAC,UAAiC,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,KAAK,OAAO,CACnE,CACJ,CAAC;QACJ,CAAC;QAED,IAAI,QAAQ,EAAE,CAAC;YACb,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,eAAe,CAAC;AACzB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,0BAA0B,CACxC,KAA0B,EAC1B,sBAA8B,CAAC;IAE/B,MAAM,eAAe,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAEnD,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC5B,MAAM,UAAU,GAAG,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAClD,MAAM,cAAc,GAClB,UAAU,IAAI,CAAC;YACf,UAAU,GAAG,eAAe,CAAC,MAAM,GAAG,mBAAmB,CAAC;QAE5D,IAAI,cAAc,EAAE,CAAC;YACnB,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBAChC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAC7B,CAAC,WAAkC,EAAE,EAAE;oBACrC,IACE,WAAW,CAAC,IAAI,KAAK,aAAa;wBAClC,SAAS,IAAI,WAAW;wBACxB,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC;wBACjC,WAAW,CAAC,OAAmC,CAAC,IAAI,CACnD,CAAC,UAAiC,EAAE,EAAE,CACpC,UAAU,CAAC,IAAI,KAAK,OAAO,CAC9B,EACD,CAAC;wBACD,OAAO;4BACL,GAAG,WAAW;4BACd,OAAO,EAAE,kBAAkB;yBACH,CAAC;oBAC7B,CAAC;oBACD,OAAO,WAAW,CAAC;gBACrB,CAAC,CACF,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,KAAK;KACN,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CAAC,KAAsB;IAC9D,MAAM,eAAe,GAAa,EAAE,CAAC;IAErC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC5B,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,IAAI,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5C,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAgB,EAAE,EAAE;gBAC9C,yDAAyD;gBACzD,IAAI,IAAI,CAAC,gBAAgB,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;oBAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ;yBACxC,IAA8B,CAAC;oBAClC,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAC5B,QAAQ,CAAC,UAAU,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,CACpD,CAAC;gBACJ,CAAC;gBAED,0DAA0D;gBAC1D,IAAI,IAAI,CAAC,gBAAgB,EAAE,KAAK,EAAE,CAAC;oBACjC,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,EAAE,CACvD,YAAY,CAAC,UAAU,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,CACxD,CAAC;gBACJ,CAAC;gBAED,+BAA+B;gBAC/B,OAAO,IAAI,CAAC,UAAU,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC;YACzD,CAAC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,QAAQ,EAAE,CAAC;YACb,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,eAAe,CAAC;AACzB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CACvC,KAAgC;IAEhC,MAAM,eAAe,GAAa,EAAE,CAAC;IAErC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC5B,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,4CAA4C;QAC5C,IACE,MAAM,IAAI,IAAI;YACd,IAAI,CAAC,IAAI,KAAK,sBAAsB;YACpC,QAAQ,IAAI,IAAI,EAChB,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,CAAC,MAGnB,CAAC;YACF,QAAQ,GAAG,MAAM,EAAE,IAAI,KAAK,aAAa,IAAI,CAAC,CAAC,MAAM,EAAE,SAAS,CAAC;QACnE,CAAC;QAED,IAAI,QAAQ,EAAE,CAAC;YACb,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,eAAe,CAAC;AACzB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,gCAAgC,CAC9C,KAAgC,EAChC,sBAA8B,CAAC;IAE/B,MAAM,eAAe,GAAG,yBAAyB,CAAC,KAAK,CAAC,CAAC;IAEzD,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC5B,MAAM,UAAU,GAAG,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAClD,MAAM,cAAc,GAClB,UAAU,IAAI,CAAC;YACf,UAAU,GAAG,eAAe,CAAC,MAAM,GAAG,mBAAmB,CAAC;QAE5D,IAAI,cAAc,EAAE,CAAC;YACnB,yDAAyD;YACzD,IACE,MAAM,IAAI,IAAI;gBACd,IAAI,CAAC,IAAI,KAAK,sBAAsB;gBACpC,QAAQ,IAAI,IAAI,EAChB,CAAC;gBACD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAqC,CAAC;gBAC1D,IAAI,MAAM,EAAE,IAAI,KAAK,aAAa,EAAE,CAAC;oBACnC,wCAAwC;oBACvC,IAAsC,CAAC,MAAM,GAAG,kBAAkB,CAAC;gBACtE,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,KAAK;KACN,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,gCAAgC,CAC9C,KAAsB,EACtB,sBAA8B,CAAC;IAE/B,MAAM,eAAe,GAAG,yBAAyB,CAAC,KAAK,CAAC,CAAC;IAEzD,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC5B,MAAM,UAAU,GAAG,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAClD,MAAM,cAAc,GAClB,UAAU,IAAI,CAAC;YACf,UAAU,GAAG,eAAe,CAAC,MAAM,GAAG,mBAAmB,CAAC;QAE5D,IAAI,cAAc,IAAI,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9D,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAgB,EAAE,EAAE;gBAC/C,uDAAuD;gBACvD,IAAI,IAAI,CAAC,gBAAgB,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;oBAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ;yBACxC,IAA8B,CAAC;oBAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CACtC,QAAQ,CAAC,UAAU,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,CACpD,CAAC;oBACF,IAAI,QAAQ,EAAE,CAAC;wBACb,OAAO;4BACL,GAAG,IAAI;4BACP,gBAAgB,EAAE;gCAChB,GAAG,IAAI,CAAC,gBAAgB;gCACxB,IAAI,EAAE,EAA4B;gCAClC,QAAQ,EAAE;oCACR,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ;oCACjC,UAAU,EAAE,kBAAkB;iCAC/B;6BACF;yBACF,CAAC;oBACJ,CAAC;gBACH,CAAC;gBAED,wDAAwD;gBACxD,IAAI,IAAI,CAAC,gBAAgB,EAAE,KAAK,EAAE,CAAC;oBACjC,MAAM,eAAe,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,IAAI,CACtD,CAAC,YAAY,EAAE,EAAE,CACf,YAAY,CAAC,UAAU,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,CAC1D,CAAC;oBACF,IAAI,eAAe,EAAE,CAAC;wBACpB,OAAO;4BACL,GAAG,IAAI;4BACP,gBAAgB,EAAE;gCAChB,GAAG,IAAI,CAAC,gBAAgB;gCACxB,KAAK,EAAE,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,MAAM,CACvC,CAAC,YAAY,EAAE,EAAE,CACf,CAAC,YAAY,CAAC,UAAU,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,CAC3D;gCACD,QAAQ,EAAE;oCACR,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ;oCACjC,UAAU,EAAE,kBAAkB;iCAC/B;6BACF;yBACF,CAAC;oBACJ,CAAC;gBACH,CAAC;gBAED,oCAAoC;gBACpC,IAAI,IAAI,CAAC,UAAU,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACpD,OAAO;wBACL,IAAI,EAAE,kBAAkB;qBACzB,CAAC;gBACJ,CAAC;gBACD,OAAO,IAAI,CAAC;YACd,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,KAAK;KACN,CAAC;AACJ,CAAC","sourcesContent":["import {\n  AnthropicMessage,\n  AnthropicContentBlock,\n  AnthropicToolResult,\n  ResponseInputItem as OpenAIResponseInputItem,\n} from \"../../types/public/agent.js\";\nimport type {\n  Content as GoogleContent,\n  Part as GooglePart,\n} from \"@google/genai\";\n\nexport type ResponseInputItem = AnthropicMessage | AnthropicToolResult;\n\ninterface FunctionResponseData {\n  inlineData?: {\n    mimeType?: string;\n    data?: string;\n  };\n}\nexport type AnthropicResponseInputItem = AnthropicMessage | AnthropicToolResult;\nexport type SupportedInputItem =\n  | AnthropicResponseInputItem\n  | OpenAIResponseInputItem\n  | GoogleContent;\n\n/**\n * Finds all items in the conversation history that contain images\n * @param items - Array of conversation items to check\n * @returns Array of indices where images were found\n */\nexport function findItemsWithImages(items: ResponseInputItem[]): number[] {\n  const itemsWithImages: number[] = [];\n\n  items.forEach((item, index) => {\n    let hasImage = false;\n\n    if (Array.isArray(item.content)) {\n      hasImage = item.content.some(\n        (contentItem: AnthropicContentBlock) =>\n          contentItem.type === \"tool_result\" &&\n          \"content\" in contentItem &&\n          Array.isArray(contentItem.content) &&\n          (contentItem.content as AnthropicContentBlock[]).some(\n            (nestedItem: AnthropicContentBlock) => nestedItem.type === \"image\",\n          ),\n      );\n    }\n\n    if (hasImage) {\n      itemsWithImages.push(index);\n    }\n  });\n\n  return itemsWithImages;\n}\n\n/**\n * Compresses conversation history by removing images from older items\n * while keeping the most recent images intact\n * @param items - Array of conversation items to process\n * @param keepMostRecentCount - Number of most recent image-containing items to preserve (default: 2)\n * @returns Object with processed items\n */\nexport function compressConversationImages(\n  items: ResponseInputItem[],\n  keepMostRecentCount: number = 2,\n): { items: ResponseInputItem[] } {\n  const itemsWithImages = findItemsWithImages(items);\n\n  items.forEach((item, index) => {\n    const imageIndex = itemsWithImages.indexOf(index);\n    const shouldCompress =\n      imageIndex >= 0 &&\n      imageIndex < itemsWithImages.length - keepMostRecentCount;\n\n    if (shouldCompress) {\n      if (Array.isArray(item.content)) {\n        item.content = item.content.map(\n          (contentItem: AnthropicContentBlock) => {\n            if (\n              contentItem.type === \"tool_result\" &&\n              \"content\" in contentItem &&\n              Array.isArray(contentItem.content) &&\n              (contentItem.content as AnthropicContentBlock[]).some(\n                (nestedItem: AnthropicContentBlock) =>\n                  nestedItem.type === \"image\",\n              )\n            ) {\n              return {\n                ...contentItem,\n                content: \"screenshot taken\",\n              } as AnthropicContentBlock;\n            }\n            return contentItem;\n          },\n        );\n      }\n    }\n  });\n\n  return {\n    items,\n  };\n}\n\n/**\n * Finds all items in the conversation history that contain images (Google format)\n * @param items - Array of conversation items to check\n * @returns Array of indices where images were found\n */\nexport function findGoogleItemsWithImages(items: GoogleContent[]): number[] {\n  const itemsWithImages: number[] = [];\n\n  items.forEach((item, index) => {\n    let hasImage = false;\n\n    if (item.parts && Array.isArray(item.parts)) {\n      hasImage = item.parts.some((part: GooglePart) => {\n        // Check for functionResponse with data containing images\n        if (part.functionResponse?.response?.data) {\n          const data = part.functionResponse.response\n            .data as FunctionResponseData[];\n          return data.some((dataItem) =>\n            dataItem.inlineData?.mimeType?.startsWith(\"image/\"),\n          );\n        }\n\n        // Check for functionResponse with parts containing images\n        if (part.functionResponse?.parts) {\n          return part.functionResponse.parts.some((responsePart) =>\n            responsePart.inlineData?.mimeType?.startsWith(\"image/\"),\n          );\n        }\n\n        // Check for direct inline data\n        return part.inlineData?.mimeType?.startsWith(\"image/\");\n      });\n    }\n\n    if (hasImage) {\n      itemsWithImages.push(index);\n    }\n  });\n\n  return itemsWithImages;\n}\n\n/**\n * Finds all items in the conversation history that contain images (OpenAI format)\n * @param items - Array of conversation items to check\n * @returns Array of indices where images were found\n */\nexport function findOpenAIItemsWithImages(\n  items: OpenAIResponseInputItem[],\n): number[] {\n  const itemsWithImages: number[] = [];\n\n  items.forEach((item, index) => {\n    let hasImage = false;\n\n    // Check for computer_call_output with image\n    if (\n      \"type\" in item &&\n      item.type === \"computer_call_output\" &&\n      \"output\" in item\n    ) {\n      const output = item.output as unknown as {\n        type: string;\n        image_url: string;\n      };\n      hasImage = output?.type === \"input_image\" && !!output?.image_url;\n    }\n\n    if (hasImage) {\n      itemsWithImages.push(index);\n    }\n  });\n\n  return itemsWithImages;\n}\n\n/**\n * Compresses OpenAI conversation history by removing images from older items\n * while keeping the most recent images intact\n * @param items - Array of conversation items to process\n * @param keepMostRecentCount - Number of most recent image-containing items to preserve (default: 2)\n * @returns Object with processed items\n */\nexport function compressOpenAIConversationImages(\n  items: OpenAIResponseInputItem[],\n  keepMostRecentCount: number = 2,\n): { items: OpenAIResponseInputItem[] } {\n  const itemsWithImages = findOpenAIItemsWithImages(items);\n\n  items.forEach((item, index) => {\n    const imageIndex = itemsWithImages.indexOf(index);\n    const shouldCompress =\n      imageIndex >= 0 &&\n      imageIndex < itemsWithImages.length - keepMostRecentCount;\n\n    if (shouldCompress) {\n      // For computer_call_output with image, replace with text\n      if (\n        \"type\" in item &&\n        item.type === \"computer_call_output\" &&\n        \"output\" in item\n      ) {\n        const output = item.output as unknown as { type: string };\n        if (output?.type === \"input_image\") {\n          // Replace the image with a text message\n          (item as unknown as { output: string }).output = \"screenshot taken\";\n        }\n      }\n    }\n  });\n\n  return {\n    items,\n  };\n}\n\n/**\n * Compresses Google conversation history by removing images from older items\n * while keeping the most recent images intact\n * @param items - Array of conversation items to process\n * @param keepMostRecentCount - Number of most recent image-containing items to preserve (default: 2)\n * @returns Object with processed items\n */\nexport function compressGoogleConversationImages(\n  items: GoogleContent[],\n  keepMostRecentCount: number = 2,\n): { items: GoogleContent[] } {\n  const itemsWithImages = findGoogleItemsWithImages(items);\n\n  items.forEach((item, index) => {\n    const imageIndex = itemsWithImages.indexOf(index);\n    const shouldCompress =\n      imageIndex >= 0 &&\n      imageIndex < itemsWithImages.length - keepMostRecentCount;\n\n    if (shouldCompress && item.parts && Array.isArray(item.parts)) {\n      item.parts = item.parts.map((part: GooglePart) => {\n        // Replace functionResponse with data containing images\n        if (part.functionResponse?.response?.data) {\n          const data = part.functionResponse.response\n            .data as FunctionResponseData[];\n          const hasImage = data.some((dataItem) =>\n            dataItem.inlineData?.mimeType?.startsWith(\"image/\"),\n          );\n          if (hasImage) {\n            return {\n              ...part,\n              functionResponse: {\n                ...part.functionResponse,\n                data: [] as FunctionResponseData[],\n                response: {\n                  ...part.functionResponse.response,\n                  compressed: \"screenshot taken\",\n                },\n              },\n            };\n          }\n        }\n\n        // Replace functionResponse with parts containing images\n        if (part.functionResponse?.parts) {\n          const hasImageInParts = part.functionResponse.parts.some(\n            (responsePart) =>\n              responsePart.inlineData?.mimeType?.startsWith(\"image/\"),\n          );\n          if (hasImageInParts) {\n            return {\n              ...part,\n              functionResponse: {\n                ...part.functionResponse,\n                parts: part.functionResponse.parts.filter(\n                  (responsePart) =>\n                    !responsePart.inlineData?.mimeType?.startsWith(\"image/\"),\n                ),\n                response: {\n                  ...part.functionResponse.response,\n                  compressed: \"screenshot taken\",\n                },\n              },\n            };\n          }\n        }\n\n        // Replace direct inline data images\n        if (part.inlineData?.mimeType?.startsWith(\"image/\")) {\n          return {\n            text: \"screenshot taken\",\n          };\n        }\n        return part;\n      });\n    }\n  });\n\n  return {\n    items,\n  };\n}\n"]}

package/dist/esm/lib/v3/agent/utils/messageProcessing.d.ts

@@ -0,0 +1,12 @@
+import type { ModelMessage } from "ai";
+/**
+ * Compress old screenshot/ariaTree data in messages in-place.
+ *
+ * Strategy:
+ * - Keep only the 2 most recent vision results (screenshots OR vision action tools like click/type/etc)
+ * - Keep only the 1 most recent ariaTree (replace older ones with placeholder)
+ *
+ * @param messages - The messages array to modify in-place
+ * @returns Number of items compressed
+ */
+export declare function processMessages(messages: ModelMessage[]): number;