@browserbasehq/orca 3.0.9-alpha-4 → 3.1.0-headers

package/dist/cjs/lib/v3/agent/utils/handleDoneToolCall.js

@@ -0,0 +1,108 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleDoneToolCall = handleDoneToolCall;
+const ai_1 = require("ai");
+const zod_1 = require("zod");
+const ai_2 = require("ai");
+const utils_js_1 = require("../../../utils.js");
+function buildBaseDoneSchema(factory) {
+    return factory.object({
+        reasoning: factory
+            .string()
+            .describe("Brief summary of what actions were taken and the outcome"),
+        taskComplete: factory
+            .boolean()
+            .describe("true if the task was fully completed, false otherwise"),
+    });
+}
+/**
+ * Force a done tool call at the end of an agent run.
+ * This ensures we always get a structured final response,
+ * even if the main loop ended without calling done.
+ */
+async function handleDoneToolCall(options) {
+    const { model, inputMessages, instruction, outputSchema, logger } = options;
+    logger({
+        category: "agent",
+        message: "Agent calling tool: done",
+        level: 1,
+    });
+    // Use the same Zod version as the user's outputSchema to avoid v3/v4 mixing
+    const factory = outputSchema
+        ? (0, utils_js_1.getZFactory)(outputSchema)
+        : zod_1.z;
+    const baseDoneSchema = buildBaseDoneSchema(factory);
+    // Merge base done schema with user-provided output schema if present
+    const doneToolSchema = outputSchema
+        ? baseDoneSchema.extend({
+            output: outputSchema.describe("The specific data the user requested from this task"),
+        })
+        : baseDoneSchema;
+    const outputInstructions = outputSchema
+        ? `\n\nThe user also requested the following information from this task. Provide it in the "output" field:\n${JSON.stringify(Object.fromEntries(Object.entries(outputSchema.shape).map(([key, value]) => [
+            key,
+            value.description || "no description",
+        ])), null, 2)}`
+        : "";
+    const systemPrompt = `You are a web automation assistant that was tasked with completing a task.
+
+The task was:
+"${instruction}"
+
+Review what was accomplished and provide your final assessment in whether the task was completed successfully. you have been provided with the history of the actions taken so far, use this to determine if the task was completed successfully.${outputInstructions}
+
+Call the "done" tool with:
+1. A brief summary of what was done
+2. Whether the task was completed successfully${outputSchema ? "\n3. The requested output data based on what you found" : ""}`;
+    const doneTool = (0, ai_2.tool)({
+        description: outputSchema
+            ? "Complete the task with your assessment and the requested output data."
+            : "Complete the task with your final assessment.",
+        inputSchema: doneToolSchema,
+        execute: async (params) => {
+            return { success: true, ...params };
+        },
+    });
+    const userPrompt = {
+        role: "user",
+        content: outputSchema
+            ? "Provide your final assessment and the requested output data."
+            : "Provide your final assessment.",
+    };
+    const result = await (0, ai_1.generateText)({
+        model,
+        system: systemPrompt,
+        messages: [...inputMessages, userPrompt],
+        tools: { done: doneTool },
+        toolChoice: { type: "tool", toolName: "done" },
+        providerOptions: {
+            google: { mediaResolution: "MEDIA_RESOLUTION_HIGH" },
+            openai: { store: false },
+        },
+    });
+    const doneToolCall = result.toolCalls.find((tc) => tc.toolName === "done");
+    const outputMessages = [
+        userPrompt,
+        ...(result.response?.messages || []),
+    ];
+    if (!doneToolCall) {
+        return {
+            reasoning: result.text || "Task execution completed",
+            taskComplete: false,
+            messages: outputMessages,
+        };
+    }
+    const input = doneToolCall.input;
+    logger({
+        category: "agent",
+        message: `Task completed`,
+        level: 1,
+    });
+    return {
+        reasoning: input.reasoning,
+        taskComplete: input.taskComplete,
+        messages: outputMessages,
+        output: input.output,
+    };
+}
+//# sourceMappingURL=handleDoneToolCall.js.map

package/dist/cjs/lib/v3/agent/utils/handleDoneToolCall.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"handleDoneToolCall.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/handleDoneToolCall.ts"],"names":[],"mappings":";;AA+BA,gDAmHC;AAlJD,2BAAwE;AACxE,6BAAwB;AACxB,2BAA0B;AAG1B,gDAAgD;AAUhD,SAAS,mBAAmB,CAAC,OAAiB;IAC5C,OAAO,OAAO,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,OAAO;aACf,MAAM,EAAE;aACR,QAAQ,CAAC,0DAA0D,CAAC;QACvE,YAAY,EAAE,OAAO;aAClB,OAAO,EAAE;aACT,QAAQ,CAAC,uDAAuD,CAAC;KACrE,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACI,KAAK,UAAU,kBAAkB,CAAC,OAMxC;IACC,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAE5E,MAAM,CAAC;QACL,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,0BAA0B;QACnC,KAAK,EAAE,CAAC;KACT,CAAC,CAAC;IACH,4EAA4E;IAC5E,MAAM,OAAO,GAAG,YAAY;QAC1B,CAAC,CAAC,IAAA,sBAAW,EAAC,YAAkC,CAAC;QACjD,CAAC,CAAC,OAAC,CAAC;IACN,MAAM,cAAc,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAEpD,qEAAqE;IACrE,MAAM,cAAc,GAAG,YAAY;QACjC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,YAAY,CAAC,QAAQ,CAC3B,qDAAqD,CACtD;SACF,CAAC;QACJ,CAAC,CAAC,cAAc,CAAC;IAEnB,MAAM,kBAAkB,GAAG,YAAY;QACrC,CAAC,CAAC,4GAA4G,IAAI,CAAC,SAAS,CACxH,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,GAAG,CACpC,CAAC,CAAC,GAAG,EAAE,KAAK,CAA+B,EAAE,EAAE,CAAC;YAC9C,GAAG;YACH,KAAK,CAAC,WAAW,IAAI,gBAAgB;SACtC,CACF,CACF,EACD,IAAI,EACJ,CAAC,CACF,EAAE;QACL,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,YAAY,GAAG;;;GAGpB,WAAW;;mPAEqO,kBAAkB;;;;gDAIrN,YAAY,CAAC,CAAC,CAAC,wDAAwD,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IAE7H,MAAM,QAAQ,GAAG,IAAA,SAAI,EAAC;QACpB,WAAW,EAAE,YAAY;YACvB,CAAC,CAAC,uEAAuE;YACzE,CAAC,CAAC,+CAA+C;QACnD,WAAW,EAAE,cAAc;QAC3B,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YACxB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC;QACtC,CAAC;KACF,CAAC,CAAC;IAEH,MAAM,UAAU,GAAiB;QAC/B,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,YAAY;YACnB,CAAC,CAAC,8DAA8D;YAChE,CAAC,CAAC,gCAAgC;KACrC,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,IAAA,iBAAY,EAAC;QAChC,KAAK;QACL,MAAM,EAAE,YAAY;QACpB,QAAQ,EAAE,CAAC,GAAG,aAAa,EAAE,UAAU,CAAC;QACxC,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAa;QACpC,UAAU,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE;QAC9C,eAAe,EAAE;YACf,MAAM,EAAE,EAAE,eAAe,EAAE,uBAAuB,EAAE;YACpD,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE;SACzB;KACF,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC;IAC3E,MAAM,cAAc,GAAmB;QACrC,UAAU;QACV,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,IAAI,EAAE,CAAC;KACrC,CAAC;IAEF,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,OAAO;YACL,SAAS,EAAE,MAAM,CAAC,IAAI,IAAI,0BAA0B;YACpD,YAAY,EAAE,KAAK;YACnB,QAAQ,EAAE,cAAc;SACzB,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,YAAY,CAAC,KAI1B,CAAC;IACF,MAAM,CAAC;QACL,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,gBAAgB;QACzB,KAAK,EAAE,CAAC;KACT,CAAC,CAAC;IAEH,OAAO;QACL,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,QAAQ,EAAE,cAAc;QACxB,MAAM,EAAE,KAAK,CAAC,MAAM;KACrB,CAAC;AACJ,CAAC","sourcesContent":["import { generateText, ModelMessage, LanguageModel, ToolSet } from \"ai\";\nimport { z } from \"zod\";\nimport { tool } from \"ai\";\nimport { LogLine } from \"../../types/public/logs.js\";\nimport { StagehandZodObject } from \"../../zodCompat.js\";\nimport { getZFactory } from \"../../../utils.js\";\nimport type { StagehandZodSchema } from \"../../zodCompat.js\";\n\ninterface DoneResult {\n  reasoning: string;\n  taskComplete: boolean;\n  messages: ModelMessage[];\n  output?: Record<string, unknown>;\n}\n\nfunction buildBaseDoneSchema(factory: typeof z) {\n  return factory.object({\n    reasoning: factory\n      .string()\n      .describe(\"Brief summary of what actions were taken and the outcome\"),\n    taskComplete: factory\n      .boolean()\n      .describe(\"true if the task was fully completed, false otherwise\"),\n  });\n}\n\n/**\n * Force a done tool call at the end of an agent run.\n * This ensures we always get a structured final response,\n * even if the main loop ended without calling done.\n */\nexport async function handleDoneToolCall(options: {\n  model: LanguageModel;\n  inputMessages: ModelMessage[];\n  instruction: string;\n  outputSchema?: StagehandZodObject;\n  logger: (message: LogLine) => void;\n}): Promise<DoneResult> {\n  const { model, inputMessages, instruction, outputSchema, logger } = options;\n\n  logger({\n    category: \"agent\",\n    message: \"Agent calling tool: done\",\n    level: 1,\n  });\n  // Use the same Zod version as the user's outputSchema to avoid v3/v4 mixing\n  const factory = outputSchema\n    ? getZFactory(outputSchema as StagehandZodSchema)\n    : z;\n  const baseDoneSchema = buildBaseDoneSchema(factory);\n\n  // Merge base done schema with user-provided output schema if present\n  const doneToolSchema = outputSchema\n    ? baseDoneSchema.extend({\n        output: outputSchema.describe(\n          \"The specific data the user requested from this task\",\n        ),\n      })\n    : baseDoneSchema;\n\n  const outputInstructions = outputSchema\n    ? `\\n\\nThe user also requested the following information from this task. Provide it in the \"output\" field:\\n${JSON.stringify(\n        Object.fromEntries(\n          Object.entries(outputSchema.shape).map(\n            ([key, value]: [string, StagehandZodSchema]) => [\n              key,\n              value.description || \"no description\",\n            ],\n          ),\n        ),\n        null,\n        2,\n      )}`\n    : \"\";\n\n  const systemPrompt = `You are a web automation assistant that was tasked with completing a task.\n\nThe task was:\n\"${instruction}\"\n\nReview what was accomplished and provide your final assessment in whether the task was completed successfully. you have been provided with the history of the actions taken so far, use this to determine if the task was completed successfully.${outputInstructions}\n\nCall the \"done\" tool with:\n1. A brief summary of what was done\n2. Whether the task was completed successfully${outputSchema ? \"\\n3. The requested output data based on what you found\" : \"\"}`;\n\n  const doneTool = tool({\n    description: outputSchema\n      ? \"Complete the task with your assessment and the requested output data.\"\n      : \"Complete the task with your final assessment.\",\n    inputSchema: doneToolSchema,\n    execute: async (params) => {\n      return { success: true, ...params };\n    },\n  });\n\n  const userPrompt: ModelMessage = {\n    role: \"user\",\n    content: outputSchema\n      ? \"Provide your final assessment and the requested output data.\"\n      : \"Provide your final assessment.\",\n  };\n\n  const result = await generateText({\n    model,\n    system: systemPrompt,\n    messages: [...inputMessages, userPrompt],\n    tools: { done: doneTool } as ToolSet,\n    toolChoice: { type: \"tool\", toolName: \"done\" },\n    providerOptions: {\n      google: { mediaResolution: \"MEDIA_RESOLUTION_HIGH\" },\n      openai: { store: false },\n    },\n  });\n\n  const doneToolCall = result.toolCalls.find((tc) => tc.toolName === \"done\");\n  const outputMessages: ModelMessage[] = [\n    userPrompt,\n    ...(result.response?.messages || []),\n  ];\n\n  if (!doneToolCall) {\n    return {\n      reasoning: result.text || \"Task execution completed\",\n      taskComplete: false,\n      messages: outputMessages,\n    };\n  }\n\n  const input = doneToolCall.input as {\n    reasoning: string;\n    taskComplete: boolean;\n    output?: Record<string, unknown>;\n  };\n  logger({\n    category: \"agent\",\n    message: `Task completed`,\n    level: 1,\n  });\n\n  return {\n    reasoning: input.reasoning,\n    taskComplete: input.taskComplete,\n    messages: outputMessages,\n    output: input.output,\n  };\n}\n"]}

package/dist/cjs/lib/v3/agent/utils/imageCompression.d.ts

@@ -0,0 +1,53 @@
+import { AnthropicMessage, AnthropicToolResult, ResponseInputItem as OpenAIResponseInputItem } from "../../types/public/agent.js";
+import type { Content as GoogleContent } from "@google/genai";
+export type ResponseInputItem = AnthropicMessage | AnthropicToolResult;
+export type AnthropicResponseInputItem = AnthropicMessage | AnthropicToolResult;
+export type SupportedInputItem = AnthropicResponseInputItem | OpenAIResponseInputItem | GoogleContent;
+/**
+ * Finds all items in the conversation history that contain images
+ * @param items - Array of conversation items to check
+ * @returns Array of indices where images were found
+ */
+export declare function findItemsWithImages(items: ResponseInputItem[]): number[];
+/**
+ * Compresses conversation history by removing images from older items
+ * while keeping the most recent images intact
+ * @param items - Array of conversation items to process
+ * @param keepMostRecentCount - Number of most recent image-containing items to preserve (default: 2)
+ * @returns Object with processed items
+ */
+export declare function compressConversationImages(items: ResponseInputItem[], keepMostRecentCount?: number): {
+    items: ResponseInputItem[];
+};
+/**
+ * Finds all items in the conversation history that contain images (Google format)
+ * @param items - Array of conversation items to check
+ * @returns Array of indices where images were found
+ */
+export declare function findGoogleItemsWithImages(items: GoogleContent[]): number[];
+/**
+ * Finds all items in the conversation history that contain images (OpenAI format)
+ * @param items - Array of conversation items to check
+ * @returns Array of indices where images were found
+ */
+export declare function findOpenAIItemsWithImages(items: OpenAIResponseInputItem[]): number[];
+/**
+ * Compresses OpenAI conversation history by removing images from older items
+ * while keeping the most recent images intact
+ * @param items - Array of conversation items to process
+ * @param keepMostRecentCount - Number of most recent image-containing items to preserve (default: 2)
+ * @returns Object with processed items
+ */
+export declare function compressOpenAIConversationImages(items: OpenAIResponseInputItem[], keepMostRecentCount?: number): {
+    items: OpenAIResponseInputItem[];
+};
+/**
+ * Compresses Google conversation history by removing images from older items
+ * while keeping the most recent images intact
+ * @param items - Array of conversation items to process
+ * @param keepMostRecentCount - Number of most recent image-containing items to preserve (default: 2)
+ * @returns Object with processed items
+ */
+export declare function compressGoogleConversationImages(items: GoogleContent[], keepMostRecentCount?: number): {
+    items: GoogleContent[];
+};

package/dist/cjs/lib/v3/agent/utils/imageCompression.js

@@ -0,0 +1,212 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.findItemsWithImages = findItemsWithImages;
+exports.compressConversationImages = compressConversationImages;
+exports.findGoogleItemsWithImages = findGoogleItemsWithImages;
+exports.findOpenAIItemsWithImages = findOpenAIItemsWithImages;
+exports.compressOpenAIConversationImages = compressOpenAIConversationImages;
+exports.compressGoogleConversationImages = compressGoogleConversationImages;
+/**
+ * Finds all items in the conversation history that contain images
+ * @param items - Array of conversation items to check
+ * @returns Array of indices where images were found
+ */
+function findItemsWithImages(items) {
+    const itemsWithImages = [];
+    items.forEach((item, index) => {
+        let hasImage = false;
+        if (Array.isArray(item.content)) {
+            hasImage = item.content.some((contentItem) => contentItem.type === "tool_result" &&
+                "content" in contentItem &&
+                Array.isArray(contentItem.content) &&
+                contentItem.content.some((nestedItem) => nestedItem.type === "image"));
+        }
+        if (hasImage) {
+            itemsWithImages.push(index);
+        }
+    });
+    return itemsWithImages;
+}
+/**
+ * Compresses conversation history by removing images from older items
+ * while keeping the most recent images intact
+ * @param items - Array of conversation items to process
+ * @param keepMostRecentCount - Number of most recent image-containing items to preserve (default: 2)
+ * @returns Object with processed items
+ */
+function compressConversationImages(items, keepMostRecentCount = 2) {
+    const itemsWithImages = findItemsWithImages(items);
+    items.forEach((item, index) => {
+        const imageIndex = itemsWithImages.indexOf(index);
+        const shouldCompress = imageIndex >= 0 &&
+            imageIndex < itemsWithImages.length - keepMostRecentCount;
+        if (shouldCompress) {
+            if (Array.isArray(item.content)) {
+                item.content = item.content.map((contentItem) => {
+                    if (contentItem.type === "tool_result" &&
+                        "content" in contentItem &&
+                        Array.isArray(contentItem.content) &&
+                        contentItem.content.some((nestedItem) => nestedItem.type === "image")) {
+                        return {
+                            ...contentItem,
+                            content: "screenshot taken",
+                        };
+                    }
+                    return contentItem;
+                });
+            }
+        }
+    });
+    return {
+        items,
+    };
+}
+/**
+ * Finds all items in the conversation history that contain images (Google format)
+ * @param items - Array of conversation items to check
+ * @returns Array of indices where images were found
+ */
+function findGoogleItemsWithImages(items) {
+    const itemsWithImages = [];
+    items.forEach((item, index) => {
+        let hasImage = false;
+        if (item.parts && Array.isArray(item.parts)) {
+            hasImage = item.parts.some((part) => {
+                // Check for functionResponse with data containing images
+                if (part.functionResponse?.response?.data) {
+                    const data = part.functionResponse.response
+                        .data;
+                    return data.some((dataItem) => dataItem.inlineData?.mimeType?.startsWith("image/"));
+                }
+                // Check for functionResponse with parts containing images
+                if (part.functionResponse?.parts) {
+                    return part.functionResponse.parts.some((responsePart) => responsePart.inlineData?.mimeType?.startsWith("image/"));
+                }
+                // Check for direct inline data
+                return part.inlineData?.mimeType?.startsWith("image/");
+            });
+        }
+        if (hasImage) {
+            itemsWithImages.push(index);
+        }
+    });
+    return itemsWithImages;
+}
+/**
+ * Finds all items in the conversation history that contain images (OpenAI format)
+ * @param items - Array of conversation items to check
+ * @returns Array of indices where images were found
+ */
+function findOpenAIItemsWithImages(items) {
+    const itemsWithImages = [];
+    items.forEach((item, index) => {
+        let hasImage = false;
+        // Check for computer_call_output with image
+        if ("type" in item &&
+            item.type === "computer_call_output" &&
+            "output" in item) {
+            const output = item.output;
+            hasImage = output?.type === "input_image" && !!output?.image_url;
+        }
+        if (hasImage) {
+            itemsWithImages.push(index);
+        }
+    });
+    return itemsWithImages;
+}
+/**
+ * Compresses OpenAI conversation history by removing images from older items
+ * while keeping the most recent images intact
+ * @param items - Array of conversation items to process
+ * @param keepMostRecentCount - Number of most recent image-containing items to preserve (default: 2)
+ * @returns Object with processed items
+ */
+function compressOpenAIConversationImages(items, keepMostRecentCount = 2) {
+    const itemsWithImages = findOpenAIItemsWithImages(items);
+    items.forEach((item, index) => {
+        const imageIndex = itemsWithImages.indexOf(index);
+        const shouldCompress = imageIndex >= 0 &&
+            imageIndex < itemsWithImages.length - keepMostRecentCount;
+        if (shouldCompress) {
+            // For computer_call_output with image, replace with text
+            if ("type" in item &&
+                item.type === "computer_call_output" &&
+                "output" in item) {
+                const output = item.output;
+                if (output?.type === "input_image") {
+                    // Replace the image with a text message
+                    item.output = "screenshot taken";
+                }
+            }
+        }
+    });
+    return {
+        items,
+    };
+}
+/**
+ * Compresses Google conversation history by removing images from older items
+ * while keeping the most recent images intact
+ * @param items - Array of conversation items to process
+ * @param keepMostRecentCount - Number of most recent image-containing items to preserve (default: 2)
+ * @returns Object with processed items
+ */
+function compressGoogleConversationImages(items, keepMostRecentCount = 2) {
+    const itemsWithImages = findGoogleItemsWithImages(items);
+    items.forEach((item, index) => {
+        const imageIndex = itemsWithImages.indexOf(index);
+        const shouldCompress = imageIndex >= 0 &&
+            imageIndex < itemsWithImages.length - keepMostRecentCount;
+        if (shouldCompress && item.parts && Array.isArray(item.parts)) {
+            item.parts = item.parts.map((part) => {
+                // Replace functionResponse with data containing images
+                if (part.functionResponse?.response?.data) {
+                    const data = part.functionResponse.response
+                        .data;
+                    const hasImage = data.some((dataItem) => dataItem.inlineData?.mimeType?.startsWith("image/"));
+                    if (hasImage) {
+                        return {
+                            ...part,
+                            functionResponse: {
+                                ...part.functionResponse,
+                                data: [],
+                                response: {
+                                    ...part.functionResponse.response,
+                                    compressed: "screenshot taken",
+                                },
+                            },
+                        };
+                    }
+                }
+                // Replace functionResponse with parts containing images
+                if (part.functionResponse?.parts) {
+                    const hasImageInParts = part.functionResponse.parts.some((responsePart) => responsePart.inlineData?.mimeType?.startsWith("image/"));
+                    if (hasImageInParts) {
+                        return {
+                            ...part,
+                            functionResponse: {
+                                ...part.functionResponse,
+                                parts: part.functionResponse.parts.filter((responsePart) => !responsePart.inlineData?.mimeType?.startsWith("image/")),
+                                response: {
+                                    ...part.functionResponse.response,
+                                    compressed: "screenshot taken",
+                                },
+                            },
+                        };
+                    }
+                }
+                // Replace direct inline data images
+                if (part.inlineData?.mimeType?.startsWith("image/")) {
+                    return {
+                        text: "screenshot taken",
+                    };
+                }
+                return part;
+            });
+        }
+    });
+    return {
+        items,
+    };
+}
+//# sourceMappingURL=imageCompression.js.map

package/dist/cjs/lib/v3/agent/utils/imageCompression.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"imageCompression.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/imageCompression.ts"],"names":[],"mappings":";;AA8BA,kDAwBC;AASD,gEAwCC;AAOD,8DAmCC;AAOD,8DA2BC;AASD,4EA+BC;AASD,4EA0EC;AArRD;;;;GAIG;AACH,SAAgB,mBAAmB,CAAC,KAA0B;IAC5D,MAAM,eAAe,GAAa,EAAE,CAAC;IAErC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC5B,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAChC,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAC1B,CAAC,WAAkC,EAAE,EAAE,CACrC,WAAW,CAAC,IAAI,KAAK,aAAa;gBAClC,SAAS,IAAI,WAAW;gBACxB,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC;gBACjC,WAAW,CAAC,OAAmC,CAAC,IAAI,CACnD,CAAC,UAAiC,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,KAAK,OAAO,CACnE,CACJ,CAAC;QACJ,CAAC;QAED,IAAI,QAAQ,EAAE,CAAC;YACb,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,eAAe,CAAC;AACzB,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,0BAA0B,CACxC,KAA0B,EAC1B,sBAA8B,CAAC;IAE/B,MAAM,eAAe,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAEnD,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC5B,MAAM,UAAU,GAAG,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAClD,MAAM,cAAc,GAClB,UAAU,IAAI,CAAC;YACf,UAAU,GAAG,eAAe,CAAC,MAAM,GAAG,mBAAmB,CAAC;QAE5D,IAAI,cAAc,EAAE,CAAC;YACnB,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBAChC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAC7B,CAAC,WAAkC,EAAE,EAAE;oBACrC,IACE,WAAW,CAAC,IAAI,KAAK,aAAa;wBAClC,SAAS,IAAI,WAAW;wBACxB,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC;wBACjC,WAAW,CAAC,OAAmC,CAAC,IAAI,CACnD,CAAC,UAAiC,EAAE,EAAE,CACpC,UAAU,CAAC,IAAI,KAAK,OAAO,CAC9B,EACD,CAAC;wBACD,OAAO;4BACL,GAAG,WAAW;4BACd,OAAO,EAAE,kBAAkB;yBACH,CAAC;oBAC7B,CAAC;oBACD,OAAO,WAAW,CAAC;gBACrB,CAAC,CACF,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,KAAK;KACN,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAgB,yBAAyB,CAAC,KAAsB;IAC9D,MAAM,eAAe,GAAa,EAAE,CAAC;IAErC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC5B,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,IAAI,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5C,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAgB,EAAE,EAAE;gBAC9C,yDAAyD;gBACzD,IAAI,IAAI,CAAC,gBAAgB,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;oBAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ;yBACxC,IAA8B,CAAC;oBAClC,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAC5B,QAAQ,CAAC,UAAU,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,CACpD,CAAC;gBACJ,CAAC;gBAED,0DAA0D;gBAC1D,IAAI,IAAI,CAAC,gBAAgB,EAAE,KAAK,EAAE,CAAC;oBACjC,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,EAAE,CACvD,YAAY,CAAC,UAAU,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,CACxD,CAAC;gBACJ,CAAC;gBAED,+BAA+B;gBAC/B,OAAO,IAAI,CAAC,UAAU,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC;YACzD,CAAC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,QAAQ,EAAE,CAAC;YACb,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,eAAe,CAAC;AACzB,CAAC;AAED;;;;GAIG;AACH,SAAgB,yBAAyB,CACvC,KAAgC;IAEhC,MAAM,eAAe,GAAa,EAAE,CAAC;IAErC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC5B,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,4CAA4C;QAC5C,IACE,MAAM,IAAI,IAAI;YACd,IAAI,CAAC,IAAI,KAAK,sBAAsB;YACpC,QAAQ,IAAI,IAAI,EAChB,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,CAAC,MAGnB,CAAC;YACF,QAAQ,GAAG,MAAM,EAAE,IAAI,KAAK,aAAa,IAAI,CAAC,CAAC,MAAM,EAAE,SAAS,CAAC;QACnE,CAAC;QAED,IAAI,QAAQ,EAAE,CAAC;YACb,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,eAAe,CAAC;AACzB,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,gCAAgC,CAC9C,KAAgC,EAChC,sBAA8B,CAAC;IAE/B,MAAM,eAAe,GAAG,yBAAyB,CAAC,KAAK,CAAC,CAAC;IAEzD,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC5B,MAAM,UAAU,GAAG,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAClD,MAAM,cAAc,GAClB,UAAU,IAAI,CAAC;YACf,UAAU,GAAG,eAAe,CAAC,MAAM,GAAG,mBAAmB,CAAC;QAE5D,IAAI,cAAc,EAAE,CAAC;YACnB,yDAAyD;YACzD,IACE,MAAM,IAAI,IAAI;gBACd,IAAI,CAAC,IAAI,KAAK,sBAAsB;gBACpC,QAAQ,IAAI,IAAI,EAChB,CAAC;gBACD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAqC,CAAC;gBAC1D,IAAI,MAAM,EAAE,IAAI,KAAK,aAAa,EAAE,CAAC;oBACnC,wCAAwC;oBACvC,IAAsC,CAAC,MAAM,GAAG,kBAAkB,CAAC;gBACtE,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,KAAK;KACN,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,gCAAgC,CAC9C,KAAsB,EACtB,sBAA8B,CAAC;IAE/B,MAAM,eAAe,GAAG,yBAAyB,CAAC,KAAK,CAAC,CAAC;IAEzD,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC5B,MAAM,UAAU,GAAG,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAClD,MAAM,cAAc,GAClB,UAAU,IAAI,CAAC;YACf,UAAU,GAAG,eAAe,CAAC,MAAM,GAAG,mBAAmB,CAAC;QAE5D,IAAI,cAAc,IAAI,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9D,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAgB,EAAE,EAAE;gBAC/C,uDAAuD;gBACvD,IAAI,IAAI,CAAC,gBAAgB,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;oBAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ;yBACxC,IAA8B,CAAC;oBAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CACtC,QAAQ,CAAC,UAAU,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,CACpD,CAAC;oBACF,IAAI,QAAQ,EAAE,CAAC;wBACb,OAAO;4BACL,GAAG,IAAI;4BACP,gBAAgB,EAAE;gCAChB,GAAG,IAAI,CAAC,gBAAgB;gCACxB,IAAI,EAAE,EAA4B;gCAClC,QAAQ,EAAE;oCACR,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ;oCACjC,UAAU,EAAE,kBAAkB;iCAC/B;6BACF;yBACF,CAAC;oBACJ,CAAC;gBACH,CAAC;gBAED,wDAAwD;gBACxD,IAAI,IAAI,CAAC,gBAAgB,EAAE,KAAK,EAAE,CAAC;oBACjC,MAAM,eAAe,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,IAAI,CACtD,CAAC,YAAY,EAAE,EAAE,CACf,YAAY,CAAC,UAAU,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,CAC1D,CAAC;oBACF,IAAI,eAAe,EAAE,CAAC;wBACpB,OAAO;4BACL,GAAG,IAAI;4BACP,gBAAgB,EAAE;gCAChB,GAAG,IAAI,CAAC,gBAAgB;gCACxB,KAAK,EAAE,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,MAAM,CACvC,CAAC,YAAY,EAAE,EAAE,CACf,CAAC,YAAY,CAAC,UAAU,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,CAC3D;gCACD,QAAQ,EAAE;oCACR,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ;oCACjC,UAAU,EAAE,kBAAkB;iCAC/B;6BACF;yBACF,CAAC;oBACJ,CAAC;gBACH,CAAC;gBAED,oCAAoC;gBACpC,IAAI,IAAI,CAAC,UAAU,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACpD,OAAO;wBACL,IAAI,EAAE,kBAAkB;qBACzB,CAAC;gBACJ,CAAC;gBACD,OAAO,IAAI,CAAC;YACd,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,KAAK;KACN,CAAC;AACJ,CAAC","sourcesContent":["import {\n  AnthropicMessage,\n  AnthropicContentBlock,\n  AnthropicToolResult,\n  ResponseInputItem as OpenAIResponseInputItem,\n} from \"../../types/public/agent.js\";\nimport type {\n  Content as GoogleContent,\n  Part as GooglePart,\n} from \"@google/genai\";\n\nexport type ResponseInputItem = AnthropicMessage | AnthropicToolResult;\n\ninterface FunctionResponseData {\n  inlineData?: {\n    mimeType?: string;\n    data?: string;\n  };\n}\nexport type AnthropicResponseInputItem = AnthropicMessage | AnthropicToolResult;\nexport type SupportedInputItem =\n  | AnthropicResponseInputItem\n  | OpenAIResponseInputItem\n  | GoogleContent;\n\n/**\n * Finds all items in the conversation history that contain images\n * @param items - Array of conversation items to check\n * @returns Array of indices where images were found\n */\nexport function findItemsWithImages(items: ResponseInputItem[]): number[] {\n  const itemsWithImages: number[] = [];\n\n  items.forEach((item, index) => {\n    let hasImage = false;\n\n    if (Array.isArray(item.content)) {\n      hasImage = item.content.some(\n        (contentItem: AnthropicContentBlock) =>\n          contentItem.type === \"tool_result\" &&\n          \"content\" in contentItem &&\n          Array.isArray(contentItem.content) &&\n          (contentItem.content as AnthropicContentBlock[]).some(\n            (nestedItem: AnthropicContentBlock) => nestedItem.type === \"image\",\n          ),\n      );\n    }\n\n    if (hasImage) {\n      itemsWithImages.push(index);\n    }\n  });\n\n  return itemsWithImages;\n}\n\n/**\n * Compresses conversation history by removing images from older items\n * while keeping the most recent images intact\n * @param items - Array of conversation items to process\n * @param keepMostRecentCount - Number of most recent image-containing items to preserve (default: 2)\n * @returns Object with processed items\n */\nexport function compressConversationImages(\n  items: ResponseInputItem[],\n  keepMostRecentCount: number = 2,\n): { items: ResponseInputItem[] } {\n  const itemsWithImages = findItemsWithImages(items);\n\n  items.forEach((item, index) => {\n    const imageIndex = itemsWithImages.indexOf(index);\n    const shouldCompress =\n      imageIndex >= 0 &&\n      imageIndex < itemsWithImages.length - keepMostRecentCount;\n\n    if (shouldCompress) {\n      if (Array.isArray(item.content)) {\n        item.content = item.content.map(\n          (contentItem: AnthropicContentBlock) => {\n            if (\n              contentItem.type === \"tool_result\" &&\n              \"content\" in contentItem &&\n              Array.isArray(contentItem.content) &&\n              (contentItem.content as AnthropicContentBlock[]).some(\n                (nestedItem: AnthropicContentBlock) =>\n                  nestedItem.type === \"image\",\n              )\n            ) {\n              return {\n                ...contentItem,\n                content: \"screenshot taken\",\n              } as AnthropicContentBlock;\n            }\n            return contentItem;\n          },\n        );\n      }\n    }\n  });\n\n  return {\n    items,\n  };\n}\n\n/**\n * Finds all items in the conversation history that contain images (Google format)\n * @param items - Array of conversation items to check\n * @returns Array of indices where images were found\n */\nexport function findGoogleItemsWithImages(items: GoogleContent[]): number[] {\n  const itemsWithImages: number[] = [];\n\n  items.forEach((item, index) => {\n    let hasImage = false;\n\n    if (item.parts && Array.isArray(item.parts)) {\n      hasImage = item.parts.some((part: GooglePart) => {\n        // Check for functionResponse with data containing images\n        if (part.functionResponse?.response?.data) {\n          const data = part.functionResponse.response\n            .data as FunctionResponseData[];\n          return data.some((dataItem) =>\n            dataItem.inlineData?.mimeType?.startsWith(\"image/\"),\n          );\n        }\n\n        // Check for functionResponse with parts containing images\n        if (part.functionResponse?.parts) {\n          return part.functionResponse.parts.some((responsePart) =>\n            responsePart.inlineData?.mimeType?.startsWith(\"image/\"),\n          );\n        }\n\n        // Check for direct inline data\n        return part.inlineData?.mimeType?.startsWith(\"image/\");\n      });\n    }\n\n    if (hasImage) {\n      itemsWithImages.push(index);\n    }\n  });\n\n  return itemsWithImages;\n}\n\n/**\n * Finds all items in the conversation history that contain images (OpenAI format)\n * @param items - Array of conversation items to check\n * @returns Array of indices where images were found\n */\nexport function findOpenAIItemsWithImages(\n  items: OpenAIResponseInputItem[],\n): number[] {\n  const itemsWithImages: number[] = [];\n\n  items.forEach((item, index) => {\n    let hasImage = false;\n\n    // Check for computer_call_output with image\n    if (\n      \"type\" in item &&\n      item.type === \"computer_call_output\" &&\n      \"output\" in item\n    ) {\n      const output = item.output as unknown as {\n        type: string;\n        image_url: string;\n      };\n      hasImage = output?.type === \"input_image\" && !!output?.image_url;\n    }\n\n    if (hasImage) {\n      itemsWithImages.push(index);\n    }\n  });\n\n  return itemsWithImages;\n}\n\n/**\n * Compresses OpenAI conversation history by removing images from older items\n * while keeping the most recent images intact\n * @param items - Array of conversation items to process\n * @param keepMostRecentCount - Number of most recent image-containing items to preserve (default: 2)\n * @returns Object with processed items\n */\nexport function compressOpenAIConversationImages(\n  items: OpenAIResponseInputItem[],\n  keepMostRecentCount: number = 2,\n): { items: OpenAIResponseInputItem[] } {\n  const itemsWithImages = findOpenAIItemsWithImages(items);\n\n  items.forEach((item, index) => {\n    const imageIndex = itemsWithImages.indexOf(index);\n    const shouldCompress =\n      imageIndex >= 0 &&\n      imageIndex < itemsWithImages.length - keepMostRecentCount;\n\n    if (shouldCompress) {\n      // For computer_call_output with image, replace with text\n      if (\n        \"type\" in item &&\n        item.type === \"computer_call_output\" &&\n        \"output\" in item\n      ) {\n        const output = item.output as unknown as { type: string };\n        if (output?.type === \"input_image\") {\n          // Replace the image with a text message\n          (item as unknown as { output: string }).output = \"screenshot taken\";\n        }\n      }\n    }\n  });\n\n  return {\n    items,\n  };\n}\n\n/**\n * Compresses Google conversation history by removing images from older items\n * while keeping the most recent images intact\n * @param items - Array of conversation items to process\n * @param keepMostRecentCount - Number of most recent image-containing items to preserve (default: 2)\n * @returns Object with processed items\n */\nexport function compressGoogleConversationImages(\n  items: GoogleContent[],\n  keepMostRecentCount: number = 2,\n): { items: GoogleContent[] } {\n  const itemsWithImages = findGoogleItemsWithImages(items);\n\n  items.forEach((item, index) => {\n    const imageIndex = itemsWithImages.indexOf(index);\n    const shouldCompress =\n      imageIndex >= 0 &&\n      imageIndex < itemsWithImages.length - keepMostRecentCount;\n\n    if (shouldCompress && item.parts && Array.isArray(item.parts)) {\n      item.parts = item.parts.map((part: GooglePart) => {\n        // Replace functionResponse with data containing images\n        if (part.functionResponse?.response?.data) {\n          const data = part.functionResponse.response\n            .data as FunctionResponseData[];\n          const hasImage = data.some((dataItem) =>\n            dataItem.inlineData?.mimeType?.startsWith(\"image/\"),\n          );\n          if (hasImage) {\n            return {\n              ...part,\n              functionResponse: {\n                ...part.functionResponse,\n                data: [] as FunctionResponseData[],\n                response: {\n                  ...part.functionResponse.response,\n                  compressed: \"screenshot taken\",\n                },\n              },\n            };\n          }\n        }\n\n        // Replace functionResponse with parts containing images\n        if (part.functionResponse?.parts) {\n          const hasImageInParts = part.functionResponse.parts.some(\n            (responsePart) =>\n              responsePart.inlineData?.mimeType?.startsWith(\"image/\"),\n          );\n          if (hasImageInParts) {\n            return {\n              ...part,\n              functionResponse: {\n                ...part.functionResponse,\n                parts: part.functionResponse.parts.filter(\n                  (responsePart) =>\n                    !responsePart.inlineData?.mimeType?.startsWith(\"image/\"),\n                ),\n                response: {\n                  ...part.functionResponse.response,\n                  compressed: \"screenshot taken\",\n                },\n              },\n            };\n          }\n        }\n\n        // Replace direct inline data images\n        if (part.inlineData?.mimeType?.startsWith(\"image/\")) {\n          return {\n            text: \"screenshot taken\",\n          };\n        }\n        return part;\n      });\n    }\n  });\n\n  return {\n    items,\n  };\n}\n"]}

package/dist/cjs/lib/v3/agent/utils/messageProcessing.d.ts

@@ -0,0 +1,12 @@
+import type { ModelMessage } from "ai";
+/**
+ * Compress old screenshot/ariaTree data in messages in-place.
+ *
+ * Strategy:
+ * - Keep only the 2 most recent vision results (screenshots OR vision action tools like click/type/etc)
+ * - Keep only the 1 most recent ariaTree (replace older ones with placeholder)
+ *
+ * @param messages - The messages array to modify in-place
+ * @returns Number of items compressed
+ */
+export declare function processMessages(messages: ModelMessage[]): number;

package/dist/cjs/lib/v3/agent/utils/messageProcessing.js

@@ -0,0 +1,167 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.processMessages = processMessages;
+// Vision action tools that include screenshots in their results
+const VISION_ACTION_TOOLS = [
+    "click",
+    "type",
+    "dragAndDrop",
+    "wait",
+    "fillFormVision",
+    "scroll",
+];
+function isToolMessage(message) {
+    return (!!message &&
+        typeof message === "object" &&
+        message.role === "tool" &&
+        Array.isArray(message.content));
+}
+function isScreenshotPart(part) {
+    return (!!part &&
+        typeof part === "object" &&
+        part.toolName === "screenshot");
+}
+function isVisionActionPart(part) {
+    if (!part || typeof part !== "object")
+        return false;
+    const toolName = part.toolName;
+    return typeof toolName === "string" && VISION_ACTION_TOOLS.includes(toolName);
+}
+function isVisionPart(part) {
+    return isScreenshotPart(part) || isVisionActionPart(part);
+}
+function isAriaTreePart(part) {
+    return (!!part &&
+        typeof part === "object" &&
+        part.toolName === "ariaTree");
+}
+/**
+ * Compress old screenshot/ariaTree data in messages in-place.
+ *
+ * Strategy:
+ * - Keep only the 2 most recent vision results (screenshots OR vision action tools like click/type/etc)
+ * - Keep only the 1 most recent ariaTree (replace older ones with placeholder)
+ *
+ * @param messages - The messages array to modify in-place
+ * @returns Number of items compressed
+ */
+function processMessages(messages) {
+    let compressedCount = 0;
+    // Find indices of all vision-related tool results (screenshots + vision actions)
+    // and ariaTree results
+    const visionIndices = [];
+    const ariaTreeIndices = [];
+    for (let i = 0; i < messages.length; i++) {
+        const message = messages[i];
+        if (isToolMessage(message)) {
+            const content = message.content;
+            if (content.some(isVisionPart)) {
+                visionIndices.push(i);
+            }
+            if (content.some(isAriaTreePart)) {
+                ariaTreeIndices.push(i);
+            }
+        }
+    }
+    // Compress old vision results (keep 2 most recent across all vision tools)
+    if (visionIndices.length > 2) {
+        const toCompress = visionIndices.slice(0, visionIndices.length - 2);
+        for (const index of toCompress) {
+            const message = messages[index];
+            if (isToolMessage(message)) {
+                // Both functions are safe to call - they only modify their respective part types
+                compressScreenshotMessage(message);
+                compressVisionActionMessage(message);
+                compressedCount++;
+            }
+        }
+    }
+    // Compress old ariaTree results (keep 1 most recent)
+    if (ariaTreeIndices.length > 1) {
+        const toCompress = ariaTreeIndices.slice(0, ariaTreeIndices.length - 1);
+        for (const idx of toCompress) {
+            const message = messages[idx];
+            if (isToolMessage(message)) {
+                compressAriaTreeMessage(message);
+                compressedCount++;
+            }
+        }
+    }
+    return compressedCount;
+}
+/**
+ * Check if output has type "content" (array-based value format).
+ * Only outputs with type "content" should have array values.
+ */
+function isContentTypeOutput(output) {
+    return output.type === "content";
+}
+/**
+ * Compress screenshot message content in-place.
+ * Only modifies outputs with type "content" to maintain schema validity.
+ * Replaces entire output object to ensure type/value consistency.
+ */
+function compressScreenshotMessage(message) {
+    for (const part of message.content) {
+        if (isScreenshotPart(part)) {
+            const typedPart = part;
+            // Only compress if output exists and has type "content"
+            if (typedPart.output && isContentTypeOutput(typedPart.output)) {
+                // Replace entire output to ensure type/value consistency
+                typedPart.output = {
+                    type: "content",
+                    value: [{ type: "text", text: "screenshot taken" }],
+                };
+            }
+        }
+    }
+}
+/**
+ * Compress vision action message content in-place by removing the screenshot
+ * but keeping the action result text.
+ * Only modifies outputs with type "content" to maintain schema validity.
+ */
+function compressVisionActionMessage(message) {
+    for (const part of message.content) {
+        if (isVisionActionPart(part)) {
+            const typedPart = part;
+            // Only compress if output is type "content" (array-based value)
+            if (typedPart.output &&
+                isContentTypeOutput(typedPart.output) &&
+                Array.isArray(typedPart.output.value)) {
+                // Filter out media content but keep text results
+                const filteredValue = typedPart.output.value.filter((item) => item && typeof item === "object" && item.type !== "media");
+                // Replace entire output to ensure type/value consistency
+                typedPart.output = {
+                    type: "content",
+                    value: filteredValue,
+                };
+            }
+        }
+    }
+}
+/**
+ * Compress ariaTree message content in-place.
+ * Only modifies outputs with type "content" to maintain schema validity.
+ * Replaces entire output object to ensure type/value consistency.
+ */
+function compressAriaTreeMessage(message) {
+    for (const part of message.content) {
+        if (isAriaTreePart(part)) {
+            const typedPart = part;
+            // Only compress if output exists and has type "content"
+            if (typedPart.output && isContentTypeOutput(typedPart.output)) {
+                typedPart.output = {
+                    type: "content",
+                    value: [
+                        {
+                            type: "text",
+                            text: "ARIA tree extracted for context of page elements",
+                        },
+                    ],
+                };
+            }
+        }
+    }
+}
+//# sourceMappingURL=messageProcessing.js.map

package/dist/cjs/lib/v3/agent/utils/messageProcessing.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"messageProcessing.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/messageProcessing.ts"],"names":[],"mappings":";;AA2DA,0CAgDC;AAzGD,gEAAgE;AAChE,MAAM,mBAAmB,GAAG;IAC1B,OAAO;IACP,MAAM;IACN,aAAa;IACb,MAAM;IACN,gBAAgB;IAChB,QAAQ;CACT,CAAC;AAEF,SAAS,aAAa,CACpB,OAAgB;IAEhB,OAAO,CACL,CAAC,CAAC,OAAO;QACT,OAAO,OAAO,KAAK,QAAQ;QAC1B,OAA8B,CAAC,IAAI,KAAK,MAAM;QAC/C,KAAK,CAAC,OAAO,CAAE,OAAiC,CAAC,OAAO,CAAC,CAC1D,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAa;IACrC,OAAO,CACL,CAAC,CAAC,IAAI;QACN,OAAO,IAAI,KAAK,QAAQ;QACvB,IAA+B,CAAC,QAAQ,KAAK,YAAY,CAC3D,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAa;IACvC,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACpD,MAAM,QAAQ,GAAI,IAA+B,CAAC,QAAQ,CAAC;IAC3D,OAAO,OAAO,QAAQ,KAAK,QAAQ,IAAI,mBAAmB,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,YAAY,CAAC,IAAa;IACjC,OAAO,gBAAgB,CAAC,IAAI,CAAC,IAAI,kBAAkB,CAAC,IAAI,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,cAAc,CAAC,IAAa;IACnC,OAAO,CACL,CAAC,CAAC,IAAI;QACN,OAAO,IAAI,KAAK,QAAQ;QACvB,IAA+B,CAAC,QAAQ,KAAK,UAAU,CACzD,CAAC;AACJ,CAAC;AAED;;;;;;;;;GASG;AACH,SAAgB,eAAe,CAAC,QAAwB;IACtD,IAAI,eAAe,GAAG,CAAC,CAAC;IAExB,iFAAiF;IACjF,uBAAuB;IACvB,MAAM,aAAa,GAAa,EAAE,CAAC;IACnC,MAAM,eAAe,GAAa,EAAE,CAAC;IAErC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAoB,CAAC;YAC7C,IAAI,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;gBAC/B,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACxB,CAAC;YACD,IAAI,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;gBACjC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAED,2EAA2E;IAC3E,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,MAAM,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACpE,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YAC/B,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;YAChC,IAAI,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC3B,iFAAiF;gBACjF,yBAAyB,CAAC,OAAO,CAAC,CAAC;gBACnC,2BAA2B,CAAC,OAAO,CAAC,CAAC;gBACrC,eAAe,EAAE,CAAC;YACpB,CAAC;QACH,CAAC;IACH,CAAC;IAED,qDAAqD;IACrD,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/B,MAAM,UAAU,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACxE,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;YAC9B,IAAI,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC3B,uBAAuB,CAAC,OAAO,CAAC,CAAC;gBACjC,eAAe,EAAE,CAAC;YACpB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,eAAe,CAAC;AACzB,CAAC;AAkBD;;;GAGG;AACH,SAAS,mBAAmB,CAAC,MAG5B;IACC,OAAO,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC;AACnC,CAAC;AAED;;;;GAIG;AACH,SAAS,yBAAyB,CAAC,OAGlC;IACC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACnC,IAAI,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,IAAsB,CAAC;YACzC,wDAAwD;YACxD,IAAI,SAAS,CAAC,MAAM,IAAI,mBAAmB,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9D,yDAAyD;gBACzD,SAAS,CAAC,MAAM,GAAG;oBACjB,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,kBAAkB,EAAE,CAAC;iBACpD,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,2BAA2B,CAAC,OAGpC;IACC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACnC,IAAI,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,IAAsB,CAAC;YAEzC,gEAAgE;YAChE,IACE,SAAS,CAAC,MAAM;gBAChB,mBAAmB,CAAC,SAAS,CAAC,MAAM,CAAC;gBACrC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,EACrC,CAAC;gBACD,iDAAiD;gBACjD,MAAM,aAAa,GACjB,SAAS,CAAC,MAAM,CAAC,KAClB,CAAC,MAAM,CACN,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,CACpE,CAAC;gBACF,yDAAyD;gBACzD,SAAS,CAAC,MAAM,GAAG;oBACjB,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,aAAa;iBACrB,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,uBAAuB,CAAC,OAGhC;IACC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACnC,IAAI,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,MAAM,SAAS,GAAG,IAAsB,CAAC;YACzC,wDAAwD;YACxD,IAAI,SAAS,CAAC,MAAM,IAAI,mBAAmB,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9D,SAAS,CAAC,MAAM,GAAG;oBACjB,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE;wBACL;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,kDAAkD;yBACzD;qBACF;iBACF,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC","sourcesContent":["import type { ModelMessage } from \"ai\";\n\n// Vision action tools that include screenshots in their results\nconst VISION_ACTION_TOOLS = [\n  \"click\",\n  \"type\",\n  \"dragAndDrop\",\n  \"wait\",\n  \"fillFormVision\",\n  \"scroll\",\n];\n\nfunction isToolMessage(\n  message: unknown,\n): message is { role: \"tool\"; content: unknown[] } {\n  return (\n    !!message &&\n    typeof message === \"object\" &&\n    (message as { role?: unknown }).role === \"tool\" &&\n    Array.isArray((message as { content?: unknown }).content)\n  );\n}\n\nfunction isScreenshotPart(part: unknown): boolean {\n  return (\n    !!part &&\n    typeof part === \"object\" &&\n    (part as { toolName?: unknown }).toolName === \"screenshot\"\n  );\n}\n\nfunction isVisionActionPart(part: unknown): boolean {\n  if (!part || typeof part !== \"object\") return false;\n  const toolName = (part as { toolName?: unknown }).toolName;\n  return typeof toolName === \"string\" && VISION_ACTION_TOOLS.includes(toolName);\n}\n\nfunction isVisionPart(part: unknown): boolean {\n  return isScreenshotPart(part) || isVisionActionPart(part);\n}\n\nfunction isAriaTreePart(part: unknown): boolean {\n  return (\n    !!part &&\n    typeof part === \"object\" &&\n    (part as { toolName?: unknown }).toolName === \"ariaTree\"\n  );\n}\n\n/**\n * Compress old screenshot/ariaTree data in messages in-place.\n *\n * Strategy:\n * - Keep only the 2 most recent vision results (screenshots OR vision action tools like click/type/etc)\n * - Keep only the 1 most recent ariaTree (replace older ones with placeholder)\n *\n * @param messages - The messages array to modify in-place\n * @returns Number of items compressed\n */\nexport function processMessages(messages: ModelMessage[]): number {\n  let compressedCount = 0;\n\n  // Find indices of all vision-related tool results (screenshots + vision actions)\n  // and ariaTree results\n  const visionIndices: number[] = [];\n  const ariaTreeIndices: number[] = [];\n\n  for (let i = 0; i < messages.length; i++) {\n    const message = messages[i];\n    if (isToolMessage(message)) {\n      const content = message.content as unknown[];\n      if (content.some(isVisionPart)) {\n        visionIndices.push(i);\n      }\n      if (content.some(isAriaTreePart)) {\n        ariaTreeIndices.push(i);\n      }\n    }\n  }\n\n  // Compress old vision results (keep 2 most recent across all vision tools)\n  if (visionIndices.length > 2) {\n    const toCompress = visionIndices.slice(0, visionIndices.length - 2);\n    for (const index of toCompress) {\n      const message = messages[index];\n      if (isToolMessage(message)) {\n        // Both functions are safe to call - they only modify their respective part types\n        compressScreenshotMessage(message);\n        compressVisionActionMessage(message);\n        compressedCount++;\n      }\n    }\n  }\n\n  // Compress old ariaTree results (keep 1 most recent)\n  if (ariaTreeIndices.length > 1) {\n    const toCompress = ariaTreeIndices.slice(0, ariaTreeIndices.length - 1);\n    for (const idx of toCompress) {\n      const message = messages[idx];\n      if (isToolMessage(message)) {\n        compressAriaTreeMessage(message);\n        compressedCount++;\n      }\n    }\n  }\n\n  return compressedCount;\n}\n\n/**\n * Tool result part structure from AI SDK.\n * The output field uses a discriminated union - type determines value format:\n * - type: \"content\" -> value: Array<{type: \"text\", ...} | {type: \"media\", ...}>\n * - type: \"text\" -> value: string\n * - type: \"json\" -> value: JSONValue\n * - type: \"error-text\" -> value: string\n * - type: \"error-json\" -> value: JSONValue\n */\ninterface ToolResultPart {\n  output?: {\n    type: string;\n    value?: unknown;\n  };\n}\n\n/**\n * Check if output has type \"content\" (array-based value format).\n * Only outputs with type \"content\" should have array values.\n */\nfunction isContentTypeOutput(output: {\n  type: string;\n  value?: unknown;\n}): boolean {\n  return output.type === \"content\";\n}\n\n/**\n * Compress screenshot message content in-place.\n * Only modifies outputs with type \"content\" to maintain schema validity.\n * Replaces entire output object to ensure type/value consistency.\n */\nfunction compressScreenshotMessage(message: {\n  role: \"tool\";\n  content: unknown[];\n}): void {\n  for (const part of message.content) {\n    if (isScreenshotPart(part)) {\n      const typedPart = part as ToolResultPart;\n      // Only compress if output exists and has type \"content\"\n      if (typedPart.output && isContentTypeOutput(typedPart.output)) {\n        // Replace entire output to ensure type/value consistency\n        typedPart.output = {\n          type: \"content\",\n          value: [{ type: \"text\", text: \"screenshot taken\" }],\n        };\n      }\n    }\n  }\n}\n\n/**\n * Compress vision action message content in-place by removing the screenshot\n * but keeping the action result text.\n * Only modifies outputs with type \"content\" to maintain schema validity.\n */\nfunction compressVisionActionMessage(message: {\n  role: \"tool\";\n  content: unknown[];\n}): void {\n  for (const part of message.content) {\n    if (isVisionActionPart(part)) {\n      const typedPart = part as ToolResultPart;\n\n      // Only compress if output is type \"content\" (array-based value)\n      if (\n        typedPart.output &&\n        isContentTypeOutput(typedPart.output) &&\n        Array.isArray(typedPart.output.value)\n      ) {\n        // Filter out media content but keep text results\n        const filteredValue = (\n          typedPart.output.value as Array<{ type?: string }>\n        ).filter(\n          (item) => item && typeof item === \"object\" && item.type !== \"media\",\n        );\n        // Replace entire output to ensure type/value consistency\n        typedPart.output = {\n          type: \"content\",\n          value: filteredValue,\n        };\n      }\n    }\n  }\n}\n\n/**\n * Compress ariaTree message content in-place.\n * Only modifies outputs with type \"content\" to maintain schema validity.\n * Replaces entire output object to ensure type/value consistency.\n */\nfunction compressAriaTreeMessage(message: {\n  role: \"tool\";\n  content: unknown[];\n}): void {\n  for (const part of message.content) {\n    if (isAriaTreePart(part)) {\n      const typedPart = part as ToolResultPart;\n      // Only compress if output exists and has type \"content\"\n      if (typedPart.output && isContentTypeOutput(typedPart.output)) {\n        typedPart.output = {\n          type: \"content\",\n          value: [\n            {\n              type: \"text\",\n              text: \"ARIA tree extracted for context of page elements\",\n            },\n          ],\n        };\n      }\n    }\n  }\n}\n"]}

package/dist/cjs/lib/v3/agent/utils/screenshotHandler.d.ts

@@ -0,0 +1,10 @@
+import type { Page } from "../../understudy/page.js";
+/**
+ * Waits for the page to settle and captures a screenshot.
+ * If the screenshot fails (e.g., page closed, navigation in progress),
+ * returns undefined instead of throwing - allowing the action to still succeed.
+ *
+ * @param page - The page to capture
+ * @param delayMs - Delay before capturing (default: 500ms, pass 0 to skip delay)
+ */
+export declare function waitAndCaptureScreenshot(page: Page, delayMs?: number): Promise<string | undefined>;