@browserbasehq/orca 3.4.0-preview.0 → 3.5.0-preview.1

package/dist/esm/lib/v3LegacyEvaluator.js

@@ -0,0 +1,215 @@
+/**
+ * Legacy V3 evaluator implementation.
+ *
+ * This is the behavior-preserving implementation that backs V3Evaluator when
+ * STAGEHAND_EVALUATOR_BACKEND=legacy.
+ */
+import { z } from "zod";
+import { LLMProvider } from "./v3/llm/LLMProvider.js";
+import { StagehandInvalidArgumentError } from "./v3/types/public/sdkErrors.js";
+const EvaluationSchema = z.object({
+    evaluation: z.enum(["YES", "NO"]),
+    reasoning: z.string(),
+});
+const BatchEvaluationSchema = z.array(EvaluationSchema);
+export class LegacyV3Evaluator {
+    v3;
+    modelName;
+    modelClientOptions;
+    silentLogger = () => { };
+    constructor(v3, modelName, modelClientOptions) {
+        this.v3 = v3;
+        this.modelName = modelName || "google/gemini-2.5-flash";
+        this.modelClientOptions = modelClientOptions || {
+            apiKey: process.env.GEMINI_API_KEY ||
+                process.env.GOOGLE_GENERATIVE_AI_API_KEY ||
+                "",
+        };
+    }
+    getClient() {
+        // Prefer a dedicated provider so we can override model per-evaluation
+        const provider = new LLMProvider(this.v3.logger);
+        return provider.getClient(this.modelName, this.modelClientOptions);
+    }
+    async ask(options) {
+        const { question, answer, screenshot = true, systemPrompt, screenshotDelayMs = 250, agentReasoning, } = options;
+        if (!question)
+            throw new StagehandInvalidArgumentError("Question cannot be an empty string");
+        if (!answer && !screenshot)
+            throw new StagehandInvalidArgumentError("Either answer (text) or screenshot must be provided");
+        if (Array.isArray(screenshot)) {
+            return this._evaluateWithMultipleScreenshots({
+                question,
+                answer,
+                screenshots: screenshot,
+                systemPrompt,
+                agentReasoning,
+            });
+        }
+        const defaultSystemPrompt = `You are an expert evaluator that confidently returns YES or NO based on if the original goal was achieved. You have access to  ${screenshot ? "a screenshot" : "the agents reasoning and actions throughout the task"} that you can use to evaluate the tasks completion. Provide detailed reasoning for your answer.\n          Today's date is ${new Date().toLocaleDateString()}`;
+        await new Promise((r) => setTimeout(r, screenshotDelayMs));
+        let imageBuffer;
+        if (screenshot) {
+            const page = await this.v3.context.awaitActivePage();
+            imageBuffer = await page.screenshot({ fullPage: false });
+        }
+        const llmClient = this.getClient();
+        const response = await llmClient.createChatCompletion({
+            logger: this.silentLogger,
+            options: {
+                messages: [
+                    { role: "system", content: systemPrompt || defaultSystemPrompt },
+                    {
+                        role: "user",
+                        content: [
+                            {
+                                type: "text",
+                                text: agentReasoning
+                                    ? `Question: ${question}\n\nAgent's reasoning and actions taken:\n${agentReasoning}`
+                                    : question,
+                            },
+                            ...(screenshot && imageBuffer
+                                ? [
+                                    {
+                                        type: "image_url",
+                                        image_url: {
+                                            url: `data:image/jpeg;base64,${imageBuffer.toString("base64")}`,
+                                        },
+                                    },
+                                ]
+                                : []),
+                            ...(answer
+                                ? [{ type: "text", text: `the answer is ${answer}` }]
+                                : []),
+                        ],
+                    },
+                ],
+                response_model: { name: "EvaluationResult", schema: EvaluationSchema },
+            },
+        });
+        try {
+            const result = response.data;
+            return { evaluation: result.evaluation, reasoning: result.reasoning };
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            return {
+                evaluation: "INVALID",
+                reasoning: `Failed to get structured response: ${errorMessage}`,
+            };
+        }
+    }
+    async batchAsk(options) {
+        const { questions, screenshot = true, systemPrompt = "You are an expert evaluator that returns YES or NO with a concise reasoning.", screenshotDelayMs = 250, } = options;
+        if (!questions?.length)
+            throw new StagehandInvalidArgumentError("Questions array cannot be empty");
+        await new Promise((r) => setTimeout(r, screenshotDelayMs));
+        let imageBuffer;
+        if (screenshot) {
+            const page = await this.v3.context.awaitActivePage();
+            imageBuffer = await page.screenshot({ fullPage: false });
+        }
+        const llmClient = this.getClient();
+        const formatted = questions
+            .map((item, i) => `${i + 1}. ${item.question}${item.answer ? `\n   Answer: ${item.answer}` : ""}`)
+            .join("\n\n");
+        const response = await llmClient.createChatCompletion({
+            logger: this.silentLogger,
+            options: {
+                messages: [
+                    {
+                        role: "system",
+                        content: `${systemPrompt}\n\nYou will be given multiple questions${screenshot ? " with a screenshot" : ""}. ${questions.some((q) => q.answer) ? "Some questions include answers to evaluate." : ""} Answer each question by returning an object in the specified JSON format. Return a single JSON array containing one object for each question in the order they were asked.`,
+                    },
+                    {
+                        role: "user",
+                        content: [
+                            { type: "text", text: formatted },
+                            ...(screenshot && imageBuffer
+                                ? [
+                                    {
+                                        type: "image_url",
+                                        image_url: {
+                                            url: `data:image/jpeg;base64,${imageBuffer.toString("base64")}`,
+                                        },
+                                    },
+                                ]
+                                : []),
+                        ],
+                    },
+                ],
+                response_model: {
+                    name: "BatchEvaluationResult",
+                    schema: BatchEvaluationSchema,
+                },
+            },
+        });
+        try {
+            const results = response.data;
+            return results.map((r) => ({
+                evaluation: r.evaluation,
+                reasoning: r.reasoning,
+            }));
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            return questions.map(() => ({
+                evaluation: "INVALID",
+                reasoning: `Failed to get structured response: ${errorMessage}`,
+            }));
+        }
+    }
+    async _evaluateWithMultipleScreenshots(options) {
+        const { question, answer, screenshots, agentReasoning, systemPrompt = `You are an expert evaluator that confidently returns YES or NO given a question and multiple screenshots showing the progression of a task.
+        ${agentReasoning ? "You also have access to the agent's detailed reasoning and thought process throughout the task." : ""}
+        Analyze ALL screenshots to understand the complete journey. Look for evidence of task completion across all screenshots, not just the last one.
+        Success criteria may appear at different points in the sequence (confirmation messages, intermediate states, etc).
+        ${agentReasoning ? "The agent's reasoning provides crucial context about what actions were attempted, what was observed, and the decision-making process. Use this alongside the visual evidence to make a comprehensive evaluation." : ""}
+        Today's date is ${new Date().toLocaleDateString()}`, } = options;
+        if (!question)
+            throw new StagehandInvalidArgumentError("Question cannot be an empty string");
+        if (!screenshots || screenshots.length === 0)
+            throw new StagehandInvalidArgumentError("At least one screenshot must be provided");
+        const llmClient = this.getClient();
+        const imageContents = screenshots.map((s) => ({
+            type: "image_url",
+            image_url: { url: `data:image/jpeg;base64,${s.toString("base64")}` },
+        }));
+        const response = await llmClient.createChatCompletion({
+            logger: this.silentLogger,
+            options: {
+                messages: [
+                    { role: "system", content: systemPrompt },
+                    {
+                        role: "user",
+                        content: [
+                            {
+                                type: "text",
+                                text: agentReasoning
+                                    ? `Question: ${question}\n\nAgent's reasoning and actions throughout the task:\n${agentReasoning}\n\nI'm providing ${screenshots.length} screenshots showing the progression of the task. Please analyze both the agent's reasoning and all screenshots to determine if the task was completed successfully.`
+                                    : `${question}\n\nI'm providing ${screenshots.length} screenshots showing the progression of the task. Please analyze all of them to determine if the task was completed successfully.`,
+                            },
+                            ...(answer
+                                ? [{ type: "text", text: `the answer is ${answer}` }]
+                                : []),
+                            ...imageContents,
+                        ],
+                    },
+                ],
+                response_model: { name: "EvaluationResult", schema: EvaluationSchema },
+            },
+        });
+        try {
+            const result = response.data;
+            return { evaluation: result.evaluation, reasoning: result.reasoning };
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            return {
+                evaluation: "INVALID",
+                reasoning: `Failed to get structured response: ${errorMessage}`,
+            };
+        }
+    }
+}
+//# sourceMappingURL=v3LegacyEvaluator.js.map

package/dist/esm/lib/v3LegacyEvaluator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"v3LegacyEvaluator.js","sourceRoot":"","sources":["../../../lib/v3LegacyEvaluator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAWxB,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,6BAA6B,EAAE,MAAM,gCAAgC,CAAC;AAE/E,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IACjC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;CACtB,CAAC,CAAC;AAEH,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;AAExD,MAAM,OAAO,iBAAiB;IACpB,EAAE,CAAK;IACP,SAAS,CAAiB;IAC1B,kBAAkB,CAAqC;IACvD,YAAY,GAA+B,GAAG,EAAE,GAAE,CAAC,CAAC;IAE5D,YACE,EAAM,EACN,SAA0B,EAC1B,kBAAkC;QAElC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,SAAS,GAAG,SAAS,IAAK,yBAA4C,CAAC;QAC5E,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,IAAI;YAC9C,MAAM,EACJ,OAAO,CAAC,GAAG,CAAC,cAAc;gBAC1B,OAAO,CAAC,GAAG,CAAC,4BAA4B;gBACxC,EAAE;SACL,CAAC;IACJ,CAAC;IAEO,SAAS;QACf,sEAAsE;QACtE,MAAM,QAAQ,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;QACjD,OAAO,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACrE,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,OAAwB;QAChC,MAAM,EACJ,QAAQ,EACR,MAAM,EACN,UAAU,GAAG,IAAI,EACjB,YAAY,EACZ,iBAAiB,GAAG,GAAG,EACvB,cAAc,GACf,GAAG,OAAO,CAAC;QACZ,IAAI,CAAC,QAAQ;YACX,MAAM,IAAI,6BAA6B,CACrC,oCAAoC,CACrC,CAAC;QACJ,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU;YACxB,MAAM,IAAI,6BAA6B,CACrC,qDAAqD,CACtD,CAAC;QAEJ,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,gCAAgC,CAAC;gBAC3C,QAAQ;gBACR,MAAM;gBACN,WAAW,EAAE,UAAU;gBACvB,YAAY;gBACZ,cAAc;aACf,CAAC,CAAC;QACL,CAAC;QAED,MAAM,mBAAmB,GAAG,kIAAkI,UAAU,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,sDAAsD,8HAA8H,IAAI,IAAI,EAAE,CAAC,kBAAkB,EAAE,EAAE,CAAC;QAElZ,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC;QAC3D,IAAI,WAA+B,CAAC;QACpC,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YACrD,WAAW,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEnC,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAEnD;YACA,MAAM,EAAE,IAAI,CAAC,YAAY;YACzB,OAAO,EAAE;gBACP,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,IAAI,mBAAmB,EAAE;oBAChE;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE;4BACP;gCACE,IAAI,EAAE,MAAM;gCACZ,IAAI,EAAE,cAAc;oCAClB,CAAC,CAAC,aAAa,QAAQ,6CAA6C,cAAc,EAAE;oCACpF,CAAC,CAAC,QAAQ;6BACb;4BACD,GAAG,CAAC,UAAU,IAAI,WAAW;gCAC3B,CAAC,CAAC;oCACE;wCACE,IAAI,EAAE,WAAoB;wCAC1B,SAAS,EAAE;4CACT,GAAG,EAAE,0BAA0B,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;yCAChE;qCACF;iCACF;gCACH,CAAC,CAAC,EAAE,CAAC;4BACP,GAAG,CAAC,MAAM;gCACR,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,iBAAiB,MAAM,EAAE,EAAE,CAAC;gCAC9D,CAAC,CAAC,EAAE,CAAC;yBACR;qBACF;iBACF;gBACD,cAAc,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,gBAAgB,EAAE;aACvE;SACF,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,QAAQ,CAAC,IAEvB,CAAC;YACF,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;QACxE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,OAAO;gBACL,UAAU,EAAE,SAAS;gBACrB,SAAS,EAAE,sCAAsC,YAAY,EAAE;aACvD,CAAC;QACb,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAAwB;QACrC,MAAM,EACJ,SAAS,EACT,UAAU,GAAG,IAAI,EACjB,YAAY,GAAG,8EAA8E,EAC7F,iBAAiB,GAAG,GAAG,GACxB,GAAG,OAAO,CAAC;QACZ,IAAI,CAAC,SAAS,EAAE,MAAM;YACpB,MAAM,IAAI,6BAA6B,CACrC,iCAAiC,CAClC,CAAC;QAEJ,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC;QAC3D,IAAI,WAA+B,CAAC;QACpC,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YACrD,WAAW,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEnC,MAAM,SAAS,GAAG,SAAS;aACxB,GAAG,CACF,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CACV,GAAG,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,gBAAgB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAClF;aACA,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhB,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAEnD;YACA,MAAM,EAAE,IAAI,CAAC,YAAY;YACzB,OAAO,EAAE;gBACP,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,QAAQ;wBACd,OAAO,EAAE,GAAG,YAAY,2CAA2C,UAAU,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,6CAA6C,CAAC,CAAC,CAAC,EAAE,6KAA6K;qBAChX;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE;4BACP,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE;4BACjC,GAAG,CAAC,UAAU,IAAI,WAAW;gCAC3B,CAAC,CAAC;oCACE;wCACE,IAAI,EAAE,WAAoB;wCAC1B,SAAS,EAAE;4CACT,GAAG,EAAE,0BAA0B,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;yCAChE;qCACF;iCACF;gCACH,CAAC,CAAC,EAAE,CAAC;yBACR;qBACF;iBACF;gBACD,cAAc,EAAE;oBACd,IAAI,EAAE,uBAAuB;oBAC7B,MAAM,EAAE,qBAAqB;iBAC9B;aACF;SACF,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,QAAQ,CAAC,IAExB,CAAC;YACF,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACzB,UAAU,EAAE,CAAC,CAAC,UAAU;gBACxB,SAAS,EAAE,CAAC,CAAC,SAAS;aACvB,CAAC,CAAC,CAAC;QACN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,OAAO,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;gBAC1B,UAAU,EAAE,SAAkB;gBAC9B,SAAS,EAAE,sCAAsC,YAAY,EAAE;aAChE,CAAC,CAAC,CAAC;QACN,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gCAAgC,CAAC,OAM9C;QACC,MAAM,EACJ,QAAQ,EACR,MAAM,EACN,WAAW,EACX,cAAc,EACd,YAAY,GAAG;UACX,cAAc,CAAC,CAAC,CAAC,iGAAiG,CAAC,CAAC,CAAC,EAAE;;;UAGvH,cAAc,CAAC,CAAC,CAAC,kNAAkN,CAAC,CAAC,CAAC,EAAE;0BACxN,IAAI,IAAI,EAAE,CAAC,kBAAkB,EAAE,EAAE,GACtD,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,QAAQ;YACX,MAAM,IAAI,6BAA6B,CACrC,oCAAoC,CACrC,CAAC;QACJ,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;YAC1C,MAAM,IAAI,6BAA6B,CACrC,0CAA0C,CAC3C,CAAC;QAEJ,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEnC,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC5C,IAAI,EAAE,WAAoB;YAC1B,SAAS,EAAE,EAAE,GAAG,EAAE,0BAA0B,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,EAAE;SACrE,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAEnD;YACA,MAAM,EAAE,IAAI,CAAC,YAAY;YACzB,OAAO,EAAE;gBACP,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE;oBACzC;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE;4BACP;gCACE,IAAI,EAAE,MAAM;gCACZ,IAAI,EAAE,cAAc;oCAClB,CAAC,CAAC,aAAa,QAAQ,2DAA2D,cAAc,qBAAqB,WAAW,CAAC,MAAM,sKAAsK;oCAC7S,CAAC,CAAC,GAAG,QAAQ,qBAAqB,WAAW,CAAC,MAAM,mIAAmI;6BAC1L;4BACD,GAAG,CAAC,MAAM;gCACR,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,iBAAiB,MAAM,EAAE,EAAE,CAAC;gCAC9D,CAAC,CAAC,EAAE,CAAC;4BACP,GAAG,aAAa;yBACjB;qBACF;iBACF;gBACD,cAAc,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,gBAAgB,EAAE;aACvE;SACF,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,QAAQ,CAAC,IAEvB,CAAC;YACF,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;QACxE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,OAAO;gBACL,UAAU,EAAE,SAAS;gBACrB,SAAS,EAAE,sCAAsC,YAAY,EAAE;aACvD,CAAC;QACb,CAAC;IACH,CAAC;CACF","sourcesContent":["/**\n * Legacy V3 evaluator implementation.\n *\n * This is the behavior-preserving implementation that backs V3Evaluator when\n * STAGEHAND_EVALUATOR_BACKEND=legacy.\n */\n\nimport { z } from \"zod\";\nimport type { AvailableModel, ClientOptions } from \"./v3/types/public/model.js\";\nimport type {\n  EvaluateOptions,\n  BatchAskOptions,\n  EvaluationResult,\n} from \"./v3/types/private/evaluator.js\";\nimport { LLMParsedResponse } from \"./inference.js\";\nimport { LLMResponse, LLMClient } from \"./v3/llm/LLMClient.js\";\nimport { LogLine } from \"./v3/types/public/logs.js\";\nimport { V3 } from \"./v3/v3.js\";\nimport { LLMProvider } from \"./v3/llm/LLMProvider.js\";\nimport { StagehandInvalidArgumentError } from \"./v3/types/public/sdkErrors.js\";\n\nconst EvaluationSchema = z.object({\n  evaluation: z.enum([\"YES\", \"NO\"]),\n  reasoning: z.string(),\n});\n\nconst BatchEvaluationSchema = z.array(EvaluationSchema);\n\nexport class LegacyV3Evaluator {\n  private v3: V3;\n  private modelName: AvailableModel;\n  private modelClientOptions: ClientOptions | { apiKey: string };\n  private silentLogger: (message: LogLine) => void = () => {};\n\n  constructor(\n    v3: V3,\n    modelName?: AvailableModel,\n    modelClientOptions?: ClientOptions,\n  ) {\n    this.v3 = v3;\n    this.modelName = modelName || (\"google/gemini-2.5-flash\" as AvailableModel);\n    this.modelClientOptions = modelClientOptions || {\n      apiKey:\n        process.env.GEMINI_API_KEY ||\n        process.env.GOOGLE_GENERATIVE_AI_API_KEY ||\n        \"\",\n    };\n  }\n\n  private getClient(): LLMClient {\n    // Prefer a dedicated provider so we can override model per-evaluation\n    const provider = new LLMProvider(this.v3.logger);\n    return provider.getClient(this.modelName, this.modelClientOptions);\n  }\n\n  async ask(options: EvaluateOptions): Promise<EvaluationResult> {\n    const {\n      question,\n      answer,\n      screenshot = true,\n      systemPrompt,\n      screenshotDelayMs = 250,\n      agentReasoning,\n    } = options;\n    if (!question)\n      throw new StagehandInvalidArgumentError(\n        \"Question cannot be an empty string\",\n      );\n    if (!answer && !screenshot)\n      throw new StagehandInvalidArgumentError(\n        \"Either answer (text) or screenshot must be provided\",\n      );\n\n    if (Array.isArray(screenshot)) {\n      return this._evaluateWithMultipleScreenshots({\n        question,\n        answer,\n        screenshots: screenshot,\n        systemPrompt,\n        agentReasoning,\n      });\n    }\n\n    const defaultSystemPrompt = `You are an expert evaluator that confidently returns YES or NO based on if the original goal was achieved. You have access to  ${screenshot ? \"a screenshot\" : \"the agents reasoning and actions throughout the task\"} that you can use to evaluate the tasks completion. Provide detailed reasoning for your answer.\\n          Today's date is ${new Date().toLocaleDateString()}`;\n\n    await new Promise((r) => setTimeout(r, screenshotDelayMs));\n    let imageBuffer: Buffer | undefined;\n    if (screenshot) {\n      const page = await this.v3.context.awaitActivePage();\n      imageBuffer = await page.screenshot({ fullPage: false });\n    }\n\n    const llmClient = this.getClient();\n\n    const response = await llmClient.createChatCompletion<\n      LLMParsedResponse<LLMResponse>\n    >({\n      logger: this.silentLogger,\n      options: {\n        messages: [\n          { role: \"system\", content: systemPrompt || defaultSystemPrompt },\n          {\n            role: \"user\",\n            content: [\n              {\n                type: \"text\",\n                text: agentReasoning\n                  ? `Question: ${question}\\n\\nAgent's reasoning and actions taken:\\n${agentReasoning}`\n                  : question,\n              },\n              ...(screenshot && imageBuffer\n                ? [\n                    {\n                      type: \"image_url\" as const,\n                      image_url: {\n                        url: `data:image/jpeg;base64,${imageBuffer.toString(\"base64\")}`,\n                      },\n                    },\n                  ]\n                : []),\n              ...(answer\n                ? [{ type: \"text\" as const, text: `the answer is ${answer}` }]\n                : []),\n            ],\n          },\n        ],\n        response_model: { name: \"EvaluationResult\", schema: EvaluationSchema },\n      },\n    });\n\n    try {\n      const result = response.data as unknown as z.infer<\n        typeof EvaluationSchema\n      >;\n      return { evaluation: result.evaluation, reasoning: result.reasoning };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return {\n        evaluation: \"INVALID\",\n        reasoning: `Failed to get structured response: ${errorMessage}`,\n      } as const;\n    }\n  }\n\n  async batchAsk(options: BatchAskOptions): Promise<EvaluationResult[]> {\n    const {\n      questions,\n      screenshot = true,\n      systemPrompt = \"You are an expert evaluator that returns YES or NO with a concise reasoning.\",\n      screenshotDelayMs = 250,\n    } = options;\n    if (!questions?.length)\n      throw new StagehandInvalidArgumentError(\n        \"Questions array cannot be empty\",\n      );\n\n    await new Promise((r) => setTimeout(r, screenshotDelayMs));\n    let imageBuffer: Buffer | undefined;\n    if (screenshot) {\n      const page = await this.v3.context.awaitActivePage();\n      imageBuffer = await page.screenshot({ fullPage: false });\n    }\n\n    const llmClient = this.getClient();\n\n    const formatted = questions\n      .map(\n        (item, i) =>\n          `${i + 1}. ${item.question}${item.answer ? `\\n   Answer: ${item.answer}` : \"\"}`,\n      )\n      .join(\"\\n\\n\");\n\n    const response = await llmClient.createChatCompletion<\n      LLMParsedResponse<LLMResponse>\n    >({\n      logger: this.silentLogger,\n      options: {\n        messages: [\n          {\n            role: \"system\",\n            content: `${systemPrompt}\\n\\nYou will be given multiple questions${screenshot ? \" with a screenshot\" : \"\"}. ${questions.some((q) => q.answer) ? \"Some questions include answers to evaluate.\" : \"\"} Answer each question by returning an object in the specified JSON format. Return a single JSON array containing one object for each question in the order they were asked.`,\n          },\n          {\n            role: \"user\",\n            content: [\n              { type: \"text\", text: formatted },\n              ...(screenshot && imageBuffer\n                ? [\n                    {\n                      type: \"image_url\" as const,\n                      image_url: {\n                        url: `data:image/jpeg;base64,${imageBuffer.toString(\"base64\")}`,\n                      },\n                    },\n                  ]\n                : []),\n            ],\n          },\n        ],\n        response_model: {\n          name: \"BatchEvaluationResult\",\n          schema: BatchEvaluationSchema,\n        },\n      },\n    });\n\n    try {\n      const results = response.data as unknown as z.infer<\n        typeof BatchEvaluationSchema\n      >;\n      return results.map((r) => ({\n        evaluation: r.evaluation,\n        reasoning: r.reasoning,\n      }));\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return questions.map(() => ({\n        evaluation: \"INVALID\" as const,\n        reasoning: `Failed to get structured response: ${errorMessage}`,\n      }));\n    }\n  }\n\n  private async _evaluateWithMultipleScreenshots(options: {\n    question: string;\n    answer?: string;\n    screenshots: Buffer[];\n    systemPrompt?: string;\n    agentReasoning?: string;\n  }): Promise<EvaluationResult> {\n    const {\n      question,\n      answer,\n      screenshots,\n      agentReasoning,\n      systemPrompt = `You are an expert evaluator that confidently returns YES or NO given a question and multiple screenshots showing the progression of a task.\n        ${agentReasoning ? \"You also have access to the agent's detailed reasoning and thought process throughout the task.\" : \"\"}\n        Analyze ALL screenshots to understand the complete journey. Look for evidence of task completion across all screenshots, not just the last one.\n        Success criteria may appear at different points in the sequence (confirmation messages, intermediate states, etc).\n        ${agentReasoning ? \"The agent's reasoning provides crucial context about what actions were attempted, what was observed, and the decision-making process. Use this alongside the visual evidence to make a comprehensive evaluation.\" : \"\"}\n        Today's date is ${new Date().toLocaleDateString()}`,\n    } = options;\n\n    if (!question)\n      throw new StagehandInvalidArgumentError(\n        \"Question cannot be an empty string\",\n      );\n    if (!screenshots || screenshots.length === 0)\n      throw new StagehandInvalidArgumentError(\n        \"At least one screenshot must be provided\",\n      );\n\n    const llmClient = this.getClient();\n\n    const imageContents = screenshots.map((s) => ({\n      type: \"image_url\" as const,\n      image_url: { url: `data:image/jpeg;base64,${s.toString(\"base64\")}` },\n    }));\n\n    const response = await llmClient.createChatCompletion<\n      LLMParsedResponse<LLMResponse>\n    >({\n      logger: this.silentLogger,\n      options: {\n        messages: [\n          { role: \"system\", content: systemPrompt },\n          {\n            role: \"user\",\n            content: [\n              {\n                type: \"text\",\n                text: agentReasoning\n                  ? `Question: ${question}\\n\\nAgent's reasoning and actions throughout the task:\\n${agentReasoning}\\n\\nI'm providing ${screenshots.length} screenshots showing the progression of the task. Please analyze both the agent's reasoning and all screenshots to determine if the task was completed successfully.`\n                  : `${question}\\n\\nI'm providing ${screenshots.length} screenshots showing the progression of the task. Please analyze all of them to determine if the task was completed successfully.`,\n              },\n              ...(answer\n                ? [{ type: \"text\" as const, text: `the answer is ${answer}` }]\n                : []),\n              ...imageContents,\n            ],\n          },\n        ],\n        response_model: { name: \"EvaluationResult\", schema: EvaluationSchema },\n      },\n    });\n\n    try {\n      const result = response.data as unknown as z.infer<\n        typeof EvaluationSchema\n      >;\n      return { evaluation: result.evaluation, reasoning: result.reasoning };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return {\n        evaluation: \"INVALID\",\n        reasoning: `Failed to get structured response: ${errorMessage}`,\n      } as const;\n    }\n  }\n}\n"]}

package/dist/esm/lib/version.d.ts

@@ -2,4 +2,4 @@
  * AUTO-GENERATED — DO NOT EDIT BY HAND
  *  Run `pnpm run gen-version` to refresh.
  */
-export declare const STAGEHAND_VERSION: "3.3.0";
+export declare const STAGEHAND_VERSION: "3.4.0";

package/dist/esm/lib/version.js

@@ -2,5 +2,5 @@
  * AUTO-GENERATED — DO NOT EDIT BY HAND
  *  Run `pnpm run gen-version` to refresh.
  */
-export const STAGEHAND_VERSION = "3.3.0";
+export const STAGEHAND_VERSION = "3.4.0";
 //# sourceMappingURL=version.js.map

package/dist/esm/lib/version.js.map

@@ -1 +1 @@
-{"version":3,"file":"version.js","sourceRoot":"","sources":["../../../lib/version.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,OAAgB,CAAC","sourcesContent":["/**\n * AUTO-GENERATED — DO NOT EDIT BY HAND\n *  Run `pnpm run gen-version` to refresh.\n */\nexport const STAGEHAND_VERSION = \"3.3.0\" as const;\n"]}
+{"version":3,"file":"version.js","sourceRoot":"","sources":["../../../lib/version.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,OAAgB,CAAC","sourcesContent":["/**\n * AUTO-GENERATED — DO NOT EDIT BY HAND\n *  Run `pnpm run gen-version` to refresh.\n */\nexport const STAGEHAND_VERSION = \"3.4.0\" as const;\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@browserbasehq/orca",
-  "version": "3.4.0-preview.0",
+  "version": "3.5.0-preview.1",
   "description": "An AI web browsing framework focused on simplicity and extensibility.",
   "type": "module",
   "main": "./dist/cjs/index.js",
@@ -35,9 +35,9 @@
     "build-dom-scripts": "tsx scripts/build-dom-scripts.ts",
     "build:cjs": "tsx scripts/build-cjs.ts",
     "build:esm": "tsx scripts/build-esm.ts",
-    "build": "pnpm --filter @browserbasehq/orca run --parallel \"/^build:(esm|cjs)$/\"",
+    "build": "pnpm --filter @browserbasehq/stagehand run --parallel \"/^build:(esm|cjs)$/\"",
     "example": "node --import tsx -e \"const args=process.argv.slice(1).filter(a=>a!=='--'); const [p]=args; const n=(p||'example').replace(/^\\.\\//,'').replace(/\\.ts$/i,''); import('node:path').then(path=>import(new URL(path.resolve('examples', n + '.ts'), 'file:')));\" --",
-    "test": "pnpm -w --dir ../.. exec turbo run test:core test:e2e --filter=@browserbasehq/orca --",
+    "test": "pnpm -w --dir ../.. exec turbo run test:core test:e2e --filter=@browserbasehq/stagehand --",
     "test:core": "tsx scripts/test-core.ts",
     "test:e2e": "tsx scripts/test-e2e.ts",
     "format": "prettier --write .",
@@ -47,7 +47,9 @@
   },
   "files": [
     "dist/esm",
-    "dist/cjs"
+    "dist/cjs",
+    "!dist/esm/tests",
+    "!dist/cjs/tests"
   ],
   "keywords": [
     "ai",
@@ -59,24 +61,36 @@
   "author": "Browserbase",
   "license": "MIT",
   "peerDependencies": {
-    "deepmerge": "^4.3.1",
+    "patchright-core": "^1.55.2",
+    "playwright-core": "^1.55.1",
+    "puppeteer-core": "^24.43.0",
     "zod": "^3.25.76 || ^4.2.0"
   },
+  "peerDependenciesMeta": {
+    "patchright-core": {
+      "optional": true
+    },
+    "playwright-core": {
+      "optional": true
+    },
+    "puppeteer-core": {
+      "optional": true
+    }
+  },
   "dependencies": {
     "@ai-sdk/provider": "^2.0.0",
     "@anthropic-ai/sdk": "0.39.0",
     "@browserbasehq/sdk": "^2.10.0",
     "@google/genai": "^1.22.0",
-    "@langchain/openai": "^0.4.4",
-    "@modelcontextprotocol/sdk": "^1.17.2",
+    "@modelcontextprotocol/sdk": "^1.29.0",
     "ai": "^5.0.133",
     "devtools-protocol": "^0.0.1464554",
     "fetch-cookie": "^3.1.0",
-    "openai": "^4.87.1",
+    "openai": "^4.104.0",
     "pino": "^9.6.0",
     "pino-pretty": "^13.0.0",
-    "uuid": "^11.1.0",
-    "ws": "^8.18.0",
+    "uuid": "^11.1.1",
+    "ws": "^8.21.0",
     "zod-to-json-schema": "^3.25.0"
   },
   "optionalDependencies": {
@@ -93,17 +107,12 @@
     "@ai-sdk/perplexity": "^2.0.13",
     "@ai-sdk/togetherai": "^1.0.23",
     "@ai-sdk/xai": "^2.0.26",
-    "@langchain/core": "^0.3.80",
     "bufferutil": "^4.0.9",
     "chrome-launcher": "^1.2.0",
-    "ollama-ai-provider-v2": "^1.5.0",
-    "patchright-core": "^1.55.2",
-    "playwright": "^1.52.0",
-    "playwright-core": "^1.54.1",
-    "puppeteer-core": "^22.8.0"
+    "ollama-ai-provider-v2": "^1.5.0"
   },
   "devDependencies": {
-    "@playwright/test": "^1.42.1",
+    "@playwright/test": "^1.55.1",
     "@types/adm-zip": "^0.5.7",
     "@types/jsdom": "^27.0.0",
     "@types/node": "^20.11.30",
@@ -112,12 +121,15 @@
     "adm-zip": "^0.5.16",
     "chalk": "^5.4.1",
     "eslint": "10.0.2",
-    "jsdom": "^24.0.0",
-    "playwright": "^1.52.0",
-    "playwright-core": "^1.54.1",
+    "jsdom": "^29.1.1",
+    "patchright-core": "^1.55.2",
+    "playwright": "^1.55.1",
+    "playwright-core": "^1.55.1",
     "prettier": "^3.2.5",
+    "puppeteer-core": "^24.43.0",
     "tsx": "*",
-    "vitest": "^4.0.8",
+    "vite": "^7.3.3",
+    "vitest": "^4.1.6",
     "zod": "^3.25.76 || ^4.2.0"
   },
   "repository": {

package/dist/cjs/tests/cache-variables.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/cjs/tests/cache-variables.test.js

@@ -1,221 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const vitest_1 = require("vitest");
-const ActCache_1 = require("../lib/v3/cache/ActCache");
-function createFakeStorage(entry) {
-    return {
-        enabled: true,
-        readJson: vitest_1.vi.fn().mockResolvedValue({ value: entry }),
-        writeJson: vitest_1.vi.fn().mockResolvedValue({}),
-        directory: "/tmp/cache",
-    };
-}
-(0, vitest_1.describe)("ActCache variable handling", () => {
-    (0, vitest_1.it)("cache key includes variable keys but not values", async () => {
-        const storage = {
-            enabled: true,
-            readJson: vitest_1.vi.fn(),
-            writeJson: vitest_1.vi.fn().mockResolvedValue({}),
-            directory: "/tmp/cache",
-        };
-        const cache = new ActCache_1.ActCache({
-            storage,
-            logger: vitest_1.vi.fn(),
-            getActHandler: () => null,
-            getDefaultLlmClient: () => ({}),
-            domSettleTimeoutMs: undefined,
-        });
-        const fakePage = {
-            url: vitest_1.vi.fn().mockResolvedValue("https://example.com"),
-        };
-        // First context with username="user1@example.com"
-        const context1 = await cache.prepareContext("type %username% into the email field", fakePage, { username: "user1@example.com" });
-        // Second context with username="user2@example.com"
-        const context2 = await cache.prepareContext("type %username% into the email field", fakePage, { username: "user2@example.com" });
-        // Third context with different variable key name
-        const context3 = await cache.prepareContext("type %email% into the email field", fakePage, { email: "user3@example.com" });
-        // Same instruction + same variable keys = same cache key
-        (0, vitest_1.expect)(context1?.cacheKey).toBe(context2?.cacheKey);
-        // Different variable keys = different cache key
-        (0, vitest_1.expect)(context1?.cacheKey).not.toBe(context3?.cacheKey);
-        // Verify variable keys are sorted and stored
-        (0, vitest_1.expect)(context1?.variableKeys).toEqual(["username"]);
-        (0, vitest_1.expect)(context2?.variableKeys).toEqual(["username"]);
-        (0, vitest_1.expect)(context3?.variableKeys).toEqual(["email"]);
-        // Verify variable values are preserved in context
-        (0, vitest_1.expect)(context1?.variables).toEqual({ username: "user1@example.com" });
-        (0, vitest_1.expect)(context2?.variables).toEqual({ username: "user2@example.com" });
-    });
-    (0, vitest_1.it)("replays cached actions with variable substitution", async () => {
-        // Cached action contains variable placeholder %username%
-        const action = {
-            selector: "xpath=/html/body/input[@type='email']",
-            description: "type username into email field",
-            method: "type",
-            arguments: ["%username%"], // Variable placeholder
-        };
-        const entry = {
-            version: 1,
-            instruction: "type %username% into the email field",
-            url: "https://example.com",
-            variableKeys: ["username"],
-            actions: [action],
-            actionDescription: "type username",
-            message: "done",
-        };
-        const storage = createFakeStorage(entry);
-        // Track what variables are passed to takeDeterministicAction
-        const capturedVariables = [];
-        const handler = {
-            takeDeterministicAction: vitest_1.vi
-                .fn()
-                .mockImplementation(async (_action, _page, _timeout, _client, _ensure, variables) => {
-                capturedVariables.push(variables || {});
-                return {
-                    success: true,
-                    message: "ok",
-                    actionDescription: "type username",
-                    actions: [action],
-                };
-            }),
-        };
-        const defaultClient = {};
-        const cache = new ActCache_1.ActCache({
-            storage,
-            logger: vitest_1.vi.fn(),
-            getActHandler: () => handler,
-            getDefaultLlmClient: () => defaultClient,
-            domSettleTimeoutMs: undefined,
-        });
-        // First replay with username="user1@example.com"
-        const context1 = {
-            instruction: "type %username% into the email field",
-            cacheKey: "test-key",
-            pageUrl: "https://example.com",
-            variableKeys: ["username"],
-            variables: { username: "user1@example.com" },
-        };
-        const result1 = await cache.tryReplay(context1, {});
-        (0, vitest_1.expect)(result1?.success).toBe(true);
-        (0, vitest_1.expect)(handler.takeDeterministicAction).toHaveBeenCalledTimes(1);
-        (0, vitest_1.expect)(capturedVariables[0]).toEqual({ username: "user1@example.com" });
-        // Reset
-        vitest_1.vi.clearAllMocks();
-        capturedVariables.length = 0;
-        // Second replay with username="user2@example.com"
-        const context2 = {
-            instruction: "type %username% into the email field",
-            cacheKey: "test-key", // Same cache key!
-            pageUrl: "https://example.com",
-            variableKeys: ["username"],
-            variables: { username: "user2@example.com" },
-        };
-        const result2 = await cache.tryReplay(context2, {});
-        (0, vitest_1.expect)(result2?.success).toBe(true);
-        (0, vitest_1.expect)(handler.takeDeterministicAction).toHaveBeenCalledTimes(1);
-        (0, vitest_1.expect)(capturedVariables[0]).toEqual({ username: "user2@example.com" });
-    });
-    (0, vitest_1.it)("cache miss when variable keys don't match", async () => {
-        const action = {
-            selector: "xpath=/html/body/input",
-            description: "type username",
-            method: "type",
-            arguments: ["%username%"],
-        };
-        // Cached entry expects "username" variable
-        const entry = {
-            version: 1,
-            instruction: "type %username% into the field",
-            url: "https://example.com",
-            variableKeys: ["username"],
-            actions: [action],
-        };
-        const storage = createFakeStorage(entry);
-        const cache = new ActCache_1.ActCache({
-            storage,
-            logger: vitest_1.vi.fn(),
-            getActHandler: () => null,
-            getDefaultLlmClient: () => ({}),
-            domSettleTimeoutMs: undefined,
-        });
-        // Context has different variable key "email"
-        const context = {
-            instruction: "type %email% into the field",
-            cacheKey: "test-key",
-            pageUrl: "https://example.com",
-            variableKeys: ["email"],
-            variables: { email: "test@example.com" },
-        };
-        const result = await cache.tryReplay(context, {});
-        // Should return null (cache miss) due to variable key mismatch
-        (0, vitest_1.expect)(result).toBeNull();
-    });
-    (0, vitest_1.it)("cache miss when required variables are missing", async () => {
-        const action = {
-            selector: "xpath=/html/body/input",
-            description: "type username",
-            method: "type",
-            arguments: ["%username%"],
-        };
-        const entry = {
-            version: 1,
-            instruction: "type %username% into the field",
-            url: "https://example.com",
-            variableKeys: ["username"],
-            actions: [action],
-        };
-        const storage = createFakeStorage(entry);
-        const logger = vitest_1.vi.fn();
-        const cache = new ActCache_1.ActCache({
-            storage,
-            logger,
-            getActHandler: () => null,
-            getDefaultLlmClient: () => ({}),
-            domSettleTimeoutMs: undefined,
-        });
-        // Context missing the username variable value
-        const context = {
-            instruction: "type %username% into the field",
-            cacheKey: "test-key",
-            pageUrl: "https://example.com",
-            variableKeys: ["username"],
-            variables: {}, // Missing username value!
-        };
-        const result = await cache.tryReplay(context, {});
-        // Should return null (cache miss)
-        (0, vitest_1.expect)(result).toBeNull();
-        // Should log the miss reason
-        (0, vitest_1.expect)(logger).toHaveBeenCalledWith(vitest_1.expect.objectContaining({
-            category: "cache",
-            message: "act cache miss: missing variables for replay",
-            level: 2,
-        }));
-    });
-    (0, vitest_1.it)("handles multiple variables correctly", async () => {
-        const storage = {
-            enabled: true,
-            readJson: vitest_1.vi.fn(),
-            writeJson: vitest_1.vi.fn().mockResolvedValue({}),
-            directory: "/tmp/cache",
-        };
-        const cache = new ActCache_1.ActCache({
-            storage,
-            logger: vitest_1.vi.fn(),
-            getActHandler: () => null,
-            getDefaultLlmClient: () => ({}),
-            domSettleTimeoutMs: undefined,
-        });
-        const fakePage = {
-            url: vitest_1.vi.fn().mockResolvedValue("https://example.com"),
-        };
-        // Context with multiple variables
-        const context1 = await cache.prepareContext("fill %username% and %password%", fakePage, { username: "user1", password: "pass1" });
-        const context2 = await cache.prepareContext("fill %username% and %password%", fakePage, { username: "user2", password: "pass2" });
-        // Same cache key despite different values
-        (0, vitest_1.expect)(context1?.cacheKey).toBe(context2?.cacheKey);
-        // Variable keys should be sorted
-        (0, vitest_1.expect)(context1?.variableKeys).toEqual(["password", "username"]);
-        (0, vitest_1.expect)(context2?.variableKeys).toEqual(["password", "username"]);
-    });
-});
-//# sourceMappingURL=cache-variables.test.js.map