@browserbasehq/orca 3.0.9-alpha-3 → 3.1.0-headers

package/dist/esm/lib/inference.js

@@ -0,0 +1,330 @@
+import { z } from "zod";
+import { getEnvTimeoutMs, withTimeout } from "./v3/timeoutConfig.js";
+import { buildActSystemPrompt, buildExtractSystemPrompt, buildExtractUserPrompt, buildMetadataPrompt, buildMetadataSystemPrompt, buildObserveSystemPrompt, buildObserveUserMessage, } from "./prompt.js";
+import { appendSummary, writeTimestampedTxtFile } from "./inferenceLogUtils.js";
+import { SupportedUnderstudyAction } from "./v3/types/private/handlers.js";
+function withLlmTimeout(promise, operation) {
+    return withTimeout(promise, getEnvTimeoutMs("LLM_MAX_MS"), `LLM ${operation}`);
+}
+export async function extract({ instruction, domElements, schema, llmClient, logger, userProvidedInstructions, logInferenceToFile = false, }) {
+    const metadataSchema = z.object({
+        progress: z
+            .string()
+            .describe("progress of what has been extracted so far, as concise as possible"),
+        completed: z
+            .boolean()
+            .describe("true if the goal is now accomplished. Use this conservatively, only when sure that the goal has been completed."),
+    });
+    const isUsingAnthropic = llmClient.type === "anthropic";
+    const isGPT5 = llmClient.modelName.includes("gpt-5"); // TODO: remove this as we update support for gpt-5 configuration options
+    const extractCallMessages = [
+        buildExtractSystemPrompt(isUsingAnthropic, userProvidedInstructions),
+        buildExtractUserPrompt(instruction, domElements, isUsingAnthropic),
+    ];
+    let extractCallFile = "";
+    let extractCallTimestamp = "";
+    if (logInferenceToFile) {
+        const { fileName, timestamp } = writeTimestampedTxtFile("extract_summary", "extract_call", {
+            modelCall: "extract",
+            messages: extractCallMessages,
+        });
+        extractCallFile = fileName;
+        extractCallTimestamp = timestamp;
+    }
+    const extractStartTime = Date.now();
+    const extractionResponse = await withLlmTimeout(llmClient.createChatCompletion({
+        options: {
+            messages: extractCallMessages,
+            response_model: {
+                schema,
+                name: "Extraction",
+            },
+            temperature: isGPT5 ? 1 : 0.1,
+            top_p: 1,
+            frequency_penalty: 0,
+            presence_penalty: 0,
+        },
+        logger,
+    }), "extract");
+    const extractEndTime = Date.now();
+    const { data: extractedData, usage: extractUsage } = extractionResponse;
+    let extractResponseFile;
+    if (logInferenceToFile) {
+        const { fileName } = writeTimestampedTxtFile("extract_summary", "extract_response", {
+            modelResponse: "extract",
+            rawResponse: extractedData,
+        });
+        extractResponseFile = fileName;
+        appendSummary("extract", {
+            extract_inference_type: "extract",
+            timestamp: extractCallTimestamp,
+            LLM_input_file: extractCallFile,
+            LLM_output_file: extractResponseFile,
+            prompt_tokens: extractUsage?.prompt_tokens ?? 0,
+            completion_tokens: extractUsage?.completion_tokens ?? 0,
+            reasoning_tokens: extractUsage?.reasoning_tokens ?? 0,
+            cached_input_tokens: extractUsage?.cached_input_tokens ?? 0,
+            inference_time_ms: extractEndTime - extractStartTime,
+        });
+    }
+    const metadataCallMessages = [
+        buildMetadataSystemPrompt(),
+        buildMetadataPrompt(instruction, extractedData),
+    ];
+    let metadataCallFile = "";
+    let metadataCallTimestamp = "";
+    if (logInferenceToFile) {
+        const { fileName, timestamp } = writeTimestampedTxtFile("extract_summary", "metadata_call", {
+            modelCall: "metadata",
+            messages: metadataCallMessages,
+        });
+        metadataCallFile = fileName;
+        metadataCallTimestamp = timestamp;
+    }
+    const metadataStartTime = Date.now();
+    const metadataResponse = await withLlmTimeout(llmClient.createChatCompletion({
+        options: {
+            messages: metadataCallMessages,
+            response_model: {
+                name: "Metadata",
+                schema: metadataSchema,
+            },
+            temperature: isGPT5 ? 1 : 0.1,
+            top_p: 1,
+            frequency_penalty: 0,
+            presence_penalty: 0,
+        },
+        logger,
+    }), "extract metadata");
+    const metadataEndTime = Date.now();
+    const { data: { completed: metadataResponseCompleted, progress: metadataResponseProgress, }, usage: metadataResponseUsage, } = metadataResponse;
+    let metadataResponseFile;
+    if (logInferenceToFile) {
+        const { fileName } = writeTimestampedTxtFile("extract_summary", "metadata_response", {
+            modelResponse: "metadata",
+            completed: metadataResponseCompleted,
+            progress: metadataResponseProgress,
+        });
+        metadataResponseFile = fileName;
+        appendSummary("extract", {
+            extract_inference_type: "metadata",
+            timestamp: metadataCallTimestamp,
+            LLM_input_file: metadataCallFile,
+            LLM_output_file: metadataResponseFile,
+            prompt_tokens: metadataResponseUsage?.prompt_tokens ?? 0,
+            completion_tokens: metadataResponseUsage?.completion_tokens ?? 0,
+            reasoning_tokens: metadataResponseUsage?.reasoning_tokens ?? 0,
+            cached_input_tokens: metadataResponseUsage?.cached_input_tokens ?? 0,
+            inference_time_ms: metadataEndTime - metadataStartTime,
+        });
+    }
+    const totalPromptTokens = (extractUsage?.prompt_tokens ?? 0) +
+        (metadataResponseUsage?.prompt_tokens ?? 0);
+    const totalCompletionTokens = (extractUsage?.completion_tokens ?? 0) +
+        (metadataResponseUsage?.completion_tokens ?? 0);
+    const totalInferenceTimeMs = extractEndTime - extractStartTime + (metadataEndTime - metadataStartTime);
+    const totalReasoningTokens = (extractUsage?.reasoning_tokens ?? 0) +
+        (metadataResponseUsage?.reasoning_tokens ?? 0);
+    const totalCachedInputTokens = (extractUsage?.cached_input_tokens ?? 0) +
+        (metadataResponseUsage?.cached_input_tokens ?? 0);
+    return {
+        ...extractedData,
+        metadata: {
+            completed: metadataResponseCompleted,
+            progress: metadataResponseProgress,
+        },
+        prompt_tokens: totalPromptTokens,
+        completion_tokens: totalCompletionTokens,
+        reasoning_tokens: totalReasoningTokens,
+        cached_input_tokens: totalCachedInputTokens,
+        inference_time_ms: totalInferenceTimeMs,
+    };
+}
+export async function observe({ instruction, domElements, llmClient, userProvidedInstructions, logger, logInferenceToFile = false, supportedActions, }) {
+    const isGPT5 = llmClient.modelName.includes("gpt-5"); // TODO: remove this as we update support for gpt-5 configuration options
+    const observeSchema = z.object({
+        elements: z
+            .array(z.object({
+            elementId: z
+                .string()
+                .regex(/^\d+-\d+$/)
+                .describe("the ID string associated with the element. Never include surrounding square brackets. This field must follow the format of 'number-number'."),
+            description: z
+                .string()
+                .describe("a description of the accessible element and its purpose"),
+            method: z
+                .enum(
+            // Use Object.values() for Zod v3 compatibility - z.enum() in v3 doesn't accept TypeScript enums directly
+            Object.values(SupportedUnderstudyAction))
+                .describe(`the candidate method/action to interact with the element. Select one of the available Understudy interaction methods.`),
+            arguments: z.array(z
+                .string()
+                .describe("the arguments to pass to the method. For example, for a click, the arguments are empty, but for a fill, the arguments are the value to fill in.")),
+        }))
+            .describe("an array of accessible elements that match the instruction"),
+    });
+    const messages = [
+        buildObserveSystemPrompt(userProvidedInstructions, supportedActions),
+        buildObserveUserMessage(instruction, domElements),
+    ];
+    let callTimestamp = "";
+    let callFile = "";
+    if (logInferenceToFile) {
+        const { fileName, timestamp } = writeTimestampedTxtFile(`observe_summary`, `observe_call`, {
+            modelCall: "observe",
+            messages,
+        });
+        callFile = fileName;
+        callTimestamp = timestamp;
+    }
+    const start = Date.now();
+    const rawResponse = await llmClient.createChatCompletion({
+        options: {
+            messages,
+            response_model: {
+                schema: observeSchema,
+                name: "Observation",
+            },
+            temperature: isGPT5 ? 1 : 0.1,
+            top_p: 1,
+            frequency_penalty: 0,
+            presence_penalty: 0,
+        },
+        logger,
+    });
+    const end = Date.now();
+    const usageTimeMs = end - start;
+    const { data: observeData, usage: observeUsage } = rawResponse;
+    const promptTokens = observeUsage?.prompt_tokens ?? 0;
+    const completionTokens = observeUsage?.completion_tokens ?? 0;
+    const reasoningTokens = observeUsage?.reasoning_tokens ?? 0;
+    const cachedInputTokens = observeUsage?.cached_input_tokens ?? 0;
+    let responseFile;
+    if (logInferenceToFile) {
+        const { fileName: responseFileName } = writeTimestampedTxtFile(`observe_summary`, `observe_response`, {
+            modelResponse: "observe",
+            rawResponse: observeData,
+        });
+        responseFile = responseFileName;
+        appendSummary("observe", {
+            [`observe_inference_type`]: "observe",
+            timestamp: callTimestamp,
+            LLM_input_file: callFile,
+            LLM_output_file: responseFile,
+            prompt_tokens: promptTokens,
+            completion_tokens: completionTokens,
+            reasoning_tokens: reasoningTokens,
+            cached_input_tokens: cachedInputTokens,
+            inference_time_ms: usageTimeMs,
+        });
+    }
+    const parsedElements = observeData.elements?.map((el) => {
+        const base = {
+            elementId: el.elementId,
+            description: String(el.description),
+            method: String(el.method),
+            arguments: el.arguments,
+        };
+        return base;
+    }) ?? [];
+    return {
+        elements: parsedElements,
+        prompt_tokens: promptTokens,
+        completion_tokens: completionTokens,
+        reasoning_tokens: reasoningTokens,
+        cached_input_tokens: cachedInputTokens,
+        inference_time_ms: usageTimeMs,
+    };
+}
+export async function act({ instruction, domElements, llmClient, userProvidedInstructions, logger, logInferenceToFile = false, }) {
+    const isGPT5 = llmClient.modelName.includes("gpt-5"); // TODO: remove this as we update support for gpt-5 configuration options
+    const actSchema = z.object({
+        elementId: z
+            .string()
+            .regex(/^\d+-\d+$/)
+            .describe("the ID string associated with the element. Never include surrounding square brackets. This field must follow the format of 'number-number'."),
+        description: z
+            .string()
+            .describe("a description of the accessible element and its purpose"),
+        method: z
+            .enum(
+        // Use Object.values() for Zod v3 compatibility - z.enum() in v3 doesn't accept TypeScript enums directly
+        Object.values(SupportedUnderstudyAction))
+            .describe("the candidate method/action to interact with the element. Select one of the available Understudy interaction methods."),
+        arguments: z.array(z
+            .string()
+            .describe("the arguments to pass to the method. For example, for a click, the arguments are empty, but for a fill, the arguments are the value to fill in.")),
+        twoStep: z.boolean(),
+    });
+    const messages = [
+        buildActSystemPrompt(userProvidedInstructions),
+        buildObserveUserMessage(instruction, domElements),
+    ];
+    let callTimestamp = "";
+    let callFile = "";
+    if (logInferenceToFile) {
+        const { fileName, timestamp } = writeTimestampedTxtFile(`act_summary`, `act_call`, {
+            modelCall: "act",
+            messages,
+        });
+        callFile = fileName;
+        callTimestamp = timestamp;
+    }
+    const start = Date.now();
+    const rawResponse = await llmClient.createChatCompletion({
+        options: {
+            messages,
+            response_model: {
+                schema: actSchema,
+                name: "act",
+            },
+            temperature: isGPT5 ? 1 : 0.1,
+            top_p: 1,
+            frequency_penalty: 0,
+            presence_penalty: 0,
+        },
+        logger,
+    });
+    const end = Date.now();
+    const usageTimeMs = end - start;
+    const { data: actData, usage: actUsage } = rawResponse;
+    const promptTokens = actUsage?.prompt_tokens ?? 0;
+    const completionTokens = actUsage?.completion_tokens ?? 0;
+    const reasoningTokens = actUsage?.reasoning_tokens ?? 0;
+    const cachedInputTokens = actUsage?.cached_input_tokens ?? 0;
+    let responseFile;
+    if (logInferenceToFile) {
+        const { fileName: responseFileName } = writeTimestampedTxtFile(`act_summary`, `act_response`, {
+            modelResponse: "act",
+            rawResponse: actData,
+        });
+        responseFile = responseFileName;
+        appendSummary("act", {
+            [`act_inference_type`]: "act",
+            timestamp: callTimestamp,
+            LLM_input_file: callFile,
+            LLM_output_file: responseFile,
+            prompt_tokens: promptTokens,
+            completion_tokens: completionTokens,
+            reasoning_tokens: reasoningTokens,
+            cached_input_tokens: cachedInputTokens,
+            inference_time_ms: usageTimeMs,
+        });
+    }
+    const parsedElement = {
+        elementId: actData.elementId,
+        description: String(actData.description),
+        method: String(actData.method),
+        arguments: actData.arguments,
+    };
+    return {
+        element: parsedElement,
+        prompt_tokens: promptTokens,
+        completion_tokens: completionTokens,
+        reasoning_tokens: reasoningTokens,
+        cached_input_tokens: cachedInputTokens,
+        inference_time_ms: usageTimeMs,
+        twoStep: actData.twoStep,
+    };
+}
+//# sourceMappingURL=inference.js.map

package/dist/esm/lib/inference.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"inference.js","sourceRoot":"","sources":["../../../lib/inference.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACrE,OAAO,EACL,oBAAoB,EACpB,wBAAwB,EACxB,sBAAsB,EACtB,mBAAmB,EACnB,yBAAyB,EACzB,wBAAwB,EACxB,uBAAuB,GACxB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,aAAa,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AAKhF,OAAO,EAAE,yBAAyB,EAAE,MAAM,gCAAgC,CAAC;AAK3E,SAAS,cAAc,CAAI,OAAmB,EAAE,SAAiB;IAC/D,OAAO,WAAW,CAChB,OAAO,EACP,eAAe,CAAC,YAAY,CAAC,EAC7B,OAAO,SAAS,EAAE,CACnB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAA+B,EAC1D,WAAW,EACX,WAAW,EACX,MAAM,EACN,SAAS,EACT,MAAM,EACN,wBAAwB,EACxB,kBAAkB,GAAG,KAAK,GAS3B;IACC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;QAC9B,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,oEAAoE,CACrE;QACH,SAAS,EAAE,CAAC;aACT,OAAO,EAAE;aACT,QAAQ,CACP,iHAAiH,CAClH;KACJ,CAAC,CAAC;IAKH,MAAM,gBAAgB,GAAG,SAAS,CAAC,IAAI,KAAK,WAAW,CAAC;IACxD,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,yEAAyE;IAE/H,MAAM,mBAAmB,GAAkB;QACzC,wBAAwB,CAAC,gBAAgB,EAAE,wBAAwB,CAAC;QACpE,sBAAsB,CAAC,WAAW,EAAE,WAAW,EAAE,gBAAgB,CAAC;KACnE,CAAC;IAEF,IAAI,eAAe,GAAG,EAAE,CAAC;IACzB,IAAI,oBAAoB,GAAG,EAAE,CAAC;IAC9B,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,uBAAuB,CACrD,iBAAiB,EACjB,cAAc,EACd;YACE,SAAS,EAAE,SAAS;YACpB,QAAQ,EAAE,mBAAmB;SAC9B,CACF,CAAC;QACF,eAAe,GAAG,QAAQ,CAAC;QAC3B,oBAAoB,GAAG,SAAS,CAAC;IACnC,CAAC;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACpC,MAAM,kBAAkB,GAAG,MAAM,cAAc,CAC7C,SAAS,CAAC,oBAAoB,CAAqB;QACjD,OAAO,EAAE;YACP,QAAQ,EAAE,mBAAmB;YAC7B,cAAc,EAAE;gBACd,MAAM;gBACN,IAAI,EAAE,YAAY;aACnB;YACD,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;YAC7B,KAAK,EAAE,CAAC;YACR,iBAAiB,EAAE,CAAC;YACpB,gBAAgB,EAAE,CAAC;SACpB;QACD,MAAM;KACP,CAAC,EACF,SAAS,CACV,CAAC;IACF,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAElC,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,kBAAkB,CAAC;IAExE,IAAI,mBAA2B,CAAC;IAChC,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,GAAG,uBAAuB,CAC1C,iBAAiB,EACjB,kBAAkB,EAClB;YACE,aAAa,EAAE,SAAS;YACxB,WAAW,EAAE,aAAa;SAC3B,CACF,CAAC;QACF,mBAAmB,GAAG,QAAQ,CAAC;QAE/B,aAAa,CAAC,SAAS,EAAE;YACvB,sBAAsB,EAAE,SAAS;YACjC,SAAS,EAAE,oBAAoB;YAC/B,cAAc,EAAE,eAAe;YAC/B,eAAe,EAAE,mBAAmB;YACpC,aAAa,EAAE,YAAY,EAAE,aAAa,IAAI,CAAC;YAC/C,iBAAiB,EAAE,YAAY,EAAE,iBAAiB,IAAI,CAAC;YACvD,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,IAAI,CAAC;YACrD,mBAAmB,EAAE,YAAY,EAAE,mBAAmB,IAAI,CAAC;YAC3D,iBAAiB,EAAE,cAAc,GAAG,gBAAgB;SACrD,CAAC,CAAC;IACL,CAAC;IAED,MAAM,oBAAoB,GAAkB;QAC1C,yBAAyB,EAAE;QAC3B,mBAAmB,CAAC,WAAW,EAAE,aAAa,CAAC;KAChD,CAAC;IAEF,IAAI,gBAAgB,GAAG,EAAE,CAAC;IAC1B,IAAI,qBAAqB,GAAG,EAAE,CAAC;IAC/B,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,uBAAuB,CACrD,iBAAiB,EACjB,eAAe,EACf;YACE,SAAS,EAAE,UAAU;YACrB,QAAQ,EAAE,oBAAoB;SAC/B,CACF,CAAC;QACF,gBAAgB,GAAG,QAAQ,CAAC;QAC5B,qBAAqB,GAAG,SAAS,CAAC;IACpC,CAAC;IAED,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACrC,MAAM,gBAAgB,GAAG,MAAM,cAAc,CAC3C,SAAS,CAAC,oBAAoB,CAAmB;QAC/C,OAAO,EAAE;YACP,QAAQ,EAAE,oBAAoB;YAC9B,cAAc,EAAE;gBACd,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,cAAc;aACvB;YACD,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;YAC7B,KAAK,EAAE,CAAC;YACR,iBAAiB,EAAE,CAAC;YACpB,gBAAgB,EAAE,CAAC;SACpB;QACD,MAAM;KACP,CAAC,EACF,kBAAkB,CACnB,CAAC;IACF,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEnC,MAAM,EACJ,IAAI,EAAE,EACJ,SAAS,EAAE,yBAAyB,EACpC,QAAQ,EAAE,wBAAwB,GACnC,EACD,KAAK,EAAE,qBAAqB,GAC7B,GAAG,gBAAgB,CAAC;IAErB,IAAI,oBAA4B,CAAC;IACjC,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,GAAG,uBAAuB,CAC1C,iBAAiB,EACjB,mBAAmB,EACnB;YACE,aAAa,EAAE,UAAU;YACzB,SAAS,EAAE,yBAAyB;YACpC,QAAQ,EAAE,wBAAwB;SACnC,CACF,CAAC;QACF,oBAAoB,GAAG,QAAQ,CAAC;QAEhC,aAAa,CAAC,SAAS,EAAE;YACvB,sBAAsB,EAAE,UAAU;YAClC,SAAS,EAAE,qBAAqB;YAChC,cAAc,EAAE,gBAAgB;YAChC,eAAe,EAAE,oBAAoB;YACrC,aAAa,EAAE,qBAAqB,EAAE,aAAa,IAAI,CAAC;YACxD,iBAAiB,EAAE,qBAAqB,EAAE,iBAAiB,IAAI,CAAC;YAChE,gBAAgB,EAAE,qBAAqB,EAAE,gBAAgB,IAAI,CAAC;YAC9D,mBAAmB,EAAE,qBAAqB,EAAE,mBAAmB,IAAI,CAAC;YACpE,iBAAiB,EAAE,eAAe,GAAG,iBAAiB;SACvD,CAAC,CAAC;IACL,CAAC;IAED,MAAM,iBAAiB,GACrB,CAAC,YAAY,EAAE,aAAa,IAAI,CAAC,CAAC;QAClC,CAAC,qBAAqB,EAAE,aAAa,IAAI,CAAC,CAAC,CAAC;IAE9C,MAAM,qBAAqB,GACzB,CAAC,YAAY,EAAE,iBAAiB,IAAI,CAAC,CAAC;QACtC,CAAC,qBAAqB,EAAE,iBAAiB,IAAI,CAAC,CAAC,CAAC;IAElD,MAAM,oBAAoB,GACxB,cAAc,GAAG,gBAAgB,GAAG,CAAC,eAAe,GAAG,iBAAiB,CAAC,CAAC;IAC5E,MAAM,oBAAoB,GACxB,CAAC,YAAY,EAAE,gBAAgB,IAAI,CAAC,CAAC;QACrC,CAAC,qBAAqB,EAAE,gBAAgB,IAAI,CAAC,CAAC,CAAC;IACjD,MAAM,sBAAsB,GAC1B,CAAC,YAAY,EAAE,mBAAmB,IAAI,CAAC,CAAC;QACxC,CAAC,qBAAqB,EAAE,mBAAmB,IAAI,CAAC,CAAC,CAAC;IAEpD,OAAO;QACL,GAAG,aAAa;QAChB,QAAQ,EAAE;YACR,SAAS,EAAE,yBAAyB;YACpC,QAAQ,EAAE,wBAAwB;SACnC;QACD,aAAa,EAAE,iBAAiB;QAChC,iBAAiB,EAAE,qBAAqB;QACxC,gBAAgB,EAAE,oBAAoB;QACtC,mBAAmB,EAAE,sBAAsB;QAC3C,iBAAiB,EAAE,oBAAoB;KACxC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,EAC5B,WAAW,EACX,WAAW,EACX,SAAS,EACT,wBAAwB,EACxB,MAAM,EACN,kBAAkB,GAAG,KAAK,EAC1B,gBAAgB,GASjB;IACC,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,yEAAyE;IAE/H,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;QAC7B,QAAQ,EAAE,CAAC;aACR,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;YACP,SAAS,EAAE,CAAC;iBACT,MAAM,EAAE;iBACR,KAAK,CAAC,WAAW,CAAC;iBAClB,QAAQ,CACP,6IAA6I,CAC9I;YACH,WAAW,EAAE,CAAC;iBACX,MAAM,EAAE;iBACR,QAAQ,CACP,yDAAyD,CAC1D;YACH,MAAM,EAAE,CAAC;iBACN,IAAI;YACH,yGAAyG;YACzG,MAAM,CAAC,MAAM,CAAC,yBAAyB,CAGtC,CACF;iBACA,QAAQ,CACP,uHAAuH,CACxH;YACH,SAAS,EAAE,CAAC,CAAC,KAAK,CAChB,CAAC;iBACE,MAAM,EAAE;iBACR,QAAQ,CACP,iJAAiJ,CAClJ,CACJ;SACF,CAAC,CACH;aACA,QAAQ,CAAC,4DAA4D,CAAC;KAC1E,CAAC,CAAC;IAIH,MAAM,QAAQ,GAAkB;QAC9B,wBAAwB,CAAC,wBAAwB,EAAE,gBAAgB,CAAC;QACpE,uBAAuB,CAAC,WAAW,EAAE,WAAW,CAAC;KAClD,CAAC;IAEF,IAAI,aAAa,GAAG,EAAE,CAAC;IACvB,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,uBAAuB,CACrD,iBAAiB,EACjB,cAAc,EACd;YACE,SAAS,EAAE,SAAS;YACpB,QAAQ;SACT,CACF,CAAC;QACF,QAAQ,GAAG,QAAQ,CAAC;QACpB,aAAa,GAAG,SAAS,CAAC;IAC5B,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAAkB;QACxE,OAAO,EAAE;YACP,QAAQ;YACR,cAAc,EAAE;gBACd,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,aAAa;aACpB;YACD,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;YAC7B,KAAK,EAAE,CAAC;YACR,iBAAiB,EAAE,CAAC;YACpB,gBAAgB,EAAE,CAAC;SACpB;QACD,MAAM;KACP,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,WAAW,GAAG,GAAG,GAAG,KAAK,CAAC;IAEhC,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,WAAW,CAAC;IAC/D,MAAM,YAAY,GAAG,YAAY,EAAE,aAAa,IAAI,CAAC,CAAC;IACtD,MAAM,gBAAgB,GAAG,YAAY,EAAE,iBAAiB,IAAI,CAAC,CAAC;IAC9D,MAAM,eAAe,GAAG,YAAY,EAAE,gBAAgB,IAAI,CAAC,CAAC;IAC5D,MAAM,iBAAiB,GAAG,YAAY,EAAE,mBAAmB,IAAI,CAAC,CAAC;IAEjE,IAAI,YAAoB,CAAC;IACzB,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,gBAAgB,EAAE,GAAG,uBAAuB,CAC5D,iBAAiB,EACjB,kBAAkB,EAClB;YACE,aAAa,EAAE,SAAS;YACxB,WAAW,EAAE,WAAW;SACzB,CACF,CAAC;QACF,YAAY,GAAG,gBAAgB,CAAC;QAEhC,aAAa,CAAC,SAAS,EAAE;YACvB,CAAC,wBAAwB,CAAC,EAAE,SAAS;YACrC,SAAS,EAAE,aAAa;YACxB,cAAc,EAAE,QAAQ;YACxB,eAAe,EAAE,YAAY;YAC7B,aAAa,EAAE,YAAY;YAC3B,iBAAiB,EAAE,gBAAgB;YACnC,gBAAgB,EAAE,eAAe;YACjC,mBAAmB,EAAE,iBAAiB;YACtC,iBAAiB,EAAE,WAAW;SAC/B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,cAAc,GAClB,WAAW,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QAC/B,MAAM,IAAI,GAAG;YACX,SAAS,EAAE,EAAE,CAAC,SAAS;YACvB,WAAW,EAAE,MAAM,CAAC,EAAE,CAAC,WAAW,CAAC;YACnC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;YACzB,SAAS,EAAE,EAAE,CAAC,SAAS;SACxB,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,IAAI,EAAE,CAAC;IAEX,OAAO;QACL,QAAQ,EAAE,cAAc;QACxB,aAAa,EAAE,YAAY;QAC3B,iBAAiB,EAAE,gBAAgB;QACnC,gBAAgB,EAAE,eAAe;QACjC,mBAAmB,EAAE,iBAAiB;QACtC,iBAAiB,EAAE,WAAW;KAC/B,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,GAAG,CAAC,EACxB,WAAW,EACX,WAAW,EACX,SAAS,EACT,wBAAwB,EACxB,MAAM,EACN,kBAAkB,GAAG,KAAK,GAQ3B;IACC,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,yEAAyE;IAE/H,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC;QACzB,SAAS,EAAE,CAAC;aACT,MAAM,EAAE;aACR,KAAK,CAAC,WAAW,CAAC;aAClB,QAAQ,CACP,6IAA6I,CAC9I;QACH,WAAW,EAAE,CAAC;aACX,MAAM,EAAE;aACR,QAAQ,CAAC,yDAAyD,CAAC;QACtE,MAAM,EAAE,CAAC;aACN,IAAI;QACH,yGAAyG;QACzG,MAAM,CAAC,MAAM,CAAC,yBAAyB,CAGtC,CACF;aACA,QAAQ,CACP,uHAAuH,CACxH;QACH,SAAS,EAAE,CAAC,CAAC,KAAK,CAChB,CAAC;aACE,MAAM,EAAE;aACR,QAAQ,CACP,iJAAiJ,CAClJ,CACJ;QACD,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE;KACrB,CAAC,CAAC;IAIH,MAAM,QAAQ,GAAkB;QAC9B,oBAAoB,CAAC,wBAAwB,CAAC;QAC9C,uBAAuB,CAAC,WAAW,EAAE,WAAW,CAAC;KAClD,CAAC;IAEF,IAAI,aAAa,GAAG,EAAE,CAAC;IACvB,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,uBAAuB,CACrD,aAAa,EACb,UAAU,EACV;YACE,SAAS,EAAE,KAAK;YAChB,QAAQ;SACT,CACF,CAAC;QACF,QAAQ,GAAG,QAAQ,CAAC;QACpB,aAAa,GAAG,SAAS,CAAC;IAC5B,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAAc;QACpE,OAAO,EAAE;YACP,QAAQ;YACR,cAAc,EAAE;gBACd,MAAM,EAAE,SAAS;gBACjB,IAAI,EAAE,KAAK;aACZ;YACD,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;YAC7B,KAAK,EAAE,CAAC;YACR,iBAAiB,EAAE,CAAC;YACpB,gBAAgB,EAAE,CAAC;SACpB;QACD,MAAM;KACP,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,WAAW,GAAG,GAAG,GAAG,KAAK,CAAC;IAEhC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,WAAW,CAAC;IACvD,MAAM,YAAY,GAAG,QAAQ,EAAE,aAAa,IAAI,CAAC,CAAC;IAClD,MAAM,gBAAgB,GAAG,QAAQ,EAAE,iBAAiB,IAAI,CAAC,CAAC;IAC1D,MAAM,eAAe,GAAG,QAAQ,EAAE,gBAAgB,IAAI,CAAC,CAAC;IACxD,MAAM,iBAAiB,GAAG,QAAQ,EAAE,mBAAmB,IAAI,CAAC,CAAC;IAE7D,IAAI,YAAoB,CAAC;IACzB,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,gBAAgB,EAAE,GAAG,uBAAuB,CAC5D,aAAa,EACb,cAAc,EACd;YACE,aAAa,EAAE,KAAK;YACpB,WAAW,EAAE,OAAO;SACrB,CACF,CAAC;QACF,YAAY,GAAG,gBAAgB,CAAC;QAEhC,aAAa,CAAC,KAAK,EAAE;YACnB,CAAC,oBAAoB,CAAC,EAAE,KAAK;YAC7B,SAAS,EAAE,aAAa;YACxB,cAAc,EAAE,QAAQ;YACxB,eAAe,EAAE,YAAY;YAC7B,aAAa,EAAE,YAAY;YAC3B,iBAAiB,EAAE,gBAAgB;YACnC,gBAAgB,EAAE,eAAe;YACjC,mBAAmB,EAAE,iBAAiB;YACtC,iBAAiB,EAAE,WAAW;SAC/B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,aAAa,GAAG;QACpB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,WAAW,EAAE,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC;QACxC,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;QAC9B,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC;IAEF,OAAO;QACL,OAAO,EAAE,aAAa;QACtB,aAAa,EAAE,YAAY;QAC3B,iBAAiB,EAAE,gBAAgB;QACnC,gBAAgB,EAAE,eAAe;QACjC,mBAAmB,EAAE,iBAAiB;QACtC,iBAAiB,EAAE,WAAW;QAC9B,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC;AACJ,CAAC","sourcesContent":["import { z } from \"zod\";\nimport { LogLine } from \"./v3/types/public/logs.js\";\nimport { ChatMessage, LLMClient } from \"./v3/llm/LLMClient.js\";\nimport { getEnvTimeoutMs, withTimeout } from \"./v3/timeoutConfig.js\";\nimport {\n  buildActSystemPrompt,\n  buildExtractSystemPrompt,\n  buildExtractUserPrompt,\n  buildMetadataPrompt,\n  buildMetadataSystemPrompt,\n  buildObserveSystemPrompt,\n  buildObserveUserMessage,\n} from \"./prompt.js\";\nimport { appendSummary, writeTimestampedTxtFile } from \"./inferenceLogUtils.js\";\nimport type {\n  InferStagehandSchema,\n  StagehandZodObject,\n} from \"./v3/zodCompat.js\";\nimport { SupportedUnderstudyAction } from \"./v3/types/private/handlers.js\";\n\n// Re-export for backward compatibility\nexport type { LLMParsedResponse, LLMUsage } from \"./v3/llm/LLMClient.js\";\n\nfunction withLlmTimeout<T>(promise: Promise<T>, operation: string): Promise<T> {\n  return withTimeout(\n    promise,\n    getEnvTimeoutMs(\"LLM_MAX_MS\"),\n    `LLM ${operation}`,\n  );\n}\n\nexport async function extract<T extends StagehandZodObject>({\n  instruction,\n  domElements,\n  schema,\n  llmClient,\n  logger,\n  userProvidedInstructions,\n  logInferenceToFile = false,\n}: {\n  instruction: string;\n  domElements: string;\n  schema: T;\n  llmClient: LLMClient;\n  userProvidedInstructions?: string;\n  logger: (message: LogLine) => void;\n  logInferenceToFile?: boolean;\n}) {\n  const metadataSchema = z.object({\n    progress: z\n      .string()\n      .describe(\n        \"progress of what has been extracted so far, as concise as possible\",\n      ),\n    completed: z\n      .boolean()\n      .describe(\n        \"true if the goal is now accomplished. Use this conservatively, only when sure that the goal has been completed.\",\n      ),\n  });\n\n  type ExtractionResponse = InferStagehandSchema<T>;\n  type MetadataResponse = z.infer<typeof metadataSchema>;\n\n  const isUsingAnthropic = llmClient.type === \"anthropic\";\n  const isGPT5 = llmClient.modelName.includes(\"gpt-5\"); // TODO: remove this as we update support for gpt-5 configuration options\n\n  const extractCallMessages: ChatMessage[] = [\n    buildExtractSystemPrompt(isUsingAnthropic, userProvidedInstructions),\n    buildExtractUserPrompt(instruction, domElements, isUsingAnthropic),\n  ];\n\n  let extractCallFile = \"\";\n  let extractCallTimestamp = \"\";\n  if (logInferenceToFile) {\n    const { fileName, timestamp } = writeTimestampedTxtFile(\n      \"extract_summary\",\n      \"extract_call\",\n      {\n        modelCall: \"extract\",\n        messages: extractCallMessages,\n      },\n    );\n    extractCallFile = fileName;\n    extractCallTimestamp = timestamp;\n  }\n\n  const extractStartTime = Date.now();\n  const extractionResponse = await withLlmTimeout(\n    llmClient.createChatCompletion<ExtractionResponse>({\n      options: {\n        messages: extractCallMessages,\n        response_model: {\n          schema,\n          name: \"Extraction\",\n        },\n        temperature: isGPT5 ? 1 : 0.1,\n        top_p: 1,\n        frequency_penalty: 0,\n        presence_penalty: 0,\n      },\n      logger,\n    }),\n    \"extract\",\n  );\n  const extractEndTime = Date.now();\n\n  const { data: extractedData, usage: extractUsage } = extractionResponse;\n\n  let extractResponseFile: string;\n  if (logInferenceToFile) {\n    const { fileName } = writeTimestampedTxtFile(\n      \"extract_summary\",\n      \"extract_response\",\n      {\n        modelResponse: \"extract\",\n        rawResponse: extractedData,\n      },\n    );\n    extractResponseFile = fileName;\n\n    appendSummary(\"extract\", {\n      extract_inference_type: \"extract\",\n      timestamp: extractCallTimestamp,\n      LLM_input_file: extractCallFile,\n      LLM_output_file: extractResponseFile,\n      prompt_tokens: extractUsage?.prompt_tokens ?? 0,\n      completion_tokens: extractUsage?.completion_tokens ?? 0,\n      reasoning_tokens: extractUsage?.reasoning_tokens ?? 0,\n      cached_input_tokens: extractUsage?.cached_input_tokens ?? 0,\n      inference_time_ms: extractEndTime - extractStartTime,\n    });\n  }\n\n  const metadataCallMessages: ChatMessage[] = [\n    buildMetadataSystemPrompt(),\n    buildMetadataPrompt(instruction, extractedData),\n  ];\n\n  let metadataCallFile = \"\";\n  let metadataCallTimestamp = \"\";\n  if (logInferenceToFile) {\n    const { fileName, timestamp } = writeTimestampedTxtFile(\n      \"extract_summary\",\n      \"metadata_call\",\n      {\n        modelCall: \"metadata\",\n        messages: metadataCallMessages,\n      },\n    );\n    metadataCallFile = fileName;\n    metadataCallTimestamp = timestamp;\n  }\n\n  const metadataStartTime = Date.now();\n  const metadataResponse = await withLlmTimeout(\n    llmClient.createChatCompletion<MetadataResponse>({\n      options: {\n        messages: metadataCallMessages,\n        response_model: {\n          name: \"Metadata\",\n          schema: metadataSchema,\n        },\n        temperature: isGPT5 ? 1 : 0.1,\n        top_p: 1,\n        frequency_penalty: 0,\n        presence_penalty: 0,\n      },\n      logger,\n    }),\n    \"extract metadata\",\n  );\n  const metadataEndTime = Date.now();\n\n  const {\n    data: {\n      completed: metadataResponseCompleted,\n      progress: metadataResponseProgress,\n    },\n    usage: metadataResponseUsage,\n  } = metadataResponse;\n\n  let metadataResponseFile: string;\n  if (logInferenceToFile) {\n    const { fileName } = writeTimestampedTxtFile(\n      \"extract_summary\",\n      \"metadata_response\",\n      {\n        modelResponse: \"metadata\",\n        completed: metadataResponseCompleted,\n        progress: metadataResponseProgress,\n      },\n    );\n    metadataResponseFile = fileName;\n\n    appendSummary(\"extract\", {\n      extract_inference_type: \"metadata\",\n      timestamp: metadataCallTimestamp,\n      LLM_input_file: metadataCallFile,\n      LLM_output_file: metadataResponseFile,\n      prompt_tokens: metadataResponseUsage?.prompt_tokens ?? 0,\n      completion_tokens: metadataResponseUsage?.completion_tokens ?? 0,\n      reasoning_tokens: metadataResponseUsage?.reasoning_tokens ?? 0,\n      cached_input_tokens: metadataResponseUsage?.cached_input_tokens ?? 0,\n      inference_time_ms: metadataEndTime - metadataStartTime,\n    });\n  }\n\n  const totalPromptTokens =\n    (extractUsage?.prompt_tokens ?? 0) +\n    (metadataResponseUsage?.prompt_tokens ?? 0);\n\n  const totalCompletionTokens =\n    (extractUsage?.completion_tokens ?? 0) +\n    (metadataResponseUsage?.completion_tokens ?? 0);\n\n  const totalInferenceTimeMs =\n    extractEndTime - extractStartTime + (metadataEndTime - metadataStartTime);\n  const totalReasoningTokens =\n    (extractUsage?.reasoning_tokens ?? 0) +\n    (metadataResponseUsage?.reasoning_tokens ?? 0);\n  const totalCachedInputTokens =\n    (extractUsage?.cached_input_tokens ?? 0) +\n    (metadataResponseUsage?.cached_input_tokens ?? 0);\n\n  return {\n    ...extractedData,\n    metadata: {\n      completed: metadataResponseCompleted,\n      progress: metadataResponseProgress,\n    },\n    prompt_tokens: totalPromptTokens,\n    completion_tokens: totalCompletionTokens,\n    reasoning_tokens: totalReasoningTokens,\n    cached_input_tokens: totalCachedInputTokens,\n    inference_time_ms: totalInferenceTimeMs,\n  };\n}\n\nexport async function observe({\n  instruction,\n  domElements,\n  llmClient,\n  userProvidedInstructions,\n  logger,\n  logInferenceToFile = false,\n  supportedActions,\n}: {\n  instruction: string;\n  domElements: string;\n  llmClient: LLMClient;\n  userProvidedInstructions?: string;\n  logger: (message: LogLine) => void;\n  logInferenceToFile?: boolean;\n  supportedActions?: string[];\n}) {\n  const isGPT5 = llmClient.modelName.includes(\"gpt-5\"); // TODO: remove this as we update support for gpt-5 configuration options\n\n  const observeSchema = z.object({\n    elements: z\n      .array(\n        z.object({\n          elementId: z\n            .string()\n            .regex(/^\\d+-\\d+$/)\n            .describe(\n              \"the ID string associated with the element. Never include surrounding square brackets. This field must follow the format of 'number-number'.\",\n            ),\n          description: z\n            .string()\n            .describe(\n              \"a description of the accessible element and its purpose\",\n            ),\n          method: z\n            .enum(\n              // Use Object.values() for Zod v3 compatibility - z.enum() in v3 doesn't accept TypeScript enums directly\n              Object.values(SupportedUnderstudyAction) as unknown as readonly [\n                string,\n                ...string[],\n              ],\n            )\n            .describe(\n              `the candidate method/action to interact with the element. Select one of the available Understudy interaction methods.`,\n            ),\n          arguments: z.array(\n            z\n              .string()\n              .describe(\n                \"the arguments to pass to the method. For example, for a click, the arguments are empty, but for a fill, the arguments are the value to fill in.\",\n              ),\n          ),\n        }),\n      )\n      .describe(\"an array of accessible elements that match the instruction\"),\n  });\n\n  type ObserveResponse = z.infer<typeof observeSchema>;\n\n  const messages: ChatMessage[] = [\n    buildObserveSystemPrompt(userProvidedInstructions, supportedActions),\n    buildObserveUserMessage(instruction, domElements),\n  ];\n\n  let callTimestamp = \"\";\n  let callFile = \"\";\n  if (logInferenceToFile) {\n    const { fileName, timestamp } = writeTimestampedTxtFile(\n      `observe_summary`,\n      `observe_call`,\n      {\n        modelCall: \"observe\",\n        messages,\n      },\n    );\n    callFile = fileName;\n    callTimestamp = timestamp;\n  }\n\n  const start = Date.now();\n  const rawResponse = await llmClient.createChatCompletion<ObserveResponse>({\n    options: {\n      messages,\n      response_model: {\n        schema: observeSchema,\n        name: \"Observation\",\n      },\n      temperature: isGPT5 ? 1 : 0.1,\n      top_p: 1,\n      frequency_penalty: 0,\n      presence_penalty: 0,\n    },\n    logger,\n  });\n  const end = Date.now();\n  const usageTimeMs = end - start;\n\n  const { data: observeData, usage: observeUsage } = rawResponse;\n  const promptTokens = observeUsage?.prompt_tokens ?? 0;\n  const completionTokens = observeUsage?.completion_tokens ?? 0;\n  const reasoningTokens = observeUsage?.reasoning_tokens ?? 0;\n  const cachedInputTokens = observeUsage?.cached_input_tokens ?? 0;\n\n  let responseFile: string;\n  if (logInferenceToFile) {\n    const { fileName: responseFileName } = writeTimestampedTxtFile(\n      `observe_summary`,\n      `observe_response`,\n      {\n        modelResponse: \"observe\",\n        rawResponse: observeData,\n      },\n    );\n    responseFile = responseFileName;\n\n    appendSummary(\"observe\", {\n      [`observe_inference_type`]: \"observe\",\n      timestamp: callTimestamp,\n      LLM_input_file: callFile,\n      LLM_output_file: responseFile,\n      prompt_tokens: promptTokens,\n      completion_tokens: completionTokens,\n      reasoning_tokens: reasoningTokens,\n      cached_input_tokens: cachedInputTokens,\n      inference_time_ms: usageTimeMs,\n    });\n  }\n\n  const parsedElements =\n    observeData.elements?.map((el) => {\n      const base = {\n        elementId: el.elementId,\n        description: String(el.description),\n        method: String(el.method),\n        arguments: el.arguments,\n      };\n      return base;\n    }) ?? [];\n\n  return {\n    elements: parsedElements,\n    prompt_tokens: promptTokens,\n    completion_tokens: completionTokens,\n    reasoning_tokens: reasoningTokens,\n    cached_input_tokens: cachedInputTokens,\n    inference_time_ms: usageTimeMs,\n  };\n}\n\nexport async function act({\n  instruction,\n  domElements,\n  llmClient,\n  userProvidedInstructions,\n  logger,\n  logInferenceToFile = false,\n}: {\n  instruction: string;\n  domElements: string;\n  llmClient: LLMClient;\n  userProvidedInstructions?: string;\n  logger: (message: LogLine) => void;\n  logInferenceToFile?: boolean;\n}) {\n  const isGPT5 = llmClient.modelName.includes(\"gpt-5\"); // TODO: remove this as we update support for gpt-5 configuration options\n\n  const actSchema = z.object({\n    elementId: z\n      .string()\n      .regex(/^\\d+-\\d+$/)\n      .describe(\n        \"the ID string associated with the element. Never include surrounding square brackets. This field must follow the format of 'number-number'.\",\n      ),\n    description: z\n      .string()\n      .describe(\"a description of the accessible element and its purpose\"),\n    method: z\n      .enum(\n        // Use Object.values() for Zod v3 compatibility - z.enum() in v3 doesn't accept TypeScript enums directly\n        Object.values(SupportedUnderstudyAction) as unknown as readonly [\n          string,\n          ...string[],\n        ],\n      )\n      .describe(\n        \"the candidate method/action to interact with the element. Select one of the available Understudy interaction methods.\",\n      ),\n    arguments: z.array(\n      z\n        .string()\n        .describe(\n          \"the arguments to pass to the method. For example, for a click, the arguments are empty, but for a fill, the arguments are the value to fill in.\",\n        ),\n    ),\n    twoStep: z.boolean(),\n  });\n\n  type ActResponse = z.infer<typeof actSchema>;\n\n  const messages: ChatMessage[] = [\n    buildActSystemPrompt(userProvidedInstructions),\n    buildObserveUserMessage(instruction, domElements),\n  ];\n\n  let callTimestamp = \"\";\n  let callFile = \"\";\n  if (logInferenceToFile) {\n    const { fileName, timestamp } = writeTimestampedTxtFile(\n      `act_summary`,\n      `act_call`,\n      {\n        modelCall: \"act\",\n        messages,\n      },\n    );\n    callFile = fileName;\n    callTimestamp = timestamp;\n  }\n\n  const start = Date.now();\n  const rawResponse = await llmClient.createChatCompletion<ActResponse>({\n    options: {\n      messages,\n      response_model: {\n        schema: actSchema,\n        name: \"act\",\n      },\n      temperature: isGPT5 ? 1 : 0.1,\n      top_p: 1,\n      frequency_penalty: 0,\n      presence_penalty: 0,\n    },\n    logger,\n  });\n  const end = Date.now();\n  const usageTimeMs = end - start;\n\n  const { data: actData, usage: actUsage } = rawResponse;\n  const promptTokens = actUsage?.prompt_tokens ?? 0;\n  const completionTokens = actUsage?.completion_tokens ?? 0;\n  const reasoningTokens = actUsage?.reasoning_tokens ?? 0;\n  const cachedInputTokens = actUsage?.cached_input_tokens ?? 0;\n\n  let responseFile: string;\n  if (logInferenceToFile) {\n    const { fileName: responseFileName } = writeTimestampedTxtFile(\n      `act_summary`,\n      `act_response`,\n      {\n        modelResponse: \"act\",\n        rawResponse: actData,\n      },\n    );\n    responseFile = responseFileName;\n\n    appendSummary(\"act\", {\n      [`act_inference_type`]: \"act\",\n      timestamp: callTimestamp,\n      LLM_input_file: callFile,\n      LLM_output_file: responseFile,\n      prompt_tokens: promptTokens,\n      completion_tokens: completionTokens,\n      reasoning_tokens: reasoningTokens,\n      cached_input_tokens: cachedInputTokens,\n      inference_time_ms: usageTimeMs,\n    });\n  }\n\n  const parsedElement = {\n    elementId: actData.elementId,\n    description: String(actData.description),\n    method: String(actData.method),\n    arguments: actData.arguments,\n  };\n\n  return {\n    element: parsedElement,\n    prompt_tokens: promptTokens,\n    completion_tokens: completionTokens,\n    reasoning_tokens: reasoningTokens,\n    cached_input_tokens: cachedInputTokens,\n    inference_time_ms: usageTimeMs,\n    twoStep: actData.twoStep,\n  };\n}\n"]}

package/dist/esm/lib/inferenceLogUtils.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Appends a new entry to the act_summary.json file, then writes the file back out.
+ */
+export declare function appendSummary<T>(inferenceType: string, entry: T): void;
+/**
+ * Writes `data` as JSON into a file in `directory`, using a prefix plus timestamp.
+ * Returns both the file name and the timestamp used, so you can log them.
+ */
+export declare function writeTimestampedTxtFile(directory: string, prefix: string, data: unknown): {
+    fileName: string;
+    timestamp: string;
+};

package/dist/esm/lib/inferenceLogUtils.js

@@ -0,0 +1,92 @@
+import fs from "fs";
+import path from "path";
+/**
+ * Create (or ensure) a parent directory named "inference_summary".
+ */
+function ensureInferenceSummaryDir() {
+    const inferenceDir = path.join(process.cwd(), "inference_summary");
+    if (!fs.existsSync(inferenceDir)) {
+        fs.mkdirSync(inferenceDir, { recursive: true });
+    }
+    return inferenceDir;
+}
+/**
+ * Appends a new entry to the act_summary.json file, then writes the file back out.
+ */
+export function appendSummary(inferenceType, entry) {
+    const summaryPath = getSummaryJsonPath(inferenceType);
+    const arrayKey = `${inferenceType}_summary`;
+    const existingData = readSummaryFile(inferenceType);
+    existingData[arrayKey].push(entry);
+    fs.writeFileSync(summaryPath, JSON.stringify(existingData, null, 2));
+}
+/** A simple timestamp utility for filenames. */
+function getTimestamp() {
+    return new Date()
+        .toISOString()
+        .replace(/[^0-9T]/g, "")
+        .replace("T", "_");
+}
+/**
+ * Writes `data` as JSON into a file in `directory`, using a prefix plus timestamp.
+ * Returns both the file name and the timestamp used, so you can log them.
+ */
+export function writeTimestampedTxtFile(directory, prefix, data) {
+    const baseDir = ensureInferenceSummaryDir();
+    const subDir = path.join(baseDir, directory);
+    if (!fs.existsSync(subDir)) {
+        fs.mkdirSync(subDir, { recursive: true });
+    }
+    const timestamp = getTimestamp();
+    const fileName = `${timestamp}_${prefix}.txt`;
+    const filePath = path.join(subDir, fileName);
+    fs.writeFileSync(filePath, JSON.stringify(data, null, 2).replace(/\\n/g, "\n"));
+    return { fileName, timestamp };
+}
+/**
+ * Returns the path to the `<inferenceType>_summary.json` file.
+ *
+ * For example, if `inferenceType = "act"`, this will be:
+ *   `./inference_summary/act_summary/act_summary.json`
+ */
+function getSummaryJsonPath(inferenceType) {
+    const baseDir = ensureInferenceSummaryDir();
+    const subDir = path.join(baseDir, `${inferenceType}_summary`);
+    if (!fs.existsSync(subDir)) {
+        fs.mkdirSync(subDir, { recursive: true });
+    }
+    return path.join(subDir, `${inferenceType}_summary.json`);
+}
+/**
+ * Reads the `<inferenceType>_summary.json` file, returning an object
+ * with the top-level array named `<inferenceType>_summary`, if it exists.
+ *
+ * E.g. if inferenceType is "act", we expect a shape like:
+ * {
+ *   "act_summary": [ ... ]
+ * }
+ *
+ * If the file or array is missing, returns { "<inferenceType>_summary": [] }.
+ */
+function readSummaryFile(inferenceType) {
+    const summaryPath = getSummaryJsonPath(inferenceType);
+    // The top-level array key, e.g. "act_summary", "observe_summary", "extract_summary"
+    const arrayKey = `${inferenceType}_summary`;
+    if (!fs.existsSync(summaryPath)) {
+        return { [arrayKey]: [] };
+    }
+    try {
+        const raw = fs.readFileSync(summaryPath, "utf8");
+        const parsed = JSON.parse(raw);
+        if (parsed &&
+            typeof parsed === "object" &&
+            Array.isArray(parsed[arrayKey])) {
+            return parsed;
+        }
+    }
+    catch {
+        // If we fail to parse for any reason, fall back to empty array
+    }
+    return { [arrayKey]: [] };
+}
+//# sourceMappingURL=inferenceLogUtils.js.map

package/dist/esm/lib/inferenceLogUtils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"inferenceLogUtils.js","sourceRoot":"","sources":["../../../lib/inferenceLogUtils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB;;GAEG;AACH,SAAS,yBAAyB;IAChC,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,mBAAmB,CAAC,CAAC;IACnE,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QACjC,EAAE,CAAC,SAAS,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAClD,CAAC;IACD,OAAO,YAAY,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAI,aAAqB,EAAE,KAAQ;IAC9D,MAAM,WAAW,GAAG,kBAAkB,CAAC,aAAa,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,GAAG,aAAa,UAAU,CAAC;IAE5C,MAAM,YAAY,GAAG,eAAe,CAAI,aAAa,CAAC,CAAC;IACvD,YAAY,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAEnC,EAAE,CAAC,aAAa,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACvE,CAAC;AAED,gDAAgD;AAChD,SAAS,YAAY;IACnB,OAAO,IAAI,IAAI,EAAE;SACd,WAAW,EAAE;SACb,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;AACvB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CACrC,SAAiB,EACjB,MAAc,EACd,IAAa;IAEb,MAAM,OAAO,GAAG,yBAAyB,EAAE,CAAC;IAE5C,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IAC7C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3B,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAG,GAAG,SAAS,IAAI,MAAM,MAAM,CAAC;IAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAE7C,EAAE,CAAC,aAAa,CACd,QAAQ,EACR,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CACpD,CAAC;IAEF,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;AACjC,CAAC;AAED;;;;;GAKG;AACH,SAAS,kBAAkB,CAAC,aAAqB;IAC/C,MAAM,OAAO,GAAG,yBAAyB,EAAE,CAAC;IAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,aAAa,UAAU,CAAC,CAAC;IAC9D,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3B,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,aAAa,eAAe,CAAC,CAAC;AAC5D,CAAC;AAED;;;;;;;;;;GAUG;AACH,SAAS,eAAe,CAAI,aAAqB;IAC/C,MAAM,WAAW,GAAG,kBAAkB,CAAC,aAAa,CAAC,CAAC;IAEtD,oFAAoF;IACpF,MAAM,QAAQ,GAAG,GAAG,aAAa,UAAU,CAAC;IAE5C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAChC,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC;IAC5B,CAAC;IAED,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/B,IACE,MAAM;YACN,OAAO,MAAM,KAAK,QAAQ;YAC1B,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAC/B,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,+DAA+D;IACjE,CAAC;IACD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC;AAC5B,CAAC","sourcesContent":["import fs from \"fs\";\nimport path from \"path\";\n\n/**\n * Create (or ensure) a parent directory named \"inference_summary\".\n */\nfunction ensureInferenceSummaryDir(): string {\n  const inferenceDir = path.join(process.cwd(), \"inference_summary\");\n  if (!fs.existsSync(inferenceDir)) {\n    fs.mkdirSync(inferenceDir, { recursive: true });\n  }\n  return inferenceDir;\n}\n\n/**\n * Appends a new entry to the act_summary.json file, then writes the file back out.\n */\nexport function appendSummary<T>(inferenceType: string, entry: T) {\n  const summaryPath = getSummaryJsonPath(inferenceType);\n  const arrayKey = `${inferenceType}_summary`;\n\n  const existingData = readSummaryFile<T>(inferenceType);\n  existingData[arrayKey].push(entry);\n\n  fs.writeFileSync(summaryPath, JSON.stringify(existingData, null, 2));\n}\n\n/** A simple timestamp utility for filenames. */\nfunction getTimestamp(): string {\n  return new Date()\n    .toISOString()\n    .replace(/[^0-9T]/g, \"\")\n    .replace(\"T\", \"_\");\n}\n\n/**\n * Writes `data` as JSON into a file in `directory`, using a prefix plus timestamp.\n * Returns both the file name and the timestamp used, so you can log them.\n */\nexport function writeTimestampedTxtFile(\n  directory: string,\n  prefix: string,\n  data: unknown,\n): { fileName: string; timestamp: string } {\n  const baseDir = ensureInferenceSummaryDir();\n\n  const subDir = path.join(baseDir, directory);\n  if (!fs.existsSync(subDir)) {\n    fs.mkdirSync(subDir, { recursive: true });\n  }\n\n  const timestamp = getTimestamp();\n  const fileName = `${timestamp}_${prefix}.txt`;\n  const filePath = path.join(subDir, fileName);\n\n  fs.writeFileSync(\n    filePath,\n    JSON.stringify(data, null, 2).replace(/\\\\n/g, \"\\n\"),\n  );\n\n  return { fileName, timestamp };\n}\n\n/**\n * Returns the path to the `<inferenceType>_summary.json` file.\n *\n * For example, if `inferenceType = \"act\"`, this will be:\n *   `./inference_summary/act_summary/act_summary.json`\n */\nfunction getSummaryJsonPath(inferenceType: string): string {\n  const baseDir = ensureInferenceSummaryDir();\n  const subDir = path.join(baseDir, `${inferenceType}_summary`);\n  if (!fs.existsSync(subDir)) {\n    fs.mkdirSync(subDir, { recursive: true });\n  }\n  return path.join(subDir, `${inferenceType}_summary.json`);\n}\n\n/**\n * Reads the `<inferenceType>_summary.json` file, returning an object\n * with the top-level array named `<inferenceType>_summary`, if it exists.\n *\n * E.g. if inferenceType is \"act\", we expect a shape like:\n * {\n *   \"act_summary\": [ ... ]\n * }\n *\n * If the file or array is missing, returns { \"<inferenceType>_summary\": [] }.\n */\nfunction readSummaryFile<T>(inferenceType: string): Record<string, T[]> {\n  const summaryPath = getSummaryJsonPath(inferenceType);\n\n  // The top-level array key, e.g. \"act_summary\", \"observe_summary\", \"extract_summary\"\n  const arrayKey = `${inferenceType}_summary`;\n\n  if (!fs.existsSync(summaryPath)) {\n    return { [arrayKey]: [] };\n  }\n\n  try {\n    const raw = fs.readFileSync(summaryPath, \"utf8\");\n    const parsed = JSON.parse(raw);\n    if (\n      parsed &&\n      typeof parsed === \"object\" &&\n      Array.isArray(parsed[arrayKey])\n    ) {\n      return parsed;\n    }\n  } catch {\n    // If we fail to parse for any reason, fall back to empty array\n  }\n  return { [arrayKey]: [] };\n}\n"]}

package/dist/esm/lib/logger.d.ts

@@ -0,0 +1,69 @@
+import pino from "pino";
+import { LogLine } from "./v3/types/public/logs.js";
+export interface LoggerOptions {
+    pretty?: boolean;
+    level?: pino.Level;
+    destination?: pino.DestinationStream;
+    usePino?: boolean;
+}
+/**
+ * Creates a configured Pino logger instance
+ */
+export declare function createLogger(options?: LoggerOptions): pino.Logger<never, boolean>;
+/**
+ * StagehandLogger class that wraps Pino for our specific needs
+ *
+ * LOGGING PRECEDENCE:
+ *
+ * Test environments:
+ *   - External logger provided -> external logger only.
+ *   - No external logger -> console fallback only (Pino disabled).
+ *
+ * Non-test environments:
+ *   - usePino === true -> emit via Pino and also call the external logger when present.
+ *   - usePino === false -> disable Pino; use the external logger when present, otherwise console fallback.
+ *   - usePino === undefined -> prefer the external logger when present; otherwise use Pino.
+ *
+ * SHARED PINO OPTIMIZATION:
+ * We maintain a single shared Pino instance when `usePino` is enabled.
+ * This prevents spawning a new worker thread for every Stagehand instance
+ * (which happens when `pino-pretty` transport is used), eliminating the
+ * memory/RSS growth observed when many Stagehand objects are created and
+ * disposed within the same process (e.g. a request-per-instance API).
+ */
+export declare class StagehandLogger {
+    /**
+     * Shared Pino logger instance across all StagehandLogger instances.
+     * First instance to enable Pino creates it, subsequent instances reuse it.
+     */
+    private static sharedPinoLogger;
+    private logger?;
+    private verbose;
+    private externalLogger?;
+    private usePino;
+    private isTest;
+    constructor(options?: LoggerOptions, externalLogger?: (logLine: LogLine) => void);
+    /**
+     * Set the verbosity level
+     */
+    setVerbosity(level: 0 | 1 | 2): void;
+    /**
+     * Log a message using our LogLine format
+     */
+    log(logLine: LogLine): void;
+    /**
+     * Helper to format auxiliary data for structured logging
+     */
+    private formatAuxiliaryData;
+    /**
+     * Convenience methods for different log levels
+     */
+    error(message: string, data?: Record<string, unknown>): void;
+    warn(message: string, data?: Record<string, unknown>): void;
+    info(message: string, data?: Record<string, unknown>): void;
+    debug(message: string, data?: Record<string, unknown>): void;
+    /**
+     * Convert a plain object to our auxiliary format
+     */
+    private convertToAuxiliary;
+}