@browserbasehq/orca 3.0.9-alpha-3 → 3.1.0-headers

package/dist/cjs/cli.js

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+"use strict";
+require("./lib/v3/cli.js");

package/dist/cjs/index.d.ts

@@ -0,0 +1,2 @@
+export * from "./lib/v3/index";
+export { default } from "./lib/v3/index";

package/dist/cjs/index.js

@@ -0,0 +1,2 @@
+"use strict";
+module.exports = require("./lib/v3/index.js");

package/dist/cjs/lib/inference.d.ts

@@ -0,0 +1,65 @@
+import { LogLine } from "./v3/types/public/logs.js";
+import { LLMClient } from "./v3/llm/LLMClient.js";
+import type { InferStagehandSchema, StagehandZodObject } from "./v3/zodCompat.js";
+export type { LLMParsedResponse, LLMUsage } from "./v3/llm/LLMClient.js";
+export declare function extract<T extends StagehandZodObject>({ instruction, domElements, schema, llmClient, logger, userProvidedInstructions, logInferenceToFile, }: {
+    instruction: string;
+    domElements: string;
+    schema: T;
+    llmClient: LLMClient;
+    userProvidedInstructions?: string;
+    logger: (message: LogLine) => void;
+    logInferenceToFile?: boolean;
+}): Promise<InferStagehandSchema<T> & {
+    metadata: {
+        completed: boolean;
+        progress: string;
+    };
+    prompt_tokens: number;
+    completion_tokens: number;
+    reasoning_tokens: number;
+    cached_input_tokens: number;
+    inference_time_ms: number;
+}>;
+export declare function observe({ instruction, domElements, llmClient, userProvidedInstructions, logger, logInferenceToFile, supportedActions, }: {
+    instruction: string;
+    domElements: string;
+    llmClient: LLMClient;
+    userProvidedInstructions?: string;
+    logger: (message: LogLine) => void;
+    logInferenceToFile?: boolean;
+    supportedActions?: string[];
+}): Promise<{
+    elements: {
+        elementId: string;
+        description: string;
+        method: string;
+        arguments: string[];
+    }[];
+    prompt_tokens: number;
+    completion_tokens: number;
+    reasoning_tokens: number;
+    cached_input_tokens: number;
+    inference_time_ms: number;
+}>;
+export declare function act({ instruction, domElements, llmClient, userProvidedInstructions, logger, logInferenceToFile, }: {
+    instruction: string;
+    domElements: string;
+    llmClient: LLMClient;
+    userProvidedInstructions?: string;
+    logger: (message: LogLine) => void;
+    logInferenceToFile?: boolean;
+}): Promise<{
+    element: {
+        elementId: string;
+        description: string;
+        method: string;
+        arguments: string[];
+    };
+    prompt_tokens: number;
+    completion_tokens: number;
+    reasoning_tokens: number;
+    cached_input_tokens: number;
+    inference_time_ms: number;
+    twoStep: boolean;
+}>;

package/dist/cjs/lib/inference.js

@@ -0,0 +1,335 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extract = extract;
+exports.observe = observe;
+exports.act = act;
+const zod_1 = require("zod");
+const timeoutConfig_js_1 = require("./v3/timeoutConfig.js");
+const prompt_js_1 = require("./prompt.js");
+const inferenceLogUtils_js_1 = require("./inferenceLogUtils.js");
+const handlers_js_1 = require("./v3/types/private/handlers.js");
+function withLlmTimeout(promise, operation) {
+    return (0, timeoutConfig_js_1.withTimeout)(promise, (0, timeoutConfig_js_1.getEnvTimeoutMs)("LLM_MAX_MS"), `LLM ${operation}`);
+}
+async function extract({ instruction, domElements, schema, llmClient, logger, userProvidedInstructions, logInferenceToFile = false, }) {
+    const metadataSchema = zod_1.z.object({
+        progress: zod_1.z
+            .string()
+            .describe("progress of what has been extracted so far, as concise as possible"),
+        completed: zod_1.z
+            .boolean()
+            .describe("true if the goal is now accomplished. Use this conservatively, only when sure that the goal has been completed."),
+    });
+    const isUsingAnthropic = llmClient.type === "anthropic";
+    const isGPT5 = llmClient.modelName.includes("gpt-5"); // TODO: remove this as we update support for gpt-5 configuration options
+    const extractCallMessages = [
+        (0, prompt_js_1.buildExtractSystemPrompt)(isUsingAnthropic, userProvidedInstructions),
+        (0, prompt_js_1.buildExtractUserPrompt)(instruction, domElements, isUsingAnthropic),
+    ];
+    let extractCallFile = "";
+    let extractCallTimestamp = "";
+    if (logInferenceToFile) {
+        const { fileName, timestamp } = (0, inferenceLogUtils_js_1.writeTimestampedTxtFile)("extract_summary", "extract_call", {
+            modelCall: "extract",
+            messages: extractCallMessages,
+        });
+        extractCallFile = fileName;
+        extractCallTimestamp = timestamp;
+    }
+    const extractStartTime = Date.now();
+    const extractionResponse = await withLlmTimeout(llmClient.createChatCompletion({
+        options: {
+            messages: extractCallMessages,
+            response_model: {
+                schema,
+                name: "Extraction",
+            },
+            temperature: isGPT5 ? 1 : 0.1,
+            top_p: 1,
+            frequency_penalty: 0,
+            presence_penalty: 0,
+        },
+        logger,
+    }), "extract");
+    const extractEndTime = Date.now();
+    const { data: extractedData, usage: extractUsage } = extractionResponse;
+    let extractResponseFile;
+    if (logInferenceToFile) {
+        const { fileName } = (0, inferenceLogUtils_js_1.writeTimestampedTxtFile)("extract_summary", "extract_response", {
+            modelResponse: "extract",
+            rawResponse: extractedData,
+        });
+        extractResponseFile = fileName;
+        (0, inferenceLogUtils_js_1.appendSummary)("extract", {
+            extract_inference_type: "extract",
+            timestamp: extractCallTimestamp,
+            LLM_input_file: extractCallFile,
+            LLM_output_file: extractResponseFile,
+            prompt_tokens: extractUsage?.prompt_tokens ?? 0,
+            completion_tokens: extractUsage?.completion_tokens ?? 0,
+            reasoning_tokens: extractUsage?.reasoning_tokens ?? 0,
+            cached_input_tokens: extractUsage?.cached_input_tokens ?? 0,
+            inference_time_ms: extractEndTime - extractStartTime,
+        });
+    }
+    const metadataCallMessages = [
+        (0, prompt_js_1.buildMetadataSystemPrompt)(),
+        (0, prompt_js_1.buildMetadataPrompt)(instruction, extractedData),
+    ];
+    let metadataCallFile = "";
+    let metadataCallTimestamp = "";
+    if (logInferenceToFile) {
+        const { fileName, timestamp } = (0, inferenceLogUtils_js_1.writeTimestampedTxtFile)("extract_summary", "metadata_call", {
+            modelCall: "metadata",
+            messages: metadataCallMessages,
+        });
+        metadataCallFile = fileName;
+        metadataCallTimestamp = timestamp;
+    }
+    const metadataStartTime = Date.now();
+    const metadataResponse = await withLlmTimeout(llmClient.createChatCompletion({
+        options: {
+            messages: metadataCallMessages,
+            response_model: {
+                name: "Metadata",
+                schema: metadataSchema,
+            },
+            temperature: isGPT5 ? 1 : 0.1,
+            top_p: 1,
+            frequency_penalty: 0,
+            presence_penalty: 0,
+        },
+        logger,
+    }), "extract metadata");
+    const metadataEndTime = Date.now();
+    const { data: { completed: metadataResponseCompleted, progress: metadataResponseProgress, }, usage: metadataResponseUsage, } = metadataResponse;
+    let metadataResponseFile;
+    if (logInferenceToFile) {
+        const { fileName } = (0, inferenceLogUtils_js_1.writeTimestampedTxtFile)("extract_summary", "metadata_response", {
+            modelResponse: "metadata",
+            completed: metadataResponseCompleted,
+            progress: metadataResponseProgress,
+        });
+        metadataResponseFile = fileName;
+        (0, inferenceLogUtils_js_1.appendSummary)("extract", {
+            extract_inference_type: "metadata",
+            timestamp: metadataCallTimestamp,
+            LLM_input_file: metadataCallFile,
+            LLM_output_file: metadataResponseFile,
+            prompt_tokens: metadataResponseUsage?.prompt_tokens ?? 0,
+            completion_tokens: metadataResponseUsage?.completion_tokens ?? 0,
+            reasoning_tokens: metadataResponseUsage?.reasoning_tokens ?? 0,
+            cached_input_tokens: metadataResponseUsage?.cached_input_tokens ?? 0,
+            inference_time_ms: metadataEndTime - metadataStartTime,
+        });
+    }
+    const totalPromptTokens = (extractUsage?.prompt_tokens ?? 0) +
+        (metadataResponseUsage?.prompt_tokens ?? 0);
+    const totalCompletionTokens = (extractUsage?.completion_tokens ?? 0) +
+        (metadataResponseUsage?.completion_tokens ?? 0);
+    const totalInferenceTimeMs = extractEndTime - extractStartTime + (metadataEndTime - metadataStartTime);
+    const totalReasoningTokens = (extractUsage?.reasoning_tokens ?? 0) +
+        (metadataResponseUsage?.reasoning_tokens ?? 0);
+    const totalCachedInputTokens = (extractUsage?.cached_input_tokens ?? 0) +
+        (metadataResponseUsage?.cached_input_tokens ?? 0);
+    return {
+        ...extractedData,
+        metadata: {
+            completed: metadataResponseCompleted,
+            progress: metadataResponseProgress,
+        },
+        prompt_tokens: totalPromptTokens,
+        completion_tokens: totalCompletionTokens,
+        reasoning_tokens: totalReasoningTokens,
+        cached_input_tokens: totalCachedInputTokens,
+        inference_time_ms: totalInferenceTimeMs,
+    };
+}
+async function observe({ instruction, domElements, llmClient, userProvidedInstructions, logger, logInferenceToFile = false, supportedActions, }) {
+    const isGPT5 = llmClient.modelName.includes("gpt-5"); // TODO: remove this as we update support for gpt-5 configuration options
+    const observeSchema = zod_1.z.object({
+        elements: zod_1.z
+            .array(zod_1.z.object({
+            elementId: zod_1.z
+                .string()
+                .regex(/^\d+-\d+$/)
+                .describe("the ID string associated with the element. Never include surrounding square brackets. This field must follow the format of 'number-number'."),
+            description: zod_1.z
+                .string()
+                .describe("a description of the accessible element and its purpose"),
+            method: zod_1.z
+                .enum(
+            // Use Object.values() for Zod v3 compatibility - z.enum() in v3 doesn't accept TypeScript enums directly
+            Object.values(handlers_js_1.SupportedUnderstudyAction))
+                .describe(`the candidate method/action to interact with the element. Select one of the available Understudy interaction methods.`),
+            arguments: zod_1.z.array(zod_1.z
+                .string()
+                .describe("the arguments to pass to the method. For example, for a click, the arguments are empty, but for a fill, the arguments are the value to fill in.")),
+        }))
+            .describe("an array of accessible elements that match the instruction"),
+    });
+    const messages = [
+        (0, prompt_js_1.buildObserveSystemPrompt)(userProvidedInstructions, supportedActions),
+        (0, prompt_js_1.buildObserveUserMessage)(instruction, domElements),
+    ];
+    let callTimestamp = "";
+    let callFile = "";
+    if (logInferenceToFile) {
+        const { fileName, timestamp } = (0, inferenceLogUtils_js_1.writeTimestampedTxtFile)(`observe_summary`, `observe_call`, {
+            modelCall: "observe",
+            messages,
+        });
+        callFile = fileName;
+        callTimestamp = timestamp;
+    }
+    const start = Date.now();
+    const rawResponse = await llmClient.createChatCompletion({
+        options: {
+            messages,
+            response_model: {
+                schema: observeSchema,
+                name: "Observation",
+            },
+            temperature: isGPT5 ? 1 : 0.1,
+            top_p: 1,
+            frequency_penalty: 0,
+            presence_penalty: 0,
+        },
+        logger,
+    });
+    const end = Date.now();
+    const usageTimeMs = end - start;
+    const { data: observeData, usage: observeUsage } = rawResponse;
+    const promptTokens = observeUsage?.prompt_tokens ?? 0;
+    const completionTokens = observeUsage?.completion_tokens ?? 0;
+    const reasoningTokens = observeUsage?.reasoning_tokens ?? 0;
+    const cachedInputTokens = observeUsage?.cached_input_tokens ?? 0;
+    let responseFile;
+    if (logInferenceToFile) {
+        const { fileName: responseFileName } = (0, inferenceLogUtils_js_1.writeTimestampedTxtFile)(`observe_summary`, `observe_response`, {
+            modelResponse: "observe",
+            rawResponse: observeData,
+        });
+        responseFile = responseFileName;
+        (0, inferenceLogUtils_js_1.appendSummary)("observe", {
+            [`observe_inference_type`]: "observe",
+            timestamp: callTimestamp,
+            LLM_input_file: callFile,
+            LLM_output_file: responseFile,
+            prompt_tokens: promptTokens,
+            completion_tokens: completionTokens,
+            reasoning_tokens: reasoningTokens,
+            cached_input_tokens: cachedInputTokens,
+            inference_time_ms: usageTimeMs,
+        });
+    }
+    const parsedElements = observeData.elements?.map((el) => {
+        const base = {
+            elementId: el.elementId,
+            description: String(el.description),
+            method: String(el.method),
+            arguments: el.arguments,
+        };
+        return base;
+    }) ?? [];
+    return {
+        elements: parsedElements,
+        prompt_tokens: promptTokens,
+        completion_tokens: completionTokens,
+        reasoning_tokens: reasoningTokens,
+        cached_input_tokens: cachedInputTokens,
+        inference_time_ms: usageTimeMs,
+    };
+}
+async function act({ instruction, domElements, llmClient, userProvidedInstructions, logger, logInferenceToFile = false, }) {
+    const isGPT5 = llmClient.modelName.includes("gpt-5"); // TODO: remove this as we update support for gpt-5 configuration options
+    const actSchema = zod_1.z.object({
+        elementId: zod_1.z
+            .string()
+            .regex(/^\d+-\d+$/)
+            .describe("the ID string associated with the element. Never include surrounding square brackets. This field must follow the format of 'number-number'."),
+        description: zod_1.z
+            .string()
+            .describe("a description of the accessible element and its purpose"),
+        method: zod_1.z
+            .enum(
+        // Use Object.values() for Zod v3 compatibility - z.enum() in v3 doesn't accept TypeScript enums directly
+        Object.values(handlers_js_1.SupportedUnderstudyAction))
+            .describe("the candidate method/action to interact with the element. Select one of the available Understudy interaction methods."),
+        arguments: zod_1.z.array(zod_1.z
+            .string()
+            .describe("the arguments to pass to the method. For example, for a click, the arguments are empty, but for a fill, the arguments are the value to fill in.")),
+        twoStep: zod_1.z.boolean(),
+    });
+    const messages = [
+        (0, prompt_js_1.buildActSystemPrompt)(userProvidedInstructions),
+        (0, prompt_js_1.buildObserveUserMessage)(instruction, domElements),
+    ];
+    let callTimestamp = "";
+    let callFile = "";
+    if (logInferenceToFile) {
+        const { fileName, timestamp } = (0, inferenceLogUtils_js_1.writeTimestampedTxtFile)(`act_summary`, `act_call`, {
+            modelCall: "act",
+            messages,
+        });
+        callFile = fileName;
+        callTimestamp = timestamp;
+    }
+    const start = Date.now();
+    const rawResponse = await llmClient.createChatCompletion({
+        options: {
+            messages,
+            response_model: {
+                schema: actSchema,
+                name: "act",
+            },
+            temperature: isGPT5 ? 1 : 0.1,
+            top_p: 1,
+            frequency_penalty: 0,
+            presence_penalty: 0,
+        },
+        logger,
+    });
+    const end = Date.now();
+    const usageTimeMs = end - start;
+    const { data: actData, usage: actUsage } = rawResponse;
+    const promptTokens = actUsage?.prompt_tokens ?? 0;
+    const completionTokens = actUsage?.completion_tokens ?? 0;
+    const reasoningTokens = actUsage?.reasoning_tokens ?? 0;
+    const cachedInputTokens = actUsage?.cached_input_tokens ?? 0;
+    let responseFile;
+    if (logInferenceToFile) {
+        const { fileName: responseFileName } = (0, inferenceLogUtils_js_1.writeTimestampedTxtFile)(`act_summary`, `act_response`, {
+            modelResponse: "act",
+            rawResponse: actData,
+        });
+        responseFile = responseFileName;
+        (0, inferenceLogUtils_js_1.appendSummary)("act", {
+            [`act_inference_type`]: "act",
+            timestamp: callTimestamp,
+            LLM_input_file: callFile,
+            LLM_output_file: responseFile,
+            prompt_tokens: promptTokens,
+            completion_tokens: completionTokens,
+            reasoning_tokens: reasoningTokens,
+            cached_input_tokens: cachedInputTokens,
+            inference_time_ms: usageTimeMs,
+        });
+    }
+    const parsedElement = {
+        elementId: actData.elementId,
+        description: String(actData.description),
+        method: String(actData.method),
+        arguments: actData.arguments,
+    };
+    return {
+        element: parsedElement,
+        prompt_tokens: promptTokens,
+        completion_tokens: completionTokens,
+        reasoning_tokens: reasoningTokens,
+        cached_input_tokens: cachedInputTokens,
+        inference_time_ms: usageTimeMs,
+        twoStep: actData.twoStep,
+    };
+}
+//# sourceMappingURL=inference.js.map

package/dist/cjs/lib/inference.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"inference.js","sourceRoot":"","sources":["../../../lib/inference.ts"],"names":[],"mappings":";;AA+BA,0BA8MC;AAED,0BAmJC;AAED,kBAuIC;AA3gBD,6BAAwB;AAGxB,4DAAqE;AACrE,2CAQqB;AACrB,iEAAgF;AAKhF,gEAA2E;AAK3E,SAAS,cAAc,CAAI,OAAmB,EAAE,SAAiB;IAC/D,OAAO,IAAA,8BAAW,EAChB,OAAO,EACP,IAAA,kCAAe,EAAC,YAAY,CAAC,EAC7B,OAAO,SAAS,EAAE,CACnB,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,OAAO,CAA+B,EAC1D,WAAW,EACX,WAAW,EACX,MAAM,EACN,SAAS,EACT,MAAM,EACN,wBAAwB,EACxB,kBAAkB,GAAG,KAAK,GAS3B;IACC,MAAM,cAAc,GAAG,OAAC,CAAC,MAAM,CAAC;QAC9B,QAAQ,EAAE,OAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,oEAAoE,CACrE;QACH,SAAS,EAAE,OAAC;aACT,OAAO,EAAE;aACT,QAAQ,CACP,iHAAiH,CAClH;KACJ,CAAC,CAAC;IAKH,MAAM,gBAAgB,GAAG,SAAS,CAAC,IAAI,KAAK,WAAW,CAAC;IACxD,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,yEAAyE;IAE/H,MAAM,mBAAmB,GAAkB;QACzC,IAAA,oCAAwB,EAAC,gBAAgB,EAAE,wBAAwB,CAAC;QACpE,IAAA,kCAAsB,EAAC,WAAW,EAAE,WAAW,EAAE,gBAAgB,CAAC;KACnE,CAAC;IAEF,IAAI,eAAe,GAAG,EAAE,CAAC;IACzB,IAAI,oBAAoB,GAAG,EAAE,CAAC;IAC9B,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAA,8CAAuB,EACrD,iBAAiB,EACjB,cAAc,EACd;YACE,SAAS,EAAE,SAAS;YACpB,QAAQ,EAAE,mBAAmB;SAC9B,CACF,CAAC;QACF,eAAe,GAAG,QAAQ,CAAC;QAC3B,oBAAoB,GAAG,SAAS,CAAC;IACnC,CAAC;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACpC,MAAM,kBAAkB,GAAG,MAAM,cAAc,CAC7C,SAAS,CAAC,oBAAoB,CAAqB;QACjD,OAAO,EAAE;YACP,QAAQ,EAAE,mBAAmB;YAC7B,cAAc,EAAE;gBACd,MAAM;gBACN,IAAI,EAAE,YAAY;aACnB;YACD,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;YAC7B,KAAK,EAAE,CAAC;YACR,iBAAiB,EAAE,CAAC;YACpB,gBAAgB,EAAE,CAAC;SACpB;QACD,MAAM;KACP,CAAC,EACF,SAAS,CACV,CAAC;IACF,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAElC,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,kBAAkB,CAAC;IAExE,IAAI,mBAA2B,CAAC;IAChC,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAA,8CAAuB,EAC1C,iBAAiB,EACjB,kBAAkB,EAClB;YACE,aAAa,EAAE,SAAS;YACxB,WAAW,EAAE,aAAa;SAC3B,CACF,CAAC;QACF,mBAAmB,GAAG,QAAQ,CAAC;QAE/B,IAAA,oCAAa,EAAC,SAAS,EAAE;YACvB,sBAAsB,EAAE,SAAS;YACjC,SAAS,EAAE,oBAAoB;YAC/B,cAAc,EAAE,eAAe;YAC/B,eAAe,EAAE,mBAAmB;YACpC,aAAa,EAAE,YAAY,EAAE,aAAa,IAAI,CAAC;YAC/C,iBAAiB,EAAE,YAAY,EAAE,iBAAiB,IAAI,CAAC;YACvD,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,IAAI,CAAC;YACrD,mBAAmB,EAAE,YAAY,EAAE,mBAAmB,IAAI,CAAC;YAC3D,iBAAiB,EAAE,cAAc,GAAG,gBAAgB;SACrD,CAAC,CAAC;IACL,CAAC;IAED,MAAM,oBAAoB,GAAkB;QAC1C,IAAA,qCAAyB,GAAE;QAC3B,IAAA,+BAAmB,EAAC,WAAW,EAAE,aAAa,CAAC;KAChD,CAAC;IAEF,IAAI,gBAAgB,GAAG,EAAE,CAAC;IAC1B,IAAI,qBAAqB,GAAG,EAAE,CAAC;IAC/B,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAA,8CAAuB,EACrD,iBAAiB,EACjB,eAAe,EACf;YACE,SAAS,EAAE,UAAU;YACrB,QAAQ,EAAE,oBAAoB;SAC/B,CACF,CAAC;QACF,gBAAgB,GAAG,QAAQ,CAAC;QAC5B,qBAAqB,GAAG,SAAS,CAAC;IACpC,CAAC;IAED,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACrC,MAAM,gBAAgB,GAAG,MAAM,cAAc,CAC3C,SAAS,CAAC,oBAAoB,CAAmB;QAC/C,OAAO,EAAE;YACP,QAAQ,EAAE,oBAAoB;YAC9B,cAAc,EAAE;gBACd,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,cAAc;aACvB;YACD,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;YAC7B,KAAK,EAAE,CAAC;YACR,iBAAiB,EAAE,CAAC;YACpB,gBAAgB,EAAE,CAAC;SACpB;QACD,MAAM;KACP,CAAC,EACF,kBAAkB,CACnB,CAAC;IACF,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEnC,MAAM,EACJ,IAAI,EAAE,EACJ,SAAS,EAAE,yBAAyB,EACpC,QAAQ,EAAE,wBAAwB,GACnC,EACD,KAAK,EAAE,qBAAqB,GAC7B,GAAG,gBAAgB,CAAC;IAErB,IAAI,oBAA4B,CAAC;IACjC,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAA,8CAAuB,EAC1C,iBAAiB,EACjB,mBAAmB,EACnB;YACE,aAAa,EAAE,UAAU;YACzB,SAAS,EAAE,yBAAyB;YACpC,QAAQ,EAAE,wBAAwB;SACnC,CACF,CAAC;QACF,oBAAoB,GAAG,QAAQ,CAAC;QAEhC,IAAA,oCAAa,EAAC,SAAS,EAAE;YACvB,sBAAsB,EAAE,UAAU;YAClC,SAAS,EAAE,qBAAqB;YAChC,cAAc,EAAE,gBAAgB;YAChC,eAAe,EAAE,oBAAoB;YACrC,aAAa,EAAE,qBAAqB,EAAE,aAAa,IAAI,CAAC;YACxD,iBAAiB,EAAE,qBAAqB,EAAE,iBAAiB,IAAI,CAAC;YAChE,gBAAgB,EAAE,qBAAqB,EAAE,gBAAgB,IAAI,CAAC;YAC9D,mBAAmB,EAAE,qBAAqB,EAAE,mBAAmB,IAAI,CAAC;YACpE,iBAAiB,EAAE,eAAe,GAAG,iBAAiB;SACvD,CAAC,CAAC;IACL,CAAC;IAED,MAAM,iBAAiB,GACrB,CAAC,YAAY,EAAE,aAAa,IAAI,CAAC,CAAC;QAClC,CAAC,qBAAqB,EAAE,aAAa,IAAI,CAAC,CAAC,CAAC;IAE9C,MAAM,qBAAqB,GACzB,CAAC,YAAY,EAAE,iBAAiB,IAAI,CAAC,CAAC;QACtC,CAAC,qBAAqB,EAAE,iBAAiB,IAAI,CAAC,CAAC,CAAC;IAElD,MAAM,oBAAoB,GACxB,cAAc,GAAG,gBAAgB,GAAG,CAAC,eAAe,GAAG,iBAAiB,CAAC,CAAC;IAC5E,MAAM,oBAAoB,GACxB,CAAC,YAAY,EAAE,gBAAgB,IAAI,CAAC,CAAC;QACrC,CAAC,qBAAqB,EAAE,gBAAgB,IAAI,CAAC,CAAC,CAAC;IACjD,MAAM,sBAAsB,GAC1B,CAAC,YAAY,EAAE,mBAAmB,IAAI,CAAC,CAAC;QACxC,CAAC,qBAAqB,EAAE,mBAAmB,IAAI,CAAC,CAAC,CAAC;IAEpD,OAAO;QACL,GAAG,aAAa;QAChB,QAAQ,EAAE;YACR,SAAS,EAAE,yBAAyB;YACpC,QAAQ,EAAE,wBAAwB;SACnC;QACD,aAAa,EAAE,iBAAiB;QAChC,iBAAiB,EAAE,qBAAqB;QACxC,gBAAgB,EAAE,oBAAoB;QACtC,mBAAmB,EAAE,sBAAsB;QAC3C,iBAAiB,EAAE,oBAAoB;KACxC,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,OAAO,CAAC,EAC5B,WAAW,EACX,WAAW,EACX,SAAS,EACT,wBAAwB,EACxB,MAAM,EACN,kBAAkB,GAAG,KAAK,EAC1B,gBAAgB,GASjB;IACC,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,yEAAyE;IAE/H,MAAM,aAAa,GAAG,OAAC,CAAC,MAAM,CAAC;QAC7B,QAAQ,EAAE,OAAC;aACR,KAAK,CACJ,OAAC,CAAC,MAAM,CAAC;YACP,SAAS,EAAE,OAAC;iBACT,MAAM,EAAE;iBACR,KAAK,CAAC,WAAW,CAAC;iBAClB,QAAQ,CACP,6IAA6I,CAC9I;YACH,WAAW,EAAE,OAAC;iBACX,MAAM,EAAE;iBACR,QAAQ,CACP,yDAAyD,CAC1D;YACH,MAAM,EAAE,OAAC;iBACN,IAAI;YACH,yGAAyG;YACzG,MAAM,CAAC,MAAM,CAAC,uCAAyB,CAGtC,CACF;iBACA,QAAQ,CACP,uHAAuH,CACxH;YACH,SAAS,EAAE,OAAC,CAAC,KAAK,CAChB,OAAC;iBACE,MAAM,EAAE;iBACR,QAAQ,CACP,iJAAiJ,CAClJ,CACJ;SACF,CAAC,CACH;aACA,QAAQ,CAAC,4DAA4D,CAAC;KAC1E,CAAC,CAAC;IAIH,MAAM,QAAQ,GAAkB;QAC9B,IAAA,oCAAwB,EAAC,wBAAwB,EAAE,gBAAgB,CAAC;QACpE,IAAA,mCAAuB,EAAC,WAAW,EAAE,WAAW,CAAC;KAClD,CAAC;IAEF,IAAI,aAAa,GAAG,EAAE,CAAC;IACvB,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAA,8CAAuB,EACrD,iBAAiB,EACjB,cAAc,EACd;YACE,SAAS,EAAE,SAAS;YACpB,QAAQ;SACT,CACF,CAAC;QACF,QAAQ,GAAG,QAAQ,CAAC;QACpB,aAAa,GAAG,SAAS,CAAC;IAC5B,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAAkB;QACxE,OAAO,EAAE;YACP,QAAQ;YACR,cAAc,EAAE;gBACd,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,aAAa;aACpB;YACD,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;YAC7B,KAAK,EAAE,CAAC;YACR,iBAAiB,EAAE,CAAC;YACpB,gBAAgB,EAAE,CAAC;SACpB;QACD,MAAM;KACP,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,WAAW,GAAG,GAAG,GAAG,KAAK,CAAC;IAEhC,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,WAAW,CAAC;IAC/D,MAAM,YAAY,GAAG,YAAY,EAAE,aAAa,IAAI,CAAC,CAAC;IACtD,MAAM,gBAAgB,GAAG,YAAY,EAAE,iBAAiB,IAAI,CAAC,CAAC;IAC9D,MAAM,eAAe,GAAG,YAAY,EAAE,gBAAgB,IAAI,CAAC,CAAC;IAC5D,MAAM,iBAAiB,GAAG,YAAY,EAAE,mBAAmB,IAAI,CAAC,CAAC;IAEjE,IAAI,YAAoB,CAAC;IACzB,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,gBAAgB,EAAE,GAAG,IAAA,8CAAuB,EAC5D,iBAAiB,EACjB,kBAAkB,EAClB;YACE,aAAa,EAAE,SAAS;YACxB,WAAW,EAAE,WAAW;SACzB,CACF,CAAC;QACF,YAAY,GAAG,gBAAgB,CAAC;QAEhC,IAAA,oCAAa,EAAC,SAAS,EAAE;YACvB,CAAC,wBAAwB,CAAC,EAAE,SAAS;YACrC,SAAS,EAAE,aAAa;YACxB,cAAc,EAAE,QAAQ;YACxB,eAAe,EAAE,YAAY;YAC7B,aAAa,EAAE,YAAY;YAC3B,iBAAiB,EAAE,gBAAgB;YACnC,gBAAgB,EAAE,eAAe;YACjC,mBAAmB,EAAE,iBAAiB;YACtC,iBAAiB,EAAE,WAAW;SAC/B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,cAAc,GAClB,WAAW,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QAC/B,MAAM,IAAI,GAAG;YACX,SAAS,EAAE,EAAE,CAAC,SAAS;YACvB,WAAW,EAAE,MAAM,CAAC,EAAE,CAAC,WAAW,CAAC;YACnC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;YACzB,SAAS,EAAE,EAAE,CAAC,SAAS;SACxB,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,IAAI,EAAE,CAAC;IAEX,OAAO;QACL,QAAQ,EAAE,cAAc;QACxB,aAAa,EAAE,YAAY;QAC3B,iBAAiB,EAAE,gBAAgB;QACnC,gBAAgB,EAAE,eAAe;QACjC,mBAAmB,EAAE,iBAAiB;QACtC,iBAAiB,EAAE,WAAW;KAC/B,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,GAAG,CAAC,EACxB,WAAW,EACX,WAAW,EACX,SAAS,EACT,wBAAwB,EACxB,MAAM,EACN,kBAAkB,GAAG,KAAK,GAQ3B;IACC,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,yEAAyE;IAE/H,MAAM,SAAS,GAAG,OAAC,CAAC,MAAM,CAAC;QACzB,SAAS,EAAE,OAAC;aACT,MAAM,EAAE;aACR,KAAK,CAAC,WAAW,CAAC;aAClB,QAAQ,CACP,6IAA6I,CAC9I;QACH,WAAW,EAAE,OAAC;aACX,MAAM,EAAE;aACR,QAAQ,CAAC,yDAAyD,CAAC;QACtE,MAAM,EAAE,OAAC;aACN,IAAI;QACH,yGAAyG;QACzG,MAAM,CAAC,MAAM,CAAC,uCAAyB,CAGtC,CACF;aACA,QAAQ,CACP,uHAAuH,CACxH;QACH,SAAS,EAAE,OAAC,CAAC,KAAK,CAChB,OAAC;aACE,MAAM,EAAE;aACR,QAAQ,CACP,iJAAiJ,CAClJ,CACJ;QACD,OAAO,EAAE,OAAC,CAAC,OAAO,EAAE;KACrB,CAAC,CAAC;IAIH,MAAM,QAAQ,GAAkB;QAC9B,IAAA,gCAAoB,EAAC,wBAAwB,CAAC;QAC9C,IAAA,mCAAuB,EAAC,WAAW,EAAE,WAAW,CAAC;KAClD,CAAC;IAEF,IAAI,aAAa,GAAG,EAAE,CAAC;IACvB,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAA,8CAAuB,EACrD,aAAa,EACb,UAAU,EACV;YACE,SAAS,EAAE,KAAK;YAChB,QAAQ;SACT,CACF,CAAC;QACF,QAAQ,GAAG,QAAQ,CAAC;QACpB,aAAa,GAAG,SAAS,CAAC;IAC5B,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAAc;QACpE,OAAO,EAAE;YACP,QAAQ;YACR,cAAc,EAAE;gBACd,MAAM,EAAE,SAAS;gBACjB,IAAI,EAAE,KAAK;aACZ;YACD,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;YAC7B,KAAK,EAAE,CAAC;YACR,iBAAiB,EAAE,CAAC;YACpB,gBAAgB,EAAE,CAAC;SACpB;QACD,MAAM;KACP,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,WAAW,GAAG,GAAG,GAAG,KAAK,CAAC;IAEhC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,WAAW,CAAC;IACvD,MAAM,YAAY,GAAG,QAAQ,EAAE,aAAa,IAAI,CAAC,CAAC;IAClD,MAAM,gBAAgB,GAAG,QAAQ,EAAE,iBAAiB,IAAI,CAAC,CAAC;IAC1D,MAAM,eAAe,GAAG,QAAQ,EAAE,gBAAgB,IAAI,CAAC,CAAC;IACxD,MAAM,iBAAiB,GAAG,QAAQ,EAAE,mBAAmB,IAAI,CAAC,CAAC;IAE7D,IAAI,YAAoB,CAAC;IACzB,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,gBAAgB,EAAE,GAAG,IAAA,8CAAuB,EAC5D,aAAa,EACb,cAAc,EACd;YACE,aAAa,EAAE,KAAK;YACpB,WAAW,EAAE,OAAO;SACrB,CACF,CAAC;QACF,YAAY,GAAG,gBAAgB,CAAC;QAEhC,IAAA,oCAAa,EAAC,KAAK,EAAE;YACnB,CAAC,oBAAoB,CAAC,EAAE,KAAK;YAC7B,SAAS,EAAE,aAAa;YACxB,cAAc,EAAE,QAAQ;YACxB,eAAe,EAAE,YAAY;YAC7B,aAAa,EAAE,YAAY;YAC3B,iBAAiB,EAAE,gBAAgB;YACnC,gBAAgB,EAAE,eAAe;YACjC,mBAAmB,EAAE,iBAAiB;YACtC,iBAAiB,EAAE,WAAW;SAC/B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,aAAa,GAAG;QACpB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,WAAW,EAAE,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC;QACxC,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;QAC9B,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC;IAEF,OAAO;QACL,OAAO,EAAE,aAAa;QACtB,aAAa,EAAE,YAAY;QAC3B,iBAAiB,EAAE,gBAAgB;QACnC,gBAAgB,EAAE,eAAe;QACjC,mBAAmB,EAAE,iBAAiB;QACtC,iBAAiB,EAAE,WAAW;QAC9B,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC;AACJ,CAAC","sourcesContent":["import { z } from \"zod\";\nimport { LogLine } from \"./v3/types/public/logs.js\";\nimport { ChatMessage, LLMClient } from \"./v3/llm/LLMClient.js\";\nimport { getEnvTimeoutMs, withTimeout } from \"./v3/timeoutConfig.js\";\nimport {\n  buildActSystemPrompt,\n  buildExtractSystemPrompt,\n  buildExtractUserPrompt,\n  buildMetadataPrompt,\n  buildMetadataSystemPrompt,\n  buildObserveSystemPrompt,\n  buildObserveUserMessage,\n} from \"./prompt.js\";\nimport { appendSummary, writeTimestampedTxtFile } from \"./inferenceLogUtils.js\";\nimport type {\n  InferStagehandSchema,\n  StagehandZodObject,\n} from \"./v3/zodCompat.js\";\nimport { SupportedUnderstudyAction } from \"./v3/types/private/handlers.js\";\n\n// Re-export for backward compatibility\nexport type { LLMParsedResponse, LLMUsage } from \"./v3/llm/LLMClient.js\";\n\nfunction withLlmTimeout<T>(promise: Promise<T>, operation: string): Promise<T> {\n  return withTimeout(\n    promise,\n    getEnvTimeoutMs(\"LLM_MAX_MS\"),\n    `LLM ${operation}`,\n  );\n}\n\nexport async function extract<T extends StagehandZodObject>({\n  instruction,\n  domElements,\n  schema,\n  llmClient,\n  logger,\n  userProvidedInstructions,\n  logInferenceToFile = false,\n}: {\n  instruction: string;\n  domElements: string;\n  schema: T;\n  llmClient: LLMClient;\n  userProvidedInstructions?: string;\n  logger: (message: LogLine) => void;\n  logInferenceToFile?: boolean;\n}) {\n  const metadataSchema = z.object({\n    progress: z\n      .string()\n      .describe(\n        \"progress of what has been extracted so far, as concise as possible\",\n      ),\n    completed: z\n      .boolean()\n      .describe(\n        \"true if the goal is now accomplished. Use this conservatively, only when sure that the goal has been completed.\",\n      ),\n  });\n\n  type ExtractionResponse = InferStagehandSchema<T>;\n  type MetadataResponse = z.infer<typeof metadataSchema>;\n\n  const isUsingAnthropic = llmClient.type === \"anthropic\";\n  const isGPT5 = llmClient.modelName.includes(\"gpt-5\"); // TODO: remove this as we update support for gpt-5 configuration options\n\n  const extractCallMessages: ChatMessage[] = [\n    buildExtractSystemPrompt(isUsingAnthropic, userProvidedInstructions),\n    buildExtractUserPrompt(instruction, domElements, isUsingAnthropic),\n  ];\n\n  let extractCallFile = \"\";\n  let extractCallTimestamp = \"\";\n  if (logInferenceToFile) {\n    const { fileName, timestamp } = writeTimestampedTxtFile(\n      \"extract_summary\",\n      \"extract_call\",\n      {\n        modelCall: \"extract\",\n        messages: extractCallMessages,\n      },\n    );\n    extractCallFile = fileName;\n    extractCallTimestamp = timestamp;\n  }\n\n  const extractStartTime = Date.now();\n  const extractionResponse = await withLlmTimeout(\n    llmClient.createChatCompletion<ExtractionResponse>({\n      options: {\n        messages: extractCallMessages,\n        response_model: {\n          schema,\n          name: \"Extraction\",\n        },\n        temperature: isGPT5 ? 1 : 0.1,\n        top_p: 1,\n        frequency_penalty: 0,\n        presence_penalty: 0,\n      },\n      logger,\n    }),\n    \"extract\",\n  );\n  const extractEndTime = Date.now();\n\n  const { data: extractedData, usage: extractUsage } = extractionResponse;\n\n  let extractResponseFile: string;\n  if (logInferenceToFile) {\n    const { fileName } = writeTimestampedTxtFile(\n      \"extract_summary\",\n      \"extract_response\",\n      {\n        modelResponse: \"extract\",\n        rawResponse: extractedData,\n      },\n    );\n    extractResponseFile = fileName;\n\n    appendSummary(\"extract\", {\n      extract_inference_type: \"extract\",\n      timestamp: extractCallTimestamp,\n      LLM_input_file: extractCallFile,\n      LLM_output_file: extractResponseFile,\n      prompt_tokens: extractUsage?.prompt_tokens ?? 0,\n      completion_tokens: extractUsage?.completion_tokens ?? 0,\n      reasoning_tokens: extractUsage?.reasoning_tokens ?? 0,\n      cached_input_tokens: extractUsage?.cached_input_tokens ?? 0,\n      inference_time_ms: extractEndTime - extractStartTime,\n    });\n  }\n\n  const metadataCallMessages: ChatMessage[] = [\n    buildMetadataSystemPrompt(),\n    buildMetadataPrompt(instruction, extractedData),\n  ];\n\n  let metadataCallFile = \"\";\n  let metadataCallTimestamp = \"\";\n  if (logInferenceToFile) {\n    const { fileName, timestamp } = writeTimestampedTxtFile(\n      \"extract_summary\",\n      \"metadata_call\",\n      {\n        modelCall: \"metadata\",\n        messages: metadataCallMessages,\n      },\n    );\n    metadataCallFile = fileName;\n    metadataCallTimestamp = timestamp;\n  }\n\n  const metadataStartTime = Date.now();\n  const metadataResponse = await withLlmTimeout(\n    llmClient.createChatCompletion<MetadataResponse>({\n      options: {\n        messages: metadataCallMessages,\n        response_model: {\n          name: \"Metadata\",\n          schema: metadataSchema,\n        },\n        temperature: isGPT5 ? 1 : 0.1,\n        top_p: 1,\n        frequency_penalty: 0,\n        presence_penalty: 0,\n      },\n      logger,\n    }),\n    \"extract metadata\",\n  );\n  const metadataEndTime = Date.now();\n\n  const {\n    data: {\n      completed: metadataResponseCompleted,\n      progress: metadataResponseProgress,\n    },\n    usage: metadataResponseUsage,\n  } = metadataResponse;\n\n  let metadataResponseFile: string;\n  if (logInferenceToFile) {\n    const { fileName } = writeTimestampedTxtFile(\n      \"extract_summary\",\n      \"metadata_response\",\n      {\n        modelResponse: \"metadata\",\n        completed: metadataResponseCompleted,\n        progress: metadataResponseProgress,\n      },\n    );\n    metadataResponseFile = fileName;\n\n    appendSummary(\"extract\", {\n      extract_inference_type: \"metadata\",\n      timestamp: metadataCallTimestamp,\n      LLM_input_file: metadataCallFile,\n      LLM_output_file: metadataResponseFile,\n      prompt_tokens: metadataResponseUsage?.prompt_tokens ?? 0,\n      completion_tokens: metadataResponseUsage?.completion_tokens ?? 0,\n      reasoning_tokens: metadataResponseUsage?.reasoning_tokens ?? 0,\n      cached_input_tokens: metadataResponseUsage?.cached_input_tokens ?? 0,\n      inference_time_ms: metadataEndTime - metadataStartTime,\n    });\n  }\n\n  const totalPromptTokens =\n    (extractUsage?.prompt_tokens ?? 0) +\n    (metadataResponseUsage?.prompt_tokens ?? 0);\n\n  const totalCompletionTokens =\n    (extractUsage?.completion_tokens ?? 0) +\n    (metadataResponseUsage?.completion_tokens ?? 0);\n\n  const totalInferenceTimeMs =\n    extractEndTime - extractStartTime + (metadataEndTime - metadataStartTime);\n  const totalReasoningTokens =\n    (extractUsage?.reasoning_tokens ?? 0) +\n    (metadataResponseUsage?.reasoning_tokens ?? 0);\n  const totalCachedInputTokens =\n    (extractUsage?.cached_input_tokens ?? 0) +\n    (metadataResponseUsage?.cached_input_tokens ?? 0);\n\n  return {\n    ...extractedData,\n    metadata: {\n      completed: metadataResponseCompleted,\n      progress: metadataResponseProgress,\n    },\n    prompt_tokens: totalPromptTokens,\n    completion_tokens: totalCompletionTokens,\n    reasoning_tokens: totalReasoningTokens,\n    cached_input_tokens: totalCachedInputTokens,\n    inference_time_ms: totalInferenceTimeMs,\n  };\n}\n\nexport async function observe({\n  instruction,\n  domElements,\n  llmClient,\n  userProvidedInstructions,\n  logger,\n  logInferenceToFile = false,\n  supportedActions,\n}: {\n  instruction: string;\n  domElements: string;\n  llmClient: LLMClient;\n  userProvidedInstructions?: string;\n  logger: (message: LogLine) => void;\n  logInferenceToFile?: boolean;\n  supportedActions?: string[];\n}) {\n  const isGPT5 = llmClient.modelName.includes(\"gpt-5\"); // TODO: remove this as we update support for gpt-5 configuration options\n\n  const observeSchema = z.object({\n    elements: z\n      .array(\n        z.object({\n          elementId: z\n            .string()\n            .regex(/^\\d+-\\d+$/)\n            .describe(\n              \"the ID string associated with the element. Never include surrounding square brackets. This field must follow the format of 'number-number'.\",\n            ),\n          description: z\n            .string()\n            .describe(\n              \"a description of the accessible element and its purpose\",\n            ),\n          method: z\n            .enum(\n              // Use Object.values() for Zod v3 compatibility - z.enum() in v3 doesn't accept TypeScript enums directly\n              Object.values(SupportedUnderstudyAction) as unknown as readonly [\n                string,\n                ...string[],\n              ],\n            )\n            .describe(\n              `the candidate method/action to interact with the element. Select one of the available Understudy interaction methods.`,\n            ),\n          arguments: z.array(\n            z\n              .string()\n              .describe(\n                \"the arguments to pass to the method. For example, for a click, the arguments are empty, but for a fill, the arguments are the value to fill in.\",\n              ),\n          ),\n        }),\n      )\n      .describe(\"an array of accessible elements that match the instruction\"),\n  });\n\n  type ObserveResponse = z.infer<typeof observeSchema>;\n\n  const messages: ChatMessage[] = [\n    buildObserveSystemPrompt(userProvidedInstructions, supportedActions),\n    buildObserveUserMessage(instruction, domElements),\n  ];\n\n  let callTimestamp = \"\";\n  let callFile = \"\";\n  if (logInferenceToFile) {\n    const { fileName, timestamp } = writeTimestampedTxtFile(\n      `observe_summary`,\n      `observe_call`,\n      {\n        modelCall: \"observe\",\n        messages,\n      },\n    );\n    callFile = fileName;\n    callTimestamp = timestamp;\n  }\n\n  const start = Date.now();\n  const rawResponse = await llmClient.createChatCompletion<ObserveResponse>({\n    options: {\n      messages,\n      response_model: {\n        schema: observeSchema,\n        name: \"Observation\",\n      },\n      temperature: isGPT5 ? 1 : 0.1,\n      top_p: 1,\n      frequency_penalty: 0,\n      presence_penalty: 0,\n    },\n    logger,\n  });\n  const end = Date.now();\n  const usageTimeMs = end - start;\n\n  const { data: observeData, usage: observeUsage } = rawResponse;\n  const promptTokens = observeUsage?.prompt_tokens ?? 0;\n  const completionTokens = observeUsage?.completion_tokens ?? 0;\n  const reasoningTokens = observeUsage?.reasoning_tokens ?? 0;\n  const cachedInputTokens = observeUsage?.cached_input_tokens ?? 0;\n\n  let responseFile: string;\n  if (logInferenceToFile) {\n    const { fileName: responseFileName } = writeTimestampedTxtFile(\n      `observe_summary`,\n      `observe_response`,\n      {\n        modelResponse: \"observe\",\n        rawResponse: observeData,\n      },\n    );\n    responseFile = responseFileName;\n\n    appendSummary(\"observe\", {\n      [`observe_inference_type`]: \"observe\",\n      timestamp: callTimestamp,\n      LLM_input_file: callFile,\n      LLM_output_file: responseFile,\n      prompt_tokens: promptTokens,\n      completion_tokens: completionTokens,\n      reasoning_tokens: reasoningTokens,\n      cached_input_tokens: cachedInputTokens,\n      inference_time_ms: usageTimeMs,\n    });\n  }\n\n  const parsedElements =\n    observeData.elements?.map((el) => {\n      const base = {\n        elementId: el.elementId,\n        description: String(el.description),\n        method: String(el.method),\n        arguments: el.arguments,\n      };\n      return base;\n    }) ?? [];\n\n  return {\n    elements: parsedElements,\n    prompt_tokens: promptTokens,\n    completion_tokens: completionTokens,\n    reasoning_tokens: reasoningTokens,\n    cached_input_tokens: cachedInputTokens,\n    inference_time_ms: usageTimeMs,\n  };\n}\n\nexport async function act({\n  instruction,\n  domElements,\n  llmClient,\n  userProvidedInstructions,\n  logger,\n  logInferenceToFile = false,\n}: {\n  instruction: string;\n  domElements: string;\n  llmClient: LLMClient;\n  userProvidedInstructions?: string;\n  logger: (message: LogLine) => void;\n  logInferenceToFile?: boolean;\n}) {\n  const isGPT5 = llmClient.modelName.includes(\"gpt-5\"); // TODO: remove this as we update support for gpt-5 configuration options\n\n  const actSchema = z.object({\n    elementId: z\n      .string()\n      .regex(/^\\d+-\\d+$/)\n      .describe(\n        \"the ID string associated with the element. Never include surrounding square brackets. This field must follow the format of 'number-number'.\",\n      ),\n    description: z\n      .string()\n      .describe(\"a description of the accessible element and its purpose\"),\n    method: z\n      .enum(\n        // Use Object.values() for Zod v3 compatibility - z.enum() in v3 doesn't accept TypeScript enums directly\n        Object.values(SupportedUnderstudyAction) as unknown as readonly [\n          string,\n          ...string[],\n        ],\n      )\n      .describe(\n        \"the candidate method/action to interact with the element. Select one of the available Understudy interaction methods.\",\n      ),\n    arguments: z.array(\n      z\n        .string()\n        .describe(\n          \"the arguments to pass to the method. For example, for a click, the arguments are empty, but for a fill, the arguments are the value to fill in.\",\n        ),\n    ),\n    twoStep: z.boolean(),\n  });\n\n  type ActResponse = z.infer<typeof actSchema>;\n\n  const messages: ChatMessage[] = [\n    buildActSystemPrompt(userProvidedInstructions),\n    buildObserveUserMessage(instruction, domElements),\n  ];\n\n  let callTimestamp = \"\";\n  let callFile = \"\";\n  if (logInferenceToFile) {\n    const { fileName, timestamp } = writeTimestampedTxtFile(\n      `act_summary`,\n      `act_call`,\n      {\n        modelCall: \"act\",\n        messages,\n      },\n    );\n    callFile = fileName;\n    callTimestamp = timestamp;\n  }\n\n  const start = Date.now();\n  const rawResponse = await llmClient.createChatCompletion<ActResponse>({\n    options: {\n      messages,\n      response_model: {\n        schema: actSchema,\n        name: \"act\",\n      },\n      temperature: isGPT5 ? 1 : 0.1,\n      top_p: 1,\n      frequency_penalty: 0,\n      presence_penalty: 0,\n    },\n    logger,\n  });\n  const end = Date.now();\n  const usageTimeMs = end - start;\n\n  const { data: actData, usage: actUsage } = rawResponse;\n  const promptTokens = actUsage?.prompt_tokens ?? 0;\n  const completionTokens = actUsage?.completion_tokens ?? 0;\n  const reasoningTokens = actUsage?.reasoning_tokens ?? 0;\n  const cachedInputTokens = actUsage?.cached_input_tokens ?? 0;\n\n  let responseFile: string;\n  if (logInferenceToFile) {\n    const { fileName: responseFileName } = writeTimestampedTxtFile(\n      `act_summary`,\n      `act_response`,\n      {\n        modelResponse: \"act\",\n        rawResponse: actData,\n      },\n    );\n    responseFile = responseFileName;\n\n    appendSummary(\"act\", {\n      [`act_inference_type`]: \"act\",\n      timestamp: callTimestamp,\n      LLM_input_file: callFile,\n      LLM_output_file: responseFile,\n      prompt_tokens: promptTokens,\n      completion_tokens: completionTokens,\n      reasoning_tokens: reasoningTokens,\n      cached_input_tokens: cachedInputTokens,\n      inference_time_ms: usageTimeMs,\n    });\n  }\n\n  const parsedElement = {\n    elementId: actData.elementId,\n    description: String(actData.description),\n    method: String(actData.method),\n    arguments: actData.arguments,\n  };\n\n  return {\n    element: parsedElement,\n    prompt_tokens: promptTokens,\n    completion_tokens: completionTokens,\n    reasoning_tokens: reasoningTokens,\n    cached_input_tokens: cachedInputTokens,\n    inference_time_ms: usageTimeMs,\n    twoStep: actData.twoStep,\n  };\n}\n"]}

package/dist/cjs/lib/inferenceLogUtils.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Appends a new entry to the act_summary.json file, then writes the file back out.
+ */
+export declare function appendSummary<T>(inferenceType: string, entry: T): void;
+/**
+ * Writes `data` as JSON into a file in `directory`, using a prefix plus timestamp.
+ * Returns both the file name and the timestamp used, so you can log them.
+ */
+export declare function writeTimestampedTxtFile(directory: string, prefix: string, data: unknown): {
+    fileName: string;
+    timestamp: string;
+};

package/dist/cjs/lib/inferenceLogUtils.js

@@ -0,0 +1,99 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.appendSummary = appendSummary;
+exports.writeTimestampedTxtFile = writeTimestampedTxtFile;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+/**
+ * Create (or ensure) a parent directory named "inference_summary".
+ */
+function ensureInferenceSummaryDir() {
+    const inferenceDir = path_1.default.join(process.cwd(), "inference_summary");
+    if (!fs_1.default.existsSync(inferenceDir)) {
+        fs_1.default.mkdirSync(inferenceDir, { recursive: true });
+    }
+    return inferenceDir;
+}
+/**
+ * Appends a new entry to the act_summary.json file, then writes the file back out.
+ */
+function appendSummary(inferenceType, entry) {
+    const summaryPath = getSummaryJsonPath(inferenceType);
+    const arrayKey = `${inferenceType}_summary`;
+    const existingData = readSummaryFile(inferenceType);
+    existingData[arrayKey].push(entry);
+    fs_1.default.writeFileSync(summaryPath, JSON.stringify(existingData, null, 2));
+}
+/** A simple timestamp utility for filenames. */
+function getTimestamp() {
+    return new Date()
+        .toISOString()
+        .replace(/[^0-9T]/g, "")
+        .replace("T", "_");
+}
+/**
+ * Writes `data` as JSON into a file in `directory`, using a prefix plus timestamp.
+ * Returns both the file name and the timestamp used, so you can log them.
+ */
+function writeTimestampedTxtFile(directory, prefix, data) {
+    const baseDir = ensureInferenceSummaryDir();
+    const subDir = path_1.default.join(baseDir, directory);
+    if (!fs_1.default.existsSync(subDir)) {
+        fs_1.default.mkdirSync(subDir, { recursive: true });
+    }
+    const timestamp = getTimestamp();
+    const fileName = `${timestamp}_${prefix}.txt`;
+    const filePath = path_1.default.join(subDir, fileName);
+    fs_1.default.writeFileSync(filePath, JSON.stringify(data, null, 2).replace(/\\n/g, "\n"));
+    return { fileName, timestamp };
+}
+/**
+ * Returns the path to the `<inferenceType>_summary.json` file.
+ *
+ * For example, if `inferenceType = "act"`, this will be:
+ *   `./inference_summary/act_summary/act_summary.json`
+ */
+function getSummaryJsonPath(inferenceType) {
+    const baseDir = ensureInferenceSummaryDir();
+    const subDir = path_1.default.join(baseDir, `${inferenceType}_summary`);
+    if (!fs_1.default.existsSync(subDir)) {
+        fs_1.default.mkdirSync(subDir, { recursive: true });
+    }
+    return path_1.default.join(subDir, `${inferenceType}_summary.json`);
+}
+/**
+ * Reads the `<inferenceType>_summary.json` file, returning an object
+ * with the top-level array named `<inferenceType>_summary`, if it exists.
+ *
+ * E.g. if inferenceType is "act", we expect a shape like:
+ * {
+ *   "act_summary": [ ... ]
+ * }
+ *
+ * If the file or array is missing, returns { "<inferenceType>_summary": [] }.
+ */
+function readSummaryFile(inferenceType) {
+    const summaryPath = getSummaryJsonPath(inferenceType);
+    // The top-level array key, e.g. "act_summary", "observe_summary", "extract_summary"
+    const arrayKey = `${inferenceType}_summary`;
+    if (!fs_1.default.existsSync(summaryPath)) {
+        return { [arrayKey]: [] };
+    }
+    try {
+        const raw = fs_1.default.readFileSync(summaryPath, "utf8");
+        const parsed = JSON.parse(raw);
+        if (parsed &&
+            typeof parsed === "object" &&
+            Array.isArray(parsed[arrayKey])) {
+            return parsed;
+        }
+    }
+    catch {
+        // If we fail to parse for any reason, fall back to empty array
+    }
+    return { [arrayKey]: [] };
+}
+//# sourceMappingURL=inferenceLogUtils.js.map

package/dist/cjs/lib/inferenceLogUtils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"inferenceLogUtils.js","sourceRoot":"","sources":["../../../lib/inferenceLogUtils.ts"],"names":[],"mappings":";;;;;AAiBA,sCAQC;AAcD,0DAsBC;AA7DD,4CAAoB;AACpB,gDAAwB;AAExB;;GAEG;AACH,SAAS,yBAAyB;IAChC,MAAM,YAAY,GAAG,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,mBAAmB,CAAC,CAAC;IACnE,IAAI,CAAC,YAAE,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QACjC,YAAE,CAAC,SAAS,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAClD,CAAC;IACD,OAAO,YAAY,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,SAAgB,aAAa,CAAI,aAAqB,EAAE,KAAQ;IAC9D,MAAM,WAAW,GAAG,kBAAkB,CAAC,aAAa,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,GAAG,aAAa,UAAU,CAAC;IAE5C,MAAM,YAAY,GAAG,eAAe,CAAI,aAAa,CAAC,CAAC;IACvD,YAAY,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAEnC,YAAE,CAAC,aAAa,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACvE,CAAC;AAED,gDAAgD;AAChD,SAAS,YAAY;IACnB,OAAO,IAAI,IAAI,EAAE;SACd,WAAW,EAAE;SACb,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;AACvB,CAAC;AAED;;;GAGG;AACH,SAAgB,uBAAuB,CACrC,SAAiB,EACjB,MAAc,EACd,IAAa;IAEb,MAAM,OAAO,GAAG,yBAAyB,EAAE,CAAC;IAE5C,MAAM,MAAM,GAAG,cAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IAC7C,IAAI,CAAC,YAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3B,YAAE,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAG,GAAG,SAAS,IAAI,MAAM,MAAM,CAAC;IAC9C,MAAM,QAAQ,GAAG,cAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAE7C,YAAE,CAAC,aAAa,CACd,QAAQ,EACR,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CACpD,CAAC;IAEF,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;AACjC,CAAC;AAED;;;;;GAKG;AACH,SAAS,kBAAkB,CAAC,aAAqB;IAC/C,MAAM,OAAO,GAAG,yBAAyB,EAAE,CAAC;IAC5C,MAAM,MAAM,GAAG,cAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,aAAa,UAAU,CAAC,CAAC;IAC9D,IAAI,CAAC,YAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3B,YAAE,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,cAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,aAAa,eAAe,CAAC,CAAC;AAC5D,CAAC;AAED;;;;;;;;;;GAUG;AACH,SAAS,eAAe,CAAI,aAAqB;IAC/C,MAAM,WAAW,GAAG,kBAAkB,CAAC,aAAa,CAAC,CAAC;IAEtD,oFAAoF;IACpF,MAAM,QAAQ,GAAG,GAAG,aAAa,UAAU,CAAC;IAE5C,IAAI,CAAC,YAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAChC,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC;IAC5B,CAAC;IAED,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,YAAE,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/B,IACE,MAAM;YACN,OAAO,MAAM,KAAK,QAAQ;YAC1B,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAC/B,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,+DAA+D;IACjE,CAAC;IACD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC;AAC5B,CAAC","sourcesContent":["import fs from \"fs\";\nimport path from \"path\";\n\n/**\n * Create (or ensure) a parent directory named \"inference_summary\".\n */\nfunction ensureInferenceSummaryDir(): string {\n  const inferenceDir = path.join(process.cwd(), \"inference_summary\");\n  if (!fs.existsSync(inferenceDir)) {\n    fs.mkdirSync(inferenceDir, { recursive: true });\n  }\n  return inferenceDir;\n}\n\n/**\n * Appends a new entry to the act_summary.json file, then writes the file back out.\n */\nexport function appendSummary<T>(inferenceType: string, entry: T) {\n  const summaryPath = getSummaryJsonPath(inferenceType);\n  const arrayKey = `${inferenceType}_summary`;\n\n  const existingData = readSummaryFile<T>(inferenceType);\n  existingData[arrayKey].push(entry);\n\n  fs.writeFileSync(summaryPath, JSON.stringify(existingData, null, 2));\n}\n\n/** A simple timestamp utility for filenames. */\nfunction getTimestamp(): string {\n  return new Date()\n    .toISOString()\n    .replace(/[^0-9T]/g, \"\")\n    .replace(\"T\", \"_\");\n}\n\n/**\n * Writes `data` as JSON into a file in `directory`, using a prefix plus timestamp.\n * Returns both the file name and the timestamp used, so you can log them.\n */\nexport function writeTimestampedTxtFile(\n  directory: string,\n  prefix: string,\n  data: unknown,\n): { fileName: string; timestamp: string } {\n  const baseDir = ensureInferenceSummaryDir();\n\n  const subDir = path.join(baseDir, directory);\n  if (!fs.existsSync(subDir)) {\n    fs.mkdirSync(subDir, { recursive: true });\n  }\n\n  const timestamp = getTimestamp();\n  const fileName = `${timestamp}_${prefix}.txt`;\n  const filePath = path.join(subDir, fileName);\n\n  fs.writeFileSync(\n    filePath,\n    JSON.stringify(data, null, 2).replace(/\\\\n/g, \"\\n\"),\n  );\n\n  return { fileName, timestamp };\n}\n\n/**\n * Returns the path to the `<inferenceType>_summary.json` file.\n *\n * For example, if `inferenceType = \"act\"`, this will be:\n *   `./inference_summary/act_summary/act_summary.json`\n */\nfunction getSummaryJsonPath(inferenceType: string): string {\n  const baseDir = ensureInferenceSummaryDir();\n  const subDir = path.join(baseDir, `${inferenceType}_summary`);\n  if (!fs.existsSync(subDir)) {\n    fs.mkdirSync(subDir, { recursive: true });\n  }\n  return path.join(subDir, `${inferenceType}_summary.json`);\n}\n\n/**\n * Reads the `<inferenceType>_summary.json` file, returning an object\n * with the top-level array named `<inferenceType>_summary`, if it exists.\n *\n * E.g. if inferenceType is \"act\", we expect a shape like:\n * {\n *   \"act_summary\": [ ... ]\n * }\n *\n * If the file or array is missing, returns { \"<inferenceType>_summary\": [] }.\n */\nfunction readSummaryFile<T>(inferenceType: string): Record<string, T[]> {\n  const summaryPath = getSummaryJsonPath(inferenceType);\n\n  // The top-level array key, e.g. \"act_summary\", \"observe_summary\", \"extract_summary\"\n  const arrayKey = `${inferenceType}_summary`;\n\n  if (!fs.existsSync(summaryPath)) {\n    return { [arrayKey]: [] };\n  }\n\n  try {\n    const raw = fs.readFileSync(summaryPath, \"utf8\");\n    const parsed = JSON.parse(raw);\n    if (\n      parsed &&\n      typeof parsed === \"object\" &&\n      Array.isArray(parsed[arrayKey])\n    ) {\n      return parsed;\n    }\n  } catch {\n    // If we fail to parse for any reason, fall back to empty array\n  }\n  return { [arrayKey]: [] };\n}\n"]}