@empiricalrun/playwright-utils 0.65.0 → 0.65.1

package/dist/test/scripts/pw-locator-patch/vision/query.js

@@ -1,15 +1,67 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.patchQuery = patchQuery;
-const vision_1 = require("@empiricalrun/llm/vision");
+const openai_1 = __importDefault(require("openai"));
 const api_worker_client_1 = require("../../../../api-worker-client");
+const telemetry_1 = require("../../../../telemetry");
+const QUERY_MODEL = "gpt-4o-mini";
+const SYSTEM_PROMPT = `
+You are given an image and an instruction. Use the "explanation" to
+think it through, and then respond with the "answer".`;
+async function queryScreenshot(base64Image, instruction) {
+    const openai = new openai_1.default({ maxRetries: 5, ...(0, api_worker_client_1.llmProxyOpenAIConfig)() });
+    const completion = await openai.chat.completions.create({
+        model: QUERY_MODEL,
+        messages: [
+            { role: "system", content: SYSTEM_PROMPT },
+            {
+                role: "user",
+                content: [
+                    {
+                        type: "image_url",
+                        image_url: { url: `data:image/png;base64,${base64Image}` },
+                    },
+                    { type: "text", text: `Instruction: ${instruction}` },
+                ],
+            },
+        ],
+        temperature: 0.5,
+        response_format: {
+            type: "json_schema",
+            json_schema: {
+                name: "your_response",
+                strict: true,
+                schema: {
+                    type: "object",
+                    properties: {
+                        explanation: { type: "string" },
+                        answer: { type: "string" },
+                    },
+                    required: ["explanation", "answer"],
+                    additionalProperties: false,
+                },
+            },
+        },
+    });
+    await (0, telemetry_1.trackLLMResponseGenerated)({
+        modelName: QUERY_MODEL,
+        tokensInput: completion.usage?.prompt_tokens ?? 0,
+        tokensOutput: completion.usage?.completion_tokens ?? 0,
+        chatSessionSource: "vision_query",
+    });
+    const content = completion.choices[0]?.message?.content;
+    if (!content) {
+        throw new Error("Query failed: no response content from LLM.");
+    }
+    return JSON.parse(content).answer;
+}
 function patchQuery(LocatorClass) {
     LocatorClass.prototype.query = async function (instruction) {
         await this.waitFor({ state: "visible" });
         const screenshot = await this.screenshot();
-        const answer = await (0, vision_1.query)(screenshot.toString("base64"), instruction, {
-            clientConfig: (0, api_worker_client_1.llmProxyOpenAIConfig)(),
-        });
-        return answer;
+        return queryScreenshot(screenshot.toString("base64"), instruction);
     };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/playwright-utils",
-  "version": "0.65.0",
+  "version": "0.65.1",
   "engines": {
     "node": ">=22"
   },
@@ -45,7 +45,7 @@
     "playwright-core": "1.63.0",
     "serve-handler": "^6.1.6",
     "typescript": "7.0.2",
-    "@empiricalrun/shared-types": "0.23.0"
+    "@empiricalrun/shared-types": "0.23.2"
   },
   "dependencies": {
     "async-retry": "^1.3.3",
@@ -56,8 +56,7 @@
     "openapi-fetch": "0.14.1",
     "puppeteer-extra-plugin-recaptcha": "^3.6.8",
     "ts-morph": "^28.0.0",
-    "@empiricalrun/cua": "^0.5.4",
-    "@empiricalrun/llm": "^0.28.3",
+    "@empiricalrun/cua": "^0.6.0",
     "@empiricalrun/r2-uploader": "^0.11.5",
     "@empiricalrun/reporter": "^0.30.3"
   },