@empiricalrun/playwright-utils 0.49.0 → 0.50.0

package/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @empiricalrun/playwright-utils
 
+## 0.50.0
+
+### Minor Changes
+
+- 76d0872: chore: remove older LLM clients
+
+### Patch Changes
+
+- Updated dependencies [76d0872]
+  - @empiricalrun/llm@0.27.0
+  - @empiricalrun/cua@0.4.1
+  - @empiricalrun/dashboard-client@0.3.0
+  - @empiricalrun/reporter@0.29.0
+
 ## 0.49.0
 
 ### Minor Changes

package/dist/telemetry.d.ts

@@ -7,11 +7,6 @@ interface TrackReportUploadParams {
     sizeBytes: number;
 }
 export declare function trackReportUpload(params: TrackReportUploadParams): Promise<void>;
-export declare function trackLLMResponseGenerated(opts: {
-    modelName: string;
-    tokensInput: number;
-    tokensOutput: number;
-}): Promise<void>;
 export declare function trackOverlayDismissed(opts: {
     original: string | undefined;
     description: string;

package/dist/telemetry.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../src/telemetry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,sCAAsC,CAAC;AAKtE,wBAAsB,UAAU,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAqBrE;AAED,UAAU,uBAAuB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,uBAAuB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAcf;AAED,wBAAsB,yBAAyB,CAAC,IAAI,EAAE;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,IAAI,CAAC,CAmBhB;AAED,wBAAsB,qBAAqB,CAAC,IAAI,EAAE;IAChD,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,KAAK,EAAE,OAAO,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C,IAAI,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1C,GAAG,OAAO,CAAC,IAAI,CAAC,CAsBhB"}
+{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../src/telemetry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,sCAAsC,CAAC;AAKtE,wBAAsB,UAAU,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAqBrE;AAED,UAAU,uBAAuB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,uBAAuB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAcf;AAED,wBAAsB,qBAAqB,CAAC,IAAI,EAAE;IAChD,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,KAAK,EAAE,OAAO,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C,IAAI,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1C,GAAG,OAAO,CAAC,IAAI,CAAC,CAsBhB"}

package/dist/telemetry.js

@@ -2,7 +2,6 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.trackEvent = trackEvent;
 exports.trackReportUpload = trackReportUpload;
-exports.trackLLMResponseGenerated = trackLLMResponseGenerated;
 exports.trackOverlayDismissed = trackOverlayDismissed;
 const TELEMETRY_WORKER_URL = "https://telemetry-worker.empirical-run.workers.dev";
 async function trackEvent(event) {
@@ -42,26 +41,6 @@ async function trackReportUpload(params) {
         },
     });
 }
-async function trackLLMResponseGenerated(opts) {
-    const environment = process.env.TEST_RUN_ENVIRONMENT || "unknown";
-    await trackEvent({
-        name: "llm_response_generated",
-        properties: {
-            environment,
-            model_name: opts.modelName,
-            chat_session_id: undefined,
-            chat_session_source: "to_look_right",
-            tokens_input: opts.tokensInput,
-            tokens_output: opts.tokensOutput,
-            cost_input: 0,
-            cost_output: 0,
-            tokens_cache_read: 0,
-            tokens_cache_creation: 0,
-            project_repo_name: process.env.GIT_REPO_NAME ?? "unknown",
-            repo_name: process.env.GIT_REPO_NAME ?? "unknown",
-        },
-    });
-}
 async function trackOverlayDismissed(opts) {
     const environment = process.env.TEST_RUN_ENVIRONMENT || "unknown";
     await trackEvent({

package/dist/test/expect/visual.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"visual.d.ts","sourceRoot":"","sources":["../../../src/test/expect/visual.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAE,IAAI,EAAY,MAAM,kBAAkB,CAAC;AAUhE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAsE7C,wBAAsB,WAAW,CAC/B,MAAM,EAAE,IAAI,GAAG,OAAO,EACtB,eAAe,EAAE,MAAM,GACtB,OAAO,CAAC,aAAa,CAAC,CA2FxB"}
+{"version":3,"file":"visual.d.ts","sourceRoot":"","sources":["../../../src/test/expect/visual.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAE,IAAI,EAAY,MAAM,kBAAkB,CAAC;AAShE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAsE7C,wBAAsB,WAAW,CAC/B,MAAM,EAAE,IAAI,GAAG,OAAO,EACtB,eAAe,EAAE,MAAM,GACtB,OAAO,CAAC,aAAa,CAAC,CAyFxB"}

package/dist/test/expect/visual.js

@@ -7,9 +7,9 @@ exports.toLookRight = toLookRight;
 const llm_1 = require("@empiricalrun/llm");
 const vision_1 = require("@empiricalrun/llm/vision");
 const fs_1 = __importDefault(require("fs"));
+const openai_1 = __importDefault(require("openai"));
 const os_1 = __importDefault(require("os"));
 const path_1 = __importDefault(require("path"));
-const telemetry_1 = require("../../telemetry");
 const mouse_pointer_1 = require("../scripts/mouse-pointer");
 function getCallerLine() {
     const stack = new Error().stack;
@@ -83,11 +83,9 @@ async function toLookRight(target, pageDescription) {
     const base64Image = screenshot.toString("base64");
     await (0, mouse_pointer_1.addHighlighterScriptsToPage)(page);
     const model = "gpt-4o";
-    const llm = new llm_1.LLM({
-        provider: "openai",
-        defaultModel: model,
-    });
-    const response = await llm.createChatCompletion({
+    const openai = new openai_1.default({ maxRetries: 5 });
+    const completion = await openai.chat.completions.create({
+        model,
         messages: [
             {
                 role: "system",
@@ -109,12 +107,10 @@ async function toLookRight(target, pageDescription) {
                 ],
             },
         ],
-        modelParameters: {
-            temperature: 0.1,
-            tool_choice: {
-                type: "function",
-                function: { name: "send_response" },
-            },
+        temperature: 0.1,
+        tool_choice: {
+            type: "function",
+            function: { name: "send_response" },
         },
         tools: [
             {
@@ -143,10 +139,12 @@ go through the reasons and pick the one that made you choose Fail`,
             },
         ],
     });
-    (0, telemetry_1.trackLLMResponseGenerated)({
+    const response = completion.choices[0]?.message;
+    (0, llm_1.trackLLMResponseGenerated)({
         modelName: model,
-        tokensInput: llm.promptTokens,
-        tokensOutput: llm.completionTokens,
+        tokensInput: completion.usage?.prompt_tokens ?? 0,
+        tokensOutput: completion.usage?.completion_tokens ?? 0,
+        chatSessionSource: "to_look_right",
     });
     const rawResponse = response.tool_calls[0];
     if (!("function" in rawResponse)) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/playwright-utils",
-  "version": "0.49.0",
+  "version": "0.50.0",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -27,27 +27,28 @@
     "@types/authenticator": "^1.1.4",
     "@types/babel__code-frame": "^7.0.6",
     "@types/console-log-level": "^1.4.5",
+    "@types/mime": "3.0.0",
     "@types/node": "^20.14.9",
-    "@types/serve-handler": "^6.1.4",
     "@types/pg": "^8.11.6",
-    "@types/mime": "3.0.0",
+    "@types/serve-handler": "^6.1.4",
     "playwright-core": "1.60.0",
     "serve-handler": "^6.1.6",
-    "@empiricalrun/shared-types": "0.13.0"
+    "@empiricalrun/shared-types": "0.14.0"
   },
   "dependencies": {
     "@babel/code-frame": "^7.24.7",
-    "pg": "^8.13.1",
     "async-retry": "^1.3.3",
     "authenticator": "^1.1.5",
     "console-log-level": "^1.4.1",
+    "mime": "3.0.0",
+    "openai": "^6.27.0",
+    "pg": "^8.13.1",
     "puppeteer-extra-plugin-recaptcha": "^3.6.8",
     "rimraf": "^6.0.1",
     "ts-morph": "^23.0.0",
-    "mime": "3.0.0",
-    "@empiricalrun/cua": "^0.4.0",
+    "@empiricalrun/cua": "^0.4.1",
     "@empiricalrun/dashboard-client": "^0.3.0",
-    "@empiricalrun/llm": "^0.26.0",
+    "@empiricalrun/llm": "^0.27.0",
     "@empiricalrun/r2-uploader": "^0.9.1",
     "@empiricalrun/reporter": "^0.29.0"
   },