@empiricalrun/playwright-utils 0.58.4 → 0.58.6

package/CHANGELOG.md

@@ -1,5 +1,29 @@
 # @empiricalrun/playwright-utils
 
+## 0.58.6
+
+### Patch Changes
+
+- bcf5ed7: chore: drop fallbacks for LLM proxy
+- 68091d9: feat: route LLM calls via api-worker proxy for usage attribution
+- Updated dependencies [bcf5ed7]
+- Updated dependencies [68091d9]
+- Updated dependencies [ed03fc8]
+  - @empiricalrun/llm@0.27.1
+  - @empiricalrun/cua@0.4.2
+  - @empiricalrun/reporter@0.30.0
+
+## 0.58.5
+
+### Patch Changes
+
+- 157ea19: fix: incremental blob reporter serializes config and projects like the built-in blob reporter
+- 8e8cea6: fix: incremental blob reporter passes error objects through unmodified, preserving value, cause and AggregateError sub-errors
+- ff99f5e: fix: incremental blob reporter emits step attachments at onStepEnd and relativizes step locations
+- 15e769c: feat: log pending upload counts around run end
+- Updated dependencies [15e769c]
+  - @empiricalrun/r2-uploader@0.10.2
+
 ## 0.58.4
 
 ### Patch Changes

package/dist/api-worker-client.d.ts

@@ -9,6 +9,23 @@ export type TwoCaptchaProxyResponse = components["schemas"]["TwoCaptchaProxyResp
 export type ProjectKvGetResponse = components["schemas"]["ProjectKvGetResponse"];
 export type ProjectKvSetRequest = NonNullable<paths["/api/projects/kv"]["post"]["requestBody"]>["content"]["application/json"];
 export type ProjectKvSetResponse = components["schemas"]["ProjectKvSetResponse"];
+/**
+ * OpenAI client config for the LLM calls playwright-utils makes from inside a
+ * test run — the overlay-dismissal browser agent and the toLookRight visual
+ * assertion. Both go through api-worker's proxy, which holds the provider key
+ * and forwards via the Cloudflare AI Gateway; the run worker puts
+ * EMPIRICALRUN_API_KEY and DASHBOARD_DOMAIN in the Playwright child process.
+ *
+ * There is deliberately no fallback to a direct OPENAI_API_KEY. A fallback
+ * makes the transport depend on ambient env, which hides a broken proxy behind
+ * a working direct call and lets a test that clears one credential still reach
+ * the model. Returns undefined only for a pi sandbox, whose OPENAI_BASE_URL
+ * already points at the sidecar's own keyless proxy route.
+ */
+export declare function llmProxyOpenAIConfig(): {
+    baseURL: string;
+    apiKey: string;
+} | undefined;
 export declare class HTTPError extends Error {
     status: number;
     constructor(message: string, status: number);

package/dist/api-worker-client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"api-worker-client.d.ts","sourceRoot":"","sources":["../src/api-worker-client.ts"],"names":[],"mappings":"AACA,OAAqB,EAAE,KAAK,MAAM,EAAE,MAAM,eAAe,CAAC;AAE1D,OAAO,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAExE,KAAK,mBAAmB,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;AAEzC,KAAK,iBAAiB,GAClB,kBAAkB,GAClB,sBAAsB,GACtB,yBAAyB,CAAC;AAE9B,MAAM,MAAM,oBAAoB,GAC9B,UAAU,CAAC,SAAS,CAAC,CAAC,6BAA6B,CAAC,CAAC;AACvD,MAAM,MAAM,qBAAqB,GAC/B,UAAU,CAAC,SAAS,CAAC,CAAC,8BAA8B,CAAC,CAAC;AACxD,MAAM,MAAM,sBAAsB,GAChC,UAAU,CAAC,SAAS,CAAC,CAAC,wBAAwB,CAAC,CAAC;AAClD,MAAM,MAAM,uBAAuB,GACjC,UAAU,CAAC,SAAS,CAAC,CAAC,yBAAyB,CAAC,CAAC;AACnD,MAAM,MAAM,oBAAoB,GAC9B,UAAU,CAAC,SAAS,CAAC,CAAC,sBAAsB,CAAC,CAAC;AAChD,MAAM,MAAM,mBAAmB,GAAG,WAAW,CAC3C,KAAK,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,CACjD,CAAC,SAAS,CAAC,CAAC,kBAAkB,CAAC,CAAC;AACjC,MAAM,MAAM,oBAAoB,GAC9B,UAAU,CAAC,SAAS,CAAC,CAAC,sBAAsB,CAAC,CAAC;AAEhD,qBAAa,SAAU,SAAQ,KAAK;IAClC,MAAM,EAAE,MAAM,CAAC;IAEf,YAAY,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAG1C;CACF;AAED,qBAAa,qBAAsB,SAAQ,SAAS;CAAG;AACvD,qBAAa,kBAAmB,SAAQ,SAAS;CAAG;AA+CpD,qBAAa,eAAe;IAC1B,OAAO,CAAC,OAAO,CAAS;IACxB,QAAQ,CAAC,MAAM,EAAE,mBAAmB,CAAC;IAErC,YAAY,EACV,aAAa,EACb,OAAO,GACR,GAAE;QACD,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,OAAO,CAAC,EAAE,MAAM,CAAC;KACb,EAiBL;IAED,UAAU,IAAI,MAAM,CAEnB;IAEK,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAK7D;IAEK,YAAY,CAAC,IAAI,EAAE,mBAAmB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAK3E;IAEK,iBAAiB,CACrB,IAAI,EAAE,iBAAiB,EACvB,IAAI,EAAE,oBAAoB,GACzB,OAAO,CAAC,qBAAqB,CAAC,CAKhC;IAED,MAAM,CAAC,CAAC,EACN,MAAM,EAAE;QACN,IAAI,CAAC,EAAE,OAAO,CAAC;QACf,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,QAAQ,EAAE,QAAQ,CAAC;KACpB,EACD,KAAK,EAAE,MAAM,GACZ,CAAC,CAUH;CACF"}
+{"version":3,"file":"api-worker-client.d.ts","sourceRoot":"","sources":["../src/api-worker-client.ts"],"names":[],"mappings":"AACA,OAAqB,EAAE,KAAK,MAAM,EAAE,MAAM,eAAe,CAAC;AAE1D,OAAO,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAExE,KAAK,mBAAmB,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;AAEzC,KAAK,iBAAiB,GAClB,kBAAkB,GAClB,sBAAsB,GACtB,yBAAyB,CAAC;AAE9B,MAAM,MAAM,oBAAoB,GAC9B,UAAU,CAAC,SAAS,CAAC,CAAC,6BAA6B,CAAC,CAAC;AACvD,MAAM,MAAM,qBAAqB,GAC/B,UAAU,CAAC,SAAS,CAAC,CAAC,8BAA8B,CAAC,CAAC;AACxD,MAAM,MAAM,sBAAsB,GAChC,UAAU,CAAC,SAAS,CAAC,CAAC,wBAAwB,CAAC,CAAC;AAClD,MAAM,MAAM,uBAAuB,GACjC,UAAU,CAAC,SAAS,CAAC,CAAC,yBAAyB,CAAC,CAAC;AACnD,MAAM,MAAM,oBAAoB,GAC9B,UAAU,CAAC,SAAS,CAAC,CAAC,sBAAsB,CAAC,CAAC;AAChD,MAAM,MAAM,mBAAmB,GAAG,WAAW,CAC3C,KAAK,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,CACjD,CAAC,SAAS,CAAC,CAAC,kBAAkB,CAAC,CAAC;AACjC,MAAM,MAAM,oBAAoB,GAC9B,UAAU,CAAC,SAAS,CAAC,CAAC,sBAAsB,CAAC,CAAC;AAQhD;;;;;;;;;;;;GAYG;AACH,wBAAgB,oBAAoB,IAChC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACnC,SAAS,CAgBZ;AAED,qBAAa,SAAU,SAAQ,KAAK;IAClC,MAAM,EAAE,MAAM,CAAC;IAEf,YAAY,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAG1C;CACF;AAED,qBAAa,qBAAsB,SAAQ,SAAS;CAAG;AACvD,qBAAa,kBAAmB,SAAQ,SAAS;CAAG;AA+CpD,qBAAa,eAAe;IAC1B,OAAO,CAAC,OAAO,CAAS;IACxB,QAAQ,CAAC,MAAM,EAAE,mBAAmB,CAAC;IAErC,YAAY,EACV,aAAa,EACb,OAAO,GACR,GAAE;QACD,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,OAAO,CAAC,EAAE,MAAM,CAAC;KACb,EAgBL;IAED,UAAU,IAAI,MAAM,CAEnB;IAEK,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAK7D;IAEK,YAAY,CAAC,IAAI,EAAE,mBAAmB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAK3E;IAEK,iBAAiB,CACrB,IAAI,EAAE,iBAAiB,EACvB,IAAI,EAAE,oBAAoB,GACzB,OAAO,CAAC,qBAAqB,CAAC,CAKhC;IAED,MAAM,CAAC,CAAC,EACN,MAAM,EAAE;QACN,IAAI,CAAC,EAAE,OAAO,CAAC;QACf,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,QAAQ,EAAE,QAAQ,CAAC;KACpB,EACD,KAAK,EAAE,MAAM,GACZ,CAAC,CAUH;CACF"}

package/dist/api-worker-client.js

@@ -4,8 +4,41 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ApiWorkerClient = exports.RetryableHTTPError = exports.NonRetryableHTTPError = exports.HTTPError = void 0;
+exports.llmProxyOpenAIConfig = llmProxyOpenAIConfig;
 const async_retry_1 = __importDefault(require("async-retry"));
 const openapi_fetch_1 = __importDefault(require("openapi-fetch"));
+const DEFAULT_API_WORKER_URL = "https://api.empirical.run";
+function apiWorkerBaseUrl() {
+    return process.env.DASHBOARD_DOMAIN || DEFAULT_API_WORKER_URL;
+}
+/**
+ * OpenAI client config for the LLM calls playwright-utils makes from inside a
+ * test run — the overlay-dismissal browser agent and the toLookRight visual
+ * assertion. Both go through api-worker's proxy, which holds the provider key
+ * and forwards via the Cloudflare AI Gateway; the run worker puts
+ * EMPIRICALRUN_API_KEY and DASHBOARD_DOMAIN in the Playwright child process.
+ *
+ * There is deliberately no fallback to a direct OPENAI_API_KEY. A fallback
+ * makes the transport depend on ambient env, which hides a broken proxy behind
+ * a working direct call and lets a test that clears one credential still reach
+ * the model. Returns undefined only for a pi sandbox, whose OPENAI_BASE_URL
+ * already points at the sidecar's own keyless proxy route.
+ */
+function llmProxyOpenAIConfig() {
+    if (process.env.OPENAI_BASE_URL) {
+        return undefined;
+    }
+    const apiKey = process.env.EMPIRICALRUN_API_KEY;
+    if (!apiKey) {
+        throw new Error("EMPIRICALRUN_API_KEY is required to reach Empirical's LLM proxy. Set it, " +
+            "or point OPENAI_BASE_URL at a proxy of your own — playwright-utils no " +
+            "longer calls OpenAI directly with OPENAI_API_KEY.");
+    }
+    return {
+        baseURL: `${apiWorkerBaseUrl()}/api/llm/openai/v1`,
+        apiKey,
+    };
+}
 class HTTPError extends Error {
     status;
     constructor(message, status) {
@@ -63,8 +96,7 @@ class ApiWorkerClient {
     baseUrl;
     client;
     constructor({ projectApiKey, baseUrl, } = {}) {
-        this.baseUrl =
-            baseUrl || process.env.DASHBOARD_DOMAIN || "https://api.empirical.run";
+        this.baseUrl = baseUrl || apiWorkerBaseUrl();
         const apiKey = projectApiKey || process.env.EMPIRICALRUN_API_KEY;
         if (!apiKey) {
             throw new Error("EMPIRICALRUN_API_KEY environment variable is required");

package/dist/generated/api-worker-openapi.d.ts

@@ -2433,26 +2433,6 @@ export interface paths {
         patch?: never;
         trace?: never;
     };
-    "/api/chat-sessions/{id}/sandbox/terminal-session": {
-        parameters: {
-            query?: never;
-            header?: never;
-            path?: never;
-            cookie?: never;
-        };
-        get?: never;
-        put?: never;
-        /**
-         * ⚙️ Create a sandbox terminal session
-         * @description Creates a PTY session on the sandbox and returns the WebSocket URL and token to connect to it. Super-admin only.
-         */
-        post: operations["createChatSessionSandboxTerminalSession"];
-        delete?: never;
-        options?: never;
-        head?: never;
-        patch?: never;
-        trace?: never;
-    };
     "/api/chat-sessions/{id}/session-state": {
         parameters: {
             query?: never;
@@ -2593,6 +2573,26 @@ export interface paths {
         patch?: never;
         trace?: never;
     };
+    "/api/chat-sessions/sandbox/protected-branches": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        /**
+         * ⚙️ List branches the sandbox may not push to
+         * @description Returns the project's default branch and the chat session's base branch. The sandbox git proxy blocks pushes to these. The session is derived from the sandbox-delegation principal.
+         */
+        get: operations["getChatSessionSandboxProtectedBranches"];
+        put?: never;
+        post?: never;
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
     "/api/chat-sessions/users": {
         parameters: {
             query?: never;
@@ -4729,26 +4729,6 @@ export interface paths {
         patch?: never;
         trace?: never;
     };
-    "/api/projects/check-prompt": {
-        parameters: {
-            query?: never;
-            header?: never;
-            path?: never;
-            cookie?: never;
-        };
-        get?: never;
-        put?: never;
-        /**
-         * ⚙️ Validate a prompt against criteria
-         * @description Uses an LLM to validate whether a prompt satisfies the given criteria.
-         */
-        post: operations["checkPrompt"];
-        delete?: never;
-        options?: never;
-        head?: never;
-        patch?: never;
-        trace?: never;
-    };
     "/api/projects/check-slug": {
         parameters: {
             query?: never;
@@ -6479,27 +6459,6 @@ export interface components {
                 test_cases_sync_error_at: string | null;
             };
         };
-        CheckPromptResponse: {
-            /** @description Response payload. */
-            data: {
-                validation: components["schemas"]["PromptValidationResult"];
-            };
-        };
-        PromptValidationResult: {
-            /**
-             * @description Validation outcome.
-             * @enum {string}
-             */
-            status: "pass" | "needs_more";
-            /** @description Human-readable feedback. */
-            assistant_message: string;
-            /** @description Requirements still missing. */
-            missing_items: string[];
-            /** @description Requirements already covered. */
-            covered_items: string[];
-            /** @description Model confidence (0-1). */
-            confidence: number;
-        };
         ProjectKvGetResponse: {
             /** @description Response payload. */
             data: {
@@ -8961,6 +8920,10 @@ export interface components {
             /** @description Human-readable error message. */
             error: string;
         };
+        SandboxProtectedBranches: {
+            /** @description Branch names the sandbox must not push to. */
+            branches: string[];
+        };
         SandboxActionSuccess: {
             /** @description Always true when the action succeeded. */
             success: boolean;
@@ -9027,14 +8990,6 @@ export interface components {
             /** @description Human-readable error message. */
             error: string;
         };
-        SandboxTerminalSession: {
-            /** @description WebSocket URL for the PTY session. */
-            ws_url: string;
-            /** @description Auth token for the terminal WebSocket. */
-            token: string;
-            /** @description Error message, or null on success. */
-            error: string | null;
-        };
         SubscribeChatSessionResponse: {
             data: {
                 already_waiting: boolean;
@@ -10184,8 +10139,6 @@ export interface components {
             id: number;
             /** @description ISO 8601 creation timestamp. */
             created_at: string;
-            /** @description Test case slug. */
-            slug: string | null;
             /** @description Playwright test ID. */
             pw_test_id: string | null;
             /** @description Test run ID. */
@@ -10703,6 +10656,23 @@ export interface components {
                 /** @description Playwright test id, if known. */
                 pw_test_id?: string | null;
             }[];
+            /** @description Scheduled dependency groups (serial blocks). */
+            dependency_groups?: {
+                /** @description Stable group id. */
+                id: string;
+                /**
+                 * @description Only Playwright serial groups today.
+                 * @enum {string}
+                 */
+                kind: "playwright_serial";
+                /** @description Members in execution order. */
+                members: {
+                    /** @description Playwright test id of the member. */
+                    test_case_id: string;
+                    /** @description 0-based execution order within the group. */
+                    dependency_order: number;
+                }[];
+            }[];
             /** @description Legacy base64-gzip manifest blob. Ignored when `tests` is present. */
             run_manifest?: string;
             /** @description Zero-based shard index, when the run is sharded. */
@@ -10772,6 +10742,8 @@ export interface components {
             last_commit_before_run_started_at: string | null;
             /** @description Snoozes matching this case at read time. */
             snooze_info: components["schemas"]["TestRunCaseSnoozeInfo"][];
+            /** @description Attempts that never ran because an earlier member of this case's Playwright serial group failed on the same attempt. Null when the case is not in a snapshotted group or when nothing was blocked. */
+            skipped_attempts: components["schemas"]["TestRunCaseSkippedAttempt"][] | null;
         };
         TestRunCaseRetryDetail: {
             /** @description Zero-based retry attempt index. */
@@ -10845,6 +10817,19 @@ export interface components {
             /** @description Environment the snooze is scoped to, if any. */
             scoped_to_environment_id: number | null;
         };
+        TestRunCaseSkippedAttempt: {
+            /** @description Attempt index this explains. */
+            retry_index: number;
+            /** @description The serial-group member that blocked this attempt. */
+            blocked_by: {
+                /** @description Playwright test id of the blocking test. */
+                pw_test_id: string | null;
+                /** @description Title of the blocking test. */
+                name: string;
+                /** @description The blocking test's outcome on this attempt. */
+                status: string;
+            };
+        };
         GetTestRunCaseResponse: {
             /** @description Response payload. */
             data: {
@@ -10886,6 +10871,8 @@ export interface components {
                 last_commit_before_run_started_at: string | null;
                 /** @description Snoozes matching this case at read time. */
                 snooze_info: components["schemas"]["TestRunCaseSnoozeInfo"][];
+                /** @description Attempts that never ran because an earlier member of this case's Playwright serial group failed on the same attempt. Null when the case is not in a snapshotted group or when nothing was blocked. */
+                skipped_attempts: components["schemas"]["TestRunCaseSkippedAttempt"][] | null;
             };
         };
         UiGetTestRunResponse: {
@@ -19253,65 +19240,6 @@ export interface operations {
             };
         };
     };
-    createChatSessionSandboxTerminalSession: {
-        parameters: {
-            query?: never;
-            header?: never;
-            path: {
-                /** @description Chat session ID. */
-                id: number;
-            };
-            cookie?: never;
-        };
-        requestBody?: never;
-        responses: {
-            /** @description Terminal session connection details. */
-            200: {
-                headers: {
-                    [name: string]: unknown;
-                };
-                content: {
-                    "application/json": components["schemas"]["SandboxTerminalSession"];
-                };
-            };
-            /** @description Invalid request. */
-            400: {
-                headers: {
-                    [name: string]: unknown;
-                };
-                content: {
-                    "application/json": components["schemas"]["ErrorResponse"];
-                };
-            };
-            /** @description Forbidden. */
-            403: {
-                headers: {
-                    [name: string]: unknown;
-                };
-                content: {
-                    "application/json": components["schemas"]["SandboxTerminalSession"];
-                };
-            };
-            /** @description Sandbox not found. */
-            404: {
-                headers: {
-                    [name: string]: unknown;
-                };
-                content: {
-                    "application/json": components["schemas"]["SandboxTerminalSession"];
-                };
-            };
-            /** @description Failed to create terminal session. */
-            500: {
-                headers: {
-                    [name: string]: unknown;
-                };
-                content: {
-                    "application/json": components["schemas"]["SandboxTerminalSession"];
-                };
-            };
-        };
-    };
     getChatSessionState: {
         parameters: {
             query?: {
@@ -19617,6 +19545,44 @@ export interface operations {
             };
         };
     };
+    getChatSessionSandboxProtectedBranches: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        requestBody?: never;
+        responses: {
+            /** @description Protected branches. */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["SandboxProtectedBranches"];
+                };
+            };
+            /** @description The caller is not a sandbox principal. */
+            400: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["SandboxErrorMessage"];
+                };
+            };
+            /** @description Chat session not found. */
+            404: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["SandboxErrorMessage"];
+                };
+            };
+        };
+    };
     listChatSessionUsers: {
         parameters: {
             query?: {
@@ -25087,53 +25053,6 @@ export interface operations {
             };
         };
     };
-    checkPrompt: {
-        parameters: {
-            query?: never;
-            header?: never;
-            path?: never;
-            cookie?: never;
-        };
-        requestBody?: {
-            content: {
-                "application/json": {
-                    /** @description Criterion the prompt must satisfy. */
-                    criteria: string;
-                    /** @description Latest user prompt to validate. */
-                    prompt: string;
-                    /**
-                     * @description Prior conversation messages for context.
-                     * @default []
-                     */
-                    messages?: {
-                        /** @enum {string} */
-                        role: "user" | "assistant";
-                        content: string;
-                    }[];
-                };
-            };
-        };
-        responses: {
-            /** @description Validation result. */
-            200: {
-                headers: {
-                    [name: string]: unknown;
-                };
-                content: {
-                    "application/json": components["schemas"]["CheckPromptResponse"];
-                };
-            };
-            /** @description Invalid request. */
-            400: {
-                headers: {
-                    [name: string]: unknown;
-                };
-                content: {
-                    "application/json": components["schemas"]["ErrorResponse"];
-                };
-            };
-        };
-    };
     checkProjectSlug: {
         parameters: {
             query: {