@empiricalrun/playwright-utils 0.58.5 → 0.59.0

package/dist/overlay-tests/cache.spec.js

@@ -20,6 +20,25 @@ const useCache = (data) => {
     }
     fs_1.default.writeFileSync(cache_1.CACHE_FILE, JSON.stringify(cacheValue, null, 2), "utf8");
 };
+// Unsets env vars for the duration of a test and puts them back exactly as they
+// were — assigning `undefined` to process.env would store the string
+// "undefined", which reads as a present (and truthy) credential.
+function clearEnv(names) {
+    const saved = names.map((name) => [name, process.env[name]]);
+    for (const name of names) {
+        delete process.env[name];
+    }
+    return () => {
+        for (const [name, value] of saved) {
+            if (value === undefined) {
+                delete process.env[name];
+            }
+            else {
+                process.env[name] = value;
+            }
+        }
+    };
+}
 fixtures_1.test.beforeEach(async ({ page }) => {
     (0, test_1.injectLocatorHighlightScripts)(page, fixtures_1.test);
 });
@@ -39,9 +58,12 @@ fixtures_1.test.afterEach(() => {
     ]);
     await page.goto(`${server.baseURL}/glass-pane.html`);
     let error;
-    // Overwriting API key to ensure we don't call OpenAI
-    const originalApiKey = process.env.OPENAI_API_KEY;
-    process.env.OPENAI_API_KEY = undefined;
+    // The only two routes the browser agent has to a model: the project API key
+    // (api-worker's proxy) and a sandbox's OPENAI_BASE_URL. Clearing both makes
+    // llmProxyOpenAIConfig throw before any request — which matters here more
+    // than elsewhere, because this is the recursive case and every extra
+    // dismissal would spawn another agent and another set of screenshots.
+    const restoreEnv = clearEnv(["EMPIRICALRUN_API_KEY", "OPENAI_BASE_URL"]);
     try {
         // Target element click will trigger the cache - which will invoke another
         // click to the same element (recursive overlay dismissal)
@@ -50,7 +72,9 @@ fixtures_1.test.afterEach(() => {
     catch (e) {
         error = e;
     }
+    finally {
+        restoreEnv();
+    }
     (0, fixtures_1.expect)(error).toBeDefined();
-    process.env.OPENAI_API_KEY = originalApiKey;
     (0, fixtures_1.expect)(error.message).toContain('<div class="glass-pane"></div> intercepts pointer events');
 });

package/dist/reporter/test-case-sync.d.ts

@@ -1,28 +1,5 @@
+import type { TestCaseSyncManifest } from "@empiricalrun/shared-types/playwright-utils";
 import type { FullConfig, Suite } from "@playwright/test/reporter";
-export interface TestCaseSyncTestCase {
-    id: string;
-    name: string;
-    file_path: string;
-    suites: string[];
-    playwright_project: string;
-    tags: string[];
-}
-export interface TestCaseDependencyGroupMember {
-    test_case_id: string;
-    dependency_order: number;
-}
-export interface TestCaseDependencyGroup {
-    id: string;
-    kind: "playwright_serial";
-    file_path: string;
-    title_path: string[];
-    playwright_project: string;
-    members: TestCaseDependencyGroupMember[];
-}
-export interface TestCaseSyncManifest {
-    version: 1;
-    test_cases: TestCaseSyncTestCase[];
-    dependency_groups: TestCaseDependencyGroup[];
-}
+export type { TestCaseDependencyGroup, TestCaseDependencyGroupMember, TestCaseSyncManifest, TestCaseSyncTestCase, } from "@empiricalrun/shared-types/playwright-utils";
 export declare function collectTestCaseSyncManifest(config: FullConfig, rootSuite: Suite): TestCaseSyncManifest;
 //# sourceMappingURL=test-case-sync.d.ts.map

package/dist/reporter/test-case-sync.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"test-case-sync.d.ts","sourceRoot":"","sources":["../../src/reporter/test-case-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,UAAU,EAEV,KAAK,EAEN,MAAM,2BAA2B,CAAC;AAMnC,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB;AAED,MAAM,WAAW,6BAA6B;IAC5C,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,uBAAuB;IACtC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,mBAAmB,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,OAAO,EAAE,6BAA6B,EAAE,CAAC;CAC1C;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,CAAC,CAAC;IACX,UAAU,EAAE,oBAAoB,EAAE,CAAC;IACnC,iBAAiB,EAAE,uBAAuB,EAAE,CAAC;CAC9C;AA+JD,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,UAAU,EAClB,SAAS,EAAE,KAAK,GACf,oBAAoB,CAuBtB"}
+{"version":3,"file":"test-case-sync.d.ts","sourceRoot":"","sources":["../../src/reporter/test-case-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEV,oBAAoB,EAErB,MAAM,6CAA6C,CAAC;AACrD,OAAO,KAAK,EACV,UAAU,EAEV,KAAK,EAEN,MAAM,2BAA2B,CAAC;AAUnC,YAAY,EACV,uBAAuB,EACvB,6BAA6B,EAC7B,oBAAoB,EACpB,oBAAoB,GACrB,MAAM,6CAA6C,CAAC;AAwRrD,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,UAAU,EAClB,SAAS,EAAE,KAAK,GACf,oBAAoB,CAyBtB"}

package/dist/reporter/test-case-sync.js

@@ -41,7 +41,7 @@ function stableGroupId(input) {
         input.column,
     ]))
         .digest("hex");
-    return `playwright_serial:${hash}`;
+    return `${input.kind}:${hash}`;
 }
 function collectDirectTestCases(rootDir, suite, state) {
     return suite.tests.map((test) => ({
@@ -53,11 +53,7 @@ function collectDirectTestCases(rootDir, suite, state) {
         tags: normalizeTags(test.tags),
     }));
 }
-function collectOutermostSerialGroup(rootDir, suite, state) {
-    if (state.serialAncestor || getSuiteParallelMode(suite) !== "serial") {
-        return null;
-    }
-    const tests = suite.allTests();
+function buildDependencyGroup(rootDir, suite, state, kind, tests) {
     if (tests.length < 2) {
         return null;
     }
@@ -71,13 +67,14 @@ function collectOutermostSerialGroup(rootDir, suite, state) {
     const location = suite.location ?? firstTest.location;
     return {
         id: stableGroupId({
+            kind,
             projectName,
             filePath,
             titlePath,
             line: location.line,
             column: location.column,
         }),
-        kind: "playwright_serial",
+        kind,
         file_path: filePath,
         title_path: titlePath,
         playwright_project: projectName,
@@ -87,14 +84,81 @@ function collectOutermostSerialGroup(rootDir, suite, state) {
         })),
     };
 }
+function collectOutermostSerialGroup(rootDir, suite, state) {
+    if (state.serialAncestor || getSuiteParallelMode(suite) !== "serial") {
+        return null;
+    }
+    return buildDependencyGroup(rootDir, suite, state, "playwright_serial", suite.allTests());
+}
+/**
+ * Tests under this suite that Playwright would place in its sequential
+ * ("general") group: everything except subtrees that opt back into
+ * parallelism. Order is declaration order, which is the order the group
+ * executes in.
+ */
+function sequentialTests(suite) {
+    const tests = [...suite.tests];
+    for (const child of suite.suites) {
+        if (getSuiteParallelMode(child) !== "parallel") {
+            tests.push(...sequentialTests(child));
+        }
+    }
+    return tests;
+}
+/**
+ * The `playwright_default` counterpart of the serial collector: the outermost
+ * scope whose tests Playwright schedules as one ordered worker group without
+ * serial semantics.
+ *
+ * Two shapes qualify (mirroring `createTestGroups` in Playwright's
+ * `runner/testGroups.ts`):
+ *
+ * - a `test.describe.configure({ mode: "default" })` suite — grouped whole,
+ *   including any nested parallel subtree, because Playwright keys the group
+ *   on the outermost sequential suite;
+ * - a file with no explicit mode in a non-fullyParallel project — Playwright's
+ *   per-file "general" group, which excludes subtrees that opt back into
+ *   `parallel`.
+ */
+function collectOutermostDefaultGroup(rootDir, suite, state) {
+    if (state.sequentialAncestor) {
+        return null;
+    }
+    const mode = getSuiteParallelMode(suite);
+    if (mode === "default") {
+        return buildDependencyGroup(rootDir, suite, state, "playwright_default", suite.allTests());
+    }
+    if (suite.type === "file" && !state.insideParallel && mode === "none") {
+        return buildDependencyGroup(rootDir, suite, state, "playwright_default", sequentialTests(suite));
+    }
+    return null;
+}
 function walkSuite(rootDir, suite, state, testCases, dependencyGroups) {
     const isDescribe = suite.type === "describe";
     const suites = isDescribe ? [...state.suites, suite.title] : state.suites;
     const nextState = { ...state, suites };
+    const mode = getSuiteParallelMode(suite);
+    // A serial group is emitted even inside a default scope: the outer default
+    // group wins for scheduling, but serial membership carries extra semantics
+    // (failure cascade, re-expansion) its consumers still need.
     const serialGroup = collectOutermostSerialGroup(rootDir, suite, nextState);
     if (serialGroup) {
         dependencyGroups.push(serialGroup);
         nextState.serialAncestor = suite;
+        if (!nextState.sequentialAncestor) {
+            nextState.sequentialAncestor = suite;
+        }
+    }
+    const defaultGroup = collectOutermostDefaultGroup(rootDir, suite, nextState);
+    if (defaultGroup) {
+        dependencyGroups.push(defaultGroup);
+        nextState.sequentialAncestor = suite;
+    }
+    // fullyParallel (project suite) or an explicit `mode: "parallel"` — every
+    // descendant is independently schedulable unless a sequential suite below
+    // says otherwise.
+    if (mode === "parallel") {
+        nextState.insideParallel = true;
     }
     testCases.push(...collectDirectTestCases(rootDir, suite, nextState));
     for (const child of suite.suites) {
@@ -112,6 +176,15 @@ function walkSuite(rootDir, suite, state, testCases, dependencyGroups) {
                     : "";
             childState.suites = [];
             childState.serialAncestor = null;
+            childState.sequentialAncestor = null;
+        }
+        else if (childState.sequentialAncestor &&
+            getSuiteParallelMode(childState.sequentialAncestor) === "none" &&
+            getSuiteParallelMode(child) === "parallel") {
+            // The scope was a plain file's "general" group, which excludes subtrees
+            // that opt back into parallel — so below this child, sequential suites
+            // start their own groups again.
+            childState.sequentialAncestor = null;
         }
         walkSuite(rootDir, child, childState, testCases, dependencyGroups);
     }
@@ -125,6 +198,8 @@ function collectTestCaseSyncManifest(config, rootSuite) {
         filePath: "",
         suites: [],
         serialAncestor: null,
+        sequentialAncestor: null,
+        insideParallel: false,
     }, testCases, dependencyGroups);
     return {
         version: 1,

package/dist/test/expect/visual.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"visual.d.ts","sourceRoot":"","sources":["../../../src/test/expect/visual.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAE,IAAI,EAAY,MAAM,kBAAkB,CAAC;AAShE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AA0E7C,wBAAsB,WAAW,CAC/B,MAAM,EAAE,IAAI,GAAG,OAAO,EACtB,eAAe,EAAE,MAAM,GACtB,OAAO,CAAC,aAAa,CAAC,CAyFxB"}
+{"version":3,"file":"visual.d.ts","sourceRoot":"","sources":["../../../src/test/expect/visual.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAE,IAAI,EAAY,MAAM,kBAAkB,CAAC;AAUhE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AA0E7C,wBAAsB,WAAW,CAC/B,MAAM,EAAE,IAAI,GAAG,OAAO,EACtB,eAAe,EAAE,MAAM,GACtB,OAAO,CAAC,aAAa,CAAC,CA2FxB"}

package/dist/test/expect/visual.js

@@ -10,6 +10,7 @@ const fs_1 = __importDefault(require("fs"));
 const openai_1 = __importDefault(require("openai"));
 const os_1 = __importDefault(require("os"));
 const path_1 = __importDefault(require("path"));
+const api_worker_client_1 = require("../../api-worker-client");
 const mouse_pointer_1 = require("../scripts/mouse-pointer");
 function getCallerLine() {
     const stack = new Error().stack;
@@ -85,7 +86,9 @@ async function toLookRight(target, pageDescription) {
     const base64Image = screenshot.toString("base64");
     await (0, mouse_pointer_1.addHighlighterScriptsToPage)(page);
     const model = "gpt-4o";
-    const openai = new openai_1.default({ maxRetries: 5 });
+    // Same api-worker proxy the overlay-dismissal agent uses, so a test run needs
+    // no OpenAI key of its own; undefined falls back to the SDK's env defaults.
+    const openai = new openai_1.default({ maxRetries: 5, ...(0, api_worker_client_1.llmProxyOpenAIConfig)() });
     const completion = await openai.chat.completions.create({
         model,
         messages: [

package/dist/test/scripts/pw-locator-patch/dismiss-overlays/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/test/scripts/pw-locator-patch/dismiss-overlays/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAGxC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAG1C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAK9C,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS,WAKhE;AAED,qBAAa,mBAAmB;IAOlB,OAAO,CAAC,QAAQ,CAAC,IAAI;IANjC,OAAO,CAAC,iBAAiB,CAIhB;IAET,YAA6B,IAAI,EAAE,IAAI,EAAI;IAErC,WAAW,kBAIhB;IAEK,GAAG,CACP,aAAa,EAAE,GAAG,EAClB,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,IAAI,EACvC,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC,CAuDf;YAEa,QAAQ;CAiCvB;AAED,wBAAgB,0BAA0B,CACxC,YAAY,EAAE,MAAM,GACnB,cAAc,GAAG,SAAS,CAiC5B"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/test/scripts/pw-locator-patch/dismiss-overlays/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAIxC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAG1C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAK9C,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS,WAKhE;AAED,qBAAa,mBAAmB;IAOlB,OAAO,CAAC,QAAQ,CAAC,IAAI;IANjC,OAAO,CAAC,iBAAiB,CAIhB;IAET,YAA6B,IAAI,EAAE,IAAI,EAAI;IAErC,WAAW,kBAIhB;IAEK,GAAG,CACP,aAAa,EAAE,GAAG,EAClB,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,IAAI,EACvC,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC,CAuDf;YAEa,QAAQ;CAqCvB;AAED,wBAAgB,0BAA0B,CACxC,YAAY,EAAE,MAAM,GACnB,cAAc,GAAG,SAAS,CAiC5B"}

package/dist/test/scripts/pw-locator-patch/dismiss-overlays/index.js

@@ -4,6 +4,7 @@ exports.OverlayDismissAgent = void 0;
 exports.isErrorSupported = isErrorSupported;
 exports.extractInterceptingElement = extractInterceptingElement;
 const cua_1 = require("@empiricalrun/cua");
+const api_worker_client_1 = require("../../../../api-worker-client");
 const telemetry_1 = require("../../../../telemetry");
 const cache_1 = require("./cache");
 const prompt_1 = require("./prompt");
@@ -99,6 +100,10 @@ class OverlayDismissAgent {
             // already on the current page. Including page_goto causes the model
             // to waste turns trying to navigate instead of clicking dismiss.
             includePageGotoTool: false,
+            // Route the model call through api-worker (Cloudflare AI Gateway) on the
+            // project API key, so test-run containers carry no OpenAI key. Undefined
+            // on a dev machine or in a pi sandbox — see the helper.
+            openAIClientConfig: (0, api_worker_client_1.llmProxyOpenAIConfig)(),
         });
         this.cachingCandidates.push({
             dom: element?.interceptor,

package/dist/test/scripts/pw-locator-patch/vision/query.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"query.d.ts","sourceRoot":"","sources":["../../../../../src/test/scripts/pw-locator-patch/vision/query.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAEhD,wBAAgB,UAAU,CAAC,YAAY,EAAE,QAAQ,GAAG;IAAE,SAAS,EAAE,OAAO,CAAA;CAAE,QAOzE"}
+{"version":3,"file":"query.d.ts","sourceRoot":"","sources":["../../../../../src/test/scripts/pw-locator-patch/vision/query.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAIhD,wBAAgB,UAAU,CAAC,YAAY,EAAE,QAAQ,GAAG;IAAE,SAAS,EAAE,OAAO,CAAA;CAAE,QASzE"}

package/dist/test/scripts/pw-locator-patch/vision/query.js

@@ -2,11 +2,14 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.patchQuery = patchQuery;
 const vision_1 = require("@empiricalrun/llm/vision");
+const api_worker_client_1 = require("../../../../api-worker-client");
 function patchQuery(LocatorClass) {
     LocatorClass.prototype.query = async function (instruction) {
         await this.waitFor({ state: "visible" });
         const screenshot = await this.screenshot();
-        const answer = await (0, vision_1.query)(screenshot.toString("base64"), instruction);
+        const answer = await (0, vision_1.query)(screenshot.toString("base64"), instruction, {
+            clientConfig: (0, api_worker_client_1.llmProxyOpenAIConfig)(),
+        });
         return answer;
     };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/playwright-utils",
-  "version": "0.58.5",
+  "version": "0.59.0",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -41,7 +41,7 @@
     "openapi-typescript": "7.6.1",
     "playwright-core": "1.62.1",
     "serve-handler": "^6.1.6",
-    "@empiricalrun/shared-types": "0.15.4"
+    "@empiricalrun/shared-types": "0.16.0"
   },
   "dependencies": {
     "@babel/code-frame": "^7.24.7",
@@ -54,10 +54,10 @@
     "puppeteer-extra-plugin-recaptcha": "^3.6.8",
     "rimraf": "^6.0.1",
     "ts-morph": "^23.0.0",
-    "@empiricalrun/cua": "^0.4.1",
-    "@empiricalrun/llm": "^0.27.0",
+    "@empiricalrun/cua": "^0.4.2",
+    "@empiricalrun/llm": "^0.27.1",
     "@empiricalrun/r2-uploader": "^0.10.2",
-    "@empiricalrun/reporter": "^0.29.1"
+    "@empiricalrun/reporter": "^0.30.0"
   },
   "scripts": {
     "dev": "pnpm gen:api-worker-openapi && tsgo --build --watch",