@empiricalrun/playwright-utils 0.48.2 → 0.48.3

package/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @empiricalrun/playwright-utils
 
+## 0.48.3
+
+### Patch Changes
+
+- d2acb85: feat: attach visual assertion screenshots to test info
+- 46e7acd: feat: telemetry for visual assertions
+
 ## 0.48.2
 
 ### Patch Changes

package/dist/overlay-tests/visual.spec.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=visual.spec.d.ts.map

package/dist/overlay-tests/visual.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"visual.spec.d.ts","sourceRoot":"","sources":["../../src/overlay-tests/visual.spec.ts"],"names":[],"mappings":""}

package/dist/overlay-tests/visual.spec.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const test_1 = require("../test");
+const fixtures_1 = require("./fixtures");
+const expect = (0, test_1.extendExpect)(fixtures_1.expect);
+(0, fixtures_1.test)("toLookRight on page attaches screenshot", async ({ page, server, }, testInfo) => {
+    await page.goto(`${server.baseURL}/visual-test.html`);
+    await expect(page).toLookRight("A welcome page with a heading and a paragraph of text");
+    const pageAttachment = testInfo.attachments.find((a) => a.name.startsWith("toLookRight-"));
+    expect(pageAttachment).toBeDefined();
+    expect(pageAttachment.name).toBe("toLookRight-L11-a-welcome-page-with-a-heading-and-a-paragraph-of-text");
+    expect(pageAttachment.contentType).toBe("image/png");
+    expect(pageAttachment.body.length).toBeGreaterThan(0);
+});
+(0, fixtures_1.test)("toLookRight on locator attaches screenshot", async ({ page, server, }, testInfo) => {
+    await page.goto(`${server.baseURL}/visual-test.html`);
+    const heading = page.locator("h1");
+    await expect(heading).toLookRight("A heading that says Welcome to the App");
+    const locatorAttachment = testInfo.attachments.find((a) => a.name.startsWith("toLookRight-"));
+    expect(locatorAttachment).toBeDefined();
+    expect(locatorAttachment.name).toBe("toLookRight-L31-a-heading-that-says-welcome-to-the-app");
+    expect(locatorAttachment.contentType).toBe("image/png");
+    expect(locatorAttachment.body.length).toBeGreaterThan(0);
+});

package/dist/telemetry.d.ts

@@ -7,6 +7,11 @@ interface TrackReportUploadParams {
     sizeBytes: number;
 }
 export declare function trackReportUpload(params: TrackReportUploadParams): Promise<void>;
+export declare function trackLLMResponseGenerated(opts: {
+    modelName: string;
+    tokensInput: number;
+    tokensOutput: number;
+}): Promise<void>;
 export declare function trackOverlayDismissed(opts: {
     original: string | undefined;
     description: string;

package/dist/telemetry.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../src/telemetry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,sCAAsC,CAAC;AAKtE,wBAAsB,UAAU,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAqBrE;AAED,UAAU,uBAAuB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,uBAAuB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAcf;AAED,wBAAsB,qBAAqB,CAAC,IAAI,EAAE;IAChD,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,KAAK,EAAE,OAAO,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C,IAAI,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1C,GAAG,OAAO,CAAC,IAAI,CAAC,CAsBhB"}
+{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../src/telemetry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,sCAAsC,CAAC;AAKtE,wBAAsB,UAAU,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAqBrE;AAED,UAAU,uBAAuB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,uBAAuB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAcf;AAED,wBAAsB,yBAAyB,CAAC,IAAI,EAAE;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,IAAI,CAAC,CAmBhB;AAED,wBAAsB,qBAAqB,CAAC,IAAI,EAAE;IAChD,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,KAAK,EAAE,OAAO,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C,IAAI,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1C,GAAG,OAAO,CAAC,IAAI,CAAC,CAsBhB"}

package/dist/telemetry.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.trackEvent = trackEvent;
 exports.trackReportUpload = trackReportUpload;
+exports.trackLLMResponseGenerated = trackLLMResponseGenerated;
 exports.trackOverlayDismissed = trackOverlayDismissed;
 const TELEMETRY_WORKER_URL = "https://telemetry-worker.empirical-run.workers.dev";
 async function trackEvent(event) {
@@ -41,6 +42,26 @@ async function trackReportUpload(params) {
         },
     });
 }
+async function trackLLMResponseGenerated(opts) {
+    const environment = process.env.TEST_RUN_ENVIRONMENT || "unknown";
+    await trackEvent({
+        name: "llm_response_generated",
+        properties: {
+            environment,
+            model_name: opts.modelName,
+            chat_session_id: undefined,
+            chat_session_source: "to_look_right",
+            tokens_input: opts.tokensInput,
+            tokens_output: opts.tokensOutput,
+            cost_input: 0,
+            cost_output: 0,
+            tokens_cache_read: 0,
+            tokens_cache_creation: 0,
+            project_repo_name: process.env.GIT_REPO_NAME ?? "unknown",
+            repo_name: process.env.GIT_REPO_NAME ?? "unknown",
+        },
+    });
+}
 async function trackOverlayDismissed(opts) {
     const environment = process.env.TEST_RUN_ENVIRONMENT || "unknown";
     await trackEvent({

package/dist/test/expect/types.d.ts

@@ -1,5 +1,6 @@
 export type MatcherResult = {
     message: () => string;
     pass: boolean;
+    screenshot?: Buffer;
 };
 //# sourceMappingURL=types.d.ts.map

package/dist/test/expect/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/test/expect/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,aAAa,GAAG;IAC1B,OAAO,EAAE,MAAM,MAAM,CAAC;IACtB,IAAI,EAAE,OAAO,CAAC;CACf,CAAC"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/test/expect/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,aAAa,GAAG;IAC1B,OAAO,EAAE,MAAM,MAAM,CAAC;IACtB,IAAI,EAAE,OAAO,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,CAAC"}

package/dist/test/expect/visual.d.ts

@@ -1,4 +1,4 @@
-import type { Page } from "@playwright/test";
+import type { Locator, Page } from "@playwright/test";
 import type { MatcherResult } from "./types";
-export declare function toLookRight(page: Page, pageDescription: string): Promise<MatcherResult>;
+export declare function toLookRight(target: Page | Locator, pageDescription: string): Promise<MatcherResult>;
 //# sourceMappingURL=visual.d.ts.map

package/dist/test/expect/visual.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"visual.d.ts","sourceRoot":"","sources":["../../../src/test/expect/visual.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAM7C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AA0B7C,wBAAsB,WAAW,CAC/B,IAAI,EAAE,IAAI,EACV,eAAe,EAAE,MAAM,GACtB,OAAO,CAAC,aAAa,CAAC,CAkFxB"}
+{"version":3,"file":"visual.d.ts","sourceRoot":"","sources":["../../../src/test/expect/visual.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAE,IAAI,EAAY,MAAM,kBAAkB,CAAC;AAOhE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AA8D7C,wBAAsB,WAAW,CAC/B,MAAM,EAAE,IAAI,GAAG,OAAO,EACtB,eAAe,EAAE,MAAM,GACtB,OAAO,CAAC,aAAa,CAAC,CA2FxB"}

package/dist/test/expect/visual.js

@@ -3,7 +3,41 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.toLookRight = toLookRight;
 const llm_1 = require("@empiricalrun/llm");
 const vision_1 = require("@empiricalrun/llm/vision");
+const telemetry_1 = require("../../telemetry");
 const mouse_pointer_1 = require("../scripts/mouse-pointer");
+function getCallerLine() {
+    const stack = new Error().stack;
+    if (!stack)
+        return "unknown";
+    const frames = stack.split("\n").slice(1);
+    // Find the test file frame (e.g. .spec.ts, .test.ts)
+    const testFrame = frames.find((f) => /\.(spec|test)\.[jt]sx?:/.test(f));
+    if (!testFrame)
+        return "unknown";
+    const match = testFrame.match(/:(\d+):\d+\)?$/);
+    return match?.[1] ?? "unknown";
+}
+function attachScreenshot(screenshot, pageDescription) {
+    try {
+        // test.info() is only available during a Playwright test run
+        const { test } = require("@playwright/test");
+        const testInfo = test.info();
+        const slug = pageDescription
+            .slice(0, 60)
+            .trim()
+            .toLowerCase()
+            .replace(/[^a-z0-9]+/g, "-")
+            .replace(/(^-|-$)/g, "");
+        const line = getCallerLine();
+        testInfo.attach(`toLookRight-L${line}-${slug}`, {
+            body: screenshot,
+            contentType: "image/png",
+        });
+    }
+    catch {
+        // Not in a Playwright test context, skip attachment
+    }
+}
 const SYSTEM_PROMPT = `
 You are a software QA tester who specializes in visual testing. You look at a
 UI screenshot and evaluate whether the UI is rendered correctly. While you don't
@@ -27,14 +61,20 @@ issues or page loading bugs.
 You are to respond to your visual evaluation with a Pass or Fail, and provide
 a reason for your evaluation.
 `;
-async function toLookRight(page, pageDescription) {
+function isLocator(target) {
+    return "page" in target && typeof target.page === "function";
+}
+async function toLookRight(target, pageDescription) {
+    const page = isLocator(target) ? target.page() : target;
     await (0, mouse_pointer_1.removeMousePointerHighlighter)(page);
-    const screenshot = await page.screenshot();
+    const screenshot = await target.screenshot();
+    attachScreenshot(screenshot, pageDescription);
     const base64Image = screenshot.toString("base64");
     await (0, mouse_pointer_1.addHighlighterScriptsToPage)(page);
+    const model = "gpt-4o";
     const llm = new llm_1.LLM({
         provider: "openai",
-        defaultModel: "gpt-4o",
+        defaultModel: model,
     });
     const response = await llm.createChatCompletion({
         messages: [
@@ -92,6 +132,11 @@ go through the reasons and pick the one that made you choose Fail`,
             },
         ],
     });
+    (0, telemetry_1.trackLLMResponseGenerated)({
+        modelName: model,
+        tokensInput: llm.promptTokens,
+        tokensOutput: llm.completionTokens,
+    });
     const rawResponse = response.tool_calls[0];
     if (!("function" in rawResponse)) {
         throw new Error("Expected a function tool call in the response, but got: " +
@@ -101,5 +146,6 @@ go through the reasons and pick the one that made you choose Fail`,
     return {
         message: () => result.most_critical_reason,
         pass: result.result === "Pass",
+        screenshot,
     };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/playwright-utils",
-  "version": "0.48.2",
+  "version": "0.48.3",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -43,11 +43,11 @@
     "puppeteer-extra-plugin-recaptcha": "^3.6.8",
     "rimraf": "^6.0.1",
     "ts-morph": "^23.0.0",
-    "@empiricalrun/cua": "^0.3.0",
     "@empiricalrun/dashboard-client": "^0.3.0",
-    "@empiricalrun/llm": "^0.26.0",
     "@empiricalrun/r2-uploader": "^0.9.1",
-    "@empiricalrun/reporter": "^0.28.1"
+    "@empiricalrun/llm": "^0.26.0",
+    "@empiricalrun/reporter": "^0.28.1",
+    "@empiricalrun/cua": "^0.3.0"
   },
   "scripts": {
     "dev": "tsc --build --watch",

package/tsconfig.tsbuildinfo

@@ -1 +1 @@
-{"root":["./src/email.ts","./src/inbox-client.ts","./src/index.ts","./src/kv.ts","./src/logger.ts","./src/mailosaur-client.ts","./src/playwright-extensions.ts","./src/postgres.ts","./src/telemetry.ts","./src/types.ts","./src/webhook.ts","./src/auth/google.ts","./src/auth/index.ts","./src/auth/types.ts","./src/captcha/index.ts","./src/config/index.ts","./src/config/proxy.ts","./src/config/devices/types.ts","./src/overlay-tests/cache.spec.ts","./src/overlay-tests/click.spec.ts","./src/overlay-tests/fixtures.ts","./src/overlay-tests/patch.spec.ts","./src/reporter/attachment-cleanup-test-reporter.ts","./src/reporter/attachment-cleanup.ts","./src/reporter/blob-utils.ts","./src/reporter/empirical-reporter.ts","./src/reporter/failing-line.ts","./src/reporter/harness.ts","./src/reporter/ibr-utils.ts","./src/reporter/incremental-blob-reporter.ts","./src/reporter/lifecycle-events.ts","./src/reporter/local-test.ts","./src/reporter/reporter-state.ts","./src/reporter/uploader.ts","./src/reporter/util.ts","./src/test/constants.ts","./src/test/coverage.ts","./src/test/index.ts","./src/test/types.ts","./src/test/video-labels.ts","./src/test/expect/index.ts","./src/test/expect/types.ts","./src/test/expect/visual.ts","./src/test/expect/webhook.ts","./src/test/scripts/agent-capabilities.ts","./src/test/scripts/index.ts","./src/test/scripts/locator-highlights.ts","./src/test/scripts/locator-vision.ts","./src/test/scripts/mouse-pointer.ts","./src/test/scripts/types.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/cache.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/index.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/prompt.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/types.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/utils.ts","./src/test/scripts/pw-locator-patch/highlight/click.ts","./src/test/scripts/pw-locator-patch/highlight/expect.ts","./src/test/scripts/pw-locator-patch/highlight/hover.ts","./src/test/scripts/pw-locator-patch/highlight/inner-text.ts","./src/test/scripts/pw-locator-patch/highlight/input-value.ts","./src/test/scripts/pw-locator-patch/highlight/is-checked.ts","./src/test/scripts/pw-locator-patch/highlight/is-disabled.ts","./src/test/scripts/pw-locator-patch/highlight/is-editable.ts","./src/test/scripts/pw-locator-patch/highlight/text-content.ts","./src/test/scripts/pw-locator-patch/utils/index.ts","./src/test/scripts/pw-locator-patch/vision/query.ts"],"version":"5.8.3"}
+{"root":["./src/email.ts","./src/inbox-client.ts","./src/index.ts","./src/kv.ts","./src/logger.ts","./src/mailosaur-client.ts","./src/playwright-extensions.ts","./src/postgres.ts","./src/telemetry.ts","./src/types.ts","./src/webhook.ts","./src/auth/google.ts","./src/auth/index.ts","./src/auth/types.ts","./src/captcha/index.ts","./src/config/index.ts","./src/config/proxy.ts","./src/config/devices/types.ts","./src/overlay-tests/cache.spec.ts","./src/overlay-tests/click.spec.ts","./src/overlay-tests/fixtures.ts","./src/overlay-tests/patch.spec.ts","./src/overlay-tests/visual.spec.ts","./src/reporter/attachment-cleanup-test-reporter.ts","./src/reporter/attachment-cleanup.ts","./src/reporter/blob-utils.ts","./src/reporter/empirical-reporter.ts","./src/reporter/failing-line.ts","./src/reporter/harness.ts","./src/reporter/ibr-utils.ts","./src/reporter/incremental-blob-reporter.ts","./src/reporter/lifecycle-events.ts","./src/reporter/local-test.ts","./src/reporter/reporter-state.ts","./src/reporter/uploader.ts","./src/reporter/util.ts","./src/test/constants.ts","./src/test/coverage.ts","./src/test/index.ts","./src/test/types.ts","./src/test/video-labels.ts","./src/test/expect/index.ts","./src/test/expect/types.ts","./src/test/expect/visual.ts","./src/test/expect/webhook.ts","./src/test/scripts/agent-capabilities.ts","./src/test/scripts/index.ts","./src/test/scripts/locator-highlights.ts","./src/test/scripts/locator-vision.ts","./src/test/scripts/mouse-pointer.ts","./src/test/scripts/types.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/cache.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/index.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/prompt.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/types.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/utils.ts","./src/test/scripts/pw-locator-patch/highlight/click.ts","./src/test/scripts/pw-locator-patch/highlight/expect.ts","./src/test/scripts/pw-locator-patch/highlight/hover.ts","./src/test/scripts/pw-locator-patch/highlight/inner-text.ts","./src/test/scripts/pw-locator-patch/highlight/input-value.ts","./src/test/scripts/pw-locator-patch/highlight/is-checked.ts","./src/test/scripts/pw-locator-patch/highlight/is-disabled.ts","./src/test/scripts/pw-locator-patch/highlight/is-editable.ts","./src/test/scripts/pw-locator-patch/highlight/text-content.ts","./src/test/scripts/pw-locator-patch/utils/index.ts","./src/test/scripts/pw-locator-patch/vision/query.ts"],"version":"5.8.3"}