@piwitests/reporter 0.41.0 → 0.43.0

package/dist/internal/capture/capture-fixtures.js

@@ -399,7 +399,7 @@ function installPickerOverlay(arg) {
   banner.style.cssText = `position:fixed;top:12px;left:50%;transform:translateX(-50%);z-index:${Z + 2};background:#111827;color:#f9fafb;font:13px/1.5 system-ui,sans-serif;border:1px solid #312e81;padding:10px 16px;border-radius:10px;box-shadow:0 4px 24px rgba(0,0,0,.5);max-width:min(680px,86vw);`;
   const hlTokens = (expr) => {
     const escHtml = (s) => s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
-    const re = /('(?:\\.|[^'])*'|"(?:\\.|[^"])*")|([A-Za-z_$][\w$]*)(?=\s*\()|([A-Za-z_$][\w$]*)(?=\s*:)|(true|false|null|\d+)|([{}(),.])/g;
+    const re = /('(?:\\.|[^'\\])*'|"(?:\\.|[^"\\])*")|([A-Za-z_$][\w$]*)(?=\s*\()|([A-Za-z_$][\w$]*)(?=\s*:)|(true|false|null|\d+)|([{}(),.])/g;
     let html = "";
     let last = 0;
     let m;
@@ -942,7 +942,7 @@ function showPickerChoices(arg) {
   panel.style.cssText = "background:#111827;color:#f9fafb;border-radius:10px;padding:20px;max-width:640px;width:90vw;max-height:70vh;overflow:auto;box-shadow:0 8px 40px rgba(0,0,0,.5);";
   const hlLocator = (expr) => {
     const escHtml = (s) => s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
-    const re = /('(?:\\.|[^'])*'|"(?:\\.|[^"])*")|([A-Za-z_$][\w$]*)(?=\s*\()|([A-Za-z_$][\w$]*)(?=\s*:)|(true|false|null|\d+)|([{}(),.])/g;
+    const re = /('(?:\\.|[^'\\])*'|"(?:\\.|[^"\\])*")|([A-Za-z_$][\w$]*)(?=\s*\()|([A-Za-z_$][\w$]*)(?=\s*:)|(true|false|null|\d+)|([{}(),.])/g;
     let html = "";
     let last = 0;
     let m;
@@ -2045,6 +2045,7 @@ var DEFAULTS = {
   streamingBatchSize: 5,
   streamingBatchDelay: 2e3,
   maxStreamBufferBytes: 100 * 1024 * 1024,
+  submitTimeout: 15 * 60 * 1e3,
   failOnFlakyTests: false,
   username: null,
   password: null,
@@ -2066,6 +2067,7 @@ var PIWI_ENV_KEYS = {
   streamingBatchSize: "PIWI_STREAMING_BATCH_SIZE",
   streamingBatchDelay: "PIWI_STREAMING_BATCH_DELAY",
   maxStreamBufferBytes: "PIWI_MAX_STREAM_BUFFER_BYTES",
+  submitTimeout: "PIWI_SUBMIT_TIMEOUT",
   liveFileUploads: "PIWI_LIVE_FILE_UPLOADS",
   failOnFlakyTests: "PIWI_FAIL_ON_FLAKY_TESTS",
   uploadTraces: "PIWI_UPLOAD_TRACES",
@@ -2115,6 +2117,7 @@ var ENV_FALLBACK_SPECS = [
   { option: "streamingBatchSize", env: PIWI_ENV_KEYS.streamingBatchSize, kind: "number" },
   { option: "streamingBatchDelay", env: PIWI_ENV_KEYS.streamingBatchDelay, kind: "number" },
   { option: "maxStreamBufferBytes", env: PIWI_ENV_KEYS.maxStreamBufferBytes, kind: "number" },
+  { option: "submitTimeout", env: PIWI_ENV_KEYS.submitTimeout, kind: "number" },
   { option: "liveFileUploads", env: PIWI_ENV_KEYS.liveFileUploads, kind: "bool" },
   { option: "failOnFlakyTests", env: PIWI_ENV_KEYS.failOnFlakyTests, kind: "bool" },
   { option: "uploadTraces", env: PIWI_ENV_KEYS.uploadTraces, kind: "bool" },
@@ -2791,15 +2794,19 @@ function extractSelector(text) {
   return text.slice(start, start + 80);
 }
 var LEAF_METHODS = new Set(LOCATOR_BUILDER_METHODS);
-function extractTopFrame(text) {
+function extractStackFrames(text) {
   const frameRe = /^\s+at (?:.*? \()?([^()\s][^()]*?):(\d+):(\d+)\)?\s*$/gm;
+  const frames = [];
   let m;
   while ((m = frameRe.exec(text)) !== null) {
     const file = m[1].replace(/\\/g, "/");
     if (file.includes("node_modules") || file.startsWith("node:")) continue;
-    return { file, line: Number(m[2]), column: Number(m[3]) };
+    frames.push({ file, line: Number(m[2]), column: Number(m[3]) });
   }
-  return null;
+  return frames;
+}
+function extractTopFrame(text) {
+  return extractStackFrames(text)[0] ?? null;
 }
 function extractTopFrameFile(text) {
   return extractTopFrame(text)?.file ?? null;
@@ -2813,6 +2820,42 @@ var LONG_HEX_RE = /\b[0-9a-f]{8,}\b/gi;
 var SHORT_HEX_RE = /\b(?=[0-9a-f]*[a-f])(?=[0-9a-f]*[0-9])[0-9a-f]{6,7}\b/gi;
 var URL_RE = /\bhttps?:\/\/[^\s'"`)]+/gi;
 var EMAIL_RE = /\b[\w.+-]+@[\w-]+(?:\.[\w-]+)+\b/gi;
+var SELECTOR_OPTION_KEY_RE = /\b(name|hasText|hasNotText|has|placeholder|label|title|alt|exact)\s*:\s*(['"`])/gi;
+var LINE_TERMINATORS = /* @__PURE__ */ new Set(["\n", "\r", "\u2028", "\u2029"]);
+function closingQuoteIndex(text, quote) {
+  const close = new Int32Array(text.length + 1).fill(-1);
+  for (let p = text.length - 1; p >= 0; p--) {
+    const ch = text[p];
+    if (ch === quote) {
+      close[p] = p;
+      continue;
+    }
+    const escaped = ch === "\\" && p + 1 < text.length && !LINE_TERMINATORS.has(text[p + 1]) ? close[p + 2] : -1;
+    close[p] = escaped !== -1 ? escaped : close[p + 1];
+  }
+  return close;
+}
+function maskSelectorOptions(text) {
+  const closers = /* @__PURE__ */ new Map();
+  let out = "";
+  let copied = 0;
+  SELECTOR_OPTION_KEY_RE.lastIndex = 0;
+  let match;
+  while (match = SELECTOR_OPTION_KEY_RE.exec(text)) {
+    const quote = match[2];
+    let close = closers.get(quote);
+    if (!close) closers.set(quote, close = closingQuoteIndex(text, quote));
+    const end = close[SELECTOR_OPTION_KEY_RE.lastIndex];
+    if (end === -1) {
+      SELECTOR_OPTION_KEY_RE.lastIndex = match.index + 1;
+      continue;
+    }
+    out += `${text.slice(copied, match.index)}${match[1]}: <STR>`;
+    copied = end + 1;
+    SELECTOR_OPTION_KEY_RE.lastIndex = copied;
+  }
+  return out + text.slice(copied);
+}
 function classifyError(text) {
   if (/strict mode violation/i.test(text)) return "strict-mode";
   if (/\bexpect\(|\bexpect\.|Expected (?:string|substring|pattern|value)|\.toHave|\.toBe|\.toContain|\.toEqual/.test(text))
@@ -2829,7 +2872,7 @@ function maskVolatile(text) {
 function extractErrorSignature(rawError) {
   const text = stripAnsi(rawError);
   const errorType = classifyError(text);
-  const normalizedMessage = maskVolatile(extractMessageHead(text));
+  const normalizedMessage = maskVolatile(maskSelectorOptions(extractMessageHead(text)));
   const selector = extractSelector(text);
   const topFrameFile = extractTopFrameFile(text);
   const signature = (normalizedMessage.split("\n")[0] || "").slice(0, 200) || "Unknown error";
@@ -3212,7 +3255,7 @@ function inspectionGateFromTestInfo(testInfo, enabled = process.env.PIWI_INSPECT
 
 // src/internal/capture/pick-on-failure.ts
 var path5 = __toESM(require("path"));
-var ANSI_RE2 = /\[[0-9;]*m/g;
+var ANSI_RE2 = /\x1b\[[0-9;]*m/g;
 function parseLeafLocatorExpression(rawExpr) {
   const leaf = parseLeafLocatorCall(rawExpr);
   return leaf ? { method: leaf.method, args: locatorCallValues(leaf) } : null;

package/dist/internal/capture/locator-healing.d.ts

@@ -85,12 +85,6 @@ declare const EXPECT_CAPTURE_EXPRESSIONS: ReadonlySet<string>;
 declare function extractAccessibleName(ariaSnapshot: string | null): string | null;
 /** Tags whose accessible name comes from an associated `<label>`. */
 declare const FORM_FIELD_TAGS: ReadonlySet<string>;
-/**
- * Roles whose accessible name is computed from the element's own content. For
- * these, an element carrying neither `aria-label` nor `aria-labelledby` is
- * named by its trimmed text — which the probe already returns.
- */
-declare const NAME_FROM_CONTENT_ROLES: ReadonlySet<string>;
 /** The subset of a probe result the accessible-name shortcut reads. */
 interface NameSource {
     tagName: string;
@@ -153,4 +147,4 @@ declare function suggestLocatorsFromAria(failed: FailedLocatorInfo, ariaSnapshot
  */
 declare function captureCallerLocation(stack?: string): string | null;
 
-export { ACTION_METHODS, CHAIN_METHODS, EXPECT_CAPTURE_EXPRESSIONS, EXPECT_METHOD, FORM_FIELD_TAGS, type FailedLocatorInfo, LOCATOR_CREATING_CHAINS, LOCATOR_METHODS, type LocatorSuggestion, NAME_FROM_CONTENT_ROLES, type NameSource, captureCallerLocation, dedupeSnapshotsByLocation, exactAccessibleName, extractAccessibleName, renderFailing, suggestLocatorsFromAria };
+export { ACTION_METHODS, CHAIN_METHODS, EXPECT_CAPTURE_EXPRESSIONS, EXPECT_METHOD, FORM_FIELD_TAGS, type FailedLocatorInfo, LOCATOR_CREATING_CHAINS, LOCATOR_METHODS, type LocatorSuggestion, type NameSource, captureCallerLocation, dedupeSnapshotsByLocation, exactAccessibleName, extractAccessibleName, renderFailing, suggestLocatorsFromAria };

package/dist/internal/capture/locator-healing.js

@@ -39,7 +39,6 @@ __export(locator_healing_exports, {
   INPUT_TYPE_TO_ROLE: () => INPUT_TYPE_TO_ROLE,
   LOCATOR_CREATING_CHAINS: () => LOCATOR_CREATING_CHAINS,
   LOCATOR_METHODS: () => LOCATOR_METHODS,
-  NAME_FROM_CONTENT_ROLES: () => NAME_FROM_CONTENT_ROLES,
   TAG_TO_ROLE: () => TAG_TO_ROLE,
   approximateAccessibleName: () => approximateAccessibleName,
   captureCallerLocation: () => captureCallerLocation,
@@ -760,7 +759,6 @@ function captureCallerLocation(stack = new Error().stack ?? "") {
   INPUT_TYPE_TO_ROLE,
   LOCATOR_CREATING_CHAINS,
   LOCATOR_METHODS,
-  NAME_FROM_CONTENT_ROLES,
   TAG_TO_ROLE,
   approximateAccessibleName,
   captureCallerLocation,

package/dist/internal/capture/pick-on-failure.js

@@ -343,7 +343,7 @@ function installPickerOverlay(arg) {
   banner.style.cssText = `position:fixed;top:12px;left:50%;transform:translateX(-50%);z-index:${Z + 2};background:#111827;color:#f9fafb;font:13px/1.5 system-ui,sans-serif;border:1px solid #312e81;padding:10px 16px;border-radius:10px;box-shadow:0 4px 24px rgba(0,0,0,.5);max-width:min(680px,86vw);`;
   const hlTokens = (expr) => {
     const escHtml = (s) => s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
-    const re = /('(?:\\.|[^'])*'|"(?:\\.|[^"])*")|([A-Za-z_$][\w$]*)(?=\s*\()|([A-Za-z_$][\w$]*)(?=\s*:)|(true|false|null|\d+)|([{}(),.])/g;
+    const re = /('(?:\\.|[^'\\])*'|"(?:\\.|[^"\\])*")|([A-Za-z_$][\w$]*)(?=\s*\()|([A-Za-z_$][\w$]*)(?=\s*:)|(true|false|null|\d+)|([{}(),.])/g;
     let html = "";
     let last = 0;
     let m;
@@ -886,7 +886,7 @@ function showPickerChoices(arg) {
   panel.style.cssText = "background:#111827;color:#f9fafb;border-radius:10px;padding:20px;max-width:640px;width:90vw;max-height:70vh;overflow:auto;box-shadow:0 8px 40px rgba(0,0,0,.5);";
   const hlLocator = (expr) => {
     const escHtml = (s) => s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
-    const re = /('(?:\\.|[^'])*'|"(?:\\.|[^"])*")|([A-Za-z_$][\w$]*)(?=\s*\()|([A-Za-z_$][\w$]*)(?=\s*:)|(true|false|null|\d+)|([{}(),.])/g;
+    const re = /('(?:\\.|[^'\\])*'|"(?:\\.|[^"\\])*")|([A-Za-z_$][\w$]*)(?=\s*\()|([A-Za-z_$][\w$]*)(?=\s*:)|(true|false|null|\d+)|([{}(),.])/g;
     let html = "";
     let last = 0;
     let m;
@@ -1412,7 +1412,7 @@ function renderFailing(failed) {
 }
 
 // src/internal/capture/pick-on-failure.ts
-var ANSI_RE = /\[[0-9;]*m/g;
+var ANSI_RE = /\x1b\[[0-9;]*m/g;
 function parseLeafLocatorExpression(rawExpr) {
   const leaf = parseLeafLocatorCall(rawExpr);
   return leaf ? { method: leaf.method, args: locatorCallValues(leaf) } : null;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@piwitests/reporter",
-  "version": "0.41.0",
+  "version": "0.43.0",
   "description": "Playwright reporter that streams results, traces and HTML reports to a Piwi Dashboard instance",
   "url": "https://github.com/PiwiTests/platform",
   "homepage": "https://piwitests.dev",

package/templates/skills/investigate-failure/SKILL.md

@@ -27,7 +27,7 @@ Prefer the **Piwi MCP server** if it is connected to this agent (tools named `li
 
 7. **Verify.** Re-run the affected spec(s): `npx playwright test <file>`. Confirm they pass and the new run is green in the dashboard (`get_run_insights` compares against the last green run: regressions cleared, nothing new broken).
 
-8. **Close the loop (optional).** With packages/reporter/admin access, `set_cluster_status` marks the cluster resolved with a note so it drops off the triage queue.
+8. **Close the loop (optional).** With reporter or admin access, `set_cluster_status` marks the cluster resolved with a note so it drops off the triage queue.
 
 ## Guardrails
 

package/templates/skills/setup-piwi/SKILL.md

@@ -47,7 +47,7 @@ change it to import from the fixtures file instead, e.g. `import { test, expect
 
 ## Authentication
 
-If the dashboard has auth enabled, runs need an API key. Have the user create one in the dashboard (**Settings → Users → API keys**; keys start with `pd_`), then put it in `.env` as `PIWI_API_KEY=pd_...` and make sure `.env` is git-ignored. Never hardcode a key into `playwright.config.ts` or commit it. In CI, pass it as a secret (`PIWI_API_KEY`).
+If the dashboard has auth enabled, runs need an API key. Have the user create one in the dashboard (**Settings → Account → API keys**; keys start with `pd_`), then put it in `.env` as `PIWI_API_KEY=pd_...` and make sure `.env` is git-ignored. Never hardcode a key into `playwright.config.ts` or commit it. In CI, pass it as a secret (`PIWI_API_KEY`).
 
 ## Verify it worked
 

package/templates/skills/stabilize-flaky-tests/SKILL.md

@@ -1,36 +1,51 @@
 ---
 name: stabilize-flaky-tests
-description: Find the flakiest Playwright tests from Piwi's flaky analysis and fix the root cause of the intermittency, ranked by impact. Use when the user asks to "fix flaky tests", "reduce flakiness", "why is this test flaky", or wants to clean up an unreliable suite.
+description: Find the flakiest Playwright tests from Piwi's flaky analysis, make each one fail on demand with Piwi's Flake Lab, fix what the reproducing condition points at, and prove the fix under that same condition. Use when the user asks to "fix flaky tests", "reduce flakiness", "why is this test flaky", or wants to clean up an unreliable suite.
 ---
 
 # Stabilize flaky tests with Piwi
 
-Piwi scores every test's flakiness over its whole run history and ranks it by impact, so you spend effort on the tests that actually cost the team — not whichever one failed most recently. This skill picks the worst offenders, fixes the *cause* of the intermittency, and confirms the score improves.
+Piwi scores every test's flakiness over its whole run history and ranks it by impact, so you spend effort on the tests that actually cost the team — not whichever one failed most recently. For each test it also ranks **suspects** from that history (a route that is slower when it fails, a test running alongside, load), and its **Flake Lab** replays each suspect as a condition next to a control until the failure reproduces. A reproduced condition proves the cause, and the same condition later proves the fix. This skill follows that loop: rank, reproduce, fix, verify.
 
 ## How you reach Piwi
 
-Prefer the **Piwi MCP server** if it is connected (`list_flaky_tests`, `get_test_stability_trend`, `get_test_case`, `get_spec_health`). Otherwise use the dashboard's **Flaky** tab for the project and work from what it shows plus the repo.
+Prefer the **Piwi MCP server** if it is connected (`list_flaky_tests`, `get_flake_profile`, `plan_flake_experiment`, `get_test_case`, `get_test_stability_trend`). Otherwise use the dashboard's **Flaky** tab and the test's **Flakiness** tab, and work from what they show plus the repo. The lab itself is a command you run from the project root: `npx @piwitests/reporter flake`. It needs the Piwi capture fixtures in the specs (`piwiFixtures`) and a dashboard URL and reporter API key (`PIWI_DASHBOARD_URL`, `PIWI_API_KEY`, or the project's `.env`).
 
 ## Steps
 
-1. **Rank the flaky tests.** Call `list_flaky_tests` for the project. It returns each test's flaky score, impact ranking, and a **root-cause category**. Take the top few by impact — do not try to fix the whole list at once.
+1. **Rank the flaky tests.** Call `list_flaky_tests` for the project and take the top few by impact. Do not try to fix the whole list at once.
 
-2. **Understand one test's pattern.** For a chosen test, call `get_test_stability_trend` (is it getting worse, or already stable again?) and `get_test_case` (recent pass/fail history). Read a representative failing execution (`get_test_run_case` / `explain_failure`) for the error, steps, console, and network at the moment it flaked.
+2. **Read the suspects.** For a chosen test, call `get_flake_profile`. Each suspect carries raw counts ("slow in 7 of 8 failures and 3 of 44 passes"), the condition that would test it, and, once the lab has run, its latest result (`lab`). `experiments` lists what the lab already found; if an experiment reproduced the test, skip to step 4. Read one failing execution (`get_test_run_case` / `explain_failure`) for the error at the moment it flaked.
 
-3. **Identify the cause.** Common categories and their fixes:
-   - **Timing / race** — replace fixed waits and manual sleeps with Playwright web-first assertions (`await expect(locator).toBeVisible()`), await the network/UI state the test depends on, not a timeout.
-   - **Locator instability** — a selector that resolves ambiguously or intermittently; heal it (see the `apply-locator-healing` skill) or scope it.
-   - **Test-order / shared state** — leaking storage, cookies, or a shared backend row; isolate setup, use a fresh context, or a unique fixture per test.
-   - **Slow / unreliable endpoint** — check `get_network_requests`; if a real endpoint is intermittently slow or 500s, that is a product bug, not a test bug — report it rather than papering over it with retries.
+3. **Reproduce it.** Run the lab on the test (`plan_flake_experiment` gives the exact commands and the arms it will run, with an estimate):
 
-4. **Fix the root cause.** Make the change in the spec (or the app, when the flake is a real defect). Do **not** "fix" flakiness by adding retries or increasing timeouts — that hides it; the goal is a test that passes deterministically.
+   ```bash
+   npx @piwitests/reporter flake <testCaseId>
+   ```
 
-5. **Verify.** Re-run the test several times to shake out intermittency: `npx playwright test <file> --repeat-each=5` (raise the count for stubborn ones). It should pass every time. Over the next runs, confirm the flaky score falls in the dashboard (`get_test_stability_trend`).
+   It runs a control, then one arm per suspect, retries off, and counts only failures with the same error as in CI. Exit code `0` means an arm reproduced the failure ("reproduced by delaying GET /api/cart to 1.8 s: 3/4 against 0/10, p = 0.011"); `1` means nothing did; `2` is an error (read its message: often the fixtures are not in use). Use `--suspect <n>` to test one suspect, `--all` to also try every condition at once. If the output warns that this checkout differs from the commit of the failures, say so in your report.
 
-6. **Report.** For each test you touched: the root-cause category, the change, and the repeat-run result. List any remaining high-impact flaky tests so the user can decide whether to continue.
+   If nothing reproduces, do not guess a fix from the suspects alone. Report the arms and their counts, and fall back to reading the failure and the attempts diff.
+
+4. **Fix what the reproducing condition points at.** The condition names the cause:
+   - **A delayed route** — the test (or the app) acts before that response arrives. Wait for the response or for the UI state it produces (`await page.waitForResponse(...)`, `await expect(locator).toHaveText(...)`), not for a timeout.
+   - **A failed route** — the app does not handle that error. If the endpoint really fails in CI, that is a product finding: report it rather than hiding it in the test.
+   - **Another test alongside or just before** — shared state: a backend row, storage, a fixture both use. Isolate it (unique data per test, a fresh context, its own fixture).
+   - **CPU load** — a race the load exposes; the fix is the same as a delayed route: await the state the test depends on.
+   Make the change in the spec, or in the app when the flake is a real defect.
+
+5. **Verify the fix under the same condition.** Run:
+
+   ```bash
+   npx @piwitests/reporter flake verify <testCaseId>
+   ```
+
+   It reruns the arm that reproduced the failure, and its control, for enough runs to say the fix holds at the rate it failed. Exit code `0` means the fix held; `1` means it still fails (or too few runs passed to say). "It passed ten times in a row" is not a verification; this is.
+
+6. **Report.** For each test you touched: the reproducing condition and its counts, the change, and the verify result. List any remaining high-impact flaky tests, and any test the lab could not reproduce, so the user can decide whether to continue.
 
 ## Guardrails
 
-- Fix the cause, never the symptom — no blanket `test.retry`, no bumped global timeout, no `waitForTimeout` sprinkles.
+- Fix the cause, never the symptom — no blanket `test.retry`, no bumped global or per-test timeout, no `waitForTimeout` sprinkles. A longer timeout can make `flake verify` pass without fixing anything; do not do it.
 - A test that flakes because a *real* endpoint is unreliable is a product finding; surface it instead of hiding it in the test.
-- Stabilize a few high-impact tests well rather than lightly touching many. Prove each one with repeated runs before moving on.
+- Stabilize a few high-impact tests well rather than lightly touching many. Prove each one with `piwi flake verify` before moving on.