@piwitests/reporter 0.40.0 → 0.42.0

package/dist/internal/capture/locator-healing.js

@@ -259,8 +259,9 @@ function isAutoGenerated(value) {
   }
   if (/^(emotion-|styled-|css-|sc-)/.test(value)) return true;
   if (value.startsWith("ng-")) return true;
-  if (/^(radix-|headlessui-|mui-|mantine-|chakra-)/i.test(value)) return true;
+  if (/^(radix-|headlessui-|mui-|mantine-|chakra-|reka-|bits-|downshift-|react-aria|el-id-)/i.test(value)) return true;
   if (/^:r[0-9a-z]+:$/i.test(value) || /^«r[0-9a-z]+»$/i.test(value)) return true;
+  if (/(^|[-_:])v-\d+(-\d+)+$/.test(value) || /(^|[-_:])v-\d+(-\d+)+[-_:]/.test(value)) return true;
   return false;
 }
 var TEST_DATA_ATTRS = /* @__PURE__ */ new Set(["data-test", "data-test-id", "data-qa", "data-qa-id", "data-cy", "data-e2e"]);

package/dist/internal/capture/pick-on-failure.js

@@ -472,6 +472,11 @@ function installPickerOverlay(arg) {
     placeLabel(r);
     foot.textContent = "click to pick \xB7 \u2191 parent \xB7 \u2193 child \xB7 Esc skip";
   };
+  const redescribe = () => {
+    labeled = null;
+    if (current()) refresh();
+  };
+  g.__piwiRedescribe = redescribe;
   const stop = (e) => {
     e.preventDefault();
     e.stopImmediatePropagation();
@@ -526,6 +531,7 @@ function installPickerOverlay(arg) {
     highlight.remove();
     label.remove();
     banner.remove();
+    if (g.__piwiRedescribe === redescribe) delete g.__piwiRedescribe;
   };
   const reportPicked = (el) => {
     removePickingListeners();
@@ -1056,8 +1062,9 @@ function isAutoGenerated(value) {
   }
   if (/^(emotion-|styled-|css-|sc-)/.test(value)) return true;
   if (value.startsWith("ng-")) return true;
-  if (/^(radix-|headlessui-|mui-|mantine-|chakra-)/i.test(value)) return true;
+  if (/^(radix-|headlessui-|mui-|mantine-|chakra-|reka-|bits-|downshift-|react-aria|el-id-)/i.test(value)) return true;
   if (/^:r[0-9a-z]+:$/i.test(value) || /^«r[0-9a-z]+»$/i.test(value)) return true;
+  if (/(^|[-_:])v-\d+(-\d+)+$/.test(value) || /(^|[-_:])v-\d+(-\d+)+[-_:]/.test(value)) return true;
   return false;
 }
 var TEST_DATA_ATTRS = /* @__PURE__ */ new Set(["data-test", "data-test-id", "data-qa", "data-qa-id", "data-cy", "data-e2e"]);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@piwitests/reporter",
-  "version": "0.40.0",
+  "version": "0.42.0",
   "description": "Playwright reporter that streams results, traces and HTML reports to a Piwi Dashboard instance",
   "url": "https://github.com/PiwiTests/platform",
   "homepage": "https://piwitests.dev",

package/templates/skills/fix-a-reported-bug/SKILL.md

@@ -0,0 +1,47 @@
+---
+name: fix-a-reported-bug
+description: Fix a bug someone reported from Piwi Picker, starting from its failing test. Use when the user names a Piwi bug report ("bug #37"), asks to "fix the reported bug", "reproduce this bug report", or pastes a link to /bug-reports/<id>.
+---
+
+# Fix a reported bug
+
+A bug report sent from Piwi Picker is already a test: the steps someone took, the assertion that says what the page
+should show, and the value it showed instead. This skill turns it into a committed failing test, reproduces the bug,
+fixes it, and proves the fix with that same test.
+
+## How you reach Piwi
+
+Prefer the **Piwi MCP server** if it is connected: `get_bug_report` (the steps in words, what was expected, the
+evidence, the reproductions, the tests that visit the page) and `render_steps` with `bugReportId` (the failing test).
+Without it, the CLI does the same with the dashboard's URL and an API key: `npx @piwitests/reporter bug <id>`.
+
+## Steps
+
+1. **Read the report.** `get_bug_report { id }`. Note the page, each expected result with the value the page showed,
+   the reporter's note, the failed requests and console errors. The evidence often names the cause (a `POST … 500`).
+
+2. **Write the failing test.** `npx @piwitests/reporter bug <id> --write` writes the spec to the project's bugs folder
+   (`tests/bugs` unless the project names another) and runs it once. It is marked `test.fail()` and annotated
+   `piwi:bug <id>`, so it passes while the bug exists. Exit code 0 means the bug reproduces here. Exit code 1 means it
+   does not: read the output. A step that finds no element means the page or the data differ here (a user, a cart, a
+   flag the report assumed): make the test's setup provide them, in the project's style, rather than deleting steps.
+
+3. **Look at the tests that already visit the page** (`missedBy` in `get_bug_report`). If one of them reaches the
+   element, the missing piece is often one assertion in that test; say so in your report, and keep the bug's own spec
+   as the regression test.
+
+4. **Fix the bug** in the application code. Keep the change to what the evidence and the expected result point at.
+
+5. **Prove the fix.** Remove the `test.fail()` line from the bug's spec and run it: it must pass. Run the tests that
+   visit the page too (`npx playwright test <their files>`), so the fix breaks nothing around it.
+
+6. **Report.** The cause, the fix, the spec file, and the two runs. Once the spec runs in CI without `test.fail()`, Piwi
+   closes the report on its own; before that, a run where the spec still carries `test.fail()` and passes marks it
+   "looks fixed".
+
+## Guardrails
+
+- Never change the expected value in the spec to make it pass: it is what the reporter said the page should show. If
+  you believe the reporter was wrong, stop and ask.
+- Keep `piwi:bug <id>` on the test: it is how the report follows the fix.
+- One bug, one spec. Do not fold the reproduction into an unrelated test.

package/templates/skills/investigate-failure/SKILL.md

@@ -27,7 +27,7 @@ Prefer the **Piwi MCP server** if it is connected to this agent (tools named `li
 
 7. **Verify.** Re-run the affected spec(s): `npx playwright test <file>`. Confirm they pass and the new run is green in the dashboard (`get_run_insights` compares against the last green run: regressions cleared, nothing new broken).
 
-8. **Close the loop (optional).** With packages/reporter/admin access, `set_cluster_status` marks the cluster resolved with a note so it drops off the triage queue.
+8. **Close the loop (optional).** With reporter or admin access, `set_cluster_status` marks the cluster resolved with a note so it drops off the triage queue.
 
 ## Guardrails
 

package/templates/skills/run-the-right-tests/SKILL.md

@@ -9,10 +9,22 @@ Running the whole suite for every change is slow, and hand-maintained grep patte
 
 ## How you reach Piwi
 
-Prefer the **Piwi MCP server** if it is connected (`list_selections`, `resolve_selection`, `preview_selection`). Otherwise use the reporter CLI (`npx @piwitests/reporter select|run`) with `PIWI_DASHBOARD_URL` / `PIWI_API_KEY` / `PIWI_PROJECT_NAME` set, or the dashboard's **Selections** tab.
+Prefer the **Piwi MCP server** if it is connected (`list_selections`, `resolve_selection`, `preview_selection`,
+`predict_locator_breaks`). Otherwise use the reporter CLI (`npx @piwitests/reporter select|run|preflight`) with `PIWI_DASHBOARD_URL` / `PIWI_API_KEY` / `PIWI_PROJECT_NAME` set, or the dashboard's **Selections** tab.
 
 ## Steps
 
+0. **After a UI change, check the locators it breaks first.** Renaming a label, an `aria-label`, a test id, a
+   placeholder or a translation breaks every test that finds the element by it. Run
+   ```
+   npx @piwitests/reporter preflight --fix
+   ```
+   It diffs the working tree against `HEAD`, lists the locators the change breaks with their tests, and rewrites the
+   ones whose call site holds the string (keeping the quotes). Edit the ones it lists as "edit by hand" yourself, then
+   run the tests it names with `npx @piwitests/reporter preflight --run`. Over MCP, `predict_locator_breaks` takes
+   your `git diff` and returns the same breaks and edits. A prediction is not a failure: say which breaks you fixed and
+   which you left, and why.
+
 1. **See what selections exist.** Call `list_selections` for the project. Every project has the built-ins `failed` (tests whose latest run failed) and `quarantine-free`, plus any the team saved (often `smoke`). Pick the one that matches the intent:
    - About to make a broad change, or want a quick confidence check → `smoke` (or `preview_selection` with `{ "include": [{ "tags": ["smoke"] }] }`).
    - Iterating on a fix → `failed`, or a `{ "include": [{ "failedInLastRuns": 5 }] }` preview.

package/templates/skills/setup-piwi/SKILL.md

@@ -47,7 +47,7 @@ change it to import from the fixtures file instead, e.g. `import { test, expect
 
 ## Authentication
 
-If the dashboard has auth enabled, runs need an API key. Have the user create one in the dashboard (**Settings → Users → API keys**; keys start with `pd_`), then put it in `.env` as `PIWI_API_KEY=pd_...` and make sure `.env` is git-ignored. Never hardcode a key into `playwright.config.ts` or commit it. In CI, pass it as a secret (`PIWI_API_KEY`).
+If the dashboard has auth enabled, runs need an API key. Have the user create one in the dashboard (**Settings → Account → API keys**; keys start with `pd_`), then put it in `.env` as `PIWI_API_KEY=pd_...` and make sure `.env` is git-ignored. Never hardcode a key into `playwright.config.ts` or commit it. In CI, pass it as a secret (`PIWI_API_KEY`).
 
 ## Verify it worked
 

package/templates/skills/stabilize-flaky-tests/SKILL.md

@@ -1,36 +1,51 @@
 ---
 name: stabilize-flaky-tests
-description: Find the flakiest Playwright tests from Piwi's flaky analysis and fix the root cause of the intermittency, ranked by impact. Use when the user asks to "fix flaky tests", "reduce flakiness", "why is this test flaky", or wants to clean up an unreliable suite.
+description: Find the flakiest Playwright tests from Piwi's flaky analysis, make each one fail on demand with Piwi's Flake Lab, fix what the reproducing condition points at, and prove the fix under that same condition. Use when the user asks to "fix flaky tests", "reduce flakiness", "why is this test flaky", or wants to clean up an unreliable suite.
 ---
 
 # Stabilize flaky tests with Piwi
 
-Piwi scores every test's flakiness over its whole run history and ranks it by impact, so you spend effort on the tests that actually cost the team — not whichever one failed most recently. This skill picks the worst offenders, fixes the *cause* of the intermittency, and confirms the score improves.
+Piwi scores every test's flakiness over its whole run history and ranks it by impact, so you spend effort on the tests that actually cost the team — not whichever one failed most recently. For each test it also ranks **suspects** from that history (a route that is slower when it fails, a test running alongside, load), and its **Flake Lab** replays each suspect as a condition next to a control until the failure reproduces. A reproduced condition proves the cause, and the same condition later proves the fix. This skill follows that loop: rank, reproduce, fix, verify.
 
 ## How you reach Piwi
 
-Prefer the **Piwi MCP server** if it is connected (`list_flaky_tests`, `get_test_stability_trend`, `get_test_case`, `get_spec_health`). Otherwise use the dashboard's **Flaky** tab for the project and work from what it shows plus the repo.
+Prefer the **Piwi MCP server** if it is connected (`list_flaky_tests`, `get_flake_profile`, `plan_flake_experiment`, `get_test_case`, `get_test_stability_trend`). Otherwise use the dashboard's **Flaky** tab and the test's **Flakiness** tab, and work from what they show plus the repo. The lab itself is a command you run from the project root: `npx @piwitests/reporter flake`. It needs the Piwi capture fixtures in the specs (`piwiFixtures`) and a dashboard URL and reporter API key (`PIWI_DASHBOARD_URL`, `PIWI_API_KEY`, or the project's `.env`).
 
 ## Steps
 
-1. **Rank the flaky tests.** Call `list_flaky_tests` for the project. It returns each test's flaky score, impact ranking, and a **root-cause category**. Take the top few by impact — do not try to fix the whole list at once.
+1. **Rank the flaky tests.** Call `list_flaky_tests` for the project and take the top few by impact. Do not try to fix the whole list at once.
 
-2. **Understand one test's pattern.** For a chosen test, call `get_test_stability_trend` (is it getting worse, or already stable again?) and `get_test_case` (recent pass/fail history). Read a representative failing execution (`get_test_run_case` / `explain_failure`) for the error, steps, console, and network at the moment it flaked.
+2. **Read the suspects.** For a chosen test, call `get_flake_profile`. Each suspect carries raw counts ("slow in 7 of 8 failures and 3 of 44 passes"), the condition that would test it, and, once the lab has run, its latest result (`lab`). `experiments` lists what the lab already found; if an experiment reproduced the test, skip to step 4. Read one failing execution (`get_test_run_case` / `explain_failure`) for the error at the moment it flaked.
 
-3. **Identify the cause.** Common categories and their fixes:
-   - **Timing / race** — replace fixed waits and manual sleeps with Playwright web-first assertions (`await expect(locator).toBeVisible()`), await the network/UI state the test depends on, not a timeout.
-   - **Locator instability** — a selector that resolves ambiguously or intermittently; heal it (see the `apply-locator-healing` skill) or scope it.
-   - **Test-order / shared state** — leaking storage, cookies, or a shared backend row; isolate setup, use a fresh context, or a unique fixture per test.
-   - **Slow / unreliable endpoint** — check `get_network_requests`; if a real endpoint is intermittently slow or 500s, that is a product bug, not a test bug — report it rather than papering over it with retries.
+3. **Reproduce it.** Run the lab on the test (`plan_flake_experiment` gives the exact commands and the arms it will run, with an estimate):
 
-4. **Fix the root cause.** Make the change in the spec (or the app, when the flake is a real defect). Do **not** "fix" flakiness by adding retries or increasing timeouts — that hides it; the goal is a test that passes deterministically.
+   ```bash
+   npx @piwitests/reporter flake <testCaseId>
+   ```
 
-5. **Verify.** Re-run the test several times to shake out intermittency: `npx playwright test <file> --repeat-each=5` (raise the count for stubborn ones). It should pass every time. Over the next runs, confirm the flaky score falls in the dashboard (`get_test_stability_trend`).
+   It runs a control, then one arm per suspect, retries off, and counts only failures with the same error as in CI. Exit code `0` means an arm reproduced the failure ("reproduced by delaying GET /api/cart to 1.8 s: 3/4 against 0/10, p = 0.011"); `1` means nothing did; `2` is an error (read its message: often the fixtures are not in use). Use `--suspect <n>` to test one suspect, `--all` to also try every condition at once. If the output warns that this checkout differs from the commit of the failures, say so in your report.
 
-6. **Report.** For each test you touched: the root-cause category, the change, and the repeat-run result. List any remaining high-impact flaky tests so the user can decide whether to continue.
+   If nothing reproduces, do not guess a fix from the suspects alone. Report the arms and their counts, and fall back to reading the failure and the attempts diff.
+
+4. **Fix what the reproducing condition points at.** The condition names the cause:
+   - **A delayed route** — the test (or the app) acts before that response arrives. Wait for the response or for the UI state it produces (`await page.waitForResponse(...)`, `await expect(locator).toHaveText(...)`), not for a timeout.
+   - **A failed route** — the app does not handle that error. If the endpoint really fails in CI, that is a product finding: report it rather than hiding it in the test.
+   - **Another test alongside or just before** — shared state: a backend row, storage, a fixture both use. Isolate it (unique data per test, a fresh context, its own fixture).
+   - **CPU load** — a race the load exposes; the fix is the same as a delayed route: await the state the test depends on.
+   Make the change in the spec, or in the app when the flake is a real defect.
+
+5. **Verify the fix under the same condition.** Run:
+
+   ```bash
+   npx @piwitests/reporter flake verify <testCaseId>
+   ```
+
+   It reruns the arm that reproduced the failure, and its control, for enough runs to say the fix holds at the rate it failed. Exit code `0` means the fix held; `1` means it still fails (or too few runs passed to say). "It passed ten times in a row" is not a verification; this is.
+
+6. **Report.** For each test you touched: the reproducing condition and its counts, the change, and the verify result. List any remaining high-impact flaky tests, and any test the lab could not reproduce, so the user can decide whether to continue.
 
 ## Guardrails
 
-- Fix the cause, never the symptom — no blanket `test.retry`, no bumped global timeout, no `waitForTimeout` sprinkles.
+- Fix the cause, never the symptom — no blanket `test.retry`, no bumped global or per-test timeout, no `waitForTimeout` sprinkles. A longer timeout can make `flake verify` pass without fixing anything; do not do it.
 - A test that flakes because a *real* endpoint is unreliable is a product finding; surface it instead of hiding it in the test.
-- Stabilize a few high-impact tests well rather than lightly touching many. Prove each one with repeated runs before moving on.
+- Stabilize a few high-impact tests well rather than lightly touching many. Prove each one with `piwi flake verify` before moving on.