@piwitests/reporter 0.56.0 → 0.58.0

package/dist/cli/index.js

@@ -873,6 +873,8 @@ var PIWI_ENV_KEYS = {
   codeReachRoots: "PIWI_CODE_REACH_ROOTS",
   coverageBrowser: "PIWI_COVERAGE_BROWSER",
   coverageServer: "PIWI_COVERAGE_SERVER",
+  externalIdPattern: "PIWI_EXTERNAL_ID_PATTERN",
+  externalIdOnDuplicate: "PIWI_EXTERNAL_ID_ON_DUPLICATE",
   captureServerTraces: "PIWI_CAPTURE_SERVER_TRACES",
   captureResources: "PIWI_CAPTURE_RESOURCES",
   leakCheck: "PIWI_LEAK_CHECK",
@@ -1137,13 +1139,18 @@ var ASSERTION_MATCHERS = [
   "toBeHidden",
   "toBeEnabled",
   "toBeDisabled",
-  "toHaveURL"
+  "toHaveURL",
+  "toContainText",
+  "toHaveCount",
+  "toBeChecked"
 ];
 var VALUE_MATCHERS = /* @__PURE__ */ new Set([
   "toHaveText",
   "toHaveValue",
   "toHaveAccessibleName",
-  "toHaveURL"
+  "toHaveURL",
+  "toContainText",
+  "toHaveCount"
 ]);
 
 // ../core/src/function-match.ts
@@ -1833,7 +1840,7 @@ function renderAssertStep(step, index, ctx) {
       });
       return [`  // Step ${index + 1}: ${matcher} without an expected value.`];
     }
-    arg = quote(matcher === "toHaveURL" ? urlForCode(assertion.expected, ctx) : assertion.expected);
+    arg = matcher === "toHaveCount" ? String(Number.parseInt(assertion.expected, 10) || 0) : quote(matcher === "toHaveURL" ? urlForCode(assertion.expected, ctx) : assertion.expected);
   }
   const subject = matcher === "toHaveURL" ? ctx.page : locatorForStep(step, index, ctx);
   let line = `  await expect(${subject})${not}.${matcher}(${arg});`;
@@ -1846,10 +1853,25 @@ function renderAssertStep(step, index, ctx) {
 function fileNames(value) {
   return (value ?? "").split("\n").map((name) => name.trim().replace(/^.*[\\/]/, "")).filter(Boolean);
 }
+function renderManualStep(step, index, ctx) {
+  const description = commentText(step.manual?.description ?? "") || "a step to do by hand";
+  ctx.warnings.push({
+    step: index,
+    code: "manual-step",
+    detail: description,
+    message: `Step ${index + 1} cannot be written as code: ${description}`
+  });
+  const lines = [`  // Manual step: ${description}`];
+  const call = step.manual?.call ? commentText(step.manual.call) : "";
+  const location = step.manual?.location ? commentText(step.manual.location) : "";
+  if (call || location) lines.push(`  //   ${[call, location ? `at ${location}` : ""].filter(Boolean).join(" ")}`);
+  return lines;
+}
 function renderRawStep(step, index, ctx) {
   const { page } = ctx;
   if (step.action === "goto") return [`  await ${page}.goto(${quote(urlForCode(step.value ?? step.pageUrl, ctx))});`];
   if (step.action === "assert" || step.action === "assertVisible") return renderAssertStep(step, index, ctx);
+  if (step.action === "manual") return renderManualStep(step, index, ctx);
   if (step.action === "press" && !step.target) {
     return [`  await ${page}.keyboard.press(${quote(step.value ?? "Enter")});`];
   }
@@ -2060,6 +2082,7 @@ function expectFailLine(options) {
   return reason ? `  test.fail(); // ${reason}` : "  test.fail();";
 }
 function samePage(a, b) {
+  if (!a || !b) return true;
   const ka = pageKey(a);
   const kb = pageKey(b);
   return ka === null || kb === null || ka === kb;
@@ -2111,7 +2134,7 @@ function renderSpec(session, options = {}) {
   const pageStarts = [];
   const enterPageOf = (index) => {
     const url = steps[index]?.pageUrl;
-    if (options.testSteps !== "page" || url == null) return;
+    if (options.testSteps !== "page" || !url) return;
     const last = pageStarts[pageStarts.length - 1];
     if (!last || !samePage(last.url, url)) pageStarts.push({ line: bodyLines.length, url });
   };
@@ -2123,7 +2146,7 @@ function renderSpec(session, options = {}) {
     const pattern = pageUrlPattern(next.pageUrl);
     enterPageOf(last + 1);
     if (pattern) bodyLines.push(`  await expect(${page}).toHaveURL(${pattern});`);
-    const acts = next.action !== "assert" && next.action !== "assertVisible";
+    const acts = next.action !== "assert" && next.action !== "assertVisible" && next.action !== "manual";
     const chosen = acts && next.target ? chooseLocator(next.target, options) : null;
     if (chosen) bodyLines.push(`  await expect(${page}.${chosen.text}).toHaveCount(1);`);
   };
@@ -2225,7 +2248,11 @@ function renderSpec(session, options = {}) {
 }
 
 // ../core/src/steps.ts
-var STEPS_VERSION = 1;
+var STEPS_VERSION = 2;
+var VERSION_2_MATCHERS = /* @__PURE__ */ new Set(["toContainText", "toHaveCount", "toBeChecked"]);
+function stepsVersionFor(steps) {
+  return steps.some((s) => s.action === "manual" || !!s.assertion && VERSION_2_MATCHERS.has(s.assertion.matcher)) ? 2 : 1;
+}
 var STEPS_LIMITS = {
   steps: 200,
   alternatives: 10,
@@ -2252,7 +2279,8 @@ var ACTIONS = /* @__PURE__ */ new Set([
   "setInputFiles",
   "dragTo",
   "assertVisible",
-  "assert"
+  "assert",
+  "manual"
 ]);
 function originOf2(url) {
   try {
@@ -2277,7 +2305,7 @@ function toAbsolute(url, origin) {
 function toStepsDocument(session, meta = {}) {
   const origin = originOf2(session.startUrl) ?? originOf2(session.steps[0]?.pageUrl ?? "");
   return {
-    v: STEPS_VERSION,
+    v: stepsVersionFor(session.steps),
     title: meta.title ?? null,
     origin,
     recordedAt: session.startedAt,
@@ -2386,6 +2414,9 @@ function checkAssertion(c, v, path24) {
     return void 0;
   }
   const expected = c.text(v.expected, `${path24}.expected`, STEPS_LIMITS.valueLength, !VALUE_MATCHERS.has(matcher));
+  if (matcher === "toHaveCount" && expected != null && !/^\d{1,6}$/.test(expected)) {
+    c.fail(`${path24}.expected`, "must be a whole number of elements on a toHaveCount assertion");
+  }
   return {
     matcher,
     expected: VALUE_MATCHERS.has(matcher) ? expected : null,
@@ -2394,6 +2425,19 @@ function checkAssertion(c, v, path24) {
     note: c.text(v.note, `${path24}.note`, STEPS_LIMITS.valueLength, true)
   };
 }
+function checkManual(c, v, path24) {
+  if (!isObject(v)) {
+    c.fail(path24, "is required on a manual step");
+    return void 0;
+  }
+  const description = c.text(v.description, `${path24}.description`, STEPS_LIMITS.valueLength, false);
+  if (description != null && !description.trim()) c.fail(`${path24}.description`, "must not be empty");
+  return {
+    description: description ?? "",
+    location: c.text(v.location, `${path24}.location`, STEPS_LIMITS.textLength, true),
+    call: c.text(v.call, `${path24}.call`, STEPS_LIMITS.textLength, true)
+  };
+}
 function checkViewports(c, v, stepCount) {
   if (v == null) return [];
   if (!Array.isArray(v)) {
@@ -2466,6 +2510,9 @@ function checkStep(c, v, path24) {
     step.assertion = { matcher: "toBeVisible", expected: null, actual: null, negated: false, note: null };
   } else if (action === "assert") {
     step.assertion = checkAssertion(c, v.assertion, `${path24}.assertion`);
+  } else if (action === "manual") {
+    step.value = null;
+    step.manual = checkManual(c, v.manual, `${path24}.manual`);
   }
   return step;
 }
@@ -2481,8 +2528,8 @@ function parseSteps(input) {
   if (!isObject(value)) return { ok: false, errors: ["a steps document must be a JSON object"] };
   const c = new Checker();
   const legacy = value.v == null && typeof value.startUrl === "string";
-  if (!legacy && value.v !== STEPS_VERSION) {
-    return { ok: false, errors: [`v: this reads version ${STEPS_VERSION}, not ${JSON.stringify(value.v)}`] };
+  if (!legacy && value.v !== 1 && value.v !== STEPS_VERSION) {
+    return { ok: false, errors: [`v: this reads versions 1 to ${STEPS_VERSION}, not ${JSON.stringify(value.v)}`] };
   }
   if (!Array.isArray(value.steps)) return { ok: false, errors: ["steps: must be a list"] };
   if (value.steps.length > STEPS_LIMITS.steps) {
@@ -2505,7 +2552,7 @@ function parseSteps(input) {
     if (originText && !origin) c.fail("origin", "must be an http(s) origin");
     const viewports = checkViewports(c, value.viewports, steps.length);
     doc = {
-      v: STEPS_VERSION,
+      v: stepsVersionFor(steps),
       title: c.text(value.title, "title", STEPS_LIMITS.textLength, true),
       origin,
       recordedAt: value.recordedAt == null ? 0 : c.number(value.recordedAt, "recordedAt"),
@@ -4832,7 +4879,7 @@ Connection:
   --server-url <url>       Dashboard URL      (env PIWI_DASHBOARD_URL)
   --api-key <key>          API key            (env PIWI_API_KEY)
 
-Policy (at least one is required):
+Policy (at least one rule or warn-only flag is required):
   --require-tag <tags>     Comma-separated; every test carrying the tag must pass
   --max-failed <n>         Fail when more than n tests failed
   --max-new-regressions <n>  Fail when more than n tests newly started failing
@@ -4845,7 +4892,7 @@ Policy (at least one is required):
   --max-new-leaks <n>      Fail when more than n of those were never seen on the base branch
 
 Warn-only (reported, never fails the build):
-  --max-uncovered-changes <n>  Warn when more than n changed files have no observed test reach
+  --max-uncovered-changes <n>  Warn when more than n changed files are uncovered changes: no test reached them
   --min-patch-coverage <percent>  Warn when fewer of the changed lines ran, in the commit's combined coverage
 
 Other:

package/dist/global-setup-module.js

@@ -127,6 +127,8 @@ var PIWI_ENV_KEYS = {
   codeReachRoots: "PIWI_CODE_REACH_ROOTS",
   coverageBrowser: "PIWI_COVERAGE_BROWSER",
   coverageServer: "PIWI_COVERAGE_SERVER",
+  externalIdPattern: "PIWI_EXTERNAL_ID_PATTERN",
+  externalIdOnDuplicate: "PIWI_EXTERNAL_ID_ON_DUPLICATE",
   captureServerTraces: "PIWI_CAPTURE_SERVER_TRACES",
   captureResources: "PIWI_CAPTURE_RESOURCES",
   leakCheck: "PIWI_LEAK_CHECK",
@@ -217,6 +219,13 @@ function resolveOptions(raw) {
     coverage.browser = coverageBrowser;
   if (coverage.server === void 0 && coverageServer) coverage.server = coverageServer;
   if (coverage.browser !== void 0 || coverage.server !== void 0) mergedRaw.coverage = coverage;
+  const externalId = { ...mergedRaw.externalId };
+  const externalIdPattern = env[PIWI_ENV_KEYS.externalIdPattern];
+  const externalIdOnDuplicate = env[PIWI_ENV_KEYS.externalIdOnDuplicate];
+  if (externalId.pattern === void 0 && externalIdPattern) externalId.pattern = externalIdPattern;
+  if (externalId.onDuplicate === void 0 && externalIdOnDuplicate)
+    externalId.onDuplicate = externalIdOnDuplicate;
+  if (Object.keys(externalId).length > 0) mergedRaw.externalId = externalId;
   const opts = { ...DEFAULTS, ...mergedRaw };
   if (env[PIWI_ENV_KEYS.verbose] !== void 0) opts.verbose = env[PIWI_ENV_KEYS.verbose] === "true";
   return opts;

package/dist/index.d.ts

@@ -129,6 +129,33 @@ interface PiwiDashboardOptions {
          */
         server?: 'off' | 'files' | 'functions' | 'lines';
     };
+    /**
+     * Match each test to its history on the dashboard by an external ID, the ID the test has in your own test
+     * management tool, so a test keeps its history when it is renamed or moved. A test declares its ID with a
+     * `piwi:id` annotation (`{ type: 'piwi:id', description: '12526' }`), which wins, or the ID is read from its title
+     * with `pattern`. Two tests declaring the same ID are reported before the first test runs, and matched by file,
+     * describe path and title instead.
+     */
+    externalId?: {
+        /**
+         * Where the ID sits in a test's own title (not its describe path): `/^(\d+), /` reads `12526` from
+         * `'12526, Log in with Email'`. The ID is the first capture group, or the whole match without a group. A string
+         * is compiled with `new RegExp(value)`; one that does not compile turns external IDs off for the run, with an
+         * error, but never fails it. Can also be set with `PIWI_EXTERNAL_ID_PATTERN`.
+         */
+        pattern?: RegExp | string;
+        /**
+         * Show the title without its ID on the dashboard, next to the ID. The whole match of `pattern` is removed from
+         * the title shown; the title stored and matched on stays whole. Defaults to `false`.
+         */
+        stripFromTitle?: boolean;
+        /**
+         * What two tests declaring the same ID do: `'report'` lists them in the console and on the run page; `'fail'`
+         * also fails the run once its tests have run. Either way the run matches them by file, describe path and title.
+         * Defaults to `'report'`. Can also be set with `PIWI_EXTERNAL_ID_ON_DUPLICATE`.
+         */
+        onDuplicate?: 'report' | 'fail';
+    };
     /**
      * Capture server-side spans for each API/document request the test makes,
      * read from the `X-Piwi-Trace` response header emitted by a Piwi
@@ -457,6 +484,8 @@ declare class PiwiDashboardReporter {
     private instanceId;
     private shardInfo;
     private metadata;
+    /** Each planned test's external ID and the IDs two tests share, settled in `onBegin`. */
+    private externalIds;
     private enabled;
     /**
      * True under `playwright test --list`, where Playwright still constructs this
@@ -515,6 +544,13 @@ declare class PiwiDashboardReporter {
     printsToStdio(): boolean;
     /** Playwright reporter hook: called once at the start of the test run */
     onBegin(config: FullConfig, suite: Suite): void;
+    /**
+     * Settle each planned test's external ID before any test runs, name the IDs two tests share, and record them on
+     * the run so the dashboard lists them too.
+     */
+    private planExternalIds;
+    /** The external ID fields of a test's payload: `null` tells the dashboard the test has none to match on. */
+    private externalIdFields;
     /** Playwright reporter hook: called when an individual test begins */
     onTestBegin(test: TestCase, result: TestResult): void;
     /** Track suite-level setup steps (beforeAll/afterAll) not tied to any test */
@@ -554,8 +590,14 @@ declare class PiwiDashboardReporter {
      * interrupted) is resolved once from the overall run status by the caller.
      */
     private materializeUnrunTests;
-    /** Playwright reporter hook: called when the full test run finishes */
-    onEnd(result: FullResult): Promise<void>;
+    /**
+     * Playwright reporter hook: called when the full test run finishes. Fails the run when two tests share an external
+     * ID and `externalId.onDuplicate` is `'fail'`.
+     */
+    onEnd(result: FullResult): Promise<{
+        status?: FullResult['status'];
+    } | void>;
+    private finishRun;
 }
 
 interface ResolveSelectionResult {

package/dist/index.js

@@ -128,6 +128,8 @@ var PIWI_ENV_KEYS = {
   codeReachRoots: "PIWI_CODE_REACH_ROOTS",
   coverageBrowser: "PIWI_COVERAGE_BROWSER",
   coverageServer: "PIWI_COVERAGE_SERVER",
+  externalIdPattern: "PIWI_EXTERNAL_ID_PATTERN",
+  externalIdOnDuplicate: "PIWI_EXTERNAL_ID_ON_DUPLICATE",
   captureServerTraces: "PIWI_CAPTURE_SERVER_TRACES",
   captureResources: "PIWI_CAPTURE_RESOURCES",
   leakCheck: "PIWI_LEAK_CHECK",
@@ -247,6 +249,13 @@ function resolveOptions(raw) {
     coverage.browser = coverageBrowser;
   if (coverage.server === void 0 && coverageServer) coverage.server = coverageServer;
   if (coverage.browser !== void 0 || coverage.server !== void 0) mergedRaw.coverage = coverage;
+  const externalId = { ...mergedRaw.externalId };
+  const externalIdPattern = env[PIWI_ENV_KEYS.externalIdPattern];
+  const externalIdOnDuplicate = env[PIWI_ENV_KEYS.externalIdOnDuplicate];
+  if (externalId.pattern === void 0 && externalIdPattern) externalId.pattern = externalIdPattern;
+  if (externalId.onDuplicate === void 0 && externalIdOnDuplicate)
+    externalId.onDuplicate = externalIdOnDuplicate;
+  if (Object.keys(externalId).length > 0) mergedRaw.externalId = externalId;
   const opts = { ...DEFAULTS, ...mergedRaw };
   if (env[PIWI_ENV_KEYS.verbose] !== void 0) opts.verbose = env[PIWI_ENV_KEYS.verbose] === "true";
   return opts;
@@ -652,6 +661,9 @@ function toWireTestCase(tc) {
     tags: rest.tags ?? null,
     locks: rest.locks ?? null,
     testMeta: rest.testMeta ?? null,
+    // Kept `undefined` when unset (a `begin` event): `null` would tell the dashboard the test has no ID.
+    externalId: rest.externalId,
+    displayTitle: rest.displayTitle ?? null,
     locatorSnapshots: rest.locatorSnapshots || null,
     locatorPages: rest.locatorPages || null,
     codeReach: rest.codeReach || null,
@@ -4549,6 +4561,30 @@ function runFilesOf(answer) {
   }
   return files;
 }
+function instanceOf(answer) {
+  const instance = isRecord2(answer) ? answer.instance : null;
+  if (!isRecord2(instance) || typeof instance.id !== "string" || !REQUEST_ID.test(instance.id)) return null;
+  const startedAt = typeof instance.startedAt === "string" && !Number.isNaN(Date.parse(instance.startedAt)) ? new Date(instance.startedAt).toISOString() : null;
+  return { id: instance.id, startedAt };
+}
+function processKeyOf(id) {
+  const dot = id.lastIndexOf(".");
+  return dot > 0 ? id.slice(0, dot) : null;
+}
+function withTestCoverage(files, tests) {
+  if (!tests || tests.files.length === 0) return files;
+  const byKey = new Map(files.map((file) => [`${file.runtime}\0${file.path}`, { ...file }]));
+  for (const ran of tests.files) {
+    const key = `${ran.runtime}\0${ran.path}`;
+    const file = byKey.get(key) ?? { path: ran.path, runtime: ran.runtime };
+    const functions = normalizeFunctionRanges([...file.functions ?? [], ...ran.functions ?? []]);
+    const lines = normalizeLineRanges([...file.lines ?? [], ...ran.lines ?? []]);
+    if (functions.length > 0) file.functions = functions;
+    if (lines.length > 0) file.lines = lines;
+    byKey.set(key, file);
+  }
+  return [...byKey.values()];
+}
 function isCollectAnswer(answer) {
   return isRecord2(answer) && answer.v === 1 && Array.isArray(answer.groups);
 }
@@ -4600,11 +4636,16 @@ var ServerCoverageCollector = class {
   }
   /**
    * The run-level coverage of an origin, with each server file's executable
-   * lines at `lines` granularity; null when the backend did not answer with it.
+   * lines at `lines` granularity, and the lines and functions the tests'
+   * coverage named added to it. The gap says why part of it is missing: the
+   * backend restarted during the run (its answer names another process than
+   * the run's first request ids), or it did not answer with run-level coverage.
    */
   async fetchRun(origin) {
     const state = this.stateOf(origin);
     const query = `?granularity=${encodeURIComponent(this.options.granularity)}`;
+    let answer = null;
+    let failure = null;
     for (const base of this.pathBases(origin)) {
       const outcome = await this.call(
         `${origin}${base}${RUN_PATH}${query}`,
@@ -4612,21 +4653,47 @@ var ServerCoverageCollector = class {
         this.limits.runTimeoutMs
       );
       if (outcome.kind === "failed") {
-        this.reportFailure(state, `Could not fetch the run's server coverage from ${origin}: ${outcome.reason}.`);
-        return null;
+        failure = outcome.reason;
+        break;
       }
       const answered = outcome.ok ? runFilesOf(outcome.json) : null;
       if (answered) {
         state.base = base;
-        const files = await this.verifyChecksums(
-          answered,
-          fileChecksums(isRecord2(outcome.json) ? outcome.json.files : null)
-        );
-        return this.options.blobs ? this.options.blobs.withBlobs(files.files) : files.files;
+        answer = { files: answered, json: outcome.json };
+        break;
       }
     }
-    this.reportFailure(state, `Could not fetch the run's server coverage: ${this.noEndpoint(origin, RUN_PATH)}.`);
-    return null;
+    const gap = answer ? this.restartGap(origin, state, instanceOf(answer.json)) : { origin, reason: "unreachable" };
+    if (!answer) failure ??= this.noEndpoint(origin, RUN_PATH);
+    if (gap && !this.closer.signal.aborted) this.logger.warn(this.gapMessage(gap, failure));
+    const checksums = fileChecksums(answer?.json.files ?? null, new Map(state.testChecksums));
+    const files = await this.verifyChecksums(withTestCoverage(answer?.files ?? [], state.testCoverage), checksums);
+    let out = files.files;
+    try {
+      if (this.options.blobs) out = await this.options.blobs.withBlobs(out);
+    } catch {
+    }
+    return { files: out, gap };
+  }
+  /**
+   * The gap of an origin whose run-level coverage `instance` answered: a
+   * restart when it, or a collection answer, names another process than the
+   * run's first request ids; null when the backend names no process.
+   */
+  restartGap(origin, state, instance) {
+    if (!instance) return null;
+    const keys = [...state.processKeys];
+    const others = [...state.instances.values()].filter((known) => known.id !== instance.id);
+    if (keys.every((key) => key === instance.id) && others.length === 0) return null;
+    const at = keys[0] === instance.id ? others[others.length - 1]?.startedAt ?? null : instance.startedAt;
+    return { origin, reason: "restarted", ...at ? { at } : {} };
+  }
+  gapMessage(gap, failure) {
+    if (gap.reason === "unreachable") {
+      return `Could not fetch the run's server coverage from ${gap.origin} (${failure}): the run's server coverage is marked incomplete, and keeps the lines the tests' coverage named.`;
+    }
+    const when = gap.at ? ` at about ${gap.at}` : "";
+    return `The backend at ${gap.origin} restarted during the run${when}: the run's server coverage is marked incomplete. It keeps the lines the tests' coverage named; what the earlier process ran outside a request is lost.`;
   }
   /** Abort every call in flight and fail every later one; the tests still waiting resolve incomplete. */
   close() {
@@ -4648,7 +4715,11 @@ var ServerCoverageCollector = class {
         timer: null,
         base: null,
         limit: createLimiter(this.limits.concurrency),
-        warned: false
+        warned: false,
+        processKeys: /* @__PURE__ */ new Set(),
+        instances: /* @__PURE__ */ new Map(),
+        testCoverage: null,
+        testChecksums: /* @__PURE__ */ new Map()
       };
       this.originStates.set(origin, state);
     }
@@ -4718,6 +4789,13 @@ var ServerCoverageCollector = class {
   }
   /** Send one call and hand each group's answer to its test. */
   async send(origin, groups) {
+    const state = this.stateOf(origin);
+    for (const group of groups) {
+      for (const id of group.ids) {
+        const key = processKeyOf(id);
+        if (key !== null) state.processKeys.add(key);
+      }
+    }
     let answer = null;
     try {
       const body = JSON.stringify({
@@ -4728,6 +4806,8 @@ var ServerCoverageCollector = class {
     } catch (error) {
       this.logger.debug(`Server coverage collection from ${origin} failed: ${errorMessage(error)}`);
     }
+    const instance = instanceOf(answer);
+    if (instance && !state.instances.has(instance.id)) state.instances.set(instance.id, instance);
     const answered = /* @__PURE__ */ new Map();
     for (const group of answer?.groups ?? []) {
       if (isRecord2(group) && typeof group.key === "string") answered.set(group.key, group);
@@ -4739,9 +4819,11 @@ var ServerCoverageCollector = class {
       } else {
         if (Array.isArray(result.missing) && result.missing.length > 0) group.entry.incomplete = true;
         fileChecksums(result.files, group.entry.checksums);
+        fileChecksums(result.files, state.testChecksums);
         const capture = sanitizeCoverage({ v: 1, files: result.files });
         if (capture) {
           group.entry.captures.push(capture);
+          state.testCoverage = mergeCoverage(state.testCoverage, capture);
           if (capture.incomplete) group.entry.incomplete = true;
         }
       }
@@ -6032,6 +6114,82 @@ function parseTestMetadata(annotations) {
   }
   return Object.keys(meta).length > 0 ? meta : null;
 }
+var MAX_EXTERNAL_ID_CHARS = 120;
+function normalizeExternalId(value) {
+  if (typeof value !== "string") return void 0;
+  const trimmed = value.trim();
+  if (!trimmed || trimmed.length > MAX_EXTERNAL_ID_CHARS) return void 0;
+  for (let i = 0; i < trimmed.length; i++) {
+    const code = trimmed.charCodeAt(i);
+    if (code < 32 || code === 127) return void 0;
+  }
+  return trimmed;
+}
+function parseExternalId(annotations) {
+  if (!Array.isArray(annotations)) return void 0;
+  let id;
+  for (const entry of annotations) {
+    if (!entry || typeof entry !== "object") continue;
+    const { type, description } = entry;
+    if (typeof type !== "string" || type.trim().toLowerCase() !== `${PIWI_ANNOTATION_PREFIX}id`) continue;
+    const value = normalizeExternalId(description);
+    if (value) id = value;
+  }
+  return id;
+}
+function compileExternalIdPattern(pattern) {
+  if (typeof pattern === "string") return new RegExp(pattern);
+  return new RegExp(pattern.source, pattern.flags.replace(/[gy]/g, ""));
+}
+function readExternalIdFromTitle(title, pattern) {
+  const match = pattern.exec(title);
+  if (!match) return null;
+  const externalId = normalizeExternalId(match.length > 1 ? match[1] : match[0]);
+  if (!externalId) return null;
+  const rest = (title.slice(0, match.index) + title.slice(match.index + match[0].length)).trim();
+  return rest ? { externalId, displayTitle: rest } : { externalId };
+}
+var MAX_EXTERNAL_ID_DUPLICATES = 50;
+var MAX_EXTERNAL_ID_DUPLICATE_TESTS = 10;
+function duplicateTestKey(test) {
+  return `${test.filePath}\0${(test.suitePath ?? []).join("")}\0${test.title}`;
+}
+function readDuplicateTest(raw) {
+  if (!raw || typeof raw !== "object") return null;
+  const { filePath: filePath2, suitePath, title, line } = raw;
+  if (typeof filePath2 !== "string" || typeof title !== "string") return null;
+  const test = { filePath: filePath2, title };
+  if (Array.isArray(suitePath) && suitePath.every((s) => typeof s === "string") && suitePath.length > 0)
+    test.suitePath = suitePath;
+  if (typeof line === "number" && Number.isFinite(line)) test.line = line;
+  return test;
+}
+function mergeExternalIdDuplicates(...lists) {
+  const byId = /* @__PURE__ */ new Map();
+  for (const list of lists) {
+    if (!Array.isArray(list)) continue;
+    for (const entry of list) {
+      if (!entry || typeof entry !== "object") continue;
+      const externalId = normalizeExternalId(entry.externalId);
+      const tests = entry.tests;
+      if (!externalId || !Array.isArray(tests)) continue;
+      const read = tests.map(readDuplicateTest).filter((test) => test !== null);
+      if (read.length === 0) continue;
+      let known = byId.get(externalId);
+      if (!known) {
+        if (byId.size >= MAX_EXTERNAL_ID_DUPLICATES) continue;
+        known = /* @__PURE__ */ new Map();
+        byId.set(externalId, known);
+      }
+      for (const test of read) {
+        if (known.size >= MAX_EXTERNAL_ID_DUPLICATE_TESTS) break;
+        const key = duplicateTestKey(test);
+        if (!known.has(key)) known.set(key, test);
+      }
+    }
+  }
+  return [...byId].map(([externalId, tests]) => ({ externalId, tests: [...tests.values()] }));
+}
 
 // src/internal/collect/test-meta.ts
 function collectTestTags(test) {
@@ -6045,6 +6203,84 @@ function collectTestMetadata(annotations) {
   return parseTestMetadata(annotations);
 }
 
+// src/internal/collect/external-ids.ts
+function naturalKey(test) {
+  return `${test.filePath}\0${test.suitePath.join("")}\0${test.title}`;
+}
+function duplicateTest(test) {
+  return {
+    filePath: test.filePath,
+    ...test.suitePath.length ? { suitePath: test.suitePath } : {},
+    title: test.title,
+    line: test.line
+  };
+}
+function planExternalIds(tests, options = {}) {
+  let pattern = null;
+  let patternError;
+  if (options.pattern !== void 0 && options.pattern !== "") {
+    try {
+      pattern = compileExternalIdPattern(options.pattern);
+    } catch (error) {
+      patternError = error instanceof Error ? error.message : String(error);
+    }
+  }
+  const found = /* @__PURE__ */ new Map();
+  const holders = /* @__PURE__ */ new Map();
+  for (const test of tests) {
+    let id = null;
+    const declared = parseExternalId(test.annotations);
+    if (declared) id = { externalId: declared };
+    else if (pattern) {
+      const read = readExternalIdFromTitle(test.title, pattern);
+      if (read) id = options.stripFromTitle ? read : { externalId: read.externalId };
+    }
+    if (!id) continue;
+    found.set(test.testId, id);
+    let keys = holders.get(id.externalId);
+    if (!keys) holders.set(id.externalId, keys = /* @__PURE__ */ new Map());
+    const key = naturalKey(test);
+    if (!keys.has(key)) keys.set(key, test);
+  }
+  const duplicates = [];
+  const duplicateIds = /* @__PURE__ */ new Set();
+  for (const [externalId, keys] of holders) {
+    if (keys.size < 2) continue;
+    duplicateIds.add(externalId);
+    duplicates.push({ externalId, tests: [...keys.values()].map(duplicateTest) });
+  }
+  const byTestId = /* @__PURE__ */ new Map();
+  for (const [testId, id] of found) if (!duplicateIds.has(id.externalId)) byTestId.set(testId, id);
+  return {
+    byTestId,
+    duplicates: mergeExternalIdDuplicates(duplicates),
+    ...patternError ? { patternError } : {}
+  };
+}
+function formatExternalIdDuplicates(duplicates, onDuplicate) {
+  if (duplicates.length === 0) return [];
+  const lines = [];
+  for (const { externalId, tests } of duplicates) {
+    lines.push(
+      `External ID ${externalId} is declared by ${tests.length} tests; this run matches them by file, describe path and title:`
+    );
+    for (const test of tests) {
+      const where = test.line != null ? `${test.filePath}:${test.line}` : test.filePath;
+      lines.push(`  ${where}  ${[...test.suitePath ?? [], test.title].join(" \u203A ")}`);
+    }
+  }
+  lines.push(
+    onDuplicate === "fail" ? "Give each test its own ID: the run fails while two tests share one (externalId.onDuplicate: fail)." : "Give each test its own ID, or set externalId.onDuplicate: 'fail' to fail the run instead."
+  );
+  return lines;
+}
+function resolveOnDuplicate(raw) {
+  if (raw === void 0 || raw === null || raw === "") return { onDuplicate: "report" };
+  const value = String(raw).trim().toLowerCase();
+  if (value === "report" || value === "fail") return { onDuplicate: value };
+  return { onDuplicate: "report", invalid: String(raw) };
+}
+
 // ../core/src/error-text.ts
 function joinErrorMessages(errors) {
   if (!errors || errors.length === 0) return null;
@@ -10496,17 +10732,26 @@ var CoverageCollection = class {
   }
   /**
    * Send the run-level coverage of every origin, and the browser's, to the
-   * dashboard run, in requests within its limits. A failure is logged once,
-   * never thrown.
+   * dashboard run, in requests within its limits; the first names the origins
+   * whose server coverage is incomplete (`incomplete`), and is sent for them
+   * alone when no file is. A failure is logged once, never thrown.
    */
   async sendRunCoverage(httpClient, runId, auth) {
     if (!this.collector && !this.browserBlobs) return;
     this.fetchRunCoverage();
-    const server = (await Promise.all(this.runFiles.values())).flatMap((result) => result ?? []);
+    const runs = await Promise.all(this.runFiles.values());
+    const server = runs.flatMap((result) => result?.files ?? []);
+    const gaps = runs.flatMap((result) => result?.gap ? [result.gap] : []);
     const files = [...server, ...await this.browserRunFiles()];
-    for (const chunk of chunkRunCoverage(files)) {
+    const chunks = chunkRunCoverage(files);
+    if (chunks.length === 0 && gaps.length > 0) chunks.push([]);
+    for (const [i, chunk] of chunks.entries()) {
       try {
-        await httpClient.putJSON(`/api/test-runs/${runId}/coverage`, { v: 1, files: chunk }, auth);
+        await httpClient.putJSON(
+          `/api/test-runs/${runId}/coverage`,
+          { v: 1, files: chunk, ...i === 0 && gaps.length > 0 ? { incomplete: gaps } : {} },
+          auth
+        );
       } catch (error) {
         this.logger.warn(`Could not send the run's coverage to the dashboard: ${errorMessage(error)}`);
         return;
@@ -10553,6 +10798,8 @@ var PiwiDashboardReporter = class _PiwiDashboardReporter {
     this.instanceId = "";
     this.shardInfo = null;
     this.metadata = {};
+    /** Each planned test's external ID and the IDs two tests share, settled in `onBegin`. */
+    this.externalIds = null;
     this.isFullRun = true;
     this.filterDetails = null;
     /** Configured `maxFailures` (0 = unlimited) — disambiguates an interrupted run's unrun reason. */
@@ -10701,6 +10948,7 @@ var PiwiDashboardReporter = class _PiwiDashboardReporter {
       this.metadata = flakeRunMetadata(stamp, this.metadata);
     }
     this.plannedTests = suite.allTests();
+    this.planExternalIds();
     this.shardInfo = resolveShardInfo(config);
     if (this.shardInfo) {
       this.logger.info(`Shard ${this.shardInfo.current}/${this.shardInfo.total} detected`);
@@ -10745,6 +10993,44 @@ var PiwiDashboardReporter = class _PiwiDashboardReporter {
       this.plannedTests.length
     );
   }
+  /**
+   * Settle each planned test's external ID before any test runs, name the IDs two tests share, and record them on
+   * the run so the dashboard lists them too.
+   */
+  planExternalIds() {
+    const options = this.options.externalId ?? {};
+    const plan = planExternalIds(
+      this.plannedTests.map((test) => ({
+        testId: test.id,
+        filePath: testFile(test, this.testPathBase),
+        suitePath: this.metadataCollector.getSuiteInfo(test).suitePath ?? [],
+        title: test.title,
+        line: test.location.line,
+        annotations: test.annotations ?? []
+      })),
+      options
+    );
+    if (plan.patternError) {
+      this.logger.error(
+        `externalId.pattern is not a valid regular expression, so no ID is read from titles: ${plan.patternError}`
+      );
+    }
+    const { onDuplicate, invalid } = resolveOnDuplicate(options.onDuplicate);
+    if (invalid !== void 0) {
+      this.logger.warn(
+        `externalId.onDuplicate is "${invalid}"; expected 'report' or 'fail', so duplicates are reported.`
+      );
+    }
+    for (const line of formatExternalIdDuplicates(plan.duplicates, onDuplicate)) this.logger.warn(line);
+    if (plan.duplicates.length > 0) this.metadata = { ...this.metadata, externalIdDuplicates: plan.duplicates };
+    this.externalIds = plan;
+  }
+  /** The external ID fields of a test's payload: `null` tells the dashboard the test has none to match on. */
+  externalIdFields(test) {
+    if (!this.externalIds) return {};
+    const id = this.externalIds.byTestId.get(test.id);
+    return { externalId: id?.externalId ?? null, displayTitle: id?.displayTitle ?? null };
+  }
   /** Playwright reporter hook: called when an individual test begins */
   onTestBegin(test, result) {
     this.lastTestWatch?.begin(test.id);
@@ -10872,6 +11158,7 @@ var PiwiDashboardReporter = class _PiwiDashboardReporter {
       tags: tags.length ? tags : null,
       locks: locks.length ? locks : null,
       testMeta: collectTestMetadata(annotations),
+      ...this.externalIdFields(test),
       // An annotation-less skip reclassified to `didnotrun` is a serial-group
       // cascade: an earlier test failed and Playwright skipped the rest.
       didNotRunReason: status === "didnotrun" ? "previous-failure" : null,
@@ -11074,6 +11361,7 @@ var PiwiDashboardReporter = class _PiwiDashboardReporter {
         tags: tags.length ? tags : null,
         locks: locks.length ? locks : null,
         testMeta: collectTestMetadata(declaredAnnotations),
+        ...this.externalIdFields(test),
         didNotRunReason: reason,
         expectedStatus: test.expectedStatus ?? null
       };
@@ -11085,8 +11373,23 @@ var PiwiDashboardReporter = class _PiwiDashboardReporter {
       }
     }
   }
-  /** Playwright reporter hook: called when the full test run finishes */
+  /**
+   * Playwright reporter hook: called when the full test run finishes. Fails the run when two tests share an external
+   * ID and `externalId.onDuplicate` is `'fail'`.
+   */
   async onEnd(result) {
+    const duplicates = this.externalIds?.duplicates.length ?? 0;
+    const failsOnDuplicates = duplicates > 0 && resolveOnDuplicate(this.options.externalId?.onDuplicate).onDuplicate === "fail";
+    const overridden = failsOnDuplicates && result?.status === "passed";
+    await this.finishRun(overridden ? { ...result, status: "failed" } : result);
+    if (failsOnDuplicates) {
+      this.logger.error(
+        `Failing the run: ${duplicates} external ID${duplicates === 1 ? " is" : "s are"} declared by more than one test.`
+      );
+      return { status: "failed" };
+    }
+  }
+  async finishRun(result) {
     if (this.listMode || !this.enabled) return;
     if (this.startTime === null) {
       this.logger.debug("The run stopped before any test began \u2014 nothing to report.");

package/dist/internal/capture/capture-fixtures.js

@@ -2648,6 +2648,8 @@ var PIWI_ENV_KEYS = {
   codeReachRoots: "PIWI_CODE_REACH_ROOTS",
   coverageBrowser: "PIWI_COVERAGE_BROWSER",
   coverageServer: "PIWI_COVERAGE_SERVER",
+  externalIdPattern: "PIWI_EXTERNAL_ID_PATTERN",
+  externalIdOnDuplicate: "PIWI_EXTERNAL_ID_ON_DUPLICATE",
   captureServerTraces: "PIWI_CAPTURE_SERVER_TRACES",
   captureResources: "PIWI_CAPTURE_RESOURCES",
   leakCheck: "PIWI_LEAK_CHECK",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@piwitests/reporter",
-  "version": "0.56.0",
+  "version": "0.58.0",
   "description": "Playwright reporter that streams results, traces and HTML reports to a Piwi Dashboard instance",
   "url": "https://github.com/PiwiTests/platform",
   "homepage": "https://piwitests.dev",

package/templates/skills/write-the-missing-test/SKILL.md

@@ -21,8 +21,9 @@ view, or the reporter CLI with `PIWI_DASHBOARD_URL` / `PIWI_API_KEY` /
 ## Steps
 
 1. **Find the gaps.** On a change, call `get_change_coverage` for the current
-   branch first — the uncovered changed files are the gaps that matter now.
-   Otherwise call `list_scenario_gaps` for the project. Filter to what you are
+   branch first — the files whose `reach` is `unreached` (the uncovered
+   changes) and the `lines.notRun` of the files it did reach are what the
+   change left untested. Otherwise call `list_scenario_gaps` for the project. Filter to what you are
    working on with `class`, `feature`, `minScore` or `pr`. Read the classes:
    - **blind-spot** — no trusted test reaches it (a route, a page, a control, a
      new error path).