@piwitests/reporter 0.57.0 → 0.58.0

package/dist/cli/index.js

@@ -873,6 +873,8 @@ var PIWI_ENV_KEYS = {
   codeReachRoots: "PIWI_CODE_REACH_ROOTS",
   coverageBrowser: "PIWI_COVERAGE_BROWSER",
   coverageServer: "PIWI_COVERAGE_SERVER",
+  externalIdPattern: "PIWI_EXTERNAL_ID_PATTERN",
+  externalIdOnDuplicate: "PIWI_EXTERNAL_ID_ON_DUPLICATE",
   captureServerTraces: "PIWI_CAPTURE_SERVER_TRACES",
   captureResources: "PIWI_CAPTURE_RESOURCES",
   leakCheck: "PIWI_LEAK_CHECK",
@@ -4877,7 +4879,7 @@ Connection:
   --server-url <url>       Dashboard URL      (env PIWI_DASHBOARD_URL)
   --api-key <key>          API key            (env PIWI_API_KEY)
 
-Policy (at least one is required):
+Policy (at least one rule or warn-only flag is required):
   --require-tag <tags>     Comma-separated; every test carrying the tag must pass
   --max-failed <n>         Fail when more than n tests failed
   --max-new-regressions <n>  Fail when more than n tests newly started failing
@@ -4890,7 +4892,7 @@ Policy (at least one is required):
   --max-new-leaks <n>      Fail when more than n of those were never seen on the base branch
 
 Warn-only (reported, never fails the build):
-  --max-uncovered-changes <n>  Warn when more than n changed files have no observed test reach
+  --max-uncovered-changes <n>  Warn when more than n changed files are uncovered changes: no test reached them
   --min-patch-coverage <percent>  Warn when fewer of the changed lines ran, in the commit's combined coverage
 
 Other:

package/dist/global-setup-module.js

@@ -127,6 +127,8 @@ var PIWI_ENV_KEYS = {
   codeReachRoots: "PIWI_CODE_REACH_ROOTS",
   coverageBrowser: "PIWI_COVERAGE_BROWSER",
   coverageServer: "PIWI_COVERAGE_SERVER",
+  externalIdPattern: "PIWI_EXTERNAL_ID_PATTERN",
+  externalIdOnDuplicate: "PIWI_EXTERNAL_ID_ON_DUPLICATE",
   captureServerTraces: "PIWI_CAPTURE_SERVER_TRACES",
   captureResources: "PIWI_CAPTURE_RESOURCES",
   leakCheck: "PIWI_LEAK_CHECK",
@@ -217,6 +219,13 @@ function resolveOptions(raw) {
     coverage.browser = coverageBrowser;
   if (coverage.server === void 0 && coverageServer) coverage.server = coverageServer;
   if (coverage.browser !== void 0 || coverage.server !== void 0) mergedRaw.coverage = coverage;
+  const externalId = { ...mergedRaw.externalId };
+  const externalIdPattern = env[PIWI_ENV_KEYS.externalIdPattern];
+  const externalIdOnDuplicate = env[PIWI_ENV_KEYS.externalIdOnDuplicate];
+  if (externalId.pattern === void 0 && externalIdPattern) externalId.pattern = externalIdPattern;
+  if (externalId.onDuplicate === void 0 && externalIdOnDuplicate)
+    externalId.onDuplicate = externalIdOnDuplicate;
+  if (Object.keys(externalId).length > 0) mergedRaw.externalId = externalId;
   const opts = { ...DEFAULTS, ...mergedRaw };
   if (env[PIWI_ENV_KEYS.verbose] !== void 0) opts.verbose = env[PIWI_ENV_KEYS.verbose] === "true";
   return opts;

package/dist/index.d.ts

@@ -129,6 +129,33 @@ interface PiwiDashboardOptions {
          */
         server?: 'off' | 'files' | 'functions' | 'lines';
     };
+    /**
+     * Match each test to its history on the dashboard by an external ID, the ID the test has in your own test
+     * management tool, so a test keeps its history when it is renamed or moved. A test declares its ID with a
+     * `piwi:id` annotation (`{ type: 'piwi:id', description: '12526' }`), which wins, or the ID is read from its title
+     * with `pattern`. Two tests declaring the same ID are reported before the first test runs, and matched by file,
+     * describe path and title instead.
+     */
+    externalId?: {
+        /**
+         * Where the ID sits in a test's own title (not its describe path): `/^(\d+), /` reads `12526` from
+         * `'12526, Log in with Email'`. The ID is the first capture group, or the whole match without a group. A string
+         * is compiled with `new RegExp(value)`; one that does not compile turns external IDs off for the run, with an
+         * error, but never fails it. Can also be set with `PIWI_EXTERNAL_ID_PATTERN`.
+         */
+        pattern?: RegExp | string;
+        /**
+         * Show the title without its ID on the dashboard, next to the ID. The whole match of `pattern` is removed from
+         * the title shown; the title stored and matched on stays whole. Defaults to `false`.
+         */
+        stripFromTitle?: boolean;
+        /**
+         * What two tests declaring the same ID do: `'report'` lists them in the console and on the run page; `'fail'`
+         * also fails the run once its tests have run. Either way the run matches them by file, describe path and title.
+         * Defaults to `'report'`. Can also be set with `PIWI_EXTERNAL_ID_ON_DUPLICATE`.
+         */
+        onDuplicate?: 'report' | 'fail';
+    };
     /**
      * Capture server-side spans for each API/document request the test makes,
      * read from the `X-Piwi-Trace` response header emitted by a Piwi
@@ -457,6 +484,8 @@ declare class PiwiDashboardReporter {
     private instanceId;
     private shardInfo;
     private metadata;
+    /** Each planned test's external ID and the IDs two tests share, settled in `onBegin`. */
+    private externalIds;
     private enabled;
     /**
      * True under `playwright test --list`, where Playwright still constructs this
@@ -515,6 +544,13 @@ declare class PiwiDashboardReporter {
     printsToStdio(): boolean;
     /** Playwright reporter hook: called once at the start of the test run */
     onBegin(config: FullConfig, suite: Suite): void;
+    /**
+     * Settle each planned test's external ID before any test runs, name the IDs two tests share, and record them on
+     * the run so the dashboard lists them too.
+     */
+    private planExternalIds;
+    /** The external ID fields of a test's payload: `null` tells the dashboard the test has none to match on. */
+    private externalIdFields;
     /** Playwright reporter hook: called when an individual test begins */
     onTestBegin(test: TestCase, result: TestResult): void;
     /** Track suite-level setup steps (beforeAll/afterAll) not tied to any test */
@@ -554,8 +590,14 @@ declare class PiwiDashboardReporter {
      * interrupted) is resolved once from the overall run status by the caller.
      */
     private materializeUnrunTests;
-    /** Playwright reporter hook: called when the full test run finishes */
-    onEnd(result: FullResult): Promise<void>;
+    /**
+     * Playwright reporter hook: called when the full test run finishes. Fails the run when two tests share an external
+     * ID and `externalId.onDuplicate` is `'fail'`.
+     */
+    onEnd(result: FullResult): Promise<{
+        status?: FullResult['status'];
+    } | void>;
+    private finishRun;
 }
 
 interface ResolveSelectionResult {

package/dist/index.js

@@ -128,6 +128,8 @@ var PIWI_ENV_KEYS = {
   codeReachRoots: "PIWI_CODE_REACH_ROOTS",
   coverageBrowser: "PIWI_COVERAGE_BROWSER",
   coverageServer: "PIWI_COVERAGE_SERVER",
+  externalIdPattern: "PIWI_EXTERNAL_ID_PATTERN",
+  externalIdOnDuplicate: "PIWI_EXTERNAL_ID_ON_DUPLICATE",
   captureServerTraces: "PIWI_CAPTURE_SERVER_TRACES",
   captureResources: "PIWI_CAPTURE_RESOURCES",
   leakCheck: "PIWI_LEAK_CHECK",
@@ -247,6 +249,13 @@ function resolveOptions(raw) {
     coverage.browser = coverageBrowser;
   if (coverage.server === void 0 && coverageServer) coverage.server = coverageServer;
   if (coverage.browser !== void 0 || coverage.server !== void 0) mergedRaw.coverage = coverage;
+  const externalId = { ...mergedRaw.externalId };
+  const externalIdPattern = env[PIWI_ENV_KEYS.externalIdPattern];
+  const externalIdOnDuplicate = env[PIWI_ENV_KEYS.externalIdOnDuplicate];
+  if (externalId.pattern === void 0 && externalIdPattern) externalId.pattern = externalIdPattern;
+  if (externalId.onDuplicate === void 0 && externalIdOnDuplicate)
+    externalId.onDuplicate = externalIdOnDuplicate;
+  if (Object.keys(externalId).length > 0) mergedRaw.externalId = externalId;
   const opts = { ...DEFAULTS, ...mergedRaw };
   if (env[PIWI_ENV_KEYS.verbose] !== void 0) opts.verbose = env[PIWI_ENV_KEYS.verbose] === "true";
   return opts;
@@ -652,6 +661,9 @@ function toWireTestCase(tc) {
     tags: rest.tags ?? null,
     locks: rest.locks ?? null,
     testMeta: rest.testMeta ?? null,
+    // Kept `undefined` when unset (a `begin` event): `null` would tell the dashboard the test has no ID.
+    externalId: rest.externalId,
+    displayTitle: rest.displayTitle ?? null,
     locatorSnapshots: rest.locatorSnapshots || null,
     locatorPages: rest.locatorPages || null,
     codeReach: rest.codeReach || null,
@@ -6102,6 +6114,82 @@ function parseTestMetadata(annotations) {
   }
   return Object.keys(meta).length > 0 ? meta : null;
 }
+var MAX_EXTERNAL_ID_CHARS = 120;
+function normalizeExternalId(value) {
+  if (typeof value !== "string") return void 0;
+  const trimmed = value.trim();
+  if (!trimmed || trimmed.length > MAX_EXTERNAL_ID_CHARS) return void 0;
+  for (let i = 0; i < trimmed.length; i++) {
+    const code = trimmed.charCodeAt(i);
+    if (code < 32 || code === 127) return void 0;
+  }
+  return trimmed;
+}
+function parseExternalId(annotations) {
+  if (!Array.isArray(annotations)) return void 0;
+  let id;
+  for (const entry of annotations) {
+    if (!entry || typeof entry !== "object") continue;
+    const { type, description } = entry;
+    if (typeof type !== "string" || type.trim().toLowerCase() !== `${PIWI_ANNOTATION_PREFIX}id`) continue;
+    const value = normalizeExternalId(description);
+    if (value) id = value;
+  }
+  return id;
+}
+function compileExternalIdPattern(pattern) {
+  if (typeof pattern === "string") return new RegExp(pattern);
+  return new RegExp(pattern.source, pattern.flags.replace(/[gy]/g, ""));
+}
+function readExternalIdFromTitle(title, pattern) {
+  const match = pattern.exec(title);
+  if (!match) return null;
+  const externalId = normalizeExternalId(match.length > 1 ? match[1] : match[0]);
+  if (!externalId) return null;
+  const rest = (title.slice(0, match.index) + title.slice(match.index + match[0].length)).trim();
+  return rest ? { externalId, displayTitle: rest } : { externalId };
+}
+var MAX_EXTERNAL_ID_DUPLICATES = 50;
+var MAX_EXTERNAL_ID_DUPLICATE_TESTS = 10;
+function duplicateTestKey(test) {
+  return `${test.filePath}\0${(test.suitePath ?? []).join("")}\0${test.title}`;
+}
+function readDuplicateTest(raw) {
+  if (!raw || typeof raw !== "object") return null;
+  const { filePath: filePath2, suitePath, title, line } = raw;
+  if (typeof filePath2 !== "string" || typeof title !== "string") return null;
+  const test = { filePath: filePath2, title };
+  if (Array.isArray(suitePath) && suitePath.every((s) => typeof s === "string") && suitePath.length > 0)
+    test.suitePath = suitePath;
+  if (typeof line === "number" && Number.isFinite(line)) test.line = line;
+  return test;
+}
+function mergeExternalIdDuplicates(...lists) {
+  const byId = /* @__PURE__ */ new Map();
+  for (const list of lists) {
+    if (!Array.isArray(list)) continue;
+    for (const entry of list) {
+      if (!entry || typeof entry !== "object") continue;
+      const externalId = normalizeExternalId(entry.externalId);
+      const tests = entry.tests;
+      if (!externalId || !Array.isArray(tests)) continue;
+      const read = tests.map(readDuplicateTest).filter((test) => test !== null);
+      if (read.length === 0) continue;
+      let known = byId.get(externalId);
+      if (!known) {
+        if (byId.size >= MAX_EXTERNAL_ID_DUPLICATES) continue;
+        known = /* @__PURE__ */ new Map();
+        byId.set(externalId, known);
+      }
+      for (const test of read) {
+        if (known.size >= MAX_EXTERNAL_ID_DUPLICATE_TESTS) break;
+        const key = duplicateTestKey(test);
+        if (!known.has(key)) known.set(key, test);
+      }
+    }
+  }
+  return [...byId].map(([externalId, tests]) => ({ externalId, tests: [...tests.values()] }));
+}
 
 // src/internal/collect/test-meta.ts
 function collectTestTags(test) {
@@ -6115,6 +6203,84 @@ function collectTestMetadata(annotations) {
   return parseTestMetadata(annotations);
 }
 
+// src/internal/collect/external-ids.ts
+function naturalKey(test) {
+  return `${test.filePath}\0${test.suitePath.join("")}\0${test.title}`;
+}
+function duplicateTest(test) {
+  return {
+    filePath: test.filePath,
+    ...test.suitePath.length ? { suitePath: test.suitePath } : {},
+    title: test.title,
+    line: test.line
+  };
+}
+function planExternalIds(tests, options = {}) {
+  let pattern = null;
+  let patternError;
+  if (options.pattern !== void 0 && options.pattern !== "") {
+    try {
+      pattern = compileExternalIdPattern(options.pattern);
+    } catch (error) {
+      patternError = error instanceof Error ? error.message : String(error);
+    }
+  }
+  const found = /* @__PURE__ */ new Map();
+  const holders = /* @__PURE__ */ new Map();
+  for (const test of tests) {
+    let id = null;
+    const declared = parseExternalId(test.annotations);
+    if (declared) id = { externalId: declared };
+    else if (pattern) {
+      const read = readExternalIdFromTitle(test.title, pattern);
+      if (read) id = options.stripFromTitle ? read : { externalId: read.externalId };
+    }
+    if (!id) continue;
+    found.set(test.testId, id);
+    let keys = holders.get(id.externalId);
+    if (!keys) holders.set(id.externalId, keys = /* @__PURE__ */ new Map());
+    const key = naturalKey(test);
+    if (!keys.has(key)) keys.set(key, test);
+  }
+  const duplicates = [];
+  const duplicateIds = /* @__PURE__ */ new Set();
+  for (const [externalId, keys] of holders) {
+    if (keys.size < 2) continue;
+    duplicateIds.add(externalId);
+    duplicates.push({ externalId, tests: [...keys.values()].map(duplicateTest) });
+  }
+  const byTestId = /* @__PURE__ */ new Map();
+  for (const [testId, id] of found) if (!duplicateIds.has(id.externalId)) byTestId.set(testId, id);
+  return {
+    byTestId,
+    duplicates: mergeExternalIdDuplicates(duplicates),
+    ...patternError ? { patternError } : {}
+  };
+}
+function formatExternalIdDuplicates(duplicates, onDuplicate) {
+  if (duplicates.length === 0) return [];
+  const lines = [];
+  for (const { externalId, tests } of duplicates) {
+    lines.push(
+      `External ID ${externalId} is declared by ${tests.length} tests; this run matches them by file, describe path and title:`
+    );
+    for (const test of tests) {
+      const where = test.line != null ? `${test.filePath}:${test.line}` : test.filePath;
+      lines.push(`  ${where}  ${[...test.suitePath ?? [], test.title].join(" \u203A ")}`);
+    }
+  }
+  lines.push(
+    onDuplicate === "fail" ? "Give each test its own ID: the run fails while two tests share one (externalId.onDuplicate: fail)." : "Give each test its own ID, or set externalId.onDuplicate: 'fail' to fail the run instead."
+  );
+  return lines;
+}
+function resolveOnDuplicate(raw) {
+  if (raw === void 0 || raw === null || raw === "") return { onDuplicate: "report" };
+  const value = String(raw).trim().toLowerCase();
+  if (value === "report" || value === "fail") return { onDuplicate: value };
+  return { onDuplicate: "report", invalid: String(raw) };
+}
+
 // ../core/src/error-text.ts
 function joinErrorMessages(errors) {
   if (!errors || errors.length === 0) return null;
@@ -10632,6 +10798,8 @@ var PiwiDashboardReporter = class _PiwiDashboardReporter {
     this.instanceId = "";
     this.shardInfo = null;
     this.metadata = {};
+    /** Each planned test's external ID and the IDs two tests share, settled in `onBegin`. */
+    this.externalIds = null;
     this.isFullRun = true;
     this.filterDetails = null;
     /** Configured `maxFailures` (0 = unlimited) — disambiguates an interrupted run's unrun reason. */
@@ -10780,6 +10948,7 @@ var PiwiDashboardReporter = class _PiwiDashboardReporter {
       this.metadata = flakeRunMetadata(stamp, this.metadata);
     }
     this.plannedTests = suite.allTests();
+    this.planExternalIds();
     this.shardInfo = resolveShardInfo(config);
     if (this.shardInfo) {
       this.logger.info(`Shard ${this.shardInfo.current}/${this.shardInfo.total} detected`);
@@ -10824,6 +10993,44 @@ var PiwiDashboardReporter = class _PiwiDashboardReporter {
       this.plannedTests.length
     );
   }
+  /**
+   * Settle each planned test's external ID before any test runs, name the IDs two tests share, and record them on
+   * the run so the dashboard lists them too.
+   */
+  planExternalIds() {
+    const options = this.options.externalId ?? {};
+    const plan = planExternalIds(
+      this.plannedTests.map((test) => ({
+        testId: test.id,
+        filePath: testFile(test, this.testPathBase),
+        suitePath: this.metadataCollector.getSuiteInfo(test).suitePath ?? [],
+        title: test.title,
+        line: test.location.line,
+        annotations: test.annotations ?? []
+      })),
+      options
+    );
+    if (plan.patternError) {
+      this.logger.error(
+        `externalId.pattern is not a valid regular expression, so no ID is read from titles: ${plan.patternError}`
+      );
+    }
+    const { onDuplicate, invalid } = resolveOnDuplicate(options.onDuplicate);
+    if (invalid !== void 0) {
+      this.logger.warn(
+        `externalId.onDuplicate is "${invalid}"; expected 'report' or 'fail', so duplicates are reported.`
+      );
+    }
+    for (const line of formatExternalIdDuplicates(plan.duplicates, onDuplicate)) this.logger.warn(line);
+    if (plan.duplicates.length > 0) this.metadata = { ...this.metadata, externalIdDuplicates: plan.duplicates };
+    this.externalIds = plan;
+  }
+  /** The external ID fields of a test's payload: `null` tells the dashboard the test has none to match on. */
+  externalIdFields(test) {
+    if (!this.externalIds) return {};
+    const id = this.externalIds.byTestId.get(test.id);
+    return { externalId: id?.externalId ?? null, displayTitle: id?.displayTitle ?? null };
+  }
   /** Playwright reporter hook: called when an individual test begins */
   onTestBegin(test, result) {
     this.lastTestWatch?.begin(test.id);
@@ -10951,6 +11158,7 @@ var PiwiDashboardReporter = class _PiwiDashboardReporter {
       tags: tags.length ? tags : null,
       locks: locks.length ? locks : null,
       testMeta: collectTestMetadata(annotations),
+      ...this.externalIdFields(test),
       // An annotation-less skip reclassified to `didnotrun` is a serial-group
       // cascade: an earlier test failed and Playwright skipped the rest.
       didNotRunReason: status === "didnotrun" ? "previous-failure" : null,
@@ -11153,6 +11361,7 @@ var PiwiDashboardReporter = class _PiwiDashboardReporter {
         tags: tags.length ? tags : null,
         locks: locks.length ? locks : null,
         testMeta: collectTestMetadata(declaredAnnotations),
+        ...this.externalIdFields(test),
         didNotRunReason: reason,
         expectedStatus: test.expectedStatus ?? null
       };
@@ -11164,8 +11373,23 @@ var PiwiDashboardReporter = class _PiwiDashboardReporter {
       }
     }
   }
-  /** Playwright reporter hook: called when the full test run finishes */
+  /**
+   * Playwright reporter hook: called when the full test run finishes. Fails the run when two tests share an external
+   * ID and `externalId.onDuplicate` is `'fail'`.
+   */
   async onEnd(result) {
+    const duplicates = this.externalIds?.duplicates.length ?? 0;
+    const failsOnDuplicates = duplicates > 0 && resolveOnDuplicate(this.options.externalId?.onDuplicate).onDuplicate === "fail";
+    const overridden = failsOnDuplicates && result?.status === "passed";
+    await this.finishRun(overridden ? { ...result, status: "failed" } : result);
+    if (failsOnDuplicates) {
+      this.logger.error(
+        `Failing the run: ${duplicates} external ID${duplicates === 1 ? " is" : "s are"} declared by more than one test.`
+      );
+      return { status: "failed" };
+    }
+  }
+  async finishRun(result) {
     if (this.listMode || !this.enabled) return;
     if (this.startTime === null) {
       this.logger.debug("The run stopped before any test began \u2014 nothing to report.");

package/dist/internal/capture/capture-fixtures.js

@@ -2648,6 +2648,8 @@ var PIWI_ENV_KEYS = {
   codeReachRoots: "PIWI_CODE_REACH_ROOTS",
   coverageBrowser: "PIWI_COVERAGE_BROWSER",
   coverageServer: "PIWI_COVERAGE_SERVER",
+  externalIdPattern: "PIWI_EXTERNAL_ID_PATTERN",
+  externalIdOnDuplicate: "PIWI_EXTERNAL_ID_ON_DUPLICATE",
   captureServerTraces: "PIWI_CAPTURE_SERVER_TRACES",
   captureResources: "PIWI_CAPTURE_RESOURCES",
   leakCheck: "PIWI_LEAK_CHECK",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@piwitests/reporter",
-  "version": "0.57.0",
+  "version": "0.58.0",
   "description": "Playwright reporter that streams results, traces and HTML reports to a Piwi Dashboard instance",
   "url": "https://github.com/PiwiTests/platform",
   "homepage": "https://piwitests.dev",

package/templates/skills/write-the-missing-test/SKILL.md

@@ -21,8 +21,9 @@ view, or the reporter CLI with `PIWI_DASHBOARD_URL` / `PIWI_API_KEY` /
 ## Steps
 
 1. **Find the gaps.** On a change, call `get_change_coverage` for the current
-   branch first — the uncovered changed files are the gaps that matter now.
-   Otherwise call `list_scenario_gaps` for the project. Filter to what you are
+   branch first — the files whose `reach` is `unreached` (the uncovered
+   changes) and the `lines.notRun` of the files it did reach are what the
+   change left untested. Otherwise call `list_scenario_gaps` for the project. Filter to what you are
    working on with `class`, `feature`, `minScore` or `pr`. Read the classes:
    - **blind-spot** — no trusted test reaches it (a route, a page, a control, a
      new error path).