@sentinelqa/playwright-reporter 0.1.28 → 0.1.30

package/README.md

@@ -12,6 +12,8 @@ Works locally out of the box with no account required.
 Optionally upload runs to Sentinel Cloud for CI history and AI failure analysis.
 
 ![Sentinel Report Example](./docs/screenshot.png)
+![Run-to-Run Diff](./docs/run_diff.png)
+![CLI Quick Diagnosis](./docs/CLI.png)
 
 ## Features
 

package/dist/localReport.d.ts

@@ -21,6 +21,7 @@ type CopiedArtifact = {
 };
 type ReportTest = {
     id: string;
+    matchKey: string;
     title: string;
     titlePath: string[];
     file: string | null;
@@ -51,6 +52,7 @@ type RunSnapshot = {
     totals: LocalReportSummary;
     tests: Array<{
         id: string;
+        matchKey: string;
         title: string;
         status: string;
         signal: string | null;

package/dist/localReport.js

@@ -95,6 +95,23 @@ const safeSlug = (value) => {
         .replace(/^-+|-+$/g, "")
         .slice(0, 64) || "artifact");
 };
+const buildTitlePath = (baseTitles, test) => {
+    const title = typeof test?.title === "string" ? test.title : null;
+    if (!title)
+        return baseTitles.filter(Boolean);
+    if (baseTitles[baseTitles.length - 1] === title)
+        return baseTitles.filter(Boolean);
+    return [...baseTitles, title].filter(Boolean);
+};
+const buildTestIdentity = (test, titlePath) => {
+    const file = test?.location?.file || "unknown";
+    const project = test?.projectName || "default";
+    const joined = titlePath.join(" > ");
+    return {
+        id: [file, project, joined].join("::"),
+        matchKey: [file, joined].join("::")
+    };
+};
 const formatDuration = (durationMs) => {
     if (!Number.isFinite(durationMs) || durationMs <= 0)
         return "0 ms";
@@ -233,7 +250,7 @@ const copyArtifact = (sourcePath, kind, reportDir, usedRelativePaths, testId) =>
         testId
     };
 };
-const createReportTest = (test, titlePath, diagnosisById) => {
+const createReportTest = (test, titlePath) => {
     const results = Array.isArray(test?.results) ? test.results : [];
     const lastResult = results.length > 0 ? results[results.length - 1] : null;
     const errors = results.flatMap((result) => Array.isArray(result?.errors)
@@ -242,30 +259,31 @@ const createReportTest = (test, titlePath, diagnosisById) => {
             .filter(Boolean)
         : []);
     const duration = results.reduce((total, result) => total + (Number(result?.duration) || 0), 0);
-    const id = [
-        test?.location?.file || "unknown",
-        test?.projectName || "default",
-        titlePath.join(" > ")
-    ].join("::");
+    const identity = buildTestIdentity(test, titlePath);
+    const status = normalizeTestStatus(test?.status || lastResult?.status || "unknown");
+    const primaryError = errors[0] || "";
     return {
-        id,
+        id: identity.id,
+        matchKey: identity.matchKey,
         title: test?.title || titlePath[titlePath.length - 1] || "Untitled test",
         titlePath,
         file: test?.location?.file || null,
         projectName: test?.projectName || null,
-        status: normalizeTestStatus(test?.status || lastResult?.status || "unknown"),
+        status,
         duration,
         errors,
-        diagnosis: diagnosisById.get(titlePath.join(" > ")) || null,
+        diagnosis: ["failed", "timedOut", "interrupted"].includes(status) && primaryError
+            ? (0, quickDiagnosis_1.parseFailureFacts)(test?.title || titlePath[titlePath.length - 1] || "Untitled test", titlePath, primaryError, status)
+            : null,
         artifacts: []
     };
 };
-const collectTests = (node, diagnosisById, parentTitles = []) => {
+const collectTests = (node, parentTitles = []) => {
     const nextTitles = node?.title ? [...parentTitles, node.title] : parentTitles;
     const collected = [];
     if (Array.isArray(node?.tests)) {
         for (const test of node.tests) {
-            collected.push(createReportTest(test, [...nextTitles, test?.title].filter(Boolean), diagnosisById));
+            collected.push(createReportTest(test, buildTitlePath(nextTitles, test)));
         }
     }
     if (Array.isArray(node?.specs)) {
@@ -273,13 +291,13 @@ const collectTests = (node, diagnosisById, parentTitles = []) => {
             const specTitles = [...nextTitles, spec?.title].filter(Boolean);
             const specTests = Array.isArray(spec?.tests) ? spec.tests : [];
             for (const test of specTests) {
-                collected.push(createReportTest(test, specTitles, diagnosisById));
+                collected.push(createReportTest(test, buildTitlePath(specTitles, test)));
             }
         }
     }
     if (Array.isArray(node?.suites)) {
         for (const suite of node.suites) {
-            collected.push(...collectTests(suite, diagnosisById, nextTitles));
+            collected.push(...collectTests(suite, nextTitles));
         }
     }
     return collected;
@@ -289,12 +307,8 @@ const collectTestRefs = (node, parentTitles = []) => {
     const refs = [];
     if (Array.isArray(node?.tests)) {
         for (const test of node.tests) {
-            const titlePath = [...nextTitles, test?.title].filter(Boolean);
-            const id = [
-                test?.location?.file || "unknown",
-                test?.projectName || "default",
-                titlePath.join(" > ")
-            ].join("::");
+            const titlePath = buildTitlePath(nextTitles, test);
+            const id = buildTestIdentity(test, titlePath).id;
             refs.push({ id, resultList: Array.isArray(test?.results) ? test.results : [] });
         }
     }
@@ -302,11 +316,7 @@ const collectTestRefs = (node, parentTitles = []) => {
         for (const spec of node.specs) {
             const titlePath = [...nextTitles, spec?.title].filter(Boolean);
             for (const test of Array.isArray(spec?.tests) ? spec.tests : []) {
-                const id = [
-                    test?.location?.file || "unknown",
-                    test?.projectName || "default",
-                    titlePath.join(" > ")
-                ].join("::");
+                const id = buildTestIdentity(test, buildTitlePath(titlePath, test)).id;
                 refs.push({ id, resultList: Array.isArray(test?.results) ? test.results : [] });
             }
         }
@@ -331,14 +341,6 @@ const summarizeTests = (tests) => {
         return summary;
     }, { total: 0, failed: 0, passed: 0, skipped: 0 });
 };
-const buildDiagnosisMap = (playwrightJsonPath) => {
-    const failures = (0, quickDiagnosis_1.collectFailureFacts)(playwrightJsonPath);
-    const map = new Map();
-    for (const failure of failures) {
-        map.set(failure.titlePath.join(" > "), failure);
-    }
-    return map;
-};
 const getFailureTests = (tests) => tests.filter((test) => ["failed", "timedOut", "interrupted"].includes(test.status));
 const groupSimilarFailures = (tests) => {
     const groups = new Map();
@@ -356,6 +358,22 @@ const groupSimilarFailures = (tests) => {
         }
         groups.get(key).tests.push(test);
     }
+    return Array.from(groups.values())
+        .filter((group) => group.tests.length > 1)
+        .sort((a, b) => b.tests.length - a.tests.length);
+};
+const groupFailureDigest = (tests) => {
+    const groups = new Map();
+    for (const test of getFailureTests(tests)) {
+        const summary = test.diagnosis
+            ? (0, quickDiagnosis_1.describeFailure)(test.diagnosis)
+            : (test.errors[0]?.split(/\r?\n/)[0]?.trim() || "Open the failure details to inspect the exact Playwright error.");
+        const key = summary;
+        if (!groups.has(key)) {
+            groups.set(key, { key, summary, tests: [] });
+        }
+        groups.get(key).tests.push(test);
+    }
     return Array.from(groups.values()).sort((a, b) => b.tests.length - a.tests.length);
 };
 const buildRunSnapshot = (tests, summary) => ({
@@ -365,6 +383,7 @@ const buildRunSnapshot = (tests, summary) => ({
     totals: summary,
     tests: tests.map((test) => ({
         id: test.id,
+        matchKey: test.matchKey,
         title: test.titlePath.join(" > ") || test.title,
         status: test.status,
         signal: test.diagnosis?.signal || null,
@@ -407,18 +426,21 @@ const buildRunDiff = (tests, snapshot) => {
         || readSnapshot(path_1.default.resolve(process.cwd(), ".sentinel", "latest.json"));
     if (!previous || previous.generatedAt === snapshot.generatedAt)
         return null;
-    const previousById = new Map(previous.tests.map((test) => [test.id, test]));
+    const currentById = new Map(tests.map((test) => [test.id, test]));
+    const currentByMatchKey = new Map(tests.map((test) => [test.matchKey, test]));
     const currentFailures = getFailureTests(tests);
     const previousFailures = previous.tests.filter((test) => ["failed", "timedOut", "interrupted"].includes(test.status));
     const previousFailureIds = new Set(previousFailures.map((test) => test.id));
+    const previousFailureMatchKeys = new Set(previousFailures.map((test) => (typeof test.matchKey === "string" ? test.matchKey : test.id)));
     return {
         label: previous.branch === snapshot.branch ? `Compared to previous ${snapshot.branch} run` : "Compared to previous run",
-        newFailures: currentFailures.filter((test) => !previousFailureIds.has(test.id)),
+        newFailures: currentFailures.filter((test) => !previousFailureIds.has(test.id) && !previousFailureMatchKeys.has(test.matchKey)),
         fixedTests: previousFailures.filter((test) => {
-            const current = tests.find((entry) => entry.id === test.id);
+            const current = currentById.get(test.id) ||
+                currentByMatchKey.get(typeof test.matchKey === "string" ? test.matchKey : test.id);
             return current && !["failed", "timedOut", "interrupted"].includes(current.status);
         }),
-        stillFailing: currentFailures.filter((test) => previousFailureIds.has(test.id))
+        stillFailing: currentFailures.filter((test) => previousFailureIds.has(test.id) || previousFailureMatchKeys.has(test.matchKey))
     };
 };
 const renderArtifact = (artifact) => {
@@ -593,22 +615,30 @@ const renderFailureDigest = (tests) => {
     if (!failedTests.length) {
         return `<div class="empty-state">No failed tests were detected in this run.</div>`;
     }
+    const digestGroups = groupFailureDigest(tests);
     return `
     <div class="digest-grid">
-      ${failedTests
-        .map((test) => {
-        const diagnosis = test.diagnosis;
-        const title = escapeHtml(test.title);
-        const summary = escapeHtml(diagnosis ? (0, quickDiagnosis_1.describeFailure)(diagnosis) : "The failure could not be summarized from the captured evidence.");
-        const debugSummary = escapeHtml(diagnosis
-            ? (0, quickDiagnosis_1.buildDebugSummary)(diagnosis)
-            : `Test: ${test.title}\nDiagnosis: Review trace and error details in the expanded card.`);
+      ${digestGroups
+        .map((group) => {
+        const primary = group.tests[0];
+        const diagnosis = primary.diagnosis;
+        const debugSummary = escapeHtml(group.tests.length > 1
+            ? [
+                `Grouped failure summary: ${group.summary}`,
+                ...group.tests.map((test) => `- ${test.title}`)
+            ].join("\n")
+            : diagnosis
+                ? (0, quickDiagnosis_1.buildDebugSummary)(diagnosis)
+                : `Test: ${primary.title}\nDiagnosis: Review trace and error details in the expanded card.`);
+        const uniqueLocators = Array.from(new Set(group.tests.map((test) => test.diagnosis?.locator).filter(Boolean)));
         return `
             <article class="digest-card">
               <div class="digest-head">
                 <div>
-                  <span class="artifact-kind">${escapeHtml(test.status)}</span>
-                  <h3>${title}</h3>
+                  <span class="artifact-kind">${escapeHtml(primary.status)}</span>
+                  <h3>${group.tests.length > 1
+            ? `${group.tests.length} tests share this failure`
+            : escapeHtml(primary.title)}</h3>
                 </div>
                 <button
                   type="button"
@@ -619,12 +649,20 @@ const renderFailureDigest = (tests) => {
                   Copy summary
                 </button>
               </div>
-              <p>${summary}</p>
+              <p>${escapeHtml(group.summary)}</p>
               <div class="fact-row">
-                ${diagnosis?.locator ? `<span class="fact-chip">Locator: ${escapeHtml(diagnosis.locator)}</span>` : ""}
-                ${diagnosis?.expected ? `<span class="fact-chip">Expected: ${escapeHtml(diagnosis.expected)}</span>` : ""}
-                ${diagnosis?.received ? `<span class="fact-chip">Observed: ${escapeHtml(diagnosis.received)}</span>` : ""}
+                ${diagnosis?.expected && diagnosis?.received
+            ? `<span class="fact-chip">Expected: ${escapeHtml(diagnosis.expected)}</span><span class="fact-chip">Observed: ${escapeHtml(diagnosis.received)}</span>`
+            : ""}
+                ${uniqueLocators.length === 1
+            ? `<span class="fact-chip">Locator: ${escapeHtml(uniqueLocators[0])}</span>`
+            : uniqueLocators.length > 1
+                ? `<span class="fact-chip">${uniqueLocators.length} locators involved</span>`
+                : ""}
               </div>
+              ${group.tests.length > 1
+            ? `<ul class="group-list">${group.tests.map((test) => `<li>${escapeHtml(test.title)}</li>`).join("\n")}</ul>`
+            : ""}
             </article>
           `;
     })
@@ -634,7 +672,7 @@ const renderFailureDigest = (tests) => {
 };
 const renderSimilarFailureGroups = (groups) => {
     if (!groups.length) {
-        return `<div class="empty-state">No failure groups were detected for this run.</div>`;
+        return `<div class="empty-state">No repeated failure fingerprint was detected in this run.</div>`;
     }
     return groups
         .map((group) => `
@@ -1223,7 +1261,7 @@ const buildHtml = (tests, summary, extraArtifacts, runDiff) => {
           <h2>Similar Failures</h2>
           <div class="failed-count">${similarGroups.length} groups</div>
         </div>
-        <p>Failures are grouped by the most likely common cause so you can separate one bug from many symptoms.</p>
+        <p>Failures are grouped only when they share the same fingerprint, so one repeated issue is easier to spot.</p>
         ${renderSimilarFailureGroups(similarGroups)}
       </section>
 
@@ -1400,8 +1438,7 @@ function generateLocalDebugReport(options) {
     const reportJsonRaw = fs_1.default.readFileSync(options.playwrightJsonPath, "utf8");
     const reportJson = JSON.parse(reportJsonRaw);
     const reportRoot = { suites: reportJson?.suites || [] };
-    const diagnosisById = buildDiagnosisMap(options.playwrightJsonPath);
-    const tests = collectTests(reportRoot, diagnosisById);
+    const tests = collectTests(reportRoot);
     const testsById = new Map(tests.map((test) => [test.id, test]));
     const claimedSourcePaths = new Set();
     const attachArtifactToTest = (sourcePath, testId) => {

package/dist/quickDiagnosis.d.ts

@@ -15,6 +15,7 @@ export type FailureFacts = {
     status: string;
 };
 export declare const collectFailureFacts: (playwrightJsonPath: string) => FailureFacts[];
+export declare const parseFailureFacts: (title: string, titlePath: string[], message: string, status: string) => FailureFacts;
 export declare const describeFailure: (failure: FailureFacts) => string;
 export declare const buildDebugSummary: (failure: FailureFacts) => string;
 export declare const buildSimilarityKey: (failure: FailureFacts) => string;

package/dist/quickDiagnosis.js

@@ -3,8 +3,18 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.buildQuickDiagnosis = exports.summarizeSignal = exports.buildSimilarityKey = exports.buildDebugSummary = exports.describeFailure = exports.collectFailureFacts = void 0;
+exports.buildQuickDiagnosis = exports.summarizeSignal = exports.buildSimilarityKey = exports.buildDebugSummary = exports.describeFailure = exports.parseFailureFacts = exports.collectFailureFacts = void 0;
 const node_fs_1 = __importDefault(require("node:fs"));
+const normalizeMessageFingerprint = (message) => stripAnsi(message)
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .slice(0, 3)
+    .join(" | ")
+    .replace(/\b\d+ms\b/gi, "<ms>")
+    .replace(/:\d+:\d+/g, ":<line>:<col>")
+    .replace(/\s+/g, " ")
+    .slice(0, 200);
 const stripAnsi = (value) => value.replace(/\u001b\[[0-9;]*m/g, "");
 const toMessage = (result) => {
     const direct = result.error?.message ||
@@ -18,11 +28,11 @@ const toMessage = (result) => {
 };
 const classifySignal = (message) => {
     const lower = message.toLowerCase();
-    if (/timeout|timed out|waiting for/.test(lower))
-        return "timeout";
     if (/expected substring|expected string|received string|tohavetext|tocontaintext/.test(lower)) {
         return "assertion_mismatch";
     }
+    if (/timeout|timed out|waiting for/.test(lower))
+        return "timeout";
     if (/resolved to 0 elements|locator.*not found|never appeared|strict mode violation/.test(lower)) {
         return "locator_not_found";
     }
@@ -120,24 +130,26 @@ const collectFailureFacts = (playwrightJsonPath) => {
         const raw = node_fs_1.default.readFileSync(playwrightJsonPath, "utf8");
         const parsed = JSON.parse(raw);
         const failedCases = flattenFailedCases(parsed);
-        return failedCases.map((failed) => ({
-            title: shortenTitle(failed.title),
-            titlePath: failed.title.split(" > ").filter(Boolean),
-            message: failed.message,
-            signal: failed.signal,
-            locator: extractLocator(failed.message),
-            expected: extractExpected(failed.message),
-            received: extractReceived(failed.message),
-            timeoutMs: extractTimeoutMs(failed.message),
-            lastUrl: extractLastUrl(failed.message),
-            status: "failed"
-        }));
+        return failedCases.map((failed) => (0, exports.parseFailureFacts)(shortenTitle(failed.title), failed.title.split(" > ").filter(Boolean), failed.message, "failed"));
     }
     catch {
         return [];
     }
 };
 exports.collectFailureFacts = collectFailureFacts;
+const parseFailureFacts = (title, titlePath, message, status) => ({
+    title,
+    titlePath,
+    message,
+    signal: classifySignal(message),
+    locator: extractLocator(message),
+    expected: extractExpected(message),
+    received: extractReceived(message),
+    timeoutMs: extractTimeoutMs(message),
+    lastUrl: extractLastUrl(message),
+    status
+});
+exports.parseFailureFacts = parseFailureFacts;
 const describeFailure = (failure) => {
     if (failure.signal === "assertion_mismatch" && failure.locator && failure.expected && failure.received) {
         return `${failure.locator} showed "${failure.received}" instead of "${failure.expected}" before timeout.`;
@@ -179,9 +191,15 @@ const buildDebugSummary = (failure) => {
 };
 exports.buildDebugSummary = buildDebugSummary;
 const buildSimilarityKey = (failure) => {
-    const locator = failure.locator || "unknown-locator";
-    const expected = failure.expected || "unknown-expected";
-    return `${failure.signal}|${locator}|${expected}`;
+    if (failure.locator || failure.expected || failure.received) {
+        return [
+            failure.signal,
+            failure.locator || "unknown-locator",
+            failure.expected || "unknown-expected",
+            failure.received || "unknown-received"
+        ].join("|");
+    }
+    return `${failure.signal}|${normalizeMessageFingerprint(failure.message)}`;
 };
 exports.buildSimilarityKey = buildSimilarityKey;
 exports.summarizeSignal = signalSummary;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sentinelqa/playwright-reporter",
-  "version": "0.1.28",
+  "version": "0.1.30",
   "private": false,
   "description": "Playwright reporter for CI debugging with optional Sentinel cloud dashboards",
   "license": "MIT",