@sentinelqa/playwright-reporter 0.1.47 → 0.1.51

package/dist/terminalSummary.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.buildPassingRunSummary = void 0;
+exports.buildPassingRunSummary = exports.recordReporterHistorySnapshot = void 0;
 const node_fs_1 = __importDefault(require("node:fs"));
 const node_path_1 = __importDefault(require("node:path"));
 const HISTORY_DIR = node_path_1.default.join(".sentinel", "reporter-history");
@@ -39,7 +39,7 @@ const getCurrentGitSha = () => {
 const normalizeStatus = (status) => {
     if (status === "failed" || status === "timedOut" || status === "interrupted")
         return "failed";
-    if (status === "passed")
+    if (status === "passed" || status === "flaky")
         return "passed";
     return "skipped";
 };
@@ -51,6 +51,13 @@ const formatDuration = (durationMs) => {
         return `${seconds}s`;
     return `${minutes}m ${seconds}s`;
 };
+const formatShortDuration = (durationMs) => {
+    if (durationMs === null || !Number.isFinite(durationMs))
+        return null;
+    if (durationMs < 1000)
+        return `${Math.round(durationMs)}ms`;
+    return `${(durationMs / 1000).toFixed(durationMs >= 10000 ? 0 : 1)}s`;
+};
 const median = (values) => {
     if (values.length === 0)
         return null;
@@ -60,7 +67,50 @@ const median = (values) => {
         return (sorted[mid - 1] + sorted[mid]) / 2;
     return sorted[mid];
 };
-const buildMatchKey = (file, projectName, titlePath) => [file || "unknown", projectName || "default", titlePath.join(" > ")].join("::");
+const medianAbsoluteDeviation = (values) => {
+    const med = median(values);
+    if (med === null)
+        return null;
+    return median(values.map((value) => Math.abs(value - med)));
+};
+const cleanTitlePath = (parts) => {
+    const normalized = parts.map((part) => part.trim()).filter(Boolean);
+    const withoutUnnamed = normalized.filter((part) => part !== "Unnamed test");
+    return withoutUnnamed.length ? withoutUnnamed : normalized;
+};
+const formatHistoryTitle = (value) => {
+    if (!value)
+        return null;
+    return cleanTitlePath(value.split(" > ")).join(" > ") || value;
+};
+const buildMatchKey = (file, projectName, titlePath) => [file || "unknown", projectName || "default", cleanTitlePath(titlePath).join(" > ")].join("::");
+const collectWallDurationMs = (node) => {
+    let minStart = Number.POSITIVE_INFINITY;
+    let maxEnd = 0;
+    const walk = (current) => {
+        if (!current)
+            return;
+        for (const child of current.suites || [])
+            walk(child);
+        for (const child of current.specs || [])
+            walk(child);
+        for (const test of current.tests || []) {
+            for (const result of test.results || []) {
+                if (!result?.startTime || typeof result.duration !== 'number')
+                    continue;
+                const start = Date.parse(result.startTime);
+                if (!Number.isFinite(start))
+                    continue;
+                minStart = Math.min(minStart, start);
+                maxEnd = Math.max(maxEnd, start + result.duration);
+            }
+        }
+    };
+    walk(node);
+    if (!Number.isFinite(minStart) || maxEnd <= minStart)
+        return null;
+    return maxEnd - minStart;
+};
 const collectTests = (node, ancestors = []) => {
     const currentTitlePath = node.title ? [...ancestors, node.title] : ancestors;
     const tests = [];
@@ -71,17 +121,15 @@ const collectTests = (node, ancestors = []) => {
     const specTests = Array.isArray(node.tests) ? node.tests : [];
     for (const test of specTests) {
         const results = Array.isArray(test.results) ? test.results : [];
-        let finalStatus = normalizeStatus(test.results?.[test.results.length - 1]?.status);
-        let durationMs = 0;
+        const finalStatus = normalizeStatus(test.results?.[test.results.length - 1]?.status);
+        const finalDurationMs = typeof results[results.length - 1]?.duration === "number" ? results[results.length - 1].duration : 0;
+        let durationMs = finalDurationMs;
         let retries = 0;
         for (const result of results) {
-            durationMs += typeof result.duration === "number" ? result.duration : 0;
             if (typeof result.retry === "number")
                 retries = Math.max(retries, result.retry);
-            if (normalizeStatus(result.status) === "failed")
-                finalStatus = "failed";
         }
-        const titlePath = [...currentTitlePath, test.title || "Unnamed test"].filter(Boolean);
+        const titlePath = cleanTitlePath([...currentTitlePath, test.title || "Unnamed test"]);
         const file = node.file || node.location?.file || test.location?.file || "";
         const projectName = test.projectName || "";
         tests.push({
@@ -89,7 +137,8 @@ const collectTests = (node, ancestors = []) => {
             title: titlePath.join(" > "),
             status: finalStatus,
             durationMs,
-            retries
+            retries,
+            timeoutMs: typeof test.timeout === "number" ? test.timeout : null
         });
     }
     return tests;
@@ -106,6 +155,7 @@ const buildSnapshot = (playwrightJsonPath) => {
         generatedAt: new Date().toISOString(),
         branch: getCurrentBranch(),
         gitSha: getCurrentGitSha(),
+        wallDurationMs: collectWallDurationMs(parsed),
         totalTests: tests.length,
         passedCount,
         failedCount: failedTests.length,
@@ -132,17 +182,24 @@ const writeSnapshot = (snapshot) => {
     const fileName = `${snapshot.generatedAt.replace(/[:.]/g, "-")}-${snapshot.gitSha}.json`;
     node_fs_1.default.writeFileSync(node_path_1.default.resolve(process.cwd(), HISTORY_DIR, fileName), JSON.stringify(snapshot, null, 2), "utf8");
 };
-const buildPassingRunSummary = (playwrightJsonPath) => {
+const recordReporterHistorySnapshot = (playwrightJsonPath) => {
+    const snapshot = buildSnapshot(playwrightJsonPath);
+    if (!snapshot)
+        return null;
+    writeSnapshot(snapshot);
+    return snapshot;
+};
+exports.recordReporterHistorySnapshot = recordReporterHistorySnapshot;
+const buildPassingRunSummary = (playwrightJsonPath, options) => {
     const snapshot = buildSnapshot(playwrightJsonPath);
     if (!snapshot)
         return null;
     const previousRuns = listSnapshots(snapshot.branch);
     writeSnapshot(snapshot);
-    if (snapshot.failedCount > 0)
+    if (snapshot.failedCount > 0 || snapshot.totalTests === 0 || snapshot.passedCount === 0)
         return null;
     let passStreak = 1;
     let lastFailureRunsAgo = null;
-    let lastFailureTitle = null;
     for (let i = 0; i < previousRuns.length; i += 1) {
         const previous = previousRuns[i];
         if (previous.failedCount === 0) {
@@ -150,11 +207,12 @@ const buildPassingRunSummary = (playwrightJsonPath) => {
             continue;
         }
         lastFailureRunsAgo = i + 1;
-        lastFailureTitle = previous.failures[0] || null;
         break;
     }
     const retryPassedCount = snapshot.retryPassedCount;
     const durationSamples = new Map();
+    const recentDurationSamples = new Map();
+    const slowPassCounts = new Map();
     for (const previous of previousRuns.slice(0, 20)) {
         for (const test of previous.tests || []) {
             if (test.status !== "passed" || test.durationMs <= 0)
@@ -162,48 +220,148 @@ const buildPassingRunSummary = (playwrightJsonPath) => {
             const bucket = durationSamples.get(test.matchKey) || [];
             bucket.push(test.durationMs);
             durationSamples.set(test.matchKey, bucket);
+            if (durationSamples.get(test.matchKey).length <= 5) {
+                const recentBucket = recentDurationSamples.get(test.matchKey) || [];
+                recentBucket.push(test.durationMs);
+                recentDurationSamples.set(test.matchKey, recentBucket);
+            }
         }
     }
     let slowestRisk = null;
+    const historicalRetryCounts = new Map();
+    const historicalFailureCounts = new Map();
+    for (const previous of previousRuns.slice(0, 10)) {
+        for (const test of previous.tests || []) {
+            if (test.status === "passed" && test.retries > 0) {
+                historicalRetryCounts.set(test.matchKey, (historicalRetryCounts.get(test.matchKey) || 0) + 1);
+            }
+            if (test.status === "failed") {
+                historicalFailureCounts.set(test.matchKey, (historicalFailureCounts.get(test.matchKey) || 0) + 1);
+            }
+            if (test.status === "passed" && test.durationMs > 0) {
+                const all = durationSamples.get(test.matchKey) || [];
+                const historicalMedian = median(all);
+                if (historicalMedian && test.durationMs / historicalMedian >= 1.35) {
+                    slowPassCounts.set(test.matchKey, (slowPassCounts.get(test.matchKey) || 0) + 1);
+                }
+            }
+        }
+    }
+    const nearFailureCandidates = [];
     for (const test of snapshot.tests) {
         if (test.status !== "passed" || test.durationMs <= 0)
             continue;
         const samples = durationSamples.get(test.matchKey) || [];
         const med = median(samples);
-        if (!med || med < 1000)
-            continue;
-        const ratio = test.durationMs / med;
-        if (ratio < 1.8)
-            continue;
-        if (!slowestRisk || ratio > slowestRisk.ratio) {
+        const variability = medianAbsoluteDeviation(samples);
+        const recentMedian = median(recentDurationSamples.get(test.matchKey) || []);
+        const hasReliableMedian = samples.length >= 5 && Boolean(med) && (med || 0) >= 250;
+        const hasReliableRecentMedian = (recentDurationSamples.get(test.matchKey) || []).length >= 3 && Boolean(recentMedian) && Boolean(med) && (med || 0) >= 250;
+        const varianceFloor = Math.max(400, variability ? variability * 3 : 400);
+        const absoluteDelta = hasReliableMedian && med ? test.durationMs - med : null;
+        const ratio = hasReliableMedian && med && absoluteDelta !== null && absoluteDelta >= varianceFloor ? test.durationMs / med : null;
+        const recentTrendDelta = hasReliableRecentMedian && recentMedian && med ? recentMedian - med : null;
+        const recentTrendRatio = hasReliableRecentMedian &&
+            recentMedian &&
+            med &&
+            recentTrendDelta !== null &&
+            recentTrendDelta >= varianceFloor
+            ? recentMedian / med
+            : null;
+        if (ratio && ratio >= 1.8 && (!slowestRisk || ratio > slowestRisk.ratio)) {
             slowestRisk = { title: test.title, ratio };
         }
+        const historicalRetries = historicalRetryCounts.get(test.matchKey) || 0;
+        const historicalFailures = historicalFailureCounts.get(test.matchKey) || 0;
+        const repeatedSlowPasses = slowPassCounts.get(test.matchKey) || 0;
+        const timeoutUtilization = test.timeoutMs && test.timeoutMs > 0 ? test.durationMs / test.timeoutMs : null;
+        const recentFailurePressure = historicalFailures >= 5 && lastFailureRunsAgo !== null && lastFailureRunsAgo <= 3;
+        const currentRunSignal = Boolean(test.retries > 0 ||
+            (ratio && ratio >= 1.8) ||
+            (timeoutUtilization && timeoutUtilization >= 0.9));
+        const persistentPassSignal = Boolean(historicalRetries >= 2 ||
+            repeatedSlowPasses >= 3 ||
+            (recentTrendRatio && recentTrendRatio >= 1.5));
+        const score = (test.retries > 0 ? 4 : 0) +
+            (ratio && ratio >= 1.8 ? 2 : 0) +
+            (ratio && ratio >= 2.2 ? 1 : 0) +
+            (recentTrendRatio && recentTrendRatio >= 1.5 ? 1 : 0) +
+            (repeatedSlowPasses >= 2 ? 2 : repeatedSlowPasses > 0 ? 1 : 0) +
+            (timeoutUtilization && timeoutUtilization >= 0.65 ? 1 : 0) +
+            (timeoutUtilization && timeoutUtilization >= 0.85 ? 1 : 0) +
+            (historicalRetries >= 2 ? 2 : historicalRetries > 0 ? 1 : 0) +
+            (recentFailurePressure ? 1 : 0);
+        const strongSignal = Boolean(currentRunSignal ||
+            persistentPassSignal);
+        const primaryReason = test.retries > 0
+            ? `passed after ${test.retries} retr${test.retries === 1 ? 'y' : 'ies'} in this run`
+            : historicalRetries >= 2
+                ? `needed retries in ${historicalRetries} of the last 10 passing runs`
+                : recentFailurePressure
+                    ? `failed in ${historicalFailures} of the last 10 runs`
+                    : ratio && ratio >= 1.8
+                        ? `took ${formatShortDuration(test.durationMs)} vs ${formatShortDuration(med)} recent median (${ratio.toFixed(1)}x)`
+                        : repeatedSlowPasses >= 2
+                            ? `has been unusually slow in ${repeatedSlowPasses} of the last 10 passing runs`
+                            : timeoutUtilization && timeoutUtilization >= 0.85
+                                ? `used ${(timeoutUtilization * 100).toFixed(0)}% of its timeout budget`
+                                : recentTrendRatio && recentTrendRatio >= 1.5
+                                    ? `is trending slower over recent runs`
+                                    : `has weak instability signals`;
+        if (score > 0) {
+            nearFailureCandidates.push({
+                title: test.title,
+                ratio,
+                recentTrendRatio,
+                retries: test.retries,
+                historicalRetries,
+                historicalFailures,
+                repeatedSlowPasses,
+                timeoutUtilization,
+                medianDurationMs: med,
+                currentDurationMs: test.durationMs,
+                score,
+                strongSignal,
+                currentRunSignal,
+                primaryReason
+            });
+        }
     }
-    const flakyLookingCount = retryPassedCount + (slowestRisk ? 1 : 0);
-    const lines = [
-        `- ${snapshot.passedCount} tests passed in ${formatDuration(snapshot.tests.reduce((sum, test) => sum + test.durationMs, 0))}`
-    ];
-    if (retryPassedCount > 0)
-        lines.push(`- Retries detected: ${retryPassedCount}`);
-    if (flakyLookingCount > 0)
-        lines.push(`- Flaky-looking tests: ${flakyLookingCount}`);
-    if (passStreak > 1)
-        lines.push(`- Pass streak: ${passStreak} runs`);
-    if (lastFailureRunsAgo !== null)
-        lines.push(`- Last failure: ${lastFailureRunsAgo} runs ago`);
-    lines.push(`- Debug readiness: traces, screenshots, video enabled`);
-    const risks = [];
-    if (slowestRisk) {
-        risks.push(`- ${slowestRisk.title} is ${slowestRisk.ratio.toFixed(1)}x slower than its recent median`);
-    }
-    if (retryPassedCount > 0) {
-        risks.push(`- ${retryPassedCount} tests passed after retry`);
+    nearFailureCandidates.sort((left, right) => Number(right.currentRunSignal) - Number(left.currentRunSignal) ||
+        right.score - left.score ||
+        (right.historicalFailures || 0) - (left.historicalFailures || 0) ||
+        (right.timeoutUtilization || 0) - (left.timeoutUtilization || 0) ||
+        (right.ratio || 0) - (left.ratio || 0));
+    const strongNearFailures = nearFailureCandidates.filter((candidate) => candidate.strongSignal &&
+        (candidate.currentRunSignal ||
+            candidate.historicalRetries >= 2 ||
+            candidate.repeatedSlowPasses >= 3 ||
+            (candidate.recentTrendRatio !== null && candidate.recentTrendRatio >= 1.5)) &&
+        candidate.score >= 3);
+    const topNearFailures = strongNearFailures.slice(0, 2);
+    const flakyLookingCount = strongNearFailures.length;
+    const hasActiveRisk = flakyLookingCount > 0;
+    const totalDurationMs = snapshot.tests.reduce((sum, test) => sum + test.durationMs, 0);
+    const displayedRunDurationMs = options?.observedRunDurationMs || snapshot.wallDurationMs || totalDurationMs;
+    const lines = ["All tests passed", `${snapshot.passedCount} tests in ${formatDuration(displayedRunDurationMs)}`];
+    if (hasActiveRisk && topNearFailures[0]) {
+        lines.push(`At risk: ${topNearFailures[0].title} ${topNearFailures[0].primaryReason}`);
+        lines.push(topNearFailures[0].retries > 0 || topNearFailures[0].historicalRetries >= 2
+            ? "Why this matters: Similar timing and retry patterns often turn into flaky failures"
+            : "Why this matters: Performance regressions often lead to flaky failures");
+        if (lastFailureRunsAgo !== null)
+            lines.push(`Last failure: ${lastFailureRunsAgo} runs ago`);
+        lines.push(topNearFailures[0].retries > 0
+            ? "Recommendation: inspect retry behavior or timing around this test"
+            : "Recommendation: monitor the next runs or investigate the slowdown");
     }
-    if (lastFailureTitle) {
-        risks.push(`- Last failure was in ${lastFailureTitle}`);
+    else {
+        lines.push("No anomalies detected");
+        if (lastFailureRunsAgo !== null)
+            lines.push(`Last failure: ${lastFailureRunsAgo} runs ago`);
+        lines.push("Status: Stable");
     }
-    if (lines.length <= 2 && risks.length === 0)
-        return null;
-    return { lines, risks };
+    lines.push("Artifacts ready: traces, screenshots, video");
+    return { lines };
 };
 exports.buildPassingRunSummary = buildPassingRunSummary;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sentinelqa/playwright-reporter",
-  "version": "0.1.47",
+  "version": "0.1.51",
   "private": false,
   "description": "Playwright reporter for CI debugging with optional Sentinel cloud dashboards",
   "license": "MIT",
@@ -39,7 +39,7 @@
     "@playwright/test": ">=1.40.0"
   },
   "dependencies": {
-    "@sentinelqa/uploader": "file:../SentinelQA_skeleton/packages/uploader"
+    "@sentinelqa/uploader": "^0.1.35"
   },
   "devDependencies": {
     "@types/node": "^20.19.32",