@sentinelqa/playwright-reporter 0.1.45 → 0.1.50

package/dist/reporter.d.ts

@@ -9,6 +9,7 @@ type ReporterOptions = {
 declare class SentinelReporter {
     private failedCount;
     private totalCount;
+    private startedAt;
     private options;
     constructor(options: ReporterOptions);
     onBegin(config: any, suite: any): void;

package/dist/reporter.js

@@ -2,6 +2,9 @@
 const node_1 = require("@sentinelqa/uploader/node");
 const env_1 = require("./env");
 const quickDiagnosis_1 = require("./quickDiagnosis");
+const terminalSummary_1 = require("./terminalSummary");
+const runHistory_1 = require("./runHistory");
+const telemetry_1 = require("./telemetry");
 const { sentinelCaptureFailureContextFromReporter } = require("@sentinelqa/uploader/playwright");
 const colorize = (value, code) => {
     if (!process.stdout.isTTY)
@@ -11,14 +14,43 @@ const colorize = (value, code) => {
 const green = (value) => colorize(value, "32");
 const yellow = (value) => colorize(value, "33");
 const dim = (value) => colorize(value, "2");
+const readFinalFailedCount = (playwrightJsonPath) => {
+    try {
+        const parsed = JSON.parse(require("node:fs").readFileSync(playwrightJsonPath, "utf8"));
+        let failed = 0;
+        const walk = (node) => {
+            if (!node)
+                return;
+            for (const child of node.suites || [])
+                walk(child);
+            for (const child of node.specs || [])
+                walk(child);
+            for (const test of node.tests || []) {
+                const results = Array.isArray(test.results) ? test.results : [];
+                const finalStatus = results[results.length - 1]?.status || null;
+                if (finalStatus === "failed" || finalStatus === "timedOut" || finalStatus === "interrupted") {
+                    failed += 1;
+                }
+            }
+        };
+        walk(parsed);
+        return failed;
+    }
+    catch {
+        return null;
+    }
+};
 class SentinelReporter {
     constructor(options) {
         this.failedCount = 0;
         this.totalCount = 0;
+        this.startedAt = Date.now();
         (0, env_1.loadSentinelEnv)();
         this.options = options;
     }
     onBegin(config, suite) {
+        this.startedAt = Date.now();
+        (0, telemetry_1.emitReporterTelemetry)();
         this.totalCount = typeof suite?.allTests === "function" ? suite.allTests().length : 0;
         if (config?.projects?.length && !this.options.project) {
             this.options.project = config.projects[0]?.name || null;
@@ -50,14 +82,41 @@ class SentinelReporter {
             !hasCiEnv &&
             !localUploadEnabled;
         const quickDiagnosis = (0, quickDiagnosis_1.buildQuickDiagnosis)(this.options.playwrightJsonPath);
+        const finalFailedCount = readFinalFailedCount(this.options.playwrightJsonPath);
+        const effectiveFailedCount = typeof finalFailedCount === "number" ? finalFailedCount : this.failedCount;
+        const passingSummary = effectiveFailedCount === 0
+            ? (0, terminalSummary_1.buildPassingRunSummary)(this.options.playwrightJsonPath, { observedRunDurationMs: Date.now() - this.startedAt })
+            : null;
+        const failedRunHistory = effectiveFailedCount > 0 ? (0, runHistory_1.buildFailedRunHistorySummary)(this.options.playwrightJsonPath) : null;
+        if (effectiveFailedCount > 0) {
+            (0, terminalSummary_1.recordReporterHistorySnapshot)(this.options.playwrightJsonPath);
+        }
         console.log("");
-        if (quickDiagnosis?.lines.length) {
-            console.log(yellow("Quick diagnosis"));
-            for (const line of quickDiagnosis.lines) {
-                console.log(`  ${dim(line)}`);
+        if (passingSummary) {
+            console.log(green("Sentinel run summary"));
+            for (const line of passingSummary.lines) {
+                console.log(`  ${line}`);
+            }
+            if (passingSummary.risks.length > 0) {
+                console.log("");
+                console.log(yellow("Potential risks"));
+                for (const line of passingSummary.risks) {
+                    console.log(`  ${line}`);
+                }
+            }
+            if (!hasWorkspaceToken) {
+                console.log("");
+                console.log("Activate a free workspace at sentinelqa.com/register");
             }
             console.log("");
         }
+        if (effectiveFailedCount > 0) {
+            (0, telemetry_1.emitFailedRunTelemetry)();
+        }
+        await (0, telemetry_1.flushTelemetry)();
+        if (effectiveFailedCount === 0) {
+            return;
+        }
         if (hasWorkspaceToken && !hasCiEnv && !localUploadEnabled) {
             console.log([
                 "Sentinel: Upload skipped.",
@@ -74,12 +133,18 @@ class SentinelReporter {
             console.log(green("✔ Artifacts collected"));
         }
         console.log("");
-        console.log("Uploading hosted debugging report to Sentinel...");
-        if (usingImplicitLocalPublicMode) {
-            console.log(dim("Local upload env not set. Falling back to local metadata for a public hosted report."));
+        if (hasWorkspaceToken) {
+            console.log("Uploading hosted debugging report to Sentinel...");
+        }
+        else if (usingImplicitLocalPublicMode) {
+            console.log("Creating free hosted debug report with Sentinel...");
+            console.log(dim("No API key detected. Using the free public report flow for this local run."));
+        }
+        else {
+            console.log("Creating free hosted debug report with Sentinel...");
         }
         console.log("");
-        const upload = await (0, node_1.runSentinelUpload)({
+        const upload = (await (0, node_1.runSentinelUpload)({
             playwrightJsonPath: this.options.playwrightJsonPath,
             playwrightReportDir: this.options.playwrightReportDir,
             testResultsDir: this.options.testResultsDir,
@@ -90,20 +155,46 @@ class SentinelReporter {
                 SENTINEL_REPORTER_SILENT: "1",
                 SENTINEL_UPLOAD_LOCAL: usingImplicitLocalPublicMode ? "1" : process.env.SENTINEL_UPLOAD_LOCAL
             }
-        });
+        }));
         if (upload.exitCode !== 0) {
             throw new Error(`Sentinel upload failed with exit code ${upload.exitCode}`);
         }
+        const backendReady = upload.diagnosis?.status === "ready" && Boolean(upload.diagnosis?.lines?.length);
+        const diagnosis = backendReady ? upload.diagnosis : quickDiagnosis;
+        if (diagnosis?.lines.length) {
+            console.log("");
+            console.log(yellow("Sentinel diagnosis"));
+            for (const line of diagnosis.lines) {
+                console.log(`  ${dim(line)}`);
+            }
+            if (diagnosis.footer?.length) {
+                console.log("");
+                for (const line of diagnosis.footer) {
+                    console.log(`  ${dim(line)}`);
+                }
+            }
+        }
+        const historyLines = failedRunHistory?.lines || [];
+        const normalizedHistory = backendReady
+            ? historyLines.filter((line) => !line.includes("Seen before:"))
+            : historyLines;
+        if (normalizedHistory.length) {
+            console.log("");
+            console.log(yellow("Run context"));
+            for (const line of normalizedHistory) {
+                console.log(`  ${dim(line)}`);
+            }
+        }
         console.log("");
-        console.log("Sentinel report");
+        console.log("View full debug report");
         console.log(`  ${upload.shareRunUrl || upload.internalRunUrl}`);
         if (upload.shareLabel) {
             console.log(`  ${dim(upload.shareLabel)}`);
         }
         if (!hasWorkspaceToken) {
             console.log("");
-            console.log("Upgrade for free to get full AI debugging suggestions");
-            console.log(`  ${dim("https://app.sentinelqa.com/register")}`);
+            console.log("Create a free workspace to keep reports private, compare runs, and unlock deeper AI debugging");
+            console.log(`  ${dim("https://sentinelqa.com/register")}`);
         }
     }
 }

package/dist/runHistory.d.ts

@@ -3,5 +3,9 @@ type RunDiffSummary = {
     fixedTests: number;
     stillFailing: number;
 };
+export type FailedRunHistorySummary = {
+    lines: string[];
+};
 export declare const buildRunDiffSummary: (playwrightJsonPath: string) => RunDiffSummary | null;
+export declare const buildFailedRunHistorySummary: (playwrightJsonPath: string) => FailedRunHistorySummary | null;
 export {};

package/dist/runHistory.js

@@ -3,11 +3,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.buildRunDiffSummary = void 0;
+exports.buildFailedRunHistorySummary = exports.buildRunDiffSummary = void 0;
 const node_fs_1 = __importDefault(require("node:fs"));
 const node_path_1 = __importDefault(require("node:path"));
 const quickDiagnosis_1 = require("./quickDiagnosis");
 const SENTINEL_HISTORY_DIR = node_path_1.default.join(".sentinel", "history");
+const REPORTER_HISTORY_DIR = node_path_1.default.join(".sentinel", "reporter-history");
 const ensureDir = (dirPath) => {
     if (!node_fs_1.default.existsSync(dirPath)) {
         node_fs_1.default.mkdirSync(dirPath, { recursive: true });
@@ -76,6 +77,16 @@ const normalizeFailures = (snapshot) => {
     }
     return [];
 };
+const readJson = (filePath) => {
+    if (!node_fs_1.default.existsSync(filePath))
+        return null;
+    try {
+        return JSON.parse(node_fs_1.default.readFileSync(filePath, "utf8"));
+    }
+    catch {
+        return null;
+    }
+};
 const readSnapshot = (filePath) => {
     if (!node_fs_1.default.existsSync(filePath))
         return null;
@@ -86,6 +97,58 @@ const readSnapshot = (filePath) => {
         return null;
     }
 };
+const listSnapshots = (branch) => {
+    const historyDir = node_path_1.default.resolve(process.cwd(), SENTINEL_HISTORY_DIR);
+    if (!node_fs_1.default.existsSync(historyDir))
+        return [];
+    return node_fs_1.default
+        .readdirSync(historyDir)
+        .filter((file) => file.endsWith(".json"))
+        .map((file) => readSnapshot(node_path_1.default.join(historyDir, file)))
+        .filter((value) => Boolean(value))
+        .map((snapshot) => ({
+        generatedAt: typeof snapshot.generatedAt === "string" ? snapshot.generatedAt : new Date(0).toISOString(),
+        branch: typeof snapshot.branch === "string" ? snapshot.branch : branch,
+        gitSha: typeof snapshot.gitSha === "string" ? snapshot.gitSha : "unknown",
+        failures: normalizeFailures(snapshot)
+    }))
+        .filter((snapshot) => snapshot.branch === branch)
+        .sort((a, b) => b.generatedAt.localeCompare(a.generatedAt));
+};
+const isMeaningfulReporterSnapshot = (snapshot) => {
+    if (!snapshot)
+        return false;
+    if (snapshot.totalTests <= 0)
+        return false;
+    if ((snapshot.passedCount || 0) === 0 && (snapshot.failedCount || 0) === 0)
+        return false;
+    return true;
+};
+const listReporterSnapshots = (branch) => {
+    const historyDir = node_path_1.default.resolve(process.cwd(), REPORTER_HISTORY_DIR);
+    if (!node_fs_1.default.existsSync(historyDir))
+        return [];
+    return node_fs_1.default
+        .readdirSync(historyDir)
+        .filter((file) => file.endsWith(".json"))
+        .map((file) => readJson(node_path_1.default.join(historyDir, file)))
+        .filter((value) => Boolean(value))
+        .filter((snapshot) => snapshot.branch === branch)
+        .filter((snapshot) => isMeaningfulReporterSnapshot(snapshot))
+        .sort((a, b) => b.generatedAt.localeCompare(a.generatedAt));
+};
+const normalizeReporterFailures = (snapshot) => {
+    if (!snapshot)
+        return [];
+    return (snapshot.tests || [])
+        .filter((test) => test && test.status === "failed")
+        .map((test) => ({
+        id: typeof test.title === "string" ? test.title : test.matchKey,
+        matchKey: typeof test.matchKey === "string" ? test.matchKey : (typeof test.title === "string" ? test.title : "unknown"),
+        title: typeof test.title === "string" ? test.title : test.matchKey,
+        status: "failed"
+    }));
+};
 const writeSnapshot = (snapshot) => {
     ensureDir(node_path_1.default.resolve(process.cwd(), ".sentinel"));
     ensureDir(node_path_1.default.resolve(process.cwd(), SENTINEL_HISTORY_DIR));
@@ -96,6 +159,7 @@ const writeSnapshot = (snapshot) => {
         node_fs_1.default.writeFileSync(node_path_1.default.resolve(process.cwd(), pointerPath), JSON.stringify({ path: historyPath, ...snapshot }, null, 2), "utf8");
     }
 };
+const matchesFailure = (left, right) => left.id === right.id || left.matchKey === right.matchKey;
 const buildRunDiffSummary = (playwrightJsonPath) => {
     const snapshot = buildSnapshot(playwrightJsonPath);
     const previous = readSnapshot(node_path_1.default.resolve(process.cwd(), ".sentinel", `latest-${snapshot.branch}.json`)) ||
@@ -105,12 +169,58 @@ const buildRunDiffSummary = (playwrightJsonPath) => {
     const currentFailureMatchKeys = new Set(snapshot.failures.map((test) => test.matchKey));
     const diff = previous && previous.generatedAt !== snapshot.generatedAt
         ? {
-            newFailures: snapshot.failures.filter((test) => !previousFailures.some((prev) => prev.id === test.id || prev.matchKey === test.matchKey)).length,
+            newFailures: snapshot.failures.filter((test) => !previousFailures.some((prev) => matchesFailure(prev, test))).length,
             fixedTests: previousFailures.filter((test) => !currentFailureIds.has(test.id) && !currentFailureMatchKeys.has(test.matchKey)).length,
-            stillFailing: snapshot.failures.filter((test) => previousFailures.some((prev) => prev.id === test.id || prev.matchKey === test.matchKey)).length
+            stillFailing: snapshot.failures.filter((test) => previousFailures.some((prev) => matchesFailure(prev, test))).length
         }
         : null;
     writeSnapshot(snapshot);
     return diff;
 };
 exports.buildRunDiffSummary = buildRunDiffSummary;
+const buildFailedRunHistorySummary = (playwrightJsonPath) => {
+    const snapshot = buildSnapshot(playwrightJsonPath);
+    if (snapshot.failures.length === 0)
+        return null;
+    const reporterSnapshots = listReporterSnapshots(snapshot.branch);
+    const previousRun = reporterSnapshots[0] || null;
+    const previousFailures = normalizeReporterFailures(previousRun);
+    let passStreakBeforeFailure = 0;
+    for (const previous of reporterSnapshots) {
+        if ((previous.failedCount || 0) === 0) {
+            passStreakBeforeFailure += 1;
+            continue;
+        }
+        break;
+    }
+    const newFailures = snapshot.failures.filter((failure) => !previousFailures.some((prev) => matchesFailure(prev, failure))).length;
+    const fixedTests = previousFailures.filter((failure) => !snapshot.failures.some((current) => matchesFailure(current, failure))).length;
+    const stillFailing = snapshot.failures.filter((failure) => previousFailures.some((prev) => matchesFailure(prev, failure))).length;
+    const failingReporterRuns = reporterSnapshots.filter((run) => (run.failedCount || 0) > 0);
+    const recurringFailures = snapshot.failures
+        .map((failure) => ({
+        failure,
+        occurrences: failingReporterRuns.filter((run) => normalizeReporterFailures(run).some((prev) => matchesFailure(prev, failure))).length
+    }))
+        .sort((a, b) => b.occurrences - a.occurrences);
+    const topRecurring = recurringFailures.find((item) => item.occurrences > 0) || null;
+    writeSnapshot(snapshot);
+    const lines = [];
+    if (passStreakBeforeFailure > 0) {
+        lines.push(`- First failure after ${passStreakBeforeFailure} passing runs`);
+    }
+    if (previousRun && (newFailures > 0 || fixedTests > 0 || stillFailing > 0)) {
+        const previousWasGreen = (previousRun.failedCount || 0) === 0;
+        if (previousWasGreen) {
+            lines.push(`- The immediately previous run was green. Compared to that previous run: ${newFailures} newly failing in this run, ${stillFailing} still failing, ${fixedTests} no longer failing`);
+        }
+        else {
+            lines.push(`- Compared to the immediately previous run: ${newFailures} newly failing in this run, ${stillFailing} still failing, ${fixedTests} no longer failing`);
+        }
+    }
+    if (topRecurring) {
+        lines.push(`- Recurring across ${topRecurring.occurrences + 1} recorded failed runs in local history (${topRecurring.failure.title})`);
+    }
+    return lines.length ? { lines } : null;
+};
+exports.buildFailedRunHistorySummary = buildFailedRunHistorySummary;

package/dist/telemetry.d.ts

@@ -0,0 +1,3 @@
+export declare const emitReporterTelemetry: () => void;
+export declare const emitFailedRunTelemetry: () => void;
+export declare const flushTelemetry: (timeoutMs?: number) => Promise<void>;

package/dist/telemetry.js

@@ -0,0 +1,173 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.flushTelemetry = exports.emitFailedRunTelemetry = exports.emitReporterTelemetry = void 0;
+const crypto_1 = __importDefault(require("crypto"));
+const http_1 = __importDefault(require("http"));
+const https_1 = __importDefault(require("https"));
+const child_process_1 = require("child_process");
+const DEFAULT_APP_URL = "https://app.sentinelqa.com";
+const PACKAGE_VERSION = (() => {
+    try {
+        return require("../package.json").version || "0.0.0";
+    }
+    catch {
+        return "0.0.0";
+    }
+})();
+const globalState = globalThis;
+const readEnv = (key) => {
+    const value = process.env[key];
+    return value && value.trim().length > 0 ? value.trim() : null;
+};
+const isTruthyEnv = (key) => {
+    const value = readEnv(key);
+    if (!value)
+        return false;
+    return ["1", "true", "yes", "on"].includes(value.toLowerCase());
+};
+const debugLog = (message) => {
+    if (!isTruthyEnv("SENTINEL_TELEMETRY_DEBUG"))
+        return;
+    console.log(`[sentinel-telemetry] ${message}`);
+};
+const gitOutput = (args) => {
+    try {
+        return (0, child_process_1.execFileSync)("git", args, {
+            cwd: process.cwd(),
+            stdio: ["ignore", "pipe", "ignore"],
+            encoding: "utf8"
+        }).trim();
+    }
+    catch {
+        return null;
+    }
+};
+const getCiProvider = () => {
+    if (readEnv("GITHUB_ACTIONS") === "true")
+        return "github";
+    if (readEnv("GITLAB_CI") === "true" || readEnv("CI_PROJECT_ID"))
+        return "gitlab";
+    if (readEnv("CIRCLECI") === "true")
+        return "circleci";
+    return "local";
+};
+const detectRepoIdentity = (provider) => {
+    if (provider === "github") {
+        const repo = readEnv("GITHUB_REPOSITORY");
+        if (repo)
+            return repo.toLowerCase();
+    }
+    if (provider === "gitlab") {
+        const repo = readEnv("CI_PROJECT_PATH") || readEnv("CI_PROJECT_URL");
+        if (repo)
+            return repo.toLowerCase();
+    }
+    if (provider === "circleci") {
+        const user = readEnv("CIRCLE_PROJECT_USERNAME");
+        const repo = readEnv("CIRCLE_PROJECT_REPONAME");
+        if (user && repo)
+            return `${user}/${repo}`.toLowerCase();
+    }
+    const remote = gitOutput(["config", "--get", "remote.origin.url"]) ||
+        gitOutput(["remote", "get-url", "origin"]);
+    if (remote)
+        return remote.toLowerCase();
+    return process.cwd().toLowerCase();
+};
+const buildPayload = (eventType) => {
+    const ciProvider = getCiProvider();
+    const repoHash = crypto_1.default
+        .createHash("sha256")
+        .update(detectRepoIdentity(ciProvider))
+        .digest("hex");
+    return {
+        repoHash,
+        eventType,
+        environment: ciProvider === "local" ? "local" : "ci",
+        ciProvider,
+        mode: readEnv("SENTINEL_TOKEN") ? "workspace" : "public",
+        version: PACKAGE_VERSION
+    };
+};
+const getPendingTelemetry = () => {
+    if (!globalState.__sentinelTelemetryPending) {
+        globalState.__sentinelTelemetryPending = [];
+    }
+    return globalState.__sentinelTelemetryPending;
+};
+const postJson = (url, payload) => new Promise((resolve) => {
+    const target = new URL(url);
+    const client = target.protocol === "https:" ? https_1.default : http_1.default;
+    const body = JSON.stringify(payload);
+    const req = client.request({
+        method: "POST",
+        hostname: target.hostname,
+        port: target.port,
+        path: `${target.pathname}${target.search}`,
+        headers: {
+            "content-type": "application/json",
+            "content-length": Buffer.byteLength(body).toString()
+        }
+    }, (res) => {
+        debugLog(`POST ${target.origin}${target.pathname} -> ${res.statusCode || 0}`);
+        res.resume();
+        res.on("end", () => resolve());
+    });
+    req.setTimeout(1200, () => {
+        debugLog(`timeout posting to ${target.origin}${target.pathname}`);
+        req.destroy();
+        resolve();
+    });
+    req.on("error", (error) => {
+        debugLog(`error posting to ${target.origin}${target.pathname}: ${error.message}`);
+        resolve();
+    });
+    req.write(body);
+    req.end();
+});
+const emitTelemetry = (eventType) => {
+    if (isTruthyEnv("SENTINEL_TELEMETRY_DISABLED"))
+        return;
+    const appUrl = readEnv("SENTINEL_APP_URL") || DEFAULT_APP_URL;
+    const payload = buildPayload(eventType);
+    debugLog(`event=${eventType} dest=${appUrl}/api/telemetry/uploader env=${payload.environment} provider=${payload.ciProvider} mode=${payload.mode} version=${payload.version} repo=${payload.repoHash.slice(0, 12)}`);
+    try {
+        const pending = getPendingTelemetry();
+        const request = postJson(`${appUrl}/api/telemetry/uploader`, payload).finally(() => {
+            const index = pending.indexOf(request);
+            if (index !== -1)
+                pending.splice(index, 1);
+        });
+        pending.push(request);
+    }
+    catch {
+        // Telemetry must never affect test execution.
+    }
+};
+const emitReporterTelemetry = () => {
+    if (globalState.__sentinelReporterTelemetrySent)
+        return;
+    globalState.__sentinelReporterTelemetrySent = true;
+    emitTelemetry("seen");
+};
+exports.emitReporterTelemetry = emitReporterTelemetry;
+const emitFailedRunTelemetry = () => {
+    if (globalState.__sentinelReporterFailedTelemetrySent)
+        return;
+    globalState.__sentinelReporterFailedTelemetrySent = true;
+    emitTelemetry("failed_run");
+};
+exports.emitFailedRunTelemetry = emitFailedRunTelemetry;
+const flushTelemetry = async (timeoutMs = 1500) => {
+    const pending = [...getPendingTelemetry()];
+    if (pending.length === 0)
+        return;
+    await Promise.race([
+        Promise.allSettled(pending).then(() => undefined),
+        new Promise((resolve) => setTimeout(resolve, timeoutMs))
+    ]);
+};
+exports.flushTelemetry = flushTelemetry;

package/dist/terminalSummary.d.ts

@@ -0,0 +1,31 @@
+type HistoryTest = {
+    matchKey: string;
+    title: string;
+    status: string;
+    durationMs: number;
+    retries: number;
+    timeoutMs: number | null;
+};
+type HistorySnapshot = {
+    generatedAt: string;
+    branch: string;
+    gitSha: string;
+    wallDurationMs: number | null;
+    totalTests: number;
+    passedCount: number;
+    failedCount: number;
+    skippedCount: number;
+    retryPassedCount: number;
+    failures: string[];
+    tests: HistoryTest[];
+};
+export type PassingRunSummary = {
+    lines: string[];
+    risks: string[];
+};
+type PassingRunSummaryOptions = {
+    observedRunDurationMs?: number | null;
+};
+export declare const recordReporterHistorySnapshot: (playwrightJsonPath: string) => HistorySnapshot;
+export declare const buildPassingRunSummary: (playwrightJsonPath: string, options?: PassingRunSummaryOptions) => PassingRunSummary | null;
+export {};