@sentinelqa/playwright-reporter 0.1.44 → 0.1.47

package/README.md

@@ -2,7 +2,7 @@
 
 After every failed run, reporter prints a shareable debugging link:
 
-Sample run: https://app.sentinelqa.com/share/run/9iT2dC8sLRhuaQ2rU_pn81m_y6BTPSetjOgEL0KKLhE
+Sample run: https://app.sentinelqa.com/share/1f343d91-be17-4c14-b1b9-2d4e8ef448d2
 
 Open it to inspect failures instantly or share it in Slack, PRs, or GitHub issues.
 

package/dist/reporter.js

@@ -2,6 +2,8 @@
 const node_1 = require("@sentinelqa/uploader/node");
 const env_1 = require("./env");
 const quickDiagnosis_1 = require("./quickDiagnosis");
+const terminalSummary_1 = require("./terminalSummary");
+const telemetry_1 = require("./telemetry");
 const { sentinelCaptureFailureContextFromReporter } = require("@sentinelqa/uploader/playwright");
 const colorize = (value, code) => {
     if (!process.stdout.isTTY)
@@ -19,6 +21,7 @@ class SentinelReporter {
         this.options = options;
     }
     onBegin(config, suite) {
+        (0, telemetry_1.emitReporterTelemetry)();
         this.totalCount = typeof suite?.allTests === "function" ? suite.allTests().length : 0;
         if (config?.projects?.length && !this.options.project) {
             this.options.project = config.projects[0]?.name || null;
@@ -50,6 +53,7 @@ class SentinelReporter {
             !hasCiEnv &&
             !localUploadEnabled;
         const quickDiagnosis = (0, quickDiagnosis_1.buildQuickDiagnosis)(this.options.playwrightJsonPath);
+        const passingSummary = (0, terminalSummary_1.buildPassingRunSummary)(this.options.playwrightJsonPath);
         console.log("");
         if (quickDiagnosis?.lines.length) {
             console.log(yellow("Quick diagnosis"));
@@ -58,6 +62,32 @@ class SentinelReporter {
             }
             console.log("");
         }
+        if (passingSummary) {
+            console.log(green("Sentinel run summary"));
+            for (const line of passingSummary.lines) {
+                console.log(`  ${line}`);
+            }
+            if (passingSummary.risks.length > 0) {
+                console.log("");
+                console.log(yellow("Potential risks"));
+                for (const line of passingSummary.risks) {
+                    console.log(`  ${line}`);
+                }
+            }
+            if (!hasWorkspaceToken) {
+                console.log("");
+                console.log("Activate a free workspace at sentinelqa.com/register");
+            }
+            console.log("");
+        }
+        if (this.failedCount === 0) {
+            await (0, telemetry_1.flushTelemetry)();
+            return;
+        }
+        if (this.failedCount > 0) {
+            (0, telemetry_1.emitFailedRunTelemetry)();
+        }
+        await (0, telemetry_1.flushTelemetry)();
         if (hasWorkspaceToken && !hasCiEnv && !localUploadEnabled) {
             console.log([
                 "Sentinel: Upload skipped.",
@@ -103,7 +133,7 @@ class SentinelReporter {
         if (!hasWorkspaceToken) {
             console.log("");
             console.log("Upgrade for free to get full AI debugging suggestions");
-            console.log(`  ${dim("https://app.sentinelqa.com/register")}`);
+            console.log(`  ${dim("https://sentinelqa.com/register")}`);
         }
     }
 }

package/dist/telemetry.d.ts

@@ -0,0 +1,3 @@
+export declare const emitReporterTelemetry: () => void;
+export declare const emitFailedRunTelemetry: () => void;
+export declare const flushTelemetry: (timeoutMs?: number) => Promise<void>;

package/dist/telemetry.js

@@ -0,0 +1,173 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.flushTelemetry = exports.emitFailedRunTelemetry = exports.emitReporterTelemetry = void 0;
+const crypto_1 = __importDefault(require("crypto"));
+const http_1 = __importDefault(require("http"));
+const https_1 = __importDefault(require("https"));
+const child_process_1 = require("child_process");
+const DEFAULT_APP_URL = "https://app.sentinelqa.com";
+const PACKAGE_VERSION = (() => {
+    try {
+        return require("../package.json").version || "0.0.0";
+    }
+    catch {
+        return "0.0.0";
+    }
+})();
+const globalState = globalThis;
+const readEnv = (key) => {
+    const value = process.env[key];
+    return value && value.trim().length > 0 ? value.trim() : null;
+};
+const isTruthyEnv = (key) => {
+    const value = readEnv(key);
+    if (!value)
+        return false;
+    return ["1", "true", "yes", "on"].includes(value.toLowerCase());
+};
+const debugLog = (message) => {
+    if (!isTruthyEnv("SENTINEL_TELEMETRY_DEBUG"))
+        return;
+    console.log(`[sentinel-telemetry] ${message}`);
+};
+const gitOutput = (args) => {
+    try {
+        return (0, child_process_1.execFileSync)("git", args, {
+            cwd: process.cwd(),
+            stdio: ["ignore", "pipe", "ignore"],
+            encoding: "utf8"
+        }).trim();
+    }
+    catch {
+        return null;
+    }
+};
+const getCiProvider = () => {
+    if (readEnv("GITHUB_ACTIONS") === "true")
+        return "github";
+    if (readEnv("GITLAB_CI") === "true" || readEnv("CI_PROJECT_ID"))
+        return "gitlab";
+    if (readEnv("CIRCLECI") === "true")
+        return "circleci";
+    return "local";
+};
+const detectRepoIdentity = (provider) => {
+    if (provider === "github") {
+        const repo = readEnv("GITHUB_REPOSITORY");
+        if (repo)
+            return repo.toLowerCase();
+    }
+    if (provider === "gitlab") {
+        const repo = readEnv("CI_PROJECT_PATH") || readEnv("CI_PROJECT_URL");
+        if (repo)
+            return repo.toLowerCase();
+    }
+    if (provider === "circleci") {
+        const user = readEnv("CIRCLE_PROJECT_USERNAME");
+        const repo = readEnv("CIRCLE_PROJECT_REPONAME");
+        if (user && repo)
+            return `${user}/${repo}`.toLowerCase();
+    }
+    const remote = gitOutput(["config", "--get", "remote.origin.url"]) ||
+        gitOutput(["remote", "get-url", "origin"]);
+    if (remote)
+        return remote.toLowerCase();
+    return process.cwd().toLowerCase();
+};
+const buildPayload = (eventType) => {
+    const ciProvider = getCiProvider();
+    const repoHash = crypto_1.default
+        .createHash("sha256")
+        .update(detectRepoIdentity(ciProvider))
+        .digest("hex");
+    return {
+        repoHash,
+        eventType,
+        environment: ciProvider === "local" ? "local" : "ci",
+        ciProvider,
+        mode: readEnv("SENTINEL_TOKEN") ? "workspace" : "public",
+        version: PACKAGE_VERSION
+    };
+};
+const getPendingTelemetry = () => {
+    if (!globalState.__sentinelTelemetryPending) {
+        globalState.__sentinelTelemetryPending = [];
+    }
+    return globalState.__sentinelTelemetryPending;
+};
+const postJson = (url, payload) => new Promise((resolve) => {
+    const target = new URL(url);
+    const client = target.protocol === "https:" ? https_1.default : http_1.default;
+    const body = JSON.stringify(payload);
+    const req = client.request({
+        method: "POST",
+        hostname: target.hostname,
+        port: target.port,
+        path: `${target.pathname}${target.search}`,
+        headers: {
+            "content-type": "application/json",
+            "content-length": Buffer.byteLength(body).toString()
+        }
+    }, (res) => {
+        debugLog(`POST ${target.origin}${target.pathname} -> ${res.statusCode || 0}`);
+        res.resume();
+        res.on("end", () => resolve());
+    });
+    req.setTimeout(1200, () => {
+        debugLog(`timeout posting to ${target.origin}${target.pathname}`);
+        req.destroy();
+        resolve();
+    });
+    req.on("error", (error) => {
+        debugLog(`error posting to ${target.origin}${target.pathname}: ${error.message}`);
+        resolve();
+    });
+    req.write(body);
+    req.end();
+});
+const emitTelemetry = (eventType) => {
+    if (isTruthyEnv("SENTINEL_TELEMETRY_DISABLED"))
+        return;
+    const appUrl = readEnv("SENTINEL_APP_URL") || DEFAULT_APP_URL;
+    const payload = buildPayload(eventType);
+    debugLog(`event=${eventType} dest=${appUrl}/api/telemetry/uploader env=${payload.environment} provider=${payload.ciProvider} mode=${payload.mode} version=${payload.version} repo=${payload.repoHash.slice(0, 12)}`);
+    try {
+        const pending = getPendingTelemetry();
+        const request = postJson(`${appUrl}/api/telemetry/uploader`, payload).finally(() => {
+            const index = pending.indexOf(request);
+            if (index !== -1)
+                pending.splice(index, 1);
+        });
+        pending.push(request);
+    }
+    catch {
+        // Telemetry must never affect test execution.
+    }
+};
+const emitReporterTelemetry = () => {
+    if (globalState.__sentinelReporterTelemetrySent)
+        return;
+    globalState.__sentinelReporterTelemetrySent = true;
+    emitTelemetry("seen");
+};
+exports.emitReporterTelemetry = emitReporterTelemetry;
+const emitFailedRunTelemetry = () => {
+    if (globalState.__sentinelReporterFailedTelemetrySent)
+        return;
+    globalState.__sentinelReporterFailedTelemetrySent = true;
+    emitTelemetry("failed_run");
+};
+exports.emitFailedRunTelemetry = emitFailedRunTelemetry;
+const flushTelemetry = async (timeoutMs = 1500) => {
+    const pending = [...getPendingTelemetry()];
+    if (pending.length === 0)
+        return;
+    await Promise.race([
+        Promise.allSettled(pending).then(() => undefined),
+        new Promise((resolve) => setTimeout(resolve, timeoutMs))
+    ]);
+};
+exports.flushTelemetry = flushTelemetry;

package/dist/terminalSummary.d.ts

@@ -0,0 +1,5 @@
+export type PassingRunSummary = {
+    lines: string[];
+    risks: string[];
+};
+export declare const buildPassingRunSummary: (playwrightJsonPath: string) => PassingRunSummary | null;

package/dist/terminalSummary.js

@@ -0,0 +1,209 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildPassingRunSummary = void 0;
+const node_fs_1 = __importDefault(require("node:fs"));
+const node_path_1 = __importDefault(require("node:path"));
+const HISTORY_DIR = node_path_1.default.join(".sentinel", "reporter-history");
+const ensureDir = (dirPath) => {
+    if (!node_fs_1.default.existsSync(dirPath))
+        node_fs_1.default.mkdirSync(dirPath, { recursive: true });
+};
+const readJson = (filePath) => {
+    if (!node_fs_1.default.existsSync(filePath))
+        return null;
+    try {
+        return JSON.parse(node_fs_1.default.readFileSync(filePath, "utf8"));
+    }
+    catch {
+        return null;
+    }
+};
+const getCurrentBranch = () => {
+    const fromEnv = process.env.GITHUB_REF_NAME ||
+        process.env.CI_COMMIT_REF_NAME ||
+        process.env.CI_COMMIT_BRANCH ||
+        process.env.BRANCH_NAME ||
+        null;
+    return fromEnv && fromEnv.trim() ? fromEnv.trim() : "main";
+};
+const getCurrentGitSha = () => {
+    const fromEnv = process.env.GITHUB_SHA ||
+        process.env.CI_COMMIT_SHA ||
+        process.env.VERCEL_GIT_COMMIT_SHA ||
+        null;
+    return fromEnv && fromEnv.trim() ? fromEnv.trim() : "unknown";
+};
+const normalizeStatus = (status) => {
+    if (status === "failed" || status === "timedOut" || status === "interrupted")
+        return "failed";
+    if (status === "passed")
+        return "passed";
+    return "skipped";
+};
+const formatDuration = (durationMs) => {
+    const totalSeconds = Math.max(0, Math.round(durationMs / 1000));
+    const minutes = Math.floor(totalSeconds / 60);
+    const seconds = totalSeconds % 60;
+    if (minutes === 0)
+        return `${seconds}s`;
+    return `${minutes}m ${seconds}s`;
+};
+const median = (values) => {
+    if (values.length === 0)
+        return null;
+    const sorted = [...values].sort((a, b) => a - b);
+    const mid = Math.floor(sorted.length / 2);
+    if (sorted.length % 2 === 0)
+        return (sorted[mid - 1] + sorted[mid]) / 2;
+    return sorted[mid];
+};
+const buildMatchKey = (file, projectName, titlePath) => [file || "unknown", projectName || "default", titlePath.join(" > ")].join("::");
+const collectTests = (node, ancestors = []) => {
+    const currentTitlePath = node.title ? [...ancestors, node.title] : ancestors;
+    const tests = [];
+    for (const child of node.suites || [])
+        tests.push(...collectTests(child, currentTitlePath));
+    for (const child of node.specs || [])
+        tests.push(...collectTests(child, currentTitlePath));
+    const specTests = Array.isArray(node.tests) ? node.tests : [];
+    for (const test of specTests) {
+        const results = Array.isArray(test.results) ? test.results : [];
+        let finalStatus = normalizeStatus(test.results?.[test.results.length - 1]?.status);
+        let durationMs = 0;
+        let retries = 0;
+        for (const result of results) {
+            durationMs += typeof result.duration === "number" ? result.duration : 0;
+            if (typeof result.retry === "number")
+                retries = Math.max(retries, result.retry);
+            if (normalizeStatus(result.status) === "failed")
+                finalStatus = "failed";
+        }
+        const titlePath = [...currentTitlePath, test.title || "Unnamed test"].filter(Boolean);
+        const file = node.file || node.location?.file || test.location?.file || "";
+        const projectName = test.projectName || "";
+        tests.push({
+            matchKey: buildMatchKey(file, projectName, titlePath),
+            title: titlePath.join(" > "),
+            status: finalStatus,
+            durationMs,
+            retries
+        });
+    }
+    return tests;
+};
+const buildSnapshot = (playwrightJsonPath) => {
+    const parsed = readJson(playwrightJsonPath);
+    if (!parsed)
+        return null;
+    const tests = collectTests(parsed);
+    const failedTests = tests.filter((test) => test.status === "failed");
+    const passedCount = tests.filter((test) => test.status === "passed").length;
+    const skippedCount = tests.filter((test) => test.status === "skipped").length;
+    return {
+        generatedAt: new Date().toISOString(),
+        branch: getCurrentBranch(),
+        gitSha: getCurrentGitSha(),
+        totalTests: tests.length,
+        passedCount,
+        failedCount: failedTests.length,
+        skippedCount,
+        retryPassedCount: tests.filter((test) => test.status === "passed" && test.retries > 0).length,
+        failures: failedTests.map((test) => test.title),
+        tests
+    };
+};
+const listSnapshots = (branch) => {
+    const historyDir = node_path_1.default.resolve(process.cwd(), HISTORY_DIR);
+    if (!node_fs_1.default.existsSync(historyDir))
+        return [];
+    return node_fs_1.default
+        .readdirSync(historyDir)
+        .filter((file) => file.endsWith(".json"))
+        .map((file) => readJson(node_path_1.default.join(historyDir, file)))
+        .filter((value) => Boolean(value))
+        .filter((snapshot) => snapshot.branch === branch)
+        .sort((a, b) => b.generatedAt.localeCompare(a.generatedAt));
+};
+const writeSnapshot = (snapshot) => {
+    ensureDir(node_path_1.default.resolve(process.cwd(), HISTORY_DIR));
+    const fileName = `${snapshot.generatedAt.replace(/[:.]/g, "-")}-${snapshot.gitSha}.json`;
+    node_fs_1.default.writeFileSync(node_path_1.default.resolve(process.cwd(), HISTORY_DIR, fileName), JSON.stringify(snapshot, null, 2), "utf8");
+};
+const buildPassingRunSummary = (playwrightJsonPath) => {
+    const snapshot = buildSnapshot(playwrightJsonPath);
+    if (!snapshot)
+        return null;
+    const previousRuns = listSnapshots(snapshot.branch);
+    writeSnapshot(snapshot);
+    if (snapshot.failedCount > 0)
+        return null;
+    let passStreak = 1;
+    let lastFailureRunsAgo = null;
+    let lastFailureTitle = null;
+    for (let i = 0; i < previousRuns.length; i += 1) {
+        const previous = previousRuns[i];
+        if (previous.failedCount === 0) {
+            passStreak += 1;
+            continue;
+        }
+        lastFailureRunsAgo = i + 1;
+        lastFailureTitle = previous.failures[0] || null;
+        break;
+    }
+    const retryPassedCount = snapshot.retryPassedCount;
+    const durationSamples = new Map();
+    for (const previous of previousRuns.slice(0, 20)) {
+        for (const test of previous.tests || []) {
+            if (test.status !== "passed" || test.durationMs <= 0)
+                continue;
+            const bucket = durationSamples.get(test.matchKey) || [];
+            bucket.push(test.durationMs);
+            durationSamples.set(test.matchKey, bucket);
+        }
+    }
+    let slowestRisk = null;
+    for (const test of snapshot.tests) {
+        if (test.status !== "passed" || test.durationMs <= 0)
+            continue;
+        const samples = durationSamples.get(test.matchKey) || [];
+        const med = median(samples);
+        if (!med || med < 1000)
+            continue;
+        const ratio = test.durationMs / med;
+        if (ratio < 1.8)
+            continue;
+        if (!slowestRisk || ratio > slowestRisk.ratio) {
+            slowestRisk = { title: test.title, ratio };
+        }
+    }
+    const flakyLookingCount = retryPassedCount + (slowestRisk ? 1 : 0);
+    const lines = [
+        `- ${snapshot.passedCount} tests passed in ${formatDuration(snapshot.tests.reduce((sum, test) => sum + test.durationMs, 0))}`
+    ];
+    if (retryPassedCount > 0)
+        lines.push(`- Retries detected: ${retryPassedCount}`);
+    if (flakyLookingCount > 0)
+        lines.push(`- Flaky-looking tests: ${flakyLookingCount}`);
+    if (passStreak > 1)
+        lines.push(`- Pass streak: ${passStreak} runs`);
+    if (lastFailureRunsAgo !== null)
+        lines.push(`- Last failure: ${lastFailureRunsAgo} runs ago`);
+    lines.push(`- Debug readiness: traces, screenshots, video enabled`);
+    const risks = [];
+    if (slowestRisk) {
+        risks.push(`- ${slowestRisk.title} is ${slowestRisk.ratio.toFixed(1)}x slower than its recent median`);
+    }
+    if (retryPassedCount > 0) {
+        risks.push(`- ${retryPassedCount} tests passed after retry`);
+    }
+    if (lastFailureTitle) {
+        risks.push(`- Last failure was in ${lastFailureTitle}`);
+    }
+    if (lines.length <= 2 && risks.length === 0)
+        return null;
+    return { lines, risks };
+};
+exports.buildPassingRunSummary = buildPassingRunSummary;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sentinelqa/playwright-reporter",
-  "version": "0.1.44",
+  "version": "0.1.47",
   "private": false,
   "description": "Playwright reporter for CI debugging with optional Sentinel cloud dashboards",
   "license": "MIT",
@@ -39,7 +39,7 @@
     "@playwright/test": ">=1.40.0"
   },
   "dependencies": {
-    "@sentinelqa/uploader": "^0.1.24"
+    "@sentinelqa/uploader": "file:../SentinelQA_skeleton/packages/uploader"
   },
   "devDependencies": {
     "@types/node": "^20.19.32",