@fixsense/cypress-plugin 0.2.0 → 0.2.4

package/README.md

@@ -55,9 +55,28 @@ actually wrong, so a green suite stays quiet.
 On GitHub Actions the same table is written to the job summary, so it appears
 on the run's own page instead of scrolling past in the log.
 
+On GitHub Actions it also writes an annotation for each flaky or broken test,
+so the finding shows up inline on the pull request, carrying how many of the
+last runs the test misbehaved in.
+
 `.fixsense/` is git-ignored for you: committing it would mean a diff on every
 run and merge conflicts in a file nobody reads.
 
+### Keeping the history in CI
+
+Every CI job starts from a clean checkout, so the history is gone before the
+next run reads it. The plugin says so in one line instead of staying silent.
+On GitHub Actions, cache the directory between runs:
+
+```yaml
+- name: Keep the test history between runs
+  uses: actions/cache@v4
+  with:
+    path: .fixsense
+    key: fixsense-history-${{ github.run_id }}
+    restore-keys: fixsense-history-
+```
+
 ### Why an account, then
 
 The local history lives on one machine and dies with the CI container. The

package/dist/index.d.ts

@@ -40,4 +40,4 @@ export declare function registerFixSense(on: On, options?: FixSenseCypressOption
 export declare function sendRun(tests: TestRecord[], startedAt: Date, options: FixSenseCypressOptions): Promise<void>;
 export default registerFixSense;
 export * from "./payload";
-export { computeFlakiness, readHistory, recordAndReport, renderMarkdown, renderSummary, scoreHistory, HISTORY_DIR, HISTORY_FILE, MAX_RUNS, } from "./local";
+export { computeFlakiness, readHistory, recordAndReport, renderMarkdown, renderSummary, renderAnnotations, scoreHistory, HISTORY_DIR, HISTORY_FILE, MAX_RUNS, } from "./local";

package/dist/index.js

@@ -23,12 +23,13 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.MAX_RUNS = exports.HISTORY_FILE = exports.HISTORY_DIR = exports.scoreHistory = exports.renderSummary = exports.renderMarkdown = exports.recordAndReport = exports.readHistory = exports.computeFlakiness = void 0;
+exports.MAX_RUNS = exports.HISTORY_FILE = exports.HISTORY_DIR = exports.scoreHistory = exports.renderAnnotations = exports.renderSummary = exports.renderMarkdown = exports.recordAndReport = exports.readHistory = exports.computeFlakiness = void 0;
 exports.registerFixSense = registerFixSense;
 exports.sendRun = sendRun;
 const payload_1 = require("./payload");
 const local_1 = require("./local");
 const DEFAULT_API_URL = "https://fix-sense.com";
+const DOCS_URL = "https://fix-sense.com/docs/integrations/cypress-plugin#keeping-the-history-in-ci";
 const DEFAULT_TIMEOUT_MS = 10_000;
 /**
  * Register the plugin from `setupNodeEvents`.
@@ -60,16 +61,19 @@ async function sendRun(tests, startedAt, options) {
         return;
     // The local history first, because it needs no account and is the only
     // part most first-time users will ever see.
+    const apiKey = options.apiKey ?? process.env.FIXSENSE_API_KEY;
     if (options.local !== false) {
         try {
-            (0, local_1.recordAndReport)(options.cwd ?? process.cwd(), tests);
+            (0, local_1.recordAndReport)(options.cwd ?? process.cwd(), tests, new Date(), {
+                connected: Boolean(apiKey),
+                docsUrl: DOCS_URL,
+            });
         }
         catch (error) {
             const reason = error instanceof Error ? error.message : String(error);
             console.error(`[fixsense] Local history not updated: ${reason}`);
         }
     }
-    const apiKey = options.apiKey ?? process.env.FIXSENSE_API_KEY;
     // Silent without a key: a developer running Cypress locally has none and
     // does not need telling on every run.
     if (!apiKey)
@@ -119,6 +123,7 @@ Object.defineProperty(exports, "readHistory", { enumerable: true, get: function
 Object.defineProperty(exports, "recordAndReport", { enumerable: true, get: function () { return local_2.recordAndReport; } });
 Object.defineProperty(exports, "renderMarkdown", { enumerable: true, get: function () { return local_2.renderMarkdown; } });
 Object.defineProperty(exports, "renderSummary", { enumerable: true, get: function () { return local_2.renderSummary; } });
+Object.defineProperty(exports, "renderAnnotations", { enumerable: true, get: function () { return local_2.renderAnnotations; } });
 Object.defineProperty(exports, "scoreHistory", { enumerable: true, get: function () { return local_2.scoreHistory; } });
 Object.defineProperty(exports, "HISTORY_DIR", { enumerable: true, get: function () { return local_2.HISTORY_DIR; } });
 Object.defineProperty(exports, "HISTORY_FILE", { enumerable: true, get: function () { return local_2.HISTORY_FILE; } });

package/dist/local.d.ts

@@ -20,6 +20,8 @@ export type Outcome = "expected" | "unexpected" | "flaky" | "skipped";
 export interface StoredTest {
     title: string;
     file: string | null;
+    /** Where the test is declared, so a GitHub annotation can point at it. */
+    line?: number | null;
     outcome: Outcome;
 }
 export interface StoredRun {
@@ -32,6 +34,7 @@ export interface LocalHistory {
 }
 export declare const HISTORY_DIR = ".fixsense";
 export declare const HISTORY_FILE = "history.json";
+export declare const SITE_URL = "https://fix-sense.com";
 /**
  * How many runs are kept.
  *
@@ -43,6 +46,10 @@ export declare const MAX_RUNS = 50;
 export declare const MIN_RUNS_FOR_VERDICT = 5;
 /** At or above this score a test is called flaky. */
 export declare const FLAKY_SCORE_THRESHOLD = 70;
+/** Runs the runner itself called flaky before that alone is a verdict. */
+export declare const MIN_FLAKY_RUNS = 2;
+/** And how often, so a long-settled test is not judged on old history. */
+export declare const MIN_FLAKY_RATE = 0.2;
 export type Verdict = "stable" | "broken" | "flaky" | "suspect";
 export interface FlakinessReport {
     score: number;
@@ -81,6 +88,8 @@ export declare function writeHistory(cwd: string, history: LocalHistory): void;
 export interface TestVerdict extends FlakinessReport {
     title: string;
     file: string | null;
+    /** The line this test is declared on, when the runner reports it. */
+    line: number | null;
     /** How many of the stored runs this test failed or needed a retry in. */
     badRuns: number;
 }
@@ -93,9 +102,9 @@ export declare function scoreHistory(history: LocalHistory): TestVerdict[];
  * so this stays empty unless the history actually found something. "suspect"
  * is deliberately not reported: it means the tool is not sure yet.
  */
-export declare function renderSummary(verdicts: TestVerdict[], totalRuns: number): string;
+export declare function renderSummary(verdicts: TestVerdict[], totalRuns: number, connected?: boolean): string;
 /** The same summary as GitHub-flavoured Markdown, for the run's own page. */
-export declare function renderMarkdown(verdicts: TestVerdict[], totalRuns: number): string;
+export declare function renderMarkdown(verdicts: TestVerdict[], totalRuns: number, connected?: boolean): string;
 /**
  * Write the Markdown where GitHub Actions renders it on the run's page.
  *
@@ -103,9 +112,59 @@ export declare function renderMarkdown(verdicts: TestVerdict[], totalRuns: numbe
  * page it is the first thing a reviewer sees.
  */
 export declare function writeStepSummary(markdown: string): void;
+/**
+ * GitHub workflow annotations for the tests that earned a verdict.
+ *
+ * This is the one thing a single-run flaky reporter does that we did not:
+ * `::warning file=…,line=…::` makes GitHub draw the finding inline on the exact
+ * line in the pull request's "Files changed" tab, not only in the job log.
+ * Ours carries what a single run cannot — how many of the last N runs the test
+ * misbehaved in — so the annotation is a measurement, not a one-off retry.
+ *
+ * A `flaky` verdict is a warning: the test is noise to quarantine. A `broken`
+ * verdict — fails every run — is an error: a real failure hiding behind a
+ * retry, and the louder level is the point. "suspect" is never annotated, for
+ * the same reason it is never summarised: the tool is not sure yet.
+ */
+export declare function renderAnnotations(verdicts: TestVerdict[], totalRuns: number): string[];
+/** True when we are inside a GitHub Actions job, where annotations render. */
+export declare function isGitHubActions(env: Record<string, string | undefined>): boolean;
+/**
+ * Emit the annotations as workflow commands on stdout, where the runner reads
+ * them. Each must be alone on its line, so this writes them directly rather
+ * than folding them into the human summary.
+ */
+export declare function writeAnnotations(lines: string[]): void;
+/**
+ * One entry per test per run, whatever the runner reported.
+ *
+ * A run that listed the same test twice would count as two runs for it, and
+ * a retried test would read as several times as flaky as it is. The last
+ * entry wins: for a runner that reports each attempt, it is the one that
+ * knows how the test finally ended.
+ */
+export declare function dedupe(tests: StoredTest[]): StoredTest[];
+/** Every supported CI sets CI=true, except Azure Pipelines, which sets TF_BUILD. */
+export declare function isCi(env: Record<string, string | undefined>): boolean;
+/**
+ * Said when a CI job finds no earlier runs to compare with.
+ *
+ * A CI job starts from a clean checkout, so without a cache the history holds
+ * one run every time, never reaches a verdict, and the package stays silent
+ * forever — which reads as "it does nothing". One line saying why is the
+ * difference between a team fixing that and a team uninstalling.
+ */
+export declare function renderNoHistoryHint(docsUrl: string): string;
+export interface ReportContext {
+    /** An API key is set, so the history is also kept by the service. */
+    connected?: boolean;
+    /** Where the package's own docs explain keeping the history in CI. */
+    docsUrl?: string;
+    env?: Record<string, string | undefined>;
+}
 /**
  * Record this run and report what the history now says.
  *
  * Returns the text that was printed, which is what the tests assert on.
  */
-export declare function recordAndReport(cwd: string, tests: StoredTest[], now?: Date): string;
+export declare function recordAndReport(cwd: string, tests: StoredTest[], now?: Date, context?: ReportContext): string;

package/dist/local.js

@@ -18,7 +18,7 @@
  * asserts the two copies are the same file, so they cannot drift.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.FLAKY_SCORE_THRESHOLD = exports.MIN_RUNS_FOR_VERDICT = exports.MAX_RUNS = exports.HISTORY_FILE = exports.HISTORY_DIR = void 0;
+exports.MIN_FLAKY_RATE = exports.MIN_FLAKY_RUNS = exports.FLAKY_SCORE_THRESHOLD = exports.MIN_RUNS_FOR_VERDICT = exports.MAX_RUNS = exports.SITE_URL = exports.HISTORY_FILE = exports.HISTORY_DIR = void 0;
 exports.countTransitions = countTransitions;
 exports.computeFlakiness = computeFlakiness;
 exports.historyPath = historyPath;
@@ -30,11 +30,18 @@ exports.scoreHistory = scoreHistory;
 exports.renderSummary = renderSummary;
 exports.renderMarkdown = renderMarkdown;
 exports.writeStepSummary = writeStepSummary;
+exports.renderAnnotations = renderAnnotations;
+exports.isGitHubActions = isGitHubActions;
+exports.writeAnnotations = writeAnnotations;
+exports.dedupe = dedupe;
+exports.isCi = isCi;
+exports.renderNoHistoryHint = renderNoHistoryHint;
 exports.recordAndReport = recordAndReport;
 const fs_1 = require("fs");
 const path_1 = require("path");
 exports.HISTORY_DIR = ".fixsense";
 exports.HISTORY_FILE = "history.json";
+exports.SITE_URL = "https://fix-sense.com";
 /**
  * How many runs are kept.
  *
@@ -46,6 +53,10 @@ exports.MAX_RUNS = 50;
 exports.MIN_RUNS_FOR_VERDICT = 5;
 /** At or above this score a test is called flaky. */
 exports.FLAKY_SCORE_THRESHOLD = 70;
+/** Runs the runner itself called flaky before that alone is a verdict. */
+exports.MIN_FLAKY_RUNS = 2;
+/** And how often, so a long-settled test is not judged on old history. */
+exports.MIN_FLAKY_RATE = 0.2;
 function toPassFail(outcome) {
     if (outcome === "skipped")
         return null;
@@ -88,12 +99,19 @@ function computeFlakiness(history) {
     const selfReportedFlakyRate = selfReportedFlaky / history.length;
     const transitions = countTransitions(sequence);
     const transitionRate = runs > 1 ? transitions / (runs - 1) : 0;
+    // The score says how bad; this says whether we are sure. A run the runner
+    // itself called flaky is direct evidence — the test failed and then passed
+    // inside one run, which nothing but flakiness explains. A test that did
+    // that in a fifth of its runs is flaky even though the score, weighted to
+    // rank the worst offenders first, leaves it below the threshold. Two such
+    // runs are required so a single retry on a bad afternoon is not a verdict.
+    const isDirectlyEvidencedFlaky = selfReportedFlaky >= exports.MIN_FLAKY_RUNS && selfReportedFlakyRate >= exports.MIN_FLAKY_RATE;
     const score = Math.round(Math.min(100, selfReportedFlakyRate * 100 * 0.75 + transitionRate * 100 * 0.25));
     let verdict;
     if (runs < exports.MIN_RUNS_FOR_VERDICT) {
         verdict = score > 0 ? "suspect" : "stable";
     }
-    else if (score >= exports.FLAKY_SCORE_THRESHOLD) {
+    else if (score >= exports.FLAKY_SCORE_THRESHOLD || isDirectlyEvidencedFlaky) {
         verdict = "flaky";
     }
     else if (failureRate >= 0.9 && transitionRate <= 0.1 && selfReportedFlakyRate <= 0.1) {
@@ -179,7 +197,11 @@ function scoreHistory(history) {
     for (const run of history.runs) {
         for (const test of run.tests ?? []) {
             const key = `${test.file ?? ""}\u0000${test.title}`;
-            const entry = byTest.get(key) ?? { file: test.file ?? null, outcomes: [] };
+            const entry = byTest.get(key) ?? { file: test.file ?? null, line: test.line ?? null, outcomes: [] };
+            // The newest run wins on location: a test that moved is reported where
+            // it now lives, not where it first failed.
+            if (test.line != null)
+                entry.line = test.line;
             entry.outcomes.push(test.outcome);
             byTest.set(key, entry);
         }
@@ -191,6 +213,7 @@ function scoreHistory(history) {
             ...report,
             title: key.split("\u0000")[1] ?? "",
             file: entry.file,
+            line: entry.line,
             badRuns: entry.outcomes.filter((o) => o === "flaky" || o === "unexpected").length,
         });
     }
@@ -203,7 +226,7 @@ function scoreHistory(history) {
  * so this stays empty unless the history actually found something. "suspect"
  * is deliberately not reported: it means the tool is not sure yet.
  */
-function renderSummary(verdicts, totalRuns) {
+function renderSummary(verdicts, totalRuns, connected = false) {
     const flaky = verdicts.filter((v) => v.verdict === "flaky");
     const broken = verdicts.filter((v) => v.verdict === "broken");
     if (flaky.length === 0 && broken.length === 0)
@@ -221,10 +244,13 @@ function renderSummary(verdicts, totalRuns) {
             lines.push(`  ${v.title} — failed in ${v.badRuns} of ${v.runs} runs`);
         }
     }
+    if (!connected) {
+        lines.push(`Why they fail, and the fix as a pull request: ${exports.SITE_URL}`);
+    }
     return lines.join("\n");
 }
 /** The same summary as GitHub-flavoured Markdown, for the run's own page. */
-function renderMarkdown(verdicts, totalRuns) {
+function renderMarkdown(verdicts, totalRuns, connected = false) {
     const interesting = verdicts.filter((v) => v.verdict === "flaky" || v.verdict === "broken");
     if (interesting.length === 0)
         return "";
@@ -238,6 +264,14 @@ function renderMarkdown(verdicts, totalRuns) {
         "| --- | --- | --- | --- |",
         rows,
         "",
+        // The run page is read by the whole team, not only whoever installed the
+        // package, so this is where the rest of them learn where the table came from.
+        ...(connected
+            ? []
+            : [
+                `<sub>Recorded by [FixSense](${exports.SITE_URL}?utm_source=step-summary) — it also says whether the test or the application is at fault, and opens the fix.</sub>`,
+                "",
+            ]),
     ].join("\n");
 }
 /**
@@ -257,22 +291,127 @@ function writeStepSummary(markdown) {
         // The step summary is a nicety; losing it is not worth a word.
     }
 }
+/** A `::warning`/`::error` workflow command cannot carry a raw newline or `::`. */
+function escapeAnnotation(message) {
+    return message.replace(/%/g, "%25").replace(/\r/g, "%0D").replace(/\n/g, "%0A");
+}
+/**
+ * GitHub workflow annotations for the tests that earned a verdict.
+ *
+ * This is the one thing a single-run flaky reporter does that we did not:
+ * `::warning file=…,line=…::` makes GitHub draw the finding inline on the exact
+ * line in the pull request's "Files changed" tab, not only in the job log.
+ * Ours carries what a single run cannot — how many of the last N runs the test
+ * misbehaved in — so the annotation is a measurement, not a one-off retry.
+ *
+ * A `flaky` verdict is a warning: the test is noise to quarantine. A `broken`
+ * verdict — fails every run — is an error: a real failure hiding behind a
+ * retry, and the louder level is the point. "suspect" is never annotated, for
+ * the same reason it is never summarised: the tool is not sure yet.
+ */
+function renderAnnotations(verdicts, totalRuns) {
+    const lines = [];
+    for (const v of verdicts) {
+        if (v.verdict !== "flaky" && v.verdict !== "broken")
+            continue;
+        const level = v.verdict === "broken" ? "error" : "warning";
+        const what = v.verdict === "broken"
+            ? `fails every run — a real failure, not flakiness`
+            : `flaky in ${v.badRuns} of ${v.runs} runs (score ${v.score})`;
+        const props = [
+            v.file ? `file=${v.file}` : null,
+            v.line != null ? `line=${v.line}` : null,
+            `title=FixSense: ${escapeAnnotation(v.verdict)} test`,
+        ]
+            .filter(Boolean)
+            .join(",");
+        lines.push(`::${level} ${props}::${escapeAnnotation(`${v.title} — ${what}, across the last ${totalRuns} runs`)}`);
+    }
+    return lines;
+}
+/** True when we are inside a GitHub Actions job, where annotations render. */
+function isGitHubActions(env) {
+    return env.GITHUB_ACTIONS === "true";
+}
+/**
+ * Emit the annotations as workflow commands on stdout, where the runner reads
+ * them. Each must be alone on its line, so this writes them directly rather
+ * than folding them into the human summary.
+ */
+function writeAnnotations(lines) {
+    for (const line of lines)
+        console.log(line);
+}
+/**
+ * One entry per test per run, whatever the runner reported.
+ *
+ * A run that listed the same test twice would count as two runs for it, and
+ * a retried test would read as several times as flaky as it is. The last
+ * entry wins: for a runner that reports each attempt, it is the one that
+ * knows how the test finally ended.
+ */
+function dedupe(tests) {
+    const byKey = new Map();
+    for (const t of tests) {
+        byKey.set(`${t.file ?? ""}\u0000${t.title}`, {
+            title: t.title,
+            file: t.file,
+            line: t.line ?? null,
+            outcome: t.outcome,
+        });
+    }
+    return [...byKey.values()];
+}
+/** Every supported CI sets CI=true, except Azure Pipelines, which sets TF_BUILD. */
+function isCi(env) {
+    return Boolean(env.CI && env.CI !== "false") || env.TF_BUILD === "True";
+}
+/**
+ * Said when a CI job finds no earlier runs to compare with.
+ *
+ * A CI job starts from a clean checkout, so without a cache the history holds
+ * one run every time, never reaches a verdict, and the package stays silent
+ * forever — which reads as "it does nothing". One line saying why is the
+ * difference between a team fixing that and a team uninstalling.
+ */
+function renderNoHistoryHint(docsUrl) {
+    return [
+        "FixSense: recorded this run, but found no earlier runs to compare it with.",
+        "  CI starts every job clean, so the history is lost between runs.",
+        `  Cache .fixsense/ between runs, or set FIXSENSE_API_KEY: ${docsUrl}`,
+    ].join("\n");
+}
 /**
  * Record this run and report what the history now says.
  *
  * Returns the text that was printed, which is what the tests assert on.
  */
-function recordAndReport(cwd, tests, now = new Date()) {
+function recordAndReport(cwd, tests, now = new Date(), context = {}) {
     const history = appendRun(readHistory(cwd), {
         at: now.toISOString(),
-        tests: tests.map((t) => ({ title: t.title, file: t.file, outcome: t.outcome })),
+        tests: dedupe(tests),
     });
     writeHistory(cwd, history);
+    const connected = context.connected === true;
+    if (history.runs.length === 1 &&
+        !connected &&
+        context.docsUrl &&
+        isCi(context.env ?? process.env)) {
+        const hint = renderNoHistoryHint(context.docsUrl);
+        console.log(`\n${hint}\n`);
+        return hint;
+    }
     const verdicts = scoreHistory(history);
-    const summary = renderSummary(verdicts, history.runs.length);
+    const summary = renderSummary(verdicts, history.runs.length, connected);
     if (summary) {
         console.log(`\n${summary}\n`);
-        writeStepSummary(renderMarkdown(verdicts, history.runs.length));
+        writeStepSummary(renderMarkdown(verdicts, history.runs.length, connected));
+        // On GitHub, also pin each verdict to its line in the PR. The summary and
+        // the annotations carry the same findings to two different places a
+        // reviewer looks: the run page, and the diff.
+        if (isGitHubActions(context.env ?? process.env)) {
+            writeAnnotations(renderAnnotations(verdicts, history.runs.length));
+        }
     }
     return summary;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fixsense/cypress-plugin",
-  "version": "0.2.0",
+  "version": "0.2.4",
   "description": "Cypress plugin that records test outcomes across runs so flaky tests can be identified over time.",
   "license": "MIT",
   "main": "./dist/index.js",
@@ -13,7 +13,14 @@
   "keywords": [
     "cypress",
     "cypress-plugin",
+    "flaky",
+    "flaky-test",
     "flaky-tests",
+    "flaky-test-detection",
+    "test-history",
+    "retry",
+    "quarantine",
+    "e2e",
     "ci",
     "test-reporting"
   ],
@@ -22,7 +29,7 @@
     "url": "https://github.com/CopilotMe/fix-sense.git",
     "directory": "packages/cypress-plugin"
   },
-  "homepage": "https://fix-sense.com",
+  "homepage": "https://fix-sense.com/docs/integrations/cypress-plugin",
   "scripts": {
     "build": "tsc -p tsconfig.json",
     "prepublishOnly": "npm run build"