@fixsense/cypress-plugin 0.1.0 → 0.2.1

package/README.md

@@ -32,8 +32,39 @@ export default defineConfig({
 });
 ```
 
-Set `FIXSENSE_API_KEY` in CI. Without it the plugin does nothing, so local
-runs are unaffected.
+That is the whole setup. No account, no key.
+
+## What you get, with no account
+
+After each run the plugin appends what every test did to
+`.fixsense/history.json` in your project, and once it has seen enough runs it
+says what the history found:
+
+```
+FixSense: 2 tests behave flakily across the last 12 runs
+  checkout › pays with a card — failed or retried in 4 of 12 runs
+  login › remembers the session — failed or retried in 3 of 12 runs
+```
+
+A test that fails *every* run is reported separately, as a bug rather than
+flakiness — quarantining that one is how a real failure reaches production.
+
+Nothing is printed until there are at least five runs and something is
+actually wrong, so a green suite stays quiet.
+
+On GitHub Actions the same table is written to the job summary, so it appears
+on the run's own page instead of scrolling past in the log.
+
+`.fixsense/` is git-ignored for you: committing it would mean a diff on every
+run and merge conflicts in a file nobody reads.
+
+### Why an account, then
+
+The local history lives on one machine and dies with the CI container. The
+hosted service keeps it across machines, branches and runs, says whether the
+test or the application is at fault, and opens the fix as a pull request. Set
+`FIXSENSE_API_KEY` in CI and the same runs are recorded there too; without it
+nothing leaves your machine.
 
 ## Options
 
@@ -44,13 +75,15 @@ registerFixSense(on, {
   enabled: true,
   timeoutMs: 10000,
   repoFullName: 'acme/web',             // when CI variables are absent
+  local: true,                          // keep .fixsense/history.json
+  cwd: process.cwd(),                   // where that history lives
 });
 ```
 
 GitHub Actions, GitLab CI, Bitbucket Pipelines and Azure Pipelines are
 detected automatically.
 
-## What it sends
+## What it sends to FixSense (only with a key)
 
 Per test: the describe/it title chain, the spec file, the outcome, the retry
 count, the duration, and the error message truncated to 2000 characters.

package/dist/index.d.ts

@@ -18,6 +18,15 @@ export interface FixSenseCypressOptions {
     timeoutMs?: number;
     /** Override repo detection when CI variables are absent or wrong. */
     repoFullName?: string;
+    /**
+     * Set false to stop keeping the local run history in `.fixsense/`.
+     *
+     * The history is what makes the plugin useful without an account, so it is
+     * on by default; uploading to FixSense remains opt-in via the API key.
+     */
+    local?: boolean;
+    /** Where the local history lives. Defaults to the process's directory. */
+    cwd?: string;
 }
 /** The subset of Cypress's plugin event registrar we use. */
 type On = (event: string, handler: (...args: never[]) => void | Promise<void>) => void;
@@ -31,3 +40,4 @@ export declare function registerFixSense(on: On, options?: FixSenseCypressOption
 export declare function sendRun(tests: TestRecord[], startedAt: Date, options: FixSenseCypressOptions): Promise<void>;
 export default registerFixSense;
 export * from "./payload";
+export { computeFlakiness, readHistory, recordAndReport, renderMarkdown, renderSummary, scoreHistory, HISTORY_DIR, HISTORY_FILE, MAX_RUNS, } from "./local";

package/dist/index.js

@@ -23,9 +23,11 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.MAX_RUNS = exports.HISTORY_FILE = exports.HISTORY_DIR = exports.scoreHistory = exports.renderSummary = exports.renderMarkdown = exports.recordAndReport = exports.readHistory = exports.computeFlakiness = void 0;
 exports.registerFixSense = registerFixSense;
 exports.sendRun = sendRun;
 const payload_1 = require("./payload");
+const local_1 = require("./local");
 const DEFAULT_API_URL = "https://fix-sense.com";
 const DEFAULT_TIMEOUT_MS = 10_000;
 /**
@@ -54,13 +56,24 @@ function registerFixSense(on, options = {}) {
 async function sendRun(tests, startedAt, options) {
     if (options.enabled === false)
         return;
+    if (tests.length === 0)
+        return;
+    // The local history first, because it needs no account and is the only
+    // part most first-time users will ever see.
+    if (options.local !== false) {
+        try {
+            (0, local_1.recordAndReport)(options.cwd ?? process.cwd(), tests);
+        }
+        catch (error) {
+            const reason = error instanceof Error ? error.message : String(error);
+            console.error(`[fixsense] Local history not updated: ${reason}`);
+        }
+    }
     const apiKey = options.apiKey ?? process.env.FIXSENSE_API_KEY;
     // Silent without a key: a developer running Cypress locally has none and
     // does not need telling on every run.
     if (!apiKey)
         return;
-    if (tests.length === 0)
-        return;
     const context = (0, payload_1.detectContext)(process.env);
     const payload = {
         schemaVersion: 1,
@@ -98,3 +111,15 @@ async function sendRun(tests, startedAt, options) {
 }
 exports.default = registerFixSense;
 __exportStar(require("./payload"), exports);
+// Named rather than a star: both modules describe an outcome, and one
+// ambiguous export is worse than nine explicit ones.
+var local_2 = require("./local");
+Object.defineProperty(exports, "computeFlakiness", { enumerable: true, get: function () { return local_2.computeFlakiness; } });
+Object.defineProperty(exports, "readHistory", { enumerable: true, get: function () { return local_2.readHistory; } });
+Object.defineProperty(exports, "recordAndReport", { enumerable: true, get: function () { return local_2.recordAndReport; } });
+Object.defineProperty(exports, "renderMarkdown", { enumerable: true, get: function () { return local_2.renderMarkdown; } });
+Object.defineProperty(exports, "renderSummary", { enumerable: true, get: function () { return local_2.renderSummary; } });
+Object.defineProperty(exports, "scoreHistory", { enumerable: true, get: function () { return local_2.scoreHistory; } });
+Object.defineProperty(exports, "HISTORY_DIR", { enumerable: true, get: function () { return local_2.HISTORY_DIR; } });
+Object.defineProperty(exports, "HISTORY_FILE", { enumerable: true, get: function () { return local_2.HISTORY_FILE; } });
+Object.defineProperty(exports, "MAX_RUNS", { enumerable: true, get: function () { return local_2.MAX_RUNS; } });

package/dist/local.d.ts

@@ -0,0 +1,124 @@
+/**
+ * The local run history: what the package does for someone with no account.
+ *
+ * A test runner is stateless between runs, so "this test has failed three
+ * times this week" is a sentence neither Playwright nor Cypress can say. That
+ * does not require a server — only a file. This module keeps one in the
+ * project being tested, scores it, and prints what it found.
+ *
+ * The hosted product is what this cannot be: history that survives the CI
+ * container, is shared across machines and branches, says whether the test or
+ * the application is at fault, and opens the fix. Local mode makes the
+ * problem visible; the account solves it.
+ *
+ * This file is byte-identical in @fixsense/playwright-reporter and
+ * @fixsense/cypress-plugin — each package installs standalone with no FixSense
+ * dependency, which is the whole reason a team adds one. The parity test
+ * asserts the two copies are the same file, so they cannot drift.
+ */
+export type Outcome = "expected" | "unexpected" | "flaky" | "skipped";
+export interface StoredTest {
+    title: string;
+    file: string | null;
+    outcome: Outcome;
+}
+export interface StoredRun {
+    at: string;
+    tests: StoredTest[];
+}
+export interface LocalHistory {
+    version: 1;
+    runs: StoredRun[];
+}
+export declare const HISTORY_DIR = ".fixsense";
+export declare const HISTORY_FILE = "history.json";
+/**
+ * How many runs are kept.
+ *
+ * Enough that a weekly pattern is visible, few enough that the file stays
+ * small in a repository someone did not ask to carry a database.
+ */
+export declare const MAX_RUNS = 50;
+/** Below this, any verdict is guesswork — the same floor the service uses. */
+export declare const MIN_RUNS_FOR_VERDICT = 5;
+/** At or above this score a test is called flaky. */
+export declare const FLAKY_SCORE_THRESHOLD = 70;
+/** Runs the runner itself called flaky before that alone is a verdict. */
+export declare const MIN_FLAKY_RUNS = 2;
+/** And how often, so a long-settled test is not judged on old history. */
+export declare const MIN_FLAKY_RATE = 0.2;
+export type Verdict = "stable" | "broken" | "flaky" | "suspect";
+export interface FlakinessReport {
+    score: number;
+    verdict: Verdict;
+    runs: number;
+    failureRate: number;
+    selfReportedFlakyRate: number;
+    transitionRate: number;
+}
+export declare function countTransitions(outcomes: Array<"pass" | "fail">): number;
+/**
+ * The same scoring the hosted service applies, so a team sees one answer
+ * whether they read it here or in the dashboard.
+ */
+export declare function computeFlakiness(history: Array<{
+    outcome: Outcome;
+}>): FlakinessReport;
+export declare function historyPath(cwd: string): string;
+/**
+ * A history file that cannot be read is treated as no history.
+ *
+ * It is a cache of past runs, not data anyone would mourn, and refusing to
+ * run because of it would break the first rule: never fail someone's build.
+ */
+export declare function readHistory(cwd: string): LocalHistory;
+export declare function appendRun(history: LocalHistory, run: StoredRun): LocalHistory;
+/**
+ * Keep the directory out of git.
+ *
+ * Committing it would mean every developer's every run produces a diff, and
+ * a merge conflict in a file nobody reads. Only written when absent, so a
+ * team that decides otherwise is not overruled on the next run.
+ */
+export declare function ensureIgnored(cwd: string): void;
+export declare function writeHistory(cwd: string, history: LocalHistory): void;
+export interface TestVerdict extends FlakinessReport {
+    title: string;
+    file: string | null;
+    /** How many of the stored runs this test failed or needed a retry in. */
+    badRuns: number;
+}
+/** Group the stored runs by test and score each one. */
+export declare function scoreHistory(history: LocalHistory): TestVerdict[];
+/**
+ * The summary a reader sees, or nothing at all.
+ *
+ * A line printed after every green run is a line teams learn to scroll past,
+ * so this stays empty unless the history actually found something. "suspect"
+ * is deliberately not reported: it means the tool is not sure yet.
+ */
+export declare function renderSummary(verdicts: TestVerdict[], totalRuns: number): string;
+/** The same summary as GitHub-flavoured Markdown, for the run's own page. */
+export declare function renderMarkdown(verdicts: TestVerdict[], totalRuns: number): string;
+/**
+ * Write the Markdown where GitHub Actions renders it on the run's page.
+ *
+ * In the job log the summary scrolls away under the test output; on the run
+ * page it is the first thing a reviewer sees.
+ */
+export declare function writeStepSummary(markdown: string): void;
+/**
+ * One entry per test per run, whatever the runner reported.
+ *
+ * A run that listed the same test twice would count as two runs for it, and
+ * a retried test would read as several times as flaky as it is. The last
+ * entry wins: for a runner that reports each attempt, it is the one that
+ * knows how the test finally ended.
+ */
+export declare function dedupe(tests: StoredTest[]): StoredTest[];
+/**
+ * Record this run and report what the history now says.
+ *
+ * Returns the text that was printed, which is what the tests assert on.
+ */
+export declare function recordAndReport(cwd: string, tests: StoredTest[], now?: Date): string;

package/dist/local.js

@@ -0,0 +1,309 @@
+"use strict";
+/**
+ * The local run history: what the package does for someone with no account.
+ *
+ * A test runner is stateless between runs, so "this test has failed three
+ * times this week" is a sentence neither Playwright nor Cypress can say. That
+ * does not require a server — only a file. This module keeps one in the
+ * project being tested, scores it, and prints what it found.
+ *
+ * The hosted product is what this cannot be: history that survives the CI
+ * container, is shared across machines and branches, says whether the test or
+ * the application is at fault, and opens the fix. Local mode makes the
+ * problem visible; the account solves it.
+ *
+ * This file is byte-identical in @fixsense/playwright-reporter and
+ * @fixsense/cypress-plugin — each package installs standalone with no FixSense
+ * dependency, which is the whole reason a team adds one. The parity test
+ * asserts the two copies are the same file, so they cannot drift.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MIN_FLAKY_RATE = exports.MIN_FLAKY_RUNS = exports.FLAKY_SCORE_THRESHOLD = exports.MIN_RUNS_FOR_VERDICT = exports.MAX_RUNS = exports.HISTORY_FILE = exports.HISTORY_DIR = void 0;
+exports.countTransitions = countTransitions;
+exports.computeFlakiness = computeFlakiness;
+exports.historyPath = historyPath;
+exports.readHistory = readHistory;
+exports.appendRun = appendRun;
+exports.ensureIgnored = ensureIgnored;
+exports.writeHistory = writeHistory;
+exports.scoreHistory = scoreHistory;
+exports.renderSummary = renderSummary;
+exports.renderMarkdown = renderMarkdown;
+exports.writeStepSummary = writeStepSummary;
+exports.dedupe = dedupe;
+exports.recordAndReport = recordAndReport;
+const fs_1 = require("fs");
+const path_1 = require("path");
+exports.HISTORY_DIR = ".fixsense";
+exports.HISTORY_FILE = "history.json";
+/**
+ * How many runs are kept.
+ *
+ * Enough that a weekly pattern is visible, few enough that the file stays
+ * small in a repository someone did not ask to carry a database.
+ */
+exports.MAX_RUNS = 50;
+/** Below this, any verdict is guesswork — the same floor the service uses. */
+exports.MIN_RUNS_FOR_VERDICT = 5;
+/** At or above this score a test is called flaky. */
+exports.FLAKY_SCORE_THRESHOLD = 70;
+/** Runs the runner itself called flaky before that alone is a verdict. */
+exports.MIN_FLAKY_RUNS = 2;
+/** And how often, so a long-settled test is not judged on old history. */
+exports.MIN_FLAKY_RATE = 0.2;
+function toPassFail(outcome) {
+    if (outcome === "skipped")
+        return null;
+    // A run the runner called flaky ended green, but it is not a pass: it
+    // needed a retry to get there, which is the behaviour being measured.
+    if (outcome === "flaky")
+        return "fail";
+    return outcome === "expected" ? "pass" : "fail";
+}
+function countTransitions(outcomes) {
+    let transitions = 0;
+    for (let i = 1; i < outcomes.length; i++) {
+        if (outcomes[i] !== outcomes[i - 1])
+            transitions++;
+    }
+    return transitions;
+}
+/**
+ * The same scoring the hosted service applies, so a team sees one answer
+ * whether they read it here or in the dashboard.
+ */
+function computeFlakiness(history) {
+    const sequence = history
+        .map((h) => toPassFail(h.outcome))
+        .filter((o) => o !== null);
+    const runs = sequence.length;
+    if (runs === 0) {
+        return {
+            score: 0,
+            verdict: "stable",
+            runs: 0,
+            failureRate: 0,
+            selfReportedFlakyRate: 0,
+            transitionRate: 0,
+        };
+    }
+    const failures = sequence.filter((o) => o === "fail").length;
+    const failureRate = failures / runs;
+    const selfReportedFlaky = history.filter((h) => h.outcome === "flaky").length;
+    const selfReportedFlakyRate = selfReportedFlaky / history.length;
+    const transitions = countTransitions(sequence);
+    const transitionRate = runs > 1 ? transitions / (runs - 1) : 0;
+    // The score says how bad; this says whether we are sure. A run the runner
+    // itself called flaky is direct evidence — the test failed and then passed
+    // inside one run, which nothing but flakiness explains. A test that did
+    // that in a fifth of its runs is flaky even though the score, weighted to
+    // rank the worst offenders first, leaves it below the threshold. Two such
+    // runs are required so a single retry on a bad afternoon is not a verdict.
+    const isDirectlyEvidencedFlaky = selfReportedFlaky >= exports.MIN_FLAKY_RUNS && selfReportedFlakyRate >= exports.MIN_FLAKY_RATE;
+    const score = Math.round(Math.min(100, selfReportedFlakyRate * 100 * 0.75 + transitionRate * 100 * 0.25));
+    let verdict;
+    if (runs < exports.MIN_RUNS_FOR_VERDICT) {
+        verdict = score > 0 ? "suspect" : "stable";
+    }
+    else if (score >= exports.FLAKY_SCORE_THRESHOLD || isDirectlyEvidencedFlaky) {
+        verdict = "flaky";
+    }
+    else if (failureRate >= 0.9 && transitionRate <= 0.1 && selfReportedFlakyRate <= 0.1) {
+        // Fails nearly always and almost never changes its mind: a failing test,
+        // not a flaky one. Quarantining this is how a real bug reaches production.
+        verdict = "broken";
+    }
+    else if (score > 0) {
+        verdict = "suspect";
+    }
+    else {
+        verdict = "stable";
+    }
+    return {
+        score,
+        verdict,
+        runs,
+        failureRate: Number(failureRate.toFixed(3)),
+        selfReportedFlakyRate: Number(selfReportedFlakyRate.toFixed(3)),
+        transitionRate: Number(transitionRate.toFixed(3)),
+    };
+}
+function historyPath(cwd) {
+    return (0, path_1.join)(cwd, exports.HISTORY_DIR, exports.HISTORY_FILE);
+}
+/**
+ * A history file that cannot be read is treated as no history.
+ *
+ * It is a cache of past runs, not data anyone would mourn, and refusing to
+ * run because of it would break the first rule: never fail someone's build.
+ */
+function readHistory(cwd) {
+    try {
+        const raw = (0, fs_1.readFileSync)(historyPath(cwd), "utf8");
+        const parsed = JSON.parse(raw);
+        if (!parsed || !Array.isArray(parsed.runs))
+            return { version: 1, runs: [] };
+        return { version: 1, runs: parsed.runs };
+    }
+    catch {
+        return { version: 1, runs: [] };
+    }
+}
+function appendRun(history, run) {
+    // Oldest first, so the newest run is the last element and the window drops
+    // the oldest runs rather than the ones that matter.
+    return { version: 1, runs: [...history.runs, run].slice(-exports.MAX_RUNS) };
+}
+/**
+ * Keep the directory out of git.
+ *
+ * Committing it would mean every developer's every run produces a diff, and
+ * a merge conflict in a file nobody reads. Only written when absent, so a
+ * team that decides otherwise is not overruled on the next run.
+ */
+function ensureIgnored(cwd) {
+    try {
+        const ignorePath = (0, path_1.join)(cwd, exports.HISTORY_DIR, ".gitignore");
+        if ((0, fs_1.existsSync)(ignorePath))
+            return;
+        (0, fs_1.mkdirSync)((0, path_1.dirname)(ignorePath), { recursive: true });
+        (0, fs_1.writeFileSync)(ignorePath, "*\n", "utf8");
+    }
+    catch {
+        // Not being able to write the ignore file is not a reason to stop.
+    }
+}
+function writeHistory(cwd, history) {
+    try {
+        const path = historyPath(cwd);
+        (0, fs_1.mkdirSync)((0, path_1.dirname)(path), { recursive: true });
+        (0, fs_1.writeFileSync)(path, `${JSON.stringify(history)}\n`, "utf8");
+        ensureIgnored(cwd);
+    }
+    catch (error) {
+        const reason = error instanceof Error ? error.message : String(error);
+        console.error(`[fixsense] Local history not saved: ${reason}`);
+    }
+}
+/** Group the stored runs by test and score each one. */
+function scoreHistory(history) {
+    const byTest = new Map();
+    for (const run of history.runs) {
+        for (const test of run.tests ?? []) {
+            const key = `${test.file ?? ""}\u0000${test.title}`;
+            const entry = byTest.get(key) ?? { file: test.file ?? null, outcomes: [] };
+            entry.outcomes.push(test.outcome);
+            byTest.set(key, entry);
+        }
+    }
+    const verdicts = [];
+    for (const [key, entry] of byTest) {
+        const report = computeFlakiness(entry.outcomes.map((outcome) => ({ outcome })));
+        verdicts.push({
+            ...report,
+            title: key.split("\u0000")[1] ?? "",
+            file: entry.file,
+            badRuns: entry.outcomes.filter((o) => o === "flaky" || o === "unexpected").length,
+        });
+    }
+    return verdicts.sort((a, b) => b.score - a.score);
+}
+/**
+ * The summary a reader sees, or nothing at all.
+ *
+ * A line printed after every green run is a line teams learn to scroll past,
+ * so this stays empty unless the history actually found something. "suspect"
+ * is deliberately not reported: it means the tool is not sure yet.
+ */
+function renderSummary(verdicts, totalRuns) {
+    const flaky = verdicts.filter((v) => v.verdict === "flaky");
+    const broken = verdicts.filter((v) => v.verdict === "broken");
+    if (flaky.length === 0 && broken.length === 0)
+        return "";
+    const lines = [];
+    if (flaky.length > 0) {
+        lines.push(`FixSense: ${flaky.length} ${flaky.length === 1 ? "test behaves" : "tests behave"} flakily across the last ${totalRuns} runs`);
+        for (const v of flaky) {
+            lines.push(`  ${v.title} — failed or retried in ${v.badRuns} of ${v.runs} runs`);
+        }
+    }
+    if (broken.length > 0) {
+        lines.push(`FixSense: ${broken.length} ${broken.length === 1 ? "test fails" : "tests fail"} every run — a bug, not flakiness`);
+        for (const v of broken) {
+            lines.push(`  ${v.title} — failed in ${v.badRuns} of ${v.runs} runs`);
+        }
+    }
+    return lines.join("\n");
+}
+/** The same summary as GitHub-flavoured Markdown, for the run's own page. */
+function renderMarkdown(verdicts, totalRuns) {
+    const interesting = verdicts.filter((v) => v.verdict === "flaky" || v.verdict === "broken");
+    if (interesting.length === 0)
+        return "";
+    const rows = interesting
+        .map((v) => `| \`${v.title}\` | ${v.verdict} | ${v.badRuns} of ${v.runs} | ${v.score} |`)
+        .join("\n");
+    return [
+        `### FixSense — what these tests did over the last ${totalRuns} runs`,
+        "",
+        "| Test | Verdict | Bad runs | Score |",
+        "| --- | --- | --- | --- |",
+        rows,
+        "",
+    ].join("\n");
+}
+/**
+ * Write the Markdown where GitHub Actions renders it on the run's page.
+ *
+ * In the job log the summary scrolls away under the test output; on the run
+ * page it is the first thing a reviewer sees.
+ */
+function writeStepSummary(markdown) {
+    const path = process.env.GITHUB_STEP_SUMMARY;
+    if (!path || !markdown)
+        return;
+    try {
+        (0, fs_1.appendFileSync)(path, `${markdown}\n`, "utf8");
+    }
+    catch {
+        // The step summary is a nicety; losing it is not worth a word.
+    }
+}
+/**
+ * One entry per test per run, whatever the runner reported.
+ *
+ * A run that listed the same test twice would count as two runs for it, and
+ * a retried test would read as several times as flaky as it is. The last
+ * entry wins: for a runner that reports each attempt, it is the one that
+ * knows how the test finally ended.
+ */
+function dedupe(tests) {
+    const byKey = new Map();
+    for (const t of tests) {
+        byKey.set(`${t.file ?? ""}\u0000${t.title}`, {
+            title: t.title,
+            file: t.file,
+            outcome: t.outcome,
+        });
+    }
+    return [...byKey.values()];
+}
+/**
+ * Record this run and report what the history now says.
+ *
+ * Returns the text that was printed, which is what the tests assert on.
+ */
+function recordAndReport(cwd, tests, now = new Date()) {
+    const history = appendRun(readHistory(cwd), {
+        at: now.toISOString(),
+        tests: dedupe(tests),
+    });
+    writeHistory(cwd, history);
+    const verdicts = scoreHistory(history);
+    const summary = renderSummary(verdicts, history.runs.length);
+    if (summary) {
+        console.log(`\n${summary}\n`);
+        writeStepSummary(renderMarkdown(verdicts, history.runs.length));
+    }
+    return summary;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fixsense/cypress-plugin",
-  "version": "0.1.0",
+  "version": "0.2.1",
   "description": "Cypress plugin that records test outcomes across runs so flaky tests can be identified over time.",
   "license": "MIT",
   "main": "./dist/index.js",