@fixsense/cypress-plugin 0.2.0 → 0.2.1

package/dist/local.d.ts

@@ -43,6 +43,10 @@ export declare const MAX_RUNS = 50;
 export declare const MIN_RUNS_FOR_VERDICT = 5;
 /** At or above this score a test is called flaky. */
 export declare const FLAKY_SCORE_THRESHOLD = 70;
+/** Runs the runner itself called flaky before that alone is a verdict. */
+export declare const MIN_FLAKY_RUNS = 2;
+/** And how often, so a long-settled test is not judged on old history. */
+export declare const MIN_FLAKY_RATE = 0.2;
 export type Verdict = "stable" | "broken" | "flaky" | "suspect";
 export interface FlakinessReport {
     score: number;
@@ -103,6 +107,15 @@ export declare function renderMarkdown(verdicts: TestVerdict[], totalRuns: numbe
  * page it is the first thing a reviewer sees.
  */
 export declare function writeStepSummary(markdown: string): void;
+/**
+ * One entry per test per run, whatever the runner reported.
+ *
+ * A run that listed the same test twice would count as two runs for it, and
+ * a retried test would read as several times as flaky as it is. The last
+ * entry wins: for a runner that reports each attempt, it is the one that
+ * knows how the test finally ended.
+ */
+export declare function dedupe(tests: StoredTest[]): StoredTest[];
 /**
  * Record this run and report what the history now says.
  *

package/dist/local.js

@@ -18,7 +18,7 @@
  * asserts the two copies are the same file, so they cannot drift.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.FLAKY_SCORE_THRESHOLD = exports.MIN_RUNS_FOR_VERDICT = exports.MAX_RUNS = exports.HISTORY_FILE = exports.HISTORY_DIR = void 0;
+exports.MIN_FLAKY_RATE = exports.MIN_FLAKY_RUNS = exports.FLAKY_SCORE_THRESHOLD = exports.MIN_RUNS_FOR_VERDICT = exports.MAX_RUNS = exports.HISTORY_FILE = exports.HISTORY_DIR = void 0;
 exports.countTransitions = countTransitions;
 exports.computeFlakiness = computeFlakiness;
 exports.historyPath = historyPath;
@@ -30,6 +30,7 @@ exports.scoreHistory = scoreHistory;
 exports.renderSummary = renderSummary;
 exports.renderMarkdown = renderMarkdown;
 exports.writeStepSummary = writeStepSummary;
+exports.dedupe = dedupe;
 exports.recordAndReport = recordAndReport;
 const fs_1 = require("fs");
 const path_1 = require("path");
@@ -46,6 +47,10 @@ exports.MAX_RUNS = 50;
 exports.MIN_RUNS_FOR_VERDICT = 5;
 /** At or above this score a test is called flaky. */
 exports.FLAKY_SCORE_THRESHOLD = 70;
+/** Runs the runner itself called flaky before that alone is a verdict. */
+exports.MIN_FLAKY_RUNS = 2;
+/** And how often, so a long-settled test is not judged on old history. */
+exports.MIN_FLAKY_RATE = 0.2;
 function toPassFail(outcome) {
     if (outcome === "skipped")
         return null;
@@ -88,12 +93,19 @@ function computeFlakiness(history) {
     const selfReportedFlakyRate = selfReportedFlaky / history.length;
     const transitions = countTransitions(sequence);
     const transitionRate = runs > 1 ? transitions / (runs - 1) : 0;
+    // The score says how bad; this says whether we are sure. A run the runner
+    // itself called flaky is direct evidence — the test failed and then passed
+    // inside one run, which nothing but flakiness explains. A test that did
+    // that in a fifth of its runs is flaky even though the score, weighted to
+    // rank the worst offenders first, leaves it below the threshold. Two such
+    // runs are required so a single retry on a bad afternoon is not a verdict.
+    const isDirectlyEvidencedFlaky = selfReportedFlaky >= exports.MIN_FLAKY_RUNS && selfReportedFlakyRate >= exports.MIN_FLAKY_RATE;
     const score = Math.round(Math.min(100, selfReportedFlakyRate * 100 * 0.75 + transitionRate * 100 * 0.25));
     let verdict;
     if (runs < exports.MIN_RUNS_FOR_VERDICT) {
         verdict = score > 0 ? "suspect" : "stable";
     }
-    else if (score >= exports.FLAKY_SCORE_THRESHOLD) {
+    else if (score >= exports.FLAKY_SCORE_THRESHOLD || isDirectlyEvidencedFlaky) {
         verdict = "flaky";
     }
     else if (failureRate >= 0.9 && transitionRate <= 0.1 && selfReportedFlakyRate <= 0.1) {
@@ -257,6 +269,25 @@ function writeStepSummary(markdown) {
         // The step summary is a nicety; losing it is not worth a word.
     }
 }
+/**
+ * One entry per test per run, whatever the runner reported.
+ *
+ * A run that listed the same test twice would count as two runs for it, and
+ * a retried test would read as several times as flaky as it is. The last
+ * entry wins: for a runner that reports each attempt, it is the one that
+ * knows how the test finally ended.
+ */
+function dedupe(tests) {
+    const byKey = new Map();
+    for (const t of tests) {
+        byKey.set(`${t.file ?? ""}\u0000${t.title}`, {
+            title: t.title,
+            file: t.file,
+            outcome: t.outcome,
+        });
+    }
+    return [...byKey.values()];
+}
 /**
  * Record this run and report what the history now says.
  *
@@ -265,7 +296,7 @@ function writeStepSummary(markdown) {
 function recordAndReport(cwd, tests, now = new Date()) {
     const history = appendRun(readHistory(cwd), {
         at: now.toISOString(),
-        tests: tests.map((t) => ({ title: t.title, file: t.file, outcome: t.outcome })),
+        tests: dedupe(tests),
     });
     writeHistory(cwd, history);
     const verdicts = scoreHistory(history);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fixsense/cypress-plugin",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "Cypress plugin that records test outcomes across runs so flaky tests can be identified over time.",
   "license": "MIT",
   "main": "./dist/index.js",