@aplaytest/flaky 0.1.0

package/dist/index.js

@@ -0,0 +1,16 @@
+/**
+ * @aplaytest/flaky — statistical flake detection and root-cause classification.
+ *
+ * Entirely deterministic. Every number here is measured from recorded
+ * attempts; no model is called anywhere in this package. A model may later be
+ * asked to phrase a verdict, never to reach one — a flake score that cannot be
+ * reproduced is a flake score nobody will act on.
+ */
+export { scoreTest, isFlaky, usableAttempts, wilsonLowerBound, DEFAULT_SCORE_CONFIG, } from './score.js';
+export { extractFeatures, correlation } from './features.js';
+export { classifyFlake, shouldRetry, FLAKE_CLASSES } from './classify.js';
+export { analyzeTest, analyzeAll, groupByTestAndProject, DEFAULT_ANALYZE_CONFIG } from './analyze.js';
+export { evaluateQuarantinePolicy, effectiveBudget, daysUntilExpiry, expiryFor, renderQuarantineComment, buildQuarantineEntry, DEFAULT_QUARANTINE_POLICY, DEFAULT_QUARANTINE_REASON, UNCLASSIFIED_CAUSE, } from './quarantine.js';
+export { quarantineCodemod, releaseCodemod, QUARANTINE_TAG } from './codemod.js';
+export { bisect, interpret, DEFAULT_BISECT_OPTIONS } from './bisect.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EACL,SAAS,EACT,OAAO,EACP,cAAc,EACd,gBAAgB,EAChB,oBAAoB,GACrB,MAAM,YAAY,CAAC;AAGpB,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAG7D,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAG1E,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAC;AAGtG,OAAO,EACL,wBAAwB,EACxB,eAAe,EACf,eAAe,EACf,SAAS,EACT,uBAAuB,EACvB,oBAAoB,EACpB,yBAAyB,EACzB,yBAAyB,EACzB,kBAAkB,GACnB,MAAM,iBAAiB,CAAC;AASzB,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAGjF,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC"}

package/dist/quarantine.d.ts

@@ -0,0 +1,76 @@
+/**
+ * Quarantine policy.
+ *
+ * A quarantine without an expiry is a deletion with extra steps, and a
+ * quarantine list without a budget becomes the default response to any red
+ * build. Both limits are enforced here so the convention is mechanical rather
+ * than aspirational — "fix or delete promptly" is not a thing a policy
+ * document can make true.
+ */
+export interface QuarantineEntry {
+    readonly testId: string;
+    /** Null means every project. */
+    readonly project: string | null;
+    readonly title: string;
+    readonly reason: string;
+    readonly flakeScore: number;
+    readonly rootCause: string;
+    readonly issueUrl: string | null;
+    readonly createdAt: string;
+    readonly expiresAt: string;
+    /** Recorded when an expiry is extended, so extensions are auditable. */
+    readonly justification: string | null;
+}
+export interface QuarantinePolicy {
+    readonly expiryDays: number;
+    readonly maxTests: number;
+    /** Cap as a share of the suite; the effective cap is the larger of the two. */
+    readonly maxRatio: number;
+}
+export declare const DEFAULT_QUARANTINE_POLICY: QuarantinePolicy;
+export declare const DEFAULT_QUARANTINE_REASON = "flaky";
+export declare const UNCLASSIFIED_CAUSE = "unclassified";
+export interface BuildQuarantineEntryInput {
+    readonly title: string;
+    readonly testId?: string | undefined;
+    readonly project?: string | null | undefined;
+    readonly reason?: string | undefined;
+    readonly flakeScore?: number | undefined;
+    readonly rootCause?: string | undefined;
+    readonly issueUrl?: string | null | undefined;
+    readonly policy?: QuarantinePolicy | undefined;
+    readonly now?: number | undefined;
+}
+export declare function buildQuarantineEntry(input: BuildQuarantineEntryInput): QuarantineEntry;
+export interface PolicyViolation {
+    readonly kind: 'expired' | 'budget-exceeded';
+    readonly message: string;
+    readonly testIds: readonly string[];
+}
+export interface PolicyResult {
+    readonly ok: boolean;
+    readonly violations: readonly PolicyViolation[];
+    readonly active: number;
+    readonly budget: number;
+    readonly expired: readonly QuarantineEntry[];
+    /** Sorted soonest-first, for the "expiring shortly" warning. */
+    readonly expiringSoon: readonly QuarantineEntry[];
+}
+export declare function effectiveBudget(suiteSize: number, policy: QuarantinePolicy): number;
+export declare function daysUntilExpiry(entry: QuarantineEntry, now?: number): number;
+export declare function expiryFor(policy: QuarantinePolicy, now?: number): string;
+/**
+ * Evaluate the whole quarantine list. Returns violations rather than throwing,
+ * so a caller can print them all at once instead of failing on the first —
+ * being told about one problem at a time is how a CI gate becomes hated.
+ */
+export declare function evaluateQuarantinePolicy(entries: readonly QuarantineEntry[], suiteSize: number, policy?: QuarantinePolicy, now?: number): PolicyResult;
+/**
+ * The tag comment written above a quarantined test.
+ *
+ * Self-documenting and greppable on purpose: whoever finds this in six months
+ * should not have to open a dashboard to learn why it is here or when it
+ * should have been dealt with.
+ */
+export declare function renderQuarantineComment(entry: QuarantineEntry, atestVersion: string): string[];
+//# sourceMappingURL=quarantine.d.ts.map

package/dist/quarantine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"quarantine.d.ts","sourceRoot":"","sources":["../src/quarantine.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,gCAAgC;IAChC,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,wEAAwE;IACxE,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CACvC;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,+EAA+E;IAC/E,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED,eAAO,MAAM,yBAAyB,EAAE,gBAIvC,CAAC;AAEF,eAAO,MAAM,yBAAyB,UAAU,CAAC;AACjD,eAAO,MAAM,kBAAkB,iBAAiB,CAAC;AAEjD,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC7C,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IAC9C,QAAQ,CAAC,MAAM,CAAC,EAAE,gBAAgB,GAAG,SAAS,CAAC;IAC/C,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACnC;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,yBAAyB,GAAG,eAAe,CAetF;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,IAAI,EAAE,SAAS,GAAG,iBAAiB,CAAC;IAC7C,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;CACrC;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,UAAU,EAAE,SAAS,eAAe,EAAE,CAAC;IAChD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,SAAS,eAAe,EAAE,CAAC;IAC7C,gEAAgE;IAChE,QAAQ,CAAC,YAAY,EAAE,SAAS,eAAe,EAAE,CAAC;CACnD;AAKD,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,GAAG,MAAM,CAEnF;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,eAAe,EAAE,GAAG,GAAE,MAAmB,GAAG,MAAM,CAExF;AAED,wBAAgB,SAAS,CAAC,MAAM,EAAE,gBAAgB,EAAE,GAAG,GAAE,MAAmB,GAAG,MAAM,CAEpF;AAED;;;;GAIG;AACH,wBAAgB,wBAAwB,CACtC,OAAO,EAAE,SAAS,eAAe,EAAE,EACnC,SAAS,EAAE,MAAM,EACjB,MAAM,GAAE,gBAA4C,EACpD,GAAG,GAAE,MAAmB,GACvB,YAAY,CA0Cd;AAED;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,GAAG,MAAM,EAAE,CAO9F"}

package/dist/quarantine.js

@@ -0,0 +1,101 @@
+/**
+ * Quarantine policy.
+ *
+ * A quarantine without an expiry is a deletion with extra steps, and a
+ * quarantine list without a budget becomes the default response to any red
+ * build. Both limits are enforced here so the convention is mechanical rather
+ * than aspirational — "fix or delete promptly" is not a thing a policy
+ * document can make true.
+ */
+export const DEFAULT_QUARANTINE_POLICY = {
+    expiryDays: 14,
+    maxTests: 5,
+    maxRatio: 0.02,
+};
+export const DEFAULT_QUARANTINE_REASON = 'flaky';
+export const UNCLASSIFIED_CAUSE = 'unclassified';
+export function buildQuarantineEntry(input) {
+    const now = input.now ?? Date.now();
+    const policy = input.policy ?? DEFAULT_QUARANTINE_POLICY;
+    return {
+        testId: input.testId ?? input.title,
+        project: input.project ?? null,
+        title: input.title,
+        reason: input.reason ?? DEFAULT_QUARANTINE_REASON,
+        flakeScore: input.flakeScore ?? 0,
+        rootCause: input.rootCause ?? UNCLASSIFIED_CAUSE,
+        issueUrl: input.issueUrl ?? null,
+        createdAt: new Date(now).toISOString(),
+        expiresAt: expiryFor(policy, now),
+        justification: null,
+    };
+}
+const MS_PER_DAY = 86_400_000;
+const EXPIRING_SOON_DAYS = 3;
+export function effectiveBudget(suiteSize, policy) {
+    return Math.max(policy.maxTests, Math.floor(suiteSize * policy.maxRatio));
+}
+export function daysUntilExpiry(entry, now = Date.now()) {
+    return (Date.parse(entry.expiresAt) - now) / MS_PER_DAY;
+}
+export function expiryFor(policy, now = Date.now()) {
+    return new Date(now + policy.expiryDays * MS_PER_DAY).toISOString();
+}
+/**
+ * Evaluate the whole quarantine list. Returns violations rather than throwing,
+ * so a caller can print them all at once instead of failing on the first —
+ * being told about one problem at a time is how a CI gate becomes hated.
+ */
+export function evaluateQuarantinePolicy(entries, suiteSize, policy = DEFAULT_QUARANTINE_POLICY, now = Date.now()) {
+    const violations = [];
+    const expired = entries.filter(e => daysUntilExpiry(e, now) < 0);
+    if (expired.length > 0) {
+        violations.push({
+            kind: 'expired',
+            message: `${expired.length} quarantine${expired.length === 1 ? '' : 's'} past expiry. ` +
+                'Quarantines expire so they get fixed: release the test, extend with a written ' +
+                'justification, or delete it.',
+            testIds: expired.map(e => e.testId),
+        });
+    }
+    const budget = effectiveBudget(suiteSize, policy);
+    if (entries.length > budget) {
+        violations.push({
+            kind: 'budget-exceeded',
+            message: `${entries.length} quarantined tests exceeds the budget of ${budget} ` +
+                `(max(${policy.maxTests}, ${(policy.maxRatio * 100).toFixed(0)}% of ${suiteSize}). ` +
+                'Fix one before quarantining another.',
+            testIds: entries.map(e => e.testId),
+        });
+    }
+    const expiringSoon = entries
+        .filter(e => {
+        const days = daysUntilExpiry(e, now);
+        return days >= 0 && days <= EXPIRING_SOON_DAYS;
+    })
+        .sort((a, b) => Date.parse(a.expiresAt) - Date.parse(b.expiresAt));
+    return {
+        ok: violations.length === 0,
+        violations,
+        active: entries.length,
+        budget,
+        expired,
+        expiringSoon,
+    };
+}
+/**
+ * The tag comment written above a quarantined test.
+ *
+ * Self-documenting and greppable on purpose: whoever finds this in six months
+ * should not have to open a dashboard to learn why it is here or when it
+ * should have been dealt with.
+ */
+export function renderQuarantineComment(entry, atestVersion) {
+    return [
+        `@quarantine ${entry.reason}`,
+        `flakeScore ${entry.flakeScore.toFixed(2)} · class ${entry.rootCause}`,
+        `expires ${entry.expiresAt.slice(0, 10)}  ·  ${entry.issueUrl ?? 'no issue linked'}`,
+        `added by atest ${atestVersion}`,
+    ];
+}
+//# sourceMappingURL=quarantine.js.map

package/dist/quarantine.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"quarantine.js","sourceRoot":"","sources":["../src/quarantine.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAwBH,MAAM,CAAC,MAAM,yBAAyB,GAAqB;IACzD,UAAU,EAAE,EAAE;IACd,QAAQ,EAAE,CAAC;IACX,QAAQ,EAAE,IAAI;CACf,CAAC;AAEF,MAAM,CAAC,MAAM,yBAAyB,GAAG,OAAO,CAAC;AACjD,MAAM,CAAC,MAAM,kBAAkB,GAAG,cAAc,CAAC;AAcjD,MAAM,UAAU,oBAAoB,CAAC,KAAgC;IACnE,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;IACpC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,yBAAyB,CAAC;IACzD,OAAO;QACL,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,KAAK;QACnC,OAAO,EAAE,KAAK,CAAC,OAAO,IAAI,IAAI;QAC9B,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,yBAAyB;QACjD,UAAU,EAAE,KAAK,CAAC,UAAU,IAAI,CAAC;QACjC,SAAS,EAAE,KAAK,CAAC,SAAS,IAAI,kBAAkB;QAChD,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,IAAI;QAChC,SAAS,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE;QACtC,SAAS,EAAE,SAAS,CAAC,MAAM,EAAE,GAAG,CAAC;QACjC,aAAa,EAAE,IAAI;KACpB,CAAC;AACJ,CAAC;AAkBD,MAAM,UAAU,GAAG,UAAU,CAAC;AAC9B,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAE7B,MAAM,UAAU,eAAe,CAAC,SAAiB,EAAE,MAAwB;IACzE,OAAO,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;AAC5E,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAAsB,EAAE,MAAc,IAAI,CAAC,GAAG,EAAE;IAC9E,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC,GAAG,UAAU,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,MAAwB,EAAE,MAAc,IAAI,CAAC,GAAG,EAAE;IAC1E,OAAO,IAAI,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC,WAAW,EAAE,CAAC;AACtE,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,wBAAwB,CACtC,OAAmC,EACnC,SAAiB,EACjB,SAA2B,yBAAyB,EACpD,MAAc,IAAI,CAAC,GAAG,EAAE;IAExB,MAAM,UAAU,GAAsB,EAAE,CAAC;IAEzC,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,eAAe,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IACjE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,UAAU,CAAC,IAAI,CAAC;YACd,IAAI,EAAE,SAAS;YACf,OAAO,EACL,GAAG,OAAO,CAAC,MAAM,cAAc,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,gBAAgB;gBAC9E,gFAAgF;gBAChF,8BAA8B;YAChC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;SACpC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAG,eAAe,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAClD,IAAI,OAAO,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;QAC5B,UAAU,CAAC,IAAI,CAAC;YACd,IAAI,EAAE,iBAAiB;YACvB,OAAO,EACL,GAAG,OAAO,CAAC,MAAM,4CAA4C,MAAM,GAAG;gBACtE,QAAQ,MAAM,CAAC,QAAQ,KAAK,CAAC,MAAM,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,SAAS,KAAK;gBACpF,sCAAsC;YACxC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;SACpC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,YAAY,GAAG,OAAO;SACzB,MAAM,CAAC,CAAC,CAAC,EAAE;QACV,MAAM,IAAI,GAAG,eAAe,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACrC,OAAO,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,kBAAkB,CAAC;IACjD,CAAC,CAAC;SACD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;IAErE,OAAO;QACL,EAAE,EAAE,UAAU,CAAC,MAAM,KAAK,CAAC;QAC3B,UAAU;QACV,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM;QACN,OAAO;QACP,YAAY;KACb,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,uBAAuB,CAAC,KAAsB,EAAE,YAAoB;IAClF,OAAO;QACL,eAAe,KAAK,CAAC,MAAM,EAAE;QAC7B,cAAc,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,KAAK,CAAC,SAAS,EAAE;QACtE,WAAW,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,KAAK,CAAC,QAAQ,IAAI,iBAAiB,EAAE;QACpF,kBAAkB,YAAY,EAAE;KACjC,CAAC;AACJ,CAAC"}

package/dist/score.d.ts

@@ -0,0 +1,85 @@
+/**
+ * Flakiness scoring.
+ *
+ * A test is flaky when, holding commit, environment and project fixed, its
+ * outcome is non-deterministic. Anything else is a bug, an environment
+ * problem, or a real regression — and calling those "flaky" is how suites rot.
+ *
+ * Raw failure rate is a bad metric on both axes. It is unstable at low `n`,
+ * and it treats a six-month-old failure the same as this morning's. Two terms
+ * fix that:
+ *
+ *   Wilson lower bound   — stays conservative while observations are few, so a
+ *                          single failure in twelve runs does not brand a test.
+ *   Transition density   — distinguishes "was broken, then fixed" (a run of
+ *                          failures, low density) from genuine non-determinism
+ *                          (alternating outcomes, high density).
+ *
+ * Neither term alone is sufficient, which the tests demonstrate with worked
+ * patterns. No model is involved anywhere in this file.
+ */
+import { type HistoricalAttempt } from '@aplaytest/core';
+export type ScoreConfidence = 'low' | 'medium' | 'high';
+export interface FlakeScore {
+    /** 0..1. Flag above `threshold` once `rawN >= minRuns`. */
+    readonly score: number;
+    /**
+     * Wilson lower bound on the rate of the MINORITY outcome — how confident we
+     * are that both outcomes genuinely occur.
+     *
+     * Not the failure rate. A test that fails 12 times out of 12 has a very high
+     * failure rate and zero instability: it is deterministically broken, not
+     * flaky. Scoring on the failure rate labelled exactly that case FLAKY at
+     * 0.45, which sends a real breakage to the quarantine list instead of to
+     * whoever broke it.
+     */
+    readonly instability: number;
+    readonly transitionDensity: number;
+    /** Sum of recency weights — the effective sample size. */
+    readonly weightedN: number;
+    readonly rawN: number;
+    readonly failures: number;
+    readonly confidence: ScoreConfidence;
+    /** True when there were too few conclusive attempts to say anything. */
+    readonly insufficientData: boolean;
+}
+export interface ScoreConfig {
+    readonly minRuns: number;
+    readonly halfLifeDays: number;
+}
+export declare const DEFAULT_SCORE_CONFIG: ScoreConfig;
+/**
+ * Attempts that can testify about a test's determinism.
+ *
+ * `infra` failures (a crashed browser, a dropped port-forward) and skips are
+ * excluded: they are facts about the environment, not about the test, and
+ * counting them would manufacture flakiness during an outage.
+ */
+export declare function usableAttempts(attempts: readonly HistoricalAttempt[]): HistoricalAttempt[];
+/** Wilson score interval, lower bound. Conservative at small n by design. */
+export declare function wilsonLowerBound(successes: number, total: number, z?: number): number;
+/**
+ * Score a test's flakiness from its recorded attempts.
+ *
+ * Combines a Wilson lower bound on the **minority** outcome with transition
+ * density. A test that fails 12 of 12 is deterministically broken, not flaky
+ * — scoring on raw failure rate labelled that case FLAKY, which sends a real
+ * breakage to the quarantine list.
+ *
+ * @param attempts - History for one (test, project), oldest or newest first.
+ * @param config - Thresholds, half-life, and minimum sample size.
+ * @param now - Clock used for recency weighting. Injected so tests are stable.
+ * @returns A score in 0..1, plus the measurements that produced it. Check
+ *   `insufficientData` before acting on `score`.
+ *
+ * @example
+ * ```ts
+ * const score = scoreTest(attempts);
+ * if (!score.insufficientData && score.score > 0.15) {
+ *   // candidate for classification, not yet for quarantine
+ * }
+ * ```
+ */
+export declare function scoreTest(attempts: readonly HistoricalAttempt[], config?: ScoreConfig, now?: number): FlakeScore;
+export declare function isFlaky(score: FlakeScore, threshold: number): boolean;
+//# sourceMappingURL=score.d.ts.map

package/dist/score.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"score.d.ts","sourceRoot":"","sources":["../src/score.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAqC,KAAK,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAE5F,MAAM,MAAM,eAAe,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AAExD,MAAM,WAAW,UAAU;IACzB,2DAA2D;IAC3D,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;;;;;;;OASG;IACH,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,0DAA0D;IAC1D,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,UAAU,EAAE,eAAe,CAAC;IACrC,wEAAwE;IACxE,QAAQ,CAAC,gBAAgB,EAAE,OAAO,CAAC;CACpC;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AAED,eAAO,MAAM,oBAAoB,EAAE,WAGlC,CAAC;AAiBF;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,SAAS,iBAAiB,EAAE,GAAG,iBAAiB,EAAE,CAO1F;AAED,6EAA6E;AAC7E,wBAAgB,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,SAAI,GAAG,MAAM,CAOhF;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,SAAS,CACvB,QAAQ,EAAE,SAAS,iBAAiB,EAAE,EACtC,MAAM,GAAE,WAAkC,EAC1C,GAAG,GAAE,MAAmB,GACvB,UAAU,CAuDZ;AAED,wBAAgB,OAAO,CAAC,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAErE"}

package/dist/score.js

@@ -0,0 +1,140 @@
+/**
+ * Flakiness scoring.
+ *
+ * A test is flaky when, holding commit, environment and project fixed, its
+ * outcome is non-deterministic. Anything else is a bug, an environment
+ * problem, or a real regression — and calling those "flaky" is how suites rot.
+ *
+ * Raw failure rate is a bad metric on both axes. It is unstable at low `n`,
+ * and it treats a six-month-old failure the same as this morning's. Two terms
+ * fix that:
+ *
+ *   Wilson lower bound   — stays conservative while observations are few, so a
+ *                          single failure in twelve runs does not brand a test.
+ *   Transition density   — distinguishes "was broken, then fixed" (a run of
+ *                          failures, low density) from genuine non-determinism
+ *                          (alternating outcomes, high density).
+ *
+ * Neither term alone is sufficient, which the tests demonstrate with worked
+ * patterns. No model is involved anywhere in this file.
+ */
+import { countsTowardFlakeStats, isFailure } from '@aplaytest/core';
+export const DEFAULT_SCORE_CONFIG = {
+    minRuns: 10,
+    halfLifeDays: 7,
+};
+/** 95% confidence. */
+const Z = 1.96;
+const MS_PER_DAY = 86_400_000;
+const EMPTY = {
+    score: 0,
+    instability: 0,
+    transitionDensity: 0,
+    weightedN: 0,
+    rawN: 0,
+    failures: 0,
+    confidence: 'low',
+    insufficientData: true,
+};
+/**
+ * Attempts that can testify about a test's determinism.
+ *
+ * `infra` failures (a crashed browser, a dropped port-forward) and skips are
+ * excluded: they are facts about the environment, not about the test, and
+ * counting them would manufacture flakiness during an outage.
+ */
+export function usableAttempts(attempts) {
+    return attempts.filter(a => a.outcome !== 'skipped' &&
+        a.outcome !== 'interrupted' &&
+        (a.failureKind === null || countsTowardFlakeStats(a.failureKind)));
+}
+/** Wilson score interval, lower bound. Conservative at small n by design. */
+export function wilsonLowerBound(successes, total, z = Z) {
+    if (total <= 0)
+        return 0;
+    const p = successes / total;
+    const denominator = 1 + (z * z) / total;
+    const centre = p + (z * z) / (2 * total);
+    const margin = z * Math.sqrt((p * (1 - p) + (z * z) / (4 * total)) / total);
+    return Math.max(0, (centre - margin) / denominator);
+}
+/**
+ * Score a test's flakiness from its recorded attempts.
+ *
+ * Combines a Wilson lower bound on the **minority** outcome with transition
+ * density. A test that fails 12 of 12 is deterministically broken, not flaky
+ * — scoring on raw failure rate labelled that case FLAKY, which sends a real
+ * breakage to the quarantine list.
+ *
+ * @param attempts - History for one (test, project), oldest or newest first.
+ * @param config - Thresholds, half-life, and minimum sample size.
+ * @param now - Clock used for recency weighting. Injected so tests are stable.
+ * @returns A score in 0..1, plus the measurements that produced it. Check
+ *   `insufficientData` before acting on `score`.
+ *
+ * @example
+ * ```ts
+ * const score = scoreTest(attempts);
+ * if (!score.insufficientData && score.score > 0.15) {
+ *   // candidate for classification, not yet for quarantine
+ * }
+ * ```
+ */
+export function scoreTest(attempts, config = DEFAULT_SCORE_CONFIG, now = Date.now()) {
+    const usable = usableAttempts(attempts);
+    const rawN = usable.length;
+    const failures = usable.filter(a => isFailure(a.outcome)).length;
+    if (rawN < config.minRuns) {
+        return { ...EMPTY, rawN, failures };
+    }
+    // Exponential recency decay: a failure one half-life ago counts half as much
+    // as one today, so a fixed test stops being penalised for its history.
+    const halfLifeMs = config.halfLifeDays * MS_PER_DAY;
+    const weightOf = (attempt) => {
+        const age = now - Date.parse(attempt.startedAt);
+        if (!Number.isFinite(age))
+            return 1;
+        return Math.pow(0.5, Math.max(0, age) / halfLifeMs);
+    };
+    let weightedN = 0;
+    let weightedFailures = 0;
+    for (const attempt of usable) {
+        const weight = weightOf(attempt);
+        weightedN += weight;
+        if (isFailure(attempt.outcome))
+            weightedFailures += weight;
+    }
+    const weightedPasses = weightedN - weightedFailures;
+    // The MINORITY outcome is what makes a test flaky. All-fail and all-pass are
+    // both perfectly deterministic; instability peaks when the two are balanced
+    // and is zero at either extreme, regardless of which one dominates.
+    const minorityOutcome = Math.min(weightedFailures, weightedPasses);
+    const instability = weightedN > 0 ? wilsonLowerBound(minorityOutcome, weightedN) : 0;
+    // Ordered oldest-first so a "flip" means a genuine change of outcome over
+    // time rather than an artefact of query ordering.
+    const ordered = [...usable].sort((a, b) => Date.parse(a.startedAt) - Date.parse(b.startedAt));
+    let flips = 0;
+    for (let i = 1; i < ordered.length; i++) {
+        const previous = ordered[i - 1];
+        const current = ordered[i];
+        if (previous === undefined || current === undefined)
+            continue;
+        if (isFailure(previous.outcome) !== isFailure(current.outcome))
+            flips += 1;
+    }
+    const transitionDensity = ordered.length > 1 ? flips / (ordered.length - 1) : 0;
+    return {
+        score: 0.6 * instability + 0.4 * transitionDensity,
+        instability,
+        transitionDensity,
+        weightedN,
+        rawN,
+        failures,
+        confidence: rawN >= 30 ? 'high' : rawN >= 15 ? 'medium' : 'low',
+        insufficientData: false,
+    };
+}
+export function isFlaky(score, threshold) {
+    return !score.insufficientData && score.score > threshold;
+}
+//# sourceMappingURL=score.js.map

package/dist/score.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"score.js","sourceRoot":"","sources":["../src/score.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,sBAAsB,EAAE,SAAS,EAA0B,MAAM,iBAAiB,CAAC;AAiC5F,MAAM,CAAC,MAAM,oBAAoB,GAAgB;IAC/C,OAAO,EAAE,EAAE;IACX,YAAY,EAAE,CAAC;CAChB,CAAC;AAEF,sBAAsB;AACtB,MAAM,CAAC,GAAG,IAAI,CAAC;AACf,MAAM,UAAU,GAAG,UAAU,CAAC;AAE9B,MAAM,KAAK,GAAe;IACxB,KAAK,EAAE,CAAC;IACR,WAAW,EAAE,CAAC;IACd,iBAAiB,EAAE,CAAC;IACpB,SAAS,EAAE,CAAC;IACZ,IAAI,EAAE,CAAC;IACP,QAAQ,EAAE,CAAC;IACX,UAAU,EAAE,KAAK;IACjB,gBAAgB,EAAE,IAAI;CACvB,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAAC,QAAsC;IACnE,OAAO,QAAQ,CAAC,MAAM,CACpB,CAAC,CAAC,EAAE,CACF,CAAC,CAAC,OAAO,KAAK,SAAS;QACvB,CAAC,CAAC,OAAO,KAAK,aAAa;QAC3B,CAAC,CAAC,CAAC,WAAW,KAAK,IAAI,IAAI,sBAAsB,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CACpE,CAAC;AACJ,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,gBAAgB,CAAC,SAAiB,EAAE,KAAa,EAAE,CAAC,GAAG,CAAC;IACtE,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACzB,MAAM,CAAC,GAAG,SAAS,GAAG,KAAK,CAAC;IAC5B,MAAM,WAAW,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC;IACxC,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC;IAC5E,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,WAAW,CAAC,CAAC;AACtD,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,UAAU,SAAS,CACvB,QAAsC,EACtC,SAAsB,oBAAoB,EAC1C,MAAc,IAAI,CAAC,GAAG,EAAE;IAExB,MAAM,MAAM,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC;IAC3B,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;IAEjE,IAAI,IAAI,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;QAC1B,OAAO,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;IACtC,CAAC;IAED,6EAA6E;IAC7E,uEAAuE;IACvE,MAAM,UAAU,GAAG,MAAM,CAAC,YAAY,GAAG,UAAU,CAAC;IACpD,MAAM,QAAQ,GAAG,CAAC,OAA0B,EAAU,EAAE;QACtD,MAAM,GAAG,GAAG,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,OAAO,CAAC,CAAC;QACpC,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC;IACtD,CAAC,CAAC;IAEF,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,KAAK,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;QACjC,SAAS,IAAI,MAAM,CAAC;QACpB,IAAI,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC;YAAE,gBAAgB,IAAI,MAAM,CAAC;IAC7D,CAAC;IACD,MAAM,cAAc,GAAG,SAAS,GAAG,gBAAgB,CAAC;IAEpD,6EAA6E;IAC7E,4EAA4E;IAC5E,oEAAoE;IACpE,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,cAAc,CAAC,CAAC;IACnE,MAAM,WAAW,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,eAAe,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAErF,0EAA0E;IAC1E,kDAAkD;IAClD,MAAM,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;IAC9F,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAChC,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,QAAQ,KAAK,SAAS,IAAI,OAAO,KAAK,SAAS;YAAE,SAAS;QAC9D,IAAI,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC;YAAE,KAAK,IAAI,CAAC,CAAC;IAC7E,CAAC;IACD,MAAM,iBAAiB,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEhF,OAAO;QACL,KAAK,EAAE,GAAG,GAAG,WAAW,GAAG,GAAG,GAAG,iBAAiB;QAClD,WAAW;QACX,iBAAiB;QACjB,SAAS;QACT,IAAI;QACJ,QAAQ;QACR,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK;QAC/D,gBAAgB,EAAE,KAAK;KACxB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,KAAiB,EAAE,SAAiB;IAC1D,OAAO,CAAC,KAAK,CAAC,gBAAgB,IAAI,KAAK,CAAC,KAAK,GAAG,SAAS,CAAC;AAC5D,CAAC"}

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@aplaytest/flaky",
+  "version": "0.1.0",
+  "description": "Statistical flake detection, root-cause classification, and quarantine policy",
+  "license": "MIT",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsc --build",
+    "test": "vitest run"
+  },
+  "dependencies": {
+    "@aplaytest/core": "^0.1.0",
+    "ts-morph": "^28.0.0",
+    "@aplaytest/runner-playwright": "^0.1.0"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/ianoflynnautomation/aplaytest.git",
+    "directory": "packages/flaky"
+  },
+  "homepage": "https://github.com/ianoflynnautomation/aplaytest#readme",
+  "bugs": {
+    "url": "https://github.com/ianoflynnautomation/aplaytest/issues"
+  }
+}