@crewhaus/tool-buildperf 0.7.0

package/dist/lib/flaky.js

@@ -0,0 +1,346 @@
+/**
+ * What repeated test runs can and cannot establish.
+ *
+ * Three failures in five runs is not a 60% failure rate. It is a point
+ * estimate drawn from five observations, and the Wilson interval around it
+ * runs from 0.231 to 0.882 — a range that spans "mildly annoying" and "the
+ * test is broken" without distinguishing them. Quarantining on the point
+ * estimate is quarantining on noise, so every rate here travels with its
+ * interval (from `@crewhaus/tool-math`'s kernel) and the quarantine
+ * recommendation keys on the interval's LOWER bound, never on the estimate.
+ *
+ * The other thing five runs cannot do is separate nondeterminism from order
+ * dependence. A test that fails only when another test ran first is not
+ * flaky; it is deterministic in a coordinate the runner did not vary. If the
+ * input carries execution order, this module looks for the predecessor that
+ * distinguishes the failing runs from the passing ones. If every run used the
+ * SAME order, it says the question is unanswerable rather than answering
+ * "nondeterministic" — which is the wrong answer given confidently, and the
+ * whole reason a team spends a week chasing the wrong test.
+ */
+import { statsKernel } from "@crewhaus/tool-math";
+export const TEST_STATUSES = ["pass", "fail", "error", "skip"];
+export class FlakyError extends Error {
+    name = "FlakyError";
+}
+/** A failure is a failure; `error` is one that did not reach an assertion. */
+const isFailure = (status) => status === "fail" || status === "error";
+/**
+ * Apply the caller's substitutions and report how many times each fired.
+ *
+ * A mask that rewrites text invisibly can merge two genuinely different
+ * failures into one group, which reads as "it always fails the same way" when
+ * it does not — so the counts come back with the result, the same rule
+ * `GoldenCompare` follows for its normalizers.
+ *
+ * There are deliberately no built-in masks. `ErrorCluster` in
+ * `@crewhaus/tool-obs` already owns shape-based masking of error text (URLs,
+ * uuids, timestamps, paths, prefixed ids, hex blobs, numbers); a second set
+ * here would drift from it, and the two tools would then disagree about
+ * whether two failures are the same failure. Pass its fingerprint in as
+ * `signature`, or declare the substitutions this suite needs.
+ */
+export function applyMasks(text, masks, counts) {
+    let out = text;
+    for (const mask of masks) {
+        const flags = mask.flags ?? "";
+        let re;
+        try {
+            re = new RegExp(mask.pattern, flags.includes("g") ? flags : `${flags}g`);
+        }
+        catch (err) {
+            throw new FlakyError(`mask /${mask.pattern}/ is not a valid regular expression: ${err.message}`);
+        }
+        const fired = out.match(re)?.length ?? 0;
+        if (fired > 0) {
+            out = out.replace(re, mask.with ?? "<masked>");
+            counts[mask.pattern] = (counts[mask.pattern] ?? 0) + fired;
+        }
+    }
+    return out.replace(/\s+/g, " ").trim();
+}
+/**
+ * The smallest run count whose all-failing Wilson lower bound clears
+ * `target`. Answers "how many consecutive failures before I may call this
+ * deterministic?" — at 95% confidence against a 0.9 target the answer is in
+ * the dozens, which is why three-for-three is evidence and not proof.
+ */
+export function runsForLowerBound(target, z = statsKernel.Z_95) {
+    for (let n = 1; n <= 1_000; n++) {
+        const interval = statsKernel.wilsonScoreInterval(n, n, z);
+        if (interval !== null && interval.lower >= target)
+            return n;
+    }
+    return null;
+}
+function orderAnalysis(occurrences, orderVaried, suiteSize) {
+    const failing = occurrences.filter((o) => isFailure(o.status));
+    const passing = occurrences.filter((o) => o.status === "pass");
+    const positions = {
+        medianPositionWhenFailing: statsKernel.median(failing.map((o) => o.position)),
+        medianPositionWhenPassing: statsKernel.median(passing.map((o) => o.position)),
+    };
+    if (suiteSize < 2) {
+        return {
+            decidable: false,
+            suspects: [],
+            ...positions,
+            note: "each run contained a single test, so nothing could have run before this one",
+        };
+    }
+    if (!orderVaried) {
+        return {
+            decidable: false,
+            suspects: [],
+            ...positions,
+            note: "every run executed the tests in the same order, so an order-dependent failure and a nondeterministic one are indistinguishable here — re-run with the suite shuffled (and the seed recorded) before concluding either",
+        };
+    }
+    if (failing.length === 0 || passing.length === 0) {
+        return {
+            decidable: false,
+            suspects: [],
+            ...positions,
+            note: "this test did not both pass and fail, so there is no contrast between orders to learn from",
+        };
+    }
+    // `orderVaried` is a claim about the SUITE. A suite that merely gained or
+    // lost a test, or shuffled only the tail, varies its sequence while this
+    // test keeps the identical prefix in every run — and an identical prefix is
+    // the suite-wide unvaried case all over again: "fails after this prefix" and
+    // "fails at random" are the same observation. Taking the suite's flag for
+    // this test's would report "the failures do not line up with a predecessor"
+    // about a comparison that was never available.
+    const prefixes = new Set(occurrences.map((o) => JSON.stringify([...o.predecessors].sort())));
+    if (prefixes.size < 2) {
+        return {
+            decidable: false,
+            suspects: [],
+            ...positions,
+            note: "the suite's order varied, but this test ran after exactly the same set of tests in every run, so nothing here separates an order-dependent failure from a nondeterministic one",
+        };
+    }
+    // Intersect the predecessors of every failing run, then subtract everything
+    // that EVER preceded a passing run. What survives ran before the test each
+    // time it failed and never when it passed: a shortlist to bisect, not a
+    // cause.
+    let shared = null;
+    for (const occurrence of failing) {
+        if (shared === null) {
+            shared = new Set(occurrence.predecessors);
+            continue;
+        }
+        for (const id of [...shared]) {
+            if (!occurrence.predecessors.has(id))
+                shared.delete(id);
+        }
+    }
+    const suspects = [...(shared ?? new Set())]
+        .filter((id) => !passing.some((o) => o.predecessors.has(id)))
+        .sort();
+    return {
+        decidable: true,
+        suspects,
+        ...positions,
+        note: suspects.length === 0
+            ? "run order varied and no test ran before this one in every failure and in none of the passes; the failures do not line up with a predecessor"
+            : `${suspects.length} test(s) ran before this one in every failing run and in none of the passing runs; re-run this test after each of them alone to confirm`,
+    };
+}
+function groupFailures(occurrences, masks, counts) {
+    const groups = new Map();
+    for (const occurrence of occurrences) {
+        if (!isFailure(occurrence.status))
+            continue;
+        const raw = occurrence.error ?? "";
+        // No text and no signature means the run recorded a failure without
+        // saying anything about it; that is not a group of one, it is an unknown.
+        if (occurrence.signature === undefined && raw === "")
+            continue;
+        const key = occurrence.signature ?? applyMasks(raw, masks, counts);
+        const row = groups.get(key);
+        if (row === undefined) {
+            groups.set(key, { count: 1, example: raw, runs: [occurrence.runId] });
+            continue;
+        }
+        row.count += 1;
+        row.runs.push(occurrence.runId);
+    }
+    return [...groups.entries()]
+        .map(([signature, row]) => ({
+        signature,
+        count: row.count,
+        example: row.example,
+        runs: row.runs,
+    }))
+        .sort((a, b) => b.count - a.count || (a.signature < b.signature ? -1 : 1));
+}
+export function detectFlaky(runs, options = {}) {
+    if (runs.length === 0)
+        throw new FlakyError("no runs were given; there is nothing to classify");
+    const z = options.z ?? statsKernel.Z_95;
+    const quarantineLowerBound = options.quarantineLowerBound ?? 0.05;
+    if (!(quarantineLowerBound >= 0 && quarantineLowerBound <= 1)) {
+        throw new FlakyError(`quarantineLowerBound must be a probability between 0 and 1, got ${quarantineLowerBound}`);
+    }
+    const masks = options.masks ?? [];
+    const maskCounts = {};
+    const occurrences = new Map();
+    const sequences = [];
+    for (const [runIndex, run] of runs.entries()) {
+        const runId = run.id ?? `run-${runIndex + 1}`;
+        const seenInRun = new Set();
+        const before = new Set();
+        for (const [position, test] of run.tests.entries()) {
+            if (seenInRun.has(test.id)) {
+                throw new FlakyError(`test "${test.id}" appears twice in ${runId}; a run reports one outcome per test, and two rows cannot both be it`);
+            }
+            seenInRun.add(test.id);
+            const list = occurrences.get(test.id) ?? [];
+            list.push({
+                runIndex,
+                runId,
+                position,
+                status: test.status,
+                predecessors: new Set(before),
+                ...(test.error === undefined ? {} : { error: test.error }),
+                ...(test.signature === undefined ? {} : { signature: test.signature }),
+            });
+            occurrences.set(test.id, list);
+            before.add(test.id);
+        }
+        // JSON, not a delimiter: a test id is caller-supplied and could contain
+        // whatever separator character we picked, which would make two different
+        // orders compare equal and silently disable the order analysis.
+        sequences.push(JSON.stringify(run.tests.map((t) => t.id)));
+    }
+    const orderVaried = new Set(sequences).size > 1;
+    const seedsDeclared = [
+        ...new Set(runs.map((r) => r.seed).filter((s) => s !== undefined)),
+    ].sort();
+    const suiteSize = Math.max(...runs.map((r) => r.tests.length));
+    const tests = [...occurrences.keys()].sort().map((id) => {
+        const list = occurrences.get(id);
+        const failures = list.filter((o) => isFailure(o.status)).length;
+        const passes = list.filter((o) => o.status === "pass").length;
+        const skipped = list.filter((o) => o.status === "skip").length;
+        // A skipped test is not a pass. Counting it as one is how a suite that
+        // stopped running a test reports it as stable for a year.
+        const runsObserved = failures + passes;
+        const absent = runs.length - list.length;
+        const interval = statsKernel.wilsonScoreInterval(failures, runsObserved, z);
+        const failureRate = {
+            pointEstimate: interval?.pointEstimate ?? null,
+            lower: interval?.lower ?? null,
+            upper: interval?.upper ?? null,
+            width: interval?.width ?? null,
+            z,
+        };
+        const order = orderAnalysis(list, orderVaried, suiteSize);
+        const failureGroups = groupFailures(list, masks, maskCounts);
+        // A failure that recorded no text and no signature is in no group, so one
+        // group does not mean one failure mode — it means one failure mode among
+        // the failures that said anything. Answering "yes, always the same way"
+        // from that subset is the cheap wrong answer: it is what sends someone
+        // after a single root cause for two different bugs.
+        const ungroupedFailures = list.filter((o) => isFailure(o.status) && o.signature === undefined && (o.error ?? "") === "").length;
+        const sameFailureEveryTime = failures === 0 || failureGroups.length === 0 || ungroupedFailures > 0
+            ? null
+            : failureGroups.length === 1;
+        const classification = runsObserved === 0
+            ? "not-observed"
+            : failures === 0
+                ? "passing"
+                : failures === runsObserved
+                    ? "failing-every-run"
+                    : "flaky";
+        let recommendation = "keep";
+        let why;
+        if (classification === "not-observed") {
+            why = `never ran: skipped in ${skipped} run(s) and absent from ${absent}`;
+        }
+        else if (classification === "passing") {
+            why = `passed all ${runsObserved} run(s) it was observed in`;
+            // Passing every time is not the same as being stable. With two or three
+            // observations the interval still reaches past a coin flip, so quote it
+            // rather than implying a clean bill of health.
+            if (interval !== null && interval.upper > 0.5) {
+                why += `; with only ${runsObserved} observation(s) its failure rate could still be as high as ${(interval.upper * 100).toFixed(0)}%`;
+            }
+        }
+        else if (classification === "failing-every-run") {
+            recommendation = "fix";
+            why = `failed in all ${runsObserved} run(s)`;
+            if (!orderVaried && suiteSize > 1) {
+                why +=
+                    "; every run used the same order, so this is equally consistent with a deterministic failure and with one that only happens after the tests that preceded it";
+            }
+            if (interval !== null) {
+                why += `; ${runsObserved}-for-${runsObserved} puts the failure rate at ${(interval.lower * 100).toFixed(0)}% or above, not at 100%`;
+            }
+        }
+        else {
+            const lower = interval?.lower ?? 0;
+            const upper = interval?.upper ?? 1;
+            recommendation = lower >= quarantineLowerBound ? "quarantine" : "insufficient-evidence";
+            why = `failed ${failures} of ${runsObserved} run(s); the ${(lower * 100).toFixed(1)}%-${(upper * 100).toFixed(1)}% interval ${recommendation === "quarantine"
+                ? `clears the ${(quarantineLowerBound * 100).toFixed(1)}% quarantine floor`
+                : `does not clear the ${(quarantineLowerBound * 100).toFixed(1)}% quarantine floor — run it more before acting on the point estimate`}`;
+            if (order.suspects.length > 0) {
+                why += `; it may be order-dependent rather than nondeterministic (${order.suspects.length} candidate predecessor(s))`;
+            }
+        }
+        return {
+            id,
+            classification,
+            runsObserved,
+            failures,
+            passes,
+            skipped,
+            absent,
+            failureRate,
+            order,
+            failureGroups,
+            sameFailureEveryTime,
+            recommendation,
+            why,
+        };
+    });
+    const summary = {
+        passing: 0,
+        flaky: 0,
+        "failing-every-run": 0,
+        "not-observed": 0,
+    };
+    for (const test of tests)
+        summary[test.classification] += 1;
+    // One run can show a failure. It cannot show a flake: every classification
+    // above needs at least two observations of one test to mean anything.
+    const verdict = runs.length < 2
+        ? "inconclusive"
+        : summary.flaky > 0
+            ? "flaky"
+            : summary["failing-every-run"] > 0
+                ? "failing"
+                : "clean";
+    return {
+        runs: runs.length,
+        ordering: {
+            varied: orderVaried,
+            seedsDeclared,
+            note: orderVaried
+                ? `${new Set(sequences).size} distinct execution order(s) across ${runs.length} run(s)`
+                : seedsDeclared.length > 1
+                    ? "the runs declare different seeds but executed the tests in the same order, so the seed never reached the suite"
+                    : "every run executed the tests in the same order; order-dependent failures cannot be distinguished from nondeterministic ones in this input",
+        },
+        tests,
+        summary,
+        masksApplied: maskCounts,
+        runsForDeterminismClaim: runsForLowerBound(0.9, z),
+        verdict,
+        note: runs.length < 2
+            ? "a single run cannot distinguish a flaky test from a deterministic one: re-run the suite and pass every run"
+            : `${tests.length} test(s) across ${runs.length} run(s); rates are Wilson intervals at z=${z}, not raw fail/run ratios`,
+    };
+}

package/dist/lib/size.d.ts

@@ -0,0 +1,114 @@
+export declare const COMPRESSION_ALGORITHMS: readonly ["none", "gzip", "brotli"];
+export type CompressionAlgorithm = (typeof COMPRESSION_ALGORITHMS)[number];
+/**
+ * Everything that determines the compressed byte count, stated rather than
+ * assumed.
+ *
+ * `tuning` carries the settings below the level that also move the output —
+ * zlib's memLevel and strategy, brotli's window and mode. They are pinned
+ * explicitly when compressing instead of left to the library defaults,
+ * because a default that changes under a runtime upgrade would silently make
+ * old baselines incomparable while still LOOKING comparable.
+ */
+export type CompressionParameters = {
+    readonly algorithm: CompressionAlgorithm;
+    /** Compression level; `null` only for `none`, where there is no level. */
+    readonly level: number | null;
+    readonly tuning: Readonly<Record<string, number>>;
+};
+export declare class SizeError extends Error {
+    readonly name = "SizeError";
+}
+/**
+ * Fill in and validate the parameters for a run.
+ *
+ * A level outside the algorithm's range is a refusal, not a clamp: gzip
+ * silently accepting 11 (a brotli quality) and compressing at 9 would produce
+ * a report claiming a level it did not use, which is the exact failure this
+ * module exists to prevent.
+ */
+export declare function compressionParameters(algorithm: CompressionAlgorithm, level?: number): CompressionParameters;
+/**
+ * The string two parameter sets must agree on for a comparison to mean
+ * anything. Sorted, so key order in a stored baseline cannot change it.
+ */
+export declare function comparabilityKey(p: CompressionParameters): string;
+/**
+ * Why these two sets of parameters cannot be compared, or `null` when they
+ * can. The message names the fields that differ, because "incomparable" on
+ * its own sends the reader to the source.
+ */
+export declare function parametersMismatch(baseline: CompressionParameters, head: CompressionParameters): string | null;
+/**
+ * Why this baseline's ENTRIES cannot be differenced against a run measured
+ * under `algorithm`, or `null` when they can.
+ *
+ * `parametersMismatch` guards the compressor settings. This guards the
+ * column. Every entry carries two numbers — a raw size and a compressed one —
+ * and which of them a delta is taken on is decided by the algorithm, so a
+ * baseline that declares gzip but stores no `compressedBytes` makes the
+ * obvious `compressed ?? raw` fallback subtract a gzip size from a raw one.
+ * That lands near -70% on real bundles: not a number a reader distrusts, a
+ * number a reader celebrates, and it hides whatever the change actually did.
+ * The reverse — a compressed column under algorithm "none" — fabricates a
+ * regression the same way, so both directions are refused rather than
+ * silently filled in.
+ */
+export declare function baselineBasisMismatch(algorithm: CompressionAlgorithm, entries: ReadonlyArray<{
+    readonly path: string;
+    readonly compressedBytes?: number | null;
+}>): string | null;
+/** Compressed byte count under `parameters`; the raw length for `none`. */
+export declare function compressedSize(bytes: Buffer, parameters: CompressionParameters): number;
+export declare const JOIN_MODES: readonly ["exact", "auto", "custom"];
+export type JoinMode = (typeof JOIN_MODES)[number];
+/**
+ * Whether a filename segment looks like a content hash rather than a name.
+ *
+ * Deliberately conservative in one direction only: `chunk`, `min`, `esm` and
+ * `vendor` must never be eaten, because normalizing a real name merges two
+ * different artifacts into one row. Pure hex needs 6 characters (webpack's
+ * shortest common `[contenthash:6]`); a mixed alphabet needs 8 AND both a
+ * digit and a letter (Vite's base64url ids, e.g. `BXaGz2Qm`). A lowercase
+ * word of 6+ letters that happens to be all hex — `decade`, `defaced` — is
+ * the known false positive, which is why every entry reports the key it was
+ * joined on rather than just the delta.
+ */
+export declare function looksLikeHash(segment: string): boolean;
+export type JoinKeyResult = {
+    readonly key: string;
+    readonly normalized: boolean;
+};
+/**
+ * The key an entry joins on.
+ *
+ * `exact` is the default and joins on the path as written. It is the right
+ * choice for a build with stable filenames and the wrong one for a hashed
+ * build, where it matches NOTHING on every run — which reads as "every file
+ * is new", not as "the join is broken". `auto` strips hash-shaped segments
+ * from the basename; `custom` applies the caller's own pattern, which is the
+ * escape hatch for a bundler whose scheme the heuristic does not recognise.
+ */
+export declare function joinKey(path: string, mode: JoinMode, custom?: RegExp): JoinKeyResult;
+/**
+ * Long enough for any real budget, short enough that the matcher's recursion
+ * depth (one frame per token) cannot reach the stack limit.
+ */
+export declare const MAX_GLOB_LENGTH = 512;
+export type GlobMatcher = (path: string) => boolean;
+/**
+ * Compile a pattern once, match many paths with it.
+ *
+ * This deliberately does NOT compile to a regular expression. `*` and `**`
+ * become `[^/]*` and `.*`, and a backtracking engine handed several of those
+ * separated by literals explores every way of splitting the string between
+ * them: `*a*a*a…*b` — a 49-character pattern — against sixty `a`s does not
+ * finish, and a budget pattern is caller-supplied. The state here is only
+ * (token index, offset in the path), so memoising it bounds the search at
+ * tokens x length cells no matter how the wildcards are arranged.
+ *
+ * Callers with many paths should hold the matcher: recompiling per path
+ * multiplies the tokenizer by the size of the build.
+ */
+export declare function compileGlob(pattern: string): GlobMatcher;
+export declare function globMatch(pattern: string, path: string): boolean;