@holmes-lab/holmes-kit 0.2.1 → 0.3.1

package/dist/holmes/review/replay-calibration.js

@@ -0,0 +1,83 @@
+"use strict";
+// @implements A-SPEC-357
+/**
+ * Does the confidence number mean anything?
+ *
+ * Goal Phase D asks for the closed loop, and the mechanism has been verified for a while — what is
+ * missing is data (`.ax/evidence` is n=0). Filling that directory with synthetic records would be
+ * worse than leaving it empty: it is a user-visible governance ledger the user deliberately chose
+ * not to git-ignore, and calibration computed from replay would read as calibration observed in use.
+ * So this measures in a temporary ledger and says plainly what it measured.
+ *
+ * The question it CAN answer honestly is not the one `computeCalibration` scores. That function
+ * grades classification correctness, and point-in-time replay carries no ground truth for "new
+ * requirement vs change to an existing one" — inventing that label would be scoring our own guess
+ * against our own guess. What replay does carry is the files the commit actually touched, so the
+ * answerable question is whether a HIGHER confidence goes with a MORE accurate answer.
+ *
+ * The prior is not neutral. The confidence formula is
+ * `0.55 + 0.15·(direct symbols exist) + 0.15·(tests found) + 0.1·(history found)` — it counts
+ * whether evidence is PRESENT, never whether it is any good. A number built that way can be
+ * expected to correlate weakly with accuracy, and if it does, saying so is the finding: a confidence
+ * that does not track correctness sells unearned certainty to whoever reads it.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.confidenceCalibration = confidenceCalibration;
+exports.correlate = correlate;
+exports.agreeingSignals = agreeingSignals;
+function confidenceCalibration(obs) {
+    const n = obs.length;
+    const confidences = obs.map((o) => o.confidence);
+    const distinct = new Set(confidences).size;
+    if (n === 0) {
+        return { n: 0, correlation: null, distinctConfidences: 0, meanAccuracyHigh: null, meanAccuracyLow: null, meanSquaredGap: null };
+    }
+    const mean = (xs) => xs.reduce((a, b) => a + b, 0) / xs.length;
+    const accs = obs.map((o) => o.accuracy);
+    const mc = mean(confidences), ma = mean(accs);
+    let num = 0, dc = 0, da = 0;
+    for (const o of obs) {
+        num += (o.confidence - mc) * (o.accuracy - ma);
+        dc += (o.confidence - mc) ** 2;
+        da += (o.accuracy - ma) ** 2;
+    }
+    // A constant series has zero variance and no correlation to report. Returning 0 would say "no
+    // relationship measured"; null says "no relationship measurable", which is the truth.
+    const correlation = dc === 0 || da === 0 ? null : num / Math.sqrt(dc * da);
+    const sorted = [...confidences].sort((a, b) => a - b);
+    const median = sorted[Math.floor(sorted.length / 2)];
+    const high = obs.filter((o) => o.confidence >= median).map((o) => o.accuracy);
+    const low = obs.filter((o) => o.confidence < median).map((o) => o.accuracy);
+    return {
+        n,
+        correlation,
+        distinctConfidences: distinct,
+        meanAccuracyHigh: high.length === 0 ? null : mean(high),
+        meanAccuracyLow: low.length === 0 ? null : mean(low),
+        meanSquaredGap: mean(obs.map((o) => (o.confidence - o.accuracy) ** 2)),
+    };
+}
+function correlate(pairs) {
+    if (pairs.length < 2)
+        return null;
+    const mean = (xs) => xs.reduce((a, b) => a + b, 0) / xs.length;
+    const mx = mean(pairs.map((p) => p.x)), my = mean(pairs.map((p) => p.y));
+    let num = 0, dx = 0, dy = 0;
+    for (const p of pairs) {
+        num += (p.x - mx) * (p.y - my);
+        dx += (p.x - mx) ** 2;
+        dy += (p.y - my) ** 2;
+    }
+    return dx === 0 || dy === 0 ? null : num / Math.sqrt(dx * dy);
+}
+/** Signals that agree in sign across corpora AND clear the threshold in both. */
+function agreeingSignals(a, b, threshold) {
+    const byName = new Map(b.map((s) => [s.signal, s]));
+    return a.filter((s) => {
+        const other = byName.get(s.signal);
+        if (s.correlation === null || other?.correlation === undefined || other.correlation === null)
+            return false;
+        return Math.sign(s.correlation) === Math.sign(other.correlation)
+            && Math.abs(s.correlation) >= threshold && Math.abs(other.correlation) >= threshold;
+    }).map((s) => s.signal);
+}

package/dist/holmes/review/replay-corpus.d.ts

@@ -0,0 +1,135 @@
+/**
+ * A repository the replay benchmark can be run against.
+ *
+ * Every verdict reached in this work — three declines of contract nodes, four of the downstream
+ * direction, a ceiling of 0.956, an achievement of 0.914 — came from ONE repository. While
+ * generalisation is unmeasured, "world top tier" is not a claim that can be checked. Making the
+ * corpus a value is the first step to a second measurement, and the layout differences are exactly
+ * what the old inline logic hard-coded: `src/holmes` paths and a `.test.` filename convention.
+ */
+export interface ReplayCorpus {
+    /** Absolute path to the repository root. */
+    root: string;
+    /**
+     * git pathspecs limiting which files this corpus considers at all.
+     *
+     * A LIST, not one path: the pathspec filters the file list git prints, so a repository that keeps
+     * tests in a sibling directory needs both included or its tests never appear and every commit
+     * looks source-only. Measured — `src` alone yielded 0 cases for a repo with 251 of them.
+     */
+    sourcePathspec: string[];
+    /** git pathspec for the spec store, materialised alongside the source for point-in-time replay. */
+    specPathspec: string;
+    /** Which of the changed files are tests. Layout differs per repository — that is the point. */
+    isTest: (file: string) => boolean;
+    /** Extensions that count as source at all. */
+    isSource: (file: string) => boolean;
+    /**
+     * @implements A-SPEC-443
+     * Where the case window STARTS. Absent means `HEAD`, which is what every caller had before.
+     *
+     * A window defined as "the newest N commits" measures something different every time the
+     * repository grows, so a digit pinned against it is not a floor. Measured 2026-08-30: the second
+     * corpus gained five commits in a day and `run-replay` lost a six-decimal assertion, 0.7996 ->
+     * 0.7663, with no product change. Re-pinning to the new digits would have hidden the fragility
+     * rather than removed it — and the same slide can hide a REAL regression just as easily.
+     */
+    pin?: string;
+}
+export interface ReplayCase {
+    commit: string;
+    subject: string;
+    /** Non-test source files the commit changed. */
+    files: string[];
+    /** Test files the commit changed. */
+    tests: string[];
+}
+export declare const HOLMES_CORPUS: ReplayCorpus;
+/**
+ * The second corpus. Measured 2026-08-29: 269 source files, 329 commits, 251 eligible cases, 959
+ * truth files, and its own `.ax/specs`. Tests live in `tests/` rather than beside the source, which
+ * is precisely the assumption that had to stop being hard-coded.
+ *
+ * Absent on any machine but this one, and callers skip rather than fail — a benchmark that requires
+ * someone else's checkout is a benchmark nobody else can run.
+ */
+export declare const JARVIS_CORPUS: ReplayCorpus;
+/**
+ * A third party's repository, with no spec store at all.
+ *
+ * The first two corpora share two properties that turned out to matter: one of them is my own work,
+ * and both carry `.ax/specs`. A verdict recorded in the activation log — "the advantage is entirely
+ * the spec layer" — rested on that. Most projects have no specs, so the question that verdict left
+ * open is what the harness is worth without them.
+ *
+ * Measured: it beats name matching on every axis here, with `specs: []` passed to the analyzer.
+ * Whatever the remaining layers are worth, they are worth something on their own.
+ */
+export declare const GEMINI_CLI_CORPUS: ReplayCorpus;
+/**
+ * @implements A-SPEC-426
+ * Measured 2026-08-29 over 29 commits touching 1 to 12 source files, through the product path with
+ * `specs: []`. The opponent is the NAME-ONLY baseline — not a person. A human comparison on this
+ * corpus is a separate measurement and is not claimed here.
+ */
+export declare const GEMINI_CLI_MEASUREMENT: {
+    readonly cases: 29;
+    readonly specsAvailable: 0;
+    readonly opponent: "name-only baseline (rankBaseline)";
+    readonly shipped: {
+        readonly top1: 0.241;
+        readonly top5Recall: 0.369;
+        readonly top10Recall: 0.483;
+        readonly top10Precision: 0.083;
+    };
+    readonly baseline: {
+        readonly top1: 0.172;
+        readonly top5Recall: 0.186;
+        readonly top10Recall: 0.302;
+        readonly top10Precision: 0.052;
+    };
+};
+/**
+ * The commits this corpus offers as replay cases: those that changed BOTH source and tests, most
+ * recent first. A commit touching only one side cannot score both axes.
+ */
+export declare function casesFor(corpus: ReplayCorpus, limit: number, offset?: number): ReplayCase[];
+/**
+ * @implements A-SPEC-364
+ * A deterministic sub-sample of a spec store, for holding a repository fixed while varying only how
+ * many specs it has.
+ *
+ * The advantage over a no-graph baseline is 2.5x on this repository (585 specs) and a wash on jarvis
+ * (54). Reading that as a DENSITY effect is, so far, a correlation: the two corpora differ in
+ * language, size, subject and commit style as well. Thinning one corpus is the controlled version of
+ * the same question — everything else stays exactly where it is.
+ *
+ * Evenly spaced over id-sorted specs, never random: a random sample makes each run a different
+ * experiment, and the first thing anyone would ask of a surprising curve is to see it again.
+ * Sampling by stride rather than by prefix matters too — spec ids here run in authoring order, so a
+ * prefix would select the oldest slices and confound "fewer specs" with "older specs".
+ */
+export declare function thinSpecs<T extends {
+    id: string;
+}>(specs: readonly T[], fraction: number): T[];
+/**
+ * @implements A-SPEC-387
+ * Skip the most recent commits, and space the rest out.
+ *
+ * An audit found the benchmark measuring the benchmarker. On this repository, answering "the ten
+ * most recently changed files" — no text, no graph, no query at all — scores Top-10 recall **0.885**
+ * and Top-1 **0.650**, beating every retrieval mechanism tried. The reason is what the 60 most recent
+ * commits ARE: consecutive slices from this session and the ones before it, each touching the same
+ * handful of files. Temporal locality that extreme inflates any recency-flavoured signal, and it is
+ * an artefact of who wrote the commits rather than a property of software maintenance.
+ *
+ * The second corpus does not have it — recency alone scores 0.678 there against a 0.662 baseline —
+ * which is what makes the contamination visible rather than invisible.
+ *
+ * `skipRecent` steps past the working set of whoever ran the benchmark last; `stride` spaces the
+ * remaining cases so that consecutive commits from one feature push are not all sampled.
+ */
+export declare function casesAvoidingRecent(corpus: ReplayCorpus, limit: number, opts: {
+    skipRecent: number;
+    stride: number;
+}): ReplayCase[];

package/dist/holmes/review/replay-corpus.js

@@ -0,0 +1,210 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GEMINI_CLI_MEASUREMENT = exports.GEMINI_CLI_CORPUS = exports.JARVIS_CORPUS = exports.HOLMES_CORPUS = void 0;
+exports.casesFor = casesFor;
+exports.thinSpecs = thinSpecs;
+exports.casesAvoidingRecent = casesAvoidingRecent;
+// @implements A-SPEC-345
+// @implements A-SPEC-387
+const os = __importStar(require("node:os"));
+const path = __importStar(require("node:path"));
+const node_child_process_1 = require("node:child_process");
+const maintenance_analyze_1 = require("../mcp/maintenance-analyze");
+const TS_FAMILY = (f) => f.endsWith('.ts') && !f.endsWith('.d.ts');
+exports.HOLMES_CORPUS = {
+    root: path.resolve(__dirname, '../../..'),
+    // @implements A-SPEC-443 — pinned so the floors stay floors. Our own repository moves too; the
+    // day it broke was only the day the OTHER corpus moved first.
+    pin: '9a2f304f12215561584285b45e4b326156b0a484',
+    sourcePathspec: ['src/holmes'],
+    specPathspec: '.ax/specs',
+    isTest: (f) => f.includes('.test.'),
+    isSource: TS_FAMILY,
+};
+/**
+ * The second corpus. Measured 2026-08-29: 269 source files, 329 commits, 251 eligible cases, 959
+ * truth files, and its own `.ax/specs`. Tests live in `tests/` rather than beside the source, which
+ * is precisely the assumption that had to stop being hard-coded.
+ *
+ * Absent on any machine but this one, and callers skip rather than fail — a benchmark that requires
+ * someone else's checkout is a benchmark nobody else can run.
+ */
+exports.JARVIS_CORPUS = {
+    root: path.join(os.homedir(), 'ai_project', 'jarvis'),
+    // @implements A-SPEC-443 — the last commit before 2026-08-30, the day five landed and moved a
+    // six-decimal assertion with no product change.
+    pin: '7b58e40ba3723b9d85390fdadf1d407cdf7bb9ec',
+    sourcePathspec: ['src', 'tests'],
+    specPathspec: '.ax/specs',
+    isTest: (f) => f.startsWith('tests/') || f.includes('.test.') || f.includes('.spec.'),
+    isSource: (f) => (f.endsWith('.ts') || f.endsWith('.py') || f.endsWith('.js')) && !f.endsWith('.d.ts'),
+};
+// @implements A-SPEC-426
+/**
+ * A third party's repository, with no spec store at all.
+ *
+ * The first two corpora share two properties that turned out to matter: one of them is my own work,
+ * and both carry `.ax/specs`. A verdict recorded in the activation log — "the advantage is entirely
+ * the spec layer" — rested on that. Most projects have no specs, so the question that verdict left
+ * open is what the harness is worth without them.
+ *
+ * Measured: it beats name matching on every axis here, with `specs: []` passed to the analyzer.
+ * Whatever the remaining layers are worth, they are worth something on their own.
+ */
+exports.GEMINI_CLI_CORPUS = {
+    root: path.join(os.homedir(), 'ai_project', 'GeminiAcp', 'reference-gemini-cli'),
+    sourcePathspec: ['packages'],
+    specPathspec: '.ax/specs',
+    isTest: (f) => f.includes('.test.') || f.includes('.spec.'),
+    isSource: (f) => (f.endsWith('.ts') || f.endsWith('.tsx')) && !f.endsWith('.d.ts'),
+};
+/**
+ * @implements A-SPEC-426
+ * Measured 2026-08-29 over 29 commits touching 1 to 12 source files, through the product path with
+ * `specs: []`. The opponent is the NAME-ONLY baseline — not a person. A human comparison on this
+ * corpus is a separate measurement and is not claimed here.
+ */
+exports.GEMINI_CLI_MEASUREMENT = {
+    cases: 29,
+    specsAvailable: 0,
+    opponent: 'name-only baseline (rankBaseline)',
+    shipped: { top1: 0.241, top5Recall: 0.369, top10Recall: 0.483, top10Precision: 0.083 },
+    baseline: { top1: 0.172, top5Recall: 0.186, top10Recall: 0.302, top10Precision: 0.052 },
+};
+/**
+ * The commits this corpus offers as replay cases: those that changed BOTH source and tests, most
+ * recent first. A commit touching only one side cannot score both axes.
+ */
+// @implements A-SPEC-485 — the offset window: the holdout discipline needs cases the main
+// measurement never touched. Default 0 keeps the old signature byte-identical for every pin.
+function casesFor(corpus, limit, offset = 0) {
+    // @implements A-SPEC-443 — an absent corpus and a bad pin are DIFFERENT events, and answering
+    // both with `[]` lets a mis-typed pin masquerade as "this machine does not have that repository".
+    // A benchmark that silently measures nothing is worse than one that fails, so the corpus is
+    // proved present FIRST; only after that does a pin failure become an error.
+    const here = (() => {
+        try {
+            (0, node_child_process_1.execFileSync)('git', ['-C', corpus.root, 'rev-parse', '--git-dir'], { encoding: 'utf8', stdio: 'pipe' });
+            return true;
+        }
+        catch {
+            return false;
+        }
+    })();
+    if (!here)
+        return [];
+    const from = corpus.pin ?? 'HEAD';
+    let raw;
+    try {
+        raw = (0, node_child_process_1.execFileSync)('git', ['-C', corpus.root, 'log', from, '--no-merges', '--format=%H\x1f%s',
+            '--name-only', '--', ...corpus.sourcePathspec], { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
+    }
+    catch (e) {
+        // The repository is here, so this is a configuration error, not a missing corpus.
+        throw new Error(`replay corpus ${corpus.root}: cannot read history from "${from}" — `
+            + 'the pin names no commit in this repository (was history rewritten?). '
+            + `Underlying: ${e instanceof Error ? e.message : String(e)}`);
+    }
+    const all = [];
+    let current = null;
+    for (const line of raw.split('\n')) {
+        if (line.includes('\x1f')) {
+            const [commit, subject] = line.split('\x1f');
+            current = { commit, subject, files: [], tests: [] };
+            all.push(current);
+        }
+        else if (current && line.trim() !== '') {
+            // @implements A-SPEC-274 — git quotes non-ASCII paths; unquoting is not optional here.
+            const file = (0, maintenance_analyze_1.unquoteGitPath)(line.trim());
+            if (!corpus.isSource(file))
+                continue;
+            (corpus.isTest(file) ? current.tests : current.files).push(file);
+        }
+    }
+    return all.filter((c) => c.files.length > 0 && c.tests.length > 0).slice(offset, offset + limit);
+}
+/**
+ * @implements A-SPEC-364
+ * A deterministic sub-sample of a spec store, for holding a repository fixed while varying only how
+ * many specs it has.
+ *
+ * The advantage over a no-graph baseline is 2.5x on this repository (585 specs) and a wash on jarvis
+ * (54). Reading that as a DENSITY effect is, so far, a correlation: the two corpora differ in
+ * language, size, subject and commit style as well. Thinning one corpus is the controlled version of
+ * the same question — everything else stays exactly where it is.
+ *
+ * Evenly spaced over id-sorted specs, never random: a random sample makes each run a different
+ * experiment, and the first thing anyone would ask of a surprising curve is to see it again.
+ * Sampling by stride rather than by prefix matters too — spec ids here run in authoring order, so a
+ * prefix would select the oldest slices and confound "fewer specs" with "older specs".
+ */
+function thinSpecs(specs, fraction) {
+    if (fraction >= 1)
+        return [...specs];
+    if (fraction <= 0)
+        return [];
+    const sorted = [...specs].sort((a, b) => a.id.localeCompare(b.id));
+    const keep = Math.max(1, Math.round(sorted.length * fraction));
+    const stride = sorted.length / keep;
+    const out = [];
+    for (let i = 0; i < keep; i++)
+        out.push(sorted[Math.min(sorted.length - 1, Math.floor(i * stride))]);
+    return [...new Set(out)];
+}
+/**
+ * @implements A-SPEC-387
+ * Skip the most recent commits, and space the rest out.
+ *
+ * An audit found the benchmark measuring the benchmarker. On this repository, answering "the ten
+ * most recently changed files" — no text, no graph, no query at all — scores Top-10 recall **0.885**
+ * and Top-1 **0.650**, beating every retrieval mechanism tried. The reason is what the 60 most recent
+ * commits ARE: consecutive slices from this session and the ones before it, each touching the same
+ * handful of files. Temporal locality that extreme inflates any recency-flavoured signal, and it is
+ * an artefact of who wrote the commits rather than a property of software maintenance.
+ *
+ * The second corpus does not have it — recency alone scores 0.678 there against a 0.662 baseline —
+ * which is what makes the contamination visible rather than invisible.
+ *
+ * `skipRecent` steps past the working set of whoever ran the benchmark last; `stride` spaces the
+ * remaining cases so that consecutive commits from one feature push are not all sampled.
+ */
+function casesAvoidingRecent(corpus, limit, opts) {
+    const skip = Math.max(0, Math.floor(opts.skipRecent));
+    const stride = Math.max(1, Math.floor(opts.stride));
+    // Ask for enough raw cases to survive both the skip and the stride, then take what was wanted.
+    const pool = casesFor(corpus, skip + limit * stride + stride);
+    return pool.slice(skip).filter((_, i) => i % stride === 0).slice(0, limit);
+}