@holmes-lab/holmes-kit 0.2.0 → 0.3.0

package/dist/holmes/review/graph-verifier.js

@@ -0,0 +1,62 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.corroborated = corroborated;
+exports.verifyByGraph = verifyByGraph;
+/** Files reachable from `seed` by a call edge in either direction, or by a shared spec anchor. */
+function corroborated(seed, scanned) {
+    const out = new Set([seed]);
+    const byPath = new Map(scanned.map((f) => [f.sourcePath, f]));
+    const seedFile = byPath.get(seed);
+    if (!seedFile)
+        return out;
+    const declaredIn = new Map();
+    for (const f of scanned) {
+        for (const sym of f.symbols) {
+            const bare = sym.qualifiedName.includes('.')
+                ? sym.qualifiedName.slice(sym.qualifiedName.lastIndexOf('.') + 1) : sym.qualifiedName;
+            for (const key of [sym.qualifiedName, bare])
+                if (!declaredIn.has(key))
+                    declaredIn.set(key, f.sourcePath);
+        }
+    }
+    // Outgoing calls from the seed.
+    for (const e of seedFile.edges ?? []) {
+        if (e.rel !== 'calls')
+            continue;
+        const target = declaredIn.get(e.to);
+        if (target)
+            out.add(target);
+    }
+    // Incoming calls to the seed's symbols, and files sharing one of its spec anchors.
+    const seedSymbols = new Set(seedFile.symbols.flatMap((s) => {
+        const bare = s.qualifiedName.includes('.') ? s.qualifiedName.slice(s.qualifiedName.lastIndexOf('.') + 1) : s.qualifiedName;
+        return [s.qualifiedName, bare];
+    }));
+    const seedAnchors = new Set(seedFile.implementsSpecs ?? []);
+    for (const f of scanned) {
+        if (f.sourcePath === seed)
+            continue;
+        if ((f.edges ?? []).some((e) => e.rel === 'calls' && seedSymbols.has(e.to))) {
+            out.add(f.sourcePath);
+            continue;
+        }
+        if (seedAnchors.size > 0 && (f.implementsSpecs ?? []).some((id) => seedAnchors.has(id)))
+            out.add(f.sourcePath);
+    }
+    return out;
+}
+/**
+ * Demote candidates the graph cannot connect to the top-ranked one.
+ *
+ * The top candidate is taken as the anchor of the answer, which is the assumption that makes this
+ * cheap and also the one that can sink it: if the first hit is wrong, corroboration is measured
+ * against the wrong place. That is why the demotion is partial and the list is never truncated.
+ */
+function verifyByGraph(ranked, scanned, opts) {
+    if (ranked.length === 0 || opts.demotion >= 1)
+        return [...ranked];
+    const ok = corroborated(ranked[0].file, scanned);
+    return ranked
+        .map((h) => ({ file: h.file, score: ok.has(h.file) ? h.score : h.score * opts.demotion }))
+        .sort((a, b) => b.score - a.score || a.file.localeCompare(b.file));
+}

package/dist/holmes/review/hop-ablation.d.ts

@@ -0,0 +1,100 @@
+/**
+ * What the knowledge-graph hop actually contributes, isolated.
+ *
+ * A-SPEC-407 put a one-hop proximity list into the rank fusion and was adopted because Top-5 recall
+ * rose on the clean slice of both corpora. Two things were wrong with that as evidence. It was
+ * measured on a hand-built arm rather than through `analyzeMaintenance`, and it was never checked
+ * against a different set of cases — the exact mistake this repository's own record warns about,
+ * twice.
+ *
+ * So this is the ablation done properly: the SHIPPED path, `HOP_SEEDS` at 5 and at 0, over three
+ * slices. The result is that the contribution flips sign. It is worth +0.061 and +0.040 of Top-5
+ * recall on the two slices the adoption was based on, and −0.047 on a third that nobody had
+ * measured. Weighted over all 92 cases it is +0.017 — positive, and small enough that "the graph
+ * improves the ranking" is only conditionally true.
+ *
+ * The hop is kept: net positive, and two of three slices favour it. What is NOT kept is the
+ * impression that it reliably helps. A later change to this axis has to be measured on all three.
+ */
+export interface HopArm {
+    top1: number;
+    top5Recall: number;
+    top10Recall: number;
+    top10Precision: number;
+    /** Truth files inside the forty exposed to the caller — the admission axis, as opposed to order. */
+    truthInsideExposed: number;
+}
+export interface HopSlice {
+    slice: string;
+    cases: number;
+    withHop: HopArm;
+    withoutHop: HopArm;
+}
+/** Measured 2026-08-29 through analyzeMaintenance, HOP_SEEDS 5 against 0. */
+export declare const HOP_ABLATION: readonly HopSlice[];
+/** Top-5 recall the hop is worth on one slice. Negative means the hop cost recall there. */
+export declare function hopContribution(slice: HopSlice): number;
+/** Case-weighted mean contribution — the single number, with its smallness visible. */
+export declare function weightedHopContribution(slices: readonly HopSlice[]): number;
+/**
+ * Why the jarvis `union recall` floor fell from 0.8813 to 0.8508 when the hop went in.
+ *
+ * It was recorded as an open question, with a hypothesis: the hop pushes lexically-matched files out
+ * of the top ten, which shrinks the symbols the call closure is seeded from, so `impactedFiles`
+ * loses answers. Measured on the pinned case set with HOP_SEEDS at 5 and 0, that hypothesis is
+ * WRONG — not one answer left through the closure.
+ *
+ * What actually happened: the hop dropped two answers that had been in `ranked` and added three.
+ * File-for-file that is a net gain. The macro metric fell because both losses landed on cases whose
+ * change touched exactly ONE file, and a singleton case forfeits a whole 1.0 of per-case recall,
+ * while three gains spread across many-file cases earn only fractions.
+ *
+ * The lesson is about the metric, not the ranker: on a corpus where a change touches anywhere from
+ * one file to sixty-four, a per-case mean and a per-file count can move in opposite directions, and
+ * neither is the whole answer.
+ */
+export interface UnionDrop {
+    slice: string;
+    lostFiles: number;
+    lostFromRanked: number;
+    lostFromImpacted: number;
+    lostOnSingletonCases: number;
+    gainedFiles: number;
+    macroUnionRecallWithHop: number;
+    macroUnionRecallWithoutHop: number;
+    /** The other axis, which fell for the reason A-SPEC-378 already named. */
+    impactRecallWithHop: number;
+    impactRecallWithoutHop: number;
+}
+/** Measured 2026-08-29 on `casesFor(JARVIS_CORPUS, 30)` — the set the floors are pinned on. */
+export declare const UNION_DROP: UnionDrop;
+/** Net answers the hop put into the union, counted as files rather than averaged over cases. */
+export declare function unionFileDelta(d: UnionDrop): number;
+/**
+ * What the rank fusion (A-SPEC-405) is worth, measured the same way the hop was.
+ *
+ * A-SPEC-405 was approved under a criterion of its own writing — "no existing figure falls on either
+ * clean slice" — and then adopted with jarvis Top-10 precision down from 0.145 to 0.135. I recorded
+ * the violation and left the revert decision open. This is the number that decision needs, taken
+ * through the product path rather than the arm the original used.
+ *
+ * The violation is REAL and is confirmed here, and a second cost the original never measured shows
+ * up with it: on jarvis the fusion also leaves two fewer answers inside the forty the caller sees.
+ *
+ * It is kept anyway, and the trade is stated rather than smoothed: +0.172 and +0.069 of Top-5 recall
+ * against −0.012 of precision and two answers. Nobody should read this as the criterion being met.
+ */
+export interface FusionSlice {
+    slice: string;
+    cases: number;
+    withFusion: HopArm;
+    withoutFusion: HopArm;
+}
+/** Measured 2026-08-29 through analyzeMaintenance, `fuseTail()` on against off. */
+export declare const FUSION_ABLATION: readonly FusionSlice[];
+/** Where the fusion COSTS something, named rather than averaged away. */
+export declare function fusionCosts(slices: readonly FusionSlice[]): Array<{
+    slice: string;
+    metric: string;
+    delta: number;
+}>;

package/dist/holmes/review/hop-ablation.js

@@ -0,0 +1,89 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FUSION_ABLATION = exports.UNION_DROP = exports.HOP_ABLATION = void 0;
+exports.hopContribution = hopContribution;
+exports.weightedHopContribution = weightedHopContribution;
+exports.unionFileDelta = unionFileDelta;
+exports.fusionCosts = fusionCosts;
+/** Measured 2026-08-29 through analyzeMaintenance, HOP_SEEDS 5 against 0. */
+exports.HOP_ABLATION = [
+    {
+        slice: 'holmes 0-29',
+        cases: 30,
+        withHop: { top1: 0.300, top5Recall: 0.554, top10Recall: 0.634, top10Precision: 0.133, truthInsideExposed: 47 },
+        withoutHop: { top1: 0.300, top5Recall: 0.493, top10Recall: 0.633, top10Precision: 0.127, truthInsideExposed: 45 },
+    },
+    {
+        slice: 'jarvis 0-29',
+        cases: 30,
+        withHop: { top1: 0.533, top5Recall: 0.528, top10Recall: 0.612, top10Precision: 0.133, truthInsideExposed: 52 },
+        withoutHop: { top1: 0.533, top5Recall: 0.488, top10Recall: 0.612, top10Precision: 0.135, truthInsideExposed: 53 },
+    },
+    {
+        // The slice the adoption never saw. It is the one that disagrees.
+        slice: 'jarvis 32-63',
+        cases: 32,
+        withHop: { top1: 0.469, top5Recall: 0.463, top10Recall: 0.611, top10Precision: 0.204, truthInsideExposed: 87 },
+        withoutHop: { top1: 0.469, top5Recall: 0.510, top10Recall: 0.627, top10Precision: 0.204, truthInsideExposed: 88 },
+    },
+];
+/** Top-5 recall the hop is worth on one slice. Negative means the hop cost recall there. */
+function hopContribution(slice) {
+    return slice.withHop.top5Recall - slice.withoutHop.top5Recall;
+}
+/** Case-weighted mean contribution — the single number, with its smallness visible. */
+function weightedHopContribution(slices) {
+    const cases = slices.reduce((a, s) => a + s.cases, 0);
+    if (cases === 0)
+        return 0;
+    return slices.reduce((a, s) => a + hopContribution(s) * s.cases, 0) / cases;
+}
+/** Measured 2026-08-29 on `casesFor(JARVIS_CORPUS, 30)` — the set the floors are pinned on. */
+exports.UNION_DROP = {
+    slice: 'jarvis newest-30',
+    lostFiles: 2,
+    lostFromRanked: 2,
+    lostFromImpacted: 0,
+    lostOnSingletonCases: 2,
+    gainedFiles: 3,
+    macroUnionRecallWithHop: 0.8507936507936509,
+    macroUnionRecallWithoutHop: 0.8813492063492064,
+    impactRecallWithHop: 0.0980392156862745,
+    impactRecallWithoutHop: 0.20588235294117646,
+};
+/** Net answers the hop put into the union, counted as files rather than averaged over cases. */
+function unionFileDelta(d) {
+    return d.gainedFiles - d.lostFiles;
+}
+/** Measured 2026-08-29 through analyzeMaintenance, `fuseTail()` on against off. */
+exports.FUSION_ABLATION = [
+    {
+        slice: 'holmes clean-30',
+        cases: 30,
+        withFusion: { top1: 0.300, top5Recall: 0.554, top10Recall: 0.634, top10Precision: 0.133, truthInsideExposed: 47 },
+        withoutFusion: { top1: 0.300, top5Recall: 0.382, top10Recall: 0.604, top10Precision: 0.117, truthInsideExposed: 44 },
+    },
+    {
+        slice: 'jarvis clean-30',
+        cases: 30,
+        withFusion: { top1: 0.533, top5Recall: 0.528, top10Recall: 0.612, top10Precision: 0.133, truthInsideExposed: 52 },
+        withoutFusion: { top1: 0.533, top5Recall: 0.459, top10Recall: 0.612, top10Precision: 0.145, truthInsideExposed: 54 },
+    },
+];
+/** Where the fusion COSTS something, named rather than averaged away. */
+function fusionCosts(slices) {
+    const out = [];
+    for (const s of slices) {
+        const pairs = [
+            ['top1', s.withFusion.top1, s.withoutFusion.top1],
+            ['top5Recall', s.withFusion.top5Recall, s.withoutFusion.top5Recall],
+            ['top10Recall', s.withFusion.top10Recall, s.withoutFusion.top10Recall],
+            ['top10Precision', s.withFusion.top10Precision, s.withoutFusion.top10Precision],
+            ['truthInsideExposed', s.withFusion.truthInsideExposed, s.withoutFusion.truthInsideExposed],
+        ];
+        for (const [metric, a, b] of pairs)
+            if (a < b)
+                out.push({ slice: s.slice, metric, delta: a - b });
+    }
+    return out;
+}

package/dist/holmes/review/manual-baseline.d.ts

@@ -0,0 +1,209 @@
+/**
+ * The "strong model with plain file search" baseline (Goal Phase B).
+ *
+ * Every other arm in this directory compares the harness against ANOTHER PROGRAM. That answers "is
+ * the spec layer worth its cost", never the Goal's actual claim: that the harness helps an agent
+ * beat what a competent developer reaches by reading and grepping. So this arm is me — given only
+ * the request text and a materialized tree, ranking by grep counts over the query terms, with the
+ * truth WITHHELD until the picks were written to disk.
+ *
+ * The rule this slice was approved under: if the baseline beats the shipped harness, the claim of
+ * surpassing a human does not hold, and that fact gets recorded rather than buried. It did beat it
+ * on recall. The record below is that measurement, kept so a later change has to move it.
+ *
+ * The `shipped` lists below are the pipeline AS IT WAS when this baseline ran (S-404). A-SPEC-405
+ * later changed the re-ranking in response to what this record showed; re-scoring the same eight
+ * cases through the new pipeline gives Top-5 recall 0.175 -> 0.217 and Top-10 recall 0.471 -> 0.533,
+ * still short of the 0.404 / 0.554 reached here by reading and grepping. The record is deliberately
+ * NOT refreshed: it is the measurement that motivated the change, and rewriting it would erase the
+ * comparison it exists to preserve.
+ *
+ * n=8. A sample this small cannot separate a real gap from noise; report the n alongside the
+ * numbers and do not treat a large difference here as a conclusion.
+ */
+import { type ReplayOutcome } from './evaluation-metrics';
+export interface ManualBaselineCase {
+    /** Commit subject with spec citations stripped — the only input the picks were made from. */
+    query: string;
+    /** Files chosen by grep-ranked reading, frozen before the truth was read. */
+    picks: string[];
+    /** What the shipped pipeline returned for the same query on the same tree. */
+    shipped: string[];
+    truthFiles: string[];
+}
+export interface ManualBaselineRecord {
+    corpus: string;
+    measuredOn: string;
+    caveat: string;
+    cases: ManualBaselineCase[];
+}
+export declare const MANUAL_BASELINE_RECORD: ManualBaselineRecord;
+export declare const MANUAL_BASELINE_N: number;
+/** Score the frozen picks with the same metric code every other arm uses. */
+export declare function manualBaselineOutcomes(record: ManualBaselineRecord): ReplayOutcome[];
+export declare function shippedOutcomes(record: ManualBaselineRecord): ReplayOutcome[];
+/**
+ * Per-metric verdict, computed from the record rather than asserted by hand: `true` only where the
+ * shipped harness is strictly ahead of the grep-and-read baseline.
+ */
+export declare function shippedBeatsManual(record: ManualBaselineRecord): {
+    top1: boolean;
+    top5Recall: boolean;
+    top10Recall: boolean;
+    top10Precision: boolean;
+};
+/**
+ * @implements A-SPEC-408
+ * The second blind set. Set 1 turned out to be markedly harder than the corpus average — Top-10
+ * recall 0.404 against 0.688 for the other 22 cases of the same 30 — so its verdict could have been
+ * a property of those eight commits rather than of the harness. These eight were drawn the same way
+ * from further down the same slice, their truth was unread until the picks were on disk, and the
+ * asymmetry came out in the same direction: the harness leads Top-1, reading and grepping leads
+ * recall.
+ *
+ * One difference from set 1 that matters when reading the two together: these `shipped` lists come
+ * from the pipeline AFTER A-SPEC-405/406/407, while set 1's are the S-404-era pipeline, kept
+ * deliberately unrefreshed. So compare each set's two rows to each other, never set 1's harness row
+ * to set 2's.
+ */
+export declare const MANUAL_BASELINE_SET_2: ManualBaselineRecord;
+/**
+ * @implements A-SPEC-410
+ * The comparison once BOTH sides get the same thing.
+ *
+ * A-SPEC-404 and 408 compared the harness ALONE against a person reading and grepping, so the
+ * presence of a judgement step was tangled up with the tools. Two asymmetries were then removed,
+ * one per round, each on a set whose truth was unread until the picks were frozen:
+ *
+ *   set "evidence-shown" (cases 32-39): the harness arm had been given file PATHS only while the
+ *     grep arm saw occurrence counts. Fixed by actually showing what A-SPEC-409 carries — lexical
+ *     score, matched symbols, via-specs, commit prose, graph-hop count, recency.
+ *   set "two-step" (cases 40-47): the grep arm judged TWICE (choosing the search terms, then the
+ *     files) while the harness arm judged once. Fixed by letting the harness be re-queried after
+ *     seeing its first result, which is what an agent actually does.
+ *
+ * Both sets: width 20 on each side, picks frozen before the truth was read.
+ *
+ * Pooled result (n=16). The verdict differs BY AXIS and is stated that way:
+ *   Top-1        harness+judgement 0.688  >  grep+judgement 0.625
+ *   Top-5 recall harness+judgement 0.712  >  grep+judgement 0.677
+ *   Top-10 recall harness+judgement 0.764 <  grep+judgement 0.821
+ *   harness ALONE 0.438 / 0.505 / 0.651 — behind the human method on every recall axis.
+ *
+ * So the harness plus a judging caller leads at the top of the list and still trails on deep
+ * recall, which is the axis the Goal is worded around. "Surpasses a human developer" does not hold.
+ */
+export interface FairCase {
+    set: string;
+    query: string;
+    /** The harness's own top ten, no judgement applied. */
+    harnessAlone: string[];
+    /** Ten chosen from the harness's evidence-annotated pool. */
+    harnessJudged: string[];
+    /** Ten chosen from grep output, the same judgement, the other tool. */
+    grepJudged: string[];
+    truthFiles: string[];
+}
+export declare const FAIR_COMPARISON: {
+    measuredOn: string;
+    caveat: string;
+    cases: FairCase[];
+};
+/**
+ * The third corpus, measured against a person instead of a program.
+ *
+ * A-SPEC-426 put the harness against a name-only matcher on this repository -- no spec store, a
+ * third party's code -- and it won all four axes. That opponent was a program, and the note there
+ * said the human comparison was still owed. This is it, under the A-SPEC-410 protocol: width 40 on
+ * both sides, the harness's evidence actually shown, picks frozen before the truth was read, eight
+ * cases whose answers I had never seen.
+ *
+ * grep and judgement win every axis, and not narrowly.
+ *
+ * The cause is not the reranking. Of nineteen truth files, NINE were anywhere in the forty the
+ * harness exposed, and two cases offered none at all. The clearest example: for "properly format
+ * markdown in the AskUser tool", the answer is packages/core/src/tools/ask-user.ts; one grep found
+ * it immediately and it appears nowhere in the harness's forty. Reranking cannot recover a file
+ * that was never offered.
+ *
+ * Read together with A-SPEC-426 rather than against it. Both are true, and the pair is the finding:
+ * beating the automated baseline does not imply beating a person.
+ */
+export interface ThirdCorpusCase extends FairCase {
+    pool40: string[];
+}
+export declare const THIRD_CORPUS_COMPARISON: {
+    corpus: string;
+    opponent: string;
+    note: string;
+    caveat: string;
+    harnessAloneTop10RecallHere: number;
+    harnessAloneTop10RecallOn29: number;
+    cases: ThirdCorpusCase[];
+};
+/** How much of the truth the exposed pool held at all -- the ceiling any reranking could reach. */
+export declare function poolCoverage(r: {
+    cases: ThirdCorpusCase[];
+}): {
+    truthTotal: number;
+    inPool: number;
+    casesWithNothing: number;
+};
+/**
+ * What A-SPEC-428's window guarantee bought, measured two ways.
+ *
+ * Controlled first: the SAME eight cases A-SPEC-427 lost on, re-run after the change. Nine of
+ * nineteen answers were inside the exposed forty before; fourteen after. Two cases offered nothing
+ * at all; one after. Answers the lexical layer ranked in its top ten but the pipeline never showed:
+ * five before, none after. Same commits, same queries — the increase belongs to the change.
+ *
+ * Then a fresh blind set of eight, truth unread until the picks were frozen. The gap to a person
+ * with grep is much smaller than it was, and precision now favours the harness alone. It is still a
+ * gap: grep and judgement lead both recall axes.
+ *
+ * One result here goes the other way and is kept because it does: on this set MY reranking was
+ * WORSE than the harness's own ordering (Top-10 recall 0.446 against 0.527). The judgement step is
+ * not a free improvement, and the earlier sets where it helped did not establish that it always
+ * would.
+ */
+export declare const WINDOW_GUARANTEE_GAIN: {
+    readonly sameCases: {
+        readonly corpus: "reference-gemini-cli, the eight cases of A-SPEC-427";
+        readonly truthTotal: 19;
+        readonly inPoolBefore: 9;
+        readonly inPoolAfter: 14;
+        readonly casesWithNothingBefore: 2;
+        readonly casesWithNothingAfter: 1;
+        readonly lexTop10BuriedBefore: 5;
+        readonly lexTop10BuriedAfter: 0;
+    };
+    readonly freshSet: {
+        readonly corpus: "reference-gemini-cli, eight cases whose truth was unread until the picks were frozen";
+        readonly cases: 8;
+        readonly harnessAlone: {
+            readonly top1: 0.25;
+            readonly top5Recall: 0.286;
+            readonly top10Recall: 0.527;
+            readonly top10Precision: 0.125;
+        };
+        readonly grepJudged: {
+            readonly top1: 0.25;
+            readonly top5Recall: 0.39;
+            readonly top10Recall: 0.595;
+            readonly top10Precision: 0.112;
+        };
+        readonly harnessJudged: {
+            readonly top1: 0.125;
+            readonly top5Recall: 0.348;
+            readonly top10Recall: 0.446;
+            readonly top10Precision: 0.1;
+        };
+        readonly poolCoverage: {
+            readonly inPool: 13;
+            readonly truthTotal: 18;
+            readonly casesWithNothing: 1;
+        };
+    };
+    readonly earlierSetHarnessAloneTop10Recall: 0.268;
+    readonly caveat: "두 세트는 커밋이 다르므로 직접 비교하지 않는다 — 새 세트가 더 쉽다(하네스 단독 0.527 대 0.268). not comparable.";
+};