@holmes-lab/holmes-kit 0.2.1 → 0.3.1

package/dist/holmes/review/run-replay.d.ts

@@ -0,0 +1,260 @@
+import { evaluationMetrics, impactMetrics, ceilingMetrics } from './evaluation-metrics';
+import { type ReplayCorpus } from './replay-corpus';
+import { type PprArmConfig } from '../assoc/assoc-arm';
+export interface ReplayResult {
+    corpus: string;
+    cases: number;
+    /** Cases whose parent tree could not be materialised — unmeasured, never scored zero. */
+    unmeasured: number;
+    casesWithoutSpecs: number;
+    metrics: ReturnType<typeof evaluationMetrics>;
+    impact: ReturnType<typeof impactMetrics>;
+    /**
+     * @implements A-SPEC-470 — the Phase-B external yardstick for the impact axis: the frozen
+     * string-property heuristic (same-dir / stem-share / prefix-depth), scored on the same cases.
+     * Beating it is the NECESSARY condition for any better-than-a-person phrasing here.
+     */
+    impactBaseline: ReturnType<typeof impactMetrics>;
+    /** Ranking ∪ impact, the set a caller actually receives (A-SPEC-312). */
+    union: ReturnType<typeof impactMetrics>;
+    /**
+     * The denominators. A recall without its ceiling cannot say whether a corpus scored higher because
+     * the analysis did better or because its co-changed files were better connected to begin with —
+     * and those two readings call for opposite decisions. `impactCeiling` walks CODE-to-CODE edges
+     * only (A-SPEC-307: the permissive walk reaches through SPEC nodes, a path no impact algorithm
+     * should follow, and flatters the denominator). `unionCeiling` is the weaker requirement the
+     * lexical axis actually has — the file merely had to be in the parent-time scan.
+     */
+    impactCeiling: ReturnType<typeof ceilingMetrics>;
+    unionCeiling: ReturnType<typeof ceilingMetrics>;
+    impactAchievement: number | null;
+    unionAchievement: number | null;
+    /**
+     * The Phase C arm, scored on the SAME cases and the same ground truth as `metrics` — added beside
+     * the product's ranking, never substituted for it, so both numbers are reported together.
+     */
+    idfArm: ReturnType<typeof evaluationMetrics>;
+    /**
+     * @implements A-SPEC-467 — the PPR associative-recall arm, present only when configs were asked
+     * for. rerank and admit are scored SEPARATELY: admission is a twice-measured failure mode (the
+     * spec layer lost 33% precision to it; caller closure added 59 files with zero correct), so a
+     * single blended number would hide exactly the cost that killed the predecessors.
+     */
+    pprArms?: Array<{
+        label: string;
+        rerank: ReturnType<typeof evaluationMetrics>;
+        admit: ReturnType<typeof evaluationMetrics>;
+    }>;
+    /**
+     * @implements A-SPEC-468 — the impact-axis grading arm: same cases, same expected sets as
+     * `impact`, predictions are PPR-mass top-k with the seed file excluded. recall AND precision are
+     * both carried — the closure's failure was precision-less expansion, so a recall-only number
+     * would repeat the caller-closure mistake.
+     */
+    pprImpact?: Array<{
+        label: string;
+        k: number;
+        recall: number | null;
+        precision: number | null;
+    }>;
+    /**
+     * @implements A-SPEC-471 — the precision frontier: the ADOPTED ranking with only the emission
+     * rule (alpha x k) swept, present only when asked. Measures how far mass alone can climb toward
+     * the human high-precision profile, and at what recall cost.
+     */
+    impactFrontier?: Array<{
+        alpha: number;
+        k: number;
+        recall: number | null;
+        precision: number | null;
+    }>;
+    /**
+     * @implements A-SPEC-472 — cumulative macro-averaged trajectories of the three iteration arms
+     * (A static, B re-seed, C judged expansion) under one emission budget and one oracle. Present
+     * only when asked; the oracle feedback is an optimistic upper bound and is scored as such.
+     */
+    iterative?: Array<{
+        arm: 'A' | 'B' | 'C' | 'D';
+        round: number;
+        recall: number | null;
+        precision: number | null;
+        emitted: number;
+    }>;
+    /**
+     * @implements A-SPEC-475 — the semantic localization arm: an INJECTED embedder (API, local or
+     * cache — the harness is only a consumer of vectors) ranks parent-time files against the commit
+     * subject. rerank and admit are scored separately, the same discipline every arm here follows.
+     */
+    semanticArm?: {
+        rerank: ReturnType<typeof evaluationMetrics>;
+        admit: ReturnType<typeof evaluationMetrics>;
+    };
+    /**
+     * @implements A-SPEC-488 — per-gate simulation of the PRODUCT rerank policy: a case scored
+     * with the sem order when its gate opens, with the product order otherwise. Present only when
+     * semanticArm.gates is asked; the plain semanticArm rows are untouched.
+     */
+    semanticGates?: Array<{
+        gate: string;
+        metrics: ReturnType<typeof evaluationMetrics>;
+    }>;
+    /**
+     * @implements A-SPEC-493 — the union answer EXTENDED by the semantic admit head: a set union,
+     * so recall is monotonically ≥ the product union's by construction; what this block measures
+     * is the size of the ceiling gain and the dilution cost the agent would absorb.
+     */
+    unionPlusAdmit?: Array<{
+        k: number;
+        recall: number | null;
+        precision: number | null;
+    }>;
+    /**
+     * @implements A-SPEC-479 — pre-emission verification of the union answer: an independent
+     * per-candidate verdict (semantic cosine or lexical IDF overlap, swept over tau) behind an
+     * explicit head guard. Present only when asked. Measures whether precision can be bought with
+     * the recall budget the union surface holds above the modeled human band — the S-488 frontier
+     * proved rank mass alone cannot climb there.
+     */
+    unionVerify?: Array<{
+        signal: 'sem' | 'lex';
+        tau: number;
+        guard: number;
+        recall: number | null;
+        precision: number | null;
+    }>;
+    /**
+     * @implements A-SPEC-480 — structural-coherence verification of the union tail: keep the top-h
+     * head, accept a tail candidate only if it shares a CODE edge or a spec anchor with the head.
+     * Evidence the ranker never consumed (the emission's internal relations), on the parent-time
+     * graph only. Present only when asked.
+     */
+    unionCoherence?: Array<{
+        mode: 'edge' | 'spec' | 'any';
+        h: number;
+        recall: number | null;
+        precision: number | null;
+    }>;
+    /**
+     * @implements A-SPEC-481 — position-independent verdicts over the emission's own structure:
+     * degree (keep members coupled to ≥ d others) and cluster (keep the largest coherent component,
+     * ties as a union). No head to be wrong about — S-498's head-error mode is removed by design;
+     * the guard stays the only positional device. Present only when asked.
+     */
+    coherenceRobust?: Array<{
+        family: 'degree' | 'cluster';
+        rel: 'edge' | 'any';
+        d: number | null;
+        guard: number;
+        recall: number | null;
+        precision: number | null;
+    }>;
+    /**
+     * @implements A-SPEC-482 — anchor + consensus combinations of the S-498/499 published bests:
+     * OR recovers truths a wrong head cut, AND refines coincidental couplings away. The last
+     * file-granularity combination; a miss here declares the evidence class exhausted.
+     */
+    anchoredConsensus?: Array<{
+        mode: 'edge' | 'spec' | 'any';
+        h: number;
+        d: number;
+        comb: 'or' | 'and';
+        recall: number | null;
+        precision: number | null;
+    }>;
+    /**
+     * @implements A-SPEC-483 — content-level verification of the union tail: does the request's
+     * rare vocabulary occur in the candidate's BODY, and do candidate and head reference each
+     * other by name? Bodies come from the materialized parent tree only. Present only when asked.
+     */
+    contentVerify?: Array<{
+        signal: string;
+        guard: number;
+        recall: number | null;
+        precision: number | null;
+    }>;
+    elapsedMs: number;
+}
+/**
+ * Run the point-in-time replay against ANY corpus.
+ *
+ * Lifted verbatim from the benchmark that lived inside `maintenance-analyze.test.ts`, where it was
+ * bound to this repository's paths. Every verdict in this work came from that single corpus, so the
+ * measurements could not be falsified by a second one — this is what makes them falsifiable.
+ *
+ * Equivalence is the load-bearing property of the extraction: this repository's numbers must come
+ * out unchanged, or subsequent comparisons measure the refactor instead of the corpora.
+ */
+export declare function runReplay(corpus: ReplayCorpus, limit: number, opts?: {
+    pprConfigs?: PprArmConfig[];
+    impactFrontier?: boolean;
+    iterativeArms?: boolean;
+    /** @implements A-SPEC-488 — gates:true adds the gate-simulation rows; absent = unchanged. */
+    semanticArm?: {
+        embedBatch: (texts: string[], kind: 'query' | 'doc') => Promise<number[][]>;
+        gates?: boolean;
+    };
+    unionVerify?: {
+        embedBatch: (texts: string[], kind: 'query' | 'doc') => Promise<number[][]>;
+        semTaus?: number[];
+    };
+    unionCoherence?: boolean;
+    coherenceRobust?: boolean;
+    anchoredConsensus?: boolean;
+    contentVerify?: boolean;
+    /** @implements A-SPEC-485 — holdout window start; default 0 is the pinned main window. */
+    offset?: number;
+    /**
+     * @implements A-SPEC-487 — per-case dump for the blind judgment protocol. The pins stay on
+     * the 1-pass result; the dump carries the 2-pass (semantic-injected) output when
+     * productSemantic is present. The RUNNER owns blinding: truth goes to a separate file.
+     */
+    caseDump?: (row: {
+        commit: string;
+        subject: string;
+        candidates: Array<{
+            file: string;
+            score: number;
+            symbols: string[];
+            evidence: string[];
+        }>;
+        rankedImpact: Array<{
+            file: string;
+            score: number;
+            semCos?: number;
+        }>;
+        truthFiles: string[];
+        /** @implements A-SPEC-489 — present only when dumpBodies was asked. */
+        bodies?: Record<string, string>;
+    }) => void;
+    /** @implements A-SPEC-487 — 2-pass semantic injection for the dump only, never the pins. */
+    productSemantic?: {
+        embedBatch: (texts: string[], kind: 'query' | 'doc') => Promise<number[][]>;
+        label: string;
+    };
+    /**
+     * @implements A-SPEC-489 — body EXCERPTS for the content-access judgment protocol: the top
+     * `top` union-order candidates get their first `chars` characters read from the MATERIALIZED
+     * parent tree (never the present one). Meaningful only alongside caseDump; feeds no metric.
+     */
+    dumpBodies?: {
+        top: number;
+        chars: number;
+        /** @implements A-SPEC-490 — drop leading license blocks before excerpting; default false keeps S-507 bytes. */
+        skipLicense?: boolean;
+    };
+}): Promise<ReplayResult>;
+/**
+ * @implements A-SPEC-475
+ * One case of the semantic arm, pure given an embedder: documents follow the S-491 convention
+ * (`sourcePath + qualified symbol names`, 2000 chars) so numbers stay comparable across the
+ * traceability and localization experiments; the query is the commit subject as-is. rerank reuses
+ * `rerankPool` — the pool-stays-fixed invariant is shared code, not a re-promise.
+ */
+export declare function semanticCaseRanking(files: ReadonlyArray<{
+    sourcePath: string;
+    symbolNames: readonly string[];
+}>, subject: string, pool: readonly string[], topN: number, embedBatch: (texts: string[], kind: 'query' | 'doc') => Promise<number[][]>): Promise<{
+    rerank: string[];
+    admit: string[];
+    docTexts: string[];
+}>;