@compaction/cli 0.1.2 → 0.2.0

package/dist/core/context-store-eval-cases.js

@@ -0,0 +1,331 @@
+import { makeContextItem } from "./context-store.js";
+/** Label a fixture `sufficiency` — gated against the sufficiency thresholds. */
+function sufficiency(fixture) {
+    return { ...fixture, role: "sufficiency" };
+}
+/** Label a fixture `adversarial` — NOT gated; a regression anchor with an expected outcome. */
+function adversarial(fixture, expected) {
+    return { ...fixture, role: "adversarial", expected };
+}
+/**
+ * Canonical fixtures for the V0.4 context-store eval harness (rung-1).
+ *
+ * These are SYNTHETIC fixtures — hand-authored stores + queries with known task-critical
+ * items — used to MEASURE deterministic retrieval/assembly behavior. They are not captured
+ * sessions and carry no real-world claim. `created_at` values are fixed so the suite is
+ * fully deterministic (apart from observed latency).
+ */
+const T0 = "2026-06-20T09:00:00.000Z";
+const T1 = "2026-06-21T09:00:00.000Z";
+const T2 = "2026-06-22T09:00:00.000Z";
+function fact(content, source_pointer, created_at) {
+    return makeContextItem({
+        content,
+        source_pointer,
+        source_trace_id: source_pointer.split(" ")[0]?.replace("trace=", ""),
+        recoverability: true,
+        created_at
+    });
+}
+/** Fixture A: the task-critical item is lexically obvious and fits the budget. */
+const lexicalRecall = {
+    name: "lexical-recall",
+    description: "Task-critical item is lexically related to the query and fits the budget.",
+    store: [
+        fact("the auth token must be refreshed before every batch upload", "trace=A message=m1", T0),
+        fact("the user prefers a restrained blue palette in the TUI", "trace=A message=m2", T1),
+        fact("retry the upload with exponential backoff on 503", "trace=A message=m3", T2)
+    ],
+    query: { text: "how do we handle the auth token before an upload batch" },
+    tokenBudget: 1000,
+    goldItems: ["the auth token must be refreshed before every batch upload"],
+    requiredSourcePointers: ["trace=A message=m1"],
+    continuationRequirements: ["auth token", "batch upload"]
+};
+/**
+ * Fixture B: the source-pointer boost is the DECIDING factor. The task-critical item
+ * (trace B) has weak lexical overlap with the query, while a non-gold distractor (trace C)
+ * has high lexical overlap. The budget fits only one item — so without the trace-id boost
+ * the distractor would win the slot and the task-critical item would be lost. The boost is
+ * what makes the right item survive.
+ */
+const goldDecision = "decision: keep the public surface at eight commands for rung-1";
+const sourcePointerRecall = {
+    name: "source-pointer-recall",
+    description: "Under budget pressure, the exact trace-id hint is the deciding factor that keeps a lexically-weak task-critical item.",
+    store: [
+        fact(goldDecision, "trace=B message=m1", T0),
+        // Distractor: high lexical overlap with the query, but NOT task-critical (trace C, unboosted).
+        fact("what did we decide about the command surface here today", "trace=C message=m1", T1)
+    ],
+    query: { text: "what did we decide about the command surface", trace_ids: ["B"] },
+    // Budget fits exactly one item — forces the boost to decide which survives.
+    tokenBudget: makeContextItem({ content: goldDecision, created_at: T0 }).estimated_tokens,
+    goldItems: [goldDecision],
+    requiredSourcePointers: ["trace=B message=m1"],
+    continuationRequirements: ["eight commands"]
+};
+/** Fixture C: the budget forces a drop — the higher-ranked task-critical item must win. */
+const budgetPressure = {
+    name: "budget-pressure",
+    description: "Under a tight budget the task-critical (higher-ranked) item must survive.",
+    store: [
+        fact("critical: the migration must run before the API deploy", "trace=D message=m1", T0),
+        fact("aside: someone mentioned the office coffee machine is broken", "trace=D message=m2", T1)
+    ],
+    query: { text: "what must happen before the api deploy migration" },
+    // Budget large enough for one short item only.
+    tokenBudget: makeContextItem({ content: "critical: the migration must run before the API deploy", created_at: T0 })
+        .estimated_tokens,
+    goldItems: ["critical: the migration must run before the API deploy"],
+    requiredSourcePointers: ["trace=D message=m1"],
+    continuationRequirements: ["migration", "api deploy"]
+};
+export const CONTEXT_STORE_EVAL_FIXTURES = [
+    sufficiency(lexicalRecall),
+    sufficiency(sourcePointerRecall),
+    sufficiency(budgetPressure)
+];
+// ---------------------------------------------------------------------------
+// Richer / adversarial fixtures (V0.4 rung-1, 2026-06-24).
+//
+// PURPOSE: make the eval harness HARDER TO GAME before any threshold is set. Unlike the
+// healthy baseline above (every axis passes), several of these fixtures are designed so a
+// SPECIFIC axis legitimately FAILS — proving the harness reports honest sub-1.0 metrics and
+// cannot be satisfied by a degenerate retriever. Expected per-axis outcomes live next to the
+// assertions in `tests/core/context-store-eval-richer.test.ts`. No thresholds, no claim.
+// ---------------------------------------------------------------------------
+/** Flexible item builder — source pointer / recoverability are optional (for the negative cases). */
+function mk(content, opts = {}) {
+    return makeContextItem({
+        content,
+        source_pointer: opts.pointer,
+        source_trace_id: opts.trace ?? (opts.pointer ? opts.pointer.split(" ")[0]?.replace("trace=", "") : undefined),
+        recoverability: opts.recoverable ?? true,
+        created_at: opts.created_at ?? T0
+    });
+}
+function tokensOf(content) {
+    return makeContextItem({ content, created_at: T0 }).estimated_tokens;
+}
+/** R1 — task-critical item survives budget pressure against several genuinely-retrieved competitors. */
+const budgetPressureMulti = {
+    name: "budget-pressure-multi",
+    description: "Gold outranks several genuinely-retrieved competitors and wins the only budget slot.",
+    store: [
+        mk("critical: rotate the signing key before the release cutover", { pointer: "trace=E message=m1" }),
+        mk("rotate the release plan before cutover today", { pointer: "trace=E message=m2", created_at: T1 }),
+        mk("the signing key was discussed", { pointer: "trace=E message=m3", created_at: T2 })
+    ],
+    query: { text: "rotate the signing key before release cutover" },
+    tokenBudget: tokensOf("critical: rotate the signing key before the release cutover"),
+    goldItems: ["critical: rotate the signing key before the release cutover"],
+    requiredSourcePointers: ["trace=E message=m1"],
+    continuationRequirements: ["signing key", "release cutover"]
+};
+/** R2 — source coverage FAILS: a second required source is dropped under budget, gold still survives. */
+const sourceCoverageMiss = {
+    name: "source-coverage-miss",
+    description: "Two sources are required but the budget fits only the gold item; the other required source is missing.",
+    store: [
+        mk("the api key rotation runbook lives in the wiki", { pointer: "trace=A message=m1" }),
+        mk("the staging database snapshot policy", { pointer: "trace=B message=m1", created_at: T1 })
+    ],
+    query: { text: "where is the api key rotation runbook" },
+    tokenBudget: tokensOf("the api key rotation runbook lives in the wiki"),
+    goldItems: ["the api key rotation runbook lives in the wiki"],
+    requiredSourcePointers: ["trace=A message=m1", "trace=B message=m1"],
+    continuationRequirements: ["rotation runbook"]
+};
+/** R3 — continuation success FAILS: one required marker is absent from the assembled text. */
+const continuationMarkerAbsent = {
+    name: "continuation-marker-absent",
+    description: "The gold item is assembled but a required continuation marker is not present in it.",
+    store: [mk("deploy step: run the smoke test then promote the build", { pointer: "trace=F message=m1" })],
+    query: { text: "what is the deploy step for promote" },
+    tokenBudget: 1000,
+    goldItems: ["deploy step: run the smoke test then promote the build"],
+    requiredSourcePointers: ["trace=F message=m1"],
+    // "smoke test" is present; "rollback plan" is absent -> continuation_success = 0.5.
+    continuationRequirements: ["smoke test", "rollback plan"]
+};
+// R4/R5 share a store: a high-lexical-overlap distractor (trace C) vs a lexically-weak gold (trace B).
+const trapGold = "rollback procedure: restore the retained original artifact";
+const trapDistractor = "which note covers the deploy step we discussed";
+const lexicalTrapStore = [
+    mk(trapGold, { pointer: "trace=B message=m1" }),
+    mk(trapDistractor, { pointer: "trace=C message=m1", created_at: T1 })
+];
+/** R4 — lexical trap, source pointer SAVES it: the trace-id boost lifts the gold over the distractor. */
+const lexicalTrapPointerSaves = {
+    name: "lexical-trap-pointer-saves",
+    description: "An irrelevant high-overlap distractor would win, but the trace-id hint lifts the gold into the slot.",
+    store: lexicalTrapStore,
+    query: { text: "which note covers the deploy rollback step", trace_ids: ["B"] },
+    tokenBudget: tokensOf(trapGold),
+    goldItems: [trapGold],
+    requiredSourcePointers: ["trace=B message=m1"],
+    continuationRequirements: ["retained original"]
+};
+/** R5 — lexical trap, NO pointer: the irrelevant distractor wins the slot, gold is lost (recall 0). */
+const lexicalTrapNoPointer = {
+    name: "lexical-trap-no-pointer",
+    description: "Without a source-pointer hint the high-overlap distractor wins the slot and the gold is legitimately lost.",
+    store: lexicalTrapStore,
+    query: { text: "which note covers the deploy step" },
+    tokenBudget: tokensOf(trapDistractor),
+    goldItems: [trapGold],
+    requiredSourcePointers: ["trace=B message=m1"],
+    continuationRequirements: ["retained original"]
+};
+/** R6 — duplicate/redundant context must NOT inflate success: 3 byte-identical golds collapse to one. */
+const goldCanary = "the canary must pass before full rollout";
+const duplicateNoInflation = {
+    name: "duplicate-no-inflation",
+    description: "Three byte-identical copies of the gold plus one filler assemble to two distinct items, not four.",
+    store: [
+        mk(goldCanary, { pointer: "trace=G message=m1" }),
+        mk(goldCanary, { pointer: "trace=G message=m1", created_at: T1 }),
+        mk(goldCanary, { pointer: "trace=G message=m1", created_at: T2 }),
+        mk("full rollout schedule for the canary release", { pointer: "trace=H message=m1", created_at: T2 })
+    ],
+    query: { text: "when does the canary pass before rollout" },
+    tokenBudget: 1000,
+    goldItems: [goldCanary],
+    requiredSourcePointers: ["trace=G message=m1"],
+    continuationRequirements: ["canary"]
+};
+/** R7 — recoverability DROPS when an assembled item lacks a source pointer. */
+const recoverabilityMissingPointers = {
+    name: "recoverability-missing-pointers",
+    description: "Two task-critical items are assembled; one carries no source pointer, so recoverability is 0.5.",
+    store: [
+        mk("retain the original artifact for rollback", { pointer: "trace=A message=m1" }),
+        // No source pointer: flagged recoverable but cannot actually be recovered (no pointer to recover to).
+        mk("the diff summary should be reviewed before apply", { recoverable: true, created_at: T1 })
+    ],
+    query: { text: "what to retain and review before apply rollback" },
+    tokenBudget: 1000,
+    goldItems: ["retain the original artifact for rollback", "the diff summary should be reviewed before apply"],
+    requiredSourcePointers: ["trace=A message=m1"],
+    continuationRequirements: ["retain", "review"]
+};
+export const CONTEXT_STORE_EVAL_RICHER_FIXTURES = [
+    sufficiency(budgetPressureMulti),
+    adversarial(sourceCoverageMiss, { recall: 1, source_coverage: 0.5, continuation_success: 1, recoverability: 1 }),
+    adversarial(continuationMarkerAbsent, { recall: 1, source_coverage: 1, continuation_success: 0.5, recoverability: 1 }),
+    sufficiency(lexicalTrapPointerSaves),
+    adversarial(lexicalTrapNoPointer, { recall: 0, source_coverage: 0, continuation_success: 0, recoverability: 1 }),
+    sufficiency(duplicateNoInflation),
+    adversarial(recoverabilityMissingPointers, { recall: 1, source_coverage: 1, continuation_success: 1, recoverability: 0.5 })
+];
+// ---------------------------------------------------------------------------
+// Edge-class fixtures (V0.4 rung-1, 2026-06-24) — the missing classes the accepted
+// threshold note (docs/design/v0.4-context-store-thresholds.md) asked for BEFORE any
+// threshold is implemented:
+//   - multi-gold        -> makes recall/coverage FRACTIONAL (today they are binary), so a
+//                          future bar can be calibrated against real partials.
+//   - empty store       -> makes zero-context behavior explicit and honest.
+//   - budget / limit     -> recall + coverage degrade for the RIGHT, distinguishable reason
+//                          (token budget vs retrieval `limit`).
+//   - adversarial anchor -> a low score is the CORRECT behavior (the info is not in the store).
+// Fixtures/tests only. No thresholds, no claim, no command.
+// ---------------------------------------------------------------------------
+// Three equally-relevant task-critical items (each fully contains the query terms, so they
+// tie on lexical overlap and order deterministically by recency). Used by the multi-gold and
+// budget/limit fixtures so exactly which items survive is deterministic.
+const mgQuery = "rollout checklist: migration, signing, oncall";
+const mgA = "rollout checklist migration signing oncall — run the database migration first";
+const mgB = "rollout checklist migration signing oncall — rotate the signing key next";
+const mgC = "rollout checklist migration signing oncall — page the oncall engineer last";
+/** F1 — multi-gold, PARTIAL recall: budget fits two of three task-critical items. */
+const multiGoldPartialRecall = {
+    name: "multi-gold-partial-recall",
+    description: "Three equally-relevant gold items, budget fits two -> recall is a real fraction (not 0/1).",
+    store: [
+        mk(mgA, { pointer: "trace=MG message=a", created_at: T0 }),
+        mk(mgB, { pointer: "trace=MG message=b", created_at: T1 }),
+        mk(mgC, { pointer: "trace=MG message=c", created_at: T2 })
+    ],
+    query: { text: mgQuery },
+    // Room for the two newest (mgC, mgB by recency tie-break); mgA is dropped.
+    tokenBudget: tokensOf(mgC) + tokensOf(mgB),
+    goldItems: [mgA, mgB, mgC],
+    // No required sources / markers here: isolate PARTIAL RECALL as the signal.
+    requiredSourcePointers: [],
+    continuationRequirements: []
+};
+/** F2 — empty store: zero-context behavior is explicit and honest (no crash, no inflation). */
+const emptyStore = {
+    name: "empty-store",
+    description: "Nothing has been accumulated yet; retrieval/assembly must report honest zeros.",
+    store: [],
+    query: { text: "what did we decide about the migration", trace_ids: ["Z"] },
+    tokenBudget: 1000,
+    goldItems: ["a task-critical fact that is not in the store"],
+    requiredSourcePointers: ["trace=Z message=m1"],
+    continuationRequirements: ["migration decision"]
+};
+/** F3 — budget pressure: recall AND coverage degrade together, for the budget reason. */
+const budgetDegradesRecallAndCoverage = {
+    name: "budget-degrades-recall-and-coverage",
+    description: "All three sources are required but the budget fits two -> recall and coverage both drop.",
+    store: [
+        mk(mgA, { pointer: "trace=MG message=a", created_at: T0 }),
+        mk(mgB, { pointer: "trace=MG message=b", created_at: T1 }),
+        mk(mgC, { pointer: "trace=MG message=c", created_at: T2 })
+    ],
+    query: { text: mgQuery },
+    tokenBudget: tokensOf(mgC) + tokensOf(mgB),
+    goldItems: [mgA, mgB, mgC],
+    // All three required -> the dropped item's source is uncovered (coverage degrades WITH recall).
+    requiredSourcePointers: ["trace=MG message=a", "trace=MG message=b", "trace=MG message=c"],
+    continuationRequirements: []
+};
+/** F4 — retrieval `limit` omits one gold: recall degrades because of the LIMIT, not the budget. */
+const limitOmitsGold = {
+    name: "limit-omits-gold",
+    description: "Budget is generous but retrieval limit=2 caps the candidate set -> one gold never reaches assembly.",
+    store: [
+        mk(mgA, { pointer: "trace=MG message=a", created_at: T0 }),
+        mk(mgB, { pointer: "trace=MG message=b", created_at: T1 }),
+        mk(mgC, { pointer: "trace=MG message=c", created_at: T2 })
+    ],
+    query: { text: mgQuery },
+    tokenBudget: 1000, // generous: nothing is dropped BY BUDGET
+    retrievalOptions: { limit: 2 },
+    goldItems: [mgA, mgB, mgC],
+    requiredSourcePointers: [],
+    continuationRequirements: []
+};
+/** F5 — adversarial anchor: a low score is the CORRECT behavior because the info isn't stored. */
+const adversarialCorrectLowScore = {
+    name: "adversarial-correct-low-score",
+    description: "The store holds only unrelated items; the task-critical fact is absent, so recall 0 is correct (not a retriever failure).",
+    store: [
+        mk("the office wifi password was rotated on tuesday", { pointer: "trace=N message=m1", created_at: T0 }),
+        mk("lunch is catered on fridays this quarter", { pointer: "trace=N message=m2", created_at: T1 })
+    ],
+    query: { text: "what is the database migration rollback procedure" },
+    tokenBudget: 1000,
+    // The required fact was never captured into the store -> a faithful retriever cannot surface it.
+    goldItems: ["the database migration rollback procedure is to restore the retained original"],
+    requiredSourcePointers: ["trace=DB message=m1"],
+    continuationRequirements: ["rollback procedure"]
+};
+export const CONTEXT_STORE_EVAL_EDGE_FIXTURES = [
+    // All edge fixtures are adversarial anchors: their partial/zero scores are the CORRECT
+    // behavior (budget / limit / empty / absent-info), not a sufficiency failure.
+    adversarial(multiGoldPartialRecall, { recall: 2 / 3, source_coverage: 1, continuation_success: 1, recoverability: 1 }),
+    adversarial(emptyStore, { recall: 0, source_coverage: 0, continuation_success: 0, recoverability: 1 }),
+    adversarial(budgetDegradesRecallAndCoverage, { recall: 2 / 3, source_coverage: 2 / 3, continuation_success: 1, recoverability: 1 }),
+    adversarial(limitOmitsGold, { recall: 2 / 3, source_coverage: 1, continuation_success: 1, recoverability: 1 }),
+    adversarial(adversarialCorrectLowScore, { recall: 0, source_coverage: 0, continuation_success: 0, recoverability: 1 })
+];
+/** Every fixture (healthy + richer + edge), for the internal sufficiency report. */
+export const CONTEXT_STORE_ALL_EVAL_FIXTURES = [
+    ...CONTEXT_STORE_EVAL_FIXTURES,
+    ...CONTEXT_STORE_EVAL_RICHER_FIXTURES,
+    ...CONTEXT_STORE_EVAL_EDGE_FIXTURES
+];
+//# sourceMappingURL=context-store-eval-cases.js.map

package/dist/core/context-store-eval.d.ts

@@ -0,0 +1,140 @@
+import { type ContextStoreItem, type RetrievalOptions, type RetrievalQuery } from "./context-store.js";
+/**
+ * V0.4 context-store eval harness — rung-1 Increment 2 (local, deterministic, fixture-based).
+ *
+ * PURPOSE: MEASURE whether deterministic retrieval + assembly surfaces the *sufficient*
+ * active context on fixtures, along four axes from the accepted design doc
+ * (`docs/design/v0.4-context-store-retrieval.md`):
+ *   - recall              — did the task-critical source-pointed items survive into the
+ *                           assembled context?
+ *   - source coverage     — are the required sources represented in the assembled context?
+ *   - latency             — observed retrieval + assembly wall time (a measurement, not a
+ *                           gate; it varies by machine).
+ *   - continuation success — can a fresh agent continue from the assembled context?
+ *                           **FIXTURE-BASED** (required markers present), NOT live model
+ *                           replay (true model replay is a future, gated axis — mirrors the
+ *                           engine harness's `task_check` honesty note).
+ * Plus a local recoverability signal (fraction of assembled items carrying a recoverable
+ * source pointer — the substrate invariant).
+ *
+ * WHAT THIS IS NOT (load-bearing):
+ * - This harness MAKES NO CLAIM. It reports per-fixture + aggregate metrics; it does NOT
+ *   decide "sufficient" and bakes in NO pass/fail thresholds. The thresholds that would gate
+ *   a rung-6 large-memory claim are an explicit OPEN founder decision (design doc open
+ *   question 4) — deliberately not encoded here.
+ * - NO model, NO embeddings, NO network. Continuation success is a deterministic substring
+ *   check against the assembled text, NOT model re-execution.
+ * - NO 300M / large-memory / retrieval-correctness claim is produced by running this.
+ */
+export declare const CONTEXT_STORE_EVAL_VERSION = "context-store-eval-v1";
+/** A monotonic clock returning milliseconds. Injected so latency is testable/deterministic. */
+export type Clock = () => number;
+/** The four quality axes a fixture is scored on (latency is a separate measurement). */
+export interface AxisScores {
+    recall: number;
+    source_coverage: number;
+    continuation_success: number;
+    recoverability: number;
+}
+/**
+ * Threshold role of a fixture:
+ * - `sufficiency` — the task-critical context SHOULD be surfaced; the fixture is GATED against
+ *   the sufficiency thresholds (see context-store-sufficiency.ts).
+ * - `adversarial` — a low/partial score is the CORRECT behavior; the fixture is NOT gated, it is
+ *   a regression ANCHOR (its measured scores must keep matching `expected`).
+ * Unlabeled fixtures default to `sufficiency`.
+ */
+export type ContextStoreFixtureRole = "sufficiency" | "adversarial";
+export interface ContextStoreEvalFixture {
+    /** Stable fixture name. */
+    name: string;
+    /** Optional human description of what the fixture exercises. */
+    description?: string;
+    /** Threshold role (default `sufficiency`). Metadata only — `evaluate` ignores it. */
+    role?: ContextStoreFixtureRole;
+    /** For `adversarial` fixtures: the expected (often low/partial) per-axis anchor outcome. */
+    expected?: Partial<AxisScores>;
+    /** The store under evaluation (the accumulated items). */
+    store: ContextStoreItem[];
+    /** The retrieval query (current step's context + optional source hints). */
+    query: RetrievalQuery;
+    /** Token budget for assembly (local estimate). */
+    tokenBudget: number;
+    /** Optional retrieval options (weights, limit). */
+    retrievalOptions?: RetrievalOptions;
+    /**
+     * Task-critical item CONTENT that MUST survive into the assembled context. Compared by
+     * normalized content hash (so authors write the text, not the hash).
+     */
+    goldItems: string[];
+    /** Source pointers that MUST be represented among the assembled items (optional). */
+    requiredSourcePointers?: string[];
+    /**
+     * FIXTURE-BASED continuation requirements: markers/substrings that MUST appear in the
+     * assembled context text for a fresh agent to continue. NOT live model replay.
+     */
+    continuationRequirements: string[];
+}
+export interface ContextStoreEvalResult {
+    version: string;
+    name: string;
+    /** [0,1] — fraction of gold (task-critical) items present in the assembled context. */
+    recall: number;
+    /** [0,1] — fraction of required sources represented (1 when none are required). */
+    source_coverage: number;
+    /** [0,1] — fraction of FIXTURE-BASED continuation markers present in the assembled text. */
+    continuation_success: number;
+    /**
+     * [0,1] — fraction of assembled items that are recoverable, i.e. carry BOTH a recoverable
+     * flag AND a source pointer to recover to (1 if nothing was assembled — vacuously true).
+     */
+    recoverability: number;
+    /** Observed retrieval + assembly time in ms (measurement, machine-dependent). */
+    latency_ms: number;
+    /** Local token estimate of the assembled context. */
+    assembled_token_estimate: number;
+    /** Whether all retrieved items fit under the budget. */
+    within_budget: boolean;
+    /** Raw counts behind the ratios (for transparency). */
+    detail: {
+        gold_total: number;
+        gold_present: number;
+        required_sources_total: number;
+        required_sources_covered: number;
+        continuation_total: number;
+        continuation_met: number;
+        assembled_item_count: number;
+    };
+}
+/**
+ * Evaluate a single fixture. Deterministic apart from `latency_ms` (wall time). Pure with
+ * respect to the inputs — no model, no network.
+ */
+export declare function evaluateContextStoreFixture(fixture: ContextStoreEvalFixture, options?: {
+    clock?: Clock;
+}): ContextStoreEvalResult;
+export interface ContextStoreEvalAggregate {
+    version: string;
+    fixture_count: number;
+    /** Mean across fixtures (each in [0,1]). */
+    mean_recall: number;
+    mean_source_coverage: number;
+    mean_continuation_success: number;
+    mean_recoverability: number;
+    /** Mean observed latency (ms) — measurement, not a gate. */
+    mean_latency_ms: number;
+    results: ContextStoreEvalResult[];
+}
+/**
+ * Evaluate a suite of fixtures and roll up the per-axis means. REPORTS metrics only — no
+ * pass/fail verdict, no thresholds (those are a gated founder decision, by design).
+ */
+export declare function evaluateContextStoreFixtures(fixtures: ContextStoreEvalFixture[], options?: {
+    clock?: Clock;
+}): ContextStoreEvalAggregate;
+/**
+ * Markdown report for a fixture suite. States the four axes, the honest non-claim, and the
+ * fixture-based-continuation caveat explicitly — so a reader cannot mistake these metrics for
+ * a large-memory / retrieval-correctness claim.
+ */
+export declare function formatContextStoreEvalMarkdown(aggregate: ContextStoreEvalAggregate): string;

package/dist/core/context-store-eval.js

@@ -0,0 +1,138 @@
+import { contentHash } from "./content-hash.js";
+import { assembleContext, retrieveContextItems } from "./context-store.js";
+/**
+ * V0.4 context-store eval harness — rung-1 Increment 2 (local, deterministic, fixture-based).
+ *
+ * PURPOSE: MEASURE whether deterministic retrieval + assembly surfaces the *sufficient*
+ * active context on fixtures, along four axes from the accepted design doc
+ * (`docs/design/v0.4-context-store-retrieval.md`):
+ *   - recall              — did the task-critical source-pointed items survive into the
+ *                           assembled context?
+ *   - source coverage     — are the required sources represented in the assembled context?
+ *   - latency             — observed retrieval + assembly wall time (a measurement, not a
+ *                           gate; it varies by machine).
+ *   - continuation success — can a fresh agent continue from the assembled context?
+ *                           **FIXTURE-BASED** (required markers present), NOT live model
+ *                           replay (true model replay is a future, gated axis — mirrors the
+ *                           engine harness's `task_check` honesty note).
+ * Plus a local recoverability signal (fraction of assembled items carrying a recoverable
+ * source pointer — the substrate invariant).
+ *
+ * WHAT THIS IS NOT (load-bearing):
+ * - This harness MAKES NO CLAIM. It reports per-fixture + aggregate metrics; it does NOT
+ *   decide "sufficient" and bakes in NO pass/fail thresholds. The thresholds that would gate
+ *   a rung-6 large-memory claim are an explicit OPEN founder decision (design doc open
+ *   question 4) — deliberately not encoded here.
+ * - NO model, NO embeddings, NO network. Continuation success is a deterministic substring
+ *   check against the assembled text, NOT model re-execution.
+ * - NO 300M / large-memory / retrieval-correctness claim is produced by running this.
+ */
+export const CONTEXT_STORE_EVAL_VERSION = "context-store-eval-v1";
+function defaultClock() {
+    return globalThis.performance.now();
+}
+function normalizeContent(content) {
+    return content.trim().replace(/\s+/g, " ");
+}
+function ratio(numerator, denominator, whenEmpty) {
+    return denominator === 0 ? whenEmpty : numerator / denominator;
+}
+/**
+ * Evaluate a single fixture. Deterministic apart from `latency_ms` (wall time). Pure with
+ * respect to the inputs — no model, no network.
+ */
+export function evaluateContextStoreFixture(fixture, options = {}) {
+    const clock = options.clock ?? defaultClock;
+    const start = clock();
+    const ranked = retrieveContextItems(fixture.store, fixture.query, fixture.retrievalOptions);
+    const assembled = assembleContext(ranked, { tokenBudget: fixture.tokenBudget });
+    const latency_ms = clock() - start;
+    const includedHashes = new Set(assembled.included.map((item) => item.content_sha256));
+    const includedPointers = new Set(assembled.included.map((item) => item.source_pointer).filter((p) => p !== undefined));
+    const goldHashes = fixture.goldItems.map((content) => contentHash(normalizeContent(content)));
+    const goldPresent = goldHashes.filter((hash) => includedHashes.has(hash)).length;
+    const requiredSources = fixture.requiredSourcePointers ?? [];
+    const sourcesCovered = requiredSources.filter((pointer) => includedPointers.has(pointer)).length;
+    const assembledLower = assembled.text.toLowerCase();
+    const continuationMet = fixture.continuationRequirements.filter((marker) => assembledLower.includes(marker.toLowerCase())).length;
+    // Recoverable requires BOTH a recoverable flag AND a source pointer to recover *to* — an
+    // item flagged recoverable but carrying no source pointer cannot actually be recovered.
+    const recoverableCount = assembled.included.filter((item) => item.recoverability === true && item.source_pointer !== undefined).length;
+    return {
+        version: CONTEXT_STORE_EVAL_VERSION,
+        name: fixture.name,
+        recall: ratio(goldPresent, goldHashes.length, 1),
+        source_coverage: ratio(sourcesCovered, requiredSources.length, 1),
+        continuation_success: ratio(continuationMet, fixture.continuationRequirements.length, 1),
+        recoverability: ratio(recoverableCount, assembled.included.length, 1),
+        latency_ms,
+        assembled_token_estimate: assembled.estimated_tokens,
+        within_budget: assembled.within_budget,
+        detail: {
+            gold_total: goldHashes.length,
+            gold_present: goldPresent,
+            required_sources_total: requiredSources.length,
+            required_sources_covered: sourcesCovered,
+            continuation_total: fixture.continuationRequirements.length,
+            continuation_met: continuationMet,
+            assembled_item_count: assembled.included.length
+        }
+    };
+}
+function mean(values) {
+    return values.length === 0 ? 0 : values.reduce((sum, v) => sum + v, 0) / values.length;
+}
+/**
+ * Evaluate a suite of fixtures and roll up the per-axis means. REPORTS metrics only — no
+ * pass/fail verdict, no thresholds (those are a gated founder decision, by design).
+ */
+export function evaluateContextStoreFixtures(fixtures, options = {}) {
+    const results = fixtures.map((fixture) => evaluateContextStoreFixture(fixture, options));
+    return {
+        version: CONTEXT_STORE_EVAL_VERSION,
+        fixture_count: results.length,
+        mean_recall: mean(results.map((r) => r.recall)),
+        mean_source_coverage: mean(results.map((r) => r.source_coverage)),
+        mean_continuation_success: mean(results.map((r) => r.continuation_success)),
+        mean_recoverability: mean(results.map((r) => r.recoverability)),
+        mean_latency_ms: mean(results.map((r) => r.latency_ms)),
+        results
+    };
+}
+function pct(value) {
+    return `${(value * 100).toFixed(1)}%`;
+}
+/**
+ * Markdown report for a fixture suite. States the four axes, the honest non-claim, and the
+ * fixture-based-continuation caveat explicitly — so a reader cannot mistake these metrics for
+ * a large-memory / retrieval-correctness claim.
+ */
+export function formatContextStoreEvalMarkdown(aggregate) {
+    const lines = [
+        "## Context-store eval (rung-1, local-deterministic, fixture-based)",
+        "",
+        `Fixtures: ${aggregate.fixture_count}`,
+        "",
+        "| Axis | Mean |",
+        "|---|---|",
+        `| Recall | ${pct(aggregate.mean_recall)} |`,
+        `| Source coverage | ${pct(aggregate.mean_source_coverage)} |`,
+        `| Continuation success (fixture-based) | ${pct(aggregate.mean_continuation_success)} |`,
+        `| Recoverability | ${pct(aggregate.mean_recoverability)} |`,
+        `| Latency (mean, ms) | ${aggregate.mean_latency_ms.toFixed(2)} |`,
+        "",
+        "### Per fixture",
+        "| Fixture | Recall | Source cov. | Continuation | Recoverability | Latency (ms) | Within budget |",
+        "|---|---|---|---|---|---|---|",
+        ...aggregate.results.map((r) => `| ${r.name} | ${pct(r.recall)} | ${pct(r.source_coverage)} | ${pct(r.continuation_success)} | ` +
+            `${pct(r.recoverability)} | ${r.latency_ms.toFixed(2)} | ${r.within_budget ? "yes" : "no"} |`),
+        "",
+        "**What these numbers are NOT:** this harness reports fixture metrics only. It bakes in NO",
+        "pass/fail thresholds and makes NO large-memory / 300M / retrieval-correctness claim — the",
+        "thresholds that would gate such a claim are an open, gated decision. Continuation success is",
+        "FIXTURE-BASED (required markers present in the assembled text), NOT live model re-execution.",
+        "Latency is an observed measurement and varies by machine."
+    ];
+    return lines.join("\n");
+}
+//# sourceMappingURL=context-store-eval.js.map