@oscharko-dev/keiko-local-knowledge 0.2.0

package/dist/evaluations/index.d.ts

@@ -0,0 +1,7 @@
+export { ALL_FIXTURES, EVAL_EMBEDDING_IDENTITY, EVAL_TOPIC_BOOST, ambiguousQueryFixture, broadQueryDiversityFixture, contextBudgetFixture, multiCapsuleFixture, multiPageFixture, noEvidenceFixture, singleTopicFixture, staleIndexFixture, structuredFileFixture, sourceIsolationFixture, wrongScopeFixture, } from "./fixtures.js";
+export { citationRequirementForUnit, scoreCitationQuality, scoreContextBudgetFit, scoreMeanReciprocalRank, scoreNdcg, scoreNoEvidenceAccuracy, scorePrecision, scoreRecall, scoreSourceIsolation, type CitationRequirementKey, } from "./dimensions.js";
+export { createScriptedEmbeddingAdapter, fnv1a32, withTopicMarker, type ScriptedEmbeddingAdapterOptions, } from "./scripted-embedding-adapter.js";
+export { renderRetrievalEvalQualityGateReport } from "./report.js";
+export { runRetrievalEval, type RunRetrievalEvalDeps } from "./runner.js";
+export { PASS_THRESHOLDS, type EvalCapsuleSpec, type EvalChunkSpec, type EvalDocumentSpec, type EvalParsedUnitSpec, type EvalParsedUnitWithoutDocId, type EvalRetrievalScope, type EvalSourceSpec, type ModelJudgedRetrievalEvalInput, type ModelJudgedRetrievalEvalJudge, type ModelJudgedRetrievalEvalScores, type RetrievalEvalDimensionScores, type RetrievalEvalFixture, type RetrievalEvalQuery, type RetrievalEvalScorecard, type RetrievalEvalThresholds, } from "./types.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/evaluations/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/evaluations/index.ts"],"names":[],"mappings":"AAKA,OAAO,EACL,YAAY,EACZ,uBAAuB,EACvB,gBAAgB,EAChB,qBAAqB,EACrB,0BAA0B,EAC1B,oBAAoB,EACpB,mBAAmB,EACnB,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,iBAAiB,GAClB,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,0BAA0B,EAC1B,oBAAoB,EACpB,qBAAqB,EACrB,uBAAuB,EACvB,SAAS,EACT,uBAAuB,EACvB,cAAc,EACd,WAAW,EACX,oBAAoB,EACpB,KAAK,sBAAsB,GAC5B,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EACL,8BAA8B,EAC9B,OAAO,EACP,eAAe,EACf,KAAK,+BAA+B,GACrC,MAAM,iCAAiC,CAAC;AAEzC,OAAO,EAAE,oCAAoC,EAAE,MAAM,aAAa,CAAC;AACnE,OAAO,EAAE,gBAAgB,EAAE,KAAK,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAE1E,OAAO,EACL,eAAe,EACf,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,gBAAgB,EACrB,KAAK,kBAAkB,EACvB,KAAK,0BAA0B,EAC/B,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,KAAK,6BAA6B,EAClC,KAAK,6BAA6B,EAClC,KAAK,8BAA8B,EACnC,KAAK,4BAA4B,EACjC,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,GAC7B,MAAM,YAAY,CAAC"}

package/dist/evaluations/index.js

@@ -0,0 +1,10 @@
+// Public surface of the retrieval evaluation harness (Epic #189, Issue #268). The harness
+// is composed by the package barrel in ../index.ts; consumers outside this package never
+// import from this subdirectory directly (ADR-0019 direction rule 3e + the trust-8
+// test-support naming convention).
+export { ALL_FIXTURES, EVAL_EMBEDDING_IDENTITY, EVAL_TOPIC_BOOST, ambiguousQueryFixture, broadQueryDiversityFixture, contextBudgetFixture, multiCapsuleFixture, multiPageFixture, noEvidenceFixture, singleTopicFixture, staleIndexFixture, structuredFileFixture, sourceIsolationFixture, wrongScopeFixture, } from "./fixtures.js";
+export { citationRequirementForUnit, scoreCitationQuality, scoreContextBudgetFit, scoreMeanReciprocalRank, scoreNdcg, scoreNoEvidenceAccuracy, scorePrecision, scoreRecall, scoreSourceIsolation, } from "./dimensions.js";
+export { createScriptedEmbeddingAdapter, fnv1a32, withTopicMarker, } from "./scripted-embedding-adapter.js";
+export { renderRetrievalEvalQualityGateReport } from "./report.js";
+export { runRetrievalEval } from "./runner.js";
+export { PASS_THRESHOLDS, } from "./types.js";

package/dist/evaluations/report.d.ts

@@ -0,0 +1,3 @@
+import type { RetrievalEvalScorecard } from "./types.js";
+export declare function renderRetrievalEvalQualityGateReport(scorecards: readonly RetrievalEvalScorecard[]): string;
+//# sourceMappingURL=report.d.ts.map

package/dist/evaluations/report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/evaluations/report.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAMzD,wBAAgB,oCAAoC,CAClD,UAAU,EAAE,SAAS,sBAAsB,EAAE,GAC5C,MAAM,CAwBR"}

package/dist/evaluations/report.js

@@ -0,0 +1,31 @@
+// Markdown renderer for Issue #268 quality-gate evidence. The report is built entirely
+// from synthetic scorecards, so it is safe to attach to epic closure evidence without
+// leaking customer content, credentials, or runtime logs.
+function format(value) {
+    return value.toFixed(3);
+}
+export function renderRetrievalEvalQualityGateReport(scorecards) {
+    const header = [
+        "# Local Knowledge Retrieval Quality Gate",
+        "",
+        "| Fixture | Recall | Precision | MRR | nDCG | Isolation | Citation | No-evidence | Context budget | Latency | Pass |",
+        "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |",
+    ];
+    const rows = scorecards.map((scorecard) => {
+        const d = scorecard.dimensions;
+        return `| ${[
+            scorecard.fixtureId,
+            format(d.recall),
+            format(d.precision),
+            format(d.meanReciprocalRank),
+            format(d.ndcg),
+            format(d.sourceIsolation),
+            format(d.citationQuality),
+            format(d.noEvidenceAccuracy),
+            format(d.contextBudgetFit),
+            format(d.latencyMs),
+            scorecard.passed ? "PASS" : "FAIL",
+        ].join(" | ")} |`;
+    });
+    return [...header, ...rows].join("\n");
+}

package/dist/evaluations/runner-seed.d.ts

@@ -0,0 +1,12 @@
+import type { EmbeddingModelIdentity } from "@oscharko-dev/keiko-contracts";
+import type { KnowledgeStore } from "../store.js";
+import { type CitationRequirementKey } from "./dimensions.js";
+import type { RetrievalEvalFixture } from "./types.js";
+export interface SeededFixture {
+    readonly chunkUnitKinds: ReadonlyMap<string, CitationRequirementKey>;
+    readonly chunkTokenCounts: ReadonlyMap<string, number>;
+    readonly topicBoosts: Readonly<Record<string, number>>;
+    readonly identity: EmbeddingModelIdentity;
+}
+export declare function seedFixture(store: KnowledgeStore, fixture: RetrievalEvalFixture): SeededFixture;
+//# sourceMappingURL=runner-seed.d.ts.map

package/dist/evaluations/runner-seed.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner-seed.d.ts","sourceRoot":"","sources":["../../src/evaluations/runner-seed.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAEV,sBAAsB,EAGvB,MAAM,+BAA+B,CAAC;AAOvC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,OAAO,EAA8B,KAAK,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AAC1F,OAAO,KAAK,EAIV,oBAAoB,EACrB,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,cAAc,EAAE,WAAW,CAAC,MAAM,EAAE,sBAAsB,CAAC,CAAC;IACrE,QAAQ,CAAC,gBAAgB,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAGvD,QAAQ,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAEvD,QAAQ,CAAC,QAAQ,EAAE,sBAAsB,CAAC;CAC3C;AAwLD,wBAAgB,WAAW,CAAC,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,oBAAoB,GAAG,aAAa,CAoB/F"}

package/dist/evaluations/runner-seed.js

@@ -0,0 +1,175 @@
+// Fixture seeding helpers for the eval runner (Epic #189, Issue #268). Extracted from
+// `runner.ts` so each file stays under the 400-LOC budget. Materialises a fixture's
+// capsules / sources / documents / parsed-units / chunks rows into a fresh store; the
+// runner then embeds the chunks separately through `embedChunkBatch`.
+//
+// Topic boosts are aggregated here too: every chunk and every query that declares a
+// `topic` contributes a boost of 1.0 to the map handed to the scripted adapter. Boosts of
+// 1.0 produce the pure topic vector for marked inputs — strong enough that the
+// ground-truth chunk always dominates the cosine ranking for its query.
+import { createCapsule } from "../capsule-lifecycle.js";
+import { createCapsuleSet, getCapsuleSet } from "../capsule-set-lifecycle.js";
+import { insertChunkRow } from "../chunking/chunker-persist.js";
+import { insertDocumentRow, insertParsedUnitRow } from "../discovery/persist.js";
+import { addSourceToCapsule } from "../source-lifecycle.js";
+import { citationRequirementForUnit } from "./dimensions.js";
+function chunkParsedUnitId(documentId, parsedUnitId) {
+    return `unit-${documentId}-${parsedUnitId}`;
+}
+function sameEmbeddingIdentity(left, right) {
+    return (left.provider === right.provider &&
+        left.modelId === right.modelId &&
+        left.modelRevision === right.modelRevision &&
+        left.vectorDimensions === right.vectorDimensions &&
+        left.vectorMetric === right.vectorMetric);
+}
+function seedCapsule(store, capsule) {
+    createCapsule(store, {
+        id: capsule.id,
+        displayName: capsule.displayName,
+        tags: [],
+        retrievalEffort: "default",
+        outputMode: "answers",
+        answerGroundingPolicy: capsule.answerGroundingPolicy,
+        embeddingModelIdentity: capsule.embeddingModelIdentity,
+        lifecycleState: "draft",
+        storageReference: `eval/${String(capsule.id)}`,
+    });
+}
+function seedSource(store, capsuleId, source) {
+    addSourceToCapsule(store, capsuleId, {
+        id: source.id,
+        displayName: `Source ${String(source.id)}`,
+        tags: [],
+        scope: { kind: "folder", rootPath: "/srv/docs", recursive: true },
+    });
+}
+function composeParsedUnit(documentId, unit) {
+    return { ...unit.unit, documentId: documentId };
+}
+function seedDocument(store, capsule, source, doc) {
+    insertDocumentRow(store._internal.db, {
+        id: doc.id,
+        capsuleId: capsule.id,
+        sourceId: String(source.id),
+        documentPath: `docs/${doc.safeDisplayName}`,
+        sizeBytes: 1024,
+        mediaType: doc.mediaType ?? "text/plain",
+        contentHash: "a".repeat(64),
+        parserId: doc.parserId ?? "text",
+        parserVersion: doc.parserVersion ?? "1",
+        lastExtractedAt: 1_700_000_000_000,
+        status: "extracted",
+        safeDisplayName: doc.safeDisplayName,
+    });
+    for (const parsedUnit of doc.parsedUnits) {
+        insertParsedUnitRow(store._internal.db, store._internal.contentCipher, capsule.id, chunkParsedUnitId(String(doc.id), parsedUnit.id), composeParsedUnit(String(doc.id), parsedUnit));
+    }
+}
+function resolveChunkUnit(doc, chunk) {
+    if (doc.parsedUnits.length === 0) {
+        throw new Error(`eval document ${String(doc.id)} must declare at least one parsed unit`);
+    }
+    if (chunk.parsedUnitId === undefined) {
+        const first = doc.parsedUnits[0];
+        if (first === undefined)
+            throw new Error("unreachable");
+        return first;
+    }
+    const resolved = doc.parsedUnits.find((unit) => unit.id === chunk.parsedUnitId);
+    if (resolved === undefined) {
+        throw new Error(`eval chunk ${String(chunk.id)} references unknown parsed unit ${chunk.parsedUnitId}`);
+    }
+    return resolved;
+}
+function seedChunks(store, capsule, source, doc, chunkUnitKinds, chunkTokenCounts) {
+    let orderIndex = 0;
+    for (const chunk of doc.chunks) {
+        const parsedUnit = resolveChunkUnit(doc, chunk);
+        const composedUnit = composeParsedUnit(String(doc.id), parsedUnit);
+        insertChunkRow(store._internal.db, {
+            id: chunk.id,
+            capsuleId: capsule.id,
+            sourceId: source.id,
+            documentId: doc.id,
+            parsedUnitId: chunkParsedUnitId(String(doc.id), parsedUnit.id),
+            orderIndex,
+            tokenCount: chunk.text.length,
+            safeExcerptHash: "b".repeat(64),
+            chunkingStrategyVersion: "issue-195-v1",
+            // Synthetic span over the chunk's own text. Inert for the eval harness (it retrieves
+            // pre-seeded vectors rather than re-slicing source text) but satisfies the v8 columns.
+            characterStart: 0,
+            characterEnd: chunk.text.length,
+        });
+        chunkUnitKinds.set(String(chunk.id), citationRequirementForUnit(composedUnit));
+        chunkTokenCounts.set(String(chunk.id), chunk.text.length);
+        orderIndex += 1;
+    }
+}
+function collectTopicBoosts(fixture) {
+    const boosts = {};
+    for (const capsule of fixture.capsules) {
+        for (const source of capsule.sources) {
+            for (const doc of source.documents) {
+                for (const chunk of doc.chunks) {
+                    if (chunk.topic !== undefined)
+                        boosts[chunk.topic] = 1.0;
+                }
+            }
+        }
+    }
+    for (const query of fixture.queries) {
+        if (query.topic !== undefined)
+            boosts[query.topic] = 1.0;
+    }
+    return boosts;
+}
+function seedCapsuleSets(store, fixture) {
+    for (const query of fixture.queries) {
+        if (query.scope.kind !== "capsule-set")
+            continue;
+        const id = query.scope.capsuleSetId;
+        if (getCapsuleSet(store, id) !== undefined)
+            continue;
+        createCapsuleSet(store, {
+            id,
+            displayName: `Set ${query.scope.capsuleSetId}`,
+            tags: [],
+            capsuleIds: query.scope.capsuleIds,
+        });
+    }
+}
+function validateFixtureIdentity(fixture) {
+    const first = fixture.capsules[0];
+    if (first === undefined) {
+        throw new Error("fixture must declare at least one capsule");
+    }
+    for (const capsule of fixture.capsules) {
+        if (!sameEmbeddingIdentity(first.embeddingModelIdentity, capsule.embeddingModelIdentity)) {
+            throw new Error(`fixture ${fixture.id} mixes embedding identities; eval runner requires one identity per run`);
+        }
+    }
+    return first.embeddingModelIdentity;
+}
+export function seedFixture(store, fixture) {
+    const chunkUnitKinds = new Map();
+    const chunkTokenCounts = new Map();
+    for (const capsule of fixture.capsules) {
+        seedCapsule(store, capsule);
+        for (const source of capsule.sources) {
+            seedSource(store, capsule.id, source);
+            for (const doc of source.documents) {
+                seedDocument(store, capsule, source, doc);
+                seedChunks(store, capsule, source, doc, chunkUnitKinds, chunkTokenCounts);
+            }
+        }
+    }
+    seedCapsuleSets(store, fixture);
+    return {
+        chunkUnitKinds,
+        chunkTokenCounts,
+        topicBoosts: collectTopicBoosts(fixture),
+        identity: validateFixtureIdentity(fixture),
+    };
+}

package/dist/evaluations/runner.d.ts

@@ -0,0 +1,8 @@
+import type { ModelJudgedRetrievalEvalJudge, RetrievalEvalFixture, RetrievalEvalScorecard } from "./types.js";
+export interface RunRetrievalEvalDeps {
+    readonly now?: () => number;
+    readonly runId?: string;
+    readonly modelJudge?: ModelJudgedRetrievalEvalJudge;
+}
+export declare function runRetrievalEval(fixture: RetrievalEvalFixture, deps?: RunRetrievalEvalDeps): Promise<RetrievalEvalScorecard>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/evaluations/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/evaluations/runner.ts"],"names":[],"mappings":"AA6CA,OAAO,KAAK,EAEV,6BAA6B,EAE7B,oBAAoB,EAEpB,sBAAsB,EACvB,MAAM,YAAY,CAAC;AAMpB,MAAM,WAAW,oBAAoB;IAGnC,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IAG5B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAGxB,QAAQ,CAAC,UAAU,CAAC,EAAE,6BAA6B,CAAC;CACrD;AAkPD,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,oBAAoB,EAC7B,IAAI,GAAE,oBAAyB,GAC9B,OAAO,CAAC,sBAAsB,CAAC,CAkBjC"}

package/dist/evaluations/runner.js

@@ -0,0 +1,205 @@
+// Retrieval evaluation runner (Epic #189, Issue #268). Materialises a `RetrievalEvalFixture`
+// into a fresh temporary SQLite store on disk, runs every query through `runLocalKnowledgeRetrieval`
+// (#199) UNCHANGED, scores each query against the deterministic guardrail dimensions, and returns an immutable
+// `RetrievalEvalScorecard`.
+//
+// Determinism contract:
+//   - The default `now()` is a monotonic counter starting at 0. Two runs of the same fixture
+//     therefore produce byte-identical scorecards (the latency dimension counts ticks of this
+//     counter, not wall-clock milliseconds).
+//   - A caller that wants real wall-clock latency passes its own `now: () => performance.now()`
+//     — but doing so DROPS the byte-identical guarantee and is incompatible with the audit
+//     ledger's manifest equality check.
+//   - The store path uses `mkdtempSync` (different per process) but the store contents are
+//     discarded at teardown; nothing about the temp path leaks into the scorecard.
+//
+// Seeding is implemented in `runner-seed.ts` so each file stays under the 400-LOC budget.
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { embedChunkBatch } from "../indexing/embedding-batcher.js";
+import { runLocalKnowledgeRetrieval } from "../retrieval/index.js";
+import { openKnowledgeStore } from "../store.js";
+import { scoreCitationQuality, scoreContextBudgetFit, scoreMeanReciprocalRank, scoreNdcg, scoreNoEvidenceAccuracy, scorePrecision, scoreRecall, scoreSourceIsolation, } from "./dimensions.js";
+import { seedFixture } from "./runner-seed.js";
+import { createScriptedEmbeddingAdapter, withTopicMarker } from "./scripted-embedding-adapter.js";
+import { PASS_THRESHOLDS } from "./types.js";
+function collectCapsuleChunks(capsule) {
+    const out = [];
+    for (const source of capsule.sources) {
+        for (const doc of source.documents) {
+            for (const chunk of doc.chunks) {
+                const text = chunk.topic !== undefined ? withTopicMarker(chunk.text, chunk.topic) : chunk.text;
+                out.push({
+                    id: chunk.id,
+                    capsuleId: capsule.id,
+                    sourceId: source.id,
+                    documentId: doc.id,
+                    text,
+                });
+            }
+        }
+    }
+    return out;
+}
+async function embedAllChunks(store, fixture, seeded, now) {
+    const adapter = createScriptedEmbeddingAdapter({
+        identity: seeded.identity,
+        topicBoosts: seeded.topicBoosts,
+    });
+    let storageCounter = 0;
+    const idSource = () => {
+        storageCounter += 1;
+        return `eval-storage-${String(storageCounter)}`;
+    };
+    for (const capsule of fixture.capsules) {
+        const chunks = collectCapsuleChunks(capsule);
+        const result = await embedChunkBatch(chunks, {
+            adapter,
+            store,
+            pinnedIdentity: capsule.embeddingModelIdentity,
+            concurrency: 1,
+            now,
+            idSource,
+        });
+        if (result.errors.length > 0) {
+            const codes = result.errors.map((e) => e.code).join(",");
+            throw new Error(`embedding seeding failed for capsule ${String(capsule.id)}: ${codes}`);
+        }
+    }
+}
+function scopeCapsuleIds(query) {
+    if (query.scope.kind === "capsule")
+        return [query.scope.capsuleId];
+    return query.scope.capsuleIds;
+}
+function buildRetrievalQuery(query, queryText) {
+    const baseQuery = {
+        text: queryText,
+        ...(query.topK !== undefined ? { topK: query.topK } : {}),
+        // For the no-evidence fixture we apply a very high minScore so unrelated chunks are
+        // dropped. The fixture's query carries no topic marker, so the cosine of its vector
+        // with any topic-boosted chunk is far below 0.99.
+        ...(query.expectedNoEvidence === true ? { minScore: 0.99 } : {}),
+    };
+    if (query.scope.kind === "capsule") {
+        return { ...baseQuery, capsuleId: query.scope.capsuleId };
+    }
+    return { ...baseQuery, capsuleSetId: query.scope.capsuleSetId };
+}
+async function runOneQuery(store, query, seeded, now) {
+    // Wrap the query text in the topic marker so the scripted adapter applies the same
+    // topic boost it used at seed time.
+    const queryText = query.topic !== undefined ? withTopicMarker(query.text, query.topic) : query.text;
+    const adapter = createScriptedEmbeddingAdapter({
+        identity: query.queryEmbeddingIdentity ?? seeded.identity,
+        topicBoosts: seeded.topicBoosts,
+    });
+    const retrievalQuery = buildRetrievalQuery(query, queryText);
+    const start = now();
+    const result = await runLocalKnowledgeRetrieval({ store, embeddingAdapter: adapter }, retrievalQuery);
+    const end = now();
+    const expected = query.expectedChunkIds ?? [];
+    const expectedNoEvidence = query.expectedNoEvidence === true;
+    return {
+        query,
+        references: result.references,
+        noEvidence: result.noEvidence,
+        ...(result.reason !== undefined ? { reason: result.reason } : {}),
+        scores: {
+            recall: scoreRecall(result.references, expected),
+            precision: scorePrecision(result.references, expected),
+            meanReciprocalRank: scoreMeanReciprocalRank(result.references, expected),
+            ndcg: scoreNdcg(result.references, expected),
+            sourceIsolation: scoreSourceIsolation(result.references, scopeCapsuleIds(query)),
+            citationQuality: scoreCitationQuality(result.references, seeded.chunkUnitKinds),
+            noEvidenceAccuracy: scoreNoEvidenceAccuracy(result.noEvidence, expectedNoEvidence, result.reason, query.expectedNoEvidenceReason),
+            contextBudgetFit: scoreContextBudgetFit(result.references, seeded.chunkTokenCounts, query.contextBudgetTokens),
+            latencyTicks: end - start,
+        },
+    };
+}
+// ─── Aggregation ─────────────────────────────────────────────────────────────
+async function runModelJudge(modelJudge, fixture, perQuery) {
+    if (modelJudge === undefined)
+        return undefined;
+    const judged = [];
+    for (const evaluation of perQuery) {
+        judged.push(await modelJudge.judge({
+            fixtureId: fixture.id,
+            queryId: evaluation.query.id,
+            queryText: evaluation.query.text,
+            references: evaluation.references,
+            noEvidence: evaluation.noEvidence,
+            ...(evaluation.reason !== undefined ? { reason: evaluation.reason } : {}),
+        }));
+    }
+    return {
+        groundedness: meanOf(judged.map((item) => item.groundedness)),
+        faithfulness: meanOf(judged.map((item) => item.faithfulness)),
+    };
+}
+function meanOf(values) {
+    if (values.length === 0)
+        return 0;
+    let sum = 0;
+    for (const v of values)
+        sum += v;
+    return sum / values.length;
+}
+function buildScorecard(fixture, runId, perQuery, modelJudged) {
+    const dimensions = {
+        recall: meanOf(perQuery.map((q) => q.scores.recall)),
+        precision: meanOf(perQuery.map((q) => q.scores.precision)),
+        meanReciprocalRank: meanOf(perQuery.map((q) => q.scores.meanReciprocalRank)),
+        ndcg: meanOf(perQuery.map((q) => q.scores.ndcg)),
+        sourceIsolation: meanOf(perQuery.map((q) => q.scores.sourceIsolation)),
+        citationQuality: meanOf(perQuery.map((q) => q.scores.citationQuality)),
+        noEvidenceAccuracy: meanOf(perQuery.map((q) => q.scores.noEvidenceAccuracy)),
+        contextBudgetFit: meanOf(perQuery.map((q) => q.scores.contextBudgetFit)),
+        latencyMs: perQuery.reduce((acc, q) => acc + q.scores.latencyTicks, 0),
+    };
+    const passed = dimensions.recall >= PASS_THRESHOLDS.recall &&
+        dimensions.precision >= PASS_THRESHOLDS.precision &&
+        dimensions.meanReciprocalRank >= PASS_THRESHOLDS.meanReciprocalRank &&
+        dimensions.ndcg >= PASS_THRESHOLDS.ndcg &&
+        dimensions.sourceIsolation >= PASS_THRESHOLDS.sourceIsolation &&
+        dimensions.citationQuality >= PASS_THRESHOLDS.citationQuality &&
+        dimensions.noEvidenceAccuracy >= PASS_THRESHOLDS.noEvidenceAccuracy &&
+        dimensions.contextBudgetFit >= PASS_THRESHOLDS.contextBudgetFit;
+    return modelJudged === undefined
+        ? { fixtureId: fixture.id, runId, dimensions, passed }
+        : { fixtureId: fixture.id, runId, dimensions, passed, modelJudged };
+}
+// ─── Default clock ───────────────────────────────────────────────────────────
+// A monotonic integer counter created fresh per call. Returns 0 on first invocation, 1 on
+// second, etc. Latency for a query is therefore exactly the number of `now()` reads inside
+// the query (`runOneQuery` reads it twice, so every query reports `latencyTicks = 1`).
+function defaultClock() {
+    let counter = -1;
+    return () => {
+        counter += 1;
+        return counter;
+    };
+}
+// ─── Public entrypoint ───────────────────────────────────────────────────────
+export async function runRetrievalEval(fixture, deps = {}) {
+    const now = deps.now ?? defaultClock();
+    const runId = deps.runId ?? `eval-${fixture.id}`;
+    const dir = mkdtempSync(join(tmpdir(), "keiko-eval-"));
+    const store = openKnowledgeStore({ dbPath: join(dir, "eval.db") });
+    try {
+        const seeded = seedFixture(store, fixture);
+        await embedAllChunks(store, fixture, seeded, now);
+        const perQuery = [];
+        for (const query of fixture.queries) {
+            perQuery.push(await runOneQuery(store, query, seeded, now));
+        }
+        const modelJudged = await runModelJudge(deps.modelJudge, fixture, perQuery);
+        return buildScorecard(fixture, runId, perQuery, modelJudged);
+    }
+    finally {
+        store.close();
+        rmSync(dir, { recursive: true, force: true });
+    }
+}

package/dist/evaluations/scripted-embedding-adapter.d.ts

@@ -0,0 +1,13 @@
+import type { EmbeddingModelIdentity } from "@oscharko-dev/keiko-contracts";
+import type { OpenAIEmbeddingAdapter } from "@oscharko-dev/keiko-model-gateway";
+export declare function fnv1a32(input: string): number;
+export interface ScriptedEmbeddingAdapterOptions {
+    readonly identity: EmbeddingModelIdentity;
+    readonly topicBoosts?: Readonly<Record<string, number>>;
+    readonly endpoint?: string;
+    readonly apiKey?: string;
+    readonly apiKeyHeaderName?: string;
+}
+export declare function createScriptedEmbeddingAdapter(options: ScriptedEmbeddingAdapterOptions): OpenAIEmbeddingAdapter;
+export declare function withTopicMarker(text: string, topic: string): string;
+//# sourceMappingURL=scripted-embedding-adapter.d.ts.map

package/dist/evaluations/scripted-embedding-adapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scripted-embedding-adapter.d.ts","sourceRoot":"","sources":["../../src/evaluations/scripted-embedding-adapter.ts"],"names":[],"mappings":"AA0BA,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AAC5E,OAAO,KAAK,EACV,sBAAsB,EAGvB,MAAM,mCAAmC,CAAC;AAW3C,wBAAgB,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAQ7C;AA4ED,MAAM,WAAW,+BAA+B;IAK9C,QAAQ,CAAC,QAAQ,EAAE,sBAAsB,CAAC;IAI1C,QAAQ,CAAC,WAAW,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAGxD,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CACpC;AA+BD,wBAAgB,8BAA8B,CAC5C,OAAO,EAAE,+BAA+B,GACvC,sBAAsB,CA6BxB;AAKD,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAKnE"}

package/dist/evaluations/scripted-embedding-adapter.js

@@ -0,0 +1,163 @@
+// Scripted (offline, deterministic) embedding adapter for the retrieval eval harness
+// (Epic #189, Issue #268). Implements `OpenAIEmbeddingAdapter` so the harness can plug
+// directly into #199's retrieval runner — same code path as production, only the model
+// call is replaced.
+//
+// Determinism guarantees (load-bearing for byte-identical scorecards):
+//   - No `Date.now()`, `Math.random()`, `performance.now()`, or `Date()` reads.
+//   - No global mutable state. The adapter holds only the immutable identity + the
+//     constructor's `topicSalts` map; nothing mutates after construction.
+//   - No `fetch` import, no network IO of any kind. The `request` method always resolves
+//     synchronously through `Promise.resolve`.
+//
+// Vector layout: a `vectorDimensions`-wide `Float32Array` filled by an FNV-1a hash of the
+// input string (32-bit, see RFC reference text). Lane 0 carries a normalised input length
+// signal so two strings of clearly different length never collide on the leading lane —
+// this matters because the hash collision space on short strings is small, and we want
+// the cosine of two distinct inputs to stay strictly < 1.
+//
+// Topic salt: every chunk in a fixture may declare an optional `topic`. The scripted
+// adapter accepts a map `{ [topic]: boost }` — for any request whose text contains a known
+// topic marker (a `[[topic]]` envelope in the input string), the corresponding boost is
+// blended into the produced vector. This lets a fixture make the ground-truth chunk for a
+// query verifiably the top result without depending on real semantic similarity. The
+// marker is parsed; the marker itself is stripped from the FNV input so two queries with
+// the same body but different topics still hash to similar (not identical) vectors.
+// ─── FNV-1a 32-bit ───────────────────────────────────────────────────────────
+// We use FNV-1a over UTF-16 code units (`charCodeAt`) so the hash is locale-independent
+// and identical across every JS engine the package targets. The literals come from
+// http://www.isthe.com/chongo/tech/comp/fnv/ — `OFFSET_BASIS = 2166136261` (0x811c9dc5),
+// `PRIME = 16777619` (0x01000193). Multiplication uses `Math.imul` so it stays exact in
+// 32-bit even on values that would otherwise round above 2^53.
+const FNV_OFFSET_BASIS = 0x811c9dc5;
+const FNV_PRIME = 0x01000193;
+export function fnv1a32(input) {
+    let hash = FNV_OFFSET_BASIS;
+    for (let i = 0; i < input.length; i += 1) {
+        hash ^= input.charCodeAt(i);
+        hash = Math.imul(hash, FNV_PRIME);
+    }
+    // Coerce to unsigned 32-bit so callers see a consistent positive integer.
+    return hash >>> 0;
+}
+// ─── Topic markers ───────────────────────────────────────────────────────────
+// A fixture embeds a topic marker inside the text it asks the adapter to embed. The format
+// `[[topic:NAME]]` is chosen so it is impossible to occur in natural text the harness
+// might emit by accident — the runner injects it explicitly when seeding chunks. The
+// adapter strips the marker before hashing so the FNV component remains stable as topics
+// are added or renamed.
+const TOPIC_MARKER_PATTERN = /\[\[topic:([a-zA-Z0-9_-]+)\]\]/g;
+const VALID_TOPIC_PATTERN = /^[a-zA-Z0-9_-]+$/;
+function extractTopics(input) {
+    const topics = [];
+    // Reset state on the shared regex by constructing a fresh local copy — the `/g` flag
+    // makes the constant stateful and we never want the next call to inherit `lastIndex`.
+    const re = new RegExp(TOPIC_MARKER_PATTERN.source, "g");
+    let match = re.exec(input);
+    while (match !== null) {
+        const captured = match[1];
+        if (captured !== undefined)
+            topics.push(captured);
+        match = re.exec(input);
+    }
+    const stripped = input.replace(re, "");
+    return { topics, stripped };
+}
+// ─── Vector synthesis ────────────────────────────────────────────────────────
+// Lane 0: normalised input length (clipped to `[0, 1]` so it interacts well with cosine).
+// Lanes 1..dim-1: a permutation of the FNV hash mixed with the lane index. The mix uses
+// `Math.imul` for 32-bit arithmetic, then squashes to `[-1, 1]` so cosine remains in its
+// usual range. A topic salt adds a deterministic "topic vector" — derived from the
+// topic-name hash — scaled by the configured boost.
+const LENGTH_NORMALISATION_DIVISOR = 1024;
+function laneFromHash(hash, laneIndex) {
+    // Mix lane index into the hash so each lane has a different but deterministic value.
+    const mixed = Math.imul(hash ^ laneIndex, FNV_PRIME) >>> 0;
+    // Map a 32-bit unsigned int into `[-1, 1)` by treating it as a float in `[0, 1)` then
+    // shifting + scaling. `0xffffffff + 1` is exact in float64.
+    return (mixed / 0x100000000) * 2 - 1;
+}
+function fillBaseVector(vector, hash, stripped) {
+    // Lane 0 is the length signal; clip to 1.0 so an extremely long input still produces a
+    // value in `[0, 1]`. The divisor is generous enough that natural fixture inputs all
+    // land below 1.0 and discriminate by length.
+    vector[0] = Math.min(stripped.length / LENGTH_NORMALISATION_DIVISOR, 1);
+    for (let i = 1; i < vector.length; i += 1) {
+        vector[i] = laneFromHash(hash, i);
+    }
+}
+function applyTopicBoost(vector, topic, boost) {
+    const topicHash = fnv1a32(`topic:${topic}`);
+    // Skip lane 0 — it is the length signal and we never want the boost to make two inputs
+    // of different length collide on the leading lane.
+    for (let i = 1; i < vector.length; i += 1) {
+        const topicLane = laneFromHash(topicHash, i);
+        // Blend: each lane becomes `(1 - boost) * base + boost * topicLane`. With `boost = 1`
+        // the vector is the pure topic vector — two inputs sharing a topic become identical
+        // on every lane except lane 0, which still records length. The resulting cosine is
+        // dominated by the topic lanes (1..dim-1), which is exactly the property a fixture
+        // needs to make ground-truth assertions deterministic.
+        const current = vector[i] ?? 0;
+        vector[i] = (1 - boost) * current + boost * topicLane;
+    }
+}
+function clampBoost(value) {
+    if (Number.isNaN(value))
+        return 0;
+    if (value < 0)
+        return 0;
+    if (value > 1)
+        return 1;
+    return value;
+}
+function resolveBoost(topics, boosts) {
+    // First topic wins. A fixture that wants multiple topics layered should pre-blend them
+    // into a single named topic — keeping a single boost per request keeps the cosine math
+    // easy to reason about for the determinism test.
+    for (const topic of topics) {
+        if (Object.prototype.hasOwnProperty.call(boosts, topic)) {
+            const raw = boosts[topic];
+            if (raw === undefined)
+                continue;
+            return { topic, boost: clampBoost(raw) };
+        }
+    }
+    return undefined;
+}
+export function createScriptedEmbeddingAdapter(options) {
+    const { identity } = options;
+    const topicBoosts = options.topicBoosts ?? {};
+    const endpoint = options.endpoint ?? "https://scripted.local/v1";
+    const apiKey = options.apiKey ?? "scripted-test-key";
+    const request = async (req) => {
+        const { topics, stripped } = extractTopics(req.input);
+        const hash = fnv1a32(stripped);
+        const vector = new Float32Array(identity.vectorDimensions);
+        fillBaseVector(vector, hash, stripped);
+        const blend = resolveBoost(topics, topicBoosts);
+        if (blend !== undefined)
+            applyTopicBoost(vector, blend.topic, blend.boost);
+        const successValue = {
+            vector,
+            modelId: identity.modelId,
+            ...(identity.modelRevision !== undefined ? { modelRevision: identity.modelRevision } : {}),
+        };
+        return Promise.resolve({ ok: true, value: successValue });
+    };
+    return {
+        endpoint,
+        apiKey,
+        ...(options.apiKeyHeaderName !== undefined
+            ? { apiKeyHeaderName: options.apiKeyHeaderName }
+            : {}),
+        request,
+    };
+}
+// ─── Marker helpers ──────────────────────────────────────────────────────────
+// Exported so fixtures + runner can apply markers without hard-coding the format.
+export function withTopicMarker(text, topic) {
+    if (!VALID_TOPIC_PATTERN.test(topic)) {
+        throw new Error(`invalid eval topic marker: ${topic}`);
+    }
+    return `[[topic:${topic}]]${text}`;
+}