@oscharko-dev/keiko-local-knowledge 0.2.0

package/dist/evaluations/dimensions.js

@@ -0,0 +1,191 @@
+// Pure scoring functions for the retrieval evaluation harness (Epic #189, Issue #268).
+// Each function takes structural inputs (no store, no IO) and returns a number in
+// `[0, 1]` — that range invariant is enforced by every branch and is load-bearing for the
+// aggregator in `runner.ts`, which averages dimensions across queries.
+//
+// Vacuous cases:
+//   - `scoreRecall` with empty `expected` ⇒ 1.0 (nothing was missed).
+//   - `scorePrecision` with empty `returned` ⇒ 1.0 (no false positives possible). Callers
+//     that want "no refs is a failure" should reach for `scoreNoEvidenceAccuracy` instead.
+//   - `scoreSourceIsolation` with empty `returned` ⇒ 1.0 (no leak possible).
+//   - `scoreCitationQuality` with empty `references` ⇒ 1.0 (no malformed citations).
+//
+// These vacuous-1.0 conventions are deliberate: a fixture that expects no evidence should
+// not be penalised on recall/precision/etc. The `noEvidenceAccuracy` dimension is the one
+// that discriminates "expected nothing AND got nothing" vs "expected nothing but got
+// something" vs "expected something but got nothing".
+// ─── Recall ──────────────────────────────────────────────────────────────────
+// `expected ∩ returned / |expected|`. We compare on `chunkId` because `RetrievalReference`
+// carries the chunk id verbatim and that is the smallest discriminator the fixtures use.
+export function scoreRecall(returned, expected) {
+    if (expected.length === 0)
+        return 1;
+    const returnedIds = new Set();
+    for (const ref of returned)
+        returnedIds.add(String(ref.chunkId));
+    let hits = 0;
+    for (const id of expected) {
+        if (returnedIds.has(String(id)))
+            hits += 1;
+    }
+    return hits / expected.length;
+}
+// ─── Precision ───────────────────────────────────────────────────────────────
+// `expected ∩ returned / |returned|`.
+export function scorePrecision(returned, expected) {
+    if (returned.length === 0)
+        return 1;
+    const expectedIds = new Set();
+    for (const id of expected)
+        expectedIds.add(String(id));
+    let hits = 0;
+    for (const ref of returned) {
+        if (expectedIds.has(String(ref.chunkId)))
+            hits += 1;
+    }
+    return hits / returned.length;
+}
+// ─── Ranking quality ────────────────────────────────────────────────────────
+// MRR captures how quickly the first relevant chunk appears. Recall/precision alone can pass even
+// when a relevant chunk is buried late in the retrieved list; MRR makes that regression visible.
+export function scoreMeanReciprocalRank(returned, expected) {
+    if (expected.length === 0)
+        return 1;
+    const expectedIds = new Set();
+    for (const id of expected)
+        expectedIds.add(String(id));
+    for (let index = 0; index < returned.length; index += 1) {
+        if (expectedIds.has(String(returned[index]?.chunkId))) {
+            return 1 / (index + 1);
+        }
+    }
+    return 0;
+}
+function discountedGain(rankIndex) {
+    return 1 / Math.log2(rankIndex + 2);
+}
+// Binary relevance nDCG@returned.length. This rewards relevant chunks appearing earlier while
+// staying deterministic and independent of model-judged graded relevance.
+export function scoreNdcg(returned, expected) {
+    if (expected.length === 0)
+        return 1;
+    if (returned.length === 0)
+        return 0;
+    const expectedIds = new Set();
+    for (const id of expected)
+        expectedIds.add(String(id));
+    let dcg = 0;
+    for (let index = 0; index < returned.length; index += 1) {
+        if (expectedIds.has(String(returned[index]?.chunkId))) {
+            dcg += discountedGain(index);
+        }
+    }
+    const idealCount = Math.min(expectedIds.size, returned.length);
+    let ideal = 0;
+    for (let index = 0; index < idealCount; index += 1) {
+        ideal += discountedGain(index);
+    }
+    return ideal === 0 ? 0 : dcg / ideal;
+}
+// ─── Source isolation ────────────────────────────────────────────────────────
+// A retrieval is source-isolated iff every returned reference belongs to a capsule that is
+// in `scopeCapsuleIds`. A single leak across the capsule boundary drops the score to 0 —
+// the issue brief calls source isolation a hard tenant-isolation guarantee, so partial
+// credit would dilute its meaning. The scope is normalised to a string set so the function
+// stays agnostic to the branded `KnowledgeCapsuleId` newtype.
+export function scoreSourceIsolation(returned, scopeCapsuleIds) {
+    if (returned.length === 0)
+        return 1;
+    const allowed = new Set();
+    for (const id of scopeCapsuleIds)
+        allowed.add(String(id));
+    for (const ref of returned) {
+        if (!allowed.has(String(ref.capsuleId)))
+            return 0;
+    }
+    return 1;
+}
+function isPageCitationWellFormed(reference) {
+    return reference.citation.pageNumber !== undefined;
+}
+function isSectionCitationWellFormed(reference) {
+    const path = reference.citation.sectionPath;
+    return path !== undefined && path.length > 0;
+}
+function isSpanCitationWellFormed(reference) {
+    return (reference.citation.characterStart !== undefined && reference.citation.characterEnd !== undefined);
+}
+function isCitationWellFormed(input) {
+    switch (input.unitKind) {
+        case "page":
+            return isPageCitationWellFormed(input.reference);
+        case "section":
+        case "html-block":
+            return isSectionCitationWellFormed(input.reference);
+        case "json-path":
+        case "csv-row":
+            return isSpanCitationWellFormed(input.reference);
+        case "unsupported-media":
+            return true;
+    }
+}
+// A fixture passes per-chunk unit-kind metadata through to this function. We cannot infer
+// the unit kind from a `CitationReference` alone — the contract permits any subset of the
+// optional fields to be present — so the runner threads `chunkUnitKinds` through.
+export function scoreCitationQuality(references, chunkUnitKinds) {
+    if (references.length === 0)
+        return 1;
+    let wellFormed = 0;
+    for (const reference of references) {
+        const unitKind = chunkUnitKinds.get(String(reference.chunkId));
+        // A reference for which we have no unit-kind metadata is treated as well-formed —
+        // there is nothing concrete to check against and penalising it would conflate
+        // "missing test metadata" with "missing citation field".
+        if (unitKind === undefined) {
+            wellFormed += 1;
+            continue;
+        }
+        if (isCitationWellFormed({ reference, unitKind }))
+            wellFormed += 1;
+    }
+    return wellFormed / references.length;
+}
+// Map a contract `ParsedUnit` kind to the requirement key. Exported so the runner can
+// build the `chunkUnitKinds` map from its fixture seed without re-declaring the union.
+export function citationRequirementForUnit(unit) {
+    return unit.kind;
+}
+// ─── No-evidence accuracy ────────────────────────────────────────────────────
+// Binary: `1.0` when the actual result matches the expected no-evidence flag, `0.0`
+// otherwise. The function returns the typed literal so a downstream `passed` check can
+// use `===` without worrying about floating-point comparisons.
+export function scoreNoEvidenceAccuracy(actualNoEvidence, expectedNoEvidence, actualReason, expectedReason) {
+    if (actualNoEvidence !== expectedNoEvidence)
+        return 0;
+    if (expectedReason !== undefined) {
+        return actualReason === expectedReason ? 1 : 0;
+    }
+    return 1;
+}
+// ─── Context-budget fit ──────────────────────────────────────────────────────
+// `1.0` when the retrieved chunk-token total fits within the configured budget. When it
+// exceeds the budget we return the bounded ratio `budget / used`, which keeps the score in
+// `[0, 1]` while preserving how far over budget the retrieval spilled. Queries without a
+// configured budget are treated vacuously as 1.0 because there is no concrete fit target.
+export function scoreContextBudgetFit(references, chunkTokenCounts, budgetTokens) {
+    if (budgetTokens === undefined)
+        return 1;
+    if (references.length === 0)
+        return 1;
+    if (budgetTokens <= 0)
+        return 0;
+    let used = 0;
+    for (const reference of references) {
+        used += chunkTokenCounts.get(String(reference.chunkId)) ?? 0;
+    }
+    if (used <= 0)
+        return 1;
+    if (used <= budgetTokens)
+        return 1;
+    return budgetTokens / used;
+}

package/dist/evaluations/fixtures.d.ts

@@ -0,0 +1,18 @@
+import type { EmbeddingModelIdentity } from "@oscharko-dev/keiko-contracts";
+import type { RetrievalEvalFixture } from "./types.js";
+export declare const EVAL_EMBEDDING_IDENTITY: EmbeddingModelIdentity;
+export declare const STALE_QUERY_EMBEDDING_IDENTITY: EmbeddingModelIdentity;
+export declare const EVAL_TOPIC_BOOST = 1;
+export declare const singleTopicFixture: RetrievalEvalFixture;
+export declare const multiCapsuleFixture: RetrievalEvalFixture;
+export declare const noEvidenceFixture: RetrievalEvalFixture;
+export declare const ambiguousQueryFixture: RetrievalEvalFixture;
+export declare const sourceIsolationFixture: RetrievalEvalFixture;
+export declare const wrongScopeFixture: RetrievalEvalFixture;
+export declare const multiPageFixture: RetrievalEvalFixture;
+export declare const structuredFileFixture: RetrievalEvalFixture;
+export declare const contextBudgetFixture: RetrievalEvalFixture;
+export declare const staleIndexFixture: RetrievalEvalFixture;
+export declare const broadQueryDiversityFixture: RetrievalEvalFixture;
+export declare const ALL_FIXTURES: readonly RetrievalEvalFixture[];
+//# sourceMappingURL=fixtures.d.ts.map

package/dist/evaluations/fixtures.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fixtures.d.ts","sourceRoot":"","sources":["../../src/evaluations/fixtures.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAGV,sBAAsB,EAGvB,MAAM,+BAA+B,CAAC;AAEvC,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAEvD,eAAO,MAAM,uBAAuB,EAAE,sBAKrC,CAAC;AAEF,eAAO,MAAM,8BAA8B,EAAE,sBAK5C,CAAC;AAEF,eAAO,MAAM,gBAAgB,IAAM,CAAC;AAepC,eAAO,MAAM,kBAAkB,EAAE,oBAiDhC,CAAC;AAEF,eAAO,MAAM,mBAAmB,EAAE,oBAqFjC,CAAC;AAEF,eAAO,MAAM,iBAAiB,EAAE,oBA+C/B,CAAC;AAEF,eAAO,MAAM,qBAAqB,EAAE,oBAiDnC,CAAC;AAEF,eAAO,MAAM,sBAAsB,EAAE,oBA2EpC,CAAC;AAEF,eAAO,MAAM,iBAAiB,EAAE,oBA2E/B,CAAC;AAEF,eAAO,MAAM,gBAAgB,EAAE,oBAoE9B,CAAC;AAEF,eAAO,MAAM,qBAAqB,EAAE,oBAkLnC,CAAC;AAEF,eAAO,MAAM,oBAAoB,EAAE,oBAiDlC,CAAC;AAEF,eAAO,MAAM,iBAAiB,EAAE,oBA8C/B,CAAC;AAEF,eAAO,MAAM,0BAA0B,EAAE,oBA6ExC,CAAC;AAEF,eAAO,MAAM,YAAY,EAAE,SAAS,oBAAoB,EAY9C,CAAC"}