@jcode.labs/ragmir 2.17.2 → 2.18.0

package/dist/evaluate.js

@@ -1,145 +1,332 @@
+import { createHash } from "node:crypto";
 import { readFile, stat } from "node:fs/promises";
 import path from "node:path";
 import { z } from "zod";
-import { recordAccess } from "./access-log.js";
+import { flushAccessLog, recordAccess } from "./access-log.js";
 import { loadConfig } from "./config.js";
+import { retainEmbeddingModel } from "./embeddings.js";
 import { RagmirError } from "./errors.js";
+import { acquireGenerationReadLease } from "./generation-retention.js";
+import { withIndexWriteLock } from "./index-write-lock.js";
 import { operationSignal, throwIfAborted } from "./operation.js";
-import { search } from "./query.js";
-export const MAX_GOLDEN_FILE_BYTES = 1_048_576;
-export const MAX_GOLDEN_CASES = 100;
+import { fingerprintIndexManifest, fingerprintQualityReport } from "./quality-report.js";
+import { searchWithConfig } from "./query.js";
+import { rankingPolicyFingerprint, rankingPolicyFor } from "./ranking.js";
+import { closeIndexReadSnapshot, closeStoreConnection, connectStore, indexReadSnapshotCurrent, loadIndexReadSnapshot, readIndexManifest, writeIndexManifest, } from "./store.js";
+export const MAX_GOLDEN_FILE_BYTES = 16_777_216;
+export const MAX_GOLDEN_CASES = 1_000;
 export const MAX_GOLDEN_QUERY_CHARACTERS = 20_000;
 export const MAX_GOLDEN_EXPECTED_VALUES = 100;
 export const MAX_GOLDEN_EXPECTED_VALUE_CHARACTERS = 500;
+const EVALUATION_DEPTH = 10;
+const DEFAULT_MINIMUM_VERIFICATION_CASES = 100;
 const expectedValueSchema = z.string().min(1).max(MAX_GOLDEN_EXPECTED_VALUE_CHARACTERS);
+const unitMetricSchema = z.number().min(0).max(1);
+const relevanceJudgmentSchema = z
+    .object({
+    kind: z.enum(["path", "citation"]),
+    value: expectedValueSchema,
+    relevance: z.union([z.literal(0), z.literal(1), z.literal(2), z.literal(3)]),
+})
+    .strict();
+const qualityThresholdSchema = z
+    .object({
+    recallAt1: unitMetricSchema.optional(),
+    recallAt3: unitMetricSchema.optional(),
+    recallAt5: unitMetricSchema.optional(),
+    recallAt10: unitMetricSchema.optional(),
+    precisionAt5: unitMetricSchema.optional(),
+    meanReciprocalRankAt10: unitMetricSchema.optional(),
+    ndcgAt10: unitMetricSchema.optional(),
+    exactCitationRate: unitMetricSchema.optional(),
+    maximumFalsePositiveRate: unitMetricSchema.optional(),
+})
+    .strict();
 const goldenQuerySchema = z
     .object({
     id: z.string().min(1).optional(),
     query: z.string().min(1).max(MAX_GOLDEN_QUERY_CHARACTERS),
-    expectedPaths: z.array(expectedValueSchema).min(1).max(MAX_GOLDEN_EXPECTED_VALUES),
+    expectedPaths: z.array(expectedValueSchema).max(MAX_GOLDEN_EXPECTED_VALUES).default([]),
     expectedCitations: z
         .array(expectedValueSchema)
         .min(1)
         .max(MAX_GOLDEN_EXPECTED_VALUES)
         .optional(),
+    answerable: z.boolean().optional(),
+    category: z.string().min(1).max(100).optional(),
+    locale: z.string().min(2).max(50).optional(),
+    relevanceJudgments: z.array(relevanceJudgmentSchema).max(MAX_GOLDEN_EXPECTED_VALUES).optional(),
+    maximumVectorDistance: z.number().nonnegative().optional(),
     includePaths: z.array(z.string().min(1).max(500)).max(20).optional(),
     excludePaths: z.array(z.string().min(1).max(500)).max(20).optional(),
     contextPaths: z.array(z.string().min(1).max(500)).max(20).optional(),
     topK: z.number().int().positive().optional(),
+})
+    .strict()
+    .superRefine((value, context) => {
+    const answerable = value.answerable ?? true;
+    const positiveJudgments = (value.relevanceJudgments ?? []).filter((judgment) => judgment.relevance > 0);
+    if (answerable &&
+        value.expectedPaths.length === 0 &&
+        value.expectedCitations === undefined &&
+        positiveJudgments.length === 0) {
+        context.addIssue({
+            code: "custom",
+            message: "Answerable golden queries require at least one relevant path or citation.",
+            path: ["expectedPaths"],
+        });
+    }
+    if (!answerable && (value.expectedPaths.length > 0 || positiveJudgments.length > 0)) {
+        context.addIssue({
+            code: "custom",
+            message: "Unanswerable golden queries cannot declare relevant evidence.",
+            path: ["answerable"],
+        });
+    }
+});
+const wrappedGoldenFileSchema = z
+    .object({
+    topK: z.number().int().positive().optional(),
+    minimumCasesForVerification: z.number().int().positive().max(MAX_GOLDEN_CASES).optional(),
+    thresholds: qualityThresholdSchema.optional(),
+    queries: z.array(goldenQuerySchema).min(1).max(MAX_GOLDEN_CASES),
 })
     .strict();
 const goldenFileSchema = z.union([
     z.array(goldenQuerySchema).min(1).max(MAX_GOLDEN_CASES),
-    z
-        .object({
-        topK: z.number().int().positive().optional(),
-        queries: z.array(goldenQuerySchema).min(1).max(MAX_GOLDEN_CASES),
-    })
-        .strict(),
+    wrappedGoldenFileSchema,
 ]);
+async function createEvaluationClient(config, signal) {
+    const connection = await connectStore(config);
+    const releaseEmbeddingModel = retainEmbeddingModel(config);
+    let snapshot;
+    try {
+        snapshot = await loadEvaluationSnapshot(config, connection);
+        const generationLease = await acquireGenerationReadLease(snapshot.tableName, config);
+        return evaluationClient(config, connection, snapshot, generationLease, signal, () => releaseEmbeddingModel());
+    }
+    catch (error) {
+        if (snapshot) {
+            closeIndexReadSnapshot(snapshot, config);
+        }
+        closeStoreConnection(connection, config);
+        await releaseEmbeddingModel();
+        throw error;
+    }
+}
+function evaluationClient(config, connection, snapshot, generationLease, signal, releaseEmbeddingModel) {
+    let closed = false;
+    return {
+        snapshot,
+        search(query, options) {
+            return searchWithConfig(query, options, config, connection, signal, snapshot, true);
+        },
+        async close() {
+            if (closed) {
+                return;
+            }
+            closed = true;
+            try {
+                closeIndexReadSnapshot(snapshot, config);
+            }
+            finally {
+                try {
+                    await generationLease.release();
+                }
+                finally {
+                    try {
+                        closeStoreConnection(connection, config);
+                    }
+                    finally {
+                        await releaseEmbeddingModel();
+                    }
+                }
+            }
+        },
+    };
+}
+async function loadEvaluationSnapshot(config, connection) {
+    for (let attempt = 0; attempt < 3; attempt += 1) {
+        const snapshot = await loadIndexReadSnapshot(config, connection);
+        const manifest = await readIndexManifest(config);
+        const current = await indexReadSnapshotCurrent(snapshot, config);
+        const matchingPresence = (snapshot.manifest === null) === (manifest === null);
+        const manifestTableName = manifest?.tableName ?? config.tableName;
+        if (current && matchingPresence && manifestTableName === snapshot.tableName) {
+            return { ...snapshot, manifest: manifest ? Object.freeze({ ...manifest }) : null };
+        }
+        closeIndexReadSnapshot(snapshot, config);
+    }
+    throw new Error("Index generation changed repeatedly while opening an evaluation snapshot. Retry evaluation.");
+}
 export async function evaluateGoldenQueries(options) {
     const signal = operationSignal(options);
     throwIfAborted(signal);
     const cwd = path.resolve(String(options.cwd ?? process.cwd()));
     const config = await loadConfig(cwd);
+    return evaluateGoldenQueriesWithConfig(options, config);
+}
+export async function evaluateGoldenQueriesWithConfig(options, config) {
+    const signal = operationSignal(options);
+    throwIfAborted(signal);
+    const cwd = path.resolve(String(options.cwd ?? config.projectRoot));
+    const activeRankingPolicyFingerprint = rankingPolicyFingerprint(rankingPolicyFor(config.embeddingProvider, config.retrievalProfile));
     throwIfAborted(signal);
     const goldenPath = path.resolve(cwd, String(options.goldenPath));
     const goldenFile = await readGoldenFile(goldenPath, signal);
-    throwIfAborted(signal);
     const defaultTopK = boundedTopK(options.topK ?? goldenFile.topK ?? 3, options.maxTopK);
+    const caseDetailLimit = evaluationCaseDetailLimit(options.caseDetailLimit);
     const cases = [];
-    for (const goldenQuery of goldenFile.queries) {
-        throwIfAborted(signal);
-        const topK = boundedTopK(goldenQuery.topK ?? defaultTopK, options.maxTopK);
-        const startedAt = performance.now();
-        const results = await search(goldenQuery.query, {
-            cwd,
-            topK,
-            ...(signal === undefined ? {} : { signal }),
-            ...(goldenQuery.includePaths === undefined ? {} : { includePaths: goldenQuery.includePaths }),
-            ...(goldenQuery.excludePaths === undefined ? {} : { excludePaths: goldenQuery.excludePaths }),
-            ...(goldenQuery.contextPaths === undefined ? {} : { contextPaths: goldenQuery.contextPaths }),
-        });
+    const evaluationConcurrency = Math.max(1, Math.min(config.workloadLimits.search.concurrency, config.workloadLimits.embedding.concurrency + config.workloadLimits.embedding.maxQueue));
+    const evaluationClient = await createEvaluationClient(config, signal);
+    const initialManifest = evaluationClient.snapshot.manifest;
+    const indexFingerprint = initialManifest
+        ? fingerprintIndexManifest(initialManifest)
+        : "missing-index";
+    try {
         throwIfAborted(signal);
-        const latencyMs = performance.now() - startedAt;
-        const returnedPaths = results.map((result) => result.relativePath);
-        const returnedCitations = results.map(citationForResult);
-        const expectedCitations = goldenQuery.expectedCitations ?? [];
-        const requiresExactCitation = expectedCitations.length > 0;
-        const expectedValues = requiresExactCitation ? expectedCitations : goldenQuery.expectedPaths;
-        const matchedPaths = returnedPaths.filter((resultPath) => goldenQuery.expectedPaths.includes(resultPath));
-        const matchedCitations = returnedCitations.filter((citation) => expectedCitations.includes(citation));
-        const bestRank = requiresExactCitation
-            ? returnedCitations.findIndex((citation) => expectedCitations.includes(citation)) + 1
-            : returnedPaths.findIndex((resultPath) => goldenQuery.expectedPaths.includes(resultPath)) + 1;
-        const reciprocalRank = bestRank > 0 ? 1 / bestRank : 0;
-        const returnedValues = requiresExactCitation ? returnedCitations : returnedPaths;
-        const uniqueReturnedValues = [...new Set(returnedValues.slice(0, topK))];
-        const expectedSet = new Set(expectedValues);
-        const matchedRelevant = uniqueReturnedValues.filter((value) => expectedSet.has(value));
-        const recall = expectedSet.size === 0 ? 0 : matchedRelevant.length / expectedSet.size;
-        const precision = uniqueReturnedValues.length === 0 ? 0 : matchedRelevant.length / uniqueReturnedValues.length;
-        const ndcg = ndcgAtK(returnedValues, expectedValues, topK);
-        const result = {
-            query: goldenQuery.query,
-            expectedPaths: goldenQuery.expectedPaths,
-            topK,
-            returnedPaths,
-            returnedCitations,
-            matchedPaths,
-            matchedCitations,
-            hit: requiresExactCitation ? matchedCitations.length > 0 : matchedPaths.length > 0,
-            bestRank: bestRank > 0 ? bestRank : null,
-            reciprocalRank,
-            recall,
-            precision,
-            ndcg,
-            latencyMs,
-        };
-        if (goldenQuery.id !== undefined) {
-            result.id = goldenQuery.id;
+        for (let offset = 0; offset < goldenFile.queries.length; offset += evaluationConcurrency) {
+            throwIfAborted(signal);
+            const batch = goldenFile.queries.slice(offset, offset + evaluationConcurrency);
+            cases.push(...(await Promise.all(batch.map(async (goldenQuery) => {
+                const topK = boundedTopK(goldenQuery.topK ?? defaultTopK, options.maxTopK);
+                const evaluationDepth = boundedTopK(Math.max(topK, EVALUATION_DEPTH), options.maxTopK);
+                const startedAt = performance.now();
+                const results = await evaluationClient.search(goldenQuery.query, {
+                    cwd,
+                    topK: evaluationDepth,
+                    ...(signal === undefined ? {} : { signal }),
+                    ...(goldenQuery.includePaths === undefined
+                        ? {}
+                        : { includePaths: goldenQuery.includePaths }),
+                    ...(goldenQuery.excludePaths === undefined
+                        ? {}
+                        : { excludePaths: goldenQuery.excludePaths }),
+                    ...(goldenQuery.contextPaths === undefined
+                        ? {}
+                        : { contextPaths: goldenQuery.contextPaths }),
+                });
+                throwIfAborted(signal);
+                return evaluateCase(goldenQuery, applyVectorDistancePolicy(results, goldenQuery.maximumVectorDistance), topK, performance.now() - startedAt);
+            }))));
         }
-        if (goldenQuery.expectedCitations !== undefined) {
-            result.expectedCitations = goldenQuery.expectedCitations;
-        }
-        if (goldenQuery.includePaths !== undefined) {
-            result.includePaths = goldenQuery.includePaths;
-        }
-        if (goldenQuery.excludePaths !== undefined) {
-            result.excludePaths = goldenQuery.excludePaths;
-        }
-        if (goldenQuery.contextPaths !== undefined) {
-            result.contextPaths = goldenQuery.contextPaths;
-        }
-        cases.push(result);
     }
+    finally {
+        await evaluationClient.close();
+    }
+    const answerableCases = cases.filter((result) => result.answerable);
+    const unanswerableCases = cases.filter((result) => !result.answerable);
+    const citationCases = answerableCases.filter((result) => result.exactCitationHit !== null);
+    const gradedCases = answerableCases.filter((result) => result.relevanceJudgments.some((judgment) => judgment.relevance > 1));
+    const metrics = evaluationMetrics(answerableCases, unanswerableCases, citationCases);
+    const thresholds = { ...goldenFile.thresholds, ...options.thresholds };
+    const gates = qualityGates(metrics, thresholds);
+    const passed = gates.every((gate) => gate.passed);
+    const requiredThresholdSet = completeThresholdSet(thresholds);
+    const relativeGoldenPath = projectRelativePath(config.projectRoot, goldenPath);
+    const verificationEligible = initialManifest !== null &&
+        (initialManifest.staleFiles?.length ?? 0) === 0 &&
+        relativeGoldenPath !== null &&
+        cases.length >= goldenFile.minimumCasesForVerification &&
+        answerableCases.length > 0 &&
+        unanswerableCases.length > 0 &&
+        citationCases.length > 0 &&
+        gradedCases.length > 0 &&
+        (options.maxTopK === undefined || options.maxTopK >= EVALUATION_DEPTH) &&
+        requiredThresholdSet !== null;
+    const storedReport = verificationEligible &&
+        passed &&
+        options.persistCompatibleReport !== false &&
+        initialManifest !== null &&
+        relativeGoldenPath !== null &&
+        requiredThresholdSet !== null
+        ? await persistQualityReport({
+            config,
+            indexFingerprint,
+            goldenPath: relativeGoldenPath,
+            goldenFingerprint: goldenFile.fingerprint,
+            rankingPolicyFingerprint: activeRankingPolicyFingerprint,
+            metrics,
+            thresholds: requiredThresholdSet,
+            total: cases.length,
+            signal,
+        })
+        : null;
     const hits = cases.filter((result) => result.hit).length;
     const latencies = cases.map((result) => result.latencyMs).sort((a, b) => a - b);
+    const recall = mean(answerableCases.map((result) => result.recall));
+    const precision = mean(answerableCases.map((result) => result.precision));
+    const meanReciprocalRank = mean(answerableCases.map((result) => result.reciprocalRank));
+    const ndcg = mean(answerableCases.map((result) => result.ndcg));
+    const abstentionAccuracy = unanswerableCases.length === 0
+        ? null
+        : unanswerableCases.filter((result) => result.abstained).length / unanswerableCases.length;
     throwIfAborted(signal);
     await recordAccess(config, {
         action: "evaluate",
         topK: defaultTopK,
         resultCount: cases.length,
     });
+    await flushAccessLog(config);
     throwIfAborted(signal);
+    const returnedCases = caseDetailLimit === undefined ? cases : cases.slice(0, caseDetailLimit);
+    const omittedCases = cases.length - returnedCases.length;
     return {
         goldenPath,
         embeddingProvider: config.embeddingProvider,
         embeddingModel: config.embeddingModel,
+        embeddingModelRevision: config.embeddingModelRevision,
+        embeddingModelDigest: config.embeddingModelDigest,
+        retrievalProfile: config.retrievalProfile,
+        rankingPolicyFingerprint: activeRankingPolicyFingerprint,
+        indexFingerprint,
+        goldenFingerprint: goldenFile.fingerprint,
         topK: defaultTopK,
         total: cases.length,
         hits,
         misses: cases.length - hits,
         hitRate: hits / cases.length,
-        recall: mean(cases.map((result) => result.recall)),
-        precision: mean(cases.map((result) => result.precision)),
-        meanReciprocalRank: mean(cases.map((result) => result.reciprocalRank)),
-        ndcg: mean(cases.map((result) => result.ndcg)),
+        recall,
+        precision,
+        meanReciprocalRank,
+        ndcg,
+        recallAt: {
+            1: metrics.recallAt1,
+            3: metrics.recallAt3,
+            5: metrics.recallAt5,
+            10: metrics.recallAt10,
+        },
+        precisionAt5: metrics.precisionAt5,
+        meanReciprocalRankAt10: metrics.meanReciprocalRankAt10,
+        ndcgAt10: metrics.ndcgAt10,
+        exactCitationRate: metrics.exactCitationRate,
+        falsePositiveRate: metrics.falsePositiveRate,
+        abstentionAccuracy,
+        thresholds,
+        gates,
+        passed,
+        verificationEligible,
+        reportStored: storedReport !== null,
+        qualityReportFingerprint: storedReport?.qualityReportFingerprint ?? null,
+        groups: {
+            categories: groupResults(cases, (result) => result.category),
+            locales: groupResults(cases, (result) => result.locale),
+        },
         p50LatencyMs: percentile(latencies, 0.5),
         p95LatencyMs: percentile(latencies, 0.95),
-        cases,
+        cases: returnedCases,
+        ...(omittedCases === 0 ? {} : { omittedCases }),
     };
 }
+function evaluationCaseDetailLimit(value) {
+    if (value === undefined) {
+        return undefined;
+    }
+    if (!Number.isSafeInteger(value) || value < 0) {
+        throw new Error("Evaluation case detail limit must be a non-negative integer.");
+    }
+    return value;
+}
 async function readGoldenFile(goldenPath, signal) {
     let size;
     try {
@@ -162,62 +349,376 @@ async function readGoldenFile(goldenPath, signal) {
         throw error;
     }
     throwIfAborted(signal);
+    const fingerprint = createHash("sha256").update(raw).digest("hex");
     const parsed = goldenFileSchema.parse(JSON.parse(raw));
     if (Array.isArray(parsed)) {
-        return { queries: parsed.map(normalizeGoldenQuery) };
-    }
-    const result = { queries: parsed.queries.map(normalizeGoldenQuery) };
-    if (parsed.topK !== undefined) {
-        return { ...result, topK: parsed.topK };
+        return {
+            minimumCasesForVerification: DEFAULT_MINIMUM_VERIFICATION_CASES,
+            thresholds: {},
+            queries: parsed.map(normalizeGoldenQuery),
+            fingerprint,
+        };
     }
-    return result;
+    return {
+        ...(parsed.topK === undefined ? {} : { topK: parsed.topK }),
+        minimumCasesForVerification: parsed.minimumCasesForVerification ?? DEFAULT_MINIMUM_VERIFICATION_CASES,
+        thresholds: normalizeThresholds(parsed.thresholds),
+        queries: parsed.queries.map(normalizeGoldenQuery),
+        fingerprint,
+    };
 }
 function normalizeGoldenQuery(value) {
-    const result = {
+    return {
         query: value.query,
         expectedPaths: value.expectedPaths,
+        answerable: value.answerable ?? true,
+        relevanceJudgments: value.relevanceJudgments ?? [],
+        ...(value.expectedCitations === undefined
+            ? {}
+            : { expectedCitations: value.expectedCitations }),
+        ...(value.id === undefined ? {} : { id: value.id }),
+        ...(value.category === undefined ? {} : { category: value.category }),
+        ...(value.locale === undefined ? {} : { locale: value.locale }),
+        ...(value.maximumVectorDistance === undefined
+            ? {}
+            : { maximumVectorDistance: value.maximumVectorDistance }),
+        ...(value.topK === undefined ? {} : { topK: value.topK }),
+        ...(value.includePaths === undefined ? {} : { includePaths: value.includePaths }),
+        ...(value.excludePaths === undefined ? {} : { excludePaths: value.excludePaths }),
+        ...(value.contextPaths === undefined ? {} : { contextPaths: value.contextPaths }),
+    };
+}
+function evaluateCase(goldenQuery, results, topK, latencyMs) {
+    const answerable = goldenQuery.answerable ?? true;
+    const returnedPaths = results.map((result) => result.relativePath);
+    const returnedCitations = results.map(citationForResult);
+    const pathJudgments = judgmentsFor("path", goldenQuery.relevanceJudgments ?? [], goldenQuery.expectedPaths);
+    const citationJudgments = judgmentsFor("citation", goldenQuery.relevanceJudgments ?? [], goldenQuery.expectedCitations ?? []);
+    const requiresExactCitation = citationJudgments.some((judgment) => judgment.relevance > 0);
+    const primaryJudgments = requiresExactCitation ? citationJudgments : pathJudgments;
+    const returnedValues = requiresExactCitation ? returnedCitations : returnedPaths;
+    const matchedPaths = uniqueAtK(returnedPaths, returnedPaths.length).filter((resultPath) => pathJudgments.some((judgment) => judgment.value === resultPath && judgment.relevance > 0));
+    const matchedCitations = uniqueAtK(returnedCitations, returnedCitations.length).filter((citation) => citationJudgments.some((judgment) => judgment.value === citation && judgment.relevance > 0));
+    const bestRank = firstRelevantRank(returnedValues, primaryJudgments, topK);
+    const reciprocalRank = bestRank === null ? 0 : 1 / bestRank;
+    const recallAt = {
+        1: recallAtK(returnedValues, primaryJudgments, 1),
+        3: recallAtK(returnedValues, primaryJudgments, 3),
+        5: recallAtK(returnedValues, primaryJudgments, 5),
+        10: recallAtK(returnedValues, primaryJudgments, 10),
     };
-    if (value.expectedCitations !== undefined) {
-        result.expectedCitations = value.expectedCitations;
+    const abstained = returnedValues.length === 0;
+    const falsePositive = !answerable && !abstained;
+    const pathHit = matchedPaths.length > 0;
+    const exactCitationHit = requiresExactCitation ? matchedCitations.length > 0 : null;
+    const hit = answerable ? (requiresExactCitation ? exactCitationHit === true : pathHit) : abstained;
+    const relevanceJudgments = [...pathJudgments, ...citationJudgments];
+    const result = {
+        query: goldenQuery.query,
+        expectedPaths: goldenQuery.expectedPaths,
+        topK,
+        returnedPaths,
+        returnedCitations,
+        matchedPaths,
+        matchedCitations,
+        answerable,
+        relevanceJudgments,
+        abstained,
+        falsePositive,
+        pathHit,
+        exactCitationHit,
+        hit,
+        bestRank,
+        reciprocalRank,
+        recall: answerable ? recallAtK(returnedValues, primaryJudgments, topK) : Number(abstained),
+        precision: answerable ? precisionAtK(returnedValues, primaryJudgments, topK, false) : 0,
+        ndcg: answerable ? gradedNdcgAtK(returnedValues, primaryJudgments, topK) : Number(abstained),
+        recallAt,
+        precisionAt5: answerable ? precisionAtK(returnedValues, primaryJudgments, 5, true) : 0,
+        reciprocalRankAt10: answerable ? reciprocalRankAtK(returnedValues, primaryJudgments, 10) : 0,
+        ndcgAt10: answerable ? gradedNdcgAtK(returnedValues, primaryJudgments, 10) : 0,
+        latencyMs,
+    };
+    if (goldenQuery.id !== undefined) {
+        result.id = goldenQuery.id;
+    }
+    if (goldenQuery.category !== undefined) {
+        result.category = goldenQuery.category;
     }
-    if (value.id !== undefined) {
-        result.id = value.id;
+    if (goldenQuery.locale !== undefined) {
+        result.locale = goldenQuery.locale;
     }
-    if (value.topK !== undefined) {
-        result.topK = value.topK;
+    if (goldenQuery.expectedCitations !== undefined) {
+        result.expectedCitations = goldenQuery.expectedCitations;
     }
-    if (value.includePaths !== undefined) {
-        result.includePaths = value.includePaths;
+    if (goldenQuery.includePaths !== undefined) {
+        result.includePaths = goldenQuery.includePaths;
     }
-    if (value.excludePaths !== undefined) {
-        result.excludePaths = value.excludePaths;
+    if (goldenQuery.excludePaths !== undefined) {
+        result.excludePaths = goldenQuery.excludePaths;
     }
-    if (value.contextPaths !== undefined) {
-        result.contextPaths = value.contextPaths;
+    if (goldenQuery.contextPaths !== undefined) {
+        result.contextPaths = goldenQuery.contextPaths;
     }
     return result;
 }
-function boundedTopK(topK, maxTopK) {
-    return maxTopK === undefined ? topK : Math.min(topK, maxTopK);
+function judgmentsFor(kind, declared, legacyExpectedValues) {
+    const explicit = declared.filter((judgment) => judgment.kind === kind);
+    if (explicit.length > 0) {
+        return explicit;
+    }
+    return legacyExpectedValues.map((value) => ({ kind, value, relevance: 1 }));
 }
-function citationForResult(result) {
-    return result.citation;
+function applyVectorDistancePolicy(results, maximumVectorDistance) {
+    if (maximumVectorDistance === undefined) {
+        return results;
+    }
+    return results.filter((result) => result.distance !== null && result.distance <= maximumVectorDistance);
+}
+function evaluationMetrics(answerableCases, unanswerableCases, citationCases) {
+    return {
+        recallAt1: mean(answerableCases.map((result) => result.recallAt[1])),
+        recallAt3: mean(answerableCases.map((result) => result.recallAt[3])),
+        recallAt5: mean(answerableCases.map((result) => result.recallAt[5])),
+        recallAt10: mean(answerableCases.map((result) => result.recallAt[10])),
+        precisionAt5: mean(answerableCases.map((result) => result.precisionAt5)),
+        meanReciprocalRankAt10: mean(answerableCases.map((result) => result.reciprocalRankAt10)),
+        ndcgAt10: mean(answerableCases.map((result) => result.ndcgAt10)),
+        exactCitationRate: citationCases.length === 0
+            ? null
+            : citationCases.filter((result) => result.exactCitationHit === true).length /
+                citationCases.length,
+        falsePositiveRate: unanswerableCases.length === 0
+            ? null
+            : unanswerableCases.filter((result) => result.falsePositive).length /
+                unanswerableCases.length,
+    };
 }
-function ndcgAtK(returned, expected, topK) {
-    const expectedSet = new Set(expected);
-    const returnedAtK = returned.slice(0, topK);
+function qualityGates(metrics, thresholds) {
+    const gates = [];
+    addMinimumGate(gates, "recallAt1", thresholds.recallAt1, metrics.recallAt1);
+    addMinimumGate(gates, "recallAt3", thresholds.recallAt3, metrics.recallAt3);
+    addMinimumGate(gates, "recallAt5", thresholds.recallAt5, metrics.recallAt5);
+    addMinimumGate(gates, "recallAt10", thresholds.recallAt10, metrics.recallAt10);
+    addMinimumGate(gates, "precisionAt5", thresholds.precisionAt5, metrics.precisionAt5);
+    addMinimumGate(gates, "meanReciprocalRankAt10", thresholds.meanReciprocalRankAt10, metrics.meanReciprocalRankAt10);
+    addMinimumGate(gates, "ndcgAt10", thresholds.ndcgAt10, metrics.ndcgAt10);
+    addMinimumGate(gates, "exactCitationRate", thresholds.exactCitationRate, metrics.exactCitationRate);
+    addMaximumGate(gates, "maximumFalsePositiveRate", thresholds.maximumFalsePositiveRate, metrics.falsePositiveRate);
+    return gates;
+}
+function addMinimumGate(gates, metric, threshold, actual) {
+    if (threshold === undefined) {
+        return;
+    }
+    gates.push({
+        metric,
+        direction: "minimum",
+        threshold,
+        actual,
+        applicable: actual !== null,
+        passed: actual !== null && actual >= threshold,
+    });
+}
+function addMaximumGate(gates, metric, threshold, actual) {
+    if (threshold === undefined) {
+        return;
+    }
+    gates.push({
+        metric,
+        direction: "maximum",
+        threshold,
+        actual,
+        applicable: actual !== null,
+        passed: actual !== null && actual <= threshold,
+    });
+}
+function completeThresholdSet(thresholds) {
+    if (thresholds.recallAt1 === undefined ||
+        thresholds.recallAt3 === undefined ||
+        thresholds.recallAt5 === undefined ||
+        thresholds.recallAt10 === undefined ||
+        thresholds.precisionAt5 === undefined ||
+        thresholds.meanReciprocalRankAt10 === undefined ||
+        thresholds.ndcgAt10 === undefined ||
+        thresholds.exactCitationRate === undefined ||
+        thresholds.maximumFalsePositiveRate === undefined) {
+        return null;
+    }
+    return {
+        recallAt1: thresholds.recallAt1,
+        recallAt3: thresholds.recallAt3,
+        recallAt5: thresholds.recallAt5,
+        recallAt10: thresholds.recallAt10,
+        precisionAt5: thresholds.precisionAt5,
+        meanReciprocalRankAt10: thresholds.meanReciprocalRankAt10,
+        ndcgAt10: thresholds.ndcgAt10,
+        exactCitationRate: thresholds.exactCitationRate,
+        maximumFalsePositiveRate: thresholds.maximumFalsePositiveRate,
+    };
+}
+async function persistQualityReport(input) {
+    return withIndexWriteLock(input.config.storageDir, input.signal, async () => {
+        const activeManifest = await readIndexManifest(input.config);
+        if (!activeManifest ||
+            fingerprintIndexManifest(activeManifest) !== input.indexFingerprint ||
+            input.metrics.exactCitationRate === null ||
+            input.metrics.falsePositiveRate === null) {
+            return null;
+        }
+        const unsignedReport = {
+            schemaVersion: 3,
+            createdAt: new Date().toISOString(),
+            goldenPath: input.goldenPath,
+            goldenFingerprint: input.goldenFingerprint,
+            indexFingerprint: input.indexFingerprint,
+            indexPolicyFingerprint: activeManifest.indexPolicyFingerprint ?? "",
+            embeddingProvider: input.config.embeddingProvider,
+            embeddingModel: input.config.embeddingModel,
+            embeddingModelRevision: input.config.embeddingModelRevision,
+            embeddingModelDigest: input.config.embeddingModelDigest,
+            retrievalProfile: input.config.retrievalProfile,
+            rankingPolicyFingerprint: input.rankingPolicyFingerprint,
+            total: input.total,
+            metrics: {
+                recallAt1: input.metrics.recallAt1,
+                recallAt3: input.metrics.recallAt3,
+                recallAt5: input.metrics.recallAt5,
+                recallAt10: input.metrics.recallAt10,
+                precisionAt5: input.metrics.precisionAt5,
+                meanReciprocalRankAt10: input.metrics.meanReciprocalRankAt10,
+                ndcgAt10: input.metrics.ndcgAt10,
+                exactCitationRate: input.metrics.exactCitationRate,
+                falsePositiveRate: input.metrics.falsePositiveRate,
+            },
+            thresholds: input.thresholds,
+            passed: true,
+            verificationEligible: true,
+        };
+        const qualityReport = {
+            ...unsignedReport,
+            qualityReportFingerprint: fingerprintQualityReport(unsignedReport),
+        };
+        await writeIndexManifest({ ...activeManifest, qualityReport }, input.config);
+        return qualityReport;
+    });
+}
+function groupResults(cases, groupFor) {
+    const grouped = new Map();
+    for (const result of cases) {
+        const group = groupFor(result);
+        if (group === undefined) {
+            continue;
+        }
+        const members = grouped.get(group) ?? [];
+        members.push(result);
+        grouped.set(group, members);
+    }
+    return Object.fromEntries([...grouped.entries()]
+        .sort(([left], [right]) => left.localeCompare(right))
+        .map(([group, members]) => [group, summarizeGroup(members)]));
+}
+function summarizeGroup(cases) {
+    const answerable = cases.filter((result) => result.answerable);
+    const unanswerable = cases.filter((result) => !result.answerable);
+    return {
+        total: cases.length,
+        answerable: answerable.length,
+        unanswerable: unanswerable.length,
+        recallAt10: mean(answerable.map((result) => result.recallAt[10])),
+        precisionAt5: mean(answerable.map((result) => result.precisionAt5)),
+        meanReciprocalRankAt10: mean(answerable.map((result) => result.reciprocalRankAt10)),
+        ndcgAt10: mean(answerable.map((result) => result.ndcgAt10)),
+        falsePositiveRate: unanswerable.length === 0
+            ? 0
+            : unanswerable.filter((result) => result.falsePositive).length / unanswerable.length,
+    };
+}
+function projectRelativePath(projectRoot, targetPath) {
+    const relativePath = path.relative(projectRoot, targetPath);
+    if (relativePath === ".." ||
+        relativePath.startsWith(`..${path.sep}`) ||
+        path.isAbsolute(relativePath)) {
+        return null;
+    }
+    return relativePath.split(path.sep).join("/");
+}
+function recallAtK(returned, judgments, topK) {
+    const relevant = new Set(judgments.filter((judgment) => judgment.relevance > 0).map((judgment) => judgment.value));
+    if (relevant.size === 0) {
+        return 0;
+    }
+    return uniqueAtK(returned, topK).filter((value) => relevant.has(value)).length / relevant.size;
+}
+function precisionAtK(returned, judgments, topK, fixedDenominator) {
+    const values = uniqueAtK(returned, topK);
+    const relevant = new Set(judgments.filter((judgment) => judgment.relevance > 0).map((judgment) => judgment.value));
+    const denominator = fixedDenominator ? topK : values.length;
+    if (denominator === 0) {
+        return 0;
+    }
+    return values.filter((value) => relevant.has(value)).length / denominator;
+}
+function reciprocalRankAtK(returned, judgments, topK) {
+    const rank = firstRelevantRank(returned, judgments, topK);
+    return rank === null ? 0 : 1 / rank;
+}
+function firstRelevantRank(returned, judgments, topK) {
+    const relevant = new Set(judgments.filter((judgment) => judgment.relevance > 0).map((judgment) => judgment.value));
+    const index = returned.slice(0, topK).findIndex((value) => relevant.has(value));
+    return index < 0 ? null : index + 1;
+}
+function gradedNdcgAtK(returned, judgments, topK) {
+    const relevance = new Map(judgments.map((judgment) => [judgment.value, judgment.relevance]));
     const seen = new Set();
-    const dcg = returnedAtK.reduce((score, value, index) => {
-        if (!expectedSet.has(value) || seen.has(value)) {
+    const dcg = returned.slice(0, topK).reduce((score, value, index) => {
+        if (seen.has(value)) {
             return score;
         }
         seen.add(value);
-        return score + 1 / Math.log2(index + 2);
+        const grade = relevance.get(value) ?? 0;
+        return score + (2 ** grade - 1) / Math.log2(index + 2);
     }, 0);
-    const idealMatches = Math.min(expectedSet.size, topK);
-    const idealDcg = Array.from({ length: idealMatches }, (_value, index) => 1 / Math.log2(index + 2)).reduce((score, gain) => score + gain, 0);
+    const idealGrades = [
+        ...new Map(judgments.map((judgment) => [judgment.value, judgment.relevance])).values(),
+    ]
+        .filter((grade) => grade > 0)
+        .sort((left, right) => right - left)
+        .slice(0, topK);
+    const idealDcg = idealGrades.reduce((score, grade, index) => score + (2 ** grade - 1) / Math.log2(index + 2), 0);
     return idealDcg === 0 ? 0 : dcg / idealDcg;
 }
+function normalizeThresholds(value) {
+    if (value === undefined) {
+        return {};
+    }
+    return {
+        ...(value.recallAt1 === undefined ? {} : { recallAt1: value.recallAt1 }),
+        ...(value.recallAt3 === undefined ? {} : { recallAt3: value.recallAt3 }),
+        ...(value.recallAt5 === undefined ? {} : { recallAt5: value.recallAt5 }),
+        ...(value.recallAt10 === undefined ? {} : { recallAt10: value.recallAt10 }),
+        ...(value.precisionAt5 === undefined ? {} : { precisionAt5: value.precisionAt5 }),
+        ...(value.meanReciprocalRankAt10 === undefined
+            ? {}
+            : { meanReciprocalRankAt10: value.meanReciprocalRankAt10 }),
+        ...(value.ndcgAt10 === undefined ? {} : { ndcgAt10: value.ndcgAt10 }),
+        ...(value.exactCitationRate === undefined
+            ? {}
+            : { exactCitationRate: value.exactCitationRate }),
+        ...(value.maximumFalsePositiveRate === undefined
+            ? {}
+            : { maximumFalsePositiveRate: value.maximumFalsePositiveRate }),
+    };
+}
+function uniqueAtK(values, topK) {
+    return [...new Set(values.slice(0, topK))];
+}
+function boundedTopK(topK, maxTopK) {
+    return maxTopK === undefined ? topK : Math.min(topK, maxTopK);
+}
+function citationForResult(result) {
+    return result.citation;
+}
 function percentile(sortedValues, quantile) {
     if (sortedValues.length === 0) {
         return 0;