@dailephd/my-dev-kit-lab 0.7.2 → 0.8.0

package/dist/src/commands/runExperimentRunCommand.js

@@ -3,8 +3,9 @@ import path from "node:path";
 import { parseAgentCommandTemplate } from "../agents/index.js";
 import { parseAgentId } from "../agents/agentRegistry.js";
 import { readBenchmarkProjectProfiles, readEvaluationCases } from "../evaluation/index.js";
+import { summarizeRetrievalGroundTruthIssuesSafely, validateRetrievalPrecisionRecallCorpus } from "../evaluation/retrievalQuality/index.js";
 import { loadLocalRepositorySubject } from "../evaluation/localRepositorySubject/index.js";
-import { contextStrategyComparisonPlugin, contextWindowScalingPlugin, createDefaultExperimentPluginRegistry, getWarmIndexCampaignPreset, incrementalChangeStalenessPlugin, parseContextBudgetsCliValue, parseWarmIndexCampaignPresetId, resolveExperimentTarget, runExperiment, warmIndexReusePlugin } from "../experiments/index.js";
+import { contextStrategyComparisonPlugin, contextWindowScalingPlugin, createDefaultExperimentPluginRegistry, getWarmIndexCampaignPreset, incrementalChangeStalenessPlugin, parseContextBudgetsCliValue, parseWarmIndexCampaignPresetId, resolveExperimentTarget, retrievalPrecisionRecallPlugin, runExperiment, warmIndexReusePlugin } from "../experiments/index.js";
 import { prepareSyntheticContextWindowScalingInputs } from "../experiments/plugins/contextWindowScaling/index.js";
 import { projectExternalLocalTarget, projectRunForExternalLocalPersistence, } from "../experiments/plugins/contextWindowScaling/localSubjectPrivacy.js";
 import { assertWorkRootOutsideTarget } from "../experiments/plugins/contextWindowScaling/localSubjectScratch.js";
@@ -51,6 +52,13 @@ const CONTEXT_WINDOW_SCALING_ALLOWED_FLAGS = [
     "--context-budgets",
     "--kit-command"
 ];
+// retrieval-precision-recall owns one frozen bundled corpus (the warm-index benchmark cases) and its project profiles,
+// both resolved from the package root, never the cwd. It accepts only these options; every other flag is rejected.
+const RETRIEVAL_PRECISION_RECALL_CASES_RESOURCE = "benchmarks/contracts/warm-index-benchmark-cases.json";
+const RETRIEVAL_PRECISION_RECALL_ALLOWED_FLAGS = ["--experiment", "--out", "--case", "--benchmark-project", "--kit-command"];
+// External-local mode (--target with --local-subject-config) is a different subject source: the local-subject config owns
+// the case set, so the bundled corpus filters are not accepted there.
+const RETRIEVAL_PRECISION_RECALL_EXTERNAL_ALLOWED_FLAGS = ["--experiment", "--out", "--target", "--local-subject-config", "--kit-command"];
 export async function runExperimentRunCommandFromArgs(argv, options = {}) {
     try {
         const args = parseRunExperimentArgs(argv);
@@ -300,16 +308,43 @@ export function parseRunExperimentArgs(argv) {
     const KIT_COMMAND_PLUGIN_IDS = [
         warmIndexReusePlugin.metadata.id,
         incrementalChangeStalenessPlugin.metadata.id,
-        contextWindowScalingPlugin.metadata.id
+        contextWindowScalingPlugin.metadata.id,
+        retrievalPrecisionRecallPlugin.metadata.id
     ];
+    if (experimentId === retrievalPrecisionRecallPlugin.metadata.id) {
+        // Mode matrix, no precedence rules: neither flag = bundled; both = external-local; exactly one is rejected.
+        if (targetPath !== undefined && localSubjectConfigPath === undefined) {
+            throw new Error(`External ${experimentId} targets require --local-subject-config.`);
+        }
+        if (localSubjectConfigPath !== undefined && targetPath === undefined) {
+            throw new Error(`--local-subject-config requires an external --target for ${experimentId}.`);
+        }
+        const externalMode = targetPath !== undefined;
+        if (externalMode && (seenFlags.includes("--case") || seenFlags.includes("--benchmark-project"))) {
+            throw new Error("--case and --benchmark-project cannot be combined with --local-subject-config; the local subject config owns the case set.");
+        }
+        const allowedFlags = externalMode ? RETRIEVAL_PRECISION_RECALL_EXTERNAL_ALLOWED_FLAGS : RETRIEVAL_PRECISION_RECALL_ALLOWED_FLAGS;
+        const unsupported = [...new Set(seenFlags.filter((flag) => !allowedFlags.includes(flag)))];
+        if (unsupported.length > 0) {
+            throw new Error(`${unsupported.join(", ")} ${unsupported.length === 1 ? "is" : "are"} not supported for --experiment ${experimentId}${externalMode ? " in external-local mode" : ""}; supported options: ${allowedFlags.join(", ")}.`);
+        }
+        if (seenFlags.includes("--case") && caseIds.length === 0) {
+            throw new Error("--case must list at least one case id.");
+        }
+        if (seenFlags.includes("--benchmark-project") && benchmarkProjects.length === 0) {
+            throw new Error("--benchmark-project must list at least one benchmark project id.");
+        }
+    }
     if (contextBudgets !== undefined && experimentId !== contextWindowScalingPlugin.metadata.id) {
         throw new Error(`--context-budgets is only supported for --experiment ${contextWindowScalingPlugin.metadata.id}.`);
     }
     if (syntheticConfigPath !== undefined && experimentId !== contextWindowScalingPlugin.metadata.id) {
         throw new Error(`--synthetic-config is only supported for --experiment ${contextWindowScalingPlugin.metadata.id}.`);
     }
-    if (localSubjectConfigPath !== undefined && experimentId !== contextWindowScalingPlugin.metadata.id) {
-        throw new Error(`--local-subject-config is only supported for --experiment ${contextWindowScalingPlugin.metadata.id}.`);
+    if (localSubjectConfigPath !== undefined &&
+        experimentId !== contextWindowScalingPlugin.metadata.id &&
+        experimentId !== retrievalPrecisionRecallPlugin.metadata.id) {
+        throw new Error(`--local-subject-config is only supported for --experiment ${contextWindowScalingPlugin.metadata.id} or ${retrievalPrecisionRecallPlugin.metadata.id}.`);
     }
     if (experimentId === contextWindowScalingPlugin.metadata.id) {
         const unsupported = [...new Set(seenFlags.filter((flag) => !CONTEXT_WINDOW_SCALING_ALLOWED_FLAGS.includes(flag)))];
@@ -426,8 +461,64 @@ async function loadPluginInputs(args, toolRoot, context, outputRoot) {
     if (args.experimentId === contextWindowScalingPlugin.metadata.id) {
         return loadContextWindowScalingInputs(args, toolRoot, context, outputRoot);
     }
+    if (args.experimentId === retrievalPrecisionRecallPlugin.metadata.id) {
+        return args.localSubjectConfigPath !== undefined
+            ? loadRetrievalPrecisionRecallLocalSubjectInputs(args, toolRoot, context, outputRoot)
+            : loadRetrievalPrecisionRecallInputs(args, toolRoot, context);
+    }
     return undefined;
 }
+// External-local retrieval-precision-recall: the existing local-subject loader owns the repository, inventory and case
+// adapter. The only additions are the physical output-root check before anything is created and a ground-truth gate
+// whose message carries logical case ids and counts only (an external subject may have private fact ids and names).
+async function loadRetrievalPrecisionRecallLocalSubjectInputs(args, toolRoot, context, outputRoot) {
+    const validation = retrievalPrecisionRecallPlugin.validateConfig(args.config);
+    if (!validation.valid || !validation.config) {
+        throw new Error(`Invalid retrieval precision/recall config: ${validation.errors.join("; ")}`);
+    }
+    if (outputRoot === undefined || args.localSubjectConfigPath === undefined) {
+        throw new Error("Internal error: external-local runs require a resolved experiment output root and config path.");
+    }
+    const target = resolveExperimentTarget(args.targetPath, toolRoot);
+    if (target.isSelf || target.kind !== "external-local") {
+        throw new Error("Local-repository subject mode requires an external-local target; --target must not be the Lab repository.");
+    }
+    // Fail closed before anything is created: the output root (and the private scratch beneath it) must be physically outside the target.
+    try {
+        await assertWorkRootOutsideTarget(outputRoot, target.targetRoot);
+    }
+    catch {
+        throw new Error("Experiment output root must not be inside the external target project.");
+    }
+    const configPath = path.resolve(context.invocationCwd, args.localSubjectConfigPath);
+    const localSubjectConfig = await readJsonConfigFile("--local-subject-config", configPath, args.localSubjectConfigPath);
+    const localSubject = await loadLocalRepositorySubject({ config: localSubjectConfig, repositoryPath: target.targetRoot });
+    const groundTruthIssues = summarizeRetrievalGroundTruthIssuesSafely(localSubject.evaluationCases);
+    if (groundTruthIssues.length > 0) {
+        throw new Error(`Invalid retrieval-precision-recall ground truth for the local subject (${groundTruthIssues
+            .map((issue) => `case ${issue.caseId}: ${issue.issueCount} issue${issue.issueCount === 1 ? "" : "s"}`)
+            .join("; ")}); details withheld.`);
+    }
+    return { cases: localSubject.evaluationCases, localSubject, env: process.env };
+}
+// The frozen bundled corpus is read through the established resource resolver and validation path, then must satisfy
+// the retrieval-precision-recall ground-truth completeness rules. Selection filters are applied by the plugin.
+async function loadRetrievalPrecisionRecallInputs(args, toolRoot, context) {
+    const validation = retrievalPrecisionRecallPlugin.validateConfig(args.config);
+    if (!validation.valid || !validation.config) {
+        throw new Error(`Invalid retrieval precision/recall config: ${validation.errors.join("; ")}`);
+    }
+    const projectProfiles = await readBenchmarkProjectProfiles(resolvePackageResource(context, DEFAULT_PROJECT_PROFILES_RESOURCE), toolRoot);
+    const cases = await readEvaluationCases(resolvePackageResource(context, RETRIEVAL_PRECISION_RECALL_CASES_RESOURCE), toolRoot, {
+        projectProfiles,
+        requireProjectProfileRef: true
+    });
+    const corpusErrors = validateRetrievalPrecisionRecallCorpus(cases);
+    if (corpusErrors.length > 0) {
+        throw new Error(`Invalid retrieval-precision-recall corpus: ${corpusErrors.join(" ")}`);
+    }
+    return { cases };
+}
 // --case is a selection over the frozen bundled catalog (catalog order, not argument order);
 // the rest of the CLI config is the plugin's own ContextWindowScalingConfig.
 function pluginConfigFor(args) {

package/dist/src/evaluation/benchmarkMetadata.js

@@ -2,6 +2,7 @@ import { existsSync } from "node:fs";
 import { readFile } from "node:fs/promises";
 import path from "node:path";
 import { PROJECT_COMPLEXITY_FORMULA, calculateProjectComplexityScore, roundToTwo } from "./projectComplexity.js";
+import { hasFactMapping, interpretFactContextTarget } from "./retrievalQuality/factContextTargets.js";
 import { TASK_LOCALITIES } from "./types.js";
 export const REQUIRED_BENCHMARK_PROJECT_IDS = [
     "todo-ts",
@@ -90,6 +91,18 @@ export function validateAnswerKey(answerKey, label) {
             errors.push(`${label}: answerKey.minimumCorrectFacts is not satisfiable.`);
         }
     }
+    if (Array.isArray(candidate.expectedContextTargets) && Array.isArray(candidate.expectedFacts)) {
+        // Only targets that opt into fact mapping with `factIds` are validated here; legacy targets stay valid as-is.
+        const knownFactIds = new Set(candidate.expectedFacts.flatMap((fact) => (fact && typeof fact === "object" && typeof fact.id === "string" && fact.id.length > 0 ? [fact.id] : [])));
+        candidate.expectedContextTargets.forEach((target, index) => {
+            if (!hasFactMapping(target))
+                return;
+            const interpretation = interpretFactContextTarget(target, knownFactIds);
+            if (!interpretation.ok) {
+                errors.push(`${label}: answerKey.expectedContextTargets[${index}]: ${interpretation.problem}.`);
+            }
+        });
+    }
     if (Array.isArray(candidate.expectedFiles) && candidate.expectedFiles.length === 0) {
         errors.push(`${label}: answerKey.expectedFiles must not be empty.`);
     }

package/dist/src/evaluation/retrievalQuality/buildRetrievalEvidence.js

@@ -0,0 +1,292 @@
+import { compareCodeUnits } from "../indexSnapshot.js";
+import { RETRIEVAL_COMMAND_FAMILIES } from "./types.js";
+const MAX_PATH_LENGTH = 1024;
+const REASON_COMMAND_FAILED = "command-failed";
+const REASON_NOT_JSON = "output-is-not-json";
+const REASON_UNSUPPORTED = "output-is-not-the-supported-upstream-schema";
+const REASON_REJECTED = "unsafe-or-invalid-identity-rejected";
+const REASON_NO_SELECTED_FILE = "no-selected-file-for-source-attribution";
+const REASON_SELECTION_UNTRUSTED = "selection-not-backed-by-supported-search-output";
+const REASON_DUPLICATE = "repeated-command-family-ignored";
+/**
+ * Normalizes a repository-relative path to the single platform-neutral form (forward slashes, no leading
+ * "./"). Returns null for anything that is not a safe repository-relative path: absolute, drive-qualified,
+ * UNC, traversing, or containing empty segments. It never repairs an unsafe value into a plausible one.
+ *
+ * Same contract as the context-window-scaling plugin's `normalizeRepositoryRelativePath`; kept local so the
+ * evaluation layer does not import from an experiment plugin. The unit tests assert the two agree.
+ */
+export function normalizeRetrievedRepositoryPath(value) {
+    if (typeof value !== "string" || value.length === 0 || value.length > MAX_PATH_LENGTH || value.includes("\0")) {
+        return null;
+    }
+    let normalized = value.replace(/\\/g, "/");
+    while (normalized.startsWith("./")) {
+        normalized = normalized.slice(2);
+    }
+    if (normalized.length === 0 || normalized.startsWith("/") || /^[A-Za-z]:/.test(normalized)) {
+        return null;
+    }
+    if (normalized.split("/").some((segment) => segment === ".." || segment === "")) {
+        return null;
+    }
+    return normalized;
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function parseJson(text) {
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return undefined;
+    }
+}
+function symbolKey(name, nodeId, file) {
+    return nodeId !== undefined ? `id\0${nodeId}` : `name\0${file ?? ""}\0${name}`;
+}
+/**
+ * Reads one upstream node-like entry. Only `file` and `symbol` kinds are evidence; edges and every other
+ * kind carry no explicit file/symbol identity of their own and are ignored. Fields are read by their
+ * documented names only; identifiers embedded inside other strings are never parsed.
+ */
+function readNode(entry, idKey) {
+    if (!isRecord(entry) || typeof entry.kind !== "string") {
+        return { type: "rejected" };
+    }
+    if (entry.kind === "file") {
+        const path = normalizeRetrievedRepositoryPath(entry.path);
+        return path === null ? { type: "rejected" } : { type: "file", path };
+    }
+    if (entry.kind === "symbol") {
+        const name = typeof entry.symbolName === "string" && entry.symbolName.length > 0 ? entry.symbolName : entry.label;
+        if (typeof name !== "string" || name.length === 0) {
+            return { type: "rejected" };
+        }
+        let file;
+        if (entry.path !== undefined) {
+            const normalized = normalizeRetrievedRepositoryPath(entry.path);
+            if (normalized === null) {
+                return { type: "rejected" };
+            }
+            file = normalized;
+        }
+        const rawId = entry[idKey];
+        if (rawId !== undefined && (typeof rawId !== "string" || rawId.length === 0)) {
+            return { type: "rejected" };
+        }
+        return { type: "symbol", name, nodeId: rawId, file };
+    }
+    return { type: "ignored" };
+}
+function interpretEntries(family, entries, idKey, files, symbols) {
+    const result = { parseState: "parsed", files: new Set(), symbolKeys: new Set(), rejected: 0 };
+    for (const entry of entries) {
+        const reading = readNode(entry, idKey);
+        if (reading.type === "ignored")
+            continue;
+        if (reading.type === "rejected") {
+            result.rejected += 1;
+            continue;
+        }
+        if (reading.type === "file") {
+            addFile(files, reading.path, family);
+            result.files.add(reading.path);
+            continue;
+        }
+        const key = symbolKey(reading.name, reading.nodeId, reading.file);
+        const existing = symbols.get(key);
+        if (existing) {
+            existing.surfacedBy.add(family);
+        }
+        else {
+            symbols.set(key, { name: reading.name, nodeId: reading.nodeId, file: reading.file, surfacedBy: new Set([family]) });
+        }
+        result.symbolKeys.add(key);
+        if (reading.file !== undefined) {
+            // A symbol node carries an explicit file identity, so that file is surfaced by the same command.
+            addFile(files, reading.file, family);
+            result.files.add(reading.file);
+        }
+    }
+    if (result.rejected > 0) {
+        result.reason = REASON_REJECTED;
+    }
+    return result;
+}
+function addFile(files, path, family) {
+    const existing = files.get(path);
+    if (existing) {
+        existing.add(family);
+    }
+    else {
+        files.set(path, new Set([family]));
+    }
+}
+function failedInterpretation(parseState, reason) {
+    return { parseState, files: new Set(), symbolKeys: new Set(), rejected: 0, reason };
+}
+function interpretStructured(command, files, symbols) {
+    const payload = parseJson(command.stdout);
+    if (!isRecord(payload)) {
+        return failedInterpretation(payload === undefined ? "malformed" : "unsupported-schema", payload === undefined ? REASON_NOT_JSON : REASON_UNSUPPORTED);
+    }
+    if (command.family === "search") {
+        const supported = payload.artifactKind === "my-dev-kit-v1-search-result" &&
+            typeof payload.version === "string" &&
+            payload.version.startsWith("1.") &&
+            Array.isArray(payload.results);
+        return supported
+            ? interpretEntries("search", payload.results, "nodeId", files, symbols)
+            : failedInterpretation("unsupported-schema", REASON_UNSUPPORTED);
+    }
+    if (command.family === "lookup") {
+        const supported = payload.status === "found" && isRecord(payload.node) && Array.isArray(payload.neighbors);
+        return supported
+            ? interpretEntries("lookup", [payload.node, ...payload.neighbors], "id", files, symbols)
+            : failedInterpretation("unsupported-schema", REASON_UNSUPPORTED);
+    }
+    const supported = payload.artifactKind === "my-dev-kit-v1-graph-slice" && typeof payload.version === "string" && payload.version.startsWith("1.") && Array.isArray(payload.nodes);
+    return supported
+        ? interpretEntries("slice", payload.nodes, "id", files, symbols)
+        : failedInterpretation("unsupported-schema", REASON_UNSUPPORTED);
+}
+/**
+ * Source output is context text. Evidence is attributed only from the trusted selection the lifecycle
+ * passed to the source command: its selected file, and the symbol already surfaced by structured output
+ * under the same node id. Nothing is read from the source body.
+ */
+function attributeSource(selection, searchInterpreted, files, symbols) {
+    if (!searchInterpreted) {
+        // The selection was derived from search output; without supported search evidence it is not trusted.
+        return failedInterpretation("unattributable", REASON_SELECTION_UNTRUSTED);
+    }
+    if (selection?.file === undefined) {
+        return failedInterpretation("unattributable", REASON_NO_SELECTED_FILE);
+    }
+    const path = normalizeRetrievedRepositoryPath(selection.file);
+    if (path === null) {
+        return { ...failedInterpretation("unattributable", REASON_REJECTED), rejected: 1 };
+    }
+    const result = { parseState: "selection-attributed", files: new Set([path]), symbolKeys: new Set(), rejected: 0 };
+    addFile(files, path, "source");
+    if (selection.nodeId !== undefined) {
+        const key = symbolKey("", selection.nodeId, undefined);
+        const known = symbols.get(key);
+        if (known) {
+            known.surfacedBy.add("source");
+            result.symbolKeys.add(key);
+        }
+    }
+    return result;
+}
+function orderedProvenance(provenance) {
+    return RETRIEVAL_COMMAND_FAMILIES.filter((family) => provenance.has(family));
+}
+function isFullyInterpreted(entry) {
+    return (entry.parseState === "parsed" || entry.parseState === "selection-attributed") && entry.rejectedIdentityCount === 0;
+}
+/**
+ * Pure policy: builds normalized retrieval evidence from the commands the lifecycle executed.
+ * Deterministic: output ordering depends only on code-unit comparison of identities and the fixed
+ * search -> lookup -> slice -> source family order, never on input order or object enumeration.
+ */
+export function buildRetrievalEvidence(input) {
+    const files = new Map();
+    const symbols = new Map();
+    const interpreted = new Map();
+    const duplicates = [];
+    const ordered = RETRIEVAL_COMMAND_FAMILIES.flatMap((family) => input.commands.filter((command) => command.family === family));
+    // Symbol attribution for source depends on structured evidence, so structured families run first (they do
+    // by family order); source is last.
+    for (const command of ordered) {
+        if (interpreted.has(command.family)) {
+            duplicates.push(command.family);
+            continue;
+        }
+        let outcome;
+        if (!command.ok) {
+            outcome = failedInterpretation("command-failed", REASON_COMMAND_FAILED);
+        }
+        else if (command.family === "source") {
+            outcome = attributeSource(input.selection, interpreted.get("search")?.parseState === "parsed", files, symbols);
+        }
+        else {
+            outcome = interpretStructured(command, files, symbols);
+        }
+        interpreted.set(command.family, {
+            family: command.family,
+            succeeded: command.ok,
+            parseState: outcome.parseState,
+            fileEvidenceCount: outcome.files.size,
+            symbolEvidenceCount: outcome.symbolKeys.size,
+            rejectedIdentityCount: outcome.rejected,
+            ...(outcome.reason !== undefined ? { reason: outcome.reason } : {})
+        });
+    }
+    const commands = RETRIEVAL_COMMAND_FAMILIES.flatMap((family) => {
+        const entry = interpreted.get(family);
+        return entry ? [entry] : [];
+    });
+    const fileEvidence = [...files.entries()]
+        .map(([path, provenance]) => ({ path, surfacedBy: orderedProvenance(provenance) }))
+        .sort((left, right) => compareCodeUnits(left.path, right.path));
+    const symbolEvidence = [...symbols.values()]
+        .map((symbol) => ({
+        name: symbol.name,
+        ...(symbol.nodeId !== undefined ? { nodeId: symbol.nodeId } : {}),
+        ...(symbol.file !== undefined ? { file: symbol.file } : {}),
+        surfacedBy: orderedProvenance(symbol.surfacedBy)
+    }))
+        .sort((left, right) => compareCodeUnits(left.file ?? "", right.file ?? "") ||
+        compareCodeUnits(left.name, right.name) ||
+        compareCodeUnits(left.nodeId ?? "", right.nodeId ?? ""));
+    const search = interpreted.get("search");
+    let availability;
+    let availabilityReason;
+    if (commands.length === 0) {
+        availability = "unavailable";
+        availabilityReason = "no-retrieval-commands-executed";
+    }
+    else if (!search) {
+        availability = "unavailable";
+        availabilityReason = "search-command-not-executed";
+    }
+    else if (search.parseState !== "parsed") {
+        availability = "unavailable";
+        availabilityReason = "search-evidence-unavailable";
+    }
+    else if (duplicates.length > 0 || !commands.every(isFullyInterpreted)) {
+        availability = "partial";
+        availabilityReason = "some-executed-commands-were-not-fully-interpreted";
+    }
+    else {
+        availability = "available";
+    }
+    return {
+        schemaVersion: "retrieval-evidence-v1",
+        availability,
+        ...(availabilityReason !== undefined ? { availabilityReason } : {}),
+        files: fileEvidence,
+        symbols: symbolEvidence,
+        commands: commands.map((entry) => duplicates.includes(entry.family) && entry.reason === undefined ? { ...entry, reason: REASON_DUPLICATE } : entry)
+    };
+}
+/**
+ * Thin adapter from the lifecycle's measured commands to the pure builder. Only the four retrieval
+ * families are considered; `index`, `version`, and any other command are ignored. Source stdout is dropped
+ * here on purpose.
+ */
+export function buildRetrievalEvidenceFromCommands(commands, selection) {
+    const inputs = [];
+    for (const command of commands) {
+        if (command.commandId === "source") {
+            inputs.push({ family: "source", ok: command.ok });
+        }
+        else if (command.commandId === "search" || command.commandId === "lookup" || command.commandId === "slice") {
+            inputs.push({ family: command.commandId, ok: command.ok, stdout: command.stdout });
+        }
+    }
+    return buildRetrievalEvidence({ commands: inputs, selection });
+}

package/dist/src/evaluation/retrievalQuality/corpusCompleteness.js

@@ -0,0 +1,122 @@
+import { normalizeRetrievedRepositoryPath } from "./buildRetrievalEvidence.js";
+import { hasFactMapping, interpretFactContextTarget } from "./factContextTargets.js";
+const asRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value) ? value : undefined;
+function normalizedSet(values, normalize) {
+    if (!Array.isArray(values))
+        return undefined;
+    const result = new Set();
+    for (const value of values) {
+        const normalized = normalize(value);
+        if (normalized === null)
+            return undefined;
+        result.add(normalized);
+    }
+    return result;
+}
+const plainString = (value) => (typeof value === "string" && value.length > 0 ? value : null);
+/**
+ * Ground-truth completeness for one retrieval-precision-recall case. Returns bounded error strings that never
+ * echo unsafe values. This rule set applies only to corpora that opt into retrieval-precision-recall; it is not
+ * a global answer-key rule.
+ */
+export function validateRetrievalPrecisionRecallCase(evaluationCase) {
+    const label = `retrieval-precision-recall case ${typeof evaluationCase.id === "string" ? evaluationCase.id : "<unknown>"}`;
+    const answerKey = asRecord(evaluationCase.answerKey);
+    if (!answerKey)
+        return [`${label}: answerKey is required.`];
+    const errors = [];
+    const expectedFiles = normalizedSet(answerKey.expectedFiles, normalizeRetrievedRepositoryPath);
+    const expectedSymbols = normalizedSet(answerKey.expectedSymbols, plainString);
+    if (expectedFiles === undefined || expectedFiles.size === 0)
+        errors.push(`${label}: answerKey.expectedFiles must be a nonempty array of safe repository-relative paths.`);
+    if (expectedSymbols === undefined || expectedSymbols.size === 0)
+        errors.push(`${label}: answerKey.expectedSymbols must be a nonempty array of nonempty strings.`);
+    const topFiles = normalizedSet(evaluationCase.expectedFiles, normalizeRetrievedRepositoryPath);
+    const topSymbols = normalizedSet(evaluationCase.expectedSymbols, plainString);
+    const sameSet = (left, right) => left !== undefined && right !== undefined && left.size === right.size && [...left].every((value) => right.has(value));
+    if (expectedFiles !== undefined && !sameSet(topFiles, expectedFiles))
+        errors.push(`${label}: expectedFiles must agree with answerKey.expectedFiles.`);
+    if (expectedSymbols !== undefined && !sameSet(topSymbols, expectedSymbols))
+        errors.push(`${label}: expectedSymbols must agree with answerKey.expectedSymbols.`);
+    const facts = answerKey.expectedFacts;
+    const factIds = [];
+    if (!Array.isArray(facts) || facts.length === 0) {
+        errors.push(`${label}: answerKey.expectedFacts must be a nonempty array.`);
+    }
+    else {
+        for (const fact of facts) {
+            const id = asRecord(fact)?.id;
+            if (typeof id !== "string" || id.length === 0 || factIds.includes(id)) {
+                errors.push(`${label}: answerKey.expectedFacts must have unique nonempty ids.`);
+                break;
+            }
+            factIds.push(id);
+        }
+    }
+    const targets = answerKey.expectedContextTargets;
+    if (!Array.isArray(targets) || targets.length === 0) {
+        errors.push(`${label}: answerKey.expectedContextTargets is required for fact coverage.`);
+        return errors;
+    }
+    const known = new Set(factIds);
+    const requiredMappedFacts = new Set();
+    targets.forEach((target, index) => {
+        const targetLabel = `${label}: answerKey.expectedContextTargets[${index}]`;
+        const record = asRecord(target);
+        if (!record) {
+            errors.push(`${targetLabel}: target must be an object.`);
+            return;
+        }
+        if (hasFactMapping(target)) {
+            const interpretation = interpretFactContextTarget(target, known);
+            if (!interpretation.ok) {
+                errors.push(`${targetLabel}: ${interpretation.problem}.`);
+                return;
+            }
+            if (interpretation.target.required)
+                interpretation.target.factIds.forEach((id) => requiredMappedFacts.add(id));
+        }
+        // No hidden expectation outside the answer key, mapped or not.
+        const file = normalizeRetrievedRepositoryPath(record.file);
+        if (file === null) {
+            errors.push(`${targetLabel}: file must be a safe repository-relative path.`);
+        }
+        else if (expectedFiles !== undefined && !expectedFiles.has(file)) {
+            errors.push(`${targetLabel}: file must be listed in answerKey.expectedFiles.`);
+        }
+        if (Array.isArray(record.symbols) && expectedSymbols !== undefined) {
+            for (const symbol of record.symbols) {
+                if (typeof symbol !== "string" || !expectedSymbols.has(symbol)) {
+                    errors.push(`${targetLabel}: every symbol must be listed in answerKey.expectedSymbols.`);
+                    break;
+                }
+            }
+        }
+    });
+    for (const id of factIds) {
+        if (!requiredMappedFacts.has(id))
+            errors.push(`${label}: expected fact ${id} has no required fact-mapped context target.`);
+    }
+    return errors;
+}
+/** Validates every case of a production retrieval-precision-recall corpus; case ids must be unique. */
+export function validateRetrievalPrecisionRecallCorpus(cases) {
+    const errors = [];
+    const seen = new Set();
+    for (const evaluationCase of cases) {
+        if (seen.has(evaluationCase.id))
+            errors.push(`retrieval-precision-recall corpus: duplicate case id ${evaluationCase.id}.`);
+        seen.add(evaluationCase.id);
+        errors.push(...validateRetrievalPrecisionRecallCase(evaluationCase));
+    }
+    return errors;
+}
+/**
+ * Per-case issue counts without any identity text. Use this, never the detailed messages, when reporting about an
+ * external private subject: the detailed messages may name expected facts.
+ */
+export function summarizeRetrievalGroundTruthIssuesSafely(cases) {
+    return cases
+        .map((evaluationCase) => ({ caseId: evaluationCase.id, issueCount: validateRetrievalPrecisionRecallCase(evaluationCase).length }))
+        .filter((entry) => entry.issueCount > 0);
+}

package/dist/src/evaluation/retrievalQuality/factContextTargets.js

@@ -0,0 +1,44 @@
+import { normalizeRetrievedRepositoryPath } from "./buildRetrievalEvidence.js";
+/** True when the target participates in fact mapping, i.e. it carries a `factIds` field. */
+export function hasFactMapping(target) {
+    return typeof target === "object" && target !== null && !Array.isArray(target) && target.factIds !== undefined;
+}
+/**
+ * Single owner of the `ExpectedContextTarget.factIds` contract. Pure. Problems are fixed bounded strings and
+ * never echo the offending value. Unsafe file identities are rejected, never repaired.
+ */
+export function interpretFactContextTarget(target, knownFactIds) {
+    if (typeof target !== "object" || target === null || Array.isArray(target)) {
+        return { ok: false, problem: "target must be an object" };
+    }
+    const record = target;
+    const file = normalizeRetrievedRepositoryPath(record.file);
+    if (file === null) {
+        return { ok: false, problem: "file must be a safe repository-relative path" };
+    }
+    const rawFactIds = record.factIds;
+    if (!Array.isArray(rawFactIds) || rawFactIds.length === 0 || rawFactIds.some((id) => typeof id !== "string" || id.length === 0)) {
+        return { ok: false, problem: "factIds must be a nonempty array of nonempty strings" };
+    }
+    const factIds = rawFactIds;
+    if (new Set(factIds).size !== factIds.length) {
+        return { ok: false, problem: "factIds must not contain duplicates" };
+    }
+    if (factIds.some((id) => !knownFactIds.has(id))) {
+        return { ok: false, problem: "factIds must reference existing expectedFacts ids" };
+    }
+    let symbols = [];
+    if (record.symbols !== undefined) {
+        if (!Array.isArray(record.symbols) || record.symbols.some((symbol) => typeof symbol !== "string" || symbol.length === 0)) {
+            return { ok: false, problem: "symbols must be an array of nonempty strings" };
+        }
+        symbols = record.symbols;
+        if (new Set(symbols).size !== symbols.length) {
+            return { ok: false, problem: "symbols must not contain duplicates" };
+        }
+    }
+    if (record.required !== undefined && typeof record.required !== "boolean") {
+        return { ok: false, problem: "required must be a boolean when present" };
+    }
+    return { ok: true, target: { file, symbols: [...symbols], required: record.required !== false, factIds: [...factIds] } };
+}

package/dist/src/evaluation/retrievalQuality/index.js

@@ -0,0 +1,5 @@
+export * from "./types.js";
+export * from "./buildRetrievalEvidence.js";
+export * from "./factContextTargets.js";
+export * from "./metrics.js";
+export * from "./corpusCompleteness.js";