@dailephd/my-dev-kit-lab 0.7.1 → 0.8.0

package/dist/src/commands/runExperimentRunCommand.js

@@ -3,8 +3,12 @@ import path from "node:path";
 import { parseAgentCommandTemplate } from "../agents/index.js";
 import { parseAgentId } from "../agents/agentRegistry.js";
 import { readBenchmarkProjectProfiles, readEvaluationCases } from "../evaluation/index.js";
-import { contextStrategyComparisonPlugin, contextWindowScalingPlugin, createDefaultExperimentPluginRegistry, getWarmIndexCampaignPreset, incrementalChangeStalenessPlugin, parseContextBudgetsCliValue, parseWarmIndexCampaignPresetId, resolveExperimentTarget, runExperiment, warmIndexReusePlugin } from "../experiments/index.js";
+import { summarizeRetrievalGroundTruthIssuesSafely, validateRetrievalPrecisionRecallCorpus } from "../evaluation/retrievalQuality/index.js";
+import { loadLocalRepositorySubject } from "../evaluation/localRepositorySubject/index.js";
+import { contextStrategyComparisonPlugin, contextWindowScalingPlugin, createDefaultExperimentPluginRegistry, getWarmIndexCampaignPreset, incrementalChangeStalenessPlugin, parseContextBudgetsCliValue, parseWarmIndexCampaignPresetId, resolveExperimentTarget, retrievalPrecisionRecallPlugin, runExperiment, warmIndexReusePlugin } from "../experiments/index.js";
 import { prepareSyntheticContextWindowScalingInputs } from "../experiments/plugins/contextWindowScaling/index.js";
+import { projectExternalLocalTarget, projectRunForExternalLocalPersistence, } from "../experiments/plugins/contextWindowScaling/localSubjectPrivacy.js";
+import { assertWorkRootOutsideTarget } from "../experiments/plugins/contextWindowScaling/localSubjectScratch.js";
 import { buildDefaultExperimentOutputRoot } from "../experiments/outputPaths.js";
 import { parsePromptComplexityLevel, parsePromptStrategy } from "../prompts/index.js";
 import { writePluginExperimentReports } from "../report/index.js";
@@ -38,7 +42,23 @@ const DEFAULT_PROJECT_PROFILES_RESOURCE = "benchmarks/contracts/benchmark-projec
 // catalog's package-relative targetRoot, so both resolve from the package root, never the cwd.
 const CONTEXT_WINDOW_SCALING_CASES_RESOURCE = "benchmarks/contracts/context-window-scaling-cases.json";
 // Options the context-window-scaling plugin accepts; every other flag is rejected, not ignored.
-const CONTEXT_WINDOW_SCALING_ALLOWED_FLAGS = ["--experiment", "--out", "--case", "--synthetic-config", "--context-budgets", "--kit-command"];
+const CONTEXT_WINDOW_SCALING_ALLOWED_FLAGS = [
+    "--experiment",
+    "--out",
+    "--target",
+    "--case",
+    "--synthetic-config",
+    "--local-subject-config",
+    "--context-budgets",
+    "--kit-command"
+];
+// retrieval-precision-recall owns one frozen bundled corpus (the warm-index benchmark cases) and its project profiles,
+// both resolved from the package root, never the cwd. It accepts only these options; every other flag is rejected.
+const RETRIEVAL_PRECISION_RECALL_CASES_RESOURCE = "benchmarks/contracts/warm-index-benchmark-cases.json";
+const RETRIEVAL_PRECISION_RECALL_ALLOWED_FLAGS = ["--experiment", "--out", "--case", "--benchmark-project", "--kit-command"];
+// External-local mode (--target with --local-subject-config) is a different subject source: the local-subject config owns
+// the case set, so the bundled corpus filters are not accepted there.
+const RETRIEVAL_PRECISION_RECALL_EXTERNAL_ALLOWED_FLAGS = ["--experiment", "--out", "--target", "--local-subject-config", "--kit-command"];
 export async function runExperimentRunCommandFromArgs(argv, options = {}) {
     try {
         const args = parseRunExperimentArgs(argv);
@@ -53,8 +73,8 @@ export async function runExperimentRunCommandFromArgs(argv, options = {}) {
             experimentId: args.experimentId,
             runId
         });
-        if (args.syntheticConfigPath !== undefined && outputRoot === undefined) {
-            // Synthetic generation needs the concrete root before the runner runs; use the exact default the
+        if ((args.syntheticConfigPath !== undefined || args.localSubjectConfigPath !== undefined) && outputRoot === undefined) {
+            // Synthetic generation and external-local runs need the concrete root before the runner runs; use the exact default the
             // runner itself would compute for this self-targeted invocation.
             outputRoot = buildDefaultExperimentOutputRoot({
                 toolRoot,
@@ -74,22 +94,53 @@ export async function runExperimentRunCommandFromArgs(argv, options = {}) {
             toolRoot,
             runId
         });
+        const localSubject = readLoadedLocalSubject(inputs);
+        // External-local runs persist only a projected run: no target, tool, or output machine-local paths.
+        const persistedRun = localSubject
+            ? projectRunForExternalLocalPersistence(result, projectExternalLocalTarget(localSubject.manifest))
+            : result;
+        if (localSubject && result.status === "failed") {
+            // A failed local run has no execution evidence, so no report is produced and none could be mistaken for a
+            // completed experiment. The safe failure description (codes and kinds only) is the whole outcome.
+            console.log([
+                `Experiment: ${result.pluginId}`,
+                `Run ID: ${result.runId}`,
+                "Status: failed",
+                "Mode: external-local repository subject",
+                `Subject: ${localSubject.subjectId}`,
+                ...result.failures.map((failure) => `Failure: ${failure.message}`)
+            ].join("\n"));
+            return 1;
+        }
         const reports = await writePluginExperimentReports({
-            run: result,
+            run: persistedRun,
             plugin: registry.describe(result.pluginId),
-            outputRoot: String(result.metadata?.outputRoot ?? "")
+            outputRoot: String(result.metadata?.outputRoot ?? ""),
+            redactOutputRoot: localSubject !== undefined
         });
-        const outputLines = [
-            `Experiment: ${result.pluginId}`,
-            `Run ID: ${result.runId}`,
-            `Status: ${result.status}`,
-            `Mode: ${result.target.isSelf ? "self" : "external target"}`,
-            `Tool root: ${result.target.toolRoot}`,
-            `Target root: ${result.target.targetRoot}`,
-            `Output: ${String(result.metadata?.outputRoot ?? "")}`,
-            `Report JSON: ${reports.outputPaths.jsonPath}`,
-            `Report HTML: ${reports.outputPaths.htmlPath}`
-        ];
+        const outputLines = localSubject
+            ? [
+                `Experiment: ${result.pluginId}`,
+                `Run ID: ${result.runId}`,
+                `Status: ${result.status}`,
+                "Mode: external-local repository subject",
+                `Subject: ${localSubject.subjectId}`,
+                `Output: ${String(result.metadata?.outputRoot ?? "")}`,
+                `Report JSON: ${reports.outputPaths.jsonPath}`,
+                `Report HTML: ${reports.outputPaths.htmlPath}`,
+                ...result.failures.map((failure) => `Failure: ${failure.message}`)
+            ]
+            : [
+                `Experiment: ${result.pluginId}`,
+                `Run ID: ${result.runId}`,
+                `Status: ${result.status}`,
+                `Mode: ${result.target.isSelf ? "self" : "external target"}`,
+                `Tool root: ${result.target.toolRoot}`,
+                `Target root: ${result.target.targetRoot}`,
+                `Output: ${String(result.metadata?.outputRoot ?? "")}`,
+                `Report JSON: ${reports.outputPaths.jsonPath}`,
+                `Report HTML: ${reports.outputPaths.htmlPath}`
+            ];
         // Automatic presentation (report already produced above) applies only to warm-index-reuse
         // campaign runs; legacy fake-agent warm-index runs and context-strategy-comparison stay
         // report-only, unchanged.
@@ -161,6 +212,7 @@ export function parseRunExperimentArgs(argv) {
     let campaignPreset;
     let contextBudgets;
     let syntheticConfigPath;
+    let localSubjectConfigPath;
     const seenFlags = [];
     const commandTemplates = {};
     for (let index = 0; index < argv.length; index += 1) {
@@ -188,6 +240,12 @@ export function parseRunExperimentArgs(argv) {
             }
             syntheticConfigPath = readRequiredValue(argv, ++index, "--synthetic-config");
         }
+        else if (arg === "--local-subject-config") {
+            if (localSubjectConfigPath !== undefined) {
+                throw new Error("--local-subject-config may be supplied only once.");
+            }
+            localSubjectConfigPath = readRequiredValue(argv, ++index, "--local-subject-config");
+        }
         else if (arg === "--case") {
             caseIds.push(...splitList(readRequiredValue(argv, ++index, "--case")));
         }
@@ -250,14 +308,44 @@ export function parseRunExperimentArgs(argv) {
     const KIT_COMMAND_PLUGIN_IDS = [
         warmIndexReusePlugin.metadata.id,
         incrementalChangeStalenessPlugin.metadata.id,
-        contextWindowScalingPlugin.metadata.id
+        contextWindowScalingPlugin.metadata.id,
+        retrievalPrecisionRecallPlugin.metadata.id
     ];
+    if (experimentId === retrievalPrecisionRecallPlugin.metadata.id) {
+        // Mode matrix, no precedence rules: neither flag = bundled; both = external-local; exactly one is rejected.
+        if (targetPath !== undefined && localSubjectConfigPath === undefined) {
+            throw new Error(`External ${experimentId} targets require --local-subject-config.`);
+        }
+        if (localSubjectConfigPath !== undefined && targetPath === undefined) {
+            throw new Error(`--local-subject-config requires an external --target for ${experimentId}.`);
+        }
+        const externalMode = targetPath !== undefined;
+        if (externalMode && (seenFlags.includes("--case") || seenFlags.includes("--benchmark-project"))) {
+            throw new Error("--case and --benchmark-project cannot be combined with --local-subject-config; the local subject config owns the case set.");
+        }
+        const allowedFlags = externalMode ? RETRIEVAL_PRECISION_RECALL_EXTERNAL_ALLOWED_FLAGS : RETRIEVAL_PRECISION_RECALL_ALLOWED_FLAGS;
+        const unsupported = [...new Set(seenFlags.filter((flag) => !allowedFlags.includes(flag)))];
+        if (unsupported.length > 0) {
+            throw new Error(`${unsupported.join(", ")} ${unsupported.length === 1 ? "is" : "are"} not supported for --experiment ${experimentId}${externalMode ? " in external-local mode" : ""}; supported options: ${allowedFlags.join(", ")}.`);
+        }
+        if (seenFlags.includes("--case") && caseIds.length === 0) {
+            throw new Error("--case must list at least one case id.");
+        }
+        if (seenFlags.includes("--benchmark-project") && benchmarkProjects.length === 0) {
+            throw new Error("--benchmark-project must list at least one benchmark project id.");
+        }
+    }
     if (contextBudgets !== undefined && experimentId !== contextWindowScalingPlugin.metadata.id) {
         throw new Error(`--context-budgets is only supported for --experiment ${contextWindowScalingPlugin.metadata.id}.`);
     }
     if (syntheticConfigPath !== undefined && experimentId !== contextWindowScalingPlugin.metadata.id) {
         throw new Error(`--synthetic-config is only supported for --experiment ${contextWindowScalingPlugin.metadata.id}.`);
     }
+    if (localSubjectConfigPath !== undefined &&
+        experimentId !== contextWindowScalingPlugin.metadata.id &&
+        experimentId !== retrievalPrecisionRecallPlugin.metadata.id) {
+        throw new Error(`--local-subject-config is only supported for --experiment ${contextWindowScalingPlugin.metadata.id} or ${retrievalPrecisionRecallPlugin.metadata.id}.`);
+    }
     if (experimentId === contextWindowScalingPlugin.metadata.id) {
         const unsupported = [...new Set(seenFlags.filter((flag) => !CONTEXT_WINDOW_SCALING_ALLOWED_FLAGS.includes(flag)))];
         if (unsupported.length > 0) {
@@ -269,6 +357,23 @@ export function parseRunExperimentArgs(argv) {
         if (syntheticConfigPath !== undefined && seenFlags.includes("--case")) {
             throw new Error("--case and --synthetic-config are mutually exclusive; the synthetic config owns the generated case set.");
         }
+        // Subject-mode matrix: bundled (no source flags), synthetic (--synthetic-config), external-local
+        // (--target with --local-subject-config). Every other combination is rejected, never given a precedence.
+        if (localSubjectConfigPath !== undefined && syntheticConfigPath !== undefined) {
+            throw new Error("--synthetic-config and --local-subject-config are mutually exclusive; choose one subject source.");
+        }
+        if (localSubjectConfigPath !== undefined && seenFlags.includes("--case")) {
+            throw new Error("--case cannot be combined with --local-subject-config; the local subject config owns the case set.");
+        }
+        if (localSubjectConfigPath !== undefined && targetPath === undefined) {
+            throw new Error(`--local-subject-config requires an external --target for ${experimentId}.`);
+        }
+        if (syntheticConfigPath !== undefined && targetPath !== undefined) {
+            throw new Error("--synthetic-config cannot be combined with --target; synthetic subjects are generated beneath the experiment output directory.");
+        }
+        if (targetPath !== undefined && localSubjectConfigPath === undefined) {
+            throw new Error(`External ${experimentId} targets require --local-subject-config.`);
+        }
     }
     if (kitCommand !== undefined && !KIT_COMMAND_PLUGIN_IDS.includes(experimentId)) {
         throw new Error(`--kit-command is only supported for --experiment ${KIT_COMMAND_PLUGIN_IDS.join(" or ")}.`);
@@ -292,6 +397,7 @@ export function parseRunExperimentArgs(argv) {
         targetPath,
         outDir,
         syntheticConfigPath,
+        localSubjectConfigPath,
         config: withoutUndefined({
             casesPath,
             projectProfilesPath,
@@ -355,8 +461,64 @@ async function loadPluginInputs(args, toolRoot, context, outputRoot) {
     if (args.experimentId === contextWindowScalingPlugin.metadata.id) {
         return loadContextWindowScalingInputs(args, toolRoot, context, outputRoot);
     }
+    if (args.experimentId === retrievalPrecisionRecallPlugin.metadata.id) {
+        return args.localSubjectConfigPath !== undefined
+            ? loadRetrievalPrecisionRecallLocalSubjectInputs(args, toolRoot, context, outputRoot)
+            : loadRetrievalPrecisionRecallInputs(args, toolRoot, context);
+    }
     return undefined;
 }
+// External-local retrieval-precision-recall: the existing local-subject loader owns the repository, inventory and case
+// adapter. The only additions are the physical output-root check before anything is created and a ground-truth gate
+// whose message carries logical case ids and counts only (an external subject may have private fact ids and names).
+async function loadRetrievalPrecisionRecallLocalSubjectInputs(args, toolRoot, context, outputRoot) {
+    const validation = retrievalPrecisionRecallPlugin.validateConfig(args.config);
+    if (!validation.valid || !validation.config) {
+        throw new Error(`Invalid retrieval precision/recall config: ${validation.errors.join("; ")}`);
+    }
+    if (outputRoot === undefined || args.localSubjectConfigPath === undefined) {
+        throw new Error("Internal error: external-local runs require a resolved experiment output root and config path.");
+    }
+    const target = resolveExperimentTarget(args.targetPath, toolRoot);
+    if (target.isSelf || target.kind !== "external-local") {
+        throw new Error("Local-repository subject mode requires an external-local target; --target must not be the Lab repository.");
+    }
+    // Fail closed before anything is created: the output root (and the private scratch beneath it) must be physically outside the target.
+    try {
+        await assertWorkRootOutsideTarget(outputRoot, target.targetRoot);
+    }
+    catch {
+        throw new Error("Experiment output root must not be inside the external target project.");
+    }
+    const configPath = path.resolve(context.invocationCwd, args.localSubjectConfigPath);
+    const localSubjectConfig = await readJsonConfigFile("--local-subject-config", configPath, args.localSubjectConfigPath);
+    const localSubject = await loadLocalRepositorySubject({ config: localSubjectConfig, repositoryPath: target.targetRoot });
+    const groundTruthIssues = summarizeRetrievalGroundTruthIssuesSafely(localSubject.evaluationCases);
+    if (groundTruthIssues.length > 0) {
+        throw new Error(`Invalid retrieval-precision-recall ground truth for the local subject (${groundTruthIssues
+            .map((issue) => `case ${issue.caseId}: ${issue.issueCount} issue${issue.issueCount === 1 ? "" : "s"}`)
+            .join("; ")}); details withheld.`);
+    }
+    return { cases: localSubject.evaluationCases, localSubject, env: process.env };
+}
+// The frozen bundled corpus is read through the established resource resolver and validation path, then must satisfy
+// the retrieval-precision-recall ground-truth completeness rules. Selection filters are applied by the plugin.
+async function loadRetrievalPrecisionRecallInputs(args, toolRoot, context) {
+    const validation = retrievalPrecisionRecallPlugin.validateConfig(args.config);
+    if (!validation.valid || !validation.config) {
+        throw new Error(`Invalid retrieval precision/recall config: ${validation.errors.join("; ")}`);
+    }
+    const projectProfiles = await readBenchmarkProjectProfiles(resolvePackageResource(context, DEFAULT_PROJECT_PROFILES_RESOURCE), toolRoot);
+    const cases = await readEvaluationCases(resolvePackageResource(context, RETRIEVAL_PRECISION_RECALL_CASES_RESOURCE), toolRoot, {
+        projectProfiles,
+        requireProjectProfileRef: true
+    });
+    const corpusErrors = validateRetrievalPrecisionRecallCorpus(cases);
+    if (corpusErrors.length > 0) {
+        throw new Error(`Invalid retrieval-precision-recall corpus: ${corpusErrors.join(" ")}`);
+    }
+    return { cases };
+}
 // --case is a selection over the frozen bundled catalog (catalog order, not argument order);
 // the rest of the CLI config is the plugin's own ContextWindowScalingConfig.
 function pluginConfigFor(args) {
@@ -370,6 +532,27 @@ async function loadContextWindowScalingInputs(args, toolRoot, context, outputRoo
     if (!validation.valid || !validation.config) {
         throw new Error(`Invalid context window scaling config: ${validation.errors.join("; ")}`);
     }
+    if (args.localSubjectConfigPath !== undefined) {
+        if (outputRoot === undefined) {
+            throw new Error("Internal error: external-local runs require a resolved experiment output root.");
+        }
+        const target = resolveExperimentTarget(args.targetPath, toolRoot);
+        if (target.isSelf || target.kind !== "external-local") {
+            throw new Error("Local-repository subject mode requires an external-local target; --target must not be the Lab repository.");
+        }
+        // Fail closed before anything is created: the output root (and the private scratch beneath it) must be outside the target.
+        try {
+            await assertWorkRootOutsideTarget(outputRoot, target.targetRoot);
+        }
+        catch {
+            throw new Error("Experiment output root must not be inside the external target project.");
+        }
+        const configPath = path.resolve(context.invocationCwd, args.localSubjectConfigPath);
+        const localSubjectConfig = await readJsonConfigFile("--local-subject-config", configPath, args.localSubjectConfigPath);
+        // The selected --target is the physical root handed to the Batch 1 loader; the config carries no path.
+        const localSubject = await loadLocalRepositorySubject({ config: localSubjectConfig, repositoryPath: target.targetRoot });
+        return { cases: localSubject.evaluationCases, localSubject, env: process.env };
+    }
     if (args.syntheticConfigPath !== undefined) {
         if (outputRoot === undefined) {
             throw new Error("Internal error: synthetic generation requires a resolved experiment output root.");
@@ -396,20 +579,27 @@ async function loadContextWindowScalingInputs(args, toolRoot, context, outputRoo
 }
 // The command only reads and parses the caller-supplied file; schema validation belongs to the synthetic planner.
 async function readSyntheticConfigJson(resolvedPath, displayPath) {
+    return readJsonConfigFile("--synthetic-config", resolvedPath, displayPath);
+}
+async function readJsonConfigFile(flag, resolvedPath, displayPath) {
     let text;
     try {
         text = await readFile(resolvedPath, "utf8");
     }
     catch (error) {
-        throw new Error(`Cannot read --synthetic-config ${displayPath}: ${error instanceof Error ? error.message : String(error)}`);
+        throw new Error(`Cannot read ${flag} ${displayPath}: ${error instanceof Error ? error.message : String(error)}`);
     }
     try {
         return JSON.parse(text);
     }
     catch (error) {
-        throw new Error(`--synthetic-config ${displayPath} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`);
+        throw new Error(`${flag} ${displayPath} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`);
     }
 }
+function readLoadedLocalSubject(inputs) {
+    const value = inputs?.localSubject;
+    return value && typeof value === "object" ? value : undefined;
+}
 // Shared by both registered plugins: explicit paths resolve against toolRoot, defaults come from
 // bundled package resources so they do not depend on the invocation cwd.
 async function loadCasesAndProjectProfiles(args, toolRoot, context) {

package/dist/src/core/fileGlobs.js

@@ -98,3 +98,47 @@ export function collectFilesForGlobs(targetRoot, globs) {
         .sort(([a], [b]) => a.localeCompare(b))
         .map(([relativePath, absolutePath]) => ({ relativePath, absolutePath }));
 }
+/**
+ * Pure counterpart of `collectFilesForGlobs` for a caller-supplied, already validated list of repository-relative
+ * file paths. It applies the same glob matching and the same walk exclusions (excluded directory names below the
+ * glob base directory, temp files) and the same ordering, but never touches the filesystem, so the caller decides
+ * which files may be opened at all. Unlike the walking variant it does not require the base directory to exist.
+ */
+export function selectRelativePathsForGlobs(relativePaths, globs) {
+    const selected = new Set();
+    for (const globPattern of globs) {
+        if (!globPattern || typeof globPattern !== "string") {
+            throw new Error("Invalid glob pattern.");
+        }
+        const baseDir = baseDirectoryFromGlob(globPattern);
+        if (baseDir.split("/").includes("..") || path.isAbsolute(baseDir)) {
+            throw new Error(`Resolved path escapes target root: ${globPattern}`);
+        }
+        // Linear scan for the trailing "/" run (a regular expression here is a polynomial-backtracking risk on
+        // glob text that comes from user configuration).
+        let baseEnd = baseDir.length;
+        while (baseEnd > 0 && baseDir.charCodeAt(baseEnd - 1) === 47)
+            baseEnd -= 1;
+        const normalizedBase = baseDir === "." ? "" : baseDir.slice(0, baseEnd);
+        for (const relativePath of relativePaths) {
+            const normalizedPath = relativePath.replace(/\\/g, "/");
+            if (normalizedPath === normalizedBase) {
+                // An exact file base is used as-is by the walking variant (no walk exclusions apply).
+                if (matchesGlob(normalizedPath, globPattern))
+                    selected.add(normalizedPath);
+                continue;
+            }
+            if (normalizedBase !== "" && !normalizedPath.startsWith(`${normalizedBase}/`))
+                continue;
+            const belowBase = normalizedBase === "" ? normalizedPath : normalizedPath.slice(normalizedBase.length + 1);
+            const directorySegments = belowBase.split("/").slice(0, -1);
+            if (directorySegments.some((segment) => excludedDirNames.has(segment)))
+                continue;
+            if (isTempFile(normalizedPath))
+                continue;
+            if (matchesGlob(normalizedPath, globPattern))
+                selected.add(normalizedPath);
+        }
+    }
+    return [...selected].sort((a, b) => a.localeCompare(b));
+}

package/dist/src/evaluation/benchmarkMetadata.js

@@ -2,6 +2,7 @@ import { existsSync } from "node:fs";
 import { readFile } from "node:fs/promises";
 import path from "node:path";
 import { PROJECT_COMPLEXITY_FORMULA, calculateProjectComplexityScore, roundToTwo } from "./projectComplexity.js";
+import { hasFactMapping, interpretFactContextTarget } from "./retrievalQuality/factContextTargets.js";
 import { TASK_LOCALITIES } from "./types.js";
 export const REQUIRED_BENCHMARK_PROJECT_IDS = [
     "todo-ts",
@@ -90,6 +91,18 @@ export function validateAnswerKey(answerKey, label) {
             errors.push(`${label}: answerKey.minimumCorrectFacts is not satisfiable.`);
         }
     }
+    if (Array.isArray(candidate.expectedContextTargets) && Array.isArray(candidate.expectedFacts)) {
+        // Only targets that opt into fact mapping with `factIds` are validated here; legacy targets stay valid as-is.
+        const knownFactIds = new Set(candidate.expectedFacts.flatMap((fact) => (fact && typeof fact === "object" && typeof fact.id === "string" && fact.id.length > 0 ? [fact.id] : [])));
+        candidate.expectedContextTargets.forEach((target, index) => {
+            if (!hasFactMapping(target))
+                return;
+            const interpretation = interpretFactContextTarget(target, knownFactIds);
+            if (!interpretation.ok) {
+                errors.push(`${label}: answerKey.expectedContextTargets[${index}]: ${interpretation.problem}.`);
+            }
+        });
+    }
     if (Array.isArray(candidate.expectedFiles) && candidate.expectedFiles.length === 0) {
         errors.push(`${label}: answerKey.expectedFiles must not be empty.`);
     }