@dailephd/my-dev-kit-lab 0.7.2 → 0.8.0

package/dist/src/evaluation/retrievalQuality/metrics.js

@@ -0,0 +1,233 @@
+import { compareCodeUnits } from "../indexSnapshot.js";
+import { normalizeRetrievedRepositoryPath } from "./buildRetrievalEvidence.js";
+import { hasFactMapping, interpretFactContextTarget } from "./factContextTargets.js";
+const sortedUnique = (values) => [...new Set(values)].sort(compareCodeUnits);
+function ratio(numerator, denominator, notApplicableReason) {
+    if (denominator === 0) {
+        return { availability: "not-applicable", numerator: null, denominator: null, value: null, reason: notApplicableReason };
+    }
+    return { availability: "available", numerator, denominator, value: numerator / denominator, reason: null };
+}
+function unavailableRatio(reason) {
+    return { availability: "unavailable", numerator: null, denominator: null, value: null, reason };
+}
+function normalizeExpectedFiles(values) {
+    if (!Array.isArray(values))
+        return { ok: false, reason: "expected-files-missing" };
+    const normalized = [];
+    for (const value of values) {
+        const path = normalizeRetrievedRepositoryPath(value);
+        if (path === null)
+            return { ok: false, reason: "unsafe-expected-file-identity" };
+        normalized.push(path);
+    }
+    return { ok: true, values: sortedUnique(normalized) };
+}
+function normalizeExpectedSymbols(values) {
+    if (!Array.isArray(values))
+        return { ok: false, reason: "expected-symbols-missing" };
+    if (values.some((value) => typeof value !== "string" || value.length === 0))
+        return { ok: false, reason: "invalid-expected-symbol" };
+    return { ok: true, values: sortedUnique(values) };
+}
+function sameSet(left, right) {
+    return left.length === right.length && left.every((value, index) => value === right[index]);
+}
+/**
+ * The answer key is the semantic source of relevance. A top-level expectation that is also present must agree
+ * with it after normalization; disagreement is never silently reconciled.
+ */
+function resolveExpectation(topLevel, answerKeyValues, hasAnswerKey, normalize, mismatchReason) {
+    if (!hasAnswerKey)
+        return normalize(topLevel);
+    const fromKey = normalize(answerKeyValues);
+    if (!fromKey.ok)
+        return fromKey;
+    if (Array.isArray(topLevel)) {
+        const fromTop = normalize(topLevel);
+        if (!fromTop.ok)
+            return fromTop;
+        if (!sameSet(fromTop.values, fromKey.values))
+            return { ok: false, reason: mismatchReason };
+    }
+    return fromKey;
+}
+const expectationStatus = (resolved) => resolved.ok ? { availability: "available", reason: null } : { availability: "unavailable", reason: resolved.reason };
+/** Symbol name -> normalized files it was retrieved from. Symbols without file identity cannot satisfy a file-specific target. */
+function symbolFilesByName(evidence) {
+    const byName = new Map();
+    for (const symbol of evidence.symbols) {
+        if (symbol.file === undefined)
+            continue;
+        const files = byName.get(symbol.name) ?? new Set();
+        files.add(symbol.file);
+        byName.set(symbol.name, files);
+    }
+    return byName;
+}
+function calculateFactCoverage(answerKey, evidence, retrievedFiles) {
+    if (!answerKey || typeof answerKey !== "object")
+        return { kind: "unavailable", reason: "answer-key-missing" };
+    const facts = answerKey.expectedFacts;
+    if (!Array.isArray(facts))
+        return { kind: "unavailable", reason: "expected-facts-invalid" };
+    const factIds = [];
+    for (const fact of facts) {
+        const id = typeof fact === "object" && fact !== null ? fact.id : undefined;
+        if (typeof id !== "string" || id.length === 0 || factIds.includes(id)) {
+            return { kind: "unavailable", reason: "expected-facts-invalid" };
+        }
+        factIds.push(id);
+    }
+    if (factIds.length === 0)
+        return { kind: "not-applicable", reason: "no-expected-facts" };
+    const targets = answerKey.expectedContextTargets;
+    if (!Array.isArray(targets))
+        return { kind: "unavailable", reason: "fact-context-mapping-unavailable" };
+    const known = new Set(factIds);
+    const targetsByFact = new Map(factIds.map((id) => [id, []]));
+    for (const target of targets) {
+        if (!hasFactMapping(target))
+            continue;
+        const required = target.required !== false;
+        const interpretation = interpretFactContextTarget(target, known);
+        if (!interpretation.ok) {
+            // An invalid optional target cannot gate coverage; an invalid required one makes the mapping untrustworthy.
+            if (required)
+                return { kind: "unavailable", reason: "fact-context-mapping-invalid" };
+            continue;
+        }
+        if (!interpretation.target.required)
+            continue;
+        for (const id of interpretation.target.factIds)
+            targetsByFact.get(id)?.push(interpretation.target);
+    }
+    if (factIds.some((id) => (targetsByFact.get(id) ?? []).length === 0)) {
+        return { kind: "unavailable", reason: "fact-context-mapping-unavailable" };
+    }
+    const symbolFiles = symbolFilesByName(evidence);
+    const isCovered = (target) => retrievedFiles.has(target.file) && target.symbols.every((symbol) => symbolFiles.get(symbol)?.has(target.file) === true);
+    const covered = factIds.filter((id) => (targetsByFact.get(id) ?? []).every(isCovered));
+    const coveredSet = new Set(covered);
+    return {
+        kind: "computed",
+        covered: sortedUnique(covered),
+        uncovered: sortedUnique(factIds.filter((id) => !coveredSet.has(id))),
+        total: factIds.length
+    };
+}
+function evidenceReason(evidence) {
+    if (evidence === undefined)
+        return "retrieval-evidence-missing";
+    if (evidence.availability === "partial")
+        return "retrieval-evidence-partial";
+    if (evidence.availability === "unavailable")
+        return "retrieval-evidence-unavailable";
+    return null;
+}
+/**
+ * Pure per-case retrieval-quality calculation. Formulas are planner-frozen:
+ * file precision = |R_file ∩ E_file| / |R_file|; file recall = |R_file ∩ E_file| / |E_file|;
+ * irrelevant context ratio = |R_file - E_file| / |R_file|; symbol precision/recall use the same set formulas
+ * over exact unique symbol names; fact coverage = covered facts / expected facts, unweighted and derived only
+ * from explicit `factIds` target mappings. No input is mutated and nothing is read from disk or text.
+ * Unavailable or not-applicable results are never converted to zero.
+ */
+export function calculateRetrievalQualityMetrics(input) {
+    const { evaluationCase, retrieval } = input;
+    const evidence = retrieval.retrievalEvidence;
+    const evidenceBlocked = evidenceReason(evidence);
+    const answerKey = evaluationCase.answerKey;
+    const hasAnswerKey = answerKey !== undefined && answerKey !== null && typeof answerKey === "object";
+    const expectedFiles = resolveExpectation(evaluationCase.expectedFiles, hasAnswerKey ? answerKey.expectedFiles : undefined, hasAnswerKey, normalizeExpectedFiles, "expected-files-answer-key-mismatch");
+    const expectedSymbols = resolveExpectation(evaluationCase.expectedSymbols, hasAnswerKey ? answerKey.expectedSymbols : undefined, hasAnswerKey, normalizeExpectedSymbols, "expected-symbols-answer-key-mismatch");
+    const usable = evidenceBlocked === null && evidence !== undefined ? evidence : undefined;
+    const retrievedFiles = usable ? sortedUnique(usable.files.map((file) => file.path)) : undefined;
+    const retrievedSymbols = usable ? sortedUnique(usable.symbols.map((symbol) => symbol.name)) : undefined;
+    // ---- files
+    let file = {
+        relevantRetrievedFiles: null,
+        irrelevantRetrievedFiles: null,
+        missedFiles: null,
+        missedFileCount: null,
+        precision: unavailableRatio(evidenceBlocked ?? (expectedFiles.ok ? "unavailable" : expectedFiles.reason)),
+        recall: unavailableRatio(evidenceBlocked ?? (expectedFiles.ok ? "unavailable" : expectedFiles.reason))
+    };
+    let irrelevantContextRatio = unavailableRatio(evidenceBlocked ?? (expectedFiles.ok ? "unavailable" : expectedFiles.reason));
+    if (retrievedFiles !== undefined && expectedFiles.ok) {
+        const expected = new Set(expectedFiles.values);
+        const retrieved = new Set(retrievedFiles);
+        const relevant = retrievedFiles.filter((path) => expected.has(path));
+        const irrelevant = retrievedFiles.filter((path) => !expected.has(path));
+        const missed = expectedFiles.values.filter((path) => !retrieved.has(path));
+        file = {
+            relevantRetrievedFiles: relevant,
+            irrelevantRetrievedFiles: irrelevant,
+            missedFiles: missed,
+            missedFileCount: missed.length,
+            precision: ratio(relevant.length, retrievedFiles.length, "no-retrieved-files"),
+            recall: ratio(relevant.length, expectedFiles.values.length, "no-expected-files")
+        };
+        irrelevantContextRatio = ratio(irrelevant.length, retrievedFiles.length, "no-retrieved-files");
+    }
+    // ---- symbols (exact unique names)
+    let symbol = {
+        relevantRetrievedSymbols: null,
+        irrelevantRetrievedSymbols: null,
+        missedSymbols: null,
+        missedSymbolCount: null,
+        precision: unavailableRatio(evidenceBlocked ?? (expectedSymbols.ok ? "unavailable" : expectedSymbols.reason)),
+        recall: unavailableRatio(evidenceBlocked ?? (expectedSymbols.ok ? "unavailable" : expectedSymbols.reason))
+    };
+    if (retrievedSymbols !== undefined && expectedSymbols.ok) {
+        const expected = new Set(expectedSymbols.values);
+        const retrieved = new Set(retrievedSymbols);
+        const relevant = retrievedSymbols.filter((name) => expected.has(name));
+        const missed = expectedSymbols.values.filter((name) => !retrieved.has(name));
+        symbol = {
+            relevantRetrievedSymbols: relevant,
+            irrelevantRetrievedSymbols: retrievedSymbols.filter((name) => !expected.has(name)),
+            missedSymbols: missed,
+            missedSymbolCount: missed.length,
+            precision: ratio(relevant.length, retrievedSymbols.length, "no-retrieved-symbols"),
+            recall: ratio(relevant.length, expectedSymbols.values.length, "no-expected-symbols")
+        };
+    }
+    // ---- facts
+    let fact = {
+        coveredFactIds: null,
+        uncoveredFactIds: null,
+        uncoveredFactCount: null,
+        coverage: unavailableRatio(evidenceBlocked ?? "unavailable")
+    };
+    if (usable !== undefined && retrievedFiles !== undefined) {
+        const outcome = calculateFactCoverage(answerKey, usable, new Set(retrievedFiles));
+        if (outcome.kind === "computed") {
+            fact = {
+                coveredFactIds: outcome.covered,
+                uncoveredFactIds: outcome.uncovered,
+                uncoveredFactCount: outcome.uncovered.length,
+                coverage: ratio(outcome.covered.length, outcome.total, "no-expected-facts")
+            };
+        }
+        else {
+            fact = { ...fact, coverage: outcome.kind === "unavailable" ? unavailableRatio(outcome.reason) : { availability: "not-applicable", numerator: null, denominator: null, value: null, reason: outcome.reason } };
+        }
+    }
+    const tokensValid = typeof retrieval.totalEstimatedTokens === "number" && Number.isFinite(retrieval.totalEstimatedTokens) && retrieval.totalEstimatedTokens >= 0;
+    return {
+        schemaVersion: "retrieval-quality-metrics-v1",
+        caseId: evaluationCase.id,
+        evidence: {
+            availability: evidence === undefined ? "missing" : evidence.availability,
+            reason: evidence === undefined ? "retrieval-evidence-missing" : (evidence.availabilityReason ?? null)
+        },
+        expectations: { files: expectationStatus(expectedFiles), symbols: expectationStatus(expectedSymbols) },
+        file,
+        symbol,
+        fact,
+        irrelevantContextRatio,
+        retrievedTokenCount: tokensValid ? retrieval.totalEstimatedTokens : null,
+        tokenCountMethod: tokensValid && typeof retrieval.tokenCountMethod === "string" ? retrieval.tokenCountMethod : null
+    };
+}

package/dist/src/evaluation/retrievalQuality/types.js

@@ -0,0 +1,9 @@
+/**
+ * Normalized retrieval-evidence contract (V1).
+ *
+ * Describes which files and symbols the existing my-dev-kit search -> lookup -> slice -> source
+ * lifecycle actually surfaced. It is observation only: it carries no relevance judgement and no
+ * score. It deliberately contains no source text, stdout/stderr bodies, or machine-local paths.
+ */
+/** Retrieval command families in fixed lifecycle order. */
+export const RETRIEVAL_COMMAND_FAMILIES = ["search", "lookup", "slice", "source"];

package/dist/src/evaluation/runMyDevKitRetrieval.js

@@ -2,6 +2,7 @@ import path from "node:path";
 import { countEstimatedTokens, countTextChars, tokenCountMethod } from "../core/countTokens.js";
 import { runMeasuredCommand } from "../core/runMeasuredCommand.js";
 import { interpretToolVersionOutput } from "./indexSnapshot.js";
+import { buildRetrievalEvidenceFromCommands } from "./retrievalQuality/buildRetrievalEvidence.js";
 const VERSION_PROBE_TIMEOUT_MS = 30_000;
 function parseJsonIfPossible(text) {
     try {
@@ -52,6 +53,7 @@ function skippedRetrieval(caseId, warnings, commands, durationMs) {
         contextText: "",
         filesRead: [],
         commands,
+        retrievalEvidence: buildRetrievalEvidenceFromCommands(commands),
         durationMs
     };
 }
@@ -337,6 +339,7 @@ export async function runMyDevKitRetrievalFromIndex(options) {
         selectedNodeId,
         selectedFile,
         selectedSymbol,
+        retrievalEvidence: buildRetrievalEvidenceFromCommands(commands, { nodeId: selectedNodeId, file: selectedFile }),
         durationMs: Date.now() - started
     };
 }

package/dist/src/experiments/defaultRegistry.js

@@ -2,6 +2,7 @@ import { ExperimentPluginRegistry } from "./registry.js";
 import { contextStrategyComparisonPlugin } from "./plugins/contextStrategyComparison/index.js";
 import { contextWindowScalingPlugin } from "./plugins/contextWindowScaling/index.js";
 import { incrementalChangeStalenessPlugin } from "./plugins/incrementalChangeStaleness/index.js";
+import { retrievalPrecisionRecallPlugin } from "./plugins/retrievalPrecisionRecall/index.js";
 import { warmIndexReusePlugin } from "./plugins/warmIndexReuse/index.js";
 export function createDefaultExperimentPluginRegistry() {
     const registry = new ExperimentPluginRegistry();
@@ -9,5 +10,6 @@ export function createDefaultExperimentPluginRegistry() {
     registry.register(warmIndexReusePlugin);
     registry.register(incrementalChangeStalenessPlugin);
     registry.register(contextWindowScalingPlugin);
+    registry.register(retrievalPrecisionRecallPlugin);
     return registry;
 }

package/dist/src/experiments/plugins/index.js

@@ -2,3 +2,4 @@ export * from "./contextStrategyComparison/index.js";
 export * from "./contextWindowScaling/index.js";
 export * from "./incrementalChangeStaleness/index.js";
 export * from "./warmIndexReuse/index.js";
+export * from "./retrievalPrecisionRecall/index.js";

package/dist/src/experiments/plugins/retrievalPrecisionRecall/config.js

@@ -0,0 +1,39 @@
+import { invalidExperimentConfig, isPlainObject, mergeConfig, validExperimentConfig } from "../../config.js";
+export const defaultRetrievalPrecisionRecallConfig = {
+    outDir: "lab-output/retrieval-precision-recall",
+    kitCommand: "npx @dailephd/my-dev-kit@latest"
+};
+const SUPPORTED_FIELDS = ["outDir", "kitCommand", "caseIds", "benchmarkProjects"];
+export const retrievalPrecisionRecallConfigDefinition = {
+    fields: [
+        { name: "outDir", type: "string", required: true, description: "Experiment output root." },
+        { name: "kitCommand", type: "string", description: "my-dev-kit command used for index and retrieval." },
+        { name: "caseIds", type: "array", description: "Bundled evaluation case IDs to include (corpus order is preserved)." },
+        { name: "benchmarkProjects", type: "array", description: "Bundled benchmark project IDs to include (corpus order is preserved)." }
+    ]
+};
+export function validateRetrievalPrecisionRecallConfig(config) {
+    if (config !== undefined && !isPlainObject(config)) {
+        return invalidExperimentConfig(["retrieval precision/recall config must be an object."]);
+    }
+    const errors = [];
+    const unsupported = Object.keys(config ?? {}).filter((key) => !SUPPORTED_FIELDS.includes(key));
+    if (unsupported.length > 0) {
+        errors.push(`Unsupported retrieval-precision-recall config field(s): ${unsupported.sort().join(", ")}.`);
+    }
+    const normalized = mergeConfig(defaultRetrievalPrecisionRecallConfig, config);
+    for (const field of ["outDir", "kitCommand"]) {
+        const value = normalized[field];
+        if (typeof value !== "string" || !value.trim())
+            errors.push(`${field} must be a non-empty string.`);
+    }
+    for (const field of ["caseIds", "benchmarkProjects"]) {
+        const value = normalized[field];
+        if (value === undefined)
+            continue;
+        if (!Array.isArray(value) || value.length === 0 || !value.every((item) => typeof item === "string" && item.trim())) {
+            errors.push(`${field} must be a non-empty array of non-empty strings when provided.`);
+        }
+    }
+    return errors.length > 0 ? invalidExperimentConfig(errors) : validExperimentConfig(normalized);
+}

package/dist/src/experiments/plugins/retrievalPrecisionRecall/execution.js

@@ -0,0 +1,152 @@
+import path from "node:path";
+import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { buildMyDevKitIndex, runMyDevKitRetrievalFromIndex } from "../../../evaluation/runMyDevKitRetrieval.js";
+import { calculateRetrievalQualityMetrics, validateRetrievalPrecisionRecallCase } from "../../../evaluation/retrievalQuality/index.js";
+import { groupWarmIndexCases, taskOutputSegment } from "../warmIndexReuse/selection.js";
+const MAX_MESSAGE_LENGTH = 300;
+const MAX_WARNINGS = 20;
+/** Bounds a message and replaces machine-local paths so persisted errors never carry them. */
+export function boundedRetrievalSafeMessage(message, knownRoots = []) {
+    let text = message;
+    for (const root of knownRoots) {
+        if (!root)
+            continue;
+        text = text.split(root).join("<path>").split(root.replace(/\\/g, "/")).join("<path>");
+    }
+    text = text
+        .replace(/[A-Za-z]:[\\/][^\s"'`]*/g, "<path>")
+        .replace(/(?:^|(?<=[\s"'`(]))\/(?:Users|home|tmp|var|private|opt|mnt)\/[^\s"'`]*/g, "<path>")
+        .replace(/[\u0000-\u001f\u007f]/g, " ");
+    return text.length > MAX_MESSAGE_LENGTH ? `${text.slice(0, MAX_MESSAGE_LENGTH)}...` : text;
+}
+const errorText = (error) => (error instanceof Error ? error.message : String(error));
+function failedCase(evaluationCase, code, message) {
+    return {
+        caseId: evaluationCase.id,
+        caseName: evaluationCase.title,
+        benchmarkProject: evaluationCase.benchmarkProject,
+        taskLocality: evaluationCase.taskLocality ?? null,
+        status: "failed",
+        retrieval: null,
+        quality: null,
+        errors: [{ code, message }]
+    };
+}
+export function summarizeRetrieval(result) {
+    const evidence = result.retrievalEvidence;
+    return {
+        skipped: result.skipped,
+        durationMs: result.durationMs,
+        totalEstimatedTokens: result.totalEstimatedTokens,
+        tokenCountMethod: result.tokenCountMethod,
+        warnings: result.warnings.slice(0, MAX_WARNINGS).map((warning) => boundedRetrievalSafeMessage(warning)),
+        evidenceAvailability: evidence?.availability ?? "missing",
+        evidenceAvailabilityReason: evidence?.availabilityReason ?? (evidence === undefined ? "retrieval-evidence-missing" : null),
+        retrievedFileCount: evidence?.files.length ?? null,
+        retrievedSymbolCount: evidence?.symbols.length ?? null,
+        commands: evidence ? evidence.commands.map((command) => ({ ...command })) : []
+    };
+}
+/**
+ * One full index per benchmark project, then one retrieval per case from that project's prepared index, using the
+ * existing lifecycle owners. A failed project fails only its own cases. No agent runs. Only bounded, context-free
+ * evidence is returned: retrieval results (and their contextText) never leave this function.
+ */
+export async function executeRetrievalPrecisionRecall(options) {
+    const buildIndex = options.dependencies?.buildIndex ?? buildMyDevKitIndex;
+    const retrieveFromIndex = options.dependencies?.retrieveFromIndex ?? runMyDevKitRetrievalFromIndex;
+    const results = new Map();
+    const knownRoots = [options.outputRoot, ...options.cases.map((evaluationCase) => evaluationCase.absoluteTargetRoot)];
+    const validCases = [];
+    for (const evaluationCase of options.cases) {
+        const groundTruthErrors = validateRetrievalPrecisionRecallCase(evaluationCase);
+        if (groundTruthErrors.length > 0) {
+            results.set(evaluationCase.id, failedCase(evaluationCase, "ground-truth-invalid", boundedRetrievalSafeMessage(groundTruthErrors.join(" "), knownRoots)));
+        }
+        else {
+            validCases.push(evaluationCase);
+        }
+    }
+    for (const group of groupWarmIndexCases(validCases)) {
+        if (group.structuralErrors.length > 0) {
+            for (const evaluationCase of group.cases) {
+                results.set(evaluationCase.id, failedCase(evaluationCase, "project-group-inconsistent", "Cases of one benchmark project disagree on target root or source roots."));
+            }
+            continue;
+        }
+        const indexDir = resolveWithinRoot(options.outputRoot, path.join("indexes", group.projectSegment));
+        const commandsDir = resolveWithinRoot(options.outputRoot, path.join("commands", group.projectSegment));
+        let indexFailure;
+        try {
+            // requireKit: false keeps a failed index as measurable evidence instead of throwing.
+            const build = await buildIndex({
+                target: { absoluteTargetRoot: group.targetRoot, sourceRoots: [...group.sourceRoots] },
+                kitCommand: options.kitCommand,
+                indexDir,
+                commandsDir: path.join(commandsDir, "index"),
+                requireKit: false
+            });
+            if (!build.ok)
+                indexFailure = build.warnings.join(" ") || "my-dev-kit index setup failed.";
+        }
+        catch (error) {
+            indexFailure = errorText(error);
+        }
+        if (indexFailure !== undefined) {
+            const message = boundedRetrievalSafeMessage(indexFailure, knownRoots);
+            for (const evaluationCase of group.cases) {
+                results.set(evaluationCase.id, failedCase(evaluationCase, "project-index-failed", message));
+            }
+            continue;
+        }
+        for (const evaluationCase of group.cases) {
+            results.set(evaluationCase.id, await executeCase({
+                evaluationCase,
+                kitCommand: options.kitCommand,
+                indexDir,
+                commandsDir: path.join(commandsDir, taskOutputSegment(evaluationCase.id)),
+                retrieveFromIndex,
+                knownRoots
+            }));
+        }
+    }
+    // Selection order is the corpus order after filtering, regardless of how projects were grouped.
+    return options.cases.map((evaluationCase) => results.get(evaluationCase.id));
+}
+async function executeCase(args) {
+    const { evaluationCase } = args;
+    let retrieval;
+    try {
+        retrieval = await args.retrieveFromIndex({
+            evaluationCase,
+            kitCommand: args.kitCommand,
+            indexDir: args.indexDir,
+            commandsDir: args.commandsDir,
+            requireKit: false
+        });
+    }
+    catch (error) {
+        return failedCase(evaluationCase, "retrieval-failed", boundedRetrievalSafeMessage(errorText(error), args.knownRoots));
+    }
+    return buildCaseEvidenceFromRetrieval(evaluationCase, retrieval);
+}
+/**
+ * The single place a returned retrieval becomes bounded case evidence: Batch 2 metrics are calculated here and the
+ * case is COMPLETED only when the retrieval evidence is available, otherwise PARTIAL. Shared by the bundled and the
+ * external-local execution paths so neither duplicates the status rule or the metric call.
+ */
+export function buildCaseEvidenceFromRetrieval(evaluationCase, retrieval) {
+    const quality = calculateRetrievalQualityMetrics({ evaluationCase, retrieval });
+    const status = retrieval.retrievalEvidence?.availability === "available" ? "completed" : "partial";
+    const errors = [];
+    return {
+        caseId: evaluationCase.id,
+        caseName: evaluationCase.title,
+        benchmarkProject: evaluationCase.benchmarkProject,
+        taskLocality: evaluationCase.taskLocality ?? null,
+        status,
+        retrieval: summarizeRetrieval(retrieval),
+        quality,
+        errors
+    };
+}

package/dist/src/experiments/plugins/retrievalPrecisionRecall/executionArtifact.js

@@ -0,0 +1,15 @@
+export const RETRIEVAL_PRECISION_RECALL_EXECUTION_ARTIFACT_FILE = "retrieval-precision-recall-execution.json";
+export const RETRIEVAL_PRECISION_RECALL_EXECUTION_SCHEMA_VERSION = "my-dev-kit-lab-retrieval-precision-recall-execution-v1";
+export function buildRetrievalPrecisionRecallExecutionArtifact(args) {
+    return {
+        schemaVersion: RETRIEVAL_PRECISION_RECALL_EXECUTION_SCHEMA_VERSION,
+        runId: args.runId,
+        pluginId: args.pluginId,
+        pluginSchemaVersion: args.pluginSchemaVersion,
+        startedAt: args.startedAt,
+        completedAt: args.completedAt,
+        tokenCountMethod: args.cases.find((entry) => entry.retrieval !== null)?.retrieval?.tokenCountMethod ?? null,
+        cases: args.cases.map((entry) => structuredClone(entry)),
+        aggregate: structuredClone(args.aggregate)
+    };
+}

package/dist/src/experiments/plugins/retrievalPrecisionRecall/index.js

@@ -0,0 +1,9 @@
+export * from "./config.js";
+export * from "./execution.js";
+export * from "./localSubjectExecution.js";
+export * from "./localSubjectPrivacy.js";
+export * from "./executionArtifact.js";
+export * from "./metadata.js";
+export * from "./metrics.js";
+export * from "./plugin.js";
+export * from "./types.js";

package/dist/src/experiments/plugins/retrievalPrecisionRecall/localSubjectExecution.js

@@ -0,0 +1,139 @@
+import path from "node:path";
+import { relativeWithinRoot } from "../../../core/pathSafety.js";
+import { normalizeRetrievedRepositoryPath, validateRetrievalPrecisionRecallCase } from "../../../evaluation/retrievalQuality/index.js";
+import { buildMyDevKitIndex, runMyDevKitRetrievalFromIndex } from "../../../evaluation/runMyDevKitRetrieval.js";
+import { captureTargetSnapshot, compareTargetSnapshots } from "../../../evaluation/targetImmutability/index.js";
+import { LocalSubjectExecutionError } from "../contextWindowScaling/localSubjectErrors.js";
+import { deriveGuidedIndexExclusions } from "../contextWindowScaling/localSubjectExclusions.js";
+import { assertWorkRootOutsideTarget, createPrivateScratch, defaultPrivateScratchIo, removePrivateScratch } from "../contextWindowScaling/localSubjectScratch.js";
+import { buildCaseEvidenceFromRetrieval } from "./execution.js";
+function toRepositoryRelative(repositoryRoot, file) {
+    if (path.isAbsolute(file)) {
+        try {
+            return relativeWithinRoot(repositoryRoot, file);
+        }
+        catch {
+            return file.replace(/\\/g, "/");
+        }
+    }
+    return normalizeRetrievedRepositoryPath(file) ?? file.replace(/\\/g, "/");
+}
+/**
+ * Counts distinct retrieved identities outside the eligible subject universe. It inspects every file identity the
+ * retrieval exposes: evidence files, evidence symbol files, the selected file, and the files read. Only the count
+ * leaves this function; an offending identity is never named.
+ */
+export function countRetrievedIdentitiesOutsideEligibleUniverse(retrieval, eligibleFiles, repositoryRoot) {
+    const identities = new Set();
+    for (const file of retrieval.retrievalEvidence?.files ?? [])
+        identities.add(toRepositoryRelative(repositoryRoot, file.path));
+    for (const symbol of retrieval.retrievalEvidence?.symbols ?? []) {
+        if (symbol.file !== undefined)
+            identities.add(toRepositoryRelative(repositoryRoot, symbol.file));
+    }
+    if (retrieval.selectedFile)
+        identities.add(toRepositoryRelative(repositoryRoot, retrieval.selectedFile));
+    for (const file of retrieval.filesRead)
+        identities.add(toRepositoryRelative(repositoryRoot, file));
+    return [...identities].filter((identity) => !eligibleFiles.has(identity)).length;
+}
+/**
+ * Internal programmatic seam: runs a loaded local subject through the existing index and retrieval owners, one
+ * private index per configured case (each case keeps its own exact source roots), with a private scratch outside the
+ * target and pre/post target-immutability enforcement. Order: validate work root, derive exclusions, validate
+ * ground truth, capture before, create scratch, execute cases, capture after, compare, remove scratch, then decide.
+ * Success is returned only when every safety gate passed. A mutated target is never reverted.
+ *
+ * Every issue message is safe by construction: fixed text plus logical case ids and counts, never a file name,
+ * symbol, fact id, path, or caught error text.
+ */
+export async function executeLocalRepositorySubjectRetrievalPrecisionRecall(args) {
+    const { subject } = args;
+    const buildIndex = args.dependencies?.buildIndex ?? buildMyDevKitIndex;
+    const retrieveFromIndex = args.dependencies?.retrieveFromIndex ?? runMyDevKitRetrievalFromIndex;
+    await assertWorkRootOutsideTarget(args.workRoot, subject.repositoryRoot);
+    const guidedExclusions = deriveGuidedIndexExclusions({
+        eligibleFiles: subject.eligibleFiles,
+        excludedFiles: [...subject.runtimeSafetyExclusions.gitIgnoredFiles, ...subject.runtimeSafetyExclusions.oversizedFiles]
+    });
+    const invalidCaseIds = subject.evaluationCases.filter((evaluationCase) => validateRetrievalPrecisionRecallCase(evaluationCase).length > 0).map((evaluationCase) => evaluationCase.id);
+    if (invalidCaseIds.length > 0) {
+        throw new LocalSubjectExecutionError([
+            { code: "GROUND_TRUTH_INVALID", message: `retrieval ground truth is incomplete for case(s): ${invalidCaseIds.join(", ")}.` }
+        ]);
+    }
+    const snapshotConfig = { targetRootPath: subject.repositoryRoot, relativeFilePaths: subject.eligibleFiles };
+    const snapshotOptions = { externalLocalSafe: { maxHashedFileBytes: subject.manifest.safetyPolicy.maxFileBytes } };
+    const before = await captureTargetSnapshot(snapshotConfig, snapshotOptions);
+    if (!before.ok) {
+        throw new LocalSubjectExecutionError([{ code: "BEFORE_SNAPSHOT_FAILED", message: `target snapshot before execution failed (${before.code}).` }]);
+    }
+    const scratch = await createPrivateScratch(args.workRoot, subject.repositoryRoot);
+    const issues = [];
+    const caseEvidence = [];
+    const eligible = new Set(subject.eligibleFiles);
+    for (const [position, evaluationCase] of subject.evaluationCases.entries()) {
+        // Short, fixed segments: private case or repository names never become directory names.
+        const indexDir = path.join(scratch.path, `i${position + 1}`);
+        const commandsDir = path.join(scratch.path, `c${position + 1}`);
+        try {
+            // requireKit: false keeps a failed index as a result we can classify instead of an arbitrary thrown message.
+            const build = await buildIndex({
+                target: { absoluteTargetRoot: evaluationCase.absoluteTargetRoot, sourceRoots: [...evaluationCase.sourceRoots] },
+                kitCommand: args.kitCommand,
+                indexDir,
+                commandsDir: path.join(commandsDir, "index"),
+                requireKit: false,
+                excludePaths: guidedExclusions
+            });
+            if (!build.ok) {
+                issues.push({ code: "EXECUTION_FAILED", message: `case ${evaluationCase.id}: the my-dev-kit index could not be prepared.` });
+                break;
+            }
+            const retrieval = await retrieveFromIndex({
+                evaluationCase,
+                kitCommand: args.kitCommand,
+                indexDir,
+                commandsDir: path.join(commandsDir, "retrieval"),
+                requireKit: false
+            });
+            const outside = countRetrievedIdentitiesOutsideEligibleUniverse(retrieval, eligible, subject.repositoryRoot);
+            if (outside > 0) {
+                issues.push({
+                    code: "RETRIEVAL_OUTSIDE_ELIGIBLE_UNIVERSE",
+                    message: `case ${evaluationCase.id}: retrieval exposed ${outside} file identit${outside === 1 ? "y" : "ies"} outside the eligible subject universe.`
+                });
+                break;
+            }
+            caseEvidence.push(buildCaseEvidenceFromRetrieval(evaluationCase, retrieval));
+        }
+        catch {
+            issues.push({ code: "EXECUTION_FAILED", message: `case ${evaluationCase.id}: execution threw before completing (details withheld).` });
+            break;
+        }
+    }
+    let comparison = null;
+    const after = await captureTargetSnapshot(snapshotConfig, snapshotOptions);
+    if (!after.ok) {
+        issues.push({ code: "AFTER_SNAPSHOT_FAILED", message: `target snapshot after execution failed (${after.code}).` });
+    }
+    else {
+        comparison = compareTargetSnapshots(before.snapshot, after.snapshot);
+        if (comparison.status === "mutated") {
+            issues.push({ code: "TARGET_MUTATED", message: "the target changed during execution; it was not restored." });
+        }
+    }
+    const cleanupFailure = await removePrivateScratch(scratch, args.scratchIo ?? defaultPrivateScratchIo);
+    if (cleanupFailure) {
+        issues.push({ code: "SCRATCH_CLEANUP_FAILED", message: "the private scratch could not be removed." });
+    }
+    if (issues.length > 0 || !comparison) {
+        throw new LocalSubjectExecutionError(issues.length > 0 ? issues : [{ code: "EXECUTION_FAILED", message: "execution did not complete." }], {
+            immutability: comparison
+        });
+    }
+    return {
+        caseEvidence,
+        immutability: { status: "unchanged", preExistingGitStatusEntryCount: comparison.preExistingGitStatusEntryCount, newMutationCount: 0 }
+    };
+}