@dailephd/my-dev-kit-lab 0.7.2 → 0.8.0

package/dist/src/experiments/plugins/retrievalPrecisionRecall/localSubjectPrivacy.js

@@ -0,0 +1,119 @@
+import { redactedFileList } from "../contextWindowScaling/localSubjectPrivacy.js";
+/** Single fixed placeholder for a withheld case title. */
+export const REDACTED_CASE_TITLE = "<redacted case title>";
+/** The one V1 marker stamped on every externally projected case, so placeholders are never mistaken for real values. */
+export const RETRIEVAL_IDENTITY_REDACTION = {
+    fileIdentities: "redacted",
+    symbolIdentities: "redacted",
+    factIdentities: "redacted",
+    warningText: "redacted",
+    caseTitle: "redacted"
+};
+/** One opaque placeholder per identity: counts stay exact; null stays null; an empty list stays empty. */
+function placeholders(kind, count) {
+    return Array.from({ length: count }, (_, index) => `<redacted ${kind} ${index + 1}>`);
+}
+const filePlaceholders = (list) => (list === null ? null : redactedFileList(list.length));
+const symbolPlaceholders = (list) => (list === null ? null : placeholders("symbol", list.length));
+const factPlaceholders = (list) => (list === null ? null : placeholders("fact", list.length));
+/** Plugin-owned text per error code; a caught Error.message is never copied into durable external evidence. */
+const FIXED_ERROR_TEXT = {
+    "ground-truth-invalid": "The case ground truth was invalid.",
+    "project-group-inconsistent": "The case source configuration was inconsistent.",
+    "project-index-failed": "The my-dev-kit index could not be prepared.",
+    "retrieval-failed": "The retrieval did not complete."
+};
+/** Only identity lists change; every number, availability and fixed-vocabulary reason is carried through untouched. */
+function projectQuality(quality) {
+    return {
+        ...quality,
+        file: {
+            ...quality.file,
+            relevantRetrievedFiles: filePlaceholders(quality.file.relevantRetrievedFiles),
+            irrelevantRetrievedFiles: filePlaceholders(quality.file.irrelevantRetrievedFiles),
+            missedFiles: filePlaceholders(quality.file.missedFiles)
+        },
+        symbol: {
+            ...quality.symbol,
+            relevantRetrievedSymbols: symbolPlaceholders(quality.symbol.relevantRetrievedSymbols),
+            irrelevantRetrievedSymbols: symbolPlaceholders(quality.symbol.irrelevantRetrievedSymbols),
+            missedSymbols: symbolPlaceholders(quality.symbol.missedSymbols)
+        },
+        fact: {
+            ...quality.fact,
+            coveredFactIds: factPlaceholders(quality.fact.coveredFactIds),
+            uncoveredFactIds: factPlaceholders(quality.fact.uncoveredFactIds)
+        }
+    };
+}
+/**
+ * Projects case evidence for durable persistence of an external-local run. Redacts file, symbol and fact identity
+ * lists, the case title, and all retrieval warning prose; keeps logical ids, locality, statuses, numbers and
+ * fixed-vocabulary evidence. It performs no metric calculation.
+ */
+export function projectRetrievalEvidenceForExternalLocalPersistence(evidence) {
+    return evidence.map((entry) => {
+        const cloned = structuredClone(entry);
+        return {
+            ...cloned,
+            caseName: REDACTED_CASE_TITLE,
+            retrieval: cloned.retrieval === null
+                ? null
+                : { ...cloned.retrieval, warnings: placeholders("warning", cloned.retrieval.warnings.length) },
+            quality: cloned.quality === null ? null : projectQuality(cloned.quality),
+            errors: cloned.errors.map((error) => ({ code: error.code, message: FIXED_ERROR_TEXT[error.code] })),
+            identityRedaction: { ...RETRIEVAL_IDENTITY_REDACTION }
+        };
+    });
+}
+/**
+ * Defense in depth: fails closed if a known private file path, case title, or warning text survived projection.
+ * It deliberately does not substring-search short symbol or fact identifiers, which would collide with ordinary words.
+ */
+export function assertExternalLocalProjectionIsPrivate(projected, privateValues) {
+    const serialized = JSON.stringify(projected);
+    const candidates = [...privateValues.filePaths, ...privateValues.caseTitles, ...privateValues.warnings].filter((value) => value.length > 0);
+    const leaked = candidates.some((value) => serialized.includes(value) || serialized.includes(JSON.stringify(value).slice(1, -1)));
+    if (leaked) {
+        throw new Error("External-local privacy projection check failed; no durable output was written.");
+    }
+}
+const GENERIC_ISSUE_TEXT = {
+    WORK_ROOT_INSIDE_TARGET: "the output directory must be outside the local subject repository",
+    BEFORE_SNAPSHOT_FAILED: "the target snapshot before execution could not be captured",
+    AFTER_SNAPSHOT_FAILED: "the target snapshot after execution could not be captured",
+    SCRATCH_CLEANUP_FAILED: "the private scratch could not be removed",
+    GUIDED_EXCLUSION_LIMIT: "the guided-index exclusions exceeded the safe limit",
+    GUIDED_EXCLUSION_UNREPRESENTABLE: "the guided-index exclusions could not be expressed exactly"
+};
+/**
+ * Safe, persistable description of an external-local failure: issue codes, logical case ids, counts and mutation
+ * kinds only. The issue messages built by the execution seam are safe by construction and are included for the
+ * three case-scoped codes; nothing else (no file name, symbol, fact id, path, or caught error text) can appear.
+ */
+export function describeRetrievalLocalSubjectFailureForPersistence(error) {
+    const parts = [];
+    const seen = new Set();
+    for (const issue of error.issues) {
+        if (issue.code === "EXECUTION_FAILED" || issue.code === "GROUND_TRUTH_INVALID" || issue.code === "RETRIEVAL_OUTSIDE_ELIGIBLE_UNIVERSE") {
+            if (!seen.has(issue.message)) {
+                seen.add(issue.message);
+                parts.push(issue.message.replace(/\.$/, ""));
+            }
+            continue;
+        }
+        if (seen.has(issue.code))
+            continue;
+        seen.add(issue.code);
+        if (issue.code === "TARGET_MUTATED") {
+            const kinds = [...new Set((error.immutability?.mutations ?? []).map((mutation) => mutation.kind))].sort();
+            parts.push(`the target changed during execution${kinds.length > 0 ? ` (${kinds.join(", ")})` : ""}; it was not restored`);
+        }
+        else {
+            parts.push(GENERIC_ISSUE_TEXT[issue.code] ?? `issue ${issue.code}`);
+        }
+    }
+    return `Local subject execution failed (${error.code}): ${parts.join("; ")}.`;
+}
+/** Fixed message for any non-LocalSubjectExecutionError thrown inside the external path: no caught text is exposed. */
+export const UNEXPECTED_EXTERNAL_FAILURE_MESSAGE = "Local subject execution failed (EXECUTION_FAILED): execution failed before completing (details withheld).";

package/dist/src/experiments/plugins/retrievalPrecisionRecall/metadata.js

@@ -0,0 +1,21 @@
+export const RETRIEVAL_PRECISION_RECALL_PLUGIN_ID = "retrieval-precision-recall";
+export const RETRIEVAL_PRECISION_RECALL_VARIANT_ID = "my-dev-kit-retrieval";
+/** The frozen bundled corpus; the public plugin offers no way to substitute another. */
+export const RETRIEVAL_PRECISION_RECALL_CASES_RESOURCE = "benchmarks/contracts/warm-index-benchmark-cases.json";
+export const RETRIEVAL_PRECISION_RECALL_PROJECT_PROFILES_RESOURCE = "benchmarks/contracts/benchmark-project-profiles.json";
+export const retrievalPrecisionRecallMetadata = {
+    id: RETRIEVAL_PRECISION_RECALL_PLUGIN_ID,
+    name: "Retrieval Precision/Recall",
+    description: "Measure deterministic file, symbol, fact and irrelevant-context retrieval quality for the existing my-dev-kit retrieval lifecycle without agents.",
+    schemaVersion: "1.0.0",
+    status: "experimental",
+    supportedTargets: ["self", "external-local"],
+    supportedOutputs: ["json", "html", "text", "artifact"]
+};
+export const RETRIEVAL_PRECISION_RECALL_VARIANTS = [
+    {
+        id: RETRIEVAL_PRECISION_RECALL_VARIANT_ID,
+        name: "my-dev-kit retrieval",
+        description: "search, then lookup, slice and source for the top candidate, from one index prepared per benchmark project."
+    }
+];

package/dist/src/experiments/plugins/retrievalPrecisionRecall/metrics.js

@@ -0,0 +1,158 @@
+import { RETRIEVAL_PRECISION_RECALL_RATIO_KEYS } from "./types.js";
+const UNAVAILABLE_RATIO = {
+    availability: "unavailable",
+    numerator: null,
+    denominator: null,
+    value: null,
+    reason: "no-retrieval-measurement"
+};
+/** Per-case ratio selector. A failed case has no quality result and is counted as unavailable, never as zero. */
+export function retrievalRatioOf(quality, key) {
+    if (quality === null)
+        return UNAVAILABLE_RATIO;
+    switch (key) {
+        case "filePrecision":
+            return quality.file.precision;
+        case "fileRecall":
+            return quality.file.recall;
+        case "symbolPrecision":
+            return quality.symbol.precision;
+        case "symbolRecall":
+            return quality.symbol.recall;
+        case "factCoverage":
+            return quality.fact.coverage;
+        case "irrelevantContextRatio":
+            return quality.irrelevantContextRatio;
+    }
+}
+function aggregateRatio(metrics) {
+    const available = metrics.filter((metric) => metric.availability === "available" && typeof metric.value === "number");
+    return {
+        availableCount: available.length,
+        unavailableCount: metrics.filter((metric) => metric.availability === "unavailable").length,
+        notApplicableCount: metrics.filter((metric) => metric.availability === "not-applicable").length,
+        meanValue: available.length === 0 ? null : available.reduce((sum, metric) => sum + metric.value, 0) / available.length
+    };
+}
+function aggregateTokens(cases) {
+    const counts = cases
+        .map((entry) => entry.quality?.retrievedTokenCount)
+        .filter((count) => typeof count === "number" && Number.isFinite(count) && count >= 0);
+    const total = counts.reduce((sum, count) => sum + count, 0);
+    return {
+        availableCount: counts.length,
+        unavailableCount: cases.length - counts.length,
+        totalTokens: counts.length === 0 ? null : total,
+        meanTokens: counts.length === 0 ? null : total / counts.length,
+        minTokens: counts.length === 0 ? null : Math.min(...counts),
+        maxTokens: counts.length === 0 ? null : Math.max(...counts)
+    };
+}
+const countStatus = (cases, status) => cases.filter((entry) => entry.status === status).length;
+/**
+ * Pure macro aggregation over per-case Batch 2 results. It never recalculates a per-case metric, never weights by
+ * denominators, and never produces a score, winner, ranking or threshold.
+ */
+export function aggregateRetrievalPrecisionRecall(cases) {
+    const ratios = Object.fromEntries(RETRIEVAL_PRECISION_RECALL_RATIO_KEYS.map((key) => [key, aggregateRatio(cases.map((entry) => retrievalRatioOf(entry.quality, key)))]));
+    const sumLists = (select) => {
+        let total = 0;
+        let evidenceCases = 0;
+        for (const entry of cases) {
+            const list = entry.quality === null ? null : select(entry.quality);
+            if (list === null)
+                continue;
+            total += list.length;
+            evidenceCases += 1;
+        }
+        return { total, evidenceCases };
+    };
+    const missedFiles = sumLists((quality) => quality.file.missedFiles);
+    const irrelevantFiles = sumLists((quality) => quality.file.irrelevantRetrievedFiles);
+    const missedSymbols = sumLists((quality) => quality.symbol.missedSymbols);
+    const irrelevantSymbols = sumLists((quality) => quality.symbol.irrelevantRetrievedSymbols);
+    const uncoveredFacts = sumLists((quality) => quality.fact.uncoveredFactIds);
+    const evidenceCount = (availability) => cases.filter((entry) => entry.retrieval?.evidenceAvailability === availability).length;
+    return {
+        runSummary: {
+            projectCount: new Set(cases.map((entry) => entry.benchmarkProject)).size,
+            caseCount: cases.length,
+            completedCaseCount: countStatus(cases, "completed"),
+            partialCaseCount: countStatus(cases, "partial"),
+            failedCaseCount: countStatus(cases, "failed"),
+            retrievalEvidenceAvailableCaseCount: evidenceCount("available"),
+            retrievalEvidencePartialCaseCount: evidenceCount("partial"),
+            retrievalEvidenceUnavailableCaseCount: cases.filter((entry) => entry.retrieval !== null && entry.retrieval.evidenceAvailability !== "available" && entry.retrieval.evidenceAvailability !== "partial").length
+        },
+        ratios,
+        tokens: aggregateTokens(cases),
+        occurrences: {
+            totalMissedFileOccurrences: missedFiles.total,
+            totalMissedSymbolOccurrences: missedSymbols.total,
+            totalUncoveredFactOccurrences: uncoveredFacts.total,
+            totalIrrelevantRetrievedFileOccurrences: irrelevantFiles.total,
+            totalIrrelevantRetrievedSymbolOccurrences: irrelevantSymbols.total,
+            fileOccurrenceEvidenceCaseCount: missedFiles.evidenceCases,
+            symbolOccurrenceEvidenceCaseCount: missedSymbols.evidenceCases,
+            factOccurrenceEvidenceCaseCount: uncoveredFacts.evidenceCases
+        }
+    };
+}
+const RATIO_METRIC_IDS = {
+    filePrecision: { id: "file-precision", name: "File precision" },
+    fileRecall: { id: "file-recall", name: "File recall" },
+    symbolPrecision: { id: "symbol-precision", name: "Symbol precision" },
+    symbolRecall: { id: "symbol-recall", name: "Symbol recall" },
+    factCoverage: { id: "fact-coverage", name: "Fact coverage" },
+    irrelevantContextRatio: { id: "irrelevant-context-ratio", name: "Irrelevant context ratio" }
+};
+/** Run-level generic metric projection. Ratios stay canonical (0..1, never percent); null means no available evidence. */
+export function toRetrievalPrecisionRecallRunMetrics(aggregate) {
+    const summary = aggregate.runSummary;
+    const occurrences = aggregate.occurrences;
+    return [
+        { id: "case-count", name: "Cases", value: summary.caseCount, unit: "count" },
+        { id: "completed-case-count", name: "Completed cases", value: summary.completedCaseCount, unit: "count" },
+        { id: "partial-case-count", name: "Partial cases", value: summary.partialCaseCount, unit: "count" },
+        { id: "failed-case-count", name: "Failed cases", value: summary.failedCaseCount, unit: "count" },
+        ...RETRIEVAL_PRECISION_RECALL_RATIO_KEYS.map((key) => ({
+            id: `mean-${RATIO_METRIC_IDS[key].id}`,
+            name: `Mean ${RATIO_METRIC_IDS[key].name.toLowerCase()}`,
+            value: aggregate.ratios[key].meanValue,
+            unit: "ratio"
+        })),
+        { id: "total-retrieved-tokens", name: "Total retrieved tokens", value: aggregate.tokens.totalTokens, unit: "tokens" },
+        { id: "mean-retrieved-tokens", name: "Mean retrieved tokens", value: aggregate.tokens.meanTokens, unit: "tokens" },
+        { id: "total-missed-file-occurrences", name: "Total missed file occurrences", value: occurrences.totalMissedFileOccurrences, unit: "count" },
+        { id: "total-missed-symbol-occurrences", name: "Total missed symbol occurrences", value: occurrences.totalMissedSymbolOccurrences, unit: "count" },
+        { id: "total-uncovered-fact-occurrences", name: "Total uncovered fact occurrences", value: occurrences.totalUncoveredFactOccurrences, unit: "count" },
+        { id: "total-irrelevant-file-occurrences", name: "Total irrelevant file occurrences", value: occurrences.totalIrrelevantRetrievedFileOccurrences, unit: "count" }
+    ];
+}
+/** Per-case metric projection for the single variant; unavailable and not-applicable ratios project as null. */
+export function toRetrievalPrecisionRecallOutcomeMetrics(quality, variantId, caseId) {
+    const base = { variantId, caseId };
+    const ratio = (key) => ({
+        ...base,
+        id: RATIO_METRIC_IDS[key].id,
+        name: RATIO_METRIC_IDS[key].name,
+        value: retrievalRatioOf(quality, key).value,
+        unit: "ratio"
+    });
+    const count = (id, name, value) => ({
+        ...base,
+        id,
+        name,
+        value: value ?? null,
+        unit: "count"
+    });
+    return [
+        ...RETRIEVAL_PRECISION_RECALL_RATIO_KEYS.map(ratio),
+        { ...base, id: "retrieved-token-count", name: "Retrieved token count", value: quality?.retrievedTokenCount ?? null, unit: "tokens" },
+        count("missed-file-count", "Missed file count", quality?.file.missedFileCount),
+        count("missed-symbol-count", "Missed symbol count", quality?.symbol.missedSymbolCount),
+        count("uncovered-fact-count", "Uncovered fact count", quality?.fact.uncoveredFactCount),
+        count("irrelevant-retrieved-file-count", "Irrelevant retrieved file count", quality?.file.irrelevantRetrievedFiles?.length),
+        count("irrelevant-retrieved-symbol-count", "Irrelevant retrieved symbol count", quality?.symbol.irrelevantRetrievedSymbols?.length)
+    ];
+}

package/dist/src/experiments/plugins/retrievalPrecisionRecall/plugin.js

@@ -0,0 +1,253 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { serializeLocalRepositorySubjectManifest } from "../../../evaluation/localRepositorySubject/index.js";
+import { summarizeExperimentRun } from "../../results.js";
+import { LocalSubjectExecutionError } from "../contextWindowScaling/localSubjectErrors.js";
+import { projectExternalLocalTarget } from "../contextWindowScaling/localSubjectPrivacy.js";
+import { isSamePhysicalDirectory } from "../contextWindowScaling/localSubjectScratch.js";
+import { selectWarmIndexCases } from "../warmIndexReuse/selection.js";
+import { defaultRetrievalPrecisionRecallConfig, retrievalPrecisionRecallConfigDefinition, validateRetrievalPrecisionRecallConfig } from "./config.js";
+import { executeRetrievalPrecisionRecall } from "./execution.js";
+import { executeLocalRepositorySubjectRetrievalPrecisionRecall } from "./localSubjectExecution.js";
+import { assertExternalLocalProjectionIsPrivate, describeRetrievalLocalSubjectFailureForPersistence, projectRetrievalEvidenceForExternalLocalPersistence, UNEXPECTED_EXTERNAL_FAILURE_MESSAGE } from "./localSubjectPrivacy.js";
+import { buildRetrievalPrecisionRecallExecutionArtifact, RETRIEVAL_PRECISION_RECALL_EXECUTION_ARTIFACT_FILE } from "./executionArtifact.js";
+import { aggregateRetrievalPrecisionRecall, retrievalRatioOf, toRetrievalPrecisionRecallOutcomeMetrics, toRetrievalPrecisionRecallRunMetrics } from "./metrics.js";
+import { RETRIEVAL_PRECISION_RECALL_PLUGIN_ID, RETRIEVAL_PRECISION_RECALL_VARIANTS, RETRIEVAL_PRECISION_RECALL_VARIANT_ID, retrievalPrecisionRecallMetadata } from "./metadata.js";
+import { RETRIEVAL_PRECISION_RECALL_RATIO_KEYS } from "./types.js";
+/** Privacy-safe Batch 1 manifest persisted beside the execution artifact for external-local runs (same file as context-window-scaling). */
+export const RETRIEVAL_PRECISION_RECALL_LOCAL_SUBJECT_MANIFEST_FILE = "local-repository-subject-manifest.json";
+/** An external target must always arrive with a loaded local repository subject; there is no bundled fallback against it. */
+export const RETRIEVAL_PRECISION_RECALL_EXTERNAL_REQUIRES_SUBJECT_MESSAGE = "External-local retrieval-precision-recall targets require a loaded local repository subject (--local-subject-config).";
+export const retrievalPrecisionRecallPlugin = {
+    metadata: retrievalPrecisionRecallMetadata,
+    defaultConfig: defaultRetrievalPrecisionRecallConfig,
+    configDefinition: retrievalPrecisionRecallConfigDefinition,
+    supportedVariants: RETRIEVAL_PRECISION_RECALL_VARIANTS.map((variant) => variant.id),
+    validateConfig: validateRetrievalPrecisionRecallConfig,
+    async run(context) {
+        const startedAt = context.startedAt.toISOString();
+        // The runner does not enforce supportedTargets, so the plugin fails closed on its own.
+        const localSubject = readLocalSubjectInput(context.inputs);
+        if (localSubject)
+            return runLocalSubject(context, localSubject, startedAt);
+        if (context.target.kind !== "self" || context.target.isSelf !== true) {
+            // Never fall back to the bundled corpus against an external repository.
+            throw new Error(RETRIEVAL_PRECISION_RECALL_EXTERNAL_REQUIRES_SUBJECT_MESSAGE);
+        }
+        const cases = selectWarmIndexCases(readCasesInput(context.inputs), context.config);
+        const outDir = context.outputRoot ?? path.resolve(context.toolRoot, context.config.outDir);
+        await mkdir(outDir, { recursive: true });
+        const caseEvidence = await executeRetrievalPrecisionRecall({
+            cases,
+            kitCommand: context.config.kitCommand,
+            outputRoot: outDir,
+            dependencies: readDependenciesInput(context.inputs)
+        });
+        // Scientific truth is calculated before persistence; the artifact and every report consume it unchanged.
+        const aggregate = aggregateRetrievalPrecisionRecall(caseEvidence);
+        const completedAt = new Date().toISOString();
+        const artifact = buildRetrievalPrecisionRecallExecutionArtifact({
+            runId: context.runId,
+            pluginId: RETRIEVAL_PRECISION_RECALL_PLUGIN_ID,
+            pluginSchemaVersion: retrievalPrecisionRecallMetadata.schemaVersion,
+            startedAt,
+            completedAt,
+            cases: caseEvidence,
+            aggregate
+        });
+        const artifactPath = resolveWithinRoot(outDir, RETRIEVAL_PRECISION_RECALL_EXECUTION_ARTIFACT_FILE);
+        await writeFile(artifactPath, `${JSON.stringify(artifact, null, 2)}\n`, "utf8");
+        return mapRetrievalPrecisionRecallToRun({
+            runId: context.runId,
+            startedAt,
+            completedAt,
+            target: context.target,
+            caseEvidence,
+            aggregate,
+            artifactPath
+        });
+    },
+    summarize(result) {
+        return result.summary ?? summarizeExperimentRun(result);
+    }
+};
+function readLocalSubjectInput(inputs) {
+    const value = inputs?.localSubject;
+    return value && typeof value === "object" ? value : undefined;
+}
+/**
+ * External-local path. The safe seam owns execution, immutability and scratch cleanup and either returns evidence or
+ * throws; only after it succeeds is anything durable written: the case evidence is privacy-projected, the aggregate
+ * (calculated from the unprojected numbers) is preserved, and the artifact and manifest are written, in that order.
+ */
+async function runLocalSubject(context, subject, startedAt) {
+    if (context.target.isSelf || context.target.kind !== "external-local") {
+        throw new Error("Local-repository subject mode requires an external-local target.");
+    }
+    if (context.config.caseIds !== undefined || context.config.benchmarkProjects !== undefined) {
+        throw new Error("Case and benchmark-project filters are not supported for an external local repository subject; the subject config owns the case set.");
+    }
+    if (!(await isSamePhysicalDirectory(context.target.targetRoot, subject.repositoryRoot))) {
+        throw new Error("The selected --target is not the repository the local subject was loaded from.");
+    }
+    const outDir = context.outputRoot;
+    if (!outDir)
+        throw new Error("Local-repository subject mode requires an experiment output directory.");
+    let localResult;
+    try {
+        localResult = await executeLocalRepositorySubjectRetrievalPrecisionRecall({
+            subject,
+            kitCommand: context.config.kitCommand,
+            workRoot: outDir,
+            dependencies: readDependenciesInput(context.inputs)
+        });
+    }
+    catch (error) {
+        if (error instanceof LocalSubjectExecutionError)
+            throw new Error(describeRetrievalLocalSubjectFailureForPersistence(error));
+        throw new Error(UNEXPECTED_EXTERNAL_FAILURE_MESSAGE);
+    }
+    // Scientific numbers are calculated once from the unprojected evidence; projection only withholds identities.
+    const aggregate = aggregateRetrievalPrecisionRecall(localResult.caseEvidence);
+    const projected = projectRetrievalEvidenceForExternalLocalPersistence(localResult.caseEvidence);
+    assertExternalLocalProjectionIsPrivate(projected, {
+        filePaths: [...subject.eligibleFiles, ...subject.runtimeSafetyExclusions.gitIgnoredFiles, ...subject.runtimeSafetyExclusions.oversizedFiles],
+        caseTitles: subject.evaluationCases.map((evaluationCase) => evaluationCase.title),
+        warnings: localResult.caseEvidence.flatMap((entry) => entry.retrieval?.warnings ?? [])
+    });
+    await mkdir(outDir, { recursive: true });
+    const completedAt = new Date().toISOString();
+    const artifact = buildRetrievalPrecisionRecallExecutionArtifact({
+        runId: context.runId,
+        pluginId: RETRIEVAL_PRECISION_RECALL_PLUGIN_ID,
+        pluginSchemaVersion: retrievalPrecisionRecallMetadata.schemaVersion,
+        startedAt,
+        completedAt,
+        cases: projected,
+        aggregate
+    });
+    const artifactPath = resolveWithinRoot(outDir, RETRIEVAL_PRECISION_RECALL_EXECUTION_ARTIFACT_FILE);
+    await writeFile(artifactPath, `${JSON.stringify(artifact, null, 2)}
+`, "utf8");
+    const manifestPath = resolveWithinRoot(outDir, RETRIEVAL_PRECISION_RECALL_LOCAL_SUBJECT_MANIFEST_FILE);
+    await writeFile(manifestPath, serializeLocalRepositorySubjectManifest(subject.manifest), "utf8");
+    const run = mapRetrievalPrecisionRecallToRun({
+        runId: context.runId,
+        startedAt,
+        completedAt,
+        target: projectExternalLocalTarget(subject.manifest),
+        caseEvidence: projected,
+        aggregate,
+        artifactPath
+    });
+    run.artifacts.push({
+        id: "local-repository-subject-manifest",
+        label: "Privacy-safe local repository subject manifest",
+        path: manifestPath,
+        kind: "artifact",
+        mimeType: "application/json",
+        description: "Privacy-safe logical subject, Git, safety and aggregate inventory metadata without source or private identity lists."
+    });
+    return run;
+}
+export function mapRetrievalPrecisionRecallToRun(args) {
+    const cases = args.caseEvidence.map((entry) => ({
+        id: entry.caseId,
+        name: entry.caseName,
+        outcomes: [buildOutcome(entry)],
+        metadata: { benchmarkProject: entry.benchmarkProject, taskLocality: entry.taskLocality }
+    }));
+    const failures = args.caseEvidence.flatMap((entry) => entry.errors.map((error) => ({
+        code: error.code,
+        message: error.message,
+        variantId: RETRIEVAL_PRECISION_RECALL_VARIANT_ID,
+        caseId: entry.caseId,
+        recoverable: true
+    })));
+    const run = {
+        runId: args.runId,
+        pluginId: RETRIEVAL_PRECISION_RECALL_PLUGIN_ID,
+        startedAt: args.startedAt,
+        completedAt: args.completedAt,
+        status: aggregateStatus(args.caseEvidence.map((entry) => entry.status)),
+        target: args.target,
+        variants: RETRIEVAL_PRECISION_RECALL_VARIANTS.map((variant) => ({ ...variant })),
+        cases,
+        metrics: toRetrievalPrecisionRecallRunMetrics(args.aggregate),
+        artifacts: [
+            {
+                id: "retrieval-precision-recall-execution",
+                label: "Retrieval precision/recall execution evidence",
+                path: args.artifactPath,
+                kind: "artifact",
+                mimeType: "application/json",
+                description: "Per-case retrieval-quality evidence and aggregate without context text or raw command output."
+            }
+        ],
+        warnings: [],
+        failures,
+        metadata: { executionArtifactPath: args.artifactPath },
+        caseExecutionEvidence: args.caseEvidence.map((entry) => structuredClone(entry)),
+        aggregate: structuredClone(args.aggregate)
+    };
+    run.summary = summarizeExperimentRun(run);
+    return run;
+}
+function buildOutcome(entry) {
+    const warnings = (entry.retrieval?.warnings ?? []).map((message) => ({
+        code: "retrieval-warning",
+        message,
+        variantId: RETRIEVAL_PRECISION_RECALL_VARIANT_ID,
+        caseId: entry.caseId
+    }));
+    const failures = entry.errors.map((error) => ({
+        code: error.code,
+        message: error.message,
+        variantId: RETRIEVAL_PRECISION_RECALL_VARIANT_ID,
+        caseId: entry.caseId,
+        recoverable: true
+    }));
+    // Availability and reason stay visible even though an unavailable/not-applicable ratio projects as a null value.
+    const metricAvailability = {};
+    for (const key of RETRIEVAL_PRECISION_RECALL_RATIO_KEYS) {
+        const ratio = retrievalRatioOf(entry.quality, key);
+        metricAvailability[key] = { availability: ratio.availability, reason: ratio.reason };
+    }
+    return {
+        id: `${entry.caseId}:${RETRIEVAL_PRECISION_RECALL_VARIANT_ID}`,
+        caseId: entry.caseId,
+        variantId: RETRIEVAL_PRECISION_RECALL_VARIANT_ID,
+        status: entry.status,
+        metrics: toRetrievalPrecisionRecallOutcomeMetrics(entry.quality, RETRIEVAL_PRECISION_RECALL_VARIANT_ID, entry.caseId),
+        artifacts: [],
+        warnings,
+        failures,
+        metadata: {
+            benchmarkProject: entry.benchmarkProject,
+            evidenceAvailability: entry.retrieval?.evidenceAvailability ?? null,
+            metricAvailability
+        }
+    };
+}
+/** Same aggregation rule as the generic per-case summary: uniform statuses win, mixed is partial. */
+function aggregateStatus(statuses) {
+    if (statuses.length === 0)
+        return "skipped";
+    for (const status of ["completed", "skipped", "failed"]) {
+        if (statuses.every((value) => value === status))
+            return status;
+    }
+    return "partial";
+}
+function readCasesInput(inputs) {
+    const value = inputs?.cases;
+    if (!Array.isArray(value)) {
+        throw new Error("Retrieval precision/recall requires cases input.");
+    }
+    return value;
+}
+function readDependenciesInput(inputs) {
+    const value = inputs?.retrievalDependencies;
+    return value && typeof value === "object" && !Array.isArray(value) ? value : undefined;
+}

package/dist/src/experiments/plugins/retrievalPrecisionRecall/types.js

@@ -0,0 +1,8 @@
+export const RETRIEVAL_PRECISION_RECALL_RATIO_KEYS = [
+    "filePrecision",
+    "fileRecall",
+    "symbolPrecision",
+    "symbolRecall",
+    "factCoverage",
+    "irrelevantContextRatio"
+];

package/dist/src/report/experiments/buildPluginExperimentReport.js

@@ -3,9 +3,12 @@ import { buildContextStrategyComparisonV043Report } from "./buildContextStrategy
 import { buildWarmIndexReuseReport } from "./buildWarmIndexReuseReport.js";
 import { buildIncrementalChangeStalenessPluginReport } from "./buildIncrementalChangeStalenessPluginReport.js";
 import { buildContextWindowScalingReport } from "./buildContextWindowScalingReport.js";
+import { buildRetrievalPrecisionRecallReport } from "./buildRetrievalPrecisionRecallReport.js";
 // Bulk context-window-scaling evidence is presented by the typed report section; the complete
 // evidence stays in context-window-scaling-execution.json.
 const CONTEXT_WINDOW_SCALING_BULK_KEYS = ["executionEvidence", "aggregate"];
+// Same for retrieval-precision-recall: the typed section presents the evidence; the execution artifact keeps all of it.
+const RETRIEVAL_PRECISION_RECALL_BULK_KEYS = ["caseExecutionEvidence", "aggregate"];
 const V043_BULK_ARRAY_KEYS = [
     "v043StageContextExecutions",
     "v043StageContextEvaluations",
@@ -18,6 +21,7 @@ export function buildPluginExperimentReport(args) {
     const warmIndexReuse = buildWarmIndexReuseReport(args.run);
     const incrementalChangeStaleness = buildIncrementalChangeStalenessPluginReport(args.run);
     const contextWindowScaling = buildContextWindowScalingReport(args.run);
+    const retrievalPrecisionRecall = buildRetrievalPrecisionRecallReport(args.run);
     const rawRun = { ...args.run, artifacts: relativizeArtifacts(args.run.artifacts, outputRoot) };
     for (const key of V043_BULK_ARRAY_KEYS) {
         delete rawRun[key];
@@ -27,6 +31,11 @@ export function buildPluginExperimentReport(args) {
             delete rawRun[key];
         }
     }
+    if (retrievalPrecisionRecall) {
+        for (const key of RETRIEVAL_PRECISION_RECALL_BULK_KEYS) {
+            delete rawRun[key];
+        }
+    }
     return {
         metadata: {
             generatedAt: args.generatedAt ?? new Date().toISOString(),
@@ -53,8 +62,9 @@ export function buildPluginExperimentReport(args) {
         warmIndexReuse,
         incrementalChangeStaleness,
         contextWindowScaling,
+        retrievalPrecisionRecall,
         contextStrategyComparisonV043,
-        interpretation: buildInterpretation(args.run, contextStrategyComparisonV043, warmIndexReuse, contextWindowScaling),
+        interpretation: buildInterpretation(args.run, contextStrategyComparisonV043, warmIndexReuse, contextWindowScaling, retrievalPrecisionRecall),
         rawRun,
     };
 }
@@ -133,7 +143,16 @@ function affectedNeighborhoodSentence(report) {
         `reindex recommendations were recommended=${summary.recommendedReindexCount}, not-indicated=${summary.notIndicatedReindexCount}, unknown=${summary.unknownReindexRecommendationCount}. ` +
         "Affected-neighborhood evidence uses a one-hop baseline graph around confirmed changed indexed files/symbols. It is observational and does not alter retrieval or execution status.");
 }
-function buildInterpretation(run, contextStrategyComparisonV043, warmIndexReuse, contextWindowScaling) {
+function buildInterpretation(run, contextStrategyComparisonV043, warmIndexReuse, contextWindowScaling, retrievalPrecisionRecall) {
+    if (retrievalPrecisionRecall) {
+        const summary = retrievalPrecisionRecall.runSummary;
+        return {
+            summary: `Retrieval quality was measured over ${summary.caseCount} case${summary.caseCount === 1 ? "" : "s"} (${summary.completedCaseCount} completed, ${summary.partialCaseCount} partial, ${summary.failedCaseCount} failed). ` +
+                "Available case-level file, symbol and fact metrics are summarized without ranking strategies. " +
+                "Unavailable and not-applicable evidence is excluded from macro means rather than treated as zero.",
+            recommendedNextStep: "Review per-case missed and irrelevant context together with availability before drawing conclusions."
+        };
+    }
     if (contextWindowScaling) {
         const summary = contextWindowScaling.runSummary;
         return {