@dailephd/my-dev-kit-lab 0.7.1 → 0.8.0

package/dist/src/experiments/plugins/retrievalPrecisionRecall/plugin.js

@@ -0,0 +1,253 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { serializeLocalRepositorySubjectManifest } from "../../../evaluation/localRepositorySubject/index.js";
+import { summarizeExperimentRun } from "../../results.js";
+import { LocalSubjectExecutionError } from "../contextWindowScaling/localSubjectErrors.js";
+import { projectExternalLocalTarget } from "../contextWindowScaling/localSubjectPrivacy.js";
+import { isSamePhysicalDirectory } from "../contextWindowScaling/localSubjectScratch.js";
+import { selectWarmIndexCases } from "../warmIndexReuse/selection.js";
+import { defaultRetrievalPrecisionRecallConfig, retrievalPrecisionRecallConfigDefinition, validateRetrievalPrecisionRecallConfig } from "./config.js";
+import { executeRetrievalPrecisionRecall } from "./execution.js";
+import { executeLocalRepositorySubjectRetrievalPrecisionRecall } from "./localSubjectExecution.js";
+import { assertExternalLocalProjectionIsPrivate, describeRetrievalLocalSubjectFailureForPersistence, projectRetrievalEvidenceForExternalLocalPersistence, UNEXPECTED_EXTERNAL_FAILURE_MESSAGE } from "./localSubjectPrivacy.js";
+import { buildRetrievalPrecisionRecallExecutionArtifact, RETRIEVAL_PRECISION_RECALL_EXECUTION_ARTIFACT_FILE } from "./executionArtifact.js";
+import { aggregateRetrievalPrecisionRecall, retrievalRatioOf, toRetrievalPrecisionRecallOutcomeMetrics, toRetrievalPrecisionRecallRunMetrics } from "./metrics.js";
+import { RETRIEVAL_PRECISION_RECALL_PLUGIN_ID, RETRIEVAL_PRECISION_RECALL_VARIANTS, RETRIEVAL_PRECISION_RECALL_VARIANT_ID, retrievalPrecisionRecallMetadata } from "./metadata.js";
+import { RETRIEVAL_PRECISION_RECALL_RATIO_KEYS } from "./types.js";
+/** Privacy-safe Batch 1 manifest persisted beside the execution artifact for external-local runs (same file as context-window-scaling). */
+export const RETRIEVAL_PRECISION_RECALL_LOCAL_SUBJECT_MANIFEST_FILE = "local-repository-subject-manifest.json";
+/** An external target must always arrive with a loaded local repository subject; there is no bundled fallback against it. */
+export const RETRIEVAL_PRECISION_RECALL_EXTERNAL_REQUIRES_SUBJECT_MESSAGE = "External-local retrieval-precision-recall targets require a loaded local repository subject (--local-subject-config).";
+export const retrievalPrecisionRecallPlugin = {
+    metadata: retrievalPrecisionRecallMetadata,
+    defaultConfig: defaultRetrievalPrecisionRecallConfig,
+    configDefinition: retrievalPrecisionRecallConfigDefinition,
+    supportedVariants: RETRIEVAL_PRECISION_RECALL_VARIANTS.map((variant) => variant.id),
+    validateConfig: validateRetrievalPrecisionRecallConfig,
+    async run(context) {
+        const startedAt = context.startedAt.toISOString();
+        // The runner does not enforce supportedTargets, so the plugin fails closed on its own.
+        const localSubject = readLocalSubjectInput(context.inputs);
+        if (localSubject)
+            return runLocalSubject(context, localSubject, startedAt);
+        if (context.target.kind !== "self" || context.target.isSelf !== true) {
+            // Never fall back to the bundled corpus against an external repository.
+            throw new Error(RETRIEVAL_PRECISION_RECALL_EXTERNAL_REQUIRES_SUBJECT_MESSAGE);
+        }
+        const cases = selectWarmIndexCases(readCasesInput(context.inputs), context.config);
+        const outDir = context.outputRoot ?? path.resolve(context.toolRoot, context.config.outDir);
+        await mkdir(outDir, { recursive: true });
+        const caseEvidence = await executeRetrievalPrecisionRecall({
+            cases,
+            kitCommand: context.config.kitCommand,
+            outputRoot: outDir,
+            dependencies: readDependenciesInput(context.inputs)
+        });
+        // Scientific truth is calculated before persistence; the artifact and every report consume it unchanged.
+        const aggregate = aggregateRetrievalPrecisionRecall(caseEvidence);
+        const completedAt = new Date().toISOString();
+        const artifact = buildRetrievalPrecisionRecallExecutionArtifact({
+            runId: context.runId,
+            pluginId: RETRIEVAL_PRECISION_RECALL_PLUGIN_ID,
+            pluginSchemaVersion: retrievalPrecisionRecallMetadata.schemaVersion,
+            startedAt,
+            completedAt,
+            cases: caseEvidence,
+            aggregate
+        });
+        const artifactPath = resolveWithinRoot(outDir, RETRIEVAL_PRECISION_RECALL_EXECUTION_ARTIFACT_FILE);
+        await writeFile(artifactPath, `${JSON.stringify(artifact, null, 2)}\n`, "utf8");
+        return mapRetrievalPrecisionRecallToRun({
+            runId: context.runId,
+            startedAt,
+            completedAt,
+            target: context.target,
+            caseEvidence,
+            aggregate,
+            artifactPath
+        });
+    },
+    summarize(result) {
+        return result.summary ?? summarizeExperimentRun(result);
+    }
+};
+function readLocalSubjectInput(inputs) {
+    const value = inputs?.localSubject;
+    return value && typeof value === "object" ? value : undefined;
+}
+/**
+ * External-local path. The safe seam owns execution, immutability and scratch cleanup and either returns evidence or
+ * throws; only after it succeeds is anything durable written: the case evidence is privacy-projected, the aggregate
+ * (calculated from the unprojected numbers) is preserved, and the artifact and manifest are written, in that order.
+ */
+async function runLocalSubject(context, subject, startedAt) {
+    if (context.target.isSelf || context.target.kind !== "external-local") {
+        throw new Error("Local-repository subject mode requires an external-local target.");
+    }
+    if (context.config.caseIds !== undefined || context.config.benchmarkProjects !== undefined) {
+        throw new Error("Case and benchmark-project filters are not supported for an external local repository subject; the subject config owns the case set.");
+    }
+    if (!(await isSamePhysicalDirectory(context.target.targetRoot, subject.repositoryRoot))) {
+        throw new Error("The selected --target is not the repository the local subject was loaded from.");
+    }
+    const outDir = context.outputRoot;
+    if (!outDir)
+        throw new Error("Local-repository subject mode requires an experiment output directory.");
+    let localResult;
+    try {
+        localResult = await executeLocalRepositorySubjectRetrievalPrecisionRecall({
+            subject,
+            kitCommand: context.config.kitCommand,
+            workRoot: outDir,
+            dependencies: readDependenciesInput(context.inputs)
+        });
+    }
+    catch (error) {
+        if (error instanceof LocalSubjectExecutionError)
+            throw new Error(describeRetrievalLocalSubjectFailureForPersistence(error));
+        throw new Error(UNEXPECTED_EXTERNAL_FAILURE_MESSAGE);
+    }
+    // Scientific numbers are calculated once from the unprojected evidence; projection only withholds identities.
+    const aggregate = aggregateRetrievalPrecisionRecall(localResult.caseEvidence);
+    const projected = projectRetrievalEvidenceForExternalLocalPersistence(localResult.caseEvidence);
+    assertExternalLocalProjectionIsPrivate(projected, {
+        filePaths: [...subject.eligibleFiles, ...subject.runtimeSafetyExclusions.gitIgnoredFiles, ...subject.runtimeSafetyExclusions.oversizedFiles],
+        caseTitles: subject.evaluationCases.map((evaluationCase) => evaluationCase.title),
+        warnings: localResult.caseEvidence.flatMap((entry) => entry.retrieval?.warnings ?? [])
+    });
+    await mkdir(outDir, { recursive: true });
+    const completedAt = new Date().toISOString();
+    const artifact = buildRetrievalPrecisionRecallExecutionArtifact({
+        runId: context.runId,
+        pluginId: RETRIEVAL_PRECISION_RECALL_PLUGIN_ID,
+        pluginSchemaVersion: retrievalPrecisionRecallMetadata.schemaVersion,
+        startedAt,
+        completedAt,
+        cases: projected,
+        aggregate
+    });
+    const artifactPath = resolveWithinRoot(outDir, RETRIEVAL_PRECISION_RECALL_EXECUTION_ARTIFACT_FILE);
+    await writeFile(artifactPath, `${JSON.stringify(artifact, null, 2)}
+`, "utf8");
+    const manifestPath = resolveWithinRoot(outDir, RETRIEVAL_PRECISION_RECALL_LOCAL_SUBJECT_MANIFEST_FILE);
+    await writeFile(manifestPath, serializeLocalRepositorySubjectManifest(subject.manifest), "utf8");
+    const run = mapRetrievalPrecisionRecallToRun({
+        runId: context.runId,
+        startedAt,
+        completedAt,
+        target: projectExternalLocalTarget(subject.manifest),
+        caseEvidence: projected,
+        aggregate,
+        artifactPath
+    });
+    run.artifacts.push({
+        id: "local-repository-subject-manifest",
+        label: "Privacy-safe local repository subject manifest",
+        path: manifestPath,
+        kind: "artifact",
+        mimeType: "application/json",
+        description: "Privacy-safe logical subject, Git, safety and aggregate inventory metadata without source or private identity lists."
+    });
+    return run;
+}
+export function mapRetrievalPrecisionRecallToRun(args) {
+    const cases = args.caseEvidence.map((entry) => ({
+        id: entry.caseId,
+        name: entry.caseName,
+        outcomes: [buildOutcome(entry)],
+        metadata: { benchmarkProject: entry.benchmarkProject, taskLocality: entry.taskLocality }
+    }));
+    const failures = args.caseEvidence.flatMap((entry) => entry.errors.map((error) => ({
+        code: error.code,
+        message: error.message,
+        variantId: RETRIEVAL_PRECISION_RECALL_VARIANT_ID,
+        caseId: entry.caseId,
+        recoverable: true
+    })));
+    const run = {
+        runId: args.runId,
+        pluginId: RETRIEVAL_PRECISION_RECALL_PLUGIN_ID,
+        startedAt: args.startedAt,
+        completedAt: args.completedAt,
+        status: aggregateStatus(args.caseEvidence.map((entry) => entry.status)),
+        target: args.target,
+        variants: RETRIEVAL_PRECISION_RECALL_VARIANTS.map((variant) => ({ ...variant })),
+        cases,
+        metrics: toRetrievalPrecisionRecallRunMetrics(args.aggregate),
+        artifacts: [
+            {
+                id: "retrieval-precision-recall-execution",
+                label: "Retrieval precision/recall execution evidence",
+                path: args.artifactPath,
+                kind: "artifact",
+                mimeType: "application/json",
+                description: "Per-case retrieval-quality evidence and aggregate without context text or raw command output."
+            }
+        ],
+        warnings: [],
+        failures,
+        metadata: { executionArtifactPath: args.artifactPath },
+        caseExecutionEvidence: args.caseEvidence.map((entry) => structuredClone(entry)),
+        aggregate: structuredClone(args.aggregate)
+    };
+    run.summary = summarizeExperimentRun(run);
+    return run;
+}
+function buildOutcome(entry) {
+    const warnings = (entry.retrieval?.warnings ?? []).map((message) => ({
+        code: "retrieval-warning",
+        message,
+        variantId: RETRIEVAL_PRECISION_RECALL_VARIANT_ID,
+        caseId: entry.caseId
+    }));
+    const failures = entry.errors.map((error) => ({
+        code: error.code,
+        message: error.message,
+        variantId: RETRIEVAL_PRECISION_RECALL_VARIANT_ID,
+        caseId: entry.caseId,
+        recoverable: true
+    }));
+    // Availability and reason stay visible even though an unavailable/not-applicable ratio projects as a null value.
+    const metricAvailability = {};
+    for (const key of RETRIEVAL_PRECISION_RECALL_RATIO_KEYS) {
+        const ratio = retrievalRatioOf(entry.quality, key);
+        metricAvailability[key] = { availability: ratio.availability, reason: ratio.reason };
+    }
+    return {
+        id: `${entry.caseId}:${RETRIEVAL_PRECISION_RECALL_VARIANT_ID}`,
+        caseId: entry.caseId,
+        variantId: RETRIEVAL_PRECISION_RECALL_VARIANT_ID,
+        status: entry.status,
+        metrics: toRetrievalPrecisionRecallOutcomeMetrics(entry.quality, RETRIEVAL_PRECISION_RECALL_VARIANT_ID, entry.caseId),
+        artifacts: [],
+        warnings,
+        failures,
+        metadata: {
+            benchmarkProject: entry.benchmarkProject,
+            evidenceAvailability: entry.retrieval?.evidenceAvailability ?? null,
+            metricAvailability
+        }
+    };
+}
+/** Same aggregation rule as the generic per-case summary: uniform statuses win, mixed is partial. */
+function aggregateStatus(statuses) {
+    if (statuses.length === 0)
+        return "skipped";
+    for (const status of ["completed", "skipped", "failed"]) {
+        if (statuses.every((value) => value === status))
+            return status;
+    }
+    return "partial";
+}
+function readCasesInput(inputs) {
+    const value = inputs?.cases;
+    if (!Array.isArray(value)) {
+        throw new Error("Retrieval precision/recall requires cases input.");
+    }
+    return value;
+}
+function readDependenciesInput(inputs) {
+    const value = inputs?.retrievalDependencies;
+    return value && typeof value === "object" && !Array.isArray(value) ? value : undefined;
+}

package/dist/src/experiments/plugins/retrievalPrecisionRecall/types.js

@@ -0,0 +1,8 @@
+export const RETRIEVAL_PRECISION_RECALL_RATIO_KEYS = [
+    "filePrecision",
+    "fileRecall",
+    "symbolPrecision",
+    "symbolRecall",
+    "factCoverage",
+    "irrelevantContextRatio"
+];

package/dist/src/report/experiments/buildPluginExperimentReport.js

@@ -3,9 +3,12 @@ import { buildContextStrategyComparisonV043Report } from "./buildContextStrategy
 import { buildWarmIndexReuseReport } from "./buildWarmIndexReuseReport.js";
 import { buildIncrementalChangeStalenessPluginReport } from "./buildIncrementalChangeStalenessPluginReport.js";
 import { buildContextWindowScalingReport } from "./buildContextWindowScalingReport.js";
+import { buildRetrievalPrecisionRecallReport } from "./buildRetrievalPrecisionRecallReport.js";
 // Bulk context-window-scaling evidence is presented by the typed report section; the complete
 // evidence stays in context-window-scaling-execution.json.
 const CONTEXT_WINDOW_SCALING_BULK_KEYS = ["executionEvidence", "aggregate"];
+// Same for retrieval-precision-recall: the typed section presents the evidence; the execution artifact keeps all of it.
+const RETRIEVAL_PRECISION_RECALL_BULK_KEYS = ["caseExecutionEvidence", "aggregate"];
 const V043_BULK_ARRAY_KEYS = [
     "v043StageContextExecutions",
     "v043StageContextEvaluations",
@@ -18,6 +21,7 @@ export function buildPluginExperimentReport(args) {
     const warmIndexReuse = buildWarmIndexReuseReport(args.run);
     const incrementalChangeStaleness = buildIncrementalChangeStalenessPluginReport(args.run);
     const contextWindowScaling = buildContextWindowScalingReport(args.run);
+    const retrievalPrecisionRecall = buildRetrievalPrecisionRecallReport(args.run);
     const rawRun = { ...args.run, artifacts: relativizeArtifacts(args.run.artifacts, outputRoot) };
     for (const key of V043_BULK_ARRAY_KEYS) {
         delete rawRun[key];
@@ -27,6 +31,11 @@ export function buildPluginExperimentReport(args) {
             delete rawRun[key];
         }
     }
+    if (retrievalPrecisionRecall) {
+        for (const key of RETRIEVAL_PRECISION_RECALL_BULK_KEYS) {
+            delete rawRun[key];
+        }
+    }
     return {
         metadata: {
             generatedAt: args.generatedAt ?? new Date().toISOString(),
@@ -53,8 +62,9 @@ export function buildPluginExperimentReport(args) {
         warmIndexReuse,
         incrementalChangeStaleness,
         contextWindowScaling,
+        retrievalPrecisionRecall,
         contextStrategyComparisonV043,
-        interpretation: buildInterpretation(args.run, contextStrategyComparisonV043, warmIndexReuse, contextWindowScaling),
+        interpretation: buildInterpretation(args.run, contextStrategyComparisonV043, warmIndexReuse, contextWindowScaling, retrievalPrecisionRecall),
         rawRun,
     };
 }
@@ -133,7 +143,16 @@ function affectedNeighborhoodSentence(report) {
         `reindex recommendations were recommended=${summary.recommendedReindexCount}, not-indicated=${summary.notIndicatedReindexCount}, unknown=${summary.unknownReindexRecommendationCount}. ` +
         "Affected-neighborhood evidence uses a one-hop baseline graph around confirmed changed indexed files/symbols. It is observational and does not alter retrieval or execution status.");
 }
-function buildInterpretation(run, contextStrategyComparisonV043, warmIndexReuse, contextWindowScaling) {
+function buildInterpretation(run, contextStrategyComparisonV043, warmIndexReuse, contextWindowScaling, retrievalPrecisionRecall) {
+    if (retrievalPrecisionRecall) {
+        const summary = retrievalPrecisionRecall.runSummary;
+        return {
+            summary: `Retrieval quality was measured over ${summary.caseCount} case${summary.caseCount === 1 ? "" : "s"} (${summary.completedCaseCount} completed, ${summary.partialCaseCount} partial, ${summary.failedCaseCount} failed). ` +
+                "Available case-level file, symbol and fact metrics are summarized without ranking strategies. " +
+                "Unavailable and not-applicable evidence is excluded from macro means rather than treated as zero.",
+            recommendedNextStep: "Review per-case missed and irrelevant context together with availability before drawing conclusions."
+        };
+    }
     if (contextWindowScaling) {
         const summary = contextWindowScaling.runSummary;
         return {

package/dist/src/report/experiments/buildRetrievalPrecisionRecallReport.js

@@ -0,0 +1,83 @@
+import { RETRIEVAL_PRECISION_RECALL_PLUGIN_ID } from "../../experiments/plugins/retrievalPrecisionRecall/metadata.js";
+import { retrievalRatioOf } from "../../experiments/plugins/retrievalPrecisionRecall/metrics.js";
+import { RETRIEVAL_PRECISION_RECALL_RATIO_KEYS } from "../../experiments/plugins/retrievalPrecisionRecall/types.js";
+import { MAX_RETRIEVAL_REPORT_IDENTITIES, RETRIEVAL_PRECISION_RECALL_REPORT_SCHEMA_VERSION } from "./retrievalPrecisionRecallReportModel.js";
+/** Fixed wording; the report states observed evidence only and never ranks or scores anything. */
+export const RETRIEVAL_PRECISION_RECALL_LIMITATIONS = [
+    "File and symbol precision and recall are set-based measures, not ranked metrics.",
+    "Symbol scoring uses exact unique symbol names, because the answer key lists names rather than node identities.",
+    "Fact coverage uses explicit fact-to-context mappings in the answer key; it does not judge generated answers or interpret fact text.",
+    "Irrelevant context ratio is file-identity based and is the complement of file precision when both are available.",
+    "Retrieved token count measures the selected retrieval context payload; it is not a relevant versus irrelevant token attribution.",
+    "The current lifecycle expands only the top search candidate (lookup, slice and source).",
+    "Unavailable and not-applicable values are not zeros and are excluded from macro means.",
+    "The experiment uses no real agents.",
+    "v0.8.0 does not compare retrieval strategies."
+];
+function boundedList(items) {
+    if (items === null)
+        return null;
+    const displayed = items.slice(0, MAX_RETRIEVAL_REPORT_IDENTITIES);
+    return {
+        totalCount: items.length,
+        displayed: [...displayed],
+        displayedCount: displayed.length,
+        omittedCount: items.length - displayed.length
+    };
+}
+/**
+ * Presents the per-case metric results and aggregate already calculated on the run. It performs no precision, recall,
+ * coverage, ratio or mean calculation of its own; identity lists are only bounded for display.
+ */
+export function buildRetrievalPrecisionRecallReport(run) {
+    if (run.pluginId !== RETRIEVAL_PRECISION_RECALL_PLUGIN_ID)
+        return null;
+    const candidate = run;
+    const evidence = candidate.caseExecutionEvidence;
+    const aggregate = candidate.aggregate;
+    if (!evidence || !aggregate) {
+        // A run that failed before producing evidence (for example an unknown case id or a non-self target) still gets the
+        // generic report with its failure; only a non-failed run with missing evidence is a broken report source.
+        if (run.status === "failed")
+            return null;
+        throw new Error("Invalid retrieval-precision-recall report source: case execution evidence and aggregate are required.");
+    }
+    if (aggregate.runSummary.caseCount !== evidence.length) {
+        throw new Error("Invalid retrieval-precision-recall report source: aggregate and case execution evidence are inconsistent.");
+    }
+    const cases = evidence.map((entry) => {
+        const quality = entry.quality;
+        const metrics = Object.fromEntries(RETRIEVAL_PRECISION_RECALL_RATIO_KEYS.map((key) => {
+            const ratio = retrievalRatioOf(quality, key);
+            return [key, { availability: ratio.availability, numerator: ratio.numerator, denominator: ratio.denominator, value: ratio.value, reason: ratio.reason }];
+        }));
+        return {
+            caseId: entry.caseId,
+            caseName: entry.caseName,
+            benchmarkProject: entry.benchmarkProject,
+            taskLocality: entry.taskLocality,
+            status: entry.status,
+            evidenceAvailability: entry.retrieval?.evidenceAvailability ?? null,
+            metrics,
+            retrievedTokenCount: quality?.retrievedTokenCount ?? null,
+            tokenCountMethod: quality?.tokenCountMethod ?? null,
+            missedFiles: boundedList(quality?.file.missedFiles ?? null),
+            irrelevantRetrievedFiles: boundedList(quality?.file.irrelevantRetrievedFiles ?? null),
+            missedSymbols: boundedList(quality?.symbol.missedSymbols ?? null),
+            irrelevantRetrievedSymbols: boundedList(quality?.symbol.irrelevantRetrievedSymbols ?? null),
+            uncoveredFactIds: boundedList(quality?.fact.uncoveredFactIds ?? null),
+            errors: entry.errors.map((error) => `${error.code}: ${error.message}`),
+            identityRedaction: entry.identityRedaction ? { ...entry.identityRedaction } : null
+        };
+    });
+    return {
+        schemaVersion: RETRIEVAL_PRECISION_RECALL_REPORT_SCHEMA_VERSION,
+        tokenCountMethod: evidence.find((entry) => entry.quality?.tokenCountMethod != null)?.quality?.tokenCountMethod ?? null,
+        runSummary: { ...aggregate.runSummary },
+        ratioSummaries: structuredClone(aggregate.ratios),
+        tokenSummary: { ...aggregate.tokens },
+        occurrenceSummary: { ...aggregate.occurrences },
+        cases,
+        limitations: [...RETRIEVAL_PRECISION_RECALL_LIMITATIONS]
+    };
+}

package/dist/src/report/experiments/index.js

@@ -24,3 +24,7 @@ export * from "./contextWindowScalingReportModel.js";
 export { buildContextWindowScalingReport, CONTEXT_WINDOW_SCALING_INTERPRETATION } from "./buildContextWindowScalingReport.js";
 export { renderContextWindowScalingHtml } from "./renderContextWindowScalingHtml.js";
 export { renderContextWindowScalingTextLines } from "./renderContextWindowScalingText.js";
+export * from "./retrievalPrecisionRecallReportModel.js";
+export { buildRetrievalPrecisionRecallReport, RETRIEVAL_PRECISION_RECALL_LIMITATIONS } from "./buildRetrievalPrecisionRecallReport.js";
+export { renderRetrievalPrecisionRecallHtml } from "./renderRetrievalPrecisionRecallHtml.js";
+export { renderRetrievalPrecisionRecallTextLines } from "./renderRetrievalPrecisionRecallText.js";

package/dist/src/report/experiments/renderPluginExperimentReportHtml.js

@@ -2,7 +2,9 @@ import { renderContextStrategyComparisonV043Html } from "./renderContextStrategy
 import { renderWarmIndexReuseHtml } from "./renderWarmIndexReuseHtml.js";
 import { renderIncrementalChangeStalenessHtml } from "./renderIncrementalChangeStalenessHtml.js";
 import { renderContextWindowScalingHtml } from "./renderContextWindowScalingHtml.js";
+import { renderRetrievalPrecisionRecallHtml } from "./renderRetrievalPrecisionRecallHtml.js";
 export function renderPluginExperimentReportHtml(report) {
+    const redactedTarget = report.target.privacyProjection === "external-local-redacted";
     return `<!DOCTYPE html>
 <html lang="en">
 <head>
@@ -49,10 +51,10 @@ export function renderPluginExperimentReportHtml(report) {
         ["Mode", report.target.mode],
         ["Tool root", report.target.toolRoot],
         ["Target root", report.target.targetRoot],
-        ["Target package", report.target.packageName ? `${report.target.packageName}@${report.target.packageVersion ?? "unknown"}` : "unavailable"],
+        ["Target package", redactedTarget ? "redacted" : report.target.packageName ? `${report.target.packageName}@${report.target.packageVersion ?? "unknown"}` : "unavailable"],
         ["Target branch/commit", report.target.hasGit ? `${report.target.branch ?? "unknown"} / ${report.target.commit ?? "unknown"}` : "unavailable"],
-        ["Has package.json", String(report.target.hasPackageJson)],
-        ["Has lockfile", String(report.target.hasLockfile)],
+        ["Has package.json", redactedTarget ? "redacted" : String(report.target.hasPackageJson)],
+        ["Has lockfile", redactedTarget ? "redacted" : String(report.target.hasLockfile)],
         ["Output root", report.metadata.outputRoot ?? "unavailable"],
     ])}
   </section>
@@ -105,6 +107,8 @@ export function renderPluginExperimentReportHtml(report) {
 
   ${renderContextWindowScalingHtml(report.contextWindowScaling)}
 
+  ${renderRetrievalPrecisionRecallHtml(report.retrievalPrecisionRecall ?? null)}
+
   <section>
     <h2>Warnings, Skips, And Failures</h2>
     ${report.findings.length ? table(["Severity", "Code", "Message", "Variant", "Case"], report.findings.map((finding) => [

package/dist/src/report/experiments/renderPluginExperimentReportText.js

@@ -3,6 +3,7 @@ import { isV2ReportSection } from "./buildIncrementalChangeStalenessReportV2.js"
 import { renderIncrementalChangeStalenessSectionV2 } from "./renderIncrementalChangeStalenessTextV2.js";
 import { REINDEX_RECOMMENDATION_EXPLANATIONS } from "./buildWarmIndexReuseReport.js";
 import { renderContextWindowScalingTextLines } from "./renderContextWindowScalingText.js";
+import { renderRetrievalPrecisionRecallTextLines } from "./renderRetrievalPrecisionRecallText.js";
 function sanitizeScalar(value) {
     const text = String(value);
     let result = "";
@@ -671,6 +672,14 @@ export function renderPluginExperimentReportText(report) {
     else {
         lines.push(...renderContextWindowScalingTextLines(report.contextWindowScaling));
     }
+    pushSection(lines, "Retrieval Precision/Recall");
+    const retrievalPrecisionRecall = report.retrievalPrecisionRecall ?? null;
+    if (retrievalPrecisionRecall === null) {
+        lines.push("Not applicable to this plugin.");
+    }
+    else {
+        lines.push(...renderRetrievalPrecisionRecallTextLines(retrievalPrecisionRecall));
+    }
     pushSection(lines, "Warnings, Skips, And Failures");
     pushDashList(lines, report.findings.map((finding) => `${finding.severity} ${finding.code} ${finding.message} variant=${finding.variantId ?? ""} case=${finding.caseId ?? ""}`));
     pushSection(lines, "Artifacts");

package/dist/src/report/experiments/renderRetrievalPrecisionRecallHtml.js

@@ -0,0 +1,96 @@
+import { RETRIEVAL_PRECISION_RECALL_RATIO_KEYS } from "../../experiments/plugins/retrievalPrecisionRecall/types.js";
+import { RETRIEVAL_IDENTITY_REDACTION_EXPLANATION, formatRetrievalCount, formatRetrievalList, formatRetrievalMean, formatRetrievalMetric, RETRIEVAL_RATIO_LABELS } from "./renderRetrievalPrecisionRecallText.js";
+export function renderRetrievalPrecisionRecallHtml(section) {
+    if (section === null) {
+        return `<section>
+    <h2>Retrieval Precision/Recall</h2>
+    <p>Not applicable to this plugin.</p>
+  </section>`;
+    }
+    const tokens = section.tokenSummary;
+    const occurrences = section.occurrenceSummary;
+    return `<section>
+    <h2>Retrieval Precision/Recall</h2>
+    <p class="muted">Measures which expected files, symbols and facts the my-dev-kit retrieval lifecycle surfaced, and which irrelevant files it surfaced, without agents. It is descriptive and does not rank anything.</p>
+    ${table(["Field", "Value"], [
+        ["Schema", section.schemaVersion],
+        ["Token count method", section.tokenCountMethod ?? "unavailable"],
+        ...Object.entries(section.runSummary).map(([label, value]) => [label, String(value)])
+    ])}
+    ${section.cases.some((entry) => entry.identityRedaction !== null) ? `<p><strong>Identity redaction.</strong> ${escapeHtml(RETRIEVAL_IDENTITY_REDACTION_EXPLANATION)}</p>` : ""}
+    <h3>Macro Averages</h3>
+    <p class="muted">Mean over available per-case values only. Unavailable and not-applicable cases are counted separately and are never treated as zero.</p>
+    ${table(["Metric", "Available", "Unavailable", "Not applicable", "Mean"], RETRIEVAL_PRECISION_RECALL_RATIO_KEYS.map((key) => {
+        const summary = section.ratioSummaries[key];
+        return [RETRIEVAL_RATIO_LABELS[key], String(summary.availableCount), String(summary.unavailableCount), String(summary.notApplicableCount), formatRetrievalMean(summary.meanValue)];
+    }))}
+    <h3>Retrieved Token Summary</h3>
+    <p class="muted">Measured retrieval payload size; not a relevant versus irrelevant token attribution.</p>
+    ${table(["Available", "Unavailable", "Total", "Mean", "Min", "Max"], [[
+            String(tokens.availableCount),
+            String(tokens.unavailableCount),
+            formatRetrievalCount(tokens.totalTokens),
+            formatRetrievalMean(tokens.meanTokens),
+            formatRetrievalCount(tokens.minTokens),
+            formatRetrievalCount(tokens.maxTokens)
+        ]])}
+    <h3>Missed And Irrelevant Occurrence Totals</h3>
+    <p class="muted">Summed per case; identical paths in different cases or projects are not deduplicated.</p>
+    ${table(["Missed files", "Irrelevant files", "Cases with file evidence", "Missed symbols", "Irrelevant symbols", "Cases with symbol evidence", "Uncovered facts", "Cases with fact evidence"], [[
+            String(occurrences.totalMissedFileOccurrences),
+            String(occurrences.totalIrrelevantRetrievedFileOccurrences),
+            String(occurrences.fileOccurrenceEvidenceCaseCount),
+            String(occurrences.totalMissedSymbolOccurrences),
+            String(occurrences.totalIrrelevantRetrievedSymbolOccurrences),
+            String(occurrences.symbolOccurrenceEvidenceCaseCount),
+            String(occurrences.totalUncoveredFactOccurrences),
+            String(occurrences.factOccurrenceEvidenceCaseCount)
+        ]])}
+    <h3>Per-Case Results</h3>
+    ${table(["Case", "Project", "Locality", "Status", "Evidence", ...RETRIEVAL_PRECISION_RECALL_RATIO_KEYS.map((key) => RETRIEVAL_RATIO_LABELS[key]), "Retrieved tokens"], section.cases.map((entry) => [
+        entry.caseId,
+        entry.benchmarkProject,
+        entry.taskLocality ?? "unavailable",
+        entry.status,
+        entry.evidenceAvailability ?? "unavailable",
+        ...RETRIEVAL_PRECISION_RECALL_RATIO_KEYS.map((key) => formatRetrievalMetric(entry.metrics[key])),
+        formatRetrievalCount(entry.retrievedTokenCount)
+    ]))}
+    <h3>Per-Case Missed And Irrelevant Context</h3>
+    <p class="muted">Displayed lists are bounded; totals are exact.</p>
+    ${section.cases
+        .map((entry) => {
+        const lists = [
+            ["Missed files", entry.missedFiles],
+            ["Irrelevant retrieved files", entry.irrelevantRetrievedFiles],
+            ["Missed symbols", entry.missedSymbols],
+            ["Irrelevant retrieved symbols", entry.irrelevantRetrievedSymbols],
+            ["Uncovered fact IDs", entry.uncoveredFactIds]
+        ];
+        return `<h4><code>${escapeHtml(entry.caseId)}</code> (${escapeHtml(entry.status)})</h4>
+    ${entry.identityRedaction !== null ? `<p class="muted">Identity redaction: files, symbols, facts, warnings and case title withheld.</p>` : ""}
+    ${lists
+            .map(([label, list]) => `<p>${escapeHtml(label)}: ${escapeHtml(formatRetrievalList(list))}</p>${list && list.displayed.length > 0 ? listHtml(list.displayed) : ""}`)
+            .join("\n    ")}${entry.errors.length > 0 ? `\n    ${listHtml(entry.errors)}` : ""}`;
+    })
+        .join("\n    ")}
+    <h3>Interpretation Limits</h3>
+    ${listHtml(section.limitations)}
+  </section>`;
+}
+function listHtml(items) {
+    return `<ul>${items.map((item) => `<li>${escapeHtml(item)}</li>`).join("")}</ul>`;
+}
+function table(headers, rows) {
+    return `<table><thead><tr>${headers.map((header) => `<th>${escapeHtml(header)}</th>`).join("")}</tr></thead><tbody>${rows
+        .map((row) => `<tr>${row.map((cell) => `<td>${escapeHtml(cell)}</td>`).join("")}</tr>`)
+        .join("")}</tbody></table>`;
+}
+function escapeHtml(value) {
+    return String(value ?? "")
+        .replaceAll("&", "&amp;")
+        .replaceAll("<", "&lt;")
+        .replaceAll(">", "&gt;")
+        .replaceAll('"', "&quot;")
+        .replaceAll("'", "&#39;");
+}