@dailephd/my-dev-kit-lab 0.7.2 → 0.8.0

package/dist/src/report/experiments/buildRetrievalPrecisionRecallReport.js

@@ -0,0 +1,83 @@
+import { RETRIEVAL_PRECISION_RECALL_PLUGIN_ID } from "../../experiments/plugins/retrievalPrecisionRecall/metadata.js";
+import { retrievalRatioOf } from "../../experiments/plugins/retrievalPrecisionRecall/metrics.js";
+import { RETRIEVAL_PRECISION_RECALL_RATIO_KEYS } from "../../experiments/plugins/retrievalPrecisionRecall/types.js";
+import { MAX_RETRIEVAL_REPORT_IDENTITIES, RETRIEVAL_PRECISION_RECALL_REPORT_SCHEMA_VERSION } from "./retrievalPrecisionRecallReportModel.js";
+/** Fixed wording; the report states observed evidence only and never ranks or scores anything. */
+export const RETRIEVAL_PRECISION_RECALL_LIMITATIONS = [
+    "File and symbol precision and recall are set-based measures, not ranked metrics.",
+    "Symbol scoring uses exact unique symbol names, because the answer key lists names rather than node identities.",
+    "Fact coverage uses explicit fact-to-context mappings in the answer key; it does not judge generated answers or interpret fact text.",
+    "Irrelevant context ratio is file-identity based and is the complement of file precision when both are available.",
+    "Retrieved token count measures the selected retrieval context payload; it is not a relevant versus irrelevant token attribution.",
+    "The current lifecycle expands only the top search candidate (lookup, slice and source).",
+    "Unavailable and not-applicable values are not zeros and are excluded from macro means.",
+    "The experiment uses no real agents.",
+    "v0.8.0 does not compare retrieval strategies."
+];
+function boundedList(items) {
+    if (items === null)
+        return null;
+    const displayed = items.slice(0, MAX_RETRIEVAL_REPORT_IDENTITIES);
+    return {
+        totalCount: items.length,
+        displayed: [...displayed],
+        displayedCount: displayed.length,
+        omittedCount: items.length - displayed.length
+    };
+}
+/**
+ * Presents the per-case metric results and aggregate already calculated on the run. It performs no precision, recall,
+ * coverage, ratio or mean calculation of its own; identity lists are only bounded for display.
+ */
+export function buildRetrievalPrecisionRecallReport(run) {
+    if (run.pluginId !== RETRIEVAL_PRECISION_RECALL_PLUGIN_ID)
+        return null;
+    const candidate = run;
+    const evidence = candidate.caseExecutionEvidence;
+    const aggregate = candidate.aggregate;
+    if (!evidence || !aggregate) {
+        // A run that failed before producing evidence (for example an unknown case id or a non-self target) still gets the
+        // generic report with its failure; only a non-failed run with missing evidence is a broken report source.
+        if (run.status === "failed")
+            return null;
+        throw new Error("Invalid retrieval-precision-recall report source: case execution evidence and aggregate are required.");
+    }
+    if (aggregate.runSummary.caseCount !== evidence.length) {
+        throw new Error("Invalid retrieval-precision-recall report source: aggregate and case execution evidence are inconsistent.");
+    }
+    const cases = evidence.map((entry) => {
+        const quality = entry.quality;
+        const metrics = Object.fromEntries(RETRIEVAL_PRECISION_RECALL_RATIO_KEYS.map((key) => {
+            const ratio = retrievalRatioOf(quality, key);
+            return [key, { availability: ratio.availability, numerator: ratio.numerator, denominator: ratio.denominator, value: ratio.value, reason: ratio.reason }];
+        }));
+        return {
+            caseId: entry.caseId,
+            caseName: entry.caseName,
+            benchmarkProject: entry.benchmarkProject,
+            taskLocality: entry.taskLocality,
+            status: entry.status,
+            evidenceAvailability: entry.retrieval?.evidenceAvailability ?? null,
+            metrics,
+            retrievedTokenCount: quality?.retrievedTokenCount ?? null,
+            tokenCountMethod: quality?.tokenCountMethod ?? null,
+            missedFiles: boundedList(quality?.file.missedFiles ?? null),
+            irrelevantRetrievedFiles: boundedList(quality?.file.irrelevantRetrievedFiles ?? null),
+            missedSymbols: boundedList(quality?.symbol.missedSymbols ?? null),
+            irrelevantRetrievedSymbols: boundedList(quality?.symbol.irrelevantRetrievedSymbols ?? null),
+            uncoveredFactIds: boundedList(quality?.fact.uncoveredFactIds ?? null),
+            errors: entry.errors.map((error) => `${error.code}: ${error.message}`),
+            identityRedaction: entry.identityRedaction ? { ...entry.identityRedaction } : null
+        };
+    });
+    return {
+        schemaVersion: RETRIEVAL_PRECISION_RECALL_REPORT_SCHEMA_VERSION,
+        tokenCountMethod: evidence.find((entry) => entry.quality?.tokenCountMethod != null)?.quality?.tokenCountMethod ?? null,
+        runSummary: { ...aggregate.runSummary },
+        ratioSummaries: structuredClone(aggregate.ratios),
+        tokenSummary: { ...aggregate.tokens },
+        occurrenceSummary: { ...aggregate.occurrences },
+        cases,
+        limitations: [...RETRIEVAL_PRECISION_RECALL_LIMITATIONS]
+    };
+}

package/dist/src/report/experiments/index.js

@@ -24,3 +24,7 @@ export * from "./contextWindowScalingReportModel.js";
 export { buildContextWindowScalingReport, CONTEXT_WINDOW_SCALING_INTERPRETATION } from "./buildContextWindowScalingReport.js";
 export { renderContextWindowScalingHtml } from "./renderContextWindowScalingHtml.js";
 export { renderContextWindowScalingTextLines } from "./renderContextWindowScalingText.js";
+export * from "./retrievalPrecisionRecallReportModel.js";
+export { buildRetrievalPrecisionRecallReport, RETRIEVAL_PRECISION_RECALL_LIMITATIONS } from "./buildRetrievalPrecisionRecallReport.js";
+export { renderRetrievalPrecisionRecallHtml } from "./renderRetrievalPrecisionRecallHtml.js";
+export { renderRetrievalPrecisionRecallTextLines } from "./renderRetrievalPrecisionRecallText.js";

package/dist/src/report/experiments/renderPluginExperimentReportHtml.js

@@ -2,6 +2,7 @@ import { renderContextStrategyComparisonV043Html } from "./renderContextStrategy
 import { renderWarmIndexReuseHtml } from "./renderWarmIndexReuseHtml.js";
 import { renderIncrementalChangeStalenessHtml } from "./renderIncrementalChangeStalenessHtml.js";
 import { renderContextWindowScalingHtml } from "./renderContextWindowScalingHtml.js";
+import { renderRetrievalPrecisionRecallHtml } from "./renderRetrievalPrecisionRecallHtml.js";
 export function renderPluginExperimentReportHtml(report) {
     const redactedTarget = report.target.privacyProjection === "external-local-redacted";
     return `<!DOCTYPE html>
@@ -106,6 +107,8 @@ export function renderPluginExperimentReportHtml(report) {
 
   ${renderContextWindowScalingHtml(report.contextWindowScaling)}
 
+  ${renderRetrievalPrecisionRecallHtml(report.retrievalPrecisionRecall ?? null)}
+
   <section>
     <h2>Warnings, Skips, And Failures</h2>
     ${report.findings.length ? table(["Severity", "Code", "Message", "Variant", "Case"], report.findings.map((finding) => [

package/dist/src/report/experiments/renderPluginExperimentReportText.js

@@ -3,6 +3,7 @@ import { isV2ReportSection } from "./buildIncrementalChangeStalenessReportV2.js"
 import { renderIncrementalChangeStalenessSectionV2 } from "./renderIncrementalChangeStalenessTextV2.js";
 import { REINDEX_RECOMMENDATION_EXPLANATIONS } from "./buildWarmIndexReuseReport.js";
 import { renderContextWindowScalingTextLines } from "./renderContextWindowScalingText.js";
+import { renderRetrievalPrecisionRecallTextLines } from "./renderRetrievalPrecisionRecallText.js";
 function sanitizeScalar(value) {
     const text = String(value);
     let result = "";
@@ -671,6 +672,14 @@ export function renderPluginExperimentReportText(report) {
     else {
         lines.push(...renderContextWindowScalingTextLines(report.contextWindowScaling));
     }
+    pushSection(lines, "Retrieval Precision/Recall");
+    const retrievalPrecisionRecall = report.retrievalPrecisionRecall ?? null;
+    if (retrievalPrecisionRecall === null) {
+        lines.push("Not applicable to this plugin.");
+    }
+    else {
+        lines.push(...renderRetrievalPrecisionRecallTextLines(retrievalPrecisionRecall));
+    }
     pushSection(lines, "Warnings, Skips, And Failures");
     pushDashList(lines, report.findings.map((finding) => `${finding.severity} ${finding.code} ${finding.message} variant=${finding.variantId ?? ""} case=${finding.caseId ?? ""}`));
     pushSection(lines, "Artifacts");

package/dist/src/report/experiments/renderRetrievalPrecisionRecallHtml.js

@@ -0,0 +1,96 @@
+import { RETRIEVAL_PRECISION_RECALL_RATIO_KEYS } from "../../experiments/plugins/retrievalPrecisionRecall/types.js";
+import { RETRIEVAL_IDENTITY_REDACTION_EXPLANATION, formatRetrievalCount, formatRetrievalList, formatRetrievalMean, formatRetrievalMetric, RETRIEVAL_RATIO_LABELS } from "./renderRetrievalPrecisionRecallText.js";
+export function renderRetrievalPrecisionRecallHtml(section) {
+    if (section === null) {
+        return `<section>
+    <h2>Retrieval Precision/Recall</h2>
+    <p>Not applicable to this plugin.</p>
+  </section>`;
+    }
+    const tokens = section.tokenSummary;
+    const occurrences = section.occurrenceSummary;
+    return `<section>
+    <h2>Retrieval Precision/Recall</h2>
+    <p class="muted">Measures which expected files, symbols and facts the my-dev-kit retrieval lifecycle surfaced, and which irrelevant files it surfaced, without agents. It is descriptive and does not rank anything.</p>
+    ${table(["Field", "Value"], [
+        ["Schema", section.schemaVersion],
+        ["Token count method", section.tokenCountMethod ?? "unavailable"],
+        ...Object.entries(section.runSummary).map(([label, value]) => [label, String(value)])
+    ])}
+    ${section.cases.some((entry) => entry.identityRedaction !== null) ? `<p><strong>Identity redaction.</strong> ${escapeHtml(RETRIEVAL_IDENTITY_REDACTION_EXPLANATION)}</p>` : ""}
+    <h3>Macro Averages</h3>
+    <p class="muted">Mean over available per-case values only. Unavailable and not-applicable cases are counted separately and are never treated as zero.</p>
+    ${table(["Metric", "Available", "Unavailable", "Not applicable", "Mean"], RETRIEVAL_PRECISION_RECALL_RATIO_KEYS.map((key) => {
+        const summary = section.ratioSummaries[key];
+        return [RETRIEVAL_RATIO_LABELS[key], String(summary.availableCount), String(summary.unavailableCount), String(summary.notApplicableCount), formatRetrievalMean(summary.meanValue)];
+    }))}
+    <h3>Retrieved Token Summary</h3>
+    <p class="muted">Measured retrieval payload size; not a relevant versus irrelevant token attribution.</p>
+    ${table(["Available", "Unavailable", "Total", "Mean", "Min", "Max"], [[
+            String(tokens.availableCount),
+            String(tokens.unavailableCount),
+            formatRetrievalCount(tokens.totalTokens),
+            formatRetrievalMean(tokens.meanTokens),
+            formatRetrievalCount(tokens.minTokens),
+            formatRetrievalCount(tokens.maxTokens)
+        ]])}
+    <h3>Missed And Irrelevant Occurrence Totals</h3>
+    <p class="muted">Summed per case; identical paths in different cases or projects are not deduplicated.</p>
+    ${table(["Missed files", "Irrelevant files", "Cases with file evidence", "Missed symbols", "Irrelevant symbols", "Cases with symbol evidence", "Uncovered facts", "Cases with fact evidence"], [[
+            String(occurrences.totalMissedFileOccurrences),
+            String(occurrences.totalIrrelevantRetrievedFileOccurrences),
+            String(occurrences.fileOccurrenceEvidenceCaseCount),
+            String(occurrences.totalMissedSymbolOccurrences),
+            String(occurrences.totalIrrelevantRetrievedSymbolOccurrences),
+            String(occurrences.symbolOccurrenceEvidenceCaseCount),
+            String(occurrences.totalUncoveredFactOccurrences),
+            String(occurrences.factOccurrenceEvidenceCaseCount)
+        ]])}
+    <h3>Per-Case Results</h3>
+    ${table(["Case", "Project", "Locality", "Status", "Evidence", ...RETRIEVAL_PRECISION_RECALL_RATIO_KEYS.map((key) => RETRIEVAL_RATIO_LABELS[key]), "Retrieved tokens"], section.cases.map((entry) => [
+        entry.caseId,
+        entry.benchmarkProject,
+        entry.taskLocality ?? "unavailable",
+        entry.status,
+        entry.evidenceAvailability ?? "unavailable",
+        ...RETRIEVAL_PRECISION_RECALL_RATIO_KEYS.map((key) => formatRetrievalMetric(entry.metrics[key])),
+        formatRetrievalCount(entry.retrievedTokenCount)
+    ]))}
+    <h3>Per-Case Missed And Irrelevant Context</h3>
+    <p class="muted">Displayed lists are bounded; totals are exact.</p>
+    ${section.cases
+        .map((entry) => {
+        const lists = [
+            ["Missed files", entry.missedFiles],
+            ["Irrelevant retrieved files", entry.irrelevantRetrievedFiles],
+            ["Missed symbols", entry.missedSymbols],
+            ["Irrelevant retrieved symbols", entry.irrelevantRetrievedSymbols],
+            ["Uncovered fact IDs", entry.uncoveredFactIds]
+        ];
+        return `<h4><code>${escapeHtml(entry.caseId)}</code> (${escapeHtml(entry.status)})</h4>
+    ${entry.identityRedaction !== null ? `<p class="muted">Identity redaction: files, symbols, facts, warnings and case title withheld.</p>` : ""}
+    ${lists
+            .map(([label, list]) => `<p>${escapeHtml(label)}: ${escapeHtml(formatRetrievalList(list))}</p>${list && list.displayed.length > 0 ? listHtml(list.displayed) : ""}`)
+            .join("\n    ")}${entry.errors.length > 0 ? `\n    ${listHtml(entry.errors)}` : ""}`;
+    })
+        .join("\n    ")}
+    <h3>Interpretation Limits</h3>
+    ${listHtml(section.limitations)}
+  </section>`;
+}
+function listHtml(items) {
+    return `<ul>${items.map((item) => `<li>${escapeHtml(item)}</li>`).join("")}</ul>`;
+}
+function table(headers, rows) {
+    return `<table><thead><tr>${headers.map((header) => `<th>${escapeHtml(header)}</th>`).join("")}</tr></thead><tbody>${rows
+        .map((row) => `<tr>${row.map((cell) => `<td>${escapeHtml(cell)}</td>`).join("")}</tr>`)
+        .join("")}</tbody></table>`;
+}
+function escapeHtml(value) {
+    return String(value ?? "")
+        .replaceAll("&", "&amp;")
+        .replaceAll("<", "&lt;")
+        .replaceAll(">", "&gt;")
+        .replaceAll('"', "&quot;")
+        .replaceAll("'", "&#39;");
+}

package/dist/src/report/experiments/renderRetrievalPrecisionRecallText.js

@@ -0,0 +1,121 @@
+import { RETRIEVAL_PRECISION_RECALL_RATIO_KEYS } from "../../experiments/plugins/retrievalPrecisionRecall/types.js";
+export const RETRIEVAL_RATIO_LABELS = {
+    filePrecision: "File precision",
+    fileRecall: "File recall",
+    symbolPrecision: "Symbol precision",
+    symbolRecall: "Symbol recall",
+    factCoverage: "Fact coverage",
+    irrelevantContextRatio: "Irrelevant context ratio"
+};
+// Render-time formatting only; the JSON report keeps exact unrounded values. Unavailable and not-applicable are
+// always named, never shown as a number.
+export function formatRetrievalMetric(metric) {
+    if (metric.availability === "available" && metric.value !== null) {
+        return `${metric.value.toFixed(4)} (${metric.numerator}/${metric.denominator})`;
+    }
+    return `${metric.availability}${metric.reason ? ` (${metric.reason})` : ""}`;
+}
+/** Explains the placeholders; shown only when at least one case was externally projected. */
+export const RETRIEVAL_IDENTITY_REDACTION_EXPLANATION = "External-local run: file, symbol and fact identities, warning text and case titles are withheld. A placeholder such as <redacted file 1> stands for exactly one withheld identity; it is not a file name and not an absence of files. Counts and metric values are exact.";
+export function formatRetrievalMean(value) {
+    return value === null ? "unavailable" : value.toFixed(4);
+}
+export function formatRetrievalCount(value) {
+    return value === null ? "unavailable" : String(value);
+}
+export function formatRetrievalList(list) {
+    if (list === null)
+        return "unavailable";
+    return `${list.totalCount} (displayed ${list.displayedCount}, omitted ${list.omittedCount})`;
+}
+function clean(value) {
+    return String(value).replace(/[\u0000-\u001f\u007f]/g, " ");
+}
+function row(parts) {
+    return `- ${parts.map(([label, value]) => `${label}: ${clean(value)}`).join(" | ")}`;
+}
+/** Lines for the "Retrieval Precision/Recall" section; every value comes from the report model. */
+export function renderRetrievalPrecisionRecallTextLines(section) {
+    const lines = [];
+    lines.push(`Schema: ${clean(section.schemaVersion)}`);
+    lines.push(`Token Count Method: ${clean(section.tokenCountMethod ?? "unavailable")}`);
+    if (section.cases.some((entry) => entry.identityRedaction !== null)) {
+        lines.push(`Identity Redaction: ${RETRIEVAL_IDENTITY_REDACTION_EXPLANATION}`);
+    }
+    lines.push("Run Summary:");
+    for (const [label, value] of Object.entries(section.runSummary)) {
+        lines.push(`${label}: ${value}`);
+    }
+    lines.push("Macro Averages (mean over available per-case values only; unavailable and not-applicable are counted, never zero):");
+    for (const key of RETRIEVAL_PRECISION_RECALL_RATIO_KEYS) {
+        const summary = section.ratioSummaries[key];
+        lines.push(row([
+            ["Metric", RETRIEVAL_RATIO_LABELS[key]],
+            ["Available", summary.availableCount],
+            ["Unavailable", summary.unavailableCount],
+            ["Not Applicable", summary.notApplicableCount],
+            ["Mean", formatRetrievalMean(summary.meanValue)]
+        ]));
+    }
+    lines.push("Retrieved Token Summary (measured payload, not relevant/irrelevant attribution):");
+    const tokens = section.tokenSummary;
+    lines.push(row([
+        ["Available", tokens.availableCount],
+        ["Unavailable", tokens.unavailableCount],
+        ["Total", formatRetrievalCount(tokens.totalTokens)],
+        ["Mean", formatRetrievalMean(tokens.meanTokens)],
+        ["Min", formatRetrievalCount(tokens.minTokens)],
+        ["Max", formatRetrievalCount(tokens.maxTokens)]
+    ]));
+    lines.push("Missed And Irrelevant Occurrence Totals (summed per case; not deduplicated across cases or projects):");
+    const occurrences = section.occurrenceSummary;
+    lines.push(row([
+        ["Missed Files", occurrences.totalMissedFileOccurrences],
+        ["Irrelevant Files", occurrences.totalIrrelevantRetrievedFileOccurrences],
+        ["Cases With File Evidence", occurrences.fileOccurrenceEvidenceCaseCount],
+        ["Missed Symbols", occurrences.totalMissedSymbolOccurrences],
+        ["Irrelevant Symbols", occurrences.totalIrrelevantRetrievedSymbolOccurrences],
+        ["Cases With Symbol Evidence", occurrences.symbolOccurrenceEvidenceCaseCount],
+        ["Uncovered Facts", occurrences.totalUncoveredFactOccurrences],
+        ["Cases With Fact Evidence", occurrences.factOccurrenceEvidenceCaseCount]
+    ]));
+    lines.push("Per-Case Results (one row per case):");
+    for (const entry of section.cases) {
+        lines.push(row([
+            ["Case", entry.caseId],
+            ["Project", entry.benchmarkProject],
+            ["Locality", entry.taskLocality ?? "unavailable"],
+            ["Status", entry.status],
+            ["Evidence", entry.evidenceAvailability ?? "unavailable"],
+            ...RETRIEVAL_PRECISION_RECALL_RATIO_KEYS.map((key) => [RETRIEVAL_RATIO_LABELS[key], formatRetrievalMetric(entry.metrics[key])]),
+            ["Retrieved Tokens", formatRetrievalCount(entry.retrievedTokenCount)]
+        ]));
+    }
+    lines.push("Per-Case Missed And Irrelevant Context (displayed lists are bounded; totals are exact):");
+    for (const entry of section.cases) {
+        lines.push(`Case ${clean(entry.caseId)} (${clean(entry.status)})`);
+        if (entry.identityRedaction !== null)
+            lines.push("  Identity redaction: files, symbols, facts, warnings and case title withheld");
+        const lists = [
+            ["Missed Files", entry.missedFiles],
+            ["Irrelevant Retrieved Files", entry.irrelevantRetrievedFiles],
+            ["Missed Symbols", entry.missedSymbols],
+            ["Irrelevant Retrieved Symbols", entry.irrelevantRetrievedSymbols],
+            ["Uncovered Fact IDs", entry.uncoveredFactIds]
+        ];
+        for (const [label, list] of lists) {
+            lines.push(`  ${label}: ${formatRetrievalList(list)}`);
+            for (const item of list?.displayed ?? []) {
+                lines.push(`    - ${clean(item)}`);
+            }
+        }
+        for (const error of entry.errors) {
+            lines.push(`  Error: ${clean(error)}`);
+        }
+    }
+    lines.push("Interpretation Limits:");
+    for (const limitation of section.limitations) {
+        lines.push(`- ${clean(limitation)}`);
+    }
+    return lines;
+}

package/dist/src/report/experiments/retrievalPrecisionRecallReportModel.js

@@ -0,0 +1,3 @@
+export const RETRIEVAL_PRECISION_RECALL_REPORT_SCHEMA_VERSION = "my-dev-kit-lab-retrieval-precision-recall-report-v1";
+/** Fixed human-facing bound for every identity list; the execution artifact keeps the complete lists. */
+export const MAX_RETRIEVAL_REPORT_IDENTITIES = 20;

package/docs/METRICS.md

@@ -1,6 +1,6 @@
 # Metrics
 
-This document is the canonical metric glossary for my-dev-kit-lab. It defines every metric that appears in benchmark profiles, prompt variants, controlled experiment artifacts, and rendered reports across the registered experiment plugins (`context-strategy-comparison`, `warm-index-reuse`, and `incremental-change-staleness`).
+This document is the canonical metric glossary for my-dev-kit-lab. It defines every metric that appears in benchmark profiles, prompt variants, controlled experiment artifacts, and rendered reports across the registered experiment plugins (`context-strategy-comparison`, `warm-index-reuse`, `incremental-change-staleness`, `context-window-scaling`, and `retrieval-precision-recall`).
 
 Related documentation:
 - [ARCHITECTURE.md](ARCHITECTURE.md) — how metrics flow through the pipeline
@@ -620,7 +620,7 @@ The generic `ExperimentMetric` projection emits run-level case and budget counts
 
 The report artifact schema is `my-dev-kit-lab-context-window-scaling-report-v1`. It includes run summaries, budget-treatment summaries, per-treatment context measurements, budget cells, relevant-file summary, and fixed interpretation limits. The execution artifact is `context-window-scaling-execution.json` (`my-dev-kit-lab-context-window-scaling-execution-v1`) and does not embed context text.
 
-**Synthetic-repository runs (v0.7.1).** The synthetic input introduces no new scientific metric: fit, utilization, correctness, success, and omitted-relevant-file semantics, the execution artifact schema, the report schema, and the plot set are identical to the bundled-corpus run. Each generated repository's `synthetic-repository-manifest.json` records generation evidence (per-file byte length, character count, SHA-256, generated-content estimated tokens, aggregate totals, and `repositoryContentIdentity`); these are reproducibility measurements, not context-window outcome metrics. The raw-full-file treatment context is measured by the existing baseline, which adds file headers and separators, so its estimated tokens differ from the manifest's generated-content estimate. Retrieval precision/recall remains v0.8.0 scope.
+**Synthetic-repository runs (v0.7.1).** The synthetic input introduces no new scientific metric: fit, utilization, correctness, success, and omitted-relevant-file semantics, the execution artifact schema, the report schema, and the plot set are identical to the bundled-corpus run. Each generated repository's `synthetic-repository-manifest.json` records generation evidence (per-file byte length, character count, SHA-256, generated-content estimated tokens, aggregate totals, and `repositoryContentIdentity`); these are reproducibility measurements, not context-window outcome metrics. The raw-full-file treatment context is measured by the existing baseline, which adds file headers and separators, so its estimated tokens differ from the manifest's generated-content estimate. Retrieval precision/recall is documented separately in the v0.8.0 section below.
 
 **Local-repository runs (v0.7.2).** Running over an explicitly selected local Git repository introduces no new scientific metric: fit, utilization, correctness, success, and omitted-relevant-file semantics, the execution artifact and report schemas, and the plot set are the same. In this mode file identity lists are replaced by numbered placeholders (`<redacted file N>`) with the same length as the real list, and each treatment carries `fileIdentityRedaction: "redacted"`; a placeholder is a withheld identity, not an empty result, and an empty list or `null` keeps its usual meaning. Operational compatibility with a repository, the 1 MiB file-size limit, and the Git-ignore exclusions are safety policy, not quality measures: they do not imply retrieval precision, recall, ranking quality, or a retrieval winner.
 
@@ -691,3 +691,33 @@ Availability: the first three are always present for a validated incremental tre
 `no-observed-regression-relative-to-full` means only that the executed scenario showed no regression under those two evidence dimensions. It does not establish equivalence or safety and does not show that full refresh is unnecessary. `not-comparable-as-partial-refresh` says the run gives no evidence about the requested partial behavior. There is no winner, ranking, composite score, safety score, safe-to-skip-refresh flag, or stale-risk percentage. Ordinary per-treatment correctness `score` remains the existing deterministic fake-agent evidence.
 
 **Report presentation.** `report.incrementalChangeStaleness` (schema `my-dev-kit-lab-incremental-change-staleness-report-v2`) holds the summary counts (including per-treatment applied-partial, fallback, and classification counts), per-scenario lifecycle, four treatments and three comparisons identified by explicit treatment IDs, and fixed limitations text. Unavailable evidence is written as `unavailable` or `not applicable`, never zero. Historical V1 reports keep their original two-treatment presentation.
+
+## Retrieval precision/recall evidence (v0.8.0)
+
+Status: released (v0.8.0). The `retrieval-precision-recall` plugin measures one retrieval per case (the existing my-dev-kit search, lookup, slice, and source lifecycle) against a deterministic answer key. It invokes no agent, and every value below is calculated by `calculateRetrievalQualityMetrics` and persisted in `retrieval-precision-recall-execution.json`; reports render only the persisted values.
+
+**Availability.** Each ratio metric is in exactly one of three states. `available` carries a numerator, denominator, and value. `not-applicable` means the denominator is zero, so the metric does not describe that case. `unavailable` means required evidence was missing, partial, or invalid, and carries a fixed reason. Unavailable and not-applicable are never converted to zero, and an `available` zero is a real measured zero.
+
+**File precision.** `relevant retrieved files / retrieved files`. Available when the retrieval evidence is trustworthy and at least one file was retrieved. When no file was retrieved it is `not-applicable`, not zero.
+
+**File recall.** `relevant retrieved files / expected files`. An available retrieval that returned nothing against a nonempty expected set has recall `0`. With no expected files it is `not-applicable`.
+
+**Symbol precision and recall.** The same set formulas over exact, case-sensitive unique symbol names: precision is `relevant retrieved symbol names / retrieved symbol names`, and recall is `relevant retrieved symbol names / expected symbol names`. There is no fuzzy or semantic matching, and several retrieved identities with the same name count once.
+
+**Fact coverage.** `covered expected facts / all expected facts`, unweighted. A fact is covered only through the answer key's explicit `factIds`-to-context mappings (`expectedContextTargets` entries that list the fact IDs they support): a target is satisfied when its file was retrieved and each of its symbols was retrieved from that file. A fact mapped to several required targets is covered only when all of them are satisfied (conjunctive). There is no fact-prose matching, no embeddings, and no model judge. If a required fact has no usable mapping, coverage is `unavailable` rather than guessed.
+
+**Irrelevant context ratio.** `irrelevant retrieved files / retrieved files`. It is calculated over file identities only. When both are available it is the diagnostic complement of file precision, so it is not an independent ranking metric. It makes no claim about which tokens were relevant.
+
+**Retrieved token count.** The existing selected retrieval context's `totalEstimatedTokens`, calculated with the repository's character-based estimator (`ceil(characters / 4)`). It is not provider telemetry, not billing tokens, and not a per-file relevant-token attribution.
+
+**Missed context.** Reported as three separate lists, never combined into a mixed-unit score: missed files, missed symbols, and uncovered fact IDs (each with a count).
+
+**Aggregation.** `aggregateRetrievalPrecisionRecall` takes macro means over available per-case values only. An available zero stays in the mean; unavailable and not-applicable cases are counted separately and excluded from it. The aggregate has no weighting, score, winner, ranking, or threshold.
+
+**External-local runs.** Numeric values and counts are identical in external-local mode. File, symbol, and fact identity lists are replaced by numbered placeholders of the same length (`<redacted file N>`, `<redacted symbol N>`, `<redacted fact N>`), case titles and retrieval warning text are replaced by fixed placeholders, and the artifact carries a fixed identity-redaction marker. A placeholder is a withheld identity, not an empty result.
+
+**Not implemented in v0.8.0.** Ranked retrieval metrics such as MRR, NDCG, MAP, and ranked precision; retrieval-strategy comparison and winners; and query-strategy comparison. Those remain future scope.
+
+**Sources.** Standard set-based precision and recall come from established information-retrieval evaluation: Manning, Raghavan, and Schütze, *Introduction to Information Retrieval*, "Evaluation of unranked retrieval sets" (Stanford online edition, <https://nlp.stanford.edu/IR-book/html/htmledition/evaluation-of-unranked-retrieval-sets-1.html>), and the National Institute of Standards and Technology's Text REtrieval Conference (TREC) material on relevance judgments, which treats relevance as a judged, answer-key property of a test collection (<https://trec.nist.gov/data/reljudge_eng.html>; program overview at <https://trec.nist.gov/>). Applying those formulas to file identities and exact symbol-name identities is this Lab's deterministic operationalization. Fact coverage (through the Lab's explicit `factIds` mapping contract) and irrelevant context ratio are Lab-specific deterministic diagnostics, not standardized TREC ranking metrics, and neither source defines the `factIds` contract.
+
+See [COMMANDS.md](COMMANDS.md#retrieval-precision-recall-v080), [WORKFLOWS.md](WORKFLOWS.md#retrieval-precision-recall-experiment-v080), and [ARCHITECTURE.md](ARCHITECTURE.md#retrieval-precisionrecall-architecture-v080).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dailephd/my-dev-kit-lab",
-  "version": "0.7.2",
+  "version": "0.8.0",
   "type": "module",
   "description": "Evidence, benchmark, and evaluation companion for my-dev-kit.",
   "bin": {