@dailephd/my-dev-kit-lab 0.7.2 → 0.8.1

package/dist/src/report/experiments/renderPluginExperimentReportText.js

@@ -3,6 +3,8 @@ import { isV2ReportSection } from "./buildIncrementalChangeStalenessReportV2.js"
 import { renderIncrementalChangeStalenessSectionV2 } from "./renderIncrementalChangeStalenessTextV2.js";
 import { REINDEX_RECOMMENDATION_EXPLANATIONS } from "./buildWarmIndexReuseReport.js";
 import { renderContextWindowScalingTextLines } from "./renderContextWindowScalingText.js";
+import { renderRetrievalPrecisionRecallTextLines } from "./renderRetrievalPrecisionRecallText.js";
+import { renderRetrievalQueryStrategyComparisonTextLines } from "./renderRetrievalQueryStrategyComparisonText.js";
 function sanitizeScalar(value) {
     const text = String(value);
     let result = "";
@@ -671,6 +673,22 @@ export function renderPluginExperimentReportText(report) {
     else {
         lines.push(...renderContextWindowScalingTextLines(report.contextWindowScaling));
     }
+    pushSection(lines, "Retrieval Precision/Recall");
+    const retrievalPrecisionRecall = report.retrievalPrecisionRecall ?? null;
+    if (retrievalPrecisionRecall === null) {
+        lines.push("Not applicable to this plugin.");
+    }
+    else {
+        lines.push(...renderRetrievalPrecisionRecallTextLines(retrievalPrecisionRecall));
+    }
+    pushSection(lines, "Retrieval Query Strategy Comparison");
+    const retrievalQueryStrategyComparison = report.retrievalQueryStrategyComparison ?? null;
+    if (retrievalQueryStrategyComparison === null) {
+        lines.push("Not applicable to this plugin.");
+    }
+    else {
+        lines.push(...renderRetrievalQueryStrategyComparisonTextLines(retrievalQueryStrategyComparison));
+    }
     pushSection(lines, "Warnings, Skips, And Failures");
     pushDashList(lines, report.findings.map((finding) => `${finding.severity} ${finding.code} ${finding.message} variant=${finding.variantId ?? ""} case=${finding.caseId ?? ""}`));
     pushSection(lines, "Artifacts");

package/dist/src/report/experiments/renderRetrievalPrecisionRecallHtml.js

@@ -0,0 +1,96 @@
+import { RETRIEVAL_PRECISION_RECALL_RATIO_KEYS } from "../../experiments/plugins/retrievalPrecisionRecall/types.js";
+import { RETRIEVAL_IDENTITY_REDACTION_EXPLANATION, formatRetrievalCount, formatRetrievalList, formatRetrievalMean, formatRetrievalMetric, RETRIEVAL_RATIO_LABELS } from "./renderRetrievalPrecisionRecallText.js";
+export function renderRetrievalPrecisionRecallHtml(section) {
+    if (section === null) {
+        return `<section>
+    <h2>Retrieval Precision/Recall</h2>
+    <p>Not applicable to this plugin.</p>
+  </section>`;
+    }
+    const tokens = section.tokenSummary;
+    const occurrences = section.occurrenceSummary;
+    return `<section>
+    <h2>Retrieval Precision/Recall</h2>
+    <p class="muted">Measures which expected files, symbols and facts the my-dev-kit retrieval lifecycle surfaced, and which irrelevant files it surfaced, without agents. It is descriptive and does not rank anything.</p>
+    ${table(["Field", "Value"], [
+        ["Schema", section.schemaVersion],
+        ["Token count method", section.tokenCountMethod ?? "unavailable"],
+        ...Object.entries(section.runSummary).map(([label, value]) => [label, String(value)])
+    ])}
+    ${section.cases.some((entry) => entry.identityRedaction !== null) ? `<p><strong>Identity redaction.</strong> ${escapeHtml(RETRIEVAL_IDENTITY_REDACTION_EXPLANATION)}</p>` : ""}
+    <h3>Macro Averages</h3>
+    <p class="muted">Mean over available per-case values only. Unavailable and not-applicable cases are counted separately and are never treated as zero.</p>
+    ${table(["Metric", "Available", "Unavailable", "Not applicable", "Mean"], RETRIEVAL_PRECISION_RECALL_RATIO_KEYS.map((key) => {
+        const summary = section.ratioSummaries[key];
+        return [RETRIEVAL_RATIO_LABELS[key], String(summary.availableCount), String(summary.unavailableCount), String(summary.notApplicableCount), formatRetrievalMean(summary.meanValue)];
+    }))}
+    <h3>Retrieved Token Summary</h3>
+    <p class="muted">Measured retrieval payload size; not a relevant versus irrelevant token attribution.</p>
+    ${table(["Available", "Unavailable", "Total", "Mean", "Min", "Max"], [[
+            String(tokens.availableCount),
+            String(tokens.unavailableCount),
+            formatRetrievalCount(tokens.totalTokens),
+            formatRetrievalMean(tokens.meanTokens),
+            formatRetrievalCount(tokens.minTokens),
+            formatRetrievalCount(tokens.maxTokens)
+        ]])}
+    <h3>Missed And Irrelevant Occurrence Totals</h3>
+    <p class="muted">Summed per case; identical paths in different cases or projects are not deduplicated.</p>
+    ${table(["Missed files", "Irrelevant files", "Cases with file evidence", "Missed symbols", "Irrelevant symbols", "Cases with symbol evidence", "Uncovered facts", "Cases with fact evidence"], [[
+            String(occurrences.totalMissedFileOccurrences),
+            String(occurrences.totalIrrelevantRetrievedFileOccurrences),
+            String(occurrences.fileOccurrenceEvidenceCaseCount),
+            String(occurrences.totalMissedSymbolOccurrences),
+            String(occurrences.totalIrrelevantRetrievedSymbolOccurrences),
+            String(occurrences.symbolOccurrenceEvidenceCaseCount),
+            String(occurrences.totalUncoveredFactOccurrences),
+            String(occurrences.factOccurrenceEvidenceCaseCount)
+        ]])}
+    <h3>Per-Case Results</h3>
+    ${table(["Case", "Project", "Locality", "Status", "Evidence", ...RETRIEVAL_PRECISION_RECALL_RATIO_KEYS.map((key) => RETRIEVAL_RATIO_LABELS[key]), "Retrieved tokens"], section.cases.map((entry) => [
+        entry.caseId,
+        entry.benchmarkProject,
+        entry.taskLocality ?? "unavailable",
+        entry.status,
+        entry.evidenceAvailability ?? "unavailable",
+        ...RETRIEVAL_PRECISION_RECALL_RATIO_KEYS.map((key) => formatRetrievalMetric(entry.metrics[key])),
+        formatRetrievalCount(entry.retrievedTokenCount)
+    ]))}
+    <h3>Per-Case Missed And Irrelevant Context</h3>
+    <p class="muted">Displayed lists are bounded; totals are exact.</p>
+    ${section.cases
+        .map((entry) => {
+        const lists = [
+            ["Missed files", entry.missedFiles],
+            ["Irrelevant retrieved files", entry.irrelevantRetrievedFiles],
+            ["Missed symbols", entry.missedSymbols],
+            ["Irrelevant retrieved symbols", entry.irrelevantRetrievedSymbols],
+            ["Uncovered fact IDs", entry.uncoveredFactIds]
+        ];
+        return `<h4><code>${escapeHtml(entry.caseId)}</code> (${escapeHtml(entry.status)})</h4>
+    ${entry.identityRedaction !== null ? `<p class="muted">Identity redaction: files, symbols, facts, warnings and case title withheld.</p>` : ""}
+    ${lists
+            .map(([label, list]) => `<p>${escapeHtml(label)}: ${escapeHtml(formatRetrievalList(list))}</p>${list && list.displayed.length > 0 ? listHtml(list.displayed) : ""}`)
+            .join("\n    ")}${entry.errors.length > 0 ? `\n    ${listHtml(entry.errors)}` : ""}`;
+    })
+        .join("\n    ")}
+    <h3>Interpretation Limits</h3>
+    ${listHtml(section.limitations)}
+  </section>`;
+}
+function listHtml(items) {
+    return `<ul>${items.map((item) => `<li>${escapeHtml(item)}</li>`).join("")}</ul>`;
+}
+function table(headers, rows) {
+    return `<table><thead><tr>${headers.map((header) => `<th>${escapeHtml(header)}</th>`).join("")}</tr></thead><tbody>${rows
+        .map((row) => `<tr>${row.map((cell) => `<td>${escapeHtml(cell)}</td>`).join("")}</tr>`)
+        .join("")}</tbody></table>`;
+}
+function escapeHtml(value) {
+    return String(value ?? "")
+        .replaceAll("&", "&amp;")
+        .replaceAll("<", "&lt;")
+        .replaceAll(">", "&gt;")
+        .replaceAll('"', "&quot;")
+        .replaceAll("'", "&#39;");
+}

package/dist/src/report/experiments/renderRetrievalPrecisionRecallText.js

@@ -0,0 +1,121 @@
+import { RETRIEVAL_PRECISION_RECALL_RATIO_KEYS } from "../../experiments/plugins/retrievalPrecisionRecall/types.js";
+export const RETRIEVAL_RATIO_LABELS = {
+    filePrecision: "File precision",
+    fileRecall: "File recall",
+    symbolPrecision: "Symbol precision",
+    symbolRecall: "Symbol recall",
+    factCoverage: "Fact coverage",
+    irrelevantContextRatio: "Irrelevant context ratio"
+};
+// Render-time formatting only; the JSON report keeps exact unrounded values. Unavailable and not-applicable are
+// always named, never shown as a number.
+export function formatRetrievalMetric(metric) {
+    if (metric.availability === "available" && metric.value !== null) {
+        return `${metric.value.toFixed(4)} (${metric.numerator}/${metric.denominator})`;
+    }
+    return `${metric.availability}${metric.reason ? ` (${metric.reason})` : ""}`;
+}
+/** Explains the placeholders; shown only when at least one case was externally projected. */
+export const RETRIEVAL_IDENTITY_REDACTION_EXPLANATION = "External-local run: file, symbol and fact identities, warning text and case titles are withheld. A placeholder such as <redacted file 1> stands for exactly one withheld identity; it is not a file name and not an absence of files. Counts and metric values are exact.";
+export function formatRetrievalMean(value) {
+    return value === null ? "unavailable" : value.toFixed(4);
+}
+export function formatRetrievalCount(value) {
+    return value === null ? "unavailable" : String(value);
+}
+export function formatRetrievalList(list) {
+    if (list === null)
+        return "unavailable";
+    return `${list.totalCount} (displayed ${list.displayedCount}, omitted ${list.omittedCount})`;
+}
+function clean(value) {
+    return String(value).replace(/[\u0000-\u001f\u007f]/g, " ");
+}
+function row(parts) {
+    return `- ${parts.map(([label, value]) => `${label}: ${clean(value)}`).join(" | ")}`;
+}
+/** Lines for the "Retrieval Precision/Recall" section; every value comes from the report model. */
+export function renderRetrievalPrecisionRecallTextLines(section) {
+    const lines = [];
+    lines.push(`Schema: ${clean(section.schemaVersion)}`);
+    lines.push(`Token Count Method: ${clean(section.tokenCountMethod ?? "unavailable")}`);
+    if (section.cases.some((entry) => entry.identityRedaction !== null)) {
+        lines.push(`Identity Redaction: ${RETRIEVAL_IDENTITY_REDACTION_EXPLANATION}`);
+    }
+    lines.push("Run Summary:");
+    for (const [label, value] of Object.entries(section.runSummary)) {
+        lines.push(`${label}: ${value}`);
+    }
+    lines.push("Macro Averages (mean over available per-case values only; unavailable and not-applicable are counted, never zero):");
+    for (const key of RETRIEVAL_PRECISION_RECALL_RATIO_KEYS) {
+        const summary = section.ratioSummaries[key];
+        lines.push(row([
+            ["Metric", RETRIEVAL_RATIO_LABELS[key]],
+            ["Available", summary.availableCount],
+            ["Unavailable", summary.unavailableCount],
+            ["Not Applicable", summary.notApplicableCount],
+            ["Mean", formatRetrievalMean(summary.meanValue)]
+        ]));
+    }
+    lines.push("Retrieved Token Summary (measured payload, not relevant/irrelevant attribution):");
+    const tokens = section.tokenSummary;
+    lines.push(row([
+        ["Available", tokens.availableCount],
+        ["Unavailable", tokens.unavailableCount],
+        ["Total", formatRetrievalCount(tokens.totalTokens)],
+        ["Mean", formatRetrievalMean(tokens.meanTokens)],
+        ["Min", formatRetrievalCount(tokens.minTokens)],
+        ["Max", formatRetrievalCount(tokens.maxTokens)]
+    ]));
+    lines.push("Missed And Irrelevant Occurrence Totals (summed per case; not deduplicated across cases or projects):");
+    const occurrences = section.occurrenceSummary;
+    lines.push(row([
+        ["Missed Files", occurrences.totalMissedFileOccurrences],
+        ["Irrelevant Files", occurrences.totalIrrelevantRetrievedFileOccurrences],
+        ["Cases With File Evidence", occurrences.fileOccurrenceEvidenceCaseCount],
+        ["Missed Symbols", occurrences.totalMissedSymbolOccurrences],
+        ["Irrelevant Symbols", occurrences.totalIrrelevantRetrievedSymbolOccurrences],
+        ["Cases With Symbol Evidence", occurrences.symbolOccurrenceEvidenceCaseCount],
+        ["Uncovered Facts", occurrences.totalUncoveredFactOccurrences],
+        ["Cases With Fact Evidence", occurrences.factOccurrenceEvidenceCaseCount]
+    ]));
+    lines.push("Per-Case Results (one row per case):");
+    for (const entry of section.cases) {
+        lines.push(row([
+            ["Case", entry.caseId],
+            ["Project", entry.benchmarkProject],
+            ["Locality", entry.taskLocality ?? "unavailable"],
+            ["Status", entry.status],
+            ["Evidence", entry.evidenceAvailability ?? "unavailable"],
+            ...RETRIEVAL_PRECISION_RECALL_RATIO_KEYS.map((key) => [RETRIEVAL_RATIO_LABELS[key], formatRetrievalMetric(entry.metrics[key])]),
+            ["Retrieved Tokens", formatRetrievalCount(entry.retrievedTokenCount)]
+        ]));
+    }
+    lines.push("Per-Case Missed And Irrelevant Context (displayed lists are bounded; totals are exact):");
+    for (const entry of section.cases) {
+        lines.push(`Case ${clean(entry.caseId)} (${clean(entry.status)})`);
+        if (entry.identityRedaction !== null)
+            lines.push("  Identity redaction: files, symbols, facts, warnings and case title withheld");
+        const lists = [
+            ["Missed Files", entry.missedFiles],
+            ["Irrelevant Retrieved Files", entry.irrelevantRetrievedFiles],
+            ["Missed Symbols", entry.missedSymbols],
+            ["Irrelevant Retrieved Symbols", entry.irrelevantRetrievedSymbols],
+            ["Uncovered Fact IDs", entry.uncoveredFactIds]
+        ];
+        for (const [label, list] of lists) {
+            lines.push(`  ${label}: ${formatRetrievalList(list)}`);
+            for (const item of list?.displayed ?? []) {
+                lines.push(`    - ${clean(item)}`);
+            }
+        }
+        for (const error of entry.errors) {
+            lines.push(`  Error: ${clean(error)}`);
+        }
+    }
+    lines.push("Interpretation Limits:");
+    for (const limitation of section.limitations) {
+        lines.push(`- ${clean(limitation)}`);
+    }
+    return lines;
+}

package/dist/src/report/experiments/renderRetrievalQueryStrategyComparisonHtml.js

@@ -0,0 +1,96 @@
+import { RETRIEVAL_QUERY_STRATEGY_EXTERNAL_PRIVACY_NOTE, describeRetrievalQueryStrategyScope, formatRetrievalQueryStrategyList, formatRetrievalQueryStrategyMetric, formatRetrievalQueryStrategyNumber } from "./renderRetrievalQueryStrategyComparisonText.js";
+export function renderRetrievalQueryStrategyComparisonHtml(section) {
+    if (section === null)
+        return "";
+    const methodology = section.methodology;
+    const objective = (value, digits) => (value === undefined ? "unavailable" : digits ? value.toFixed(4) : String(value));
+    return `<section>
+    <h2>Retrieval Query Strategy Comparison</h2>
+    <h3>Methodology</h3>
+    ${table(["Field", "Value"], [
+        ["Schema", section.schemaVersion],
+        ["File F1", methodology.fileF1],
+        ["Symbol F1", methodology.symbolF1],
+        ["Aggregation", methodology.aggregation],
+        ["Multi-objective comparison", methodology.multiObjectiveComparison],
+        ["Unique-best rule", methodology.uniqueBestRule],
+        ["Strategy order", section.strategyOrder.join(", ")]
+    ])}
+    ${section.identityRedaction !== null ? `<p><strong>Identity redaction.</strong> ${escapeHtml(RETRIEVAL_QUERY_STRATEGY_EXTERNAL_PRIVACY_NOTE)}</p>` : ""}
+    <h3>Task-Type Comparison</h3>
+    ${table(["Scope", "Cases", "Matched Cases", "Excluded Cases", "Interpretation", "Best Strategy", "Pareto Front"], section.scopes.map((scope) => [
+        scope.scopeId,
+        String(scope.caseCount),
+        String(scope.comparisonCaseCount),
+        String(scope.excludedCaseCount),
+        scope.interpretation,
+        scope.bestStrategyId ?? "none",
+        formatRetrievalQueryStrategyList(scope.paretoFrontStrategyIds)
+    ]))}
+    ${listHtml(section.scopes.map(describeRetrievalQueryStrategyScope))}
+    <h3>Per-Scope Strategy Objectives</h3>
+    ${section.scopes
+        .map((scope) => `<h4>${escapeHtml(scope.scopeId)}</h4>
+    ${table(["Strategy", "Completed", "Partial", "Failed", "Mean File F1", "Mean Symbol F1", "Mean Fact Coverage", "Mean Retrieved Tokens"], scope.strategySummaries.map((summary) => [
+        summary.strategyId,
+        String(summary.completedCaseCount),
+        String(summary.partialCaseCount),
+        String(summary.failedCaseCount),
+        objective(summary.objectives?.meanFileF1, true),
+        objective(summary.objectives?.meanSymbolF1, true),
+        objective(summary.objectives?.meanFactCoverage, true),
+        objective(summary.objectives?.meanRetrievedTokenCount, false)
+    ]))}`)
+        .join("\n    ")}
+    <h3>Per-Case Treatments</h3>
+    ${table([
+        "Case",
+        "Task Type",
+        "Strategy",
+        "Status",
+        "Evidence",
+        "File Precision",
+        "File Recall",
+        "File F1",
+        "Symbol Precision",
+        "Symbol Recall",
+        "Symbol F1",
+        "Fact Coverage",
+        "Irrelevant Context Ratio",
+        "Retrieved Tokens"
+    ], section.cases.flatMap((entry) => entry.treatments.map((treatment) => [
+        entry.caseId,
+        entry.taskLocality ?? "unavailable",
+        treatment.strategyId,
+        treatment.executionStatus,
+        treatment.evidenceAvailability ?? "unavailable",
+        formatRetrievalQueryStrategyMetric(treatment.filePrecision),
+        formatRetrievalQueryStrategyMetric(treatment.fileRecall),
+        formatRetrievalQueryStrategyMetric(treatment.fileF1),
+        formatRetrievalQueryStrategyMetric(treatment.symbolPrecision),
+        formatRetrievalQueryStrategyMetric(treatment.symbolRecall),
+        formatRetrievalQueryStrategyMetric(treatment.symbolF1),
+        formatRetrievalQueryStrategyMetric(treatment.factCoverage),
+        formatRetrievalQueryStrategyMetric(treatment.irrelevantContextRatio),
+        formatRetrievalQueryStrategyNumber(treatment.retrievedTokenCount)
+    ])))}
+    <h3>Interpretation Limits</h3>
+    ${listHtml(section.limitations)}
+  </section>`;
+}
+function listHtml(items) {
+    return `<ul>${items.map((item) => `<li>${escapeHtml(item)}</li>`).join("")}</ul>`;
+}
+function table(headers, rows) {
+    return `<table><thead><tr>${headers.map((header) => `<th>${escapeHtml(header)}</th>`).join("")}</tr></thead><tbody>${rows
+        .map((row) => `<tr>${row.map((cell) => `<td>${escapeHtml(cell)}</td>`).join("")}</tr>`)
+        .join("")}</tbody></table>`;
+}
+function escapeHtml(value) {
+    return String(value ?? "")
+        .replaceAll("&", "&amp;")
+        .replaceAll("<", "&lt;")
+        .replaceAll(">", "&gt;")
+        .replaceAll('"', "&quot;")
+        .replaceAll("'", "&#39;");
+}

package/dist/src/report/experiments/renderRetrievalQueryStrategyComparisonText.js

@@ -0,0 +1,58 @@
+export const RETRIEVAL_QUERY_STRATEGY_EXTERNAL_PRIVACY_NOTE = "External-local report: private file, symbol, fact, semantic-node, warning, and case-title identities are withheld; numeric scientific results and strategy interpretations are preserved.";
+/** Available values keep four decimals; unavailable/not-applicable are never shown as zero. */
+export function formatRetrievalQueryStrategyMetric(metric) {
+    if (metric.availability === "available" && metric.value !== null)
+        return metric.value.toFixed(4);
+    return `${metric.availability}${metric.reason ? ` (${metric.reason})` : ""}`;
+}
+export const formatRetrievalQueryStrategyNumber = (value) => (value === null ? "unavailable" : String(value));
+const formatMean = (value) => (value === undefined ? "unavailable" : value.toFixed(4));
+const formatTokenMean = (value) => (value === undefined ? "unavailable" : String(value));
+export const formatRetrievalQueryStrategyList = (items) => (items.length === 0 ? "none" : items.join(", "));
+/** One human sentence per scope: a best strategy only for a single-member front; otherwise an explicit tradeoff. */
+export function describeRetrievalQueryStrategyScope(scope) {
+    if (scope.interpretation === "unique-best") {
+        return `${scope.scopeId}: unique-best ${scope.bestStrategyId} (single-member Pareto front).`;
+    }
+    if (scope.interpretation === "tradeoff") {
+        return `${scope.scopeId}: No Single Best strategy; Pareto front (nondominated, not ranked): ${formatRetrievalQueryStrategyList(scope.paretoFrontStrategyIds)}.`;
+    }
+    return `${scope.scopeId}: unavailable; no matched complete-case comparison.`;
+}
+export function renderRetrievalQueryStrategyComparisonTextLines(section) {
+    const lines = [];
+    const methodology = section.methodology;
+    lines.push(`Schema: ${section.schemaVersion}`);
+    lines.push(`File F1: ${methodology.fileF1}`);
+    lines.push(`Symbol F1: ${methodology.symbolF1}`);
+    lines.push(`Aggregation: ${methodology.aggregation}`);
+    lines.push(`Multi-objective comparison: ${methodology.multiObjectiveComparison}`);
+    lines.push(`Unique-best rule: ${methodology.uniqueBestRule}`);
+    lines.push(`Strategy order: ${section.strategyOrder.join(", ")}`);
+    if (section.identityRedaction !== null)
+        lines.push(RETRIEVAL_QUERY_STRATEGY_EXTERNAL_PRIVACY_NOTE);
+    lines.push("", "Task-Type Comparison");
+    for (const scope of section.scopes) {
+        lines.push(`- scope=${scope.scopeId} cases=${scope.caseCount} matched=${scope.comparisonCaseCount} excluded=${scope.excludedCaseCount} interpretation=${scope.interpretation} best=${scope.bestStrategyId ?? "none"} pareto=${formatRetrievalQueryStrategyList(scope.paretoFrontStrategyIds)}`);
+    }
+    for (const scope of section.scopes)
+        lines.push(describeRetrievalQueryStrategyScope(scope));
+    lines.push("", "Per-Scope Strategy Objectives");
+    for (const scope of section.scopes) {
+        lines.push(`Scope ${scope.scopeId}:`);
+        for (const summary of scope.strategySummaries) {
+            const objectives = summary.objectives;
+            lines.push(`- ${summary.strategyId} completed=${summary.completedCaseCount} partial=${summary.partialCaseCount} failed=${summary.failedCaseCount} meanFileF1=${formatMean(objectives?.meanFileF1)} meanSymbolF1=${formatMean(objectives?.meanSymbolF1)} meanFactCoverage=${formatMean(objectives?.meanFactCoverage)} meanRetrievedTokens=${formatTokenMean(objectives?.meanRetrievedTokenCount)}`);
+        }
+    }
+    lines.push("", "Per-Case Treatments");
+    for (const entry of section.cases) {
+        for (const treatment of entry.treatments) {
+            lines.push(`- case=${entry.caseId} taskType=${entry.taskLocality ?? "unavailable"} strategy=${treatment.strategyId} status=${treatment.executionStatus} evidence=${treatment.evidenceAvailability ?? "unavailable"} filePrecision=${formatRetrievalQueryStrategyMetric(treatment.filePrecision)} fileRecall=${formatRetrievalQueryStrategyMetric(treatment.fileRecall)} fileF1=${formatRetrievalQueryStrategyMetric(treatment.fileF1)} symbolPrecision=${formatRetrievalQueryStrategyMetric(treatment.symbolPrecision)} symbolRecall=${formatRetrievalQueryStrategyMetric(treatment.symbolRecall)} symbolF1=${formatRetrievalQueryStrategyMetric(treatment.symbolF1)} factCoverage=${formatRetrievalQueryStrategyMetric(treatment.factCoverage)} irrelevantContextRatio=${formatRetrievalQueryStrategyMetric(treatment.irrelevantContextRatio)} retrievedTokens=${formatRetrievalQueryStrategyNumber(treatment.retrievedTokenCount)}`);
+        }
+    }
+    lines.push("", "Interpretation Limits");
+    for (const limitation of section.limitations)
+        lines.push(`- ${limitation}`);
+    return lines;
+}

package/dist/src/report/experiments/retrievalPrecisionRecallReportModel.js

@@ -0,0 +1,3 @@
+export const RETRIEVAL_PRECISION_RECALL_REPORT_SCHEMA_VERSION = "my-dev-kit-lab-retrieval-precision-recall-report-v1";
+/** Fixed human-facing bound for every identity list; the execution artifact keeps the complete lists. */
+export const MAX_RETRIEVAL_REPORT_IDENTITIES = 20;

package/dist/src/report/experiments/retrievalQueryStrategyComparisonReportModel.js

@@ -0,0 +1 @@
+export const RETRIEVAL_QUERY_STRATEGY_COMPARISON_REPORT_SCHEMA_VERSION = "my-dev-kit-lab-retrieval-query-strategy-comparison-report-v1";

package/docs/METRICS.md

@@ -1,6 +1,6 @@
 # Metrics
 
-This document is the canonical metric glossary for my-dev-kit-lab. It defines every metric that appears in benchmark profiles, prompt variants, controlled experiment artifacts, and rendered reports across the registered experiment plugins (`context-strategy-comparison`, `warm-index-reuse`, and `incremental-change-staleness`).
+This document is the canonical metric glossary for my-dev-kit-lab. It defines every metric that appears in benchmark profiles, prompt variants, controlled experiment artifacts, and rendered reports across the registered experiment plugins (`context-strategy-comparison`, `warm-index-reuse`, `incremental-change-staleness`, `context-window-scaling`, `retrieval-precision-recall`, and `retrieval-query-strategy-comparison`).
 
 Related documentation:
 - [ARCHITECTURE.md](ARCHITECTURE.md) — how metrics flow through the pipeline
@@ -620,7 +620,7 @@ The generic `ExperimentMetric` projection emits run-level case and budget counts
 
 The report artifact schema is `my-dev-kit-lab-context-window-scaling-report-v1`. It includes run summaries, budget-treatment summaries, per-treatment context measurements, budget cells, relevant-file summary, and fixed interpretation limits. The execution artifact is `context-window-scaling-execution.json` (`my-dev-kit-lab-context-window-scaling-execution-v1`) and does not embed context text.
 
-**Synthetic-repository runs (v0.7.1).** The synthetic input introduces no new scientific metric: fit, utilization, correctness, success, and omitted-relevant-file semantics, the execution artifact schema, the report schema, and the plot set are identical to the bundled-corpus run. Each generated repository's `synthetic-repository-manifest.json` records generation evidence (per-file byte length, character count, SHA-256, generated-content estimated tokens, aggregate totals, and `repositoryContentIdentity`); these are reproducibility measurements, not context-window outcome metrics. The raw-full-file treatment context is measured by the existing baseline, which adds file headers and separators, so its estimated tokens differ from the manifest's generated-content estimate. Retrieval precision/recall remains v0.8.0 scope.
+**Synthetic-repository runs (v0.7.1).** The synthetic input introduces no new scientific metric: fit, utilization, correctness, success, and omitted-relevant-file semantics, the execution artifact schema, the report schema, and the plot set are identical to the bundled-corpus run. Each generated repository's `synthetic-repository-manifest.json` records generation evidence (per-file byte length, character count, SHA-256, generated-content estimated tokens, aggregate totals, and `repositoryContentIdentity`); these are reproducibility measurements, not context-window outcome metrics. The raw-full-file treatment context is measured by the existing baseline, which adds file headers and separators, so its estimated tokens differ from the manifest's generated-content estimate. Retrieval precision/recall is documented separately in the v0.8.0 section below.
 
 **Local-repository runs (v0.7.2).** Running over an explicitly selected local Git repository introduces no new scientific metric: fit, utilization, correctness, success, and omitted-relevant-file semantics, the execution artifact and report schemas, and the plot set are the same. In this mode file identity lists are replaced by numbered placeholders (`<redacted file N>`) with the same length as the real list, and each treatment carries `fileIdentityRedaction: "redacted"`; a placeholder is a withheld identity, not an empty result, and an empty list or `null` keeps its usual meaning. Operational compatibility with a repository, the 1 MiB file-size limit, and the Git-ignore exclusions are safety policy, not quality measures: they do not imply retrieval precision, recall, ranking quality, or a retrieval winner.
 
@@ -691,3 +691,153 @@ Availability: the first three are always present for a validated incremental tre
 `no-observed-regression-relative-to-full` means only that the executed scenario showed no regression under those two evidence dimensions. It does not establish equivalence or safety and does not show that full refresh is unnecessary. `not-comparable-as-partial-refresh` says the run gives no evidence about the requested partial behavior. There is no winner, ranking, composite score, safety score, safe-to-skip-refresh flag, or stale-risk percentage. Ordinary per-treatment correctness `score` remains the existing deterministic fake-agent evidence.
 
 **Report presentation.** `report.incrementalChangeStaleness` (schema `my-dev-kit-lab-incremental-change-staleness-report-v2`) holds the summary counts (including per-treatment applied-partial, fallback, and classification counts), per-scenario lifecycle, four treatments and three comparisons identified by explicit treatment IDs, and fixed limitations text. Unavailable evidence is written as `unavailable` or `not applicable`, never zero. Historical V1 reports keep their original two-treatment presentation.
+
+## Retrieval precision/recall evidence (v0.8.0)
+
+Status: released (v0.8.0). The `retrieval-precision-recall` plugin measures one retrieval per case (the existing my-dev-kit search, lookup, slice, and source lifecycle) against a deterministic answer key. It invokes no agent, and every value below is calculated by `calculateRetrievalQualityMetrics` and persisted in `retrieval-precision-recall-execution.json`; reports render only the persisted values.
+
+**Availability.** Each ratio metric is in exactly one of three states. `available` carries a numerator, denominator, and value. `not-applicable` means the denominator is zero, so the metric does not describe that case. `unavailable` means required evidence was missing, partial, or invalid, and carries a fixed reason. Unavailable and not-applicable are never converted to zero, and an `available` zero is a real measured zero.
+
+**File precision.** `relevant retrieved files / retrieved files`. Available when the retrieval evidence is trustworthy and at least one file was retrieved. When no file was retrieved it is `not-applicable`, not zero.
+
+**File recall.** `relevant retrieved files / expected files`. An available retrieval that returned nothing against a nonempty expected set has recall `0`. With no expected files it is `not-applicable`.
+
+**Symbol precision and recall.** The same set formulas over exact, case-sensitive unique symbol names: precision is `relevant retrieved symbol names / retrieved symbol names`, and recall is `relevant retrieved symbol names / expected symbol names`. There is no fuzzy or semantic matching, and several retrieved identities with the same name count once.
+
+**Fact coverage.** `covered expected facts / all expected facts`, unweighted. A fact is covered only through the answer key's explicit `factIds`-to-context mappings (`expectedContextTargets` entries that list the fact IDs they support): a target is satisfied when its file was retrieved and each of its symbols was retrieved from that file. A fact mapped to several required targets is covered only when all of them are satisfied (conjunctive). There is no fact-prose matching, no embeddings, and no model judge. If a required fact has no usable mapping, coverage is `unavailable` rather than guessed.
+
+**Irrelevant context ratio.** `irrelevant retrieved files / retrieved files`. It is calculated over file identities only. When both are available it is the diagnostic complement of file precision, so it is not an independent ranking metric. It makes no claim about which tokens were relevant.
+
+**Retrieved token count.** The existing selected retrieval context's `totalEstimatedTokens`, calculated with the repository's character-based estimator (`ceil(characters / 4)`). It is not provider telemetry, not billing tokens, and not a per-file relevant-token attribution.
+
+**Missed context.** Reported as three separate lists, never combined into a mixed-unit score: missed files, missed symbols, and uncovered fact IDs (each with a count).
+
+**Aggregation.** `aggregateRetrievalPrecisionRecall` takes macro means over available per-case values only. An available zero stays in the mean; unavailable and not-applicable cases are counted separately and excluded from it. The aggregate has no weighting, score, winner, ranking, or threshold.
+
+**External-local runs.** Numeric values and counts are identical in external-local mode. File, symbol, and fact identity lists are replaced by numbered placeholders of the same length (`<redacted file N>`, `<redacted symbol N>`, `<redacted fact N>`), case titles and retrieval warning text are replaced by fixed placeholders, and the artifact carries a fixed identity-redaction marker. A placeholder is a withheld identity, not an empty result.
+
+**Not implemented in v0.8.0.** Ranked retrieval metrics such as MRR, NDCG, MAP, and ranked precision; retrieval-strategy comparison and winners; and query-strategy comparison. Strategy comparison was later added by the separate `retrieval-query-strategy-comparison` plugin documented below; ranked metrics remain outside the Lab's scope.
+
+**Sources.** Standard set-based precision and recall come from established information-retrieval evaluation: Manning, Raghavan, and Schütze, *Introduction to Information Retrieval*, "Evaluation of unranked retrieval sets" (Stanford online edition, <https://nlp.stanford.edu/IR-book/html/htmledition/evaluation-of-unranked-retrieval-sets-1.html>), and the National Institute of Standards and Technology's Text REtrieval Conference (TREC) material on relevance judgments, which treats relevance as a judged, answer-key property of a test collection (<https://trec.nist.gov/data/reljudge_eng.html>; program overview at <https://trec.nist.gov/>). Applying those formulas to file identities and exact symbol-name identities is this Lab's deterministic operationalization. Fact coverage (through the Lab's explicit `factIds` mapping contract) and irrelevant context ratio are Lab-specific deterministic diagnostics, not standardized TREC ranking metrics, and neither source defines the `factIds` contract.
+
+See [COMMANDS.md](COMMANDS.md#retrieval-precision-recall-v080), [WORKFLOWS.md](WORKFLOWS.md#retrieval-precision-recall-experiment-v080), and [ARCHITECTURE.md](ARCHITECTURE.md#retrieval-precisionrecall-architecture-v080).
+
+## Retrieval query strategy comparison metrics (v0.8.1)
+
+Status: released (v0.8.1). The `retrieval-query-strategy-comparison` plugin runs seven fixed retrieval strategies for each case, with no agent, and compares them. This section is the authoritative definition of its scientific values. Every value is calculated once by the plugin's analysis and persisted in `retrieval-query-strategy-comparison-analysis.json`; reports and generic metrics present those precomputed values and never recalculate them.
+
+The seven strategies, in canonical order, are `keyword-search`, `symbol-lookup`, `graph-neighborhood`, `source-slice`, `data-model-graph`, `model-view-lineage`, and `combined-graph-guided`.
+
+### Inherited retrieval-quality metrics
+
+v0.8.1 reuses, without change, the v0.8.0 definitions (see [Retrieval precision/recall evidence](#retrieval-precisionrecall-evidence-v080)):
+
+- File precision `|R_file ∩ E_file| / |R_file|` and file recall `|R_file ∩ E_file| / |E_file|`.
+- Symbol precision `|R_symbol ∩ E_symbol| / |R_symbol|` and symbol recall `|R_symbol ∩ E_symbol| / |E_symbol|`, over exact, case-sensitive unique symbol names.
+- Fact coverage: covered expected facts divided by expected facts, using only the answer key's explicit `factIds`-to-context mappings.
+- Irrelevant context ratio `|R_file - E_file| / |R_file|`.
+- Retrieved token count: the measured retrieval payload size, `ceil(character count / 4)` through the existing estimator.
+
+The strategy-neutral retrieval evidence is adapted to the same generic identity-evidence metric owner that scores the v0.8.0 retrieval evidence, so no command provenance changes any numerator, denominator, identity definition, fact mapping, zero-denominator rule, or availability rule. The ratio metrics keep the `available`, `unavailable`, and `not-applicable` states; unavailable and not-applicable values are never converted to zero.
+
+### Balanced file F1
+
+Let `TP_file` be the number of relevant retrieved files, `FP_file` the number of irrelevant retrieved files, and `FN_file` the number of missed files. Then:
+
+```text
+F1_file = 2 * TP_file / (2 * TP_file + FP_file + FN_file)
+```
+
+Availability: if any of the three identity sets is unavailable, F1 is `unavailable` with the reason `quality-identity-sets-unavailable`. If the denominator is zero (no relevant, irrelevant, or missed files at all), F1 is `not-applicable` with the reason `no-positive-or-retrieved-identities`. Otherwise it is `available`, with the numerator `2 * TP_file` and the denominator `2 * TP_file + FP_file + FN_file`.
+
+An important edge case: when nothing is retrieved and the expected relevant set is nonempty, `TP = 0`, `FP = 0`, and `FN > 0`, so `F1 = 0` as an available value. It is not `not-applicable`, because retrieval performance against a nonempty relevant set is measurable. The Lab computes F1 from these counts, not by averaging stored precision and recall.
+
+### Balanced symbol F1
+
+```text
+F1_symbol = 2 * TP_symbol / (2 * TP_symbol + FP_symbol + FN_symbol)
+```
+
+`TP_symbol`, `FP_symbol`, and `FN_symbol` are the numbers of relevant retrieved, irrelevant retrieved, and missed symbols, with the identical availability rules and the exact unique symbol-name identity inherited from v0.8.0. There is no fuzzy name matching, semantic matching, node-ID matching, alias handling, or case folding.
+
+**Source for F1.** Manning, Christopher D.; Raghavan, Prabhakar; Schütze, Hinrich. *Introduction to Information Retrieval*. Cambridge University Press. Chapter 8, "Evaluation in information retrieval." <https://nlp.stanford.edu/IR-book/pdf/08eval.pdf>. The source establishes that balanced F1 is the harmonic mean of precision and recall, `F1 = 2PR / (P + R)`. The count form above is the equivalent implementation form the Lab uses so that an empty retrieval against a nonempty relevant set is well defined. The source does not specify the Lab's `unavailable` and `not-applicable` policy, which is project-owned.
+
+### Matched complete cases
+
+A case participates in cross-strategy aggregation for a scope only when all seven strategies have all four primary objectives available: file F1, symbol F1, fact coverage, and a valid retrieved token count. If any strategy lacks any one of them, the case is excluded from every strategy's aggregate for that scope, so no strategy averages over more cases than another. A treatment's own scientific evidence is still retained when its case is excluded; the rule applies only to cross-strategy aggregation and best-strategy interpretation.
+
+### Macro means
+
+For each scope and strategy, over the matched comparison cases only:
+
+```text
+meanFileF1              = sum(case file F1)           / matched comparison case count
+meanSymbolF1            = sum(case symbol F1)         / matched comparison case count
+meanFactCoverage        = sum(case fact coverage)     / matched comparison case count
+meanRetrievedTokenCount = sum(case retrieved tokens)  / matched comparison case count
+```
+
+Stored values are not rounded. Every included task contributes equally: there is no denominator weighting, no micro averaging, and no weighting by expected files, expected symbols, facts, token count, or task locality.
+
+### Task-type scopes
+
+The scopes, in report order, are `overall`, `localized`, `cross-module`, and `broad-change`. `overall` contains every selected case; the other three reuse the existing `taskLocality` metadata (`localized`, `cross-module`, `broad-change`). A case with null or unknown `taskLocality` participates only in `overall`. The Lab does not infer task type from query text.
+
+### Primary objective vector
+
+For a strategy in a scope the objective vector is:
+
+```text
+O(strategy, scope) = {
+  meanFileF1: maximize,
+  meanSymbolF1: maximize,
+  meanFactCoverage: maximize,
+  meanRetrievedTokenCount: minimize
+}
+```
+
+Only these four objectives determine dominance. File and symbol precision, recall, irrelevant context ratio, duration, and missed counts remain visible diagnostic metrics but are not additional dominance dimensions: F1 already combines precision and recall, and the irrelevant context ratio is algebraically coupled to file precision when both are defined.
+
+### Pareto dominance
+
+Strategy A dominates strategy B if and only if all of the following hold:
+
+```text
+A.meanFileF1 >= B.meanFileF1
+AND A.meanSymbolF1 >= B.meanSymbolF1
+AND A.meanFactCoverage >= B.meanFactCoverage
+AND A.meanRetrievedTokenCount <= B.meanRetrievedTokenCount
+AND at least one of those comparisons is strict
+```
+
+There is no epsilon, no tolerance band, no rounding, and no weighting. Two strategies with identical vectors do not dominate each other. The Pareto front of a scope is every strategy that no other strategy dominates.
+
+**Source for Pareto dominance.** Emmerich, Michael T. M.; Deutz, André H. "A tutorial on multiobjective optimization: fundamentals and evolutionary methods." *Natural Computing* 17, 585–609 (2018). DOI 10.1007/s11047-018-9685-y. <https://link.springer.com/article/10.1007/s11047-018-9685-y>. It is cited only for the general multiobjective and Pareto framework. The Lab's exact four objectives and its unique-best interpretation are project-owned design choices.
+
+### Best-strategy interpretation
+
+This rule is project-owned:
+
+- `comparisonCaseCount = 0`: the interpretation is `unavailable`, the Pareto front is empty, and `bestStrategyId` is `null`.
+- The Pareto front has exactly one strategy: the interpretation is `unique-best` and `bestStrategyId` is that strategy.
+- The Pareto front has more than one strategy: the interpretation is `tradeoff` and `bestStrategyId` is `null`.
+
+No hidden tie breaker exists: token count, F1, strategy order, alphabetical order, and task locality never select a winner. "No single best strategy" is a valid result.
+
+### No composite score or ranking
+
+The plugin does not calculate a weighted composite score, a scalar winner score, a total ordinal ranking, MRR, NDCG, MAP, a statistical significance test, or a confidence interval. No scientifically justified weights exist for combining the four objectives, so the Lab exposes tradeoffs instead of hiding them in an arbitrary scalar. The Pareto-front array is emitted in canonical strategy order for deterministic serialization; that order is not a performance order, and the first entry is not the best strategy.
+
+### External-local runs: science before redaction
+
+In external-local mode the scientific analysis runs first, on the real in-memory file, symbol, and fact identities. Only afterward are file, symbol, fact, semantic-node, warning, and case-title identities replaced for persistence. Privacy placeholders are therefore never used to calculate precision, recall, F1, fact coverage, means, Pareto fronts, or `bestStrategyId`, and every numeric value and strategy interpretation is identical to the unredacted analysis.
+
+### Token caveat
+
+Retrieved token count is `ceil(character count / 4)` through the existing estimator. It is context-size evidence. It is not provider billing telemetry, not tokenizer-exact model usage, and not a relevant-token attribution.
+
+### Where each value lives
+
+`retrieval-query-strategy-comparison-execution.json` holds execution evidence only. `retrieval-query-strategy-comparison-analysis.json` is the scientific truth owner for the metrics, objective vectors, matched scopes, Pareto fronts, and best-strategy interpretation. `report.json`, `report.html`, and `report.txt` present the precomputed analysis and do not recalculate it. The generic experiment metrics (per-treatment values and per-scope objective means) are projections of the same precomputed values.
+
+See [COMMANDS.md](COMMANDS.md#retrieval-query-strategy-comparison-v081), [WORKFLOWS.md](WORKFLOWS.md#retrieval-query-strategy-comparison-experiment-v081), and [ARCHITECTURE.md](ARCHITECTURE.md#retrieval-query-strategy-comparison-architecture-v081).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dailephd/my-dev-kit-lab",
-  "version": "0.7.2",
+  "version": "0.8.1",
   "type": "module",
   "description": "Evidence, benchmark, and evaluation companion for my-dev-kit.",
   "bin": {