@dailephd/my-dev-kit-lab 0.7.2 → 0.8.1

package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/metrics.js

@@ -0,0 +1,60 @@
+import { RETRIEVAL_QUERY_STRATEGY_IDS } from "../../../evaluation/retrievalQueryStrategies.js";
+/** Projects already-calculated scientific values onto generic metrics. Owns no formula. */
+export function toRetrievalQueryStrategyOutcomeMetrics(treatment, caseId) {
+    const base = { variantId: treatment.strategyId, caseId };
+    const quality = treatment.quality;
+    const ratio = (id, name, metric) => ({
+        ...base,
+        id,
+        name,
+        value: metric?.value ?? null,
+        unit: "ratio"
+    });
+    const count = (id, name, value) => ({
+        ...base,
+        id,
+        name,
+        value: value ?? null,
+        unit: "count"
+    });
+    return [
+        ratio("file-precision", "File precision", quality?.file.precision),
+        ratio("file-recall", "File recall", quality?.file.recall),
+        ratio("file-f1", "File F1", treatment.fileF1),
+        ratio("symbol-precision", "Symbol precision", quality?.symbol.precision),
+        ratio("symbol-recall", "Symbol recall", quality?.symbol.recall),
+        ratio("symbol-f1", "Symbol F1", treatment.symbolF1),
+        ratio("fact-coverage", "Fact coverage", quality?.fact.coverage),
+        ratio("irrelevant-context-ratio", "Irrelevant context ratio", quality?.irrelevantContextRatio),
+        { ...base, id: "retrieved-token-count", name: "Retrieved token count", value: quality?.retrievedTokenCount ?? null, unit: "tokens" },
+        count("missed-file-count", "Missed file count", quality?.file.missedFileCount),
+        count("missed-symbol-count", "Missed symbol count", quality?.symbol.missedSymbolCount),
+        count("uncovered-fact-count", "Uncovered fact count", quality?.fact.uncoveredFactCount),
+        count("irrelevant-retrieved-file-count", "Irrelevant retrieved file count", quality?.file.irrelevantRetrievedFiles?.length),
+        count("irrelevant-retrieved-symbol-count", "Irrelevant retrieved symbol count", quality?.symbol.irrelevantRetrievedSymbols?.length)
+    ];
+}
+/** Per scope: one matched-case count, then four objective means per strategy in canonical order. No best/rank/score metric. */
+export function toRetrievalQueryStrategyRunMetrics(analysis) {
+    const metrics = [];
+    for (const scope of analysis.scopes) {
+        const { scopeId } = scope;
+        metrics.push({
+            id: `${scopeId}-comparison-case-count`,
+            name: `${scopeId} matched comparison cases`,
+            value: scope.comparisonCaseCount,
+            unit: "count"
+        });
+        for (const strategyId of RETRIEVAL_QUERY_STRATEGY_IDS) {
+            const objectives = scope.strategySummaries.find((summary) => summary.strategyId === strategyId)?.objectives ?? null;
+            metrics.push({ id: `${scopeId}-mean-file-f1`, name: `${scopeId} mean file F1`, value: objectives?.meanFileF1 ?? null, unit: "ratio", variantId: strategyId }, { id: `${scopeId}-mean-symbol-f1`, name: `${scopeId} mean symbol F1`, value: objectives?.meanSymbolF1 ?? null, unit: "ratio", variantId: strategyId }, { id: `${scopeId}-mean-fact-coverage`, name: `${scopeId} mean fact coverage`, value: objectives?.meanFactCoverage ?? null, unit: "ratio", variantId: strategyId }, {
+                id: `${scopeId}-mean-retrieved-token-count`,
+                name: `${scopeId} mean retrieved token count`,
+                value: objectives?.meanRetrievedTokenCount ?? null,
+                unit: "tokens",
+                variantId: strategyId
+            });
+        }
+    }
+    return metrics;
+}

package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/plugin.js

@@ -0,0 +1,292 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { serializeLocalRepositorySubjectManifest } from "../../../evaluation/localRepositorySubject/index.js";
+import { summarizeExperimentRun } from "../../results.js";
+import { LocalSubjectExecutionError } from "../contextWindowScaling/localSubjectErrors.js";
+import { projectExternalLocalTarget } from "../contextWindowScaling/localSubjectPrivacy.js";
+import { isSamePhysicalDirectory } from "../contextWindowScaling/localSubjectScratch.js";
+import { selectWarmIndexCases } from "../warmIndexReuse/selection.js";
+import { defaultRetrievalQueryStrategyComparisonConfig, retrievalQueryStrategyComparisonConfigDefinition, validateRetrievalQueryStrategyComparisonConfig } from "./config.js";
+import { analyzeRetrievalQueryStrategyComparison } from "./analysis.js";
+import { buildRetrievalQueryStrategyComparisonAnalysisArtifact, RETRIEVAL_QUERY_STRATEGY_COMPARISON_ANALYSIS_ARTIFACT_FILE } from "./analysisArtifact.js";
+import { executeRetrievalQueryStrategyCaseFromPreparedIndex, executeRetrievalQueryStrategyComparison } from "./execution.js";
+import { executeLocalRepositorySubjectRetrievalQueryStrategyComparison } from "./localSubjectExecution.js";
+import { assertRetrievalQueryStrategyExternalProjectionIsPrivate, describeRetrievalQueryStrategyLocalSubjectFailureForPersistence, projectRetrievalQueryStrategyAnalysisForExternalLocalPersistence, projectRetrievalQueryStrategyExecutionForExternalLocalPersistence, RETRIEVAL_QUERY_STRATEGY_UNEXPECTED_EXTERNAL_FAILURE_MESSAGE } from "./localSubjectPrivacy.js";
+import { buildRetrievalQueryStrategyComparisonExecutionArtifact, RETRIEVAL_QUERY_STRATEGY_COMPARISON_EXECUTION_ARTIFACT_FILE } from "./executionArtifact.js";
+import { RETRIEVAL_QUERY_STRATEGY_COMPARISON_PLUGIN_ID, RETRIEVAL_QUERY_STRATEGY_COMPARISON_VARIANTS, retrievalQueryStrategyComparisonMetadata } from "./metadata.js";
+import { toRetrievalQueryStrategyOutcomeMetrics, toRetrievalQueryStrategyRunMetrics } from "./metrics.js";
+/** Privacy-safe subject manifest persisted beside the artifacts for external-local runs (same file as the other local-subject plugins). */
+export const RETRIEVAL_QUERY_STRATEGY_COMPARISON_LOCAL_SUBJECT_MANIFEST_FILE = "local-repository-subject-manifest.json";
+/** An external target must always arrive with a loaded local repository subject; there is no bundled fallback against it. */
+export const RETRIEVAL_QUERY_STRATEGY_COMPARISON_EXTERNAL_REQUIRES_SUBJECT_MESSAGE = "External-local retrieval-query-strategy-comparison targets require a loaded local repository subject (--local-subject-config).";
+export const retrievalQueryStrategyComparisonPlugin = {
+    metadata: retrievalQueryStrategyComparisonMetadata,
+    defaultConfig: defaultRetrievalQueryStrategyComparisonConfig,
+    configDefinition: retrievalQueryStrategyComparisonConfigDefinition,
+    supportedVariants: RETRIEVAL_QUERY_STRATEGY_COMPARISON_VARIANTS.map((variant) => variant.id),
+    validateConfig: validateRetrievalQueryStrategyComparisonConfig,
+    async run(context) {
+        const startedAt = context.startedAt.toISOString();
+        const localSubject = readLocalSubjectInput(context.inputs);
+        if (localSubject)
+            return runLocalSubject(context, localSubject, startedAt);
+        // The runner does not enforce supportedTargets; never fall back to the bundled corpus against an external repository.
+        if (context.target.kind !== "self" || context.target.isSelf !== true) {
+            throw new Error(RETRIEVAL_QUERY_STRATEGY_COMPARISON_EXTERNAL_REQUIRES_SUBJECT_MESSAGE);
+        }
+        const cases = selectWarmIndexCases(readCasesInput(context.inputs), context.config);
+        const outDir = context.outputRoot ?? path.resolve(context.toolRoot, context.config.outDir);
+        await mkdir(outDir, { recursive: true });
+        const caseEvidence = await executeRetrievalQueryStrategyComparison({
+            cases,
+            kitCommand: context.config.kitCommand,
+            outputRoot: outDir,
+            dependencies: readDependenciesInput(context.inputs)
+        });
+        // Scientific analysis is calculated in memory before persistence; reports later consume it, never recalculate it.
+        const analysis = analyzeRetrievalQueryStrategyComparison(cases, caseEvidence);
+        const completedAt = new Date().toISOString();
+        const artifact = buildRetrievalQueryStrategyComparisonExecutionArtifact({
+            runId: context.runId,
+            pluginId: RETRIEVAL_QUERY_STRATEGY_COMPARISON_PLUGIN_ID,
+            pluginSchemaVersion: retrievalQueryStrategyComparisonMetadata.schemaVersion,
+            startedAt,
+            completedAt,
+            cases: caseEvidence
+        });
+        const analysisArtifact = buildRetrievalQueryStrategyComparisonAnalysisArtifact({
+            runId: context.runId,
+            pluginId: RETRIEVAL_QUERY_STRATEGY_COMPARISON_PLUGIN_ID,
+            pluginSchemaVersion: retrievalQueryStrategyComparisonMetadata.schemaVersion,
+            startedAt,
+            completedAt,
+            analysis
+        });
+        const executionArtifactPath = resolveWithinRoot(outDir, RETRIEVAL_QUERY_STRATEGY_COMPARISON_EXECUTION_ARTIFACT_FILE);
+        const analysisArtifactPath = resolveWithinRoot(outDir, RETRIEVAL_QUERY_STRATEGY_COMPARISON_ANALYSIS_ARTIFACT_FILE);
+        await writeFile(executionArtifactPath, `${JSON.stringify(artifact, null, 2)}\n`, "utf8");
+        await writeFile(analysisArtifactPath, `${JSON.stringify(analysisArtifact, null, 2)}\n`, "utf8");
+        return mapRetrievalQueryStrategyComparisonToRun({
+            runId: context.runId,
+            startedAt,
+            completedAt,
+            target: context.target,
+            caseEvidence,
+            analysis,
+            executionArtifactPath,
+            analysisArtifactPath
+        });
+    },
+    summarize(result) {
+        return result.summary ?? summarizeExperimentRun(result);
+    }
+};
+function readLocalSubjectInput(inputs) {
+    const value = inputs?.localSubject;
+    return value && typeof value === "object" ? value : undefined;
+}
+/** Test seams for the external path derive from the same optional dependency input the bundled path reads. */
+function readLocalSubjectDependencies(inputs) {
+    const injected = readDependenciesInput(inputs);
+    if (!injected)
+        return undefined;
+    const { buildIndex, runCoreStrategy, runSemanticStrategy } = injected;
+    return {
+        ...(buildIndex ? { buildIndex } : {}),
+        ...(runCoreStrategy || runSemanticStrategy
+            ? {
+                executeCase: (options) => executeRetrievalQueryStrategyCaseFromPreparedIndex({
+                    ...options,
+                    dependencies: { ...(runCoreStrategy ? { runCoreStrategy } : {}), ...(runSemanticStrategy ? { runSemanticStrategy } : {}) }
+                })
+            }
+            : {})
+    };
+}
+/**
+ * External-local path. The safe seam owns execution, immutability and scratch cleanup and either returns evidence or
+ * throws. Scientific analysis is calculated from the real, unprojected identities; only then is the evidence
+ * privacy-projected and asserted private. Nothing durable is written before every gate has passed, and then in this
+ * order: execution artifact, analysis artifact, manifest.
+ */
+async function runLocalSubject(context, subject, startedAt) {
+    if (context.target.isSelf || context.target.kind !== "external-local") {
+        throw new Error("Local-repository subject mode requires an external-local target.");
+    }
+    if (context.config.caseIds !== undefined || context.config.benchmarkProjects !== undefined) {
+        throw new Error("Case and benchmark-project filters are not supported for an external local repository subject; the subject config owns the case set.");
+    }
+    if (!(await isSamePhysicalDirectory(context.target.targetRoot, subject.repositoryRoot))) {
+        throw new Error("The selected --target is not the repository the local subject was loaded from.");
+    }
+    const outDir = context.outputRoot;
+    if (!outDir)
+        throw new Error("Local-repository subject mode requires an experiment output directory.");
+    let localResult;
+    try {
+        localResult = await executeLocalRepositorySubjectRetrievalQueryStrategyComparison({
+            subject,
+            kitCommand: context.config.kitCommand,
+            workRoot: outDir,
+            dependencies: readLocalSubjectDependencies(context.inputs)
+        });
+    }
+    catch (error) {
+        if (error instanceof LocalSubjectExecutionError)
+            throw new Error(describeRetrievalQueryStrategyLocalSubjectFailureForPersistence(error));
+        throw new Error(RETRIEVAL_QUERY_STRATEGY_UNEXPECTED_EXTERNAL_FAILURE_MESSAGE);
+    }
+    // Science first, on real identities; placeholders are never scientific identities.
+    const analysis = analyzeRetrievalQueryStrategyComparison(subject.evaluationCases, localResult.caseEvidence);
+    const projectedExecution = projectRetrievalQueryStrategyExecutionForExternalLocalPersistence(localResult.caseEvidence);
+    const projectedAnalysis = projectRetrievalQueryStrategyAnalysisForExternalLocalPersistence(analysis);
+    const privateValues = {
+        filePaths: [...subject.eligibleFiles, ...subject.runtimeSafetyExclusions.gitIgnoredFiles, ...subject.runtimeSafetyExclusions.oversizedFiles],
+        caseTitles: subject.evaluationCases.map((evaluationCase) => evaluationCase.title),
+        warnings: localResult.caseEvidence.flatMap((entry) => entry.treatments.flatMap((treatment) => treatment.retrieval?.warnings ?? [])),
+        privateRoots: [subject.repositoryRoot, outDir]
+    };
+    assertRetrievalQueryStrategyExternalProjectionIsPrivate(projectedExecution, privateValues);
+    assertRetrievalQueryStrategyExternalProjectionIsPrivate(projectedAnalysis, privateValues);
+    await mkdir(outDir, { recursive: true });
+    const completedAt = new Date().toISOString();
+    const executionArtifact = buildRetrievalQueryStrategyComparisonExecutionArtifact({
+        runId: context.runId,
+        pluginId: RETRIEVAL_QUERY_STRATEGY_COMPARISON_PLUGIN_ID,
+        pluginSchemaVersion: retrievalQueryStrategyComparisonMetadata.schemaVersion,
+        startedAt,
+        completedAt,
+        cases: projectedExecution
+    });
+    const analysisArtifact = buildRetrievalQueryStrategyComparisonAnalysisArtifact({
+        runId: context.runId,
+        pluginId: RETRIEVAL_QUERY_STRATEGY_COMPARISON_PLUGIN_ID,
+        pluginSchemaVersion: retrievalQueryStrategyComparisonMetadata.schemaVersion,
+        startedAt,
+        completedAt,
+        analysis: projectedAnalysis
+    });
+    await writeFile(resolveWithinRoot(outDir, RETRIEVAL_QUERY_STRATEGY_COMPARISON_EXECUTION_ARTIFACT_FILE), `${JSON.stringify(executionArtifact, null, 2)}\n`, "utf8");
+    await writeFile(resolveWithinRoot(outDir, RETRIEVAL_QUERY_STRATEGY_COMPARISON_ANALYSIS_ARTIFACT_FILE), `${JSON.stringify(analysisArtifact, null, 2)}\n`, "utf8");
+    await writeFile(resolveWithinRoot(outDir, RETRIEVAL_QUERY_STRATEGY_COMPARISON_LOCAL_SUBJECT_MANIFEST_FILE), serializeLocalRepositorySubjectManifest(subject.manifest), "utf8");
+    // Durable external run: logical target and basenames only; metrics come from the unchanged numeric analysis.
+    const run = mapRetrievalQueryStrategyComparisonToRun({
+        runId: context.runId,
+        startedAt,
+        completedAt,
+        target: projectExternalLocalTarget(subject.manifest),
+        caseEvidence: projectedExecution,
+        analysis: projectedAnalysis,
+        executionArtifactPath: RETRIEVAL_QUERY_STRATEGY_COMPARISON_EXECUTION_ARTIFACT_FILE,
+        analysisArtifactPath: RETRIEVAL_QUERY_STRATEGY_COMPARISON_ANALYSIS_ARTIFACT_FILE
+    });
+    run.artifacts.push({
+        id: "local-repository-subject-manifest",
+        label: "Privacy-safe local repository subject manifest",
+        path: RETRIEVAL_QUERY_STRATEGY_COMPARISON_LOCAL_SUBJECT_MANIFEST_FILE,
+        kind: "artifact",
+        mimeType: "application/json",
+        description: "Privacy-safe logical subject, Git, safety and aggregate inventory metadata without source or private identity lists."
+    });
+    return run;
+}
+export function mapRetrievalQueryStrategyComparisonToRun(args) {
+    const cases = args.caseEvidence.map((entry, caseIndex) => ({
+        id: entry.caseId,
+        name: entry.caseName,
+        outcomes: entry.treatments.map((treatment, treatmentIndex) => buildOutcome(entry, treatment, args.analysis.cases[caseIndex].treatments[treatmentIndex])),
+        metadata: { benchmarkProject: entry.benchmarkProject, taskLocality: entry.taskLocality }
+    }));
+    const run = {
+        runId: args.runId,
+        pluginId: RETRIEVAL_QUERY_STRATEGY_COMPARISON_PLUGIN_ID,
+        startedAt: args.startedAt,
+        completedAt: args.completedAt,
+        status: aggregateTreatmentStatus(args.caseEvidence.flatMap((entry) => entry.treatments.map((treatment) => treatment.status))),
+        target: args.target,
+        variants: RETRIEVAL_QUERY_STRATEGY_COMPARISON_VARIANTS.map((variant) => ({ ...variant })),
+        cases,
+        metrics: toRetrievalQueryStrategyRunMetrics(args.analysis),
+        artifacts: [
+            {
+                id: "retrieval-query-strategy-comparison-execution",
+                label: "Retrieval query strategy comparison execution evidence",
+                path: args.executionArtifactPath,
+                kind: "artifact",
+                mimeType: "application/json",
+                description: "Matched per-case execution evidence for all seven retrieval query strategies without context text or raw command output."
+            },
+            {
+                id: "retrieval-query-strategy-comparison-analysis",
+                label: "Retrieval query strategy comparison scientific analysis",
+                path: args.analysisArtifactPath,
+                kind: "artifact",
+                mimeType: "application/json",
+                description: "Planner-defined retrieval-quality, F1, matched task-type aggregates, and Pareto comparison for the seven retrieval query strategies."
+            }
+        ],
+        warnings: [],
+        // Outcome failures are the authoritative per-treatment failure location.
+        failures: [],
+        metadata: { executionArtifactPath: args.executionArtifactPath, analysisArtifactPath: args.analysisArtifactPath },
+        caseExecutionEvidence: structuredClone(args.caseEvidence),
+        analysis: structuredClone(args.analysis)
+    };
+    run.summary = summarizeExperimentRun(run);
+    return run;
+}
+function buildOutcome(caseEvidence, treatment, treatmentAnalysis) {
+    const { caseId, benchmarkProject, taskLocality } = caseEvidence;
+    const { strategyId } = treatment;
+    return {
+        id: `${caseId}:${strategyId}`,
+        caseId,
+        variantId: strategyId,
+        status: treatment.status,
+        metrics: toRetrievalQueryStrategyOutcomeMetrics(treatmentAnalysis, caseId),
+        artifacts: [],
+        warnings: (treatment.retrieval?.warnings ?? []).map((message) => ({
+            code: "retrieval-warning",
+            message,
+            variantId: strategyId,
+            caseId
+        })),
+        failures: treatment.errors.map((error) => ({
+            code: error.code,
+            message: error.message,
+            variantId: strategyId,
+            caseId,
+            recoverable: true
+        })),
+        metadata: {
+            benchmarkProject,
+            taskLocality,
+            evidenceAvailability: treatment.retrieval?.evidenceAvailability ?? null
+        }
+    };
+}
+function aggregateTreatmentStatus(statuses) {
+    if (statuses.length === 0)
+        return "skipped";
+    if (statuses.every((status) => status === "completed"))
+        return "completed";
+    if (statuses.every((status) => status === "skipped"))
+        return "skipped";
+    if (statuses.every((status) => status === "failed"))
+        return "failed";
+    return "partial";
+}
+function readCasesInput(inputs) {
+    const value = inputs?.cases;
+    if (!Array.isArray(value)) {
+        throw new Error("Retrieval query strategy comparison requires cases input.");
+    }
+    return value;
+}
+function readDependenciesInput(inputs) {
+    const value = inputs?.retrievalDependencies;
+    return value && typeof value === "object" && !Array.isArray(value)
+        ? value
+        : undefined;
+}

package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/types.js

@@ -0,0 +1 @@
+export {};

package/dist/src/report/experiments/buildPluginExperimentReport.js

@@ -3,9 +3,15 @@ import { buildContextStrategyComparisonV043Report } from "./buildContextStrategy
 import { buildWarmIndexReuseReport } from "./buildWarmIndexReuseReport.js";
 import { buildIncrementalChangeStalenessPluginReport } from "./buildIncrementalChangeStalenessPluginReport.js";
 import { buildContextWindowScalingReport } from "./buildContextWindowScalingReport.js";
+import { buildRetrievalPrecisionRecallReport } from "./buildRetrievalPrecisionRecallReport.js";
+import { buildRetrievalQueryStrategyComparisonReport } from "./buildRetrievalQueryStrategyComparisonReport.js";
 // Bulk context-window-scaling evidence is presented by the typed report section; the complete
 // evidence stays in context-window-scaling-execution.json.
 const CONTEXT_WINDOW_SCALING_BULK_KEYS = ["executionEvidence", "aggregate"];
+// Same for retrieval-precision-recall: the typed section presents the evidence; the execution artifact keeps all of it.
+const RETRIEVAL_PRECISION_RECALL_BULK_KEYS = ["caseExecutionEvidence", "aggregate"];
+// Same for retrieval-query-strategy-comparison: the typed section presents execution evidence and analysis.
+const RETRIEVAL_QUERY_STRATEGY_COMPARISON_BULK_KEYS = ["caseExecutionEvidence", "analysis"];
 const V043_BULK_ARRAY_KEYS = [
     "v043StageContextExecutions",
     "v043StageContextEvaluations",
@@ -18,6 +24,8 @@ export function buildPluginExperimentReport(args) {
     const warmIndexReuse = buildWarmIndexReuseReport(args.run);
     const incrementalChangeStaleness = buildIncrementalChangeStalenessPluginReport(args.run);
     const contextWindowScaling = buildContextWindowScalingReport(args.run);
+    const retrievalPrecisionRecall = buildRetrievalPrecisionRecallReport(args.run);
+    const retrievalQueryStrategyComparison = buildRetrievalQueryStrategyComparisonReport(args.run);
     const rawRun = { ...args.run, artifacts: relativizeArtifacts(args.run.artifacts, outputRoot) };
     for (const key of V043_BULK_ARRAY_KEYS) {
         delete rawRun[key];
@@ -27,6 +35,16 @@ export function buildPluginExperimentReport(args) {
             delete rawRun[key];
         }
     }
+    if (retrievalPrecisionRecall) {
+        for (const key of RETRIEVAL_PRECISION_RECALL_BULK_KEYS) {
+            delete rawRun[key];
+        }
+    }
+    if (retrievalQueryStrategyComparison) {
+        for (const key of RETRIEVAL_QUERY_STRATEGY_COMPARISON_BULK_KEYS) {
+            delete rawRun[key];
+        }
+    }
     return {
         metadata: {
             generatedAt: args.generatedAt ?? new Date().toISOString(),
@@ -53,8 +71,10 @@ export function buildPluginExperimentReport(args) {
         warmIndexReuse,
         incrementalChangeStaleness,
         contextWindowScaling,
+        retrievalPrecisionRecall,
+        retrievalQueryStrategyComparison,
         contextStrategyComparisonV043,
-        interpretation: buildInterpretation(args.run, contextStrategyComparisonV043, warmIndexReuse, contextWindowScaling),
+        interpretation: buildInterpretation(args.run, contextStrategyComparisonV043, warmIndexReuse, contextWindowScaling, retrievalPrecisionRecall, retrievalQueryStrategyComparison),
         rawRun,
     };
 }
@@ -133,7 +153,29 @@ function affectedNeighborhoodSentence(report) {
         `reindex recommendations were recommended=${summary.recommendedReindexCount}, not-indicated=${summary.notIndicatedReindexCount}, unknown=${summary.unknownReindexRecommendationCount}. ` +
         "Affected-neighborhood evidence uses a one-hop baseline graph around confirmed changed indexed files/symbols. It is observational and does not alter retrieval or execution status.");
 }
-function buildInterpretation(run, contextStrategyComparisonV043, warmIndexReuse, contextWindowScaling) {
+function buildInterpretation(run, contextStrategyComparisonV043, warmIndexReuse, contextWindowScaling, retrievalPrecisionRecall, retrievalQueryStrategyComparison) {
+    if (retrievalQueryStrategyComparison) {
+        const overall = retrievalQueryStrategyComparison.scopes.find((scope) => scope.scopeId === "overall");
+        const taskTypes = "Task-type results are reported separately for localized, cross-module, and broad-change cases.";
+        const summary = overall?.interpretation === "unique-best"
+            ? `Overall matched comparison: ${overall.bestStrategyId} is the unique nondominated strategy across ${overall.comparisonCaseCount} matched case(s). ${taskTypes}`
+            : overall?.interpretation === "tradeoff"
+                ? `Overall matched comparison has no single best strategy across ${overall.comparisonCaseCount} matched case(s); the Pareto front is ${overall.paretoFrontStrategyIds.join(", ")}. ${taskTypes}`
+                : "No matched complete-case retrieval-strategy comparison was available for the overall scope. Review treatment availability and excluded cases before drawing conclusions.";
+        return {
+            summary,
+            recommendedNextStep: "Review the task-type scope table, Pareto fronts, and per-case treatment metrics; do not treat Pareto-front order as a ranking."
+        };
+    }
+    if (retrievalPrecisionRecall) {
+        const summary = retrievalPrecisionRecall.runSummary;
+        return {
+            summary: `Retrieval quality was measured over ${summary.caseCount} case${summary.caseCount === 1 ? "" : "s"} (${summary.completedCaseCount} completed, ${summary.partialCaseCount} partial, ${summary.failedCaseCount} failed). ` +
+                "Available case-level file, symbol and fact metrics are summarized without ranking strategies. " +
+                "Unavailable and not-applicable evidence is excluded from macro means rather than treated as zero.",
+            recommendedNextStep: "Review per-case missed and irrelevant context together with availability before drawing conclusions."
+        };
+    }
     if (contextWindowScaling) {
         const summary = contextWindowScaling.runSummary;
         return {

package/dist/src/report/experiments/buildRetrievalPrecisionRecallReport.js

@@ -0,0 +1,83 @@
+import { RETRIEVAL_PRECISION_RECALL_PLUGIN_ID } from "../../experiments/plugins/retrievalPrecisionRecall/metadata.js";
+import { retrievalRatioOf } from "../../experiments/plugins/retrievalPrecisionRecall/metrics.js";
+import { RETRIEVAL_PRECISION_RECALL_RATIO_KEYS } from "../../experiments/plugins/retrievalPrecisionRecall/types.js";
+import { MAX_RETRIEVAL_REPORT_IDENTITIES, RETRIEVAL_PRECISION_RECALL_REPORT_SCHEMA_VERSION } from "./retrievalPrecisionRecallReportModel.js";
+/** Fixed wording; the report states observed evidence only and never ranks or scores anything. */
+export const RETRIEVAL_PRECISION_RECALL_LIMITATIONS = [
+    "File and symbol precision and recall are set-based measures, not ranked metrics.",
+    "Symbol scoring uses exact unique symbol names, because the answer key lists names rather than node identities.",
+    "Fact coverage uses explicit fact-to-context mappings in the answer key; it does not judge generated answers or interpret fact text.",
+    "Irrelevant context ratio is file-identity based and is the complement of file precision when both are available.",
+    "Retrieved token count measures the selected retrieval context payload; it is not a relevant versus irrelevant token attribution.",
+    "The current lifecycle expands only the top search candidate (lookup, slice and source).",
+    "Unavailable and not-applicable values are not zeros and are excluded from macro means.",
+    "The experiment uses no real agents.",
+    "v0.8.0 does not compare retrieval strategies."
+];
+function boundedList(items) {
+    if (items === null)
+        return null;
+    const displayed = items.slice(0, MAX_RETRIEVAL_REPORT_IDENTITIES);
+    return {
+        totalCount: items.length,
+        displayed: [...displayed],
+        displayedCount: displayed.length,
+        omittedCount: items.length - displayed.length
+    };
+}
+/**
+ * Presents the per-case metric results and aggregate already calculated on the run. It performs no precision, recall,
+ * coverage, ratio or mean calculation of its own; identity lists are only bounded for display.
+ */
+export function buildRetrievalPrecisionRecallReport(run) {
+    if (run.pluginId !== RETRIEVAL_PRECISION_RECALL_PLUGIN_ID)
+        return null;
+    const candidate = run;
+    const evidence = candidate.caseExecutionEvidence;
+    const aggregate = candidate.aggregate;
+    if (!evidence || !aggregate) {
+        // A run that failed before producing evidence (for example an unknown case id or a non-self target) still gets the
+        // generic report with its failure; only a non-failed run with missing evidence is a broken report source.
+        if (run.status === "failed")
+            return null;
+        throw new Error("Invalid retrieval-precision-recall report source: case execution evidence and aggregate are required.");
+    }
+    if (aggregate.runSummary.caseCount !== evidence.length) {
+        throw new Error("Invalid retrieval-precision-recall report source: aggregate and case execution evidence are inconsistent.");
+    }
+    const cases = evidence.map((entry) => {
+        const quality = entry.quality;
+        const metrics = Object.fromEntries(RETRIEVAL_PRECISION_RECALL_RATIO_KEYS.map((key) => {
+            const ratio = retrievalRatioOf(quality, key);
+            return [key, { availability: ratio.availability, numerator: ratio.numerator, denominator: ratio.denominator, value: ratio.value, reason: ratio.reason }];
+        }));
+        return {
+            caseId: entry.caseId,
+            caseName: entry.caseName,
+            benchmarkProject: entry.benchmarkProject,
+            taskLocality: entry.taskLocality,
+            status: entry.status,
+            evidenceAvailability: entry.retrieval?.evidenceAvailability ?? null,
+            metrics,
+            retrievedTokenCount: quality?.retrievedTokenCount ?? null,
+            tokenCountMethod: quality?.tokenCountMethod ?? null,
+            missedFiles: boundedList(quality?.file.missedFiles ?? null),
+            irrelevantRetrievedFiles: boundedList(quality?.file.irrelevantRetrievedFiles ?? null),
+            missedSymbols: boundedList(quality?.symbol.missedSymbols ?? null),
+            irrelevantRetrievedSymbols: boundedList(quality?.symbol.irrelevantRetrievedSymbols ?? null),
+            uncoveredFactIds: boundedList(quality?.fact.uncoveredFactIds ?? null),
+            errors: entry.errors.map((error) => `${error.code}: ${error.message}`),
+            identityRedaction: entry.identityRedaction ? { ...entry.identityRedaction } : null
+        };
+    });
+    return {
+        schemaVersion: RETRIEVAL_PRECISION_RECALL_REPORT_SCHEMA_VERSION,
+        tokenCountMethod: evidence.find((entry) => entry.quality?.tokenCountMethod != null)?.quality?.tokenCountMethod ?? null,
+        runSummary: { ...aggregate.runSummary },
+        ratioSummaries: structuredClone(aggregate.ratios),
+        tokenSummary: { ...aggregate.tokens },
+        occurrenceSummary: { ...aggregate.occurrences },
+        cases,
+        limitations: [...RETRIEVAL_PRECISION_RECALL_LIMITATIONS]
+    };
+}

package/dist/src/report/experiments/buildRetrievalQueryStrategyComparisonReport.js

@@ -0,0 +1,91 @@
+import { RETRIEVAL_QUERY_STRATEGY_IDS } from "../../evaluation/retrievalQueryStrategies.js";
+import { RETRIEVAL_QUERY_STRATEGY_COMPARISON_METHODOLOGY } from "../../experiments/plugins/retrievalQueryStrategyComparison/analysisArtifact.js";
+import { RETRIEVAL_QUERY_STRATEGY_COMPARISON_PLUGIN_ID } from "../../experiments/plugins/retrievalQueryStrategyComparison/metadata.js";
+import { RETRIEVAL_QUERY_STRATEGY_COMPARISON_REPORT_SCHEMA_VERSION } from "./retrievalQueryStrategyComparisonReportModel.js";
+/** Fixed wording: observed evidence only. It never claims a strategy is generally superior. */
+export const RETRIEVAL_QUERY_STRATEGY_COMPARISON_LIMITATIONS = [
+    "Retrieval treatments are deterministic my-dev-kit workflows; no coding agent or learned judge participates.",
+    "File and symbol precision and recall retain the v0.8.0 set-based identity semantics.",
+    "Balanced file and symbol F1 use 2TP / (2TP + FP + FN).",
+    "Cross-strategy scope means use matched complete cases only: a task is excluded from every strategy aggregate when any strategy lacks a primary objective.",
+    "Scope means are unweighted macro means over task instances.",
+    "Pareto dominance compares mean file F1, mean symbol F1, mean fact coverage, and mean retrieved token count without scalar weights.",
+    "A unique best strategy is reported only when the Pareto front has exactly one member; multiple nondominated strategies are reported as a tradeoff, not forcibly ranked.",
+    "Task-type scopes reuse the existing taskLocality labels: localized, cross-module, and broad-change.",
+    "Retrieved token count is a context-size estimate from the selected retrieval payload, not provider billing telemetry.",
+    "Unavailable and not-applicable metric values are not converted to zero.",
+    "External-local durable reports withhold private file, symbol, fact, semantic-node, warning, and case-title identities."
+];
+const NO_RETRIEVAL_MEASUREMENT = {
+    availability: "unavailable",
+    numerator: null,
+    denominator: null,
+    value: null,
+    reason: "no-retrieval-measurement"
+};
+const noMeasurement = () => ({ ...NO_RETRIEVAL_MEASUREMENT });
+const copyMetric = (metric) => ({
+    availability: metric.availability,
+    numerator: metric.numerator,
+    denominator: metric.denominator,
+    value: metric.value,
+    reason: metric.reason
+});
+/**
+ * Presentation-only adapter over the already-calculated `run.analysis`. It recalculates no precision, recall, F1, fact
+ * coverage, mean, Pareto front or best strategy; it only copies values and reads persisted execution status.
+ */
+export function buildRetrievalQueryStrategyComparisonReport(run) {
+    if (run.pluginId !== RETRIEVAL_QUERY_STRATEGY_COMPARISON_PLUGIN_ID)
+        return null;
+    const candidate = run;
+    const analysis = candidate.analysis;
+    const execution = candidate.caseExecutionEvidence;
+    if (!analysis || !execution) {
+        // A run that failed before analysis exists still gets the generic failure report.
+        if (run.status === "failed")
+            return null;
+        throw new Error("Invalid retrieval-query-strategy-comparison report source: execution evidence and analysis are required.");
+    }
+    const inconsistent = analysis.cases.length !== execution.length ||
+        analysis.cases.some((entry, index) => entry.caseId !== execution[index].caseId ||
+            entry.treatments.length !== RETRIEVAL_QUERY_STRATEGY_IDS.length ||
+            entry.treatments.some((treatment, position) => treatment.strategyId !== RETRIEVAL_QUERY_STRATEGY_IDS[position]) ||
+            execution[index].treatments.length !== RETRIEVAL_QUERY_STRATEGY_IDS.length);
+    if (inconsistent) {
+        throw new Error("Invalid retrieval-query-strategy-comparison report source: execution evidence and analysis are inconsistent.");
+    }
+    const cases = analysis.cases.map((entry, caseIndex) => ({
+        caseId: entry.caseId,
+        caseName: execution[caseIndex].caseName,
+        benchmarkProject: entry.benchmarkProject,
+        taskLocality: entry.taskLocality,
+        treatments: entry.treatments.map((treatment, position) => {
+            const quality = treatment.quality;
+            return {
+                strategyId: treatment.strategyId,
+                executionStatus: treatment.executionStatus,
+                evidenceAvailability: execution[caseIndex].treatments[position].retrieval?.evidenceAvailability ?? null,
+                filePrecision: quality ? copyMetric(quality.file.precision) : noMeasurement(),
+                fileRecall: quality ? copyMetric(quality.file.recall) : noMeasurement(),
+                fileF1: copyMetric(treatment.fileF1),
+                symbolPrecision: quality ? copyMetric(quality.symbol.precision) : noMeasurement(),
+                symbolRecall: quality ? copyMetric(quality.symbol.recall) : noMeasurement(),
+                symbolF1: copyMetric(treatment.symbolF1),
+                factCoverage: quality ? copyMetric(quality.fact.coverage) : noMeasurement(),
+                irrelevantContextRatio: quality ? copyMetric(quality.irrelevantContextRatio) : noMeasurement(),
+                retrievedTokenCount: quality?.retrievedTokenCount ?? null,
+                tokenCountMethod: quality?.tokenCountMethod ?? null
+            };
+        })
+    }));
+    return {
+        schemaVersion: RETRIEVAL_QUERY_STRATEGY_COMPARISON_REPORT_SCHEMA_VERSION,
+        methodology: structuredClone(RETRIEVAL_QUERY_STRATEGY_COMPARISON_METHODOLOGY),
+        strategyOrder: [...RETRIEVAL_QUERY_STRATEGY_IDS],
+        identityRedaction: run.target.privacyProjection === "external-local-redacted" ? "external-local-redacted" : null,
+        scopes: structuredClone(analysis.scopes),
+        cases,
+        limitations: [...RETRIEVAL_QUERY_STRATEGY_COMPARISON_LIMITATIONS]
+    };
+}

package/dist/src/report/experiments/index.js

@@ -24,3 +24,11 @@ export * from "./contextWindowScalingReportModel.js";
 export { buildContextWindowScalingReport, CONTEXT_WINDOW_SCALING_INTERPRETATION } from "./buildContextWindowScalingReport.js";
 export { renderContextWindowScalingHtml } from "./renderContextWindowScalingHtml.js";
 export { renderContextWindowScalingTextLines } from "./renderContextWindowScalingText.js";
+export * from "./retrievalPrecisionRecallReportModel.js";
+export { buildRetrievalPrecisionRecallReport, RETRIEVAL_PRECISION_RECALL_LIMITATIONS } from "./buildRetrievalPrecisionRecallReport.js";
+export { renderRetrievalPrecisionRecallHtml } from "./renderRetrievalPrecisionRecallHtml.js";
+export { renderRetrievalPrecisionRecallTextLines } from "./renderRetrievalPrecisionRecallText.js";
+export * from "./retrievalQueryStrategyComparisonReportModel.js";
+export { buildRetrievalQueryStrategyComparisonReport, RETRIEVAL_QUERY_STRATEGY_COMPARISON_LIMITATIONS } from "./buildRetrievalQueryStrategyComparisonReport.js";
+export { renderRetrievalQueryStrategyComparisonHtml } from "./renderRetrievalQueryStrategyComparisonHtml.js";
+export { renderRetrievalQueryStrategyComparisonTextLines } from "./renderRetrievalQueryStrategyComparisonText.js";

package/dist/src/report/experiments/renderPluginExperimentReportHtml.js

@@ -2,6 +2,8 @@ import { renderContextStrategyComparisonV043Html } from "./renderContextStrategy
 import { renderWarmIndexReuseHtml } from "./renderWarmIndexReuseHtml.js";
 import { renderIncrementalChangeStalenessHtml } from "./renderIncrementalChangeStalenessHtml.js";
 import { renderContextWindowScalingHtml } from "./renderContextWindowScalingHtml.js";
+import { renderRetrievalPrecisionRecallHtml } from "./renderRetrievalPrecisionRecallHtml.js";
+import { renderRetrievalQueryStrategyComparisonHtml } from "./renderRetrievalQueryStrategyComparisonHtml.js";
 export function renderPluginExperimentReportHtml(report) {
     const redactedTarget = report.target.privacyProjection === "external-local-redacted";
     return `<!DOCTYPE html>
@@ -106,6 +108,10 @@ export function renderPluginExperimentReportHtml(report) {
 
   ${renderContextWindowScalingHtml(report.contextWindowScaling)}
 
+  ${renderRetrievalPrecisionRecallHtml(report.retrievalPrecisionRecall ?? null)}
+
+  ${renderRetrievalQueryStrategyComparisonHtml(report.retrievalQueryStrategyComparison ?? null)}
+
   <section>
     <h2>Warnings, Skips, And Failures</h2>
     ${report.findings.length ? table(["Severity", "Code", "Message", "Variant", "Case"], report.findings.map((finding) => [