@dailephd/my-dev-kit-lab 0.8.0 → 0.8.2

package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/plugin.js

@@ -0,0 +1,292 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { serializeLocalRepositorySubjectManifest } from "../../../evaluation/localRepositorySubject/index.js";
+import { summarizeExperimentRun } from "../../results.js";
+import { LocalSubjectExecutionError } from "../contextWindowScaling/localSubjectErrors.js";
+import { projectExternalLocalTarget } from "../contextWindowScaling/localSubjectPrivacy.js";
+import { isSamePhysicalDirectory } from "../contextWindowScaling/localSubjectScratch.js";
+import { selectWarmIndexCases } from "../warmIndexReuse/selection.js";
+import { defaultRetrievalQueryStrategyComparisonConfig, retrievalQueryStrategyComparisonConfigDefinition, validateRetrievalQueryStrategyComparisonConfig } from "./config.js";
+import { analyzeRetrievalQueryStrategyComparison } from "./analysis.js";
+import { buildRetrievalQueryStrategyComparisonAnalysisArtifact, RETRIEVAL_QUERY_STRATEGY_COMPARISON_ANALYSIS_ARTIFACT_FILE } from "./analysisArtifact.js";
+import { executeRetrievalQueryStrategyCaseFromPreparedIndex, executeRetrievalQueryStrategyComparison } from "./execution.js";
+import { executeLocalRepositorySubjectRetrievalQueryStrategyComparison } from "./localSubjectExecution.js";
+import { assertRetrievalQueryStrategyExternalProjectionIsPrivate, describeRetrievalQueryStrategyLocalSubjectFailureForPersistence, projectRetrievalQueryStrategyAnalysisForExternalLocalPersistence, projectRetrievalQueryStrategyExecutionForExternalLocalPersistence, RETRIEVAL_QUERY_STRATEGY_UNEXPECTED_EXTERNAL_FAILURE_MESSAGE } from "./localSubjectPrivacy.js";
+import { buildRetrievalQueryStrategyComparisonExecutionArtifact, RETRIEVAL_QUERY_STRATEGY_COMPARISON_EXECUTION_ARTIFACT_FILE } from "./executionArtifact.js";
+import { RETRIEVAL_QUERY_STRATEGY_COMPARISON_PLUGIN_ID, RETRIEVAL_QUERY_STRATEGY_COMPARISON_VARIANTS, retrievalQueryStrategyComparisonMetadata } from "./metadata.js";
+import { toRetrievalQueryStrategyOutcomeMetrics, toRetrievalQueryStrategyRunMetrics } from "./metrics.js";
+/** Privacy-safe subject manifest persisted beside the artifacts for external-local runs (same file as the other local-subject plugins). */
+export const RETRIEVAL_QUERY_STRATEGY_COMPARISON_LOCAL_SUBJECT_MANIFEST_FILE = "local-repository-subject-manifest.json";
+/** An external target must always arrive with a loaded local repository subject; there is no bundled fallback against it. */
+export const RETRIEVAL_QUERY_STRATEGY_COMPARISON_EXTERNAL_REQUIRES_SUBJECT_MESSAGE = "External-local retrieval-query-strategy-comparison targets require a loaded local repository subject (--local-subject-config).";
+export const retrievalQueryStrategyComparisonPlugin = {
+    metadata: retrievalQueryStrategyComparisonMetadata,
+    defaultConfig: defaultRetrievalQueryStrategyComparisonConfig,
+    configDefinition: retrievalQueryStrategyComparisonConfigDefinition,
+    supportedVariants: RETRIEVAL_QUERY_STRATEGY_COMPARISON_VARIANTS.map((variant) => variant.id),
+    validateConfig: validateRetrievalQueryStrategyComparisonConfig,
+    async run(context) {
+        const startedAt = context.startedAt.toISOString();
+        const localSubject = readLocalSubjectInput(context.inputs);
+        if (localSubject)
+            return runLocalSubject(context, localSubject, startedAt);
+        // The runner does not enforce supportedTargets; never fall back to the bundled corpus against an external repository.
+        if (context.target.kind !== "self" || context.target.isSelf !== true) {
+            throw new Error(RETRIEVAL_QUERY_STRATEGY_COMPARISON_EXTERNAL_REQUIRES_SUBJECT_MESSAGE);
+        }
+        const cases = selectWarmIndexCases(readCasesInput(context.inputs), context.config);
+        const outDir = context.outputRoot ?? path.resolve(context.toolRoot, context.config.outDir);
+        await mkdir(outDir, { recursive: true });
+        const caseEvidence = await executeRetrievalQueryStrategyComparison({
+            cases,
+            kitCommand: context.config.kitCommand,
+            outputRoot: outDir,
+            dependencies: readDependenciesInput(context.inputs)
+        });
+        // Scientific analysis is calculated in memory before persistence; reports later consume it, never recalculate it.
+        const analysis = analyzeRetrievalQueryStrategyComparison(cases, caseEvidence);
+        const completedAt = new Date().toISOString();
+        const artifact = buildRetrievalQueryStrategyComparisonExecutionArtifact({
+            runId: context.runId,
+            pluginId: RETRIEVAL_QUERY_STRATEGY_COMPARISON_PLUGIN_ID,
+            pluginSchemaVersion: retrievalQueryStrategyComparisonMetadata.schemaVersion,
+            startedAt,
+            completedAt,
+            cases: caseEvidence
+        });
+        const analysisArtifact = buildRetrievalQueryStrategyComparisonAnalysisArtifact({
+            runId: context.runId,
+            pluginId: RETRIEVAL_QUERY_STRATEGY_COMPARISON_PLUGIN_ID,
+            pluginSchemaVersion: retrievalQueryStrategyComparisonMetadata.schemaVersion,
+            startedAt,
+            completedAt,
+            analysis
+        });
+        const executionArtifactPath = resolveWithinRoot(outDir, RETRIEVAL_QUERY_STRATEGY_COMPARISON_EXECUTION_ARTIFACT_FILE);
+        const analysisArtifactPath = resolveWithinRoot(outDir, RETRIEVAL_QUERY_STRATEGY_COMPARISON_ANALYSIS_ARTIFACT_FILE);
+        await writeFile(executionArtifactPath, `${JSON.stringify(artifact, null, 2)}\n`, "utf8");
+        await writeFile(analysisArtifactPath, `${JSON.stringify(analysisArtifact, null, 2)}\n`, "utf8");
+        return mapRetrievalQueryStrategyComparisonToRun({
+            runId: context.runId,
+            startedAt,
+            completedAt,
+            target: context.target,
+            caseEvidence,
+            analysis,
+            executionArtifactPath,
+            analysisArtifactPath
+        });
+    },
+    summarize(result) {
+        return result.summary ?? summarizeExperimentRun(result);
+    }
+};
+function readLocalSubjectInput(inputs) {
+    const value = inputs?.localSubject;
+    return value && typeof value === "object" ? value : undefined;
+}
+/** Test seams for the external path derive from the same optional dependency input the bundled path reads. */
+function readLocalSubjectDependencies(inputs) {
+    const injected = readDependenciesInput(inputs);
+    if (!injected)
+        return undefined;
+    const { buildIndex, runCoreStrategy, runSemanticStrategy } = injected;
+    return {
+        ...(buildIndex ? { buildIndex } : {}),
+        ...(runCoreStrategy || runSemanticStrategy
+            ? {
+                executeCase: (options) => executeRetrievalQueryStrategyCaseFromPreparedIndex({
+                    ...options,
+                    dependencies: { ...(runCoreStrategy ? { runCoreStrategy } : {}), ...(runSemanticStrategy ? { runSemanticStrategy } : {}) }
+                })
+            }
+            : {})
+    };
+}
+/**
+ * External-local path. The safe seam owns execution, immutability and scratch cleanup and either returns evidence or
+ * throws. Scientific analysis is calculated from the real, unprojected identities; only then is the evidence
+ * privacy-projected and asserted private. Nothing durable is written before every gate has passed, and then in this
+ * order: execution artifact, analysis artifact, manifest.
+ */
+async function runLocalSubject(context, subject, startedAt) {
+    if (context.target.isSelf || context.target.kind !== "external-local") {
+        throw new Error("Local-repository subject mode requires an external-local target.");
+    }
+    if (context.config.caseIds !== undefined || context.config.benchmarkProjects !== undefined) {
+        throw new Error("Case and benchmark-project filters are not supported for an external local repository subject; the subject config owns the case set.");
+    }
+    if (!(await isSamePhysicalDirectory(context.target.targetRoot, subject.repositoryRoot))) {
+        throw new Error("The selected --target is not the repository the local subject was loaded from.");
+    }
+    const outDir = context.outputRoot;
+    if (!outDir)
+        throw new Error("Local-repository subject mode requires an experiment output directory.");
+    let localResult;
+    try {
+        localResult = await executeLocalRepositorySubjectRetrievalQueryStrategyComparison({
+            subject,
+            kitCommand: context.config.kitCommand,
+            workRoot: outDir,
+            dependencies: readLocalSubjectDependencies(context.inputs)
+        });
+    }
+    catch (error) {
+        if (error instanceof LocalSubjectExecutionError)
+            throw new Error(describeRetrievalQueryStrategyLocalSubjectFailureForPersistence(error));
+        throw new Error(RETRIEVAL_QUERY_STRATEGY_UNEXPECTED_EXTERNAL_FAILURE_MESSAGE);
+    }
+    // Science first, on real identities; placeholders are never scientific identities.
+    const analysis = analyzeRetrievalQueryStrategyComparison(subject.evaluationCases, localResult.caseEvidence);
+    const projectedExecution = projectRetrievalQueryStrategyExecutionForExternalLocalPersistence(localResult.caseEvidence);
+    const projectedAnalysis = projectRetrievalQueryStrategyAnalysisForExternalLocalPersistence(analysis);
+    const privateValues = {
+        filePaths: [...subject.eligibleFiles, ...subject.runtimeSafetyExclusions.gitIgnoredFiles, ...subject.runtimeSafetyExclusions.oversizedFiles],
+        caseTitles: subject.evaluationCases.map((evaluationCase) => evaluationCase.title),
+        warnings: localResult.caseEvidence.flatMap((entry) => entry.treatments.flatMap((treatment) => treatment.retrieval?.warnings ?? [])),
+        privateRoots: [subject.repositoryRoot, outDir]
+    };
+    assertRetrievalQueryStrategyExternalProjectionIsPrivate(projectedExecution, privateValues);
+    assertRetrievalQueryStrategyExternalProjectionIsPrivate(projectedAnalysis, privateValues);
+    await mkdir(outDir, { recursive: true });
+    const completedAt = new Date().toISOString();
+    const executionArtifact = buildRetrievalQueryStrategyComparisonExecutionArtifact({
+        runId: context.runId,
+        pluginId: RETRIEVAL_QUERY_STRATEGY_COMPARISON_PLUGIN_ID,
+        pluginSchemaVersion: retrievalQueryStrategyComparisonMetadata.schemaVersion,
+        startedAt,
+        completedAt,
+        cases: projectedExecution
+    });
+    const analysisArtifact = buildRetrievalQueryStrategyComparisonAnalysisArtifact({
+        runId: context.runId,
+        pluginId: RETRIEVAL_QUERY_STRATEGY_COMPARISON_PLUGIN_ID,
+        pluginSchemaVersion: retrievalQueryStrategyComparisonMetadata.schemaVersion,
+        startedAt,
+        completedAt,
+        analysis: projectedAnalysis
+    });
+    await writeFile(resolveWithinRoot(outDir, RETRIEVAL_QUERY_STRATEGY_COMPARISON_EXECUTION_ARTIFACT_FILE), `${JSON.stringify(executionArtifact, null, 2)}\n`, "utf8");
+    await writeFile(resolveWithinRoot(outDir, RETRIEVAL_QUERY_STRATEGY_COMPARISON_ANALYSIS_ARTIFACT_FILE), `${JSON.stringify(analysisArtifact, null, 2)}\n`, "utf8");
+    await writeFile(resolveWithinRoot(outDir, RETRIEVAL_QUERY_STRATEGY_COMPARISON_LOCAL_SUBJECT_MANIFEST_FILE), serializeLocalRepositorySubjectManifest(subject.manifest), "utf8");
+    // Durable external run: logical target and basenames only; metrics come from the unchanged numeric analysis.
+    const run = mapRetrievalQueryStrategyComparisonToRun({
+        runId: context.runId,
+        startedAt,
+        completedAt,
+        target: projectExternalLocalTarget(subject.manifest),
+        caseEvidence: projectedExecution,
+        analysis: projectedAnalysis,
+        executionArtifactPath: RETRIEVAL_QUERY_STRATEGY_COMPARISON_EXECUTION_ARTIFACT_FILE,
+        analysisArtifactPath: RETRIEVAL_QUERY_STRATEGY_COMPARISON_ANALYSIS_ARTIFACT_FILE
+    });
+    run.artifacts.push({
+        id: "local-repository-subject-manifest",
+        label: "Privacy-safe local repository subject manifest",
+        path: RETRIEVAL_QUERY_STRATEGY_COMPARISON_LOCAL_SUBJECT_MANIFEST_FILE,
+        kind: "artifact",
+        mimeType: "application/json",
+        description: "Privacy-safe logical subject, Git, safety and aggregate inventory metadata without source or private identity lists."
+    });
+    return run;
+}
+export function mapRetrievalQueryStrategyComparisonToRun(args) {
+    const cases = args.caseEvidence.map((entry, caseIndex) => ({
+        id: entry.caseId,
+        name: entry.caseName,
+        outcomes: entry.treatments.map((treatment, treatmentIndex) => buildOutcome(entry, treatment, args.analysis.cases[caseIndex].treatments[treatmentIndex])),
+        metadata: { benchmarkProject: entry.benchmarkProject, taskLocality: entry.taskLocality }
+    }));
+    const run = {
+        runId: args.runId,
+        pluginId: RETRIEVAL_QUERY_STRATEGY_COMPARISON_PLUGIN_ID,
+        startedAt: args.startedAt,
+        completedAt: args.completedAt,
+        status: aggregateTreatmentStatus(args.caseEvidence.flatMap((entry) => entry.treatments.map((treatment) => treatment.status))),
+        target: args.target,
+        variants: RETRIEVAL_QUERY_STRATEGY_COMPARISON_VARIANTS.map((variant) => ({ ...variant })),
+        cases,
+        metrics: toRetrievalQueryStrategyRunMetrics(args.analysis),
+        artifacts: [
+            {
+                id: "retrieval-query-strategy-comparison-execution",
+                label: "Retrieval query strategy comparison execution evidence",
+                path: args.executionArtifactPath,
+                kind: "artifact",
+                mimeType: "application/json",
+                description: "Matched per-case execution evidence for all seven retrieval query strategies without context text or raw command output."
+            },
+            {
+                id: "retrieval-query-strategy-comparison-analysis",
+                label: "Retrieval query strategy comparison scientific analysis",
+                path: args.analysisArtifactPath,
+                kind: "artifact",
+                mimeType: "application/json",
+                description: "Planner-defined retrieval-quality, F1, matched task-type aggregates, and Pareto comparison for the seven retrieval query strategies."
+            }
+        ],
+        warnings: [],
+        // Outcome failures are the authoritative per-treatment failure location.
+        failures: [],
+        metadata: { executionArtifactPath: args.executionArtifactPath, analysisArtifactPath: args.analysisArtifactPath },
+        caseExecutionEvidence: structuredClone(args.caseEvidence),
+        analysis: structuredClone(args.analysis)
+    };
+    run.summary = summarizeExperimentRun(run);
+    return run;
+}
+function buildOutcome(caseEvidence, treatment, treatmentAnalysis) {
+    const { caseId, benchmarkProject, taskLocality } = caseEvidence;
+    const { strategyId } = treatment;
+    return {
+        id: `${caseId}:${strategyId}`,
+        caseId,
+        variantId: strategyId,
+        status: treatment.status,
+        metrics: toRetrievalQueryStrategyOutcomeMetrics(treatmentAnalysis, caseId),
+        artifacts: [],
+        warnings: (treatment.retrieval?.warnings ?? []).map((message) => ({
+            code: "retrieval-warning",
+            message,
+            variantId: strategyId,
+            caseId
+        })),
+        failures: treatment.errors.map((error) => ({
+            code: error.code,
+            message: error.message,
+            variantId: strategyId,
+            caseId,
+            recoverable: true
+        })),
+        metadata: {
+            benchmarkProject,
+            taskLocality,
+            evidenceAvailability: treatment.retrieval?.evidenceAvailability ?? null
+        }
+    };
+}
+function aggregateTreatmentStatus(statuses) {
+    if (statuses.length === 0)
+        return "skipped";
+    if (statuses.every((status) => status === "completed"))
+        return "completed";
+    if (statuses.every((status) => status === "skipped"))
+        return "skipped";
+    if (statuses.every((status) => status === "failed"))
+        return "failed";
+    return "partial";
+}
+function readCasesInput(inputs) {
+    const value = inputs?.cases;
+    if (!Array.isArray(value)) {
+        throw new Error("Retrieval query strategy comparison requires cases input.");
+    }
+    return value;
+}
+function readDependenciesInput(inputs) {
+    const value = inputs?.retrievalDependencies;
+    return value && typeof value === "object" && !Array.isArray(value)
+        ? value
+        : undefined;
+}

package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/types.js

@@ -0,0 +1 @@
+export {};

package/dist/src/report/experiments/buildContextPackGenerationReport.js

@@ -0,0 +1,171 @@
+import path from "node:path";
+import { CONTEXT_PACK_GENERATION_METHODOLOGY } from "../../experiments/plugins/contextPackGeneration/analysisArtifact.js";
+import { CONTEXT_PACK_GENERATION_PLUGIN_ID, CONTEXT_PACK_GENERATION_TREATMENT_IDS } from "../../experiments/plugins/contextPackGeneration/metadata.js";
+import { CONTEXT_PACK_GENERATION_REPORT_SCHEMA_VERSION, PACK_PREVIEW_MAX_CALL_RELATIONSHIPS, PACK_PREVIEW_MAX_EVIDENCE_NOTES, PACK_PREVIEW_MAX_FILES, PACK_PREVIEW_MAX_SOURCE_LINES_PER_SLICE, PACK_PREVIEW_MAX_SOURCE_SLICES, PACK_PREVIEW_MAX_SYMBOLS, PACK_PREVIEW_MAX_TESTS } from "./contextPackGenerationReportModel.js";
+/** Fixed wording: observed evidence only. */
+export const CONTEXT_PACK_GENERATION_LIMITATIONS = [
+    "This is a context-pack experiment over deterministic my-dev-kit workflows; it is not a coding-agent success evaluation.",
+    "Estimated tokens come from the existing Lab estimator over rendered text, not provider billing telemetry.",
+    "Unavailable and not-applicable values are not converted to zero.",
+    "Scope means use matched complete cases only and are unweighted macro means over task instances.",
+    "No composite score, ranking, or winning treatment is produced; treatments are reported side by side.",
+    "Tests and call relationships in the pack are descriptive and are not scored.",
+    "The context pack is an experimental Lab artifact, not the production my-dev-kit context API.",
+    "The pack preview is bounded for display only and does not change the stored pack or any measured size."
+];
+const copyMetric = (metric) => ({
+    availability: metric.availability,
+    numerator: metric.numerator,
+    denominator: metric.denominator,
+    value: metric.value,
+    reason: metric.reason
+});
+const NO_MEASUREMENT = { availability: "unavailable", numerator: null, denominator: null, value: null, reason: "no-retrieval-measurement" };
+function bounded(source, limit, project) {
+    return { items: source.slice(0, limit).map(project), totalCount: source.length, omittedCount: Math.max(0, source.length - limit) };
+}
+/** Display truncation of a stored slice: first lines only. The pack's own `truncated` flag is copied untouched. */
+function previewSlice(slice) {
+    const lines = slice.text.split("\n");
+    const shown = lines.slice(0, PACK_PREVIEW_MAX_SOURCE_LINES_PER_SLICE);
+    return {
+        file: slice.file,
+        symbolName: slice.symbolName,
+        startLine: slice.startLine,
+        endLine: slice.endLine,
+        lineCount: slice.lineCount,
+        boundaryKnown: slice.boundaryKnown,
+        truncated: slice.truncated,
+        previewText: shown.join("\n"),
+        previewLineCount: shown.length,
+        previewTruncated: lines.length > shown.length
+    };
+}
+/** External-local evidence carries a redaction marker and no pack body; the preview is counts and availability only. */
+function buildRedactedPreview(entry, redaction) {
+    const packTreatment = entry.treatments.find((treatment) => treatment.treatmentId === "context-pack");
+    return {
+        caseId: entry.caseId,
+        status: "redacted-external-local",
+        packArtifactPath: null,
+        packAvailability: packTreatment?.availability ?? null,
+        sections: (packTreatment?.sections ?? []).map((section) => ({
+            id: section.id,
+            availability: section.availability,
+            reason: section.reason,
+            itemCount: section.itemCount,
+            estimatedTokens: section.estimatedTokens,
+            truncatedCount: section.truncatedCount ?? null
+        })),
+        task: null,
+        files: null,
+        symbols: null,
+        sourceSlices: null,
+        callRelationships: null,
+        tests: null,
+        evidenceNotes: null,
+        redaction: { ...redaction }
+    };
+}
+function buildPreview(entry, pack) {
+    if (entry.identityRedaction)
+        return buildRedactedPreview(entry, entry.identityRedaction);
+    const packTreatment = entry.treatments.find((treatment) => treatment.treatmentId === "context-pack");
+    const packArtifactPath = packTreatment?.packArtifactPath ?? null;
+    const base = {
+        caseId: entry.caseId,
+        packArtifactPath,
+        // Section summaries are persisted execution evidence; they never come from the preview truncation.
+        sections: (packTreatment?.sections ?? []).map((section) => ({
+            id: section.id,
+            availability: section.availability,
+            reason: section.reason,
+            itemCount: section.itemCount,
+            estimatedTokens: section.estimatedTokens
+        }))
+    };
+    if (!pack) {
+        return {
+            ...base,
+            status: packArtifactPath === null ? "no-pack-produced" : "pack-artifact-unavailable",
+            packAvailability: packTreatment?.availability ?? null,
+            task: null,
+            files: null,
+            symbols: null,
+            sourceSlices: null,
+            callRelationships: null,
+            tests: null,
+            evidenceNotes: null
+        };
+    }
+    return {
+        ...base,
+        status: "available",
+        packAvailability: pack.availability,
+        task: { title: pack.task.title, summary: pack.task.summary },
+        files: bounded(pack.files, PACK_PREVIEW_MAX_FILES, (file) => ({ path: file.path, rank: file.rank, reason: file.reason })),
+        symbols: bounded(pack.symbols, PACK_PREVIEW_MAX_SYMBOLS, (symbol) => ({ name: symbol.name, file: symbol.file, line: symbol.line, nodeId: symbol.nodeId })),
+        sourceSlices: bounded(pack.sourceSlices, PACK_PREVIEW_MAX_SOURCE_SLICES, previewSlice),
+        callRelationships: bounded(pack.callRelationships.filter((relationship) => relationship.kind === "calls"), PACK_PREVIEW_MAX_CALL_RELATIONSHIPS, (relationship) => ({ fromNodeId: relationship.fromNodeId, toNodeId: relationship.toNodeId })),
+        tests: bounded(pack.tests, PACK_PREVIEW_MAX_TESTS, (test) => ({ path: test.path, how: test.how })),
+        evidenceNotes: bounded(pack.evidenceNotes, PACK_PREVIEW_MAX_EVIDENCE_NOTES, (code) => code)
+    };
+}
+/**
+ * Presentation-only adapter over the already-calculated `run.analysis` and `run.caseExecutionEvidence`. It recalculates no
+ * F1, fact coverage, token size, saving, matched-case membership, mean or delta; it copies persisted values. `contextPacks`
+ * are the persisted per-case pack artifacts, loaded by the caller; only display truncation is derived from them.
+ */
+export function buildContextPackGenerationReport(run, contextPacks = new Map()) {
+    if (run.pluginId !== CONTEXT_PACK_GENERATION_PLUGIN_ID)
+        return null;
+    const candidate = run;
+    const analysis = candidate.analysis;
+    const execution = candidate.caseExecutionEvidence;
+    if (!analysis || !execution) {
+        // A run that failed before analysis exists still gets the generic failure report.
+        if (run.status === "failed")
+            return null;
+        throw new Error("Invalid context-pack-generation report source: execution evidence and analysis are required.");
+    }
+    const inconsistent = analysis.cases.length !== execution.length ||
+        analysis.cases.some((entry, index) => entry.caseId !== execution[index].caseId ||
+            entry.treatments.length !== CONTEXT_PACK_GENERATION_TREATMENT_IDS.length ||
+            entry.treatments.some((treatment, position) => treatment.treatmentId !== CONTEXT_PACK_GENERATION_TREATMENT_IDS[position]));
+    if (inconsistent) {
+        throw new Error("Invalid context-pack-generation report source: execution evidence and analysis are inconsistent.");
+    }
+    const cases = analysis.cases.map((entry, caseIndex) => ({
+        caseId: entry.caseId,
+        caseName: execution[caseIndex].caseName,
+        benchmarkProject: entry.benchmarkProject,
+        taskLocality: entry.taskLocality,
+        treatments: entry.treatments.map((treatment) => ({
+            treatmentId: treatment.treatmentId,
+            executionStatus: treatment.executionStatus,
+            availability: treatment.availability,
+            fileF1: copyMetric(treatment.fileF1),
+            symbolF1: copyMetric(treatment.symbolF1),
+            factCoverage: treatment.quality ? copyMetric(treatment.quality.fact.coverage) : { ...NO_MEASUREMENT },
+            estimatedTokens: treatment.estimatedTokens,
+            tokenCountMethod: treatment.tokenCountMethod
+        })),
+        comparison: { ...entry.comparison }
+    }));
+    return {
+        schemaVersion: CONTEXT_PACK_GENERATION_REPORT_SCHEMA_VERSION,
+        experiment: { pluginId: run.pluginId, target: run.target.isSelf ? "self" : "external target" },
+        treatmentOrder: [...CONTEXT_PACK_GENERATION_TREATMENT_IDS],
+        methodology: { ...CONTEXT_PACK_GENERATION_METHODOLOGY },
+        cases,
+        scopes: structuredClone(analysis.scopes),
+        previews: execution.map((entry) => buildPreview(entry, contextPacks.get(entry.caseId))),
+        // Execution/analysis artifacts live directly in the output root; never expose a machine-local absolute path.
+        artifacts: run.artifacts.map((artifact) => ({
+            id: artifact.id,
+            path: artifact.path ? (path.isAbsolute(artifact.path) ? path.basename(artifact.path) : artifact.path) : null,
+            caseId: artifact.caseId ?? null
+        })),
+        limitations: [...CONTEXT_PACK_GENERATION_LIMITATIONS]
+    };
+}

package/dist/src/report/experiments/buildPluginExperimentReport.js

@@ -4,11 +4,17 @@ import { buildWarmIndexReuseReport } from "./buildWarmIndexReuseReport.js";
 import { buildIncrementalChangeStalenessPluginReport } from "./buildIncrementalChangeStalenessPluginReport.js";
 import { buildContextWindowScalingReport } from "./buildContextWindowScalingReport.js";
 import { buildRetrievalPrecisionRecallReport } from "./buildRetrievalPrecisionRecallReport.js";
+import { buildRetrievalQueryStrategyComparisonReport } from "./buildRetrievalQueryStrategyComparisonReport.js";
+import { buildContextPackGenerationReport } from "./buildContextPackGenerationReport.js";
 // Bulk context-window-scaling evidence is presented by the typed report section; the complete
 // evidence stays in context-window-scaling-execution.json.
 const CONTEXT_WINDOW_SCALING_BULK_KEYS = ["executionEvidence", "aggregate"];
 // Same for retrieval-precision-recall: the typed section presents the evidence; the execution artifact keeps all of it.
 const RETRIEVAL_PRECISION_RECALL_BULK_KEYS = ["caseExecutionEvidence", "aggregate"];
+// Same for retrieval-query-strategy-comparison: the typed section presents execution evidence and analysis.
+const RETRIEVAL_QUERY_STRATEGY_COMPARISON_BULK_KEYS = ["caseExecutionEvidence", "analysis"];
+// Same for context-pack-generation: the typed section presents execution evidence and analysis.
+const CONTEXT_PACK_GENERATION_BULK_KEYS = ["caseExecutionEvidence", "analysis"];
 const V043_BULK_ARRAY_KEYS = [
     "v043StageContextExecutions",
     "v043StageContextEvaluations",
@@ -22,6 +28,8 @@ export function buildPluginExperimentReport(args) {
     const incrementalChangeStaleness = buildIncrementalChangeStalenessPluginReport(args.run);
     const contextWindowScaling = buildContextWindowScalingReport(args.run);
     const retrievalPrecisionRecall = buildRetrievalPrecisionRecallReport(args.run);
+    const retrievalQueryStrategyComparison = buildRetrievalQueryStrategyComparisonReport(args.run);
+    const contextPackGeneration = buildContextPackGenerationReport(args.run, args.contextPacks);
     const rawRun = { ...args.run, artifacts: relativizeArtifacts(args.run.artifacts, outputRoot) };
     for (const key of V043_BULK_ARRAY_KEYS) {
         delete rawRun[key];
@@ -36,6 +44,16 @@ export function buildPluginExperimentReport(args) {
             delete rawRun[key];
         }
     }
+    if (retrievalQueryStrategyComparison) {
+        for (const key of RETRIEVAL_QUERY_STRATEGY_COMPARISON_BULK_KEYS) {
+            delete rawRun[key];
+        }
+    }
+    if (contextPackGeneration) {
+        for (const key of CONTEXT_PACK_GENERATION_BULK_KEYS) {
+            delete rawRun[key];
+        }
+    }
     return {
         metadata: {
             generatedAt: args.generatedAt ?? new Date().toISOString(),
@@ -63,8 +81,10 @@ export function buildPluginExperimentReport(args) {
         incrementalChangeStaleness,
         contextWindowScaling,
         retrievalPrecisionRecall,
+        retrievalQueryStrategyComparison,
+        contextPackGeneration,
         contextStrategyComparisonV043,
-        interpretation: buildInterpretation(args.run, contextStrategyComparisonV043, warmIndexReuse, contextWindowScaling, retrievalPrecisionRecall),
+        interpretation: buildInterpretation(args.run, contextStrategyComparisonV043, warmIndexReuse, contextWindowScaling, retrievalPrecisionRecall, retrievalQueryStrategyComparison, contextPackGeneration),
         rawRun,
     };
 }
@@ -143,7 +163,32 @@ function affectedNeighborhoodSentence(report) {
         `reindex recommendations were recommended=${summary.recommendedReindexCount}, not-indicated=${summary.notIndicatedReindexCount}, unknown=${summary.unknownReindexRecommendationCount}. ` +
         "Affected-neighborhood evidence uses a one-hop baseline graph around confirmed changed indexed files/symbols. It is observational and does not alter retrieval or execution status.");
 }
-function buildInterpretation(run, contextStrategyComparisonV043, warmIndexReuse, contextWindowScaling, retrievalPrecisionRecall) {
+function buildInterpretation(run, contextStrategyComparisonV043, warmIndexReuse, contextWindowScaling, retrievalPrecisionRecall, retrievalQueryStrategyComparison, contextPackGeneration = null) {
+    if (contextPackGeneration) {
+        const overall = contextPackGeneration.scopes.find((scope) => scope.scopeId === "overall");
+        const saved = overall?.tokenSavings;
+        const deltas = overall?.pairedDeltas;
+        const summary = overall && saved && deltas
+            ? `Overall matched comparison over ${overall.includedCaseCount} of ${overall.caseCount} case(s): the context pack used a mean of ${saved.meanTokensSaved} fewer estimated tokens than raw full-file context (${saved.percentSavedOfMeans}% of the raw mean; a negative value means more tokens); mean fact coverage delta ${deltas.meanFactCoverageDelta}, mean file F1 delta ${deltas.meanFileF1Delta}, mean symbol F1 delta ${deltas.meanSymbolF1Delta}. The treatments are reported side by side without a composite score.`
+            : "No matched complete-case context-pack comparison was available for the overall scope. Review treatment availability and excluded cases before drawing conclusions.";
+        return {
+            summary,
+            recommendedNextStep: "Review per-case coverage and size, the scope aggregates, and the bounded pack preview; unavailable values are never treated as zero."
+        };
+    }
+    if (retrievalQueryStrategyComparison) {
+        const overall = retrievalQueryStrategyComparison.scopes.find((scope) => scope.scopeId === "overall");
+        const taskTypes = "Task-type results are reported separately for localized, cross-module, and broad-change cases.";
+        const summary = overall?.interpretation === "unique-best"
+            ? `Overall matched comparison: ${overall.bestStrategyId} is the unique nondominated strategy across ${overall.comparisonCaseCount} matched case(s). ${taskTypes}`
+            : overall?.interpretation === "tradeoff"
+                ? `Overall matched comparison has no single best strategy across ${overall.comparisonCaseCount} matched case(s); the Pareto front is ${overall.paretoFrontStrategyIds.join(", ")}. ${taskTypes}`
+                : "No matched complete-case retrieval-strategy comparison was available for the overall scope. Review treatment availability and excluded cases before drawing conclusions.";
+        return {
+            summary,
+            recommendedNextStep: "Review the task-type scope table, Pareto fronts, and per-case treatment metrics; do not treat Pareto-front order as a ranking."
+        };
+    }
     if (retrievalPrecisionRecall) {
         const summary = retrievalPrecisionRecall.runSummary;
         return {

package/dist/src/report/experiments/buildRetrievalQueryStrategyComparisonReport.js

@@ -0,0 +1,91 @@
+import { RETRIEVAL_QUERY_STRATEGY_IDS } from "../../evaluation/retrievalQueryStrategies.js";
+import { RETRIEVAL_QUERY_STRATEGY_COMPARISON_METHODOLOGY } from "../../experiments/plugins/retrievalQueryStrategyComparison/analysisArtifact.js";
+import { RETRIEVAL_QUERY_STRATEGY_COMPARISON_PLUGIN_ID } from "../../experiments/plugins/retrievalQueryStrategyComparison/metadata.js";
+import { RETRIEVAL_QUERY_STRATEGY_COMPARISON_REPORT_SCHEMA_VERSION } from "./retrievalQueryStrategyComparisonReportModel.js";
+/** Fixed wording: observed evidence only. It never claims a strategy is generally superior. */
+export const RETRIEVAL_QUERY_STRATEGY_COMPARISON_LIMITATIONS = [
+    "Retrieval treatments are deterministic my-dev-kit workflows; no coding agent or learned judge participates.",
+    "File and symbol precision and recall retain the v0.8.0 set-based identity semantics.",
+    "Balanced file and symbol F1 use 2TP / (2TP + FP + FN).",
+    "Cross-strategy scope means use matched complete cases only: a task is excluded from every strategy aggregate when any strategy lacks a primary objective.",
+    "Scope means are unweighted macro means over task instances.",
+    "Pareto dominance compares mean file F1, mean symbol F1, mean fact coverage, and mean retrieved token count without scalar weights.",
+    "A unique best strategy is reported only when the Pareto front has exactly one member; multiple nondominated strategies are reported as a tradeoff, not forcibly ranked.",
+    "Task-type scopes reuse the existing taskLocality labels: localized, cross-module, and broad-change.",
+    "Retrieved token count is a context-size estimate from the selected retrieval payload, not provider billing telemetry.",
+    "Unavailable and not-applicable metric values are not converted to zero.",
+    "External-local durable reports withhold private file, symbol, fact, semantic-node, warning, and case-title identities."
+];
+const NO_RETRIEVAL_MEASUREMENT = {
+    availability: "unavailable",
+    numerator: null,
+    denominator: null,
+    value: null,
+    reason: "no-retrieval-measurement"
+};
+const noMeasurement = () => ({ ...NO_RETRIEVAL_MEASUREMENT });
+const copyMetric = (metric) => ({
+    availability: metric.availability,
+    numerator: metric.numerator,
+    denominator: metric.denominator,
+    value: metric.value,
+    reason: metric.reason
+});
+/**
+ * Presentation-only adapter over the already-calculated `run.analysis`. It recalculates no precision, recall, F1, fact
+ * coverage, mean, Pareto front or best strategy; it only copies values and reads persisted execution status.
+ */
+export function buildRetrievalQueryStrategyComparisonReport(run) {
+    if (run.pluginId !== RETRIEVAL_QUERY_STRATEGY_COMPARISON_PLUGIN_ID)
+        return null;
+    const candidate = run;
+    const analysis = candidate.analysis;
+    const execution = candidate.caseExecutionEvidence;
+    if (!analysis || !execution) {
+        // A run that failed before analysis exists still gets the generic failure report.
+        if (run.status === "failed")
+            return null;
+        throw new Error("Invalid retrieval-query-strategy-comparison report source: execution evidence and analysis are required.");
+    }
+    const inconsistent = analysis.cases.length !== execution.length ||
+        analysis.cases.some((entry, index) => entry.caseId !== execution[index].caseId ||
+            entry.treatments.length !== RETRIEVAL_QUERY_STRATEGY_IDS.length ||
+            entry.treatments.some((treatment, position) => treatment.strategyId !== RETRIEVAL_QUERY_STRATEGY_IDS[position]) ||
+            execution[index].treatments.length !== RETRIEVAL_QUERY_STRATEGY_IDS.length);
+    if (inconsistent) {
+        throw new Error("Invalid retrieval-query-strategy-comparison report source: execution evidence and analysis are inconsistent.");
+    }
+    const cases = analysis.cases.map((entry, caseIndex) => ({
+        caseId: entry.caseId,
+        caseName: execution[caseIndex].caseName,
+        benchmarkProject: entry.benchmarkProject,
+        taskLocality: entry.taskLocality,
+        treatments: entry.treatments.map((treatment, position) => {
+            const quality = treatment.quality;
+            return {
+                strategyId: treatment.strategyId,
+                executionStatus: treatment.executionStatus,
+                evidenceAvailability: execution[caseIndex].treatments[position].retrieval?.evidenceAvailability ?? null,
+                filePrecision: quality ? copyMetric(quality.file.precision) : noMeasurement(),
+                fileRecall: quality ? copyMetric(quality.file.recall) : noMeasurement(),
+                fileF1: copyMetric(treatment.fileF1),
+                symbolPrecision: quality ? copyMetric(quality.symbol.precision) : noMeasurement(),
+                symbolRecall: quality ? copyMetric(quality.symbol.recall) : noMeasurement(),
+                symbolF1: copyMetric(treatment.symbolF1),
+                factCoverage: quality ? copyMetric(quality.fact.coverage) : noMeasurement(),
+                irrelevantContextRatio: quality ? copyMetric(quality.irrelevantContextRatio) : noMeasurement(),
+                retrievedTokenCount: quality?.retrievedTokenCount ?? null,
+                tokenCountMethod: quality?.tokenCountMethod ?? null
+            };
+        })
+    }));
+    return {
+        schemaVersion: RETRIEVAL_QUERY_STRATEGY_COMPARISON_REPORT_SCHEMA_VERSION,
+        methodology: structuredClone(RETRIEVAL_QUERY_STRATEGY_COMPARISON_METHODOLOGY),
+        strategyOrder: [...RETRIEVAL_QUERY_STRATEGY_IDS],
+        identityRedaction: run.target.privacyProjection === "external-local-redacted" ? "external-local-redacted" : null,
+        scopes: structuredClone(analysis.scopes),
+        cases,
+        limitations: [...RETRIEVAL_QUERY_STRATEGY_COMPARISON_LIMITATIONS]
+    };
+}

package/dist/src/report/experiments/contextPackGenerationReportModel.js

@@ -0,0 +1,9 @@
+export const CONTEXT_PACK_GENERATION_REPORT_SCHEMA_VERSION = "my-dev-kit-lab-context-pack-generation-report-v1";
+/** Report-presentation limits only. They never change the experimental pack limits and are not CLI configuration. */
+export const PACK_PREVIEW_MAX_FILES = 5;
+export const PACK_PREVIEW_MAX_SYMBOLS = 5;
+export const PACK_PREVIEW_MAX_SOURCE_SLICES = 3;
+export const PACK_PREVIEW_MAX_SOURCE_LINES_PER_SLICE = 12;
+export const PACK_PREVIEW_MAX_CALL_RELATIONSHIPS = 5;
+export const PACK_PREVIEW_MAX_TESTS = 5;
+export const PACK_PREVIEW_MAX_EVIDENCE_NOTES = 5;