@dailephd/my-dev-kit-lab 0.8.0 → 0.8.2

package/dist/src/experiments/plugins/contextPackGeneration/renderContextPack.js

@@ -0,0 +1,61 @@
+/** Stable source boundary; it cannot be confused with Markdown code-fence nesting. */
+export const CONTEXT_PACK_SOURCE_BEGIN = "<<<SOURCE";
+export const CONTEXT_PACK_SOURCE_END = "SOURCE>>>";
+export const CONTEXT_PACK_SECTION_HEADINGS = {
+    task: "## Task",
+    files: "## Relevant files",
+    symbols: "## Relevant symbols",
+    sourceSlices: "## Source slices",
+    callRelationships: "## Call relationships",
+    tests: "## Tests",
+    evidenceNotes: "## Evidence notes"
+};
+const lf = (text) => text.replace(/\r\n?/g, "\n");
+function renderBody(id, lines, availability) {
+    const heading = CONTEXT_PACK_SECTION_HEADINGS[id];
+    if (lines.length > 0)
+        return [heading, ...lines].join("\n");
+    const { availability: state, reason } = availability[id];
+    const suffix = reason === null ? "" : `: ${reason}`;
+    const placeholder = state === "available" ? "(none)" : `(${state}${suffix})`;
+    return `${heading}\n${placeholder}`;
+}
+function renderSlice(slice) {
+    return [
+        `file: ${slice.file}`,
+        `lines: ${slice.startLine}-${slice.endLine}`,
+        `boundary: ${slice.boundaryKnown ? "known" : "unknown"}`,
+        `truncated: ${slice.truncated ? "yes" : "no"}`,
+        CONTEXT_PACK_SOURCE_BEGIN,
+        lf(slice.text),
+        CONTEXT_PACK_SOURCE_END
+    ].join("\n");
+}
+/** Deterministic, language-agnostic rendering. Source text is verbatim apart from line-ending normalization. */
+export function renderContextPack(input) {
+    const availability = input.sectionAvailability;
+    const sections = {
+        task: `${CONTEXT_PACK_SECTION_HEADINGS.task}\nTitle: ${lf(input.task.title)}\nSummary: ${lf(input.task.summary)}`,
+        files: renderBody("files", input.files.map((file) => `- ${file.path}`), availability),
+        symbols: renderBody("symbols", input.symbols.map((symbol) => (symbol.file === null ? `- ${symbol.name}` : `- ${symbol.name} (${symbol.file})`)), availability),
+        sourceSlices: renderSourceSlices(input.sourceSlices, availability),
+        callRelationships: renderBody("callRelationships", input.callRelationships.map((call) => `${call.fromNodeId} -> ${call.toNodeId}`), availability),
+        tests: renderBody("tests", input.tests.map((test) => `- ${test.path}`), availability),
+        evidenceNotes: renderBody("evidenceNotes", input.evidenceNotes.map((code) => `- ${code}`), availability)
+    };
+    const renderedText = `${[
+        sections.task,
+        sections.files,
+        sections.symbols,
+        sections.sourceSlices,
+        sections.callRelationships,
+        sections.tests,
+        sections.evidenceNotes
+    ].join("\n\n")}\n`;
+    return { sections, renderedText };
+}
+function renderSourceSlices(slices, availability) {
+    if (slices.length === 0)
+        return renderBody("sourceSlices", [], availability);
+    return `${CONTEXT_PACK_SECTION_HEADINGS.sourceSlices}\n${slices.map(renderSlice).join("\n\n")}`;
+}

package/dist/src/experiments/plugins/contextPackGeneration/types.js

@@ -0,0 +1,29 @@
+/** Domain schema of the Lab-owned experimental context pack. It is a measurement artifact, not a production contract. */
+export const CONTEXT_PACK_SCHEMA_VERSION = "my-dev-kit-lab-context-pack-experiment-v1";
+/** Logical section order of the pack. */
+export const CONTEXT_PACK_SECTION_IDS = [
+    "task",
+    "files",
+    "symbols",
+    "sourceSlices",
+    "callRelationships",
+    "tests",
+    "evidenceNotes"
+];
+/**
+ * Closed fixed vocabulary. Only codes produced by the pure Batch 1 domain exist; no free-form warning text, and no
+ * execution, indexing, privacy or persistence failure codes.
+ */
+export const CONTEXT_PACK_EVIDENCE_NOTE_CODES = [
+    "symbol-end-unknown",
+    "source-slice-cap-reached",
+    "total-source-line-cap-reached",
+    "source-slice-count-cap-reached",
+    "seed-cap-reached",
+    "file-cap-reached",
+    "symbol-cap-reached",
+    "test-file-cap-reached",
+    "call-relationship-cap-reached",
+    "invalid-repository-path-ignored",
+    "invalid-source-slice-ignored"
+];

package/dist/src/experiments/plugins/contextWindowScaling/localSubjectPrivacy.js

@@ -83,6 +83,8 @@ export function projectRunForExternalLocalPersistence(run, target) {
         metadata.outputRoot = EXTERNAL_LOCAL_REDACTED_VALUE;
     if (typeof metadata.executionArtifactPath === "string")
         metadata.executionArtifactPath = path.basename(metadata.executionArtifactPath);
+    if (typeof metadata.analysisArtifactPath === "string")
+        metadata.analysisArtifactPath = path.basename(metadata.analysisArtifactPath);
     return { ...run, target, metadata: metadata };
 }
 const GENERIC_ISSUE_TEXT = {

package/dist/src/experiments/plugins/index.js

@@ -3,3 +3,4 @@ export * from "./contextWindowScaling/index.js";
 export * from "./incrementalChangeStaleness/index.js";
 export * from "./warmIndexReuse/index.js";
 export * from "./retrievalPrecisionRecall/index.js";
+export * from "./retrievalQueryStrategyComparison/index.js";

package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/analysis.js

@@ -0,0 +1,172 @@
+import { RETRIEVAL_QUERY_STRATEGY_IDS } from "../../../evaluation/retrievalQueryStrategies.js";
+import { calculateRetrievalQualityMetricsFromIdentityEvidence } from "../../../evaluation/retrievalQuality/metrics.js";
+import { RETRIEVAL_QUERY_STRATEGY_COMPARISON_SCOPE_IDS } from "./analysisTypes.js";
+function toRetrievalQualityIdentityEvidence(evidence) {
+    return {
+        availability: evidence.availability,
+        ...(evidence.availabilityReason !== null ? { availabilityReason: evidence.availabilityReason } : {}),
+        files: evidence.files.map((file) => file.path),
+        symbols: evidence.symbols.map((symbol) => ({
+            name: symbol.name,
+            ...(symbol.file !== null ? { file: symbol.file } : {})
+        }))
+    };
+}
+/**
+ * Balanced F1 from identity-set sizes: 2TP / (2TP + FP + FN). Well defined when nothing was retrieved but
+ * relevant identities exist (F1 = 0). Never averages precision and recall.
+ */
+export function calculateBalancedIdentityF1(relevant, irrelevant, missed) {
+    if (relevant === null || irrelevant === null || missed === null) {
+        return { availability: "unavailable", numerator: null, denominator: null, value: null, reason: "quality-identity-sets-unavailable" };
+    }
+    const numerator = 2 * relevant.length;
+    const denominator = numerator + irrelevant.length + missed.length;
+    if (denominator === 0) {
+        return { availability: "not-applicable", numerator: null, denominator: null, value: null, reason: "no-positive-or-retrieved-identities" };
+    }
+    return { availability: "available", numerator, denominator, value: numerator / denominator, reason: null };
+}
+const noMeasurementF1 = () => ({
+    availability: "unavailable",
+    numerator: null,
+    denominator: null,
+    value: null,
+    reason: "no-retrieval-measurement"
+});
+/** Scores already-completed retrieval evidence. It never influences retrieval. */
+export function analyzeRetrievalQueryStrategyTreatment(evaluationCase, treatment) {
+    if (treatment.evidence === null || treatment.retrieval === null) {
+        return {
+            strategyId: treatment.strategyId,
+            executionStatus: treatment.status,
+            quality: null,
+            fileF1: noMeasurementF1(),
+            symbolF1: noMeasurementF1()
+        };
+    }
+    const quality = calculateRetrievalQualityMetricsFromIdentityEvidence({
+        evaluationCase,
+        retrieval: {
+            identityEvidence: toRetrievalQualityIdentityEvidence(treatment.evidence),
+            totalEstimatedTokens: treatment.retrieval.totalEstimatedTokens,
+            tokenCountMethod: treatment.retrieval.tokenCountMethod
+        }
+    });
+    return {
+        strategyId: treatment.strategyId,
+        executionStatus: treatment.status,
+        quality,
+        fileF1: calculateBalancedIdentityF1(quality.file.relevantRetrievedFiles, quality.file.irrelevantRetrievedFiles, quality.file.missedFiles),
+        symbolF1: calculateBalancedIdentityF1(quality.symbol.relevantRetrievedSymbols, quality.symbol.irrelevantRetrievedSymbols, quality.symbol.missedSymbols)
+    };
+}
+export function analyzeRetrievalQueryStrategyCase(evaluationCase, executionCase) {
+    if (evaluationCase.id !== executionCase.caseId) {
+        throw new Error("Retrieval query strategy analysis case identity mismatch.");
+    }
+    const order = executionCase.treatments.map((treatment) => treatment.strategyId);
+    if (order.length !== RETRIEVAL_QUERY_STRATEGY_IDS.length || order.some((id, index) => id !== RETRIEVAL_QUERY_STRATEGY_IDS[index])) {
+        throw new Error("Retrieval query strategy analysis requires the seven canonical treatments in canonical order.");
+    }
+    return {
+        caseId: executionCase.caseId,
+        benchmarkProject: executionCase.benchmarkProject,
+        taskLocality: executionCase.taskLocality,
+        treatments: executionCase.treatments.map((treatment) => analyzeRetrievalQueryStrategyTreatment(evaluationCase, treatment))
+    };
+}
+function isTreatmentComparable(treatment) {
+    const quality = treatment.quality;
+    return (quality !== null &&
+        treatment.fileF1.availability === "available" &&
+        treatment.symbolF1.availability === "available" &&
+        quality.fact.coverage.availability === "available" &&
+        typeof quality.retrievedTokenCount === "number" &&
+        Number.isFinite(quality.retrievedTokenCount) &&
+        quality.retrievedTokenCount >= 0);
+}
+/** Matched design: a case counts only when all seven strategies expose all four objectives. */
+export function isCaseComparableAcrossAllStrategies(caseAnalysis) {
+    return caseAnalysis.treatments.length === RETRIEVAL_QUERY_STRATEGY_IDS.length && caseAnalysis.treatments.every(isTreatmentComparable);
+}
+function casesForScope(cases, scopeId) {
+    return scopeId === "overall" ? [...cases] : cases.filter((entry) => entry.taskLocality === scopeId);
+}
+/** No epsilon, tolerance, rounding or weighting. */
+export function paretoDominatesRetrievalStrategy(left, right) {
+    const noWorse = left.meanFileF1 >= right.meanFileF1 &&
+        left.meanSymbolF1 >= right.meanSymbolF1 &&
+        left.meanFactCoverage >= right.meanFactCoverage &&
+        left.meanRetrievedTokenCount <= right.meanRetrievedTokenCount;
+    const strictlyBetter = left.meanFileF1 > right.meanFileF1 ||
+        left.meanSymbolF1 > right.meanSymbolF1 ||
+        left.meanFactCoverage > right.meanFactCoverage ||
+        left.meanRetrievedTokenCount < right.meanRetrievedTokenCount;
+    return noWorse && strictlyBetter;
+}
+/** Every nondominated strategy in the order given (canonical strategy order); never a performance ranking. */
+export function findRetrievalStrategyParetoFront(summaries) {
+    const candidates = summaries.filter((summary) => summary.objectives !== null);
+    return candidates
+        .filter((candidate) => !candidates.some((other) => other !== candidate && paretoDominatesRetrievalStrategy(other.objectives, candidate.objectives)))
+        .map((candidate) => candidate.strategyId);
+}
+export function aggregateRetrievalQueryStrategyScope(scopeId, cases) {
+    const scopeCases = casesForScope(cases, scopeId);
+    const comparisonCases = scopeCases.filter(isCaseComparableAcrossAllStrategies);
+    const strategySummaries = RETRIEVAL_QUERY_STRATEGY_IDS.map((strategyId, index) => {
+        const treatmentsOfScope = scopeCases.map((entry) => entry.treatments[index]);
+        const countStatus = (status) => treatmentsOfScope.filter((treatment) => treatment.executionStatus === status).length;
+        let objectives = null;
+        if (comparisonCases.length > 0) {
+            const values = comparisonCases.map((entry) => {
+                const treatment = entry.treatments[index];
+                return {
+                    fileF1: treatment.fileF1.value,
+                    symbolF1: treatment.symbolF1.value,
+                    factCoverage: treatment.quality.fact.coverage.value,
+                    retrievedTokens: treatment.quality.retrievedTokenCount
+                };
+            });
+            const mean = (select) => values.reduce((sum, value) => sum + select(value), 0) / values.length;
+            objectives = {
+                meanFileF1: mean((value) => value.fileF1),
+                meanSymbolF1: mean((value) => value.symbolF1),
+                meanFactCoverage: mean((value) => value.factCoverage),
+                meanRetrievedTokenCount: mean((value) => value.retrievedTokens)
+            };
+        }
+        return {
+            strategyId,
+            completedCaseCount: countStatus("completed"),
+            partialCaseCount: countStatus("partial"),
+            failedCaseCount: countStatus("failed"),
+            objectives
+        };
+    });
+    const paretoFrontStrategyIds = comparisonCases.length === 0 ? [] : findRetrievalStrategyParetoFront(strategySummaries);
+    const unique = paretoFrontStrategyIds.length === 1;
+    return {
+        scopeId,
+        caseCount: scopeCases.length,
+        comparisonCaseCount: comparisonCases.length,
+        excludedCaseCount: scopeCases.length - comparisonCases.length,
+        comparisonCaseIds: comparisonCases.map((entry) => entry.caseId),
+        strategySummaries,
+        paretoFrontStrategyIds,
+        bestStrategyId: unique ? paretoFrontStrategyIds[0] : null,
+        interpretation: comparisonCases.length === 0 ? "unavailable" : unique ? "unique-best" : "tradeoff"
+    };
+}
+export function analyzeRetrievalQueryStrategyComparison(evaluationCases, executionCases) {
+    if (evaluationCases.length !== executionCases.length ||
+        evaluationCases.some((evaluationCase, index) => evaluationCase.id !== executionCases[index].caseId)) {
+        throw new Error("Retrieval query strategy analysis requires matching selected case IDs in the same order.");
+    }
+    const cases = evaluationCases.map((evaluationCase, index) => analyzeRetrievalQueryStrategyCase(evaluationCase, executionCases[index]));
+    return {
+        cases,
+        scopes: RETRIEVAL_QUERY_STRATEGY_COMPARISON_SCOPE_IDS.map((scopeId) => aggregateRetrievalQueryStrategyScope(scopeId, cases))
+    };
+}

package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/analysisArtifact.js

@@ -0,0 +1,28 @@
+export const RETRIEVAL_QUERY_STRATEGY_COMPARISON_ANALYSIS_ARTIFACT_FILE = "retrieval-query-strategy-comparison-analysis.json";
+export const RETRIEVAL_QUERY_STRATEGY_COMPARISON_ANALYSIS_SCHEMA_VERSION = "my-dev-kit-lab-retrieval-query-strategy-comparison-analysis-v1";
+/** Single owner of the frozen methodology; the artifact and the report both use it. */
+export const RETRIEVAL_QUERY_STRATEGY_COMPARISON_METHODOLOGY = {
+    fileF1: "balanced-f1",
+    symbolF1: "balanced-f1",
+    aggregation: "matched-complete-case-macro-mean",
+    multiObjectiveComparison: "pareto-dominance",
+    uniqueBestRule: "single-member-pareto-front",
+    objectiveDirections: {
+        meanFileF1: "maximize",
+        meanSymbolF1: "maximize",
+        meanFactCoverage: "maximize",
+        meanRetrievedTokenCount: "minimize"
+    }
+};
+export function buildRetrievalQueryStrategyComparisonAnalysisArtifact(args) {
+    return {
+        schemaVersion: RETRIEVAL_QUERY_STRATEGY_COMPARISON_ANALYSIS_SCHEMA_VERSION,
+        runId: args.runId,
+        pluginId: args.pluginId,
+        pluginSchemaVersion: args.pluginSchemaVersion,
+        startedAt: args.startedAt,
+        completedAt: args.completedAt,
+        methodology: structuredClone(RETRIEVAL_QUERY_STRATEGY_COMPARISON_METHODOLOGY),
+        analysis: structuredClone(args.analysis)
+    };
+}

package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/analysisTypes.js

@@ -0,0 +1,2 @@
+/** Comparison scopes, in output order. Named scopes reuse the existing TASK_LOCALITIES taxonomy. */
+export const RETRIEVAL_QUERY_STRATEGY_COMPARISON_SCOPE_IDS = ["overall", "localized", "cross-module", "broad-change"];

package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/config.js

@@ -0,0 +1,39 @@
+import { invalidExperimentConfig, isPlainObject, mergeConfig, validExperimentConfig } from "../../config.js";
+export const defaultRetrievalQueryStrategyComparisonConfig = {
+    outDir: "lab-output/retrieval-query-strategy-comparison",
+    kitCommand: "npx @dailephd/my-dev-kit@latest"
+};
+const SUPPORTED_FIELDS = ["outDir", "kitCommand", "caseIds", "benchmarkProjects"];
+export const retrievalQueryStrategyComparisonConfigDefinition = {
+    fields: [
+        { name: "outDir", type: "string", required: true, description: "Experiment output root." },
+        { name: "kitCommand", type: "string", description: "my-dev-kit command used for index and retrieval." },
+        { name: "caseIds", type: "array", description: "Bundled evaluation case IDs to include; corpus order is preserved." },
+        { name: "benchmarkProjects", type: "array", description: "Bundled benchmark project IDs to include; corpus order is preserved." }
+    ]
+};
+export function validateRetrievalQueryStrategyComparisonConfig(config) {
+    if (config !== undefined && !isPlainObject(config)) {
+        return invalidExperimentConfig(["retrieval query strategy comparison config must be an object."]);
+    }
+    const errors = [];
+    const unsupported = Object.keys(config ?? {}).filter((key) => !SUPPORTED_FIELDS.includes(key));
+    if (unsupported.length > 0) {
+        errors.push(`Unsupported retrieval-query-strategy-comparison config field(s): ${unsupported.sort().join(", ")}.`);
+    }
+    const normalized = mergeConfig(defaultRetrievalQueryStrategyComparisonConfig, config);
+    for (const field of ["outDir", "kitCommand"]) {
+        const value = normalized[field];
+        if (typeof value !== "string" || !value.trim())
+            errors.push(`${field} must be a non-empty string.`);
+    }
+    for (const field of ["caseIds", "benchmarkProjects"]) {
+        const value = normalized[field];
+        if (value === undefined)
+            continue;
+        if (!Array.isArray(value) || value.length === 0 || !value.every((item) => typeof item === "string" && item.trim())) {
+            errors.push(`${field} must be a non-empty array of non-empty strings when provided.`);
+        }
+    }
+    return errors.length > 0 ? invalidExperimentConfig(errors) : validExperimentConfig(normalized);
+}

package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/execution.js

@@ -0,0 +1,225 @@
+import { cp, rm } from "node:fs/promises";
+import path from "node:path";
+import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { validateRetrievalPrecisionRecallCase } from "../../../evaluation/retrievalQuality/index.js";
+import { RETRIEVAL_QUERY_STRATEGY_IDS, isCoreGraphRetrievalQueryStrategyId } from "../../../evaluation/retrievalQueryStrategies.js";
+import { buildMyDevKitIndex, runMyDevKitRetrievalStrategyFromIndex } from "../../../evaluation/runMyDevKitRetrieval.js";
+import { runSemanticRetrievalStrategyFromIndex } from "../../../evaluation/runSemanticRetrievalStrategy.js";
+import { groupWarmIndexCases, taskOutputSegment } from "../warmIndexReuse/selection.js";
+import { boundedRetrievalSafeMessage } from "../retrievalPrecisionRecall/execution.js";
+const MAX_WARNINGS = 20;
+export const SEMANTIC_INDEX_CLEANUP_FAILED_WARNING = "semantic-index-cleanup-failed";
+function failedTreatment(strategyId, code, message) {
+    return { strategyId, status: "failed", retrieval: null, evidence: null, errors: [{ code, message }] };
+}
+/**
+ * The single normalization point from an executed strategy result into bounded comparison evidence. It takes no
+ * contextText and inspects no raw command output. An available observation (including an available empty one)
+ * is COMPLETED; partial or unavailable evidence from a measured result is PARTIAL.
+ */
+export function buildQueryStrategyTreatmentEvidence(strategyId, result) {
+    const evidence = result.queryStrategyEvidence;
+    return {
+        strategyId,
+        status: evidence.availability === "available" ? "completed" : "partial",
+        retrieval: {
+            skipped: result.skipped,
+            durationMs: result.durationMs,
+            totalEstimatedTokens: result.totalEstimatedTokens,
+            tokenCountMethod: result.tokenCountMethod,
+            warnings: result.warnings.slice(0, MAX_WARNINGS).map((warning) => boundedRetrievalSafeMessage(warning)),
+            evidenceAvailability: evidence.availability,
+            evidenceAvailabilityReason: evidence.availabilityReason,
+            retrievedFileCount: evidence.files.length,
+            retrievedSymbolCount: evidence.symbols.length,
+            steps: evidence.steps.map((step) => ({ ...step }))
+        },
+        evidence: structuredClone(evidence),
+        errors: []
+    };
+}
+function failedCase(evaluationCase, code, message) {
+    return {
+        caseId: evaluationCase.id,
+        caseName: evaluationCase.title,
+        benchmarkProject: evaluationCase.benchmarkProject,
+        taskLocality: evaluationCase.taskLocality ?? null,
+        treatments: RETRIEVAL_QUERY_STRATEGY_IDS.map((strategyId) => failedTreatment(strategyId, code, message))
+    };
+}
+/**
+ * Matched seven-treatment execution. One base index per benchmark project; the five core strategies share it;
+ * each semantic case/treatment gets a private copy of it because upstream `data-model` writes derived artifacts
+ * into the index directory. No agent runs and no quality metric is calculated. Only bounded, context-free
+ * evidence is returned.
+ */
+export async function executeRetrievalQueryStrategyComparison(options) {
+    const dependencies = {
+        buildIndex: options.dependencies?.buildIndex ?? buildMyDevKitIndex,
+        runCoreStrategy: options.dependencies?.runCoreStrategy ?? runMyDevKitRetrievalStrategyFromIndex,
+        runSemanticStrategy: options.dependencies?.runSemanticStrategy ?? runSemanticRetrievalStrategyFromIndex
+    };
+    const results = new Map();
+    const validCases = [];
+    for (const evaluationCase of options.cases) {
+        if (validateRetrievalPrecisionRecallCase(evaluationCase).length > 0) {
+            results.set(evaluationCase.id, failedCase(evaluationCase, "ground-truth-invalid", "Retrieval comparison ground truth is invalid for this case."));
+        }
+        else {
+            validCases.push(evaluationCase);
+        }
+    }
+    for (const group of groupWarmIndexCases(validCases)) {
+        if (group.structuralErrors.length > 0) {
+            for (const evaluationCase of group.cases) {
+                results.set(evaluationCase.id, failedCase(evaluationCase, "project-group-inconsistent", "Cases of one benchmark project disagree on target root or source roots."));
+            }
+            continue;
+        }
+        const baseIndexDir = resolveWithinRoot(options.outputRoot, path.join("indexes", group.projectSegment, "base"));
+        const projectCommandsDir = resolveWithinRoot(options.outputRoot, path.join("commands", group.projectSegment));
+        let indexReady = false;
+        try {
+            // requireKit: false keeps a failed index as measurable evidence instead of throwing. Full, ordinary index only.
+            const build = await dependencies.buildIndex({
+                target: { absoluteTargetRoot: group.targetRoot, sourceRoots: [...group.sourceRoots] },
+                kitCommand: options.kitCommand,
+                indexDir: baseIndexDir,
+                commandsDir: path.join(projectCommandsDir, "index"),
+                requireKit: false
+            });
+            indexReady = build.ok;
+        }
+        catch {
+            indexReady = false;
+        }
+        if (!indexReady) {
+            for (const evaluationCase of group.cases) {
+                results.set(evaluationCase.id, failedCase(evaluationCase, "project-index-failed", "The benchmark-project index could not be prepared."));
+            }
+            continue;
+        }
+        for (const evaluationCase of group.cases) {
+            const caseSegment = taskOutputSegment(evaluationCase.id);
+            results.set(evaluationCase.id, await executeRetrievalQueryStrategyCaseFromPreparedIndex({
+                evaluationCase,
+                kitCommand: options.kitCommand,
+                baseIndexDir,
+                commandsDir: resolveWithinRoot(projectCommandsDir, caseSegment),
+                semanticIndexesRoot: resolveWithinRoot(options.outputRoot, path.join("strategy-indexes", group.projectSegment, caseSegment)),
+                dependencies
+            }));
+        }
+    }
+    // Selection order is the corpus order after filtering, regardless of how projects were grouped.
+    return options.cases.map((evaluationCase) => results.get(evaluationCase.id));
+}
+/**
+ * Executes exactly seven treatments for one already-validated case against one already-prepared base index. Core
+ * treatments share `baseIndexDir`; each semantic treatment gets its own fresh recursive copy under
+ * `semanticIndexesRoot`. It builds no index, validates no ground truth, and calculates no scientific analysis.
+ */
+export async function executeRetrievalQueryStrategyCaseFromPreparedIndex(options) {
+    const { evaluationCase } = options;
+    const dependencies = {
+        runCoreStrategy: options.dependencies?.runCoreStrategy ?? runMyDevKitRetrievalStrategyFromIndex,
+        runSemanticStrategy: options.dependencies?.runSemanticStrategy ?? runSemanticRetrievalStrategyFromIndex
+    };
+    const treatments = [];
+    // Strictly sequential, in the frozen strategy order: deterministic chronology and failure ordering.
+    for (const strategyId of RETRIEVAL_QUERY_STRATEGY_IDS) {
+        const commandsDir = resolveWithinRoot(options.commandsDir, strategyId);
+        treatments.push(isCoreGraphRetrievalQueryStrategyId(strategyId)
+            ? await executeCoreTreatment({
+                strategyId,
+                evaluationCase,
+                kitCommand: options.kitCommand,
+                baseIndexDir: options.baseIndexDir,
+                commandsDir,
+                dependencies
+            })
+            : await executeSemanticTreatment({
+                strategyId,
+                evaluationCase,
+                kitCommand: options.kitCommand,
+                baseIndexDir: options.baseIndexDir,
+                treatmentIndexDir: resolveWithinRoot(options.semanticIndexesRoot, strategyId),
+                commandsDir,
+                dependencies
+            }));
+    }
+    return {
+        caseId: evaluationCase.id,
+        caseName: evaluationCase.title,
+        benchmarkProject: evaluationCase.benchmarkProject,
+        taskLocality: evaluationCase.taskLocality ?? null,
+        treatments
+    };
+}
+const SEMANTIC_INDEX_CLEANUP_FAILED_MESSAGE = "The isolated semantic-strategy index could not be removed.";
+const RETRIEVAL_THREW_MESSAGE = "The retrieval strategy threw before producing a measurement.";
+async function executeCoreTreatment(args) {
+    try {
+        const result = await args.dependencies.runCoreStrategy({
+            strategyId: args.strategyId,
+            evaluationCase: args.evaluationCase,
+            kitCommand: args.kitCommand,
+            indexDir: args.baseIndexDir,
+            commandsDir: args.commandsDir,
+            requireKit: false
+        });
+        const evidence = result.queryStrategyEvidence;
+        if (!evidence)
+            return failedTreatment(args.strategyId, "retrieval-failed", RETRIEVAL_THREW_MESSAGE);
+        return buildQueryStrategyTreatmentEvidence(args.strategyId, { ...result, queryStrategyEvidence: evidence });
+    }
+    catch {
+        return failedTreatment(args.strategyId, "retrieval-failed", RETRIEVAL_THREW_MESSAGE);
+    }
+}
+async function executeSemanticTreatment(args) {
+    try {
+        // Fresh copy of the untouched base index; the destination must not pre-exist.
+        await rm(args.treatmentIndexDir, { recursive: true, force: true });
+        await cp(args.baseIndexDir, args.treatmentIndexDir, { recursive: true });
+    }
+    catch {
+        await rm(args.treatmentIndexDir, { recursive: true, force: true }).catch(() => undefined);
+        return failedTreatment(args.strategyId, "strategy-index-copy-failed", "The isolated semantic-strategy index could not be prepared.");
+    }
+    let treatment;
+    let cleanupFailed = false;
+    try {
+        const result = await args.dependencies.runSemanticStrategy({
+            strategyId: args.strategyId,
+            evaluationCase: args.evaluationCase,
+            kitCommand: args.kitCommand,
+            indexDir: args.treatmentIndexDir,
+            commandsDir: args.commandsDir,
+            requireKit: false
+        });
+        treatment = buildQueryStrategyTreatmentEvidence(args.strategyId, result);
+    }
+    catch {
+        treatment = failedTreatment(args.strategyId, "retrieval-failed", RETRIEVAL_THREW_MESSAGE);
+    }
+    finally {
+        try {
+            await rm(args.treatmentIndexDir, { recursive: true, force: true });
+        }
+        catch {
+            // A cleanup failure never invalidates a measurement; it is recorded as a fixed, path-free warning.
+            cleanupFailed = true;
+        }
+    }
+    if (cleanupFailed) {
+        if (treatment.retrieval) {
+            treatment.retrieval.warnings = [...treatment.retrieval.warnings, SEMANTIC_INDEX_CLEANUP_FAILED_WARNING];
+        }
+        else {
+            // No measurement exists: keep the primary failure and make the leftover copied index auditable (path-free).
+            treatment.errors = [...treatment.errors, { code: "strategy-index-cleanup-failed", message: SEMANTIC_INDEX_CLEANUP_FAILED_MESSAGE }];
+        }
+    }
+    return treatment;
+}

package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/executionArtifact.js

@@ -0,0 +1,15 @@
+import { RETRIEVAL_QUERY_STRATEGY_IDS } from "../../../evaluation/retrievalQueryStrategies.js";
+export const RETRIEVAL_QUERY_STRATEGY_COMPARISON_EXECUTION_ARTIFACT_FILE = "retrieval-query-strategy-comparison-execution.json";
+export const RETRIEVAL_QUERY_STRATEGY_COMPARISON_EXECUTION_SCHEMA_VERSION = "my-dev-kit-lab-retrieval-query-strategy-comparison-execution-v1";
+export function buildRetrievalQueryStrategyComparisonExecutionArtifact(args) {
+    return {
+        schemaVersion: RETRIEVAL_QUERY_STRATEGY_COMPARISON_EXECUTION_SCHEMA_VERSION,
+        runId: args.runId,
+        pluginId: args.pluginId,
+        pluginSchemaVersion: args.pluginSchemaVersion,
+        startedAt: args.startedAt,
+        completedAt: args.completedAt,
+        strategyOrder: [...RETRIEVAL_QUERY_STRATEGY_IDS],
+        cases: structuredClone(args.cases)
+    };
+}

package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/index.js

@@ -0,0 +1,12 @@
+export * from "./analysis.js";
+export * from "./analysisArtifact.js";
+export * from "./analysisTypes.js";
+export * from "./config.js";
+export * from "./execution.js";
+export * from "./executionArtifact.js";
+export * from "./localSubjectExecution.js";
+export * from "./localSubjectPrivacy.js";
+export * from "./metadata.js";
+export * from "./metrics.js";
+export * from "./plugin.js";
+export * from "./types.js";