@dailephd/my-dev-kit-lab 0.8.0 → 0.8.1

package/dist/src/evaluation/runSemanticRetrievalStrategy.js

@@ -0,0 +1,303 @@
+/**
+ * Semantic retrieval strategies (`data-model-graph`, `model-view-lineage`) executed against an
+ * already prepared my-dev-kit index through `data-model` queries.
+ *
+ * The target is chosen from the task query and the upstream data-model graph only; the answer key is
+ * never read. Output is normalized into `retrieval-query-strategy-evidence-v1`, never into
+ * `retrieval-evidence-v1`, because these commands are not search/lookup/slice/source.
+ */
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { countEstimatedTokens, countTextChars, tokenCountMethod } from "../core/countTokens.js";
+import { runMeasuredCommand } from "../core/runMeasuredCommand.js";
+import { compareCodeUnits } from "./indexSnapshot.js";
+import { normalizeRetrievedRepositoryPath } from "./retrievalQuality/buildRetrievalEvidence.js";
+import { canonicalizeQueryStrategyIdentities } from "./retrievalQueryStrategyEvidence.js";
+const REJECTED_REASON = "some-semantic-identities-were-rejected";
+function isRecord(value) {
+    return !!value && typeof value === "object" && !Array.isArray(value);
+}
+function parseJsonIfPossible(text) {
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Lowercases, splits on every run of non ASCII alphanumerics, drops tokens shorter than two
+ * characters, deduplicates, and sorts in code-unit order. No stemming, synonyms, or stop words.
+ */
+export function tokenizeSemanticRetrievalQuery(query) {
+    const tokens = query
+        .toLowerCase()
+        .split(/[^A-Za-z0-9]+/)
+        .filter((token) => token.length >= 2);
+    return [...new Set(tokens)].sort(compareCodeUnits);
+}
+/** Count of unique query tokens that occur exactly as label tokens. */
+export function scoreSemanticLabel(queryTokens, label) {
+    const labelTokens = new Set(tokenizeSemanticRetrievalQuery(label));
+    let score = 0;
+    for (const token of new Set(queryTokens)) {
+        if (labelTokens.has(token))
+            score += 1;
+    }
+    return score;
+}
+function step(kind, succeeded, evidenceAvailable, reason) {
+    return { kind, succeeded, evidenceAvailable, reason };
+}
+function buildEvidence(strategyId, availability, availabilityReason, identities, steps) {
+    const canonical = canonicalizeQueryStrategyIdentities(identities.files.map((file) => ({ path: file })), identities.symbols);
+    return {
+        schemaVersion: "retrieval-query-strategy-evidence-v1",
+        strategyId,
+        availability,
+        availabilityReason,
+        files: canonical.files,
+        symbols: canonical.symbols,
+        steps
+    };
+}
+function unavailableEvidence(strategyId, reason, steps) {
+    return buildEvidence(strategyId, "unavailable", reason, { files: [], symbols: [] }, steps);
+}
+async function readDataModelGraph(indexDir) {
+    let text;
+    try {
+        text = await readFile(path.join(indexDir, "data-model-graph.json"), "utf8");
+    }
+    catch {
+        return { ok: false, reason: "data-model-graph-artifact-missing" };
+    }
+    const payload = parseJsonIfPossible(text);
+    if (!isRecord(payload))
+        return { ok: false, reason: "data-model-graph-artifact-malformed" };
+    if (payload.artifactKind !== "my-dev-kit-v1-data-model-graph" ||
+        typeof payload.schemaVersion !== "string" ||
+        !payload.schemaVersion.startsWith("1.")) {
+        return { ok: false, reason: "data-model-graph-schema-unsupported" };
+    }
+    if (!Array.isArray(payload.nodes) || !Array.isArray(payload.edges)) {
+        return { ok: false, reason: "data-model-graph-artifact-malformed" };
+    }
+    const nodes = [];
+    for (const node of payload.nodes) {
+        if (isRecord(node) &&
+            typeof node.id === "string" &&
+            typeof node.label === "string" &&
+            (node.kind === "entity" || node.kind === "field")) {
+            nodes.push({ id: node.id, kind: node.kind, label: node.label });
+        }
+    }
+    return { ok: true, nodes };
+}
+/** Highest score wins; ties break on node id in code-unit order. No candidate means no match. */
+function selectSemanticCandidate(queryTokens, nodes) {
+    let best;
+    for (const node of nodes) {
+        const score = scoreSemanticLabel(queryTokens, node.label);
+        if (score <= 0)
+            continue;
+        if (!best || score > best.score || (score === best.score && compareCodeUnits(node.id, best.node.id) < 0)) {
+            best = { node, score };
+        }
+    }
+    return best?.node;
+}
+/** Normalizes `filePath` of every ref; unsafe values are counted, never kept. */
+function collectSafeFiles(refs) {
+    const files = [];
+    let rejected = 0;
+    for (const ref of refs) {
+        const normalized = isRecord(ref) ? normalizeRetrievedRepositoryPath(ref.filePath) : null;
+        if (normalized === null) {
+            rejected += 1;
+        }
+        else {
+            files.push(normalized);
+        }
+    }
+    return { files, rejected };
+}
+function uniqueFileOrNull(files) {
+    const unique = [...new Set(files)];
+    return unique.length === 1 ? unique[0] : null;
+}
+function readEntityOutput(payload, expectedMode) {
+    if (!isRecord(payload) || payload.status !== "ok" || typeof payload.mode !== "string")
+        return "malformed";
+    if (payload.mode !== expectedMode)
+        return "mismatched";
+    return payload;
+}
+function readIdentity(value) {
+    return isRecord(value) && typeof value.id === "string" && typeof value.name === "string"
+        ? { id: value.id, name: value.name }
+        : undefined;
+}
+function buildDataModelGraphStrategyEvidence(selectedNode, lookupPayload, generationCommand, lookupCommand) {
+    const strategyId = "data-model-graph";
+    const generationStep = step("data-model", generationCommand.ok, generationCommand.ok, null);
+    const failed = (reason) => unavailableEvidence(strategyId, reason, [generationStep, step("data-model", lookupCommand.ok, false, reason)]);
+    if (!lookupCommand.ok)
+        return failed("data-model-lookup-command-failed");
+    const expectedMode = selectedNode.kind === "entity" ? "entity" : "field";
+    const output = readEntityOutput(lookupPayload, expectedMode);
+    if (output === "malformed")
+        return failed("data-model-lookup-output-malformed");
+    if (output === "mismatched")
+        return failed("data-model-lookup-output-mismatched");
+    const selected = readIdentity(expectedMode === "entity" ? output.entity : output.field);
+    if (!selected || !Array.isArray(output.sourceRefs))
+        return failed("data-model-lookup-output-malformed");
+    const { files, rejected } = collectSafeFiles(output.sourceRefs);
+    const symbol = {
+        name: selected.name,
+        nodeId: selected.id,
+        file: uniqueFileOrNull(files)
+    };
+    const partial = rejected > 0;
+    return buildEvidence(strategyId, partial ? "partial" : "available", partial ? REJECTED_REASON : null, { files, symbols: [symbol] }, [generationStep, step("data-model", true, !partial, partial ? REJECTED_REASON : null)]);
+}
+function buildModelViewLineageStrategyEvidence(tracePayload, generationCommand, traceCommand, expectedMode) {
+    const strategyId = "model-view-lineage";
+    const generationStep = step("data-model", generationCommand.ok, generationCommand.ok, null);
+    const failed = (reason) => unavailableEvidence(strategyId, reason, [generationStep, step("model-view-lineage", traceCommand.ok, false, reason)]);
+    if (!traceCommand.ok)
+        return failed("model-view-lineage-command-failed");
+    const output = readEntityOutput(tracePayload, expectedMode);
+    if (output === "malformed")
+        return failed("model-view-lineage-output-malformed");
+    if (output === "mismatched")
+        return failed("model-view-lineage-output-mismatched");
+    const lineage = output.lineage;
+    if (!readIdentity(output.entity) || !isRecord(lineage) || !Array.isArray(lineage.nodes) || !Array.isArray(lineage.edges)) {
+        return failed("model-view-lineage-output-malformed");
+    }
+    const files = [];
+    const symbols = [];
+    let rejected = 0;
+    for (const node of lineage.nodes) {
+        if (!isRecord(node) || typeof node.id !== "string" || typeof node.label !== "string") {
+            rejected += 1;
+            continue;
+        }
+        let nodeFiles = [];
+        if (Array.isArray(node.evidenceRefs)) {
+            const collected = collectSafeFiles(node.evidenceRefs);
+            nodeFiles = collected.files;
+            rejected += collected.rejected;
+        }
+        else if (node.evidenceRefs !== undefined) {
+            rejected += 1;
+        }
+        files.push(...nodeFiles);
+        symbols.push({ name: node.label, nodeId: node.id, file: uniqueFileOrNull(nodeFiles) });
+    }
+    const partial = rejected > 0;
+    return buildEvidence(strategyId, partial ? "partial" : "available", partial ? REJECTED_REASON : null, { files, symbols }, [generationStep, step("model-view-lineage", true, !partial, partial ? REJECTED_REASON : null)]);
+}
+async function runDataModelCommand(options, commandId, args, commands) {
+    const command = await runMeasuredCommand({
+        commandId,
+        commandString: options.kitCommand,
+        cwd: process.cwd(),
+        outDir: options.commandsDir,
+        extraArgs: ["data-model", "--index", options.indexDir, ...args, "--json"]
+    });
+    commands.push(command);
+    if (!command.ok && options.requireKit) {
+        throw new Error(command.error || `my-dev-kit data-model failed with exit code ${command.exitCode}`);
+    }
+    return command;
+}
+function buildResult(options, started, commands, evidence, contextText) {
+    const warnings = [];
+    if (evidence.availability === "unavailable") {
+        warnings.push(`my-dev-kit semantic retrieval was unavailable: ${evidence.availabilityReason}.`);
+    }
+    else if (evidence.availability === "partial") {
+        warnings.push(`my-dev-kit semantic retrieval was partially interpreted: ${evidence.availabilityReason}.`);
+    }
+    const text = evidence.availability === "unavailable" ? "" : contextText;
+    return {
+        caseId: options.evaluationCase.id,
+        strategyId: options.strategyId,
+        skipped: text.length === 0,
+        warnings,
+        totalChars: countTextChars(text),
+        totalEstimatedTokens: countEstimatedTokens(text),
+        tokenCountMethod,
+        contextText: text,
+        commands,
+        queryStrategyEvidence: evidence,
+        durationMs: Date.now() - started
+    };
+}
+/** Shared by both strategies: generate artifacts, validate the graph, select one semantic node. */
+async function resolveSemanticTarget(options, commands) {
+    const generation = await runDataModelCommand(options, "data-model", [], commands);
+    if (!generation.ok) {
+        return {
+            kind: "done",
+            evidence: unavailableEvidence(options.strategyId, "data-model-command-failed", [
+                step("data-model", false, false, "data-model-command-failed")
+            ])
+        };
+    }
+    const generationStep = step("data-model", true, true, null);
+    const graph = await readDataModelGraph(options.indexDir);
+    if (!graph.ok) {
+        return {
+            kind: "done",
+            evidence: unavailableEvidence(options.strategyId, graph.reason, [step("data-model", true, false, graph.reason)])
+        };
+    }
+    const node = selectSemanticCandidate(tokenizeSemanticRetrievalQuery(options.evaluationCase.query), graph.nodes);
+    if (!node) {
+        return {
+            kind: "done",
+            evidence: buildEvidence(options.strategyId, "available", null, { files: [], symbols: [] }, [generationStep])
+        };
+    }
+    return { kind: "node", node, generation };
+}
+async function executeDataModelGraphStrategy(options, started) {
+    const commands = [];
+    const resolved = await resolveSemanticTarget(options, commands);
+    if (resolved.kind === "done")
+        return buildResult(options, started, commands, resolved.evidence, "");
+    const selector = resolved.node.kind === "entity" ? ["--entity", resolved.node.label] : ["--field", resolved.node.label];
+    const lookup = await runDataModelCommand(options, "data-model-selected", selector, commands);
+    const evidence = buildDataModelGraphStrategyEvidence(resolved.node, lookup.ok ? parseJsonIfPossible(lookup.stdout) : undefined, resolved.generation, lookup);
+    return buildResult(options, started, commands, evidence, lookup.stdout);
+}
+async function executeModelViewLineageStrategy(options, started) {
+    const commands = [];
+    const resolved = await resolveSemanticTarget(options, commands);
+    if (resolved.kind === "done")
+        return buildResult(options, started, commands, resolved.evidence, "");
+    const isEntity = resolved.node.kind === "entity";
+    const selector = isEntity ? ["--trace-view", resolved.node.label] : ["--field", resolved.node.label, "--trace-view"];
+    const trace = await runDataModelCommand(options, "model-view-lineage", selector, commands);
+    const evidence = buildModelViewLineageStrategyEvidence(trace.ok ? parseJsonIfPossible(trace.stdout) : undefined, resolved.generation, trace, isEntity ? "trace-entity" : "trace-field");
+    return buildResult(options, started, commands, evidence, trace.stdout);
+}
+/**
+ * Executes one semantic retrieval strategy against an already prepared index. Never invokes `index`,
+ * never reads the answer key, never runs agents.
+ */
+export async function runSemanticRetrievalStrategyFromIndex(options) {
+    const started = Date.now();
+    switch (options.strategyId) {
+        case "data-model-graph":
+            return executeDataModelGraphStrategy(options, started);
+        case "model-view-lineage":
+            return executeModelViewLineageStrategy(options, started);
+        default:
+            throw new Error(`Unsupported semantic retrieval strategy: ${String(options.strategyId)}`);
+    }
+}

package/dist/src/experiments/defaultRegistry.js

@@ -3,6 +3,7 @@ import { contextStrategyComparisonPlugin } from "./plugins/contextStrategyCompar
 import { contextWindowScalingPlugin } from "./plugins/contextWindowScaling/index.js";
 import { incrementalChangeStalenessPlugin } from "./plugins/incrementalChangeStaleness/index.js";
 import { retrievalPrecisionRecallPlugin } from "./plugins/retrievalPrecisionRecall/index.js";
+import { retrievalQueryStrategyComparisonPlugin } from "./plugins/retrievalQueryStrategyComparison/index.js";
 import { warmIndexReusePlugin } from "./plugins/warmIndexReuse/index.js";
 export function createDefaultExperimentPluginRegistry() {
     const registry = new ExperimentPluginRegistry();
@@ -11,5 +12,6 @@ export function createDefaultExperimentPluginRegistry() {
     registry.register(incrementalChangeStalenessPlugin);
     registry.register(contextWindowScalingPlugin);
     registry.register(retrievalPrecisionRecallPlugin);
+    registry.register(retrievalQueryStrategyComparisonPlugin);
     return registry;
 }

package/dist/src/experiments/plugins/contextWindowScaling/localSubjectPrivacy.js

@@ -83,6 +83,8 @@ export function projectRunForExternalLocalPersistence(run, target) {
         metadata.outputRoot = EXTERNAL_LOCAL_REDACTED_VALUE;
     if (typeof metadata.executionArtifactPath === "string")
         metadata.executionArtifactPath = path.basename(metadata.executionArtifactPath);
+    if (typeof metadata.analysisArtifactPath === "string")
+        metadata.analysisArtifactPath = path.basename(metadata.analysisArtifactPath);
     return { ...run, target, metadata: metadata };
 }
 const GENERIC_ISSUE_TEXT = {

package/dist/src/experiments/plugins/index.js

@@ -3,3 +3,4 @@ export * from "./contextWindowScaling/index.js";
 export * from "./incrementalChangeStaleness/index.js";
 export * from "./warmIndexReuse/index.js";
 export * from "./retrievalPrecisionRecall/index.js";
+export * from "./retrievalQueryStrategyComparison/index.js";

package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/analysis.js

@@ -0,0 +1,172 @@
+import { RETRIEVAL_QUERY_STRATEGY_IDS } from "../../../evaluation/retrievalQueryStrategies.js";
+import { calculateRetrievalQualityMetricsFromIdentityEvidence } from "../../../evaluation/retrievalQuality/metrics.js";
+import { RETRIEVAL_QUERY_STRATEGY_COMPARISON_SCOPE_IDS } from "./analysisTypes.js";
+function toRetrievalQualityIdentityEvidence(evidence) {
+    return {
+        availability: evidence.availability,
+        ...(evidence.availabilityReason !== null ? { availabilityReason: evidence.availabilityReason } : {}),
+        files: evidence.files.map((file) => file.path),
+        symbols: evidence.symbols.map((symbol) => ({
+            name: symbol.name,
+            ...(symbol.file !== null ? { file: symbol.file } : {})
+        }))
+    };
+}
+/**
+ * Balanced F1 from identity-set sizes: 2TP / (2TP + FP + FN). Well defined when nothing was retrieved but
+ * relevant identities exist (F1 = 0). Never averages precision and recall.
+ */
+export function calculateBalancedIdentityF1(relevant, irrelevant, missed) {
+    if (relevant === null || irrelevant === null || missed === null) {
+        return { availability: "unavailable", numerator: null, denominator: null, value: null, reason: "quality-identity-sets-unavailable" };
+    }
+    const numerator = 2 * relevant.length;
+    const denominator = numerator + irrelevant.length + missed.length;
+    if (denominator === 0) {
+        return { availability: "not-applicable", numerator: null, denominator: null, value: null, reason: "no-positive-or-retrieved-identities" };
+    }
+    return { availability: "available", numerator, denominator, value: numerator / denominator, reason: null };
+}
+const noMeasurementF1 = () => ({
+    availability: "unavailable",
+    numerator: null,
+    denominator: null,
+    value: null,
+    reason: "no-retrieval-measurement"
+});
+/** Scores already-completed retrieval evidence. It never influences retrieval. */
+export function analyzeRetrievalQueryStrategyTreatment(evaluationCase, treatment) {
+    if (treatment.evidence === null || treatment.retrieval === null) {
+        return {
+            strategyId: treatment.strategyId,
+            executionStatus: treatment.status,
+            quality: null,
+            fileF1: noMeasurementF1(),
+            symbolF1: noMeasurementF1()
+        };
+    }
+    const quality = calculateRetrievalQualityMetricsFromIdentityEvidence({
+        evaluationCase,
+        retrieval: {
+            identityEvidence: toRetrievalQualityIdentityEvidence(treatment.evidence),
+            totalEstimatedTokens: treatment.retrieval.totalEstimatedTokens,
+            tokenCountMethod: treatment.retrieval.tokenCountMethod
+        }
+    });
+    return {
+        strategyId: treatment.strategyId,
+        executionStatus: treatment.status,
+        quality,
+        fileF1: calculateBalancedIdentityF1(quality.file.relevantRetrievedFiles, quality.file.irrelevantRetrievedFiles, quality.file.missedFiles),
+        symbolF1: calculateBalancedIdentityF1(quality.symbol.relevantRetrievedSymbols, quality.symbol.irrelevantRetrievedSymbols, quality.symbol.missedSymbols)
+    };
+}
+export function analyzeRetrievalQueryStrategyCase(evaluationCase, executionCase) {
+    if (evaluationCase.id !== executionCase.caseId) {
+        throw new Error("Retrieval query strategy analysis case identity mismatch.");
+    }
+    const order = executionCase.treatments.map((treatment) => treatment.strategyId);
+    if (order.length !== RETRIEVAL_QUERY_STRATEGY_IDS.length || order.some((id, index) => id !== RETRIEVAL_QUERY_STRATEGY_IDS[index])) {
+        throw new Error("Retrieval query strategy analysis requires the seven canonical treatments in canonical order.");
+    }
+    return {
+        caseId: executionCase.caseId,
+        benchmarkProject: executionCase.benchmarkProject,
+        taskLocality: executionCase.taskLocality,
+        treatments: executionCase.treatments.map((treatment) => analyzeRetrievalQueryStrategyTreatment(evaluationCase, treatment))
+    };
+}
+function isTreatmentComparable(treatment) {
+    const quality = treatment.quality;
+    return (quality !== null &&
+        treatment.fileF1.availability === "available" &&
+        treatment.symbolF1.availability === "available" &&
+        quality.fact.coverage.availability === "available" &&
+        typeof quality.retrievedTokenCount === "number" &&
+        Number.isFinite(quality.retrievedTokenCount) &&
+        quality.retrievedTokenCount >= 0);
+}
+/** Matched design: a case counts only when all seven strategies expose all four objectives. */
+export function isCaseComparableAcrossAllStrategies(caseAnalysis) {
+    return caseAnalysis.treatments.length === RETRIEVAL_QUERY_STRATEGY_IDS.length && caseAnalysis.treatments.every(isTreatmentComparable);
+}
+function casesForScope(cases, scopeId) {
+    return scopeId === "overall" ? [...cases] : cases.filter((entry) => entry.taskLocality === scopeId);
+}
+/** No epsilon, tolerance, rounding or weighting. */
+export function paretoDominatesRetrievalStrategy(left, right) {
+    const noWorse = left.meanFileF1 >= right.meanFileF1 &&
+        left.meanSymbolF1 >= right.meanSymbolF1 &&
+        left.meanFactCoverage >= right.meanFactCoverage &&
+        left.meanRetrievedTokenCount <= right.meanRetrievedTokenCount;
+    const strictlyBetter = left.meanFileF1 > right.meanFileF1 ||
+        left.meanSymbolF1 > right.meanSymbolF1 ||
+        left.meanFactCoverage > right.meanFactCoverage ||
+        left.meanRetrievedTokenCount < right.meanRetrievedTokenCount;
+    return noWorse && strictlyBetter;
+}
+/** Every nondominated strategy in the order given (canonical strategy order); never a performance ranking. */
+export function findRetrievalStrategyParetoFront(summaries) {
+    const candidates = summaries.filter((summary) => summary.objectives !== null);
+    return candidates
+        .filter((candidate) => !candidates.some((other) => other !== candidate && paretoDominatesRetrievalStrategy(other.objectives, candidate.objectives)))
+        .map((candidate) => candidate.strategyId);
+}
+export function aggregateRetrievalQueryStrategyScope(scopeId, cases) {
+    const scopeCases = casesForScope(cases, scopeId);
+    const comparisonCases = scopeCases.filter(isCaseComparableAcrossAllStrategies);
+    const strategySummaries = RETRIEVAL_QUERY_STRATEGY_IDS.map((strategyId, index) => {
+        const treatmentsOfScope = scopeCases.map((entry) => entry.treatments[index]);
+        const countStatus = (status) => treatmentsOfScope.filter((treatment) => treatment.executionStatus === status).length;
+        let objectives = null;
+        if (comparisonCases.length > 0) {
+            const values = comparisonCases.map((entry) => {
+                const treatment = entry.treatments[index];
+                return {
+                    fileF1: treatment.fileF1.value,
+                    symbolF1: treatment.symbolF1.value,
+                    factCoverage: treatment.quality.fact.coverage.value,
+                    retrievedTokens: treatment.quality.retrievedTokenCount
+                };
+            });
+            const mean = (select) => values.reduce((sum, value) => sum + select(value), 0) / values.length;
+            objectives = {
+                meanFileF1: mean((value) => value.fileF1),
+                meanSymbolF1: mean((value) => value.symbolF1),
+                meanFactCoverage: mean((value) => value.factCoverage),
+                meanRetrievedTokenCount: mean((value) => value.retrievedTokens)
+            };
+        }
+        return {
+            strategyId,
+            completedCaseCount: countStatus("completed"),
+            partialCaseCount: countStatus("partial"),
+            failedCaseCount: countStatus("failed"),
+            objectives
+        };
+    });
+    const paretoFrontStrategyIds = comparisonCases.length === 0 ? [] : findRetrievalStrategyParetoFront(strategySummaries);
+    const unique = paretoFrontStrategyIds.length === 1;
+    return {
+        scopeId,
+        caseCount: scopeCases.length,
+        comparisonCaseCount: comparisonCases.length,
+        excludedCaseCount: scopeCases.length - comparisonCases.length,
+        comparisonCaseIds: comparisonCases.map((entry) => entry.caseId),
+        strategySummaries,
+        paretoFrontStrategyIds,
+        bestStrategyId: unique ? paretoFrontStrategyIds[0] : null,
+        interpretation: comparisonCases.length === 0 ? "unavailable" : unique ? "unique-best" : "tradeoff"
+    };
+}
+export function analyzeRetrievalQueryStrategyComparison(evaluationCases, executionCases) {
+    if (evaluationCases.length !== executionCases.length ||
+        evaluationCases.some((evaluationCase, index) => evaluationCase.id !== executionCases[index].caseId)) {
+        throw new Error("Retrieval query strategy analysis requires matching selected case IDs in the same order.");
+    }
+    const cases = evaluationCases.map((evaluationCase, index) => analyzeRetrievalQueryStrategyCase(evaluationCase, executionCases[index]));
+    return {
+        cases,
+        scopes: RETRIEVAL_QUERY_STRATEGY_COMPARISON_SCOPE_IDS.map((scopeId) => aggregateRetrievalQueryStrategyScope(scopeId, cases))
+    };
+}

package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/analysisArtifact.js

@@ -0,0 +1,28 @@
+export const RETRIEVAL_QUERY_STRATEGY_COMPARISON_ANALYSIS_ARTIFACT_FILE = "retrieval-query-strategy-comparison-analysis.json";
+export const RETRIEVAL_QUERY_STRATEGY_COMPARISON_ANALYSIS_SCHEMA_VERSION = "my-dev-kit-lab-retrieval-query-strategy-comparison-analysis-v1";
+/** Single owner of the frozen methodology; the artifact and the report both use it. */
+export const RETRIEVAL_QUERY_STRATEGY_COMPARISON_METHODOLOGY = {
+    fileF1: "balanced-f1",
+    symbolF1: "balanced-f1",
+    aggregation: "matched-complete-case-macro-mean",
+    multiObjectiveComparison: "pareto-dominance",
+    uniqueBestRule: "single-member-pareto-front",
+    objectiveDirections: {
+        meanFileF1: "maximize",
+        meanSymbolF1: "maximize",
+        meanFactCoverage: "maximize",
+        meanRetrievedTokenCount: "minimize"
+    }
+};
+export function buildRetrievalQueryStrategyComparisonAnalysisArtifact(args) {
+    return {
+        schemaVersion: RETRIEVAL_QUERY_STRATEGY_COMPARISON_ANALYSIS_SCHEMA_VERSION,
+        runId: args.runId,
+        pluginId: args.pluginId,
+        pluginSchemaVersion: args.pluginSchemaVersion,
+        startedAt: args.startedAt,
+        completedAt: args.completedAt,
+        methodology: structuredClone(RETRIEVAL_QUERY_STRATEGY_COMPARISON_METHODOLOGY),
+        analysis: structuredClone(args.analysis)
+    };
+}

package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/analysisTypes.js

@@ -0,0 +1,2 @@
+/** Comparison scopes, in output order. Named scopes reuse the existing TASK_LOCALITIES taxonomy. */
+export const RETRIEVAL_QUERY_STRATEGY_COMPARISON_SCOPE_IDS = ["overall", "localized", "cross-module", "broad-change"];

package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/config.js

@@ -0,0 +1,39 @@
+import { invalidExperimentConfig, isPlainObject, mergeConfig, validExperimentConfig } from "../../config.js";
+export const defaultRetrievalQueryStrategyComparisonConfig = {
+    outDir: "lab-output/retrieval-query-strategy-comparison",
+    kitCommand: "npx @dailephd/my-dev-kit@latest"
+};
+const SUPPORTED_FIELDS = ["outDir", "kitCommand", "caseIds", "benchmarkProjects"];
+export const retrievalQueryStrategyComparisonConfigDefinition = {
+    fields: [
+        { name: "outDir", type: "string", required: true, description: "Experiment output root." },
+        { name: "kitCommand", type: "string", description: "my-dev-kit command used for index and retrieval." },
+        { name: "caseIds", type: "array", description: "Bundled evaluation case IDs to include; corpus order is preserved." },
+        { name: "benchmarkProjects", type: "array", description: "Bundled benchmark project IDs to include; corpus order is preserved." }
+    ]
+};
+export function validateRetrievalQueryStrategyComparisonConfig(config) {
+    if (config !== undefined && !isPlainObject(config)) {
+        return invalidExperimentConfig(["retrieval query strategy comparison config must be an object."]);
+    }
+    const errors = [];
+    const unsupported = Object.keys(config ?? {}).filter((key) => !SUPPORTED_FIELDS.includes(key));
+    if (unsupported.length > 0) {
+        errors.push(`Unsupported retrieval-query-strategy-comparison config field(s): ${unsupported.sort().join(", ")}.`);
+    }
+    const normalized = mergeConfig(defaultRetrievalQueryStrategyComparisonConfig, config);
+    for (const field of ["outDir", "kitCommand"]) {
+        const value = normalized[field];
+        if (typeof value !== "string" || !value.trim())
+            errors.push(`${field} must be a non-empty string.`);
+    }
+    for (const field of ["caseIds", "benchmarkProjects"]) {
+        const value = normalized[field];
+        if (value === undefined)
+            continue;
+        if (!Array.isArray(value) || value.length === 0 || !value.every((item) => typeof item === "string" && item.trim())) {
+            errors.push(`${field} must be a non-empty array of non-empty strings when provided.`);
+        }
+    }
+    return errors.length > 0 ? invalidExperimentConfig(errors) : validExperimentConfig(normalized);
+}