@dailephd/my-dev-kit-lab 0.7.2 → 0.8.1

package/dist/src/experiments/plugins/retrievalPrecisionRecall/plugin.js

@@ -0,0 +1,253 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { serializeLocalRepositorySubjectManifest } from "../../../evaluation/localRepositorySubject/index.js";
+import { summarizeExperimentRun } from "../../results.js";
+import { LocalSubjectExecutionError } from "../contextWindowScaling/localSubjectErrors.js";
+import { projectExternalLocalTarget } from "../contextWindowScaling/localSubjectPrivacy.js";
+import { isSamePhysicalDirectory } from "../contextWindowScaling/localSubjectScratch.js";
+import { selectWarmIndexCases } from "../warmIndexReuse/selection.js";
+import { defaultRetrievalPrecisionRecallConfig, retrievalPrecisionRecallConfigDefinition, validateRetrievalPrecisionRecallConfig } from "./config.js";
+import { executeRetrievalPrecisionRecall } from "./execution.js";
+import { executeLocalRepositorySubjectRetrievalPrecisionRecall } from "./localSubjectExecution.js";
+import { assertExternalLocalProjectionIsPrivate, describeRetrievalLocalSubjectFailureForPersistence, projectRetrievalEvidenceForExternalLocalPersistence, UNEXPECTED_EXTERNAL_FAILURE_MESSAGE } from "./localSubjectPrivacy.js";
+import { buildRetrievalPrecisionRecallExecutionArtifact, RETRIEVAL_PRECISION_RECALL_EXECUTION_ARTIFACT_FILE } from "./executionArtifact.js";
+import { aggregateRetrievalPrecisionRecall, retrievalRatioOf, toRetrievalPrecisionRecallOutcomeMetrics, toRetrievalPrecisionRecallRunMetrics } from "./metrics.js";
+import { RETRIEVAL_PRECISION_RECALL_PLUGIN_ID, RETRIEVAL_PRECISION_RECALL_VARIANTS, RETRIEVAL_PRECISION_RECALL_VARIANT_ID, retrievalPrecisionRecallMetadata } from "./metadata.js";
+import { RETRIEVAL_PRECISION_RECALL_RATIO_KEYS } from "./types.js";
+/** Privacy-safe Batch 1 manifest persisted beside the execution artifact for external-local runs (same file as context-window-scaling). */
+export const RETRIEVAL_PRECISION_RECALL_LOCAL_SUBJECT_MANIFEST_FILE = "local-repository-subject-manifest.json";
+/** An external target must always arrive with a loaded local repository subject; there is no bundled fallback against it. */
+export const RETRIEVAL_PRECISION_RECALL_EXTERNAL_REQUIRES_SUBJECT_MESSAGE = "External-local retrieval-precision-recall targets require a loaded local repository subject (--local-subject-config).";
+export const retrievalPrecisionRecallPlugin = {
+    metadata: retrievalPrecisionRecallMetadata,
+    defaultConfig: defaultRetrievalPrecisionRecallConfig,
+    configDefinition: retrievalPrecisionRecallConfigDefinition,
+    supportedVariants: RETRIEVAL_PRECISION_RECALL_VARIANTS.map((variant) => variant.id),
+    validateConfig: validateRetrievalPrecisionRecallConfig,
+    async run(context) {
+        const startedAt = context.startedAt.toISOString();
+        // The runner does not enforce supportedTargets, so the plugin fails closed on its own.
+        const localSubject = readLocalSubjectInput(context.inputs);
+        if (localSubject)
+            return runLocalSubject(context, localSubject, startedAt);
+        if (context.target.kind !== "self" || context.target.isSelf !== true) {
+            // Never fall back to the bundled corpus against an external repository.
+            throw new Error(RETRIEVAL_PRECISION_RECALL_EXTERNAL_REQUIRES_SUBJECT_MESSAGE);
+        }
+        const cases = selectWarmIndexCases(readCasesInput(context.inputs), context.config);
+        const outDir = context.outputRoot ?? path.resolve(context.toolRoot, context.config.outDir);
+        await mkdir(outDir, { recursive: true });
+        const caseEvidence = await executeRetrievalPrecisionRecall({
+            cases,
+            kitCommand: context.config.kitCommand,
+            outputRoot: outDir,
+            dependencies: readDependenciesInput(context.inputs)
+        });
+        // Scientific truth is calculated before persistence; the artifact and every report consume it unchanged.
+        const aggregate = aggregateRetrievalPrecisionRecall(caseEvidence);
+        const completedAt = new Date().toISOString();
+        const artifact = buildRetrievalPrecisionRecallExecutionArtifact({
+            runId: context.runId,
+            pluginId: RETRIEVAL_PRECISION_RECALL_PLUGIN_ID,
+            pluginSchemaVersion: retrievalPrecisionRecallMetadata.schemaVersion,
+            startedAt,
+            completedAt,
+            cases: caseEvidence,
+            aggregate
+        });
+        const artifactPath = resolveWithinRoot(outDir, RETRIEVAL_PRECISION_RECALL_EXECUTION_ARTIFACT_FILE);
+        await writeFile(artifactPath, `${JSON.stringify(artifact, null, 2)}\n`, "utf8");
+        return mapRetrievalPrecisionRecallToRun({
+            runId: context.runId,
+            startedAt,
+            completedAt,
+            target: context.target,
+            caseEvidence,
+            aggregate,
+            artifactPath
+        });
+    },
+    summarize(result) {
+        return result.summary ?? summarizeExperimentRun(result);
+    }
+};
+function readLocalSubjectInput(inputs) {
+    const value = inputs?.localSubject;
+    return value && typeof value === "object" ? value : undefined;
+}
+/**
+ * External-local path. The safe seam owns execution, immutability and scratch cleanup and either returns evidence or
+ * throws; only after it succeeds is anything durable written: the case evidence is privacy-projected, the aggregate
+ * (calculated from the unprojected numbers) is preserved, and the artifact and manifest are written, in that order.
+ */
+async function runLocalSubject(context, subject, startedAt) {
+    if (context.target.isSelf || context.target.kind !== "external-local") {
+        throw new Error("Local-repository subject mode requires an external-local target.");
+    }
+    if (context.config.caseIds !== undefined || context.config.benchmarkProjects !== undefined) {
+        throw new Error("Case and benchmark-project filters are not supported for an external local repository subject; the subject config owns the case set.");
+    }
+    if (!(await isSamePhysicalDirectory(context.target.targetRoot, subject.repositoryRoot))) {
+        throw new Error("The selected --target is not the repository the local subject was loaded from.");
+    }
+    const outDir = context.outputRoot;
+    if (!outDir)
+        throw new Error("Local-repository subject mode requires an experiment output directory.");
+    let localResult;
+    try {
+        localResult = await executeLocalRepositorySubjectRetrievalPrecisionRecall({
+            subject,
+            kitCommand: context.config.kitCommand,
+            workRoot: outDir,
+            dependencies: readDependenciesInput(context.inputs)
+        });
+    }
+    catch (error) {
+        if (error instanceof LocalSubjectExecutionError)
+            throw new Error(describeRetrievalLocalSubjectFailureForPersistence(error));
+        throw new Error(UNEXPECTED_EXTERNAL_FAILURE_MESSAGE);
+    }
+    // Scientific numbers are calculated once from the unprojected evidence; projection only withholds identities.
+    const aggregate = aggregateRetrievalPrecisionRecall(localResult.caseEvidence);
+    const projected = projectRetrievalEvidenceForExternalLocalPersistence(localResult.caseEvidence);
+    assertExternalLocalProjectionIsPrivate(projected, {
+        filePaths: [...subject.eligibleFiles, ...subject.runtimeSafetyExclusions.gitIgnoredFiles, ...subject.runtimeSafetyExclusions.oversizedFiles],
+        caseTitles: subject.evaluationCases.map((evaluationCase) => evaluationCase.title),
+        warnings: localResult.caseEvidence.flatMap((entry) => entry.retrieval?.warnings ?? [])
+    });
+    await mkdir(outDir, { recursive: true });
+    const completedAt = new Date().toISOString();
+    const artifact = buildRetrievalPrecisionRecallExecutionArtifact({
+        runId: context.runId,
+        pluginId: RETRIEVAL_PRECISION_RECALL_PLUGIN_ID,
+        pluginSchemaVersion: retrievalPrecisionRecallMetadata.schemaVersion,
+        startedAt,
+        completedAt,
+        cases: projected,
+        aggregate
+    });
+    const artifactPath = resolveWithinRoot(outDir, RETRIEVAL_PRECISION_RECALL_EXECUTION_ARTIFACT_FILE);
+    await writeFile(artifactPath, `${JSON.stringify(artifact, null, 2)}
+`, "utf8");
+    const manifestPath = resolveWithinRoot(outDir, RETRIEVAL_PRECISION_RECALL_LOCAL_SUBJECT_MANIFEST_FILE);
+    await writeFile(manifestPath, serializeLocalRepositorySubjectManifest(subject.manifest), "utf8");
+    const run = mapRetrievalPrecisionRecallToRun({
+        runId: context.runId,
+        startedAt,
+        completedAt,
+        target: projectExternalLocalTarget(subject.manifest),
+        caseEvidence: projected,
+        aggregate,
+        artifactPath
+    });
+    run.artifacts.push({
+        id: "local-repository-subject-manifest",
+        label: "Privacy-safe local repository subject manifest",
+        path: manifestPath,
+        kind: "artifact",
+        mimeType: "application/json",
+        description: "Privacy-safe logical subject, Git, safety and aggregate inventory metadata without source or private identity lists."
+    });
+    return run;
+}
+export function mapRetrievalPrecisionRecallToRun(args) {
+    const cases = args.caseEvidence.map((entry) => ({
+        id: entry.caseId,
+        name: entry.caseName,
+        outcomes: [buildOutcome(entry)],
+        metadata: { benchmarkProject: entry.benchmarkProject, taskLocality: entry.taskLocality }
+    }));
+    const failures = args.caseEvidence.flatMap((entry) => entry.errors.map((error) => ({
+        code: error.code,
+        message: error.message,
+        variantId: RETRIEVAL_PRECISION_RECALL_VARIANT_ID,
+        caseId: entry.caseId,
+        recoverable: true
+    })));
+    const run = {
+        runId: args.runId,
+        pluginId: RETRIEVAL_PRECISION_RECALL_PLUGIN_ID,
+        startedAt: args.startedAt,
+        completedAt: args.completedAt,
+        status: aggregateStatus(args.caseEvidence.map((entry) => entry.status)),
+        target: args.target,
+        variants: RETRIEVAL_PRECISION_RECALL_VARIANTS.map((variant) => ({ ...variant })),
+        cases,
+        metrics: toRetrievalPrecisionRecallRunMetrics(args.aggregate),
+        artifacts: [
+            {
+                id: "retrieval-precision-recall-execution",
+                label: "Retrieval precision/recall execution evidence",
+                path: args.artifactPath,
+                kind: "artifact",
+                mimeType: "application/json",
+                description: "Per-case retrieval-quality evidence and aggregate without context text or raw command output."
+            }
+        ],
+        warnings: [],
+        failures,
+        metadata: { executionArtifactPath: args.artifactPath },
+        caseExecutionEvidence: args.caseEvidence.map((entry) => structuredClone(entry)),
+        aggregate: structuredClone(args.aggregate)
+    };
+    run.summary = summarizeExperimentRun(run);
+    return run;
+}
+function buildOutcome(entry) {
+    const warnings = (entry.retrieval?.warnings ?? []).map((message) => ({
+        code: "retrieval-warning",
+        message,
+        variantId: RETRIEVAL_PRECISION_RECALL_VARIANT_ID,
+        caseId: entry.caseId
+    }));
+    const failures = entry.errors.map((error) => ({
+        code: error.code,
+        message: error.message,
+        variantId: RETRIEVAL_PRECISION_RECALL_VARIANT_ID,
+        caseId: entry.caseId,
+        recoverable: true
+    }));
+    // Availability and reason stay visible even though an unavailable/not-applicable ratio projects as a null value.
+    const metricAvailability = {};
+    for (const key of RETRIEVAL_PRECISION_RECALL_RATIO_KEYS) {
+        const ratio = retrievalRatioOf(entry.quality, key);
+        metricAvailability[key] = { availability: ratio.availability, reason: ratio.reason };
+    }
+    return {
+        id: `${entry.caseId}:${RETRIEVAL_PRECISION_RECALL_VARIANT_ID}`,
+        caseId: entry.caseId,
+        variantId: RETRIEVAL_PRECISION_RECALL_VARIANT_ID,
+        status: entry.status,
+        metrics: toRetrievalPrecisionRecallOutcomeMetrics(entry.quality, RETRIEVAL_PRECISION_RECALL_VARIANT_ID, entry.caseId),
+        artifacts: [],
+        warnings,
+        failures,
+        metadata: {
+            benchmarkProject: entry.benchmarkProject,
+            evidenceAvailability: entry.retrieval?.evidenceAvailability ?? null,
+            metricAvailability
+        }
+    };
+}
+/** Same aggregation rule as the generic per-case summary: uniform statuses win, mixed is partial. */
+function aggregateStatus(statuses) {
+    if (statuses.length === 0)
+        return "skipped";
+    for (const status of ["completed", "skipped", "failed"]) {
+        if (statuses.every((value) => value === status))
+            return status;
+    }
+    return "partial";
+}
+function readCasesInput(inputs) {
+    const value = inputs?.cases;
+    if (!Array.isArray(value)) {
+        throw new Error("Retrieval precision/recall requires cases input.");
+    }
+    return value;
+}
+function readDependenciesInput(inputs) {
+    const value = inputs?.retrievalDependencies;
+    return value && typeof value === "object" && !Array.isArray(value) ? value : undefined;
+}

package/dist/src/experiments/plugins/retrievalPrecisionRecall/types.js

@@ -0,0 +1,8 @@
+export const RETRIEVAL_PRECISION_RECALL_RATIO_KEYS = [
+    "filePrecision",
+    "fileRecall",
+    "symbolPrecision",
+    "symbolRecall",
+    "factCoverage",
+    "irrelevantContextRatio"
+];

package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/analysis.js

@@ -0,0 +1,172 @@
+import { RETRIEVAL_QUERY_STRATEGY_IDS } from "../../../evaluation/retrievalQueryStrategies.js";
+import { calculateRetrievalQualityMetricsFromIdentityEvidence } from "../../../evaluation/retrievalQuality/metrics.js";
+import { RETRIEVAL_QUERY_STRATEGY_COMPARISON_SCOPE_IDS } from "./analysisTypes.js";
+function toRetrievalQualityIdentityEvidence(evidence) {
+    return {
+        availability: evidence.availability,
+        ...(evidence.availabilityReason !== null ? { availabilityReason: evidence.availabilityReason } : {}),
+        files: evidence.files.map((file) => file.path),
+        symbols: evidence.symbols.map((symbol) => ({
+            name: symbol.name,
+            ...(symbol.file !== null ? { file: symbol.file } : {})
+        }))
+    };
+}
+/**
+ * Balanced F1 from identity-set sizes: 2TP / (2TP + FP + FN). Well defined when nothing was retrieved but
+ * relevant identities exist (F1 = 0). Never averages precision and recall.
+ */
+export function calculateBalancedIdentityF1(relevant, irrelevant, missed) {
+    if (relevant === null || irrelevant === null || missed === null) {
+        return { availability: "unavailable", numerator: null, denominator: null, value: null, reason: "quality-identity-sets-unavailable" };
+    }
+    const numerator = 2 * relevant.length;
+    const denominator = numerator + irrelevant.length + missed.length;
+    if (denominator === 0) {
+        return { availability: "not-applicable", numerator: null, denominator: null, value: null, reason: "no-positive-or-retrieved-identities" };
+    }
+    return { availability: "available", numerator, denominator, value: numerator / denominator, reason: null };
+}
+const noMeasurementF1 = () => ({
+    availability: "unavailable",
+    numerator: null,
+    denominator: null,
+    value: null,
+    reason: "no-retrieval-measurement"
+});
+/** Scores already-completed retrieval evidence. It never influences retrieval. */
+export function analyzeRetrievalQueryStrategyTreatment(evaluationCase, treatment) {
+    if (treatment.evidence === null || treatment.retrieval === null) {
+        return {
+            strategyId: treatment.strategyId,
+            executionStatus: treatment.status,
+            quality: null,
+            fileF1: noMeasurementF1(),
+            symbolF1: noMeasurementF1()
+        };
+    }
+    const quality = calculateRetrievalQualityMetricsFromIdentityEvidence({
+        evaluationCase,
+        retrieval: {
+            identityEvidence: toRetrievalQualityIdentityEvidence(treatment.evidence),
+            totalEstimatedTokens: treatment.retrieval.totalEstimatedTokens,
+            tokenCountMethod: treatment.retrieval.tokenCountMethod
+        }
+    });
+    return {
+        strategyId: treatment.strategyId,
+        executionStatus: treatment.status,
+        quality,
+        fileF1: calculateBalancedIdentityF1(quality.file.relevantRetrievedFiles, quality.file.irrelevantRetrievedFiles, quality.file.missedFiles),
+        symbolF1: calculateBalancedIdentityF1(quality.symbol.relevantRetrievedSymbols, quality.symbol.irrelevantRetrievedSymbols, quality.symbol.missedSymbols)
+    };
+}
+export function analyzeRetrievalQueryStrategyCase(evaluationCase, executionCase) {
+    if (evaluationCase.id !== executionCase.caseId) {
+        throw new Error("Retrieval query strategy analysis case identity mismatch.");
+    }
+    const order = executionCase.treatments.map((treatment) => treatment.strategyId);
+    if (order.length !== RETRIEVAL_QUERY_STRATEGY_IDS.length || order.some((id, index) => id !== RETRIEVAL_QUERY_STRATEGY_IDS[index])) {
+        throw new Error("Retrieval query strategy analysis requires the seven canonical treatments in canonical order.");
+    }
+    return {
+        caseId: executionCase.caseId,
+        benchmarkProject: executionCase.benchmarkProject,
+        taskLocality: executionCase.taskLocality,
+        treatments: executionCase.treatments.map((treatment) => analyzeRetrievalQueryStrategyTreatment(evaluationCase, treatment))
+    };
+}
+function isTreatmentComparable(treatment) {
+    const quality = treatment.quality;
+    return (quality !== null &&
+        treatment.fileF1.availability === "available" &&
+        treatment.symbolF1.availability === "available" &&
+        quality.fact.coverage.availability === "available" &&
+        typeof quality.retrievedTokenCount === "number" &&
+        Number.isFinite(quality.retrievedTokenCount) &&
+        quality.retrievedTokenCount >= 0);
+}
+/** Matched design: a case counts only when all seven strategies expose all four objectives. */
+export function isCaseComparableAcrossAllStrategies(caseAnalysis) {
+    return caseAnalysis.treatments.length === RETRIEVAL_QUERY_STRATEGY_IDS.length && caseAnalysis.treatments.every(isTreatmentComparable);
+}
+function casesForScope(cases, scopeId) {
+    return scopeId === "overall" ? [...cases] : cases.filter((entry) => entry.taskLocality === scopeId);
+}
+/** No epsilon, tolerance, rounding or weighting. */
+export function paretoDominatesRetrievalStrategy(left, right) {
+    const noWorse = left.meanFileF1 >= right.meanFileF1 &&
+        left.meanSymbolF1 >= right.meanSymbolF1 &&
+        left.meanFactCoverage >= right.meanFactCoverage &&
+        left.meanRetrievedTokenCount <= right.meanRetrievedTokenCount;
+    const strictlyBetter = left.meanFileF1 > right.meanFileF1 ||
+        left.meanSymbolF1 > right.meanSymbolF1 ||
+        left.meanFactCoverage > right.meanFactCoverage ||
+        left.meanRetrievedTokenCount < right.meanRetrievedTokenCount;
+    return noWorse && strictlyBetter;
+}
+/** Every nondominated strategy in the order given (canonical strategy order); never a performance ranking. */
+export function findRetrievalStrategyParetoFront(summaries) {
+    const candidates = summaries.filter((summary) => summary.objectives !== null);
+    return candidates
+        .filter((candidate) => !candidates.some((other) => other !== candidate && paretoDominatesRetrievalStrategy(other.objectives, candidate.objectives)))
+        .map((candidate) => candidate.strategyId);
+}
+export function aggregateRetrievalQueryStrategyScope(scopeId, cases) {
+    const scopeCases = casesForScope(cases, scopeId);
+    const comparisonCases = scopeCases.filter(isCaseComparableAcrossAllStrategies);
+    const strategySummaries = RETRIEVAL_QUERY_STRATEGY_IDS.map((strategyId, index) => {
+        const treatmentsOfScope = scopeCases.map((entry) => entry.treatments[index]);
+        const countStatus = (status) => treatmentsOfScope.filter((treatment) => treatment.executionStatus === status).length;
+        let objectives = null;
+        if (comparisonCases.length > 0) {
+            const values = comparisonCases.map((entry) => {
+                const treatment = entry.treatments[index];
+                return {
+                    fileF1: treatment.fileF1.value,
+                    symbolF1: treatment.symbolF1.value,
+                    factCoverage: treatment.quality.fact.coverage.value,
+                    retrievedTokens: treatment.quality.retrievedTokenCount
+                };
+            });
+            const mean = (select) => values.reduce((sum, value) => sum + select(value), 0) / values.length;
+            objectives = {
+                meanFileF1: mean((value) => value.fileF1),
+                meanSymbolF1: mean((value) => value.symbolF1),
+                meanFactCoverage: mean((value) => value.factCoverage),
+                meanRetrievedTokenCount: mean((value) => value.retrievedTokens)
+            };
+        }
+        return {
+            strategyId,
+            completedCaseCount: countStatus("completed"),
+            partialCaseCount: countStatus("partial"),
+            failedCaseCount: countStatus("failed"),
+            objectives
+        };
+    });
+    const paretoFrontStrategyIds = comparisonCases.length === 0 ? [] : findRetrievalStrategyParetoFront(strategySummaries);
+    const unique = paretoFrontStrategyIds.length === 1;
+    return {
+        scopeId,
+        caseCount: scopeCases.length,
+        comparisonCaseCount: comparisonCases.length,
+        excludedCaseCount: scopeCases.length - comparisonCases.length,
+        comparisonCaseIds: comparisonCases.map((entry) => entry.caseId),
+        strategySummaries,
+        paretoFrontStrategyIds,
+        bestStrategyId: unique ? paretoFrontStrategyIds[0] : null,
+        interpretation: comparisonCases.length === 0 ? "unavailable" : unique ? "unique-best" : "tradeoff"
+    };
+}
+export function analyzeRetrievalQueryStrategyComparison(evaluationCases, executionCases) {
+    if (evaluationCases.length !== executionCases.length ||
+        evaluationCases.some((evaluationCase, index) => evaluationCase.id !== executionCases[index].caseId)) {
+        throw new Error("Retrieval query strategy analysis requires matching selected case IDs in the same order.");
+    }
+    const cases = evaluationCases.map((evaluationCase, index) => analyzeRetrievalQueryStrategyCase(evaluationCase, executionCases[index]));
+    return {
+        cases,
+        scopes: RETRIEVAL_QUERY_STRATEGY_COMPARISON_SCOPE_IDS.map((scopeId) => aggregateRetrievalQueryStrategyScope(scopeId, cases))
+    };
+}

package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/analysisArtifact.js

@@ -0,0 +1,28 @@
+export const RETRIEVAL_QUERY_STRATEGY_COMPARISON_ANALYSIS_ARTIFACT_FILE = "retrieval-query-strategy-comparison-analysis.json";
+export const RETRIEVAL_QUERY_STRATEGY_COMPARISON_ANALYSIS_SCHEMA_VERSION = "my-dev-kit-lab-retrieval-query-strategy-comparison-analysis-v1";
+/** Single owner of the frozen methodology; the artifact and the report both use it. */
+export const RETRIEVAL_QUERY_STRATEGY_COMPARISON_METHODOLOGY = {
+    fileF1: "balanced-f1",
+    symbolF1: "balanced-f1",
+    aggregation: "matched-complete-case-macro-mean",
+    multiObjectiveComparison: "pareto-dominance",
+    uniqueBestRule: "single-member-pareto-front",
+    objectiveDirections: {
+        meanFileF1: "maximize",
+        meanSymbolF1: "maximize",
+        meanFactCoverage: "maximize",
+        meanRetrievedTokenCount: "minimize"
+    }
+};
+export function buildRetrievalQueryStrategyComparisonAnalysisArtifact(args) {
+    return {
+        schemaVersion: RETRIEVAL_QUERY_STRATEGY_COMPARISON_ANALYSIS_SCHEMA_VERSION,
+        runId: args.runId,
+        pluginId: args.pluginId,
+        pluginSchemaVersion: args.pluginSchemaVersion,
+        startedAt: args.startedAt,
+        completedAt: args.completedAt,
+        methodology: structuredClone(RETRIEVAL_QUERY_STRATEGY_COMPARISON_METHODOLOGY),
+        analysis: structuredClone(args.analysis)
+    };
+}

package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/analysisTypes.js

@@ -0,0 +1,2 @@
+/** Comparison scopes, in output order. Named scopes reuse the existing TASK_LOCALITIES taxonomy. */
+export const RETRIEVAL_QUERY_STRATEGY_COMPARISON_SCOPE_IDS = ["overall", "localized", "cross-module", "broad-change"];

package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/config.js

@@ -0,0 +1,39 @@
+import { invalidExperimentConfig, isPlainObject, mergeConfig, validExperimentConfig } from "../../config.js";
+export const defaultRetrievalQueryStrategyComparisonConfig = {
+    outDir: "lab-output/retrieval-query-strategy-comparison",
+    kitCommand: "npx @dailephd/my-dev-kit@latest"
+};
+const SUPPORTED_FIELDS = ["outDir", "kitCommand", "caseIds", "benchmarkProjects"];
+export const retrievalQueryStrategyComparisonConfigDefinition = {
+    fields: [
+        { name: "outDir", type: "string", required: true, description: "Experiment output root." },
+        { name: "kitCommand", type: "string", description: "my-dev-kit command used for index and retrieval." },
+        { name: "caseIds", type: "array", description: "Bundled evaluation case IDs to include; corpus order is preserved." },
+        { name: "benchmarkProjects", type: "array", description: "Bundled benchmark project IDs to include; corpus order is preserved." }
+    ]
+};
+export function validateRetrievalQueryStrategyComparisonConfig(config) {
+    if (config !== undefined && !isPlainObject(config)) {
+        return invalidExperimentConfig(["retrieval query strategy comparison config must be an object."]);
+    }
+    const errors = [];
+    const unsupported = Object.keys(config ?? {}).filter((key) => !SUPPORTED_FIELDS.includes(key));
+    if (unsupported.length > 0) {
+        errors.push(`Unsupported retrieval-query-strategy-comparison config field(s): ${unsupported.sort().join(", ")}.`);
+    }
+    const normalized = mergeConfig(defaultRetrievalQueryStrategyComparisonConfig, config);
+    for (const field of ["outDir", "kitCommand"]) {
+        const value = normalized[field];
+        if (typeof value !== "string" || !value.trim())
+            errors.push(`${field} must be a non-empty string.`);
+    }
+    for (const field of ["caseIds", "benchmarkProjects"]) {
+        const value = normalized[field];
+        if (value === undefined)
+            continue;
+        if (!Array.isArray(value) || value.length === 0 || !value.every((item) => typeof item === "string" && item.trim())) {
+            errors.push(`${field} must be a non-empty array of non-empty strings when provided.`);
+        }
+    }
+    return errors.length > 0 ? invalidExperimentConfig(errors) : validExperimentConfig(normalized);
+}