@dailephd/my-dev-kit-lab 0.5.0 → 0.5.2

package/dist/src/evaluation/benchmarkMetadata.js

@@ -1,7 +1,8 @@
 import { existsSync } from "node:fs";
 import { readFile } from "node:fs/promises";
 import path from "node:path";
-import { PROJECT_COMPLEXITY_FORMULA, calculateProjectComplexityScore } from "./projectComplexity.js";
+import { PROJECT_COMPLEXITY_FORMULA, calculateProjectComplexityScore, roundToTwo } from "./projectComplexity.js";
+import { TASK_LOCALITIES } from "./types.js";
 export const REQUIRED_BENCHMARK_PROJECT_IDS = [
     "todo-ts",
     "todo-python",
@@ -11,6 +12,8 @@ export const REQUIRED_BENCHMARK_PROJECT_IDS = [
     "task-analytics-large-mixed"
 ];
 export const VALID_COMPLEXITY_LEVELS = new Set(["small", "medium", "large", "mixed-language"]);
+export const REQUIRED_WARM_INDEX_BENCHMARK_PROJECT_IDS = ["task-workflow-medium-ts", "task-analytics-large-mixed"];
+export const MIN_WARM_INDEX_TASKS_PER_PROJECT = 5;
 export async function readBenchmarkProjectProfiles(profilesPath, repoRoot = process.cwd()) {
     let parsed;
     try {
@@ -95,6 +98,193 @@ export function validateAnswerKey(answerKey, label) {
     }
     return errors;
 }
+export function validateTaskLocality(value, label) {
+    if (typeof value === "string" && TASK_LOCALITIES.includes(value)) {
+        return [];
+    }
+    return [`${label}: taskLocality must be one of ${TASK_LOCALITIES.join(", ")} (received ${JSON.stringify(value)}).`];
+}
+export function deriveWarmIndexTaskStats(cases, projectId) {
+    const projectCases = cases.filter((benchmarkCase) => benchmarkCase?.benchmarkProject === projectId);
+    if (projectCases.length === 0) {
+        return { taskCount: 0, expectedRelevantFilesAverage: 0, expectedRelevantSymbolsAverage: 0 };
+    }
+    const fileCounts = projectCases.map((benchmarkCase) => (Array.isArray(benchmarkCase.expectedFiles) ? benchmarkCase.expectedFiles.length : 0));
+    const symbolCounts = projectCases.map((benchmarkCase) => Array.isArray(benchmarkCase.expectedSymbols) ? benchmarkCase.expectedSymbols.length : 0);
+    return {
+        taskCount: projectCases.length,
+        expectedRelevantFilesAverage: roundToTwo(fileCounts.reduce((total, count) => total + count, 0) / projectCases.length),
+        expectedRelevantSymbolsAverage: roundToTwo(symbolCounts.reduce((total, count) => total + count, 0) / projectCases.length)
+    };
+}
+export function validateWarmIndexBenchmarkCases(cases, profiles, repoRoot = process.cwd()) {
+    const errors = [];
+    const profilesById = new Map(profiles.map((profile) => [profile.projectId, profile]));
+    const ids = new Set();
+    const sourceRootsByProject = new Map();
+    cases.forEach((benchmarkCase, index) => {
+        if (!benchmarkCase || typeof benchmarkCase !== "object") {
+            errors.push(`warm-index case at index ${index}: must be an object.`);
+            return;
+        }
+        const id = typeof benchmarkCase.id === "string" && benchmarkCase.id.length > 0 ? benchmarkCase.id : undefined;
+        const label = id === undefined ? `warm-index case at index ${index}` : `warm-index case ${id}`;
+        if (id === undefined) {
+            errors.push(`${label}: id must be a nonempty string.`);
+        }
+        else if (ids.has(id)) {
+            errors.push(`${label}: duplicate case id.`);
+        }
+        else {
+            ids.add(id);
+        }
+        const projectId = typeof benchmarkCase.benchmarkProject === "string" && benchmarkCase.benchmarkProject.length > 0
+            ? benchmarkCase.benchmarkProject
+            : undefined;
+        if (projectId === undefined) {
+            errors.push(`${label}: benchmarkProject must be a nonempty string.`);
+        }
+        if (typeof benchmarkCase.projectProfileRef !== "string" || benchmarkCase.projectProfileRef.length === 0) {
+            errors.push(`${label}: missing projectProfileRef.`);
+        }
+        else if (benchmarkCase.projectProfileRef !== projectId) {
+            errors.push(`${label}: projectProfileRef ${benchmarkCase.projectProfileRef} must equal benchmarkProject ${projectId ?? "<missing>"}.`);
+        }
+        const profile = projectId === undefined ? undefined : profilesById.get(projectId);
+        if (projectId !== undefined && profile === undefined) {
+            errors.push(`${label}: unknown benchmark project profile ${projectId}.`);
+        }
+        if (benchmarkCase.taskLocality === undefined) {
+            errors.push(`${label}: missing taskLocality; expected one of ${TASK_LOCALITIES.join(", ")}.`);
+        }
+        else {
+            errors.push(...validateTaskLocality(benchmarkCase.taskLocality, label));
+        }
+        const targetRoot = typeof benchmarkCase.targetRoot === "string" && benchmarkCase.targetRoot.length > 0 ? benchmarkCase.targetRoot : undefined;
+        if (targetRoot === undefined) {
+            errors.push(`${label}: targetRoot must be a nonempty string.`);
+        }
+        else if (profile !== undefined && normalizeRelativeRoot(targetRoot) !== normalizeRelativeRoot(profile.rootPath ?? "")) {
+            errors.push(`${label}: targetRoot ${targetRoot} does not match profile ${profile.projectId} rootPath ${profile.rootPath}.`);
+        }
+        if (!Array.isArray(benchmarkCase.sourceRoots) || benchmarkCase.sourceRoots.length === 0) {
+            errors.push(`${label}: sourceRoots must be a nonempty array.`);
+        }
+        else {
+            if (profile !== undefined) {
+                errors.push(...validateWarmIndexSourceRoots(benchmarkCase.sourceRoots, profile, label));
+            }
+            if (projectId !== undefined) {
+                const reference = sourceRootsByProject.get(projectId);
+                if (reference === undefined) {
+                    sourceRootsByProject.set(projectId, { caseLabel: label, sourceRoots: benchmarkCase.sourceRoots });
+                }
+                else if (!sameOrderedValues(reference.sourceRoots, benchmarkCase.sourceRoots)) {
+                    errors.push(`${label}: sourceRoots ${JSON.stringify(benchmarkCase.sourceRoots)} must match the ordered sourceRoots ${JSON.stringify(reference.sourceRoots)} of ${reference.caseLabel} for project ${projectId}.`);
+                }
+            }
+        }
+        const expectedFiles = Array.isArray(benchmarkCase.expectedFiles) ? benchmarkCase.expectedFiles : undefined;
+        const expectedSymbols = Array.isArray(benchmarkCase.expectedSymbols) ? benchmarkCase.expectedSymbols : undefined;
+        if (expectedFiles === undefined || expectedFiles.length === 0) {
+            errors.push(`${label}: expectedFiles must be a nonempty array.`);
+        }
+        if (expectedSymbols === undefined || expectedSymbols.length === 0) {
+            errors.push(`${label}: expectedSymbols must be a nonempty array.`);
+        }
+        if (benchmarkCase.answerKey === undefined) {
+            errors.push(`${label}: missing answerKey.`);
+        }
+        else {
+            errors.push(...validateAnswerKey(benchmarkCase.answerKey, label));
+            const answerKey = benchmarkCase.answerKey;
+            if (answerKey && typeof answerKey === "object") {
+                if (expectedFiles !== undefined && Array.isArray(answerKey.expectedFiles) && !sameOrderedValues(expectedFiles, answerKey.expectedFiles)) {
+                    errors.push(`${label}: expectedFiles must exactly match answerKey.expectedFiles (same values in the same order).`);
+                }
+                if (expectedSymbols !== undefined &&
+                    Array.isArray(answerKey.expectedSymbols) &&
+                    !sameOrderedValues(expectedSymbols, answerKey.expectedSymbols)) {
+                    errors.push(`${label}: expectedSymbols must exactly match answerKey.expectedSymbols (same values in the same order).`);
+                }
+            }
+        }
+        if (expectedFiles !== undefined && targetRoot !== undefined) {
+            for (const expectedFile of expectedFiles) {
+                if (typeof expectedFile !== "string" || expectedFile.length === 0 || path.isAbsolute(expectedFile) || expectedFile.includes("..")) {
+                    errors.push(`${label}: expected file must be a safe relative path: ${String(expectedFile)}.`);
+                }
+                else if (!existsSync(path.resolve(repoRoot, targetRoot, expectedFile))) {
+                    errors.push(`${label}: expected file does not exist in ${projectId ?? targetRoot}: ${expectedFile}.`);
+                }
+            }
+        }
+    });
+    const representedProjectIds = [
+        ...new Set(cases
+            .map((benchmarkCase) => benchmarkCase?.benchmarkProject)
+            .filter((projectId) => typeof projectId === "string" && projectId.length > 0))
+    ].sort();
+    for (const projectId of representedProjectIds) {
+        const profile = profilesById.get(projectId);
+        if (profile === undefined || !profile.complexityMetrics || typeof profile.complexityMetrics !== "object") {
+            continue;
+        }
+        const derived = deriveWarmIndexTaskStats(cases, projectId);
+        for (const field of ["taskCount", "expectedRelevantFilesAverage", "expectedRelevantSymbolsAverage"]) {
+            if (profile.complexityMetrics[field] !== derived[field]) {
+                errors.push(`profile ${projectId}: complexityMetrics.${field} ${profile.complexityMetrics[field]} does not match warm-index corpus value ${derived[field]}.`);
+            }
+        }
+    }
+    return errors;
+}
+export function validateWarmIndexBenchmarkSuiteCoverage(cases) {
+    const errors = [];
+    for (const projectId of REQUIRED_WARM_INDEX_BENCHMARK_PROJECT_IDS) {
+        const projectCases = cases.filter((benchmarkCase) => benchmarkCase?.benchmarkProject === projectId);
+        if (projectCases.length === 0) {
+            errors.push(`warm-index suite: missing required benchmark project ${projectId}.`);
+            continue;
+        }
+        if (projectCases.length < MIN_WARM_INDEX_TASKS_PER_PROJECT) {
+            errors.push(`warm-index suite: project ${projectId} has ${projectCases.length} cases; at least ${MIN_WARM_INDEX_TASKS_PER_PROJECT} are required.`);
+        }
+        for (const locality of TASK_LOCALITIES) {
+            if (!projectCases.some((benchmarkCase) => benchmarkCase.taskLocality === locality)) {
+                errors.push(`warm-index suite: project ${projectId} has no ${locality} case.`);
+            }
+        }
+    }
+    return errors;
+}
+function validateWarmIndexSourceRoots(sourceRoots, profile, label) {
+    const errors = [];
+    const allowedRoots = [...(profile.sourceRoots ?? []), ...(profile.testRoots ?? [])];
+    const seen = new Set();
+    for (const sourceRoot of sourceRoots) {
+        if (seen.has(sourceRoot)) {
+            errors.push(`${label}: duplicate source root ${JSON.stringify(sourceRoot)}.`);
+            continue;
+        }
+        seen.add(sourceRoot);
+        if (typeof sourceRoot !== "string" || !allowedRoots.includes(sourceRoot)) {
+            errors.push(`${label}: unknown source root ${JSON.stringify(sourceRoot)} for profile ${profile.projectId}; allowed roots: ${allowedRoots.join(", ")}.`);
+        }
+    }
+    for (const allowedRoot of allowedRoots) {
+        if (!seen.has(allowedRoot)) {
+            errors.push(`${label}: missing profile source/test root ${allowedRoot} for profile ${profile.projectId}.`);
+        }
+    }
+    return errors;
+}
+function normalizeRelativeRoot(value) {
+    return path.posix.normalize(value.replace(/\\/g, "/")).replace(/\/+$/, "");
+}
+function sameOrderedValues(left, right) {
+    return left.length === right.length && left.every((value, index) => value === right[index]);
+}
 export function validateBenchmarkProjectProfiles(profiles, repoRoot = process.cwd()) {
     const errors = [];
     const ids = new Set(profiles.map((profile) => profile.projectId));

package/dist/src/evaluation/projectComplexity.js

@@ -73,7 +73,7 @@ export function computeProjectComplexityMetrics(projectRoot, fileTree, taskStats
 function normalize(value, cap) {
     return Math.min(value / cap, 1);
 }
-function roundToTwo(value) {
+export function roundToTwo(value) {
     return Math.round(value * 100) / 100;
 }
 function sum(values) {

package/dist/src/evaluation/readEvaluationCases.js

@@ -1,7 +1,7 @@
 import { readFile } from "node:fs/promises";
 import path from "node:path";
 import { resolveWithinRoot } from "../core/pathSafety.js";
-import { validateAnswerKey } from "./benchmarkMetadata.js";
+import { validateAnswerKey, validateTaskLocality } from "./benchmarkMetadata.js";
 export async function readEvaluationCases(casesPath, repoRoot = process.cwd(), options = {}) {
     let parsed;
     try {
@@ -36,6 +36,12 @@ export async function readEvaluationCases(casesPath, repoRoot = process.cwd(), o
             throw new Error(`Duplicate evaluation case id: ${input.id}`);
         }
         ids.add(input.id);
+        if (candidate.taskLocality !== undefined) {
+            const localityErrors = validateTaskLocality(candidate.taskLocality, `evaluation case ${input.id}`);
+            if (localityErrors.length > 0) {
+                throw new Error(localityErrors.join("\n"));
+            }
+        }
         if (candidate.answerKey !== undefined) {
             const answerKeyErrors = validateAnswerKey(candidate.answerKey, `evaluation case ${input.id}`);
             if (answerKeyErrors.length > 0) {

package/dist/src/evaluation/types.js

@@ -1 +1 @@
-export {};
+export const TASK_LOCALITIES = ["localized", "cross-module", "broad-change"];

package/dist/src/experiments/plugins/warmIndexReuse/agentEvaluation.js

@@ -0,0 +1,233 @@
+import { mkdtemp, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { runAgentPrompt } from "../../../agents/index.js";
+import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { classifyAgentRunOutcome } from "../../../evaluation/classifyAgentRunOutcome.js";
+import { parseAgentAnswer } from "../../../evaluation/parseAgentAnswer.js";
+import { scoreCorrectness } from "../../../evaluation/scoreCorrectness.js";
+import { generatePromptVariants } from "../../../prompts/index.js";
+import { buildWarmIndexRealAgentPrompt } from "./realAgentPrompt.js";
+import { taskOutputSegment } from "./selection.js";
+// Classified statuses for which the scorer evaluated an actual agent answer.
+const SCOREABLE_STATUSES = new Set(["completed", "invalid-output"]);
+const SIDE_STRATEGIES = {
+    "raw-full-file": "raw-full-file",
+    "warm-index-reuse": "my-dev-kit-guided",
+};
+export async function evaluateWarmIndexFakeAgents(options) {
+    const casesById = new Map(options.cases.map((evaluationCase) => [evaluationCase.id, evaluationCase]));
+    const result = [];
+    for (const project of options.projects) {
+        const tasks = [];
+        for (const task of project.tasks) {
+            const evaluationCase = casesById.get(task.caseId);
+            const run = (variantId) => evaluateFakeSide({
+                evaluationCase,
+                caseId: task.caseId,
+                variantId,
+                projectProfiles: options.projectProfiles,
+                outDir: resolveWithinRoot(options.outputRoot, path.join("agents", project.projectSegment, taskOutputSegment(task.caseId), variantId)),
+                cwd: options.cwd,
+                env: options.env,
+            });
+            tasks.push({
+                caseId: task.caseId,
+                raw: task.rawBaseline ? await run("raw-full-file") : null,
+                warm: task.warmRetrieval ? await run("warm-index-reuse") : null,
+            });
+        }
+        result.push({ benchmarkProject: project.benchmarkProject, tasks });
+    }
+    return result;
+}
+async function evaluateFakeSide(args) {
+    const promptStrategy = SIDE_STRATEGIES[args.variantId];
+    const base = { variantId: args.variantId, agentId: "fake-agent", promptStrategy };
+    try {
+        if (!args.evaluationCase) {
+            throw new Error(`Evaluation case ${args.caseId} is not available for fake-agent evaluation.`);
+        }
+        const [promptVariant] = generatePromptVariants({
+            cases: [args.evaluationCase],
+            projectProfiles: [...args.projectProfiles],
+            strategies: [promptStrategy],
+            complexityLevels: ["short"],
+        });
+        if (!promptVariant) {
+            throw new Error(`Failed to generate a fake-agent prompt for case ${args.caseId}.`);
+        }
+        const agentRunResult = await runAgentPrompt({
+            runId: `${args.caseId}.fake-agent.${args.variantId}`,
+            agentId: "fake-agent",
+            promptVariant,
+            promptText: promptVariant.promptText,
+            cwd: args.cwd,
+            outDir: args.outDir,
+            env: args.env,
+        });
+        return buildSideEvidenceFromRun({ base, caseId: args.caseId, outDir: args.outDir, agentRunResult, expectedAnswerKey: promptVariant.expectedAnswerKey });
+    }
+    catch (error) {
+        return buildFailedSideEvidence(base, error);
+    }
+}
+/**
+ * Real-agent campaign evaluator (v0.5.2 Batch 3): consumes the exact context text the warm-index
+ * execution stage already produced for one side. Never calls my-dev-kit, never independently
+ * inspects the repository, and never runs a side with no usable context.
+ */
+export async function evaluateWarmIndexRealAgentCampaign(options) {
+    const casesById = new Map(options.cases.map((evaluationCase) => [evaluationCase.id, evaluationCase]));
+    const result = [];
+    for (const project of options.projects) {
+        const tasks = [];
+        for (const task of project.tasks) {
+            const evaluationCase = casesById.get(task.caseId);
+            const rawContext = task.rawBaseline?.contextText;
+            const rawEligible = typeof rawContext === "string" && rawContext.trim().length > 0;
+            const warmContext = task.warmRetrieval?.contextText;
+            const warmEligible = task.warmRetrieval !== undefined &&
+                task.warmRetrieval.skipped === false &&
+                typeof warmContext === "string" &&
+                warmContext.trim().length > 0;
+            const runSide = (variantId, contextText) => evaluateRealSide({
+                evaluationCase,
+                caseId: task.caseId,
+                variantId,
+                contextText,
+                projectProfiles: options.projectProfiles,
+                outDir: resolveWithinRoot(options.outputRoot, path.join("agents", project.projectSegment, taskOutputSegment(task.caseId), variantId)),
+                agentId: options.agentId,
+                timeoutMs: options.timeoutMs,
+                env: options.env,
+            });
+            tasks.push({
+                caseId: task.caseId,
+                raw: rawEligible ? await runSide("raw-full-file", rawContext) : null,
+                warm: warmEligible ? await runSide("warm-index-reuse", warmContext) : null,
+            });
+        }
+        result.push({ benchmarkProject: project.benchmarkProject, tasks });
+    }
+    return result;
+}
+async function evaluateRealSide(args) {
+    const promptStrategy = SIDE_STRATEGIES[args.variantId];
+    const base = { variantId: args.variantId, agentId: args.agentId, promptStrategy };
+    let neutralCwd;
+    try {
+        if (!args.evaluationCase) {
+            throw new Error(`Evaluation case ${args.caseId} is not available for real-agent evaluation.`);
+        }
+        // Generated only for internal scoring/metadata (expectedAnswerKey, promptComplexityLevel); its
+        // promptText is never sent to the provider.
+        const [promptVariant] = generatePromptVariants({
+            cases: [args.evaluationCase],
+            projectProfiles: [...args.projectProfiles],
+            strategies: [promptStrategy],
+            complexityLevels: ["short"],
+        });
+        if (!promptVariant) {
+            throw new Error(`Failed to generate an internal scoring prompt variant for case ${args.caseId}.`);
+        }
+        const realPromptText = buildWarmIndexRealAgentPrompt({
+            evaluationCase: args.evaluationCase,
+            variantId: args.variantId,
+            contextText: args.contextText,
+        });
+        // Neutral working directory: the provider must not run from the repository, the benchmark
+        // target, the installed package, or the agent artifact directory.
+        neutralCwd = await mkdtemp(path.join(os.tmpdir(), "my-dev-kit-lab-warm-agent-"));
+        const agentRunResult = await runAgentPrompt({
+            runId: `${args.caseId}.${args.agentId}.${args.variantId}`,
+            agentId: args.agentId,
+            promptVariant,
+            promptText: realPromptText,
+            promptTransport: "stdin",
+            timeoutMs: args.timeoutMs,
+            requireAvailable: false,
+            env: args.env,
+            cwd: neutralCwd,
+            outDir: args.outDir,
+        });
+        return buildSideEvidenceFromRun({
+            base,
+            caseId: args.caseId,
+            outDir: args.outDir,
+            agentRunResult,
+            expectedAnswerKey: promptVariant.expectedAnswerKey,
+        });
+    }
+    catch (error) {
+        return buildFailedSideEvidence(base, error);
+    }
+    finally {
+        if (neutralCwd) {
+            try {
+                await rm(neutralCwd, { recursive: true, force: true });
+            }
+            catch {
+                // Best-effort cleanup only: { force: true } suppresses a missing path, but not an
+                // EPERM/EBUSY from an OS still holding this directory open -- for example a timed-out
+                // provider process whose kill did not (yet) fully release it as its own cwd. A cleanup
+                // failure here must never replace or hide the side's already-classified evidence.
+            }
+        }
+    }
+}
+// Shared scoring/classification/evidence assembly for both the fake-agent and real-agent paths.
+function buildSideEvidenceFromRun(args) {
+    const { base, caseId, outDir, agentRunResult, expectedAnswerKey } = args;
+    const parsedAnswer = parseAgentAnswer({
+        text: agentRunResult.finalAnswerText,
+        answerKey: expectedAnswerKey,
+        tokenUsage: agentRunResult.tokenUsage,
+    });
+    const classification = classifyAgentRunOutcome({ agentRunResult, parsedAnswer });
+    const correctness = scoreCorrectness({
+        caseId,
+        answerKey: expectedAnswerKey,
+        parsedAnswer,
+        status: classification.status,
+    });
+    const scoreable = SCOREABLE_STATUSES.has(classification.status) && Number.isFinite(correctness.correctnessScore);
+    const totalTokens = agentRunResult.tokenUsage.totalTokens;
+    return {
+        ...base,
+        status: classification.status,
+        correctness: {
+            available: scoreable,
+            score: scoreable ? correctness.correctnessScore : null,
+            passed: scoreable ? correctness.passed : null,
+            failureReasons: [...correctness.failureReasons],
+        },
+        tokenUsage: {
+            totalTokens: typeof totalTokens === "number" && Number.isFinite(totalTokens) ? totalTokens : null,
+            source: agentRunResult.tokenUsageSource,
+            reliability: agentRunResult.tokenUsageReliability,
+        },
+        durationMs: agentRunResult.durationMs,
+        warnings: [...classification.warnings],
+        errors: [...classification.errors],
+        artifactPaths: {
+            promptPath: path.join(outDir, "prompt.txt"),
+            agentRunResultPath: path.join(outDir, "agent-run-result.json"),
+            stdoutPath: agentRunResult.stdoutPath,
+            stderrPath: agentRunResult.stderrPath,
+            telemetryPath: agentRunResult.telemetryPath,
+        },
+    };
+}
+function buildFailedSideEvidence(base, error) {
+    return {
+        ...base,
+        status: "failed",
+        correctness: { available: false, score: null, passed: null, failureReasons: [] },
+        tokenUsage: { totalTokens: null, source: "unavailable", reliability: "unavailable" },
+        durationMs: null,
+        warnings: [],
+        errors: [error instanceof Error ? error.message : String(error)],
+        artifactPaths: {},
+    };
+}

package/dist/src/experiments/plugins/warmIndexReuse/campaignPresets.js

@@ -0,0 +1,83 @@
+// v0.5.2 Batch 1 -- frozen warm-index real-agent campaign preset contract.
+//
+// This module owns preset identity, provider assignment, bundled resource paths, the frozen case
+// list per preset, and the default per-agent timeout. It contains no CLI parsing, no report
+// rendering, and no provider execution logic; those remain owned elsewhere.
+export const WARM_INDEX_CAMPAIGN_PRESET_IDS = ["codex-full", "claude-full", "codex-timeout-isolation"];
+const CASES_RESOURCE_PATH = "benchmarks/contracts/warm-index-benchmark-cases.json";
+const PROJECT_PROFILES_RESOURCE_PATH = "benchmarks/contracts/benchmark-project-profiles.json";
+const DEFAULT_TIMEOUT_MS = 240_000;
+// Exact v0.5.1 12-case corpus, in corpus order. Frozen; do not reorder or extend in Batch 1.
+const FULL_CAMPAIGN_CASE_IDS = Object.freeze([
+    "warm-medium-import-dedupe",
+    "warm-medium-create-project-task",
+    "warm-medium-complete-idempotent",
+    "warm-medium-composite-filter",
+    "warm-medium-project-summary",
+    "warm-medium-broad-workflow-map",
+    "warm-large-health-label",
+    "warm-large-ts-analytics-snapshot",
+    "warm-large-ts-leaderboard",
+    "warm-large-python-parser-metrics",
+    "warm-large-python-pipeline",
+    "warm-large-broad-analytics-comparison",
+]);
+// Exactly the three large/mixed cases covering cross-module, localized, and broad-change.
+const TIMEOUT_ISOLATION_CASE_IDS = Object.freeze([
+    "warm-large-health-label",
+    "warm-large-ts-leaderboard",
+    "warm-large-broad-analytics-comparison",
+]);
+const WARM_INDEX_CAMPAIGN_PRESETS = Object.freeze({
+    "codex-full": Object.freeze({
+        id: "codex-full",
+        agentId: "codex",
+        casesResourcePath: CASES_RESOURCE_PATH,
+        projectProfilesResourcePath: PROJECT_PROFILES_RESOURCE_PATH,
+        caseIds: FULL_CAMPAIGN_CASE_IDS,
+        defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
+    }),
+    "claude-full": Object.freeze({
+        id: "claude-full",
+        agentId: "claude",
+        casesResourcePath: CASES_RESOURCE_PATH,
+        projectProfilesResourcePath: PROJECT_PROFILES_RESOURCE_PATH,
+        caseIds: FULL_CAMPAIGN_CASE_IDS,
+        defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
+    }),
+    "codex-timeout-isolation": Object.freeze({
+        id: "codex-timeout-isolation",
+        agentId: "codex",
+        casesResourcePath: CASES_RESOURCE_PATH,
+        projectProfilesResourcePath: PROJECT_PROFILES_RESOURCE_PATH,
+        caseIds: TIMEOUT_ISOLATION_CASE_IDS,
+        defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
+    }),
+});
+/** Fails explicitly for any value outside the exact three frozen preset IDs. */
+export function parseWarmIndexCampaignPresetId(value) {
+    if (WARM_INDEX_CAMPAIGN_PRESET_IDS.includes(value)) {
+        return value;
+    }
+    throw new Error(`Unknown warm-index campaign preset: ${value}. Supported presets: ${WARM_INDEX_CAMPAIGN_PRESET_IDS.join(", ")}.`);
+}
+export function getWarmIndexCampaignPreset(id) {
+    const preset = WARM_INDEX_CAMPAIGN_PRESETS[id];
+    if (!preset) {
+        throw new Error(`Unknown warm-index campaign preset: ${id}. Supported presets: ${WARM_INDEX_CAMPAIGN_PRESET_IDS.join(", ")}.`);
+    }
+    return preset;
+}
+/**
+ * Resolves the effective per-agent timeout for a campaign: the preset default when no override was
+ * requested, or an explicit finite positive integer override.
+ */
+export function resolveWarmIndexCampaignTimeoutMs(preset, requestedTimeoutMs) {
+    if (requestedTimeoutMs === undefined) {
+        return preset.defaultTimeoutMs;
+    }
+    if (!Number.isFinite(requestedTimeoutMs) || !Number.isInteger(requestedTimeoutMs) || requestedTimeoutMs <= 0) {
+        throw new Error(`Campaign timeoutMs must be a finite positive integer; received ${requestedTimeoutMs}.`);
+    }
+    return requestedTimeoutMs;
+}

package/dist/src/experiments/plugins/warmIndexReuse/config.js

@@ -1,4 +1,5 @@
 import { invalidExperimentConfig, isPlainObject, mergeConfig, validExperimentConfig } from "../../config.js";
+import { WARM_INDEX_CAMPAIGN_PRESET_IDS } from "./campaignPresets.js";
 export const defaultWarmIndexReuseConfig = {
     casesPath: "examples/token-savings-cases.json",
     projectProfilesPath: "benchmarks/contracts/benchmark-project-profiles.json",
@@ -12,6 +13,9 @@ const SUPPORTED_FIELDS = [
     "kitCommand",
     "caseIds",
     "benchmarkProjects",
+    "campaignPreset",
+    "includeRealAgents",
+    "timeoutMs",
 ];
 export const warmIndexReuseConfigDefinition = {
     fields: [
@@ -21,6 +25,21 @@ export const warmIndexReuseConfigDefinition = {
         { name: "kitCommand", type: "string", description: "my-dev-kit command used for index and retrieval." },
         { name: "caseIds", type: "array", description: "Evaluation case IDs to include." },
         { name: "benchmarkProjects", type: "array", description: "Benchmark project IDs to include." },
+        {
+            name: "campaignPreset",
+            type: "string",
+            description: `Real-agent campaign preset id: ${WARM_INDEX_CAMPAIGN_PRESET_IDS.join(", ")}. Absent selects the legacy deterministic fake-agent path.`,
+        },
+        {
+            name: "includeRealAgents",
+            type: "boolean",
+            description: "Required (must be true) together with campaignPreset; rejected without it.",
+        },
+        {
+            name: "timeoutMs",
+            type: "number",
+            description: "Optional per-agent timeout override (ms) for a campaign; defaults to the preset's 240000 ms. Rejected without campaignPreset.",
+        },
     ],
 };
 /**
@@ -51,6 +70,38 @@ export function validateWarmIndexReuseConfig(config) {
             errors.push(`${field} must be a non-empty array of non-empty strings when provided.`);
         }
     }
+    // Campaign fields are validated against the RAW supplied object, not the merged/normalized one,
+    // so legacy defaults (casesPath/projectProfilesPath) never look like an explicit override.
+    const raw = isPlainObject(config) ? config : {};
+    const rawCampaignPreset = raw.campaignPreset;
+    if (rawCampaignPreset !== undefined) {
+        if (typeof rawCampaignPreset !== "string" || !WARM_INDEX_CAMPAIGN_PRESET_IDS.includes(rawCampaignPreset)) {
+            errors.push(`campaignPreset must be one of: ${WARM_INDEX_CAMPAIGN_PRESET_IDS.join(", ")}.`);
+        }
+        if (raw.casesPath !== undefined) {
+            errors.push("campaignPreset cannot be combined with an explicit casesPath; the preset owns the production corpus.");
+        }
+        if (raw.projectProfilesPath !== undefined) {
+            errors.push("campaignPreset cannot be combined with an explicit projectProfilesPath; the preset owns the project profiles.");
+        }
+        if (raw.includeRealAgents !== true) {
+            errors.push("campaignPreset requires includeRealAgents to be exactly true.");
+        }
+        if (raw.timeoutMs !== undefined) {
+            const timeoutMs = raw.timeoutMs;
+            if (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs) || !Number.isInteger(timeoutMs) || timeoutMs <= 0) {
+                errors.push("timeoutMs must be a finite positive integer when campaignPreset is set.");
+            }
+        }
+    }
+    else {
+        if (raw.includeRealAgents !== undefined) {
+            errors.push("includeRealAgents is only supported together with campaignPreset.");
+        }
+        if (raw.timeoutMs !== undefined) {
+            errors.push("timeoutMs is only supported together with campaignPreset.");
+        }
+    }
     if (errors.length > 0) {
         return invalidExperimentConfig(errors);
     }