@dailephd/my-dev-kit-lab 0.5.1 → 0.6.0

package/dist/src/evaluation/runMyDevKitRetrieval.js

@@ -1,6 +1,8 @@
 import path from "node:path";
 import { countEstimatedTokens, countTextChars, tokenCountMethod } from "../core/countTokens.js";
 import { runMeasuredCommand } from "../core/runMeasuredCommand.js";
+import { interpretToolVersionOutput } from "./indexSnapshot.js";
+const VERSION_PROBE_TIMEOUT_MS = 30_000;
 function parseJsonIfPossible(text) {
     try {
         return JSON.parse(text);
@@ -89,6 +91,38 @@ export async function buildMyDevKitIndex(options) {
         command
     };
 }
+/**
+ * Runs `<kit-command> --version` once through the shared measured-command runner and returns
+ * bounded tool evidence. Kept separate from `buildMyDevKitIndex` so it never counts toward index
+ * duration. An unsupported, failing, slow, or empty probe is explicit `unavailable` evidence; it
+ * never throws and never fails the caller.
+ */
+export async function probeMyDevKitVersion(options) {
+    try {
+        const command = await runMeasuredCommand({
+            commandId: "version",
+            commandString: options.kitCommand,
+            cwd: process.cwd(),
+            outDir: options.commandsDir,
+            extraArgs: ["--version"],
+            timeoutMs: VERSION_PROBE_TIMEOUT_MS
+        });
+        return interpretToolVersionOutput({
+            ok: command.ok,
+            exitCode: command.exitCode,
+            stdout: command.stdout,
+            error: command.error
+        });
+    }
+    catch (error) {
+        return interpretToolVersionOutput({
+            ok: false,
+            exitCode: null,
+            stdout: "",
+            error: error instanceof Error ? error.message : String(error)
+        });
+    }
+}
 /**
  * Runs search -> lookup -> slice -> source against an already prepared index. Never invokes
  * `index`; `durationMs` covers retrieval only.

package/dist/src/experiments/plugins/warmIndexReuse/agentEvaluation.js

@@ -0,0 +1,233 @@
+import { mkdtemp, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { runAgentPrompt } from "../../../agents/index.js";
+import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { classifyAgentRunOutcome } from "../../../evaluation/classifyAgentRunOutcome.js";
+import { parseAgentAnswer } from "../../../evaluation/parseAgentAnswer.js";
+import { scoreCorrectness } from "../../../evaluation/scoreCorrectness.js";
+import { generatePromptVariants } from "../../../prompts/index.js";
+import { buildWarmIndexRealAgentPrompt } from "./realAgentPrompt.js";
+import { taskOutputSegment } from "./selection.js";
+// Classified statuses for which the scorer evaluated an actual agent answer.
+const SCOREABLE_STATUSES = new Set(["completed", "invalid-output"]);
+const SIDE_STRATEGIES = {
+    "raw-full-file": "raw-full-file",
+    "warm-index-reuse": "my-dev-kit-guided",
+};
+export async function evaluateWarmIndexFakeAgents(options) {
+    const casesById = new Map(options.cases.map((evaluationCase) => [evaluationCase.id, evaluationCase]));
+    const result = [];
+    for (const project of options.projects) {
+        const tasks = [];
+        for (const task of project.tasks) {
+            const evaluationCase = casesById.get(task.caseId);
+            const run = (variantId) => evaluateFakeSide({
+                evaluationCase,
+                caseId: task.caseId,
+                variantId,
+                projectProfiles: options.projectProfiles,
+                outDir: resolveWithinRoot(options.outputRoot, path.join("agents", project.projectSegment, taskOutputSegment(task.caseId), variantId)),
+                cwd: options.cwd,
+                env: options.env,
+            });
+            tasks.push({
+                caseId: task.caseId,
+                raw: task.rawBaseline ? await run("raw-full-file") : null,
+                warm: task.warmRetrieval ? await run("warm-index-reuse") : null,
+            });
+        }
+        result.push({ benchmarkProject: project.benchmarkProject, tasks });
+    }
+    return result;
+}
+async function evaluateFakeSide(args) {
+    const promptStrategy = SIDE_STRATEGIES[args.variantId];
+    const base = { variantId: args.variantId, agentId: "fake-agent", promptStrategy };
+    try {
+        if (!args.evaluationCase) {
+            throw new Error(`Evaluation case ${args.caseId} is not available for fake-agent evaluation.`);
+        }
+        const [promptVariant] = generatePromptVariants({
+            cases: [args.evaluationCase],
+            projectProfiles: [...args.projectProfiles],
+            strategies: [promptStrategy],
+            complexityLevels: ["short"],
+        });
+        if (!promptVariant) {
+            throw new Error(`Failed to generate a fake-agent prompt for case ${args.caseId}.`);
+        }
+        const agentRunResult = await runAgentPrompt({
+            runId: `${args.caseId}.fake-agent.${args.variantId}`,
+            agentId: "fake-agent",
+            promptVariant,
+            promptText: promptVariant.promptText,
+            cwd: args.cwd,
+            outDir: args.outDir,
+            env: args.env,
+        });
+        return buildSideEvidenceFromRun({ base, caseId: args.caseId, outDir: args.outDir, agentRunResult, expectedAnswerKey: promptVariant.expectedAnswerKey });
+    }
+    catch (error) {
+        return buildFailedSideEvidence(base, error);
+    }
+}
+/**
+ * Real-agent campaign evaluator (v0.5.2 Batch 3): consumes the exact context text the warm-index
+ * execution stage already produced for one side. Never calls my-dev-kit, never independently
+ * inspects the repository, and never runs a side with no usable context.
+ */
+export async function evaluateWarmIndexRealAgentCampaign(options) {
+    const casesById = new Map(options.cases.map((evaluationCase) => [evaluationCase.id, evaluationCase]));
+    const result = [];
+    for (const project of options.projects) {
+        const tasks = [];
+        for (const task of project.tasks) {
+            const evaluationCase = casesById.get(task.caseId);
+            const rawContext = task.rawBaseline?.contextText;
+            const rawEligible = typeof rawContext === "string" && rawContext.trim().length > 0;
+            const warmContext = task.warmRetrieval?.contextText;
+            const warmEligible = task.warmRetrieval !== undefined &&
+                task.warmRetrieval.skipped === false &&
+                typeof warmContext === "string" &&
+                warmContext.trim().length > 0;
+            const runSide = (variantId, contextText) => evaluateRealSide({
+                evaluationCase,
+                caseId: task.caseId,
+                variantId,
+                contextText,
+                projectProfiles: options.projectProfiles,
+                outDir: resolveWithinRoot(options.outputRoot, path.join("agents", project.projectSegment, taskOutputSegment(task.caseId), variantId)),
+                agentId: options.agentId,
+                timeoutMs: options.timeoutMs,
+                env: options.env,
+            });
+            tasks.push({
+                caseId: task.caseId,
+                raw: rawEligible ? await runSide("raw-full-file", rawContext) : null,
+                warm: warmEligible ? await runSide("warm-index-reuse", warmContext) : null,
+            });
+        }
+        result.push({ benchmarkProject: project.benchmarkProject, tasks });
+    }
+    return result;
+}
+async function evaluateRealSide(args) {
+    const promptStrategy = SIDE_STRATEGIES[args.variantId];
+    const base = { variantId: args.variantId, agentId: args.agentId, promptStrategy };
+    let neutralCwd;
+    try {
+        if (!args.evaluationCase) {
+            throw new Error(`Evaluation case ${args.caseId} is not available for real-agent evaluation.`);
+        }
+        // Generated only for internal scoring/metadata (expectedAnswerKey, promptComplexityLevel); its
+        // promptText is never sent to the provider.
+        const [promptVariant] = generatePromptVariants({
+            cases: [args.evaluationCase],
+            projectProfiles: [...args.projectProfiles],
+            strategies: [promptStrategy],
+            complexityLevels: ["short"],
+        });
+        if (!promptVariant) {
+            throw new Error(`Failed to generate an internal scoring prompt variant for case ${args.caseId}.`);
+        }
+        const realPromptText = buildWarmIndexRealAgentPrompt({
+            evaluationCase: args.evaluationCase,
+            variantId: args.variantId,
+            contextText: args.contextText,
+        });
+        // Neutral working directory: the provider must not run from the repository, the benchmark
+        // target, the installed package, or the agent artifact directory.
+        neutralCwd = await mkdtemp(path.join(os.tmpdir(), "my-dev-kit-lab-warm-agent-"));
+        const agentRunResult = await runAgentPrompt({
+            runId: `${args.caseId}.${args.agentId}.${args.variantId}`,
+            agentId: args.agentId,
+            promptVariant,
+            promptText: realPromptText,
+            promptTransport: "stdin",
+            timeoutMs: args.timeoutMs,
+            requireAvailable: false,
+            env: args.env,
+            cwd: neutralCwd,
+            outDir: args.outDir,
+        });
+        return buildSideEvidenceFromRun({
+            base,
+            caseId: args.caseId,
+            outDir: args.outDir,
+            agentRunResult,
+            expectedAnswerKey: promptVariant.expectedAnswerKey,
+        });
+    }
+    catch (error) {
+        return buildFailedSideEvidence(base, error);
+    }
+    finally {
+        if (neutralCwd) {
+            try {
+                await rm(neutralCwd, { recursive: true, force: true });
+            }
+            catch {
+                // Best-effort cleanup only: { force: true } suppresses a missing path, but not an
+                // EPERM/EBUSY from an OS still holding this directory open -- for example a timed-out
+                // provider process whose kill did not (yet) fully release it as its own cwd. A cleanup
+                // failure here must never replace or hide the side's already-classified evidence.
+            }
+        }
+    }
+}
+// Shared scoring/classification/evidence assembly for both the fake-agent and real-agent paths.
+function buildSideEvidenceFromRun(args) {
+    const { base, caseId, outDir, agentRunResult, expectedAnswerKey } = args;
+    const parsedAnswer = parseAgentAnswer({
+        text: agentRunResult.finalAnswerText,
+        answerKey: expectedAnswerKey,
+        tokenUsage: agentRunResult.tokenUsage,
+    });
+    const classification = classifyAgentRunOutcome({ agentRunResult, parsedAnswer });
+    const correctness = scoreCorrectness({
+        caseId,
+        answerKey: expectedAnswerKey,
+        parsedAnswer,
+        status: classification.status,
+    });
+    const scoreable = SCOREABLE_STATUSES.has(classification.status) && Number.isFinite(correctness.correctnessScore);
+    const totalTokens = agentRunResult.tokenUsage.totalTokens;
+    return {
+        ...base,
+        status: classification.status,
+        correctness: {
+            available: scoreable,
+            score: scoreable ? correctness.correctnessScore : null,
+            passed: scoreable ? correctness.passed : null,
+            failureReasons: [...correctness.failureReasons],
+        },
+        tokenUsage: {
+            totalTokens: typeof totalTokens === "number" && Number.isFinite(totalTokens) ? totalTokens : null,
+            source: agentRunResult.tokenUsageSource,
+            reliability: agentRunResult.tokenUsageReliability,
+        },
+        durationMs: agentRunResult.durationMs,
+        warnings: [...classification.warnings],
+        errors: [...classification.errors],
+        artifactPaths: {
+            promptPath: path.join(outDir, "prompt.txt"),
+            agentRunResultPath: path.join(outDir, "agent-run-result.json"),
+            stdoutPath: agentRunResult.stdoutPath,
+            stderrPath: agentRunResult.stderrPath,
+            telemetryPath: agentRunResult.telemetryPath,
+        },
+    };
+}
+function buildFailedSideEvidence(base, error) {
+    return {
+        ...base,
+        status: "failed",
+        correctness: { available: false, score: null, passed: null, failureReasons: [] },
+        tokenUsage: { totalTokens: null, source: "unavailable", reliability: "unavailable" },
+        durationMs: null,
+        warnings: [],
+        errors: [error instanceof Error ? error.message : String(error)],
+        artifactPaths: {},
+    };
+}

package/dist/src/experiments/plugins/warmIndexReuse/campaignPresets.js

@@ -0,0 +1,83 @@
+// v0.5.2 Batch 1 -- frozen warm-index real-agent campaign preset contract.
+//
+// This module owns preset identity, provider assignment, bundled resource paths, the frozen case
+// list per preset, and the default per-agent timeout. It contains no CLI parsing, no report
+// rendering, and no provider execution logic; those remain owned elsewhere.
+export const WARM_INDEX_CAMPAIGN_PRESET_IDS = ["codex-full", "claude-full", "codex-timeout-isolation"];
+const CASES_RESOURCE_PATH = "benchmarks/contracts/warm-index-benchmark-cases.json";
+const PROJECT_PROFILES_RESOURCE_PATH = "benchmarks/contracts/benchmark-project-profiles.json";
+const DEFAULT_TIMEOUT_MS = 240_000;
+// Exact v0.5.1 12-case corpus, in corpus order. Frozen; do not reorder or extend in Batch 1.
+const FULL_CAMPAIGN_CASE_IDS = Object.freeze([
+    "warm-medium-import-dedupe",
+    "warm-medium-create-project-task",
+    "warm-medium-complete-idempotent",
+    "warm-medium-composite-filter",
+    "warm-medium-project-summary",
+    "warm-medium-broad-workflow-map",
+    "warm-large-health-label",
+    "warm-large-ts-analytics-snapshot",
+    "warm-large-ts-leaderboard",
+    "warm-large-python-parser-metrics",
+    "warm-large-python-pipeline",
+    "warm-large-broad-analytics-comparison",
+]);
+// Exactly the three large/mixed cases covering cross-module, localized, and broad-change.
+const TIMEOUT_ISOLATION_CASE_IDS = Object.freeze([
+    "warm-large-health-label",
+    "warm-large-ts-leaderboard",
+    "warm-large-broad-analytics-comparison",
+]);
+const WARM_INDEX_CAMPAIGN_PRESETS = Object.freeze({
+    "codex-full": Object.freeze({
+        id: "codex-full",
+        agentId: "codex",
+        casesResourcePath: CASES_RESOURCE_PATH,
+        projectProfilesResourcePath: PROJECT_PROFILES_RESOURCE_PATH,
+        caseIds: FULL_CAMPAIGN_CASE_IDS,
+        defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
+    }),
+    "claude-full": Object.freeze({
+        id: "claude-full",
+        agentId: "claude",
+        casesResourcePath: CASES_RESOURCE_PATH,
+        projectProfilesResourcePath: PROJECT_PROFILES_RESOURCE_PATH,
+        caseIds: FULL_CAMPAIGN_CASE_IDS,
+        defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
+    }),
+    "codex-timeout-isolation": Object.freeze({
+        id: "codex-timeout-isolation",
+        agentId: "codex",
+        casesResourcePath: CASES_RESOURCE_PATH,
+        projectProfilesResourcePath: PROJECT_PROFILES_RESOURCE_PATH,
+        caseIds: TIMEOUT_ISOLATION_CASE_IDS,
+        defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
+    }),
+});
+/** Fails explicitly for any value outside the exact three frozen preset IDs. */
+export function parseWarmIndexCampaignPresetId(value) {
+    if (WARM_INDEX_CAMPAIGN_PRESET_IDS.includes(value)) {
+        return value;
+    }
+    throw new Error(`Unknown warm-index campaign preset: ${value}. Supported presets: ${WARM_INDEX_CAMPAIGN_PRESET_IDS.join(", ")}.`);
+}
+export function getWarmIndexCampaignPreset(id) {
+    const preset = WARM_INDEX_CAMPAIGN_PRESETS[id];
+    if (!preset) {
+        throw new Error(`Unknown warm-index campaign preset: ${id}. Supported presets: ${WARM_INDEX_CAMPAIGN_PRESET_IDS.join(", ")}.`);
+    }
+    return preset;
+}
+/**
+ * Resolves the effective per-agent timeout for a campaign: the preset default when no override was
+ * requested, or an explicit finite positive integer override.
+ */
+export function resolveWarmIndexCampaignTimeoutMs(preset, requestedTimeoutMs) {
+    if (requestedTimeoutMs === undefined) {
+        return preset.defaultTimeoutMs;
+    }
+    if (!Number.isFinite(requestedTimeoutMs) || !Number.isInteger(requestedTimeoutMs) || requestedTimeoutMs <= 0) {
+        throw new Error(`Campaign timeoutMs must be a finite positive integer; received ${requestedTimeoutMs}.`);
+    }
+    return requestedTimeoutMs;
+}

package/dist/src/experiments/plugins/warmIndexReuse/config.js

@@ -1,4 +1,5 @@
 import { invalidExperimentConfig, isPlainObject, mergeConfig, validExperimentConfig } from "../../config.js";
+import { WARM_INDEX_CAMPAIGN_PRESET_IDS } from "./campaignPresets.js";
 export const defaultWarmIndexReuseConfig = {
     casesPath: "examples/token-savings-cases.json",
     projectProfilesPath: "benchmarks/contracts/benchmark-project-profiles.json",
@@ -12,6 +13,9 @@ const SUPPORTED_FIELDS = [
     "kitCommand",
     "caseIds",
     "benchmarkProjects",
+    "campaignPreset",
+    "includeRealAgents",
+    "timeoutMs",
 ];
 export const warmIndexReuseConfigDefinition = {
     fields: [
@@ -21,6 +25,21 @@ export const warmIndexReuseConfigDefinition = {
         { name: "kitCommand", type: "string", description: "my-dev-kit command used for index and retrieval." },
         { name: "caseIds", type: "array", description: "Evaluation case IDs to include." },
         { name: "benchmarkProjects", type: "array", description: "Benchmark project IDs to include." },
+        {
+            name: "campaignPreset",
+            type: "string",
+            description: `Real-agent campaign preset id: ${WARM_INDEX_CAMPAIGN_PRESET_IDS.join(", ")}. Absent selects the legacy deterministic fake-agent path.`,
+        },
+        {
+            name: "includeRealAgents",
+            type: "boolean",
+            description: "Required (must be true) together with campaignPreset; rejected without it.",
+        },
+        {
+            name: "timeoutMs",
+            type: "number",
+            description: "Optional per-agent timeout override (ms) for a campaign; defaults to the preset's 240000 ms. Rejected without campaignPreset.",
+        },
     ],
 };
 /**
@@ -51,6 +70,38 @@ export function validateWarmIndexReuseConfig(config) {
             errors.push(`${field} must be a non-empty array of non-empty strings when provided.`);
         }
     }
+    // Campaign fields are validated against the RAW supplied object, not the merged/normalized one,
+    // so legacy defaults (casesPath/projectProfilesPath) never look like an explicit override.
+    const raw = isPlainObject(config) ? config : {};
+    const rawCampaignPreset = raw.campaignPreset;
+    if (rawCampaignPreset !== undefined) {
+        if (typeof rawCampaignPreset !== "string" || !WARM_INDEX_CAMPAIGN_PRESET_IDS.includes(rawCampaignPreset)) {
+            errors.push(`campaignPreset must be one of: ${WARM_INDEX_CAMPAIGN_PRESET_IDS.join(", ")}.`);
+        }
+        if (raw.casesPath !== undefined) {
+            errors.push("campaignPreset cannot be combined with an explicit casesPath; the preset owns the production corpus.");
+        }
+        if (raw.projectProfilesPath !== undefined) {
+            errors.push("campaignPreset cannot be combined with an explicit projectProfilesPath; the preset owns the project profiles.");
+        }
+        if (raw.includeRealAgents !== true) {
+            errors.push("campaignPreset requires includeRealAgents to be exactly true.");
+        }
+        if (raw.timeoutMs !== undefined) {
+            const timeoutMs = raw.timeoutMs;
+            if (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs) || !Number.isInteger(timeoutMs) || timeoutMs <= 0) {
+                errors.push("timeoutMs must be a finite positive integer when campaignPreset is set.");
+            }
+        }
+    }
+    else {
+        if (raw.includeRealAgents !== undefined) {
+            errors.push("includeRealAgents is only supported together with campaignPreset.");
+        }
+        if (raw.timeoutMs !== undefined) {
+            errors.push("timeoutMs is only supported together with campaignPreset.");
+        }
+    }
     if (errors.length > 0) {
         return invalidExperimentConfig(errors);
     }

package/dist/src/experiments/plugins/warmIndexReuse/execution.js

@@ -1,5 +1,6 @@
 import path from "node:path";
 import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { assessIndexFreshness } from "../../../evaluation/indexFreshness.js";
 import { runMyDevKitRetrievalFromIndex } from "../../../evaluation/runMyDevKitRetrieval.js";
 import { runRawFullFileBaseline } from "../../../evaluation/runRawFullFileBaseline.js";
 import { groupWarmIndexCases, taskOutputSegment } from "./selection.js";
@@ -90,8 +91,15 @@ async function executeTask(args) {
     }
     let warmRetrieval;
     let warmStatus = "failed";
+    let indexFreshness;
     if (!args.session) {
         errors.push(args.setupFailure ?? { side: "warm", code: "warm-index-setup-failed", message: "No warm index session." });
+        // No snapshot exists, so freshness is conservatively unknown rather than assumed.
+        indexFreshness = await assessIndexFreshness({
+            snapshot: null,
+            targetRoot: evaluationCase.absoluteTargetRoot,
+            sourceRoots: evaluationCase.sourceRoots,
+        });
     }
     else {
         try {
@@ -101,6 +109,12 @@ async function executeTask(args) {
             errors.push({ side: "warm", code: "warm-session-target-mismatch", message: errorMessage(error) });
         }
         if (!errors.some((error) => error.side === "warm")) {
+            // Observational only: measured immediately before reuse, and retrieval below runs regardless.
+            indexFreshness = await assessIndexFreshness({
+                snapshot: args.session.indexSnapshot,
+                targetRoot: args.session.targetRoot,
+                sourceRoots: args.session.sourceRoots,
+            });
             try {
                 // requireKit: false retains subordinate command evidence; a skipped result caused by a
                 // failed command is classified as failed below rather than as an honest skip.
@@ -134,6 +148,7 @@ async function executeTask(args) {
         warmStatus,
         rawBaseline,
         warmRetrieval,
+        indexFreshness,
         warnings,
         errors,
     };

package/dist/src/experiments/plugins/warmIndexReuse/executionArtifact.js

@@ -46,6 +46,7 @@ function summarizeTask(task) {
                 commands: warm.commands.map(summarizeCommand),
             }
             : null,
+        indexFreshness: task.indexFreshness ? structuredClone(task.indexFreshness) : null,
         warnings: [...task.warnings],
         errors: task.errors.map((error) => ({ ...error })),
     };
@@ -62,6 +63,7 @@ export function summarizeProjectExecution(project) {
         sessionPrepared: project.session !== undefined,
         buildDurationMs: project.build ? project.build.durationMs : null,
         indexCommand: project.build ? summarizeCommand(project.build.command) : null,
+        indexSnapshot: project.session ? structuredClone(project.session.indexSnapshot) : null,
         tasks: project.tasks.map(summarizeTask),
         warnings: [...project.warnings],
         errors: [...project.errors],

package/dist/src/experiments/plugins/warmIndexReuse/fakeAgentEvaluation.js

@@ -1,118 +1,6 @@
-import path from "node:path";
-import { runAgentPrompt } from "../../../agents/index.js";
-import { resolveWithinRoot } from "../../../core/pathSafety.js";
-import { classifyAgentRunOutcome } from "../../../evaluation/classifyAgentRunOutcome.js";
-import { parseAgentAnswer } from "../../../evaluation/parseAgentAnswer.js";
-import { scoreCorrectness } from "../../../evaluation/scoreCorrectness.js";
-import { generatePromptVariants } from "../../../prompts/index.js";
-import { taskOutputSegment } from "./selection.js";
-// Classified statuses for which the scorer evaluated an actual agent answer.
-const SCOREABLE_STATUSES = new Set(["completed", "invalid-output"]);
-const SIDE_STRATEGIES = {
-    "raw-full-file": "raw-full-file",
-    "warm-index-reuse": "my-dev-kit-guided",
-};
-export async function evaluateWarmIndexFakeAgents(options) {
-    const casesById = new Map(options.cases.map((evaluationCase) => [evaluationCase.id, evaluationCase]));
-    const result = [];
-    for (const project of options.projects) {
-        const tasks = [];
-        for (const task of project.tasks) {
-            const evaluationCase = casesById.get(task.caseId);
-            const run = (variantId) => evaluateSide({
-                evaluationCase,
-                caseId: task.caseId,
-                variantId,
-                projectProfiles: options.projectProfiles,
-                outDir: resolveWithinRoot(options.outputRoot, path.join("agents", project.projectSegment, taskOutputSegment(task.caseId), variantId)),
-                cwd: options.cwd,
-                env: options.env,
-            });
-            tasks.push({
-                caseId: task.caseId,
-                raw: task.rawBaseline ? await run("raw-full-file") : null,
-                warm: task.warmRetrieval ? await run("warm-index-reuse") : null,
-            });
-        }
-        result.push({ benchmarkProject: project.benchmarkProject, tasks });
-    }
-    return result;
-}
-async function evaluateSide(args) {
-    const promptStrategy = SIDE_STRATEGIES[args.variantId];
-    const base = { variantId: args.variantId, agentId: "fake-agent", promptStrategy };
-    try {
-        if (!args.evaluationCase) {
-            throw new Error(`Evaluation case ${args.caseId} is not available for fake-agent evaluation.`);
-        }
-        const [promptVariant] = generatePromptVariants({
-            cases: [args.evaluationCase],
-            projectProfiles: [...args.projectProfiles],
-            strategies: [promptStrategy],
-            complexityLevels: ["short"],
-        });
-        if (!promptVariant) {
-            throw new Error(`Failed to generate a fake-agent prompt for case ${args.caseId}.`);
-        }
-        const agentRunResult = await runAgentPrompt({
-            runId: `${args.caseId}.fake-agent.${args.variantId}`,
-            agentId: "fake-agent",
-            promptVariant,
-            promptText: promptVariant.promptText,
-            cwd: args.cwd,
-            outDir: args.outDir,
-            env: args.env,
-        });
-        const parsedAnswer = parseAgentAnswer({
-            text: agentRunResult.finalAnswerText,
-            answerKey: promptVariant.expectedAnswerKey,
-            tokenUsage: agentRunResult.tokenUsage,
-        });
-        const classification = classifyAgentRunOutcome({ agentRunResult, parsedAnswer });
-        const correctness = scoreCorrectness({
-            caseId: args.caseId,
-            answerKey: promptVariant.expectedAnswerKey,
-            parsedAnswer,
-            status: classification.status,
-        });
-        const scoreable = SCOREABLE_STATUSES.has(classification.status) && Number.isFinite(correctness.correctnessScore);
-        const totalTokens = agentRunResult.tokenUsage.totalTokens;
-        return {
-            ...base,
-            status: classification.status,
-            correctness: {
-                available: scoreable,
-                score: scoreable ? correctness.correctnessScore : null,
-                passed: scoreable ? correctness.passed : null,
-                failureReasons: [...correctness.failureReasons],
-            },
-            tokenUsage: {
-                totalTokens: typeof totalTokens === "number" && Number.isFinite(totalTokens) ? totalTokens : null,
-                source: agentRunResult.tokenUsageSource,
-                reliability: agentRunResult.tokenUsageReliability,
-            },
-            durationMs: agentRunResult.durationMs,
-            warnings: [...classification.warnings],
-            errors: [...classification.errors],
-            artifactPaths: {
-                promptPath: path.join(args.outDir, "prompt.txt"),
-                agentRunResultPath: path.join(args.outDir, "agent-run-result.json"),
-                stdoutPath: agentRunResult.stdoutPath,
-                stderrPath: agentRunResult.stderrPath,
-                telemetryPath: agentRunResult.telemetryPath,
-            },
-        };
-    }
-    catch (error) {
-        return {
-            ...base,
-            status: "failed",
-            correctness: { available: false, score: null, passed: null, failureReasons: [] },
-            tokenUsage: { totalTokens: null, source: "unavailable", reliability: "unavailable" },
-            durationMs: null,
-            warnings: [],
-            errors: [error instanceof Error ? error.message : String(error)],
-            artifactPaths: {},
-        };
-    }
-}
+// v0.5.2 Batch 3 -- thin compatibility re-export. The shared bounded evidence types,
+// scoring/classification pipeline, and both the fake-agent and real-agent campaign evaluators now
+// live in one owner: ./agentEvaluation.ts. This file exists only so existing direct imports (for
+// example src/experiments/plugins/warmIndexReuse/metrics.ts and
+// src/report/experiments/buildWarmIndexReuseReport.ts) and existing tests keep working unchanged.
+export { evaluateWarmIndexFakeAgents, } from "./agentEvaluation.js";

package/dist/src/experiments/plugins/warmIndexReuse/index.js

@@ -1,8 +1,10 @@
+export * from "./agentEvaluation.js";
+export * from "./campaignPresets.js";
 export * from "./config.js";
 export * from "./execution.js";
 export * from "./executionArtifact.js";
-export * from "./fakeAgentEvaluation.js";
 export * from "./metrics.js";
 export * from "./plugin.js";
+export * from "./realAgentPrompt.js";
 export * from "./selection.js";
 export * from "./warmIndexSession.js";