@dailephd/my-dev-kit-lab 0.5.1 → 0.5.2

package/dist/src/core/runMeasuredCommand.js

@@ -41,13 +41,15 @@ export async function runMeasuredCommand(options) {
         let spawnError;
         let timedOut = false;
         let timeout;
+        let killGraceTimeout;
+        let settled = false;
         let child;
         try {
             child = spawn(executable, args, {
                 cwd: options.cwd,
                 env: { ...process.env, ...options.env },
                 shell: false,
-                stdio: ["ignore", "pipe", "pipe"]
+                stdio: options.stdinText !== undefined ? ["pipe", "pipe", "pipe"] : ["ignore", "pipe", "pipe"]
             });
         }
         catch (error) {
@@ -80,6 +82,7 @@ export async function runMeasuredCommand(options) {
             ]).then(() => resolve(measured));
             return;
         }
+        // stdout/stderr are always piped in both stdio branches above.
         child.stdout.on("data", (chunk) => {
             stdout += String(chunk);
         });
@@ -89,17 +92,40 @@ export async function runMeasuredCommand(options) {
         child.on("error", (error) => {
             spawnError = error.message;
         });
-        if (options.timeoutMs !== undefined) {
-            timeout = setTimeout(() => {
-                timedOut = true;
-                spawnError = `Command timed out after ${options.timeoutMs}ms.`;
-                forceTerminateProcess(child.pid);
-            }, options.timeoutMs);
+        if (options.stdinText !== undefined) {
+            // A child that exits or closes stdin before we write must not crash the process (EPIPE).
+            child.stdin?.on("error", (error) => {
+                spawnError = spawnError ?? `Failed to write to command stdin: ${error.message}`;
+            });
+            child.stdin?.write(options.stdinText, "utf8");
+            child.stdin?.end();
         }
-        child.on("close", async (exitCode) => {
-            if (timeout) {
+        // forceTerminateProcess is fire-and-forget (it never confirms the target process tree actually
+        // exited: on Windows it spawns a detached `taskkill /T /F` and does not await or check its
+        // result). That is normally sufficient, but a grandchild process spawned through an
+        // intermediate shim (for example a Windows .cmd launcher) can occasionally outlive an
+        // already-killed intermediate process. Without a bounded fallback, a caller whose kill silently
+        // failed would await this promise forever instead of receiving the already-known timeout error.
+        const KILL_GRACE_MS = 3000;
+        const settle = async (exitCode) => {
+            if (settled)
+                return;
+            settled = true;
+            if (timeout)
                 clearTimeout(timeout);
-            }
+            if (killGraceTimeout)
+                clearTimeout(killGraceTimeout);
+            // The caller's `scripts/cli.ts` entry point sets `process.exitCode` rather than calling
+            // `process.exit()`, so the process only exits once Node's event loop actually drains. A
+            // still-open child stdio pipe -- the same situation the kill-grace fallback above exists for
+            // -- would otherwise keep this process alive forever even though this promise has already
+            // resolved. Detaching is safe to do unconditionally: once "close" has fired the streams are
+            // already ended, so this is a harmless no-op on the normal (non-fallback) path.
+            child.stdout?.destroy();
+            child.stderr?.destroy();
+            child.stdin?.destroy();
+            child.removeAllListeners();
+            child.unref();
             const endedAt = new Date().toISOString();
             const durationMs = Date.now() - started;
             await writeArtifact(stdoutPath, stdout);
@@ -125,6 +151,19 @@ export async function runMeasuredCommand(options) {
             };
             await writeArtifact(telemetryPath, JSON.stringify(measured, null, 2));
             resolve(measured);
+        };
+        if (options.timeoutMs !== undefined) {
+            timeout = setTimeout(() => {
+                timedOut = true;
+                spawnError = `Command timed out after ${options.timeoutMs}ms.`;
+                forceTerminateProcess(child.pid);
+                killGraceTimeout = setTimeout(() => {
+                    void settle(null);
+                }, KILL_GRACE_MS);
+            }, options.timeoutMs);
+        }
+        child.on("close", (exitCode) => {
+            void settle(exitCode);
         });
     });
     return result;

package/dist/src/experiments/plugins/warmIndexReuse/agentEvaluation.js

@@ -0,0 +1,233 @@
+import { mkdtemp, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { runAgentPrompt } from "../../../agents/index.js";
+import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { classifyAgentRunOutcome } from "../../../evaluation/classifyAgentRunOutcome.js";
+import { parseAgentAnswer } from "../../../evaluation/parseAgentAnswer.js";
+import { scoreCorrectness } from "../../../evaluation/scoreCorrectness.js";
+import { generatePromptVariants } from "../../../prompts/index.js";
+import { buildWarmIndexRealAgentPrompt } from "./realAgentPrompt.js";
+import { taskOutputSegment } from "./selection.js";
+// Classified statuses for which the scorer evaluated an actual agent answer.
+const SCOREABLE_STATUSES = new Set(["completed", "invalid-output"]);
+const SIDE_STRATEGIES = {
+    "raw-full-file": "raw-full-file",
+    "warm-index-reuse": "my-dev-kit-guided",
+};
+export async function evaluateWarmIndexFakeAgents(options) {
+    const casesById = new Map(options.cases.map((evaluationCase) => [evaluationCase.id, evaluationCase]));
+    const result = [];
+    for (const project of options.projects) {
+        const tasks = [];
+        for (const task of project.tasks) {
+            const evaluationCase = casesById.get(task.caseId);
+            const run = (variantId) => evaluateFakeSide({
+                evaluationCase,
+                caseId: task.caseId,
+                variantId,
+                projectProfiles: options.projectProfiles,
+                outDir: resolveWithinRoot(options.outputRoot, path.join("agents", project.projectSegment, taskOutputSegment(task.caseId), variantId)),
+                cwd: options.cwd,
+                env: options.env,
+            });
+            tasks.push({
+                caseId: task.caseId,
+                raw: task.rawBaseline ? await run("raw-full-file") : null,
+                warm: task.warmRetrieval ? await run("warm-index-reuse") : null,
+            });
+        }
+        result.push({ benchmarkProject: project.benchmarkProject, tasks });
+    }
+    return result;
+}
+async function evaluateFakeSide(args) {
+    const promptStrategy = SIDE_STRATEGIES[args.variantId];
+    const base = { variantId: args.variantId, agentId: "fake-agent", promptStrategy };
+    try {
+        if (!args.evaluationCase) {
+            throw new Error(`Evaluation case ${args.caseId} is not available for fake-agent evaluation.`);
+        }
+        const [promptVariant] = generatePromptVariants({
+            cases: [args.evaluationCase],
+            projectProfiles: [...args.projectProfiles],
+            strategies: [promptStrategy],
+            complexityLevels: ["short"],
+        });
+        if (!promptVariant) {
+            throw new Error(`Failed to generate a fake-agent prompt for case ${args.caseId}.`);
+        }
+        const agentRunResult = await runAgentPrompt({
+            runId: `${args.caseId}.fake-agent.${args.variantId}`,
+            agentId: "fake-agent",
+            promptVariant,
+            promptText: promptVariant.promptText,
+            cwd: args.cwd,
+            outDir: args.outDir,
+            env: args.env,
+        });
+        return buildSideEvidenceFromRun({ base, caseId: args.caseId, outDir: args.outDir, agentRunResult, expectedAnswerKey: promptVariant.expectedAnswerKey });
+    }
+    catch (error) {
+        return buildFailedSideEvidence(base, error);
+    }
+}
+/**
+ * Real-agent campaign evaluator (v0.5.2 Batch 3): consumes the exact context text the warm-index
+ * execution stage already produced for one side. Never calls my-dev-kit, never independently
+ * inspects the repository, and never runs a side with no usable context.
+ */
+export async function evaluateWarmIndexRealAgentCampaign(options) {
+    const casesById = new Map(options.cases.map((evaluationCase) => [evaluationCase.id, evaluationCase]));
+    const result = [];
+    for (const project of options.projects) {
+        const tasks = [];
+        for (const task of project.tasks) {
+            const evaluationCase = casesById.get(task.caseId);
+            const rawContext = task.rawBaseline?.contextText;
+            const rawEligible = typeof rawContext === "string" && rawContext.trim().length > 0;
+            const warmContext = task.warmRetrieval?.contextText;
+            const warmEligible = task.warmRetrieval !== undefined &&
+                task.warmRetrieval.skipped === false &&
+                typeof warmContext === "string" &&
+                warmContext.trim().length > 0;
+            const runSide = (variantId, contextText) => evaluateRealSide({
+                evaluationCase,
+                caseId: task.caseId,
+                variantId,
+                contextText,
+                projectProfiles: options.projectProfiles,
+                outDir: resolveWithinRoot(options.outputRoot, path.join("agents", project.projectSegment, taskOutputSegment(task.caseId), variantId)),
+                agentId: options.agentId,
+                timeoutMs: options.timeoutMs,
+                env: options.env,
+            });
+            tasks.push({
+                caseId: task.caseId,
+                raw: rawEligible ? await runSide("raw-full-file", rawContext) : null,
+                warm: warmEligible ? await runSide("warm-index-reuse", warmContext) : null,
+            });
+        }
+        result.push({ benchmarkProject: project.benchmarkProject, tasks });
+    }
+    return result;
+}
+async function evaluateRealSide(args) {
+    const promptStrategy = SIDE_STRATEGIES[args.variantId];
+    const base = { variantId: args.variantId, agentId: args.agentId, promptStrategy };
+    let neutralCwd;
+    try {
+        if (!args.evaluationCase) {
+            throw new Error(`Evaluation case ${args.caseId} is not available for real-agent evaluation.`);
+        }
+        // Generated only for internal scoring/metadata (expectedAnswerKey, promptComplexityLevel); its
+        // promptText is never sent to the provider.
+        const [promptVariant] = generatePromptVariants({
+            cases: [args.evaluationCase],
+            projectProfiles: [...args.projectProfiles],
+            strategies: [promptStrategy],
+            complexityLevels: ["short"],
+        });
+        if (!promptVariant) {
+            throw new Error(`Failed to generate an internal scoring prompt variant for case ${args.caseId}.`);
+        }
+        const realPromptText = buildWarmIndexRealAgentPrompt({
+            evaluationCase: args.evaluationCase,
+            variantId: args.variantId,
+            contextText: args.contextText,
+        });
+        // Neutral working directory: the provider must not run from the repository, the benchmark
+        // target, the installed package, or the agent artifact directory.
+        neutralCwd = await mkdtemp(path.join(os.tmpdir(), "my-dev-kit-lab-warm-agent-"));
+        const agentRunResult = await runAgentPrompt({
+            runId: `${args.caseId}.${args.agentId}.${args.variantId}`,
+            agentId: args.agentId,
+            promptVariant,
+            promptText: realPromptText,
+            promptTransport: "stdin",
+            timeoutMs: args.timeoutMs,
+            requireAvailable: false,
+            env: args.env,
+            cwd: neutralCwd,
+            outDir: args.outDir,
+        });
+        return buildSideEvidenceFromRun({
+            base,
+            caseId: args.caseId,
+            outDir: args.outDir,
+            agentRunResult,
+            expectedAnswerKey: promptVariant.expectedAnswerKey,
+        });
+    }
+    catch (error) {
+        return buildFailedSideEvidence(base, error);
+    }
+    finally {
+        if (neutralCwd) {
+            try {
+                await rm(neutralCwd, { recursive: true, force: true });
+            }
+            catch {
+                // Best-effort cleanup only: { force: true } suppresses a missing path, but not an
+                // EPERM/EBUSY from an OS still holding this directory open -- for example a timed-out
+                // provider process whose kill did not (yet) fully release it as its own cwd. A cleanup
+                // failure here must never replace or hide the side's already-classified evidence.
+            }
+        }
+    }
+}
+// Shared scoring/classification/evidence assembly for both the fake-agent and real-agent paths.
+function buildSideEvidenceFromRun(args) {
+    const { base, caseId, outDir, agentRunResult, expectedAnswerKey } = args;
+    const parsedAnswer = parseAgentAnswer({
+        text: agentRunResult.finalAnswerText,
+        answerKey: expectedAnswerKey,
+        tokenUsage: agentRunResult.tokenUsage,
+    });
+    const classification = classifyAgentRunOutcome({ agentRunResult, parsedAnswer });
+    const correctness = scoreCorrectness({
+        caseId,
+        answerKey: expectedAnswerKey,
+        parsedAnswer,
+        status: classification.status,
+    });
+    const scoreable = SCOREABLE_STATUSES.has(classification.status) && Number.isFinite(correctness.correctnessScore);
+    const totalTokens = agentRunResult.tokenUsage.totalTokens;
+    return {
+        ...base,
+        status: classification.status,
+        correctness: {
+            available: scoreable,
+            score: scoreable ? correctness.correctnessScore : null,
+            passed: scoreable ? correctness.passed : null,
+            failureReasons: [...correctness.failureReasons],
+        },
+        tokenUsage: {
+            totalTokens: typeof totalTokens === "number" && Number.isFinite(totalTokens) ? totalTokens : null,
+            source: agentRunResult.tokenUsageSource,
+            reliability: agentRunResult.tokenUsageReliability,
+        },
+        durationMs: agentRunResult.durationMs,
+        warnings: [...classification.warnings],
+        errors: [...classification.errors],
+        artifactPaths: {
+            promptPath: path.join(outDir, "prompt.txt"),
+            agentRunResultPath: path.join(outDir, "agent-run-result.json"),
+            stdoutPath: agentRunResult.stdoutPath,
+            stderrPath: agentRunResult.stderrPath,
+            telemetryPath: agentRunResult.telemetryPath,
+        },
+    };
+}
+function buildFailedSideEvidence(base, error) {
+    return {
+        ...base,
+        status: "failed",
+        correctness: { available: false, score: null, passed: null, failureReasons: [] },
+        tokenUsage: { totalTokens: null, source: "unavailable", reliability: "unavailable" },
+        durationMs: null,
+        warnings: [],
+        errors: [error instanceof Error ? error.message : String(error)],
+        artifactPaths: {},
+    };
+}

package/dist/src/experiments/plugins/warmIndexReuse/campaignPresets.js

@@ -0,0 +1,83 @@
+// v0.5.2 Batch 1 -- frozen warm-index real-agent campaign preset contract.
+//
+// This module owns preset identity, provider assignment, bundled resource paths, the frozen case
+// list per preset, and the default per-agent timeout. It contains no CLI parsing, no report
+// rendering, and no provider execution logic; those remain owned elsewhere.
+export const WARM_INDEX_CAMPAIGN_PRESET_IDS = ["codex-full", "claude-full", "codex-timeout-isolation"];
+const CASES_RESOURCE_PATH = "benchmarks/contracts/warm-index-benchmark-cases.json";
+const PROJECT_PROFILES_RESOURCE_PATH = "benchmarks/contracts/benchmark-project-profiles.json";
+const DEFAULT_TIMEOUT_MS = 240_000;
+// Exact v0.5.1 12-case corpus, in corpus order. Frozen; do not reorder or extend in Batch 1.
+const FULL_CAMPAIGN_CASE_IDS = Object.freeze([
+    "warm-medium-import-dedupe",
+    "warm-medium-create-project-task",
+    "warm-medium-complete-idempotent",
+    "warm-medium-composite-filter",
+    "warm-medium-project-summary",
+    "warm-medium-broad-workflow-map",
+    "warm-large-health-label",
+    "warm-large-ts-analytics-snapshot",
+    "warm-large-ts-leaderboard",
+    "warm-large-python-parser-metrics",
+    "warm-large-python-pipeline",
+    "warm-large-broad-analytics-comparison",
+]);
+// Exactly the three large/mixed cases covering cross-module, localized, and broad-change.
+const TIMEOUT_ISOLATION_CASE_IDS = Object.freeze([
+    "warm-large-health-label",
+    "warm-large-ts-leaderboard",
+    "warm-large-broad-analytics-comparison",
+]);
+const WARM_INDEX_CAMPAIGN_PRESETS = Object.freeze({
+    "codex-full": Object.freeze({
+        id: "codex-full",
+        agentId: "codex",
+        casesResourcePath: CASES_RESOURCE_PATH,
+        projectProfilesResourcePath: PROJECT_PROFILES_RESOURCE_PATH,
+        caseIds: FULL_CAMPAIGN_CASE_IDS,
+        defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
+    }),
+    "claude-full": Object.freeze({
+        id: "claude-full",
+        agentId: "claude",
+        casesResourcePath: CASES_RESOURCE_PATH,
+        projectProfilesResourcePath: PROJECT_PROFILES_RESOURCE_PATH,
+        caseIds: FULL_CAMPAIGN_CASE_IDS,
+        defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
+    }),
+    "codex-timeout-isolation": Object.freeze({
+        id: "codex-timeout-isolation",
+        agentId: "codex",
+        casesResourcePath: CASES_RESOURCE_PATH,
+        projectProfilesResourcePath: PROJECT_PROFILES_RESOURCE_PATH,
+        caseIds: TIMEOUT_ISOLATION_CASE_IDS,
+        defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
+    }),
+});
+/** Fails explicitly for any value outside the exact three frozen preset IDs. */
+export function parseWarmIndexCampaignPresetId(value) {
+    if (WARM_INDEX_CAMPAIGN_PRESET_IDS.includes(value)) {
+        return value;
+    }
+    throw new Error(`Unknown warm-index campaign preset: ${value}. Supported presets: ${WARM_INDEX_CAMPAIGN_PRESET_IDS.join(", ")}.`);
+}
+export function getWarmIndexCampaignPreset(id) {
+    const preset = WARM_INDEX_CAMPAIGN_PRESETS[id];
+    if (!preset) {
+        throw new Error(`Unknown warm-index campaign preset: ${id}. Supported presets: ${WARM_INDEX_CAMPAIGN_PRESET_IDS.join(", ")}.`);
+    }
+    return preset;
+}
+/**
+ * Resolves the effective per-agent timeout for a campaign: the preset default when no override was
+ * requested, or an explicit finite positive integer override.
+ */
+export function resolveWarmIndexCampaignTimeoutMs(preset, requestedTimeoutMs) {
+    if (requestedTimeoutMs === undefined) {
+        return preset.defaultTimeoutMs;
+    }
+    if (!Number.isFinite(requestedTimeoutMs) || !Number.isInteger(requestedTimeoutMs) || requestedTimeoutMs <= 0) {
+        throw new Error(`Campaign timeoutMs must be a finite positive integer; received ${requestedTimeoutMs}.`);
+    }
+    return requestedTimeoutMs;
+}

package/dist/src/experiments/plugins/warmIndexReuse/config.js

@@ -1,4 +1,5 @@
 import { invalidExperimentConfig, isPlainObject, mergeConfig, validExperimentConfig } from "../../config.js";
+import { WARM_INDEX_CAMPAIGN_PRESET_IDS } from "./campaignPresets.js";
 export const defaultWarmIndexReuseConfig = {
     casesPath: "examples/token-savings-cases.json",
     projectProfilesPath: "benchmarks/contracts/benchmark-project-profiles.json",
@@ -12,6 +13,9 @@ const SUPPORTED_FIELDS = [
     "kitCommand",
     "caseIds",
     "benchmarkProjects",
+    "campaignPreset",
+    "includeRealAgents",
+    "timeoutMs",
 ];
 export const warmIndexReuseConfigDefinition = {
     fields: [
@@ -21,6 +25,21 @@ export const warmIndexReuseConfigDefinition = {
         { name: "kitCommand", type: "string", description: "my-dev-kit command used for index and retrieval." },
         { name: "caseIds", type: "array", description: "Evaluation case IDs to include." },
         { name: "benchmarkProjects", type: "array", description: "Benchmark project IDs to include." },
+        {
+            name: "campaignPreset",
+            type: "string",
+            description: `Real-agent campaign preset id: ${WARM_INDEX_CAMPAIGN_PRESET_IDS.join(", ")}. Absent selects the legacy deterministic fake-agent path.`,
+        },
+        {
+            name: "includeRealAgents",
+            type: "boolean",
+            description: "Required (must be true) together with campaignPreset; rejected without it.",
+        },
+        {
+            name: "timeoutMs",
+            type: "number",
+            description: "Optional per-agent timeout override (ms) for a campaign; defaults to the preset's 240000 ms. Rejected without campaignPreset.",
+        },
     ],
 };
 /**
@@ -51,6 +70,38 @@ export function validateWarmIndexReuseConfig(config) {
             errors.push(`${field} must be a non-empty array of non-empty strings when provided.`);
         }
     }
+    // Campaign fields are validated against the RAW supplied object, not the merged/normalized one,
+    // so legacy defaults (casesPath/projectProfilesPath) never look like an explicit override.
+    const raw = isPlainObject(config) ? config : {};
+    const rawCampaignPreset = raw.campaignPreset;
+    if (rawCampaignPreset !== undefined) {
+        if (typeof rawCampaignPreset !== "string" || !WARM_INDEX_CAMPAIGN_PRESET_IDS.includes(rawCampaignPreset)) {
+            errors.push(`campaignPreset must be one of: ${WARM_INDEX_CAMPAIGN_PRESET_IDS.join(", ")}.`);
+        }
+        if (raw.casesPath !== undefined) {
+            errors.push("campaignPreset cannot be combined with an explicit casesPath; the preset owns the production corpus.");
+        }
+        if (raw.projectProfilesPath !== undefined) {
+            errors.push("campaignPreset cannot be combined with an explicit projectProfilesPath; the preset owns the project profiles.");
+        }
+        if (raw.includeRealAgents !== true) {
+            errors.push("campaignPreset requires includeRealAgents to be exactly true.");
+        }
+        if (raw.timeoutMs !== undefined) {
+            const timeoutMs = raw.timeoutMs;
+            if (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs) || !Number.isInteger(timeoutMs) || timeoutMs <= 0) {
+                errors.push("timeoutMs must be a finite positive integer when campaignPreset is set.");
+            }
+        }
+    }
+    else {
+        if (raw.includeRealAgents !== undefined) {
+            errors.push("includeRealAgents is only supported together with campaignPreset.");
+        }
+        if (raw.timeoutMs !== undefined) {
+            errors.push("timeoutMs is only supported together with campaignPreset.");
+        }
+    }
     if (errors.length > 0) {
         return invalidExperimentConfig(errors);
     }

package/dist/src/experiments/plugins/warmIndexReuse/fakeAgentEvaluation.js

@@ -1,118 +1,6 @@
-import path from "node:path";
-import { runAgentPrompt } from "../../../agents/index.js";
-import { resolveWithinRoot } from "../../../core/pathSafety.js";
-import { classifyAgentRunOutcome } from "../../../evaluation/classifyAgentRunOutcome.js";
-import { parseAgentAnswer } from "../../../evaluation/parseAgentAnswer.js";
-import { scoreCorrectness } from "../../../evaluation/scoreCorrectness.js";
-import { generatePromptVariants } from "../../../prompts/index.js";
-import { taskOutputSegment } from "./selection.js";
-// Classified statuses for which the scorer evaluated an actual agent answer.
-const SCOREABLE_STATUSES = new Set(["completed", "invalid-output"]);
-const SIDE_STRATEGIES = {
-    "raw-full-file": "raw-full-file",
-    "warm-index-reuse": "my-dev-kit-guided",
-};
-export async function evaluateWarmIndexFakeAgents(options) {
-    const casesById = new Map(options.cases.map((evaluationCase) => [evaluationCase.id, evaluationCase]));
-    const result = [];
-    for (const project of options.projects) {
-        const tasks = [];
-        for (const task of project.tasks) {
-            const evaluationCase = casesById.get(task.caseId);
-            const run = (variantId) => evaluateSide({
-                evaluationCase,
-                caseId: task.caseId,
-                variantId,
-                projectProfiles: options.projectProfiles,
-                outDir: resolveWithinRoot(options.outputRoot, path.join("agents", project.projectSegment, taskOutputSegment(task.caseId), variantId)),
-                cwd: options.cwd,
-                env: options.env,
-            });
-            tasks.push({
-                caseId: task.caseId,
-                raw: task.rawBaseline ? await run("raw-full-file") : null,
-                warm: task.warmRetrieval ? await run("warm-index-reuse") : null,
-            });
-        }
-        result.push({ benchmarkProject: project.benchmarkProject, tasks });
-    }
-    return result;
-}
-async function evaluateSide(args) {
-    const promptStrategy = SIDE_STRATEGIES[args.variantId];
-    const base = { variantId: args.variantId, agentId: "fake-agent", promptStrategy };
-    try {
-        if (!args.evaluationCase) {
-            throw new Error(`Evaluation case ${args.caseId} is not available for fake-agent evaluation.`);
-        }
-        const [promptVariant] = generatePromptVariants({
-            cases: [args.evaluationCase],
-            projectProfiles: [...args.projectProfiles],
-            strategies: [promptStrategy],
-            complexityLevels: ["short"],
-        });
-        if (!promptVariant) {
-            throw new Error(`Failed to generate a fake-agent prompt for case ${args.caseId}.`);
-        }
-        const agentRunResult = await runAgentPrompt({
-            runId: `${args.caseId}.fake-agent.${args.variantId}`,
-            agentId: "fake-agent",
-            promptVariant,
-            promptText: promptVariant.promptText,
-            cwd: args.cwd,
-            outDir: args.outDir,
-            env: args.env,
-        });
-        const parsedAnswer = parseAgentAnswer({
-            text: agentRunResult.finalAnswerText,
-            answerKey: promptVariant.expectedAnswerKey,
-            tokenUsage: agentRunResult.tokenUsage,
-        });
-        const classification = classifyAgentRunOutcome({ agentRunResult, parsedAnswer });
-        const correctness = scoreCorrectness({
-            caseId: args.caseId,
-            answerKey: promptVariant.expectedAnswerKey,
-            parsedAnswer,
-            status: classification.status,
-        });
-        const scoreable = SCOREABLE_STATUSES.has(classification.status) && Number.isFinite(correctness.correctnessScore);
-        const totalTokens = agentRunResult.tokenUsage.totalTokens;
-        return {
-            ...base,
-            status: classification.status,
-            correctness: {
-                available: scoreable,
-                score: scoreable ? correctness.correctnessScore : null,
-                passed: scoreable ? correctness.passed : null,
-                failureReasons: [...correctness.failureReasons],
-            },
-            tokenUsage: {
-                totalTokens: typeof totalTokens === "number" && Number.isFinite(totalTokens) ? totalTokens : null,
-                source: agentRunResult.tokenUsageSource,
-                reliability: agentRunResult.tokenUsageReliability,
-            },
-            durationMs: agentRunResult.durationMs,
-            warnings: [...classification.warnings],
-            errors: [...classification.errors],
-            artifactPaths: {
-                promptPath: path.join(args.outDir, "prompt.txt"),
-                agentRunResultPath: path.join(args.outDir, "agent-run-result.json"),
-                stdoutPath: agentRunResult.stdoutPath,
-                stderrPath: agentRunResult.stderrPath,
-                telemetryPath: agentRunResult.telemetryPath,
-            },
-        };
-    }
-    catch (error) {
-        return {
-            ...base,
-            status: "failed",
-            correctness: { available: false, score: null, passed: null, failureReasons: [] },
-            tokenUsage: { totalTokens: null, source: "unavailable", reliability: "unavailable" },
-            durationMs: null,
-            warnings: [],
-            errors: [error instanceof Error ? error.message : String(error)],
-            artifactPaths: {},
-        };
-    }
-}
+// v0.5.2 Batch 3 -- thin compatibility re-export. The shared bounded evidence types,
+// scoring/classification pipeline, and both the fake-agent and real-agent campaign evaluators now
+// live in one owner: ./agentEvaluation.ts. This file exists only so existing direct imports (for
+// example src/experiments/plugins/warmIndexReuse/metrics.ts and
+// src/report/experiments/buildWarmIndexReuseReport.ts) and existing tests keep working unchanged.
+export { evaluateWarmIndexFakeAgents, } from "./agentEvaluation.js";

package/dist/src/experiments/plugins/warmIndexReuse/index.js

@@ -1,8 +1,10 @@
+export * from "./agentEvaluation.js";
+export * from "./campaignPresets.js";
 export * from "./config.js";
 export * from "./execution.js";
 export * from "./executionArtifact.js";
-export * from "./fakeAgentEvaluation.js";
 export * from "./metrics.js";
 export * from "./plugin.js";
+export * from "./realAgentPrompt.js";
 export * from "./selection.js";
 export * from "./warmIndexSession.js";