@dailephd/my-dev-kit-lab 0.4.9 → 0.5.1

package/dist/src/experiments/plugins/warmIndexReuse/config.js

@@ -0,0 +1,58 @@
+import { invalidExperimentConfig, isPlainObject, mergeConfig, validExperimentConfig } from "../../config.js";
+export const defaultWarmIndexReuseConfig = {
+    casesPath: "examples/token-savings-cases.json",
+    projectProfilesPath: "benchmarks/contracts/benchmark-project-profiles.json",
+    outDir: "lab-output/warm-index-reuse",
+    kitCommand: "npx @dailephd/my-dev-kit@latest",
+};
+const SUPPORTED_FIELDS = [
+    "casesPath",
+    "projectProfilesPath",
+    "outDir",
+    "kitCommand",
+    "caseIds",
+    "benchmarkProjects",
+];
+export const warmIndexReuseConfigDefinition = {
+    fields: [
+        { name: "casesPath", type: "string", required: true, description: "Evaluation cases file." },
+        { name: "outDir", type: "string", required: true, description: "Experiment output root." },
+        { name: "projectProfilesPath", type: "string", description: "Benchmark project profiles file." },
+        { name: "kitCommand", type: "string", description: "my-dev-kit command used for index and retrieval." },
+        { name: "caseIds", type: "array", description: "Evaluation case IDs to include." },
+        { name: "benchmarkProjects", type: "array", description: "Benchmark project IDs to include." },
+    ],
+};
+/**
+ * Closed validation: fields outside the warm-index contract (for example agent-matrix settings)
+ * are rejected instead of being silently ignored.
+ */
+export function validateWarmIndexReuseConfig(config) {
+    if (config !== undefined && !isPlainObject(config)) {
+        return invalidExperimentConfig(["warm index reuse config must be an object."]);
+    }
+    const errors = [];
+    const unsupported = Object.keys(config ?? {}).filter((key) => !SUPPORTED_FIELDS.includes(key));
+    if (unsupported.length > 0) {
+        errors.push(`Unsupported warm-index-reuse config field(s): ${unsupported.sort().join(", ")}.`);
+    }
+    const normalized = mergeConfig(defaultWarmIndexReuseConfig, config);
+    for (const field of ["casesPath", "projectProfilesPath", "outDir", "kitCommand"]) {
+        const value = normalized[field];
+        if (typeof value !== "string" || !value.trim()) {
+            errors.push(`${field} must be a non-empty string.`);
+        }
+    }
+    for (const field of ["caseIds", "benchmarkProjects"]) {
+        const value = normalized[field];
+        if (value === undefined)
+            continue;
+        if (!Array.isArray(value) || value.length === 0 || !value.every((item) => typeof item === "string" && item.trim())) {
+            errors.push(`${field} must be a non-empty array of non-empty strings when provided.`);
+        }
+    }
+    if (errors.length > 0) {
+        return invalidExperimentConfig(errors);
+    }
+    return validExperimentConfig(normalized);
+}

package/dist/src/experiments/plugins/warmIndexReuse/execution.js

@@ -0,0 +1,158 @@
+import path from "node:path";
+import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { runMyDevKitRetrievalFromIndex } from "../../../evaluation/runMyDevKitRetrieval.js";
+import { runRawFullFileBaseline } from "../../../evaluation/runRawFullFileBaseline.js";
+import { groupWarmIndexCases, taskOutputSegment } from "./selection.js";
+import { assertWarmIndexSessionMatchesTarget, prepareWarmIndexSession, } from "./warmIndexSession.js";
+/**
+ * Runs the warm-index lifecycle for already selected cases: one index setup attempt per valid
+ * benchmark-project group, then per task one raw baseline and one retrieval against that group's
+ * session. Never indexes per task and never retries a failed index.
+ */
+export async function executeWarmIndexReuse(options) {
+    const projects = [];
+    for (const group of groupWarmIndexCases(options.cases)) {
+        const indexDir = resolveWithinRoot(options.outputRoot, path.join("indexes", group.projectSegment));
+        const commandsDir = resolveWithinRoot(options.outputRoot, path.join("commands", group.projectSegment));
+        const project = {
+            benchmarkProject: group.benchmarkProject,
+            projectSegment: group.projectSegment,
+            targetRoot: group.targetRoot,
+            sourceRoots: group.sourceRoots,
+            indexDir,
+            tasks: [],
+            status: "skipped",
+            warnings: [],
+            errors: [...group.structuralErrors],
+        };
+        let setupFailure;
+        if (group.structuralErrors.length > 0) {
+            setupFailure = {
+                side: "warm",
+                code: "warm-index-group-inconsistent",
+                message: group.structuralErrors.join(" "),
+            };
+        }
+        else {
+            try {
+                // requireKit: false keeps the measured index command as evidence when setup fails.
+                const prepared = await prepareWarmIndexSession({
+                    target: { absoluteTargetRoot: group.targetRoot, sourceRoots: [...group.sourceRoots] },
+                    kitCommand: options.kitCommand,
+                    indexDir,
+                    commandsDir: path.join(commandsDir, "index"),
+                    requireKit: false,
+                });
+                project.build = prepared.build;
+                if (prepared.ok) {
+                    project.session = prepared.session;
+                }
+                else {
+                    project.warnings.push(...prepared.warnings);
+                    const message = prepared.warnings.join(" ") || "my-dev-kit index setup failed.";
+                    project.errors.push(message);
+                    setupFailure = { side: "warm", code: "warm-index-setup-failed", message };
+                }
+            }
+            catch (error) {
+                const message = errorMessage(error);
+                project.errors.push(message);
+                setupFailure = { side: "warm", code: "warm-index-setup-failed", message };
+            }
+        }
+        for (const evaluationCase of group.cases) {
+            project.tasks.push(await executeTask({
+                evaluationCase,
+                kitCommand: options.kitCommand,
+                session: project.session,
+                setupFailure,
+                commandsDir: path.join(commandsDir, taskOutputSegment(evaluationCase.id)),
+            }));
+        }
+        project.status = aggregateStatus(project.tasks.map((task) => task.status));
+        projects.push(project);
+    }
+    return projects;
+}
+async function executeTask(args) {
+    const { evaluationCase } = args;
+    const errors = [];
+    const warnings = [];
+    let rawBaseline;
+    let rawStatus;
+    try {
+        rawBaseline = await runRawFullFileBaseline(evaluationCase);
+        rawStatus = "completed";
+    }
+    catch (error) {
+        rawStatus = "failed";
+        errors.push({ side: "raw", code: "raw-baseline-failed", message: errorMessage(error) });
+    }
+    let warmRetrieval;
+    let warmStatus = "failed";
+    if (!args.session) {
+        errors.push(args.setupFailure ?? { side: "warm", code: "warm-index-setup-failed", message: "No warm index session." });
+    }
+    else {
+        try {
+            assertWarmIndexSessionMatchesTarget(args.session, evaluationCase);
+        }
+        catch (error) {
+            errors.push({ side: "warm", code: "warm-session-target-mismatch", message: errorMessage(error) });
+        }
+        if (!errors.some((error) => error.side === "warm")) {
+            try {
+                // requireKit: false retains subordinate command evidence; a skipped result caused by a
+                // failed command is classified as failed below rather than as an honest skip.
+                warmRetrieval = await runMyDevKitRetrievalFromIndex({
+                    evaluationCase,
+                    kitCommand: args.kitCommand,
+                    indexDir: args.session.indexDir,
+                    commandsDir: args.commandsDir,
+                    requireKit: false,
+                });
+                warnings.push(...warmRetrieval.warnings);
+                warmStatus = classifyWarmRetrieval(warmRetrieval);
+                if (warmStatus === "failed") {
+                    errors.push({
+                        side: "warm",
+                        code: "warm-retrieval-failed",
+                        message: warmRetrieval.warnings.join(" ") || "my-dev-kit retrieval command failed.",
+                    });
+                }
+            }
+            catch (error) {
+                errors.push({ side: "warm", code: "warm-retrieval-failed", message: errorMessage(error) });
+            }
+        }
+    }
+    return {
+        caseId: evaluationCase.id,
+        benchmarkProject: evaluationCase.benchmarkProject,
+        status: aggregateStatus([rawStatus, warmStatus]),
+        rawStatus,
+        warmStatus,
+        rawBaseline,
+        warmRetrieval,
+        warnings,
+        errors,
+    };
+}
+function classifyWarmRetrieval(result) {
+    if (!result.skipped)
+        return "completed";
+    return result.commands.some((command) => !command.ok) ? "failed" : "skipped";
+}
+/** Same aggregation rule as the generic per-case summary: uniform statuses win, mixed is partial. */
+export function aggregateStatus(statuses) {
+    if (statuses.length === 0)
+        return "skipped";
+    for (const status of ["completed", "skipped", "failed"]) {
+        if (statuses.every((value) => value === status))
+            return status;
+    }
+    return "partial";
+}
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}

package/dist/src/experiments/plugins/warmIndexReuse/executionArtifact.js

@@ -0,0 +1,77 @@
+export const WARM_INDEX_EXECUTION_ARTIFACT_FILE = "warm-index-execution.json";
+export const WARM_INDEX_EXECUTION_SCHEMA_VERSION = "my-dev-kit-lab-warm-index-execution-v1";
+export function summarizeCommand(command) {
+    return {
+        commandId: command.commandId,
+        ok: command.ok,
+        exitCode: command.exitCode,
+        durationMs: command.durationMs,
+        stdoutPath: command.stdoutPath,
+        stderrPath: command.stderrPath,
+        telemetryPath: command.telemetryPath,
+        error: command.error ?? null,
+    };
+}
+function summarizeTask(task) {
+    const raw = task.rawBaseline;
+    const warm = task.warmRetrieval;
+    return {
+        caseId: task.caseId,
+        status: task.status,
+        rawStatus: task.rawStatus,
+        warmStatus: task.warmStatus,
+        rawBaseline: raw
+            ? {
+                targetRoot: raw.targetRoot,
+                filesIncluded: [...raw.filesIncluded],
+                totalFiles: raw.totalFiles,
+                totalChars: raw.totalChars,
+                totalEstimatedTokens: raw.totalEstimatedTokens,
+                tokenCountMethod: raw.tokenCountMethod,
+                durationMs: raw.durationMs,
+            }
+            : null,
+        warmRetrieval: warm
+            ? {
+                skipped: warm.skipped,
+                warnings: [...warm.warnings],
+                totalChars: warm.totalChars,
+                totalEstimatedTokens: warm.totalEstimatedTokens,
+                tokenCountMethod: warm.tokenCountMethod,
+                filesRead: [...warm.filesRead],
+                selectedNodeId: warm.selectedNodeId ?? null,
+                selectedFile: warm.selectedFile ?? null,
+                selectedSymbol: warm.selectedSymbol ?? null,
+                durationMs: warm.durationMs,
+                commands: warm.commands.map(summarizeCommand),
+            }
+            : null,
+        warnings: [...task.warnings],
+        errors: task.errors.map((error) => ({ ...error })),
+    };
+}
+/** Bounded, context-free project summary used for both the persisted artifact and the run record. */
+export function summarizeProjectExecution(project) {
+    return {
+        benchmarkProject: project.benchmarkProject,
+        sessionKey: project.projectSegment,
+        status: project.status,
+        targetRoot: project.targetRoot,
+        sourceRoots: [...project.sourceRoots],
+        indexDir: project.indexDir,
+        sessionPrepared: project.session !== undefined,
+        buildDurationMs: project.build ? project.build.durationMs : null,
+        indexCommand: project.build ? summarizeCommand(project.build.command) : null,
+        tasks: project.tasks.map(summarizeTask),
+        warnings: [...project.warnings],
+        errors: [...project.errors],
+    };
+}
+export function buildWarmIndexExecutionArtifact(args) {
+    return {
+        schemaVersion: WARM_INDEX_EXECUTION_SCHEMA_VERSION,
+        runId: args.runId,
+        pluginId: args.pluginId,
+        projects: args.projects.map(summarizeProjectExecution),
+    };
+}

package/dist/src/experiments/plugins/warmIndexReuse/fakeAgentEvaluation.js

@@ -0,0 +1,118 @@
+import path from "node:path";
+import { runAgentPrompt } from "../../../agents/index.js";
+import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { classifyAgentRunOutcome } from "../../../evaluation/classifyAgentRunOutcome.js";
+import { parseAgentAnswer } from "../../../evaluation/parseAgentAnswer.js";
+import { scoreCorrectness } from "../../../evaluation/scoreCorrectness.js";
+import { generatePromptVariants } from "../../../prompts/index.js";
+import { taskOutputSegment } from "./selection.js";
+// Classified statuses for which the scorer evaluated an actual agent answer.
+const SCOREABLE_STATUSES = new Set(["completed", "invalid-output"]);
+const SIDE_STRATEGIES = {
+    "raw-full-file": "raw-full-file",
+    "warm-index-reuse": "my-dev-kit-guided",
+};
+export async function evaluateWarmIndexFakeAgents(options) {
+    const casesById = new Map(options.cases.map((evaluationCase) => [evaluationCase.id, evaluationCase]));
+    const result = [];
+    for (const project of options.projects) {
+        const tasks = [];
+        for (const task of project.tasks) {
+            const evaluationCase = casesById.get(task.caseId);
+            const run = (variantId) => evaluateSide({
+                evaluationCase,
+                caseId: task.caseId,
+                variantId,
+                projectProfiles: options.projectProfiles,
+                outDir: resolveWithinRoot(options.outputRoot, path.join("agents", project.projectSegment, taskOutputSegment(task.caseId), variantId)),
+                cwd: options.cwd,
+                env: options.env,
+            });
+            tasks.push({
+                caseId: task.caseId,
+                raw: task.rawBaseline ? await run("raw-full-file") : null,
+                warm: task.warmRetrieval ? await run("warm-index-reuse") : null,
+            });
+        }
+        result.push({ benchmarkProject: project.benchmarkProject, tasks });
+    }
+    return result;
+}
+async function evaluateSide(args) {
+    const promptStrategy = SIDE_STRATEGIES[args.variantId];
+    const base = { variantId: args.variantId, agentId: "fake-agent", promptStrategy };
+    try {
+        if (!args.evaluationCase) {
+            throw new Error(`Evaluation case ${args.caseId} is not available for fake-agent evaluation.`);
+        }
+        const [promptVariant] = generatePromptVariants({
+            cases: [args.evaluationCase],
+            projectProfiles: [...args.projectProfiles],
+            strategies: [promptStrategy],
+            complexityLevels: ["short"],
+        });
+        if (!promptVariant) {
+            throw new Error(`Failed to generate a fake-agent prompt for case ${args.caseId}.`);
+        }
+        const agentRunResult = await runAgentPrompt({
+            runId: `${args.caseId}.fake-agent.${args.variantId}`,
+            agentId: "fake-agent",
+            promptVariant,
+            promptText: promptVariant.promptText,
+            cwd: args.cwd,
+            outDir: args.outDir,
+            env: args.env,
+        });
+        const parsedAnswer = parseAgentAnswer({
+            text: agentRunResult.finalAnswerText,
+            answerKey: promptVariant.expectedAnswerKey,
+            tokenUsage: agentRunResult.tokenUsage,
+        });
+        const classification = classifyAgentRunOutcome({ agentRunResult, parsedAnswer });
+        const correctness = scoreCorrectness({
+            caseId: args.caseId,
+            answerKey: promptVariant.expectedAnswerKey,
+            parsedAnswer,
+            status: classification.status,
+        });
+        const scoreable = SCOREABLE_STATUSES.has(classification.status) && Number.isFinite(correctness.correctnessScore);
+        const totalTokens = agentRunResult.tokenUsage.totalTokens;
+        return {
+            ...base,
+            status: classification.status,
+            correctness: {
+                available: scoreable,
+                score: scoreable ? correctness.correctnessScore : null,
+                passed: scoreable ? correctness.passed : null,
+                failureReasons: [...correctness.failureReasons],
+            },
+            tokenUsage: {
+                totalTokens: typeof totalTokens === "number" && Number.isFinite(totalTokens) ? totalTokens : null,
+                source: agentRunResult.tokenUsageSource,
+                reliability: agentRunResult.tokenUsageReliability,
+            },
+            durationMs: agentRunResult.durationMs,
+            warnings: [...classification.warnings],
+            errors: [...classification.errors],
+            artifactPaths: {
+                promptPath: path.join(args.outDir, "prompt.txt"),
+                agentRunResultPath: path.join(args.outDir, "agent-run-result.json"),
+                stdoutPath: agentRunResult.stdoutPath,
+                stderrPath: agentRunResult.stderrPath,
+                telemetryPath: agentRunResult.telemetryPath,
+            },
+        };
+    }
+    catch (error) {
+        return {
+            ...base,
+            status: "failed",
+            correctness: { available: false, score: null, passed: null, failureReasons: [] },
+            tokenUsage: { totalTokens: null, source: "unavailable", reliability: "unavailable" },
+            durationMs: null,
+            warnings: [],
+            errors: [error instanceof Error ? error.message : String(error)],
+            artifactPaths: {},
+        };
+    }
+}

package/dist/src/experiments/plugins/warmIndexReuse/index.js

@@ -0,0 +1,8 @@
+export * from "./config.js";
+export * from "./execution.js";
+export * from "./executionArtifact.js";
+export * from "./fakeAgentEvaluation.js";
+export * from "./metrics.js";
+export * from "./plugin.js";
+export * from "./selection.js";
+export * from "./warmIndexSession.js";