@dailephd/my-dev-kit-lab 0.4.9 → 0.5.1

package/dist/src/experiments/plugins/warmIndexReuse/metrics.js

@@ -0,0 +1,257 @@
+// ---------------------------------------------------------------------------
+// Warm-index metric owner. Every warm-index formula (amortization, cumulative
+// prefix sums, availability) is calculated here exactly once, as a pure
+// transform over the bounded Prompt 2 execution summaries. Reports render
+// these results; they never recalculate them.
+// ---------------------------------------------------------------------------
+export const WARM_INDEX_METRICS_SCHEMA_VERSION = "my-dev-kit-lab-warm-index-metrics-v1";
+const NO_AGENT_EVIDENCE_REASON = "No fake-agent evaluation evidence was supplied for this task.";
+const AGENT_NOT_RUN_REASON = "The fake agent was not run because this side produced no context evidence.";
+export function availableMetric(value, unit, source, tokenCountMethod = null) {
+    if (typeof value !== "number" || !Number.isFinite(value)) {
+        return unavailableMetric(unit, source, `Invalid input: expected a finite number but received ${String(value)}.`, tokenCountMethod);
+    }
+    return { availability: "available", value, unit, source, reason: null, tokenCountMethod };
+}
+export function unavailableMetric(unit, source, reason, tokenCountMethod = null) {
+    return { availability: "unavailable", value: null, unit, source, reason: requireReason(reason), tokenCountMethod };
+}
+export function notApplicableMetric(unit, source, reason) {
+    return { availability: "not-applicable", value: null, unit, source, reason: requireReason(reason), tokenCountMethod: null };
+}
+function requireReason(reason) {
+    if (!reason.trim()) {
+        throw new Error("Unavailable and not-applicable warm-index metrics require a reason.");
+    }
+    return reason;
+}
+/** Strict-prefix running sum: once one ordinal lacks evidence, every later ordinal stays unavailable. */
+class PrefixSum {
+    unit;
+    source;
+    total;
+    brokenReason;
+    method = null;
+    constructor(unit, source, start) {
+        this.unit = unit;
+        this.source = source;
+        this.total = "value" in start ? start.value : 0;
+        this.brokenReason = "unavailableReason" in start ? start.unavailableReason : null;
+    }
+    add(direct, label) {
+        if (this.brokenReason === null) {
+            if (direct.availability !== "available" || direct.value === null) {
+                this.brokenReason = `${label} evidence is missing or invalid; the cumulative prefix is incomplete from this task onward.`;
+            }
+            else if (this.method !== null && direct.tokenCountMethod !== this.method) {
+                this.brokenReason = `${label} uses token-count method ${String(direct.tokenCountMethod)} but earlier tasks used ${this.method}; mixed methods are not summed.`;
+            }
+            else {
+                this.method = direct.tokenCountMethod;
+                this.total += direct.value;
+            }
+        }
+        return this.brokenReason === null
+            ? availableMetric(this.total, this.unit, this.source, this.method)
+            : unavailableMetric(this.unit, this.source, this.brokenReason, this.method);
+    }
+}
+function indexBuildMetric(project) {
+    if (project.buildDurationMs === null) {
+        return unavailableMetric("ms", "measured", project.errors.length > 0
+            ? "No index setup was attempted because the project group was structurally inconsistent."
+            : "No index build evidence was recorded for this project.");
+    }
+    // A failed build is still a measured command duration; it is only excluded from warm costs below.
+    return availableMetric(project.buildDurationMs, "ms", "measured");
+}
+/**
+ * Fake-agent correctness and total tokens come only from bounded fake-agent evidence. Missing
+ * token telemetry stays unavailable; estimated context tokens are never substituted.
+ */
+function agentSideMetrics(evidence) {
+    if (evidence === undefined || evidence === null) {
+        const reason = evidence === undefined ? NO_AGENT_EVIDENCE_REASON : AGENT_NOT_RUN_REASON;
+        return {
+            agentCorrectness: unavailableMetric("score", "agent", reason),
+            agentTotalTokens: unavailableMetric("tokens", "agent", reason),
+        };
+    }
+    const correctnessScore = evidence.correctness.score;
+    const agentCorrectness = evidence.correctness.available && correctnessScore !== null
+        ? availableMetric(correctnessScore, "score", "agent")
+        : unavailableMetric("score", "agent", `Fake-agent output was not scoreable (status ${evidence.status})${evidence.errors[0] ? `: ${evidence.errors[0]}` : "."}`);
+    const agentTotalTokens = evidence.tokenUsage.totalTokens !== null
+        ? availableMetric(evidence.tokenUsage.totalTokens, "tokens", "agent")
+        : unavailableMetric("tokens", "agent", `The fake agent did not report total tokens (token usage source ${evidence.tokenUsage.source}).`);
+    return { agentCorrectness, agentTotalTokens };
+}
+function assertAgentEvidenceAligned(project, evidence) {
+    if (!evidence)
+        return;
+    const aligned = evidence.benchmarkProject === project.benchmarkProject &&
+        evidence.tasks.length === project.tasks.length &&
+        evidence.tasks.every((task, index) => task.caseId === project.tasks[index].caseId);
+    if (!aligned) {
+        throw new Error(`Fake-agent evidence is not aligned with the execution summary for ${project.benchmarkProject}.`);
+    }
+}
+export function calculateWarmIndexProjectMetrics(project, agentEvidence) {
+    assertAgentEvidenceAligned(project, agentEvidence);
+    const indexBuildDurationMs = indexBuildMetric(project);
+    const buildUsable = project.sessionPrepared && indexBuildDurationMs.availability === "available";
+    const noSessionReason = project.sessionPrepared
+        ? "Index build duration is unavailable, so warm index cost cannot be calculated."
+        : "No valid warm index session was prepared for this project, so the index build cost is not charged to warm tasks.";
+    const rawDuration = new PrefixSum("ms", "measured", { value: 0 });
+    const rawTokens = new PrefixSum("estimated-tokens", "estimated-chars-div-4", { value: 0 });
+    // The one-time build is charged exactly once, as the starting value of the warm prefix.
+    const warmDuration = new PrefixSum("ms", "derived", buildUsable ? { value: indexBuildDurationMs.value } : { unavailableReason: noSessionReason });
+    const warmTokens = new PrefixSum("estimated-tokens", "estimated-chars-div-4", { value: 0 });
+    // Fake-agent token totals accumulate separately from estimated context tokens.
+    const rawAgentTokens = new PrefixSum("tokens", "agent", { value: 0 });
+    const warmAgentTokens = new PrefixSum("tokens", "agent", { value: 0 });
+    const tasks = project.tasks.map((task, index) => {
+        const taskOrdinal = index + 1;
+        const label = (side) => `Task ${taskOrdinal} (${task.caseId}) ${side}`;
+        const raw = rawDirect(task);
+        const warm = warmDirect(task);
+        const taskAgentEvidence = agentEvidence?.tasks[index];
+        const rawAgent = agentSideMetrics(agentEvidence ? taskAgentEvidence?.raw ?? null : undefined);
+        const warmAgent = agentSideMetrics(agentEvidence ? taskAgentEvidence?.warm ?? null : undefined);
+        return {
+            caseId: task.caseId,
+            taskOrdinal,
+            raw: {
+                ...raw,
+                cumulativeDurationMs: rawDuration.add(raw.operationDurationMs, label("raw duration")),
+                cumulativeEstimatedContextTokens: rawTokens.add(raw.contextEstimatedTokens, label("raw estimated context tokens")),
+                ...rawAgent,
+                cumulativeAgentTotalTokens: rawAgentTokens.add(rawAgent.agentTotalTokens, label("raw fake-agent total tokens")),
+            },
+            warm: {
+                ...warm,
+                // Amortized build cost after N tasks = build duration / N.
+                amortizedIndexBuildDurationMs: buildUsable
+                    ? availableMetric(indexBuildDurationMs.value / taskOrdinal, "ms", "derived")
+                    : unavailableMetric("ms", "derived", noSessionReason),
+                cumulativeComponentDurationMs: warmDuration.add(warm.retrievalDurationMs, label("warm retrieval duration")),
+                cumulativeEstimatedContextTokens: warmTokens.add(warm.contextEstimatedTokens, label("warm estimated context tokens")),
+                ...warmAgent,
+                cumulativeAgentTotalTokens: warmAgentTokens.add(warmAgent.agentTotalTokens, label("warm fake-agent total tokens")),
+            },
+        };
+    });
+    return {
+        benchmarkProject: project.benchmarkProject,
+        sessionKey: project.sessionKey,
+        sessionPrepared: project.sessionPrepared,
+        indexBuildDurationMs,
+        tasks,
+    };
+}
+function rawDirect(task) {
+    const raw = task.rawBaseline;
+    if (!raw) {
+        const reason = `No raw baseline evidence was produced (raw status ${task.rawStatus}).`;
+        return {
+            contextCharacters: unavailableMetric("characters", "measured", reason),
+            contextEstimatedTokens: unavailableMetric("estimated-tokens", "estimated-chars-div-4", reason),
+            operationDurationMs: unavailableMetric("ms", "measured", reason),
+        };
+    }
+    return {
+        contextCharacters: availableMetric(raw.totalChars, "characters", "measured"),
+        contextEstimatedTokens: availableMetric(raw.totalEstimatedTokens, "estimated-tokens", "estimated-chars-div-4", raw.tokenCountMethod),
+        operationDurationMs: availableMetric(raw.durationMs, "ms", "measured"),
+    };
+}
+function warmDirect(task) {
+    const warm = task.warmRetrieval;
+    if (!warm) {
+        const reason = `No warm retrieval was run for this task (warm status ${task.warmStatus}).`;
+        return {
+            contextCharacters: unavailableMetric("characters", "measured", reason),
+            contextEstimatedTokens: unavailableMetric("estimated-tokens", "estimated-chars-div-4", reason),
+            retrievalDurationMs: unavailableMetric("ms", "measured", reason),
+        };
+    }
+    // A skipped retrieval still carries measured values; its skipped status stays on the outcome.
+    return {
+        contextCharacters: availableMetric(warm.totalChars, "characters", "measured"),
+        contextEstimatedTokens: availableMetric(warm.totalEstimatedTokens, "estimated-tokens", "estimated-chars-div-4", warm.tokenCountMethod),
+        retrievalDurationMs: availableMetric(warm.durationMs, "ms", "measured"),
+    };
+}
+/** Pure, order-preserving calculation; ordinals and cumulative state restart for every project. */
+export function calculateWarmIndexMetrics(projects, agentEvidence) {
+    if (agentEvidence && agentEvidence.length !== projects.length) {
+        throw new Error("Fake-agent evidence is not aligned with the execution summaries.");
+    }
+    return {
+        schemaVersion: WARM_INDEX_METRICS_SCHEMA_VERSION,
+        projects: projects.map((project, index) => calculateWarmIndexProjectMetrics(project, agentEvidence?.[index])),
+    };
+}
+/** Generic outcome metrics: only available numeric values are emitted, never placeholder zeros. */
+export function toRawOutcomeMetrics(task, variantId) {
+    return toExperimentMetrics(task.caseId, variantId, [
+        { id: "context-character-count", name: "Context characters", description: "Characters in the raw full-file context.", metric: task.raw.contextCharacters },
+        { id: "context-estimated-token-count", name: "Estimated context tokens", description: "Character-based estimate of raw context tokens; not provider token usage.", metric: task.raw.contextEstimatedTokens },
+        { id: "operation-duration-ms", name: "Operation duration", description: "Measured raw full-file context construction duration.", metric: task.raw.operationDurationMs },
+        { id: "cumulative-component-duration-ms", name: "Cumulative component duration", description: "Sum of raw context construction durations for tasks 1..N in this project.", metric: task.raw.cumulativeDurationMs },
+        { id: "cumulative-context-estimated-token-count", name: "Cumulative estimated context tokens", description: "Sum of raw estimated context tokens for tasks 1..N in this project; not provider token usage.", metric: task.raw.cumulativeEstimatedContextTokens },
+        ...agentSpecs(task.raw),
+    ]);
+}
+export function toWarmOutcomeMetrics(task, variantId) {
+    return toExperimentMetrics(task.caseId, variantId, [
+        { id: "context-character-count", name: "Context characters", description: "Characters in the retrieved my-dev-kit context.", metric: task.warm.contextCharacters },
+        { id: "context-estimated-token-count", name: "Estimated context tokens", description: "Character-based estimate of retrieved context tokens; not provider token usage.", metric: task.warm.contextEstimatedTokens },
+        { id: "operation-duration-ms", name: "Operation duration", description: "Measured retrieval-only duration (search, lookup, slice, source); excludes index build.", metric: task.warm.retrievalDurationMs },
+        { id: "cumulative-component-duration-ms", name: "Cumulative component duration", description: "One index build plus retrieval durations for tasks 1..N in this project; a component sum, not wall-clock latency.", metric: task.warm.cumulativeComponentDurationMs },
+        { id: "cumulative-context-estimated-token-count", name: "Cumulative estimated context tokens", description: "Sum of retrieved estimated context tokens for tasks 1..N in this project; not provider token usage.", metric: task.warm.cumulativeEstimatedContextTokens },
+        { id: "amortized-index-build-duration-ms", name: "Amortized index build duration", description: "Index build duration divided by this task's ordinal within its project.", metric: task.warm.amortizedIndexBuildDurationMs },
+        ...agentSpecs(task.warm),
+    ]);
+}
+function agentSpecs(side) {
+    return [
+        { id: "agent-correctness-score", name: "Fake-agent correctness score", description: "Deterministic fake-agent correctness from the existing benchmark answer-key scorer; not real-model correctness.", metric: side.agentCorrectness },
+        { id: "agent-total-tokens", name: "Fake-agent total tokens", description: "Simulated fake-agent harness token total; not provider billing telemetry.", metric: side.agentTotalTokens },
+        { id: "cumulative-agent-total-tokens", name: "Cumulative fake-agent total tokens", description: "Sum of simulated fake-agent token totals for tasks 1..N in this project; not provider billing telemetry.", metric: side.cumulativeAgentTotalTokens },
+    ];
+}
+function toExperimentMetrics(caseId, variantId, specs) {
+    return specs
+        .filter((spec) => spec.metric.availability === "available" && spec.metric.value !== null)
+        .map((spec) => ({
+        id: spec.id,
+        name: spec.name,
+        value: spec.metric.value,
+        unit: spec.metric.unit,
+        description: spec.description,
+        variantId,
+        caseId,
+    }));
+}
+/** Run-level metrics are limited to direct counts. */
+export function toRunLevelMetrics(metrics) {
+    return [
+        { id: "warm-index-project-count", name: "Warm-index project count", value: metrics.projects.length, unit: "count", description: "Benchmark-project groups in this run." },
+        {
+            id: "warm-index-task-count",
+            name: "Warm-index task count",
+            value: metrics.projects.reduce((sum, project) => sum + project.tasks.length, 0),
+            unit: "count",
+            description: "Tasks evaluated across all benchmark-project groups.",
+        },
+        {
+            id: "warm-index-session-prepared-project-count",
+            name: "Prepared warm-index session count",
+            value: metrics.projects.filter((project) => project.sessionPrepared).length,
+            unit: "count",
+            description: "Benchmark-project groups for which exactly one warm index session was prepared.",
+        },
+    ];
+}

package/dist/src/experiments/plugins/warmIndexReuse/plugin.js

@@ -0,0 +1,212 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { summarizeExperimentRun } from "../../results.js";
+import { defaultWarmIndexReuseConfig, validateWarmIndexReuseConfig, warmIndexReuseConfigDefinition, } from "./config.js";
+import { aggregateStatus, executeWarmIndexReuse, } from "./execution.js";
+import { buildWarmIndexExecutionArtifact, WARM_INDEX_EXECUTION_ARTIFACT_FILE, } from "./executionArtifact.js";
+import { calculateWarmIndexMetrics, toRawOutcomeMetrics, toRunLevelMetrics, toWarmOutcomeMetrics, } from "./metrics.js";
+import { selectWarmIndexCases } from "./selection.js";
+import { evaluateWarmIndexFakeAgents, } from "./fakeAgentEvaluation.js";
+export const RAW_FULL_FILE_VARIANT_ID = "raw-full-file";
+export const WARM_INDEX_REUSE_VARIANT_ID = "warm-index-reuse";
+export const warmIndexReuseMetadata = {
+    id: "warm-index-reuse",
+    name: "Warm Index Reuse",
+    description: "Measure repeated task execution where one my-dev-kit index is built once per benchmark project and reused across multiple tasks, with a matched raw-full-file baseline per task.",
+    schemaVersion: "1.0.0",
+    status: "experimental",
+    supportedTargets: ["self", "external-local"],
+    supportedOutputs: ["json", "html", "text", "plot", "artifact"],
+};
+const WARM_INDEX_VARIANTS = [
+    {
+        id: RAW_FULL_FILE_VARIANT_ID,
+        name: "Raw full file",
+        description: "Every file matched by the case's raw include globs, read once per task.",
+    },
+    {
+        id: WARM_INDEX_REUSE_VARIANT_ID,
+        name: "Warm index reuse",
+        description: "my-dev-kit retrieval against one index prepared once per benchmark project.",
+    },
+];
+export const warmIndexReusePlugin = {
+    metadata: warmIndexReuseMetadata,
+    defaultConfig: defaultWarmIndexReuseConfig,
+    configDefinition: warmIndexReuseConfigDefinition,
+    supportedVariants: WARM_INDEX_VARIANTS.map((variant) => variant.id),
+    validateConfig: validateWarmIndexReuseConfig,
+    async run(context) {
+        const startedAt = context.startedAt.toISOString();
+        const cases = selectWarmIndexCases(readCasesInput(context.inputs), context.config);
+        const projectProfiles = readProjectProfilesInput(context.inputs);
+        const outDir = context.outputRoot ?? path.resolve(context.toolRoot, context.config.outDir);
+        await mkdir(outDir, { recursive: true });
+        const projects = await executeWarmIndexReuse({
+            cases,
+            kitCommand: context.config.kitCommand,
+            outputRoot: outDir,
+        });
+        // Fake-agent evaluation runs after execution evidence exists and never repeats it.
+        const agentEvidence = await evaluateWarmIndexFakeAgents({
+            projects,
+            cases,
+            projectProfiles,
+            outputRoot: outDir,
+            cwd: context.target.targetRoot,
+            env: readEnvInput(context.inputs),
+        });
+        const artifact = buildWarmIndexExecutionArtifact({
+            runId: context.runId,
+            pluginId: warmIndexReuseMetadata.id,
+            projects,
+        });
+        const artifactPath = resolveWithinRoot(outDir, WARM_INDEX_EXECUTION_ARTIFACT_FILE);
+        await writeFile(artifactPath, `${JSON.stringify(artifact, null, 2)}\n`, "utf8");
+        const run = mapWarmIndexExecutionToRun({
+            runId: context.runId,
+            startedAt,
+            completedAt: new Date().toISOString(),
+            target: context.target,
+            cases,
+            projects,
+            projectSummaries: artifact.projects,
+            agentEvidence,
+            artifactPath,
+        });
+        return run;
+    },
+    summarize(result) {
+        return result.summary ?? summarizeExperimentRun(result);
+    },
+};
+export function mapWarmIndexExecutionToRun(args) {
+    const titles = new Map(args.cases.map((evaluationCase) => [evaluationCase.id, evaluationCase.title]));
+    const warmIndexMetrics = calculateWarmIndexMetrics(args.projectSummaries, args.agentEvidence);
+    // projectSummaries (and therefore metrics) preserve project and task order of args.projects.
+    const experimentCases = args.projects.flatMap((project, projectIndex) => project.tasks.map((task, taskIndex) => {
+        const taskMetrics = warmIndexMetrics.projects[projectIndex].tasks[taskIndex];
+        const taskAgents = args.agentEvidence[projectIndex].tasks[taskIndex];
+        return {
+            id: task.caseId,
+            name: titles.get(task.caseId) ?? task.caseId,
+            outcomes: [
+                buildRawOutcome(project, task, taskMetrics, taskAgents),
+                buildWarmOutcome(project, task, taskMetrics, taskAgents),
+            ],
+            metadata: { benchmarkProject: task.benchmarkProject, sessionKey: project.projectSegment },
+        };
+    }));
+    const warnings = args.projects.flatMap((project) => project.warnings.map((message) => ({
+        code: "warm-index-setup-warning",
+        message,
+        details: { benchmarkProject: project.benchmarkProject },
+    })));
+    const run = {
+        runId: args.runId,
+        pluginId: warmIndexReuseMetadata.id,
+        startedAt: args.startedAt,
+        completedAt: args.completedAt,
+        status: aggregateStatus(experimentCases.flatMap((experimentCase) => experimentCase.outcomes.map((outcome) => outcome.status))),
+        target: args.target,
+        variants: WARM_INDEX_VARIANTS.map((variant) => ({ ...variant })),
+        cases: experimentCases,
+        metrics: toRunLevelMetrics(warmIndexMetrics),
+        artifacts: [
+            {
+                id: "warm-index-execution",
+                label: "Warm index execution evidence",
+                path: args.artifactPath,
+                kind: "artifact",
+                mimeType: "application/json",
+                description: "Per-project index setup and per-task raw/warm execution summaries without context text.",
+            },
+        ],
+        warnings,
+        failures: [],
+        metadata: { executionArtifactPath: args.artifactPath },
+        projectExecutions: args.projectSummaries,
+        agentEvidence: args.agentEvidence,
+        warmIndexMetrics,
+    };
+    run.summary = summarizeExperimentRun(run);
+    return run;
+}
+function outcomeMetadata(project, task, agent) {
+    return {
+        benchmarkProject: task.benchmarkProject,
+        sessionKey: project.projectSegment,
+        warmSessionAvailable: project.session !== undefined,
+        taskStatus: task.status,
+        // Agent evaluation status is reported separately; it does not change the execution status.
+        agentStatus: agent ? agent.status : "not-run",
+    };
+}
+function agentWarnings(agent, variantId, caseId) {
+    if (!agent)
+        return [];
+    return agent.errors.map((message) => ({
+        code: "fake-agent-evaluation-error",
+        message,
+        variantId,
+        caseId,
+    }));
+}
+function buildRawOutcome(project, task, taskMetrics, taskAgents) {
+    return {
+        id: `${task.caseId}:${RAW_FULL_FILE_VARIANT_ID}`,
+        caseId: task.caseId,
+        variantId: RAW_FULL_FILE_VARIANT_ID,
+        status: task.rawStatus,
+        metrics: toRawOutcomeMetrics(taskMetrics, RAW_FULL_FILE_VARIANT_ID),
+        artifacts: [],
+        warnings: agentWarnings(taskAgents.raw, RAW_FULL_FILE_VARIANT_ID, task.caseId),
+        failures: toFailures(task, "raw", RAW_FULL_FILE_VARIANT_ID),
+        metadata: outcomeMetadata(project, task, taskAgents.raw),
+    };
+}
+function buildWarmOutcome(project, task, taskMetrics, taskAgents) {
+    return {
+        id: `${task.caseId}:${WARM_INDEX_REUSE_VARIANT_ID}`,
+        caseId: task.caseId,
+        variantId: WARM_INDEX_REUSE_VARIANT_ID,
+        status: task.warmStatus,
+        metrics: toWarmOutcomeMetrics(taskMetrics, WARM_INDEX_REUSE_VARIANT_ID),
+        artifacts: [],
+        warnings: [
+            ...task.warnings.map((message) => ({
+                code: "warm-retrieval-warning",
+                message,
+                variantId: WARM_INDEX_REUSE_VARIANT_ID,
+                caseId: task.caseId,
+            })),
+            ...agentWarnings(taskAgents.warm, WARM_INDEX_REUSE_VARIANT_ID, task.caseId),
+        ],
+        failures: toFailures(task, "warm", WARM_INDEX_REUSE_VARIANT_ID),
+        metadata: outcomeMetadata(project, task, taskAgents.warm),
+    };
+}
+function toFailures(task, side, variantId) {
+    return task.errors
+        .filter((error) => error.side === side)
+        .map((error) => ({ code: error.code, message: error.message, variantId, caseId: task.caseId, recoverable: true }));
+}
+function readProjectProfilesInput(inputs) {
+    const value = inputs?.projectProfiles;
+    if (!Array.isArray(value)) {
+        throw new Error("Warm index reuse requires projectProfiles input for fake-agent evaluation.");
+    }
+    return value;
+}
+function readEnvInput(inputs) {
+    const env = inputs?.env;
+    return env && typeof env === "object" && !Array.isArray(env) ? env : undefined;
+}
+function readCasesInput(inputs) {
+    const value = inputs?.cases;
+    if (!Array.isArray(value)) {
+        throw new Error("Warm index reuse requires cases input.");
+    }
+    return value;
+}

package/dist/src/experiments/plugins/warmIndexReuse/selection.js

@@ -0,0 +1,85 @@
+import path from "node:path";
+import { sanitizePathSegment } from "../../outputPaths.js";
+/**
+ * Keeps source case order, applies case-ID then benchmark-project filters, and fails clearly for
+ * unknown requested IDs or an empty selection.
+ */
+export function selectWarmIndexCases(cases, filters) {
+    const knownCaseIds = new Set(cases.map((evaluationCase) => evaluationCase.id));
+    const missingCases = (filters.caseIds ?? []).filter((caseId) => !knownCaseIds.has(caseId));
+    if (missingCases.length > 0) {
+        throw new Error(`Evaluation case not found: ${missingCases.join(", ")}`);
+    }
+    const knownProjects = new Set(cases.map((evaluationCase) => evaluationCase.benchmarkProject));
+    const missingProjects = (filters.benchmarkProjects ?? []).filter((project) => !knownProjects.has(project));
+    if (missingProjects.length > 0) {
+        throw new Error(`Benchmark project not found: ${missingProjects.join(", ")}`);
+    }
+    const selected = cases
+        .filter((evaluationCase) => !filters.caseIds?.length || filters.caseIds.includes(evaluationCase.id))
+        .filter((evaluationCase) => !filters.benchmarkProjects?.length || filters.benchmarkProjects.includes(evaluationCase.benchmarkProject));
+    if (selected.length === 0) {
+        throw new Error(`No evaluation cases matched the requested filters (cases: ${(filters.caseIds ?? []).join(", ") || "any"}; ` +
+            `benchmark projects: ${(filters.benchmarkProjects ?? []).join(", ") || "any"}).`);
+    }
+    return selected;
+}
+/**
+ * Groups cases by benchmarkProject in first-seen order, keeping task order within a group. Cases
+ * that share a project but disagree on target root or ordered source roots make the group
+ * structurally invalid rather than silently picking one configuration.
+ */
+export function groupWarmIndexCases(cases) {
+    const groups = new Map();
+    for (const evaluationCase of cases) {
+        let group = groups.get(evaluationCase.benchmarkProject);
+        if (!group) {
+            group = {
+                benchmarkProject: evaluationCase.benchmarkProject,
+                projectSegment: safeOutputSegment(evaluationCase.benchmarkProject, "benchmark project"),
+                targetRoot: evaluationCase.absoluteTargetRoot,
+                sourceRoots: Object.freeze([...evaluationCase.sourceRoots]),
+                cases: [],
+                structuralErrors: [],
+            };
+            groups.set(evaluationCase.benchmarkProject, group);
+        }
+        else {
+            if (path.resolve(group.targetRoot) !== path.resolve(evaluationCase.absoluteTargetRoot)) {
+                group.structuralErrors.push(`Case ${evaluationCase.id} target root ${evaluationCase.absoluteTargetRoot} differs from ${group.targetRoot} for benchmark project ${group.benchmarkProject}.`);
+            }
+            if (evaluationCase.sourceRoots.join("\u0000") !== group.sourceRoots.join("\u0000")) {
+                group.structuralErrors.push(`Case ${evaluationCase.id} source roots [${evaluationCase.sourceRoots.join(", ")}] differ from [${group.sourceRoots.join(", ")}] for benchmark project ${group.benchmarkProject}.`);
+            }
+        }
+        group.cases.push(evaluationCase);
+    }
+    const result = [...groups.values()];
+    assertUniqueSegments(result.map((group) => [group.benchmarkProject, group.projectSegment]), "benchmark project");
+    for (const group of result) {
+        assertUniqueSegments(group.cases.map((evaluationCase) => [evaluationCase.id, taskOutputSegment(evaluationCase.id)]), "case");
+    }
+    return result;
+}
+export function taskOutputSegment(caseId) {
+    return safeOutputSegment(caseId, "case");
+}
+// Reuses the shared experiment path-segment sanitizer and additionally rejects dot-only segments,
+// which the sanitizer preserves and which would resolve outside the intended directory.
+function safeOutputSegment(value, label) {
+    const segment = sanitizePathSegment(value);
+    if (/^\.+$/.test(segment)) {
+        throw new Error(`Unsafe ${label} ID for an output path segment: ${JSON.stringify(value)}`);
+    }
+    return segment;
+}
+function assertUniqueSegments(entries, label) {
+    const owners = new Map();
+    for (const [id, segment] of entries) {
+        const owner = owners.get(segment);
+        if (owner !== undefined && owner !== id) {
+            throw new Error(`The ${label} IDs ${owner} and ${id} map to the same output path segment "${segment}".`);
+        }
+        owners.set(segment, id);
+    }
+}

package/dist/src/experiments/plugins/warmIndexReuse/warmIndexSession.js

@@ -0,0 +1,43 @@
+import { existsSync } from "node:fs";
+import path from "node:path";
+import { buildMyDevKitIndex } from "../../../evaluation/runMyDevKitRetrieval.js";
+/**
+ * Builds the index once through the shared `buildMyDevKitIndex` owner. A build that reports
+ * success but leaves no index directory is not treated as a prepared session.
+ */
+export async function prepareWarmIndexSession(options) {
+    const build = await buildMyDevKitIndex(options);
+    if (!build.ok) {
+        return { ok: false, warnings: [...build.warnings], build };
+    }
+    if (!existsSync(build.indexDir)) {
+        const message = `my-dev-kit index reported success but index directory is missing: ${build.indexDir}`;
+        if (options.requireKit) {
+            throw new Error(message);
+        }
+        return { ok: false, warnings: [...build.warnings, message], build };
+    }
+    const session = Object.freeze({
+        indexDir: build.indexDir,
+        buildDurationMs: build.durationMs,
+        targetRoot: options.target.absoluteTargetRoot,
+        sourceRoots: Object.freeze([...options.target.sourceRoots]),
+        buildCommand: build.command
+    });
+    return { ok: true, session, build };
+}
+function normalizeRoot(root) {
+    const resolved = path.resolve(root);
+    return process.platform === "win32" ? resolved.toLowerCase() : resolved;
+}
+/** Rejects reuse of a session against a different target root or source-root configuration. */
+export function assertWarmIndexSessionMatchesTarget(session, target) {
+    if (normalizeRoot(session.targetRoot) !== normalizeRoot(target.absoluteTargetRoot)) {
+        throw new Error(`Warm index session target root ${session.targetRoot} does not match ${target.absoluteTargetRoot}.`);
+    }
+    const sameSourceRoots = session.sourceRoots.length === target.sourceRoots.length &&
+        session.sourceRoots.every((sourceRoot, index) => sourceRoot === target.sourceRoots[index]);
+    if (!sameSourceRoots) {
+        throw new Error(`Warm index session source roots [${session.sourceRoots.join(", ")}] do not match [${target.sourceRoots.join(", ")}].`);
+    }
+}