@dailephd/my-dev-kit-lab 0.6.3 → 0.7.1

package/dist/src/experiments/plugins/contextWindowScaling/plugin.js

@@ -0,0 +1,185 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { tokenCountMethod } from "../../../core/countTokens.js";
+import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { summarizeExperimentRun } from "../../results.js";
+import { aggregateStatus } from "../warmIndexReuse/execution.js";
+import { contextWindowScalingConfigDefinition, defaultContextWindowScalingConfig, validateContextWindowScalingConfig, } from "./config.js";
+import { executeContextWindowScalingCases, } from "./execution.js";
+import { buildContextWindowScalingExecutionArtifact, CONTEXT_WINDOW_SCALING_EXECUTION_ARTIFACT_FILE, } from "./executionArtifact.js";
+import { CONTEXT_WINDOW_SCALING_TREATMENT_IDS, contextWindowScalingMetadata } from "./metadata.js";
+import { aggregateContextWindowScaling, toAggregateExperimentMetrics, toOutcomeBudgetMetrics, } from "./metrics.js";
+import { resolveScalingProjectProfiles } from "./projectProfile.js";
+const VARIANTS = [
+    {
+        id: "raw-full-file",
+        name: "Raw full file",
+        description: "Every file matched by the case's raw include globs, constructed once per case.",
+    },
+    {
+        id: "my-dev-kit-guided",
+        name: "my-dev-kit guided",
+        description: "my-dev-kit retrieval context, constructed once per case.",
+    },
+];
+/** Internal-only seam: tests inject construction/evaluation owners through inputs.dependencies. */
+export const contextWindowScalingPlugin = {
+    metadata: contextWindowScalingMetadata,
+    defaultConfig: defaultContextWindowScalingConfig,
+    configDefinition: contextWindowScalingConfigDefinition,
+    supportedVariants: CONTEXT_WINDOW_SCALING_TREATMENT_IDS,
+    validateConfig: validateContextWindowScalingConfig,
+    async run(context) {
+        const startedAt = context.startedAt.toISOString();
+        const cases = readCasesInput(context.inputs);
+        const projectProfiles = resolveScalingProjectProfiles(cases, readProjectProfilesInput(context.inputs));
+        const outDir = context.outputRoot ?? path.resolve(context.toolRoot, "lab-output", "context-window-scaling");
+        await mkdir(outDir, { recursive: true });
+        const executionEvidence = await executeContextWindowScalingCases({
+            cases,
+            contextBudgets: context.config.contextBudgets,
+            kitCommand: context.config.kitCommand,
+            outputRoot: outDir,
+            projectProfiles,
+            cwd: context.target.targetRoot,
+            env: readEnvInput(context.inputs),
+            dependencies: readDependenciesInput(context.inputs),
+        });
+        const completedAt = new Date().toISOString();
+        const artifact = buildContextWindowScalingExecutionArtifact({
+            runId: context.runId,
+            pluginId: contextWindowScalingMetadata.id,
+            pluginSchemaVersion: contextWindowScalingMetadata.schemaVersion,
+            startedAt,
+            completedAt,
+            tokenCountMethod,
+            contextBudgets: context.config.contextBudgets,
+            cases: executionEvidence,
+        });
+        const artifactPath = resolveWithinRoot(outDir, CONTEXT_WINDOW_SCALING_EXECUTION_ARTIFACT_FILE);
+        await writeFile(artifactPath, `${JSON.stringify(artifact, null, 2)}\n`, "utf8");
+        return mapExecutionToRun({
+            runId: context.runId,
+            startedAt,
+            completedAt,
+            target: context.target,
+            contextBudgets: context.config.contextBudgets,
+            executionEvidence,
+            artifactPath,
+        });
+    },
+    summarize(result) {
+        return result.summary ?? summarizeExperimentRun(result);
+    },
+};
+export function mapExecutionToRun(args) {
+    const cases = args.executionEvidence.map((caseEvidence) => ({
+        id: caseEvidence.caseId,
+        name: caseEvidence.caseName,
+        outcomes: caseEvidence.treatments.map((treatment) => buildOutcome(caseEvidence, treatment)),
+        metadata: { benchmarkProject: caseEvidence.benchmarkProject },
+    }));
+    const aggregate = aggregateContextWindowScaling({ contextBudgets: args.contextBudgets, cases: args.executionEvidence });
+    const run = {
+        runId: args.runId,
+        pluginId: contextWindowScalingMetadata.id,
+        startedAt: args.startedAt,
+        completedAt: args.completedAt,
+        status: aggregateStatus(cases.flatMap((experimentCase) => experimentCase.outcomes.map((outcome) => outcome.status))),
+        target: args.target,
+        variants: VARIANTS.map((variant) => ({ ...variant })),
+        cases,
+        metrics: toAggregateExperimentMetrics(aggregate),
+        artifacts: [
+            {
+                id: "context-window-scaling-execution",
+                label: "Context window scaling execution evidence",
+                path: args.artifactPath,
+                kind: "artifact",
+                mimeType: "application/json",
+                description: "Per-case, per-treatment, per-budget fit, correctness, success, and omitted-file evidence without context text.",
+            },
+        ],
+        warnings: [],
+        failures: [],
+        metadata: { executionArtifactPath: args.artifactPath },
+        contextBudgets: [...args.contextBudgets],
+        executionEvidence: structuredClone([...args.executionEvidence]),
+        aggregate,
+    };
+    run.summary = summarizeExperimentRun(run);
+    return run;
+}
+function buildOutcome(caseEvidence, treatment) {
+    const { caseId } = caseEvidence;
+    const variantId = treatment.variantId;
+    const metrics = [];
+    if (treatment.context.estimatedTokens !== null) {
+        metrics.push({
+            id: "context-estimated-tokens",
+            name: "Context estimated tokens",
+            value: treatment.context.estimatedTokens,
+            unit: "tokens",
+            variantId,
+            caseId,
+        });
+    }
+    if (treatment.relevantFileEvidence.omittedRelevantFileCount !== null) {
+        metrics.push({
+            id: "omitted-relevant-file-count",
+            name: "Omitted relevant files",
+            value: treatment.relevantFileEvidence.omittedRelevantFileCount,
+            unit: "files",
+            variantId,
+            caseId,
+        });
+    }
+    metrics.push(...toOutcomeBudgetMetrics(treatment, caseId));
+    const warnings = [
+        ...treatment.context.warnings.map((message) => ({ code: "guided-retrieval-warning", message, variantId, caseId })),
+        ...(treatment.evaluation.status === "unavailable"
+            ? [{ code: "deterministic-evaluation-unavailable", message: treatment.evaluation.reason ?? "Evaluation unavailable.", variantId, caseId }]
+            : []),
+    ];
+    const failures = treatment.errors.map((error) => ({
+        code: error.code,
+        message: error.message,
+        variantId,
+        caseId,
+        recoverable: true,
+    }));
+    return {
+        id: `${caseId}:${variantId}`,
+        caseId,
+        variantId,
+        status: treatment.status,
+        metrics,
+        artifacts: [],
+        warnings,
+        failures,
+        metadata: {
+            benchmarkProject: caseEvidence.benchmarkProject,
+            contextStatus: treatment.context.status,
+            evaluationStatus: treatment.evaluation.status,
+        },
+    };
+}
+function readCasesInput(inputs) {
+    const value = inputs?.cases;
+    if (!Array.isArray(value) || value.length === 0) {
+        throw new Error("Context window scaling requires a non-empty cases input.");
+    }
+    return value;
+}
+function readProjectProfilesInput(inputs) {
+    const value = inputs?.projectProfiles;
+    return Array.isArray(value) ? value : [];
+}
+function readEnvInput(inputs) {
+    const env = inputs?.env;
+    return env && typeof env === "object" && !Array.isArray(env) ? env : undefined;
+}
+function readDependenciesInput(inputs) {
+    const value = inputs?.dependencies;
+    return value && typeof value === "object" ? value : undefined;
+}

package/dist/src/experiments/plugins/contextWindowScaling/projectProfile.js

@@ -0,0 +1,51 @@
+import path from "node:path";
+import { calculateProjectComplexityScore, computeProjectComplexityMetrics, PROJECT_COMPLEXITY_FORMULA, } from "../../../evaluation/projectComplexity.js";
+import { buildProjectFileTree } from "../../../evaluation/projectFileTree.js";
+/**
+ * The fixed scaling project has no entry in the shared benchmark-project-profiles contract. The
+ * existing prompt owner requires a profile per case, so a missing one is derived in memory from
+ * the existing file-tree and complexity owners. Supplied profiles always win; nothing is written.
+ */
+export function resolveScalingProjectProfiles(cases, supplied) {
+    const profiles = [...supplied];
+    const known = new Set(profiles.map((profile) => profile.projectId));
+    for (const evaluationCase of cases) {
+        const profileId = evaluationCase.projectProfileRef ?? evaluationCase.benchmarkProject;
+        if (known.has(profileId))
+            continue;
+        known.add(profileId);
+        profiles.push(deriveProfile(profileId, evaluationCase, cases));
+    }
+    return profiles;
+}
+function deriveProfile(profileId, evaluationCase, cases) {
+    const siblings = cases.filter((candidate) => (candidate.projectProfileRef ?? candidate.benchmarkProject) === profileId);
+    const fileTree = buildProjectFileTree(evaluationCase.absoluteTargetRoot);
+    const average = (values) => (values.length === 0 ? 0 : values.reduce((a, b) => a + b, 0) / values.length);
+    const complexityMetrics = computeProjectComplexityMetrics(evaluationCase.absoluteTargetRoot, fileTree, {
+        taskCount: siblings.length,
+        expectedRelevantFilesAverage: average(siblings.map((candidate) => candidate.expectedFiles.length)),
+        expectedRelevantSymbolsAverage: average(siblings.map((candidate) => candidate.expectedSymbols.length)),
+    });
+    const languages = [
+        ...new Set(fileTree.entries.map((entry) => entry.language).filter((language) => Boolean(language))),
+    ].sort();
+    return {
+        projectId: profileId,
+        displayName: profileId,
+        description: "Profile derived in memory for the context-window-scaling fixed corpus.",
+        languageMix: languages.length <= 1 ? `single-language ${languages[0] ?? "unknown"}` : "multi-language",
+        primaryLanguage: languages[0] ?? "unknown",
+        languages,
+        complexityLevel: "large",
+        complexityScore: calculateProjectComplexityScore(complexityMetrics),
+        complexityMetrics,
+        complexityFormula: PROJECT_COMPLEXITY_FORMULA,
+        rootPath: path.posix.normalize(evaluationCase.targetRoot.replace(/\\/g, "/")),
+        sourceRoots: [...evaluationCase.sourceRoots],
+        testRoots: ["tests"],
+        fileTree,
+        benchmarkPurpose: "Context-window scaling evaluation.",
+        expectedUseCases: ["context budget fit evaluation"],
+    };
+}

package/dist/src/experiments/plugins/contextWindowScaling/relevantFiles.js

@@ -0,0 +1,76 @@
+/**
+ * Bounded repository-relative identity: "\" becomes "/", leading "./" is removed, case is kept.
+ * Returns null for absolute paths, drive paths, parent traversal, empty segments, or oversized values.
+ */
+export function normalizeRepositoryRelativePath(value) {
+    if (typeof value !== "string" || value.length === 0 || value.length > 1024) {
+        return null;
+    }
+    let normalized = value.replace(/\\/g, "/");
+    while (normalized.startsWith("./")) {
+        normalized = normalized.slice(2);
+    }
+    if (normalized.length === 0 || normalized.startsWith("/") || /^[A-Za-z]:/.test(normalized)) {
+        return null;
+    }
+    if (normalized.split("/").some((segment) => segment === ".." || segment === "")) {
+        return null;
+    }
+    return normalized;
+}
+/** observedFiles === null means observed-file provenance is unavailable. */
+export function computeRelevantFileEvidence(args) {
+    if (args.expectedFiles.length === 0) {
+        return {
+            status: "not-applicable",
+            expectedRelevantFiles: [],
+            expectedRelevantFileCount: 0,
+            observedExpectedFileCount: null,
+            omittedRelevantFileCount: null,
+            omittedRelevantFiles: [],
+            reason: "No expected relevant files are defined for this case.",
+        };
+    }
+    const expected = [];
+    for (const file of args.expectedFiles) {
+        const normalized = normalizeRepositoryRelativePath(file);
+        if (normalized === null) {
+            return unavailable([...args.expectedFiles], "An expected relevant file is not a bounded repository-relative path.");
+        }
+        if (!expected.includes(normalized)) {
+            expected.push(normalized);
+        }
+    }
+    if (args.observedFiles === null) {
+        return unavailable(expected, "Observed-file provenance is unavailable.");
+    }
+    const observed = new Set();
+    for (const file of args.observedFiles) {
+        const normalized = normalizeRepositoryRelativePath(file);
+        if (normalized === null) {
+            return unavailable(expected, "An observed file is not a bounded repository-relative path.");
+        }
+        observed.add(normalized);
+    }
+    const omitted = expected.filter((file) => !observed.has(file));
+    return {
+        status: "available",
+        expectedRelevantFiles: expected,
+        expectedRelevantFileCount: expected.length,
+        observedExpectedFileCount: expected.length - omitted.length,
+        omittedRelevantFileCount: omitted.length,
+        omittedRelevantFiles: omitted,
+        reason: null,
+    };
+}
+function unavailable(expected, reason) {
+    return {
+        status: "unavailable",
+        expectedRelevantFiles: expected,
+        expectedRelevantFileCount: expected.length,
+        observedExpectedFileCount: null,
+        omittedRelevantFileCount: null,
+        omittedRelevantFiles: [],
+        reason,
+    };
+}

package/dist/src/experiments/plugins/contextWindowScaling/successEvidence.js

@@ -0,0 +1,60 @@
+import { calculateContextBudgetUtilizationPercent, classifyContextFit } from "./contextBudget.js";
+export const UNAVAILABLE_CORRECTNESS = { availability: "unavailable", score: null, pass: null };
+/** The existing scorer's own passed result is used as-is; no score threshold is defined here. */
+export function toCorrectnessEvidence(scored) {
+    if (!scored.available || scored.score === null || scored.passed === null) {
+        return { ...UNAVAILABLE_CORRECTNESS };
+    }
+    return { availability: "available", score: scored.score, pass: scored.passed };
+}
+export function deriveSuccessEvidence(args) {
+    if (args.contextFitStatus === "unavailable") {
+        return { status: "unavailable", success: null, reason: "context-unavailable" };
+    }
+    if (args.contextFitStatus === "context-too-large") {
+        return { status: "available", success: false, reason: "context-too-large" };
+    }
+    if (args.correctness.availability !== "available" || args.correctness.pass === null) {
+        return { status: "unavailable", success: null, reason: "evaluation-unavailable" };
+    }
+    return args.correctness.pass
+        ? { status: "available", success: true, reason: "correctness-pass" }
+        : { status: "available", success: false, reason: "correctness-fail" };
+}
+/**
+ * One cell per budget. `shared` is the single deterministic evaluation of this treatment's context
+ * (null when it was never requested because no budget fits or the context is unavailable).
+ */
+export function buildBudgetCell(args) {
+    if (args.contextEstimatedTokens === null) {
+        return {
+            contextBudgetTokens: args.contextBudgetTokens,
+            contextFitStatus: "unavailable",
+            contextBudgetUtilizationPercent: null,
+            evaluationStatus: "unavailable",
+            correctness: { ...UNAVAILABLE_CORRECTNESS },
+            successEvidence: deriveSuccessEvidence({ contextFitStatus: "unavailable", correctness: UNAVAILABLE_CORRECTNESS }),
+        };
+    }
+    const fit = classifyContextFit(args.contextEstimatedTokens, args.contextBudgetTokens);
+    const utilization = calculateContextBudgetUtilizationPercent(args.contextEstimatedTokens, args.contextBudgetTokens);
+    if (fit === "context-too-large") {
+        return {
+            contextBudgetTokens: args.contextBudgetTokens,
+            contextFitStatus: fit,
+            contextBudgetUtilizationPercent: utilization,
+            evaluationStatus: "not-evaluated-context-too-large",
+            correctness: { ...UNAVAILABLE_CORRECTNESS },
+            successEvidence: deriveSuccessEvidence({ contextFitStatus: fit, correctness: UNAVAILABLE_CORRECTNESS }),
+        };
+    }
+    const correctness = args.shared ?? { ...UNAVAILABLE_CORRECTNESS };
+    return {
+        contextBudgetTokens: args.contextBudgetTokens,
+        contextFitStatus: fit,
+        contextBudgetUtilizationPercent: utilization,
+        evaluationStatus: correctness.availability === "available" ? "evaluated" : "unavailable",
+        correctness: { ...correctness },
+        successEvidence: deriveSuccessEvidence({ contextFitStatus: fit, correctness }),
+    };
+}

package/dist/src/experiments/plugins/contextWindowScaling/syntheticInputs.js

@@ -0,0 +1,56 @@
+import { existsSync, mkdirSync, realpathSync } from "node:fs";
+import path from "node:path";
+import { relativeWithinRoot, resolveWithinRoot } from "../../../core/pathSafety.js";
+import { materializeSyntheticRepository, planSyntheticRepositories } from "../../../evaluation/syntheticRepository/index.js";
+import { resolveScalingProjectProfiles } from "./projectProfile.js";
+/**
+ * Stable child of the experiment output root that owns every generated repository. Batch 2 lays each case out
+ * beneath it as `<case id>/repository` plus a sibling manifest. It never collides with the execution artifact,
+ * reports, `guided/`, `agents/` or `charts/`.
+ */
+export const SYNTHETIC_REPOSITORIES_OUTPUT_SUBDIRECTORY = "synthetic-repositories";
+/**
+ * Internal input source for context-window-scaling: Batch 1 config -> plans -> Batch 2 materializations ->
+ * EvaluationCase[] -> in-memory project profiles. It owns only that sequencing; planning, rendering, manifests,
+ * profile derivation and execution all stay with their existing owners, and generated repositories are not
+ * serialized into bundled case JSON (readEvaluationCases is deliberately not used).
+ *
+ * Fail closed: an invalid or empty config, a planner failure, a materialization collision or a verification
+ * failure throws the owning typed error before any case exists. All plans are computed before the first
+ * repository is written. Repeating the call with the same config and output root reuses the existing
+ * materializations (Batch 2 idempotence) and returns the same identities and order.
+ *
+ * The caller must supply the final experiment output root; the public command resolves it before calling this
+ * and passes the same root to the generic runner.
+ */
+export function prepareSyntheticContextWindowScalingInputs(args) {
+    const plans = planSyntheticRepositories(args.syntheticRepositoryConfig);
+    const syntheticOutputRoot = resolveWithinRoot(args.outputRoot, SYNTHETIC_REPOSITORIES_OUTPUT_SUBDIRECTORY);
+    assertStaysInsideOutputRoot(args.outputRoot, syntheticOutputRoot);
+    const materializations = plans.map((plan) => {
+        const materialized = materializeSyntheticRepository(plan, syntheticOutputRoot);
+        return {
+            caseId: plan.caseId,
+            generationIdentity: plan.generationIdentity,
+            repositoryContentIdentity: materialized.manifest.repositoryContentIdentity,
+            repositoryRoot: materialized.repositoryRoot,
+            manifestPath: materialized.manifestPath,
+            evaluationCase: materialized.evaluationCase,
+            reusedExistingMaterialization: materialized.reusedExistingMaterialization,
+        };
+    });
+    const cases = materializations.map((entry) => entry.evaluationCase);
+    return { cases, projectProfiles: resolveScalingProjectProfiles(cases, []), materializations, syntheticOutputRoot };
+}
+/** A pre-existing synthetic subtree that is a link resolving elsewhere must not redirect generation out of the experiment output. */
+function assertStaysInsideOutputRoot(outputRoot, syntheticOutputRoot) {
+    mkdirSync(path.resolve(outputRoot), { recursive: true });
+    if (!existsSync(syntheticOutputRoot))
+        return;
+    try {
+        relativeWithinRoot(realpathSync(path.resolve(outputRoot)), realpathSync(syntheticOutputRoot));
+    }
+    catch {
+        throw new Error(`Synthetic repository output ${SYNTHETIC_REPOSITORIES_OUTPUT_SUBDIRECTORY} resolves outside the experiment output root.`);
+    }
+}

package/dist/src/experiments/plugins/contextWindowScaling/types.js

@@ -0,0 +1 @@
+export {};

package/dist/src/experiments/plugins/index.js

@@ -1,3 +1,4 @@
 export * from "./contextStrategyComparison/index.js";
+export * from "./contextWindowScaling/index.js";
 export * from "./incrementalChangeStaleness/index.js";
 export * from "./warmIndexReuse/index.js";

package/dist/src/experiments/plugins/warmIndexReuse/agentEvaluation.js

@@ -41,7 +41,7 @@ export async function evaluateWarmIndexFakeAgents(options) {
     }
     return result;
 }
-async function evaluateFakeSide(args) {
+export async function evaluateFakeSide(args) {
     const promptStrategy = SIDE_STRATEGIES[args.variantId];
     const base = { variantId: args.variantId, agentId: "fake-agent", promptStrategy };
     try {

package/dist/src/plots/buildContextWindowScalingPlotData.js

@@ -0,0 +1,96 @@
+import { existsSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { CONTEXT_WINDOW_SCALING_EXECUTION_ARTIFACT_FILE } from "../experiments/plugins/contextWindowScaling/executionArtifact.js";
+import { parseContextWindowScalingExecutionArtifact } from "../experiments/plugins/contextWindowScaling/executionArtifactReader.js";
+import { aggregateContextWindowScaling } from "../experiments/plugins/contextWindowScaling/metrics.js";
+import { STANDARD_CONTEXT_BUDGETS } from "../experiments/plugins/contextWindowScaling/contextBudget.js";
+// ---------------------------------------------------------------------------
+// Context-window-scaling plot data. Maps values already calculated by
+// aggregateContextWindowScaling() (carried on ContextWindowScalingRun.aggregate) to plot
+// points. Nothing is recalculated; a null aggregate value becomes a skipped point, never 0.
+// ---------------------------------------------------------------------------
+export const CONTEXT_WINDOW_SCALING_PLOT_IDS = [
+    "context-window-scaling-context-size",
+    "context-window-scaling-success-rate-by-budget",
+    "context-window-scaling-correctness-by-budget",
+];
+export const CONTEXT_WINDOW_SCALING_SKIP_REASONS = {
+    contextSize: "context-size-unavailable",
+    successRate: "success-rate-unavailable",
+    correctness: "correctness-unavailable",
+};
+/** Standard budgets read as 8k/16k/32k/64k; any other budget keeps its exact integer form. */
+export function formatContextBudgetLabel(budget) {
+    return STANDARD_CONTEXT_BUDGETS.includes(budget) ? `${budget / 1024}k` : String(budget);
+}
+/**
+ * Returns the aggregate rebuilt from `<experimentDir>/context-window-scaling-execution.json`, or null
+ * when that artifact is absent. A present artifact is validated (schema version and structure) and
+ * aggregated by the canonical aggregateContextWindowScaling(); malformed input fails, never falls back.
+ */
+export async function readContextWindowScalingPlotSource(experimentDir) {
+    const artifactPath = path.join(experimentDir, CONTEXT_WINDOW_SCALING_EXECUTION_ARTIFACT_FILE);
+    if (!existsSync(artifactPath))
+        return null;
+    let parsed;
+    try {
+        parsed = JSON.parse(await readFile(artifactPath, "utf8"));
+    }
+    catch (error) {
+        throw new Error(`Invalid context-window-scaling execution artifact (${artifactPath}): not valid JSON (${error instanceof Error ? error.message : String(error)}).`);
+    }
+    const artifact = parseContextWindowScalingExecutionArtifact(parsed);
+    return {
+        aggregate: aggregateContextWindowScaling({ contextBudgets: artifact.contextBudgets, cases: artifact.cases }),
+        generatedAt: artifact.completedAt,
+    };
+}
+export function buildContextWindowScalingPlotData(args) {
+    const { aggregate } = args;
+    const skippedPoints = [];
+    const series = (id, title, xLabel, yLabel) => ({
+        id,
+        title,
+        xLabel,
+        yLabel,
+        kind: "scatter",
+        points: [],
+        warnings: [],
+    });
+    const contextSize = series(CONTEXT_WINDOW_SCALING_PLOT_IDS[0], "Raw vs Retrieved Context Size", "Case ordinal (canonical case order)", "Estimated context tokens");
+    const successRate = series(CONTEXT_WINDOW_SCALING_PLOT_IDS[1], "Success Rate by Context Budget", "Context budget (estimated tokens)", "Success rate (percent)");
+    const correctness = series(CONTEXT_WINDOW_SCALING_PLOT_IDS[2], "Correctness by Context Budget", "Context budget (estimated tokens)", "Mean correctness score");
+    const add = (plot, x, value, group, label, reason, metadata) => {
+        if (value !== null && Number.isFinite(value)) {
+            plot.points.push({ x, y: value, group, label, metadata });
+        }
+        else {
+            skippedPoints.push({ plotId: plot.id, label, reason });
+        }
+    };
+    // Case ordinal follows the aggregate's canonical case order (first appearance of each caseId).
+    const caseOrdinals = new Map();
+    for (const summary of aggregate.caseTreatmentContextSummaries) {
+        if (!caseOrdinals.has(summary.caseId))
+            caseOrdinals.set(summary.caseId, caseOrdinals.size + 1);
+    }
+    for (const summary of aggregate.caseTreatmentContextSummaries) {
+        add(contextSize, caseOrdinals.get(summary.caseId), summary.contextStatus === "available" ? summary.estimatedTokens : null, summary.variantId, `${summary.caseId} ${summary.variantId}`, CONTEXT_WINDOW_SCALING_SKIP_REASONS.contextSize, { caseId: summary.caseId });
+    }
+    for (const summary of aggregate.budgetTreatmentSummaries) {
+        const label = `${summary.variantId} ${formatContextBudgetLabel(summary.contextBudgetTokens)}`;
+        const metadata = { contextBudgetTokens: summary.contextBudgetTokens };
+        add(successRate, summary.contextBudgetTokens, summary.successRatePercent, summary.variantId, label, CONTEXT_WINDOW_SCALING_SKIP_REASONS.successRate, metadata);
+        add(correctness, summary.contextBudgetTokens, summary.meanCorrectnessScore, summary.variantId, label, CONTEXT_WINDOW_SCALING_SKIP_REASONS.correctness, metadata);
+    }
+    const plots = [contextSize, successRate, correctness];
+    const warnings = [];
+    for (const plot of plots) {
+        if (plot.points.length === 0) {
+            plot.warnings.push("No comparable data available.");
+            warnings.push(`${plot.title}: no comparable data available.`);
+        }
+    }
+    return { generatedAt: args.generatedAt, sourceExperimentDir: args.experimentDir, plots, skippedPoints, warnings };
+}

package/dist/src/plots/index.js

@@ -1,5 +1,6 @@
 export * from "./types.js";
 export * from "./buildExperimentPlotData.js";
 export * from "./buildWarmIndexPlotData.js";
+export * from "./buildContextWindowScalingPlotData.js";
 export * from "./renderSvgChart.js";
 export * from "./writePlotArtifacts.js";

package/dist/src/plots/writePlotArtifacts.js

@@ -1,6 +1,7 @@
 import { mkdir, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { resolveWithinRoot } from "../core/pathSafety.js";
+import { buildContextWindowScalingPlotData, readContextWindowScalingPlotSource } from "./buildContextWindowScalingPlotData.js";
 import { buildExperimentPlotData } from "./buildExperimentPlotData.js";
 import { buildWarmIndexPlotData, readWarmIndexPlotSource } from "./buildWarmIndexPlotData.js";
 import { renderSvgChart } from "./renderSvgChart.js";
@@ -16,10 +17,18 @@ export async function writePlotArtifacts(options) {
     // A warm-index-reuse plugin output directory is detected by its plugin report; every other
     // directory keeps the legacy controlled-experiment plot path unchanged.
     const experimentDir = path.resolve(options.repoRoot ?? process.cwd(), options.experimentDir);
+    // A context-window-scaling directory is detected by its V1 execution artifact; evidence for more
+    // than one plugin family in the same directory is ambiguous and fails instead of choosing one.
+    const scalingSource = await readContextWindowScalingPlotSource(experimentDir);
     const warmSection = await readWarmIndexPlotSource(experimentDir);
-    const data = warmSection
-        ? buildWarmIndexPlotData({ section: warmSection, experimentDir })
-        : await buildExperimentPlotData({ experimentDir: options.experimentDir, repoRoot: options.repoRoot });
+    if (scalingSource && warmSection) {
+        throw new Error(`Ambiguous experiment directory for plots (${experimentDir}): it contains both context-window-scaling and warm-index-reuse evidence.`);
+    }
+    const data = scalingSource
+        ? buildContextWindowScalingPlotData({ aggregate: scalingSource.aggregate, experimentDir, generatedAt: scalingSource.generatedAt })
+        : warmSection
+            ? buildWarmIndexPlotData({ section: warmSection, experimentDir })
+            : await buildExperimentPlotData({ experimentDir: options.experimentDir, repoRoot: options.repoRoot });
     return writePlotArtifactsFromData({ data, outDir: options.outDir });
 }
 export async function writePlotArtifactsFromData(options) {