@dailephd/my-dev-kit-lab 0.6.3 → 0.7.0

package/dist/src/experiments/plugins/contextWindowScaling/metrics.js

@@ -0,0 +1,149 @@
+import { CONTEXT_WINDOW_SCALING_TREATMENT_IDS } from "./metadata.js";
+export function aggregateContextWindowScaling(args) {
+    const budgets = [...args.contextBudgets].sort((a, b) => a - b);
+    const treatments = [...CONTEXT_WINDOW_SCALING_TREATMENT_IDS];
+    const treatmentsOf = (variantId) => args.cases.map((caseEvidence) => ({ caseEvidence, treatment: requireTreatment(caseEvidence, variantId, budgets) }));
+    const budgetTreatmentSummaries = [];
+    for (const contextBudgetTokens of budgets) {
+        for (const variantId of treatments) {
+            const cells = treatmentsOf(variantId).map(({ treatment }) => cellFor(treatment, contextBudgetTokens));
+            budgetTreatmentSummaries.push(summarizeCells(contextBudgetTokens, variantId, cells));
+        }
+    }
+    const caseTreatmentContextSummaries = args.cases.flatMap((caseEvidence) => treatments.map((variantId) => {
+        const treatment = requireTreatment(caseEvidence, variantId, budgets);
+        return {
+            caseId: caseEvidence.caseId,
+            caseName: caseEvidence.caseName,
+            variantId,
+            contextStatus: treatment.context.status,
+            characterCount: treatment.context.characterCount,
+            estimatedTokens: treatment.context.estimatedTokens,
+            tokenCountMethod: treatment.context.tokenCountMethod,
+            observedFileCount: treatment.context.observedFiles === null ? null : treatment.context.observedFiles.length,
+            relevantFileEvidence: structuredClone(treatment.relevantFileEvidence),
+        };
+    }));
+    const allCells = budgetTreatmentSummaries;
+    const relevant = caseTreatmentContextSummaries.map((summary) => summary.relevantFileEvidence);
+    const relevantCount = (status) => relevant.filter((evidence) => evidence.status === status).length;
+    const relevantFileSummary = {
+        relevantFileEvidenceAvailableCount: relevantCount("available"),
+        relevantFileEvidenceUnavailableCount: relevantCount("unavailable"),
+        relevantFileEvidenceNotApplicableCount: relevantCount("not-applicable"),
+        totalOmittedRelevantFileCount: relevant.reduce((sum, evidence) => sum + (evidence.omittedRelevantFileCount ?? 0), 0),
+    };
+    const sumOf = (pick) => allCells.reduce((sum, summary) => sum + pick(summary), 0);
+    const evaluatedBudgetCellCount = args.cases
+        .flatMap((caseEvidence) => caseEvidence.treatments)
+        .flatMap((treatment) => treatment.budgetCells)
+        .filter((cell) => cell.evaluationStatus === "evaluated").length;
+    return {
+        budgets,
+        treatments,
+        runSummary: {
+            caseCount: args.cases.length,
+            treatmentCount: treatments.length,
+            budgetCount: budgets.length,
+            totalBudgetCellCount: sumOf((s) => s.totalCellCount),
+            evaluatedBudgetCellCount,
+            contextTooLargeCellCount: sumOf((s) => s.contextTooLargeCount),
+            contextUnavailableCellCount: sumOf((s) => s.contextUnavailableCount),
+            correctnessAvailableCellCount: sumOf((s) => s.correctnessAvailableCount),
+            correctnessUnavailableCellCount: sumOf((s) => s.correctnessUnavailableCount),
+            successEvidenceAvailableCellCount: sumOf((s) => s.successEvidenceAvailableCount),
+            successEvidenceUnavailableCellCount: sumOf((s) => s.successEvidenceUnavailableCount),
+            relevantFileEvidenceAvailableTreatmentCount: relevantFileSummary.relevantFileEvidenceAvailableCount,
+            relevantFileEvidenceUnavailableTreatmentCount: relevantFileSummary.relevantFileEvidenceUnavailableCount,
+            relevantFileEvidenceNotApplicableTreatmentCount: relevantFileSummary.relevantFileEvidenceNotApplicableCount,
+        },
+        budgetTreatmentSummaries,
+        caseTreatmentContextSummaries,
+        relevantFileSummary,
+    };
+}
+function requireTreatment(caseEvidence, variantId, budgets) {
+    const matches = caseEvidence.treatments.filter((treatment) => treatment.variantId === variantId);
+    const treatment = matches[0];
+    if (matches.length !== 1 || !treatment) {
+        throw new Error(`Invalid context-window-scaling evidence: case ${caseEvidence.caseId} must have exactly one ${variantId} treatment.`);
+    }
+    const cellBudgets = treatment.budgetCells.map((cell) => cell.contextBudgetTokens);
+    if (cellBudgets.length !== budgets.length || budgets.some((budget) => cellBudgets.filter((value) => value === budget).length !== 1)) {
+        throw new Error(`Invalid context-window-scaling evidence: case ${caseEvidence.caseId} ${variantId} cells do not match the selected budgets.`);
+    }
+    return treatment;
+}
+function cellFor(treatment, budget) {
+    return treatment.budgetCells.find((cell) => cell.contextBudgetTokens === budget);
+}
+function isFiniteNumber(value) {
+    return typeof value === "number" && Number.isFinite(value);
+}
+function summarizeCells(contextBudgetTokens, variantId, cells) {
+    const fitCount = cells.filter((cell) => cell.contextFitStatus === "fits").length;
+    const contextTooLargeCount = cells.filter((cell) => cell.contextFitStatus === "context-too-large").length;
+    const contextUnavailableCount = cells.filter((cell) => cell.contextFitStatus === "unavailable").length;
+    const successAvailable = cells.filter((cell) => cell.successEvidence.status === "available");
+    const successfulCellCount = successAvailable.filter((cell) => cell.successEvidence.success === true).length;
+    const scores = cells
+        .filter((cell) => cell.correctness.availability === "available")
+        .map((cell) => cell.correctness.score)
+        .filter(isFiniteNumber);
+    const utilizations = cells.map((cell) => cell.contextBudgetUtilizationPercent).filter(isFiniteNumber);
+    const mean = (values) => (values.length === 0 ? null : values.reduce((sum, value) => sum + value, 0) / values.length);
+    return {
+        contextBudgetTokens,
+        variantId,
+        totalCellCount: cells.length,
+        contextMeasuredCount: fitCount + contextTooLargeCount,
+        fitCount,
+        contextTooLargeCount,
+        contextUnavailableCount,
+        successfulCellCount,
+        notSuccessfulCellCount: successAvailable.length - successfulCellCount,
+        successEvidenceAvailableCount: successAvailable.length,
+        successEvidenceUnavailableCount: cells.length - successAvailable.length,
+        successRatePercent: successAvailable.length === 0 ? null : (successfulCellCount / successAvailable.length) * 100,
+        correctnessAvailableCount: scores.length,
+        correctnessUnavailableCount: cells.length - scores.length,
+        meanCorrectnessScore: mean(scores),
+        utilizationAvailableCount: utilizations.length,
+        utilizationUnavailableCount: cells.length - utilizations.length,
+        meanContextBudgetUtilizationPercent: mean(utilizations),
+        minContextBudgetUtilizationPercent: utilizations.length === 0 ? null : Math.min(...utilizations),
+        maxContextBudgetUtilizationPercent: utilizations.length === 0 ? null : Math.max(...utilizations),
+    };
+}
+/** Run-level scalar metrics: one set per budget x treatment, scoped by variantId and budget-bearing ids. */
+export function toAggregateExperimentMetrics(aggregate) {
+    const metrics = [
+        { id: "context-window-scaling-case-count", name: "Context-window-scaling case count", value: aggregate.runSummary.caseCount, unit: "count" },
+        { id: "context-window-scaling-budget-count", name: "Context budget count", value: aggregate.runSummary.budgetCount, unit: "count" },
+    ];
+    for (const summary of aggregate.budgetTreatmentSummaries) {
+        const prefix = `context-budget-${summary.contextBudgetTokens}`;
+        const scoped = (suffix, name, value, unit, description) => ({
+            id: `${prefix}-${suffix}`,
+            name: `${name} (${summary.contextBudgetTokens} token budget)`,
+            value,
+            unit,
+            ...(description ? { description } : {}),
+            variantId: summary.variantId,
+        });
+        metrics.push(scoped("fit-count", "Cases whose context fits", summary.fitCount, "count"), scoped("context-too-large-count", "Cases whose context is too large", summary.contextTooLargeCount, "count"), scoped("successful-cell-count", "Successful cells", summary.successfulCellCount, "count"), scoped("success-rate-percent", "Success rate", summary.successRatePercent, "percent", "Successful cells over success-evidence-available cells; null when none are available."), scoped("mean-correctness-score", "Mean correctness score", summary.meanCorrectnessScore, "score", "Mean of available existing correctness scores; unavailable correctness is excluded, not zero."), scoped("mean-context-budget-utilization-percent", "Mean context budget utilization", summary.meanContextBudgetUtilizationPercent, "percent", "Uncapped; may exceed 100."));
+    }
+    return metrics;
+}
+/** Per-outcome scalar metrics for one treatment, one set per budget cell. */
+export function toOutcomeBudgetMetrics(treatment, caseId) {
+    return treatment.budgetCells.flatMap((cell) => {
+        const base = { variantId: treatment.variantId, caseId };
+        const prefix = `budget-${cell.contextBudgetTokens}`;
+        return [
+            { id: `${prefix}-context-budget-utilization-percent`, name: `Context budget utilization (${cell.contextBudgetTokens})`, value: cell.contextBudgetUtilizationPercent, unit: "percent", ...base },
+            { id: `${prefix}-correctness-score`, name: `Correctness score (${cell.contextBudgetTokens})`, value: cell.correctness.score, unit: "score", ...base },
+            { id: `${prefix}-success`, name: `Success (${cell.contextBudgetTokens})`, value: cell.successEvidence.success, ...base },
+        ];
+    });
+}

package/dist/src/experiments/plugins/contextWindowScaling/plugin.js

@@ -0,0 +1,185 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { tokenCountMethod } from "../../../core/countTokens.js";
+import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { summarizeExperimentRun } from "../../results.js";
+import { aggregateStatus } from "../warmIndexReuse/execution.js";
+import { contextWindowScalingConfigDefinition, defaultContextWindowScalingConfig, validateContextWindowScalingConfig, } from "./config.js";
+import { executeContextWindowScalingCases, } from "./execution.js";
+import { buildContextWindowScalingExecutionArtifact, CONTEXT_WINDOW_SCALING_EXECUTION_ARTIFACT_FILE, } from "./executionArtifact.js";
+import { CONTEXT_WINDOW_SCALING_TREATMENT_IDS, contextWindowScalingMetadata } from "./metadata.js";
+import { aggregateContextWindowScaling, toAggregateExperimentMetrics, toOutcomeBudgetMetrics, } from "./metrics.js";
+import { resolveScalingProjectProfiles } from "./projectProfile.js";
+const VARIANTS = [
+    {
+        id: "raw-full-file",
+        name: "Raw full file",
+        description: "Every file matched by the case's raw include globs, constructed once per case.",
+    },
+    {
+        id: "my-dev-kit-guided",
+        name: "my-dev-kit guided",
+        description: "my-dev-kit retrieval context, constructed once per case.",
+    },
+];
+/** Internal-only seam: tests inject construction/evaluation owners through inputs.dependencies. */
+export const contextWindowScalingPlugin = {
+    metadata: contextWindowScalingMetadata,
+    defaultConfig: defaultContextWindowScalingConfig,
+    configDefinition: contextWindowScalingConfigDefinition,
+    supportedVariants: CONTEXT_WINDOW_SCALING_TREATMENT_IDS,
+    validateConfig: validateContextWindowScalingConfig,
+    async run(context) {
+        const startedAt = context.startedAt.toISOString();
+        const cases = readCasesInput(context.inputs);
+        const projectProfiles = resolveScalingProjectProfiles(cases, readProjectProfilesInput(context.inputs));
+        const outDir = context.outputRoot ?? path.resolve(context.toolRoot, "lab-output", "context-window-scaling");
+        await mkdir(outDir, { recursive: true });
+        const executionEvidence = await executeContextWindowScalingCases({
+            cases,
+            contextBudgets: context.config.contextBudgets,
+            kitCommand: context.config.kitCommand,
+            outputRoot: outDir,
+            projectProfiles,
+            cwd: context.target.targetRoot,
+            env: readEnvInput(context.inputs),
+            dependencies: readDependenciesInput(context.inputs),
+        });
+        const completedAt = new Date().toISOString();
+        const artifact = buildContextWindowScalingExecutionArtifact({
+            runId: context.runId,
+            pluginId: contextWindowScalingMetadata.id,
+            pluginSchemaVersion: contextWindowScalingMetadata.schemaVersion,
+            startedAt,
+            completedAt,
+            tokenCountMethod,
+            contextBudgets: context.config.contextBudgets,
+            cases: executionEvidence,
+        });
+        const artifactPath = resolveWithinRoot(outDir, CONTEXT_WINDOW_SCALING_EXECUTION_ARTIFACT_FILE);
+        await writeFile(artifactPath, `${JSON.stringify(artifact, null, 2)}\n`, "utf8");
+        return mapExecutionToRun({
+            runId: context.runId,
+            startedAt,
+            completedAt,
+            target: context.target,
+            contextBudgets: context.config.contextBudgets,
+            executionEvidence,
+            artifactPath,
+        });
+    },
+    summarize(result) {
+        return result.summary ?? summarizeExperimentRun(result);
+    },
+};
+export function mapExecutionToRun(args) {
+    const cases = args.executionEvidence.map((caseEvidence) => ({
+        id: caseEvidence.caseId,
+        name: caseEvidence.caseName,
+        outcomes: caseEvidence.treatments.map((treatment) => buildOutcome(caseEvidence, treatment)),
+        metadata: { benchmarkProject: caseEvidence.benchmarkProject },
+    }));
+    const aggregate = aggregateContextWindowScaling({ contextBudgets: args.contextBudgets, cases: args.executionEvidence });
+    const run = {
+        runId: args.runId,
+        pluginId: contextWindowScalingMetadata.id,
+        startedAt: args.startedAt,
+        completedAt: args.completedAt,
+        status: aggregateStatus(cases.flatMap((experimentCase) => experimentCase.outcomes.map((outcome) => outcome.status))),
+        target: args.target,
+        variants: VARIANTS.map((variant) => ({ ...variant })),
+        cases,
+        metrics: toAggregateExperimentMetrics(aggregate),
+        artifacts: [
+            {
+                id: "context-window-scaling-execution",
+                label: "Context window scaling execution evidence",
+                path: args.artifactPath,
+                kind: "artifact",
+                mimeType: "application/json",
+                description: "Per-case, per-treatment, per-budget fit, correctness, success, and omitted-file evidence without context text.",
+            },
+        ],
+        warnings: [],
+        failures: [],
+        metadata: { executionArtifactPath: args.artifactPath },
+        contextBudgets: [...args.contextBudgets],
+        executionEvidence: structuredClone([...args.executionEvidence]),
+        aggregate,
+    };
+    run.summary = summarizeExperimentRun(run);
+    return run;
+}
+function buildOutcome(caseEvidence, treatment) {
+    const { caseId } = caseEvidence;
+    const variantId = treatment.variantId;
+    const metrics = [];
+    if (treatment.context.estimatedTokens !== null) {
+        metrics.push({
+            id: "context-estimated-tokens",
+            name: "Context estimated tokens",
+            value: treatment.context.estimatedTokens,
+            unit: "tokens",
+            variantId,
+            caseId,
+        });
+    }
+    if (treatment.relevantFileEvidence.omittedRelevantFileCount !== null) {
+        metrics.push({
+            id: "omitted-relevant-file-count",
+            name: "Omitted relevant files",
+            value: treatment.relevantFileEvidence.omittedRelevantFileCount,
+            unit: "files",
+            variantId,
+            caseId,
+        });
+    }
+    metrics.push(...toOutcomeBudgetMetrics(treatment, caseId));
+    const warnings = [
+        ...treatment.context.warnings.map((message) => ({ code: "guided-retrieval-warning", message, variantId, caseId })),
+        ...(treatment.evaluation.status === "unavailable"
+            ? [{ code: "deterministic-evaluation-unavailable", message: treatment.evaluation.reason ?? "Evaluation unavailable.", variantId, caseId }]
+            : []),
+    ];
+    const failures = treatment.errors.map((error) => ({
+        code: error.code,
+        message: error.message,
+        variantId,
+        caseId,
+        recoverable: true,
+    }));
+    return {
+        id: `${caseId}:${variantId}`,
+        caseId,
+        variantId,
+        status: treatment.status,
+        metrics,
+        artifacts: [],
+        warnings,
+        failures,
+        metadata: {
+            benchmarkProject: caseEvidence.benchmarkProject,
+            contextStatus: treatment.context.status,
+            evaluationStatus: treatment.evaluation.status,
+        },
+    };
+}
+function readCasesInput(inputs) {
+    const value = inputs?.cases;
+    if (!Array.isArray(value) || value.length === 0) {
+        throw new Error("Context window scaling requires a non-empty cases input.");
+    }
+    return value;
+}
+function readProjectProfilesInput(inputs) {
+    const value = inputs?.projectProfiles;
+    return Array.isArray(value) ? value : [];
+}
+function readEnvInput(inputs) {
+    const env = inputs?.env;
+    return env && typeof env === "object" && !Array.isArray(env) ? env : undefined;
+}
+function readDependenciesInput(inputs) {
+    const value = inputs?.dependencies;
+    return value && typeof value === "object" ? value : undefined;
+}

package/dist/src/experiments/plugins/contextWindowScaling/projectProfile.js

@@ -0,0 +1,51 @@
+import path from "node:path";
+import { calculateProjectComplexityScore, computeProjectComplexityMetrics, PROJECT_COMPLEXITY_FORMULA, } from "../../../evaluation/projectComplexity.js";
+import { buildProjectFileTree } from "../../../evaluation/projectFileTree.js";
+/**
+ * The fixed scaling project has no entry in the shared benchmark-project-profiles contract. The
+ * existing prompt owner requires a profile per case, so a missing one is derived in memory from
+ * the existing file-tree and complexity owners. Supplied profiles always win; nothing is written.
+ */
+export function resolveScalingProjectProfiles(cases, supplied) {
+    const profiles = [...supplied];
+    const known = new Set(profiles.map((profile) => profile.projectId));
+    for (const evaluationCase of cases) {
+        const profileId = evaluationCase.projectProfileRef ?? evaluationCase.benchmarkProject;
+        if (known.has(profileId))
+            continue;
+        known.add(profileId);
+        profiles.push(deriveProfile(profileId, evaluationCase, cases));
+    }
+    return profiles;
+}
+function deriveProfile(profileId, evaluationCase, cases) {
+    const siblings = cases.filter((candidate) => (candidate.projectProfileRef ?? candidate.benchmarkProject) === profileId);
+    const fileTree = buildProjectFileTree(evaluationCase.absoluteTargetRoot);
+    const average = (values) => (values.length === 0 ? 0 : values.reduce((a, b) => a + b, 0) / values.length);
+    const complexityMetrics = computeProjectComplexityMetrics(evaluationCase.absoluteTargetRoot, fileTree, {
+        taskCount: siblings.length,
+        expectedRelevantFilesAverage: average(siblings.map((candidate) => candidate.expectedFiles.length)),
+        expectedRelevantSymbolsAverage: average(siblings.map((candidate) => candidate.expectedSymbols.length)),
+    });
+    const languages = [
+        ...new Set(fileTree.entries.map((entry) => entry.language).filter((language) => Boolean(language))),
+    ].sort();
+    return {
+        projectId: profileId,
+        displayName: profileId,
+        description: "Profile derived in memory for the context-window-scaling fixed corpus.",
+        languageMix: languages.length <= 1 ? `single-language ${languages[0] ?? "unknown"}` : "multi-language",
+        primaryLanguage: languages[0] ?? "unknown",
+        languages,
+        complexityLevel: "large",
+        complexityScore: calculateProjectComplexityScore(complexityMetrics),
+        complexityMetrics,
+        complexityFormula: PROJECT_COMPLEXITY_FORMULA,
+        rootPath: path.posix.normalize(evaluationCase.targetRoot.replace(/\\/g, "/")),
+        sourceRoots: [...evaluationCase.sourceRoots],
+        testRoots: ["tests"],
+        fileTree,
+        benchmarkPurpose: "Context-window scaling evaluation.",
+        expectedUseCases: ["context budget fit evaluation"],
+    };
+}

package/dist/src/experiments/plugins/contextWindowScaling/relevantFiles.js

@@ -0,0 +1,76 @@
+/**
+ * Bounded repository-relative identity: "\" becomes "/", leading "./" is removed, case is kept.
+ * Returns null for absolute paths, drive paths, parent traversal, empty segments, or oversized values.
+ */
+export function normalizeRepositoryRelativePath(value) {
+    if (typeof value !== "string" || value.length === 0 || value.length > 1024) {
+        return null;
+    }
+    let normalized = value.replace(/\\/g, "/");
+    while (normalized.startsWith("./")) {
+        normalized = normalized.slice(2);
+    }
+    if (normalized.length === 0 || normalized.startsWith("/") || /^[A-Za-z]:/.test(normalized)) {
+        return null;
+    }
+    if (normalized.split("/").some((segment) => segment === ".." || segment === "")) {
+        return null;
+    }
+    return normalized;
+}
+/** observedFiles === null means observed-file provenance is unavailable. */
+export function computeRelevantFileEvidence(args) {
+    if (args.expectedFiles.length === 0) {
+        return {
+            status: "not-applicable",
+            expectedRelevantFiles: [],
+            expectedRelevantFileCount: 0,
+            observedExpectedFileCount: null,
+            omittedRelevantFileCount: null,
+            omittedRelevantFiles: [],
+            reason: "No expected relevant files are defined for this case.",
+        };
+    }
+    const expected = [];
+    for (const file of args.expectedFiles) {
+        const normalized = normalizeRepositoryRelativePath(file);
+        if (normalized === null) {
+            return unavailable([...args.expectedFiles], "An expected relevant file is not a bounded repository-relative path.");
+        }
+        if (!expected.includes(normalized)) {
+            expected.push(normalized);
+        }
+    }
+    if (args.observedFiles === null) {
+        return unavailable(expected, "Observed-file provenance is unavailable.");
+    }
+    const observed = new Set();
+    for (const file of args.observedFiles) {
+        const normalized = normalizeRepositoryRelativePath(file);
+        if (normalized === null) {
+            return unavailable(expected, "An observed file is not a bounded repository-relative path.");
+        }
+        observed.add(normalized);
+    }
+    const omitted = expected.filter((file) => !observed.has(file));
+    return {
+        status: "available",
+        expectedRelevantFiles: expected,
+        expectedRelevantFileCount: expected.length,
+        observedExpectedFileCount: expected.length - omitted.length,
+        omittedRelevantFileCount: omitted.length,
+        omittedRelevantFiles: omitted,
+        reason: null,
+    };
+}
+function unavailable(expected, reason) {
+    return {
+        status: "unavailable",
+        expectedRelevantFiles: expected,
+        expectedRelevantFileCount: expected.length,
+        observedExpectedFileCount: null,
+        omittedRelevantFileCount: null,
+        omittedRelevantFiles: [],
+        reason,
+    };
+}

package/dist/src/experiments/plugins/contextWindowScaling/successEvidence.js

@@ -0,0 +1,60 @@
+import { calculateContextBudgetUtilizationPercent, classifyContextFit } from "./contextBudget.js";
+export const UNAVAILABLE_CORRECTNESS = { availability: "unavailable", score: null, pass: null };
+/** The existing scorer's own passed result is used as-is; no score threshold is defined here. */
+export function toCorrectnessEvidence(scored) {
+    if (!scored.available || scored.score === null || scored.passed === null) {
+        return { ...UNAVAILABLE_CORRECTNESS };
+    }
+    return { availability: "available", score: scored.score, pass: scored.passed };
+}
+export function deriveSuccessEvidence(args) {
+    if (args.contextFitStatus === "unavailable") {
+        return { status: "unavailable", success: null, reason: "context-unavailable" };
+    }
+    if (args.contextFitStatus === "context-too-large") {
+        return { status: "available", success: false, reason: "context-too-large" };
+    }
+    if (args.correctness.availability !== "available" || args.correctness.pass === null) {
+        return { status: "unavailable", success: null, reason: "evaluation-unavailable" };
+    }
+    return args.correctness.pass
+        ? { status: "available", success: true, reason: "correctness-pass" }
+        : { status: "available", success: false, reason: "correctness-fail" };
+}
+/**
+ * One cell per budget. `shared` is the single deterministic evaluation of this treatment's context
+ * (null when it was never requested because no budget fits or the context is unavailable).
+ */
+export function buildBudgetCell(args) {
+    if (args.contextEstimatedTokens === null) {
+        return {
+            contextBudgetTokens: args.contextBudgetTokens,
+            contextFitStatus: "unavailable",
+            contextBudgetUtilizationPercent: null,
+            evaluationStatus: "unavailable",
+            correctness: { ...UNAVAILABLE_CORRECTNESS },
+            successEvidence: deriveSuccessEvidence({ contextFitStatus: "unavailable", correctness: UNAVAILABLE_CORRECTNESS }),
+        };
+    }
+    const fit = classifyContextFit(args.contextEstimatedTokens, args.contextBudgetTokens);
+    const utilization = calculateContextBudgetUtilizationPercent(args.contextEstimatedTokens, args.contextBudgetTokens);
+    if (fit === "context-too-large") {
+        return {
+            contextBudgetTokens: args.contextBudgetTokens,
+            contextFitStatus: fit,
+            contextBudgetUtilizationPercent: utilization,
+            evaluationStatus: "not-evaluated-context-too-large",
+            correctness: { ...UNAVAILABLE_CORRECTNESS },
+            successEvidence: deriveSuccessEvidence({ contextFitStatus: fit, correctness: UNAVAILABLE_CORRECTNESS }),
+        };
+    }
+    const correctness = args.shared ?? { ...UNAVAILABLE_CORRECTNESS };
+    return {
+        contextBudgetTokens: args.contextBudgetTokens,
+        contextFitStatus: fit,
+        contextBudgetUtilizationPercent: utilization,
+        evaluationStatus: correctness.availability === "available" ? "evaluated" : "unavailable",
+        correctness: { ...correctness },
+        successEvidence: deriveSuccessEvidence({ contextFitStatus: fit, correctness }),
+    };
+}

package/dist/src/experiments/plugins/contextWindowScaling/types.js

@@ -0,0 +1 @@
+export {};

package/dist/src/experiments/plugins/index.js

@@ -1,3 +1,4 @@
 export * from "./contextStrategyComparison/index.js";
+export * from "./contextWindowScaling/index.js";
 export * from "./incrementalChangeStaleness/index.js";
 export * from "./warmIndexReuse/index.js";

package/dist/src/experiments/plugins/warmIndexReuse/agentEvaluation.js

@@ -41,7 +41,7 @@ export async function evaluateWarmIndexFakeAgents(options) {
     }
     return result;
 }
-async function evaluateFakeSide(args) {
+export async function evaluateFakeSide(args) {
     const promptStrategy = SIDE_STRATEGIES[args.variantId];
     const base = { variantId: args.variantId, agentId: "fake-agent", promptStrategy };
     try {

package/dist/src/plots/buildContextWindowScalingPlotData.js

@@ -0,0 +1,96 @@
+import { existsSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { CONTEXT_WINDOW_SCALING_EXECUTION_ARTIFACT_FILE } from "../experiments/plugins/contextWindowScaling/executionArtifact.js";
+import { parseContextWindowScalingExecutionArtifact } from "../experiments/plugins/contextWindowScaling/executionArtifactReader.js";
+import { aggregateContextWindowScaling } from "../experiments/plugins/contextWindowScaling/metrics.js";
+import { STANDARD_CONTEXT_BUDGETS } from "../experiments/plugins/contextWindowScaling/contextBudget.js";
+// ---------------------------------------------------------------------------
+// Context-window-scaling plot data. Maps values already calculated by
+// aggregateContextWindowScaling() (carried on ContextWindowScalingRun.aggregate) to plot
+// points. Nothing is recalculated; a null aggregate value becomes a skipped point, never 0.
+// ---------------------------------------------------------------------------
+export const CONTEXT_WINDOW_SCALING_PLOT_IDS = [
+    "context-window-scaling-context-size",
+    "context-window-scaling-success-rate-by-budget",
+    "context-window-scaling-correctness-by-budget",
+];
+export const CONTEXT_WINDOW_SCALING_SKIP_REASONS = {
+    contextSize: "context-size-unavailable",
+    successRate: "success-rate-unavailable",
+    correctness: "correctness-unavailable",
+};
+/** Standard budgets read as 8k/16k/32k/64k; any other budget keeps its exact integer form. */
+export function formatContextBudgetLabel(budget) {
+    return STANDARD_CONTEXT_BUDGETS.includes(budget) ? `${budget / 1024}k` : String(budget);
+}
+/**
+ * Returns the aggregate rebuilt from `<experimentDir>/context-window-scaling-execution.json`, or null
+ * when that artifact is absent. A present artifact is validated (schema version and structure) and
+ * aggregated by the canonical aggregateContextWindowScaling(); malformed input fails, never falls back.
+ */
+export async function readContextWindowScalingPlotSource(experimentDir) {
+    const artifactPath = path.join(experimentDir, CONTEXT_WINDOW_SCALING_EXECUTION_ARTIFACT_FILE);
+    if (!existsSync(artifactPath))
+        return null;
+    let parsed;
+    try {
+        parsed = JSON.parse(await readFile(artifactPath, "utf8"));
+    }
+    catch (error) {
+        throw new Error(`Invalid context-window-scaling execution artifact (${artifactPath}): not valid JSON (${error instanceof Error ? error.message : String(error)}).`);
+    }
+    const artifact = parseContextWindowScalingExecutionArtifact(parsed);
+    return {
+        aggregate: aggregateContextWindowScaling({ contextBudgets: artifact.contextBudgets, cases: artifact.cases }),
+        generatedAt: artifact.completedAt,
+    };
+}
+export function buildContextWindowScalingPlotData(args) {
+    const { aggregate } = args;
+    const skippedPoints = [];
+    const series = (id, title, xLabel, yLabel) => ({
+        id,
+        title,
+        xLabel,
+        yLabel,
+        kind: "scatter",
+        points: [],
+        warnings: [],
+    });
+    const contextSize = series(CONTEXT_WINDOW_SCALING_PLOT_IDS[0], "Raw vs Retrieved Context Size", "Case ordinal (canonical case order)", "Estimated context tokens");
+    const successRate = series(CONTEXT_WINDOW_SCALING_PLOT_IDS[1], "Success Rate by Context Budget", "Context budget (estimated tokens)", "Success rate (percent)");
+    const correctness = series(CONTEXT_WINDOW_SCALING_PLOT_IDS[2], "Correctness by Context Budget", "Context budget (estimated tokens)", "Mean correctness score");
+    const add = (plot, x, value, group, label, reason, metadata) => {
+        if (value !== null && Number.isFinite(value)) {
+            plot.points.push({ x, y: value, group, label, metadata });
+        }
+        else {
+            skippedPoints.push({ plotId: plot.id, label, reason });
+        }
+    };
+    // Case ordinal follows the aggregate's canonical case order (first appearance of each caseId).
+    const caseOrdinals = new Map();
+    for (const summary of aggregate.caseTreatmentContextSummaries) {
+        if (!caseOrdinals.has(summary.caseId))
+            caseOrdinals.set(summary.caseId, caseOrdinals.size + 1);
+    }
+    for (const summary of aggregate.caseTreatmentContextSummaries) {
+        add(contextSize, caseOrdinals.get(summary.caseId), summary.contextStatus === "available" ? summary.estimatedTokens : null, summary.variantId, `${summary.caseId} ${summary.variantId}`, CONTEXT_WINDOW_SCALING_SKIP_REASONS.contextSize, { caseId: summary.caseId });
+    }
+    for (const summary of aggregate.budgetTreatmentSummaries) {
+        const label = `${summary.variantId} ${formatContextBudgetLabel(summary.contextBudgetTokens)}`;
+        const metadata = { contextBudgetTokens: summary.contextBudgetTokens };
+        add(successRate, summary.contextBudgetTokens, summary.successRatePercent, summary.variantId, label, CONTEXT_WINDOW_SCALING_SKIP_REASONS.successRate, metadata);
+        add(correctness, summary.contextBudgetTokens, summary.meanCorrectnessScore, summary.variantId, label, CONTEXT_WINDOW_SCALING_SKIP_REASONS.correctness, metadata);
+    }
+    const plots = [contextSize, successRate, correctness];
+    const warnings = [];
+    for (const plot of plots) {
+        if (plot.points.length === 0) {
+            plot.warnings.push("No comparable data available.");
+            warnings.push(`${plot.title}: no comparable data available.`);
+        }
+    }
+    return { generatedAt: args.generatedAt, sourceExperimentDir: args.experimentDir, plots, skippedPoints, warnings };
+}

package/dist/src/plots/index.js

@@ -1,5 +1,6 @@
 export * from "./types.js";
 export * from "./buildExperimentPlotData.js";
 export * from "./buildWarmIndexPlotData.js";
+export * from "./buildContextWindowScalingPlotData.js";
 export * from "./renderSvgChart.js";
 export * from "./writePlotArtifacts.js";