@dailephd/my-dev-kit-lab 0.6.3 → 0.7.1

package/dist/src/experiments/plugins/contextWindowScaling/evaluation.js

@@ -0,0 +1,36 @@
+import path from "node:path";
+import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { evaluateFakeSide } from "../warmIndexReuse/agentEvaluation.js";
+import { taskOutputSegment } from "../warmIndexReuse/selection.js";
+import { toCorrectnessEvidence } from "./successEvidence.js";
+/**
+ * Reuses the single existing fake-agent pipeline (prompt owner, fake agent, parseAgentAnswer,
+ * classifyAgentRunOutcome, scoreCorrectness) owned by warm-index agent evaluation. The pipeline
+ * derives its prompt from the case and never receives a budget, so the result is identical for
+ * every budget the treatment context fits.
+ */
+export const evaluateTreatmentWithFakeAgent = async (args) => {
+    const side = await evaluateFakeSide({
+        evaluationCase: args.evaluationCase,
+        caseId: args.evaluationCase.id,
+        // The shared pipeline's side ids map to prompt strategies; the guided treatment uses the
+        // my-dev-kit-guided strategy. The side id is internal and is not persisted as a variant.
+        variantId: args.treatment === "raw-full-file" ? "raw-full-file" : "warm-index-reuse",
+        projectProfiles: args.projectProfiles,
+        outDir: resolveWithinRoot(args.outputRoot, path.join("agents", taskOutputSegment(args.evaluationCase.id), args.treatment)),
+        cwd: args.cwd,
+        env: args.env,
+    });
+    return {
+        agentId: "fake-agent",
+        agentStatus: side.status,
+        correctness: toCorrectnessEvidence({
+            available: side.correctness.available,
+            score: side.correctness.score,
+            passed: side.correctness.passed,
+        }),
+        failureReasons: [...side.correctness.failureReasons],
+        warnings: [...side.warnings],
+        errors: [...side.errors],
+    };
+};

package/dist/src/experiments/plugins/contextWindowScaling/execution.js

@@ -0,0 +1,187 @@
+import path from "node:path";
+import { tokenCountMethod } from "../../../core/countTokens.js";
+import { runMyDevKitRetrieval } from "../../../evaluation/runMyDevKitRetrieval.js";
+import { runRawFullFileBaseline } from "../../../evaluation/runRawFullFileBaseline.js";
+import { taskOutputSegment } from "../warmIndexReuse/selection.js";
+import { evaluateTreatmentWithFakeAgent } from "./evaluation.js";
+import { CONTEXT_WINDOW_SCALING_TREATMENT_IDS } from "./metadata.js";
+import { computeRelevantFileEvidence } from "./relevantFiles.js";
+import { buildBudgetCell, UNAVAILABLE_CORRECTNESS } from "./successEvidence.js";
+export const defaultContextWindowScalingDependencies = {
+    constructRawContext: (evaluationCase) => runRawFullFileBaseline(evaluationCase),
+    constructGuidedContext: ({ evaluationCase, kitCommand, outputDir }) => runMyDevKitRetrieval({ evaluationCase, kitCommand, outputDir, requireKit: false }),
+    evaluateTreatment: evaluateTreatmentWithFakeAgent,
+};
+/**
+ * Per case: construct each treatment context once, then evaluate that one context against every
+ * budget. A deterministic evaluation runs at most once per treatment and is shared by the budgets
+ * the context fits. Nothing is truncated, retried, or rebuilt per budget.
+ */
+export async function executeContextWindowScalingCases(args) {
+    const deps = { ...defaultContextWindowScalingDependencies, ...args.dependencies };
+    const results = [];
+    for (const evaluationCase of args.cases) {
+        const treatments = [];
+        for (const variantId of CONTEXT_WINDOW_SCALING_TREATMENT_IDS) {
+            treatments.push(await executeTreatment({ ...args, evaluationCase, variantId, deps }));
+        }
+        results.push({
+            caseId: evaluationCase.id,
+            caseName: evaluationCase.title,
+            benchmarkProject: evaluationCase.benchmarkProject,
+            targetRoot: evaluationCase.targetRoot,
+            taskLocality: evaluationCase.taskLocality ?? null,
+            treatments,
+        });
+    }
+    return results;
+}
+async function executeTreatment(args) {
+    const { evaluationCase, variantId, deps } = args;
+    const constructed = await constructContext(args);
+    const tokens = constructed.context.estimatedTokens;
+    const expectedFiles = evaluationCase.answerKey?.expectedFiles ?? evaluationCase.expectedFiles;
+    const relevantFileEvidence = computeRelevantFileEvidence({
+        expectedFiles,
+        observedFiles: constructed.context.observedFiles,
+    });
+    let evaluation;
+    const anyFits = tokens !== null && args.contextBudgets.some((budget) => tokens <= budget);
+    if (tokens === null) {
+        evaluation = emptyEvaluation("unavailable", "Treatment context is unavailable.");
+    }
+    else if (!anyFits) {
+        evaluation = emptyEvaluation("not-evaluated-no-fitting-budget", "The context is too large for every selected budget.");
+    }
+    else {
+        evaluation = await evaluateOnce(args, deps);
+    }
+    const budgetCells = args.contextBudgets.map((contextBudgetTokens) => buildBudgetCell({
+        contextBudgetTokens,
+        contextEstimatedTokens: tokens,
+        shared: evaluation.status === "evaluated" || evaluation.status === "unavailable" ? evaluation.correctness : null,
+    }));
+    return {
+        variantId,
+        status: treatmentStatus(constructed, budgetCells.map((cell) => cell.contextFitStatus === "fits" && cell.evaluationStatus === "unavailable")),
+        context: constructed.context,
+        evaluation,
+        relevantFileEvidence,
+        budgetCells,
+        errors: constructed.error ? [constructed.error] : [],
+    };
+}
+async function evaluateOnce(args, deps) {
+    try {
+        const result = await deps.evaluateTreatment({
+            evaluationCase: args.evaluationCase,
+            treatment: args.variantId,
+            projectProfiles: args.projectProfiles,
+            outputRoot: args.outputRoot,
+            cwd: args.cwd,
+            env: args.env,
+        });
+        const available = result.correctness.availability === "available";
+        return {
+            status: available ? "evaluated" : "unavailable",
+            agentId: result.agentId,
+            agentStatus: result.agentStatus,
+            evaluationCount: 1,
+            correctness: result.correctness,
+            failureReasons: result.failureReasons,
+            warnings: [...result.warnings, ...result.errors],
+            reason: available ? null : `Deterministic evaluation was not scoreable (agent status ${result.agentStatus}).`,
+        };
+    }
+    catch (error) {
+        return {
+            ...emptyEvaluation("unavailable", error instanceof Error ? error.message : String(error)),
+            evaluationCount: 1,
+        };
+    }
+}
+function emptyEvaluation(status, reason) {
+    return {
+        status,
+        agentId: null,
+        agentStatus: null,
+        evaluationCount: 0,
+        correctness: { ...UNAVAILABLE_CORRECTNESS },
+        failureReasons: [],
+        warnings: [],
+        reason,
+    };
+}
+function treatmentStatus(constructed, fittingCellUnavailable) {
+    if (constructed.error)
+        return "failed";
+    if (constructed.skipped)
+        return "skipped";
+    return fittingCellUnavailable.some(Boolean) ? "partial" : "completed";
+}
+async function constructContext(args) {
+    const { evaluationCase, variantId, deps } = args;
+    try {
+        if (variantId === "raw-full-file") {
+            const raw = await deps.constructRawContext(evaluationCase);
+            return {
+                skipped: false,
+                error: null,
+                context: {
+                    status: "available",
+                    characterCount: raw.totalChars,
+                    estimatedTokens: raw.totalEstimatedTokens,
+                    tokenCountMethod: raw.tokenCountMethod,
+                    observedFiles: [...raw.filesIncluded],
+                    warnings: [],
+                    reason: null,
+                },
+            };
+        }
+        const guided = await deps.constructGuidedContext({
+            evaluationCase,
+            kitCommand: args.kitCommand,
+            outputDir: path.join(args.outputRoot, "guided", taskOutputSegment(evaluationCase.id)),
+        });
+        if (guided.skipped) {
+            return {
+                skipped: true,
+                error: null,
+                context: unavailableContext(guided.warnings, "my-dev-kit-guided retrieval was skipped and produced no context."),
+            };
+        }
+        return {
+            skipped: false,
+            error: null,
+            context: {
+                status: "available",
+                characterCount: guided.totalChars,
+                estimatedTokens: guided.totalEstimatedTokens,
+                tokenCountMethod: guided.tokenCountMethod,
+                observedFiles: [...guided.filesRead],
+                warnings: [...guided.warnings],
+                reason: null,
+            },
+        };
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        return {
+            skipped: false,
+            error: { code: "context-construction-failed", message },
+            context: unavailableContext([], message),
+        };
+    }
+}
+function unavailableContext(warnings, reason) {
+    return {
+        status: "unavailable",
+        characterCount: null,
+        estimatedTokens: null,
+        tokenCountMethod: null,
+        observedFiles: null,
+        warnings: [...warnings],
+        reason,
+    };
+}
+export { tokenCountMethod as CONTEXT_WINDOW_SCALING_TOKEN_COUNT_METHOD };

package/dist/src/experiments/plugins/contextWindowScaling/executionArtifact.js

@@ -0,0 +1,15 @@
+export const CONTEXT_WINDOW_SCALING_EXECUTION_ARTIFACT_FILE = "context-window-scaling-execution.json";
+export const CONTEXT_WINDOW_SCALING_EXECUTION_SCHEMA_VERSION = "my-dev-kit-lab-context-window-scaling-execution-v1";
+export function buildContextWindowScalingExecutionArtifact(args) {
+    return {
+        schemaVersion: CONTEXT_WINDOW_SCALING_EXECUTION_SCHEMA_VERSION,
+        runId: args.runId,
+        pluginId: args.pluginId,
+        pluginSchemaVersion: args.pluginSchemaVersion,
+        startedAt: args.startedAt,
+        completedAt: args.completedAt,
+        estimator: { tokenCountMethod: args.tokenCountMethod },
+        contextBudgets: [...args.contextBudgets],
+        cases: structuredClone([...args.cases]),
+    };
+}

package/dist/src/experiments/plugins/contextWindowScaling/executionArtifactReader.js

@@ -0,0 +1,186 @@
+import { validateContextBudgets } from "./config.js";
+import { CONTEXT_WINDOW_SCALING_EXECUTION_SCHEMA_VERSION, } from "./executionArtifact.js";
+import { CONTEXT_WINDOW_SCALING_PLUGIN_ID, CONTEXT_WINDOW_SCALING_TREATMENT_IDS } from "./metadata.js";
+const MAX_REPORTED_PROBLEMS = 8;
+const RUN_STATUSES = ["completed", "partial", "failed", "skipped"];
+const AVAILABILITY = ["available", "unavailable"];
+const EVALUATION_STATUSES = ["evaluated", "not-evaluated-no-fitting-budget", "unavailable"];
+const RELEVANT_STATUSES = ["available", "unavailable", "not-applicable"];
+const FIT_STATUSES = ["fits", "context-too-large", "unavailable"];
+const CELL_EVALUATION_STATUSES = ["evaluated", "not-evaluated-context-too-large", "unavailable"];
+const SUCCESS_REASONS = ["correctness-pass", "correctness-fail", "context-too-large", "evaluation-unavailable", "context-unavailable"];
+const isObj = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
+const isStr = (v) => typeof v === "string";
+const isStrOrNull = (v) => v === null || typeof v === "string";
+const isCount = (v) => typeof v === "number" && Number.isSafeInteger(v) && v >= 0;
+const isCountOrNull = (v) => v === null || isCount(v);
+const isNumOrNull = (v) => v === null || (typeof v === "number" && Number.isFinite(v));
+const isStrArray = (v) => Array.isArray(v) && v.every((item) => typeof item === "string");
+const isOneOf = (v, allowed) => typeof v === "string" && allowed.includes(v);
+/**
+ * Structural validator for the persisted V1 execution artifact. It checks only what aggregation and
+ * plotting need (closed enums, types, budget/cell agreement); it never repairs or coerces values.
+ * Throws one Error listing bounded problems; returns the input, typed, when valid.
+ */
+export function parseContextWindowScalingExecutionArtifact(value) {
+    if (!isObj(value)) {
+        throw new Error("Invalid context-window-scaling execution artifact: expected a JSON object.");
+    }
+    if (value.schemaVersion !== CONTEXT_WINDOW_SCALING_EXECUTION_SCHEMA_VERSION) {
+        throw new Error(`Invalid context-window-scaling execution artifact: unsupported schemaVersion ${JSON.stringify(value.schemaVersion)}; expected ${CONTEXT_WINDOW_SCALING_EXECUTION_SCHEMA_VERSION}.`);
+    }
+    const problems = [];
+    const fail = (message) => {
+        problems.push(message);
+    };
+    const checkCorrectness = (correctness, at) => {
+        if (!isObj(correctness) ||
+            !isOneOf(correctness.availability, AVAILABILITY) ||
+            !isNumOrNull(correctness.score) ||
+            !(correctness.pass === null || typeof correctness.pass === "boolean")) {
+            fail(`${at} is malformed.`);
+        }
+    };
+    for (const field of ["runId", "pluginSchemaVersion", "startedAt", "completedAt"]) {
+        if (!isStr(value[field]))
+            fail(`${field} must be a string.`);
+    }
+    if (value.pluginId !== CONTEXT_WINDOW_SCALING_PLUGIN_ID)
+        fail(`pluginId must be ${CONTEXT_WINDOW_SCALING_PLUGIN_ID}.`);
+    if (!isObj(value.estimator) || !isStr(value.estimator.tokenCountMethod))
+        fail("estimator.tokenCountMethod must be a string.");
+    let budgets = [];
+    const budgetCheck = validateContextBudgets(value.contextBudgets);
+    if (budgetCheck.errors.length > 0) {
+        fail(`contextBudgets: ${budgetCheck.errors.join(" ")}`);
+    }
+    else if (budgetCheck.budgets.some((budget, index) => budget !== value.contextBudgets[index])) {
+        fail("contextBudgets must be in ascending order.");
+    }
+    else {
+        budgets = budgetCheck.budgets;
+    }
+    const cases = value.cases;
+    if (!Array.isArray(cases) || cases.length === 0) {
+        fail("cases must be a non-empty array.");
+    }
+    else {
+        cases.forEach((caseEvidence, caseIndex) => {
+            const at = `cases[${caseIndex}]`;
+            if (!isObj(caseEvidence))
+                return fail(`${at} must be an object.`);
+            for (const field of ["caseId", "caseName", "benchmarkProject", "targetRoot"]) {
+                if (!isStr(caseEvidence[field]))
+                    fail(`${at}.${field} must be a string.`);
+            }
+            if (!isStrOrNull(caseEvidence.taskLocality))
+                fail(`${at}.taskLocality must be a string or null.`);
+            const treatments = caseEvidence.treatments;
+            if (!Array.isArray(treatments))
+                return fail(`${at}.treatments must be an array.`);
+            const ids = treatments.map((t) => (isObj(t) ? t.variantId : undefined));
+            if (ids.length !== CONTEXT_WINDOW_SCALING_TREATMENT_IDS.length ||
+                CONTEXT_WINDOW_SCALING_TREATMENT_IDS.some((id) => ids.filter((x) => x === id).length !== 1)) {
+                fail(`${at}.treatments must contain exactly ${CONTEXT_WINDOW_SCALING_TREATMENT_IDS.join(" and ")}.`);
+            }
+            treatments.forEach((treatment, treatmentIndex) => {
+                const tat = `${at}.treatments[${treatmentIndex}]`;
+                if (!isObj(treatment))
+                    return fail(`${tat} must be an object.`);
+                if (!isOneOf(treatment.status, RUN_STATUSES))
+                    fail(`${tat}.status is not a known status.`);
+                const context = treatment.context;
+                if (!isObj(context)) {
+                    fail(`${tat}.context must be an object.`);
+                }
+                else {
+                    if (!isOneOf(context.status, AVAILABILITY))
+                        fail(`${tat}.context.status is not a known status.`);
+                    if (!isCountOrNull(context.characterCount) || !isCountOrNull(context.estimatedTokens)) {
+                        fail(`${tat}.context counts must be nonnegative integers or null.`);
+                    }
+                    if (!isStrOrNull(context.tokenCountMethod) || !isStrOrNull(context.reason))
+                        fail(`${tat}.context strings must be strings or null.`);
+                    if (!(context.observedFiles === null || isStrArray(context.observedFiles)) || !isStrArray(context.warnings)) {
+                        fail(`${tat}.context file/warning lists are malformed.`);
+                    }
+                    if (context.status === "available" && context.estimatedTokens === null)
+                        fail(`${tat}.context is available without estimatedTokens.`);
+                }
+                const evaluation = treatment.evaluation;
+                if (!isObj(evaluation)) {
+                    fail(`${tat}.evaluation must be an object.`);
+                }
+                else {
+                    if (!isOneOf(evaluation.status, EVALUATION_STATUSES))
+                        fail(`${tat}.evaluation.status is not a known status.`);
+                    if (!(evaluation.agentId === null || evaluation.agentId === "fake-agent"))
+                        fail(`${tat}.evaluation.agentId must be fake-agent or null.`);
+                    if (!isStrOrNull(evaluation.agentStatus) || !isStrOrNull(evaluation.reason))
+                        fail(`${tat}.evaluation strings must be strings or null.`);
+                    if (!isCount(evaluation.evaluationCount))
+                        fail(`${tat}.evaluation.evaluationCount must be a nonnegative integer.`);
+                    if (!isStrArray(evaluation.failureReasons) || !isStrArray(evaluation.warnings))
+                        fail(`${tat}.evaluation lists must be string arrays.`);
+                    checkCorrectness(evaluation.correctness, `${tat}.evaluation.correctness`);
+                }
+                const relevant = treatment.relevantFileEvidence;
+                if (!isObj(relevant)) {
+                    fail(`${tat}.relevantFileEvidence must be an object.`);
+                }
+                else {
+                    if (!isOneOf(relevant.status, RELEVANT_STATUSES))
+                        fail(`${tat}.relevantFileEvidence.status is not a known status.`);
+                    if (!isStrArray(relevant.expectedRelevantFiles) || !isStrArray(relevant.omittedRelevantFiles)) {
+                        fail(`${tat}.relevantFileEvidence file lists must be string arrays.`);
+                    }
+                    if (!isCount(relevant.expectedRelevantFileCount))
+                        fail(`${tat}.relevantFileEvidence.expectedRelevantFileCount must be a nonnegative integer.`);
+                    if (!isCountOrNull(relevant.observedExpectedFileCount) || !isCountOrNull(relevant.omittedRelevantFileCount)) {
+                        fail(`${tat}.relevantFileEvidence counts must be nonnegative integers or null.`);
+                    }
+                    if (!isStrOrNull(relevant.reason))
+                        fail(`${tat}.relevantFileEvidence.reason must be a string or null.`);
+                }
+                const cells = treatment.budgetCells;
+                if (!Array.isArray(cells)) {
+                    fail(`${tat}.budgetCells must be an array.`);
+                }
+                else {
+                    const cellBudgets = cells.map((cell) => (isObj(cell) ? cell.contextBudgetTokens : undefined));
+                    if (budgets.length > 0 && (cells.length !== budgets.length || budgets.some((b) => cellBudgets.filter((x) => x === b).length !== 1))) {
+                        fail(`${tat}.budgetCells do not match contextBudgets.`);
+                    }
+                    cells.forEach((cell, cellIndex) => {
+                        const cat = `${tat}.budgetCells[${cellIndex}]`;
+                        if (!isObj(cell))
+                            return fail(`${cat} must be an object.`);
+                        if (!isOneOf(cell.contextFitStatus, FIT_STATUSES))
+                            fail(`${cat}.contextFitStatus is not a known status.`);
+                        if (!isNumOrNull(cell.contextBudgetUtilizationPercent))
+                            fail(`${cat}.contextBudgetUtilizationPercent must be a number or null.`);
+                        if (!isOneOf(cell.evaluationStatus, CELL_EVALUATION_STATUSES))
+                            fail(`${cat}.evaluationStatus is not a known status.`);
+                        checkCorrectness(cell.correctness, `${cat}.correctness`);
+                        const success = cell.successEvidence;
+                        if (!isObj(success) ||
+                            !isOneOf(success.status, AVAILABILITY) ||
+                            !(success.success === null || typeof success.success === "boolean") ||
+                            !isOneOf(success.reason, SUCCESS_REASONS)) {
+                            fail(`${cat}.successEvidence is malformed.`);
+                        }
+                    });
+                }
+                if (!Array.isArray(treatment.errors) || !treatment.errors.every((e) => isObj(e) && isStr(e.code) && isStr(e.message))) {
+                    fail(`${tat}.errors must be an array of { code, message }.`);
+                }
+            });
+        });
+    }
+    if (problems.length > 0) {
+        const shown = problems.slice(0, MAX_REPORTED_PROBLEMS);
+        const extra = problems.length - shown.length;
+        throw new Error(`Invalid context-window-scaling execution artifact: ${shown.join(" ")}${extra > 0 ? ` (${extra} more problem(s) omitted.)` : ""}`);
+    }
+    return value;
+}

package/dist/src/experiments/plugins/contextWindowScaling/index.js

@@ -0,0 +1,15 @@
+export * from "./config.js";
+export * from "./cliBudgets.js";
+export * from "./contextBudget.js";
+export * from "./evaluation.js";
+export * from "./execution.js";
+export * from "./executionArtifact.js";
+export * from "./executionArtifactReader.js";
+export * from "./metadata.js";
+export * from "./metrics.js";
+export * from "./plugin.js";
+export * from "./projectProfile.js";
+export * from "./relevantFiles.js";
+export * from "./successEvidence.js";
+export * from "./types.js";
+export * from "./syntheticInputs.js";

package/dist/src/experiments/plugins/contextWindowScaling/metadata.js

@@ -0,0 +1,12 @@
+export const CONTEXT_WINDOW_SCALING_PLUGIN_ID = "context-window-scaling";
+/** Stable strategy IDs shared with existing plugins; budget is never part of variant identity. */
+export const CONTEXT_WINDOW_SCALING_TREATMENT_IDS = ["raw-full-file", "my-dev-kit-guided"];
+export const contextWindowScalingMetadata = {
+    id: CONTEXT_WINDOW_SCALING_PLUGIN_ID,
+    name: "Context Window Scaling",
+    description: "Measure raw-full-file and my-dev-kit-guided strategies under increasing deterministic estimated-context-token budgets, over the bundled fixed scaling corpus or caller-supplied deterministic synthetic repositories (--synthetic-config).",
+    schemaVersion: "1.0.0",
+    status: "experimental",
+    supportedTargets: ["self"],
+    supportedOutputs: ["json", "text", "html", "plot"],
+};

package/dist/src/experiments/plugins/contextWindowScaling/metrics.js

@@ -0,0 +1,149 @@
+import { CONTEXT_WINDOW_SCALING_TREATMENT_IDS } from "./metadata.js";
+export function aggregateContextWindowScaling(args) {
+    const budgets = [...args.contextBudgets].sort((a, b) => a - b);
+    const treatments = [...CONTEXT_WINDOW_SCALING_TREATMENT_IDS];
+    const treatmentsOf = (variantId) => args.cases.map((caseEvidence) => ({ caseEvidence, treatment: requireTreatment(caseEvidence, variantId, budgets) }));
+    const budgetTreatmentSummaries = [];
+    for (const contextBudgetTokens of budgets) {
+        for (const variantId of treatments) {
+            const cells = treatmentsOf(variantId).map(({ treatment }) => cellFor(treatment, contextBudgetTokens));
+            budgetTreatmentSummaries.push(summarizeCells(contextBudgetTokens, variantId, cells));
+        }
+    }
+    const caseTreatmentContextSummaries = args.cases.flatMap((caseEvidence) => treatments.map((variantId) => {
+        const treatment = requireTreatment(caseEvidence, variantId, budgets);
+        return {
+            caseId: caseEvidence.caseId,
+            caseName: caseEvidence.caseName,
+            variantId,
+            contextStatus: treatment.context.status,
+            characterCount: treatment.context.characterCount,
+            estimatedTokens: treatment.context.estimatedTokens,
+            tokenCountMethod: treatment.context.tokenCountMethod,
+            observedFileCount: treatment.context.observedFiles === null ? null : treatment.context.observedFiles.length,
+            relevantFileEvidence: structuredClone(treatment.relevantFileEvidence),
+        };
+    }));
+    const allCells = budgetTreatmentSummaries;
+    const relevant = caseTreatmentContextSummaries.map((summary) => summary.relevantFileEvidence);
+    const relevantCount = (status) => relevant.filter((evidence) => evidence.status === status).length;
+    const relevantFileSummary = {
+        relevantFileEvidenceAvailableCount: relevantCount("available"),
+        relevantFileEvidenceUnavailableCount: relevantCount("unavailable"),
+        relevantFileEvidenceNotApplicableCount: relevantCount("not-applicable"),
+        totalOmittedRelevantFileCount: relevant.reduce((sum, evidence) => sum + (evidence.omittedRelevantFileCount ?? 0), 0),
+    };
+    const sumOf = (pick) => allCells.reduce((sum, summary) => sum + pick(summary), 0);
+    const evaluatedBudgetCellCount = args.cases
+        .flatMap((caseEvidence) => caseEvidence.treatments)
+        .flatMap((treatment) => treatment.budgetCells)
+        .filter((cell) => cell.evaluationStatus === "evaluated").length;
+    return {
+        budgets,
+        treatments,
+        runSummary: {
+            caseCount: args.cases.length,
+            treatmentCount: treatments.length,
+            budgetCount: budgets.length,
+            totalBudgetCellCount: sumOf((s) => s.totalCellCount),
+            evaluatedBudgetCellCount,
+            contextTooLargeCellCount: sumOf((s) => s.contextTooLargeCount),
+            contextUnavailableCellCount: sumOf((s) => s.contextUnavailableCount),
+            correctnessAvailableCellCount: sumOf((s) => s.correctnessAvailableCount),
+            correctnessUnavailableCellCount: sumOf((s) => s.correctnessUnavailableCount),
+            successEvidenceAvailableCellCount: sumOf((s) => s.successEvidenceAvailableCount),
+            successEvidenceUnavailableCellCount: sumOf((s) => s.successEvidenceUnavailableCount),
+            relevantFileEvidenceAvailableTreatmentCount: relevantFileSummary.relevantFileEvidenceAvailableCount,
+            relevantFileEvidenceUnavailableTreatmentCount: relevantFileSummary.relevantFileEvidenceUnavailableCount,
+            relevantFileEvidenceNotApplicableTreatmentCount: relevantFileSummary.relevantFileEvidenceNotApplicableCount,
+        },
+        budgetTreatmentSummaries,
+        caseTreatmentContextSummaries,
+        relevantFileSummary,
+    };
+}
+function requireTreatment(caseEvidence, variantId, budgets) {
+    const matches = caseEvidence.treatments.filter((treatment) => treatment.variantId === variantId);
+    const treatment = matches[0];
+    if (matches.length !== 1 || !treatment) {
+        throw new Error(`Invalid context-window-scaling evidence: case ${caseEvidence.caseId} must have exactly one ${variantId} treatment.`);
+    }
+    const cellBudgets = treatment.budgetCells.map((cell) => cell.contextBudgetTokens);
+    if (cellBudgets.length !== budgets.length || budgets.some((budget) => cellBudgets.filter((value) => value === budget).length !== 1)) {
+        throw new Error(`Invalid context-window-scaling evidence: case ${caseEvidence.caseId} ${variantId} cells do not match the selected budgets.`);
+    }
+    return treatment;
+}
+function cellFor(treatment, budget) {
+    return treatment.budgetCells.find((cell) => cell.contextBudgetTokens === budget);
+}
+function isFiniteNumber(value) {
+    return typeof value === "number" && Number.isFinite(value);
+}
+function summarizeCells(contextBudgetTokens, variantId, cells) {
+    const fitCount = cells.filter((cell) => cell.contextFitStatus === "fits").length;
+    const contextTooLargeCount = cells.filter((cell) => cell.contextFitStatus === "context-too-large").length;
+    const contextUnavailableCount = cells.filter((cell) => cell.contextFitStatus === "unavailable").length;
+    const successAvailable = cells.filter((cell) => cell.successEvidence.status === "available");
+    const successfulCellCount = successAvailable.filter((cell) => cell.successEvidence.success === true).length;
+    const scores = cells
+        .filter((cell) => cell.correctness.availability === "available")
+        .map((cell) => cell.correctness.score)
+        .filter(isFiniteNumber);
+    const utilizations = cells.map((cell) => cell.contextBudgetUtilizationPercent).filter(isFiniteNumber);
+    const mean = (values) => (values.length === 0 ? null : values.reduce((sum, value) => sum + value, 0) / values.length);
+    return {
+        contextBudgetTokens,
+        variantId,
+        totalCellCount: cells.length,
+        contextMeasuredCount: fitCount + contextTooLargeCount,
+        fitCount,
+        contextTooLargeCount,
+        contextUnavailableCount,
+        successfulCellCount,
+        notSuccessfulCellCount: successAvailable.length - successfulCellCount,
+        successEvidenceAvailableCount: successAvailable.length,
+        successEvidenceUnavailableCount: cells.length - successAvailable.length,
+        successRatePercent: successAvailable.length === 0 ? null : (successfulCellCount / successAvailable.length) * 100,
+        correctnessAvailableCount: scores.length,
+        correctnessUnavailableCount: cells.length - scores.length,
+        meanCorrectnessScore: mean(scores),
+        utilizationAvailableCount: utilizations.length,
+        utilizationUnavailableCount: cells.length - utilizations.length,
+        meanContextBudgetUtilizationPercent: mean(utilizations),
+        minContextBudgetUtilizationPercent: utilizations.length === 0 ? null : Math.min(...utilizations),
+        maxContextBudgetUtilizationPercent: utilizations.length === 0 ? null : Math.max(...utilizations),
+    };
+}
+/** Run-level scalar metrics: one set per budget x treatment, scoped by variantId and budget-bearing ids. */
+export function toAggregateExperimentMetrics(aggregate) {
+    const metrics = [
+        { id: "context-window-scaling-case-count", name: "Context-window-scaling case count", value: aggregate.runSummary.caseCount, unit: "count" },
+        { id: "context-window-scaling-budget-count", name: "Context budget count", value: aggregate.runSummary.budgetCount, unit: "count" },
+    ];
+    for (const summary of aggregate.budgetTreatmentSummaries) {
+        const prefix = `context-budget-${summary.contextBudgetTokens}`;
+        const scoped = (suffix, name, value, unit, description) => ({
+            id: `${prefix}-${suffix}`,
+            name: `${name} (${summary.contextBudgetTokens} token budget)`,
+            value,
+            unit,
+            ...(description ? { description } : {}),
+            variantId: summary.variantId,
+        });
+        metrics.push(scoped("fit-count", "Cases whose context fits", summary.fitCount, "count"), scoped("context-too-large-count", "Cases whose context is too large", summary.contextTooLargeCount, "count"), scoped("successful-cell-count", "Successful cells", summary.successfulCellCount, "count"), scoped("success-rate-percent", "Success rate", summary.successRatePercent, "percent", "Successful cells over success-evidence-available cells; null when none are available."), scoped("mean-correctness-score", "Mean correctness score", summary.meanCorrectnessScore, "score", "Mean of available existing correctness scores; unavailable correctness is excluded, not zero."), scoped("mean-context-budget-utilization-percent", "Mean context budget utilization", summary.meanContextBudgetUtilizationPercent, "percent", "Uncapped; may exceed 100."));
+    }
+    return metrics;
+}
+/** Per-outcome scalar metrics for one treatment, one set per budget cell. */
+export function toOutcomeBudgetMetrics(treatment, caseId) {
+    return treatment.budgetCells.flatMap((cell) => {
+        const base = { variantId: treatment.variantId, caseId };
+        const prefix = `budget-${cell.contextBudgetTokens}`;
+        return [
+            { id: `${prefix}-context-budget-utilization-percent`, name: `Context budget utilization (${cell.contextBudgetTokens})`, value: cell.contextBudgetUtilizationPercent, unit: "percent", ...base },
+            { id: `${prefix}-correctness-score`, name: `Correctness score (${cell.contextBudgetTokens})`, value: cell.correctness.score, unit: "score", ...base },
+            { id: `${prefix}-success`, name: `Success (${cell.contextBudgetTokens})`, value: cell.successEvidence.success, ...base },
+        ];
+    });
+}