@dailephd/my-dev-kit-lab 0.6.2 → 0.7.0

package/dist/src/experiments/plugins/contextWindowScaling/metrics.js

@@ -0,0 +1,149 @@
+import { CONTEXT_WINDOW_SCALING_TREATMENT_IDS } from "./metadata.js";
+export function aggregateContextWindowScaling(args) {
+    const budgets = [...args.contextBudgets].sort((a, b) => a - b);
+    const treatments = [...CONTEXT_WINDOW_SCALING_TREATMENT_IDS];
+    const treatmentsOf = (variantId) => args.cases.map((caseEvidence) => ({ caseEvidence, treatment: requireTreatment(caseEvidence, variantId, budgets) }));
+    const budgetTreatmentSummaries = [];
+    for (const contextBudgetTokens of budgets) {
+        for (const variantId of treatments) {
+            const cells = treatmentsOf(variantId).map(({ treatment }) => cellFor(treatment, contextBudgetTokens));
+            budgetTreatmentSummaries.push(summarizeCells(contextBudgetTokens, variantId, cells));
+        }
+    }
+    const caseTreatmentContextSummaries = args.cases.flatMap((caseEvidence) => treatments.map((variantId) => {
+        const treatment = requireTreatment(caseEvidence, variantId, budgets);
+        return {
+            caseId: caseEvidence.caseId,
+            caseName: caseEvidence.caseName,
+            variantId,
+            contextStatus: treatment.context.status,
+            characterCount: treatment.context.characterCount,
+            estimatedTokens: treatment.context.estimatedTokens,
+            tokenCountMethod: treatment.context.tokenCountMethod,
+            observedFileCount: treatment.context.observedFiles === null ? null : treatment.context.observedFiles.length,
+            relevantFileEvidence: structuredClone(treatment.relevantFileEvidence),
+        };
+    }));
+    const allCells = budgetTreatmentSummaries;
+    const relevant = caseTreatmentContextSummaries.map((summary) => summary.relevantFileEvidence);
+    const relevantCount = (status) => relevant.filter((evidence) => evidence.status === status).length;
+    const relevantFileSummary = {
+        relevantFileEvidenceAvailableCount: relevantCount("available"),
+        relevantFileEvidenceUnavailableCount: relevantCount("unavailable"),
+        relevantFileEvidenceNotApplicableCount: relevantCount("not-applicable"),
+        totalOmittedRelevantFileCount: relevant.reduce((sum, evidence) => sum + (evidence.omittedRelevantFileCount ?? 0), 0),
+    };
+    const sumOf = (pick) => allCells.reduce((sum, summary) => sum + pick(summary), 0);
+    const evaluatedBudgetCellCount = args.cases
+        .flatMap((caseEvidence) => caseEvidence.treatments)
+        .flatMap((treatment) => treatment.budgetCells)
+        .filter((cell) => cell.evaluationStatus === "evaluated").length;
+    return {
+        budgets,
+        treatments,
+        runSummary: {
+            caseCount: args.cases.length,
+            treatmentCount: treatments.length,
+            budgetCount: budgets.length,
+            totalBudgetCellCount: sumOf((s) => s.totalCellCount),
+            evaluatedBudgetCellCount,
+            contextTooLargeCellCount: sumOf((s) => s.contextTooLargeCount),
+            contextUnavailableCellCount: sumOf((s) => s.contextUnavailableCount),
+            correctnessAvailableCellCount: sumOf((s) => s.correctnessAvailableCount),
+            correctnessUnavailableCellCount: sumOf((s) => s.correctnessUnavailableCount),
+            successEvidenceAvailableCellCount: sumOf((s) => s.successEvidenceAvailableCount),
+            successEvidenceUnavailableCellCount: sumOf((s) => s.successEvidenceUnavailableCount),
+            relevantFileEvidenceAvailableTreatmentCount: relevantFileSummary.relevantFileEvidenceAvailableCount,
+            relevantFileEvidenceUnavailableTreatmentCount: relevantFileSummary.relevantFileEvidenceUnavailableCount,
+            relevantFileEvidenceNotApplicableTreatmentCount: relevantFileSummary.relevantFileEvidenceNotApplicableCount,
+        },
+        budgetTreatmentSummaries,
+        caseTreatmentContextSummaries,
+        relevantFileSummary,
+    };
+}
+function requireTreatment(caseEvidence, variantId, budgets) {
+    const matches = caseEvidence.treatments.filter((treatment) => treatment.variantId === variantId);
+    const treatment = matches[0];
+    if (matches.length !== 1 || !treatment) {
+        throw new Error(`Invalid context-window-scaling evidence: case ${caseEvidence.caseId} must have exactly one ${variantId} treatment.`);
+    }
+    const cellBudgets = treatment.budgetCells.map((cell) => cell.contextBudgetTokens);
+    if (cellBudgets.length !== budgets.length || budgets.some((budget) => cellBudgets.filter((value) => value === budget).length !== 1)) {
+        throw new Error(`Invalid context-window-scaling evidence: case ${caseEvidence.caseId} ${variantId} cells do not match the selected budgets.`);
+    }
+    return treatment;
+}
+function cellFor(treatment, budget) {
+    return treatment.budgetCells.find((cell) => cell.contextBudgetTokens === budget);
+}
+function isFiniteNumber(value) {
+    return typeof value === "number" && Number.isFinite(value);
+}
+function summarizeCells(contextBudgetTokens, variantId, cells) {
+    const fitCount = cells.filter((cell) => cell.contextFitStatus === "fits").length;
+    const contextTooLargeCount = cells.filter((cell) => cell.contextFitStatus === "context-too-large").length;
+    const contextUnavailableCount = cells.filter((cell) => cell.contextFitStatus === "unavailable").length;
+    const successAvailable = cells.filter((cell) => cell.successEvidence.status === "available");
+    const successfulCellCount = successAvailable.filter((cell) => cell.successEvidence.success === true).length;
+    const scores = cells
+        .filter((cell) => cell.correctness.availability === "available")
+        .map((cell) => cell.correctness.score)
+        .filter(isFiniteNumber);
+    const utilizations = cells.map((cell) => cell.contextBudgetUtilizationPercent).filter(isFiniteNumber);
+    const mean = (values) => (values.length === 0 ? null : values.reduce((sum, value) => sum + value, 0) / values.length);
+    return {
+        contextBudgetTokens,
+        variantId,
+        totalCellCount: cells.length,
+        contextMeasuredCount: fitCount + contextTooLargeCount,
+        fitCount,
+        contextTooLargeCount,
+        contextUnavailableCount,
+        successfulCellCount,
+        notSuccessfulCellCount: successAvailable.length - successfulCellCount,
+        successEvidenceAvailableCount: successAvailable.length,
+        successEvidenceUnavailableCount: cells.length - successAvailable.length,
+        successRatePercent: successAvailable.length === 0 ? null : (successfulCellCount / successAvailable.length) * 100,
+        correctnessAvailableCount: scores.length,
+        correctnessUnavailableCount: cells.length - scores.length,
+        meanCorrectnessScore: mean(scores),
+        utilizationAvailableCount: utilizations.length,
+        utilizationUnavailableCount: cells.length - utilizations.length,
+        meanContextBudgetUtilizationPercent: mean(utilizations),
+        minContextBudgetUtilizationPercent: utilizations.length === 0 ? null : Math.min(...utilizations),
+        maxContextBudgetUtilizationPercent: utilizations.length === 0 ? null : Math.max(...utilizations),
+    };
+}
+/** Run-level scalar metrics: one set per budget x treatment, scoped by variantId and budget-bearing ids. */
+export function toAggregateExperimentMetrics(aggregate) {
+    const metrics = [
+        { id: "context-window-scaling-case-count", name: "Context-window-scaling case count", value: aggregate.runSummary.caseCount, unit: "count" },
+        { id: "context-window-scaling-budget-count", name: "Context budget count", value: aggregate.runSummary.budgetCount, unit: "count" },
+    ];
+    for (const summary of aggregate.budgetTreatmentSummaries) {
+        const prefix = `context-budget-${summary.contextBudgetTokens}`;
+        const scoped = (suffix, name, value, unit, description) => ({
+            id: `${prefix}-${suffix}`,
+            name: `${name} (${summary.contextBudgetTokens} token budget)`,
+            value,
+            unit,
+            ...(description ? { description } : {}),
+            variantId: summary.variantId,
+        });
+        metrics.push(scoped("fit-count", "Cases whose context fits", summary.fitCount, "count"), scoped("context-too-large-count", "Cases whose context is too large", summary.contextTooLargeCount, "count"), scoped("successful-cell-count", "Successful cells", summary.successfulCellCount, "count"), scoped("success-rate-percent", "Success rate", summary.successRatePercent, "percent", "Successful cells over success-evidence-available cells; null when none are available."), scoped("mean-correctness-score", "Mean correctness score", summary.meanCorrectnessScore, "score", "Mean of available existing correctness scores; unavailable correctness is excluded, not zero."), scoped("mean-context-budget-utilization-percent", "Mean context budget utilization", summary.meanContextBudgetUtilizationPercent, "percent", "Uncapped; may exceed 100."));
+    }
+    return metrics;
+}
+/** Per-outcome scalar metrics for one treatment, one set per budget cell. */
+export function toOutcomeBudgetMetrics(treatment, caseId) {
+    return treatment.budgetCells.flatMap((cell) => {
+        const base = { variantId: treatment.variantId, caseId };
+        const prefix = `budget-${cell.contextBudgetTokens}`;
+        return [
+            { id: `${prefix}-context-budget-utilization-percent`, name: `Context budget utilization (${cell.contextBudgetTokens})`, value: cell.contextBudgetUtilizationPercent, unit: "percent", ...base },
+            { id: `${prefix}-correctness-score`, name: `Correctness score (${cell.contextBudgetTokens})`, value: cell.correctness.score, unit: "score", ...base },
+            { id: `${prefix}-success`, name: `Success (${cell.contextBudgetTokens})`, value: cell.successEvidence.success, ...base },
+        ];
+    });
+}

package/dist/src/experiments/plugins/contextWindowScaling/plugin.js

@@ -0,0 +1,185 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { tokenCountMethod } from "../../../core/countTokens.js";
+import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { summarizeExperimentRun } from "../../results.js";
+import { aggregateStatus } from "../warmIndexReuse/execution.js";
+import { contextWindowScalingConfigDefinition, defaultContextWindowScalingConfig, validateContextWindowScalingConfig, } from "./config.js";
+import { executeContextWindowScalingCases, } from "./execution.js";
+import { buildContextWindowScalingExecutionArtifact, CONTEXT_WINDOW_SCALING_EXECUTION_ARTIFACT_FILE, } from "./executionArtifact.js";
+import { CONTEXT_WINDOW_SCALING_TREATMENT_IDS, contextWindowScalingMetadata } from "./metadata.js";
+import { aggregateContextWindowScaling, toAggregateExperimentMetrics, toOutcomeBudgetMetrics, } from "./metrics.js";
+import { resolveScalingProjectProfiles } from "./projectProfile.js";
+const VARIANTS = [
+    {
+        id: "raw-full-file",
+        name: "Raw full file",
+        description: "Every file matched by the case's raw include globs, constructed once per case.",
+    },
+    {
+        id: "my-dev-kit-guided",
+        name: "my-dev-kit guided",
+        description: "my-dev-kit retrieval context, constructed once per case.",
+    },
+];
+/** Internal-only seam: tests inject construction/evaluation owners through inputs.dependencies. */
+export const contextWindowScalingPlugin = {
+    metadata: contextWindowScalingMetadata,
+    defaultConfig: defaultContextWindowScalingConfig,
+    configDefinition: contextWindowScalingConfigDefinition,
+    supportedVariants: CONTEXT_WINDOW_SCALING_TREATMENT_IDS,
+    validateConfig: validateContextWindowScalingConfig,
+    async run(context) {
+        const startedAt = context.startedAt.toISOString();
+        const cases = readCasesInput(context.inputs);
+        const projectProfiles = resolveScalingProjectProfiles(cases, readProjectProfilesInput(context.inputs));
+        const outDir = context.outputRoot ?? path.resolve(context.toolRoot, "lab-output", "context-window-scaling");
+        await mkdir(outDir, { recursive: true });
+        const executionEvidence = await executeContextWindowScalingCases({
+            cases,
+            contextBudgets: context.config.contextBudgets,
+            kitCommand: context.config.kitCommand,
+            outputRoot: outDir,
+            projectProfiles,
+            cwd: context.target.targetRoot,
+            env: readEnvInput(context.inputs),
+            dependencies: readDependenciesInput(context.inputs),
+        });
+        const completedAt = new Date().toISOString();
+        const artifact = buildContextWindowScalingExecutionArtifact({
+            runId: context.runId,
+            pluginId: contextWindowScalingMetadata.id,
+            pluginSchemaVersion: contextWindowScalingMetadata.schemaVersion,
+            startedAt,
+            completedAt,
+            tokenCountMethod,
+            contextBudgets: context.config.contextBudgets,
+            cases: executionEvidence,
+        });
+        const artifactPath = resolveWithinRoot(outDir, CONTEXT_WINDOW_SCALING_EXECUTION_ARTIFACT_FILE);
+        await writeFile(artifactPath, `${JSON.stringify(artifact, null, 2)}\n`, "utf8");
+        return mapExecutionToRun({
+            runId: context.runId,
+            startedAt,
+            completedAt,
+            target: context.target,
+            contextBudgets: context.config.contextBudgets,
+            executionEvidence,
+            artifactPath,
+        });
+    },
+    summarize(result) {
+        return result.summary ?? summarizeExperimentRun(result);
+    },
+};
+export function mapExecutionToRun(args) {
+    const cases = args.executionEvidence.map((caseEvidence) => ({
+        id: caseEvidence.caseId,
+        name: caseEvidence.caseName,
+        outcomes: caseEvidence.treatments.map((treatment) => buildOutcome(caseEvidence, treatment)),
+        metadata: { benchmarkProject: caseEvidence.benchmarkProject },
+    }));
+    const aggregate = aggregateContextWindowScaling({ contextBudgets: args.contextBudgets, cases: args.executionEvidence });
+    const run = {
+        runId: args.runId,
+        pluginId: contextWindowScalingMetadata.id,
+        startedAt: args.startedAt,
+        completedAt: args.completedAt,
+        status: aggregateStatus(cases.flatMap((experimentCase) => experimentCase.outcomes.map((outcome) => outcome.status))),
+        target: args.target,
+        variants: VARIANTS.map((variant) => ({ ...variant })),
+        cases,
+        metrics: toAggregateExperimentMetrics(aggregate),
+        artifacts: [
+            {
+                id: "context-window-scaling-execution",
+                label: "Context window scaling execution evidence",
+                path: args.artifactPath,
+                kind: "artifact",
+                mimeType: "application/json",
+                description: "Per-case, per-treatment, per-budget fit, correctness, success, and omitted-file evidence without context text.",
+            },
+        ],
+        warnings: [],
+        failures: [],
+        metadata: { executionArtifactPath: args.artifactPath },
+        contextBudgets: [...args.contextBudgets],
+        executionEvidence: structuredClone([...args.executionEvidence]),
+        aggregate,
+    };
+    run.summary = summarizeExperimentRun(run);
+    return run;
+}
+function buildOutcome(caseEvidence, treatment) {
+    const { caseId } = caseEvidence;
+    const variantId = treatment.variantId;
+    const metrics = [];
+    if (treatment.context.estimatedTokens !== null) {
+        metrics.push({
+            id: "context-estimated-tokens",
+            name: "Context estimated tokens",
+            value: treatment.context.estimatedTokens,
+            unit: "tokens",
+            variantId,
+            caseId,
+        });
+    }
+    if (treatment.relevantFileEvidence.omittedRelevantFileCount !== null) {
+        metrics.push({
+            id: "omitted-relevant-file-count",
+            name: "Omitted relevant files",
+            value: treatment.relevantFileEvidence.omittedRelevantFileCount,
+            unit: "files",
+            variantId,
+            caseId,
+        });
+    }
+    metrics.push(...toOutcomeBudgetMetrics(treatment, caseId));
+    const warnings = [
+        ...treatment.context.warnings.map((message) => ({ code: "guided-retrieval-warning", message, variantId, caseId })),
+        ...(treatment.evaluation.status === "unavailable"
+            ? [{ code: "deterministic-evaluation-unavailable", message: treatment.evaluation.reason ?? "Evaluation unavailable.", variantId, caseId }]
+            : []),
+    ];
+    const failures = treatment.errors.map((error) => ({
+        code: error.code,
+        message: error.message,
+        variantId,
+        caseId,
+        recoverable: true,
+    }));
+    return {
+        id: `${caseId}:${variantId}`,
+        caseId,
+        variantId,
+        status: treatment.status,
+        metrics,
+        artifacts: [],
+        warnings,
+        failures,
+        metadata: {
+            benchmarkProject: caseEvidence.benchmarkProject,
+            contextStatus: treatment.context.status,
+            evaluationStatus: treatment.evaluation.status,
+        },
+    };
+}
+function readCasesInput(inputs) {
+    const value = inputs?.cases;
+    if (!Array.isArray(value) || value.length === 0) {
+        throw new Error("Context window scaling requires a non-empty cases input.");
+    }
+    return value;
+}
+function readProjectProfilesInput(inputs) {
+    const value = inputs?.projectProfiles;
+    return Array.isArray(value) ? value : [];
+}
+function readEnvInput(inputs) {
+    const env = inputs?.env;
+    return env && typeof env === "object" && !Array.isArray(env) ? env : undefined;
+}
+function readDependenciesInput(inputs) {
+    const value = inputs?.dependencies;
+    return value && typeof value === "object" ? value : undefined;
+}

package/dist/src/experiments/plugins/contextWindowScaling/projectProfile.js

@@ -0,0 +1,51 @@
+import path from "node:path";
+import { calculateProjectComplexityScore, computeProjectComplexityMetrics, PROJECT_COMPLEXITY_FORMULA, } from "../../../evaluation/projectComplexity.js";
+import { buildProjectFileTree } from "../../../evaluation/projectFileTree.js";
+/**
+ * The fixed scaling project has no entry in the shared benchmark-project-profiles contract. The
+ * existing prompt owner requires a profile per case, so a missing one is derived in memory from
+ * the existing file-tree and complexity owners. Supplied profiles always win; nothing is written.
+ */
+export function resolveScalingProjectProfiles(cases, supplied) {
+    const profiles = [...supplied];
+    const known = new Set(profiles.map((profile) => profile.projectId));
+    for (const evaluationCase of cases) {
+        const profileId = evaluationCase.projectProfileRef ?? evaluationCase.benchmarkProject;
+        if (known.has(profileId))
+            continue;
+        known.add(profileId);
+        profiles.push(deriveProfile(profileId, evaluationCase, cases));
+    }
+    return profiles;
+}
+function deriveProfile(profileId, evaluationCase, cases) {
+    const siblings = cases.filter((candidate) => (candidate.projectProfileRef ?? candidate.benchmarkProject) === profileId);
+    const fileTree = buildProjectFileTree(evaluationCase.absoluteTargetRoot);
+    const average = (values) => (values.length === 0 ? 0 : values.reduce((a, b) => a + b, 0) / values.length);
+    const complexityMetrics = computeProjectComplexityMetrics(evaluationCase.absoluteTargetRoot, fileTree, {
+        taskCount: siblings.length,
+        expectedRelevantFilesAverage: average(siblings.map((candidate) => candidate.expectedFiles.length)),
+        expectedRelevantSymbolsAverage: average(siblings.map((candidate) => candidate.expectedSymbols.length)),
+    });
+    const languages = [
+        ...new Set(fileTree.entries.map((entry) => entry.language).filter((language) => Boolean(language))),
+    ].sort();
+    return {
+        projectId: profileId,
+        displayName: profileId,
+        description: "Profile derived in memory for the context-window-scaling fixed corpus.",
+        languageMix: languages.length <= 1 ? `single-language ${languages[0] ?? "unknown"}` : "multi-language",
+        primaryLanguage: languages[0] ?? "unknown",
+        languages,
+        complexityLevel: "large",
+        complexityScore: calculateProjectComplexityScore(complexityMetrics),
+        complexityMetrics,
+        complexityFormula: PROJECT_COMPLEXITY_FORMULA,
+        rootPath: path.posix.normalize(evaluationCase.targetRoot.replace(/\\/g, "/")),
+        sourceRoots: [...evaluationCase.sourceRoots],
+        testRoots: ["tests"],
+        fileTree,
+        benchmarkPurpose: "Context-window scaling evaluation.",
+        expectedUseCases: ["context budget fit evaluation"],
+    };
+}

package/dist/src/experiments/plugins/contextWindowScaling/relevantFiles.js

@@ -0,0 +1,76 @@
+/**
+ * Bounded repository-relative identity: "\" becomes "/", leading "./" is removed, case is kept.
+ * Returns null for absolute paths, drive paths, parent traversal, empty segments, or oversized values.
+ */
+export function normalizeRepositoryRelativePath(value) {
+    if (typeof value !== "string" || value.length === 0 || value.length > 1024) {
+        return null;
+    }
+    let normalized = value.replace(/\\/g, "/");
+    while (normalized.startsWith("./")) {
+        normalized = normalized.slice(2);
+    }
+    if (normalized.length === 0 || normalized.startsWith("/") || /^[A-Za-z]:/.test(normalized)) {
+        return null;
+    }
+    if (normalized.split("/").some((segment) => segment === ".." || segment === "")) {
+        return null;
+    }
+    return normalized;
+}
+/** observedFiles === null means observed-file provenance is unavailable. */
+export function computeRelevantFileEvidence(args) {
+    if (args.expectedFiles.length === 0) {
+        return {
+            status: "not-applicable",
+            expectedRelevantFiles: [],
+            expectedRelevantFileCount: 0,
+            observedExpectedFileCount: null,
+            omittedRelevantFileCount: null,
+            omittedRelevantFiles: [],
+            reason: "No expected relevant files are defined for this case.",
+        };
+    }
+    const expected = [];
+    for (const file of args.expectedFiles) {
+        const normalized = normalizeRepositoryRelativePath(file);
+        if (normalized === null) {
+            return unavailable([...args.expectedFiles], "An expected relevant file is not a bounded repository-relative path.");
+        }
+        if (!expected.includes(normalized)) {
+            expected.push(normalized);
+        }
+    }
+    if (args.observedFiles === null) {
+        return unavailable(expected, "Observed-file provenance is unavailable.");
+    }
+    const observed = new Set();
+    for (const file of args.observedFiles) {
+        const normalized = normalizeRepositoryRelativePath(file);
+        if (normalized === null) {
+            return unavailable(expected, "An observed file is not a bounded repository-relative path.");
+        }
+        observed.add(normalized);
+    }
+    const omitted = expected.filter((file) => !observed.has(file));
+    return {
+        status: "available",
+        expectedRelevantFiles: expected,
+        expectedRelevantFileCount: expected.length,
+        observedExpectedFileCount: expected.length - omitted.length,
+        omittedRelevantFileCount: omitted.length,
+        omittedRelevantFiles: omitted,
+        reason: null,
+    };
+}
+function unavailable(expected, reason) {
+    return {
+        status: "unavailable",
+        expectedRelevantFiles: expected,
+        expectedRelevantFileCount: expected.length,
+        observedExpectedFileCount: null,
+        omittedRelevantFileCount: null,
+        omittedRelevantFiles: [],
+        reason,
+    };
+}

package/dist/src/experiments/plugins/contextWindowScaling/successEvidence.js

@@ -0,0 +1,60 @@
+import { calculateContextBudgetUtilizationPercent, classifyContextFit } from "./contextBudget.js";
+export const UNAVAILABLE_CORRECTNESS = { availability: "unavailable", score: null, pass: null };
+/** The existing scorer's own passed result is used as-is; no score threshold is defined here. */
+export function toCorrectnessEvidence(scored) {
+    if (!scored.available || scored.score === null || scored.passed === null) {
+        return { ...UNAVAILABLE_CORRECTNESS };
+    }
+    return { availability: "available", score: scored.score, pass: scored.passed };
+}
+export function deriveSuccessEvidence(args) {
+    if (args.contextFitStatus === "unavailable") {
+        return { status: "unavailable", success: null, reason: "context-unavailable" };
+    }
+    if (args.contextFitStatus === "context-too-large") {
+        return { status: "available", success: false, reason: "context-too-large" };
+    }
+    if (args.correctness.availability !== "available" || args.correctness.pass === null) {
+        return { status: "unavailable", success: null, reason: "evaluation-unavailable" };
+    }
+    return args.correctness.pass
+        ? { status: "available", success: true, reason: "correctness-pass" }
+        : { status: "available", success: false, reason: "correctness-fail" };
+}
+/**
+ * One cell per budget. `shared` is the single deterministic evaluation of this treatment's context
+ * (null when it was never requested because no budget fits or the context is unavailable).
+ */
+export function buildBudgetCell(args) {
+    if (args.contextEstimatedTokens === null) {
+        return {
+            contextBudgetTokens: args.contextBudgetTokens,
+            contextFitStatus: "unavailable",
+            contextBudgetUtilizationPercent: null,
+            evaluationStatus: "unavailable",
+            correctness: { ...UNAVAILABLE_CORRECTNESS },
+            successEvidence: deriveSuccessEvidence({ contextFitStatus: "unavailable", correctness: UNAVAILABLE_CORRECTNESS }),
+        };
+    }
+    const fit = classifyContextFit(args.contextEstimatedTokens, args.contextBudgetTokens);
+    const utilization = calculateContextBudgetUtilizationPercent(args.contextEstimatedTokens, args.contextBudgetTokens);
+    if (fit === "context-too-large") {
+        return {
+            contextBudgetTokens: args.contextBudgetTokens,
+            contextFitStatus: fit,
+            contextBudgetUtilizationPercent: utilization,
+            evaluationStatus: "not-evaluated-context-too-large",
+            correctness: { ...UNAVAILABLE_CORRECTNESS },
+            successEvidence: deriveSuccessEvidence({ contextFitStatus: fit, correctness: UNAVAILABLE_CORRECTNESS }),
+        };
+    }
+    const correctness = args.shared ?? { ...UNAVAILABLE_CORRECTNESS };
+    return {
+        contextBudgetTokens: args.contextBudgetTokens,
+        contextFitStatus: fit,
+        contextBudgetUtilizationPercent: utilization,
+        evaluationStatus: correctness.availability === "available" ? "evaluated" : "unavailable",
+        correctness: { ...correctness },
+        successEvidence: deriveSuccessEvidence({ contextFitStatus: fit, correctness }),
+    };
+}

package/dist/src/experiments/plugins/contextWindowScaling/types.js

@@ -0,0 +1 @@
+export {};

package/dist/src/experiments/plugins/incrementalChangeStaleness/comparisonV2.js

@@ -0,0 +1,49 @@
+import { compareCorrectness, compareRequiredFileEvidence } from "./comparison.js";
+function neutral(relation) {
+    if (relation === "stale-worse")
+        return "candidate-worse";
+    if (relation === "stale-better")
+        return "candidate-better";
+    return relation;
+}
+/** Same rules as the v0.6.2 correctness comparison, candidate vs reference. */
+export function compareCandidateCorrectness(candidate, reference) {
+    return neutral(compareCorrectness(candidate, reference));
+}
+/** Same rules as the v0.6.2 required-file comparison, candidate vs reference. */
+export function compareCandidateRequiredFileEvidence(candidate, reference) {
+    return neutral(compareRequiredFileEvidence(candidate, reference));
+}
+/**
+ * Frozen precedence. Rule 0 (realization gate) first: a full fallback is never evidence about
+ * partial-refresh performance. Then observed regression (either dimension worse), then inconclusive
+ * (a dimension unknown), then no observed regression (including candidate-better, which is not a
+ * winner claim). affected-neighborhood evidence and reindexRecommendation are never inputs.
+ */
+export function classifyPartialRefreshAgainstFull(args) {
+    const { realization, correctnessRelation, requiredFileEvidenceRelation } = args;
+    // RULE 0 -- realization gate.
+    if (realization === "FALLBACK_FULL") {
+        return { classification: "not-comparable-as-partial-refresh", reasonCodes: ["candidate-fell-back-to-full"] };
+    }
+    // RULE 1 -- observed regression wins even if the other dimension is unknown.
+    if (correctnessRelation === "candidate-worse" || requiredFileEvidenceRelation === "candidate-worse") {
+        const reasonCodes = [];
+        if (correctnessRelation === "candidate-worse")
+            reasonCodes.push("candidate-correctness-lower");
+        if (requiredFileEvidenceRelation === "candidate-worse")
+            reasonCodes.push("candidate-missing-required-file");
+        return { classification: "observed-regression-relative-to-full", reasonCodes };
+    }
+    // RULE 2 -- inconclusive.
+    if (correctnessRelation === "unknown" || requiredFileEvidenceRelation === "unknown") {
+        const reasonCodes = [];
+        if (correctnessRelation === "unknown")
+            reasonCodes.push("correctness-unavailable");
+        if (requiredFileEvidenceRelation === "unknown")
+            reasonCodes.push("required-file-evidence-unavailable");
+        return { classification: "inconclusive", reasonCodes };
+    }
+    // RULE 3 -- no observed regression.
+    return { classification: "no-observed-regression-relative-to-full", reasonCodes: ["no-reference-regression-observed"] };
+}

package/dist/src/experiments/plugins/incrementalChangeStaleness/config.js

@@ -1,10 +1,11 @@
 import { invalidExperimentConfig, isPlainObject, mergeConfig, validExperimentConfig } from "../../config.js";
 import { FROZEN_INCREMENTAL_CHANGE_STALENESS_SCENARIO_IDS } from "./scenarioTypes.js";
 /**
- * The frozen v0.6.2 compatibility baseline for the upstream tool. Every index
- * of one matched scenario must come from this one configured command.
+ * The upstream tool version the four-treatment experiment expects: 1.12.5 is the first published
+ * my-dev-kit that exposes the incremental refresh-scope options. Every index of one matched scenario
+ * must come from this one configured command.
  */
-export const INCREMENTAL_CHANGE_STALENESS_DEFAULT_KIT_COMMAND = "npx @dailephd/my-dev-kit@1.12.4";
+export const INCREMENTAL_CHANGE_STALENESS_DEFAULT_KIT_COMMAND = "npx @dailephd/my-dev-kit@1.12.5";
 export const defaultIncrementalChangeStalenessConfig = {
     outDir: "lab-output/incremental-change-staleness",
     kitCommand: INCREMENTAL_CHANGE_STALENESS_DEFAULT_KIT_COMMAND
@@ -17,7 +18,7 @@ export const incrementalChangeStalenessConfigDefinition = {
         {
             name: "caseIds",
             type: "array",
-            description: `Frozen scenario ids to include (${FROZEN_INCREMENTAL_CHANGE_STALENESS_SCENARIO_IDS.join(", ")}); defaults to all. Both stale-index and full-refresh always run for each selected scenario.`
+            description: `Frozen scenario ids to include (${FROZEN_INCREMENTAL_CHANGE_STALENESS_SCENARIO_IDS.join(", ")}); defaults to all. All four treatments run for each selected scenario: stale-index, changed-files-refresh, affected-neighborhood-refresh, and full-refresh.`
         }
     ]
 };

package/dist/src/experiments/plugins/incrementalChangeStaleness/disposableTarget.js

@@ -8,6 +8,24 @@ import { resolveWithinRoot } from "../../../core/pathSafety.js";
  * target copies; it never builds a treatment index or runs retrieval.
  */
 export const INCREMENTAL_CHANGE_STALENESS_TREATMENT_IDS = ["stale-index", "full-refresh"];
+/**
+ * The four v0.6.3 internal lifecycle treatment identities in their fixed order
+ * (never alphabetical). The released v0.6.2 public constant above is unchanged;
+ * this set is used only by the internal v0.6.3 lifecycle.
+ */
+export const INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS = [
+    "stale-index",
+    "changed-files-refresh",
+    "affected-neighborhood-refresh",
+    "full-refresh"
+];
+/** Treatment ID and treatment intent are separate runtime evidence. */
+export const INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_INTENTS = {
+    "stale-index": "my-dev-kit-no-refresh",
+    "changed-files-refresh": "my-dev-kit-changed-files-refresh",
+    "affected-neighborhood-refresh": "my-dev-kit-affected-neighborhood-refresh",
+    "full-refresh": "my-dev-kit-full-refresh"
+};
 /**
  * Default project-relative, gitignored runtime location for disposable
  * treatment targets. Callers may pass a different `runRoot`, but it must
@@ -38,8 +56,8 @@ function assertDisposableRootDoesNotOverlapCanonical(targetRoot, canonicalProjec
  */
 export async function createDisposableTreatmentTarget(options) {
     const { repoRoot, scenarioId, treatmentId, benchmarkProjectId, canonicalProjectRootRelative } = options;
-    if (!INCREMENTAL_CHANGE_STALENESS_TREATMENT_IDS.includes(treatmentId)) {
-        throw new Error(`Unknown treatment id: ${treatmentId}. Expected one of ${INCREMENTAL_CHANGE_STALENESS_TREATMENT_IDS.join(", ")}.`);
+    if (!INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS.includes(treatmentId)) {
+        throw new Error(`Unknown treatment id: ${treatmentId}. Expected one of ${INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS.join(", ")}.`);
     }
     if (typeof scenarioId !== "string" || scenarioId.length === 0) {
         throw new Error("scenarioId must be a nonempty string.");