@dailephd/my-dev-kit-lab 0.6.3 → 0.7.1

package/dist/src/report/experiments/buildContextWindowScalingReport.js

@@ -0,0 +1,84 @@
+import { CONTEXT_WINDOW_SCALING_PLUGIN_ID } from "../../experiments/plugins/contextWindowScaling/metadata.js";
+import { CONTEXT_WINDOW_SCALING_REPORT_SCHEMA_VERSION, MAX_REPORT_OMITTED_RELEVANT_FILES, } from "./contextWindowScalingReportModel.js";
+/** Fixed wording; the report states observed evidence only and never ranks treatments. */
+export const CONTEXT_WINDOW_SCALING_INTERPRETATION = [
+    "Estimated context tokens use the Lab deterministic heuristic (characters divided by four, rounded up); they are not provider billing telemetry or a provider model tokenizer result.",
+    "The configured budgets are experiment budgets, not claims about actual provider model context-window limits.",
+    "context-too-large is an expected measurement state under a budget and is not an operational execution failure.",
+    "When a context is too large for a budget, evaluation did not occur, so correctness is unavailable; unavailable correctness is not zero and is excluded from mean correctness.",
+    "Success combines context fit with the existing deterministic correctness pass evidence: success is true only when the context fits and correctness passes.",
+    "context-too-large is an available, not-successful outcome for the applicable budget and remains in the success-rate denominator.",
+    "Operational or evidence-unavailable cells are excluded from the success-rate denominator and are counted separately.",
+    "Omitted relevant files are benchmark-expected files that were not observed in the treatment's context provenance.",
+    "Omitted relevant files are not full retrieval precision or recall; no precision, recall, or irrelevant-context measure is calculated.",
+    "The deterministic fake-agent correctness path is context-independent in the current harness: correctness tests the existing answer-key and evaluation pipeline, not semantic model sensitivity to different context content.",
+    "Utilization is estimated context tokens divided by the budget, is not capped at 100 percent, and may exceed 100 for a context that is too large.",
+    "No composite score, winner, or ranking is calculated; the treatments are reported side by side.",
+];
+/**
+ * Presents the aggregate and cell evidence already calculated on the run. It performs no fit,
+ * success, correctness, or utilization calculation of its own.
+ */
+export function buildContextWindowScalingReport(run) {
+    if (run.pluginId !== CONTEXT_WINDOW_SCALING_PLUGIN_ID)
+        return null;
+    const candidate = run;
+    const evidence = candidate.executionEvidence;
+    const aggregate = candidate.aggregate;
+    if (!evidence || !aggregate) {
+        throw new Error("Invalid context-window-scaling report source: execution evidence and aggregate are required.");
+    }
+    if (aggregate.caseTreatmentContextSummaries.length !== evidence.length * aggregate.treatments.length) {
+        throw new Error("Invalid context-window-scaling report source: aggregate and execution evidence are inconsistent.");
+    }
+    const caseTreatmentContexts = aggregate.caseTreatmentContextSummaries.map((summary) => {
+        const relevant = summary.relevantFileEvidence;
+        return {
+            caseId: summary.caseId,
+            caseName: summary.caseName,
+            variantId: summary.variantId,
+            contextStatus: summary.contextStatus,
+            characterCount: summary.characterCount,
+            estimatedTokens: summary.estimatedTokens,
+            tokenCountMethod: summary.tokenCountMethod,
+            observedFileCount: summary.observedFileCount,
+            relevantFileStatus: relevant.status,
+            expectedRelevantFileCount: relevant.expectedRelevantFileCount,
+            observedExpectedFileCount: relevant.observedExpectedFileCount,
+            omittedRelevantFileCount: relevant.omittedRelevantFileCount,
+            omittedRelevantFiles: boundedList(relevant.omittedRelevantFiles),
+            relevantFileReason: relevant.reason,
+        };
+    });
+    const caseBudgetCells = evidence.flatMap((caseEvidence) => caseEvidence.treatments.flatMap((treatment) => treatment.budgetCells.map((cell) => ({
+        caseId: caseEvidence.caseId,
+        variantId: treatment.variantId,
+        contextEstimatedTokens: treatment.context.estimatedTokens,
+        ...structuredClone(cell),
+    }))));
+    return {
+        schemaVersion: CONTEXT_WINDOW_SCALING_REPORT_SCHEMA_VERSION,
+        budgets: [...aggregate.budgets],
+        treatments: [...aggregate.treatments],
+        tokenCountMethod: firstTokenCountMethod(aggregate.caseTreatmentContextSummaries.map((summary) => summary.tokenCountMethod)),
+        runSummary: { ...aggregate.runSummary },
+        budgetTreatmentSummaries: aggregate.budgetTreatmentSummaries.map((summary) => ({ ...summary })),
+        caseTreatmentContexts,
+        caseBudgetCells,
+        relevantFileSummary: { ...aggregate.relevantFileSummary },
+        interpretation: [...CONTEXT_WINDOW_SCALING_INTERPRETATION],
+    };
+}
+function boundedList(items) {
+    const displayed = items.slice(0, MAX_REPORT_OMITTED_RELEVANT_FILES);
+    return {
+        totalCount: items.length,
+        displayedCount: displayed.length,
+        omittedCount: items.length - displayed.length,
+        truncated: items.length > displayed.length,
+        items: [...displayed],
+    };
+}
+function firstTokenCountMethod(methods) {
+    return methods.find((method) => method !== null) ?? null;
+}

package/dist/src/report/experiments/buildPluginExperimentReport.js

@@ -2,6 +2,10 @@ import path from "node:path";
 import { buildContextStrategyComparisonV043Report } from "./buildContextStrategyComparisonV043Report.js";
 import { buildWarmIndexReuseReport } from "./buildWarmIndexReuseReport.js";
 import { buildIncrementalChangeStalenessPluginReport } from "./buildIncrementalChangeStalenessPluginReport.js";
+import { buildContextWindowScalingReport } from "./buildContextWindowScalingReport.js";
+// Bulk context-window-scaling evidence is presented by the typed report section; the complete
+// evidence stays in context-window-scaling-execution.json.
+const CONTEXT_WINDOW_SCALING_BULK_KEYS = ["executionEvidence", "aggregate"];
 const V043_BULK_ARRAY_KEYS = [
     "v043StageContextExecutions",
     "v043StageContextEvaluations",
@@ -13,10 +17,16 @@ export function buildPluginExperimentReport(args) {
     const contextStrategyComparisonV043 = buildContextStrategyComparisonV043Report(args.run);
     const warmIndexReuse = buildWarmIndexReuseReport(args.run);
     const incrementalChangeStaleness = buildIncrementalChangeStalenessPluginReport(args.run);
+    const contextWindowScaling = buildContextWindowScalingReport(args.run);
     const rawRun = { ...args.run, artifacts: relativizeArtifacts(args.run.artifacts, outputRoot) };
     for (const key of V043_BULK_ARRAY_KEYS) {
         delete rawRun[key];
     }
+    if (contextWindowScaling) {
+        for (const key of CONTEXT_WINDOW_SCALING_BULK_KEYS) {
+            delete rawRun[key];
+        }
+    }
     return {
         metadata: {
             generatedAt: args.generatedAt ?? new Date().toISOString(),
@@ -42,8 +52,9 @@ export function buildPluginExperimentReport(args) {
         findings: buildFindings(args.run),
         warmIndexReuse,
         incrementalChangeStaleness,
+        contextWindowScaling,
         contextStrategyComparisonV043,
-        interpretation: buildInterpretation(args.run, contextStrategyComparisonV043, warmIndexReuse),
+        interpretation: buildInterpretation(args.run, contextStrategyComparisonV043, warmIndexReuse, contextWindowScaling),
         rawRun,
     };
 }
@@ -122,7 +133,16 @@ function affectedNeighborhoodSentence(report) {
         `reindex recommendations were recommended=${summary.recommendedReindexCount}, not-indicated=${summary.notIndicatedReindexCount}, unknown=${summary.unknownReindexRecommendationCount}. ` +
         "Affected-neighborhood evidence uses a one-hop baseline graph around confirmed changed indexed files/symbols. It is observational and does not alter retrieval or execution status.");
 }
-function buildInterpretation(run, contextStrategyComparisonV043, warmIndexReuse) {
+function buildInterpretation(run, contextStrategyComparisonV043, warmIndexReuse, contextWindowScaling) {
+    if (contextWindowScaling) {
+        const summary = contextWindowScaling.runSummary;
+        return {
+            summary: `Context-window-scaling evidence covers ${summary.caseCount} case${summary.caseCount === 1 ? "" : "s"}, ${summary.treatmentCount} treatments, and ${summary.budgetCount} budget${summary.budgetCount === 1 ? "" : "s"} ` +
+                `(${summary.totalBudgetCellCount} budget cells): ${summary.evaluatedBudgetCellCount} evaluated, ${summary.contextTooLargeCellCount} context-too-large, ${summary.contextUnavailableCellCount} with unavailable context. ` +
+                "Context-too-large is an expected measurement state, not an execution failure. The treatments are reported side by side and are not ranked.",
+            recommendedNextStep: "Review the budget summary, per-case context sizes, and per-case budget matrix; unavailable correctness and success evidence are excluded from the rates and are never treated as zero.",
+        };
+    }
     if (run.pluginId === "warm-index-reuse" && warmIndexReuse) {
         const summary = warmIndexReuse.summary;
         const campaign = warmIndexReuse.agentCampaign;

package/dist/src/report/experiments/contextWindowScalingReportModel.js

@@ -0,0 +1,3 @@
+export const CONTEXT_WINDOW_SCALING_REPORT_SCHEMA_VERSION = "my-dev-kit-lab-context-window-scaling-report-v1";
+/** Presentation limit only; the full omitted identities stay in context-window-scaling-execution.json. */
+export const MAX_REPORT_OMITTED_RELEVANT_FILES = 20;

package/dist/src/report/experiments/index.js

@@ -20,3 +20,7 @@ export { buildIncrementalChangeStalenessPluginReport } from "./buildIncrementalC
 export { buildIncrementalChangeStalenessReportV2FromArtifact, findReportTreatment, findReportComparison, LIMITATIONS_V2 } from "./buildIncrementalChangeStalenessReportV2.js";
 export { buildIncrementalChangeStalenessReport, STALE_RISK_CLASSIFICATION_EXPLANATIONS } from "./buildIncrementalChangeStalenessReport.js";
 export { renderIncrementalChangeStalenessHtml } from "./renderIncrementalChangeStalenessHtml.js";
+export * from "./contextWindowScalingReportModel.js";
+export { buildContextWindowScalingReport, CONTEXT_WINDOW_SCALING_INTERPRETATION } from "./buildContextWindowScalingReport.js";
+export { renderContextWindowScalingHtml } from "./renderContextWindowScalingHtml.js";
+export { renderContextWindowScalingTextLines } from "./renderContextWindowScalingText.js";

package/dist/src/report/experiments/renderContextWindowScalingHtml.js

@@ -0,0 +1,99 @@
+import { formatContextWindowScalingCount, formatContextWindowScalingPercent, formatContextWindowScalingScore, } from "./renderContextWindowScalingText.js";
+export function renderContextWindowScalingHtml(section) {
+    if (section === null) {
+        return `<section>
+    <h2>Context Window Scaling</h2>
+    <p>Not applicable to this plugin.</p>
+  </section>`;
+    }
+    const summary = section.runSummary;
+    return `<section>
+    <h2>Context Window Scaling</h2>
+    ${table(["Field", "Value"], [
+        ["Schema", section.schemaVersion],
+        ["Budgets (estimated context tokens)", section.budgets.join(", ")],
+        ["Treatments", section.treatments.join(", ")],
+        ["Token count method", section.tokenCountMethod ?? "unavailable"],
+        ...Object.entries(summary).map(([label, value]) => [label, String(value)]),
+    ])}
+    <h3>Budget Summary</h3>
+    ${table(["Budget", "Treatment", "Cells", "Fits", "Too large", "Context unavailable", "Success", "Not successful", "Success unavailable", "Success rate", "Correctness available", "Mean correctness", "Mean utilization", "Min utilization", "Max utilization"], section.budgetTreatmentSummaries.map((s) => [
+        String(s.contextBudgetTokens),
+        s.variantId,
+        String(s.totalCellCount),
+        String(s.fitCount),
+        String(s.contextTooLargeCount),
+        String(s.contextUnavailableCount),
+        String(s.successfulCellCount),
+        String(s.notSuccessfulCellCount),
+        String(s.successEvidenceUnavailableCount),
+        formatContextWindowScalingPercent(s.successRatePercent),
+        String(s.correctnessAvailableCount),
+        formatContextWindowScalingScore(s.meanCorrectnessScore),
+        formatContextWindowScalingPercent(s.meanContextBudgetUtilizationPercent),
+        formatContextWindowScalingPercent(s.minContextBudgetUtilizationPercent),
+        formatContextWindowScalingPercent(s.maxContextBudgetUtilizationPercent),
+    ]))}
+    <h3>Case Context Evidence</h3>
+    <p class="muted">Context does not vary by budget, so there is one row per case and treatment.</p>
+    ${table(["Case", "Treatment", "Context", "Characters", "Estimated tokens", "Token method", "Observed files", "Relevant-file evidence", "Expected relevant files", "Observed expected files", "Omitted relevant files"], section.caseTreatmentContexts.map((c) => [
+        c.caseId,
+        c.variantId,
+        c.contextStatus,
+        formatContextWindowScalingCount(c.characterCount),
+        formatContextWindowScalingCount(c.estimatedTokens),
+        c.tokenCountMethod ?? "unavailable",
+        formatContextWindowScalingCount(c.observedFileCount),
+        c.relevantFileStatus,
+        String(c.expectedRelevantFileCount),
+        formatContextWindowScalingCount(c.observedExpectedFileCount),
+        formatContextWindowScalingCount(c.omittedRelevantFileCount),
+    ]))}
+    <h3>Omitted Relevant Files</h3>
+    <p class="muted">Benchmark-expected files not observed in the treatment context. Not retrieval precision or recall.</p>
+    ${table(["Evidence available", "Unavailable", "Not applicable", "Total omitted"], [[
+            String(section.relevantFileSummary.relevantFileEvidenceAvailableCount),
+            String(section.relevantFileSummary.relevantFileEvidenceUnavailableCount),
+            String(section.relevantFileSummary.relevantFileEvidenceNotApplicableCount),
+            String(section.relevantFileSummary.totalOmittedRelevantFileCount),
+        ]])}
+    ${section.caseTreatmentContexts
+        .filter((c) => c.relevantFileStatus === "available" && (c.omittedRelevantFileCount ?? 0) > 0)
+        .map((c) => `<p><code>${escapeHtml(c.caseId)}</code> / ${escapeHtml(c.variantId)}: omitted ${c.omittedRelevantFiles.totalCount}; displayed ${c.omittedRelevantFiles.displayedCount} of ${c.omittedRelevantFiles.totalCount}; omitted from display ${c.omittedRelevantFiles.omittedCount}; truncated ${c.omittedRelevantFiles.truncated ? "yes" : "no"}</p>${list(c.omittedRelevantFiles.items)}`)
+        .join("\n    ")}
+    <h3>Per-Case Budget Matrix</h3>
+    ${table(["Case", "Treatment", "Budget", "Estimated context tokens", "Fit", "Utilization", "Evaluation", "Correctness", "Correctness score", "Correctness pass", "Success evidence", "Success", "Success reason"], section.caseBudgetCells.map((cell) => [
+        cell.caseId,
+        cell.variantId,
+        String(cell.contextBudgetTokens),
+        formatContextWindowScalingCount(cell.contextEstimatedTokens),
+        cell.contextFitStatus,
+        formatContextWindowScalingPercent(cell.contextBudgetUtilizationPercent),
+        cell.evaluationStatus,
+        cell.correctness.availability,
+        formatContextWindowScalingScore(cell.correctness.score),
+        cell.correctness.pass === null ? "unavailable" : String(cell.correctness.pass),
+        cell.successEvidence.status,
+        cell.successEvidence.success === null ? "unavailable" : String(cell.successEvidence.success),
+        cell.successEvidence.reason,
+    ]))}
+    <h3>Interpretation And Limitations</h3>
+    ${list(section.interpretation)}
+  </section>`;
+}
+function list(items) {
+    return `<ul>${items.map((item) => `<li>${escapeHtml(item)}</li>`).join("")}</ul>`;
+}
+function table(headers, rows) {
+    return `<table><thead><tr>${headers.map((header) => `<th>${escapeHtml(header)}</th>`).join("")}</tr></thead><tbody>${rows
+        .map((row) => `<tr>${row.map((cell) => `<td>${escapeHtml(cell)}</td>`).join("")}</tr>`)
+        .join("")}</tbody></table>`;
+}
+function escapeHtml(value) {
+    return String(value ?? "")
+        .replaceAll("&", "&amp;")
+        .replaceAll("<", "&lt;")
+        .replaceAll(">", "&gt;")
+        .replaceAll('"', "&quot;")
+        .replaceAll("'", "&#39;");
+}

package/dist/src/report/experiments/renderContextWindowScalingText.js

@@ -0,0 +1,108 @@
+// Render-time formatting only; the JSON report keeps exact unrounded values.
+// Rule: percent values use 2 decimals plus "%", correctness scores use 4 decimals, null is "unavailable".
+export function formatContextWindowScalingPercent(value) {
+    return value === null ? "unavailable" : `${value.toFixed(2)}%`;
+}
+export function formatContextWindowScalingScore(value) {
+    return value === null ? "unavailable" : value.toFixed(4);
+}
+export function formatContextWindowScalingCount(value) {
+    return value === null ? "unavailable" : String(value);
+}
+function clean(value) {
+    return String(value).replace(/[\u0000-\u001f\u007f]/g, " ");
+}
+function row(parts) {
+    return `- ${parts.map(([label, value]) => `${label}: ${clean(value)}`).join(" | ")}`;
+}
+/** Lines for the "Context Window Scaling" section; every value comes from the report model. */
+export function renderContextWindowScalingTextLines(section) {
+    const lines = [];
+    lines.push(`Schema: ${clean(section.schemaVersion)}`);
+    lines.push(`Budgets (estimated context tokens): ${section.budgets.join(", ")}`);
+    lines.push(`Treatments: ${section.treatments.join(", ")}`);
+    lines.push(`Token Count Method: ${clean(section.tokenCountMethod ?? "unavailable")}`);
+    lines.push("Run Summary:");
+    const summary = section.runSummary;
+    for (const [label, value] of Object.entries(summary)) {
+        lines.push(`${label}: ${value}`);
+    }
+    lines.push("Budget Summary (one row per budget and treatment):");
+    for (const s of section.budgetTreatmentSummaries) {
+        lines.push(row([
+            ["Budget", s.contextBudgetTokens],
+            ["Treatment", s.variantId],
+            ["Cells", s.totalCellCount],
+            ["Fits", s.fitCount],
+            ["Too Large", s.contextTooLargeCount],
+            ["Context Unavailable", s.contextUnavailableCount],
+            ["Success", s.successfulCellCount],
+            ["Not Successful", s.notSuccessfulCellCount],
+            ["Success Unavailable", s.successEvidenceUnavailableCount],
+            ["Success Rate", formatContextWindowScalingPercent(s.successRatePercent)],
+            ["Correctness Available", s.correctnessAvailableCount],
+            ["Mean Correctness", formatContextWindowScalingScore(s.meanCorrectnessScore)],
+            ["Mean Utilization", formatContextWindowScalingPercent(s.meanContextBudgetUtilizationPercent)],
+            ["Min Utilization", formatContextWindowScalingPercent(s.minContextBudgetUtilizationPercent)],
+            ["Max Utilization", formatContextWindowScalingPercent(s.maxContextBudgetUtilizationPercent)],
+        ]));
+    }
+    lines.push("Case Context Evidence (one row per case and treatment; context does not vary by budget):");
+    for (const c of section.caseTreatmentContexts) {
+        lines.push(row([
+            ["Case", c.caseId],
+            ["Treatment", c.variantId],
+            ["Context", c.contextStatus],
+            ["Characters", formatContextWindowScalingCount(c.characterCount)],
+            ["Estimated Tokens", formatContextWindowScalingCount(c.estimatedTokens)],
+            ["Token Method", c.tokenCountMethod ?? "unavailable"],
+            ["Observed Files", formatContextWindowScalingCount(c.observedFileCount)],
+            ["Relevant File Evidence", c.relevantFileStatus],
+            ["Expected Relevant Files", c.expectedRelevantFileCount],
+            ["Observed Expected Files", formatContextWindowScalingCount(c.observedExpectedFileCount)],
+            ["Omitted Relevant Files", formatContextWindowScalingCount(c.omittedRelevantFileCount)],
+        ]));
+    }
+    lines.push("Omitted Relevant Files (benchmark-expected files not observed in the treatment context):");
+    lines.push(row([
+        ["Evidence Available", section.relevantFileSummary.relevantFileEvidenceAvailableCount],
+        ["Unavailable", section.relevantFileSummary.relevantFileEvidenceUnavailableCount],
+        ["Not Applicable", section.relevantFileSummary.relevantFileEvidenceNotApplicableCount],
+        ["Total Omitted", section.relevantFileSummary.totalOmittedRelevantFileCount],
+    ]));
+    for (const c of section.caseTreatmentContexts) {
+        if (c.relevantFileStatus === "available" && (c.omittedRelevantFileCount ?? 0) > 0) {
+            const list = c.omittedRelevantFiles;
+            lines.push(`${clean(c.caseId)} / ${c.variantId}: omitted ${list.totalCount}; displayed ${list.displayedCount} of ${list.totalCount}; omitted from display ${list.omittedCount}; truncated ${list.truncated ? "yes" : "no"}`);
+            for (const file of list.items) {
+                lines.push(`  - ${clean(file)}`);
+            }
+        }
+        else if (c.relevantFileStatus !== "available" && c.relevantFileReason) {
+            lines.push(`${clean(c.caseId)} / ${c.variantId}: ${c.relevantFileStatus} (${clean(c.relevantFileReason)})`);
+        }
+    }
+    lines.push("Per-Case Budget Matrix (one row per case, treatment, and budget):");
+    for (const cell of section.caseBudgetCells) {
+        lines.push(row([
+            ["Case", cell.caseId],
+            ["Treatment", cell.variantId],
+            ["Budget", cell.contextBudgetTokens],
+            ["Estimated Context Tokens", formatContextWindowScalingCount(cell.contextEstimatedTokens)],
+            ["Fit", cell.contextFitStatus],
+            ["Utilization", formatContextWindowScalingPercent(cell.contextBudgetUtilizationPercent)],
+            ["Evaluation", cell.evaluationStatus],
+            ["Correctness", cell.correctness.availability],
+            ["Correctness Score", formatContextWindowScalingScore(cell.correctness.score)],
+            ["Correctness Pass", cell.correctness.pass === null ? "unavailable" : cell.correctness.pass],
+            ["Success Evidence", cell.successEvidence.status],
+            ["Success", cell.successEvidence.success === null ? "unavailable" : cell.successEvidence.success],
+            ["Success Reason", cell.successEvidence.reason],
+        ]));
+    }
+    lines.push("Interpretation And Limitations:");
+    for (const item of section.interpretation) {
+        lines.push(`- ${clean(item)}`);
+    }
+    return lines;
+}

package/dist/src/report/experiments/renderPluginExperimentReportHtml.js

@@ -1,6 +1,7 @@
 import { renderContextStrategyComparisonV043Html } from "./renderContextStrategyComparisonV043Html.js";
 import { renderWarmIndexReuseHtml } from "./renderWarmIndexReuseHtml.js";
 import { renderIncrementalChangeStalenessHtml } from "./renderIncrementalChangeStalenessHtml.js";
+import { renderContextWindowScalingHtml } from "./renderContextWindowScalingHtml.js";
 export function renderPluginExperimentReportHtml(report) {
     return `<!DOCTYPE html>
 <html lang="en">
@@ -102,6 +103,8 @@ export function renderPluginExperimentReportHtml(report) {
 
   ${renderIncrementalChangeStalenessHtml(report.incrementalChangeStaleness)}
 
+  ${renderContextWindowScalingHtml(report.contextWindowScaling)}
+
   <section>
     <h2>Warnings, Skips, And Failures</h2>
     ${report.findings.length ? table(["Severity", "Code", "Message", "Variant", "Case"], report.findings.map((finding) => [

package/dist/src/report/experiments/renderPluginExperimentReportText.js

@@ -2,6 +2,7 @@ import { STALE_RISK_CLASSIFICATION_EXPLANATIONS } from "./buildIncrementalChange
 import { isV2ReportSection } from "./buildIncrementalChangeStalenessReportV2.js";
 import { renderIncrementalChangeStalenessSectionV2 } from "./renderIncrementalChangeStalenessTextV2.js";
 import { REINDEX_RECOMMENDATION_EXPLANATIONS } from "./buildWarmIndexReuseReport.js";
+import { renderContextWindowScalingTextLines } from "./renderContextWindowScalingText.js";
 function sanitizeScalar(value) {
     const text = String(value);
     let result = "";
@@ -663,6 +664,13 @@ export function renderPluginExperimentReportText(report) {
     renderWarmIndexReuseSection(lines, report.warmIndexReuse);
     pushSection(lines, "Incremental-Change And Staleness Evidence");
     renderIncrementalChangeStalenessSection(lines, report.incrementalChangeStaleness);
+    pushSection(lines, "Context Window Scaling");
+    if (report.contextWindowScaling === null) {
+        lines.push("Not applicable to this plugin.");
+    }
+    else {
+        lines.push(...renderContextWindowScalingTextLines(report.contextWindowScaling));
+    }
     pushSection(lines, "Warnings, Skips, And Failures");
     pushDashList(lines, report.findings.map((finding) => `${finding.severity} ${finding.code} ${finding.message} variant=${finding.variantId ?? ""} case=${finding.caseId ?? ""}`));
     pushSection(lines, "Artifacts");

package/docs/METRICS.md

@@ -596,6 +596,34 @@ All six metrics are warm-side only (they are not on the raw-full-file side, beca
 
 **Observational only.** The evidence never triggers reindexing, suppresses or alters warm retrieval, or changes execution, provider, agent, correctness, or token-evidence status. It uses only the baseline graph of the prepared index: there is no graph-diff, refreshed-index, or stale-versus-refreshed comparison.
 
+## Context-window scaling evidence (v0.7.0)
+
+Status: **released in v0.7.0**. The `context-window-scaling` plugin compares `raw-full-file` and `my-dev-kit-guided` treatment contexts against selected estimated-token budgets. By default it runs the bundled case catalog against the fixed self target; the v0.7.1 `--synthetic-config` input runs the same metrics over deterministic generated repositories without changing any formula below.
+
+### Measurement and budget-cell semantics
+
+- **Estimated tokens:** context construction records character count and an estimated token count using the Lab's deterministic `ceil(characters / 4)` heuristic. It is not a provider tokenizer result, model-window measurement, or billing telemetry.
+- **Budget values:** defaults are 8192, 16384, 32768, and 65536 tokens (shown as 8k, 16k, 32k, and 64k). Users can select other positive safe integers. Duplicate values after normalization are rejected; accepted budgets are sorted ascending.
+- **Context fit:** `fits` when estimated context tokens are less than or equal to the selected budget; `context-too-large` when they exceed it; `unavailable` when context measurement is unavailable. This classification does not truncate or reconstruct the context. Each treatment's context is built once, independent of its budgets.
+- **Budget utilization:** `contextBudgetUtilizationPercent = estimatedContextTokens / budget * 100`. It is not capped at 100 percent, so an over-budget context can have utilization greater than 100. It is unavailable when context size is unavailable.
+- **Correctness:** the existing deterministic fake-agent answer-key evaluator runs no more than once per treatment and is shared by every budget cell where that treatment's context fits. Correctness is unavailable for a `context-too-large` cell and is not converted to zero. This harness's evaluation is context-independent; it does not estimate how a real model's correctness would change with the supplied context.
+- **Success:** success is true only when the context fits and the available correctness evidence passes. A too-large context is an available unsuccessful cell with reason `context-too-large`, even though no evaluation ran. Operational or evidence-unavailable cells remain unavailable and are excluded from success-rate denominators. Thus context-too-large cells count as not successful, while unavailable cells do not.
+- **Omitted relevant files:** benchmark-expected relevant files absent from the treatment's observed context provenance. This is an evidence count/list, not retrieval precision, recall, or an irrelevant-context ratio. The report bounds displayed omitted paths and retains total/displayed/omitted counts.
+
+### Aggregate and per-cell metrics
+
+`aggregateContextWindowScaling` in `src/experiments/plugins/contextWindowScaling/metrics.ts` owns the per-budget and run summaries. `buildContextWindowScalingReport.ts` presents these results without recalculating fit, utilization, success, or correctness.
+
+For each budget and treatment, aggregate evidence includes `fitCount`, `contextTooLargeCount`, `contextUnavailableCount`, `successfulCellCount`, `notSuccessfulCellCount`, success-evidence availability counts, `successRatePercent`, correctness availability counts and `meanCorrectnessScore`, and availability counts plus mean/min/max context-budget utilization. Success rate is successful cells divided by success-evidence-available cells; it is null when none are available. Mean correctness averages only available scores and is null when none are available. Utilization summaries include only measured values and remain uncapped.
+
+The generic `ExperimentMetric` projection emits run-level case and budget counts, plus per-budget/per-treatment fit count, too-large count, successful count, success rate, mean correctness, and mean utilization. Per-case treatment outcomes carry context estimated tokens, omitted relevant file count when available, and budget-scoped utilization, correctness score, and success evidence. Unavailable values remain null/unavailable with their evidence status; no value is fabricated as zero.
+
+The report artifact schema is `my-dev-kit-lab-context-window-scaling-report-v1`. It includes run summaries, budget-treatment summaries, per-treatment context measurements, budget cells, relevant-file summary, and fixed interpretation limits. The execution artifact is `context-window-scaling-execution.json` (`my-dev-kit-lab-context-window-scaling-execution-v1`) and does not embed context text.
+
+**Synthetic-repository runs (v0.7.1).** The synthetic input introduces no new scientific metric: fit, utilization, correctness, success, and omitted-relevant-file semantics, the execution artifact schema, the report schema, and the plot set are identical to the bundled-corpus run. Each generated repository's `synthetic-repository-manifest.json` records generation evidence (per-file byte length, character count, SHA-256, generated-content estimated tokens, aggregate totals, and `repositoryContentIdentity`); these are reproducibility measurements, not context-window outcome metrics. The raw-full-file treatment context is measured by the existing baseline, which adds file headers and separators, so its estimated tokens differ from the manifest's generated-content estimate. Retrieval precision/recall remains v0.8.0 scope.
+
+There is no composite score, winner, or ranking. Budgets are experiment thresholds, not assertions about actual provider context-window capacities. See [COMMANDS.md](COMMANDS.md), [WORKFLOWS.md](WORKFLOWS.md), and [ARCHITECTURE.md](ARCHITECTURE.md).
+
 ## Incremental-change and staleness evidence (v0.6.2)
 
 The `incremental-change-staleness` plugin reuses the six existing v0.6.1 affected-neighborhood numeric metrics unchanged (`changedFileCount`, `changedSymbolCount`, `affectedNodeCount`, `affectedEdgeCount`, `taskOverlapCount`, `taskOverlapPercent`; see "Affected-neighborhood metrics (v0.6.1)" above) and adds no new numeric formula.
@@ -630,7 +658,7 @@ The `incremental-change-staleness` plugin reuses the six existing v0.6.1 affecte
 
 **Report presentation.** `report.incrementalChangeStaleness` holds `scenarioCount`, `readyScenarioCount`, `failedScenarioCount`, `observedStaleRegressionCount`, `noObservedStaleRegressionCount`, `inconclusiveCount`, per-scenario evidence (lifecycle, both treatments in order, and the comparison), and fixed limitations text. There is no overall score, grade, winner, best-treatment, or safe-to-skip-reindex field anywhere in the report or the persisted execution artifact.
 
-### Partial-refresh treatment evidence (v0.6.3, current release)
+### Partial-refresh treatment evidence (v0.6.3, retained in current release)
 
 The `incremental-change-staleness` plugin now compares four treatments (`stale-index`, `changed-files-refresh`, `affected-neighborhood-refresh`, `full-refresh`). It defines no new formula. Two evidence families must not be merged:
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dailephd/my-dev-kit-lab",
-  "version": "0.6.3",
+  "version": "0.7.1",
   "type": "module",
   "description": "Evidence, benchmark, and evaluation companion for my-dev-kit.",
   "bin": {