@dailephd/my-dev-kit-lab 0.8.1 → 0.9.0

package/dist/src/report/experiments/index.js

@@ -28,6 +28,15 @@ export * from "./retrievalPrecisionRecallReportModel.js";
 export { buildRetrievalPrecisionRecallReport, RETRIEVAL_PRECISION_RECALL_LIMITATIONS } from "./buildRetrievalPrecisionRecallReport.js";
 export { renderRetrievalPrecisionRecallHtml } from "./renderRetrievalPrecisionRecallHtml.js";
 export { renderRetrievalPrecisionRecallTextLines } from "./renderRetrievalPrecisionRecallText.js";
+export * from "./contextPackGenerationReportModel.js";
+export { buildContextPackGenerationReport, CONTEXT_PACK_GENERATION_LIMITATIONS } from "./buildContextPackGenerationReport.js";
+export { loadContextPackArtifacts } from "./loadContextPackArtifacts.js";
+export { renderContextPackGenerationHtml } from "./renderContextPackGenerationHtml.js";
+export { renderContextPackGenerationTextLines } from "./renderContextPackGenerationText.js";
+export * from "./agentSuccessRateReportModel.js";
+export { buildAgentSuccessRateReport, safeAgentSuccessArtifactReference } from "./buildAgentSuccessRateReport.js";
+export { renderAgentSuccessRateHtml } from "./renderAgentSuccessRateHtml.js";
+export { renderAgentSuccessRateTextLines } from "./renderAgentSuccessRateText.js";
 export * from "./retrievalQueryStrategyComparisonReportModel.js";
 export { buildRetrievalQueryStrategyComparisonReport, RETRIEVAL_QUERY_STRATEGY_COMPARISON_LIMITATIONS } from "./buildRetrievalQueryStrategyComparisonReport.js";
 export { renderRetrievalQueryStrategyComparisonHtml } from "./renderRetrievalQueryStrategyComparisonHtml.js";

package/dist/src/report/experiments/loadContextPackArtifacts.js

@@ -0,0 +1,43 @@
+import { readFile } from "node:fs/promises";
+import { resolveWithinRoot } from "../../core/pathSafety.js";
+import { CONTEXT_PACK_GENERATION_PLUGIN_ID } from "../../experiments/plugins/contextPackGeneration/metadata.js";
+import { CONTEXT_PACK_SCHEMA_VERSION } from "../../experiments/plugins/contextPackGeneration/types.js";
+function isContextPack(value, caseId) {
+    if (typeof value !== "object" || value === null)
+        return false;
+    const pack = value;
+    return (pack.schemaVersion === CONTEXT_PACK_SCHEMA_VERSION &&
+        pack.caseId === caseId &&
+        typeof pack.task === "object" &&
+        pack.task !== null &&
+        Array.isArray(pack.files) &&
+        Array.isArray(pack.symbols) &&
+        Array.isArray(pack.sourceSlices) &&
+        Array.isArray(pack.callRelationships) &&
+        Array.isArray(pack.tests) &&
+        Array.isArray(pack.evidenceNotes));
+}
+/**
+ * Reads the persisted per-case pack artifacts the plugin already wrote under the output root. It never reruns retrieval or
+ * reopens repository source. A missing or invalid artifact is simply absent: the report then marks that preview unavailable.
+ */
+export async function loadContextPackArtifacts(run, outputRoot) {
+    const packs = new Map();
+    if (run.pluginId !== CONTEXT_PACK_GENERATION_PLUGIN_ID)
+        return packs;
+    const evidence = run.caseExecutionEvidence ?? [];
+    for (const entry of evidence) {
+        const relativePath = entry.treatments.find((treatment) => treatment.treatmentId === "context-pack")?.packArtifactPath;
+        if (!relativePath)
+            continue;
+        try {
+            const parsed = JSON.parse(await readFile(resolveWithinRoot(outputRoot, relativePath), "utf8"));
+            if (isContextPack(parsed, entry.caseId))
+                packs.set(entry.caseId, parsed);
+        }
+        catch {
+            // Unreadable or unsafe artifact: the preview is reported as pack-artifact-unavailable, never as an empty pack.
+        }
+    }
+    return packs;
+}

package/dist/src/report/experiments/renderAgentSuccessRateHtml.js

@@ -0,0 +1,169 @@
+import { AGENT_SUCCESS_BASIS_LABEL, formatAgentSuccessMetric, formatAgentSuccessPairs, formatAgentSuccessTotal } from "./renderAgentSuccessRateText.js";
+function escapeHtml(value) {
+    return String(value ?? "")
+        .replaceAll("&", "&")
+        .replaceAll("<", "&lt;")
+        .replaceAll(">", "&gt;")
+        .replaceAll('"', "&quot;")
+        .replaceAll("'", "&#39;");
+}
+function table(headers, rows) {
+    return `<table><thead><tr>${headers.map((header) => `<th>${escapeHtml(header)}</th>`).join("")}</tr></thead><tbody>${rows
+        .map((row) => `<tr>${row.map((cell) => `<td>${escapeHtml(cell)}</td>`).join("")}</tr>`)
+        .join("")}</tbody></table>`;
+}
+/**
+ * HTML rendering of the typed section. All dynamic content is escaped, artifact references are plain text (never links
+ * or embedded files), and no value is recalculated here.
+ */
+export function renderAgentSuccessRateHtml(report) {
+    if (!report)
+        return "";
+    const realMode = report.identity.executionMode === "real-agent";
+    const m = formatAgentSuccessMetric;
+    const parts = [];
+    parts.push(`<section id="agent-success-rate">
+    <h2>Agent Success Rate</h2>
+    <h3>Experiment identity</h3>
+    ${table(["Field", "Value"], [
+        ["Run ID", report.identity.runId],
+        ["Plugin", `${report.identity.pluginId} (schema ${report.identity.pluginSchemaVersion})`],
+        ["Run status", report.identity.runStatus],
+        ["Cases", String(report.identity.caseCount)],
+        ["Treatments", report.identity.treatmentOrder.join(", ")],
+        ["Treatment outcomes", String(report.identity.treatmentOutcomeCount)]
+    ])}
+    <h3>Execution mode and provider</h3>
+    ${table(["Field", "Value"], [
+        ["Execution mode", report.identity.executionMode],
+        ["Provider", report.identity.providerId ?? "none (no coding agent was invoked)"],
+        ["Provider timeout per attempt", report.identity.timeoutMs === null ? "not applicable" : `${report.identity.timeoutMs} ms`],
+        ["Repair attempts allowed", report.identity.repairAttempts === null ? "not applicable" : String(report.identity.repairAttempts)],
+        ["Maximum attempts per treatment", String(report.identity.maxAttemptsPerTreatment)]
+    ])}
+    <h3>Scientific limitations</h3>
+    <p><strong>${escapeHtml(report.scientificStatement)}</strong></p>
+    <ul>${report.limitations.map((limitation) => `<li>${escapeHtml(limitation)}</li>`).join("")}</ul>
+    <h3>Overall result availability</h3>
+    ${table(["Field", "Value"], [
+        ["Overall", report.resultAvailability.overall],
+        ["Outcomes with a determinate verdict", `${report.resultAvailability.evaluableTreatmentOutcomes} of ${report.resultAvailability.totalTreatmentOutcomes}`],
+        ["Outcomes without a determinate verdict", String(report.resultAvailability.unavailableTreatmentOutcomes)]
+    ])}
+    ${report.resultAvailability.notes.length > 0 ? `<ul>${report.resultAvailability.notes.map((note) => `<li>${escapeHtml(note)}</li>`).join("")}</ul>` : ""}
+  </section>`);
+    const treatmentRows = report.treatments.map((treatment) => [
+        treatment.treatmentId,
+        String(treatment.evaluableCases),
+        String(treatment.initialSuccessfulCases),
+        m(treatment.initialSuccessRate),
+        String(treatment.finalSuccessfulCases),
+        m(treatment.finalSuccessRate),
+        treatment.repair ? String(treatment.repair.repairAttempts) : "not applicable",
+        treatment.repair ? String(treatment.repair.repairedCases) : "not applicable"
+    ]);
+    const comparison = report.comparison;
+    parts.push(`<section id="agent-success-rate-comparison">
+    <h2>Treatment Comparison</h2>
+    ${table(["Treatment", "Evaluable cases", "Initial successful cases", "Initial success rate", "Final successful cases", "Final success rate", "Repair attempts", "Repaired cases"], treatmentRows)}
+    ${report.treatments
+        .filter((treatment) => treatment.repair)
+        .map((treatment) => `<p class="muted">${escapeHtml(treatment.treatmentId)}: repair-eligible cases ${treatment.repair.repairEligibleCases}; cases with a repair ${treatment.repair.repairAttemptedCases}; repair success rate ${escapeHtml(m(treatment.repair.repairSuccessRate))}; mean attempts per evaluable case ${escapeHtml(m(treatment.repair.meanAttemptsPerEvaluableCase))}.</p>`)
+        .join("")}
+    ${comparison
+        ? `<h3>Matched comparison (descriptive; no winner is declared)</h3>
+    ${table(["Statement", "Value"], [
+            ["Comparison evaluable", comparison.comparisonEvaluable ? "yes" : "no"],
+            ["Observed outcome difference between treatments", comparison.observedOutcomeDifference],
+            ["Statistical or causal effect", `${comparison.statisticalEffect} (no significance or causal claim is made)`],
+            ["Final matched cases", comparison.finalMatchedCaseIds.join(", ") || "none"],
+            ["Final paired outcomes", formatAgentSuccessPairs(comparison.finalPairedOutcomes)],
+            ["Initial-attempt matched cases", comparison.initialMatchedCaseIds ? comparison.initialMatchedCaseIds.join(", ") || "none" : "not applicable"],
+            ["Initial-attempt paired outcomes", formatAgentSuccessPairs(comparison.initialPairedOutcomes)]
+        ])}
+    ${comparison.pairedDifferences.length > 0
+            ? table(["Measurement", "Basis", "Matched cases", "raw-full-file mean", "context-pack mean", "context-pack minus raw-full-file"], comparison.pairedDifferences.map((difference) => [
+                difference.id,
+                AGENT_SUCCESS_BASIS_LABEL[difference.basis] ?? difference.basis,
+                String(difference.matchedCaseIds.length),
+                m(difference.rawFullFileMean),
+                m(difference.contextPackMean),
+                m(difference.meanDifference)
+            ]))
+            : ""}`
+        : `<p class="muted">Matched comparison: not applicable (deterministic-fixture mode does not compare context treatments).</p>`}
+  </section>`);
+    parts.push(`<section id="agent-success-rate-cases">
+    <h2>Per-Case Results</h2>
+    ${table(["Case", "Project", "Locality", "Treatment", "Status", "Initial task success", "Final task success", "Attempts", "Task-check pass rate", "Regression-check pass rate", "Required-fact coverage"], report.cases.flatMap((entry) => entry.treatments.map((treatment) => [
+        entry.caseId,
+        entry.benchmarkProject,
+        entry.taskLocality,
+        treatment.treatmentId,
+        `${treatment.executionStatus} / ${treatment.evidenceAvailability}`,
+        m(treatment.initialTaskSuccess),
+        m(treatment.finalTaskSuccess),
+        String(treatment.attemptCount),
+        m(treatment.taskCheckPassRate),
+        m(treatment.regressionCheckPassRate),
+        m(treatment.requiredFactCoverage)
+    ])))}
+  </section>`);
+    parts.push(`<section id="agent-success-rate-repair-history">
+    <h2>Per-Attempt Repair History</h2>
+    ${!realMode || report.repairHistory.length === 0
+        ? `<p class="muted">${realMode ? "No attempt history was recorded." : "Not applicable: deterministic-fixture mode runs a single fixture patch and no provider attempts."}</p>`
+        : table(["Case", "Treatment", "Attempt", "Provider status", "Patch outcome", "Verification", "Task success", "Outcome", "Duration", "Tokens", "Artifacts"], report.repairHistory.flatMap((history) => history.attempts.map((attempt) => [
+            history.caseId,
+            history.treatmentId,
+            String(attempt.attemptNumber),
+            attempt.providerStatus,
+            attempt.patchOutcome,
+            attempt.verificationAvailability,
+            m(attempt.taskSuccess),
+            `${attempt.failureCategory}${attempt.repairEligible ? " (repair-eligible)" : ""}`,
+            m(attempt.providerDurationMs),
+            attempt.tokenAvailability === "available" ? m(attempt.providerTokens) : "unavailable",
+            `proposed: ${attempt.proposedPatchPath ?? "none"}; applied: ${attempt.appliedPatchPath ?? "none"}; agent: ${attempt.agentArtifactDirectory ?? "none"}`
+        ])))}
+  </section>`);
+    parts.push(`<section id="agent-success-rate-edit-quality">
+    <h2>Edit Quality And Blast Radius (final evaluated patch)</h2>
+    ${table(["Case", "Treatment", "Expected-edit coverage", "Edit-scope precision", "Unexpected changed files", "Protected mutations", "Changed files", "Total churn", "Relative churn"], report.cases.flatMap((entry) => entry.treatments.map((treatment) => [
+        entry.caseId,
+        treatment.treatmentId,
+        m(treatment.expectedEditCoverage),
+        m(treatment.editScopePrecision),
+        m(treatment.unexpectedChangedFiles),
+        m(treatment.protectedMutations),
+        `${m(treatment.changedFileCount)}${treatment.changedFiles ? ` [${treatment.changedFiles.items.join(", ")}${treatment.changedFiles.omittedCount > 0 ? `, +${treatment.changedFiles.omittedCount} more` : ""}]` : ""}`,
+        m(treatment.totalChurn),
+        m(treatment.relativeChurn)
+    ])))}
+  </section>`);
+    parts.push(`<section id="agent-success-rate-cost">
+    <h2>Duration And Token Evidence</h2>
+    ${table(["Case", "Treatment", "Final-attempt provider duration", "Final-attempt provider tokens", "Total provider duration (all attempts)", "Total provider tokens (all attempts)"], report.cases.flatMap((entry) => entry.treatments.map((treatment) => [
+        entry.caseId,
+        treatment.treatmentId,
+        m(treatment.finalAttemptProviderDurationMs),
+        m(treatment.finalAttemptProviderTokens),
+        treatment.totalProviderDurationMs ? m(treatment.totalProviderDurationMs) : "not applicable",
+        treatment.totalProviderTokens ? m(treatment.totalProviderTokens) : "not applicable"
+    ])))}
+    ${report.treatments
+        .filter((treatment) => treatment.repair)
+        .flatMap((treatment) => [
+        ...treatment.repair.providerDurationMs.map((total) => `<p class="muted">${escapeHtml(treatment.treatmentId)} provider duration [${escapeHtml(AGENT_SUCCESS_BASIS_LABEL[total.basis] ?? total.basis)}]: ${escapeHtml(formatAgentSuccessTotal(total))}</p>`),
+        ...treatment.repair.providerTokens.map((total) => `<p class="muted">${escapeHtml(treatment.treatmentId)} provider tokens [${escapeHtml(AGENT_SUCCESS_BASIS_LABEL[total.basis] ?? total.basis)}]: ${escapeHtml(formatAgentSuccessTotal(total))}</p>`)
+    ])
+        .join("")}
+  </section>`);
+    const errors = report.cases.flatMap((entry) => entry.treatments.flatMap((treatment) => treatment.errors.map((error) => [entry.caseId, treatment.treatmentId, String(error.attemptNumber), error.code, error.message])));
+    parts.push(`<section id="agent-success-rate-findings">
+    <h2>Agent Success Rate Warnings, Failures And Artifacts</h2>
+    ${report.warnings.length === 0 && errors.length === 0 ? "<p>No warnings or recorded errors.</p>" : table(["Source", "Case", "Treatment", "Attempt", "Code", "Message"], [...report.warnings.map((warning) => ["warning", "", "", "", warning.code, warning.message]), ...errors.map((error) => ["error", ...error])])}
+    ${table(["Artifact", "Reference", "Case", "Treatment"], report.artifacts.map((artifact) => [artifact.id, artifact.path ?? "[reference withheld]", artifact.caseId ?? "", artifact.variantId ?? ""]))}
+  </section>`);
+    return parts.join("\n\n  ");
+}

package/dist/src/report/experiments/renderAgentSuccessRateText.js

@@ -0,0 +1,139 @@
+/** One display form for every metric: an absent value is labelled, never rendered as zero. */
+export function formatAgentSuccessMetric(metric) {
+    if (!metric)
+        return "unavailable (not recorded)";
+    if (metric.availability === "unavailable")
+        return `unavailable (${metric.reason ?? "no reason recorded"})`;
+    if (metric.availability === "not-applicable")
+        return `not applicable (${metric.reason ?? "no reason recorded"})`;
+    const value = metric.value;
+    if (typeof value === "boolean")
+        return value ? "yes" : "no";
+    if (typeof value !== "number")
+        return "unavailable (no value)";
+    switch (metric.unit) {
+        case "ratio":
+            return value.toFixed(4);
+        case "ms":
+            return `${Math.round(value)} ms`;
+        case "tokens":
+            return `${Math.round(value)} tokens`;
+        case "lines":
+            return `${Math.round(value)} lines`;
+        default:
+            return Number.isInteger(value) ? String(value) : value.toFixed(4);
+    }
+}
+export function formatAgentSuccessTotal(total) {
+    const base = `${formatAgentSuccessMetric(total.total)}; contributing cases=${total.contributingCaseCount}, measured=${total.availableCaseCount}, unmeasured=${total.unavailableCaseCount}`;
+    return total.unavailableCaseCount > 0 && total.sumOfAvailable !== null ? `${base}; sum of measured cases only=${total.sumOfAvailable} (not a complete total)` : base;
+}
+export function formatAgentSuccessPairs(pairs) {
+    if (!pairs)
+        return "not recorded";
+    return `both succeeded=${pairs.bothSucceeded}, only raw-full-file=${pairs.onlyRawFullFileSucceeded}, only context-pack=${pairs.onlyContextPackSucceeded}, neither=${pairs.neitherSucceeded}`;
+}
+export const AGENT_SUCCESS_BASIS_LABEL = {
+    "first-attempt": "first attempt",
+    "final-attempt": "final attempt",
+    "total-across-attempts": "total across attempts"
+};
+const sanitize = (value) => String(value ?? "").replace(/[\r\n\t]+/g, " ").trim();
+/** Text rendering of the typed section. It formats already-calculated values and recalculates none of them. */
+export function renderAgentSuccessRateTextLines(report) {
+    const realMode = report.identity.executionMode === "real-agent";
+    const lines = [];
+    const section = (title) => {
+        lines.push("", title, "-".repeat(title.length));
+    };
+    lines.push("", "AGENT SUCCESS RATE", "==================");
+    section("A. Experiment identity");
+    lines.push(`Run ID: ${sanitize(report.identity.runId)}`, `Plugin: ${sanitize(report.identity.pluginId)} (schema ${sanitize(report.identity.pluginSchemaVersion)})`, `Run status: ${sanitize(report.identity.runStatus)}`, `Cases: ${report.identity.caseCount}; treatments: ${report.identity.treatmentOrder.join(", ")}; treatment outcomes: ${report.identity.treatmentOutcomeCount}`);
+    section("B. Execution mode and provider");
+    lines.push(`Execution mode: ${sanitize(report.identity.executionMode)}`, `Provider: ${report.identity.providerId ?? "none (no coding agent was invoked)"}`, `Provider timeout per attempt: ${report.identity.timeoutMs === null ? "not applicable" : `${report.identity.timeoutMs} ms`}`, `Repair attempts allowed: ${report.identity.repairAttempts === null ? "not applicable" : report.identity.repairAttempts}; maximum attempts per treatment: ${report.identity.maxAttemptsPerTreatment}`);
+    section("C. Scientific limitations");
+    lines.push(sanitize(report.scientificStatement));
+    for (const limitation of report.limitations)
+        lines.push(`- ${sanitize(limitation)}`);
+    section("D. Overall result availability");
+    const availability = report.resultAvailability;
+    lines.push(`Overall: ${availability.overall}`, `Treatment outcomes with a determinate task-success verdict: ${availability.evaluableTreatmentOutcomes} of ${availability.totalTreatmentOutcomes}; without: ${availability.unavailableTreatmentOutcomes}`);
+    for (const note of availability.notes)
+        lines.push(`- ${sanitize(note)}`);
+    section("E. Treatment comparison");
+    for (const treatment of report.treatments) {
+        lines.push(`Treatment ${treatment.treatmentId}:`, `  evaluable cases: ${treatment.evaluableCases}`, `  initial-attempt successful cases: ${treatment.initialSuccessfulCases}; initial success rate: ${formatAgentSuccessMetric(treatment.initialSuccessRate)}`, `  final successful cases: ${treatment.finalSuccessfulCases}; final success rate: ${formatAgentSuccessMetric(treatment.finalSuccessRate)}`);
+        if (treatment.repair) {
+            lines.push(`  repair-eligible cases: ${treatment.repair.repairEligibleCases}; repair attempts executed: ${treatment.repair.repairAttempts}; cases with a repair: ${treatment.repair.repairAttemptedCases}; repaired cases: ${treatment.repair.repairedCases}`, `  repair success rate: ${formatAgentSuccessMetric(treatment.repair.repairSuccessRate)}; mean attempts per evaluable case: ${formatAgentSuccessMetric(treatment.repair.meanAttemptsPerEvaluableCase)}`);
+        }
+        else {
+            lines.push("  repair: not applicable (deterministic-fixture mode; the single fixture attempt is both initial and final)");
+        }
+    }
+    if (report.comparison) {
+        const comparison = report.comparison;
+        lines.push("Matched comparison (descriptive; no winner is declared):", `  comparison evaluable: ${comparison.comparisonEvaluable ? "yes" : "no"}`, `  observed outcome difference between treatments: ${comparison.observedOutcomeDifference}`, `  statistical or causal effect: ${comparison.statisticalEffect} (no significance or causal claim is made)`, `  final matched cases: ${comparison.finalMatchedCaseIds.length > 0 ? comparison.finalMatchedCaseIds.join(", ") : "none"}`, `  final paired outcomes: ${formatAgentSuccessPairs(comparison.finalPairedOutcomes)}`);
+        if (comparison.initialMatchedCaseIds) {
+            lines.push(`  initial-attempt matched cases: ${comparison.initialMatchedCaseIds.length > 0 ? comparison.initialMatchedCaseIds.join(", ") : "none"}`, `  initial-attempt paired outcomes: ${formatAgentSuccessPairs(comparison.initialPairedOutcomes)}`);
+        }
+        for (const incomplete of comparison.finalIncompleteCases)
+            lines.push(`  excluded from final matched metrics: ${incomplete.caseId} (no determinate verdict for ${incomplete.unavailableTreatmentIds.join(", ")})`);
+        for (const difference of comparison.pairedDifferences) {
+            lines.push(`  paired difference ${difference.id} [${AGENT_SUCCESS_BASIS_LABEL[difference.basis] ?? difference.basis}] over ${difference.matchedCaseIds.length} matched case(s): raw-full-file ${formatAgentSuccessMetric(difference.rawFullFileMean)}; context-pack ${formatAgentSuccessMetric(difference.contextPackMean)}; context-pack minus raw-full-file ${formatAgentSuccessMetric(difference.meanDifference)}`);
+        }
+    }
+    else {
+        lines.push("Matched comparison: not applicable (deterministic-fixture mode does not compare context treatments).");
+    }
+    section("F. Per-case results");
+    for (const entry of report.cases) {
+        lines.push(`Case ${entry.caseId} (${sanitize(entry.caseName)}) project=${entry.benchmarkProject} locality=${entry.taskLocality}`);
+        for (const treatment of entry.treatments) {
+            lines.push(`  ${treatment.treatmentId}: status=${treatment.executionStatus} evidence=${treatment.evidenceAvailability}`, `    initial task success: ${formatAgentSuccessMetric(treatment.initialTaskSuccess)}; final task success: ${formatAgentSuccessMetric(treatment.finalTaskSuccess)}; attempts=${treatment.attemptCount}; repair attempts=${treatment.repairAttemptCount}; repair succeeded: ${formatAgentSuccessMetric(treatment.repairSucceeded)}`, `    task-check pass rate: ${formatAgentSuccessMetric(treatment.taskCheckPassRate)}; regression-check pass rate: ${formatAgentSuccessMetric(treatment.regressionCheckPassRate)}; required-fact coverage: ${formatAgentSuccessMetric(treatment.requiredFactCoverage)}`);
+        }
+    }
+    section("G. Per-attempt repair history");
+    if (!realMode || report.repairHistory.length === 0) {
+        lines.push(realMode ? "No attempt history was recorded." : "Not applicable: deterministic-fixture mode runs a single fixture patch and no provider attempts.");
+    }
+    for (const history of report.repairHistory) {
+        lines.push(`Case ${history.caseId} / ${history.treatmentId}: ${history.attempts.length} attempt(s)`);
+        for (const attempt of history.attempts) {
+            lines.push(`  attempt ${attempt.attemptNumber}: provider=${attempt.providerStatus}; patch=${attempt.patchOutcome}; verification=${attempt.verificationAvailability}; task success=${formatAgentSuccessMetric(attempt.taskSuccess)}; outcome=${attempt.failureCategory}${attempt.repairEligible ? " (repair-eligible)" : ""}`, `    duration: ${formatAgentSuccessMetric(attempt.providerDurationMs)}; tokens: ${attempt.tokenAvailability === "available" ? formatAgentSuccessMetric(attempt.providerTokens) : "unavailable"}`, `    artifacts: proposed patch=${attempt.proposedPatchPath ?? "none"}; applied patch=${attempt.appliedPatchPath ?? "none"}; agent files=${attempt.agentArtifactDirectory ?? "none"}`);
+        }
+    }
+    section("H. Edit-quality and blast-radius measurements (final evaluated patch)");
+    for (const entry of report.cases) {
+        for (const treatment of entry.treatments) {
+            lines.push(`${entry.caseId} / ${treatment.treatmentId}: expected-edit coverage=${formatAgentSuccessMetric(treatment.expectedEditCoverage)}; edit-scope precision=${formatAgentSuccessMetric(treatment.editScopePrecision)}; unexpected changed files=${formatAgentSuccessMetric(treatment.unexpectedChangedFiles)}; protected mutations=${formatAgentSuccessMetric(treatment.protectedMutations)}`, `  changed files=${formatAgentSuccessMetric(treatment.changedFileCount)}${treatment.changedFiles ? ` [${treatment.changedFiles.items.join(", ")}${treatment.changedFiles.omittedCount > 0 ? `, +${treatment.changedFiles.omittedCount} more` : ""}]` : ""}; total churn=${formatAgentSuccessMetric(treatment.totalChurn)}; relative churn=${formatAgentSuccessMetric(treatment.relativeChurn)}`);
+        }
+    }
+    section("I. Duration and token evidence");
+    for (const entry of report.cases) {
+        for (const treatment of entry.treatments) {
+            lines.push(`${entry.caseId} / ${treatment.treatmentId}: final-attempt provider duration=${formatAgentSuccessMetric(treatment.finalAttemptProviderDurationMs)}; final-attempt provider tokens=${formatAgentSuccessMetric(treatment.finalAttemptProviderTokens)}` +
+                (treatment.totalProviderDurationMs
+                    ? `; total provider duration across attempts=${formatAgentSuccessMetric(treatment.totalProviderDurationMs)}; total provider tokens across attempts=${formatAgentSuccessMetric(treatment.totalProviderTokens)}`
+                    : ""));
+        }
+    }
+    for (const treatment of report.treatments) {
+        if (!treatment.repair)
+            continue;
+        for (const total of treatment.repair.providerDurationMs)
+            lines.push(`${treatment.treatmentId} provider duration [${AGENT_SUCCESS_BASIS_LABEL[total.basis] ?? total.basis}]: ${formatAgentSuccessTotal(total)}`);
+        for (const total of treatment.repair.providerTokens)
+            lines.push(`${treatment.treatmentId} provider tokens [${AGENT_SUCCESS_BASIS_LABEL[total.basis] ?? total.basis}]: ${formatAgentSuccessTotal(total)}`);
+    }
+    section("J. Warnings, failures and artifact references");
+    const errors = report.cases.flatMap((entry) => entry.treatments.flatMap((treatment) => treatment.errors.map((error) => ({ caseId: entry.caseId, treatmentId: treatment.treatmentId, ...error }))));
+    if (report.warnings.length === 0 && errors.length === 0)
+        lines.push("No warnings or recorded errors.");
+    for (const warning of report.warnings)
+        lines.push(`warning ${sanitize(warning.code)}: ${sanitize(warning.message)}`);
+    for (const error of errors)
+        lines.push(`error ${error.caseId} / ${error.treatmentId} attempt ${error.attemptNumber} ${sanitize(error.code)}: ${sanitize(error.message)}`);
+    for (const artifact of report.artifacts)
+        lines.push(`artifact ${sanitize(artifact.id)}: ${artifact.path ?? "[reference withheld]"}`);
+    return lines;
+}

package/dist/src/report/experiments/renderContextPackGenerationHtml.js

@@ -0,0 +1,122 @@
+import { formatContextPackNumber, redactedPreviewDisplayLines } from "./renderContextPackGenerationText.js";
+import { formatRetrievalQueryStrategyMetric } from "./renderRetrievalQueryStrategyComparisonText.js";
+const formatMean = (value) => (value === undefined || value === null ? "unavailable" : value.toFixed(4));
+export function renderContextPackGenerationHtml(section) {
+    if (section === null)
+        return "";
+    return `<section>
+    <h2>Context Pack Generation</h2>
+    <h3>Methodology</h3>
+    ${table(["Field", "Value"], [
+        ["Schema", section.schemaVersion],
+        ["Experiment", `${section.experiment.pluginId} (${section.experiment.target})`],
+        ["Treatment order", section.treatmentOrder.join(", ")],
+        ["File F1", section.methodology.fileF1],
+        ["Symbol F1", section.methodology.symbolF1],
+        ["Aggregation", section.methodology.aggregation],
+        ["Treatment comparison", section.methodology.treatmentComparison],
+        ["Size measure", section.methodology.sizeMeasure]
+    ])}
+    <h3>Per-Case Treatments</h3>
+    ${table(["Case", "Project", "Task Type", "Treatment", "Status", "Availability", "File F1", "Symbol F1", "Fact Coverage", "Estimated Tokens"], section.cases.flatMap((entry) => entry.treatments.map((treatment) => [
+        entry.caseId,
+        entry.benchmarkProject,
+        entry.taskLocality ?? "unavailable",
+        treatment.treatmentId,
+        treatment.executionStatus,
+        treatment.availability ?? "unavailable",
+        formatRetrievalQueryStrategyMetric(treatment.fileF1),
+        formatRetrievalQueryStrategyMetric(treatment.symbolF1),
+        formatRetrievalQueryStrategyMetric(treatment.factCoverage),
+        formatContextPackNumber(treatment.estimatedTokens)
+    ])))}
+    <h3>Paired Comparison (context-pack minus raw-full-file)</h3>
+    ${table(["Case", "File F1 Delta", "Symbol F1 Delta", "Fact Coverage Delta", "Estimated Token Delta", "Tokens Saved", "Percent Saved"], section.cases.map((entry) => [
+        entry.caseId,
+        formatMean(entry.comparison.fileF1Delta),
+        formatMean(entry.comparison.symbolF1Delta),
+        formatMean(entry.comparison.factCoverageDelta),
+        formatContextPackNumber(entry.comparison.estimatedTokenDelta),
+        formatContextPackNumber(entry.comparison.tokensSaved),
+        formatMean(entry.comparison.percentSaved)
+    ]))}
+    <h3>Scope Aggregates</h3>
+    ${section.scopes
+        .map((scope) => `<h4>${escapeHtml(scope.scopeId)}</h4>
+    <p>Cases ${scope.caseCount}; included ${scope.includedCaseCount} (${escapeHtml(scope.includedCaseIds.join(", ") || "none")}); excluded ${scope.excludedCaseCount} (${escapeHtml(scope.excludedCaseIds.join(", ") || "none")}).</p>
+    ${table(["Treatment", "Completed", "Partial", "Failed", "Mean File F1", "Mean Symbol F1", "Mean Fact Coverage", "Mean Estimated Tokens"], scope.treatmentSummaries.map((summary) => [
+        summary.treatmentId,
+        String(summary.completedCaseCount),
+        String(summary.partialCaseCount),
+        String(summary.failedCaseCount),
+        formatMean(summary.objectives?.meanFileF1),
+        formatMean(summary.objectives?.meanSymbolF1),
+        formatMean(summary.objectives?.meanFactCoverage),
+        formatContextPackNumber(summary.objectives?.meanEstimatedTokenCount)
+    ]))}
+    ${table(["Mean File F1 Delta", "Mean Symbol F1 Delta", "Mean Fact Coverage Delta", "Mean Estimated Token Delta", "Mean Tokens Saved", "Percent Saved Of Means"], [
+        [
+            formatMean(scope.pairedDeltas?.meanFileF1Delta),
+            formatMean(scope.pairedDeltas?.meanSymbolF1Delta),
+            formatMean(scope.pairedDeltas?.meanFactCoverageDelta),
+            formatContextPackNumber(scope.pairedDeltas?.meanEstimatedTokenDelta),
+            formatContextPackNumber(scope.tokenSavings?.meanTokensSaved),
+            formatMean(scope.tokenSavings?.percentSavedOfMeans)
+        ]
+    ])}`)
+        .join("\n    ")}
+    <h3>Context Pack Preview</h3>
+    <p>Bounded for display only; stored packs and measured sizes are unchanged.</p>
+    ${section.previews.map(previewHtml).join("\n    ")}
+    <h3>Interpretation Limits</h3>
+    ${listHtml(section.limitations)}
+  </section>`;
+}
+function previewHtml(preview) {
+    if (preview.status === "redacted-external-local") {
+        return `<h4>${escapeHtml(preview.caseId)}</h4>
+    <p>Status ${escapeHtml(preview.status)}; pack availability ${escapeHtml(preview.packAvailability ?? "unavailable")}.</p>
+    ${table(["Section", "Availability", "Items", "Estimated Tokens"], preview.sections.map((s) => [s.id, s.availability, String(s.itemCount), String(s.estimatedTokens)]))}
+    ${listHtml(redactedPreviewDisplayLines(preview))}`;
+    }
+    const head = `<h4>${escapeHtml(preview.caseId)}</h4>
+    <p>Status ${escapeHtml(preview.status)}; pack availability ${escapeHtml(preview.packAvailability ?? "unavailable")}.</p>
+    ${table(["Section", "Availability", "Items", "Estimated Tokens"], preview.sections.map((s) => [s.id, s.availability, String(s.itemCount), String(s.estimatedTokens)]))}`;
+    if (preview.task === null)
+        return head;
+    const count = (list) => `shown ${list.items.length} of ${list.totalCount}; omitted ${list.omittedCount}`;
+    return `${head}
+    <h5>1. Task</h5>
+    <p><strong>${escapeHtml(preview.task.title)}</strong> ${escapeHtml(preview.task.summary)}</p>
+    <h5>2. Relevant files (${count(preview.files)})</h5>
+    ${listHtml(preview.files.items.map((file) => `${file.path} rank=${file.rank} reason=${file.reason}`))}
+    <h5>3. Relevant symbols (${count(preview.symbols)})</h5>
+    ${listHtml(preview.symbols.items.map((symbol) => `${symbol.name} file=${symbol.file ?? "unknown"} line=${symbol.line ?? "unknown"}`))}
+    <h5>4. Source slices (${count(preview.sourceSlices)})</h5>
+    ${preview
+        .sourceSlices.items.map((slice) => `<p>${escapeHtml(`${slice.file}:${slice.startLine}-${slice.endLine}`)} boundary known: ${slice.boundaryKnown}; truncated by experiment policy: ${slice.truncated}; truncated by report preview: ${slice.previewTruncated} (showing ${slice.previewLineCount} of ${slice.lineCount} lines)</p>
+    <pre>${escapeHtml(slice.previewText)}</pre>`)
+        .join("\n    ")}
+    <h5>5. Call relationships (${count(preview.callRelationships)})</h5>
+    ${listHtml(preview.callRelationships.items.map((call) => `${call.fromNodeId} calls ${call.toNodeId}`))}
+    <h5>6. Tests (${count(preview.tests)})</h5>
+    ${listHtml(preview.tests.items.map((test) => `${test.path} how=${test.how}`))}
+    <h5>7. Evidence notes (${count(preview.evidenceNotes)})</h5>
+    ${listHtml(preview.evidenceNotes.items)}`;
+}
+function listHtml(items) {
+    return `<ul>${items.map((item) => `<li>${escapeHtml(item)}</li>`).join("")}</ul>`;
+}
+function table(headers, rows) {
+    return `<table><thead><tr>${headers.map((header) => `<th>${escapeHtml(header)}</th>`).join("")}</tr></thead><tbody>${rows
+        .map((row) => `<tr>${row.map((cell) => `<td>${escapeHtml(cell)}</td>`).join("")}</tr>`)
+        .join("")}</tbody></table>`;
+}
+function escapeHtml(value) {
+    return String(value ?? "")
+        .replaceAll("&", "&amp;")
+        .replaceAll("<", "&lt;")
+        .replaceAll(">", "&gt;")
+        .replaceAll('"', "&quot;")
+        .replaceAll("'", "&#39;");
+}

package/dist/src/report/experiments/renderContextPackGenerationText.js

@@ -0,0 +1,101 @@
+import { formatRetrievalQueryStrategyMetric, formatRetrievalQueryStrategyNumber } from "./renderRetrievalQueryStrategyComparisonText.js";
+const formatMean = (value) => (value === undefined || value === null ? "unavailable" : value.toFixed(4));
+export const formatContextPackNumber = (value) => formatRetrievalQueryStrategyNumber(value ?? null);
+const omitted = (list) => `shown=${list.items.length} total=${list.totalCount} omitted=${list.omittedCount}`;
+const REDACTED_SECTION_LABELS = {
+    files: { label: "Relevant files", what: "identities redacted" },
+    symbols: { label: "Relevant symbols", what: "identities redacted" },
+    sourceSlices: { label: "Source slices", what: "content redacted" },
+    callRelationships: { label: "Call relationships", what: "identities redacted" },
+    tests: { label: "Tests", what: "identities redacted" },
+    evidenceNotes: { label: "Evidence notes", what: "counts only" }
+};
+/**
+ * Literal, fixed-text lines for an external-local preview, generated from persisted safe counts only. Shared by the text
+ * and HTML renderers so both show identical wording; no identity or source text can reach this function.
+ */
+export function redactedPreviewDisplayLines(preview) {
+    const lines = ["Task: title and summary redacted"];
+    for (const section of preview.sections) {
+        const entry = REDACTED_SECTION_LABELS[section.id];
+        if (!entry)
+            continue;
+        const truncated = section.truncatedCount !== null ? `, ${section.truncatedCount} truncated by experiment policy` : "";
+        lines.push(`${entry.label}: ${section.itemCount} ${section.itemCount === 1 ? "item" : "items"}${truncated} — ${entry.what}`);
+    }
+    return lines;
+}
+function redactedPreviewLines(preview) {
+    const lines = [`Pack preview: case=${preview.caseId} status=${preview.status} packAvailability=${preview.packAvailability ?? "unavailable"}`];
+    for (const section of preview.sections) {
+        lines.push(`  section ${section.id}: availability=${section.availability} items=${section.itemCount} estimatedTokens=${section.estimatedTokens}${section.reason ? ` reason=${section.reason}` : ""}`);
+    }
+    for (const [index, line] of redactedPreviewDisplayLines(preview).entries())
+        lines.push(`  ${index + 1}. ${line}`);
+    return lines;
+}
+function previewLines(preview) {
+    if (preview.status === "redacted-external-local")
+        return redactedPreviewLines(preview);
+    const lines = [`Pack preview: case=${preview.caseId} status=${preview.status} packAvailability=${preview.packAvailability ?? "unavailable"}`];
+    for (const section of preview.sections) {
+        lines.push(`  section ${section.id}: availability=${section.availability} items=${section.itemCount} estimatedTokens=${section.estimatedTokens}${section.reason ? ` reason=${section.reason}` : ""}`);
+    }
+    if (preview.task === null)
+        return lines;
+    lines.push(`  1. Task: ${preview.task.title}`, `     ${preview.task.summary}`);
+    lines.push(`  2. Relevant files (${omitted(preview.files)})`);
+    for (const file of preview.files.items)
+        lines.push(`     - ${file.path} rank=${file.rank} reason=${file.reason}`);
+    lines.push(`  3. Relevant symbols (${omitted(preview.symbols)})`);
+    for (const symbol of preview.symbols.items)
+        lines.push(`     - ${symbol.name} file=${symbol.file ?? "unknown"} line=${symbol.line ?? "unknown"}`);
+    lines.push(`  4. Source slices (${omitted(preview.sourceSlices)})`);
+    for (const slice of preview.sourceSlices.items) {
+        lines.push(`     - ${slice.file}:${slice.startLine}-${slice.endLine} boundaryKnown=${slice.boundaryKnown} policyTruncated=${slice.truncated} previewTruncated=${slice.previewTruncated} (showing ${slice.previewLineCount} of ${slice.lineCount} lines)`);
+        for (const sourceLine of slice.previewText.split("\n"))
+            lines.push(`         | ${sourceLine}`);
+    }
+    lines.push(`  5. Call relationships (${omitted(preview.callRelationships)})`);
+    for (const call of preview.callRelationships.items)
+        lines.push(`     - ${call.fromNodeId} calls ${call.toNodeId}`);
+    lines.push(`  6. Tests (${omitted(preview.tests)})`);
+    for (const test of preview.tests.items)
+        lines.push(`     - ${test.path} how=${test.how}`);
+    lines.push(`  7. Evidence notes (${omitted(preview.evidenceNotes)})`);
+    for (const note of preview.evidenceNotes.items)
+        lines.push(`     - ${note}`);
+    return lines;
+}
+export function renderContextPackGenerationTextLines(section) {
+    const lines = [];
+    lines.push(`Schema: ${section.schemaVersion}`);
+    lines.push(`Experiment: ${section.experiment.pluginId} target=${section.experiment.target}`);
+    lines.push(`Treatment order: ${section.treatmentOrder.join(", ")}`);
+    lines.push(`Methodology: fileF1=${section.methodology.fileF1} symbolF1=${section.methodology.symbolF1} aggregation=${section.methodology.aggregation} comparison=${section.methodology.treatmentComparison} size=${section.methodology.sizeMeasure}`);
+    lines.push("", "Per-Case Treatments");
+    for (const entry of section.cases) {
+        for (const treatment of entry.treatments) {
+            lines.push(`- case=${entry.caseId} project=${entry.benchmarkProject} taskType=${entry.taskLocality ?? "unavailable"} treatment=${treatment.treatmentId} status=${treatment.executionStatus} availability=${treatment.availability ?? "unavailable"} fileF1=${formatRetrievalQueryStrategyMetric(treatment.fileF1)} symbolF1=${formatRetrievalQueryStrategyMetric(treatment.symbolF1)} factCoverage=${formatRetrievalQueryStrategyMetric(treatment.factCoverage)} estimatedTokens=${formatContextPackNumber(treatment.estimatedTokens)}`);
+        }
+        const c = entry.comparison;
+        lines.push(`  paired case=${entry.caseId} fileF1Delta=${formatMean(c.fileF1Delta)} symbolF1Delta=${formatMean(c.symbolF1Delta)} factCoverageDelta=${formatMean(c.factCoverageDelta)} estimatedTokenDelta=${formatContextPackNumber(c.estimatedTokenDelta)} tokensSaved=${formatContextPackNumber(c.tokensSaved)} percentSaved=${formatMean(c.percentSaved)}`);
+    }
+    lines.push("", "Scope Aggregates");
+    for (const scope of section.scopes) {
+        lines.push(`Scope ${scope.scopeId}: cases=${scope.caseCount} included=${scope.includedCaseCount} excluded=${scope.excludedCaseCount} includedIds=${scope.includedCaseIds.join(", ") || "none"} excludedIds=${scope.excludedCaseIds.join(", ") || "none"}`);
+        for (const summary of scope.treatmentSummaries) {
+            const o = summary.objectives;
+            lines.push(`- ${summary.treatmentId} completed=${summary.completedCaseCount} partial=${summary.partialCaseCount} failed=${summary.failedCaseCount} meanFileF1=${formatMean(o?.meanFileF1)} meanSymbolF1=${formatMean(o?.meanSymbolF1)} meanFactCoverage=${formatMean(o?.meanFactCoverage)} meanEstimatedTokens=${formatContextPackNumber(o?.meanEstimatedTokenCount)}`);
+        }
+        const d = scope.pairedDeltas;
+        lines.push(`  paired meanFileF1Delta=${formatMean(d?.meanFileF1Delta)} meanSymbolF1Delta=${formatMean(d?.meanSymbolF1Delta)} meanFactCoverageDelta=${formatMean(d?.meanFactCoverageDelta)} meanEstimatedTokenDelta=${formatContextPackNumber(d?.meanEstimatedTokenDelta)} meanTokensSaved=${formatContextPackNumber(scope.tokenSavings?.meanTokensSaved)} percentSavedOfMeans=${formatMean(scope.tokenSavings?.percentSavedOfMeans)}`);
+    }
+    lines.push("", "Context Pack Preview (bounded for display; stored packs are unchanged)");
+    for (const preview of section.previews)
+        lines.push(...previewLines(preview));
+    lines.push("", "Interpretation Limits");
+    for (const limitation of section.limitations)
+        lines.push(`- ${limitation}`);
+    return lines;
+}

package/dist/src/report/experiments/renderPluginExperimentReportHtml.js

@@ -4,6 +4,8 @@ import { renderIncrementalChangeStalenessHtml } from "./renderIncrementalChangeS
 import { renderContextWindowScalingHtml } from "./renderContextWindowScalingHtml.js";
 import { renderRetrievalPrecisionRecallHtml } from "./renderRetrievalPrecisionRecallHtml.js";
 import { renderRetrievalQueryStrategyComparisonHtml } from "./renderRetrievalQueryStrategyComparisonHtml.js";
+import { renderContextPackGenerationHtml } from "./renderContextPackGenerationHtml.js";
+import { renderAgentSuccessRateHtml } from "./renderAgentSuccessRateHtml.js";
 export function renderPluginExperimentReportHtml(report) {
     const redactedTarget = report.target.privacyProjection === "external-local-redacted";
     return `<!DOCTYPE html>
@@ -112,6 +114,10 @@ export function renderPluginExperimentReportHtml(report) {
 
   ${renderRetrievalQueryStrategyComparisonHtml(report.retrievalQueryStrategyComparison ?? null)}
 
+  ${renderContextPackGenerationHtml(report.contextPackGeneration ?? null)}
+
+  ${renderAgentSuccessRateHtml(report.agentSuccessRate ?? null)}
+
   <section>
     <h2>Warnings, Skips, And Failures</h2>
     ${report.findings.length ? table(["Severity", "Code", "Message", "Variant", "Case"], report.findings.map((finding) => [