@dailephd/my-dev-kit-lab 0.8.1 → 0.9.0

package/dist/src/experiments/plugins/contextPackGeneration/renderContextPack.js

@@ -0,0 +1,61 @@
+/** Stable source boundary; it cannot be confused with Markdown code-fence nesting. */
+export const CONTEXT_PACK_SOURCE_BEGIN = "<<<SOURCE";
+export const CONTEXT_PACK_SOURCE_END = "SOURCE>>>";
+export const CONTEXT_PACK_SECTION_HEADINGS = {
+    task: "## Task",
+    files: "## Relevant files",
+    symbols: "## Relevant symbols",
+    sourceSlices: "## Source slices",
+    callRelationships: "## Call relationships",
+    tests: "## Tests",
+    evidenceNotes: "## Evidence notes"
+};
+const lf = (text) => text.replace(/\r\n?/g, "\n");
+function renderBody(id, lines, availability) {
+    const heading = CONTEXT_PACK_SECTION_HEADINGS[id];
+    if (lines.length > 0)
+        return [heading, ...lines].join("\n");
+    const { availability: state, reason } = availability[id];
+    const suffix = reason === null ? "" : `: ${reason}`;
+    const placeholder = state === "available" ? "(none)" : `(${state}${suffix})`;
+    return `${heading}\n${placeholder}`;
+}
+function renderSlice(slice) {
+    return [
+        `file: ${slice.file}`,
+        `lines: ${slice.startLine}-${slice.endLine}`,
+        `boundary: ${slice.boundaryKnown ? "known" : "unknown"}`,
+        `truncated: ${slice.truncated ? "yes" : "no"}`,
+        CONTEXT_PACK_SOURCE_BEGIN,
+        lf(slice.text),
+        CONTEXT_PACK_SOURCE_END
+    ].join("\n");
+}
+/** Deterministic, language-agnostic rendering. Source text is verbatim apart from line-ending normalization. */
+export function renderContextPack(input) {
+    const availability = input.sectionAvailability;
+    const sections = {
+        task: `${CONTEXT_PACK_SECTION_HEADINGS.task}\nTitle: ${lf(input.task.title)}\nSummary: ${lf(input.task.summary)}`,
+        files: renderBody("files", input.files.map((file) => `- ${file.path}`), availability),
+        symbols: renderBody("symbols", input.symbols.map((symbol) => (symbol.file === null ? `- ${symbol.name}` : `- ${symbol.name} (${symbol.file})`)), availability),
+        sourceSlices: renderSourceSlices(input.sourceSlices, availability),
+        callRelationships: renderBody("callRelationships", input.callRelationships.map((call) => `${call.fromNodeId} -> ${call.toNodeId}`), availability),
+        tests: renderBody("tests", input.tests.map((test) => `- ${test.path}`), availability),
+        evidenceNotes: renderBody("evidenceNotes", input.evidenceNotes.map((code) => `- ${code}`), availability)
+    };
+    const renderedText = `${[
+        sections.task,
+        sections.files,
+        sections.symbols,
+        sections.sourceSlices,
+        sections.callRelationships,
+        sections.tests,
+        sections.evidenceNotes
+    ].join("\n\n")}\n`;
+    return { sections, renderedText };
+}
+function renderSourceSlices(slices, availability) {
+    if (slices.length === 0)
+        return renderBody("sourceSlices", [], availability);
+    return `${CONTEXT_PACK_SECTION_HEADINGS.sourceSlices}\n${slices.map(renderSlice).join("\n\n")}`;
+}

package/dist/src/experiments/plugins/contextPackGeneration/types.js

@@ -0,0 +1,29 @@
+/** Domain schema of the Lab-owned experimental context pack. It is a measurement artifact, not a production contract. */
+export const CONTEXT_PACK_SCHEMA_VERSION = "my-dev-kit-lab-context-pack-experiment-v1";
+/** Logical section order of the pack. */
+export const CONTEXT_PACK_SECTION_IDS = [
+    "task",
+    "files",
+    "symbols",
+    "sourceSlices",
+    "callRelationships",
+    "tests",
+    "evidenceNotes"
+];
+/**
+ * Closed fixed vocabulary. Only codes produced by the pure Batch 1 domain exist; no free-form warning text, and no
+ * execution, indexing, privacy or persistence failure codes.
+ */
+export const CONTEXT_PACK_EVIDENCE_NOTE_CODES = [
+    "symbol-end-unknown",
+    "source-slice-cap-reached",
+    "total-source-line-cap-reached",
+    "source-slice-count-cap-reached",
+    "seed-cap-reached",
+    "file-cap-reached",
+    "symbol-cap-reached",
+    "test-file-cap-reached",
+    "call-relationship-cap-reached",
+    "invalid-repository-path-ignored",
+    "invalid-source-slice-ignored"
+];

package/dist/src/experiments/plugins/warmIndexReuse/plugin.js

@@ -42,9 +42,8 @@ export const warmIndexReusePlugin = {
         const startedAt = context.startedAt.toISOString();
         // A campaign preset selects campaign-aware case selection and real-agent evaluation for
         // exactly one provider; its absence preserves the legacy fake-agent path unchanged. Real-agent
-        // campaign execution is still guarded at the public experiment-run command (v0.5.2 Batch 3):
-        // this plugin-level path is exercised by programmatic/test callers only until Batch 4
-        // implements campaign-accurate reporting.
+        // execution stays explicitly gated: the config requires includeRealAgents for a preset, and the
+        // public experiment-run command rejects incompatible options before this path is reached.
         const preset = context.config.campaignPreset ? getWarmIndexCampaignPreset(context.config.campaignPreset) : undefined;
         const cases = preset
             ? selectWarmIndexCampaignCases(readCasesInput(context.inputs), preset, context.config)

package/dist/src/report/experiments/agentSuccessRateReportModel.js

@@ -0,0 +1,6 @@
+/**
+ * Typed agent-success-rate report section. Every value here is copied from the plugin's already-calculated analysis and
+ * execution evidence: the report layer rescored nothing, reran no check, reparsed no patch and read no reference patch.
+ * It holds no prompt, source-file or patch body and no raw command output.
+ */
+export const AGENT_SUCCESS_RATE_REPORT_SCHEMA_VERSION = "my-dev-kit-lab-agent-success-rate-report-v1";

package/dist/src/report/experiments/buildAgentSuccessRateReport.js

@@ -0,0 +1,255 @@
+import { attemptEvidenceOf } from "../../experiments/plugins/agentSuccessRate/attemptEvidence.js";
+import { AGENT_SUCCESS_RATE_PLUGIN_ID, AGENT_SUCCESS_RATE_REAL_AGENT_EXECUTION_MODE, AGENT_SUCCESS_RATE_TREATMENT_IDS } from "../../experiments/plugins/agentSuccessRate/metadata.js";
+import { AGENT_SUCCESS_RATE_REPORT_SCHEMA_VERSION } from "./agentSuccessRateReportModel.js";
+export const AGENT_SUCCESS_RATE_DETERMINISTIC_STATEMENT = "The fixture validates the patch evaluation pipeline and benchmark corpus. It does not measure a coding agent or the effect of context selection.";
+export const AGENT_SUCCESS_RATE_REAL_AGENT_STATEMENT = "The observed outcomes are a descriptive matched comparison of two source-context treatments under the selected provider.";
+const MAX_TEXT = 300;
+const MAX_CHANGED_FILES = 20;
+const bound = (text) => {
+    const normalized = text.replace(/\s+/g, " ").trim();
+    return normalized.length > MAX_TEXT ? `${normalized.slice(0, MAX_TEXT)}...` : normalized;
+};
+/** Copies a plugin metric. Availability, value and reason are carried through untouched. */
+function copyMetric(metric, fallbackUnit = "count") {
+    if (!metric)
+        return { availability: "unavailable", value: null, unit: fallbackUnit, reason: "the measurement was not recorded." };
+    return { availability: metric.availability, value: metric.value, unit: metric.unit, reason: metric.reason === null ? null : bound(metric.reason) };
+}
+function copyTotal(total) {
+    return {
+        basis: total.basis,
+        contributingCaseCount: total.contributingCaseCount,
+        availableCaseCount: total.availableCaseCount,
+        unavailableCaseCount: total.unavailableCaseCount,
+        sumOfAvailable: total.sumOfAvailable,
+        total: copyMetric(total.total)
+    };
+}
+function copyDifference(entry) {
+    return {
+        id: entry.id,
+        basis: entry.basis,
+        matchedCaseIds: [...entry.matchedCaseIds],
+        rawFullFileMean: copyMetric(entry.rawFullFileMean),
+        contextPackMean: copyMetric(entry.contextPackMean),
+        meanDifference: copyMetric(entry.meanDifference)
+    };
+}
+/**
+ * A relative, forward-slash artifact reference with no traversal, drive, scheme or absolute form. Anything else is
+ * withheld rather than rendered, so a report can never point outside the experiment output directory.
+ */
+export function safeAgentSuccessArtifactReference(reference) {
+    if (typeof reference !== "string" || reference.length === 0 || reference.length > 400)
+        return null;
+    if (reference.includes("\\") || reference.includes("\0") || reference.startsWith("/") || /^[A-Za-z]:/.test(reference) || /^[A-Za-z][A-Za-z0-9+.-]*:/.test(reference))
+        return null;
+    const segments = reference.split("/");
+    if (segments.some((segment) => segment === "" || segment === "." || segment === ".."))
+        return null;
+    return reference;
+}
+function patchOutcome(evidence) {
+    return evidence.patch.outcome ?? (evidence.patch.attempted ? "unrecorded" : "not-attempted");
+}
+function verificationAvailability(evidence) {
+    return evidence.postEditVerification !== null ? "available" : "unavailable";
+}
+export function buildAgentSuccessRateReport(run) {
+    if (run.pluginId !== AGENT_SUCCESS_RATE_PLUGIN_ID)
+        return null;
+    const candidate = run;
+    const analysis = candidate.analysis;
+    const execution = candidate.caseExecutionEvidence;
+    if (!analysis || !execution) {
+        // A run that failed before analysis exists still gets the generic failure report.
+        if (run.status === "failed")
+            return null;
+        throw new Error("Invalid agent-success-rate report source: execution evidence and analysis are required.");
+    }
+    const inconsistent = analysis.cases.length !== execution.length ||
+        analysis.cases.some((entry, index) => entry.caseId !== execution[index].caseId ||
+            entry.treatments.length !== AGENT_SUCCESS_RATE_TREATMENT_IDS.length ||
+            entry.treatments.some((treatment, position) => treatment.treatmentId !== AGENT_SUCCESS_RATE_TREATMENT_IDS[position]));
+    if (inconsistent) {
+        throw new Error("Invalid agent-success-rate report source: execution evidence and analysis are inconsistent.");
+    }
+    const realMode = analysis.executionMode === AGENT_SUCCESS_RATE_REAL_AGENT_EXECUTION_MODE;
+    const metadata = (run.metadata ?? {});
+    const repairAttempts = realMode && typeof metadata.repairAttempts === "number" ? metadata.repairAttempts : realMode ? 0 : null;
+    const timeoutMs = realMode && typeof metadata.timeoutMs === "number" ? metadata.timeoutMs : null;
+    const cases = analysis.cases.map((entry, caseIndex) => {
+        const evidence = execution[caseIndex];
+        return {
+            caseId: entry.caseId,
+            caseName: bound(evidence.caseName),
+            benchmarkProject: entry.benchmarkProject,
+            taskLocality: evidence.taskLocality,
+            treatments: entry.treatments.map((treatment, treatmentIndex) => {
+                const treatmentEvidence = evidence.treatments[treatmentIndex];
+                const repair = treatment.repair;
+                const changed = treatmentEvidence.change?.changedFiles.map((file) => file.relativePath).filter((file) => safeAgentSuccessArtifactReference(file) !== null) ?? null;
+                const errors = attemptEvidenceOf(treatmentEvidence).flatMap((attempt, index) => attempt.errors.slice(0, 10).map((error) => ({ attemptNumber: index + 1, code: bound(error.code), message: bound(error.message) })));
+                return {
+                    treatmentId: treatment.treatmentId,
+                    executionStatus: treatment.executionStatus,
+                    evidenceAvailability: treatment.evidenceAvailability,
+                    initialTaskSuccess: copyMetric(repair ? repair.initialAttemptTaskSuccess : treatment.metrics.taskSuccess, "boolean"),
+                    finalTaskSuccess: copyMetric(treatment.metrics.taskSuccess, "boolean"),
+                    attemptCount: repair ? repair.attemptCount : 1,
+                    repairAttemptCount: repair ? repair.repairAttemptCount : 0,
+                    repairSucceeded: repair
+                        ? copyMetric(repair.repairSucceeded, "boolean")
+                        : { availability: "not-applicable", value: null, unit: "boolean", reason: "deterministic-fixture mode has no repair." },
+                    taskCheckPassRate: copyMetric(treatment.metrics.taskCheckPassRate, "ratio"),
+                    regressionCheckPassRate: copyMetric(treatment.metrics.regressionCheckPassRate, "ratio"),
+                    requiredFactCoverage: copyMetric(treatment.metrics.requiredFactCoverage, "ratio"),
+                    expectedEditCoverage: copyMetric(treatment.metrics.expectedEditCoverage, "ratio"),
+                    editScopePrecision: copyMetric(treatment.metrics.editScopePrecision, "ratio"),
+                    unexpectedChangedFiles: copyMetric(treatment.metrics.unexpectedChangedFileCount, "count"),
+                    protectedMutations: copyMetric(treatment.metrics.protectedMutationCount, "count"),
+                    changedFileCount: copyMetric(treatment.metrics.changedFileCount, "count"),
+                    changedFiles: changed === null || treatment.metrics.changedFileCount.availability !== "available"
+                        ? null
+                        : { items: changed.slice(0, MAX_CHANGED_FILES), totalCount: changed.length, omittedCount: Math.max(0, changed.length - MAX_CHANGED_FILES) },
+                    totalChurn: copyMetric(treatment.metrics.totalChurn, "lines"),
+                    relativeChurn: copyMetric(treatment.metrics.relativeChurn, "ratio"),
+                    finalAttemptProviderDurationMs: copyMetric(treatment.metrics.agentDurationMs, "ms"),
+                    finalAttemptProviderTokens: copyMetric(treatment.metrics.agentTotalTokens, "tokens"),
+                    totalProviderDurationMs: repair ? copyMetric(repair.totalProviderDurationMs, "ms") : null,
+                    totalProviderTokens: repair ? copyMetric(repair.totalProviderTokens, "tokens") : null,
+                    errors: errors.slice(0, 20)
+                };
+            })
+        };
+    });
+    const treatments = analysis.aggregates.map((aggregate) => {
+        const repair = aggregate.repair;
+        return {
+            treatmentId: aggregate.treatmentId,
+            evaluableCases: repair ? repair.finalEvaluableCount : aggregate.evaluableCaseCount,
+            initialSuccessfulCases: repair ? repair.initialAttemptSuccessfulCount : aggregate.successfulCaseCount,
+            initialSuccessRate: copyMetric(aggregate.initialAttemptSuccessRate, "ratio"),
+            finalSuccessfulCases: repair ? repair.finalSuccessfulCount : aggregate.successfulCaseCount,
+            finalSuccessRate: copyMetric(repair ? repair.finalTaskSuccessRate : aggregate.taskSuccessRate, "ratio"),
+            repair: repair
+                ? {
+                    repairEligibleCases: repair.repairEligibleCaseCount,
+                    repairAttemptedCases: repair.repairAttemptedCaseCount,
+                    repairAttempts: repair.totalRepairAttemptCount,
+                    repairedCases: repair.repairedCaseCount,
+                    repairSuccessRate: copyMetric(repair.repairSuccessRate, "ratio"),
+                    meanAttemptsPerEvaluableCase: copyMetric(repair.meanAttemptsPerEvaluableCase, "count"),
+                    providerDurationMs: repair.providerDurationMs.map(copyTotal),
+                    providerTokens: repair.providerTokens.map(copyTotal)
+                }
+                : null
+        };
+    });
+    const repairHistory = realMode
+        ? analysis.cases.flatMap((entry, caseIndex) => entry.treatments.flatMap((treatment, treatmentIndex) => {
+            const repair = treatment.repair;
+            const evidence = execution[caseIndex].treatments[treatmentIndex];
+            if (!repair || !evidence.attempts)
+                return [];
+            return [
+                {
+                    caseId: entry.caseId,
+                    treatmentId: treatment.treatmentId,
+                    attempts: repair.attempts.map((attempt, index) => {
+                        const attemptEvidence = evidence.attempts[index];
+                        const inner = attemptEvidence.evidence;
+                        return {
+                            attemptNumber: attempt.attemptNumber,
+                            providerStatus: attempt.providerStatus,
+                            patchOutcome: patchOutcome(inner),
+                            verificationAvailability: verificationAvailability(inner),
+                            taskSuccess: copyMetric(attempt.metrics.taskSuccess, "boolean"),
+                            failureCategory: attempt.failureCategory,
+                            repairEligible: attempt.repairEligible,
+                            providerDurationMs: copyMetric(attempt.metrics.agentDurationMs, "ms"),
+                            providerTokens: copyMetric(attempt.metrics.agentTotalTokens, "tokens"),
+                            tokenAvailability: attempt.metrics.agentTotalTokens.availability === "available" ? "available" : "unavailable",
+                            proposedPatchPath: safeAgentSuccessArtifactReference(inner.proposedPatchPath),
+                            appliedPatchPath: safeAgentSuccessArtifactReference(inner.appliedPatchPath),
+                            agentArtifactDirectory: safeAgentSuccessArtifactReference(inner.realAgent?.agentArtifactDirectory ?? null)
+                        };
+                    })
+                }
+            ];
+        }))
+        : [];
+    const outcomeAvailability = analysis.cases.flatMap((entry) => entry.treatments.map((treatment) => treatment.metrics.taskSuccess.availability));
+    const evaluable = outcomeAvailability.filter((availability) => availability === "available").length;
+    const total = outcomeAvailability.length;
+    const unavailable = total - evaluable;
+    const notes = [];
+    if (unavailable > 0)
+        notes.push(`${unavailable} of ${total} treatment outcome(s) have no determinate task-success verdict; they are excluded from rates and never counted as failures.`);
+    if (realMode && analysis.comparison && analysis.comparison.incompleteCases.length > 0) {
+        notes.push(`${analysis.comparison.incompleteCases.length} case(s) are excluded from matched comparisons because a treatment verdict is unavailable.`);
+    }
+    const comparison = analysis.comparison
+        ? (() => {
+            const repairComparison = analysis.repairComparison;
+            const finalMatched = analysis.comparison.matchedCaseIds.length;
+            const evaluableComparison = repairComparison ? repairComparison.interpretation.evaluable : finalMatched > 0;
+            const finalPairs = analysis.comparison.pairedOutcomes;
+            const differs = finalPairs.onlyRawFullFileSucceeded + finalPairs.onlyContextPackSucceeded > 0;
+            return {
+                comparisonEvaluable: evaluableComparison,
+                observedOutcomeDifference: repairComparison ? repairComparison.interpretation.observedOutcomeDifference : !evaluableComparison ? "not-evaluable" : differs ? "observed" : "not-observed",
+                statisticalEffect: "not-assessed",
+                finalMatchedCaseIds: [...analysis.comparison.matchedCaseIds],
+                finalIncompleteCases: analysis.comparison.incompleteCases.map((entry) => ({ caseId: entry.caseId, unavailableTreatmentIds: [...entry.unavailableTreatmentIds] })),
+                finalPairedOutcomes: { ...finalPairs },
+                initialMatchedCaseIds: repairComparison ? [...repairComparison.initialAttempt.matchedCaseIds] : null,
+                initialIncompleteCases: repairComparison ? repairComparison.initialAttempt.incompleteCases.map((entry) => ({ caseId: entry.caseId, unavailableTreatmentIds: [...entry.unavailableTreatmentIds] })) : null,
+                initialPairedOutcomes: repairComparison ? { ...repairComparison.initialAttempt.pairedOutcomes } : null,
+                pairedDifferences: repairComparison ? repairComparison.pairedDifferences.map(copyDifference) : []
+            };
+        })()
+        : null;
+    const overall = total === 0 || evaluable === 0 ? "unavailable" : unavailable === 0 ? "complete" : "partial";
+    return {
+        schemaVersion: AGENT_SUCCESS_RATE_REPORT_SCHEMA_VERSION,
+        identity: {
+            runId: run.runId,
+            pluginId: run.pluginId,
+            pluginSchemaVersion: "1.0.0",
+            runStatus: run.status,
+            executionMode: analysis.executionMode,
+            providerId: typeof metadata.providerId === "string" ? metadata.providerId : null,
+            timeoutMs,
+            repairAttempts,
+            maxAttemptsPerTreatment: repairAttempts === null ? 1 : 1 + repairAttempts,
+            treatmentOrder: [...analysis.treatmentOrder],
+            caseCount: analysis.caseCount,
+            treatmentOutcomeCount: total
+        },
+        scientificStatement: realMode ? AGENT_SUCCESS_RATE_REAL_AGENT_STATEMENT : AGENT_SUCCESS_RATE_DETERMINISTIC_STATEMENT,
+        limitations: [
+            ...analysis.limitations.map(bound),
+            ...(realMode
+                ? [
+                    "The corpus is finite and a single run supports no generalization.",
+                    "Provider output can vary between runs, and provider usage or availability can differ between treatments.",
+                    "No causal or statistical-significance claim is made; no weighted composite score and no automatic best treatment exist."
+                ]
+                : [])
+        ],
+        resultAvailability: { overall, evaluableTreatmentOutcomes: evaluable, unavailableTreatmentOutcomes: unavailable, totalTreatmentOutcomes: total, notes },
+        treatments,
+        comparison,
+        cases,
+        repairHistory,
+        warnings: run.warnings.map((warning) => ({ code: bound(warning.code), message: bound(warning.message) })),
+        artifacts: run.artifacts.map((artifact) => ({
+            id: bound(artifact.id),
+            label: bound(artifact.label),
+            path: safeAgentSuccessArtifactReference(artifact.path),
+            caseId: artifact.caseId ?? null,
+            variantId: artifact.variantId ?? null
+        }))
+    };
+}

package/dist/src/report/experiments/buildContextPackGenerationReport.js

@@ -0,0 +1,171 @@
+import path from "node:path";
+import { CONTEXT_PACK_GENERATION_METHODOLOGY } from "../../experiments/plugins/contextPackGeneration/analysisArtifact.js";
+import { CONTEXT_PACK_GENERATION_PLUGIN_ID, CONTEXT_PACK_GENERATION_TREATMENT_IDS } from "../../experiments/plugins/contextPackGeneration/metadata.js";
+import { CONTEXT_PACK_GENERATION_REPORT_SCHEMA_VERSION, PACK_PREVIEW_MAX_CALL_RELATIONSHIPS, PACK_PREVIEW_MAX_EVIDENCE_NOTES, PACK_PREVIEW_MAX_FILES, PACK_PREVIEW_MAX_SOURCE_LINES_PER_SLICE, PACK_PREVIEW_MAX_SOURCE_SLICES, PACK_PREVIEW_MAX_SYMBOLS, PACK_PREVIEW_MAX_TESTS } from "./contextPackGenerationReportModel.js";
+/** Fixed wording: observed evidence only. */
+export const CONTEXT_PACK_GENERATION_LIMITATIONS = [
+    "This is a context-pack experiment over deterministic my-dev-kit workflows; it is not a coding-agent success evaluation.",
+    "Estimated tokens come from the existing Lab estimator over rendered text, not provider billing telemetry.",
+    "Unavailable and not-applicable values are not converted to zero.",
+    "Scope means use matched complete cases only and are unweighted macro means over task instances.",
+    "No composite score, ranking, or winning treatment is produced; treatments are reported side by side.",
+    "Tests and call relationships in the pack are descriptive and are not scored.",
+    "The context pack is an experimental Lab artifact, not the production my-dev-kit context API.",
+    "The pack preview is bounded for display only and does not change the stored pack or any measured size."
+];
+const copyMetric = (metric) => ({
+    availability: metric.availability,
+    numerator: metric.numerator,
+    denominator: metric.denominator,
+    value: metric.value,
+    reason: metric.reason
+});
+const NO_MEASUREMENT = { availability: "unavailable", numerator: null, denominator: null, value: null, reason: "no-retrieval-measurement" };
+function bounded(source, limit, project) {
+    return { items: source.slice(0, limit).map(project), totalCount: source.length, omittedCount: Math.max(0, source.length - limit) };
+}
+/** Display truncation of a stored slice: first lines only. The pack's own `truncated` flag is copied untouched. */
+function previewSlice(slice) {
+    const lines = slice.text.split("\n");
+    const shown = lines.slice(0, PACK_PREVIEW_MAX_SOURCE_LINES_PER_SLICE);
+    return {
+        file: slice.file,
+        symbolName: slice.symbolName,
+        startLine: slice.startLine,
+        endLine: slice.endLine,
+        lineCount: slice.lineCount,
+        boundaryKnown: slice.boundaryKnown,
+        truncated: slice.truncated,
+        previewText: shown.join("\n"),
+        previewLineCount: shown.length,
+        previewTruncated: lines.length > shown.length
+    };
+}
+/** External-local evidence carries a redaction marker and no pack body; the preview is counts and availability only. */
+function buildRedactedPreview(entry, redaction) {
+    const packTreatment = entry.treatments.find((treatment) => treatment.treatmentId === "context-pack");
+    return {
+        caseId: entry.caseId,
+        status: "redacted-external-local",
+        packArtifactPath: null,
+        packAvailability: packTreatment?.availability ?? null,
+        sections: (packTreatment?.sections ?? []).map((section) => ({
+            id: section.id,
+            availability: section.availability,
+            reason: section.reason,
+            itemCount: section.itemCount,
+            estimatedTokens: section.estimatedTokens,
+            truncatedCount: section.truncatedCount ?? null
+        })),
+        task: null,
+        files: null,
+        symbols: null,
+        sourceSlices: null,
+        callRelationships: null,
+        tests: null,
+        evidenceNotes: null,
+        redaction: { ...redaction }
+    };
+}
+function buildPreview(entry, pack) {
+    if (entry.identityRedaction)
+        return buildRedactedPreview(entry, entry.identityRedaction);
+    const packTreatment = entry.treatments.find((treatment) => treatment.treatmentId === "context-pack");
+    const packArtifactPath = packTreatment?.packArtifactPath ?? null;
+    const base = {
+        caseId: entry.caseId,
+        packArtifactPath,
+        // Section summaries are persisted execution evidence; they never come from the preview truncation.
+        sections: (packTreatment?.sections ?? []).map((section) => ({
+            id: section.id,
+            availability: section.availability,
+            reason: section.reason,
+            itemCount: section.itemCount,
+            estimatedTokens: section.estimatedTokens
+        }))
+    };
+    if (!pack) {
+        return {
+            ...base,
+            status: packArtifactPath === null ? "no-pack-produced" : "pack-artifact-unavailable",
+            packAvailability: packTreatment?.availability ?? null,
+            task: null,
+            files: null,
+            symbols: null,
+            sourceSlices: null,
+            callRelationships: null,
+            tests: null,
+            evidenceNotes: null
+        };
+    }
+    return {
+        ...base,
+        status: "available",
+        packAvailability: pack.availability,
+        task: { title: pack.task.title, summary: pack.task.summary },
+        files: bounded(pack.files, PACK_PREVIEW_MAX_FILES, (file) => ({ path: file.path, rank: file.rank, reason: file.reason })),
+        symbols: bounded(pack.symbols, PACK_PREVIEW_MAX_SYMBOLS, (symbol) => ({ name: symbol.name, file: symbol.file, line: symbol.line, nodeId: symbol.nodeId })),
+        sourceSlices: bounded(pack.sourceSlices, PACK_PREVIEW_MAX_SOURCE_SLICES, previewSlice),
+        callRelationships: bounded(pack.callRelationships.filter((relationship) => relationship.kind === "calls"), PACK_PREVIEW_MAX_CALL_RELATIONSHIPS, (relationship) => ({ fromNodeId: relationship.fromNodeId, toNodeId: relationship.toNodeId })),
+        tests: bounded(pack.tests, PACK_PREVIEW_MAX_TESTS, (test) => ({ path: test.path, how: test.how })),
+        evidenceNotes: bounded(pack.evidenceNotes, PACK_PREVIEW_MAX_EVIDENCE_NOTES, (code) => code)
+    };
+}
+/**
+ * Presentation-only adapter over the already-calculated `run.analysis` and `run.caseExecutionEvidence`. It recalculates no
+ * F1, fact coverage, token size, saving, matched-case membership, mean or delta; it copies persisted values. `contextPacks`
+ * are the persisted per-case pack artifacts, loaded by the caller; only display truncation is derived from them.
+ */
+export function buildContextPackGenerationReport(run, contextPacks = new Map()) {
+    if (run.pluginId !== CONTEXT_PACK_GENERATION_PLUGIN_ID)
+        return null;
+    const candidate = run;
+    const analysis = candidate.analysis;
+    const execution = candidate.caseExecutionEvidence;
+    if (!analysis || !execution) {
+        // A run that failed before analysis exists still gets the generic failure report.
+        if (run.status === "failed")
+            return null;
+        throw new Error("Invalid context-pack-generation report source: execution evidence and analysis are required.");
+    }
+    const inconsistent = analysis.cases.length !== execution.length ||
+        analysis.cases.some((entry, index) => entry.caseId !== execution[index].caseId ||
+            entry.treatments.length !== CONTEXT_PACK_GENERATION_TREATMENT_IDS.length ||
+            entry.treatments.some((treatment, position) => treatment.treatmentId !== CONTEXT_PACK_GENERATION_TREATMENT_IDS[position]));
+    if (inconsistent) {
+        throw new Error("Invalid context-pack-generation report source: execution evidence and analysis are inconsistent.");
+    }
+    const cases = analysis.cases.map((entry, caseIndex) => ({
+        caseId: entry.caseId,
+        caseName: execution[caseIndex].caseName,
+        benchmarkProject: entry.benchmarkProject,
+        taskLocality: entry.taskLocality,
+        treatments: entry.treatments.map((treatment) => ({
+            treatmentId: treatment.treatmentId,
+            executionStatus: treatment.executionStatus,
+            availability: treatment.availability,
+            fileF1: copyMetric(treatment.fileF1),
+            symbolF1: copyMetric(treatment.symbolF1),
+            factCoverage: treatment.quality ? copyMetric(treatment.quality.fact.coverage) : { ...NO_MEASUREMENT },
+            estimatedTokens: treatment.estimatedTokens,
+            tokenCountMethod: treatment.tokenCountMethod
+        })),
+        comparison: { ...entry.comparison }
+    }));
+    return {
+        schemaVersion: CONTEXT_PACK_GENERATION_REPORT_SCHEMA_VERSION,
+        experiment: { pluginId: run.pluginId, target: run.target.isSelf ? "self" : "external target" },
+        treatmentOrder: [...CONTEXT_PACK_GENERATION_TREATMENT_IDS],
+        methodology: { ...CONTEXT_PACK_GENERATION_METHODOLOGY },
+        cases,
+        scopes: structuredClone(analysis.scopes),
+        previews: execution.map((entry) => buildPreview(entry, contextPacks.get(entry.caseId))),
+        // Execution/analysis artifacts live directly in the output root; never expose a machine-local absolute path.
+        artifacts: run.artifacts.map((artifact) => ({
+            id: artifact.id,
+            path: artifact.path ? (path.isAbsolute(artifact.path) ? path.basename(artifact.path) : artifact.path) : null,
+            caseId: artifact.caseId ?? null
+        })),
+        limitations: [...CONTEXT_PACK_GENERATION_LIMITATIONS]
+    };
+}

package/dist/src/report/experiments/buildPluginExperimentReport.js

@@ -5,6 +5,8 @@ import { buildIncrementalChangeStalenessPluginReport } from "./buildIncrementalC
 import { buildContextWindowScalingReport } from "./buildContextWindowScalingReport.js";
 import { buildRetrievalPrecisionRecallReport } from "./buildRetrievalPrecisionRecallReport.js";
 import { buildRetrievalQueryStrategyComparisonReport } from "./buildRetrievalQueryStrategyComparisonReport.js";
+import { buildContextPackGenerationReport } from "./buildContextPackGenerationReport.js";
+import { AGENT_SUCCESS_RATE_DETERMINISTIC_STATEMENT, AGENT_SUCCESS_RATE_REAL_AGENT_STATEMENT, buildAgentSuccessRateReport } from "./buildAgentSuccessRateReport.js";
 // Bulk context-window-scaling evidence is presented by the typed report section; the complete
 // evidence stays in context-window-scaling-execution.json.
 const CONTEXT_WINDOW_SCALING_BULK_KEYS = ["executionEvidence", "aggregate"];
@@ -12,6 +14,10 @@ const CONTEXT_WINDOW_SCALING_BULK_KEYS = ["executionEvidence", "aggregate"];
 const RETRIEVAL_PRECISION_RECALL_BULK_KEYS = ["caseExecutionEvidence", "aggregate"];
 // Same for retrieval-query-strategy-comparison: the typed section presents execution evidence and analysis.
 const RETRIEVAL_QUERY_STRATEGY_COMPARISON_BULK_KEYS = ["caseExecutionEvidence", "analysis"];
+// Same for context-pack-generation: the typed section presents execution evidence and analysis.
+const CONTEXT_PACK_GENERATION_BULK_KEYS = ["caseExecutionEvidence", "analysis"];
+// Same for agent-success-rate: the typed section presents execution evidence and analysis.
+const AGENT_SUCCESS_RATE_BULK_KEYS = ["caseExecutionEvidence", "analysis"];
 const V043_BULK_ARRAY_KEYS = [
     "v043StageContextExecutions",
     "v043StageContextEvaluations",
@@ -26,6 +32,8 @@ export function buildPluginExperimentReport(args) {
     const contextWindowScaling = buildContextWindowScalingReport(args.run);
     const retrievalPrecisionRecall = buildRetrievalPrecisionRecallReport(args.run);
     const retrievalQueryStrategyComparison = buildRetrievalQueryStrategyComparisonReport(args.run);
+    const contextPackGeneration = buildContextPackGenerationReport(args.run, args.contextPacks);
+    const agentSuccessRate = buildAgentSuccessRateReport(args.run);
     const rawRun = { ...args.run, artifacts: relativizeArtifacts(args.run.artifacts, outputRoot) };
     for (const key of V043_BULK_ARRAY_KEYS) {
         delete rawRun[key];
@@ -45,6 +53,16 @@ export function buildPluginExperimentReport(args) {
             delete rawRun[key];
         }
     }
+    if (contextPackGeneration) {
+        for (const key of CONTEXT_PACK_GENERATION_BULK_KEYS) {
+            delete rawRun[key];
+        }
+    }
+    if (agentSuccessRate) {
+        for (const key of AGENT_SUCCESS_RATE_BULK_KEYS) {
+            delete rawRun[key];
+        }
+    }
     return {
         metadata: {
             generatedAt: args.generatedAt ?? new Date().toISOString(),
@@ -73,8 +91,10 @@ export function buildPluginExperimentReport(args) {
         contextWindowScaling,
         retrievalPrecisionRecall,
         retrievalQueryStrategyComparison,
+        contextPackGeneration,
+        agentSuccessRate,
         contextStrategyComparisonV043,
-        interpretation: buildInterpretation(args.run, contextStrategyComparisonV043, warmIndexReuse, contextWindowScaling, retrievalPrecisionRecall, retrievalQueryStrategyComparison),
+        interpretation: buildInterpretation(args.run, contextStrategyComparisonV043, warmIndexReuse, contextWindowScaling, retrievalPrecisionRecall, retrievalQueryStrategyComparison, contextPackGeneration, agentSuccessRate),
         rawRun,
     };
 }
@@ -153,7 +173,34 @@ function affectedNeighborhoodSentence(report) {
         `reindex recommendations were recommended=${summary.recommendedReindexCount}, not-indicated=${summary.notIndicatedReindexCount}, unknown=${summary.unknownReindexRecommendationCount}. ` +
         "Affected-neighborhood evidence uses a one-hop baseline graph around confirmed changed indexed files/symbols. It is observational and does not alter retrieval or execution status.");
 }
-function buildInterpretation(run, contextStrategyComparisonV043, warmIndexReuse, contextWindowScaling, retrievalPrecisionRecall, retrievalQueryStrategyComparison) {
+function buildInterpretation(run, contextStrategyComparisonV043, warmIndexReuse, contextWindowScaling, retrievalPrecisionRecall, retrievalQueryStrategyComparison, contextPackGeneration = null, agentSuccessRate = null) {
+    if (agentSuccessRate) {
+        const real = agentSuccessRate.identity.executionMode === "real-agent";
+        const [raw, pack] = agentSuccessRate.treatments;
+        const rate = (treatment, kind) => {
+            const metric = kind === "initial" ? treatment?.initialSuccessRate : treatment?.finalSuccessRate;
+            return metric && metric.availability === "available" && typeof metric.value === "number" ? metric.value.toFixed(4) : "unavailable";
+        };
+        const summary = real
+            ? `${AGENT_SUCCESS_RATE_REAL_AGENT_STATEMENT} Provider ${agentSuccessRate.identity.providerId ?? "unknown"} ran ${agentSuccessRate.identity.caseCount} case(s) with up to ${agentSuccessRate.identity.maxAttemptsPerTreatment} attempt(s) per treatment. Initial-attempt success rate: raw-full-file ${rate(raw, "initial")}, context-pack ${rate(pack, "initial")}; final success rate: raw-full-file ${rate(raw, "final")}, context-pack ${rate(pack, "final")}. No treatment is declared best and no statistical or causal claim is made.`
+            : `${AGENT_SUCCESS_RATE_DETERMINISTIC_STATEMENT} ${agentSuccessRate.identity.caseCount} case(s) were evaluated for each treatment identity.`;
+        return {
+            summary,
+            recommendedNextStep: "Review per-case results, the repair history and the scientific limitations; unavailable evidence is never counted as a failure or as zero."
+        };
+    }
+    if (contextPackGeneration) {
+        const overall = contextPackGeneration.scopes.find((scope) => scope.scopeId === "overall");
+        const saved = overall?.tokenSavings;
+        const deltas = overall?.pairedDeltas;
+        const summary = overall && saved && deltas
+            ? `Overall matched comparison over ${overall.includedCaseCount} of ${overall.caseCount} case(s): the context pack used a mean of ${saved.meanTokensSaved} fewer estimated tokens than raw full-file context (${saved.percentSavedOfMeans}% of the raw mean; a negative value means more tokens); mean fact coverage delta ${deltas.meanFactCoverageDelta}, mean file F1 delta ${deltas.meanFileF1Delta}, mean symbol F1 delta ${deltas.meanSymbolF1Delta}. The treatments are reported side by side without a composite score.`
+            : "No matched complete-case context-pack comparison was available for the overall scope. Review treatment availability and excluded cases before drawing conclusions.";
+        return {
+            summary,
+            recommendedNextStep: "Review per-case coverage and size, the scope aggregates, and the bounded pack preview; unavailable values are never treated as zero."
+        };
+    }
     if (retrievalQueryStrategyComparison) {
         const overall = retrievalQueryStrategyComparison.scopes.find((scope) => scope.scopeId === "overall");
         const taskTypes = "Task-type results are reported separately for localized, cross-module, and broad-change cases.";

package/dist/src/report/experiments/contextPackGenerationReportModel.js

@@ -0,0 +1,9 @@
+export const CONTEXT_PACK_GENERATION_REPORT_SCHEMA_VERSION = "my-dev-kit-lab-context-pack-generation-report-v1";
+/** Report-presentation limits only. They never change the experimental pack limits and are not CLI configuration. */
+export const PACK_PREVIEW_MAX_FILES = 5;
+export const PACK_PREVIEW_MAX_SYMBOLS = 5;
+export const PACK_PREVIEW_MAX_SOURCE_SLICES = 3;
+export const PACK_PREVIEW_MAX_SOURCE_LINES_PER_SLICE = 12;
+export const PACK_PREVIEW_MAX_CALL_RELATIONSHIPS = 5;
+export const PACK_PREVIEW_MAX_TESTS = 5;
+export const PACK_PREVIEW_MAX_EVIDENCE_NOTES = 5;