@dailephd/my-dev-kit-lab 0.8.2 → 0.9.0

package/dist/src/report/experiments/renderAgentSuccessRateText.js

@@ -0,0 +1,139 @@
+/** One display form for every metric: an absent value is labelled, never rendered as zero. */
+export function formatAgentSuccessMetric(metric) {
+    if (!metric)
+        return "unavailable (not recorded)";
+    if (metric.availability === "unavailable")
+        return `unavailable (${metric.reason ?? "no reason recorded"})`;
+    if (metric.availability === "not-applicable")
+        return `not applicable (${metric.reason ?? "no reason recorded"})`;
+    const value = metric.value;
+    if (typeof value === "boolean")
+        return value ? "yes" : "no";
+    if (typeof value !== "number")
+        return "unavailable (no value)";
+    switch (metric.unit) {
+        case "ratio":
+            return value.toFixed(4);
+        case "ms":
+            return `${Math.round(value)} ms`;
+        case "tokens":
+            return `${Math.round(value)} tokens`;
+        case "lines":
+            return `${Math.round(value)} lines`;
+        default:
+            return Number.isInteger(value) ? String(value) : value.toFixed(4);
+    }
+}
+export function formatAgentSuccessTotal(total) {
+    const base = `${formatAgentSuccessMetric(total.total)}; contributing cases=${total.contributingCaseCount}, measured=${total.availableCaseCount}, unmeasured=${total.unavailableCaseCount}`;
+    return total.unavailableCaseCount > 0 && total.sumOfAvailable !== null ? `${base}; sum of measured cases only=${total.sumOfAvailable} (not a complete total)` : base;
+}
+export function formatAgentSuccessPairs(pairs) {
+    if (!pairs)
+        return "not recorded";
+    return `both succeeded=${pairs.bothSucceeded}, only raw-full-file=${pairs.onlyRawFullFileSucceeded}, only context-pack=${pairs.onlyContextPackSucceeded}, neither=${pairs.neitherSucceeded}`;
+}
+export const AGENT_SUCCESS_BASIS_LABEL = {
+    "first-attempt": "first attempt",
+    "final-attempt": "final attempt",
+    "total-across-attempts": "total across attempts"
+};
+const sanitize = (value) => String(value ?? "").replace(/[\r\n\t]+/g, " ").trim();
+/** Text rendering of the typed section. It formats already-calculated values and recalculates none of them. */
+export function renderAgentSuccessRateTextLines(report) {
+    const realMode = report.identity.executionMode === "real-agent";
+    const lines = [];
+    const section = (title) => {
+        lines.push("", title, "-".repeat(title.length));
+    };
+    lines.push("", "AGENT SUCCESS RATE", "==================");
+    section("A. Experiment identity");
+    lines.push(`Run ID: ${sanitize(report.identity.runId)}`, `Plugin: ${sanitize(report.identity.pluginId)} (schema ${sanitize(report.identity.pluginSchemaVersion)})`, `Run status: ${sanitize(report.identity.runStatus)}`, `Cases: ${report.identity.caseCount}; treatments: ${report.identity.treatmentOrder.join(", ")}; treatment outcomes: ${report.identity.treatmentOutcomeCount}`);
+    section("B. Execution mode and provider");
+    lines.push(`Execution mode: ${sanitize(report.identity.executionMode)}`, `Provider: ${report.identity.providerId ?? "none (no coding agent was invoked)"}`, `Provider timeout per attempt: ${report.identity.timeoutMs === null ? "not applicable" : `${report.identity.timeoutMs} ms`}`, `Repair attempts allowed: ${report.identity.repairAttempts === null ? "not applicable" : report.identity.repairAttempts}; maximum attempts per treatment: ${report.identity.maxAttemptsPerTreatment}`);
+    section("C. Scientific limitations");
+    lines.push(sanitize(report.scientificStatement));
+    for (const limitation of report.limitations)
+        lines.push(`- ${sanitize(limitation)}`);
+    section("D. Overall result availability");
+    const availability = report.resultAvailability;
+    lines.push(`Overall: ${availability.overall}`, `Treatment outcomes with a determinate task-success verdict: ${availability.evaluableTreatmentOutcomes} of ${availability.totalTreatmentOutcomes}; without: ${availability.unavailableTreatmentOutcomes}`);
+    for (const note of availability.notes)
+        lines.push(`- ${sanitize(note)}`);
+    section("E. Treatment comparison");
+    for (const treatment of report.treatments) {
+        lines.push(`Treatment ${treatment.treatmentId}:`, `  evaluable cases: ${treatment.evaluableCases}`, `  initial-attempt successful cases: ${treatment.initialSuccessfulCases}; initial success rate: ${formatAgentSuccessMetric(treatment.initialSuccessRate)}`, `  final successful cases: ${treatment.finalSuccessfulCases}; final success rate: ${formatAgentSuccessMetric(treatment.finalSuccessRate)}`);
+        if (treatment.repair) {
+            lines.push(`  repair-eligible cases: ${treatment.repair.repairEligibleCases}; repair attempts executed: ${treatment.repair.repairAttempts}; cases with a repair: ${treatment.repair.repairAttemptedCases}; repaired cases: ${treatment.repair.repairedCases}`, `  repair success rate: ${formatAgentSuccessMetric(treatment.repair.repairSuccessRate)}; mean attempts per evaluable case: ${formatAgentSuccessMetric(treatment.repair.meanAttemptsPerEvaluableCase)}`);
+        }
+        else {
+            lines.push("  repair: not applicable (deterministic-fixture mode; the single fixture attempt is both initial and final)");
+        }
+    }
+    if (report.comparison) {
+        const comparison = report.comparison;
+        lines.push("Matched comparison (descriptive; no winner is declared):", `  comparison evaluable: ${comparison.comparisonEvaluable ? "yes" : "no"}`, `  observed outcome difference between treatments: ${comparison.observedOutcomeDifference}`, `  statistical or causal effect: ${comparison.statisticalEffect} (no significance or causal claim is made)`, `  final matched cases: ${comparison.finalMatchedCaseIds.length > 0 ? comparison.finalMatchedCaseIds.join(", ") : "none"}`, `  final paired outcomes: ${formatAgentSuccessPairs(comparison.finalPairedOutcomes)}`);
+        if (comparison.initialMatchedCaseIds) {
+            lines.push(`  initial-attempt matched cases: ${comparison.initialMatchedCaseIds.length > 0 ? comparison.initialMatchedCaseIds.join(", ") : "none"}`, `  initial-attempt paired outcomes: ${formatAgentSuccessPairs(comparison.initialPairedOutcomes)}`);
+        }
+        for (const incomplete of comparison.finalIncompleteCases)
+            lines.push(`  excluded from final matched metrics: ${incomplete.caseId} (no determinate verdict for ${incomplete.unavailableTreatmentIds.join(", ")})`);
+        for (const difference of comparison.pairedDifferences) {
+            lines.push(`  paired difference ${difference.id} [${AGENT_SUCCESS_BASIS_LABEL[difference.basis] ?? difference.basis}] over ${difference.matchedCaseIds.length} matched case(s): raw-full-file ${formatAgentSuccessMetric(difference.rawFullFileMean)}; context-pack ${formatAgentSuccessMetric(difference.contextPackMean)}; context-pack minus raw-full-file ${formatAgentSuccessMetric(difference.meanDifference)}`);
+        }
+    }
+    else {
+        lines.push("Matched comparison: not applicable (deterministic-fixture mode does not compare context treatments).");
+    }
+    section("F. Per-case results");
+    for (const entry of report.cases) {
+        lines.push(`Case ${entry.caseId} (${sanitize(entry.caseName)}) project=${entry.benchmarkProject} locality=${entry.taskLocality}`);
+        for (const treatment of entry.treatments) {
+            lines.push(`  ${treatment.treatmentId}: status=${treatment.executionStatus} evidence=${treatment.evidenceAvailability}`, `    initial task success: ${formatAgentSuccessMetric(treatment.initialTaskSuccess)}; final task success: ${formatAgentSuccessMetric(treatment.finalTaskSuccess)}; attempts=${treatment.attemptCount}; repair attempts=${treatment.repairAttemptCount}; repair succeeded: ${formatAgentSuccessMetric(treatment.repairSucceeded)}`, `    task-check pass rate: ${formatAgentSuccessMetric(treatment.taskCheckPassRate)}; regression-check pass rate: ${formatAgentSuccessMetric(treatment.regressionCheckPassRate)}; required-fact coverage: ${formatAgentSuccessMetric(treatment.requiredFactCoverage)}`);
+        }
+    }
+    section("G. Per-attempt repair history");
+    if (!realMode || report.repairHistory.length === 0) {
+        lines.push(realMode ? "No attempt history was recorded." : "Not applicable: deterministic-fixture mode runs a single fixture patch and no provider attempts.");
+    }
+    for (const history of report.repairHistory) {
+        lines.push(`Case ${history.caseId} / ${history.treatmentId}: ${history.attempts.length} attempt(s)`);
+        for (const attempt of history.attempts) {
+            lines.push(`  attempt ${attempt.attemptNumber}: provider=${attempt.providerStatus}; patch=${attempt.patchOutcome}; verification=${attempt.verificationAvailability}; task success=${formatAgentSuccessMetric(attempt.taskSuccess)}; outcome=${attempt.failureCategory}${attempt.repairEligible ? " (repair-eligible)" : ""}`, `    duration: ${formatAgentSuccessMetric(attempt.providerDurationMs)}; tokens: ${attempt.tokenAvailability === "available" ? formatAgentSuccessMetric(attempt.providerTokens) : "unavailable"}`, `    artifacts: proposed patch=${attempt.proposedPatchPath ?? "none"}; applied patch=${attempt.appliedPatchPath ?? "none"}; agent files=${attempt.agentArtifactDirectory ?? "none"}`);
+        }
+    }
+    section("H. Edit-quality and blast-radius measurements (final evaluated patch)");
+    for (const entry of report.cases) {
+        for (const treatment of entry.treatments) {
+            lines.push(`${entry.caseId} / ${treatment.treatmentId}: expected-edit coverage=${formatAgentSuccessMetric(treatment.expectedEditCoverage)}; edit-scope precision=${formatAgentSuccessMetric(treatment.editScopePrecision)}; unexpected changed files=${formatAgentSuccessMetric(treatment.unexpectedChangedFiles)}; protected mutations=${formatAgentSuccessMetric(treatment.protectedMutations)}`, `  changed files=${formatAgentSuccessMetric(treatment.changedFileCount)}${treatment.changedFiles ? ` [${treatment.changedFiles.items.join(", ")}${treatment.changedFiles.omittedCount > 0 ? `, +${treatment.changedFiles.omittedCount} more` : ""}]` : ""}; total churn=${formatAgentSuccessMetric(treatment.totalChurn)}; relative churn=${formatAgentSuccessMetric(treatment.relativeChurn)}`);
+        }
+    }
+    section("I. Duration and token evidence");
+    for (const entry of report.cases) {
+        for (const treatment of entry.treatments) {
+            lines.push(`${entry.caseId} / ${treatment.treatmentId}: final-attempt provider duration=${formatAgentSuccessMetric(treatment.finalAttemptProviderDurationMs)}; final-attempt provider tokens=${formatAgentSuccessMetric(treatment.finalAttemptProviderTokens)}` +
+                (treatment.totalProviderDurationMs
+                    ? `; total provider duration across attempts=${formatAgentSuccessMetric(treatment.totalProviderDurationMs)}; total provider tokens across attempts=${formatAgentSuccessMetric(treatment.totalProviderTokens)}`
+                    : ""));
+        }
+    }
+    for (const treatment of report.treatments) {
+        if (!treatment.repair)
+            continue;
+        for (const total of treatment.repair.providerDurationMs)
+            lines.push(`${treatment.treatmentId} provider duration [${AGENT_SUCCESS_BASIS_LABEL[total.basis] ?? total.basis}]: ${formatAgentSuccessTotal(total)}`);
+        for (const total of treatment.repair.providerTokens)
+            lines.push(`${treatment.treatmentId} provider tokens [${AGENT_SUCCESS_BASIS_LABEL[total.basis] ?? total.basis}]: ${formatAgentSuccessTotal(total)}`);
+    }
+    section("J. Warnings, failures and artifact references");
+    const errors = report.cases.flatMap((entry) => entry.treatments.flatMap((treatment) => treatment.errors.map((error) => ({ caseId: entry.caseId, treatmentId: treatment.treatmentId, ...error }))));
+    if (report.warnings.length === 0 && errors.length === 0)
+        lines.push("No warnings or recorded errors.");
+    for (const warning of report.warnings)
+        lines.push(`warning ${sanitize(warning.code)}: ${sanitize(warning.message)}`);
+    for (const error of errors)
+        lines.push(`error ${error.caseId} / ${error.treatmentId} attempt ${error.attemptNumber} ${sanitize(error.code)}: ${sanitize(error.message)}`);
+    for (const artifact of report.artifacts)
+        lines.push(`artifact ${sanitize(artifact.id)}: ${artifact.path ?? "[reference withheld]"}`);
+    return lines;
+}

package/dist/src/report/experiments/renderPluginExperimentReportHtml.js

@@ -5,6 +5,7 @@ import { renderContextWindowScalingHtml } from "./renderContextWindowScalingHtml
 import { renderRetrievalPrecisionRecallHtml } from "./renderRetrievalPrecisionRecallHtml.js";
 import { renderRetrievalQueryStrategyComparisonHtml } from "./renderRetrievalQueryStrategyComparisonHtml.js";
 import { renderContextPackGenerationHtml } from "./renderContextPackGenerationHtml.js";
+import { renderAgentSuccessRateHtml } from "./renderAgentSuccessRateHtml.js";
 export function renderPluginExperimentReportHtml(report) {
     const redactedTarget = report.target.privacyProjection === "external-local-redacted";
     return `<!DOCTYPE html>
@@ -115,6 +116,8 @@ export function renderPluginExperimentReportHtml(report) {
 
   ${renderContextPackGenerationHtml(report.contextPackGeneration ?? null)}
 
+  ${renderAgentSuccessRateHtml(report.agentSuccessRate ?? null)}
+
   <section>
     <h2>Warnings, Skips, And Failures</h2>
     ${report.findings.length ? table(["Severity", "Code", "Message", "Variant", "Case"], report.findings.map((finding) => [

package/dist/src/report/experiments/renderPluginExperimentReportText.js

@@ -6,6 +6,7 @@ import { renderContextWindowScalingTextLines } from "./renderContextWindowScalin
 import { renderRetrievalPrecisionRecallTextLines } from "./renderRetrievalPrecisionRecallText.js";
 import { renderRetrievalQueryStrategyComparisonTextLines } from "./renderRetrievalQueryStrategyComparisonText.js";
 import { renderContextPackGenerationTextLines } from "./renderContextPackGenerationText.js";
+import { renderAgentSuccessRateTextLines } from "./renderAgentSuccessRateText.js";
 function sanitizeScalar(value) {
     const text = String(value);
     let result = "";
@@ -691,6 +692,10 @@ export function renderPluginExperimentReportText(report) {
         lines.push(...renderRetrievalQueryStrategyComparisonTextLines(retrievalQueryStrategyComparison));
     }
     // Additive: other plugins keep their existing text output unchanged.
+    const agentSuccessRate = report.agentSuccessRate ?? null;
+    if (agentSuccessRate !== null) {
+        lines.push(...renderAgentSuccessRateTextLines(agentSuccessRate));
+    }
     const contextPackGeneration = report.contextPackGeneration ?? null;
     if (contextPackGeneration !== null) {
         pushSection(lines, "Context Pack Generation");

package/docs/METRICS.md

@@ -868,3 +868,73 @@ There is no composite score, winner, or ranking. Tests and call relationships ma
 In external-local mode, analysis uses real identities before the privacy projection. Redacted placeholders never enter metric calculations. Durable external-local output has no pack body; the report preview presents redacted identities.
 
 See [COMMANDS.md](COMMANDS.md#context-pack-generation-v082), [WORKFLOWS.md](WORKFLOWS.md#context-pack-generation-experiment), and [ARCHITECTURE.md](ARCHITECTURE.md#context-pack-generation-architecture-v082).
+
+## Agent-success-rate metrics (v0.9.0)
+
+Status: shipped in v0.9.0. The scientific owner of every per-treatment value is `analyzeAgentSuccessTreatment` in `src/experiments/plugins/agentSuccessRate/analysis.ts`; repair aggregation and paired differences are owned by `repairAnalysis.ts`. Values are persisted in `agent-success-rate-analysis.json` (schema `1.0.0`); `agent-success-rate-execution.json` holds the evidence they are computed from. `contextEffectEvaluated` is `false` in deterministic-fixture mode.
+
+### Availability rules
+
+Every metric is typed `available`, `unavailable`, or `not-applicable`, with a unit and, when not available, a reason.
+
+- **Unavailable** means the evidence needed to compute the value does not exist (for example the provider reported no token usage, a check timed out, the patch was not applied, or the baseline was invalid). It is never converted to zero, never counted as a failure, and excluded from rates and means.
+- **Not applicable** means the quantity is undefined by construction (for example a ratio whose denominator is zero, or `repairSucceeded` when no repair attempt ran).
+- **Invalid baseline exclusion:** a treatment whose unmodified baseline is not evaluable (it needs at least one failing task check, all regression checks passing, and no check timing out or erroring) has no task-success verdict and is excluded from rates; this is a benchmark defect, not an agent failure.
+- **Tri-state verdicts:** a verdict is true, false, or unknown. A failed check makes a conjunction false; otherwise a missing, timed-out, or errored check makes it unknown; only complete passing evidence makes it true.
+
+### Task-success metrics
+
+| Metric | Meaning and formula | Source, units, availability |
+|---|---|---|
+| `taskChecksPassed` | Every trusted task check passed on the changed state | boolean from post-edit verification; unknown if any task check is missing, timed out, or errored |
+| `regressionSafe` | Every regression check that passed on the baseline still passes | boolean; only baseline-passing regression checks are considered |
+| `requiredFactsSatisfied` | Every behavior fact marked required has all its verification checks passing | boolean |
+| `protectedIntegrity` | Protected files are byte-intact after the patch | boolean from the protected-integrity comparison |
+| `taskResolved` | `taskChecksPassed` AND `requiredFactsSatisfied` | boolean |
+| `taskSuccess` | `taskResolved` AND `regressionSafe` AND `protectedIntegrity` intact. A patch that was rejected or not applied yields false; an uncomputable verdict yields unavailable | boolean; the only success verdict; agent prose is never evidence |
+| `initialAttemptTaskSuccess` | `taskSuccess` of attempt 1 | boolean; real-agent repair evidence only |
+| `finalTaskSuccess` | `taskSuccess` of the last attempt executed | boolean; equals `taskSuccess` |
+| `repairSucceeded` | `initialAttemptTaskSuccess` is false AND `finalTaskSuccess` is true | boolean; not applicable when no repair attempt ran; unavailable if either verdict is unavailable |
+| `repairSuccessRate` (aggregate) | repaired cases / repair-attempted cases with a determinate repair verdict | ratio; not applicable when no repair attempt ran for the treatment |
+| `taskSuccessRate`, `initialAttemptSuccessRate`, `finalTaskSuccessRate` (aggregates) | successful cases / cases with a determinate verdict for that attempt basis | ratio; unavailable when no case has a determinate verdict |
+
+First versus final attempt: `initialAttemptSuccessRate` and `finalTaskSuccessRate` are separate measurements. A repaired success raises the final rate and never the initial rate. Without repair evidence a single attempt makes the two equal.
+
+### Check, fact, and edit-quality metrics
+
+| Metric | Formula | Numerator / denominator | Notes |
+|---|---|---|---|
+| `taskCheckPassRate` | passed task checks / task checks | `taskCheckPassedCount` / `taskCheckTotalCount` | unavailable unless every task check has a complete result; not applicable with zero checks |
+| `regressionCheckPassRate` | passed regression checks / regression checks | `regressionCheckPassedCount` / `regressionCheckTotalCount` | `regressionFailureCount` counts failed checks that passed on the baseline |
+| `requiredFactCoverage` | satisfied required facts / required facts | `requiredFactsSatisfiedCount` / `requiredFactsTotal` | not applicable with no required facts |
+| `factCoverage` | satisfied facts (required and optional) / all facts | satisfied / total facts | not applicable with no facts |
+| `expectedEditCoverage` | changed files that were expected / expected edit files | `expectedEditFilesChangedCount` / `expectedEditFileCount` | ratio |
+| `editScopePrecision` | changed files inside the allowed edit scope / changed files | `allowedEditChangedFileCount` / `changedFileCount` | not applicable when no file changed |
+| `unexpectedChangedFileCount` | changed files minus changed files inside the allowed scope | count | files outside the allowed scope |
+| `protectedMutationCount` | protected paths found mutated after the patch | count | unavailable if integrity could not be proven; `attemptedProtectedEditCount` counts protected paths a patch tried to edit |
+| `changedFileCount`, `addedFileCount`, `modifiedFileCount`, `deletedFileCount` | counts of captured changed files by status | count | from the changed-file capture of the final attempt |
+| `linesAdded`, `linesDeleted` | sums of per-file additions and deletions | lines | unavailable if any changed file has no line counts (binary); never estimated |
+| `totalChurn` | `linesAdded` + `linesDeleted` | lines | the final patch's churn, not the churn accumulated over repair attempts |
+| `relativeChurn` | `totalChurn` / `baselineTextLineCount` | ratio | unavailable when the baseline text-line count is unknown; not applicable at zero baseline lines |
+
+Edit-quality and blast-radius metrics describe the final attempt only. A treatment that never applied a patch has these metrics unavailable, not zero.
+
+### Time and token metrics
+
+| Metric | Meaning | Source and caveat |
+|---|---|---|
+| `agentDurationMs` | Measured wall-clock duration of the provider attempt | unavailable in deterministic mode and when no provider ran; for repair runs, `firstAttemptProviderDurationMs`, `finalAttemptProviderDurationMs`, and `totalProviderDurationMs` are reported separately |
+| `totalProviderDurationMs` | Sum of provider durations over attempts that ran | unavailable if any invoked attempt was not measured |
+| `agentTotalTokens` (provider token usage) | Provider-reported total token usage for the attempt | unavailable when the provider reported none (source and reliability are recorded); `totalProviderTokens` sums over attempts and is unavailable if any invoked attempt lacks a total |
+| `estimatedContextTokens` | Estimated size of the context supplied to the provider | a size estimate recorded in execution evidence, not provider-reported usage; never substituted for provider tokens |
+| `baselineVerificationDurationMs`, `patchPipelineDurationMs`, `postEditVerificationDurationMs`, `evaluationDurationMs` | Measured durations of the evaluator's own stages | milliseconds; `totalEvaluationDurationMs` sums attempts |
+
+Cumulative versus final: durations and tokens accumulate over all provider attempts (repair costs are included), while edit-quality and blast-radius metrics describe only the final patch.
+
+### Paired comparisons and means
+
+- Aggregate means (`meanTaskCheckPassRate`, `meanEditScopePrecision`, `meanTotalChurn`, and the other `mean*` entries) are unweighted arithmetic means over **matched** cases: cases for which the source metric is available in every treatment. Unavailable values are not converted to zero. A mean with no matched case is unavailable.
+- The matched comparison counts, over cases with a determinate verdict in both treatments, `bothSucceeded`, `onlyRawFullFileSucceeded`, `onlyContextPackSucceeded`, and `neitherSucceeded`; incomplete cases are listed separately with the unavailable treatment IDs. With repair evidence a second matched comparison uses initial-attempt success.
+- Paired differences report the mean of the `raw-full-file` values, the mean of the `context-pack` values, and the mean of `context-pack` minus `raw-full-file` over matched cases.
+- These are **descriptive**. No weighted composite score, ranking, automatic winner, causal claim, or statistical-significance test is produced (`statisticalEffect` is `not-assessed`). The corpus is finite and controlled, so results do not generalize beyond it, and a single run carries no variance estimate.
+- Provider telemetry can be unavailable, and in deterministic-fixture mode both treatments receive the same reference patch, so those results measure the pipeline only.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dailephd/my-dev-kit-lab",
-  "version": "0.8.2",
+  "version": "0.9.0",
   "type": "module",
   "description": "Evidence, benchmark, and evaluation companion for my-dev-kit.",
   "bin": {