@roackb2/heddle 0.0.36 → 0.0.38

package/dist/src/core/eval/progress.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"progress.js","sourceRoot":"","sources":["../../../../src/core/eval/progress.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAWpC,MAAM,OAAO,oBAAoB;IACtB,YAAY,CAAS;IACb,MAAM,CAAS;IACf,WAAW,CAAU;IAEtC,YAAY,IAIX;QACC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACtC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC;QAC5C,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,CAAC,KAAa,EAAE,OAAe;QACjC,IAAI,CAAC,KAAK,CAAC;YACT,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,KAAK;YACL,MAAM,EAAE,MAAM;YACd,OAAO;SACR,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CAAI,IAMd;QACC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,CAAC,KAAK,CAAC;YACT,SAAS,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE;YAC5C,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,SAAS,EAAE,CAAC;SACb,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,MAAM,CAAC;QAC/C,MAAM,QAAQ,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC;YAChC,WAAW,CAAC,GAAG,EAAE;gBACf,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;gBACzC,IAAI,CAAC,KAAK,CAAC;oBACT,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBACnC,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,MAAM,EAAE,WAAW;oBACnB,OAAO,EAAE,IAAI,CAAC,gBAAgB,IAAI,iBAAiB,IAAI,CAAC,OAAO,EAAE;oBACjE,SAAS;iBACV,CAAC,CAAC;YACL,CAAC,EAAE,WAAW,CAAC;YACjB,CAAC,CAAC,SAAS,CAAC;QAEZ,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC;YAChC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YACzC,IAAI,CAAC,KAAK,CAAC;gBACT,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,MAAM,EAAE,WAAW;gBACnB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,SAAS;aACV,CAAC,CAAC;YACH,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YACzC,IAAI,CAAC,KAAK,CAAC;gBACT,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,MAAM,EAAE,QAAQ;gBAChB,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;gBAC/D,SAAS;aACV,CAAC,CAAC;YACH,MAAM,KAAK,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,IAAI,QAAQ,EAAE,CAAC;gBACb,aAAa,CAAC,QAAQ,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,KAAwB;QACpC,cAAc,CAAC,IAAI,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACxE,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;CACF;AAED,SAAS,kBAAkB,CAAC,KAAwB;IAClD,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,aAAa,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC;IAC5F,OAAO,IAAI,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,OAAO,GAAG,OAAO,IAAI,CAAC;AAC3E,CAAC;AAED,SAAS,aAAa,CAAC,SAAiB;IACtC,IAAI,SAAS,GAAG,IAAI,EAAE,CAAC;QACrB,OAAO,GAAG,SAAS,IAAI,CAAC;IAC1B,CAAC;IACD,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC;AAC5C,CAAC"}

package/dist/src/core/eval/report-writer.d.ts

@@ -0,0 +1,7 @@
+import type { EvalSuiteReport } from './schema.js';
+export declare function writeEvalSuiteReport(report: EvalSuiteReport): {
+    jsonPath: string;
+    markdownPath: string;
+};
+export declare function formatEvalSuiteMarkdown(report: EvalSuiteReport): string;
+//# sourceMappingURL=report-writer.d.ts.map

package/dist/src/core/eval/report-writer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report-writer.d.ts","sourceRoot":"","sources":["../../../../src/core/eval/report-writer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAiB,eAAe,EAAE,MAAM,aAAa,CAAC;AAElE,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,eAAe,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,CAOxG;AAED,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,CAqBvE"}

package/dist/src/core/eval/report-writer.js

@@ -0,0 +1,159 @@
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { relative, join } from 'node:path';
+export function writeEvalSuiteReport(report) {
+    mkdirSync(report.resultsDir, { recursive: true });
+    const jsonPath = join(report.resultsDir, 'report.json');
+    const markdownPath = join(report.resultsDir, 'report.md');
+    writeFileSync(jsonPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
+    writeFileSync(markdownPath, formatEvalSuiteMarkdown(report), 'utf8');
+    return { jsonPath, markdownPath };
+}
+export function formatEvalSuiteMarkdown(report) {
+    const passed = report.results.filter((result) => result.status === 'passed').length;
+    const lines = [
+        '# Heddle Agent Eval Report',
+        '',
+        `Target: ${report.target}`,
+        `Started: ${report.startedAt}`,
+        `Finished: ${report.finishedAt}`,
+        `Results: ${passed}/${report.results.length} passed`,
+        '',
+        '| Case | Status | Model | Checks | Outcome | Turns | Mutations | Verification After Mutation |',
+        '| --- | --- | --- | ---: | --- | ---: | ---: | ---: |',
+        ...report.results.map(formatSummaryRow),
+        '',
+    ];
+    for (const result of report.results) {
+        lines.push(...formatRunDetail(result, report.resultsDir), '');
+    }
+    return `${lines.join('\n').trimEnd()}\n`;
+}
+function formatSummaryRow(result) {
+    const passedChecks = result.checks.filter((check) => check.passed).length;
+    return [
+        result.caseId,
+        result.status,
+        result.model ?? 'default',
+        `${passedChecks}/${result.checks.length}`,
+        result.metrics.outcome ?? `exit ${result.agent.exitCode ?? 'unknown'}`,
+        String(result.metrics.assistantTurns),
+        String(result.metrics.mutations),
+        String(result.metrics.verificationCommandsAfterMutation),
+    ].map(escapeCell).join(' | ').replace(/^/, '| ').replace(/$/, ' |');
+}
+function formatRunDetail(result, resultsDir) {
+    const traceFiles = result.artifacts.traceFiles.map((path) => formatPath(path, resultsDir));
+    const lines = [
+        `## ${result.caseId}`,
+        '',
+        '| Field | Value |',
+        '| --- | --- |',
+        `| Status | ${escapeCell(result.status)} |`,
+        `| Model | ${escapeCell(result.model ?? 'default')} |`,
+        `| Max steps | ${escapeCell(String(result.maxSteps ?? 'default'))} |`,
+        `| Agent exit | ${escapeCell(`${result.agent.exitCode ?? 'unknown'}${result.agent.timedOut ? ' (timed out)' : ''}`)} |`,
+        `| Fixture | ${escapeCell(formatFixture(result))} |`,
+        `| Workspace | \`${escapeCell(formatPath(result.workspaceRoot, resultsDir))}\` |`,
+        `| Output | \`${escapeCell(formatPath(result.outputDir, resultsDir))}\` |`,
+        `| Diff | \`${escapeCell(formatPath(result.artifacts.gitDiffPath, resultsDir))}\` |`,
+        `| Diff stat | \`${escapeCell(formatPath(result.artifacts.gitDiffStatPath, resultsDir))}\` |`,
+        `| Changed files JSON | \`${escapeCell(formatPath(result.artifacts.changedFilesPath, resultsDir))}\` |`,
+        `| Git status | \`${escapeCell(formatPath(result.artifacts.gitStatusPath, resultsDir))}\` |`,
+        `| Progress | ${result.artifacts.progressPath ? `\`${escapeCell(formatPath(result.artifacts.progressPath, resultsDir))}\`` : 'none'} |`,
+        `| Session catalog | ${result.artifacts.sessionCatalogPath ? `\`${escapeCell(formatPath(result.artifacts.sessionCatalogPath, resultsDir))}\`` : 'none'} |`,
+        `| Trace files | ${traceFiles.length ? traceFiles.map((path) => `\`${escapeCell(path)}\``).join('<br>') : 'none'} |`,
+        '',
+        '### Milestone Review',
+        '',
+    ];
+    lines.push(...formatReviewSection(result), '', '### Changed Files', '');
+    if (result.artifacts.changedFiles.length === 0) {
+        lines.push('- none');
+    }
+    else {
+        lines.push('| File | Status | + | - |', '| --- | --- | ---: | ---: |');
+        for (const file of result.artifacts.changedFiles) {
+            lines.push(`| ${escapeCell(file.path)} | ${escapeCell(file.status)} | ${file.additions ?? ''} | ${file.deletions ?? ''} |`);
+        }
+    }
+    lines.push('', '### Post-Run Checks', '');
+    if (result.checks.length === 0) {
+        lines.push('- none');
+    }
+    else {
+        for (const check of result.checks) {
+            lines.push(`- ${check.passed ? 'PASS' : 'FAIL'} ${check.name}: \`${check.command}\` (${check.exitCode ?? 'unknown'}, ${check.durationMs}ms)`);
+        }
+    }
+    lines.push('', '### Metrics', '', '| Metric | Value |', '| --- | ---: |', `| Assistant turns | ${result.metrics.assistantTurns} |`, `| Tool calls | ${result.metrics.toolCalls} |`, `| Mutations | ${result.metrics.mutations} |`, `| Verification after mutation | ${result.metrics.verificationCommandsAfterMutation} |`, `| Approvals requested | ${result.metrics.approvalsRequested} |`, `| Approvals resolved | ${result.metrics.approvalsResolved} |`, `| Tool errors | ${result.metrics.toolErrors} |`);
+    lines.push('', '### Agent Verification Commands', '');
+    if (result.metrics.verificationCommandDetails.length === 0) {
+        lines.push('- none detected after first mutation');
+    }
+    else {
+        for (const command of result.metrics.verificationCommandDetails) {
+            lines.push(`- \`${escapeCell(command)}\``);
+        }
+    }
+    lines.push('', '### Rubric', '');
+    if (result.review.requiredOutcomes.length === 0 && result.review.humanQuestions.length === 0) {
+        lines.push('- none');
+    }
+    else {
+        for (const outcome of result.review.requiredOutcomes) {
+            lines.push(`- [ ] ${outcome}`);
+        }
+        for (const question of result.review.humanQuestions) {
+            lines.push(`- [ ] ${question}`);
+        }
+    }
+    if (result.metrics.summary) {
+        lines.push('', '### Final Summary', '', result.metrics.summary);
+    }
+    return lines;
+}
+function formatReviewSection(result) {
+    const lines = [];
+    if (result.review.milestone) {
+        lines.push(`Milestone: ${result.review.milestone}`);
+    }
+    if (result.review.intent) {
+        lines.push('', result.review.intent);
+    }
+    lines.push('', '| Review Field | Items |', '| --- | --- |');
+    lines.push(`| Required outcomes | ${formatListCell(result.review.requiredOutcomes)} |`);
+    lines.push(`| Allowed scope | ${formatListCell(result.review.allowedScope)} |`);
+    lines.push(`| Out of scope | ${formatListCell(result.review.outOfScope)} |`);
+    lines.push(`| Human questions | ${formatListCell(result.review.humanQuestions)} |`);
+    return lines;
+}
+function formatListCell(items) {
+    return items.length ? items.map((item) => `- ${escapeCell(item)}`).join('<br>') : 'none';
+}
+function formatFixture(result) {
+    if (result.fixture.type === 'git-worktree') {
+        return [
+            'git-worktree',
+            result.fixture.ref ? `ref ${result.fixture.ref}` : undefined,
+            result.fixture.resolvedRef ? `commit ${shortSha(result.fixture.resolvedRef)}` : undefined,
+            result.fixture.baselineCommit && result.fixture.baselineCommit !== result.fixture.resolvedRef ?
+                `baseline ${shortSha(result.fixture.baselineCommit)}`
+                : undefined,
+        ].filter(Boolean).join(', ');
+    }
+    return result.fixture.baselineCommit ? `inline, baseline ${shortSha(result.fixture.baselineCommit)}` : 'inline';
+}
+function shortSha(value) {
+    return value.slice(0, 12);
+}
+function escapeCell(value) {
+    return value.replaceAll('|', '\\|').replaceAll('\n', ' ');
+}
+function formatPath(path, basePath) {
+    if (path === basePath) {
+        return '.';
+    }
+    const relativePath = relative(basePath, path);
+    return relativePath && !relativePath.startsWith('..') ? relativePath : path;
+}
+//# sourceMappingURL=report-writer.js.map

package/dist/src/core/eval/report-writer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report-writer.js","sourceRoot":"","sources":["../../../../src/core/eval/report-writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAG3C,MAAM,UAAU,oBAAoB,CAAC,MAAuB;IAC1D,SAAS,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;IACxD,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;IAC1D,aAAa,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACxE,aAAa,CAAC,YAAY,EAAE,uBAAuB,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;IACrE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,MAAuB;IAC7D,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;IACpF,MAAM,KAAK,GAAG;QACZ,4BAA4B;QAC5B,EAAE;QACF,WAAW,MAAM,CAAC,MAAM,EAAE;QAC1B,YAAY,MAAM,CAAC,SAAS,EAAE;QAC9B,aAAa,MAAM,CAAC,UAAU,EAAE;QAChC,YAAY,MAAM,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,SAAS;QACpD,EAAE;QACF,gGAAgG;QAChG,uDAAuD;QACvD,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;QACvC,EAAE;KACH,CAAC;IAEF,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;IAChE,CAAC;IAED,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC;AAC3C,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAqB;IAC7C,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAC1E,OAAO;QACL,MAAM,CAAC,MAAM;QACb,MAAM,CAAC,MAAM;QACb,MAAM,CAAC,KAAK,IAAI,SAAS;QACzB,GAAG,YAAY,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE;QACzC,MAAM,CAAC,OAAO,CAAC,OAAO,IAAI,QAAQ,MAAM,CAAC,KAAK,CAAC,QAAQ,IAAI,SAAS,EAAE;QACtE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC;QAChC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,iCAAiC,CAAC;KACzD,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,eAAe,CAAC,MAAqB,EAAE,UAAkB;IAChE,MAAM,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC;IAC3F,MAAM,KAAK,GAAG;QACZ,MAAM,MAAM,CAAC,MAAM,EAAE;QACrB,EAAE;QACF,mBAAmB;QACnB,eAAe;QACf,cAAc,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI;QAC3C,aAAa,UAAU,CAAC,MAAM,CAAC,KAAK,IAAI,SAAS,CAAC,IAAI;QACtD,iBAAiB,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,IAAI,SAAS,CAAC,CAAC,IAAI;QACrE,kBAAkB,UAAU,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,IAAI,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI;QACvH,eAAe,UAAU,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI;QACpD,mBAAmB,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC,MAAM;QACjF,gBAAgB,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC,MAAM;QAC1E,cAAc,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC,MAAM;QACpF,mBAAmB,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC,MAAM;QAC7F,4BAA4B,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,gBAAgB,EAAE,UAAU,CAAC,CAAC,MAAM;QACvG,oBAAoB,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC,MAAM;QAC5F,gBAAgB,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI;QACvI,uBAAuB,MAAM,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,kBAAkB,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI;QAC1J,mBAAmB,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI;QACpH,EAAE;QACF,sBAAsB;QACtB,EAAE;KACH,CAAC;IAEF,KAAK,CAAC,IAAI,CAAC,GAAG,mBAAmB,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,mBAAmB,EAAE,EAAE,CAAC,CAAC;IACxE,IAAI,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/C,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACvB,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,2BAA2B,EAAE,6BAA6B,CAAC,CAAC;QACvE,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,SAAS,CAAC,YAAY,EAAE,CAAC;YACjD,KAAK,CAAC,IAAI,CAAC,KAAK,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,SAAS,IAAI,EAAE,MAAM,IAAI,CAAC,SAAS,IAAI,EAAE,IAAI,CAAC,CAAC;QAC9H,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,qBAAqB,EAAE,EAAE,CAAC,CAAC;IAC1C,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACvB,CAAC;SAAM,CAAC;QACN,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,OAAO,OAAO,KAAK,CAAC,QAAQ,IAAI,SAAS,KAAK,KAAK,CAAC,UAAU,KAAK,CAAC,CAAC;QAChJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CACR,EAAE,EACF,aAAa,EACb,EAAE,EACF,oBAAoB,EACpB,gBAAgB,EAChB,uBAAuB,MAAM,CAAC,OAAO,CAAC,cAAc,IAAI,EACxD,kBAAkB,MAAM,CAAC,OAAO,CAAC,SAAS,IAAI,EAC9C,iBAAiB,MAAM,CAAC,OAAO,CAAC,SAAS,IAAI,EAC7C,mCAAmC,MAAM,CAAC,OAAO,CAAC,iCAAiC,IAAI,EACvF,2BAA2B,MAAM,CAAC,OAAO,CAAC,kBAAkB,IAAI,EAChE,0BAA0B,MAAM,CAAC,OAAO,CAAC,iBAAiB,IAAI,EAC9D,mBAAmB,MAAM,CAAC,OAAO,CAAC,UAAU,IAAI,CACjD,CAAC;IAEF,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,iCAAiC,EAAE,EAAE,CAAC,CAAC;IACtD,IAAI,MAAM,CAAC,OAAO,CAAC,0BAA0B,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3D,KAAK,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;IACrD,CAAC;SAAM,CAAC;QACN,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,0BAA0B,EAAE,CAAC;YAChE,KAAK,CAAC,IAAI,CAAC,OAAO,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,EAAE,EAAE,CAAC,CAAC;IACjC,IAAI,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7F,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACvB,CAAC;SAAM,CAAC;QACN,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;YACrD,KAAK,CAAC,IAAI,CAAC,SAAS,OAAO,EAAE,CAAC,CAAC;QACjC,CAAC;QACD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;YACpD,KAAK,CAAC,IAAI,CAAC,SAAS,QAAQ,EAAE,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,mBAAmB,EAAE,EAAE,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAClE,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAqB;IAChD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,cAAc,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACvC,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,0BAA0B,EAAE,eAAe,CAAC,CAAC;IAC5D,KAAK,CAAC,IAAI,CAAC,yBAAyB,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;IACxF,KAAK,CAAC,IAAI,CAAC,qBAAqB,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAChF,KAAK,CAAC,IAAI,CAAC,oBAAoB,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IAC7E,KAAK,CAAC,IAAI,CAAC,uBAAuB,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IACpF,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,KAAe;IACrC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;AAC3F,CAAC;AAED,SAAS,aAAa,CAAC,MAAqB;IAC1C,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;QAC3C,OAAO;YACL,cAAc;YACd,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS;YAC5D,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS;YACzF,MAAM,CAAC,OAAO,CAAC,cAAc,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,KAAK,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;gBAC7F,YAAY,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE;gBACvD,CAAC,CAAC,SAAS;SACZ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,oBAAoB,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;AAClH,CAAC;AAED,SAAS,QAAQ,CAAC,KAAa;IAC7B,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,UAAU,CAAC,IAAY,EAAE,QAAgB;IAChD,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtB,OAAO,GAAG,CAAC;IACb,CAAC;IAED,MAAM,YAAY,GAAG,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IAC9C,OAAO,YAAY,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC;AAC9E,CAAC"}

package/dist/src/core/eval/schema.d.ts

@@ -0,0 +1,206 @@
+import { z } from 'zod';
+export declare const evalCheckSchema: z.ZodObject<{
+    name: z.ZodOptional<z.ZodString>;
+    command: z.ZodString;
+    timeoutMs: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;
+export declare const evalSetupSchema: z.ZodObject<{
+    copyFiles: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    files: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    commands: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        name: z.ZodOptional<z.ZodString>;
+        command: z.ZodString;
+        timeoutMs: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>>>;
+    commitMessage: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+export declare const evalInlineFixtureSchema: z.ZodObject<{
+    type: z.ZodLiteral<"inline">;
+}, z.core.$strip>;
+export declare const evalGitWorktreeFixtureSchema: z.ZodObject<{
+    type: z.ZodLiteral<"git-worktree">;
+    repo: z.ZodDefault<z.ZodString>;
+    ref: z.ZodString;
+}, z.core.$strip>;
+export declare const evalFixtureSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
+    type: z.ZodLiteral<"inline">;
+}, z.core.$strip>, z.ZodObject<{
+    type: z.ZodLiteral<"git-worktree">;
+    repo: z.ZodDefault<z.ZodString>;
+    ref: z.ZodString;
+}, z.core.$strip>], "type">;
+export declare const evalMilestoneReviewSchema: z.ZodObject<{
+    milestone: z.ZodOptional<z.ZodString>;
+    intent: z.ZodOptional<z.ZodString>;
+    requiredOutcomes: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    allowedScope: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    outOfScope: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    humanQuestions: z.ZodDefault<z.ZodArray<z.ZodString>>;
+}, z.core.$strip>;
+export declare const agentEvalCaseSchema: z.ZodObject<{
+    id: z.ZodString;
+    kind: z.ZodLiteral<"coding">;
+    description: z.ZodOptional<z.ZodString>;
+    prompt: z.ZodString;
+    model: z.ZodOptional<z.ZodString>;
+    maxSteps: z.ZodOptional<z.ZodNumber>;
+    setup: z.ZodDefault<z.ZodObject<{
+        copyFiles: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+        files: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+        commands: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodOptional<z.ZodString>;
+            command: z.ZodString;
+            timeoutMs: z.ZodOptional<z.ZodNumber>;
+        }, z.core.$strip>>>;
+        commitMessage: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>;
+    fixture: z.ZodDefault<z.ZodDiscriminatedUnion<[z.ZodObject<{
+        type: z.ZodLiteral<"inline">;
+    }, z.core.$strip>, z.ZodObject<{
+        type: z.ZodLiteral<"git-worktree">;
+        repo: z.ZodDefault<z.ZodString>;
+        ref: z.ZodString;
+    }, z.core.$strip>], "type">>;
+    review: z.ZodDefault<z.ZodObject<{
+        milestone: z.ZodOptional<z.ZodString>;
+        intent: z.ZodOptional<z.ZodString>;
+        requiredOutcomes: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        allowedScope: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        outOfScope: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        humanQuestions: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    }, z.core.$strip>>;
+    checks: z.ZodDefault<z.ZodArray<z.ZodObject<{
+        name: z.ZodOptional<z.ZodString>;
+        command: z.ZodString;
+        timeoutMs: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>>>;
+    rubric: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    tags: z.ZodDefault<z.ZodArray<z.ZodString>>;
+}, z.core.$strip>;
+export declare const evalCaseSchema: z.ZodObject<{
+    id: z.ZodString;
+    kind: z.ZodLiteral<"coding">;
+    description: z.ZodOptional<z.ZodString>;
+    prompt: z.ZodString;
+    model: z.ZodOptional<z.ZodString>;
+    maxSteps: z.ZodOptional<z.ZodNumber>;
+    setup: z.ZodDefault<z.ZodObject<{
+        copyFiles: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+        files: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+        commands: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodOptional<z.ZodString>;
+            command: z.ZodString;
+            timeoutMs: z.ZodOptional<z.ZodNumber>;
+        }, z.core.$strip>>>;
+        commitMessage: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>;
+    fixture: z.ZodDefault<z.ZodDiscriminatedUnion<[z.ZodObject<{
+        type: z.ZodLiteral<"inline">;
+    }, z.core.$strip>, z.ZodObject<{
+        type: z.ZodLiteral<"git-worktree">;
+        repo: z.ZodDefault<z.ZodString>;
+        ref: z.ZodString;
+    }, z.core.$strip>], "type">>;
+    review: z.ZodDefault<z.ZodObject<{
+        milestone: z.ZodOptional<z.ZodString>;
+        intent: z.ZodOptional<z.ZodString>;
+        requiredOutcomes: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        allowedScope: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        outOfScope: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        humanQuestions: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    }, z.core.$strip>>;
+    checks: z.ZodDefault<z.ZodArray<z.ZodObject<{
+        name: z.ZodOptional<z.ZodString>;
+        command: z.ZodString;
+        timeoutMs: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>>>;
+    rubric: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    tags: z.ZodDefault<z.ZodArray<z.ZodString>>;
+}, z.core.$strip>;
+export type EvalCheck = z.infer<typeof evalCheckSchema>;
+export type EvalSetup = z.infer<typeof evalSetupSchema>;
+export type EvalFixture = z.infer<typeof evalFixtureSchema>;
+export type EvalMilestoneReview = z.infer<typeof evalMilestoneReviewSchema>;
+export type AgentEvalCase = z.infer<typeof agentEvalCaseSchema>;
+export type EvalCase = z.infer<typeof evalCaseSchema>;
+export type EvalCheckResult = {
+    name: string;
+    command: string;
+    exitCode: number | null;
+    stdout: string;
+    stderr: string;
+    durationMs: number;
+    passed: boolean;
+    timedOut: boolean;
+};
+export type EvalChangedFile = {
+    path: string;
+    status: string;
+    additions?: number;
+    deletions?: number;
+};
+export type EvalTraceMetrics = {
+    assistantTurns: number;
+    toolCalls: number;
+    toolResults: number;
+    mutations: number;
+    approvalsRequested: number;
+    approvalsResolved: number;
+    toolErrors: number;
+    verificationCommandsAfterMutation: number;
+    firstMutationStep?: number;
+    outcome?: string;
+    summary?: string;
+    toolsByName: Record<string, number>;
+    readOrSearchBeforeMutation: string[];
+    verificationCommandDetails: string[];
+};
+export type EvalRunResult = {
+    caseId: string;
+    target: string;
+    status: 'passed' | 'failed';
+    workspaceRoot: string;
+    outputDir: string;
+    fixture: {
+        type: EvalFixture['type'];
+        repo?: string;
+        ref?: string;
+        resolvedRef?: string;
+        baselineCommit?: string;
+    };
+    startedAt: string;
+    finishedAt: string;
+    durationMs: number;
+    agent: {
+        command: string[];
+        exitCode: number | null;
+        stdoutPath: string;
+        stderrPath: string;
+        timedOut: boolean;
+    };
+    artifacts: {
+        gitStatusPath: string;
+        gitDiffPath: string;
+        gitDiffStatPath: string;
+        changedFilesPath: string;
+        progressPath?: string;
+        sessionCatalogPath?: string;
+        traceFiles: string[];
+        changedFiles: EvalChangedFile[];
+    };
+    checks: EvalCheckResult[];
+    metrics: EvalTraceMetrics;
+    review: EvalMilestoneReview;
+    model?: string;
+    maxSteps?: number;
+};
+export type EvalSuiteReport = {
+    version: 1;
+    target: string;
+    repoRoot: string;
+    startedAt: string;
+    finishedAt: string;
+    resultsDir: string;
+    results: EvalRunResult[];
+};
+//# sourceMappingURL=schema.d.ts.map

package/dist/src/core/eval/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../../src/core/eval/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,eAAe;;;;iBASoG,CAAC;AAEjI,eAAO,MAAM,eAAe;;;;;;;;;iBAmB8E,CAAC;AAE3G,eAAO,MAAM,uBAAuB;;iBAG+C,CAAC;AAEpF,eAAO,MAAM,4BAA4B;;;;iBAQ8C,CAAC;AAExF,eAAO,MAAM,iBAAiB;;;;;;2BAGyC,CAAC;AAExE,eAAO,MAAM,yBAAyB;;;;;;;iBA2BoE,CAAC;AAE3G,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA2CmF,CAAC;AAEpH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAAsB,CAAC;AAElD,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAChE,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAEtD,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,iCAAiC,EAAE,MAAM,CAAC;IAC1C,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpC,0BAA0B,EAAE,MAAM,EAAE,CAAC;IACrC,0BAA0B,EAAE,MAAM,EAAE,CAAC;CACtC,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE;QACP,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;QAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE;QACL,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,OAAO,CAAC;KACnB,CAAC;IACF,SAAS,EAAE;QACT,aAAa,EAAE,MAAM,CAAC;QACtB,WAAW,EAAE,MAAM,CAAC;QACpB,eAAe,EAAE,MAAM,CAAC;QACxB,gBAAgB,EAAE,MAAM,CAAC;QACzB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,UAAU,EAAE,MAAM,EAAE,CAAC;QACrB,YAAY,EAAE,eAAe,EAAE,CAAC;KACjC,CAAC;IACF,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,OAAO,EAAE,gBAAgB,CAAC;IAC1B,MAAM,EAAE,mBAAmB,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,OAAO,EAAE,CAAC,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,aAAa,EAAE,CAAC;CAC1B,CAAC"}

package/dist/src/core/eval/schema.js

@@ -0,0 +1,104 @@
+import { z } from 'zod';
+export const evalCheckSchema = z.object({
+    name: z.string().trim().min(1)
+        .describe('Human-readable label for this check in reports. Defaults to the command when omitted.')
+        .optional(),
+    command: z.string().trim().min(1)
+        .describe('Shell command to run inside the disposable eval workspace after the agent finishes. Exit code 0 means the check passed.'),
+    timeoutMs: z.number().int().positive()
+        .describe('Optional timeout for this check command in milliseconds.')
+        .optional(),
+}).describe('A deterministic post-run command used to decide whether the agent-produced workspace passes objective validation.');
+export const evalSetupSchema = z.object({
+    copyFiles: z.record(z.string().trim().min(1).describe('Repository-relative source file path to copy from the eval harness repo.'), z.string().trim().min(1).describe('Workspace-relative destination file path to create before the eval starts.'))
+        .describe('Fixture files to copy from the eval harness repo into the disposable workspace before committing the initial Git baseline.')
+        .optional(),
+    files: z.record(z.string().trim().min(1).describe('Workspace-relative file path to create before the eval starts.'), z.string().describe('Exact UTF-8 file contents to write for the fixture file.'))
+        .describe('Fixture files to write into the disposable workspace before committing the initial Git baseline.')
+        .optional(),
+    commands: z.array(evalCheckSchema)
+        .describe('Setup commands to run after fixture files are written and before the initial Git baseline is committed.')
+        .optional(),
+    commitMessage: z.string().trim().min(1)
+        .describe('Commit message for the initial fixture Git baseline. Defaults to a generated eval fixture message.')
+        .optional(),
+}).describe('Instructions for creating the disposable repository state that the agent will work against.');
+export const evalInlineFixtureSchema = z.object({
+    type: z.literal('inline')
+        .describe('Create a small disposable Git repository from the case setup files and commands.'),
+}).describe('A synthetic disposable repository built from inline eval setup data.');
+export const evalGitWorktreeFixtureSchema = z.object({
+    type: z.literal('git-worktree')
+        .describe('Create a disposable Git worktree from an existing repository at a pinned ref.'),
+    repo: z.string().trim().min(1)
+        .describe('Repository path to create the worktree from. Relative paths are resolved from the Heddle repo root.')
+        .default('.'),
+    ref: z.string().trim().min(1)
+        .describe('Pinned target ref for the worktree, such as a release tag or commit SHA. Avoid moving HEAD for comparable evals.'),
+}).describe('A realistic disposable repository fixture created from a pinned Git ref.');
+export const evalFixtureSchema = z.discriminatedUnion('type', [
+    evalInlineFixtureSchema,
+    evalGitWorktreeFixtureSchema,
+]).describe('How to prepare the disposable workspace the agent edits.');
+export const evalMilestoneReviewSchema = z.object({
+    milestone: z.string().trim().min(1)
+        .describe('Short name for the user-intended milestone this case evaluates.')
+        .optional(),
+    intent: z.string().trim().min(1)
+        .describe('Human-readable statement of what the agent should accomplish beyond merely passing checks.')
+        .optional(),
+    requiredOutcomes: z.array(z.string().trim().min(1).describe('Observable outcome a human reviewer should look for in the final diff, trace, or answer.'))
+        .describe('Milestone outcomes expected for a high-quality completion.')
+        .default([]),
+    allowedScope: z.array(z.string().trim().min(1).describe('Files, modules, or behavior areas the agent is allowed or expected to touch.'))
+        .describe('Expected implementation scope for judging whether the diff stayed on task.')
+        .default([]),
+    outOfScope: z.array(z.string().trim().min(1).describe('Files, modules, or behavior areas that should not be changed for this case.'))
+        .describe('Boundaries a human reviewer should use to spot unrelated churn.')
+        .default([]),
+    humanQuestions: z.array(z.string().trim().min(1).describe('Question for human review after the run completes.'))
+        .describe('Review prompts that help judge task completion quality beyond deterministic checks.')
+        .default([]),
+}).describe('Human-review metadata for milestone-style eval cases where pass/fail checks are not enough.');
+export const agentEvalCaseSchema = z.object({
+    id: z.string().trim().regex(/^[a-zA-Z0-9._-]+$/, 'Use a filesystem-safe case id.')
+        .describe('Stable filesystem-safe case id used in result paths, filtering, and reports.'),
+    kind: z.literal('coding')
+        .describe('Eval case type. The first harness slice supports coding cases run through ask --new-session.'),
+    description: z.string().trim()
+        .describe('Optional short explanation of what behavior this case is meant to exercise.')
+        .optional(),
+    prompt: z.string().trim().min(1)
+        .describe('User prompt sent to Heddle in the disposable workspace. This should ask for real coding work, not just Q&A.'),
+    model: z.string().trim().min(1)
+        .describe('Optional model override for this case. The CLI-level --model takes precedence when supplied.')
+        .optional(),
+    maxSteps: z.number().int().positive()
+        .describe('Optional maximum agent loop steps for this case. The CLI-level --max-steps takes precedence when supplied.')
+        .optional(),
+    setup: evalSetupSchema
+        .describe('Disposable workspace setup for this case.')
+        .default({}),
+    fixture: evalFixtureSchema
+        .describe('Workspace fixture source. Defaults to an inline synthetic repository.')
+        .default({ type: 'inline' }),
+    review: evalMilestoneReviewSchema
+        .describe('Optional milestone-completion review guidance included in reports.')
+        .default({
+        requiredOutcomes: [],
+        allowedScope: [],
+        outOfScope: [],
+        humanQuestions: [],
+    }),
+    checks: z.array(evalCheckSchema)
+        .describe('Deterministic post-agent commands that must pass for the case to be marked passed.')
+        .default([]),
+    rubric: z.array(z.string().trim().min(1).describe('Qualitative behavior criterion for human or future LLM judging.'))
+        .describe('Non-deterministic quality criteria preserved in reports for manual or future judge review.')
+        .default([]),
+    tags: z.array(z.string().trim().min(1).describe('Free-form label for filtering or grouping eval cases.'))
+        .describe('Case labels such as bugfix, refactor, verification, multi-file, or tui.')
+        .default([]),
+}).describe('A live coding-task eval case run in a disposable Git workspace through Heddle ask/session execution.');
+export const evalCaseSchema = agentEvalCaseSchema;
+//# sourceMappingURL=schema.js.map

package/dist/src/core/eval/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../../src/core/eval/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;SAC3B,QAAQ,CAAC,uFAAuF,CAAC;SACjG,QAAQ,EAAE;IACb,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;SAC9B,QAAQ,CAAC,yHAAyH,CAAC;IACtI,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;SACnC,QAAQ,CAAC,0DAA0D,CAAC;SACpE,QAAQ,EAAE;CACd,CAAC,CAAC,QAAQ,CAAC,mHAAmH,CAAC,CAAC;AAEjI,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,SAAS,EAAE,CAAC,CAAC,MAAM,CACjB,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,0EAA0E,CAAC,EAC7G,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,4EAA4E,CAAC,CAChH;SACE,QAAQ,CAAC,4HAA4H,CAAC;SACtI,QAAQ,EAAE;IACb,KAAK,EAAE,CAAC,CAAC,MAAM,CACb,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,gEAAgE,CAAC,EACnG,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,0DAA0D,CAAC,CAChF;SACE,QAAQ,CAAC,kGAAkG,CAAC;SAC5G,QAAQ,EAAE;IACb,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC;SAC/B,QAAQ,CAAC,yGAAyG,CAAC;SACnH,QAAQ,EAAE;IACb,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;SACpC,QAAQ,CAAC,oGAAoG,CAAC;SAC9G,QAAQ,EAAE;CACd,CAAC,CAAC,QAAQ,CAAC,6FAA6F,CAAC,CAAC;AAE3G,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9C,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC;SACtB,QAAQ,CAAC,kFAAkF,CAAC;CAChG,CAAC,CAAC,QAAQ,CAAC,sEAAsE,CAAC,CAAC;AAEpF,MAAM,CAAC,MAAM,4BAA4B,GAAG,CAAC,CAAC,MAAM,CAAC;IACnD,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC;SAC5B,QAAQ,CAAC,+EAA+E,CAAC;IAC5F,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;SAC3B,QAAQ,CAAC,qGAAqG,CAAC;SAC/G,OAAO,CAAC,GAAG,CAAC;IACf,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;SAC1B,QAAQ,CAAC,kHAAkH,CAAC;CAChI,CAAC,CAAC,QAAQ,CAAC,0EAA0E,CAAC,CAAC;AAExF,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,kBAAkB,CAAC,MAAM,EAAE;IAC5D,uBAAuB;IACvB,4BAA4B;CAC7B,CAAC,CAAC,QAAQ,CAAC,0DAA0D,CAAC,CAAC;AAExE,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;SAChC,QAAQ,CAAC,iEAAiE,CAAC;SAC3E,QAAQ,EAAE;IACb,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;SAC7B,QAAQ,CAAC,4FAA4F,CAAC;SACtG,QAAQ,EAAE;IACb,gBAAgB,EAAE,CAAC,CAAC,KAAK,CACvB,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,0FAA0F,CAAC,CAC9H;SACE,QAAQ,CAAC,4DAA4D,CAAC;SACtE,OAAO,CAAC,EAAE,CAAC;IACd,YAAY,EAAE,CAAC,CAAC,KAAK,CACnB,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,8EAA8E,CAAC,CAClH;SACE,QAAQ,CAAC,4EAA4E,CAAC;SACtF,OAAO,CAAC,EAAE,CAAC;IACd,UAAU,EAAE,CAAC,CAAC,KAAK,CACjB,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,6EAA6E,CAAC,CACjH;SACE,QAAQ,CAAC,iEAAiE,CAAC;SAC3E,OAAO,CAAC,EAAE,CAAC;IACd,cAAc,EAAE,CAAC,CAAC,KAAK,CACrB,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,oDAAoD,CAAC,CACxF;SACE,QAAQ,CAAC,qFAAqF,CAAC;SAC/F,OAAO,CAAC,EAAE,CAAC;CACf,CAAC,CAAC,QAAQ,CAAC,6FAA6F,CAAC,CAAC;AAE3G,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1C,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,mBAAmB,EAAE,gCAAgC,CAAC;SAC/E,QAAQ,CAAC,8EAA8E,CAAC;IAC3F,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC;SACtB,QAAQ,CAAC,8FAA8F,CAAC;IAC3G,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE;SAC3B,QAAQ,CAAC,6EAA6E,CAAC;SACvF,QAAQ,EAAE;IACb,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;SAC7B,QAAQ,CAAC,6GAA6G,CAAC;IAC1H,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;SAC5B,QAAQ,CAAC,8FAA8F,CAAC;SACxG,QAAQ,EAAE;IACb,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;SAClC,QAAQ,CAAC,4GAA4G,CAAC;SACtH,QAAQ,EAAE;IACb,KAAK,EAAE,eAAe;SACnB,QAAQ,CAAC,2CAA2C,CAAC;SACrD,OAAO,CAAC,EAAE,CAAC;IACd,OAAO,EAAE,iBAAiB;SACvB,QAAQ,CAAC,uEAAuE,CAAC;SACjF,OAAO,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;IAC9B,MAAM,EAAE,yBAAyB;SAC9B,QAAQ,CAAC,oEAAoE,CAAC;SAC9E,OAAO,CAAC;QACP,gBAAgB,EAAE,EAAE;QACpB,YAAY,EAAE,EAAE;QAChB,UAAU,EAAE,EAAE;QACd,cAAc,EAAE,EAAE;KACnB,CAAC;IACJ,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC;SAC7B,QAAQ,CAAC,oFAAoF,CAAC;SAC9F,OAAO,CAAC,EAAE,CAAC;IACd,MAAM,EAAE,CAAC,CAAC,KAAK,CACb,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,iEAAiE,CAAC,CACrG;SACE,QAAQ,CAAC,4FAA4F,CAAC;SACtG,OAAO,CAAC,EAAE,CAAC;IACd,IAAI,EAAE,CAAC,CAAC,KAAK,CACX,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,uDAAuD,CAAC,CAC3F;SACE,QAAQ,CAAC,yEAAyE,CAAC;SACnF,OAAO,CAAC,EAAE,CAAC;CACf,CAAC,CAAC,QAAQ,CAAC,sGAAsG,CAAC,CAAC;AAEpH,MAAM,CAAC,MAAM,cAAc,GAAG,mBAAmB,CAAC"}

package/dist/src/core/eval/trace-analyzer.d.ts

@@ -0,0 +1,6 @@
+import type { TraceEvent } from '../types.js';
+import type { EvalTraceMetrics } from './schema.js';
+export declare function analyzeTraceFiles(paths: string[]): EvalTraceMetrics;
+export declare function analyzeTrace(trace: TraceEvent[]): EvalTraceMetrics;
+export declare function readTraceFile(path: string): TraceEvent[];
+//# sourceMappingURL=trace-analyzer.d.ts.map

package/dist/src/core/eval/trace-analyzer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"trace-analyzer.d.ts","sourceRoot":"","sources":["../../../../src/core/eval/trace-analyzer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAKpD,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,gBAAgB,CAEnE;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,UAAU,EAAE,GAAG,gBAAgB,CAiFlE;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU,EAAE,CAMxD"}

package/dist/src/core/eval/trace-analyzer.js

@@ -0,0 +1,106 @@
+import { existsSync, readFileSync } from 'node:fs';
+const MUTATION_TOOLS = new Set(['edit_file', 'delete_file', 'move_file', 'run_shell_mutate']);
+const READ_OR_SEARCH_TOOLS = new Set(['read_file', 'list_files', 'search_files', 'run_shell_inspect']);
+export function analyzeTraceFiles(paths) {
+    return analyzeTrace(paths.flatMap(readTraceFile));
+}
+export function analyzeTrace(trace) {
+    const toolsByName = {};
+    const readOrSearchBeforeMutation = [];
+    let assistantTurns = 0;
+    let toolCalls = 0;
+    let toolResults = 0;
+    let mutations = 0;
+    let approvalsRequested = 0;
+    let approvalsResolved = 0;
+    let toolErrors = 0;
+    let verificationCommandsAfterMutation = 0;
+    const verificationCommandDetails = [];
+    let firstMutationStep;
+    let outcome;
+    let summary;
+    for (const event of trace) {
+        if (event.type === 'assistant.turn') {
+            assistantTurns++;
+            for (const call of event.toolCalls ?? []) {
+                toolCalls++;
+                toolsByName[call.tool] = (toolsByName[call.tool] ?? 0) + 1;
+                if (READ_OR_SEARCH_TOOLS.has(call.tool) && firstMutationStep === undefined) {
+                    readOrSearchBeforeMutation.push(summarizeToolInput(call.tool, call.input));
+                }
+                if (MUTATION_TOOLS.has(call.tool)) {
+                    mutations++;
+                    firstMutationStep ??= event.step;
+                }
+                if (firstMutationStep !== undefined
+                    && (call.tool === 'run_shell_mutate' || call.tool === 'run_shell_inspect')
+                    && isVerificationCommand(call.input)) {
+                    verificationCommandsAfterMutation++;
+                    verificationCommandDetails.push(summarizeToolInput(call.tool, call.input));
+                }
+            }
+            continue;
+        }
+        if (event.type === 'tool.result') {
+            toolResults++;
+            if (!event.result.ok) {
+                toolErrors++;
+            }
+        }
+        if (event.type === 'tool.approval_requested') {
+            approvalsRequested++;
+        }
+        if (event.type === 'tool.approval_resolved') {
+            approvalsResolved++;
+        }
+        if (event.type === 'run.finished') {
+            outcome = event.outcome;
+            summary = event.summary;
+        }
+    }
+    return {
+        assistantTurns,
+        toolCalls,
+        toolResults,
+        mutations,
+        approvalsRequested,
+        approvalsResolved,
+        toolErrors,
+        verificationCommandsAfterMutation,
+        verificationCommandDetails: [...new Set(verificationCommandDetails)],
+        firstMutationStep,
+        outcome,
+        summary,
+        toolsByName,
+        readOrSearchBeforeMutation: [...new Set(readOrSearchBeforeMutation)],
+    };
+}
+export function readTraceFile(path) {
+    if (!existsSync(path)) {
+        return [];
+    }
+    const parsed = JSON.parse(readFileSync(path, 'utf8'));
+    return Array.isArray(parsed) ? parsed.filter(isTraceEvent) : [];
+}
+function isTraceEvent(value) {
+    return Boolean(value && typeof value === 'object' && !Array.isArray(value) && typeof value.type === 'string');
+}
+function summarizeToolInput(tool, input) {
+    const object = input && typeof input === 'object' && !Array.isArray(input) ? input : undefined;
+    const path = object && typeof object.path === 'string' ? object.path : undefined;
+    const query = object && typeof object.query === 'string' ? object.query : undefined;
+    const command = object && typeof object.command === 'string' ? object.command : undefined;
+    return `${tool}:${path ?? query ?? command ?? ''}`.slice(0, 160);
+}
+function isVerificationCommand(input) {
+    if (!input || typeof input !== 'object' || Array.isArray(input)) {
+        return false;
+    }
+    const command = input.command;
+    if (typeof command !== 'string') {
+        return false;
+    }
+    return /\b(yarn|npm|pnpm|vitest|jest|mocha|tsc|eslint|cargo|go|pytest|python|ruff)\b/.test(command)
+        && /\b(test|build|lint|typecheck|check|vitest|tsc|pytest|ruff)\b/.test(command);
+}
+//# sourceMappingURL=trace-analyzer.js.map