@tangle-network/browser-agent-driver 0.6.0

package/dist/supervisor/policy.js

@@ -0,0 +1,118 @@
+import { snapshotHash } from '../recovery.js';
+const DEFAULT_WINDOW = 4;
+export function detectSupervisorSignal(input) {
+    const window = Math.max(3, input.window ?? DEFAULT_WINDOW);
+    const recent = input.recentTurns.slice(-window);
+    if (recent.length < 2) {
+        return {
+            severity: 'none',
+            reasons: [],
+            repeatedActionCount: 0,
+            unchangedTurns: 0,
+            errorTurns: 0,
+            verificationFailures: 0,
+        };
+    }
+    const fingerprints = [
+        ...recent.map((turn) => pageFingerprint(turn.state)),
+        pageFingerprint(input.currentState),
+    ];
+    const unchangedTurns = trailingSameCount(fingerprints);
+    const actionSignatures = recent
+        .map((turn) => actionSignature(turn.action))
+        .filter((signature) => signature !== 'wait');
+    const repeatedActionCount = trailingSameCount(actionSignatures);
+    const errorTurns = recent.filter((turn) => Boolean(turn.error)).length;
+    const verificationFailures = recent.filter((turn) => Boolean(turn.verificationFailure)).length;
+    const remainingTurns = Math.max(0, input.maxTurns - input.currentTurn);
+    const hardReasons = [];
+    if (unchangedTurns >= 4 && repeatedActionCount >= 3) {
+        hardReasons.push('same-page repeated action loop');
+    }
+    if (errorTurns >= 3) {
+        hardReasons.push('error burst');
+    }
+    if (unchangedTurns >= 4 && verificationFailures >= 2) {
+        hardReasons.push('verification deadlock');
+    }
+    if (remainingTurns <= 3 && (unchangedTurns >= 3 || errorTurns >= 2)) {
+        hardReasons.push('endgame stall with low turn budget');
+    }
+    const softReasons = [];
+    if (unchangedTurns >= 3)
+        softReasons.push('page unchanged');
+    if (repeatedActionCount >= 2)
+        softReasons.push('action repetition');
+    if (errorTurns >= 2)
+        softReasons.push('repeat execution errors');
+    if (verificationFailures >= 1)
+        softReasons.push('verification mismatch');
+    const reasons = hardReasons.length > 0 ? hardReasons : softReasons;
+    const severity = hardReasons.length > 0 ? 'hard' : (softReasons.length > 0 ? 'soft' : 'none');
+    return {
+        severity,
+        reasons,
+        repeatedActionCount,
+        unchangedTurns,
+        errorTurns,
+        verificationFailures,
+    };
+}
+export function formatSupervisorSignal(signal) {
+    if (signal.severity === 'none')
+        return 'no stall signal';
+    return [
+        `severity=${signal.severity}`,
+        `reasons=${signal.reasons.join(', ')}`,
+        `unchanged=${signal.unchangedTurns}`,
+        `repeatedActions=${signal.repeatedActionCount}`,
+        `errors=${signal.errorTurns}`,
+        `verificationFailures=${signal.verificationFailures}`,
+    ].join(' | ');
+}
+function pageFingerprint(state) {
+    return `${state.url}#${snapshotHash(state.snapshot)}`;
+}
+function trailingSameCount(values) {
+    if (values.length === 0)
+        return 0;
+    const last = values[values.length - 1];
+    let count = 0;
+    for (let i = values.length - 1; i >= 0; i--) {
+        if (values[i] !== last)
+            break;
+        count++;
+    }
+    return count;
+}
+function actionSignature(action) {
+    switch (action.action) {
+        case 'click':
+            return `click:${action.selector}`;
+        case 'type':
+            return `type:${action.selector}:${action.text.slice(0, 64)}`;
+        case 'press':
+            return `press:${action.selector}:${action.key}`;
+        case 'hover':
+            return `hover:${action.selector}`;
+        case 'select':
+            return `select:${action.selector}:${action.value}`;
+        case 'scroll':
+            return `scroll:${action.direction}:${action.amount ?? 500}:${action.selector ?? 'viewport'}`;
+        case 'navigate':
+            return `navigate:${action.url}`;
+        case 'wait':
+            return 'wait';
+        case 'evaluate':
+            return `evaluate:${action.criteria}`;
+        case 'runScript':
+            return `runScript:${action.script.slice(0, 64)}`;
+        case 'verifyPreview':
+            return 'verifyPreview';
+        case 'complete':
+            return 'complete';
+        case 'abort':
+            return 'abort';
+    }
+}
+//# sourceMappingURL=policy.js.map

package/dist/supervisor/policy.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"policy.js","sourceRoot":"","sources":["../../src/supervisor/policy.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAW9C,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB,MAAM,UAAU,sBAAsB,CAAC,KAA4B;IACjE,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,IAAI,cAAc,CAAC,CAAC;IAC3D,MAAM,MAAM,GAAG,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC;IAChD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,OAAO;YACL,QAAQ,EAAE,MAAM;YAChB,OAAO,EAAE,EAAE;YACX,mBAAmB,EAAE,CAAC;YACtB,cAAc,EAAE,CAAC;YACjB,UAAU,EAAE,CAAC;YACb,oBAAoB,EAAE,CAAC;SACxB,CAAC;IACJ,CAAC;IAED,MAAM,YAAY,GAAG;QACnB,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpD,eAAe,CAAC,KAAK,CAAC,YAAY,CAAC;KACpC,CAAC;IACF,MAAM,cAAc,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAC;IAEvD,MAAM,gBAAgB,GAAG,MAAM;SAC5B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SAC3C,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,KAAK,MAAM,CAAC,CAAC;IAC/C,MAAM,mBAAmB,GAAG,iBAAiB,CAAC,gBAAgB,CAAC,CAAC;IAEhE,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;IACvE,MAAM,oBAAoB,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,MAAM,CAAC;IAC/F,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC;IAEvE,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,IAAI,cAAc,IAAI,CAAC,IAAI,mBAAmB,IAAI,CAAC,EAAE,CAAC;QACpD,WAAW,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;IACrD,CAAC;IACD,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;QACpB,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAClC,CAAC;IACD,IAAI,cAAc,IAAI,CAAC,IAAI,oBAAoB,IAAI,CAAC,EAAE,CAAC;QACrD,WAAW,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,cAAc,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC;QACpE,WAAW,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,IAAI,cAAc,IAAI,CAAC;QAAE,WAAW,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC5D,IAAI,mBAAmB,IAAI,CAAC;QAAE,WAAW,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACpE,IAAI,UAAU,IAAI,CAAC;QAAE,WAAW,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;IACjE,IAAI,oBAAoB,IAAI,CAAC;QAAE,WAAW,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAEzE,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC;IACnE,MAAM,QAAQ,GAAiC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAE5H,OAAO;QACL,QAAQ;QACR,OAAO;QACP,mBAAmB;QACnB,cAAc;QACd,UAAU;QACV,oBAAoB;KACrB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,MAAwB;IAC7D,IAAI,MAAM,CAAC,QAAQ,KAAK,MAAM;QAAE,OAAO,iBAAiB,CAAC;IACzD,OAAO;QACL,YAAY,MAAM,CAAC,QAAQ,EAAE;QAC7B,WAAW,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACtC,aAAa,MAAM,CAAC,cAAc,EAAE;QACpC,mBAAmB,MAAM,CAAC,mBAAmB,EAAE;QAC/C,UAAU,MAAM,CAAC,UAAU,EAAE;QAC7B,wBAAwB,MAAM,CAAC,oBAAoB,EAAE;KACtD,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,eAAe,CAAC,KAAgB;IACvC,OAAO,GAAG,KAAK,CAAC,GAAG,IAAI,YAAY,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;AACxD,CAAC;AAED,SAAS,iBAAiB,CAAC,MAAgB;IACzC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACvC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI;YAAE,MAAM;QAC9B,KAAK,EAAE,CAAC;IACV,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,eAAe,CAAC,MAAc;IACrC,QAAQ,MAAM,CAAC,MAAM,EAAE,CAAC;QACtB,KAAK,OAAO;YACV,OAAO,SAAS,MAAM,CAAC,QAAQ,EAAE,CAAC;QACpC,KAAK,MAAM;YACT,OAAO,QAAQ,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;QAC/D,KAAK,OAAO;YACV,OAAO,SAAS,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC;QAClD,KAAK,OAAO;YACV,OAAO,SAAS,MAAM,CAAC,QAAQ,EAAE,CAAC;QACpC,KAAK,QAAQ;YACX,OAAO,UAAU,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACrD,KAAK,QAAQ;YACX,OAAO,UAAU,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,MAAM,IAAI,GAAG,IAAI,MAAM,CAAC,QAAQ,IAAI,UAAU,EAAE,CAAC;QAC/F,KAAK,UAAU;YACb,OAAO,YAAY,MAAM,CAAC,GAAG,EAAE,CAAC;QAClC,KAAK,MAAM;YACT,OAAO,MAAM,CAAC;QAChB,KAAK,UAAU;YACb,OAAO,YAAY,MAAM,CAAC,QAAQ,EAAE,CAAC;QACvC,KAAK,WAAW;YACd,OAAO,aAAa,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;QACnD,KAAK,eAAe;YAClB,OAAO,eAAe,CAAC;QACzB,KAAK,UAAU;YACb,OAAO,UAAU,CAAC;QACpB,KAAK,OAAO;YACV,OAAO,OAAO,CAAC;IACnB,CAAC;AACH,CAAC"}

package/dist/test-report.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Test Report Generation — markdown, JSON, HTML output + comparison.
+ */
+import type { TestSuiteResult } from './types.js';
+export interface ReportOptions {
+    format: 'json' | 'markdown' | 'html' | 'junit';
+    /** Include last N turns per test in report */
+    includeTurns?: boolean;
+    /** Include screenshots in report */
+    includeScreenshots?: boolean;
+}
+export declare function generateReport(suite: TestSuiteResult, options: ReportOptions): string;
+/** Compare two suite results (e.g., before/after improvement) */
+export declare function compareReports(before: TestSuiteResult, after: TestSuiteResult): string;
+//# sourceMappingURL=test-report.d.ts.map

package/dist/test-report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-report.d.ts","sourceRoot":"","sources":["../src/test-report.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAGlD,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,GAAG,UAAU,GAAG,MAAM,GAAG,OAAO,CAAC;IAC/C,8CAA8C;IAC9C,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,oCAAoC;IACpC,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,eAAe,EAAE,OAAO,EAAE,aAAa,GAAG,MAAM,CAarF;AAoMD,iEAAiE;AACjE,wBAAgB,cAAc,CAAC,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,eAAe,GAAG,MAAM,CAyCtF"}

package/dist/test-report.js

@@ -0,0 +1,241 @@
+/**
+ * Test Report Generation — markdown, JSON, HTML output + comparison.
+ */
+import { generateJUnitXml } from './reporters/junit.js';
+export function generateReport(suite, options) {
+    switch (options.format) {
+        case 'json':
+            return JSON.stringify(suite, null, 2);
+        case 'markdown':
+            return generateMarkdownReport(suite, options);
+        case 'html':
+            return generateHtmlReport(suite);
+        case 'junit':
+            return generateJUnitXml(suite);
+        default:
+            throw new Error(`Unknown format: ${options.format}`);
+    }
+}
+function generateMarkdownReport(suite, options) {
+    const lines = [];
+    const { summary, results } = suite;
+    lines.push('# Test Suite Report\n');
+    lines.push(`**Model:** ${suite.model}`);
+    if (suite.runtime) {
+        lines.push(`**Runtime:** ${suite.runtime.provider}${suite.runtime.sandboxBackendType ? ` -> ${suite.runtime.sandboxBackendType}` : ''}${suite.runtime.sandboxBackendProfile ? ` (${suite.runtime.sandboxBackendProfile})` : ''}`);
+    }
+    lines.push(`**Date:** ${suite.timestamp}`);
+    lines.push(`**Pass Rate:** ${(summary.passRate * 100).toFixed(1)}% (${summary.passed}/${summary.total})`);
+    lines.push(`**Duration:** ${(summary.totalDurationMs / 1000).toFixed(1)}s\n`);
+    // Summary metrics
+    lines.push('## Summary\n');
+    lines.push('| Metric | Value |');
+    lines.push('|--------|-------|');
+    lines.push(`| Total | ${summary.total} |`);
+    lines.push(`| Passed | ${summary.passed} |`);
+    lines.push(`| Failed | ${summary.failed} |`);
+    lines.push(`| Skipped | ${summary.skipped} |`);
+    lines.push(`| Avg Turns | ${summary.avgTurns.toFixed(1)} |`);
+    lines.push(`| Avg Tokens | ${summary.avgTokens.toFixed(0)} |`);
+    lines.push(`| Avg Duration | ${(summary.avgDurationMs / 1000).toFixed(1)}s |`);
+    lines.push(`| p50 Duration | ${(summary.p50DurationMs / 1000).toFixed(1)}s |`);
+    lines.push(`| p95 Duration | ${(summary.p95DurationMs / 1000).toFixed(1)}s |`);
+    // Per-test results
+    lines.push('\n## Results\n');
+    lines.push('| Test | Category | Agent | Verified | Turns | Tokens | Duration |');
+    lines.push('|------|----------|-------|----------|-------|--------|----------|');
+    for (const r of results) {
+        const agent = r.skipped ? 'SKIP' : r.agentSuccess ? 'Y' : 'N';
+        const verified = r.skipped ? 'SKIP' : r.verified ? 'PASS' : 'FAIL';
+        const duration = r.skipped ? '-' : `${(r.durationMs / 1000).toFixed(1)}s`;
+        const category = esc(r.testCase.category || '-');
+        lines.push(`| ${esc(r.testCase.name)} | ${category} | ${agent} | ${verified} | ${r.turnsUsed} | ${r.tokensUsed} | ${duration} |`);
+    }
+    // Failure details
+    const failures = results.filter((r) => !r.skipped && !r.verified);
+    if (failures.length > 0) {
+        lines.push('\n## Failures\n');
+        for (const r of failures) {
+            lines.push(`### ${esc(r.testCase.name)} (${r.testCase.id})\n`);
+            lines.push(`- **Goal:** ${esc(r.testCase.goal.slice(0, 150))}`);
+            lines.push(`- **Agent reported:** ${r.agentSuccess ? 'success' : 'failure'}`);
+            lines.push(`- **Verdict:** ${esc(r.verdict)}`);
+            lines.push(`- **Turns:** ${r.turnsUsed}/${r.testCase.maxTurns ?? 30}`);
+            if (r.phaseTimings) {
+                const phaseSummary = [
+                    fmtPhase('navigate', r.phaseTimings.initialNavigateMs),
+                    fmtPhase('observe', r.phaseTimings.firstObserveMs),
+                    fmtPhase('decide', r.phaseTimings.firstDecideMs),
+                    fmtPhase('execute', r.phaseTimings.firstExecuteMs),
+                ].filter(Boolean).join(', ');
+                if (phaseSummary) {
+                    lines.push(`- **Phase timings:** ${phaseSummary}`);
+                }
+            }
+            if (r.startupDiagnostics) {
+                const startupSummary = [
+                    `first turn seen ${r.startupDiagnostics.firstTurnSeen ? 'yes' : 'no'}`,
+                    r.startupDiagnostics.timeToFirstTurnMs !== undefined
+                        ? `time to first turn ${(r.startupDiagnostics.timeToFirstTurnMs / 1000).toFixed(1)}s`
+                        : '',
+                    r.startupDiagnostics.zeroTurnFailureClass
+                        ? `zero-turn class ${r.startupDiagnostics.zeroTurnFailureClass}`
+                        : '',
+                ].filter(Boolean).join(', ');
+                if (startupSummary) {
+                    lines.push(`- **Startup:** ${startupSummary}`);
+                }
+            }
+            if (r.wasteMetrics) {
+                lines.push(`- **Waste:** repeated queries ${r.wasteMetrics.repeatedQueryCount}, ` +
+                    `verification rejections ${r.wasteMetrics.verificationRejectionCount}, ` +
+                    `turns after evidence ${r.wasteMetrics.turnsAfterSufficientEvidence}, ` +
+                    `error turns ${r.wasteMetrics.errorTurns}`);
+            }
+            if (r.criteriaResults?.length) {
+                lines.push('- **Criteria:**');
+                for (const c of r.criteriaResults) {
+                    const icon = c.passed ? 'PASS' : 'FAIL';
+                    const desc = c.criterion.description || c.criterion.type;
+                    lines.push(`  - [${icon}] ${esc(desc)}${c.detail ? ': ' + esc(c.detail) : ''}`);
+                }
+            }
+            // Last actions for debugging
+            const lastTurns = r.agentResult.turns.slice(-5);
+            if (lastTurns.length > 0) {
+                lines.push('- **Last actions:**');
+                for (const t of lastTurns) {
+                    const actionStr = JSON.stringify(t.action).slice(0, 100);
+                    const verified = t.verified === false ? ' [UNVERIFIED]' : '';
+                    lines.push(`  - Turn ${t.turn}: ${esc(actionStr)}${verified}`);
+                }
+            }
+            lines.push('');
+        }
+    }
+    // Optional turn logs
+    if (options.includeTurns) {
+        const nonSkipped = results.filter((r) => !r.skipped && r.agentResult.turns.length > 0);
+        if (nonSkipped.length > 0) {
+            lines.push('\n## Turn Logs\n');
+            for (const r of nonSkipped) {
+                lines.push(`### ${r.verified ? 'PASS' : 'FAIL'} ${esc(r.testCase.name)}\n`);
+                lines.push('<details><summary>Turn Log</summary>\n');
+                for (const turn of r.agentResult.turns.slice(-10)) {
+                    lines.push(`- Turn ${turn.turn}: \`${esc(JSON.stringify(turn.action))}\``);
+                    if (turn.reasoning) {
+                        lines.push(`  - Reasoning: ${esc(turn.reasoning.slice(0, 100))}`);
+                    }
+                }
+                lines.push('\n</details>\n');
+            }
+        }
+    }
+    return lines.join('\n');
+}
+function generateHtmlReport(suite) {
+    const { summary } = suite;
+    return `<!DOCTYPE html>
+<html>
+<head>
+  <title>Test Suite Report</title>
+  <style>
+    body { font-family: system-ui, sans-serif; max-width: 1200px; margin: 0 auto; padding: 20px; }
+    .passed { color: #22c55e; }
+    .failed { color: #ef4444; }
+    .skipped { color: #9ca3af; }
+    .summary { display: flex; gap: 20px; margin: 20px 0; }
+    .summary-card { padding: 20px; border-radius: 8px; background: #f3f4f6; min-width: 100px; }
+    table { width: 100%; border-collapse: collapse; margin: 20px 0; }
+    th, td { padding: 12px; text-align: left; border-bottom: 1px solid #e5e7eb; }
+    th { background: #f9fafb; font-weight: 600; }
+    .metrics { display: grid; grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); gap: 12px; margin: 16px 0; }
+    .metric { padding: 12px; background: #f9fafb; border-radius: 6px; }
+    .metric-value { font-size: 1.5em; font-weight: 700; }
+    .metric-label { font-size: 0.85em; color: #6b7280; }
+  </style>
+</head>
+<body>
+  <h1>Test Suite Report</h1>
+  <p><strong>Model:</strong> ${suite.model}${suite.runtime ? ` | <strong>Runtime:</strong> ${suite.runtime.provider}${suite.runtime.sandboxBackendType ? ` -> ${suite.runtime.sandboxBackendType}` : ''}${suite.runtime.sandboxBackendProfile ? ` (${suite.runtime.sandboxBackendProfile})` : ''}` : ''} | <strong>Date:</strong> ${suite.timestamp} | <strong>Duration:</strong> ${(summary.totalDurationMs / 1000).toFixed(1)}s</p>
+
+  <div class="summary">
+    <div class="summary-card"><div class="metric-value passed">${summary.passed}</div><div class="metric-label">Passed</div></div>
+    <div class="summary-card"><div class="metric-value failed">${summary.failed}</div><div class="metric-label">Failed</div></div>
+    <div class="summary-card"><div class="metric-value skipped">${summary.skipped}</div><div class="metric-label">Skipped</div></div>
+    <div class="summary-card"><div class="metric-value">${(summary.passRate * 100).toFixed(0)}%</div><div class="metric-label">Pass Rate</div></div>
+  </div>
+
+  <div class="metrics">
+    <div class="metric"><div class="metric-value">${summary.avgTurns.toFixed(1)}</div><div class="metric-label">Avg Turns</div></div>
+    <div class="metric"><div class="metric-value">${summary.avgTokens.toFixed(0)}</div><div class="metric-label">Avg Tokens</div></div>
+    <div class="metric"><div class="metric-value">${(summary.p50DurationMs / 1000).toFixed(1)}s</div><div class="metric-label">p50 Duration</div></div>
+    <div class="metric"><div class="metric-value">${(summary.p95DurationMs / 1000).toFixed(1)}s</div><div class="metric-label">p95 Duration</div></div>
+  </div>
+
+  <table>
+    <thead>
+      <tr><th>Test</th><th>Status</th><th>Duration</th><th>Turns</th><th>Verdict</th></tr>
+    </thead>
+    <tbody>
+      ${suite.results.map((r) => {
+        const cls = r.skipped ? 'skipped' : r.verified ? 'passed' : 'failed';
+        const label = r.skipped ? 'SKIP' : r.verified ? 'PASS' : 'FAIL';
+        return `
+        <tr>
+          <td>${r.testCase.name}</td>
+          <td class="${cls}">${label}</td>
+          <td>${r.skipped ? '-' : (r.durationMs / 1000).toFixed(1) + 's'}</td>
+          <td>${r.turnsUsed}</td>
+          <td>${esc(r.verdict)}</td>
+        </tr>`;
+    }).join('')}
+    </tbody>
+  </table>
+</body>
+</html>`;
+}
+/** Compare two suite results (e.g., before/after improvement) */
+export function compareReports(before, after) {
+    const lines = [];
+    lines.push('# Test Suite Comparison\n');
+    lines.push(`**Before:** ${before.model} (${before.timestamp})`);
+    lines.push(`**After:** ${after.model} (${after.timestamp})\n`);
+    const bSum = before.summary;
+    const aSum = after.summary;
+    lines.push('| Metric | Before | After | Delta |');
+    lines.push('|--------|--------|-------|-------|');
+    const delta = (a, b, fmt) => {
+        const d = a - b;
+        const sign = d > 0 ? '+' : '';
+        return `${sign}${fmt(d)}`;
+    };
+    const pct = (n) => `${(n * 100).toFixed(1)}%`;
+    const dec1 = (n) => n.toFixed(1);
+    const sec = (n) => `${(n / 1000).toFixed(1)}s`;
+    lines.push(`| Pass Rate | ${pct(bSum.passRate)} | ${pct(aSum.passRate)} | ${delta(aSum.passRate, bSum.passRate, pct)} |`);
+    lines.push(`| Passed | ${bSum.passed} | ${aSum.passed} | ${delta(aSum.passed, bSum.passed, dec1)} |`);
+    lines.push(`| Avg Turns | ${dec1(bSum.avgTurns)} | ${dec1(aSum.avgTurns)} | ${delta(aSum.avgTurns, bSum.avgTurns, dec1)} |`);
+    lines.push(`| Avg Duration | ${sec(bSum.avgDurationMs)} | ${sec(aSum.avgDurationMs)} | ${delta(aSum.avgDurationMs, bSum.avgDurationMs, sec)} |`);
+    // Per-test comparison
+    lines.push('\n## Per-Test Comparison\n');
+    lines.push('| Test | Before | After |');
+    lines.push('|------|--------|-------|');
+    for (const afterResult of after.results) {
+        const beforeResult = before.results.find((r) => r.testCase.id === afterResult.testCase.id);
+        const bStatus = beforeResult ? (beforeResult.verified ? 'PASS' : 'FAIL') : 'N/A';
+        const aStatus = afterResult.verified ? 'PASS' : 'FAIL';
+        lines.push(`| ${esc(afterResult.testCase.name)} | ${bStatus} | ${aStatus} |`);
+    }
+    return lines.join('\n');
+}
+/** Escape pipe and newline for markdown table cells */
+function esc(s) {
+    return s.replace(/\|/g, '\\|').replace(/\n/g, ' ');
+}
+function fmtPhase(label, value) {
+    if (value === undefined)
+        return '';
+    return `${label} ${(value / 1000).toFixed(1)}s`;
+}
+//# sourceMappingURL=test-report.js.map

package/dist/test-report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-report.js","sourceRoot":"","sources":["../src/test-report.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAUxD,MAAM,UAAU,cAAc,CAAC,KAAsB,EAAE,OAAsB;IAC3E,QAAQ,OAAO,CAAC,MAAM,EAAE,CAAC;QACvB,KAAK,MAAM;YACT,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QACxC,KAAK,UAAU;YACb,OAAO,sBAAsB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAChD,KAAK,MAAM;YACT,OAAO,kBAAkB,CAAC,KAAK,CAAC,CAAC;QACnC,KAAK,OAAO;YACV,OAAO,gBAAgB,CAAC,KAAK,CAAC,CAAC;QACjC;YACE,MAAM,IAAI,KAAK,CAAC,mBAAmB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACzD,CAAC;AACH,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAsB,EAAE,OAAsB;IAC5E,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC;IAEnC,KAAK,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IACpC,KAAK,CAAC,IAAI,CAAC,cAAc,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IACxC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,gBAAgB,KAAK,CAAC,OAAO,CAAC,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,qBAAqB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACpO,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,aAAa,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;IAC3C,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC;IAC1G,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAE9E,kBAAkB;IAClB,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACjC,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACjC,KAAK,CAAC,IAAI,CAAC,aAAa,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC;IAC3C,KAAK,CAAC,IAAI,CAAC,cAAc,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;IAC7C,KAAK,CAAC,IAAI,CAAC,cAAc,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;IAC7C,KAAK,CAAC,IAAI,CAAC,eAAe,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;IAC/C,KAAK,CAAC,IAAI,CAAC,iBAAiB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC7D,KAAK,CAAC,IAAI,CAAC,kBAAkB,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC/D,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC/E,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC/E,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAE/E,mBAAmB;IACnB,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC7B,KAAK,CAAC,IAAI,CAAC,oEAAoE,CAAC,CAAC;IACjF,KAAK,CAAC,IAAI,CAAC,oEAAoE,CAAC,CAAC;IAEjF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAC9D,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QACnE,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;QAC1E,MAAM,QAAQ,GAAG,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,IAAI,GAAG,CAAC,CAAC;QACjD,KAAK,CAAC,IAAI,CACR,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,QAAQ,MAAM,KAAK,MAAM,QAAQ,MAAM,CAAC,CAAC,SAAS,MAAM,CAAC,CAAC,UAAU,MAAM,QAAQ,IAAI,CACtH,CAAC;IACJ,CAAC;IAED,kBAAkB;IAClB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAClE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,EAAE,KAAK,CAAC,CAAC;YAC/D,KAAK,CAAC,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;YAChE,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;YAC9E,KAAK,CAAC,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAC/C,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC,CAAC;YACvE,IAAI,CAAC,CAAC,YAAY,EAAE,CAAC;gBACnB,MAAM,YAAY,GAAG;oBACnB,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,YAAY,CAAC,iBAAiB,CAAC;oBACtD,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,YAAY,CAAC,cAAc,CAAC;oBAClD,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,YAAY,CAAC,aAAa,CAAC;oBAChD,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,YAAY,CAAC,cAAc,CAAC;iBACnD,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC7B,IAAI,YAAY,EAAE,CAAC;oBACjB,KAAK,CAAC,IAAI,CAAC,wBAAwB,YAAY,EAAE,CAAC,CAAC;gBACrD,CAAC;YACH,CAAC;YACD,IAAI,CAAC,CAAC,kBAAkB,EAAE,CAAC;gBACzB,MAAM,cAAc,GAAG;oBACrB,mBAAmB,CAAC,CAAC,kBAAkB,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE;oBACtE,CAAC,CAAC,kBAAkB,CAAC,iBAAiB,KAAK,SAAS;wBAClD,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,kBAAkB,CAAC,iBAAiB,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;wBACrF,CAAC,CAAC,EAAE;oBACN,CAAC,CAAC,kBAAkB,CAAC,oBAAoB;wBACvC,CAAC,CAAC,mBAAmB,CAAC,CAAC,kBAAkB,CAAC,oBAAoB,EAAE;wBAChE,CAAC,CAAC,EAAE;iBACP,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC7B,IAAI,cAAc,EAAE,CAAC;oBACnB,KAAK,CAAC,IAAI,CAAC,kBAAkB,cAAc,EAAE,CAAC,CAAC;gBACjD,CAAC;YACH,CAAC;YACD,IAAI,CAAC,CAAC,YAAY,EAAE,CAAC;gBACnB,KAAK,CAAC,IAAI,CACR,iCAAiC,CAAC,CAAC,YAAY,CAAC,kBAAkB,IAAI;oBACtE,2BAA2B,CAAC,CAAC,YAAY,CAAC,0BAA0B,IAAI;oBACxE,wBAAwB,CAAC,CAAC,YAAY,CAAC,4BAA4B,IAAI;oBACvE,eAAe,CAAC,CAAC,YAAY,CAAC,UAAU,EAAE,CAC3C,CAAC;YACJ,CAAC;YAED,IAAI,CAAC,CAAC,eAAe,EAAE,MAAM,EAAE,CAAC;gBAC9B,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;gBAC9B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,eAAe,EAAE,CAAC;oBAClC,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;oBACxC,MAAM,IAAI,GAAG,CAAC,CAAC,SAAS,CAAC,WAAW,IAAI,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC;oBACzD,KAAK,CAAC,IAAI,CAAC,QAAQ,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAClF,CAAC;YACH,CAAC;YAED,6BAA6B;YAC7B,MAAM,SAAS,GAAG,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAChD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;gBAClC,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;oBAC1B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;oBACzD,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,KAAK,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC7D,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,SAAS,CAAC,GAAG,QAAQ,EAAE,CAAC,CAAC;gBACjE,CAAC;YACH,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,qBAAqB;IACrB,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACvF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YAC/B,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;gBAC3B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC5E,KAAK,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;gBACrD,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;oBAClD,KAAK,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,OAAO,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC;oBAC3E,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;wBACnB,KAAK,CAAC,IAAI,CAAC,kBAAkB,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;oBACpE,CAAC;gBACH,CAAC;gBACD,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAsB;IAChD,MAAM,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC;IAC1B,OAAO;;;;;;;;;;;;;;;;;;;;;;+BAsBsB,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,gCAAgC,KAAK,CAAC,OAAO,CAAC,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,qBAAqB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,6BAA6B,KAAK,CAAC,SAAS,iCAAiC,CAAC,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;;;iEAG9V,OAAO,CAAC,MAAM;iEACd,OAAO,CAAC,MAAM;kEACb,OAAO,CAAC,OAAO;0DACvB,CAAC,OAAO,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;;;;oDAIzC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;oDAC3B,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;oDAC5B,CAAC,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;oDACzC,CAAC,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;;;;;;;;QAQrF,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACxB,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;QACrE,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAChE,OAAO;;gBAEC,CAAC,CAAC,QAAQ,CAAC,IAAI;uBACR,GAAG,KAAK,KAAK;gBACpB,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG;gBACxD,CAAC,CAAC,SAAS;gBACX,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC;cAChB,CAAC;IACT,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;;;;QAIT,CAAC;AACT,CAAC;AAED,iEAAiE;AACjE,MAAM,UAAU,cAAc,CAAC,MAAuB,EAAE,KAAsB;IAC5E,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IACxC,KAAK,CAAC,IAAI,CAAC,eAAe,MAAM,CAAC,KAAK,KAAK,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC;IAChE,KAAK,CAAC,IAAI,CAAC,cAAc,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,SAAS,KAAK,CAAC,CAAC;IAE/D,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC;IAC5B,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;IAClD,KAAK,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;IAElD,MAAM,KAAK,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,GAA0B,EAAE,EAAE;QACjE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAChB,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9B,OAAO,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5B,CAAC,CAAC;IAEF,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IACtD,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACzC,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IAEvD,KAAK,CAAC,IAAI,CAAC,iBAAiB,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IAC1H,KAAK,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,MAAM,MAAM,IAAI,CAAC,MAAM,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACtG,KAAK,CAAC,IAAI,CAAC,iBAAiB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7H,KAAK,CAAC,IAAI,CAAC,oBAAoB,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,aAAa,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IAEjJ,sBAAsB;IACtB,KAAK,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;IACzC,KAAK,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IACxC,KAAK,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IAExC,KAAK,MAAM,WAAW,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QACxC,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,KAAK,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAC3F,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QACjF,MAAM,OAAO,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QACvD,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,OAAO,MAAM,OAAO,IAAI,CAAC,CAAC;IAChF,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,uDAAuD;AACvD,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;AACrD,CAAC;AAED,SAAS,QAAQ,CAAC,KAAa,EAAE,KAAc;IAC7C,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IACnC,OAAO,GAAG,KAAK,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AAClD,CAAC"}

package/dist/test-runner.d.ts

@@ -0,0 +1,99 @@
+/**
+ * Test Runner — dependency-aware orchestration with ground-truth verification.
+ *
+ * Sequential mode (default): reuses a single driver, resolves dependencies.
+ * Parallel mode: spawns up to N concurrent workers, each with its own driver.
+ */
+import type { TestCase, TestResult, TestSuiteResult, AgentConfig, Turn } from './types.js';
+import type { Driver } from './drivers/types.js';
+import type { ArtifactSink, ProgressEvent } from './artifacts/types.js';
+import { type RunAnalysis } from './memory/analyzer.js';
+import type { ProjectStore } from './memory/project-store.js';
+export interface TestRunnerOptions {
+    /** Agent configuration (model, API key, vision, etc.) */
+    config?: AgentConfig;
+    /** Default per-test timeout in ms when a case omits timeoutMs */
+    defaultTimeoutMs?: number;
+    /** Single driver for sequential execution */
+    driver?: Driver;
+    /** Factory for creating isolated drivers (required for parallel execution) */
+    driverFactory?: () => Promise<Driver>;
+    /** Max concurrent test cases (default: 1 = sequential) */
+    concurrency?: number;
+    /** Stop suite on first failure */
+    stopOnFailure?: boolean;
+    /** Enable trajectory memory (loads/saves successful runs) */
+    enableMemory?: boolean;
+    /** Path to trajectory store directory */
+    trajectoryStorePath?: string;
+    /** Project memory store — enables knowledge, selectors, and domain-scoped trajectories */
+    projectStore?: ProjectStore;
+    /** Hints from previous run analysis — injected into each agent's context */
+    feedbackHints?: string;
+    /** Capture screenshot every N turns (0 = disabled) */
+    screenshotInterval?: number;
+    /** Called when a test case starts */
+    onTestStart?: (tc: TestCase) => void;
+    /** Called when a test case completes */
+    onTestComplete?: (result: TestResult) => void;
+    /** Called after each agent turn */
+    onTurn?: (tc: TestCase, turn: Turn) => void;
+    /** Pluggable artifact storage — screenshots, video, reports flow through this */
+    artifactSink?: ArtifactSink;
+    /** Unified progress event stream for dashboards/CI/coordinators */
+    onProgress?: (event: ProgressEvent) => void;
+    /** Per-worker timeout in ms — stuck workers get force-terminated (default: none) */
+    workerTimeoutMs?: number;
+}
+interface RunTestOptions {
+    signal?: AbortSignal;
+}
+export declare class TestRunner {
+    private config;
+    private driver;
+    private driverFactory;
+    private concurrency;
+    private stopOnFailure;
+    private store;
+    private projectStore?;
+    private feedbackHints;
+    private screenshotInterval;
+    private analyzer;
+    private onTestStart?;
+    private onTestComplete?;
+    private onTurn?;
+    private artifactSink?;
+    private onProgress?;
+    private workerTimeoutMs?;
+    private defaultTimeoutMs?;
+    private pendingArtifactOps;
+    constructor(options: TestRunnerOptions);
+    /** Run a single test case */
+    runTest(testCase: TestCase, driver?: Driver, options?: RunTestOptions): Promise<TestResult>;
+    /** Run a suite of test cases with dependency resolution and optional parallelism */
+    runSuite(cases: TestCase[]): Promise<TestSuiteResult>;
+    /** Analyze a completed suite run and return structured findings */
+    analyzeSuite(suite: TestSuiteResult): RunAnalysis;
+    /** Generate optimization hints from analysis — pass to feedbackHints on the next run */
+    generateHints(analysis: RunAnalysis): string;
+    private runSequential;
+    private runParallel;
+    private verifyCriteria;
+    private verifySingleCriterion;
+    private checkDependencies;
+    private resolveTimeoutMs;
+    private makeSkippedResult;
+    private buildVerdict;
+    /**
+     * Extract knowledge from successful test runs and merge into domain knowledge.
+     * Uses the Brain's extractKnowledge method to analyze trajectories.
+     */
+    private extractAndSaveKnowledge;
+    private buildSuiteResult;
+    private buildRuntimeConfig;
+    private trackArtifactOp;
+    private flushArtifactOps;
+    private startRuntimeObservability;
+}
+export {};
+//# sourceMappingURL=test-runner.d.ts.map

package/dist/test-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-runner.d.ts","sourceRoot":"","sources":["../src/test-runner.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH,OAAO,KAAK,EACV,QAAQ,EACR,UAAU,EACV,eAAe,EAGf,WAAW,EAEX,IAAI,EACL,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAIxE,OAAO,EAAsB,KAAK,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAC5E,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAmB9D,MAAM,WAAW,iBAAiB;IAChC,yDAAyD;IACzD,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,iEAAiE;IACjE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B,6CAA6C;IAC7C,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,8EAA8E;IAC9E,aAAa,CAAC,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IAEtC,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kCAAkC;IAClC,aAAa,CAAC,EAAE,OAAO,CAAC;IAExB,6DAA6D;IAC7D,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,yCAAyC;IACzC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,0FAA0F;IAC1F,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B,4EAA4E;IAC5E,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,sDAAsD;IACtD,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B,qCAAqC;IACrC,WAAW,CAAC,EAAE,CAAC,EAAE,EAAE,QAAQ,KAAK,IAAI,CAAC;IACrC,wCAAwC;IACxC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,KAAK,IAAI,CAAC;IAC9C,mCAAmC;IACnC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,KAAK,IAAI,CAAC;IAE5C,iFAAiF;IACjF,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,mEAAmE;IACnE,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;IAC5C,oFAAoF;IACpF,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,UAAU,cAAc;IACtB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAiCD,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAc;IAC5B,OAAO,CAAC,MAAM,CAAqB;IACnC,OAAO,CAAC,aAAa,CAAsC;IAC3D,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,aAAa,CAAU;IAC/B,OAAO,CAAC,KAAK,CAAyB;IACtC,OAAO,CAAC,YAAY,CAAC,CAAe;IACpC,OAAO,CAAC,aAAa,CAAqB;IAC1C,OAAO,CAAC,kBAAkB,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAqB;IACrC,OAAO,CAAC,WAAW,CAAC,CAAyB;IAC7C,OAAO,CAAC,cAAc,CAAC,CAA+B;IACtD,OAAO,CAAC,MAAM,CAAC,CAAqC;IACpD,OAAO,CAAC,YAAY,CAAC,CAAe;IACpC,OAAO,CAAC,UAAU,CAAC,CAAiC;IACpD,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,gBAAgB,CAAC,CAAS;IAClC,OAAO,CAAC,kBAAkB,CAAoC;gBAElD,OAAO,EAAE,iBAAiB;IA+BtC,6BAA6B;IACvB,OAAO,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,UAAU,CAAC;IA8QjG,oFAAoF;IAC9E,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC;IA4F3D,mEAAmE;IACnE,YAAY,CAAC,KAAK,EAAE,eAAe,GAAG,WAAW;IAIjD,wFAAwF;IACxF,aAAa,CAAC,QAAQ,EAAE,WAAW,GAAG,MAAM;YAM9B,aAAa;YAwBb,WAAW;YA0JX,cAAc;YAqBd,qBAAqB;IAgGnC,OAAO,CAAC,iBAAiB;IAczB,OAAO,CAAC,gBAAgB;IAKxB,OAAO,CAAC,iBAAiB;IA+BzB,OAAO,CAAC,YAAY;IAmBpB;;;OAGG;YACW,uBAAuB;IAsCrC,OAAO,CAAC,gBAAgB;IA0BxB,OAAO,CAAC,kBAAkB;IAU1B,OAAO,CAAC,eAAe;YAOT,gBAAgB;YAKhB,yBAAyB;CA8LxC"}