@tangle-network/browser-agent-driver 0.6.0

package/dist/test-runner.js

@@ -0,0 +1,1075 @@
+/**
+ * Test Runner — dependency-aware orchestration with ground-truth verification.
+ *
+ * Sequential mode (default): reuses a single driver, resolves dependencies.
+ * Parallel mode: spawns up to N concurrent workers, each with its own driver.
+ */
+import { existsSync, readFileSync, unlinkSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { AgentRunner } from './runner.js';
+import { Brain } from './brain/index.js';
+import { TrajectoryStore } from './memory/store.js';
+import { TrajectoryAnalyzer } from './memory/analyzer.js';
+import { AppKnowledge } from './memory/knowledge.js';
+import { generateReport } from './test-report.js';
+const DEFAULT_MAX_TURNS = 30;
+import { loadPricing, calculateCost } from './model-pricing.js';
+function combineAbortSignals(signals) {
+    const active = signals.filter((s) => !!s);
+    if (active.length === 0)
+        return undefined;
+    if (active.length === 1)
+        return active[0];
+    const controller = new AbortController();
+    const onAbort = (event) => {
+        const target = event.target;
+        controller.abort(target.reason || 'Cancelled');
+    };
+    for (const signal of active) {
+        if (signal.aborted) {
+            controller.abort(signal.reason || 'Cancelled');
+            return controller.signal;
+        }
+        signal.addEventListener('abort', onAbort, { once: true });
+    }
+    return controller.signal;
+}
+function getOrigin(url) {
+    if (!url)
+        return undefined;
+    try {
+        return new URL(url).origin;
+    }
+    catch {
+        return undefined;
+    }
+}
+export class TestRunner {
+    config;
+    driver;
+    driverFactory;
+    concurrency;
+    stopOnFailure;
+    store;
+    projectStore;
+    feedbackHints;
+    screenshotInterval;
+    analyzer;
+    onTestStart;
+    onTestComplete;
+    onTurn;
+    artifactSink;
+    onProgress;
+    workerTimeoutMs;
+    defaultTimeoutMs;
+    pendingArtifactOps = new Set();
+    constructor(options) {
+        if (!options.driver && !options.driverFactory) {
+            throw new Error('TestRunnerOptions requires either driver or driverFactory');
+        }
+        this.config = options.config || {};
+        this.driver = options.driver;
+        this.driverFactory = options.driverFactory;
+        this.concurrency = options.concurrency ?? 1;
+        this.stopOnFailure = options.stopOnFailure ?? false;
+        this.projectStore = options.projectStore;
+        this.store = options.enableMemory
+            ? new TrajectoryStore(options.trajectoryStorePath, {
+                enableScoring: this.config.traceScoring === true,
+                ttlDays: this.config.traceTtlDays ?? 30,
+            })
+            : null;
+        // Auto-load hints from project store if available
+        this.feedbackHints = options.feedbackHints ?? options.projectStore?.loadHints() ?? undefined;
+        this.screenshotInterval = options.screenshotInterval ?? 0;
+        this.analyzer = new TrajectoryAnalyzer();
+        this.onTestStart = options.onTestStart;
+        this.onTestComplete = options.onTestComplete;
+        this.onTurn = options.onTurn;
+        this.artifactSink = options.artifactSink;
+        this.onProgress = options.onProgress;
+        this.workerTimeoutMs = options.workerTimeoutMs;
+        // Backward-compat: honor non-typed timeoutMs on config if explicitly set by library users.
+        this.defaultTimeoutMs = options.defaultTimeoutMs
+            ?? (options.config?.timeoutMs);
+    }
+    /** Run a single test case */
+    async runTest(testCase, driver, options) {
+        const activeDriver = driver || this.driver;
+        if (!activeDriver) {
+            throw new Error('No driver available — provide driver or driverFactory');
+        }
+        const startedAt = new Date();
+        const screenshots = [];
+        this.onTestStart?.(testCase);
+        this.onProgress?.({ type: 'test:start', testId: testCase.id, testName: testCase.name, workerId: 0 });
+        // Run setup hook
+        const page = activeDriver.getPage?.();
+        const runtimeObservability = await this.startRuntimeObservability(testCase, page, activeDriver);
+        let runtimeOutcome;
+        const runStartedAtMs = Date.now();
+        let timeToFirstTurnMs;
+        const observedTurns = [];
+        const partialPhaseTimings = {};
+        if (testCase.setup && page) {
+            await testCase.setup(page);
+        }
+        try {
+            // Look up reference trajectory
+            let referenceTrajectory;
+            if (this.store) {
+                const match = this.store.findBestMatch(testCase.goal, {
+                    origin: getOrigin(testCase.startUrl),
+                });
+                if (match) {
+                    referenceTrajectory = this.store.formatAsReference(match);
+                }
+            }
+            // Build combined context: reference trajectory + feedback hints
+            let combinedReference = referenceTrajectory;
+            if (this.feedbackHints) {
+                combinedReference = combinedReference
+                    ? `${combinedReference}\n\n${this.feedbackHints}`
+                    : this.feedbackHints;
+            }
+            const runner = new AgentRunner({
+                driver: activeDriver,
+                config: this.config,
+                referenceTrajectory: combinedReference,
+                projectStore: this.projectStore,
+                onTurn: (turn) => {
+                    if (timeToFirstTurnMs === undefined) {
+                        timeToFirstTurnMs = Math.max(0, Date.now() - runStartedAtMs);
+                    }
+                    observedTurns.push(structuredClone(turn));
+                    this.onTurn?.(testCase, turn);
+                    this.onProgress?.({
+                        type: 'test:turn',
+                        testId: testCase.id,
+                        turn: turn.turn,
+                        action: turn.action.action,
+                        durationMs: turn.durationMs,
+                        tokensUsed: turn.tokensUsed,
+                        modelUsed: turn.modelUsed,
+                    });
+                    if (this.screenshotInterval > 0 && turn.turn % this.screenshotInterval === 0) {
+                        if (turn.state.screenshot) {
+                            screenshots.push({ turn: turn.turn, base64: turn.state.screenshot });
+                            // Emit screenshot through artifact sink
+                            if (this.artifactSink) {
+                                const buf = Buffer.from(turn.state.screenshot, 'base64');
+                                const pending = this.artifactSink.put({
+                                    type: 'screenshot',
+                                    testId: testCase.id,
+                                    name: `turn-${String(turn.turn).padStart(3, '0')}.jpg`,
+                                    data: buf,
+                                    contentType: 'image/jpeg',
+                                    metadata: { turn: String(turn.turn), action: turn.action.action },
+                                }).then(uri => {
+                                    this.onProgress?.({ type: 'test:artifact', testId: testCase.id, artifactType: 'screenshot', uri });
+                                }).catch(() => { }); // Best-effort
+                                this.trackArtifactOp(pending);
+                            }
+                        }
+                    }
+                },
+                onPhaseTiming: (phase, durationMs) => {
+                    if (phase === 'navigate')
+                        partialPhaseTimings.initialNavigateMs ??= durationMs;
+                    if (phase === 'observe')
+                        partialPhaseTimings.firstObserveMs ??= durationMs;
+                    if (phase === 'decide')
+                        partialPhaseTimings.firstDecideMs ??= durationMs;
+                    if (phase === 'execute')
+                        partialPhaseTimings.firstExecuteMs ??= durationMs;
+                },
+            });
+            // Build goal with success description
+            let goal = testCase.goal;
+            if (testCase.successDescription) {
+                goal += `\n\nSuccess criteria: ${testCase.successDescription}`;
+            }
+            // Run with timeout + abort signal
+            const timeoutMs = this.resolveTimeoutMs(testCase);
+            const timeoutAbortController = timeoutMs && timeoutMs > 0 ? new AbortController() : undefined;
+            const combinedSignal = combineAbortSignals([timeoutAbortController?.signal, options?.signal]);
+            const runPromise = runner.run({
+                goal,
+                startUrl: testCase.startUrl,
+                allowedDomains: testCase.allowedDomains,
+                maxTurns: testCase.maxTurns ?? DEFAULT_MAX_TURNS,
+                signal: combinedSignal,
+            });
+            let agentResult;
+            if (timeoutMs && timeoutMs > 0 && timeoutAbortController) {
+                let timer;
+                const timeoutPromise = new Promise((resolve) => {
+                    timer = setTimeout(() => {
+                        const timedOutBeforeFirstTurn = timeToFirstTurnMs === undefined;
+                        const reason = timedOutBeforeFirstTurn
+                            ? `Pre-first-turn timeout after ${timeoutMs}ms`
+                            : `Test timed out after ${timeoutMs}ms`;
+                        timeoutAbortController.abort(reason);
+                        resolve({
+                            success: false,
+                            reason,
+                            turns: [...observedTurns],
+                            totalMs: timeoutMs,
+                            phaseTimings: partialPhaseTimings,
+                            startupDiagnostics: timedOutBeforeFirstTurn
+                                ? {
+                                    firstTurnSeen: false,
+                                    zeroTurnFailureClass: 'pre_first_turn_timeout',
+                                    startupReason: reason,
+                                }
+                                : {
+                                    firstTurnSeen: true,
+                                    timeToFirstTurnMs,
+                                },
+                            wasteMetrics: {
+                                repeatedQueryCount: 0,
+                                verificationRejectionCount: 0,
+                                turnsAfterSufficientEvidence: 0,
+                                errorTurns: 0,
+                            },
+                        });
+                    }, timeoutMs);
+                });
+                agentResult = await Promise.race([runPromise, timeoutPromise])
+                    .finally(() => { if (timer)
+                    clearTimeout(timer); });
+            }
+            else {
+                agentResult = await runPromise;
+            }
+            agentResult = attachStartupDiagnostics(agentResult, timeToFirstTurnMs);
+            // Ground-truth verification
+            // Default to agent outcome when no explicit criteria are provided.
+            let verified = agentResult.success;
+            let criteriaResults;
+            if (testCase.successCriteria?.length && page) {
+                criteriaResults = await this.verifyCriteria(testCase.successCriteria, page);
+                verified = agentResult.success && criteriaResults.every((c) => c.passed);
+            }
+            if (!agentResult.success) {
+                verified = false;
+            }
+            // Save trajectory if memory enabled
+            const tokensUsed = agentResult.turns.reduce((sum, t) => sum + (t.tokensUsed || 0), 0);
+            const totalInputTokens = agentResult.turns.reduce((sum, t) => sum + (t.inputTokens || 0), 0);
+            const totalOutputTokens = agentResult.turns.reduce((sum, t) => sum + (t.outputTokens || 0), 0);
+            if (this.store) {
+                this.store.save(testCase.goal, agentResult.turns, verified && agentResult.success, this.config.model || 'unknown', { origin: getOrigin(testCase.startUrl) });
+            }
+            const endedAt = new Date();
+            const verdict = this.buildVerdict(agentResult, verified, criteriaResults);
+            // Calculate cost from LiteLLM's pricing database
+            const modelName = this.config.model || 'gpt-5.4';
+            const estimatedCostUsd = calculateCost(modelName, totalInputTokens, totalOutputTokens);
+            const result = {
+                testCase,
+                agentResult,
+                agentSuccess: agentResult.success,
+                verified,
+                criteriaResults,
+                verdict,
+                turnsUsed: agentResult.turns.length,
+                tokensUsed,
+                inputTokens: totalInputTokens || undefined,
+                outputTokens: totalOutputTokens || undefined,
+                estimatedCostUsd: estimatedCostUsd || undefined,
+                durationMs: endedAt.getTime() - startedAt.getTime(),
+                phaseTimings: agentResult.phaseTimings,
+                startupDiagnostics: agentResult.startupDiagnostics,
+                wasteMetrics: agentResult.wasteMetrics,
+                startedAt,
+                endedAt,
+                screenshots: screenshots.length > 0 ? screenshots : undefined,
+                runtime: this.buildRuntimeConfig(),
+            };
+            // Emit video artifact if available
+            if (this.artifactSink && page) {
+                try {
+                    const videoPath = await page.video?.()?.path?.();
+                    if (videoPath) {
+                        const fs = await import('node:fs');
+                        if (fs.existsSync(videoPath)) {
+                            const videoData = fs.readFileSync(videoPath);
+                            const uri = await this.artifactSink.put({
+                                type: 'video',
+                                testId: testCase.id,
+                                name: 'recording.webm',
+                                data: videoData,
+                                contentType: 'video/webm',
+                                metadata: { durationMs: String(result.durationMs), turnsUsed: String(result.turnsUsed) },
+                            });
+                            this.onProgress?.({ type: 'test:artifact', testId: testCase.id, artifactType: 'video', uri });
+                        }
+                    }
+                }
+                catch {
+                    // Video capture is best-effort
+                }
+            }
+            this.onTestComplete?.(result);
+            this.onProgress?.({
+                type: 'test:complete',
+                testId: testCase.id,
+                passed: result.verified && result.agentSuccess,
+                verdict: result.verdict,
+                durationMs: result.durationMs,
+                turnsUsed: result.turnsUsed,
+                tokensUsed: result.tokensUsed,
+                estimatedCostUsd: result.estimatedCostUsd,
+            });
+            runtimeOutcome = {
+                passed: result.verified && result.agentSuccess,
+                verdict: result.verdict,
+                durationMs: result.durationMs,
+                turnsUsed: result.turnsUsed,
+            };
+            return result;
+        }
+        finally {
+            await runtimeObservability?.finalize(runtimeOutcome).catch((err) => {
+                if (this.config.debug) {
+                    console.log(`[TestRunner] Observability finalize failed for ${testCase.id}: ${err instanceof Error ? err.message : err}`);
+                }
+            });
+            if (testCase.teardown && page) {
+                await testCase.teardown(page).catch((err) => {
+                    if (this.config.debug) {
+                        console.log(`[TestRunner] Teardown failed for ${testCase.id}: ${err instanceof Error ? err.message : err}`);
+                    }
+                });
+            }
+        }
+    }
+    /** Run a suite of test cases with dependency resolution and optional parallelism */
+    async runSuite(cases) {
+        // Pre-load pricing database (async fetch, cached 24h)
+        loadPricing().catch(() => { });
+        this.onProgress?.({ type: 'suite:start', totalTests: cases.length, concurrency: this.concurrency });
+        let result;
+        if (this.concurrency > 1 && this.driverFactory) {
+            result = await this.runParallel(cases);
+        }
+        else {
+            result = await this.runSequential(cases);
+        }
+        // Emit reports as artifacts
+        if (this.artifactSink) {
+            try {
+                await this.flushArtifactOps();
+                const jsonReport = generateReport(result, { format: 'json' });
+                const jsonUri = await this.artifactSink.put({
+                    type: 'report-json',
+                    testId: 'suite',
+                    name: 'report.json',
+                    data: Buffer.from(jsonReport, 'utf-8'),
+                    contentType: 'application/json',
+                    metadata: { model: result.model, timestamp: result.timestamp },
+                });
+                this.onProgress?.({ type: 'test:artifact', testId: 'suite', artifactType: 'report-json', uri: jsonUri });
+                const mdReport = generateReport(result, { format: 'markdown', includeTurns: true });
+                const mdUri = await this.artifactSink.put({
+                    type: 'report-md',
+                    testId: 'suite',
+                    name: 'report.md',
+                    data: Buffer.from(mdReport, 'utf-8'),
+                    contentType: 'text/markdown',
+                });
+                this.onProgress?.({ type: 'test:artifact', testId: 'suite', artifactType: 'report-md', uri: mdUri });
+            }
+            catch {
+                // Report emission is best-effort
+            }
+        }
+        // Post-suite memory operations
+        if (this.projectStore) {
+            this.projectStore.saveRunSummary(result);
+            // Generate and save optimization hints
+            const analysis = this.analyzer.analyze(result);
+            const hints = this.analyzer.generateHints(analysis);
+            if (hints) {
+                this.projectStore.saveHints(hints);
+            }
+            // Extract knowledge from successful runs (async, best-effort)
+            await this.extractAndSaveKnowledge(result).catch((err) => {
+                if (this.config.debug) {
+                    console.log(`[TestRunner] Knowledge extraction failed: ${err instanceof Error ? err.message : err}`);
+                }
+            });
+        }
+        // Finalize artifact sink (writes manifest, flushes)
+        let manifestUri;
+        if (this.artifactSink) {
+            try {
+                const manifest = this.artifactSink.getManifest();
+                manifestUri = await this.artifactSink.put({
+                    type: 'report-json',
+                    testId: 'suite',
+                    name: 'manifest.json',
+                    data: Buffer.from(JSON.stringify(manifest, null, 2), 'utf-8'),
+                    contentType: 'application/json',
+                });
+                await this.artifactSink.close?.();
+            }
+            catch {
+                // Best-effort
+            }
+        }
+        const totalCostUsd = result.results.reduce((sum, r) => sum + (r.estimatedCostUsd || 0), 0);
+        this.onProgress?.({
+            type: 'suite:complete',
+            passed: result.summary.passed,
+            failed: result.summary.failed,
+            skipped: result.summary.skipped,
+            totalMs: result.summary.totalDurationMs,
+            totalCostUsd: totalCostUsd || undefined,
+            manifestUri,
+        });
+        return result;
+    }
+    /** Analyze a completed suite run and return structured findings */
+    analyzeSuite(suite) {
+        return this.analyzer.analyze(suite);
+    }
+    /** Generate optimization hints from analysis — pass to feedbackHints on the next run */
+    generateHints(analysis) {
+        return this.analyzer.generateHints(analysis);
+    }
+    // ── Sequential execution ──
+    async runSequential(cases) {
+        const sorted = [...cases].sort((a, b) => (a.priority ?? 99) - (b.priority ?? 99));
+        const results = [];
+        for (const tc of sorted) {
+            // Check dependencies
+            const skipReason = this.checkDependencies(tc, results);
+            if (skipReason) {
+                const skipped = this.makeSkippedResult(tc, skipReason);
+                results.push(skipped);
+                continue;
+            }
+            const result = await this.runTest(tc);
+            results.push(result);
+            if (!result.verified && this.stopOnFailure)
+                break;
+        }
+        return this.buildSuiteResult(results);
+    }
+    // ── Parallel execution ──
+    async runParallel(cases) {
+        const sorted = [...cases].sort((a, b) => (a.priority ?? 99) - (b.priority ?? 99));
+        const knownCaseIds = new Set(sorted.map((c) => c.id));
+        const results = [];
+        const completed = new Map();
+        const pending = new Set(sorted.map((c) => c.id));
+        let stopped = false;
+        let runningCount = 0;
+        const tryDequeue = () => {
+            for (const tc of sorted) {
+                if (!pending.has(tc.id))
+                    continue;
+                const missingDeps = (tc.dependsOn ?? []).filter((dep) => !knownCaseIds.has(dep));
+                if (missingDeps.length > 0) {
+                    pending.delete(tc.id);
+                    const skipped = this.makeSkippedResult(tc, `Dependencies not found: ${missingDeps.join(', ')}`);
+                    completed.set(tc.id, skipped);
+                    results.push(skipped);
+                    return tryDequeue();
+                }
+                // Check if dependencies are met
+                if (tc.dependsOn?.length) {
+                    const unmet = tc.dependsOn.some((dep) => {
+                        const depResult = completed.get(dep);
+                        return !depResult || !depResult.verified;
+                    });
+                    // If a dependency failed, skip this case
+                    const depFailed = tc.dependsOn.some((dep) => {
+                        const depResult = completed.get(dep);
+                        return depResult && !depResult.verified;
+                    });
+                    if (depFailed) {
+                        const failedDeps = tc.dependsOn.filter(dep => {
+                            const r = completed.get(dep);
+                            return r && !r.verified;
+                        });
+                        pending.delete(tc.id);
+                        const skipped = this.makeSkippedResult(tc, `Dependencies failed: ${failedDeps.join(', ')}`);
+                        completed.set(tc.id, skipped);
+                        results.push(skipped);
+                        return tryDequeue(); // Try next
+                    }
+                    if (unmet)
+                        continue; // Dependencies not yet resolved, try another
+                }
+                pending.delete(tc.id);
+                return tc;
+            }
+            return null;
+        };
+        const markUnresolvablePending = () => {
+            if (pending.size === 0)
+                return;
+            for (const tc of sorted) {
+                if (!pending.has(tc.id))
+                    continue;
+                pending.delete(tc.id);
+                const unresolvedDeps = (tc.dependsOn ?? []).filter((dep) => {
+                    const depResult = completed.get(dep);
+                    if (!depResult)
+                        return true;
+                    return !depResult.verified;
+                });
+                const reason = unresolvedDeps.length > 0
+                    ? `Unresolvable dependencies (cycle or unmet): ${unresolvedDeps.join(', ')}`
+                    : 'Unresolvable dependency graph (no runnable tests remain)';
+                const skipped = this.makeSkippedResult(tc, reason);
+                completed.set(tc.id, skipped);
+                results.push(skipped);
+            }
+        };
+        const worker = async () => {
+            while (!stopped) {
+                const tc = tryDequeue();
+                if (!tc) {
+                    if (pending.size === 0)
+                        return;
+                    if (runningCount === 0) {
+                        markUnresolvablePending();
+                        continue;
+                    }
+                    // Wait for dependencies to resolve
+                    await new Promise((r) => setTimeout(r, 100));
+                    continue;
+                }
+                runningCount += 1;
+                const driver = await this.driverFactory();
+                try {
+                    let result;
+                    if (this.workerTimeoutMs) {
+                        const workerAbortController = new AbortController();
+                        const runPromise = this.runTest(tc, driver, { signal: workerAbortController.signal });
+                        let timeout;
+                        const timeoutPromise = new Promise((resolve) => {
+                            timeout = setTimeout(() => {
+                                workerAbortController.abort(`Worker timed out after ${this.workerTimeoutMs}ms`);
+                                resolve(this.makeSkippedResult(tc, `Worker timed out after ${this.workerTimeoutMs}ms`));
+                            }, this.workerTimeoutMs);
+                        });
+                        try {
+                            result = await Promise.race([runPromise, timeoutPromise]);
+                            if (result.skipped) {
+                                // Keep background promise from surfacing unhandled rejections.
+                                runPromise.catch(() => { });
+                            }
+                        }
+                        finally {
+                            if (timeout)
+                                clearTimeout(timeout);
+                        }
+                    }
+                    else {
+                        result = await this.runTest(tc, driver);
+                    }
+                    completed.set(tc.id, result);
+                    results.push(result);
+                    if (!result.verified && this.stopOnFailure) {
+                        if (!stopped) {
+                            stopped = true;
+                            for (const remainingId of [...pending]) {
+                                const remaining = sorted.find((c) => c.id === remainingId);
+                                if (!remaining)
+                                    continue;
+                                pending.delete(remaining.id);
+                                const skipped = this.makeSkippedResult(remaining, `Stopped after failure in ${tc.id}`);
+                                completed.set(remaining.id, skipped);
+                                results.push(skipped);
+                            }
+                        }
+                    }
+                }
+                finally {
+                    runningCount = Math.max(0, runningCount - 1);
+                    await driver.close?.();
+                }
+            }
+        };
+        // Spawn workers
+        const workers = Array.from({ length: Math.min(this.concurrency, cases.length) }, () => worker());
+        await Promise.all(workers);
+        // Preserve original sort order in results
+        const ordered = sorted.map((tc) => results.find((r) => r.testCase.id === tc.id) || this.makeSkippedResult(tc, 'Not reached'));
+        return this.buildSuiteResult(ordered);
+    }
+    // ── Verification ──
+    async verifyCriteria(criteria, page) {
+        const results = [];
+        for (const criterion of criteria) {
+            try {
+                results.push(await this.verifySingleCriterion(criterion, page));
+            }
+            catch (err) {
+                results.push({
+                    criterion,
+                    passed: false,
+                    detail: `Error: ${err instanceof Error ? err.message : String(err)}`,
+                });
+            }
+        }
+        return results;
+    }
+    async verifySingleCriterion(criterion, page) {
+        switch (criterion.type) {
+            case 'url-contains': {
+                const url = page.url();
+                const passed = url.includes(criterion.value || '');
+                return {
+                    criterion,
+                    passed,
+                    detail: passed ? undefined : `URL "${url}" does not contain "${criterion.value}"`,
+                };
+            }
+            case 'url-matches': {
+                const url = page.url();
+                const regex = new RegExp(criterion.value || '');
+                const passed = regex.test(url);
+                return {
+                    criterion,
+                    passed,
+                    detail: passed ? undefined : `URL "${url}" does not match pattern "${criterion.value}"`,
+                };
+            }
+            case 'element-visible': {
+                if (!criterion.selector) {
+                    return { criterion, passed: false, detail: 'No selector provided' };
+                }
+                const visible = await page
+                    .locator(criterion.selector)
+                    .first()
+                    .isVisible({ timeout: 5000 })
+                    .catch(() => false);
+                return {
+                    criterion,
+                    passed: visible,
+                    detail: visible ? undefined : `Element "${criterion.selector}" is not visible`,
+                };
+            }
+            case 'element-text': {
+                if (!criterion.selector) {
+                    return { criterion, passed: false, detail: 'No selector provided' };
+                }
+                const text = await page
+                    .locator(criterion.selector)
+                    .first()
+                    .textContent({ timeout: 5000 })
+                    .catch(() => '');
+                const passed = (text || '').includes(criterion.value || '');
+                return {
+                    criterion,
+                    passed,
+                    detail: passed
+                        ? undefined
+                        : `Element "${criterion.selector}" text "${text}" does not contain "${criterion.value}"`,
+                };
+            }
+            case 'element-count': {
+                if (!criterion.selector) {
+                    return { criterion, passed: false, detail: 'No selector provided' };
+                }
+                const count = await page.locator(criterion.selector).count();
+                const expected = parseInt(criterion.value || '0', 10);
+                const passed = count >= expected;
+                return {
+                    criterion,
+                    passed,
+                    detail: passed
+                        ? undefined
+                        : `Found ${count} elements matching "${criterion.selector}", expected >= ${expected}`,
+                };
+            }
+            case 'custom': {
+                if (!criterion.check) {
+                    return { criterion, passed: false, detail: 'No check function provided' };
+                }
+                const passed = await criterion.check(page);
+                return {
+                    criterion,
+                    passed,
+                    detail: passed ? undefined : `Custom check failed: ${criterion.description || 'unknown'}`,
+                };
+            }
+            default:
+                return { criterion, passed: false, detail: `Unknown criterion type: ${criterion.type}` };
+        }
+    }
+    // ── Helpers ──
+    checkDependencies(tc, completed) {
+        if (!tc.dependsOn?.length)
+            return undefined;
+        const unmet = tc.dependsOn.filter((dep) => {
+            const depResult = completed.find((r) => r.testCase.id === dep);
+            return !depResult || !depResult.verified;
+        });
+        if (unmet.length > 0) {
+            return `Dependencies not met: ${unmet.join(', ')}`;
+        }
+        return undefined;
+    }
+    resolveTimeoutMs(testCase) {
+        if (testCase.timeoutMs !== undefined)
+            return testCase.timeoutMs;
+        return this.defaultTimeoutMs;
+    }
+    makeSkippedResult(tc, reason) {
+        const now = new Date();
+        return {
+            testCase: tc,
+            agentResult: { success: false, reason, turns: [], totalMs: 0 },
+            agentSuccess: false,
+            verified: false,
+            verdict: `Skipped: ${reason}`,
+            turnsUsed: 0,
+            tokensUsed: 0,
+            durationMs: 0,
+            phaseTimings: {},
+            startupDiagnostics: {
+                firstTurnSeen: false,
+                zeroTurnFailureClass: 'unknown',
+                startupReason: reason,
+            },
+            wasteMetrics: {
+                repeatedQueryCount: 0,
+                verificationRejectionCount: 0,
+                turnsAfterSufficientEvidence: 0,
+                errorTurns: 0,
+            },
+            startedAt: now,
+            endedAt: now,
+            skipped: true,
+            skipReason: reason,
+            runtime: this.buildRuntimeConfig(),
+        };
+    }
+    buildVerdict(agentResult, verified, criteriaResults) {
+        if (verified && agentResult.success) {
+            return agentResult.result || 'Goal achieved';
+        }
+        if (!agentResult.success) {
+            return agentResult.reason || 'Agent failed';
+        }
+        // Agent reported success but verification failed
+        const failedCriteria = criteriaResults
+            ?.filter((c) => !c.passed)
+            .map((c) => c.detail || c.criterion.description || c.criterion.type)
+            .join('; ');
+        return `Verification failed: ${failedCriteria || 'unknown criteria'}`;
+    }
+    /**
+     * Extract knowledge from successful test runs and merge into domain knowledge.
+     * Uses the Brain's extractKnowledge method to analyze trajectories.
+     */
+    async extractAndSaveKnowledge(suite) {
+        if (!this.projectStore)
+            return;
+        const successfulResults = suite.results.filter(r => r.verified && !r.skipped);
+        if (successfulResults.length === 0)
+            return;
+        const brain = new Brain(this.config);
+        for (const result of successfulResults) {
+            const startUrl = result.testCase.startUrl;
+            if (!startUrl)
+                continue;
+            // Format trajectory for extraction
+            const trajectoryText = result.agentResult.turns
+                .map((t, i) => {
+                const action = JSON.stringify(t.action);
+                const verified = t.verified ? ' [verified]' : '';
+                return `${i + 1}. URL: ${t.state.url} → ${action}${verified}`;
+            })
+                .join('\n');
+            const facts = await brain.extractKnowledge(trajectoryText, startUrl);
+            if (facts.length === 0)
+                continue;
+            // Merge into domain knowledge
+            const knowledge = new AppKnowledge(this.projectStore.getKnowledgePath(startUrl), startUrl);
+            knowledge.recordFacts(facts);
+            knowledge.save();
+            if (this.config.debug) {
+                console.log(`[TestRunner] Extracted ${facts.length} facts for ${startUrl}`);
+            }
+        }
+    }
+    buildSuiteResult(results) {
+        const nonSkipped = results.filter((r) => !r.skipped);
+        const durations = nonSkipped.map((r) => r.durationMs).sort((a, b) => a - b);
+        const passed = nonSkipped.filter((r) => r.verified && r.agentSuccess).length;
+        return {
+            model: this.config.model || 'unknown',
+            runtime: this.buildRuntimeConfig(),
+            timestamp: new Date().toISOString(),
+            results,
+            summary: {
+                total: results.length,
+                passed,
+                failed: nonSkipped.length - passed,
+                skipped: results.length - nonSkipped.length,
+                passRate: nonSkipped.length > 0 ? passed / nonSkipped.length : 0,
+                avgTurns: avg(nonSkipped.map((r) => r.turnsUsed)),
+                avgTokens: avg(nonSkipped.map((r) => r.tokensUsed)),
+                avgDurationMs: avg(durations),
+                p50DurationMs: percentile(durations, 0.5),
+                p95DurationMs: percentile(durations, 0.95),
+                totalDurationMs: durations.reduce((sum, d) => sum + d, 0),
+            },
+        };
+    }
+    buildRuntimeConfig() {
+        return {
+            provider: this.config.provider || 'openai',
+            model: this.config.model || 'unknown',
+            ...(this.config.sandboxBackendType ? { sandboxBackendType: this.config.sandboxBackendType } : {}),
+            ...(this.config.sandboxBackendProfile ? { sandboxBackendProfile: this.config.sandboxBackendProfile } : {}),
+            ...(this.config.sandboxBackendProvider ? { sandboxBackendProvider: this.config.sandboxBackendProvider } : {}),
+        };
+    }
+    trackArtifactOp(p) {
+        this.pendingArtifactOps.add(p);
+        p.finally(() => {
+            this.pendingArtifactOps.delete(p);
+        }).catch(() => { });
+    }
+    async flushArtifactOps() {
+        if (this.pendingArtifactOps.size === 0)
+            return;
+        await Promise.allSettled([...this.pendingArtifactOps]);
+    }
+    async startRuntimeObservability(testCase, page, driver) {
+        const config = this.config.observability;
+        if (config?.enabled === false || !page || typeof page.on !== 'function') {
+            return null;
+        }
+        const captureConsole = config?.captureConsole !== false;
+        const captureNetwork = config?.captureNetwork !== false;
+        const tracePolicy = config?.tracePolicy ?? 'on-failure';
+        const maxConsoleEntries = config?.maxConsoleEntries ?? 200;
+        const maxNetworkEntries = config?.maxNetworkEntries ?? 200;
+        const observabilityStartedAt = new Date().toISOString();
+        const consoleEntries = [];
+        const pageErrors = [];
+        const requestFailures = [];
+        const responseErrors = [];
+        const pushLimited = (target, entry, limit) => {
+            if (target.length >= limit)
+                return;
+            target.push(entry);
+        };
+        const onConsole = (msg) => {
+            const level = msg.type();
+            if (!['warning', 'error', 'assert'].includes(level))
+                return;
+            const location = msg.location();
+            pushLimited(consoleEntries, {
+                level,
+                text: msg.text(),
+                url: location.url || '',
+                line: String(location.lineNumber ?? ''),
+                column: String(location.columnNumber ?? ''),
+            }, maxConsoleEntries);
+        };
+        const onPageError = (err) => {
+            pushLimited(pageErrors, {
+                message: err.message,
+                stack: err.stack || '',
+            }, maxConsoleEntries);
+        };
+        const onRequestFailed = (request) => {
+            pushLimited(requestFailures, {
+                url: request.url(),
+                method: request.method(),
+                resourceType: request.resourceType(),
+                errorText: request.failure()?.errorText || '',
+            }, maxNetworkEntries);
+        };
+        const onResponse = (response) => {
+            const status = response.status();
+            if (status < 400)
+                return;
+            const request = response.request();
+            pushLimited(responseErrors, {
+                url: response.url(),
+                status: String(status),
+                statusText: response.statusText(),
+                method: request.method(),
+                resourceType: request.resourceType(),
+            }, maxNetworkEntries);
+        };
+        if (captureConsole) {
+            page.on('console', onConsole);
+            page.on('pageerror', onPageError);
+        }
+        if (captureNetwork) {
+            page.on('requestfailed', onRequestFailed);
+            page.on('response', onResponse);
+        }
+        const context = page.context?.();
+        let traceStarted = false;
+        if (tracePolicy !== 'off' && context?.tracing?.start) {
+            try {
+                await context.tracing.start({ screenshots: true, snapshots: true });
+                traceStarted = true;
+            }
+            catch {
+                traceStarted = false;
+            }
+        }
+        return {
+            finalize: async (outcome) => {
+                if (captureConsole) {
+                    if (typeof page.off === 'function') {
+                        page.off('console', onConsole);
+                        page.off('pageerror', onPageError);
+                    }
+                }
+                if (captureNetwork) {
+                    if (typeof page.off === 'function') {
+                        page.off('requestfailed', onRequestFailed);
+                        page.off('response', onResponse);
+                    }
+                }
+                let traceCaptured = false;
+                let traceArtifactUri = '';
+                let traceError = '';
+                if (traceStarted && context?.tracing?.stop) {
+                    const shouldPersistTrace = tracePolicy === 'always' || (tracePolicy === 'on-failure' && outcome?.passed === false) || outcome === undefined;
+                    try {
+                        if (shouldPersistTrace) {
+                            const tracePath = join(tmpdir(), `abd-trace-${Date.now()}-${sanitizeArtifactId(testCase.id)}.zip`);
+                            await context.tracing.stop({ path: tracePath });
+                            if (this.artifactSink && existsSync(tracePath)) {
+                                const data = readFileSync(tracePath);
+                                if (data.length > 0) {
+                                    traceCaptured = true;
+                                    traceArtifactUri = await this.artifactSink.put({
+                                        type: 'trace',
+                                        testId: testCase.id,
+                                        name: 'trace.zip',
+                                        data,
+                                        contentType: 'application/zip',
+                                        metadata: {
+                                            policy: tracePolicy,
+                                            passed: String(outcome?.passed ?? false),
+                                        },
+                                    });
+                                    this.onProgress?.({ type: 'test:artifact', testId: testCase.id, artifactType: 'trace', uri: traceArtifactUri });
+                                }
+                                unlinkSync(tracePath);
+                            }
+                        }
+                        else {
+                            await context.tracing.stop();
+                        }
+                    }
+                    catch (err) {
+                        traceError = err instanceof Error ? err.message : String(err);
+                    }
+                }
+                if (!this.artifactSink)
+                    return;
+                const driverDiagnostics = driver?.getDiagnostics?.();
+                const runtimeLog = {
+                    startedAt: observabilityStartedAt,
+                    finishedAt: new Date().toISOString(),
+                    tracePolicy,
+                    traceCaptured,
+                    traceArtifactUri,
+                    traceError,
+                    outcome: outcome
+                        ? {
+                            passed: outcome.passed,
+                            verdict: outcome.verdict,
+                            durationMs: outcome.durationMs,
+                            turnsUsed: outcome.turnsUsed,
+                        }
+                        : null,
+                    console: consoleEntries,
+                    pageErrors,
+                    requestFailures,
+                    responseErrors,
+                    driverDiagnostics: driverDiagnostics ?? null,
+                };
+                const uri = await this.artifactSink.put({
+                    type: 'runtime-log',
+                    testId: testCase.id,
+                    name: 'runtime-log.json',
+                    data: Buffer.from(JSON.stringify(runtimeLog, null, 2), 'utf-8'),
+                    contentType: 'application/json',
+                    metadata: {
+                        consoleEntries: String(consoleEntries.length),
+                        pageErrors: String(pageErrors.length),
+                        requestFailures: String(requestFailures.length),
+                        responseErrors: String(responseErrors.length),
+                        traceCaptured: String(traceCaptured),
+                    },
+                });
+                this.onProgress?.({ type: 'test:artifact', testId: testCase.id, artifactType: 'runtime-log', uri });
+            },
+        };
+    }
+}
+function attachStartupDiagnostics(agentResult, timeToFirstTurnMs) {
+    if (agentResult.startupDiagnostics)
+        return agentResult;
+    if (typeof timeToFirstTurnMs === 'number') {
+        return {
+            ...agentResult,
+            startupDiagnostics: {
+                firstTurnSeen: true,
+                timeToFirstTurnMs,
+            },
+        };
+    }
+    if ((agentResult.turns?.length ?? 0) > 0) {
+        return {
+            ...agentResult,
+            startupDiagnostics: {
+                firstTurnSeen: true,
+            },
+        };
+    }
+    return {
+        ...agentResult,
+        startupDiagnostics: {
+            firstTurnSeen: false,
+            zeroTurnFailureClass: classifyZeroTurnFailure(agentResult.reason),
+            startupReason: agentResult.reason,
+        },
+    };
+}
+function classifyZeroTurnFailure(reasonRaw) {
+    const reason = String(reasonRaw || '').toLowerCase();
+    if (!reason)
+        return 'unknown';
+    if (/pre-first-turn timeout|startup timeout/.test(reason))
+        return 'pre_first_turn_timeout';
+    if (/incorrect api key|authorization failed|rate limit|llm|api key/.test(reason))
+        return 'provider_or_credentials';
+    return 'runner_startup_error';
+}
+function sanitizeArtifactId(value) {
+    return String(value || 'test').replace(/[^a-zA-Z0-9_-]+/g, '-').slice(0, 64) || 'test';
+}
+function avg(nums) {
+    if (nums.length === 0)
+        return 0;
+    return nums.reduce((sum, n) => sum + n, 0) / nums.length;
+}
+function percentile(sorted, p) {
+    if (sorted.length === 0)
+        return 0;
+    const idx = Math.ceil(sorted.length * p) - 1;
+    return sorted[Math.max(0, Math.min(idx, sorted.length - 1))];
+}
+//# sourceMappingURL=test-runner.js.map