agent-eval-opencode 0.10.0

package/dist/lib/results.js

@@ -0,0 +1,361 @@
+/**
+ * Results storage and reporting for eval experiments.
+ */
+import { mkdirSync, writeFileSync, readdirSync, readFileSync, existsSync, statSync, unlinkSync, } from 'fs';
+import { join, dirname } from 'path';
+import chalk from 'chalk';
+import { parseTranscript } from './o11y/index.js';
+import { isNonModelFailure } from './classifier.js';
+import { readFixtureFiles } from './fixture.js';
+/**
+ * Convert AgentRunResult to EvalRunData (result + transcript).
+ */
+export function agentResultToEvalRunData(agentResult) {
+    // Collect output content from scripts and tests
+    const outputContent = {};
+    // Add EVAL.ts test output
+    if (agentResult.testResult?.output) {
+        outputContent.eval = agentResult.testResult.output;
+    }
+    // Add all script outputs (nested under 'scripts' to avoid collision)
+    if (agentResult.scriptsResults &&
+        Object.keys(agentResult.scriptsResults).length > 0) {
+        outputContent.scripts = {};
+        for (const [name, result] of Object.entries(agentResult.scriptsResults)) {
+            if (result.output) {
+                outputContent.scripts[name] = result.output;
+            }
+        }
+    }
+    return {
+        result: {
+            status: agentResult.success ? 'passed' : 'failed',
+            error: agentResult.error,
+            duration: agentResult.duration / 1000, // Convert to seconds
+        },
+        transcript: agentResult.transcript,
+        outputContent: Object.keys(outputContent).length > 0 ? outputContent : undefined,
+        generatedFiles: agentResult.generatedFiles,
+        deletedFiles: agentResult.deletedFiles,
+    };
+}
+/**
+ * Create a summary from multiple run data.
+ */
+export function createEvalSummary(name, runData) {
+    const runs = runData.map((r) => r.result);
+    const passedRuns = runs.filter((r) => r.status === 'passed').length;
+    const totalDuration = runs.reduce((sum, r) => sum + r.duration, 0);
+    return {
+        name,
+        totalRuns: runs.length,
+        passedRuns,
+        passRate: runs.length > 0 ? (passedRuns / runs.length) * 100 : 0,
+        meanDuration: runs.length > 0 ? totalDuration / runs.length : 0,
+        runs: runData,
+    };
+}
+/**
+ * Create experiment results from eval summaries.
+ */
+export function createExperimentResults(config, evals, startedAt, completedAt) {
+    return {
+        startedAt: startedAt.toISOString(),
+        completedAt: completedAt.toISOString(),
+        config,
+        evals,
+    };
+}
+/**
+ * Save experiment results to disk.
+ *
+ * Creates a directory structure per design:
+ * results/
+ *   experiment-name/
+ *     2024-01-26T12-00-00Z/
+ *       eval-1/
+ *         run-1/
+ *           result.json
+ *           transcript.json      (parsed/structured - primary format)
+ *           transcript-raw.jsonl (raw agent output - for debugging)
+ *           outputs/
+ *         summary.json
+ */
+export function saveResults(results, options) {
+    const timestamp = results.startedAt.replace(/:/g, '-');
+    const experimentDir = join(options.resultsDir, options.experimentName, timestamp);
+    // Create experiment directory
+    mkdirSync(experimentDir, { recursive: true });
+    // Save per-eval results
+    for (const evalSummary of results.evals) {
+        const evalDir = join(experimentDir, evalSummary.name);
+        mkdirSync(evalDir, { recursive: true });
+        // Save summary (simplified format per design)
+        const fingerprint = options.fingerprints?.[evalSummary.name];
+        const valid = options.validity?.[evalSummary.name];
+        const summaryForFile = {
+            totalRuns: evalSummary.totalRuns,
+            passedRuns: evalSummary.passedRuns,
+            passRate: `${evalSummary.passRate.toFixed(0)}%`,
+            meanDuration: evalSummary.meanDuration,
+        };
+        if (fingerprint) {
+            summaryForFile.fingerprint = fingerprint;
+        }
+        if (valid === false) {
+            summaryForFile.valid = false;
+        }
+        if (options.smoke) {
+            summaryForFile.smoke = true;
+        }
+        writeFileSync(join(evalDir, 'summary.json'), JSON.stringify(summaryForFile, null, 2));
+        // Save individual run results
+        for (let i = 0; i < evalSummary.runs.length; i++) {
+            const runData = evalSummary.runs[i];
+            const runDir = join(evalDir, `run-${i + 1}`);
+            mkdirSync(runDir, { recursive: true });
+            // Build the result with paths and o11y summary
+            const model = results.config.model;
+            const resultWithPaths = {
+                ...runData.result,
+                model,
+            };
+            // Save transcripts if available
+            if (runData.transcript) {
+                // Parse the raw transcript
+                const transcript = parseTranscript(runData.transcript, results.config.agent, model);
+                // Save parsed transcript as primary format (transcript.json)
+                writeFileSync(join(runDir, 'transcript.json'), JSON.stringify(transcript, null, 2));
+                resultWithPaths.transcriptPath = './transcript.json';
+                // Save raw transcript for debugging (transcript-raw.jsonl)
+                writeFileSync(join(runDir, 'transcript-raw.jsonl'), runData.transcript);
+                resultWithPaths.transcriptRawPath = './transcript-raw.jsonl';
+                // Include summary in result.json for quick access
+                resultWithPaths.o11y = transcript.summary;
+            }
+            // Save script/test outputs to outputs/
+            const outputsDir = join(runDir, 'outputs');
+            mkdirSync(outputsDir, { recursive: true });
+            if (runData.outputContent) {
+                const outputPaths = {};
+                // Save EVAL.ts test output
+                if (runData.outputContent.eval) {
+                    writeFileSync(join(outputsDir, 'eval.txt'), runData.outputContent.eval);
+                    outputPaths.eval = './outputs/eval.txt';
+                }
+                // Save npm script outputs under outputs/scripts/ to avoid collision with eval.txt
+                if (runData.outputContent.scripts) {
+                    const scriptsDir = join(outputsDir, 'scripts');
+                    mkdirSync(scriptsDir, { recursive: true });
+                    outputPaths.scripts = {};
+                    for (const [name, content] of Object.entries(runData.outputContent.scripts)) {
+                        if (content) {
+                            const fileName = `${name}.txt`;
+                            writeFileSync(join(scriptsDir, fileName), content);
+                            outputPaths.scripts[name] = `./outputs/scripts/${fileName}`;
+                        }
+                    }
+                }
+                if (outputPaths.eval ||
+                    (outputPaths.scripts &&
+                        Object.keys(outputPaths.scripts).length > 0)) {
+                    resultWithPaths.outputPaths = outputPaths;
+                }
+            }
+            // Copy project files based on copyFiles config
+            const copyFiles = results.config.copyFiles;
+            const hasGeneratedFiles = runData.generatedFiles &&
+                Object.keys(runData.generatedFiles).length > 0;
+            const hasDeletedFiles = runData.deletedFiles && runData.deletedFiles.length > 0;
+            if (copyFiles &&
+                copyFiles !== 'none' &&
+                (hasGeneratedFiles || hasDeletedFiles)) {
+                const projectDir = join(runDir, 'project');
+                mkdirSync(projectDir, { recursive: true });
+                // For 'all' mode, first copy original fixture files
+                if (copyFiles === 'all') {
+                    const fixturePath = options.fixturePaths?.[evalSummary.name];
+                    if (fixturePath) {
+                        const fixtureFiles = readFixtureFiles(fixturePath);
+                        for (const [filePath, content] of fixtureFiles) {
+                            const fullPath = join(projectDir, filePath);
+                            mkdirSync(dirname(fullPath), { recursive: true });
+                            writeFileSync(fullPath, content);
+                        }
+                    }
+                    // Remove deleted files from the copied project
+                    if (runData.deletedFiles) {
+                        for (const filePath of runData.deletedFiles) {
+                            const fullPath = join(projectDir, filePath);
+                            if (existsSync(fullPath)) {
+                                unlinkSync(fullPath);
+                            }
+                        }
+                    }
+                }
+                // Write generated files (overwrites originals in 'all' mode where paths match)
+                if (runData.generatedFiles) {
+                    for (const [filePath, content] of Object.entries(runData.generatedFiles)) {
+                        const fullPath = join(projectDir, filePath);
+                        mkdirSync(dirname(fullPath), { recursive: true });
+                        writeFileSync(fullPath, content);
+                    }
+                }
+            }
+            // Save result.json with paths and o11y summary
+            writeFileSync(join(runDir, 'result.json'), JSON.stringify(resultWithPaths, null, 2));
+        }
+    }
+    return experimentDir;
+}
+/**
+ * Format results for terminal display.
+ */
+export function formatResultsTable(results) {
+    const lines = [];
+    const separator = '─'.repeat(60);
+    lines.push('');
+    lines.push(chalk.bold('Experiment Results'));
+    lines.push(chalk.gray(separator));
+    lines.push('');
+    // Calculate overall stats
+    const totalRuns = results.evals.reduce((sum, e) => sum + e.totalRuns, 0);
+    const totalPassed = results.evals.reduce((sum, e) => sum + e.passedRuns, 0);
+    const overallPassRate = totalRuns > 0 ? (totalPassed / totalRuns) * 100 : 0;
+    for (const evalSummary of results.evals) {
+        const passIcon = evalSummary.passedRuns === evalSummary.totalRuns ? '✓' : '✗';
+        const passColor = evalSummary.passedRuns === evalSummary.totalRuns
+            ? chalk.green
+            : chalk.red;
+        lines.push(passColor(`${passIcon} ${evalSummary.name}: ${evalSummary.passedRuns}/${evalSummary.totalRuns} passed (${evalSummary.passRate.toFixed(0)}%)`));
+        lines.push(chalk.gray(`  Mean duration: ${evalSummary.meanDuration.toFixed(1)}s`));
+        lines.push('');
+    }
+    lines.push(chalk.gray(separator));
+    lines.push('');
+    const overallColor = overallPassRate === 100
+        ? chalk.green
+        : overallPassRate >= 50
+            ? chalk.yellow
+            : chalk.red;
+    lines.push(overallColor(`Overall: ${totalPassed}/${totalRuns} passed (${overallPassRate.toFixed(0)}%)`));
+    const duration = (new Date(results.completedAt).getTime() -
+        new Date(results.startedAt).getTime()) /
+        1000;
+    lines.push(chalk.gray(`Total time: ${duration.toFixed(1)}s`));
+    lines.push('');
+    return lines.join('\n');
+}
+/**
+ * Format a single eval result for terminal display (used during progress).
+ */
+export function formatRunResult(evalName, runNumber, totalRuns, result, context) {
+    const icon = result.status === 'passed' ? '✓' : '✗';
+    const color = result.status === 'passed' ? chalk.green : chalk.red;
+    const prefix = context?.experimentName
+        ? `${context.experimentName}/${evalName}`
+        : evalName;
+    let line = color(`${icon} ${prefix} [${runNumber}/${totalRuns}]`);
+    if (context?.model || context?.agent) {
+        line += chalk.gray(` (${[context.agent, context.model].filter(Boolean).join(' · ')})`);
+    }
+    line += chalk.gray(` (${result.duration.toFixed(1)}s)`);
+    if (result.error) {
+        line += chalk.red(` - ${result.error.slice(0, 200)}${result.error.length > 200 ? '...' : ''}`);
+    }
+    return line;
+}
+/**
+ * Create a progress indicator for running evals.
+ */
+export function createProgressDisplay(evalName, runNumber, totalRuns, context) {
+    const prefix = context?.experimentName
+        ? `${context.experimentName}/${evalName}`
+        : evalName;
+    const meta = [context?.agent, context?.model].filter(Boolean).join(' · ');
+    const suffix = meta ? ` [${meta}]` : '';
+    return chalk.blue(`Running ${prefix} [${runNumber}/${totalRuns}]${suffix}...`);
+}
+/**
+ * Scan existing results for an experiment to find reusable eval results.
+ *
+ * A result is reusable if:
+ * 1. Its fingerprint matches the current fingerprint
+ * 2. It is "valid" (not marked as invalid by the classifier)
+ * 3. It has passedRuns > 0 (successful result worth reusing)
+ *
+ * Scans all timestamps newest-first and returns the latest match per eval.
+ */
+export function scanReusableResults(resultsDir, experimentName, fingerprints) {
+    const reusable = new Map();
+    const experimentDir = join(resultsDir, experimentName);
+    if (!existsSync(experimentDir))
+        return reusable;
+    // Get all timestamps, sorted newest first
+    let timestamps;
+    try {
+        timestamps = readdirSync(experimentDir)
+            .filter((t) => !t.startsWith('.'))
+            .sort()
+            .reverse();
+    }
+    catch {
+        return reusable;
+    }
+    for (const timestamp of timestamps) {
+        const tsDir = join(experimentDir, timestamp);
+        if (!statSync(tsDir).isDirectory())
+            continue;
+        let evalDirs;
+        try {
+            evalDirs = readdirSync(tsDir).filter((d) => !d.startsWith('.'));
+        }
+        catch {
+            continue;
+        }
+        for (const evalDir of evalDirs) {
+            // Already found a reusable result for this eval
+            if (reusable.has(evalDir))
+                continue;
+            // Check if we have a fingerprint for this eval
+            const expectedFingerprint = fingerprints[evalDir];
+            if (!expectedFingerprint)
+                continue;
+            const summaryPath = join(tsDir, evalDir, 'summary.json');
+            try {
+                const summary = JSON.parse(readFileSync(summaryPath, 'utf-8'));
+                // Check fingerprint match
+                if (summary.fingerprint !== expectedFingerprint)
+                    continue;
+                // Check validity (valid defaults to true if not explicitly set to false)
+                if (summary.valid === false)
+                    continue;
+                // Skip smoke test results
+                if (summary.smoke === true)
+                    continue;
+                // Skip non-model failures (infra/timeout) — they should be re-run
+                if (isNonModelFailure(join(tsDir, evalDir)))
+                    continue;
+                // Check that it has completed runs (use --force to re-run failures)
+                if (summary.totalRuns <= 0)
+                    continue;
+                // Unclassified failures (0% with no classification.json) are not reusable —
+                // they were never properly processed (e.g. interrupted run) and need re-running.
+                if (summary.passedRuns === 0 &&
+                    !existsSync(join(tsDir, evalDir, 'classification.json')))
+                    continue;
+                reusable.set(evalDir, {
+                    evalName: evalDir,
+                    fingerprint: summary.fingerprint,
+                    passRate: summary.passRate,
+                    timestamp,
+                });
+            }
+            catch {
+                // Skip invalid summaries
+            }
+        }
+    }
+    return reusable;
+}
+//# sourceMappingURL=results.js.map

package/dist/lib/results.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"results.js","sourceRoot":"","sources":["../../src/lib/results.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EACN,SAAS,EACT,aAAa,EACb,WAAW,EACX,YAAY,EACZ,UAAU,EACV,QAAQ,EACR,UAAU,GACV,MAAM,IAAI,CAAC;AACZ,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,KAAK,MAAM,OAAO,CAAC;AAS1B,OAAO,EAAE,eAAe,EAAmB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAEhD;;GAEG;AACH,MAAM,UAAU,wBAAwB,CACvC,WAA2B;IAE3B,gDAAgD;IAChD,MAAM,aAAa,GAAiC,EAAE,CAAC;IAEvD,0BAA0B;IAC1B,IAAI,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;QACpC,aAAa,CAAC,IAAI,GAAG,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC;IACpD,CAAC;IAED,qEAAqE;IACrE,IACC,WAAW,CAAC,cAAc;QAC1B,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC,EACjD,CAAC;QACF,aAAa,CAAC,OAAO,GAAG,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAC1C,WAAW,CAAC,cAAc,CAC1B,EAAE,CAAC;YACH,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;gBACnB,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;YAC7C,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO;QACN,MAAM,EAAE;YACP,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YACjD,KAAK,EAAE,WAAW,CAAC,KAAK;YACxB,QAAQ,EAAE,WAAW,CAAC,QAAQ,GAAG,IAAI,EAAE,qBAAqB;SAC5D;QACD,UAAU,EAAE,WAAW,CAAC,UAAU;QAClC,aAAa,EACZ,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;QAClE,cAAc,EAAE,WAAW,CAAC,cAAc;QAC1C,YAAY,EAAE,WAAW,CAAC,YAAY;KACtC,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAChC,IAAY,EACZ,OAAsB;IAEtB,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;IACpE,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IAEnE,OAAO;QACN,IAAI;QACJ,SAAS,EAAE,IAAI,CAAC,MAAM;QACtB,UAAU;QACV,QAAQ,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QAChE,YAAY,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC/D,IAAI,EAAE,OAAO;KACb,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CACtC,MAAgC,EAChC,KAAoB,EACpB,SAAe,EACf,WAAiB;IAEjB,OAAO;QACN,SAAS,EAAE,SAAS,CAAC,WAAW,EAAE;QAClC,WAAW,EAAE,WAAW,CAAC,WAAW,EAAE;QACtC,MAAM;QACN,KAAK;KACL,CAAC;AACH,CAAC;AAoBD;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,WAAW,CAC1B,OAA0B,EAC1B,OAA2B;IAE3B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACvD,MAAM,aAAa,GAAG,IAAI,CACzB,OAAO,CAAC,UAAU,EAClB,OAAO,CAAC,cAAc,EACtB,SAAS,CACT,CAAC;IAEF,8BAA8B;IAC9B,SAAS,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9C,wBAAwB;IACxB,KAAK,MAAM,WAAW,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC;QACtD,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAExC,8CAA8C;QAC9C,MAAM,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACnD,MAAM,cAAc,GAA4B;YAC/C,SAAS,EAAE,WAAW,CAAC,SAAS;YAChC,UAAU,EAAE,WAAW,CAAC,UAAU;YAClC,QAAQ,EAAE,GAAG,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;YAC/C,YAAY,EAAE,WAAW,CAAC,YAAY;SACtC,CAAC;QACF,IAAI,WAAW,EAAE,CAAC;YACjB,cAAc,CAAC,WAAW,GAAG,WAAW,CAAC;QAC1C,CAAC;QACD,IAAI,KAAK,KAAK,KAAK,EAAE,CAAC;YACrB,cAAc,CAAC,KAAK,GAAG,KAAK,CAAC;QAC9B,CAAC;QACD,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YACnB,cAAc,CAAC,KAAK,GAAG,IAAI,CAAC;QAC7B,CAAC;QACD,aAAa,CACZ,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,EAC7B,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CACvC,CAAC;QAEF,8BAA8B;QAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAClD,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC7C,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAEvC,+CAA+C;YAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;YACnC,MAAM,eAAe,GAEjB;gBACH,GAAG,OAAO,CAAC,MAAM;gBACjB,KAAK;aACL,CAAC;YAEF,gCAAgC;YAChC,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACxB,2BAA2B;gBAC3B,MAAM,UAAU,GAAG,eAAe,CACjC,OAAO,CAAC,UAAU,EAClB,OAAO,CAAC,MAAM,CAAC,KAAK,EACpB,KAAK,CACL,CAAC;gBAEF,6DAA6D;gBAC7D,aAAa,CACZ,IAAI,CAAC,MAAM,EAAE,iBAAiB,CAAC,EAC/B,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CACnC,CAAC;gBACF,eAAe,CAAC,cAAc,GAAG,mBAAmB,CAAC;gBAErD,2DAA2D;gBAC3D,aAAa,CACZ,IAAI,CAAC,MAAM,EAAE,sBAAsB,CAAC,EACpC,OAAO,CAAC,UAAU,CAClB,CAAC;gBACF,eAAe,CAAC,iBAAiB,GAAG,wBAAwB,CAAC;gBAE7D,kDAAkD;gBAClD,eAAe,CAAC,IAAI,GAAG,UAAU,CAAC,OAAO,CAAC;YAC3C,CAAC;YAED,uCAAuC;YACvC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAC3C,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAE3C,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;gBAC3B,MAAM,WAAW,GAAiC,EAAE,CAAC;gBAErD,2BAA2B;gBAC3B,IAAI,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;oBAChC,aAAa,CACZ,IAAI,CAAC,UAAU,EAAE,UAAU,CAAC,EAC5B,OAAO,CAAC,aAAa,CAAC,IAAI,CAC1B,CAAC;oBACF,WAAW,CAAC,IAAI,GAAG,oBAAoB,CAAC;gBACzC,CAAC;gBAED,kFAAkF;gBAClF,IAAI,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;oBACnC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;oBAC/C,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC3C,WAAW,CAAC,OAAO,GAAG,EAAE,CAAC;oBACzB,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAC3C,OAAO,CAAC,aAAa,CAAC,OAAO,CAC7B,EAAE,CAAC;wBACH,IAAI,OAAO,EAAE,CAAC;4BACb,MAAM,QAAQ,GAAG,GAAG,IAAI,MAAM,CAAC;4BAC/B,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;4BACnD,WAAW,CAAC,OAAO,CAClB,IAAI,CACJ,GAAG,qBAAqB,QAAQ,EAAE,CAAC;wBACrC,CAAC;oBACF,CAAC;gBACF,CAAC;gBAED,IACC,WAAW,CAAC,IAAI;oBAChB,CAAC,WAAW,CAAC,OAAO;wBACnB,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,EAC5C,CAAC;oBACF,eAAe,CAAC,WAAW,GAAG,WAAW,CAAC;gBAC3C,CAAC;YACF,CAAC;YAED,+CAA+C;YAC/C,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;YAC3C,MAAM,iBAAiB,GACtB,OAAO,CAAC,cAAc;gBACtB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;YAChD,MAAM,eAAe,GACpB,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;YACzD,IACC,SAAS;gBACT,SAAS,KAAK,MAAM;gBACpB,CAAC,iBAAiB,IAAI,eAAe,CAAC,EACrC,CAAC;gBACF,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;gBAC3C,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAE3C,oDAAoD;gBACpD,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;oBACzB,MAAM,WAAW,GAChB,OAAO,CAAC,YAAY,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;oBAC1C,IAAI,WAAW,EAAE,CAAC;wBACjB,MAAM,YAAY,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAC;wBACnD,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,YAAY,EAAE,CAAC;4BAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;4BAC5C,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;4BAClD,aAAa,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;wBAClC,CAAC;oBACF,CAAC;oBAED,+CAA+C;oBAC/C,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;wBAC1B,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;4BAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;4BAC5C,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gCAC1B,UAAU,CAAC,QAAQ,CAAC,CAAC;4BACtB,CAAC;wBACF,CAAC;oBACF,CAAC;gBACF,CAAC;gBAED,+EAA+E;gBAC/E,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;oBAC5B,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAC/C,OAAO,CAAC,cAAc,CACtB,EAAE,CAAC;wBACH,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;wBAC5C,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;wBAClD,aAAa,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;oBAClC,CAAC;gBACF,CAAC;YACF,CAAC;YAED,+CAA+C;YAC/C,aAAa,CACZ,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAC3B,IAAI,CAAC,SAAS,CAAC,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC,CACxC,CAAC;QACH,CAAC;IACF,CAAC;IAED,OAAO,aAAa,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAA0B;IAC5D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAEjC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;IAC7C,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAClC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,0BAA0B;IAC1B,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;IACzE,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAC5E,MAAM,eAAe,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAE5E,KAAK,MAAM,WAAW,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACzC,MAAM,QAAQ,GACb,WAAW,CAAC,UAAU,KAAK,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAC9D,MAAM,SAAS,GACd,WAAW,CAAC,UAAU,KAAK,WAAW,CAAC,SAAS;YAC/C,CAAC,CAAC,KAAK,CAAC,KAAK;YACb,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;QAEd,KAAK,CAAC,IAAI,CACT,SAAS,CACR,GAAG,QAAQ,IAAI,WAAW,CAAC,IAAI,KAAK,WAAW,CAAC,UAAU,IACzD,WAAW,CAAC,SACb,YAAY,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAC/C,CACD,CAAC;QACF,KAAK,CAAC,IAAI,CACT,KAAK,CAAC,IAAI,CACT,oBAAoB,WAAW,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAC1D,CACD,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAClC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,MAAM,YAAY,GACjB,eAAe,KAAK,GAAG;QACtB,CAAC,CAAC,KAAK,CAAC,KAAK;QACb,CAAC,CAAC,eAAe,IAAI,EAAE;YACvB,CAAC,CAAC,KAAK,CAAC,MAAM;YACd,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;IACd,KAAK,CAAC,IAAI,CACT,YAAY,CACX,YAAY,WAAW,IAAI,SAAS,YAAY,eAAe,CAAC,OAAO,CACtE,CAAC,CACD,IAAI,CACL,CACD,CAAC;IAEF,MAAM,QAAQ,GACb,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE;QACvC,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;QACvC,IAAI,CAAC;IACN,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACzB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAC9B,QAAgB,EAChB,SAAiB,EACjB,SAAiB,EACjB,MAAqB,EACrB,OAAqE;IAErE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IACpD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;IACnE,MAAM,MAAM,GAAG,OAAO,EAAE,cAAc;QACrC,CAAC,CAAC,GAAG,OAAO,CAAC,cAAc,IAAI,QAAQ,EAAE;QACzC,CAAC,CAAC,QAAQ,CAAC;IAEZ,IAAI,IAAI,GAAG,KAAK,CAAC,GAAG,IAAI,IAAI,MAAM,KAAK,SAAS,IAAI,SAAS,GAAG,CAAC,CAAC;IAClE,IAAI,OAAO,EAAE,KAAK,IAAI,OAAO,EAAE,KAAK,EAAE,CAAC;QACtC,IAAI,IAAI,KAAK,CAAC,IAAI,CACjB,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAClE,CAAC;IACH,CAAC;IACD,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAExD,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QAClB,IAAI,IAAI,KAAK,CAAC,GAAG,CAChB,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAC/B,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EACrC,EAAE,CACF,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CACpC,QAAgB,EAChB,SAAiB,EACjB,SAAiB,EACjB,OAAqE;IAErE,MAAM,MAAM,GAAG,OAAO,EAAE,cAAc;QACrC,CAAC,CAAC,GAAG,OAAO,CAAC,cAAc,IAAI,QAAQ,EAAE;QACzC,CAAC,CAAC,QAAQ,CAAC;IACZ,MAAM,IAAI,GAAG,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1E,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACxC,OAAO,KAAK,CAAC,IAAI,CAChB,WAAW,MAAM,KAAK,SAAS,IAAI,SAAS,IAAI,MAAM,KAAK,CAC3D,CAAC;AACH,CAAC;AAYD;;;;;;;;;GASG;AACH,MAAM,UAAU,mBAAmB,CAClC,UAAkB,EAClB,cAAsB,EACtB,YAAoC;IAEpC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAC;IACnD,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;IAEvD,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC;QAAE,OAAO,QAAQ,CAAC;IAEhD,0CAA0C;IAC1C,IAAI,UAAoB,CAAC;IACzB,IAAI,CAAC;QACJ,UAAU,GAAG,WAAW,CAAC,aAAa,CAAC;aACrC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;aACjC,IAAI,EAAE;aACN,OAAO,EAAE,CAAC;IACb,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,QAAQ,CAAC;IACjB,CAAC;IAED,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;QAC7C,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE;YAAE,SAAS;QAE7C,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACJ,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QACjE,CAAC;QAAC,MAAM,CAAC;YACR,SAAS;QACV,CAAC;QAED,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,gDAAgD;YAChD,IAAI,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC;gBAAE,SAAS;YAEpC,+CAA+C;YAC/C,MAAM,mBAAmB,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;YAClD,IAAI,CAAC,mBAAmB;gBAAE,SAAS;YAEnC,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,cAAc,CAAC,CAAC;YACzD,IAAI,CAAC;gBACJ,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;gBAE/D,0BAA0B;gBAC1B,IAAI,OAAO,CAAC,WAAW,KAAK,mBAAmB;oBAAE,SAAS;gBAE1D,yEAAyE;gBACzE,IAAI,OAAO,CAAC,KAAK,KAAK,KAAK;oBAAE,SAAS;gBAEtC,0BAA0B;gBAC1B,IAAI,OAAO,CAAC,KAAK,KAAK,IAAI;oBAAE,SAAS;gBAErC,kEAAkE;gBAClE,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;oBAAE,SAAS;gBAEtD,oEAAoE;gBACpE,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC;oBAAE,SAAS;gBAErC,4EAA4E;gBAC5E,iFAAiF;gBACjF,IACC,OAAO,CAAC,UAAU,KAAK,CAAC;oBACxB,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,qBAAqB,CAAC,CAAC;oBAExD,SAAS;gBAEV,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE;oBACrB,QAAQ,EAAE,OAAO;oBACjB,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,QAAQ,EAAE,OAAO,CAAC,QAAQ;oBAC1B,SAAS;iBACT,CAAC,CAAC;YACJ,CAAC;YAAC,MAAM,CAAC;gBACR,yBAAyB;YAC1B,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,QAAQ,CAAC;AACjB,CAAC"}

package/dist/lib/runner.d.ts

@@ -0,0 +1,71 @@
+/**
+ * Experiment runner - orchestrates running evals against agent.
+ * Concurrency is controlled via an optional ConcurrencyLimiter shared across experiments.
+ * 429 rate-limit errors are retried with exponential backoff.
+ * With earlyExit, in-flight attempts are aborted when one passes.
+ */
+import type { ResolvedExperimentConfig, EvalFixture, EvalRunData, ExperimentResults, RunnableExperimentConfig, ProgressEvent } from './types.js';
+/**
+ * Rate-limits how many operations can START within a time window.
+ * Once started, operations run freely with no concurrency limit.
+ * Create one instance and share it across experiments to control global start rate.
+ */
+export declare class StartRateLimiter {
+    private readonly startsPerWindow;
+    private readonly windowMs;
+    private queue;
+    private started;
+    private timer;
+    constructor(startsPerWindow: number, windowMs: number);
+    /**
+     * Wait for permission to start, then return immediately.
+     * The operation runs freely after this resolves.
+     */
+    waitToStart(): Promise<void>;
+    private ensureTimer;
+}
+/**
+ * Options for running an experiment.
+ */
+export interface RunExperimentOptions {
+    /** Resolved experiment configuration */
+    config: RunnableExperimentConfig;
+    /** Fixtures to run */
+    fixtures: EvalFixture[];
+    /** API key for the agent */
+    apiKey: string;
+    /** Directory to save results */
+    resultsDir: string;
+    /** Experiment name */
+    experimentName: string;
+    /** Per-eval fingerprints (eval name -> hash) for result reuse */
+    fingerprints?: Record<string, string>;
+    /** Callback for progress updates */
+    onProgress?: (event: ProgressEvent) => void;
+    /** Whether to run in verbose mode */
+    verbose?: boolean;
+    /** Whether this is a smoke test run */
+    smoke?: boolean;
+    /** Shared rate limiter to control how many sandbox runs start per time window */
+    rateLimiter?: StartRateLimiter;
+}
+/**
+ * Run an experiment - execute all evals with configured runs concurrently.
+ * With earlyExit enabled, remaining attempts for a fixture are aborted once one passes.
+ */
+export declare function runExperiment(options: RunExperimentOptions): Promise<ExperimentResults>;
+/**
+ * Run a single eval (for testing/debugging).
+ */
+export declare function runSingleEval<T extends ResolvedExperimentConfig['model']>(fixture: EvalFixture, options: {
+    agent?: ResolvedExperimentConfig['agent'];
+    model: T;
+    timeout: number;
+    apiKey: string;
+    setup?: ResolvedExperimentConfig['setup'];
+    scripts?: string[];
+    sandbox?: ResolvedExperimentConfig['sandbox'];
+    editPrompt?: (prompt: string) => string;
+    verbose?: boolean;
+}): Promise<T extends Array<unknown> ? EvalRunData[] : EvalRunData>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/lib/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/lib/runner.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EACV,wBAAwB,EACxB,WAAW,EACX,WAAW,EAEX,iBAAiB,EACjB,wBAAwB,EACxB,aAAa,EACd,MAAM,YAAY,CAAC;AASpB;;;;GAIG;AACH,qBAAa,gBAAgB;IAMzB,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAN3B,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,OAAO,CAAK;IACpB,OAAO,CAAC,KAAK,CAA+C;gBAGzC,eAAe,EAAE,MAAM,EACvB,QAAQ,EAAE,MAAM;IAGnC;;;OAGG;IACG,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAWlC,OAAO,CAAC,WAAW;CAgBpB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,wCAAwC;IACxC,MAAM,EAAE,wBAAwB,CAAC;IACjC,sBAAsB;IACtB,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,gCAAgC;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,sBAAsB;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,iEAAiE;IACjE,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,oCAAoC;IACpC,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;IAC5C,qCAAqC;IACrC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,uCAAuC;IACvC,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,iFAAiF;IACjF,WAAW,CAAC,EAAE,gBAAgB,CAAC;CAChC;AAoBD;;;GAGG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,iBAAiB,CAAC,CA2N5B;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,CAAC,SAAS,wBAAwB,CAAC,OAAO,CAAC,EAC7E,OAAO,EAAE,WAAW,EACpB,OAAO,EAAE;IACP,KAAK,CAAC,EAAE,wBAAwB,CAAC,OAAO,CAAC,CAAC;IAC1C,KAAK,EAAE,CAAC,CAAC;IACT,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,wBAAwB,CAAC,OAAO,CAAC,CAAC;IAC1C,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,wBAAwB,CAAC,SAAS,CAAC,CAAC;IAC9C,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;IACxC,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,GACA,OAAO,CAAC,CAAC,SAAS,KAAK,CAAC,OAAO,CAAC,GAAG,WAAW,EAAE,GAAG,WAAW,CAAC,CA6BjE"}