agent-nuvira 1.14.0

package/dist/learning/benchmark.js

@@ -0,0 +1,589 @@
+/**
+ * Benchmark — Standardized model benchmarking system for coding tasks.
+ *
+ * Runs a suite of coding tasks against configured providers/models and
+ * measures: success rate, output quality, latency, and cost.
+ *
+ * Usage:
+ *   buff benchmark                          — Run all tasks against default provider
+ *   buff benchmark --provider groq          — Run against a specific provider
+ *   buff benchmark --model llama-3.3-70b    — Run against a specific model
+ *   buff benchmark --tasks quick            — Run only quick tasks
+ *   buff benchmark --budget 0.50            — Stop if costs exceed $0.50
+ *   buff benchmark list                     — List available benchmark tasks
+ *   buff benchmark results                  — Show previous benchmark results
+ *
+ * Results stored in: ~/.buff/memory/benchmarks.json
+ */
+import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
+import { join } from 'node:path';
+import { homedir } from 'node:os';
+import { estimateTokens, calculateCost } from './cost-tracker.js';
+import { logger } from '../utils/logger.js';
+// ─── Benchmark Tasks ────────────────────────────────────────────────────────
+const BENCHMARK_TASKS = [
+    // ── Easy: Code Generation ─────────────────────────────────────────────
+    {
+        id: 'hello-world',
+        title: 'Hello World Function',
+        tag: 'code-generation',
+        difficulty: 'easy',
+        prompt: 'Write a JavaScript function called `greet` that takes a name parameter and returns "Hello, {name}!". Include a default parameter value.',
+        expectedPatterns: ['function greet', 'Hello,', 'default'],
+        maxExpectedTokens: 150,
+        timeEstimate: 'quick',
+        outputLanguage: 'javascript',
+    },
+    {
+        id: 'fibonacci',
+        title: 'Fibonacci Sequence',
+        tag: 'code-generation',
+        difficulty: 'easy',
+        prompt: 'Write a Python function `fibonacci(n)` that returns the nth Fibonacci number using recursion. Include a comment explaining the base case.',
+        expectedPatterns: ['def fibonacci', 'return', '#'],
+        maxExpectedTokens: 200,
+        timeEstimate: 'quick',
+        outputLanguage: 'python',
+    },
+    {
+        id: 'fizzbuzz',
+        title: 'FizzBuzz Implementation',
+        tag: 'code-generation',
+        difficulty: 'easy',
+        prompt: 'Write a TypeScript function `fizzBuzz(n)` that prints numbers 1 to n, replacing multiples of 3 with "Fizz", multiples of 5 with "Buzz", and multiples of both with "FizzBuzz".',
+        expectedPatterns: ['Fizz', 'Buzz', 'FizzBuzz', 'function fizzBuzz'],
+        maxExpectedTokens: 300,
+        timeEstimate: 'quick',
+        outputLanguage: 'typescript',
+    },
+    // ── Medium: Code Generation ───────────────────────────────────────────
+    {
+        id: 'express-api',
+        title: 'Express.js API Route',
+        tag: 'code-generation',
+        difficulty: 'medium',
+        prompt: 'Create an Express.js route handler for a REST API endpoint that handles POST requests to /api/users. It should validate that the request body has "name" and "email" fields, and return a 201 status with the user object. Include error handling for missing fields (400 status).',
+        expectedPatterns: ['app.post', '/api/users', 'res.status', 'name', 'email'],
+        maxExpectedTokens: 500,
+        timeEstimate: 'medium',
+        outputLanguage: 'javascript',
+    },
+    {
+        id: 'typescript-generics',
+        title: 'TypeScript Generics',
+        tag: 'code-generation',
+        difficulty: 'medium',
+        prompt: 'Write a TypeScript generic function `mergeObjects<T, U>(obj1: T, obj2: U): T & U` that merges two objects. Then write a type-safe function `getProperty<T, K extends keyof T>(obj: T, key: K): T[K]`.',
+        expectedPatterns: ['<T,', 'extends', 'keyof', 'T & U'],
+        maxExpectedTokens: 400,
+        timeEstimate: 'medium',
+        outputLanguage: 'typescript',
+    },
+    {
+        id: 'react-component',
+        title: 'React Component',
+        tag: 'code-generation',
+        difficulty: 'medium',
+        prompt: 'Write a React TypeScript component called `UserList` that fetches users from `https://jsonplaceholder.typicode.com/users` using useEffect and useState. Display loading, error, and success states. Include proper TypeScript interfaces.',
+        expectedPatterns: ['useState', 'useEffect', 'interface', 'fetch', 'UserList'],
+        maxExpectedTokens: 600,
+        timeEstimate: 'medium',
+        outputLanguage: 'typescript',
+    },
+    // ── Refactoring ───────────────────────────────────────────────────────
+    {
+        id: 'refactor-callback',
+        title: 'Callback to Async/Await',
+        tag: 'refactoring',
+        difficulty: 'medium',
+        prompt: 'Refactor this callback-based function to use async/await:\n\nfunction getUserData(userId, callback) {\n  getUser(userId, (err, user) => {\n    if (err) return callback(err);\n    getPosts(user.id, (err, posts) => {\n      if (err) return callback(err);\n      getComments(posts[0].id, (err, comments) => {\n        if (err) return callback(err);\n        callback(null, { user, posts, comments });\n      });\n    });\n  });\n}',
+        expectedPatterns: ['async', 'await', 'try', 'catch'],
+        antiPatterns: ['callback'],
+        maxExpectedTokens: 400,
+        timeEstimate: 'medium',
+        outputLanguage: 'javascript',
+    },
+    {
+        id: 'refactor-class-to-func',
+        title: 'Class to Functional Component',
+        tag: 'refactoring',
+        difficulty: 'medium',
+        prompt: 'Refactor this React class component to a functional component with hooks:\n\nclass Counter extends React.Component {\n  constructor(props) {\n    super(props);\n    this.state = { count: 0 };\n  }\n  componentDidMount() { document.title = `Count: ${this.state.count}`; }\n  componentDidUpdate() { document.title = `Count: ${this.state.count}`; }\n  render() {\n    return <div><p>Count: {this.state.count}</p><button onClick={() => this.setState({ count: this.state.count + 1 })}>+</button></div>;\n  }\n}',
+        expectedPatterns: ['useState', 'useEffect', '=>'],
+        antiPatterns: ['class Counter', 'this.state', 'extends React'],
+        maxExpectedTokens: 400,
+        timeEstimate: 'medium',
+        outputLanguage: 'typescript',
+    },
+    // ── Debugging ─────────────────────────────────────────────────────────
+    {
+        id: 'debug-closure',
+        title: 'Fix Closure Bug',
+        tag: 'debugging',
+        difficulty: 'medium',
+        prompt: 'Find and fix the bug in this code:\n\nfor (var i = 0; i < 5; i++) {\n  setTimeout(function() {\n    console.log(i);\n  }, 100);\n}\n\nExpected output: 0, 1, 2, 3, 4\nActual output: 5, 5, 5, 5, 5\n\nExplain the bug and provide the fix.',
+        expectedPatterns: ['let', 'closure', 'var', 'block scope'],
+        maxExpectedTokens: 300,
+        timeEstimate: 'quick',
+        outputLanguage: 'javascript',
+    },
+    {
+        id: 'debug-null',
+        title: 'Fix Null Reference',
+        tag: 'debugging',
+        difficulty: 'easy',
+        prompt: 'Find and fix the bug: \n\nconst users = [\n  { name: "Alice", address: { city: "NYC" } },\n  { name: "Bob" },\n  { name: "Charlie", address: { city: "LA" } }\n];\n\nusers.forEach(user => console.log(user.address.city.toUpperCase()));\n\nExplain the error and provide a fix using optional chaining.',
+        expectedPatterns: ['?.', 'optional chaining', 'undefined'],
+        maxExpectedTokens: 250,
+        timeEstimate: 'quick',
+        outputLanguage: 'javascript',
+    },
+    // ── Testing ───────────────────────────────────────────────────────────
+    {
+        id: 'unit-test',
+        title: 'Write Unit Tests',
+        tag: 'testing',
+        difficulty: 'medium',
+        prompt: 'Write Vitest unit tests for this function:\n\nexport function calculateDiscount(price: number, isMember: boolean): number {\n  if (price < 0) throw new Error("Price cannot be negative");\n  if (isMember) return price * 0.9;\n  if (price > 100) return price * 0.95;\n  return price;\n}\n\nInclude tests for: normal cases, member discount, high-value discount, negative price error, and edge cases.',
+        expectedPatterns: ['describe', 'it', 'expect', 'calculateDiscount', 'throw'],
+        maxExpectedTokens: 500,
+        timeEstimate: 'medium',
+        outputLanguage: 'typescript',
+    },
+    // ── Documentation ─────────────────────────────────────────────────────
+    {
+        id: 'jsdoc',
+        title: 'Generate Documentation',
+        tag: 'documentation',
+        difficulty: 'easy',
+        prompt: 'Add JSDoc comments to this function:\n\nfunction parseConfig(configStr, options) {\n  const config = JSON.parse(configStr);\n  if (options.validate) {\n    validateConfig(config);\n  }\n  return config;\n}\n\nInclude: parameter descriptions, return type, and a usage example.',
+        expectedPatterns: ['@param', '@returns', '@example', 'parseConfig'],
+        maxExpectedTokens: 250,
+        timeEstimate: 'quick',
+        outputLanguage: 'javascript',
+    },
+    // ── Security ──────────────────────────────────────────────────────────
+    {
+        id: 'security-sql',
+        title: 'Fix SQL Injection',
+        tag: 'security',
+        difficulty: 'hard',
+        prompt: 'Identify and fix the security vulnerability in this code:\n\napp.get("/users", (req, res) => {\n  const name = req.query.name;\n  const query = `SELECT * FROM users WHERE name = "${name}"`;\n  db.query(query, (err, results) => {\n    res.json(results);\n  });\n});\n\nExplain the vulnerability and provide a secure implementation using parameterized queries.',
+        expectedPatterns: ['parameterized', 'prepared statement', 'SQL injection', '$1', '?'],
+        antiPatterns: ['string interpolation', 'template literal'],
+        maxExpectedTokens: 400,
+        timeEstimate: 'medium',
+        outputLanguage: 'javascript',
+    },
+    {
+        id: 'security-xss',
+        title: 'Fix XSS Vulnerability',
+        tag: 'security',
+        difficulty: 'medium',
+        prompt: 'Identify and fix the XSS vulnerability:\n\napp.get("/profile", (req, res) => {\n  const username = req.query.username;\n  res.send(`<h1>Welcome, ${username}!</h1>`);\n});\n\nExplain the vulnerability and show two fix approaches: (1) escaping output, (2) using a template engine with auto-escaping.',
+        expectedPatterns: ['escape', 'XSS', 'sanitize', 'textContent'],
+        antiPatterns: ['res.send', 'innerHTML'],
+        maxExpectedTokens: 400,
+        timeEstimate: 'medium',
+        outputLanguage: 'javascript',
+    },
+    // ── Optimization ──────────────────────────────────────────────────────
+    {
+        id: 'optimize-array',
+        title: 'Optimize Array Operations',
+        tag: 'optimization',
+        difficulty: 'medium',
+        prompt: 'Optimize this code for performance:\n\nfunction findDuplicates(arr) {\n  const duplicates = [];\n  for (let i = 0; i < arr.length; i++) {\n    for (let j = 0; j < arr.length; j++) {\n      if (i !== j && arr[i] === arr[j]) {\n        if (!duplicates.includes(arr[i])) {\n          duplicates.push(arr[i]);\n        }\n      }\n    }\n  }\n  return duplicates;\n}\n\nExplain why the original is slow (O(n²)) and provide an O(n) solution.',
+        expectedPatterns: ['Set', 'O(n', 'has', 'new Set'],
+        antiPatterns: ['nested loop', 'includes'],
+        maxExpectedTokens: 400,
+        timeEstimate: 'medium',
+        outputLanguage: 'javascript',
+    },
+    // ── Explanation / Comprehension ───────────────────────────────────────
+    {
+        id: 'explain-prototype',
+        title: 'Explain Prototype Chain',
+        tag: 'comprehension',
+        difficulty: 'medium',
+        prompt: 'Explain JavaScript prototype chain in 3-4 sentences. Include what `__proto__` is, how `Function.prototype` works, and give a simple example of prototypal inheritance.',
+        expectedPatterns: ['prototype', '__proto__', 'inheritance', 'Object.create'],
+        maxExpectedTokens: 300,
+        timeEstimate: 'quick',
+        outputLanguage: 'text',
+    },
+    {
+        id: 'explain-event-loop',
+        title: 'Explain Event Loop',
+        tag: 'comprehension',
+        difficulty: 'medium',
+        prompt: 'Explain the JavaScript event loop in simple terms. Cover: call stack, task queue, microtasks (Promise), and the order of execution. Give a short code example that demonstrates the order.',
+        expectedPatterns: ['call stack', 'task queue', 'microtask', 'Promise', 'event loop'],
+        maxExpectedTokens: 400,
+        timeEstimate: 'quick',
+        outputLanguage: 'text',
+    },
+    // ── Hard: Complex Code Generation ─────────────────────────────────────
+    {
+        id: 'middleware-chain',
+        title: 'Express Middleware Chain',
+        tag: 'code-generation',
+        difficulty: 'hard',
+        prompt: 'Write an Express middleware chain for a production API: (1) request logging middleware that logs method, URL, and response time, (2) authentication middleware that checks a JWT token from the Authorization header, (3) error handling middleware that catches all errors and returns consistent JSON error responses. Include TypeScript types.',
+        expectedPatterns: ['next', 'req', 'res', 'JWT', 'Authorization', 'middleware'],
+        maxExpectedTokens: 700,
+        timeEstimate: 'slow',
+        outputLanguage: 'typescript',
+    },
+    {
+        id: 'rate-limiter',
+        title: 'Rate Limiter Implementation',
+        tag: 'code-generation',
+        difficulty: 'hard',
+        prompt: 'Implement a token bucket rate limiter in TypeScript:\n- Configurable: maxTokens, refillRate (tokens per second)\n- allowRequest(): boolean method\n- Thread-safe consideration (single-threaded JS is fine)\n- Include a usage example\n- Add JSDoc comments',
+        expectedPatterns: ['class', 'token', 'bucket', 'refill', 'allowRequest', 'RateLimiter'],
+        maxExpectedTokens: 500,
+        timeEstimate: 'medium',
+        outputLanguage: 'typescript',
+    },
+    // ── Translation ───────────────────────────────────────────────────────
+    {
+        id: 'translate-py-to-ts',
+        title: 'Python to TypeScript Translation',
+        tag: 'translation',
+        difficulty: 'medium',
+        prompt: 'Translate this Python function to TypeScript:\n\ndef process_items(items: list[dict]) -> dict[str, list]:\n    result = {"valid": [], "invalid": []}\n    for item in items:\n        if item.get("active") and item.get("value", 0) > 0:\n            result["valid"].append(item)\n        else:\n            result["invalid"].append(item)\n    return result\n\nInclude proper TypeScript interfaces and types.',
+        expectedPatterns: ['interface', 'processItems', 'valid', 'invalid', 'Array'],
+        maxExpectedTokens: 400,
+        timeEstimate: 'medium',
+        outputLanguage: 'typescript',
+    },
+];
+// ─── Constants ──────────────────────────────────────────────────────────────
+const MEMORY_DIR = join(homedir(), '.buff', 'memory');
+const BENCHMARK_PATH = join(MEMORY_DIR, 'benchmarks.json');
+const CURRENT_VERSION = 1;
+const MAX_BENCHMARK_RUNS = 50;
+// ─── Helpers ────────────────────────────────────────────────────────────────
+function ensureDir() {
+    if (!existsSync(MEMORY_DIR)) {
+        mkdirSync(MEMORY_DIR, { recursive: true });
+    }
+}
+function readBenchmarkData() {
+    try {
+        ensureDir();
+        if (!existsSync(BENCHMARK_PATH)) {
+            return { runs: [], version: CURRENT_VERSION };
+        }
+        const raw = readFileSync(BENCHMARK_PATH, 'utf-8');
+        return JSON.parse(raw);
+    }
+    catch {
+        return { runs: [], version: CURRENT_VERSION };
+    }
+}
+function writeBenchmarkData(data) {
+    ensureDir();
+    writeFileSync(BENCHMARK_PATH, JSON.stringify(data, null, 2), 'utf-8');
+}
+/**
+ * Generate a quality score for a model's output based on heuristics.
+ * Scores range from 0 (poor) to 1 (excellent).
+ */
+export function scoreQuality(output, task) {
+    let score = 0.5; // Start at neutral
+    // Bonus: output matches expected patterns
+    if (task.expectedPatterns) {
+        const matched = task.expectedPatterns.filter((p) => output.toLowerCase().includes(p.toLowerCase())).length;
+        score += (matched / task.expectedPatterns.length) * 0.3;
+    }
+    // Penalty: output contains anti-patterns
+    if (task.antiPatterns) {
+        const antiMatched = task.antiPatterns.filter((p) => output.toLowerCase().includes(p.toLowerCase())).length;
+        score -= (antiMatched / task.antiPatterns.length) * 0.3;
+    }
+    // Bonus: output is not too short (meaningful response)
+    if (output.length > 100)
+        score += 0.1;
+    if (output.length > 500)
+        score += 0.1;
+    // Bonus: contains code blocks when expected
+    if (task.outputLanguage && task.outputLanguage !== 'text') {
+        if (output.includes('```'))
+            score += 0.1;
+    }
+    // Clamp between 0 and 1
+    return Math.max(0, Math.min(1, score));
+}
+/**
+ * Calculate median of an array of numbers.
+ */
+function median(values) {
+    if (values.length === 0)
+        return 0;
+    const sorted = [...values].sort((a, b) => a - b);
+    const mid = Math.floor(sorted.length / 2);
+    return sorted.length % 2 !== 0
+        ? sorted[mid]
+        : (sorted[mid - 1] + sorted[mid]) / 2;
+}
+// ─── Benchmark Runner ───────────────────────────────────────────────────────
+/**
+ * Run the benchmark suite against a provider/model.
+ */
+export async function runBenchmark(provider, providerName, model, options) {
+    // Filter tasks
+    let tasks = [...BENCHMARK_TASKS];
+    if (options?.taskIds && options.taskIds.length > 0) {
+        tasks = tasks.filter((t) => options.taskIds.includes(t.id));
+    }
+    if (options?.timeEstimate) {
+        tasks = tasks.filter((t) => t.timeEstimate === options.timeEstimate);
+    }
+    const runId = `benchmark-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`;
+    const startedAt = Date.now();
+    const results = [];
+    let totalCost = 0;
+    for (let i = 0; i < tasks.length; i++) {
+        const task = tasks[i];
+        // Check budget
+        if (options?.budget && totalCost >= options.budget) {
+            logger.info(`Budget of $${options.budget.toFixed(2)} reached. Stopping benchmark.`);
+            break;
+        }
+        options?.onProgress?.(i + 1, tasks.length, task);
+        const taskStartTime = Date.now();
+        try {
+            const output = await provider.generate(task.prompt, {
+                model,
+                temperature: 0.3, // Low temperature for deterministic results
+                maxTokens: task.maxExpectedTokens + 512, // Allow extra tokens
+            });
+            const latencyMs = Date.now() - taskStartTime;
+            const inputTokens = estimateTokens(task.prompt);
+            const outputTokens = estimateTokens(output);
+            const costUsd = calculateCost(providerName, model, inputTokens, outputTokens);
+            totalCost += costUsd;
+            const result = {
+                taskId: task.id,
+                provider: providerName,
+                model,
+                output,
+                success: true,
+                latencyMs,
+                inputTokens,
+                outputTokens,
+                costUsd,
+                qualityScore: scoreQuality(output, task),
+                timestamp: Date.now(),
+            };
+            results.push(result);
+        }
+        catch (err) {
+            const latencyMs = Date.now() - taskStartTime;
+            results.push({
+                taskId: task.id,
+                provider: providerName,
+                model,
+                output: '',
+                success: false,
+                latencyMs,
+                inputTokens: estimateTokens(task.prompt),
+                outputTokens: 0,
+                costUsd: 0,
+                qualityScore: 0,
+                error: err instanceof Error ? err.message : String(err),
+                timestamp: Date.now(),
+            });
+        }
+    }
+    const endedAt = Date.now();
+    // Compute summary
+    const passed = results.filter((r) => r.success);
+    const qualityScores = passed.map((r) => r.qualityScore);
+    const latencies = results.map((r) => r.latencyMs);
+    const run = {
+        id: runId,
+        provider: providerName,
+        model,
+        startedAt,
+        endedAt,
+        results,
+        summary: {
+            totalTasks: results.length,
+            tasksPassed: passed.length,
+            tasksFailed: results.length - passed.length,
+            avgQualityScore: qualityScores.length > 0
+                ? qualityScores.reduce((a, b) => a + b, 0) / qualityScores.length
+                : 0,
+            medianLatencyMs: median(latencies),
+            totalCostUsd: results.reduce((sum, r) => sum + r.costUsd, 0),
+            totalTokens: results.reduce((sum, r) => sum + r.inputTokens + r.outputTokens, 0),
+        },
+    };
+    // Persist
+    const data = readBenchmarkData();
+    data.runs.push(run);
+    // Prune old runs
+    if (data.runs.length > MAX_BENCHMARK_RUNS) {
+        data.runs = data.runs.slice(-MAX_BENCHMARK_RUNS);
+    }
+    writeBenchmarkData(data);
+    return run;
+}
+// ─── Report Formatting ──────────────────────────────────────────────────────
+/**
+ * Format a benchmark run as a human-readable report.
+ */
+export function formatBenchmarkReport(run) {
+    const lines = [];
+    const s = run.summary;
+    const elapsed = ((run.endedAt - run.startedAt) / 1000).toFixed(1);
+    lines.push('═'.repeat(60));
+    lines.push(`  📊  Benchmark Results: ${run.provider}/${run.model}`);
+    lines.push('═'.repeat(60));
+    lines.push('');
+    lines.push(`  Run ID: ${run.id}`);
+    lines.push(`  Duration: ${elapsed}s`);
+    lines.push(`  Tasks: ${s.tasksPassed}/${s.totalTasks} passed (${s.tasksFailed} failed)`);
+    lines.push(`  Avg quality: ${(s.avgQualityScore * 100).toFixed(1)}%`);
+    lines.push(`  Median latency: ${s.medianLatencyMs}ms`);
+    lines.push(`  Total cost: $${s.totalCostUsd.toFixed(6)}`);
+    lines.push(`  Total tokens: ${s.totalTokens.toLocaleString()}`);
+    lines.push('');
+    // Results table
+    lines.push(`  ${'─'.repeat(58)}`);
+    lines.push(`  ${'Task'.padEnd(25)} ${'Status'.padEnd(8)} ${'Quality'.padEnd(9)} ${'Latency'.padEnd(9)} ${'Cost'}`);
+    lines.push(`  ${'─'.repeat(58)}`);
+    for (const r of run.results) {
+        const status = r.success ? '✅' : '❌';
+        const quality = r.success ? `${(r.qualityScore * 100).toFixed(0)}%` : 'N/A';
+        const latency = `${r.latencyMs}ms`;
+        const cost = `$${r.costUsd.toFixed(8)}`;
+        lines.push(`  ${r.taskId.padEnd(25)} ${status.padEnd(8)} ${quality.padEnd(9)} ${latency.padEnd(9)} ${cost}`);
+    }
+    lines.push(`  ${'─'.repeat(58)}`);
+    lines.push('');
+    return lines.join('\n');
+}
+/**
+ * Format a benchmark run as JSON (for machine consumption).
+ */
+export function formatBenchmarkJSON(run) {
+    return JSON.stringify(run, null, 2);
+}
+/**
+ * Format a benchmark run as Markdown (for documentation).
+ */
+export function formatBenchmarkMarkdown(run) {
+    const s = run.summary;
+    const elapsed = ((run.endedAt - run.startedAt) / 1000).toFixed(1);
+    const lines = [
+        `# Benchmark: ${run.provider}/${run.model}`,
+        '',
+        `- **Run ID:** ${run.id}`,
+        `- **Duration:** ${elapsed}s`,
+        `- **Tasks:** ${s.tasksPassed}/${s.totalTasks} passed`,
+        `- **Avg Quality Score:** ${(s.avgQualityScore * 100).toFixed(1)}%`,
+        `- **Median Latency:** ${s.medianLatencyMs}ms`,
+        `- **Total Cost:** $${s.totalCostUsd.toFixed(6)}`,
+        `- **Total Tokens:** ${s.totalTokens.toLocaleString()}`,
+        '',
+        '## Results',
+        '',
+        '| Task | Status | Quality | Latency | Cost |',
+        '|------|--------|---------|---------|------|',
+    ];
+    for (const r of run.results) {
+        const status = r.success ? '✅ Pass' : '❌ Fail';
+        const quality = r.success ? `${(r.qualityScore * 100).toFixed(0)}%` : 'N/A';
+        lines.push(`| ${r.taskId} | ${status} | ${quality} | ${r.latencyMs}ms | $${r.costUsd.toFixed(8)} |`);
+    }
+    return lines.join('\n');
+}
+// ─── Query Functions ────────────────────────────────────────────────────────
+/**
+ * Get all available benchmark tasks.
+ */
+export function getBenchmarkTasks() {
+    return [...BENCHMARK_TASKS];
+}
+/**
+ * Get a specific benchmark task by ID.
+ */
+export function getBenchmarkTask(id) {
+    return BENCHMARK_TASKS.find((t) => t.id === id);
+}
+/**
+ * Get all past benchmark runs.
+ */
+export function getBenchmarkRuns() {
+    const data = readBenchmarkData();
+    return [...data.runs].reverse(); // Most recent first
+}
+/**
+ * Get the most recent benchmark run for a specific provider/model.
+ */
+export function getLatestBenchmarkRun(provider, model) {
+    const data = readBenchmarkData();
+    const runs = data.runs
+        .filter((r) => r.provider === provider && r.model === model)
+        .sort((a, b) => b.startedAt - a.startedAt);
+    return runs[0] || null;
+}
+/**
+ * Compare two benchmark runs side by side.
+ */
+export function compareBenchmarks(runA, runB) {
+    const a = runA.summary;
+    const b = runB.summary;
+    const winner = (aVal, bVal, higherBetter) => {
+        if (aVal === bVal)
+            return 'tie';
+        return (higherBetter ? aVal > bVal : aVal < bVal)
+            ? '← ' + runA.model
+            : runB.model + ' →';
+    };
+    const aPassRate = ((a.tasksPassed / a.totalTasks) * 100).toFixed(0);
+    const bPassRate = ((b.tasksPassed / b.totalTasks) * 100).toFixed(0);
+    const aQual = (a.avgQualityScore * 100).toFixed(1);
+    const bQual = (b.avgQualityScore * 100).toFixed(1);
+    const aLat = a.medianLatencyMs + 'ms';
+    const bLat = b.medianLatencyMs + 'ms';
+    const aCost = '$' + a.totalCostUsd.toFixed(6);
+    const bCost = '$' + b.totalCostUsd.toFixed(6);
+    const lines = [
+        '═'.repeat(60),
+        '  ⚔️  Benchmark Comparison: ' + runA.model + ' vs ' + runB.model,
+        '═'.repeat(60),
+        '',
+        '  ' + 'Metric'.padEnd(30) + runA.model.padEnd(20) + runB.model.padEnd(20) + 'Winner',
+        '  ' + '─'.repeat(70),
+        '  ' + 'Pass Rate'.padEnd(30) +
+            (aPassRate + '%').padEnd(20) +
+            (bPassRate + '%').padEnd(20) +
+            winner(a.tasksPassed / a.totalTasks, b.tasksPassed / a.totalTasks, true),
+        '  ' + 'Avg Quality'.padEnd(30) +
+            (aQual + '%').padEnd(20) +
+            (bQual + '%').padEnd(20) +
+            winner(a.avgQualityScore, b.avgQualityScore, true),
+        '  ' + 'Median Latency'.padEnd(30) +
+            aLat.padEnd(20) +
+            bLat.padEnd(20) +
+            winner(a.medianLatencyMs, b.medianLatencyMs, false),
+        '  ' + 'Total Cost'.padEnd(30) +
+            aCost.padEnd(20) +
+            bCost.padEnd(20) +
+            winner(a.totalCostUsd, b.totalCostUsd, false),
+        '',
+    ];
+    return lines.join('\n');
+}
+/**
+ * Clear all benchmark data.
+ */
+export function clearBenchmarks() {
+    writeBenchmarkData({ runs: [], version: CURRENT_VERSION });
+}
+//# sourceMappingURL=benchmark.js.map

package/dist/learning/benchmark.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../../src/learning/benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAGlC,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAkH5C,+EAA+E;AAE/E,MAAM,eAAe,GAAoB;IACvC,yEAAyE;IACzE;QACE,EAAE,EAAE,aAAa;QACjB,KAAK,EAAE,sBAAsB;QAC7B,GAAG,EAAE,iBAAiB;QACtB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,yIAAyI;QACjJ,gBAAgB,EAAE,CAAC,gBAAgB,EAAE,QAAQ,EAAE,SAAS,CAAC;QACzD,iBAAiB,EAAE,GAAG;QACtB,YAAY,EAAE,OAAO;QACrB,cAAc,EAAE,YAAY;KAC7B;IACD;QACE,EAAE,EAAE,WAAW;QACf,KAAK,EAAE,oBAAoB;QAC3B,GAAG,EAAE,iBAAiB;QACtB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,2IAA2I;QACnJ,gBAAgB,EAAE,CAAC,eAAe,EAAE,QAAQ,EAAE,GAAG,CAAC;QAClD,iBAAiB,EAAE,GAAG;QACtB,YAAY,EAAE,OAAO;QACrB,cAAc,EAAE,QAAQ;KACzB;IACD;QACE,EAAE,EAAE,UAAU;QACd,KAAK,EAAE,yBAAyB;QAChC,GAAG,EAAE,iBAAiB;QACtB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,gLAAgL;QACxL,gBAAgB,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,mBAAmB,CAAC;QACnE,iBAAiB,EAAE,GAAG;QACtB,YAAY,EAAE,OAAO;QACrB,cAAc,EAAE,YAAY;KAC7B;IAED,yEAAyE;IACzE;QACE,EAAE,EAAE,aAAa;QACjB,KAAK,EAAE,sBAAsB;QAC7B,GAAG,EAAE,iBAAiB;QACtB,UAAU,EAAE,QAAQ;QACpB,MAAM,EAAE,oRAAoR;QAC5R,gBAAgB,EAAE,CAAC,UAAU,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,CAAC;QAC3E,iBAAiB,EAAE,GAAG;QACtB,YAAY,EAAE,QAAQ;QACtB,cAAc,EAAE,YAAY;KAC7B;IACD;QACE,EAAE,EAAE,qBAAqB;QACzB,KAAK,EAAE,qBAAqB;QAC5B,GAAG,EAAE,iBAAiB;QACtB,UAAU,EAAE,QAAQ;QACpB,MAAM,EAAE,uMAAuM;QAC/M,gBAAgB,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC;QACtD,iBAAiB,EAAE,GAAG;QACtB,YAAY,EAAE,QAAQ;QACtB,cAAc,EAAE,YAAY;KAC7B;IACD;QACE,EAAE,EAAE,iBAAiB;QACrB,KAAK,EAAE,iBAAiB;QACxB,GAAG,EAAE,iBAAiB;QACtB,UAAU,EAAE,QAAQ;QACpB,MAAM,EAAE,2OAA2O;QACnP,gBAAgB,EAAE,CAAC,UAAU,EAAE,WAAW,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,CAAC;QAC7E,iBAAiB,EAAE,GAAG;QACtB,YAAY,EAAE,QAAQ;QACtB,cAAc,EAAE,YAAY;KAC7B;IAED,yEAAyE;IACzE;QACE,EAAE,EAAE,mBAAmB;QACvB,KAAK,EAAE,yBAAyB;QAChC,GAAG,EAAE,aAAa;QAClB,UAAU,EAAE,QAAQ;QACpB,MAAM,EAAE,6aAA6a;QACrb,gBAAgB,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC;QACpD,YAAY,EAAE,CAAC,UAAU,CAAC;QAC1B,iBAAiB,EAAE,GAAG;QACtB,YAAY,EAAE,QAAQ;QACtB,cAAc,EAAE,YAAY;KAC7B;IACD;QACE,EAAE,EAAE,wBAAwB;QAC5B,KAAK,EAAE,+BAA+B;QACtC,GAAG,EAAE,aAAa;QAClB,UAAU,EAAE,QAAQ;QACpB,MAAM,EAAE,2fAA2f;QACngB,gBAAgB,EAAE,CAAC,UAAU,EAAE,WAAW,EAAE,IAAI,CAAC;QACjD,YAAY,EAAE,CAAC,eAAe,EAAE,YAAY,EAAE,eAAe,CAAC;QAC9D,iBAAiB,EAAE,GAAG;QACtB,YAAY,EAAE,QAAQ;QACtB,cAAc,EAAE,YAAY;KAC7B;IAED,yEAAyE;IACzE;QACE,EAAE,EAAE,eAAe;QACnB,KAAK,EAAE,iBAAiB;QACxB,GAAG,EAAE,WAAW;QAChB,UAAU,EAAE,QAAQ;QACpB,MAAM,EAAE,4OAA4O;QACpP,gBAAgB,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,aAAa,CAAC;QAC1D,iBAAiB,EAAE,GAAG;QACtB,YAAY,EAAE,OAAO;QACrB,cAAc,EAAE,YAAY;KAC7B;IACD;QACE,EAAE,EAAE,YAAY;QAChB,KAAK,EAAE,oBAAoB;QAC3B,GAAG,EAAE,WAAW;QAChB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,2SAA2S;QACnT,gBAAgB,EAAE,CAAC,IAAI,EAAE,mBAAmB,EAAE,WAAW,CAAC;QAC1D,iBAAiB,EAAE,GAAG;QACtB,YAAY,EAAE,OAAO;QACrB,cAAc,EAAE,YAAY;KAC7B;IAED,yEAAyE;IACzE;QACE,EAAE,EAAE,WAAW;QACf,KAAK,EAAE,kBAAkB;QACzB,GAAG,EAAE,SAAS;QACd,UAAU,EAAE,QAAQ;QACpB,MAAM,EAAE,8YAA8Y;QACtZ,gBAAgB,EAAE,CAAC,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,mBAAmB,EAAE,OAAO,CAAC;QAC5E,iBAAiB,EAAE,GAAG;QACtB,YAAY,EAAE,QAAQ;QACtB,cAAc,EAAE,YAAY;KAC7B;IAED,yEAAyE;IACzE;QACE,EAAE,EAAE,OAAO;QACX,KAAK,EAAE,wBAAwB;QAC/B,GAAG,EAAE,eAAe;QACpB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,qRAAqR;QAC7R,gBAAgB,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,aAAa,CAAC;QACnE,iBAAiB,EAAE,GAAG;QACtB,YAAY,EAAE,OAAO;QACrB,cAAc,EAAE,YAAY;KAC7B;IAED,yEAAyE;IACzE;QACE,EAAE,EAAE,cAAc;QAClB,KAAK,EAAE,mBAAmB;QAC1B,GAAG,EAAE,UAAU;QACf,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,wWAAwW;QAChX,gBAAgB,EAAE,CAAC,eAAe,EAAE,oBAAoB,EAAE,eAAe,EAAE,IAAI,EAAE,GAAG,CAAC;QACrF,YAAY,EAAE,CAAC,sBAAsB,EAAE,kBAAkB,CAAC;QAC1D,iBAAiB,EAAE,GAAG;QACtB,YAAY,EAAE,QAAQ;QACtB,cAAc,EAAE,YAAY;KAC7B;IACD;QACE,EAAE,EAAE,cAAc;QAClB,KAAK,EAAE,uBAAuB;QAC9B,GAAG,EAAE,UAAU;QACf,UAAU,EAAE,QAAQ;QACpB,MAAM,EAAE,2SAA2S;QACnT,gBAAgB,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,aAAa,CAAC;QAC9D,YAAY,EAAE,CAAC,UAAU,EAAE,WAAW,CAAC;QACvC,iBAAiB,EAAE,GAAG;QACtB,YAAY,EAAE,QAAQ;QACtB,cAAc,EAAE,YAAY;KAC7B;IAED,yEAAyE;IACzE;QACE,EAAE,EAAE,gBAAgB;QACpB,KAAK,EAAE,2BAA2B;QAClC,GAAG,EAAE,cAAc;QACnB,UAAU,EAAE,QAAQ;QACpB,MAAM,EAAE,sbAAsb;QAC9b,gBAAgB,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,CAAC;QAClD,YAAY,EAAE,CAAC,aAAa,EAAE,UAAU,CAAC;QACzC,iBAAiB,EAAE,GAAG;QACtB,YAAY,EAAE,QAAQ;QACtB,cAAc,EAAE,YAAY;KAC7B;IAED,yEAAyE;IACzE;QACE,EAAE,EAAE,mBAAmB;QACvB,KAAK,EAAE,yBAAyB;QAChC,GAAG,EAAE,eAAe;QACpB,UAAU,EAAE,QAAQ;QACpB,MAAM,EAAE,wKAAwK;QAChL,gBAAgB,EAAE,CAAC,WAAW,EAAE,WAAW,EAAE,aAAa,EAAE,eAAe,CAAC;QAC5E,iBAAiB,EAAE,GAAG;QACtB,YAAY,EAAE,OAAO;QACrB,cAAc,EAAE,MAAM;KACvB;IACD;QACE,EAAE,EAAE,oBAAoB;QACxB,KAAK,EAAE,oBAAoB;QAC3B,GAAG,EAAE,eAAe;QACpB,UAAU,EAAE,QAAQ;QACpB,MAAM,EAAE,4LAA4L;QACpM,gBAAgB,EAAE,CAAC,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS,EAAE,YAAY,CAAC;QACpF,iBAAiB,EAAE,GAAG;QACtB,YAAY,EAAE,OAAO;QACrB,cAAc,EAAE,MAAM;KACvB;IAED,yEAAyE;IACzE;QACE,EAAE,EAAE,kBAAkB;QACtB,KAAK,EAAE,0BAA0B;QACjC,GAAG,EAAE,iBAAiB;QACtB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,oVAAoV;QAC5V,gBAAgB,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,eAAe,EAAE,YAAY,CAAC;QAC9E,iBAAiB,EAAE,GAAG;QACtB,YAAY,EAAE,MAAM;QACpB,cAAc,EAAE,YAAY;KAC7B;IACD;QACE,EAAE,EAAE,cAAc;QAClB,KAAK,EAAE,6BAA6B;QACpC,GAAG,EAAE,iBAAiB;QACtB,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,8PAA8P;QACtQ,gBAAgB,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,cAAc,EAAE,aAAa,CAAC;QACvF,iBAAiB,EAAE,GAAG;QACtB,YAAY,EAAE,QAAQ;QACtB,cAAc,EAAE,YAAY;KAC7B;IAED,yEAAyE;IACzE;QACE,EAAE,EAAE,oBAAoB;QACxB,KAAK,EAAE,kCAAkC;QACzC,GAAG,EAAE,aAAa;QAClB,UAAU,EAAE,QAAQ;QACpB,MAAM,EAAE,sZAAsZ;QAC9Z,gBAAgB,EAAE,CAAC,WAAW,EAAE,cAAc,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC;QAC5E,iBAAiB,EAAE,GAAG;QACtB,YAAY,EAAE,QAAQ;QACtB,cAAc,EAAE,YAAY;KAC7B;CACF,CAAC;AAEF,+EAA+E;AAE/E,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;AACtD,MAAM,cAAc,GAAG,IAAI,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAC;AAC3D,MAAM,eAAe,GAAG,CAAC,CAAC;AAE1B,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAE9B,+EAA+E;AAE/E,SAAS,SAAS;IAChB,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5B,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7C,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB;IACxB,IAAI,CAAC;QACH,SAAS,EAAE,CAAC;QACZ,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,EAAE,CAAC;YAChC,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC;QAChD,CAAC;QACD,MAAM,GAAG,GAAG,YAAY,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;QAClD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAkB,CAAC;IAC1C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC;IAChD,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAmB;IAC7C,SAAS,EAAE,CAAC;IACZ,aAAa,CAAC,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;AACxE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,MAAc,EAAE,IAAmB;IAC9D,IAAI,KAAK,GAAG,GAAG,CAAC,CAAC,mBAAmB;IAEpC,0CAA0C;IAC1C,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CACjD,MAAM,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAC/C,CAAC,MAAM,CAAC;QACT,KAAK,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC;IAC1D,CAAC;IAED,yCAAyC;IACzC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;QACtB,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CACjD,MAAM,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAC/C,CAAC,MAAM,CAAC;QACT,KAAK,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC;IAC1D,CAAC;IAED,uDAAuD;IACvD,IAAI,MAAM,CAAC,MAAM,GAAG,GAAG;QAAE,KAAK,IAAI,GAAG,CAAC;IACtC,IAAI,MAAM,CAAC,MAAM,GAAG,GAAG;QAAE,KAAK,IAAI,GAAG,CAAC;IAEtC,4CAA4C;IAC5C,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,cAAc,KAAK,MAAM,EAAE,CAAC;QAC1D,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,KAAK,IAAI,GAAG,CAAC;IAC3C,CAAC;IAED,wBAAwB;IACxB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,SAAS,MAAM,CAAC,MAAgB;IAC9B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC1C,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC;QAC5B,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;QACb,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;AAC1C,CAAC;AAED,+EAA+E;AAE/E;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,QAA2B,EAC3B,YAAoB,EACpB,KAAa,EACb,OASC;IAED,eAAe;IACf,IAAI,KAAK,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;IAEjC,IAAI,OAAO,EAAE,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnD,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,OAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,IAAI,OAAO,EAAE,YAAY,EAAE,CAAC;QAC1B,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,OAAO,CAAC,YAAY,CAAC,CAAC;IACvE,CAAC;IAED,MAAM,KAAK,GAAG,aAAa,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;IAClF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAEtB,eAAe;QACf,IAAI,OAAO,EAAE,MAAM,IAAI,SAAS,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnD,MAAM,CAAC,IAAI,CAAC,cAAc,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,+BAA+B,CAAC,CAAC;YACpF,MAAM;QACR,CAAC;QAED,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAEjD,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEjC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE;gBAClD,KAAK;gBACL,WAAW,EAAE,GAAG,EAAE,4CAA4C;gBAC9D,SAAS,EAAE,IAAI,CAAC,iBAAiB,GAAG,GAAG,EAAE,qBAAqB;aAC/D,CAAC,CAAC;YAEH,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa,CAAC;YAC7C,MAAM,WAAW,GAAG,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAChD,MAAM,YAAY,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;YAC5C,MAAM,OAAO,GAAG,aAAa,CAAC,YAAY,EAAE,KAAK,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC;YAC9E,SAAS,IAAI,OAAO,CAAC;YAErB,MAAM,MAAM,GAAoB;gBAC9B,MAAM,EAAE,IAAI,CAAC,EAAE;gBACf,QAAQ,EAAE,YAAY;gBACtB,KAAK;gBACL,MAAM;gBACN,OAAO,EAAE,IAAI;gBACb,SAAS;gBACT,WAAW;gBACX,YAAY;gBACZ,OAAO;gBACP,YAAY,EAAE,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC;gBACxC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;aACtB,CAAC;YAEF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa,CAAC;YAC7C,OAAO,CAAC,IAAI,CAAC;gBACX,MAAM,EAAE,IAAI,CAAC,EAAE;gBACf,QAAQ,EAAE,YAAY;gBACtB,KAAK;gBACL,MAAM,EAAE,EAAE;gBACV,OAAO,EAAE,KAAK;gBACd,SAAS;gBACT,WAAW,EAAE,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC;gBACxC,YAAY,EAAE,CAAC;gBACf,OAAO,EAAE,CAAC;gBACV,YAAY,EAAE,CAAC;gBACf,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;gBACvD,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;aACtB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE3B,kBAAkB;IAClB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAChD,MAAM,aAAa,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;IACxD,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAElD,MAAM,GAAG,GAAiB;QACxB,EAAE,EAAE,KAAK;QACT,QAAQ,EAAE,YAAY;QACtB,KAAK;QACL,SAAS;QACT,OAAO;QACP,OAAO;QACP,OAAO,EAAE;YACP,UAAU,EAAE,OAAO,CAAC,MAAM;YAC1B,WAAW,EAAE,MAAM,CAAC,MAAM;YAC1B,WAAW,EAAE,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;YAC3C,eAAe,EAAE,aAAa,CAAC,MAAM,GAAG,CAAC;gBACvC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM;gBACjE,CAAC,CAAC,CAAC;YACL,eAAe,EAAE,MAAM,CAAC,SAAS,CAAC;YAClC,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YAC5D,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC;SACjF;KACF,CAAC;IAEF,UAAU;IACV,MAAM,IAAI,GAAG,iBAAiB,EAAE,CAAC;IACjC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEpB,iBAAiB;IACjB,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;QAC1C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,kBAAkB,CAAC,CAAC;IACnD,CAAC;IAED,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAEzB,OAAO,GAAG,CAAC;AACb,CAAC;AAED,+EAA+E;AAE/E;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,GAAiB;IACrD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC;IACtB,MAAM,OAAO,GAAG,CAAC,CAAC,GAAG,CAAC,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAElE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,4BAA4B,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;IACpE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,aAAa,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;IAClC,KAAK,CAAC,IAAI,CAAC,eAAe,OAAO,GAAG,CAAC,CAAC;IACtC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,UAAU,YAAY,CAAC,CAAC,WAAW,UAAU,CAAC,CAAC;IACzF,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,eAAe,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACtE,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC;IACvD,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC1D,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,WAAW,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;IAChE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,gBAAgB;IAChB,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAClC,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,EAAE,CAAC,CAAC;IACnH,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAElC,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QACrC,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;QAC5E,MAAM,OAAO,GAAG,GAAG,CAAC,CAAC,SAAS,IAAI,CAAC;QACnC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QACxC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAC/G,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAClC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,GAAiB;IACnD,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACtC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CAAC,GAAiB;IACvD,MAAM,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC;IACtB,MAAM,OAAO,GAAG,CAAC,CAAC,GAAG,CAAC,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAElE,MAAM,KAAK,GAAa;QACtB,gBAAgB,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,KAAK,EAAE;QAC3C,EAAE;QACF,iBAAiB,GAAG,CAAC,EAAE,EAAE;QACzB,mBAAmB,OAAO,GAAG;QAC7B,gBAAgB,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,UAAU,SAAS;QACtD,4BAA4B,CAAC,CAAC,CAAC,eAAe,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;QACnE,yBAAyB,CAAC,CAAC,eAAe,IAAI;QAC9C,sBAAsB,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;QACjD,uBAAuB,CAAC,CAAC,WAAW,CAAC,cAAc,EAAE,EAAE;QACvD,EAAE;QACF,YAAY;QACZ,EAAE;QACF,8CAA8C;QAC9C,8CAA8C;KAC/C,CAAC;IAEF,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;QAC/C,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;QAC5E,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,MAAM,MAAM,MAAM,OAAO,MAAM,CAAC,CAAC,SAAS,SAAS,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACvG,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,+EAA+E;AAE/E;;GAEG;AACH,MAAM,UAAU,iBAAiB;IAC/B,OAAO,CAAC,GAAG,eAAe,CAAC,CAAC;AAC9B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,EAAU;IACzC,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;AAClD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB;IAC9B,MAAM,IAAI,GAAG,iBAAiB,EAAE,CAAC;IACjC,OAAO,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,oBAAoB;AACvD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,QAAgB,EAAE,KAAa;IACnE,MAAM,IAAI,GAAG,iBAAiB,EAAE,CAAC;IACjC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI;SACnB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC;SAC3D,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC;IAE7C,OAAO,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;AACzB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAkB,EAAE,IAAkB;IACtE,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC;IACvB,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC;IAEvB,MAAM,MAAM,GAAG,CAAC,IAAY,EAAE,IAAY,EAAE,YAAqB,EAAU,EAAE;QAC3E,IAAI,IAAI,KAAK,IAAI;YAAE,OAAO,KAAK,CAAC;QAChC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC;YAC/C,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK;YACnB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACxB,CAAC,CAAC;IAEF,MAAM,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACpE,MAAM,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACpE,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,eAAe,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,eAAe,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,CAAC,CAAC,eAAe,GAAG,IAAI,CAAC;IACtC,MAAM,IAAI,GAAG,CAAC,CAAC,eAAe,GAAG,IAAI,CAAC;IACtC,MAAM,KAAK,GAAG,GAAG,GAAG,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,GAAG,GAAG,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAE9C,MAAM,KAAK,GAAa;QACtB,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QACd,8BAA8B,GAAG,IAAI,CAAC,KAAK,GAAG,MAAM,GAAG,IAAI,CAAC,KAAK;QACjE,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QACd,EAAE;QACF,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,QAAQ;QACrF,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QACrB,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,UAAU,EAAE,IAAI,CAAC;QAC1E,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC;YAC7B,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;YACxB,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;YACxB,MAAM,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,eAAe,EAAE,IAAI,CAAC;QACpD,IAAI,GAAG,gBAAgB,CAAC,MAAM,CAAC,EAAE,CAAC;YAChC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YACf,MAAM,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,eAAe,EAAE,KAAK,CAAC;QACrD,IAAI,GAAG,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YAChB,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YAChB,MAAM,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,YAAY,EAAE,KAAK,CAAC;QAC/C,EAAE;KACH,CAAC;IAEF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe;IAC7B,kBAAkB,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAC;AAC7D,CAAC"}