crewly 1.5.18 → 1.5.19

package/dist/backend/backend/src/services/agent/crewly-agent/eval/run-eval-l4.js

@@ -0,0 +1,414 @@
+#!/usr/bin/env npx tsx
+/**
+ * L4 Cross-Runtime Benchmark Script
+ *
+ * Runs all L4 eval tasks (collaboration + trap/resilience) across multiple
+ * CLI runtimes and produces a comparison report.
+ *
+ * Each runtime receives the same CONTEXT.md, skill shims, and fixtures.
+ * Tool call tracking uses the `.crewly/.tool-log.jsonl` written by skill
+ * shims plus filesystem diffing.
+ *
+ * Usage:
+ *   npx tsx backend/src/services/agent/crewly-agent/eval/run-eval-l4.ts
+ *   npx tsx ... --runtime=claude-code           # single runtime
+ *   npx tsx ... --runtime=all                   # all 4 runtimes
+ *   npx tsx ... --runtime=claude-code,gemini-cli # specific runtimes
+ *   npx tsx ... --tasks=L4-C1,L4-T1             # specific tasks
+ *   npx tsx ... --verbose                        # detailed output
+ *   npx tsx ... --keep-work-dirs                 # preserve temp dirs
+ *   npx tsx ... --model=claude-sonnet-4    # override model name
+ *
+ * @module eval/run-eval-l4
+ */
+import { mkdirSync, writeFileSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { L4_TASKS } from './tasks/l4-tasks.js';
+import { createAdapter } from './adapters/index.js';
+import { scoreTask, aggregateResults } from './eval-scorer.js';
+// ---------------------------------------------------------------------------
+// Configuration
+// ---------------------------------------------------------------------------
+/** All supported runtimes for L4 benchmarking */
+const ALL_RUNTIMES = ['claude-code', 'codex-cli', 'gemini-cli', 'crewly-agent'];
+/** CLI runtimes only (excludes crewly-agent which needs a running backend) */
+const CLI_RUNTIMES = ['claude-code', 'codex-cli', 'gemini-cli'];
+const OUTPUT_DIR = join(process.cwd(), 'eval-results');
+/** Default model names per runtime (used for cost calculation) */
+const DEFAULT_MODELS = {
+    'claude-code': 'claude-sonnet-4',
+    'codex-cli': 'o3-mini',
+    'gemini-cli': 'gemini-2.5-pro',
+    'crewly-agent': 'claude-sonnet-4',
+};
+/**
+ * Parse CLI arguments into a BenchmarkConfig.
+ *
+ * @param argv - process.argv.slice(2)
+ * @returns parsed configuration
+ */
+export function parseArgs(argv) {
+    const config = {
+        runtimes: CLI_RUNTIMES,
+        taskIds: [],
+        verbose: false,
+        keepWorkDirs: false,
+        outputDir: OUTPUT_DIR,
+    };
+    for (const arg of argv) {
+        if (arg.startsWith('--runtime=')) {
+            const val = arg.split('=')[1];
+            if (val === 'all') {
+                config.runtimes = [...ALL_RUNTIMES];
+            }
+            else if (val === 'cli') {
+                config.runtimes = [...CLI_RUNTIMES];
+            }
+            else {
+                config.runtimes = val.split(',');
+            }
+        }
+        else if (arg.startsWith('--tasks=')) {
+            config.taskIds = arg.split('=')[1].split(',');
+        }
+        else if (arg.startsWith('--model=')) {
+            config.model = arg.split('=')[1];
+        }
+        else if (arg === '--verbose' || arg === '-v') {
+            config.verbose = true;
+        }
+        else if (arg === '--keep-work-dirs') {
+            config.keepWorkDirs = true;
+        }
+        else if (arg.startsWith('--output=')) {
+            config.outputDir = arg.split('=')[1];
+        }
+    }
+    return config;
+}
+// ---------------------------------------------------------------------------
+// Runtime availability check
+// ---------------------------------------------------------------------------
+/**
+ * Check which runtimes are available on this machine.
+ *
+ * @param runtimes - list of runtimes to check
+ * @returns object with available and unavailable lists
+ */
+export async function checkRuntimeAvailability(runtimes) {
+    const available = [];
+    const unavailable = [];
+    for (const id of runtimes) {
+        if (id === 'crewly-agent') {
+            // crewly-agent needs a running backend — skip availability check
+            unavailable.push({ id, reason: 'Requires running backend (use eval-runner instead)' });
+            continue;
+        }
+        try {
+            const adapter = createAdapter(id);
+            const tmpWorkDir = join(tmpdir(), `eval-check-${id}-${Date.now()}`);
+            mkdirSync(tmpWorkDir, { recursive: true });
+            try {
+                await adapter.initialize({
+                    runtimeId: id,
+                    model: DEFAULT_MODELS[id],
+                    workDir: tmpWorkDir,
+                });
+                await adapter.shutdown();
+                available.push(id);
+            }
+            finally {
+                rmSync(tmpWorkDir, { recursive: true, force: true });
+            }
+        }
+        catch (err) {
+            const reason = err instanceof Error ? err.message : String(err);
+            unavailable.push({ id, reason });
+        }
+    }
+    return { available, unavailable };
+}
+// ---------------------------------------------------------------------------
+// Single-runtime benchmark
+// ---------------------------------------------------------------------------
+/**
+ * Run all L4 tasks against a single runtime and produce a report.
+ *
+ * @param runtimeId    - which runtime to benchmark
+ * @param tasks        - L4 tasks to run
+ * @param model        - model identifier for cost calculation
+ * @param verbose      - whether to emit progress logs
+ * @param keepWorkDirs - whether to preserve work directories
+ * @returns evaluation run report
+ */
+async function benchmarkRuntime(runtimeId, tasks, model, verbose, keepWorkDirs) {
+    const adapter = createAdapter(runtimeId);
+    const tmpWorkDir = join(tmpdir(), `eval-l4-${runtimeId}-${Date.now()}`);
+    mkdirSync(tmpWorkDir, { recursive: true });
+    try {
+        await adapter.initialize({
+            runtimeId,
+            model,
+            workDir: tmpWorkDir,
+        });
+        const taskResults = [];
+        for (const task of tasks) {
+            if (verbose) {
+                console.log(`  [${runtimeId}] ${task.id}: ${task.title}`);
+            }
+            const workDir = join(tmpWorkDir, task.id);
+            mkdirSync(workDir, { recursive: true });
+            let result;
+            let verification = { passed: false, checks: [] };
+            try {
+                // Setup
+                await task.setup(workDir);
+                // Run
+                result = await adapter.runTask(task, workDir);
+                if (verbose) {
+                    const status = result.success ? '✓' : '✗';
+                    console.log(`    ${status} Runtime: ${result.durationMs}ms, ${result.toolCalls.length} tool calls`);
+                }
+                // Verify
+                verification = await task.verify(workDir, result.toolCalls);
+                if (verbose) {
+                    const vStatus = verification.passed ? 'PASS' : 'FAIL';
+                    console.log(`    Verify: ${vStatus}`);
+                    for (const c of [...verification.checks, ...(verification.collaborationChecks ?? [])]) {
+                        console.log(`      ${c.passed ? '✓' : '✗'} ${c.name}`);
+                    }
+                }
+            }
+            catch (err) {
+                const msg = err instanceof Error ? err.message : String(err);
+                if (verbose)
+                    console.log(`    ERROR: ${msg}`);
+                taskResults.push({
+                    taskId: task.id,
+                    level: task.level,
+                    title: task.title,
+                    success: false,
+                    dimensions: [],
+                    composite: 0,
+                    metrics: {
+                        durationMs: 0, steps: 0, usage: { input: 0, output: 0 },
+                        costUSD: 0, toolCallCount: 0, correctToolCalls: 0,
+                        unnecessaryToolCalls: 0, argAccuracy: 0, toolErrors: 0,
+                        humanNudges: 0, loopDetected: false, timedOut: false,
+                    },
+                    verification: { passed: false, checks: [] },
+                    error: msg,
+                });
+                continue;
+            }
+            finally {
+                if (!keepWorkDirs) {
+                    try {
+                        rmSync(workDir, { recursive: true, force: true });
+                    }
+                    catch { /* */ }
+                }
+            }
+            if (result) {
+                taskResults.push(scoreTask(task, result, verification, runtimeId, model));
+            }
+        }
+        await adapter.shutdown();
+        return aggregateResults(taskResults, runtimeId, model);
+    }
+    finally {
+        if (!keepWorkDirs) {
+            try {
+                rmSync(tmpWorkDir, { recursive: true, force: true });
+            }
+            catch { /* */ }
+        }
+    }
+}
+/**
+ * Build a cross-runtime comparison from individual reports.
+ *
+ * @param reports - array of per-runtime reports
+ * @returns structured comparison
+ */
+export function buildComparison(reports) {
+    // Ranking by composite score
+    const ranking = reports
+        .map((r) => ({
+        runtimeId: r.runtimeId,
+        model: r.model,
+        composite: Math.round(r.overallComposite * 10) / 10,
+        passRate: `${r.passedCount}/${r.totalCount}`,
+        totalCostUSD: Math.round(r.totalCostUSD * 10000) / 10000,
+        totalDurationMs: r.totalDurationMs,
+    }))
+        .sort((a, b) => b.composite - a.composite);
+    // Per-task comparison
+    const allTaskIds = new Set();
+    const taskTitles = new Map();
+    for (const r of reports) {
+        for (const tr of r.taskResults) {
+            allTaskIds.add(tr.taskId);
+            taskTitles.set(tr.taskId, tr.title);
+        }
+    }
+    const taskComparison = Array.from(allTaskIds).map((taskId) => {
+        const scores = {};
+        for (const r of reports) {
+            const tr = r.taskResults.find((t) => t.taskId === taskId);
+            scores[r.runtimeId] = tr ? Math.round(tr.composite * 10) / 10 : 0;
+        }
+        return {
+            taskId,
+            title: taskTitles.get(taskId) ?? taskId,
+            scores,
+        };
+    });
+    return {
+        timestamp: new Date().toISOString(),
+        reports,
+        ranking,
+        taskComparison,
+    };
+}
+// ---------------------------------------------------------------------------
+// Output formatting
+// ---------------------------------------------------------------------------
+/**
+ * Print a formatted comparison to stdout.
+ *
+ * @param comparison - cross-runtime comparison data
+ */
+function printComparison(comparison) {
+    const SEP = '═'.repeat(76);
+    const THIN = '─'.repeat(76);
+    console.log(`\n${SEP}`);
+    console.log('  L4 CROSS-RUNTIME BENCHMARK RESULTS');
+    console.log(`  ${comparison.timestamp}`);
+    console.log(SEP);
+    // Ranking table
+    console.log('\n  RANKING');
+    console.log(THIN);
+    console.log('  Rank  Runtime          Score   Pass Rate   Cost($)   Duration');
+    console.log(THIN);
+    comparison.ranking.forEach((r, i) => {
+        console.log(`  ${String(i + 1).padStart(4)}  ` +
+            `${r.runtimeId.padEnd(16)} ` +
+            `${String(r.composite).padStart(5)}   ` +
+            `${r.passRate.padStart(9)}   ` +
+            `${r.totalCostUSD.toFixed(4).padStart(7)}   ` +
+            `${(r.totalDurationMs / 1000).toFixed(1).padStart(7)}s`);
+    });
+    console.log(THIN);
+    // Per-task breakdown
+    console.log('\n  PER-TASK SCORES');
+    console.log(THIN);
+    const runtimeIds = comparison.ranking.map((r) => r.runtimeId);
+    const header = '  Task       ' + runtimeIds.map((r) => r.padStart(14)).join('');
+    console.log(header);
+    console.log(THIN);
+    for (const tc of comparison.taskComparison) {
+        const scores = runtimeIds.map((r) => {
+            const s = tc.scores[r] ?? 0;
+            return String(s).padStart(14);
+        }).join('');
+        console.log(`  ${tc.taskId.padEnd(11)} ${scores}`);
+    }
+    console.log(THIN);
+    // Winner
+    if (comparison.ranking.length > 0) {
+        const winner = comparison.ranking[0];
+        console.log(`\n  🏆 WINNER: ${winner.runtimeId} (${winner.model}) — ${winner.composite}/100`);
+    }
+    console.log(`${SEP}\n`);
+}
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+/**
+ * Main entry point for the benchmark script.
+ *
+ * @param argv - command line arguments (process.argv.slice(2))
+ */
+export async function main(argv = process.argv.slice(2)) {
+    const config = parseArgs(argv);
+    console.log('═'.repeat(76));
+    console.log('  L4 CROSS-RUNTIME BENCHMARK');
+    console.log(`  Runtimes: ${config.runtimes.join(', ')}`);
+    console.log(`  Tasks: ${config.taskIds.length > 0 ? config.taskIds.join(', ') : 'all L4'}`);
+    console.log('═'.repeat(76));
+    // Filter tasks
+    let tasks = L4_TASKS;
+    if (config.taskIds.length > 0) {
+        const idSet = new Set(config.taskIds);
+        tasks = tasks.filter((t) => idSet.has(t.id));
+    }
+    if (tasks.length === 0) {
+        console.error('No tasks match the filter. Available: ' + L4_TASKS.map((t) => t.id).join(', '));
+        process.exit(1);
+    }
+    console.log(`\n  Running ${tasks.length} tasks across ${config.runtimes.length} runtime(s)...`);
+    // Check availability
+    const { available, unavailable } = await checkRuntimeAvailability(config.runtimes);
+    if (unavailable.length > 0) {
+        console.log('\n  ⚠ Unavailable runtimes (will be skipped):');
+        for (const u of unavailable) {
+            console.log(`    ✗ ${u.id}: ${u.reason}`);
+        }
+    }
+    if (available.length === 0) {
+        console.error('\n  No runtimes available. Install at least one CLI:');
+        console.error('    - claude (Claude Code)');
+        console.error('    - codex (Codex CLI)');
+        console.error('    - gemini (Gemini CLI)');
+        process.exit(1);
+    }
+    console.log(`\n  ✓ Available: ${available.join(', ')}`);
+    // Run benchmarks
+    const reports = [];
+    for (const runtimeId of available) {
+        const model = config.model ?? DEFAULT_MODELS[runtimeId];
+        console.log(`\n${'─'.repeat(76)}`);
+        console.log(`  Benchmarking: ${runtimeId} (${model})`);
+        console.log('─'.repeat(76));
+        try {
+            const report = await benchmarkRuntime(runtimeId, tasks, model, config.verbose, config.keepWorkDirs);
+            reports.push(report);
+            console.log(`  ✓ ${runtimeId}: ${report.overallComposite.toFixed(1)}/100 (${report.passedCount}/${report.totalCount} passed)`);
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            console.error(`  ✗ ${runtimeId} FAILED: ${msg}`);
+        }
+    }
+    if (reports.length === 0) {
+        console.error('\nAll runtimes failed. No report generated.');
+        process.exit(1);
+    }
+    // Build comparison
+    const comparison = buildComparison(reports);
+    // Print
+    printComparison(comparison);
+    // Save
+    mkdirSync(config.outputDir, { recursive: true });
+    const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
+    const filename = `l4-comparison-${timestamp}.json`;
+    const reportPath = join(config.outputDir, filename);
+    writeFileSync(reportPath, JSON.stringify(comparison, null, 2));
+    console.log(`  Report saved: ${reportPath}`);
+    // Also save individual runtime reports
+    for (const report of reports) {
+        const rFilename = `l4-${report.runtimeId}-${timestamp}.json`;
+        writeFileSync(join(config.outputDir, rFilename), JSON.stringify(report, null, 2));
+    }
+}
+// Run if executed directly
+const isDirectExecution = process.argv[1]?.includes('run-eval-l4');
+if (isDirectExecution) {
+    main().catch((err) => {
+        console.error('Fatal:', err);
+        process.exit(1);
+    });
+}
+//# sourceMappingURL=run-eval-l4.js.map

package/dist/backend/backend/src/services/agent/crewly-agent/eval/run-eval-l4.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-eval-l4.js","sourceRoot":"","sources":["../../../../../../../../backend/src/services/agent/crewly-agent/eval/run-eval-l4.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,SAAS,EAA4B,aAAa,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACrF,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAEjC,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAS/D,8EAA8E;AAC9E,gBAAgB;AAChB,8EAA8E;AAE9E,iDAAiD;AACjD,MAAM,YAAY,GAAgB,CAAC,aAAa,EAAE,WAAW,EAAE,YAAY,EAAE,cAAc,CAAC,CAAC;AAE7F,8EAA8E;AAC9E,MAAM,YAAY,GAAgB,CAAC,aAAa,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC;AAE7E,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,CAAC,CAAC;AAEvD,kEAAkE;AAClE,MAAM,cAAc,GAA8B;IAChD,aAAa,EAAE,iBAAiB;IAChC,WAAW,EAAE,SAAS;IACtB,YAAY,EAAE,gBAAgB;IAC9B,cAAc,EAAE,iBAAiB;CAClC,CAAC;AAqBF;;;;;GAKG;AACH,MAAM,UAAU,SAAS,CAAC,IAAc;IACtC,MAAM,MAAM,GAAoB;QAC9B,QAAQ,EAAE,YAAY;QACtB,OAAO,EAAE,EAAE;QACX,OAAO,EAAE,KAAK;QACd,YAAY,EAAE,KAAK;QACnB,SAAS,EAAE,UAAU;KACtB,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,GAAG,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;YACjC,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9B,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;gBAClB,MAAM,CAAC,QAAQ,GAAG,CAAC,GAAG,YAAY,CAAC,CAAC;YACtC,CAAC;iBAAM,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;gBACzB,MAAM,CAAC,QAAQ,GAAG,CAAC,GAAG,YAAY,CAAC,CAAC;YACtC,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAgB,CAAC;YAClD,CAAC;QACH,CAAC;aAAM,IAAI,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YACtC,MAAM,CAAC,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAChD,CAAC;aAAM,IAAI,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YACtC,MAAM,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnC,CAAC;aAAM,IAAI,GAAG,KAAK,WAAW,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YAC/C,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;QACxB,CAAC;aAAM,IAAI,GAAG,KAAK,kBAAkB,EAAE,CAAC;YACtC,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC;QAC7B,CAAC;aAAM,IAAI,GAAG,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YACvC,MAAM,CAAC,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,8EAA8E;AAC9E,6BAA6B;AAC7B,8EAA8E;AAE9E;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,QAAqB;IAErB,MAAM,SAAS,GAAgB,EAAE,CAAC;IAClC,MAAM,WAAW,GAA6C,EAAE,CAAC;IAEjE,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,IAAI,EAAE,KAAK,cAAc,EAAE,CAAC;YAC1B,iEAAiE;YACjE,WAAW,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,oDAAoD,EAAE,CAAC,CAAC;YACvF,SAAS;QACX,CAAC;QAED,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,aAAa,CAAC,EAAE,CAAC,CAAC;YAClC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,cAAc,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACpE,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC3C,IAAI,CAAC;gBACH,MAAM,OAAO,CAAC,UAAU,CAAC;oBACvB,SAAS,EAAE,EAAE;oBACb,KAAK,EAAE,cAAc,CAAC,EAAE,CAAC;oBACzB,OAAO,EAAE,UAAU;iBACpB,CAAC,CAAC;gBACH,MAAM,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACzB,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACrB,CAAC;oBAAS,CAAC;gBACT,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACvD,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,MAAM,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAChE,WAAW,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAED,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC;AACpC,CAAC;AAED,8EAA8E;AAC9E,2BAA2B;AAC3B,8EAA8E;AAE9E;;;;;;;;;GASG;AACH,KAAK,UAAU,gBAAgB,CAC7B,SAAoB,EACpB,KAAiB,EACjB,KAAa,EACb,OAAgB,EAChB,YAAqB;IAErB,MAAM,OAAO,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;IAEzC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,WAAW,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACxE,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE3C,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,UAAU,CAAC;YACvB,SAAS;YACT,KAAK;YACL,OAAO,EAAE,UAAU;SACpB,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,EAAE,CAAC;QAEvB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO,CAAC,GAAG,CAAC,MAAM,SAAS,KAAK,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;YAC5D,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;YAC1C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAExC,IAAI,MAAiC,CAAC;YACtC,IAAI,YAAY,GAAqB,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;YAEnE,IAAI,CAAC;gBACH,QAAQ;gBACR,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBAE1B,MAAM;gBACN,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;gBAC9C,IAAI,OAAO,EAAE,CAAC;oBACZ,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;oBAC1C,OAAO,CAAC,GAAG,CAAC,OAAO,MAAM,aAAa,MAAM,CAAC,UAAU,OAAO,MAAM,CAAC,SAAS,CAAC,MAAM,aAAa,CAAC,CAAC;gBACtG,CAAC;gBAED,SAAS;gBACT,YAAY,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;gBAC5D,IAAI,OAAO,EAAE,CAAC;oBACZ,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;oBACtD,OAAO,CAAC,GAAG,CAAC,eAAe,OAAO,EAAE,CAAC,CAAC;oBACtC,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,GAAG,CAAC,YAAY,CAAC,mBAAmB,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;wBACtF,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;oBACzD,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7D,IAAI,OAAO;oBAAE,OAAO,CAAC,GAAG,CAAC,cAAc,GAAG,EAAE,CAAC,CAAC;gBAC9C,WAAW,CAAC,IAAI,CAAC;oBACf,MAAM,EAAE,IAAI,CAAC,EAAE;oBACf,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,OAAO,EAAE,KAAK;oBACd,UAAU,EAAE,EAAE;oBACd,SAAS,EAAE,CAAC;oBACZ,OAAO,EAAE;wBACP,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;wBACvD,OAAO,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC;wBACjD,oBAAoB,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC;wBACtD,WAAW,EAAE,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK;qBACrD;oBACD,YAAY,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE;oBAC3C,KAAK,EAAE,GAAG;iBACX,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;oBAAS,CAAC;gBACT,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClB,IAAI,CAAC;wBAAC,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;oBAAC,CAAC;oBAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC;gBAC5E,CAAC;YACH,CAAC;YAED,IAAI,MAAM,EAAE,CAAC;gBACX,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC;YAC5E,CAAC;QACH,CAAC;QAED,MAAM,OAAO,CAAC,QAAQ,EAAE,CAAC;QAEzB,OAAO,gBAAgB,CAAC,WAAW,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;IACzD,CAAC;YAAS,CAAC;QACT,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,IAAI,CAAC;gBAAC,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;AACH,CAAC;AA+BD;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAAC,OAAwB;IACtD,6BAA6B;IAC7B,MAAM,OAAO,GAAG,OAAO;SACpB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACX,SAAS,EAAE,CAAC,CAAC,SAAS;QACtB,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,gBAAgB,GAAG,EAAE,CAAC,GAAG,EAAE;QACnD,QAAQ,EAAE,GAAG,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,UAAU,EAAE;QAC5C,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,GAAG,KAAK,CAAC,GAAG,KAAK;QACxD,eAAe,EAAE,CAAC,CAAC,eAAe;KACnC,CAAC,CAAC;SACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC;IAE7C,sBAAsB;IACtB,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,KAAK,MAAM,EAAE,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/B,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;YAC1B,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAED,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QAC3D,MAAM,MAAM,GAA2B,EAAE,CAAC;QAC1C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,EAAE,GAAG,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;YAC1D,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,SAAS,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACpE,CAAC;QACD,OAAO;YACL,MAAM;YACN,KAAK,EAAE,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,MAAM;YACvC,MAAM;SACP,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,OAAO;QACP,OAAO;QACP,cAAc;KACf,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,oBAAoB;AACpB,8EAA8E;AAE9E;;;;GAIG;AACH,SAAS,eAAe,CAAC,UAAkC;IACzD,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC3B,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAE5B,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;IACxB,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;IACpD,OAAO,CAAC,GAAG,CAAC,KAAK,UAAU,CAAC,SAAS,EAAE,CAAC,CAAC;IACzC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAEjB,gBAAgB;IAChB,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IAC3B,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClB,OAAO,CAAC,GAAG,CAAC,iEAAiE,CAAC,CAAC;IAC/E,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClB,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAClC,OAAO,CAAC,GAAG,CACT,KAAK,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI;YAClC,GAAG,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG;YAC5B,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK;YACvC,GAAG,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK;YAC9B,GAAG,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK;YAC7C,GAAG,CAAC,CAAC,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CACxD,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAElB,qBAAqB;IACrB,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAElB,MAAM,UAAU,GAAG,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAC9D,MAAM,MAAM,GAAG,eAAe,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAChF,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACpB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAElB,KAAK,MAAM,EAAE,IAAI,UAAU,CAAC,cAAc,EAAE,CAAC;QAC3C,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YAClC,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC5B,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACZ,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,MAAM,EAAE,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAElB,SAAS;IACT,IAAI,UAAU,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,CAAC,SAAS,KAAK,MAAM,CAAC,KAAK,OAAO,MAAM,CAAC,SAAS,MAAM,CAAC,CAAC;IAChG,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,8EAA8E;AAC9E,OAAO;AACP,8EAA8E;AAE9E;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,OAAiB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAE/B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzD,OAAO,CAAC,GAAG,CAAC,YAAY,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC5F,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAE5B,eAAe;IACf,IAAI,KAAK,GAAG,QAAQ,CAAC;IACrB,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACtC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,KAAK,CAAC,wCAAwC,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/F,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,eAAe,KAAK,CAAC,MAAM,iBAAiB,MAAM,CAAC,QAAQ,CAAC,MAAM,gBAAgB,CAAC,CAAC;IAEhG,qBAAqB;IACrB,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,GAAG,MAAM,wBAAwB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAEnF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAC;QAC7D,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,CAAC,KAAK,CAAC,sDAAsD,CAAC,CAAC;QACtE,OAAO,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAC5C,OAAO,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;QACzC,OAAO,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC3C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,oBAAoB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAExD,iBAAiB;IACjB,MAAM,OAAO,GAAoB,EAAE,CAAC;IAEpC,KAAK,MAAM,SAAS,IAAI,SAAS,EAAE,CAAC;QAClC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,cAAc,CAAC,SAAS,CAAC,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,mBAAmB,SAAS,KAAK,KAAK,GAAG,CAAC,CAAC;QACvD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAE5B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,gBAAgB,CACnC,SAAS,EACT,KAAK,EACL,KAAK,EACL,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,YAAY,CACpB,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,OAAO,CAAC,GAAG,CAAC,OAAO,SAAS,KAAK,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,UAAU,UAAU,CAAC,CAAC;QACjI,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,OAAO,CAAC,KAAK,CAAC,OAAO,SAAS,YAAY,GAAG,EAAE,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC;QAC7D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,mBAAmB;IACnB,MAAM,UAAU,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAE5C,QAAQ;IACR,eAAe,CAAC,UAAU,CAAC,CAAC;IAE5B,OAAO;IACP,SAAS,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACjD,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9E,MAAM,QAAQ,GAAG,iBAAiB,SAAS,OAAO,CAAC;IACnD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IACpD,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/D,OAAO,CAAC,GAAG,CAAC,mBAAmB,UAAU,EAAE,CAAC,CAAC;IAE7C,uCAAuC;IACvC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,SAAS,IAAI,SAAS,OAAO,CAAC;QAC7D,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACpF,CAAC;AACH,CAAC;AAED,2BAA2B;AAC3B,MAAM,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,aAAa,CAAC,CAAC;AACnE,IAAI,iBAAiB,EAAE,CAAC;IACtB,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACnB,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAC7B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/backend/backend/src/services/agent/crewly-agent/eval/run-eval-standalone.d.ts

@@ -0,0 +1,42 @@
+#!/usr/bin/env npx tsx
+/**
+ * Crewly Eval Standalone Runner
+ *
+ * Runs eval tasks directly against the backend's POST /api/eval/run endpoint.
+ * Unlike run-eval-cli.ts which uses /api/chat/eval, this uses the dedicated
+ * eval endpoint that creates a temporary AgentRunnerService per task.
+ *
+ * If the /api/eval/run endpoint is not available (old backend), falls back
+ * to a simulated scoring based on task setup + verify against manual execution.
+ *
+ * Usage:
+ *   npx tsx backend/src/services/agent/crewly-agent/eval/run-eval-standalone.ts
+ *   npx tsx ... --level=L1              # only L1 tasks
+ *   npx tsx ... --level=L4              # only L4 tasks
+ *   npx tsx ... --tasks=L1-01,L2-03     # specific tasks
+ *   npx tsx ... --runtime=crewly-agent  # label for output
+ *
+ * @module eval/run-eval-standalone
+ */
+import { type EvalTask, type EvalVerification } from './eval-types.js';
+/**
+ * Compute per-dimension scores from verification results.
+ *
+ * D2 (Code Quality): For coding tasks, scores based on code-related
+ * verification checks. For collaboration-only tasks, gives a neutral score.
+ *
+ * D6 (Stability): Starts at 100, deducts for actual instability signals
+ * (tool errors, timeout, near-timeout) instead of using a pass/fail binary.
+ *
+ * @param verification - verification result from task.verify()
+ * @param toolCalls    - tool calls recorded during the run
+ * @param task         - the eval task definition
+ * @param durationMs   - wall-clock duration of the task run
+ * @returns per-dimension scores (0-100)
+ */
+export declare function computeDimensions(verification: EvalVerification, toolCalls: Array<{
+    toolName: string;
+    args?: unknown;
+    result?: unknown;
+}>, task: EvalTask, durationMs: number): Record<string, number>;
+//# sourceMappingURL=run-eval-standalone.d.ts.map

package/dist/backend/backend/src/services/agent/crewly-agent/eval/run-eval-standalone.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-eval-standalone.d.ts","sourceRoot":"","sources":["../../../../../../../../backend/src/services/agent/crewly-agent/eval/run-eval-standalone.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;;;;GAkBG;AAaH,OAAO,EAKL,KAAK,QAAQ,EACb,KAAK,gBAAgB,EACtB,MAAM,iBAAiB,CAAC;AAyGzB;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,iBAAiB,CAC/B,YAAY,EAAE,gBAAgB,EAC9B,SAAS,EAAE,KAAK,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC,EACxE,IAAI,EAAE,QAAQ,EACd,UAAU,EAAE,MAAM,GACjB,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAqFxB"}