crewly 1.5.18 → 1.5.19

package/dist/backend/backend/src/services/agent/crewly-agent/eval/run-eval-cli.js

@@ -0,0 +1,349 @@
+#!/usr/bin/env npx tsx
+/**
+ * Crewly Eval CLI — Quick Runner
+ *
+ * Runs L1 eval tasks against the Crewly Agent runtime via the running backend.
+ * Uses the backend's chat API to send prompts and collect results.
+ *
+ * Usage:
+ *   npx tsx backend/src/services/agent/crewly-agent/eval/run-eval-cli.ts
+ *   npx tsx backend/src/services/agent/crewly-agent/eval/run-eval-cli.ts --tasks L1-01,L1-02
+ *
+ * @module eval/run-eval-cli
+ */
+import { mkdir, writeFile, readFile, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { randomUUID } from 'node:crypto';
+import { execSync } from 'node:child_process';
+// ─── Configuration ────────────────────────────────────────────────────────
+const BACKEND_URL = process.env.BACKEND_URL || 'http://localhost:8787';
+const EVAL_SESSION = `eval-agent-${Date.now()}`;
+const OUTPUT_DIR = join(process.cwd(), 'eval-results');
+// ─── Fixture Setup ────────────────────────────────────────────────────────
+const FIXTURE_PKG = JSON.stringify({
+    name: 'eval-base-project',
+    version: '1.0.0',
+    type: 'module',
+    scripts: { build: 'tsc', test: 'echo "no tests"' },
+}, null, 2);
+const FIXTURE_INDEX = `/**
+ * Calculate the sum of numbers.
+ * @param numbers - array of numbers
+ * @returns total sum
+ */
+export function calculateTotal(numbers: number[]): number {
+  return numbers.reduce((sum, n) => sum + n, 0);
+}
+`;
+const FIXTURE_API = `/**
+ * Simple API service.
+ */
+export class ApiService {
+  private baseUrl: string;
+  constructor(baseUrl: string) { this.baseUrl = baseUrl; }
+  async get<T>(path: string): Promise<T> {
+    const res = await fetch(\`\${this.baseUrl}\${path}\`);
+    return res.json() as Promise<T>;
+  }
+}
+`;
+const FIXTURE_HELPERS = `/**
+ * Format a date as YYYY-MM-DD.
+ * @param date - input date
+ * @returns formatted string
+ */
+export function formatDate(date: Date): string {
+  return date.toISOString().slice(0, 10);
+}
+`;
+/**
+ * Create the base-project fixture in the given directory.
+ */
+async function createFixture(workDir) {
+    await mkdir(join(workDir, 'src', 'services'), { recursive: true });
+    await mkdir(join(workDir, 'src', 'utils'), { recursive: true });
+    await writeFile(join(workDir, 'package.json'), FIXTURE_PKG);
+    await writeFile(join(workDir, 'src', 'index.ts'), FIXTURE_INDEX);
+    await writeFile(join(workDir, 'src', 'services', 'api.service.ts'), FIXTURE_API);
+    await writeFile(join(workDir, 'src', 'utils', 'helpers.ts'), FIXTURE_HELPERS);
+}
+// ─── L1 Task Definitions ─────────────────────────────────────────────────
+const L1_TASKS = [
+    {
+        id: 'L1-01',
+        title: 'Read package.json, report version',
+        prompt: 'Read the file package.json in this project and tell me the version number.',
+        maxSteps: 3,
+        timeoutMs: 60_000,
+        expectedTools: ['Read'],
+        async setup(workDir) { await createFixture(workDir); },
+        async verify(workDir, output, toolCalls) {
+            const pkg = JSON.parse(await readFile(join(workDir, 'package.json'), 'utf-8'));
+            const hasVersion = output.includes(pkg.version);
+            const usedRead = toolCalls.some(t => t.toLowerCase().includes('read'));
+            return {
+                passed: hasVersion && usedRead,
+                partialScore: (hasVersion ? 0.5 : 0) + (usedRead ? 0.5 : 0),
+                checks: [
+                    { name: 'output-contains-version', passed: hasVersion, detail: hasVersion ? `Found "${pkg.version}"` : 'Version not in output' },
+                    { name: 'used-read-tool', passed: usedRead, detail: usedRead ? 'Used Read tool' : `Tools: ${toolCalls.join(', ')}` },
+                ],
+            };
+        },
+    },
+    {
+        id: 'L1-02',
+        title: 'Create hello.ts with greeting export',
+        prompt: "Create a file called hello.ts in the project root with the content: export const greeting = 'Hello';",
+        maxSteps: 3,
+        timeoutMs: 60_000,
+        expectedTools: ['Write'],
+        async setup(workDir) { await createFixture(workDir); },
+        async verify(workDir, output, toolCalls) {
+            const helloPath = join(workDir, 'hello.ts');
+            let exists = false;
+            let correct = false;
+            try {
+                const content = await readFile(helloPath, 'utf-8');
+                exists = true;
+                correct = content.includes("export const greeting = 'Hello'") || content.includes('export const greeting = "Hello"');
+            }
+            catch { /* */ }
+            const usedWrite = toolCalls.some(t => t.toLowerCase().includes('write'));
+            return {
+                passed: exists && correct,
+                partialScore: (exists ? 0.3 : 0) + (correct ? 0.4 : 0) + (usedWrite ? 0.3 : 0),
+                checks: [
+                    { name: 'file-exists', passed: exists, detail: exists ? 'hello.ts exists' : 'hello.ts not found' },
+                    { name: 'content-correct', passed: correct, detail: correct ? 'Content matches' : 'Content mismatch' },
+                    { name: 'used-write-tool', passed: usedWrite, detail: usedWrite ? 'Used Write' : `Tools: ${toolCalls.join(', ')}` },
+                ],
+            };
+        },
+    },
+    {
+        id: 'L1-03',
+        title: 'Find all .test.ts files in src/',
+        prompt: 'Find all files ending in .test.ts inside the src/ directory. List them.',
+        maxSteps: 3,
+        timeoutMs: 60_000,
+        expectedTools: ['Glob'],
+        async setup(workDir) {
+            await createFixture(workDir);
+            await writeFile(join(workDir, 'src', 'utils', 'helpers.test.ts'), 'test("x", () => {});');
+            await writeFile(join(workDir, 'src', 'index.test.ts'), 'test("y", () => {});');
+        },
+        async verify(_workDir, output, toolCalls) {
+            const found1 = output.includes('helpers.test.ts');
+            const found2 = output.includes('index.test.ts');
+            const usedGlob = toolCalls.some(t => t.toLowerCase().includes('glob'));
+            return {
+                passed: found1 && found2,
+                partialScore: (found1 ? 0.35 : 0) + (found2 ? 0.35 : 0) + (usedGlob ? 0.3 : 0),
+                checks: [
+                    { name: 'found-helpers-test', passed: found1, detail: found1 ? 'Found helpers.test.ts' : 'Missing' },
+                    { name: 'found-index-test', passed: found2, detail: found2 ? 'Found index.test.ts' : 'Missing' },
+                    { name: 'used-glob', passed: usedGlob, detail: usedGlob ? 'Used Glob' : `Tools: ${toolCalls.join(', ')}` },
+                ],
+            };
+        },
+    },
+    {
+        id: 'L1-04',
+        title: 'Search for "export class" in src/',
+        prompt: 'Search for all occurrences of "export class" in the src/ directory. Report which files and line numbers.',
+        maxSteps: 3,
+        timeoutMs: 60_000,
+        expectedTools: ['Grep'],
+        async setup(workDir) {
+            await createFixture(workDir);
+        },
+        async verify(_workDir, output, toolCalls) {
+            const foundFile = output.includes('api.service.ts') || output.includes('ApiService');
+            const usedGrep = toolCalls.some(t => t.toLowerCase().includes('grep'));
+            return {
+                passed: foundFile && usedGrep,
+                partialScore: (foundFile ? 0.5 : 0) + (usedGrep ? 0.5 : 0),
+                checks: [
+                    { name: 'found-api-service', passed: foundFile, detail: foundFile ? 'Found api.service.ts' : 'Not found' },
+                    { name: 'used-grep', passed: usedGrep, detail: usedGrep ? 'Used Grep' : `Tools: ${toolCalls.join(', ')}` },
+                ],
+            };
+        },
+    },
+    {
+        id: 'L1-05',
+        title: 'Run git status',
+        prompt: 'Run git status and show me the output.',
+        maxSteps: 3,
+        timeoutMs: 60_000,
+        expectedTools: ['Bash'],
+        async setup(workDir) {
+            await createFixture(workDir);
+            execSync('git init && git add -A && git commit -m "init"', {
+                cwd: workDir, stdio: 'pipe',
+                env: { ...process.env, GIT_AUTHOR_NAME: 'eval', GIT_AUTHOR_EMAIL: 'e@t.l', GIT_COMMITTER_NAME: 'eval', GIT_COMMITTER_EMAIL: 'e@t.l' },
+            });
+        },
+        async verify(_workDir, output, toolCalls) {
+            const hasStatus = output.includes('branch') || output.includes('clean') || output.includes('nothing to commit');
+            const usedBash = toolCalls.some(t => t.toLowerCase().includes('bash'));
+            return {
+                passed: hasStatus && usedBash,
+                partialScore: (hasStatus ? 0.5 : 0) + (usedBash ? 0.5 : 0),
+                checks: [
+                    { name: 'has-git-output', passed: hasStatus, detail: hasStatus ? 'Output has git status' : 'No git status in output' },
+                    { name: 'used-bash', passed: usedBash, detail: usedBash ? 'Used Bash' : `Tools: ${toolCalls.join(', ')}` },
+                ],
+            };
+        },
+    },
+    {
+        id: 'L1-06',
+        title: 'Change version in package.json',
+        prompt: 'Change the version in package.json from "1.0.0" to "1.1.0".',
+        maxSteps: 5,
+        timeoutMs: 60_000,
+        expectedTools: ['Edit'],
+        async setup(workDir) { await createFixture(workDir); },
+        async verify(workDir, _output, toolCalls) {
+            let correct = false;
+            let validJson = false;
+            try {
+                const content = await readFile(join(workDir, 'package.json'), 'utf-8');
+                const pkg = JSON.parse(content);
+                validJson = true;
+                correct = pkg.version === '1.1.0';
+            }
+            catch { /* */ }
+            const usedEdit = toolCalls.some(t => t.toLowerCase().includes('edit'));
+            return {
+                passed: correct && validJson,
+                partialScore: (validJson ? 0.3 : 0) + (correct ? 0.4 : 0) + (usedEdit ? 0.3 : 0),
+                checks: [
+                    { name: 'json-valid', passed: validJson, detail: validJson ? 'Valid JSON' : 'Invalid JSON' },
+                    { name: 'version-updated', passed: correct, detail: correct ? 'Version is 1.1.0' : 'Version not updated' },
+                    { name: 'used-edit', passed: usedEdit, detail: usedEdit ? 'Used Edit' : `Tools: ${toolCalls.join(', ')}` },
+                ],
+            };
+        },
+    },
+];
+// ─── Backend API Client ───────────────────────────────────────────────────
+/**
+ * Send a prompt to the Crewly Agent via the backend chat API
+ * and collect the response with tool call information.
+ */
+async function runAgentTask(prompt, workDir, timeoutMs) {
+    const startMs = Date.now();
+    // Use the agent's internal API to run a single prompt
+    const response = await fetch(`${BACKEND_URL}/api/chat/eval`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+            prompt,
+            workDir,
+            sessionName: EVAL_SESSION,
+            maxSteps: 10,
+            timeoutMs,
+        }),
+        signal: AbortSignal.timeout(timeoutMs + 10_000),
+    });
+    if (!response.ok) {
+        const error = await response.text();
+        throw new Error(`Backend API error ${response.status}: ${error}`);
+    }
+    const result = await response.json();
+    return {
+        output: result.text || '',
+        toolCalls: (result.toolCalls || []).map((tc) => tc.toolName),
+        durationMs: Date.now() - startMs,
+    };
+}
+// ─── Main ─────────────────────────────────────────────────────────────────
+async function main() {
+    const args = process.argv.slice(2);
+    const taskFilter = args.find(a => a.startsWith('--tasks='))?.split('=')[1]?.split(',');
+    let tasks = L1_TASKS;
+    if (taskFilter) {
+        tasks = tasks.filter(t => taskFilter.includes(t.id));
+    }
+    console.log('═'.repeat(72));
+    console.log('  CREWLY EVAL RUNNER — L1 Tasks');
+    console.log(`  Tasks: ${tasks.length} | Backend: ${BACKEND_URL}`);
+    console.log('═'.repeat(72));
+    const scores = [];
+    for (const task of tasks) {
+        console.log(`\n▸ ${task.id}: ${task.title}`);
+        // Create work directory
+        const workDir = join(tmpdir(), `crewly-eval-${task.id}-${randomUUID().slice(0, 8)}`);
+        await mkdir(workDir, { recursive: true });
+        try {
+            // Setup fixture
+            await task.setup(workDir);
+            console.log(`  Setup: ${workDir}`);
+            // Run task
+            const { output, toolCalls, durationMs } = await runAgentTask(`Working directory: ${workDir}\n\n${task.prompt}`, workDir, task.timeoutMs);
+            console.log(`  Duration: ${durationMs}ms | Tools: ${toolCalls.join(', ') || 'none'}`);
+            // Verify
+            const verification = await task.verify(workDir, output, toolCalls);
+            const status = verification.passed ? '✅ PASS' : '❌ FAIL';
+            console.log(`  Result: ${status} (${(verification.partialScore * 100).toFixed(0)}%)`);
+            for (const check of verification.checks) {
+                console.log(`    ${check.passed ? '✓' : '✗'} ${check.name}: ${check.detail}`);
+            }
+            scores.push({
+                taskId: task.id,
+                passed: verification.passed,
+                partialScore: verification.partialScore,
+                durationMs,
+                checks: verification.checks,
+                output: output.slice(0, 500),
+                toolsUsed: toolCalls,
+            });
+        }
+        catch (error) {
+            const msg = error instanceof Error ? error.message : String(error);
+            console.log(`  ERROR: ${msg}`);
+            scores.push({
+                taskId: task.id,
+                passed: false,
+                partialScore: 0,
+                durationMs: 0,
+                checks: [{ name: 'runner-error', passed: false, detail: msg }],
+                output: '',
+                toolsUsed: [],
+            });
+        }
+        finally {
+            // Cleanup
+            try {
+                await rm(workDir, { recursive: true, force: true });
+            }
+            catch { /* */ }
+        }
+    }
+    // Print summary
+    console.log('\n' + '═'.repeat(72));
+    console.log('  SUMMARY');
+    console.log('═'.repeat(72));
+    const passed = scores.filter(s => s.passed).length;
+    const avgScore = scores.reduce((sum, s) => sum + s.partialScore, 0) / scores.length;
+    console.log(`  Pass: ${passed}/${scores.length} | Avg Score: ${(avgScore * 100).toFixed(1)}%`);
+    for (const score of scores) {
+        const icon = score.passed ? '✅' : '❌';
+        console.log(`  ${icon} ${score.taskId}: ${(score.partialScore * 100).toFixed(0)}% (${score.durationMs}ms)`);
+    }
+    console.log('═'.repeat(72));
+    // Save results
+    await mkdir(OUTPUT_DIR, { recursive: true });
+    const filename = `eval-L1-${new Date().toISOString().slice(0, 19).replace(/:/g, '-')}.json`;
+    await writeFile(join(OUTPUT_DIR, filename), JSON.stringify({ timestamp: new Date().toISOString(), scores, summary: { passed, total: scores.length, avgScore } }, null, 2));
+    console.log(`\nResults saved to ${join(OUTPUT_DIR, filename)}`);
+}
+main().catch((err) => {
+    console.error('Fatal:', err);
+    process.exit(1);
+});
+//# sourceMappingURL=run-eval-cli.js.map

package/dist/backend/backend/src/services/agent/crewly-agent/eval/run-eval-cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-eval-cli.js","sourceRoot":"","sources":["../../../../../../../../backend/src/services/agent/crewly-agent/eval/run-eval-cli.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAiB,MAAM,kBAAkB,CAAC;AAEjF,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAE9C,6EAA6E;AAC7E,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,uBAAuB,CAAC;AACvE,MAAM,YAAY,GAAG,cAAc,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;AAChD,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,CAAC,CAAC;AA8BvD,6EAA6E;AAC7E,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC;IACjC,IAAI,EAAE,mBAAmB;IACzB,OAAO,EAAE,OAAO;IAChB,IAAI,EAAE,QAAQ;IACd,OAAO,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,iBAAiB,EAAE;CACnD,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AAEZ,MAAM,aAAa,GAAG;;;;;;;;CAQrB,CAAC;AAEF,MAAM,WAAW,GAAG;;;;;;;;;;;CAWnB,CAAC;AAEF,MAAM,eAAe,GAAG;;;;;;;;CAQvB,CAAC;AAEF;;GAEG;AACH,KAAK,UAAU,aAAa,CAAC,OAAe;IAC1C,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACnE,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChE,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,EAAE,WAAW,CAAC,CAAC;IAC5D,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,EAAE,aAAa,CAAC,CAAC;IACjE,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,gBAAgB,CAAC,EAAE,WAAW,CAAC,CAAC;IACjF,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,YAAY,CAAC,EAAE,eAAe,CAAC,CAAC;AAChF,CAAC;AAED,4EAA4E;AAC5E,MAAM,QAAQ,GAAkB;IAC9B;QACE,EAAE,EAAE,OAAO;QACX,KAAK,EAAE,mCAAmC;QAC1C,MAAM,EAAE,4EAA4E;QACpF,QAAQ,EAAE,CAAC;QACX,SAAS,EAAE,MAAM;QACjB,aAAa,EAAE,CAAC,MAAM,CAAC;QACvB,KAAK,CAAC,KAAK,CAAC,OAAO,IAAI,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACtD,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS;YACrC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;YAC/E,MAAM,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAChD,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;YACvE,OAAO;gBACL,MAAM,EAAE,UAAU,IAAI,QAAQ;gBAC9B,YAAY,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3D,MAAM,EAAE;oBACN,EAAE,IAAI,EAAE,yBAAyB,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,uBAAuB,EAAE;oBAChI,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,UAAU,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE;iBACrH;aACF,CAAC;QACJ,CAAC;KACF;IACD;QACE,EAAE,EAAE,OAAO;QACX,KAAK,EAAE,sCAAsC;QAC7C,MAAM,EAAE,sGAAsG;QAC9G,QAAQ,EAAE,CAAC;QACX,SAAS,EAAE,MAAM;QACjB,aAAa,EAAE,CAAC,OAAO,CAAC;QACxB,KAAK,CAAC,KAAK,CAAC,OAAO,IAAI,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACtD,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS;YACrC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;YAC5C,IAAI,MAAM,GAAG,KAAK,CAAC;YACnB,IAAI,OAAO,GAAG,KAAK,CAAC;YACpB,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;gBACnD,MAAM,GAAG,IAAI,CAAC;gBACd,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,iCAAiC,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,iCAAiC,CAAC,CAAC;YACvH,CAAC;YAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC;YACjB,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;YACzE,OAAO;gBACL,MAAM,EAAE,MAAM,IAAI,OAAO;gBACzB,YAAY,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC9E,MAAM,EAAE;oBACN,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,oBAAoB,EAAE;oBAClG,EAAE,IAAI,EAAE,iBAAiB,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,kBAAkB,EAAE;oBACtG,EAAE,IAAI,EAAE,iBAAiB,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE;iBACpH;aACF,CAAC;QACJ,CAAC;KACF;IACD;QACE,EAAE,EAAE,OAAO;QACX,KAAK,EAAE,iCAAiC;QACxC,MAAM,EAAE,yEAAyE;QACjF,QAAQ,EAAE,CAAC;QACX,SAAS,EAAE,MAAM;QACjB,aAAa,EAAE,CAAC,MAAM,CAAC;QACvB,KAAK,CAAC,KAAK,CAAC,OAAO;YACjB,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC;YAC7B,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,iBAAiB,CAAC,EAAE,sBAAsB,CAAC,CAAC;YAC1F,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,eAAe,CAAC,EAAE,sBAAsB,CAAC,CAAC;QACjF,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS;YACtC,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;YAClD,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;YAChD,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;YACvE,OAAO;gBACL,MAAM,EAAE,MAAM,IAAI,MAAM;gBACxB,YAAY,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC9E,MAAM,EAAE;oBACN,EAAE,IAAI,EAAE,oBAAoB,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,SAAS,EAAE;oBACpG,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,SAAS,EAAE;oBAChG,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE;iBAC3G;aACF,CAAC;QACJ,CAAC;KACF;IACD;QACE,EAAE,EAAE,OAAO;QACX,KAAK,EAAE,mCAAmC;QAC1C,MAAM,EAAE,0GAA0G;QAClH,QAAQ,EAAE,CAAC;QACX,SAAS,EAAE,MAAM;QACjB,aAAa,EAAE,CAAC,MAAM,CAAC;QACvB,KAAK,CAAC,KAAK,CAAC,OAAO;YACjB,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS;YACtC,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;YACrF,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;YACvE,OAAO;gBACL,MAAM,EAAE,SAAS,IAAI,QAAQ;gBAC7B,YAAY,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1D,MAAM,EAAE;oBACN,EAAE,IAAI,EAAE,mBAAmB,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,WAAW,EAAE;oBAC1G,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE;iBAC3G;aACF,CAAC;QACJ,CAAC;KACF;IACD;QACE,EAAE,EAAE,OAAO;QACX,KAAK,EAAE,gBAAgB;QACvB,MAAM,EAAE,wCAAwC;QAChD,QAAQ,EAAE,CAAC;QACX,SAAS,EAAE,MAAM;QACjB,aAAa,EAAE,CAAC,MAAM,CAAC;QACvB,KAAK,CAAC,KAAK,CAAC,OAAO;YACjB,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC;YAC7B,QAAQ,CAAC,gDAAgD,EAAE;gBACzD,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM;gBAC3B,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,eAAe,EAAE,MAAM,EAAE,gBAAgB,EAAE,OAAO,EAAE,kBAAkB,EAAE,MAAM,EAAE,mBAAmB,EAAE,OAAO,EAAE;aACtI,CAAC,CAAC;QACL,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS;YACtC,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;YAChH,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;YACvE,OAAO;gBACL,MAAM,EAAE,SAAS,IAAI,QAAQ;gBAC7B,YAAY,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1D,MAAM,EAAE;oBACN,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,yBAAyB,EAAE;oBACtH,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE;iBAC3G;aACF,CAAC;QACJ,CAAC;KACF;IACD;QACE,EAAE,EAAE,OAAO;QACX,KAAK,EAAE,gCAAgC;QACvC,MAAM,EAAE,6DAA6D;QACrE,QAAQ,EAAE,CAAC;QACX,SAAS,EAAE,MAAM;QACjB,aAAa,EAAE,CAAC,MAAM,CAAC;QACvB,KAAK,CAAC,KAAK,CAAC,OAAO,IAAI,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACtD,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS;YACtC,IAAI,OAAO,GAAG,KAAK,CAAC;YACpB,IAAI,SAAS,GAAG,KAAK,CAAC;YACtB,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,EAAE,OAAO,CAAC,CAAC;gBACvE,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBAChC,SAAS,GAAG,IAAI,CAAC;gBACjB,OAAO,GAAG,GAAG,CAAC,OAAO,KAAK,OAAO,CAAC;YACpC,CAAC;YAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC;YACjB,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;YACvE,OAAO;gBACL,MAAM,EAAE,OAAO,IAAI,SAAS;gBAC5B,YAAY,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAChF,MAAM,EAAE;oBACN,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,cAAc,EAAE;oBAC5F,EAAE,IAAI,EAAE,iBAAiB,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,qBAAqB,EAAE;oBAC1G,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE;iBAC3G;aACF,CAAC;QACJ,CAAC;KACF;CACF,CAAC;AAEF,6EAA6E;AAE7E;;;GAGG;AACH,KAAK,UAAU,YAAY,CACzB,MAAc,EACd,OAAe,EACf,SAAiB;IAEjB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE3B,sDAAsD;IACtD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,WAAW,gBAAgB,EAAE;QAC3D,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;QAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,MAAM;YACN,OAAO;YACP,WAAW,EAAE,YAAY;YACzB,QAAQ,EAAE,EAAE;YACZ,SAAS;SACV,CAAC;QACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,GAAG,MAAM,CAAC;KAChD,CAAC,CAAC;IAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,qBAAqB,QAAQ,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAGjC,CAAC;IAEF,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,IAAI,IAAI,EAAE;QACzB,SAAS,EAAE,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,EAAwB,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC;QAClF,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;KACjC,CAAC;AACJ,CAAC;AAED,6EAA6E;AAC7E,KAAK,UAAU,IAAI;IACjB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;IAEvF,IAAI,KAAK,GAAG,QAAQ,CAAC;IACrB,IAAI,UAAU,EAAE,CAAC;QACf,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACvD,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,YAAY,KAAK,CAAC,MAAM,eAAe,WAAW,EAAE,CAAC,CAAC;IAClE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAE5B,MAAM,MAAM,GAAgB,EAAE,CAAC;IAE/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QAE7C,wBAAwB;QACxB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,eAAe,IAAI,CAAC,EAAE,IAAI,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QACrF,MAAM,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE1C,IAAI,CAAC;YACH,gBAAgB;YAChB,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC1B,OAAO,CAAC,GAAG,CAAC,YAAY,OAAO,EAAE,CAAC,CAAC;YAEnC,WAAW;YACX,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,MAAM,YAAY,CAC1D,sBAAsB,OAAO,OAAO,IAAI,CAAC,MAAM,EAAE,EACjD,OAAO,EACP,IAAI,CAAC,SAAS,CACf,CAAC;YACF,OAAO,CAAC,GAAG,CAAC,eAAe,UAAU,eAAe,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,EAAE,CAAC,CAAC;YAEtF,SAAS;YACT,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;YACnE,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;YACzD,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,KAAK,CAAC,YAAY,CAAC,YAAY,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACtF,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;gBACxC,OAAO,CAAC,GAAG,CAAC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YAChF,CAAC;YAED,MAAM,CAAC,IAAI,CAAC;gBACV,MAAM,EAAE,IAAI,CAAC,EAAE;gBACf,MAAM,EAAE,YAAY,CAAC,MAAM;gBAC3B,YAAY,EAAE,YAAY,CAAC,YAAY;gBACvC,UAAU;gBACV,MAAM,EAAE,YAAY,CAAC,MAAM;gBAC3B,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;gBAC5B,SAAS,EAAE,SAAS;aACrB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACnE,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC;gBACV,MAAM,EAAE,IAAI,CAAC,EAAE;gBACf,MAAM,EAAE,KAAK;gBACb,YAAY,EAAE,CAAC;gBACf,UAAU,EAAE,CAAC;gBACb,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;gBAC9D,MAAM,EAAE,EAAE;gBACV,SAAS,EAAE,EAAE;aACd,CAAC,CAAC;QACL,CAAC;gBAAS,CAAC;YACT,UAAU;YACV,IAAI,CAAC;gBAAC,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC;QAC9E,CAAC;IACH,CAAC;IAED,gBAAgB;IAChB,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IACzB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACnD,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IACpF,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,IAAI,MAAM,CAAC,MAAM,iBAAiB,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAE/F,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,YAAY,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,UAAU,KAAK,CAAC,CAAC;IAC9G,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAE5B,eAAe;IACf,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7C,MAAM,QAAQ,GAAG,WAAW,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,OAAO,CAAC;IAC5F,MAAM,SAAS,CACb,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,EAC1B,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAC9H,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;AAClE,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAC7B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}

package/dist/backend/backend/src/services/agent/crewly-agent/eval/run-eval-l4.d.ts

@@ -0,0 +1,97 @@
+#!/usr/bin/env npx tsx
+/**
+ * L4 Cross-Runtime Benchmark Script
+ *
+ * Runs all L4 eval tasks (collaboration + trap/resilience) across multiple
+ * CLI runtimes and produces a comparison report.
+ *
+ * Each runtime receives the same CONTEXT.md, skill shims, and fixtures.
+ * Tool call tracking uses the `.crewly/.tool-log.jsonl` written by skill
+ * shims plus filesystem diffing.
+ *
+ * Usage:
+ *   npx tsx backend/src/services/agent/crewly-agent/eval/run-eval-l4.ts
+ *   npx tsx ... --runtime=claude-code           # single runtime
+ *   npx tsx ... --runtime=all                   # all 4 runtimes
+ *   npx tsx ... --runtime=claude-code,gemini-cli # specific runtimes
+ *   npx tsx ... --tasks=L4-C1,L4-T1             # specific tasks
+ *   npx tsx ... --verbose                        # detailed output
+ *   npx tsx ... --keep-work-dirs                 # preserve temp dirs
+ *   npx tsx ... --model=claude-sonnet-4    # override model name
+ *
+ * @module eval/run-eval-l4
+ */
+import type { EvalRunReport, RuntimeId } from './eval-types.js';
+interface BenchmarkConfig {
+    /** Runtimes to benchmark */
+    runtimes: RuntimeId[];
+    /** Task IDs to run (empty = all L4) */
+    taskIds: string[];
+    /** Model override */
+    model?: string;
+    /** Verbose output */
+    verbose: boolean;
+    /** Keep work directories after run */
+    keepWorkDirs: boolean;
+    /** Output directory */
+    outputDir: string;
+}
+/**
+ * Parse CLI arguments into a BenchmarkConfig.
+ *
+ * @param argv - process.argv.slice(2)
+ * @returns parsed configuration
+ */
+export declare function parseArgs(argv: string[]): BenchmarkConfig;
+/**
+ * Check which runtimes are available on this machine.
+ *
+ * @param runtimes - list of runtimes to check
+ * @returns object with available and unavailable lists
+ */
+export declare function checkRuntimeAvailability(runtimes: RuntimeId[]): Promise<{
+    available: RuntimeId[];
+    unavailable: Array<{
+        id: RuntimeId;
+        reason: string;
+    }>;
+}>;
+/**
+ * Comparison report across multiple runtimes.
+ */
+export interface CrossRuntimeComparison {
+    /** ISO timestamp of benchmark run */
+    timestamp: string;
+    /** Per-runtime reports */
+    reports: EvalRunReport[];
+    /** Ranking by composite score (descending) */
+    ranking: Array<{
+        runtimeId: RuntimeId;
+        model: string;
+        composite: number;
+        passRate: string;
+        totalCostUSD: number;
+        totalDurationMs: number;
+    }>;
+    /** Per-task comparison across runtimes */
+    taskComparison: Array<{
+        taskId: string;
+        title: string;
+        scores: Record<string, number>;
+    }>;
+}
+/**
+ * Build a cross-runtime comparison from individual reports.
+ *
+ * @param reports - array of per-runtime reports
+ * @returns structured comparison
+ */
+export declare function buildComparison(reports: EvalRunReport[]): CrossRuntimeComparison;
+/**
+ * Main entry point for the benchmark script.
+ *
+ * @param argv - command line arguments (process.argv.slice(2))
+ */
+export declare function main(argv?: string[]): Promise<void>;
+export {};
+//# sourceMappingURL=run-eval-l4.d.ts.map

package/dist/backend/backend/src/services/agent/crewly-agent/eval/run-eval-l4.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-eval-l4.d.ts","sourceRoot":"","sources":["../../../../../../../../backend/src/services/agent/crewly-agent/eval/run-eval-l4.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;;;;;;;GAqBG;AASH,OAAO,KAAK,EACV,aAAa,EAGb,SAAS,EAEV,MAAM,iBAAiB,CAAC;AA0BzB,UAAU,eAAe;IACvB,4BAA4B;IAC5B,QAAQ,EAAE,SAAS,EAAE,CAAC;IACtB,uCAAuC;IACvC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,qBAAqB;IACrB,OAAO,EAAE,OAAO,CAAC;IACjB,sCAAsC;IACtC,YAAY,EAAE,OAAO,CAAC;IACtB,uBAAuB;IACvB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,eAAe,CAiCzD;AAMD;;;;;GAKG;AACH,wBAAsB,wBAAwB,CAC5C,QAAQ,EAAE,SAAS,EAAE,GACpB,OAAO,CAAC;IAAE,SAAS,EAAE,SAAS,EAAE,CAAC;IAAC,WAAW,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,SAAS,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CAAE,CAAC,CAiC5F;AAiHD;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,qCAAqC;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,0BAA0B;IAC1B,OAAO,EAAE,aAAa,EAAE,CAAC;IACzB,8CAA8C;IAC9C,OAAO,EAAE,KAAK,CAAC;QACb,SAAS,EAAE,SAAS,CAAC;QACrB,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,EAAE,MAAM,CAAC;QAClB,QAAQ,EAAE,MAAM,CAAC;QACjB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC,CAAC;IACH,0CAA0C;IAC1C,cAAc,EAAE,KAAK,CAAC;QACpB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAChC,CAAC,CAAC;CACJ;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,aAAa,EAAE,GAAG,sBAAsB,CA0ChF;AAoED;;;;GAIG;AACH,wBAAsB,IAAI,CAAC,IAAI,GAAE,MAAM,EAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CA4FhF"}