crewly 1.5.18 → 1.5.20

package/dist/backend/backend/src/services/agent/crewly-agent/eval/adapters/cli-adapter.js

@@ -0,0 +1,406 @@
+/**
+ * CLI Adapter for the Evaluation Framework
+ *
+ * Generic adapter that spawns CLI-based agent runtimes (Claude Code,
+ * Codex CLI, Gemini CLI) as child processes. Each runtime receives
+ * the same CONTEXT.md, skill scripts, and fixtures — no Crewly SDK
+ * required.
+ *
+ * Tool call tracking is done via the `.crewly/.tool-log.jsonl` file
+ * written by the eval skill shims, plus filesystem diffing to infer
+ * file read/write operations.
+ *
+ * @module eval/adapters/cli-adapter
+ */
+import { execFile } from 'node:child_process';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { parseToolLog, mergeToolCallRecords } from '../tool-log-parser.js';
+/**
+ * Default CLI configurations for each supported runtime.
+ */
+export const CLI_DEFAULTS = {
+    'claude-code': {
+        cliCommand: 'claude',
+        cliFlags: ['--print', '--dangerously-skip-permissions'],
+        printMode: true,
+    },
+    'codex-cli': {
+        cliCommand: 'codex',
+        // `codex exec` is the non-interactive subcommand
+        // --full-auto = auto-approve all tool calls with sandboxed execution
+        // --skip-git-repo-check = eval workspaces aren't git repos
+        cliFlags: ['exec', '--full-auto', '--skip-git-repo-check'],
+        printMode: true,
+    },
+    'gemini-cli': {
+        cliCommand: 'gemini',
+        // -y = auto-approve all tool calls (YOLO mode)
+        // prompt is passed via -p flag (non-interactive/headless mode)
+        cliFlags: ['-y'],
+        printMode: true,
+    },
+};
+/**
+ * Take a snapshot of all files in a directory (non-recursive into node_modules).
+ *
+ * @param dir - directory to snapshot
+ * @returns file snapshot
+ */
+function takeSnapshot(dir) {
+    const files = new Map();
+    walkDir(dir, dir, files);
+    return { files };
+}
+/**
+ * Recursively walk a directory and record file metadata.
+ *
+ * @param baseDir  - root directory for relative paths
+ * @param dir      - current directory being walked
+ * @param files    - map to populate
+ */
+function walkDir(baseDir, dir, files) {
+    let entries;
+    try {
+        entries = fs.readdirSync(dir, { withFileTypes: true });
+    }
+    catch {
+        return;
+    }
+    for (const entry of entries) {
+        if (entry.name === 'node_modules' || entry.name === '.git')
+            continue;
+        const fullPath = path.join(dir, entry.name);
+        if (entry.isDirectory()) {
+            walkDir(baseDir, fullPath, files);
+        }
+        else if (entry.isFile()) {
+            try {
+                const stat = fs.statSync(fullPath);
+                const rel = path.relative(baseDir, fullPath);
+                files.set(rel, `${stat.mtimeMs}:${stat.size}`);
+            }
+            catch {
+                // Skip inaccessible files
+            }
+        }
+    }
+}
+/**
+ * Diff two file snapshots to produce inferred tool call records.
+ *
+ * New files → write_file tool calls.
+ * Modified files → edit_file tool calls.
+ * Deleted files → delete_file tool calls (rare but possible).
+ *
+ * @param before  - snapshot taken before the CLI ran
+ * @param after   - snapshot taken after the CLI ran
+ * @param workDir - absolute path for reconstructing file paths
+ * @returns inferred tool call records
+ */
+function diffSnapshots(before, after, workDir) {
+    const records = [];
+    // New or modified files
+    for (const [relPath, hash] of after.files) {
+        const beforeHash = before.files.get(relPath);
+        if (!beforeHash) {
+            records.push({
+                toolName: 'write_file',
+                args: { path: path.join(workDir, relPath) },
+                result: { inferred: true },
+            });
+        }
+        else if (beforeHash !== hash) {
+            records.push({
+                toolName: 'edit_file',
+                args: { path: path.join(workDir, relPath) },
+                result: { inferred: true },
+            });
+        }
+    }
+    // Deleted files
+    for (const relPath of before.files.keys()) {
+        if (!after.files.has(relPath)) {
+            records.push({
+                toolName: 'delete_file',
+                args: { path: path.join(workDir, relPath) },
+                result: { inferred: true },
+            });
+        }
+    }
+    return records;
+}
+/**
+ * Spawn a CLI process and capture its output.
+ *
+ * @param command   - CLI command to run
+ * @param args      - command arguments
+ * @param cwd       - working directory
+ * @param timeoutMs - timeout in milliseconds
+ * @returns process result with stdout, stderr, exit code, and timing
+ */
+function runCLIProcess(command, args, cwd, timeoutMs) {
+    return new Promise((resolve) => {
+        const startMs = Date.now();
+        let timedOut = false;
+        const proc = execFile(command, args, {
+            cwd,
+            timeout: timeoutMs,
+            maxBuffer: 10 * 1024 * 1024, // 10MB
+            env: { ...process.env, TERM: 'dumb', NO_COLOR: '1' },
+        }, (error, stdout, stderr) => {
+            const durationMs = Date.now() - startMs;
+            if (error && 'killed' in error && error.killed) {
+                timedOut = true;
+            }
+            // Detect timeout via duration even when the process exits cleanly
+            // after receiving SIGTERM (e.g. Codex CLI exits 0 on signal).
+            if (!timedOut && durationMs >= timeoutMs * 0.95) {
+                timedOut = true;
+            }
+            resolve({
+                stdout: stdout ?? '',
+                stderr: stderr ?? '',
+                exitCode: error ? error.code === 'ETIMEDOUT' ? null : 1 : 0,
+                timedOut,
+                durationMs,
+            });
+        });
+        // Close stdin immediately so CLIs that read from stdin (e.g. Codex CLI)
+        // receive EOF and proceed with the prompt argument instead of hanging.
+        proc.stdin?.end();
+        // Safety: ensure process doesn't outlive timeout
+        const killTimer = setTimeout(() => {
+            timedOut = true;
+            proc.kill('SIGTERM');
+            setTimeout(() => proc.kill('SIGKILL'), 5000);
+        }, timeoutMs + 1000);
+        proc.on('exit', () => clearTimeout(killTimer));
+    });
+}
+// ---------------------------------------------------------------------------
+// Adapter
+// ---------------------------------------------------------------------------
+/**
+ * RuntimeAdapter implementation for CLI-based agent runtimes.
+ *
+ * Supports Claude Code (`claude --print`), Codex CLI (`codex --quiet`),
+ * and Gemini CLI (`gemini --sandbox=none`).
+ *
+ * Tool call tracking combines:
+ * 1. Skill log (`.crewly/.tool-log.jsonl`) — written by bash skill shims
+ * 2. Filesystem diff — infers read/write/edit operations from file changes
+ *
+ * @example
+ * ```typescript
+ * const adapter = new CLIAdapter();
+ * await adapter.initialize({
+ *   runtimeId: 'claude-code',
+ *   model: 'claude-sonnet-4',
+ *   workDir: '/tmp/eval',
+ * });
+ * const result = await adapter.runTask(task, '/tmp/eval/task-dir');
+ * await adapter.shutdown();
+ * ```
+ */
+export class CLIAdapter {
+    config = null;
+    runtimeId = null;
+    /**
+     * Initialize the adapter with CLI-specific configuration.
+     *
+     * Merges the base AdapterConfig with CLI defaults for the specified runtime.
+     * Verifies the CLI command is accessible.
+     *
+     * @param config - adapter configuration
+     */
+    async initialize(config) {
+        const defaults = CLI_DEFAULTS[config.runtimeId];
+        if (!defaults) {
+            throw new Error(`No CLI defaults for runtime "${config.runtimeId}". ` +
+                `Supported: ${Object.keys(CLI_DEFAULTS).join(', ')}`);
+        }
+        this.runtimeId = config.runtimeId;
+        this.config = {
+            ...config,
+            ...defaults,
+            // Allow options override
+            cliCommand: config.options?.cliCommand ?? defaults.cliCommand,
+            cliFlags: config.options?.cliFlags ?? defaults.cliFlags,
+            printMode: config.options?.printMode ?? defaults.printMode,
+        };
+        // Verify CLI is accessible
+        await this.verifyCLI();
+    }
+    /**
+     * Execute a single eval task by spawning the CLI process.
+     *
+     * Steps:
+     * 1. Snapshot the work directory (before)
+     * 2. Build CLI command with task prompt
+     * 3. Spawn process, capture output, enforce timeout
+     * 4. Snapshot the work directory (after)
+     * 5. Parse skill log + diff files → tool call records
+     * 6. Return RuntimeResult
+     *
+     * @param task    - task definition
+     * @param workDir - absolute path to the task's work directory
+     * @returns runtime result
+     */
+    async runTask(task, workDir) {
+        if (!this.config) {
+            throw new Error('Adapter not initialized. Call initialize() first.');
+        }
+        // 1. Pre-snapshot
+        const beforeSnapshot = takeSnapshot(workDir);
+        // 2. Build command args
+        const args = this.buildArgs(task.prompt);
+        // 3. Run CLI process
+        const procResult = await runCLIProcess(this.config.cliCommand, args, workDir, task.timeoutMs);
+        // 4. Post-snapshot
+        const afterSnapshot = takeSnapshot(workDir);
+        // 5. Build tool call records
+        const skillRecords = parseToolLog(workDir);
+        const fsRecords = diffSnapshots(beforeSnapshot, afterSnapshot, workDir);
+        const allToolCalls = mergeToolCallRecords(fsRecords, skillRecords);
+        // 6. Build result
+        return {
+            success: procResult.exitCode === 0 && !procResult.timedOut,
+            output: procResult.stdout + (procResult.stderr ? `\n[stderr]\n${procResult.stderr}` : ''),
+            durationMs: procResult.durationMs,
+            steps: this.estimateSteps(procResult.stdout),
+            usage: this.extractUsage(procResult.stdout, procResult.stderr),
+            toolCalls: allToolCalls,
+            humanNudges: 0,
+            timedOut: procResult.timedOut,
+            loopDetected: this.detectLoop(procResult.stdout),
+            error: procResult.timedOut
+                ? 'Process timed out'
+                : procResult.exitCode !== 0
+                    ? `Process exited with code ${procResult.exitCode}`
+                    : undefined,
+        };
+    }
+    /**
+     * Shut down the adapter. No persistent resources to clean up for CLI adapters.
+     */
+    async shutdown() {
+        this.config = null;
+        this.runtimeId = null;
+    }
+    // ---------------------------------------------------------------------------
+    // Private helpers
+    // ---------------------------------------------------------------------------
+    /**
+     * Verify the CLI command is accessible by running --version or --help.
+     *
+     * @throws Error if the CLI is not found
+     */
+    async verifyCLI() {
+        if (!this.config)
+            return;
+        const result = await runCLIProcess(this.config.cliCommand, ['--version'], process.cwd(), 10_000);
+        if (result.exitCode !== 0 && !result.stdout && !result.stderr) {
+            throw new Error(`CLI "${this.config.cliCommand}" not found or not accessible. ` +
+                `Ensure it is installed and in PATH.`);
+        }
+    }
+    /**
+     * Build CLI arguments for a task prompt.
+     *
+     * Each CLI has a different way to pass the prompt:
+     * - Claude Code: `claude --print "prompt"` (positional arg)
+     * - Codex CLI: `codex exec --approval-mode full-auto "prompt"` (positional after exec)
+     * - Gemini CLI: `gemini -y -p "prompt"` (-p flag for non-interactive)
+     *
+     * @param prompt - task prompt text
+     * @returns array of CLI arguments
+     */
+    buildArgs(prompt) {
+        if (!this.config)
+            return [];
+        const args = [...this.config.cliFlags];
+        switch (this.runtimeId) {
+            case 'gemini-cli':
+                // Gemini uses -p flag for non-interactive (headless) mode
+                args.push('-p', prompt);
+                break;
+            case 'claude-code':
+            case 'codex-cli':
+            default:
+                // Claude Code and Codex accept prompt as positional argument
+                args.push(prompt);
+                break;
+        }
+        return args;
+    }
+    /**
+     * Estimate the number of reasoning steps from CLI output.
+     *
+     * Heuristic: count distinct tool use markers or action blocks.
+     *
+     * @param stdout - CLI stdout
+     * @returns estimated step count
+     */
+    estimateSteps(stdout) {
+        // Common patterns in CLI output that indicate a step
+        const patterns = [
+            /\[tool:/gi, // Claude Code tool markers
+            /⠋|⠙|⠹|⠸|⠼|⠴|⠦|⠧|⠇|⠏/g, // Spinner characters
+            /Running:/gi, // Codex action markers
+            /```/g, // Code blocks (rough proxy)
+        ];
+        let maxMatches = 0;
+        for (const pattern of patterns) {
+            const matches = stdout.match(pattern);
+            if (matches && matches.length > maxMatches) {
+                maxMatches = matches.length;
+            }
+        }
+        // Fallback: estimate from output length (1 step per ~500 chars)
+        return Math.max(1, maxMatches || Math.ceil(stdout.length / 500));
+    }
+    /**
+     * Extract token usage from CLI output.
+     *
+     * CLIs don't always report usage. Return zeros when not available.
+     *
+     * @param stdout - CLI stdout
+     * @param stderr - CLI stderr
+     * @returns token usage estimates
+     */
+    extractUsage(stdout, stderr) {
+        // Try to parse Claude Code's usage line: "Total tokens: X input, Y output"
+        const combined = stdout + stderr;
+        const usageMatch = combined.match(/(\d[\d,]+)\s*input.*?(\d[\d,]+)\s*output/i);
+        if (usageMatch) {
+            return {
+                input: parseInt(usageMatch[1].replace(/,/g, ''), 10),
+                output: parseInt(usageMatch[2].replace(/,/g, ''), 10),
+            };
+        }
+        // Estimate from output length if no explicit usage
+        // Rough: 1 token ≈ 4 chars
+        return {
+            input: Math.ceil(stdout.length / 4),
+            output: Math.ceil(stdout.length / 4),
+        };
+    }
+    /**
+     * Detect whether the CLI output indicates a loop.
+     *
+     * @param stdout - CLI stdout
+     * @returns true if loop-like patterns detected
+     */
+    detectLoop(stdout) {
+        // Look for repeated identical blocks (sign of a loop)
+        const lines = stdout.split('\n');
+        if (lines.length < 20)
+            return false;
+        // Check if the last 10 lines are identical to 10 lines before that
+        const lastBlock = lines.slice(-10).join('\n');
+        const prevBlock = lines.slice(-20, -10).join('\n');
+        return lastBlock === prevBlock && lastBlock.length > 50;
+    }
+}
+//# sourceMappingURL=cli-adapter.js.map

package/dist/backend/backend/src/services/agent/crewly-agent/eval/adapters/cli-adapter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli-adapter.js","sourceRoot":"","sources":["../../../../../../../../../backend/src/services/agent/crewly-agent/eval/adapters/cli-adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AASlC,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAmB3E;;GAEG;AACH,MAAM,CAAC,MAAM,YAAY,GAA0E;IACjG,aAAa,EAAE;QACb,UAAU,EAAE,QAAQ;QACpB,QAAQ,EAAE,CAAC,SAAS,EAAE,gCAAgC,CAAC;QACvD,SAAS,EAAE,IAAI;KAChB;IACD,WAAW,EAAE;QACX,UAAU,EAAE,OAAO;QACnB,iDAAiD;QACjD,qEAAqE;QACrE,2DAA2D;QAC3D,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,uBAAuB,CAAC;QAC1D,SAAS,EAAE,IAAI;KAChB;IACD,YAAY,EAAE;QACZ,UAAU,EAAE,QAAQ;QACpB,+CAA+C;QAC/C,+DAA+D;QAC/D,QAAQ,EAAE,CAAC,IAAI,CAAC;QAChB,SAAS,EAAE,IAAI;KAChB;CACO,CAAC;AAcX;;;;;GAKG;AACH,SAAS,YAAY,CAAC,GAAW;IAC/B,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,OAAO,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;IACzB,OAAO,EAAE,KAAK,EAAE,CAAC;AACnB,CAAC;AAED;;;;;;GAMG;AACH,SAAS,OAAO,CAAC,OAAe,EAAE,GAAW,EAAE,KAA0B;IACvE,IAAI,OAAoB,CAAC;IACzB,IAAI,CAAC;QACH,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IACzD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;IACT,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;YAAE,SAAS;QAErE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5C,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;QACpC,CAAC;aAAM,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBACnC,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;gBAC7C,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YACjD,CAAC;YAAC,MAAM,CAAC;gBACP,0BAA0B;YAC5B,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAS,aAAa,CACpB,MAAoB,EACpB,KAAmB,EACnB,OAAe;IAEf,MAAM,OAAO,GAAqB,EAAE,CAAC;IAErC,wBAAwB;IACxB,KAAK,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAC1C,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC7C,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,OAAO,CAAC,IAAI,CAAC;gBACX,QAAQ,EAAE,YAAY;gBACtB,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE;gBAC3C,MAAM,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE;aAC3B,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;YAC/B,OAAO,CAAC,IAAI,CAAC;gBACX,QAAQ,EAAE,WAAW;gBACrB,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE;gBAC3C,MAAM,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE;aAC3B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,gBAAgB;IAChB,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAC1C,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9B,OAAO,CAAC,IAAI,CAAC;gBACX,QAAQ,EAAE,aAAa;gBACvB,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE;gBAC3C,MAAM,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE;aAC3B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAsBD;;;;;;;;GAQG;AACH,SAAS,aAAa,CACpB,OAAe,EACf,IAAc,EACd,GAAW,EACX,SAAiB;IAEjB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,EAAE,IAAI,EAAE;YACnC,GAAG;YACH,OAAO,EAAE,SAAS;YAClB,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,OAAO;YACpC,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE;SACrD,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;YAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;YAExC,IAAI,KAAK,IAAI,QAAQ,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBAC/C,QAAQ,GAAG,IAAI,CAAC;YAClB,CAAC;YAED,kEAAkE;YAClE,8DAA8D;YAC9D,IAAI,CAAC,QAAQ,IAAI,UAAU,IAAI,SAAS,GAAG,IAAI,EAAE,CAAC;gBAChD,QAAQ,GAAG,IAAI,CAAC;YAClB,CAAC;YAED,OAAO,CAAC;gBACN,MAAM,EAAE,MAAM,IAAI,EAAE;gBACpB,MAAM,EAAE,MAAM,IAAI,EAAE;gBACpB,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAE,KAA+B,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACtF,QAAQ;gBACR,UAAU;aACX,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,wEAAwE;QACxE,uEAAuE;QACvE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC;QAElB,iDAAiD;QACjD,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;YAChC,QAAQ,GAAG,IAAI,CAAC;YAChB,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACrB,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,CAAC,CAAC;QAC/C,CAAC,EAAE,SAAS,GAAG,IAAI,CAAC,CAAC;QAErB,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,OAAO,UAAU;IACb,MAAM,GAA4B,IAAI,CAAC;IACvC,SAAS,GAAqB,IAAI,CAAC;IAE3C;;;;;;;OAOG;IACH,KAAK,CAAC,UAAU,CAAC,MAAqB;QACpC,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAChD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CACb,gCAAgC,MAAM,CAAC,SAAS,KAAK;gBACrD,cAAc,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACrD,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QAClC,IAAI,CAAC,MAAM,GAAG;YACZ,GAAG,MAAM;YACT,GAAG,QAAQ;YACX,yBAAyB;YACzB,UAAU,EAAG,MAAM,CAAC,OAAO,EAAE,UAAqB,IAAI,QAAQ,CAAC,UAAU;YACzE,QAAQ,EAAG,MAAM,CAAC,OAAO,EAAE,QAAqB,IAAI,QAAQ,CAAC,QAAQ;YACrE,SAAS,EAAG,MAAM,CAAC,OAAO,EAAE,SAAqB,IAAI,QAAQ,CAAC,SAAS;SACxE,CAAC;QAEF,2BAA2B;QAC3B,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;IACzB,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,KAAK,CAAC,OAAO,CAAC,IAAc,EAAE,OAAe;QAC3C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;QACvE,CAAC;QAED,kBAAkB;QAClB,MAAM,cAAc,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;QAE7C,wBAAwB;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEzC,qBAAqB;QACrB,MAAM,UAAU,GAAG,MAAM,aAAa,CACpC,IAAI,CAAC,MAAM,CAAC,UAAU,EACtB,IAAI,EACJ,OAAO,EACP,IAAI,CAAC,SAAS,CACf,CAAC;QAEF,mBAAmB;QACnB,MAAM,aAAa,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;QAE5C,6BAA6B;QAC7B,MAAM,YAAY,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,aAAa,CAAC,cAAc,EAAE,aAAa,EAAE,OAAO,CAAC,CAAC;QACxE,MAAM,YAAY,GAAG,oBAAoB,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QAEnE,kBAAkB;QAClB,OAAO;YACL,OAAO,EAAE,UAAU,CAAC,QAAQ,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ;YAC1D,MAAM,EAAE,UAAU,CAAC,MAAM,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,eAAe,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzF,UAAU,EAAE,UAAU,CAAC,UAAU;YACjC,KAAK,EAAE,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC;YAC5C,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC;YAC9D,SAAS,EAAE,YAAY;YACvB,WAAW,EAAE,CAAC;YACd,QAAQ,EAAE,UAAU,CAAC,QAAQ;YAC7B,YAAY,EAAE,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC;YAChD,KAAK,EAAE,UAAU,CAAC,QAAQ;gBACxB,CAAC,CAAC,mBAAmB;gBACrB,CAAC,CAAC,UAAU,CAAC,QAAQ,KAAK,CAAC;oBACzB,CAAC,CAAC,4BAA4B,UAAU,CAAC,QAAQ,EAAE;oBACnD,CAAC,CAAC,SAAS;SAChB,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ;QACZ,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,8EAA8E;IAC9E,kBAAkB;IAClB,8EAA8E;IAE9E;;;;OAIG;IACK,KAAK,CAAC,SAAS;QACrB,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,OAAO;QAEzB,MAAM,MAAM,GAAG,MAAM,aAAa,CAChC,IAAI,CAAC,MAAM,CAAC,UAAU,EACtB,CAAC,WAAW,CAAC,EACb,OAAO,CAAC,GAAG,EAAE,EACb,MAAM,CACP,CAAC;QAEF,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CACb,QAAQ,IAAI,CAAC,MAAM,CAAC,UAAU,iCAAiC;gBAC/D,qCAAqC,CACtC,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;;;;;;;OAUG;IACK,SAAS,CAAC,MAAc;QAC9B,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,OAAO,EAAE,CAAC;QAE5B,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAEvC,QAAQ,IAAI,CAAC,SAAS,EAAE,CAAC;YACvB,KAAK,YAAY;gBACf,0DAA0D;gBAC1D,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;gBACxB,MAAM;YACR,KAAK,aAAa,CAAC;YACnB,KAAK,WAAW,CAAC;YACjB;gBACE,6DAA6D;gBAC7D,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAClB,MAAM;QACV,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;;;OAOG;IACK,aAAa,CAAC,MAAc;QAClC,qDAAqD;QACrD,MAAM,QAAQ,GAAG;YACf,WAAW,EAAY,2BAA2B;YAClD,sBAAsB,EAAE,qBAAqB;YAC7C,YAAY,EAAW,uBAAuB;YAC9C,MAAM,EAAiB,4BAA4B;SACpD,CAAC;QAEF,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACtC,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,UAAU,EAAE,CAAC;gBAC3C,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;YAC9B,CAAC;QACH,CAAC;QAED,gEAAgE;QAChE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC;IACnE,CAAC;IAED;;;;;;;;OAQG;IACK,YAAY,CAAC,MAAc,EAAE,MAAc;QACjD,2EAA2E;QAC3E,MAAM,QAAQ,GAAG,MAAM,GAAG,MAAM,CAAC;QACjC,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC/E,IAAI,UAAU,EAAE,CAAC;YACf,OAAO;gBACL,KAAK,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;gBACpD,MAAM,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;aACtD,CAAC;QACJ,CAAC;QAED,mDAAmD;QACnD,2BAA2B;QAC3B,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;YACnC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;SACrC,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IACK,UAAU,CAAC,MAAc;QAC/B,sDAAsD;QACtD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,KAAK,CAAC,MAAM,GAAG,EAAE;YAAE,OAAO,KAAK,CAAC;QAEpC,mEAAmE;QACnE,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnD,OAAO,SAAS,KAAK,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,EAAE,CAAC;IAC1D,CAAC;CACF"}

package/dist/backend/backend/src/services/agent/crewly-agent/eval/adapters/crewly-agent-adapter.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Crewly Agent Adapter for the Evaluation Framework
+ *
+ * Wraps the {@link AgentRunnerService} to conform to the
+ * {@link RuntimeAdapter} interface so that the crewly-agent runtime
+ * can be benchmarked by the eval runner.
+ *
+ * @module crewly-agent-adapter
+ */
+import type { AdapterConfig, EvalTask, RuntimeAdapter, RuntimeResult } from '../eval-types.js';
+/**
+ * Maps Crewly runtime tool names to the canonical names expected by the eval scorer.
+ *
+ * The eval task definitions use standard names (e.g. `delegate-task`, `send-message`)
+ * but the Crewly runtime registers tools under different names. This map bridges
+ * the gap so that tool-accuracy scoring works correctly.
+ */
+export declare const TOOL_ALIAS_MAP: Readonly<Record<string, string>>;
+/**
+ * Normalize a tool name using {@link TOOL_ALIAS_MAP}.
+ *
+ * If the tool name has a known alias, the canonical name is returned.
+ * Otherwise the original name is returned unchanged.
+ *
+ * @param toolName - raw tool name from the runtime
+ * @returns canonical tool name for eval scoring
+ */
+export declare function normalizeToolName(toolName: string): string;
+/**
+ * RuntimeAdapter implementation for the crewly-agent runtime.
+ *
+ * Creates an {@link AgentRunnerService} during initialization and
+ * delegates each eval task to it.
+ *
+ * @example
+ * ```typescript
+ * const adapter = new CrewlyAgentAdapter();
+ * await adapter.initialize({ runtimeId: 'crewly-agent', model: 'claude-opus-4', workDir: '/tmp' });
+ * const result = await adapter.runTask(task, '/tmp/task-dir');
+ * await adapter.shutdown();
+ * ```
+ */
+export declare class CrewlyAgentAdapter implements RuntimeAdapter {
+    private runner;
+    private config;
+    /**
+     * Initialize the adapter by creating and setting up an AgentRunnerService.
+     *
+     * @param config - adapter configuration with model, workDir, etc.
+     */
+    initialize(config: AdapterConfig): Promise<void>;
+    /**
+     * Execute a single eval task using the crewly-agent runtime.
+     *
+     * Maps the {@link AgentRunResult} from the runner into the generic
+     * {@link RuntimeResult} expected by the eval framework.
+     *
+     * @param task    - task definition with prompt and expected tools
+     * @param workDir - absolute path to the task's work directory
+     * @returns runtime result with timing, tool calls, and cost info
+     */
+    runTask(task: EvalTask, workDir: string): Promise<RuntimeResult>;
+    /**
+     * Shut down the adapter and release resources.
+     */
+    shutdown(): Promise<void>;
+}
+//# sourceMappingURL=crewly-agent-adapter.d.ts.map

package/dist/backend/backend/src/services/agent/crewly-agent/eval/adapters/crewly-agent-adapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"crewly-agent-adapter.d.ts","sourceRoot":"","sources":["../../../../../../../../../backend/src/services/agent/crewly-agent/eval/adapters/crewly-agent-adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,OAAO,KAAK,EACV,aAAa,EACb,QAAQ,EACR,cAAc,EACd,aAAa,EACd,MAAM,kBAAkB,CAAC;AAO1B;;;;;;GAMG;AACH,eAAO,MAAM,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAMlD,CAAC;AAEX;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAE1D;AAkED;;;;;;;;;;;;;GAaG;AACH,qBAAa,kBAAmB,YAAW,cAAc;IACvD,OAAO,CAAC,MAAM,CAAmC;IACjD,OAAO,CAAC,MAAM,CAA8B;IAE5C;;;;OAIG;IACG,UAAU,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAuBtD;;;;;;;;;OASG;IACG,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;IA0DtE;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAIhC"}