crewly 1.5.18 → 1.5.20

package/dist/backend/backend/src/services/agent/crewly-agent/eval/tasks/l1-tasks.js

@@ -0,0 +1,396 @@
+/**
+ * L1 Evaluation Tasks - Basic Tool Mastery
+ *
+ * Six tasks that test an agent's ability to use individual tools
+ * correctly: read, write, glob, grep, bash, and edit.
+ *
+ * Tool name matching is case-insensitive and uses substring matching
+ * to support different runtimes (e.g. "Read" vs "read_file").
+ *
+ * @module eval/tasks/l1-tasks
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { DEFAULT_TASK_TIMEOUT_MS, MAX_TASK_STEPS } from '../eval-types.js';
+const __filename_esm = fileURLToPath(import.meta.url);
+const __dirname_esm = path.dirname(__filename_esm);
+/** Absolute path to the base project fixture used by L1 tasks */
+const FIXTURE_DIR = path.resolve(__dirname_esm, '..', 'fixtures', 'base-project');
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/**
+ * Build an {@link EvalVerification} from a list of checks.
+ * The verification passes only if every check passes.
+ *
+ * @param checks - individual check results
+ * @returns assembled verification
+ */
+function buildVerification(checks) {
+    return {
+        passed: checks.every((c) => c.passed),
+        checks,
+    };
+}
+/**
+ * Copy the fixture base-project into a work directory.
+ *
+ * @param workDir - destination directory
+ */
+async function copyFixture(workDir) {
+    fs.cpSync(FIXTURE_DIR, workDir, { recursive: true });
+}
+/**
+ * Check whether at least one tool call used a tool matching the given name.
+ * Matching is case-insensitive and uses substring/alias matching to handle
+ * different runtime tool naming conventions:
+ * - "read" matches "Read", "read_file", "ReadFile", etc.
+ * - "glob" matches "Glob", "glob_files", "GlobFiles", etc.
+ *
+ * @param toolCalls - recorded tool calls from the runtime
+ * @param toolName  - expected tool name (will match as substring)
+ * @returns true if the tool was used
+ */
+function usedTool(toolCalls, toolName) {
+    const lower = toolName.toLowerCase();
+    return toolCalls.some((tc) => {
+        const tcLower = tc.toolName.toLowerCase();
+        return tcLower.includes(lower) || lower.includes(tcLower);
+    });
+}
+/**
+ * Get the string value of a tool call argument, checking multiple possible
+ * key names (to handle different runtimes using different arg names).
+ *
+ * @param tc   - tool call record
+ * @param keys - possible argument key names
+ * @returns the string value or empty string
+ */
+function getArgValue(tc, ...keys) {
+    const args = tc.args;
+    if (!args)
+        return '';
+    for (const key of keys) {
+        if (args[key] !== undefined)
+            return String(args[key]);
+    }
+    return '';
+}
+// ---------------------------------------------------------------------------
+// L1-01: Read package.json
+// ---------------------------------------------------------------------------
+/**
+ * L1-01: Read package.json and report its version field.
+ *
+ * Tests the agent's ability to use the read tool and extract
+ * structured information from a JSON file.
+ */
+const L1_01_READ_FILE = {
+    id: 'L1-01',
+    level: 'L1',
+    title: 'Read package.json and report version',
+    prompt: 'Read the package.json file in this project and tell me the current version number.',
+    maxSteps: MAX_TASK_STEPS,
+    timeoutMs: DEFAULT_TASK_TIMEOUT_MS,
+    expectedTools: ['read'],
+    tags: ['file-ops', 'json'],
+    async setup(workDir) {
+        await copyFixture(workDir);
+    },
+    async verify(workDir, toolCalls) {
+        const usedReadTool = usedTool(toolCalls, 'read');
+        // Check if agent read a file containing "package.json" in the path
+        const readPackageJson = toolCalls.some((tc) => {
+            if (!usedTool([tc], 'read'))
+                return false;
+            const filePath = getArgValue(tc, 'path', 'file_path', 'filePath', 'file');
+            return filePath.includes('package.json');
+        });
+        const checks = [
+            {
+                name: 'used_read',
+                passed: usedReadTool,
+                message: 'Agent should use a read/read_file tool',
+            },
+            {
+                name: 'read_package_json',
+                passed: readPackageJson,
+                message: 'Agent should read package.json specifically',
+            },
+        ];
+        return buildVerification(checks);
+    },
+};
+// ---------------------------------------------------------------------------
+// L1-02: Create hello.ts
+// ---------------------------------------------------------------------------
+/**
+ * L1-02: Create a hello.ts file that exports a greeting function.
+ *
+ * Tests the agent's ability to use the write tool to create
+ * a new TypeScript source file. Checks both root and src/ paths.
+ */
+const L1_02_WRITE_FILE = {
+    id: 'L1-02',
+    level: 'L1',
+    title: 'Create hello.ts with greeting export',
+    prompt: 'Create a file called hello.ts in the project root that exports a function named greet which takes a name parameter (string) and returns a greeting string like "Hello, <name>!".',
+    maxSteps: MAX_TASK_STEPS,
+    timeoutMs: DEFAULT_TASK_TIMEOUT_MS,
+    expectedTools: ['write'],
+    tags: ['file-ops', 'typescript'],
+    async setup(workDir) {
+        await copyFixture(workDir);
+    },
+    async verify(workDir, toolCalls) {
+        // Check both root and src/ paths (agent may put it in either)
+        const rootPath = path.join(workDir, 'hello.ts');
+        const srcPath = path.join(workDir, 'src', 'hello.ts');
+        const fileExists = fs.existsSync(rootPath) || fs.existsSync(srcPath);
+        let containsExport = false;
+        let containsGreet = false;
+        const helloPath = fs.existsSync(rootPath) ? rootPath : srcPath;
+        if (fileExists) {
+            const content = fs.readFileSync(helloPath, 'utf-8');
+            containsExport = content.includes('export');
+            containsGreet = content.includes('greet');
+        }
+        const checks = [
+            {
+                name: 'used_write',
+                passed: usedTool(toolCalls, 'write'),
+                message: 'Agent should use a write/write_file tool',
+            },
+            {
+                name: 'file_created',
+                passed: fileExists,
+                message: 'hello.ts should exist (root or src/)',
+            },
+            {
+                name: 'has_export',
+                passed: containsExport,
+                message: 'File should contain an export',
+            },
+            {
+                name: 'has_greet_function',
+                passed: containsGreet,
+                message: 'File should define a greet function',
+            },
+        ];
+        return buildVerification(checks);
+    },
+};
+// ---------------------------------------------------------------------------
+// L1-03: Find test files
+// ---------------------------------------------------------------------------
+/**
+ * L1-03: Find all .test.ts files in the src/ directory.
+ *
+ * Tests the agent's ability to use glob/search for file discovery.
+ * Accepts both glob and grep tools since some runtimes may use either.
+ */
+const L1_03_GLOB_FILES = {
+    id: 'L1-03',
+    level: 'L1',
+    title: 'Find all .test.ts files in src/',
+    prompt: 'Find all files matching the pattern "*.test.ts" inside the src/ directory and list them.',
+    maxSteps: MAX_TASK_STEPS,
+    timeoutMs: DEFAULT_TASK_TIMEOUT_MS,
+    expectedTools: ['glob'],
+    tags: ['file-ops', 'discovery'],
+    async setup(workDir) {
+        await copyFixture(workDir);
+    },
+    async verify(_workDir, toolCalls) {
+        // Accept glob, grep, bash (ls/find), or any search tool
+        const usedSearchTool = usedTool(toolCalls, 'glob') ||
+            usedTool(toolCalls, 'grep') ||
+            usedTool(toolCalls, 'bash') ||
+            usedTool(toolCalls, 'search');
+        // Check if the pattern includes test.ts
+        const correctPattern = toolCalls.some((tc) => {
+            const args = tc.args;
+            if (!args)
+                return false;
+            const allArgs = JSON.stringify(args).toLowerCase();
+            return allArgs.includes('test.ts') || allArgs.includes('.test.');
+        });
+        const checks = [
+            {
+                name: 'used_search_tool',
+                passed: usedSearchTool,
+                message: 'Agent should use a glob/grep/search tool',
+            },
+            {
+                name: 'correct_pattern',
+                passed: correctPattern,
+                message: 'Search pattern should target .test.ts files',
+            },
+        ];
+        return buildVerification(checks);
+    },
+};
+// ---------------------------------------------------------------------------
+// L1-04: Search for export class
+// ---------------------------------------------------------------------------
+/**
+ * L1-04: Search for "export class" declarations in src/.
+ *
+ * Tests the agent's ability to use grep/search for content searching.
+ */
+const L1_04_GREP_SEARCH = {
+    id: 'L1-04',
+    level: 'L1',
+    title: 'Search for "export class" in src/',
+    prompt: 'Search for all occurrences of "export class" in the src/ directory and report the results.',
+    maxSteps: MAX_TASK_STEPS,
+    timeoutMs: DEFAULT_TASK_TIMEOUT_MS,
+    expectedTools: ['grep'],
+    tags: ['search', 'code-analysis'],
+    async setup(workDir) {
+        await copyFixture(workDir);
+    },
+    async verify(_workDir, toolCalls) {
+        // Accept grep, search, or bash (grep command)
+        const usedSearchTool = usedTool(toolCalls, 'grep') ||
+            usedTool(toolCalls, 'search') ||
+            usedTool(toolCalls, 'bash');
+        const correctPattern = toolCalls.some((tc) => {
+            const args = tc.args;
+            if (!args)
+                return false;
+            const allArgs = JSON.stringify(args).toLowerCase();
+            return allArgs.includes('export class') || allArgs.includes('export');
+        });
+        const checks = [
+            {
+                name: 'used_search_tool',
+                passed: usedSearchTool,
+                message: 'Agent should use a grep/search tool',
+            },
+            {
+                name: 'correct_pattern',
+                passed: correctPattern,
+                message: 'Search pattern should include "export class"',
+            },
+        ];
+        return buildVerification(checks);
+    },
+};
+// ---------------------------------------------------------------------------
+// L1-05: Show git status
+// ---------------------------------------------------------------------------
+/**
+ * L1-05: Show the current git status.
+ *
+ * Tests the agent's ability to use bash for shell commands.
+ */
+const L1_05_BASH_EXEC = {
+    id: 'L1-05',
+    level: 'L1',
+    title: 'Show git status',
+    prompt: 'Run "git status" in the project directory and show me the output.',
+    maxSteps: MAX_TASK_STEPS,
+    timeoutMs: DEFAULT_TASK_TIMEOUT_MS,
+    expectedTools: ['bash'],
+    tags: ['bash', 'git'],
+    async setup(workDir) {
+        await copyFixture(workDir);
+        // Initialize a git repo so git status works
+        const { execSync } = await import('node:child_process');
+        execSync('git init', { cwd: workDir, stdio: 'ignore' });
+        execSync('git add .', { cwd: workDir, stdio: 'ignore' });
+    },
+    async verify(_workDir, toolCalls) {
+        const usedBashTool = usedTool(toolCalls, 'bash');
+        const ranGitStatus = toolCalls.some((tc) => {
+            if (!usedTool([tc], 'bash'))
+                return false;
+            const cmd = getArgValue(tc, 'command', 'cmd');
+            return cmd.includes('git status');
+        });
+        const checks = [
+            {
+                name: 'used_bash',
+                passed: usedBashTool,
+                message: 'Agent should use a bash/exec tool',
+            },
+            {
+                name: 'ran_git_status',
+                passed: ranGitStatus,
+                message: 'Agent should run git status command',
+            },
+        ];
+        return buildVerification(checks);
+    },
+};
+// ---------------------------------------------------------------------------
+// L1-06: Edit version in package.json
+// ---------------------------------------------------------------------------
+/**
+ * L1-06: Change the version in package.json from 1.0.0 to 1.1.0.
+ *
+ * Tests the agent's ability to use edit for targeted modifications.
+ */
+const L1_06_EDIT_FILE = {
+    id: 'L1-06',
+    level: 'L1',
+    title: 'Change version in package.json 1.0.0 to 1.1.0',
+    prompt: 'Edit the package.json file and change the version from "1.0.0" to "1.1.0". Do not change anything else.',
+    maxSteps: MAX_TASK_STEPS,
+    timeoutMs: DEFAULT_TASK_TIMEOUT_MS,
+    expectedTools: ['edit'],
+    tags: ['file-ops', 'edit', 'json'],
+    async setup(workDir) {
+        await copyFixture(workDir);
+    },
+    async verify(workDir, toolCalls) {
+        const pkgPath = path.join(workDir, 'package.json');
+        let versionUpdated = false;
+        let namePreserved = false;
+        if (fs.existsSync(pkgPath)) {
+            try {
+                const content = JSON.parse(fs.readFileSync(pkgPath, 'utf-8'));
+                versionUpdated = content.version === '1.1.0';
+                namePreserved = content.name === 'eval-base-project';
+            }
+            catch {
+                // JSON parse failed
+            }
+        }
+        // Accept edit or write (agent may overwrite the whole file)
+        const usedEditTool = usedTool(toolCalls, 'edit') || usedTool(toolCalls, 'write');
+        const checks = [
+            {
+                name: 'used_edit',
+                passed: usedEditTool,
+                message: 'Agent should use an edit/write tool',
+            },
+            {
+                name: 'version_updated',
+                passed: versionUpdated,
+                message: 'Version should be changed to 1.1.0',
+            },
+            {
+                name: 'name_preserved',
+                passed: namePreserved,
+                message: 'Package name should remain eval-base-project',
+            },
+        ];
+        return buildVerification(checks);
+    },
+};
+// ---------------------------------------------------------------------------
+// Export
+// ---------------------------------------------------------------------------
+/** All L1 evaluation tasks */
+export const L1_TASKS = [
+    L1_01_READ_FILE,
+    L1_02_WRITE_FILE,
+    L1_03_GLOB_FILES,
+    L1_04_GREP_SEARCH,
+    L1_05_BASH_EXEC,
+    L1_06_EDIT_FILE,
+];
+//# sourceMappingURL=l1-tasks.js.map

package/dist/backend/backend/src/services/agent/crewly-agent/eval/tasks/l1-tasks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"l1-tasks.js","sourceRoot":"","sources":["../../../../../../../../../backend/src/services/agent/crewly-agent/eval/tasks/l1-tasks.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAIzC,OAAO,EAAE,uBAAuB,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAE3E,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACtD,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;AAEnD,iEAAiE;AACjE,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,IAAI,EAAE,UAAU,EAAE,cAAc,CAAC,CAAC;AAElF,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E;;;;;;GAMG;AACH,SAAS,iBAAiB,CAAC,MAAmB;IAC5C,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QACrC,MAAM;KACP,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,WAAW,CAAC,OAAe;IACxC,EAAE,CAAC,MAAM,CAAC,WAAW,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AACvD,CAAC;AAED;;;;;;;;;;GAUG;AACH,SAAS,QAAQ,CAAC,SAA2B,EAAE,QAAgB;IAC7D,MAAM,KAAK,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;IACrC,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE;QAC3B,MAAM,OAAO,GAAG,EAAE,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QAC1C,OAAO,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC5D,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,WAAW,CAAC,EAAkB,EAAE,GAAG,IAAc;IACxD,MAAM,IAAI,GAAG,EAAE,CAAC,IAA2C,CAAC;IAC5D,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IACrB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,SAAS;YAAE,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,8EAA8E;AAC9E,2BAA2B;AAC3B,8EAA8E;AAE9E;;;;;GAKG;AACH,MAAM,eAAe,GAAa;IAChC,EAAE,EAAE,OAAO;IACX,KAAK,EAAE,IAAI;IACX,KAAK,EAAE,sCAAsC;IAC7C,MAAM,EAAE,oFAAoF;IAC5F,QAAQ,EAAE,cAAc;IACxB,SAAS,EAAE,uBAAuB;IAClC,aAAa,EAAE,CAAC,MAAM,CAAC;IACvB,IAAI,EAAE,CAAC,UAAU,EAAE,MAAM,CAAC;IAE1B,KAAK,CAAC,KAAK,CAAC,OAAe;QACzB,MAAM,WAAW,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAe,EAAE,SAA2B;QACvD,MAAM,YAAY,GAAG,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAEjD,mEAAmE;QACnE,MAAM,eAAe,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE;YAC5C,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC;gBAAE,OAAO,KAAK,CAAC;YAC1C,MAAM,QAAQ,GAAG,WAAW,CAAC,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;YAC1E,OAAO,QAAQ,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAgB;YAC1B;gBACE,IAAI,EAAE,WAAW;gBACjB,MAAM,EAAE,YAAY;gBACpB,OAAO,EAAE,wCAAwC;aAClD;YACD;gBACE,IAAI,EAAE,mBAAmB;gBACzB,MAAM,EAAE,eAAe;gBACvB,OAAO,EAAE,6CAA6C;aACvD;SACF,CAAC;QACF,OAAO,iBAAiB,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;CACF,CAAC;AAEF,8EAA8E;AAC9E,yBAAyB;AACzB,8EAA8E;AAE9E;;;;;GAKG;AACH,MAAM,gBAAgB,GAAa;IACjC,EAAE,EAAE,OAAO;IACX,KAAK,EAAE,IAAI;IACX,KAAK,EAAE,sCAAsC;IAC7C,MAAM,EACJ,kLAAkL;IACpL,QAAQ,EAAE,cAAc;IACxB,SAAS,EAAE,uBAAuB;IAClC,aAAa,EAAE,CAAC,OAAO,CAAC;IACxB,IAAI,EAAE,CAAC,UAAU,EAAE,YAAY,CAAC;IAEhC,KAAK,CAAC,KAAK,CAAC,OAAe;QACzB,MAAM,WAAW,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAe,EAAE,SAA2B;QACvD,8DAA8D;QAC9D,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QAChD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;QACtD,MAAM,UAAU,GAAG,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAErE,IAAI,cAAc,GAAG,KAAK,CAAC;QAC3B,IAAI,aAAa,GAAG,KAAK,CAAC;QAC1B,MAAM,SAAS,GAAG,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC;QAE/D,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACpD,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAC5C,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAC5C,CAAC;QAED,MAAM,MAAM,GAAgB;YAC1B;gBACE,IAAI,EAAE,YAAY;gBAClB,MAAM,EAAE,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;gBACpC,OAAO,EAAE,0CAA0C;aACpD;YACD;gBACE,IAAI,EAAE,cAAc;gBACpB,MAAM,EAAE,UAAU;gBAClB,OAAO,EAAE,sCAAsC;aAChD;YACD;gBACE,IAAI,EAAE,YAAY;gBAClB,MAAM,EAAE,cAAc;gBACtB,OAAO,EAAE,+BAA+B;aACzC;YACD;gBACE,IAAI,EAAE,oBAAoB;gBAC1B,MAAM,EAAE,aAAa;gBACrB,OAAO,EAAE,qCAAqC;aAC/C;SACF,CAAC;QACF,OAAO,iBAAiB,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;CACF,CAAC;AAEF,8EAA8E;AAC9E,yBAAyB;AACzB,8EAA8E;AAE9E;;;;;GAKG;AACH,MAAM,gBAAgB,GAAa;IACjC,EAAE,EAAE,OAAO;IACX,KAAK,EAAE,IAAI;IACX,KAAK,EAAE,iCAAiC;IACxC,MAAM,EAAE,0FAA0F;IAClG,QAAQ,EAAE,cAAc;IACxB,SAAS,EAAE,uBAAuB;IAClC,aAAa,EAAE,CAAC,MAAM,CAAC;IACvB,IAAI,EAAE,CAAC,UAAU,EAAE,WAAW,CAAC;IAE/B,KAAK,CAAC,KAAK,CAAC,OAAe;QACzB,MAAM,WAAW,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,QAAgB,EAAE,SAA2B;QACxD,wDAAwD;QACxD,MAAM,cAAc,GAClB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;YAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;YAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;YAC3B,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAEhC,wCAAwC;QACxC,MAAM,cAAc,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE;YAC3C,MAAM,IAAI,GAAG,EAAE,CAAC,IAA2C,CAAC;YAC5D,IAAI,CAAC,IAAI;gBAAE,OAAO,KAAK,CAAC;YACxB,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;YACnD,OAAO,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACnE,CAAC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAgB;YAC1B;gBACE,IAAI,EAAE,kBAAkB;gBACxB,MAAM,EAAE,cAAc;gBACtB,OAAO,EAAE,0CAA0C;aACpD;YACD;gBACE,IAAI,EAAE,iBAAiB;gBACvB,MAAM,EAAE,cAAc;gBACtB,OAAO,EAAE,6CAA6C;aACvD;SACF,CAAC;QACF,OAAO,iBAAiB,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;CACF,CAAC;AAEF,8EAA8E;AAC9E,iCAAiC;AACjC,8EAA8E;AAE9E;;;;GAIG;AACH,MAAM,iBAAiB,GAAa;IAClC,EAAE,EAAE,OAAO;IACX,KAAK,EAAE,IAAI;IACX,KAAK,EAAE,mCAAmC;IAC1C,MAAM,EAAE,4FAA4F;IACpG,QAAQ,EAAE,cAAc;IACxB,SAAS,EAAE,uBAAuB;IAClC,aAAa,EAAE,CAAC,MAAM,CAAC;IACvB,IAAI,EAAE,CAAC,QAAQ,EAAE,eAAe,CAAC;IAEjC,KAAK,CAAC,KAAK,CAAC,OAAe;QACzB,MAAM,WAAW,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,QAAgB,EAAE,SAA2B;QACxD,8CAA8C;QAC9C,MAAM,cAAc,GAClB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;YAC3B,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC;YAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAE9B,MAAM,cAAc,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE;YAC3C,MAAM,IAAI,GAAG,EAAE,CAAC,IAA2C,CAAC;YAC5D,IAAI,CAAC,IAAI;gBAAE,OAAO,KAAK,CAAC;YACxB,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;YACnD,OAAO,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACxE,CAAC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAgB;YAC1B;gBACE,IAAI,EAAE,kBAAkB;gBACxB,MAAM,EAAE,cAAc;gBACtB,OAAO,EAAE,qCAAqC;aAC/C;YACD;gBACE,IAAI,EAAE,iBAAiB;gBACvB,MAAM,EAAE,cAAc;gBACtB,OAAO,EAAE,8CAA8C;aACxD;SACF,CAAC;QACF,OAAO,iBAAiB,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;CACF,CAAC;AAEF,8EAA8E;AAC9E,yBAAyB;AACzB,8EAA8E;AAE9E;;;;GAIG;AACH,MAAM,eAAe,GAAa;IAChC,EAAE,EAAE,OAAO;IACX,KAAK,EAAE,IAAI;IACX,KAAK,EAAE,iBAAiB;IACxB,MAAM,EAAE,mEAAmE;IAC3E,QAAQ,EAAE,cAAc;IACxB,SAAS,EAAE,uBAAuB;IAClC,aAAa,EAAE,CAAC,MAAM,CAAC;IACvB,IAAI,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC;IAErB,KAAK,CAAC,KAAK,CAAC,OAAe;QACzB,MAAM,WAAW,CAAC,OAAO,CAAC,CAAC;QAC3B,4CAA4C;QAC5C,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACxD,QAAQ,CAAC,UAAU,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;QACxD,QAAQ,CAAC,WAAW,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,QAAgB,EAAE,SAA2B;QACxD,MAAM,YAAY,GAAG,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAEjD,MAAM,YAAY,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE;YACzC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC;gBAAE,OAAO,KAAK,CAAC;YAC1C,MAAM,GAAG,GAAG,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;YAC9C,OAAO,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAgB;YAC1B;gBACE,IAAI,EAAE,WAAW;gBACjB,MAAM,EAAE,YAAY;gBACpB,OAAO,EAAE,mCAAmC;aAC7C;YACD;gBACE,IAAI,EAAE,gBAAgB;gBACtB,MAAM,EAAE,YAAY;gBACpB,OAAO,EAAE,qCAAqC;aAC/C;SACF,CAAC;QACF,OAAO,iBAAiB,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;CACF,CAAC;AAEF,8EAA8E;AAC9E,sCAAsC;AACtC,8EAA8E;AAE9E;;;;GAIG;AACH,MAAM,eAAe,GAAa;IAChC,EAAE,EAAE,OAAO;IACX,KAAK,EAAE,IAAI;IACX,KAAK,EAAE,+CAA+C;IACtD,MAAM,EACJ,yGAAyG;IAC3G,QAAQ,EAAE,cAAc;IACxB,SAAS,EAAE,uBAAuB;IAClC,aAAa,EAAE,CAAC,MAAM,CAAC;IACvB,IAAI,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,CAAC;IAElC,KAAK,CAAC,KAAK,CAAC,OAAe;QACzB,MAAM,WAAW,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAe,EAAE,SAA2B;QACvD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACnD,IAAI,cAAc,GAAG,KAAK,CAAC;QAC3B,IAAI,aAAa,GAAG,KAAK,CAAC;QAE1B,IAAI,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAA4B,CAAC;gBACzF,cAAc,GAAG,OAAO,CAAC,OAAO,KAAK,OAAO,CAAC;gBAC7C,aAAa,GAAG,OAAO,CAAC,IAAI,KAAK,mBAAmB,CAAC;YACvD,CAAC;YAAC,MAAM,CAAC;gBACP,oBAAoB;YACtB,CAAC;QACH,CAAC;QAED,4DAA4D;QAC5D,MAAM,YAAY,GAAG,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,IAAI,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAEjF,MAAM,MAAM,GAAgB;YAC1B;gBACE,IAAI,EAAE,WAAW;gBACjB,MAAM,EAAE,YAAY;gBACpB,OAAO,EAAE,qCAAqC;aAC/C;YACD;gBACE,IAAI,EAAE,iBAAiB;gBACvB,MAAM,EAAE,cAAc;gBACtB,OAAO,EAAE,oCAAoC;aAC9C;YACD;gBACE,IAAI,EAAE,gBAAgB;gBACtB,MAAM,EAAE,aAAa;gBACrB,OAAO,EAAE,8CAA8C;aACxD;SACF,CAAC;QACF,OAAO,iBAAiB,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;CACF,CAAC;AAEF,8EAA8E;AAC9E,SAAS;AACT,8EAA8E;AAE9E,8BAA8B;AAC9B,MAAM,CAAC,MAAM,QAAQ,GAAe;IAClC,eAAe;IACf,gBAAgB;IAChB,gBAAgB;IAChB,iBAAiB;IACjB,eAAe;IACf,eAAe;CAChB,CAAC"}

package/dist/backend/backend/src/services/agent/crewly-agent/eval/tasks/l2-tasks.d.ts

@@ -0,0 +1,14 @@
+/**
+ * L2 Evaluation Tasks - Multi-Step Workflows
+ *
+ * Five tasks that test an agent's ability to chain multiple tools
+ * together to complete realistic development workflows:
+ * read-then-edit, test-and-fix, multi-file refactor, git workflow,
+ * and dependency analysis.
+ *
+ * @module eval/tasks/l2-tasks
+ */
+import type { EvalTask } from '../eval-types.js';
+/** All L2 evaluation tasks */
+export declare const L2_TASKS: EvalTask[];
+//# sourceMappingURL=l2-tasks.d.ts.map

package/dist/backend/backend/src/services/agent/crewly-agent/eval/tasks/l2-tasks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"l2-tasks.d.ts","sourceRoot":"","sources":["../../../../../../../../../backend/src/services/agent/crewly-agent/eval/tasks/l2-tasks.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAOH,OAAO,KAAK,EAAa,QAAQ,EAAoB,MAAM,kBAAkB,CAAC;AA+nB9E,8BAA8B;AAC9B,eAAO,MAAM,QAAQ,EAAE,QAAQ,EAM9B,CAAC"}