crewly 1.5.18 → 1.5.19

package/dist/backend/backend/src/services/agent/crewly-agent/eval/context-generator.js

@@ -0,0 +1,299 @@
+/**
+ * Eval Context Generator
+ *
+ * Generates a CONTEXT.md file and the `.crewly/` directory structure
+ * that provides runtime-agnostic context for L4 eval tasks. Every
+ * runtime (Crewly Agent, Claude Code, Codex CLI, Gemini CLI) receives
+ * the same context package — no Crewly-specific API calls required.
+ *
+ * @module eval/context-generator
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { fileURLToPath } from 'node:url';
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+/**
+ * Default eval team configuration used by L4 tasks.
+ */
+export const DEFAULT_EVAL_TEAM = {
+    agentName: 'Sam',
+    agentRole: 'Team Lead',
+    teamName: 'Eval Product Team',
+    teamId: 'eval-team-001',
+    members: [
+        {
+            name: 'Sam',
+            role: 'Team Lead',
+            sessionName: 'eval-tl-sam',
+            agentStatus: 'active',
+            workingStatus: 'idle',
+        },
+        {
+            name: 'Leo',
+            role: 'Developer',
+            sessionName: 'eval-dev-leo',
+            agentStatus: 'active',
+            workingStatus: 'idle',
+        },
+        {
+            name: 'Max',
+            role: 'Developer',
+            sessionName: 'eval-dev-max',
+            agentStatus: 'inactive',
+            workingStatus: 'idle',
+        },
+    ],
+};
+// ---------------------------------------------------------------------------
+// CONTEXT.md generation
+// ---------------------------------------------------------------------------
+/**
+ * Generate the CONTEXT.md content from a team configuration.
+ *
+ * The generated document tells the agent who it is, who its team members
+ * are, and which bash skill scripts are available. It is intentionally
+ * runtime-agnostic — any agent that can read markdown and run bash can
+ * use it.
+ *
+ * @param config - team and agent configuration
+ * @returns the full CONTEXT.md string
+ */
+export function generateContextMarkdown(config) {
+    const memberRows = config.members
+        .map((m) => {
+        const workload = m.agentStatus === 'inactive' ? '—' : m.workingStatus;
+        return `| ${m.name} | ${m.role} | ${m.sessionName} | ${m.agentStatus} | ${workload} |`;
+    })
+        .join('\n');
+    const extraSection = config.extraInstructions
+        ? `\n## Additional Instructions\n\n${config.extraInstructions}\n`
+        : '';
+    return `# Team Collaboration Context
+
+## Your Identity
+You are **${config.agentName}**, ${config.agentRole} of the ${config.teamName}.
+
+## Your Team
+| Name | Role | Session | Status | Workload |
+|------|------|---------|--------|----------|
+${memberRows}
+
+## Available Skills
+You can run these bash scripts to coordinate with your team:
+
+- \`bash skills/delegate-task.sh '{"to":"<session>","task":"...","priority":"high"}'\`
+- \`bash skills/send-message.sh '{"to":"<session>","message":"..."}'\`
+- \`bash skills/get-team-status.sh '{}'\`
+- \`bash skills/verify-output.sh '{"checks":[{"name":"build","command":"npm run build"}]}'\`
+- \`bash skills/report-status.sh '{"status":"complete","summary":"..."}'\`
+- \`bash skills/handle-failure.sh '{"worker":"<session>","error":"...","action":"reassign"}'\`
+
+## Rules
+1. Always read CONTEXT.md and relevant files before taking action.
+2. Delegate to **active** workers only. Never assign work to inactive agents.
+3. Use the skill scripts above for all team coordination — do not hard-code API calls.
+4. Report your status via report-status when done.
+
+## Project Layout
+- Source code: \`src/\`
+- Tasks: \`.crewly/tasks/\`
+- Knowledge base: \`.crewly/knowledge/\`
+- Team config: \`.crewly/teams/team-config.json\`
+- Reports output: \`.crewly/reports/\`
+- Messages output: \`.crewly/messages/\`
+${extraSection}`;
+}
+// ---------------------------------------------------------------------------
+// Team config JSON generation
+// ---------------------------------------------------------------------------
+/**
+ * Generate the team-config.json content from a context configuration.
+ *
+ * @param config - team and agent configuration
+ * @returns serialized JSON string for team-config.json
+ */
+export function generateTeamConfig(config) {
+    const teamConfig = {
+        id: config.teamId,
+        name: config.teamName,
+        members: config.members.map((m, idx) => ({
+            id: `member-${String(idx + 1).padStart(3, '0')}`,
+            name: m.name,
+            role: m.role.toLowerCase().replace(/\s+/g, '-'),
+            sessionName: m.sessionName,
+            agentStatus: m.agentStatus,
+            workingStatus: m.workingStatus,
+        })),
+    };
+    return JSON.stringify(teamConfig, null, 2);
+}
+// ---------------------------------------------------------------------------
+// Full workspace setup
+// ---------------------------------------------------------------------------
+/**
+ * Set up a complete eval workspace with CONTEXT.md, skill scripts,
+ * and the `.crewly/` directory structure.
+ *
+ * This is the main entry point for task setup functions. It creates
+ * a self-contained directory that any runtime can work in.
+ *
+ * @param workDir - target directory to populate
+ * @param config  - team/agent configuration (defaults to DEFAULT_EVAL_TEAM)
+ * @param options - additional setup options
+ */
+export async function setupEvalWorkspace(workDir, config = DEFAULT_EVAL_TEAM, options = {}) {
+    // Create directory structure
+    const dirs = [
+        '.crewly/tasks',
+        '.crewly/knowledge',
+        '.crewly/teams',
+        '.crewly/messages',
+        '.crewly/reports',
+        'skills',
+        'src/services',
+        'src/controllers',
+        'src/utils',
+    ];
+    for (const d of dirs) {
+        fs.mkdirSync(path.join(workDir, d), { recursive: true });
+    }
+    // Write CONTEXT.md
+    fs.writeFileSync(path.join(workDir, 'CONTEXT.md'), generateContextMarkdown(config));
+    // Write team config
+    fs.writeFileSync(path.join(workDir, '.crewly', 'teams', 'team-config.json'), generateTeamConfig(config));
+    // Copy skill shims
+    copySkillShims(workDir);
+    // Write default project files unless skipProjectFiles is set
+    if (!options.skipProjectFiles) {
+        writeDefaultProjectFiles(workDir);
+    }
+    // Write knowledge base files unless skipKnowledge is set
+    if (!options.skipKnowledge) {
+        writeDefaultKnowledge(workDir);
+    }
+}
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+/**
+ * Copy skill shim scripts from the fixtures directory to the workspace.
+ *
+ * @param workDir - target workspace directory
+ */
+function copySkillShims(workDir) {
+    const fixturesSkillsDir = path.join(__dirname, 'fixtures', 'skills');
+    const targetSkillsDir = path.join(workDir, 'skills');
+    // List of skill scripts to copy
+    const scripts = [
+        '_log.sh',
+        'delegate-task.sh',
+        'send-message.sh',
+        'get-team-status.sh',
+        'verify-output.sh',
+        'report-status.sh',
+        'handle-failure.sh',
+    ];
+    for (const script of scripts) {
+        const src = path.join(fixturesSkillsDir, script);
+        const dst = path.join(targetSkillsDir, script);
+        if (fs.existsSync(src)) {
+            fs.copyFileSync(src, dst);
+            fs.chmodSync(dst, 0o755);
+        }
+    }
+}
+/**
+ * Write default project files (package.json, tsconfig.json, sample source).
+ *
+ * @param workDir - target workspace directory
+ */
+function writeDefaultProjectFiles(workDir) {
+    fs.writeFileSync(path.join(workDir, 'package.json'), JSON.stringify({
+        name: 'eval-project',
+        version: '1.0.0',
+        type: 'module',
+        scripts: {
+            build: 'tsc --noEmit',
+            test: 'echo "4 passing" && exit 0',
+        },
+    }, null, 2));
+    fs.writeFileSync(path.join(workDir, 'tsconfig.json'), JSON.stringify({
+        compilerOptions: {
+            target: 'ES2022',
+            module: 'ESNext',
+            moduleResolution: 'bundler',
+            strict: true,
+            outDir: './dist',
+            rootDir: './src',
+            declaration: true,
+            esModuleInterop: true,
+            skipLibCheck: true,
+        },
+        include: ['src/**/*.ts'],
+    }, null, 2));
+    // Sample existing service for pattern reference
+    fs.writeFileSync(path.join(workDir, 'src', 'services', 'data.service.ts'), `/**
+ * Data service for CRUD operations.
+ */
+export interface DataRecord {
+  id: string;
+  value: string;
+  createdAt: number;
+}
+
+/**
+ * Create a new data record.
+ *
+ * @param id    - unique identifier
+ * @param value - record value
+ * @returns the created record
+ */
+export function createRecord(id: string, value: string): DataRecord {
+  return { id, value, createdAt: Date.now() };
+}
+
+/**
+ * Validate a data record.
+ *
+ * @param record - the record to validate
+ * @returns true if all fields are valid
+ */
+export function validateRecord(record: DataRecord): boolean {
+  return record.id.length > 0 && record.value.length > 0 && record.createdAt > 0;
+}
+`);
+}
+/**
+ * Write default knowledge base files.
+ *
+ * @param workDir - target workspace directory
+ */
+function writeDefaultKnowledge(workDir) {
+    fs.writeFileSync(path.join(workDir, '.crewly', 'knowledge', 'project-patterns.md'), `# Project Patterns
+
+## Controller Pattern
+All controllers should:
+1. Export named functions (not classes)
+2. Accept (req, res) parameters
+3. Use try/catch for error handling
+4. Return JSON responses
+
+## Service Pattern
+All services should:
+1. Export interfaces for data types
+2. Export named functions for operations
+3. Include JSDoc comments on all exports
+4. Use TypeScript strict types (no \`any\`)
+
+## Naming Convention
+- Controllers: kebab-case with .controller.ts suffix
+- Services: kebab-case with .service.ts suffix
+- Tests: co-located as .test.ts in the same directory
+
+## Error Handling
+- Always wrap async operations in try/catch
+- Return structured error responses: { error: string, code: number }
+- Log errors before returning them
+`);
+}
+//# sourceMappingURL=context-generator.js.map

package/dist/backend/backend/src/services/agent/crewly-agent/eval/context-generator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"context-generator.js","sourceRoot":"","sources":["../../../../../../../../backend/src/services/agent/crewly-agent/eval/context-generator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAwC/D;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAsB;IAClD,SAAS,EAAE,KAAK;IAChB,SAAS,EAAE,WAAW;IACtB,QAAQ,EAAE,mBAAmB;IAC7B,MAAM,EAAE,eAAe;IACvB,OAAO,EAAE;QACP;YACE,IAAI,EAAE,KAAK;YACX,IAAI,EAAE,WAAW;YACjB,WAAW,EAAE,aAAa;YAC1B,WAAW,EAAE,QAAQ;YACrB,aAAa,EAAE,MAAM;SACtB;QACD;YACE,IAAI,EAAE,KAAK;YACX,IAAI,EAAE,WAAW;YACjB,WAAW,EAAE,cAAc;YAC3B,WAAW,EAAE,QAAQ;YACrB,aAAa,EAAE,MAAM;SACtB;QACD;YACE,IAAI,EAAE,KAAK;YACX,IAAI,EAAE,WAAW;YACjB,WAAW,EAAE,cAAc;YAC3B,WAAW,EAAE,UAAU;YACvB,aAAa,EAAE,MAAM;SACtB;KACF;CACF,CAAC;AAEF,8EAA8E;AAC9E,wBAAwB;AACxB,8EAA8E;AAE9E;;;;;;;;;;GAUG;AACH,MAAM,UAAU,uBAAuB,CAAC,MAAyB;IAC/D,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO;SAC9B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,QAAQ,GAAG,CAAC,CAAC,WAAW,KAAK,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;QACtE,OAAO,KAAK,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,WAAW,MAAM,CAAC,CAAC,WAAW,MAAM,QAAQ,IAAI,CAAC;IACzF,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,MAAM,YAAY,GAAG,MAAM,CAAC,iBAAiB;QAC3C,CAAC,CAAC,mCAAmC,MAAM,CAAC,iBAAiB,IAAI;QACjE,CAAC,CAAC,EAAE,CAAC;IAEP,OAAO;;;YAGG,MAAM,CAAC,SAAS,OAAO,MAAM,CAAC,SAAS,WAAW,MAAM,CAAC,QAAQ;;;;;EAK3E,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;EAyBV,YAAY,EAAE,CAAC;AACjB,CAAC;AAED,8EAA8E;AAC9E,8BAA8B;AAC9B,8EAA8E;AAE9E;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAAyB;IAC1D,MAAM,UAAU,GAAG;QACjB,EAAE,EAAE,MAAM,CAAC,MAAM;QACjB,IAAI,EAAE,MAAM,CAAC,QAAQ;QACrB,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;YACvC,EAAE,EAAE,UAAU,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;YAChD,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;YAC/C,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,aAAa,EAAE,CAAC,CAAC,aAAa;SAC/B,CAAC,CAAC;KACJ,CAAC;IACF,OAAO,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AAC7C,CAAC;AAED,8EAA8E;AAC9E,uBAAuB;AACvB,8EAA8E;AAE9E;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,OAAe,EACf,SAA4B,iBAAiB,EAC7C,UAAwB,EAAE;IAE1B,6BAA6B;IAC7B,MAAM,IAAI,GAAG;QACX,eAAe;QACf,mBAAmB;QACnB,eAAe;QACf,kBAAkB;QAClB,iBAAiB;QACjB,QAAQ;QACR,cAAc;QACd,iBAAiB;QACjB,WAAW;KACZ,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,mBAAmB;IACnB,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,EAChC,uBAAuB,CAAC,MAAM,CAAC,CAChC,CAAC;IAEF,oBAAoB;IACpB,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,kBAAkB,CAAC,EAC1D,kBAAkB,CAAC,MAAM,CAAC,CAC3B,CAAC;IAEF,mBAAmB;IACnB,cAAc,CAAC,OAAO,CAAC,CAAC;IAExB,6DAA6D;IAC7D,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;QAC9B,wBAAwB,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;IAED,yDAAyD;IACzD,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;QAC3B,qBAAqB,CAAC,OAAO,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAYD,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E;;;;GAIG;AACH,SAAS,cAAc,CAAC,OAAe;IACrC,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;IACrE,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAErD,gCAAgC;IAChC,MAAM,OAAO,GAAG;QACd,SAAS;QACT,kBAAkB;QAClB,iBAAiB;QACjB,oBAAoB;QACpB,kBAAkB;QAClB,kBAAkB;QAClB,mBAAmB;KACpB,CAAC;IAEF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;QACjD,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;QAC/C,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACvB,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YAC1B,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,wBAAwB,CAAC,OAAe;IAC/C,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,EAClC,IAAI,CAAC,SAAS,CACZ;QACE,IAAI,EAAE,cAAc;QACpB,OAAO,EAAE,OAAO;QAChB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE;YACP,KAAK,EAAE,cAAc;YACrB,IAAI,EAAE,4BAA4B;SACnC;KACF,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;IAEF,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EACnC,IAAI,CAAC,SAAS,CACZ;QACE,eAAe,EAAE;YACf,MAAM,EAAE,QAAQ;YAChB,MAAM,EAAE,QAAQ;YAChB,gBAAgB,EAAE,SAAS;YAC3B,MAAM,EAAE,IAAI;YACZ,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE,OAAO;YAChB,WAAW,EAAE,IAAI;YACjB,eAAe,EAAE,IAAI;YACrB,YAAY,EAAE,IAAI;SACnB;QACD,OAAO,EAAE,CAAC,aAAa,CAAC;KACzB,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;IAEF,gDAAgD;IAChD,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,iBAAiB,CAAC,EACxD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6BH,CACE,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,qBAAqB,CAAC,OAAe;IAC5C,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,qBAAqB,CAAC,EACjE;;;;;;;;;;;;;;;;;;;;;;;;;CAyBH,CACE,CAAC;AACJ,CAAC"}

package/dist/backend/backend/src/services/agent/crewly-agent/eval/eval-runner.d.ts

@@ -0,0 +1,59 @@
+/**
+ * Crewly Agent Evaluation Framework - Runner
+ *
+ * Orchestrates a full evaluation run: filters tasks, initializes the
+ * runtime adapter, executes tasks sequentially, scores results, and
+ * produces a final report.
+ *
+ * @module eval-runner
+ */
+import type { EvalLevel, EvalRunReport, EvalTask, RuntimeAdapter, RuntimeId } from './eval-types.js';
+/**
+ * Configuration for {@link runEvaluation}.
+ */
+export interface RunnerConfig {
+    /** Initialized runtime adapter */
+    adapter: RuntimeAdapter;
+    /** Runtime identifier (used in reports and file names) */
+    runtimeId: RuntimeId;
+    /** Model identifier for cost calculation */
+    model: string;
+    /** Optional list of specific task IDs to run */
+    taskIds?: string[];
+    /** Optional list of levels to include */
+    levels?: EvalLevel[];
+    /** Directory where report JSON files are written */
+    outputDir: string;
+    /** Path to a baseline report JSON for normalization */
+    baselinePath?: string;
+    /** If true, keep temporary work directories after evaluation */
+    keepWorkDirs?: boolean;
+    /** If true, emit verbose progress logs */
+    verbose?: boolean;
+}
+/**
+ * Run a full evaluation across the provided tasks.
+ *
+ * The runner performs the following steps:
+ * 1. Filter tasks by ID and/or level if specified in config
+ * 2. Initialize the runtime adapter
+ * 3. For each task: setup work dir, run via adapter, verify, score, teardown
+ * 4. Aggregate results into a run report
+ * 5. Optionally normalize against a baseline report
+ * 6. Save JSON report to outputDir
+ * 7. Print summary to stdout
+ *
+ * @param config   - runner configuration
+ * @param allTasks - full list of available eval tasks
+ * @returns the completed evaluation run report
+ *
+ * @example
+ * ```typescript
+ * const report = await runEvaluation(
+ *   { adapter, model: 'claude-opus-4', outputDir: './reports' },
+ *   ALL_TASKS,
+ * );
+ * ```
+ */
+export declare function runEvaluation(config: RunnerConfig, allTasks: EvalTask[]): Promise<EvalRunReport>;
+//# sourceMappingURL=eval-runner.d.ts.map

package/dist/backend/backend/src/services/agent/crewly-agent/eval/eval-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-runner.d.ts","sourceRoot":"","sources":["../../../../../../../../backend/src/services/agent/crewly-agent/eval/eval-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAMH,OAAO,KAAK,EACV,SAAS,EACT,aAAa,EACb,QAAQ,EAGR,cAAc,EACd,SAAS,EAEV,MAAM,iBAAiB,CAAC;AAQzB;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,kCAAkC;IAClC,OAAO,EAAE,cAAc,CAAC;IACxB,0DAA0D;IAC1D,SAAS,EAAE,SAAS,CAAC;IACrB,4CAA4C;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,gDAAgD;IAChD,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,yCAAyC;IACzC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC;IACrB,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,uDAAuD;IACvD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gEAAgE;IAChE,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,0CAA0C;IAC1C,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AA6GD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,YAAY,EACpB,QAAQ,EAAE,QAAQ,EAAE,GACnB,OAAO,CAAC,aAAa,CAAC,CA6FxB"}

package/dist/backend/backend/src/services/agent/crewly-agent/eval/eval-runner.js

@@ -0,0 +1,218 @@
+/**
+ * Crewly Agent Evaluation Framework - Runner
+ *
+ * Orchestrates a full evaluation run: filters tasks, initializes the
+ * runtime adapter, executes tasks sequentially, scores results, and
+ * produces a final report.
+ *
+ * @module eval-runner
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import * as os from 'node:os';
+import { aggregateResults, normalizeToBaseline, scoreTask } from './eval-scorer.js';
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+/**
+ * Log a message to stdout when verbose mode is enabled.
+ *
+ * @param verbose - whether verbose logging is on
+ * @param args    - values to log
+ */
+function log(verbose, ...args) {
+    if (verbose) {
+        console.log('[eval]', ...args);
+    }
+}
+/**
+ * Format a timestamp for use in filenames (ISO without colons).
+ *
+ * @returns formatted timestamp string
+ */
+function formatTimestamp() {
+    return new Date().toISOString().replace(/[:.]/g, '-');
+}
+/**
+ * Sanitize a string for use as a filename component.
+ *
+ * @param name - raw string to sanitize
+ * @returns filesystem-safe string
+ */
+function sanitizeFilename(name) {
+    return name.replace(/[^a-zA-Z0-9_-]/g, '_');
+}
+/**
+ * Create a temporary work directory for a single task.
+ *
+ * @param taskId - task identifier used in the directory name
+ * @returns absolute path to the created directory
+ */
+function setupWorkDir(taskId) {
+    const dir = path.join(os.tmpdir(), `crewly-eval-${sanitizeFilename(taskId)}-${Date.now()}`);
+    fs.mkdirSync(dir, { recursive: true });
+    return dir;
+}
+/**
+ * Build a failed {@link EvalTaskResult} for tasks that error before scoring.
+ *
+ * @param task  - the task that failed
+ * @param error - error message
+ * @returns a task result with zero scores
+ */
+function createFailedResult(task, error) {
+    return {
+        taskId: task.id,
+        level: task.level,
+        title: task.title,
+        success: false,
+        dimensions: [],
+        composite: 0,
+        metrics: {
+            durationMs: 0,
+            steps: 0,
+            usage: { input: 0, output: 0 },
+            costUSD: 0,
+            toolCallCount: 0,
+            correctToolCalls: 0,
+            unnecessaryToolCalls: 0,
+            argAccuracy: 0,
+            toolErrors: 0,
+            humanNudges: 0,
+            loopDetected: false,
+            timedOut: false,
+        },
+        verification: { passed: false, checks: [] },
+        error,
+    };
+}
+/**
+ * Print a human-readable summary of the evaluation run to stdout.
+ *
+ * @param report - the completed evaluation report
+ */
+function printSummary(report) {
+    console.log('\n========================================');
+    console.log('  Evaluation Summary');
+    console.log('========================================');
+    console.log(`  Runtime : ${report.runtimeId}`);
+    console.log(`  Model   : ${report.model}`);
+    console.log(`  Tasks   : ${report.passedCount}/${report.totalCount} passed`);
+    console.log(`  Score   : ${report.overallComposite.toFixed(1)}/100`);
+    console.log(`  Cost    : $${report.totalCostUSD.toFixed(4)}`);
+    console.log(`  Duration: ${(report.totalDurationMs / 1000).toFixed(1)}s`);
+    console.log('----------------------------------------');
+    for (const dim of report.aggregateDimensions) {
+        console.log(`  ${dim.dimensionId.padEnd(20)} ${dim.raw.toFixed(1).padStart(6)} (w=${dim.weight})`);
+    }
+    console.log('========================================\n');
+}
+// ---------------------------------------------------------------------------
+// Main runner
+// ---------------------------------------------------------------------------
+/**
+ * Run a full evaluation across the provided tasks.
+ *
+ * The runner performs the following steps:
+ * 1. Filter tasks by ID and/or level if specified in config
+ * 2. Initialize the runtime adapter
+ * 3. For each task: setup work dir, run via adapter, verify, score, teardown
+ * 4. Aggregate results into a run report
+ * 5. Optionally normalize against a baseline report
+ * 6. Save JSON report to outputDir
+ * 7. Print summary to stdout
+ *
+ * @param config   - runner configuration
+ * @param allTasks - full list of available eval tasks
+ * @returns the completed evaluation run report
+ *
+ * @example
+ * ```typescript
+ * const report = await runEvaluation(
+ *   { adapter, model: 'claude-opus-4', outputDir: './reports' },
+ *   ALL_TASKS,
+ * );
+ * ```
+ */
+export async function runEvaluation(config, allTasks) {
+    const verbose = config.verbose ?? false;
+    // -- Step 1: Filter tasks --
+    let tasks = allTasks;
+    if (config.taskIds && config.taskIds.length > 0) {
+        const idSet = new Set(config.taskIds);
+        tasks = tasks.filter((t) => idSet.has(t.id));
+    }
+    if (config.levels && config.levels.length > 0) {
+        const levelSet = new Set(config.levels);
+        tasks = tasks.filter((t) => levelSet.has(t.level));
+    }
+    log(verbose, `Running ${tasks.length} task(s) with model ${config.model}`);
+    // -- Step 2: Ensure output directory exists --
+    fs.mkdirSync(config.outputDir, { recursive: true });
+    // -- Step 3: Run each task --
+    const taskResults = [];
+    for (const task of tasks) {
+        log(verbose, `[${task.id}] ${task.title} -- starting`);
+        const workDir = setupWorkDir(task.id);
+        let result;
+        let verification = { passed: false, checks: [] };
+        try {
+            // Setup
+            await task.setup(workDir);
+            log(verbose, `[${task.id}] work dir ready: ${workDir}`);
+            // Run
+            result = await config.adapter.runTask(task, workDir);
+            log(verbose, `[${task.id}] runtime finished in ${result.durationMs}ms`);
+            // Verify
+            verification = await task.verify(workDir, result.toolCalls);
+            log(verbose, `[${task.id}] verification: ${verification.passed ? 'PASS' : 'FAIL'}`);
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            log(verbose, `[${task.id}] ERROR: ${message}`);
+            taskResults.push(createFailedResult(task, message));
+            continue;
+        }
+        finally {
+            // Teardown
+            try {
+                if (task.teardown) {
+                    await task.teardown(workDir);
+                }
+                if (!config.keepWorkDirs) {
+                    fs.rmSync(workDir, { recursive: true, force: true });
+                }
+            }
+            catch {
+                // Teardown errors are non-fatal
+            }
+        }
+        // Score
+        const taskResult = scoreTask(task, result, verification, config.runtimeId, config.model);
+        taskResults.push(taskResult);
+        log(verbose, `[${task.id}] composite score: ${taskResult.composite.toFixed(1)}`);
+    }
+    // -- Step 4: Aggregate --
+    let report = aggregateResults(taskResults, config.runtimeId, config.model);
+    // -- Step 5: Normalize to baseline if provided --
+    if (config.baselinePath) {
+        try {
+            const baselineJson = fs.readFileSync(config.baselinePath, 'utf-8');
+            const baselineReport = JSON.parse(baselineJson);
+            report = normalizeToBaseline(report, baselineReport);
+            log(verbose, 'Normalized to baseline report');
+        }
+        catch (err) {
+            log(verbose, `WARNING: Could not load baseline: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    // -- Step 6: Save report --
+    const filename = `eval-${sanitizeFilename(config.model)}-${formatTimestamp()}.json`;
+    const reportPath = path.join(config.outputDir, filename);
+    fs.writeFileSync(reportPath, JSON.stringify(report, null, 2), 'utf-8');
+    log(verbose, `Report saved to ${reportPath}`);
+    // -- Step 7: Print summary --
+    printSummary(report);
+    return report;
+}
+//# sourceMappingURL=eval-runner.js.map

package/dist/backend/backend/src/services/agent/crewly-agent/eval/eval-runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-runner.js","sourceRoot":"","sources":["../../../../../../../../backend/src/services/agent/crewly-agent/eval/eval-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAa9B,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AA8BpF,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E;;;;;GAKG;AACH,SAAS,GAAG,CAAC,OAAgB,EAAE,GAAG,IAAe;IAC/C,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,eAAe;IACtB,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;AACxD,CAAC;AAED;;;;;GAKG;AACH,SAAS,gBAAgB,CAAC,IAAY;IACpC,OAAO,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;AAC9C,CAAC;AAED;;;;;GAKG;AACH,SAAS,YAAY,CAAC,MAAc;IAClC,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,eAAe,gBAAgB,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IAC5F,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACvC,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;GAMG;AACH,SAAS,kBAAkB,CAAC,IAAc,EAAE,KAAa;IACvD,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,EAAE;QACf,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,OAAO,EAAE,KAAK;QACd,UAAU,EAAE,EAAE;QACd,SAAS,EAAE,CAAC;QACZ,OAAO,EAAE;YACP,UAAU,EAAE,CAAC;YACb,KAAK,EAAE,CAAC;YACR,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;YAC9B,OAAO,EAAE,CAAC;YACV,aAAa,EAAE,CAAC;YAChB,gBAAgB,EAAE,CAAC;YACnB,oBAAoB,EAAE,CAAC;YACvB,WAAW,EAAE,CAAC;YACd,UAAU,EAAE,CAAC;YACb,WAAW,EAAE,CAAC;YACd,YAAY,EAAE,KAAK;YACnB,QAAQ,EAAE,KAAK;SAChB;QACD,YAAY,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE;QAC3C,KAAK;KACN,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,YAAY,CAAC,MAAqB;IACzC,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;IAC1D,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IACpC,OAAO,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IAC3C,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,UAAU,SAAS,CAAC,CAAC;IAC7E,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC9D,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,MAAM,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC1E,OAAO,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC;IACxD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,mBAAmB,EAAE,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;IACrG,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;AAC5D,CAAC;AAED,8EAA8E;AAC9E,cAAc;AACd,8EAA8E;AAE9E;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAoB,EACpB,QAAoB;IAEpB,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,KAAK,CAAC;IAExC,6BAA6B;IAC7B,IAAI,KAAK,GAAG,QAAQ,CAAC;IAErB,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChD,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACtC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9C,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACxC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACrD,CAAC;IAED,GAAG,CAAC,OAAO,EAAE,WAAW,KAAK,CAAC,MAAM,uBAAuB,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IAE3E,+CAA+C;IAC/C,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEpD,8BAA8B;IAC9B,MAAM,WAAW,GAAqB,EAAE,CAAC;IAEzC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,GAAG,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,KAAK,cAAc,CAAC,CAAC;QACvD,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEtC,IAAI,MAAiC,CAAC;QACtC,IAAI,YAAY,GAAqB,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;QAEnE,IAAI,CAAC;YACH,QAAQ;YACR,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC1B,GAAG,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,EAAE,qBAAqB,OAAO,EAAE,CAAC,CAAC;YAExD,MAAM;YACN,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACrD,GAAG,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,EAAE,yBAAyB,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC;YAExE,SAAS;YACT,YAAY,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;YAC5D,GAAG,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,EAAE,mBAAmB,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;QACtF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,GAAG,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,EAAE,YAAY,OAAO,EAAE,CAAC,CAAC;YAC/C,WAAW,CAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;YACpD,SAAS;QACX,CAAC;gBAAS,CAAC;YACT,WAAW;YACX,IAAI,CAAC;gBACH,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAClB,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBAC/B,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;oBACzB,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;gBACvD,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,gCAAgC;YAClC,CAAC;QACH,CAAC;QAED,QAAQ;QACR,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QACzF,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC7B,GAAG,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,EAAE,sBAAsB,UAAU,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACnF,CAAC;IAED,0BAA0B;IAC1B,IAAI,MAAM,GAAG,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;IAE3E,kDAAkD;IAClD,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACxB,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;YACnE,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAkB,CAAC;YACjE,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;YACrD,GAAG,CAAC,OAAO,EAAE,+BAA+B,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,OAAO,EAAE,qCAAqC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACxG,CAAC;IACH,CAAC;IAED,4BAA4B;IAC5B,MAAM,QAAQ,GAAG,QAAQ,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,eAAe,EAAE,OAAO,CAAC;IACpF,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IACzD,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACvE,GAAG,CAAC,OAAO,EAAE,mBAAmB,UAAU,EAAE,CAAC,CAAC;IAE9C,8BAA8B;IAC9B,YAAY,CAAC,MAAM,CAAC,CAAC;IAErB,OAAO,MAAM,CAAC;AAChB,CAAC"}