crewly 1.6.3 → 1.6.5

package/dist/backend/backend/src/services/agent/crewly-agent/eval/context-generator.js

@@ -1,299 +0,0 @@
-/**
- * Eval Context Generator
- *
- * Generates a CONTEXT.md file and the `.crewly/` directory structure
- * that provides runtime-agnostic context for L4 eval tasks. Every
- * runtime (Crewly Agent, Claude Code, Codex CLI, Gemini CLI) receives
- * the same context package — no Crewly-specific API calls required.
- *
- * @module eval/context-generator
- */
-import * as fs from 'node:fs';
-import * as path from 'node:path';
-import { fileURLToPath } from 'node:url';
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-/**
- * Default eval team configuration used by L4 tasks.
- */
-export const DEFAULT_EVAL_TEAM = {
-    agentName: 'Sam',
-    agentRole: 'Team Lead',
-    teamName: 'Eval Product Team',
-    teamId: 'eval-team-001',
-    members: [
-        {
-            name: 'Sam',
-            role: 'Team Lead',
-            sessionName: 'eval-tl-sam',
-            agentStatus: 'active',
-            workingStatus: 'idle',
-        },
-        {
-            name: 'Leo',
-            role: 'Developer',
-            sessionName: 'eval-dev-leo',
-            agentStatus: 'active',
-            workingStatus: 'idle',
-        },
-        {
-            name: 'Max',
-            role: 'Developer',
-            sessionName: 'eval-dev-max',
-            agentStatus: 'inactive',
-            workingStatus: 'idle',
-        },
-    ],
-};
-// ---------------------------------------------------------------------------
-// CONTEXT.md generation
-// ---------------------------------------------------------------------------
-/**
- * Generate the CONTEXT.md content from a team configuration.
- *
- * The generated document tells the agent who it is, who its team members
- * are, and which bash skill scripts are available. It is intentionally
- * runtime-agnostic — any agent that can read markdown and run bash can
- * use it.
- *
- * @param config - team and agent configuration
- * @returns the full CONTEXT.md string
- */
-export function generateContextMarkdown(config) {
-    const memberRows = config.members
-        .map((m) => {
-        const workload = m.agentStatus === 'inactive' ? '—' : m.workingStatus;
-        return `| ${m.name} | ${m.role} | ${m.sessionName} | ${m.agentStatus} | ${workload} |`;
-    })
-        .join('\n');
-    const extraSection = config.extraInstructions
-        ? `\n## Additional Instructions\n\n${config.extraInstructions}\n`
-        : '';
-    return `# Team Collaboration Context
-
-## Your Identity
-You are **${config.agentName}**, ${config.agentRole} of the ${config.teamName}.
-
-## Your Team
-| Name | Role | Session | Status | Workload |
-|------|------|---------|--------|----------|
-${memberRows}
-
-## Available Skills
-You can run these bash scripts to coordinate with your team:
-
-- \`bash skills/delegate-task.sh '{"to":"<session>","task":"...","priority":"high"}'\`
-- \`bash skills/send-message.sh '{"to":"<session>","message":"..."}'\`
-- \`bash skills/get-team-status.sh '{}'\`
-- \`bash skills/verify-output.sh '{"checks":[{"name":"build","command":"npm run build"}]}'\`
-- \`bash skills/report-status.sh '{"status":"complete","summary":"..."}'\`
-- \`bash skills/handle-failure.sh '{"worker":"<session>","error":"...","action":"reassign"}'\`
-
-## Rules
-1. Always read CONTEXT.md and relevant files before taking action.
-2. Delegate to **active** workers only. Never assign work to inactive agents.
-3. Use the skill scripts above for all team coordination — do not hard-code API calls.
-4. Report your status via report-status when done.
-
-## Project Layout
-- Source code: \`src/\`
-- Tasks: \`.crewly/tasks/\`
-- Knowledge base: \`.crewly/knowledge/\`
-- Team config: \`.crewly/teams/team-config.json\`
-- Reports output: \`.crewly/reports/\`
-- Messages output: \`.crewly/messages/\`
-${extraSection}`;
-}
-// ---------------------------------------------------------------------------
-// Team config JSON generation
-// ---------------------------------------------------------------------------
-/**
- * Generate the team-config.json content from a context configuration.
- *
- * @param config - team and agent configuration
- * @returns serialized JSON string for team-config.json
- */
-export function generateTeamConfig(config) {
-    const teamConfig = {
-        id: config.teamId,
-        name: config.teamName,
-        members: config.members.map((m, idx) => ({
-            id: `member-${String(idx + 1).padStart(3, '0')}`,
-            name: m.name,
-            role: m.role.toLowerCase().replace(/\s+/g, '-'),
-            sessionName: m.sessionName,
-            agentStatus: m.agentStatus,
-            workingStatus: m.workingStatus,
-        })),
-    };
-    return JSON.stringify(teamConfig, null, 2);
-}
-// ---------------------------------------------------------------------------
-// Full workspace setup
-// ---------------------------------------------------------------------------
-/**
- * Set up a complete eval workspace with CONTEXT.md, skill scripts,
- * and the `.crewly/` directory structure.
- *
- * This is the main entry point for task setup functions. It creates
- * a self-contained directory that any runtime can work in.
- *
- * @param workDir - target directory to populate
- * @param config  - team/agent configuration (defaults to DEFAULT_EVAL_TEAM)
- * @param options - additional setup options
- */
-export async function setupEvalWorkspace(workDir, config = DEFAULT_EVAL_TEAM, options = {}) {
-    // Create directory structure
-    const dirs = [
-        '.crewly/tasks',
-        '.crewly/knowledge',
-        '.crewly/teams',
-        '.crewly/messages',
-        '.crewly/reports',
-        'skills',
-        'src/services',
-        'src/controllers',
-        'src/utils',
-    ];
-    for (const d of dirs) {
-        fs.mkdirSync(path.join(workDir, d), { recursive: true });
-    }
-    // Write CONTEXT.md
-    fs.writeFileSync(path.join(workDir, 'CONTEXT.md'), generateContextMarkdown(config));
-    // Write team config
-    fs.writeFileSync(path.join(workDir, '.crewly', 'teams', 'team-config.json'), generateTeamConfig(config));
-    // Copy skill shims
-    copySkillShims(workDir);
-    // Write default project files unless skipProjectFiles is set
-    if (!options.skipProjectFiles) {
-        writeDefaultProjectFiles(workDir);
-    }
-    // Write knowledge base files unless skipKnowledge is set
-    if (!options.skipKnowledge) {
-        writeDefaultKnowledge(workDir);
-    }
-}
-// ---------------------------------------------------------------------------
-// Internal helpers
-// ---------------------------------------------------------------------------
-/**
- * Copy skill shim scripts from the fixtures directory to the workspace.
- *
- * @param workDir - target workspace directory
- */
-function copySkillShims(workDir) {
-    const fixturesSkillsDir = path.join(__dirname, 'fixtures', 'skills');
-    const targetSkillsDir = path.join(workDir, 'skills');
-    // List of skill scripts to copy
-    const scripts = [
-        '_log.sh',
-        'delegate-task.sh',
-        'send-message.sh',
-        'get-team-status.sh',
-        'verify-output.sh',
-        'report-status.sh',
-        'handle-failure.sh',
-    ];
-    for (const script of scripts) {
-        const src = path.join(fixturesSkillsDir, script);
-        const dst = path.join(targetSkillsDir, script);
-        if (fs.existsSync(src)) {
-            fs.copyFileSync(src, dst);
-            fs.chmodSync(dst, 0o755);
-        }
-    }
-}
-/**
- * Write default project files (package.json, tsconfig.json, sample source).
- *
- * @param workDir - target workspace directory
- */
-function writeDefaultProjectFiles(workDir) {
-    fs.writeFileSync(path.join(workDir, 'package.json'), JSON.stringify({
-        name: 'eval-project',
-        version: '1.0.0',
-        type: 'module',
-        scripts: {
-            build: 'tsc --noEmit',
-            test: 'echo "4 passing" && exit 0',
-        },
-    }, null, 2));
-    fs.writeFileSync(path.join(workDir, 'tsconfig.json'), JSON.stringify({
-        compilerOptions: {
-            target: 'ES2022',
-            module: 'ESNext',
-            moduleResolution: 'bundler',
-            strict: true,
-            outDir: './dist',
-            rootDir: './src',
-            declaration: true,
-            esModuleInterop: true,
-            skipLibCheck: true,
-        },
-        include: ['src/**/*.ts'],
-    }, null, 2));
-    // Sample existing service for pattern reference
-    fs.writeFileSync(path.join(workDir, 'src', 'services', 'data.service.ts'), `/**
- * Data service for CRUD operations.
- */
-export interface DataRecord {
-  id: string;
-  value: string;
-  createdAt: number;
-}
-
-/**
- * Create a new data record.
- *
- * @param id    - unique identifier
- * @param value - record value
- * @returns the created record
- */
-export function createRecord(id: string, value: string): DataRecord {
-  return { id, value, createdAt: Date.now() };
-}
-
-/**
- * Validate a data record.
- *
- * @param record - the record to validate
- * @returns true if all fields are valid
- */
-export function validateRecord(record: DataRecord): boolean {
-  return record.id.length > 0 && record.value.length > 0 && record.createdAt > 0;
-}
-`);
-}
-/**
- * Write default knowledge base files.
- *
- * @param workDir - target workspace directory
- */
-function writeDefaultKnowledge(workDir) {
-    fs.writeFileSync(path.join(workDir, '.crewly', 'knowledge', 'project-patterns.md'), `# Project Patterns
-
-## Controller Pattern
-All controllers should:
-1. Export named functions (not classes)
-2. Accept (req, res) parameters
-3. Use try/catch for error handling
-4. Return JSON responses
-
-## Service Pattern
-All services should:
-1. Export interfaces for data types
-2. Export named functions for operations
-3. Include JSDoc comments on all exports
-4. Use TypeScript strict types (no \`any\`)
-
-## Naming Convention
-- Controllers: kebab-case with .controller.ts suffix
-- Services: kebab-case with .service.ts suffix
-- Tests: co-located as .test.ts in the same directory
-
-## Error Handling
-- Always wrap async operations in try/catch
-- Return structured error responses: { error: string, code: number }
-- Log errors before returning them
-`);
-}
-//# sourceMappingURL=context-generator.js.map

package/dist/backend/backend/src/services/agent/crewly-agent/eval/context-generator.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"context-generator.js","sourceRoot":"","sources":["../../../../../../../../backend/src/services/agent/crewly-agent/eval/context-generator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAwC/D;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAsB;IAClD,SAAS,EAAE,KAAK;IAChB,SAAS,EAAE,WAAW;IACtB,QAAQ,EAAE,mBAAmB;IAC7B,MAAM,EAAE,eAAe;IACvB,OAAO,EAAE;QACP;YACE,IAAI,EAAE,KAAK;YACX,IAAI,EAAE,WAAW;YACjB,WAAW,EAAE,aAAa;YAC1B,WAAW,EAAE,QAAQ;YACrB,aAAa,EAAE,MAAM;SACtB;QACD;YACE,IAAI,EAAE,KAAK;YACX,IAAI,EAAE,WAAW;YACjB,WAAW,EAAE,cAAc;YAC3B,WAAW,EAAE,QAAQ;YACrB,aAAa,EAAE,MAAM;SACtB;QACD;YACE,IAAI,EAAE,KAAK;YACX,IAAI,EAAE,WAAW;YACjB,WAAW,EAAE,cAAc;YAC3B,WAAW,EAAE,UAAU;YACvB,aAAa,EAAE,MAAM;SACtB;KACF;CACF,CAAC;AAEF,8EAA8E;AAC9E,wBAAwB;AACxB,8EAA8E;AAE9E;;;;;;;;;;GAUG;AACH,MAAM,UAAU,uBAAuB,CAAC,MAAyB;IAC/D,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO;SAC9B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,QAAQ,GAAG,CAAC,CAAC,WAAW,KAAK,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;QACtE,OAAO,KAAK,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,WAAW,MAAM,CAAC,CAAC,WAAW,MAAM,QAAQ,IAAI,CAAC;IACzF,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,MAAM,YAAY,GAAG,MAAM,CAAC,iBAAiB;QAC3C,CAAC,CAAC,mCAAmC,MAAM,CAAC,iBAAiB,IAAI;QACjE,CAAC,CAAC,EAAE,CAAC;IAEP,OAAO;;;YAGG,MAAM,CAAC,SAAS,OAAO,MAAM,CAAC,SAAS,WAAW,MAAM,CAAC,QAAQ;;;;;EAK3E,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;EAyBV,YAAY,EAAE,CAAC;AACjB,CAAC;AAED,8EAA8E;AAC9E,8BAA8B;AAC9B,8EAA8E;AAE9E;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAAyB;IAC1D,MAAM,UAAU,GAAG;QACjB,EAAE,EAAE,MAAM,CAAC,MAAM;QACjB,IAAI,EAAE,MAAM,CAAC,QAAQ;QACrB,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;YACvC,EAAE,EAAE,UAAU,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;YAChD,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;YAC/C,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,aAAa,EAAE,CAAC,CAAC,aAAa;SAC/B,CAAC,CAAC;KACJ,CAAC;IACF,OAAO,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AAC7C,CAAC;AAED,8EAA8E;AAC9E,uBAAuB;AACvB,8EAA8E;AAE9E;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,OAAe,EACf,SAA4B,iBAAiB,EAC7C,UAAwB,EAAE;IAE1B,6BAA6B;IAC7B,MAAM,IAAI,GAAG;QACX,eAAe;QACf,mBAAmB;QACnB,eAAe;QACf,kBAAkB;QAClB,iBAAiB;QACjB,QAAQ;QACR,cAAc;QACd,iBAAiB;QACjB,WAAW;KACZ,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,mBAAmB;IACnB,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,EAChC,uBAAuB,CAAC,MAAM,CAAC,CAChC,CAAC;IAEF,oBAAoB;IACpB,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,kBAAkB,CAAC,EAC1D,kBAAkB,CAAC,MAAM,CAAC,CAC3B,CAAC;IAEF,mBAAmB;IACnB,cAAc,CAAC,OAAO,CAAC,CAAC;IAExB,6DAA6D;IAC7D,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;QAC9B,wBAAwB,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;IAED,yDAAyD;IACzD,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;QAC3B,qBAAqB,CAAC,OAAO,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAYD,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E;;;;GAIG;AACH,SAAS,cAAc,CAAC,OAAe;IACrC,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;IACrE,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAErD,gCAAgC;IAChC,MAAM,OAAO,GAAG;QACd,SAAS;QACT,kBAAkB;QAClB,iBAAiB;QACjB,oBAAoB;QACpB,kBAAkB;QAClB,kBAAkB;QAClB,mBAAmB;KACpB,CAAC;IAEF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;QACjD,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;QAC/C,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACvB,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YAC1B,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,wBAAwB,CAAC,OAAe;IAC/C,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,EAClC,IAAI,CAAC,SAAS,CACZ;QACE,IAAI,EAAE,cAAc;QACpB,OAAO,EAAE,OAAO;QAChB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE;YACP,KAAK,EAAE,cAAc;YACrB,IAAI,EAAE,4BAA4B;SACnC;KACF,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;IAEF,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EACnC,IAAI,CAAC,SAAS,CACZ;QACE,eAAe,EAAE;YACf,MAAM,EAAE,QAAQ;YAChB,MAAM,EAAE,QAAQ;YAChB,gBAAgB,EAAE,SAAS;YAC3B,MAAM,EAAE,IAAI;YACZ,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE,OAAO;YAChB,WAAW,EAAE,IAAI;YACjB,eAAe,EAAE,IAAI;YACrB,YAAY,EAAE,IAAI;SACnB;QACD,OAAO,EAAE,CAAC,aAAa,CAAC;KACzB,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;IAEF,gDAAgD;IAChD,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,iBAAiB,CAAC,EACxD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6BH,CACE,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,qBAAqB,CAAC,OAAe;IAC5C,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,qBAAqB,CAAC,EACjE;;;;;;;;;;;;;;;;;;;;;;;;;CAyBH,CACE,CAAC;AACJ,CAAC"}

package/dist/backend/backend/src/services/agent/crewly-agent/eval/eval-runner.d.ts

@@ -1,59 +0,0 @@
-/**
- * Crewly Agent Evaluation Framework - Runner
- *
- * Orchestrates a full evaluation run: filters tasks, initializes the
- * runtime adapter, executes tasks sequentially, scores results, and
- * produces a final report.
- *
- * @module eval-runner
- */
-import type { EvalLevel, EvalRunReport, EvalTask, RuntimeAdapter, RuntimeId } from './eval-types.js';
-/**
- * Configuration for {@link runEvaluation}.
- */
-export interface RunnerConfig {
-    /** Initialized runtime adapter */
-    adapter: RuntimeAdapter;
-    /** Runtime identifier (used in reports and file names) */
-    runtimeId: RuntimeId;
-    /** Model identifier for cost calculation */
-    model: string;
-    /** Optional list of specific task IDs to run */
-    taskIds?: string[];
-    /** Optional list of levels to include */
-    levels?: EvalLevel[];
-    /** Directory where report JSON files are written */
-    outputDir: string;
-    /** Path to a baseline report JSON for normalization */
-    baselinePath?: string;
-    /** If true, keep temporary work directories after evaluation */
-    keepWorkDirs?: boolean;
-    /** If true, emit verbose progress logs */
-    verbose?: boolean;
-}
-/**
- * Run a full evaluation across the provided tasks.
- *
- * The runner performs the following steps:
- * 1. Filter tasks by ID and/or level if specified in config
- * 2. Initialize the runtime adapter
- * 3. For each task: setup work dir, run via adapter, verify, score, teardown
- * 4. Aggregate results into a run report
- * 5. Optionally normalize against a baseline report
- * 6. Save JSON report to outputDir
- * 7. Print summary to stdout
- *
- * @param config   - runner configuration
- * @param allTasks - full list of available eval tasks
- * @returns the completed evaluation run report
- *
- * @example
- * ```typescript
- * const report = await runEvaluation(
- *   { adapter, model: 'claude-opus-4', outputDir: './reports' },
- *   ALL_TASKS,
- * );
- * ```
- */
-export declare function runEvaluation(config: RunnerConfig, allTasks: EvalTask[]): Promise<EvalRunReport>;
-//# sourceMappingURL=eval-runner.d.ts.map

package/dist/backend/backend/src/services/agent/crewly-agent/eval/eval-runner.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"eval-runner.d.ts","sourceRoot":"","sources":["../../../../../../../../backend/src/services/agent/crewly-agent/eval/eval-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAMH,OAAO,KAAK,EACV,SAAS,EACT,aAAa,EACb,QAAQ,EAGR,cAAc,EACd,SAAS,EAEV,MAAM,iBAAiB,CAAC;AAQzB;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,kCAAkC;IAClC,OAAO,EAAE,cAAc,CAAC;IACxB,0DAA0D;IAC1D,SAAS,EAAE,SAAS,CAAC;IACrB,4CAA4C;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,gDAAgD;IAChD,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,yCAAyC;IACzC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC;IACrB,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,uDAAuD;IACvD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gEAAgE;IAChE,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,0CAA0C;IAC1C,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AA6GD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,YAAY,EACpB,QAAQ,EAAE,QAAQ,EAAE,GACnB,OAAO,CAAC,aAAa,CAAC,CA6FxB"}

package/dist/backend/backend/src/services/agent/crewly-agent/eval/eval-runner.js

@@ -1,218 +0,0 @@
-/**
- * Crewly Agent Evaluation Framework - Runner
- *
- * Orchestrates a full evaluation run: filters tasks, initializes the
- * runtime adapter, executes tasks sequentially, scores results, and
- * produces a final report.
- *
- * @module eval-runner
- */
-import * as fs from 'node:fs';
-import * as path from 'node:path';
-import * as os from 'node:os';
-import { aggregateResults, normalizeToBaseline, scoreTask } from './eval-scorer.js';
-// ---------------------------------------------------------------------------
-// Internal helpers
-// ---------------------------------------------------------------------------
-/**
- * Log a message to stdout when verbose mode is enabled.
- *
- * @param verbose - whether verbose logging is on
- * @param args    - values to log
- */
-function log(verbose, ...args) {
-    if (verbose) {
-        console.log('[eval]', ...args);
-    }
-}
-/**
- * Format a timestamp for use in filenames (ISO without colons).
- *
- * @returns formatted timestamp string
- */
-function formatTimestamp() {
-    return new Date().toISOString().replace(/[:.]/g, '-');
-}
-/**
- * Sanitize a string for use as a filename component.
- *
- * @param name - raw string to sanitize
- * @returns filesystem-safe string
- */
-function sanitizeFilename(name) {
-    return name.replace(/[^a-zA-Z0-9_-]/g, '_');
-}
-/**
- * Create a temporary work directory for a single task.
- *
- * @param taskId - task identifier used in the directory name
- * @returns absolute path to the created directory
- */
-function setupWorkDir(taskId) {
-    const dir = path.join(os.tmpdir(), `crewly-eval-${sanitizeFilename(taskId)}-${Date.now()}`);
-    fs.mkdirSync(dir, { recursive: true });
-    return dir;
-}
-/**
- * Build a failed {@link EvalTaskResult} for tasks that error before scoring.
- *
- * @param task  - the task that failed
- * @param error - error message
- * @returns a task result with zero scores
- */
-function createFailedResult(task, error) {
-    return {
-        taskId: task.id,
-        level: task.level,
-        title: task.title,
-        success: false,
-        dimensions: [],
-        composite: 0,
-        metrics: {
-            durationMs: 0,
-            steps: 0,
-            usage: { input: 0, output: 0 },
-            costUSD: 0,
-            toolCallCount: 0,
-            correctToolCalls: 0,
-            unnecessaryToolCalls: 0,
-            argAccuracy: 0,
-            toolErrors: 0,
-            humanNudges: 0,
-            loopDetected: false,
-            timedOut: false,
-        },
-        verification: { passed: false, checks: [] },
-        error,
-    };
-}
-/**
- * Print a human-readable summary of the evaluation run to stdout.
- *
- * @param report - the completed evaluation report
- */
-function printSummary(report) {
-    console.log('\n========================================');
-    console.log('  Evaluation Summary');
-    console.log('========================================');
-    console.log(`  Runtime : ${report.runtimeId}`);
-    console.log(`  Model   : ${report.model}`);
-    console.log(`  Tasks   : ${report.passedCount}/${report.totalCount} passed`);
-    console.log(`  Score   : ${report.overallComposite.toFixed(1)}/100`);
-    console.log(`  Cost    : $${report.totalCostUSD.toFixed(4)}`);
-    console.log(`  Duration: ${(report.totalDurationMs / 1000).toFixed(1)}s`);
-    console.log('----------------------------------------');
-    for (const dim of report.aggregateDimensions) {
-        console.log(`  ${dim.dimensionId.padEnd(20)} ${dim.raw.toFixed(1).padStart(6)} (w=${dim.weight})`);
-    }
-    console.log('========================================\n');
-}
-// ---------------------------------------------------------------------------
-// Main runner
-// ---------------------------------------------------------------------------
-/**
- * Run a full evaluation across the provided tasks.
- *
- * The runner performs the following steps:
- * 1. Filter tasks by ID and/or level if specified in config
- * 2. Initialize the runtime adapter
- * 3. For each task: setup work dir, run via adapter, verify, score, teardown
- * 4. Aggregate results into a run report
- * 5. Optionally normalize against a baseline report
- * 6. Save JSON report to outputDir
- * 7. Print summary to stdout
- *
- * @param config   - runner configuration
- * @param allTasks - full list of available eval tasks
- * @returns the completed evaluation run report
- *
- * @example
- * ```typescript
- * const report = await runEvaluation(
- *   { adapter, model: 'claude-opus-4', outputDir: './reports' },
- *   ALL_TASKS,
- * );
- * ```
- */
-export async function runEvaluation(config, allTasks) {
-    const verbose = config.verbose ?? false;
-    // -- Step 1: Filter tasks --
-    let tasks = allTasks;
-    if (config.taskIds && config.taskIds.length > 0) {
-        const idSet = new Set(config.taskIds);
-        tasks = tasks.filter((t) => idSet.has(t.id));
-    }
-    if (config.levels && config.levels.length > 0) {
-        const levelSet = new Set(config.levels);
-        tasks = tasks.filter((t) => levelSet.has(t.level));
-    }
-    log(verbose, `Running ${tasks.length} task(s) with model ${config.model}`);
-    // -- Step 2: Ensure output directory exists --
-    fs.mkdirSync(config.outputDir, { recursive: true });
-    // -- Step 3: Run each task --
-    const taskResults = [];
-    for (const task of tasks) {
-        log(verbose, `[${task.id}] ${task.title} -- starting`);
-        const workDir = setupWorkDir(task.id);
-        let result;
-        let verification = { passed: false, checks: [] };
-        try {
-            // Setup
-            await task.setup(workDir);
-            log(verbose, `[${task.id}] work dir ready: ${workDir}`);
-            // Run
-            result = await config.adapter.runTask(task, workDir);
-            log(verbose, `[${task.id}] runtime finished in ${result.durationMs}ms`);
-            // Verify
-            verification = await task.verify(workDir, result.toolCalls);
-            log(verbose, `[${task.id}] verification: ${verification.passed ? 'PASS' : 'FAIL'}`);
-        }
-        catch (err) {
-            const message = err instanceof Error ? err.message : String(err);
-            log(verbose, `[${task.id}] ERROR: ${message}`);
-            taskResults.push(createFailedResult(task, message));
-            continue;
-        }
-        finally {
-            // Teardown
-            try {
-                if (task.teardown) {
-                    await task.teardown(workDir);
-                }
-                if (!config.keepWorkDirs) {
-                    fs.rmSync(workDir, { recursive: true, force: true });
-                }
-            }
-            catch {
-                // Teardown errors are non-fatal
-            }
-        }
-        // Score
-        const taskResult = scoreTask(task, result, verification, config.runtimeId, config.model);
-        taskResults.push(taskResult);
-        log(verbose, `[${task.id}] composite score: ${taskResult.composite.toFixed(1)}`);
-    }
-    // -- Step 4: Aggregate --
-    let report = aggregateResults(taskResults, config.runtimeId, config.model);
-    // -- Step 5: Normalize to baseline if provided --
-    if (config.baselinePath) {
-        try {
-            const baselineJson = fs.readFileSync(config.baselinePath, 'utf-8');
-            const baselineReport = JSON.parse(baselineJson);
-            report = normalizeToBaseline(report, baselineReport);
-            log(verbose, 'Normalized to baseline report');
-        }
-        catch (err) {
-            log(verbose, `WARNING: Could not load baseline: ${err instanceof Error ? err.message : String(err)}`);
-        }
-    }
-    // -- Step 6: Save report --
-    const filename = `eval-${sanitizeFilename(config.model)}-${formatTimestamp()}.json`;
-    const reportPath = path.join(config.outputDir, filename);
-    fs.writeFileSync(reportPath, JSON.stringify(report, null, 2), 'utf-8');
-    log(verbose, `Report saved to ${reportPath}`);
-    // -- Step 7: Print summary --
-    printSummary(report);
-    return report;
-}
-//# sourceMappingURL=eval-runner.js.map

package/dist/backend/backend/src/services/agent/crewly-agent/eval/eval-runner.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"eval-runner.js","sourceRoot":"","sources":["../../../../../../../../backend/src/services/agent/crewly-agent/eval/eval-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAa9B,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AA8BpF,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E;;;;;GAKG;AACH,SAAS,GAAG,CAAC,OAAgB,EAAE,GAAG,IAAe;IAC/C,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,eAAe;IACtB,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;AACxD,CAAC;AAED;;;;;GAKG;AACH,SAAS,gBAAgB,CAAC,IAAY;IACpC,OAAO,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;AAC9C,CAAC;AAED;;;;;GAKG;AACH,SAAS,YAAY,CAAC,MAAc;IAClC,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,eAAe,gBAAgB,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IAC5F,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACvC,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;GAMG;AACH,SAAS,kBAAkB,CAAC,IAAc,EAAE,KAAa;IACvD,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,EAAE;QACf,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,OAAO,EAAE,KAAK;QACd,UAAU,EAAE,EAAE;QACd,SAAS,EAAE,CAAC;QACZ,OAAO,EAAE;YACP,UAAU,EAAE,CAAC;YACb,KAAK,EAAE,CAAC;YACR,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;YAC9B,OAAO,EAAE,CAAC;YACV,aAAa,EAAE,CAAC;YAChB,gBAAgB,EAAE,CAAC;YACnB,oBAAoB,EAAE,CAAC;YACvB,WAAW,EAAE,CAAC;YACd,UAAU,EAAE,CAAC;YACb,WAAW,EAAE,CAAC;YACd,YAAY,EAAE,KAAK;YACnB,QAAQ,EAAE,KAAK;SAChB;QACD,YAAY,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE;QAC3C,KAAK;KACN,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,YAAY,CAAC,MAAqB;IACzC,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;IAC1D,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IACpC,OAAO,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IAC3C,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,UAAU,SAAS,CAAC,CAAC;IAC7E,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC9D,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,MAAM,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC1E,OAAO,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC;IACxD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,mBAAmB,EAAE,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;IACrG,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;AAC5D,CAAC;AAED,8EAA8E;AAC9E,cAAc;AACd,8EAA8E;AAE9E;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAoB,EACpB,QAAoB;IAEpB,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,KAAK,CAAC;IAExC,6BAA6B;IAC7B,IAAI,KAAK,GAAG,QAAQ,CAAC;IAErB,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChD,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACtC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9C,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACxC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACrD,CAAC;IAED,GAAG,CAAC,OAAO,EAAE,WAAW,KAAK,CAAC,MAAM,uBAAuB,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IAE3E,+CAA+C;IAC/C,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEpD,8BAA8B;IAC9B,MAAM,WAAW,GAAqB,EAAE,CAAC;IAEzC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,GAAG,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,KAAK,cAAc,CAAC,CAAC;QACvD,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEtC,IAAI,MAAiC,CAAC;QACtC,IAAI,YAAY,GAAqB,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;QAEnE,IAAI,CAAC;YACH,QAAQ;YACR,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC1B,GAAG,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,EAAE,qBAAqB,OAAO,EAAE,CAAC,CAAC;YAExD,MAAM;YACN,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACrD,GAAG,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,EAAE,yBAAyB,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC;YAExE,SAAS;YACT,YAAY,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;YAC5D,GAAG,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,EAAE,mBAAmB,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;QACtF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,GAAG,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,EAAE,YAAY,OAAO,EAAE,CAAC,CAAC;YAC/C,WAAW,CAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;YACpD,SAAS;QACX,CAAC;gBAAS,CAAC;YACT,WAAW;YACX,IAAI,CAAC;gBACH,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAClB,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBAC/B,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;oBACzB,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;gBACvD,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,gCAAgC;YAClC,CAAC;QACH,CAAC;QAED,QAAQ;QACR,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QACzF,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC7B,GAAG,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,EAAE,sBAAsB,UAAU,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACnF,CAAC;IAED,0BAA0B;IAC1B,IAAI,MAAM,GAAG,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;IAE3E,kDAAkD;IAClD,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACxB,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;YACnE,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAkB,CAAC;YACjE,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;YACrD,GAAG,CAAC,OAAO,EAAE,+BAA+B,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,OAAO,EAAE,qCAAqC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACxG,CAAC;IACH,CAAC;IAED,4BAA4B;IAC5B,MAAM,QAAQ,GAAG,QAAQ,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,eAAe,EAAE,OAAO,CAAC;IACpF,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IACzD,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACvE,GAAG,CAAC,OAAO,EAAE,mBAAmB,UAAU,EAAE,CAAC,CAAC;IAE9C,8BAA8B;IAC9B,YAAY,CAAC,MAAM,CAAC,CAAC;IAErB,OAAO,MAAM,CAAC;AAChB,CAAC"}