@sylphx/flow 1.7.0 → 1.8.1

package/src/services/evaluation-service.ts

@@ -1,271 +0,0 @@
-import { spawn } from 'node:child_process';
-import fs from 'node:fs/promises';
-import path from 'node:path';
-import type { InkMonitor } from '../components/benchmark-monitor.js';
-import { DEFAULT_AGENTS, PERFORMANCE_SCORE_RANGES } from '../constants/benchmark-constants.js';
-import type { AgentTimings, AgentWork } from '../types/benchmark.js';
-import { ProcessManager } from '../utils/process-manager.js';
-
-/**
- * Evaluate agent results by running Claude to analyze agent work
- * Pure function (with side effects: file I/O, process spawning)
- */
-export async function evaluateResults(
-  outputDir: string,
-  reportDir: string | undefined,
-  monitor?: InkMonitor
-): Promise<void> {
-  // First, collect actual timing information for each agent
-  const agentTimings: AgentTimings = {};
-  const agentDirs = DEFAULT_AGENTS.map((agent) => path.join(outputDir, agent));
-
-  for (const agentDir of agentDirs) {
-    const agentName = path.basename(agentDir);
-    try {
-      // Try to read the execution-time.txt file first
-      const timingFile = path.join(agentDir, 'execution-time.txt');
-      const timingContent = await fs.readFile(timingFile, 'utf-8');
-
-      // Parse the timing information
-      const durationMatch = timingContent.match(/Duration:\s*(\d+)\s*seconds/);
-      const duration = durationMatch ? Number.parseInt(durationMatch[1], 10) : 0;
-
-      agentTimings[agentName] = { duration };
-    } catch (_error) {
-      // Fallback: try to read from timing.json
-      try {
-        const timingJsonFile = path.join(agentDir, 'timing.json');
-        const timingContent = await fs.readFile(timingJsonFile, 'utf-8');
-        const _timingData = JSON.parse(timingContent);
-
-        // If we have timing data but no duration, estimate it
-        agentTimings[agentName] = { duration: 0 }; // Unknown duration
-      } catch (_fallbackError) {
-        agentTimings[agentName] = { duration: 0 }; // No timing data available
-      }
-    }
-  }
-
-  const evaluatorPrompt = await buildEvaluationPrompt(agentTimings);
-
-  // Collect all agent work by reading their created files
-  const agentWork: AgentWork = {};
-
-  for (const agentDir of agentDirs) {
-    const agentName = path.basename(agentDir);
-    try {
-      // Read all files in agent directory
-      const files = await fs.readdir(agentDir);
-
-      // FUNCTIONAL: Build file content array instead of string accumulation
-      const fileContents: string[] = [];
-      for (const file of files) {
-        const filePath = path.join(agentDir, file);
-        const stat = await fs.stat(filePath);
-
-        if (stat.isFile()) {
-          const content = await fs.readFile(filePath, 'utf-8');
-          fileContents.push(`\n--- File: ${file} ---\n${content}\n`);
-        }
-      }
-
-      // Join at the end
-      agentWork[agentName] = `=== ${agentName} WORK ===\n\n${fileContents.join('')}`;
-    } catch (error) {
-      agentWork[agentName] = `=== ${agentName} WORK ===\n\nERROR: Could not read files - ${error}`;
-    }
-  }
-
-  // Combine all agent work into input for evaluator
-  const allWork = Object.values(agentWork).join(`\n${'='.repeat(80)}\n`);
-  const fullInput = `${evaluatorPrompt}\n\nAGENT WORK TO EVALUATE:\n${allWork}`;
-
-  // Write evaluation prompt to temp file
-  const tempEvalFile = path.join(outputDir, '.evaluation-prompt.md');
-  await fs.writeFile(tempEvalFile, fullInput);
-
-  // Add evaluation agent to monitor if available
-  if (monitor) {
-    monitor.addAgent('evaluator');
-    monitor.updateAgentStatus('evaluator', 'running');
-  }
-
-  // Run evaluation with Claude
-  const evaluationProcess = spawn(
-    'claude',
-    [
-      '--system-prompt',
-      `@${tempEvalFile}`,
-      '--dangerously-skip-permissions',
-      '--output-format',
-      'stream-json',
-      '--verbose',
-      'Please evaluate the agent work as described in the system prompt.',
-    ],
-    {
-      cwd: outputDir,
-      stdio: ['inherit', 'pipe', 'pipe'],
-      env: {
-        ...process.env,
-        FORCE_NO_PROGRESS: '1',
-        CI: '1',
-        PYTHONUNBUFFERED: '1',
-      },
-    }
-  );
-
-  // Track evaluation process for cleanup
-  ProcessManager.getInstance().trackChildProcess(evaluationProcess);
-
-  // FUNCTIONAL: Use arrays for immutable buffer accumulation
-  const evaluationOutputChunks: string[] = [];
-  let incompleteStdoutLine = '';
-
-  evaluationProcess.stdout?.on('data', (data) => {
-    const output = data.toString();
-
-    // Process complete lines only - keep incomplete data in buffer
-    const combined = incompleteStdoutLine + output;
-    const lines = combined.split('\n');
-    incompleteStdoutLine = lines.pop() || ''; // Keep last incomplete line
-
-    for (const line of lines) {
-      if (!line.trim()) {
-        continue;
-      }
-
-      try {
-        const jsonData = JSON.parse(line);
-
-        if (jsonData.type === 'assistant' && jsonData.message?.content) {
-          // Extract text content from assistant message
-          for (const content of jsonData.message.content) {
-            if (content.type === 'text') {
-              const textContent = content.text.trim();
-              if (textContent) {
-                evaluationOutputChunks.push(`${textContent}\n`);
-                // Add to monitor if available
-                monitor?.addAgentOutput('evaluator', textContent);
-              }
-            }
-          }
-        }
-      } catch (_e) {
-        // Skip invalid JSON (shouldn't happen with stream-json)
-        // For non-JSON output, add to evaluation output
-        evaluationOutputChunks.push(`${line}\n`);
-        monitor?.addAgentOutput('evaluator', line);
-      }
-    }
-  });
-
-  return new Promise((resolve, reject) => {
-    evaluationProcess.on('close', async (code) => {
-      // Update evaluator status
-      monitor?.updateAgentStatus('evaluator', code === 0 ? 'completed' : 'error');
-
-      if (code === 0) {
-        // Save report to both temp directory and optionally to project directory
-        // FUNCTIONAL: Join output chunks at the end
-        const evaluationOutput = evaluationOutputChunks.join('');
-        const tempReportPath = path.join(outputDir, 'evaluation-report.md');
-        await fs.writeFile(tempReportPath, evaluationOutput);
-
-        // Save summary of what each agent created
-        const summary = Object.entries(agentWork)
-          .map(([agent, content]) => {
-            const fileCount = (content.match(/--- File: /g) || []).length;
-            return `${agent}: ${fileCount} files created`;
-          })
-          .join('\n');
-
-        const tempSummaryPath = path.join(outputDir, 'summary.txt');
-        await fs.writeFile(tempSummaryPath, summary);
-
-        // Show completion message and display the full LLM output
-        if (monitor) {
-          monitor?.addAgentOutput('evaluator', '📊 Evaluation completed!');
-          monitor?.addAgentOutput('evaluator', `📁 Report saved to: ${tempReportPath}`);
-          monitor?.addAgentOutput('evaluator', '');
-          monitor?.addAgentOutput('evaluator', '🏆 EVALUATION RESULTS:');
-          monitor?.addAgentOutput('evaluator', '');
-
-          // Display the complete LLM evaluation output directly
-          const lines = evaluationOutput.split('\n');
-          lines.forEach((line, _index) => {
-            if (line.trim()) {
-              monitor?.addAgentOutput('evaluator', line);
-            }
-          });
-
-          monitor?.addAgentOutput('evaluator', '');
-          monitor?.addAgentOutput('evaluator', '✅ End of evaluation report');
-        }
-
-        // Clean up evaluation temp file
-        try {
-          await fs.unlink(tempEvalFile);
-        } catch (_error) {
-          // Ignore cleanup errors
-        }
-
-        // Also save to project directory if report option is provided
-        if (reportDir) {
-          const projectReportPath = path.join(process.cwd(), reportDir, 'evaluation-report.md');
-          const projectSummaryPath = path.join(process.cwd(), reportDir, 'summary.txt');
-
-          // Ensure report directory exists
-          await fs.mkdir(path.dirname(projectReportPath), { recursive: true });
-
-          await fs.writeFile(projectReportPath, evaluationOutput);
-          await fs.writeFile(projectSummaryPath, summary);
-
-          if (monitor) {
-            monitor?.addAgentOutput(
-              'evaluator',
-              `📁 Project report saved to: ${projectReportPath}`
-            );
-          }
-        }
-
-        resolve();
-      } else {
-        monitor?.addAgentOutput('evaluator', `❌ Evaluation failed with exit code ${code}`);
-        reject(new Error(`Evaluation failed with code ${code}`));
-      }
-    });
-
-    evaluationProcess.on('error', (error) => {
-      reject(error);
-    });
-  });
-}
-
-/**
- * Internal helper: Build evaluation prompt from agent timings
- */
-async function buildEvaluationPrompt(agentTimings: AgentTimings): Promise<string> {
-  // Load template from file - required file, no fallback
-  const templatePath = path.join(process.cwd(), 'templates', 'evaluation-prompt.md');
-
-  try {
-    const template = await fs.readFile(templatePath, 'utf-8');
-
-    // Generate agent performance data section
-    const performanceData = Object.entries(agentTimings)
-      .map(([agent, timing]) => {
-        const duration = timing.duration || 0;
-        const scoreRange = PERFORMANCE_SCORE_RANGES.find((range) => duration <= range.max)!;
-        return `- ${agent}: ${duration}s execution time (Performance: ${scoreRange.score}/10)`;
-      })
-      .join('\n');
-
-    // Replace template variables
-    return template.replace('{{AGENT_PERFORMANCE_DATA}}', performanceData);
-  } catch (error) {
-    const errorMessage = error instanceof Error ? error.message : String(error);
-    throw new Error(
-      `Failed to load evaluation template from ${templatePath}. Error: ${errorMessage}\n\nPlease ensure:\n1. The file exists at: ${templatePath}\n2. The file is readable (check permissions)\n3. The file contains valid markdown content`
-    );
-  }
-}

package/src/services/functional/evaluation-logic.ts

@@ -1,296 +0,0 @@
-/**
- * Business logic for evaluation service
- * Pure functions for evaluation processing
- *
- * DESIGN RATIONALE:
- * - Pure functions for data transformation
- * - Testable without file system or child processes
- * - Clear separation of I/O and logic
- * - Composable evaluation pipeline
- */
-
-import type { FileSystemError } from '../../core/functional/error-types.js';
-import { fileSystemError } from '../../core/functional/error-types.js';
-import { pipe } from '../../core/functional/pipe.js';
-import type { Result } from '../../core/functional/result.js';
-import { failure, success } from '../../core/functional/result.js';
-import { Arr, Obj, Str } from '../../utils/functional/index.js';
-
-/**
- * Domain types
- */
-
-export interface AgentTiming {
-  duration: number;
-}
-
-export interface AgentTimings {
-  [agentName: string]: AgentTiming;
-}
-
-export interface PerformanceScoreRange {
-  max: number;
-  score: number;
-  label: string;
-}
-
-export interface AgentPerformanceData {
-  agent: string;
-  duration: number;
-  score: number;
-  label: string;
-}
-
-export interface EvaluationTemplate {
-  content: string;
-  variables: string[];
-}
-
-/**
- * Pure functions for timing processing
- */
-
-/**
- * Parse duration from execution time text
- */
-export const parseDuration = (content: string): number => {
-  const match = content.match(/Duration:\s*(\d+)\s*seconds/);
-  return match ? Number.parseInt(match[1], 10) : 0;
-};
-
-/**
- * Parse timing from JSON
- */
-export const parseTimingJSON = (content: string): Result<AgentTiming, FileSystemError> => {
-  try {
-    const data = JSON.parse(content);
-    return success({ duration: data.duration || 0 });
-  } catch (error) {
-    return failure(
-      fileSystemError('Failed to parse timing JSON', '', 'read', {
-        cause: error instanceof Error ? error : undefined,
-      })
-    );
-  }
-};
-
-/**
- * Find performance score range for duration
- */
-export const findScoreRange = (
-  duration: number,
-  ranges: PerformanceScoreRange[]
-): PerformanceScoreRange => {
-  const found = ranges.find((range) => duration <= range.max);
-  return found || ranges[ranges.length - 1];
-};
-
-/**
- * Calculate agent performance data
- */
-export const calculatePerformance = (
-  agentName: string,
-  timing: AgentTiming,
-  scoreRanges: PerformanceScoreRange[]
-): AgentPerformanceData => {
-  const scoreRange = findScoreRange(timing.duration, scoreRanges);
-  return {
-    agent: agentName,
-    duration: timing.duration,
-    score: scoreRange.score,
-    label: scoreRange.label,
-  };
-};
-
-/**
- * Format agent performance as string
- */
-export const formatPerformance = (data: AgentPerformanceData): string => {
-  return `- ${data.agent}: ${data.duration}s execution time (Performance: ${data.score}/10)`;
-};
-
-/**
- * Generate performance data section
- */
-export const generatePerformanceSection = (
-  timings: AgentTimings,
-  scoreRanges: PerformanceScoreRange[]
-): string => {
-  return pipe(
-    Obj.entries(timings),
-    Arr.map(([agent, timing]) => calculatePerformance(agent, timing, scoreRanges)),
-    Arr.map(formatPerformance),
-    Str.join('\n')
-  );
-};
-
-/**
- * Template variable replacement (pure)
- */
-export const replaceTemplateVariables = (
-  template: string,
-  variables: Record<string, string>
-): string => {
-  let result = template;
-
-  for (const [key, value] of Obj.entries(variables)) {
-    const placeholder = `{{${String(key)}}}`;
-    result = Str.replaceAll(placeholder, value)(result);
-  }
-
-  return result;
-};
-
-/**
- * Parse template and find variables
- */
-export const parseTemplate = (content: string): EvaluationTemplate => {
-  const variablePattern = /\{\{([A-Z_]+)\}\}/g;
-  const variables: string[] = [];
-
-  let match;
-  while ((match = variablePattern.exec(content)) !== null) {
-    if (!variables.includes(match[1])) {
-      variables.push(match[1]);
-    }
-  }
-
-  return {
-    content,
-    variables,
-  };
-};
-
-/**
- * Build evaluation prompt from template
- */
-export const buildEvaluationPrompt = (
-  template: string,
-  timings: AgentTimings,
-  scoreRanges: PerformanceScoreRange[]
-): string => {
-  const performanceData = generatePerformanceSection(timings, scoreRanges);
-
-  return replaceTemplateVariables(template, {
-    AGENT_PERFORMANCE_DATA: performanceData,
-  });
-};
-
-/**
- * Format agent work content
- */
-export const formatAgentWork = (agentName: string, files: string[]): string => {
-  const header = `=== ${agentName} WORK ===\n\n`;
-  const fileContents = files.map((content) => content).join('\n');
-  return header + fileContents;
-};
-
-/**
- * Format file content
- */
-export const formatFileContent = (fileName: string, content: string): string => {
-  return `\n--- File: ${fileName} ---\n${content}\n`;
-};
-
-/**
- * Combine agent work sections
- */
-export const combineAgentWork = (workSections: string[]): string => {
-  const separator = `\n${'='.repeat(80)}\n`;
-  return workSections.join(separator);
-};
-
-/**
- * Build full evaluation input
- */
-export const buildEvaluationInput = (prompt: string, agentWork: string): string => {
-  return `${prompt}\n\nAGENT WORK TO EVALUATE:\n${agentWork}`;
-};
-
-/**
- * Validate evaluation template
- */
-export const validateTemplate = (template: string): Result<string, FileSystemError> => {
-  if (Str.isBlank(template)) {
-    return failure(fileSystemError('Template is empty', '', 'read'));
-  }
-
-  const parsed = parseTemplate(template);
-  if (!parsed.variables.includes('AGENT_PERFORMANCE_DATA')) {
-    return failure(
-      fileSystemError('Template missing required variable: AGENT_PERFORMANCE_DATA', '', 'read')
-    );
-  }
-
-  return success(template);
-};
-
-/**
- * Extract summary statistics from agent work
- */
-export const extractSummaryStats = (agentWork: Record<string, string>): Record<string, number> => {
-  return pipe(
-    Obj.entries(agentWork),
-    Arr.map(([agent, content]) => {
-      const fileCount = (content.match(/--- File: /g) || []).length;
-      return [agent, fileCount] as [string, number];
-    }),
-    Obj.fromEntries
-  );
-};
-
-/**
- * Format summary statistics
- */
-export const formatSummaryStats = (stats: Record<string, number>): string => {
-  return pipe(
-    Obj.entries(stats),
-    Arr.map(([agent, count]) => `${agent}: ${count} files created`),
-    Str.join('\n')
-  );
-};
-
-/**
- * Parse evaluation output (extract sections)
- */
-export const parseEvaluationOutput = (
-  output: string
-): {
-  sections: Array<{ title: string; content: string }>;
-} => {
-  const sections: Array<{ title: string; content: string }> = [];
-  const lines = Str.lines(output);
-
-  let currentSection: { title: string; content: string } | null = null;
-
-  for (const line of lines) {
-    if (line.startsWith('##')) {
-      if (currentSection) {
-        sections.push(currentSection);
-      }
-      currentSection = {
-        title: line.replace(/^##\s*/, '').trim(),
-        content: '',
-      };
-    } else if (currentSection) {
-      currentSection.content += `${line}\n`;
-    }
-  }
-
-  if (currentSection) {
-    sections.push(currentSection);
-  }
-
-  return { sections };
-};
-
-/**
- * Default performance score ranges
- */
-export const DEFAULT_SCORE_RANGES: PerformanceScoreRange[] = [
-  { max: 30, score: 10, label: 'Excellent' },
-  { max: 60, score: 8, label: 'Good' },
-  { max: 120, score: 6, label: 'Fair' },
-  { max: 240, score: 4, label: 'Slow' },
-  { max: Number.POSITIVE_INFINITY, score: 2, label: 'Very Slow' },
-];