@sylphx/flow 1.0.1 → 1.0.3

package/src/services/config-service.ts

@@ -0,0 +1,258 @@
+/**
+ * Configuration service - handles layered configuration loading
+ * Priority: local > project > home
+ */
+
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import os from 'node:os';
+import {
+  CONFIG_DIR,
+  USER_SETTINGS_FILE,
+  getProjectSettingsFile,
+  getProjectLocalSettingsFile
+} from '../config/constants.js';
+
+/**
+ * User configuration (sensitive data, saved to home directory)
+ * This is set up once and reused across projects
+ */
+export interface UserSettings {
+  claudeProvider?: string;
+  claudeProviderConfig?: {
+    ANTHROPIC_BASE_URL: string;
+    description: string;
+  };
+  claudeApiKey?: string;
+
+  // API keys for providers
+  apiKeys?: {
+    kimi?: string;     // Kimi provider
+    'z.ai'?: string;   // Z.ai proxy
+  };
+
+  // User preferences (can be changed anytime)
+  defaultProvider?: string;
+  defaultAgent?: string;
+  hasCompletedSetup?: boolean;
+
+  [key: string]: unknown;
+}
+
+/**
+ * Project configuration (shareable, saved to project directory)
+ */
+export interface ProjectSettings {
+  target?: string;
+  version?: string;
+  defaultAgent?: string;  // Can override user default per project
+
+  [key: string]: unknown;
+}
+
+/**
+ * Runtime choices (temporary, not saved)
+ * These are selected each run but can be overridden by CLI flags
+ */
+export interface RuntimeChoices {
+  provider?: string;  // Selected for this run
+  agent?: string;     // Selected for this run
+  prompt?: string;    // User prompt for this run
+
+  [key: string]: unknown;
+}
+
+export class ConfigService {
+  /**
+   * Load complete configuration (user + project + local)
+   */
+  static async loadConfiguration(cwd: string = process.cwd()): Promise<{
+    user: UserSettings;
+    project: ProjectSettings;
+    choices: RuntimeChoices;
+  }> {
+    const userSettings = await this.loadHomeSettings();
+    const projectSettings = await this.loadProjectSettings(cwd);
+    const localSettings = await this.loadLocalSettings(cwd);
+
+    // Runtime choices merge: local > project > user defaults
+    const choices: RuntimeChoices = {
+      provider: localSettings.provider || userSettings.defaultProvider,
+      agent: localSettings.agent || projectSettings.defaultAgent || userSettings.defaultAgent,
+    };
+
+    return {
+      user: userSettings,
+      project: projectSettings,
+      choices,
+    };
+  }
+
+  /**
+   * Legacy method for backward compatibility
+   */
+  static async loadSettings(cwd: string = process.cwd()): Promise<any> {
+    const config = await this.loadConfiguration(cwd);
+    return {
+      ...config.user,
+      ...config.project,
+      ...config.choices,
+    };
+  }
+
+  /**
+   * Load user global settings (mainly for API keys)
+   */
+  static async loadHomeSettings(): Promise<UserSettings> {
+    try {
+      const content = await fs.readFile(USER_SETTINGS_FILE, 'utf-8');
+      return JSON.parse(content);
+    } catch {
+      return {};
+    }
+  }
+
+  /**
+   * Save user global settings
+   */
+  static async saveHomeSettings(settings: UserSettings): Promise<void> {
+    // Ensure directory exists
+    await fs.mkdir(USER_SETTINGS_FILE.replace('/settings.json', ''), { recursive: true });
+
+    // Merge with existing settings and save
+    const existing = await this.loadHomeSettings();
+    const merged = { ...existing, ...settings };
+    await fs.writeFile(USER_SETTINGS_FILE, JSON.stringify(merged, null, 2) + '\n');
+  }
+
+  /**
+   * Check if user has completed initial setup (API keys configured)
+   */
+  static async hasInitialSetup(): Promise<boolean> {
+    const userSettings = await this.loadHomeSettings();
+    // Check if user has completed setup (either has API keys OR has explicitly chosen default)
+    return !!(userSettings.hasCompletedSetup || (userSettings.apiKeys && Object.keys(userSettings.apiKeys).length > 0));
+  }
+
+  /**
+   * Get available providers (those with API keys configured)
+   * Always includes 'default' (no override)
+   */
+  static getAvailableProviders(userSettings: UserSettings): string[] {
+    const providers: string[] = ['default']; // Always available
+    if (userSettings.apiKeys?.kimi) providers.push('kimi');
+    if (userSettings.apiKeys?.['z.ai']) providers.push('z.ai');
+    return providers;
+  }
+
+  /**
+   * Load project-level settings
+   */
+  static async loadProjectSettings(cwd: string = process.cwd()): Promise<ProjectSettings> {
+    try {
+      const configPath = getProjectSettingsFile(cwd);
+      const content = await fs.readFile(configPath, 'utf-8');
+      return JSON.parse(content);
+    } catch {
+      return {};
+    }
+  }
+
+  /**
+   * Save project-level settings
+   */
+  static async saveProjectSettings(settings: ProjectSettings, cwd: string = process.cwd()): Promise<void> {
+    // Ensure directory exists
+    const configDir = path.join(cwd, CONFIG_DIR);
+    await fs.mkdir(configDir, { recursive: true });
+
+    // Merge with existing settings and save
+    const existing = await this.loadProjectSettings(cwd);
+    const merged = { ...existing, ...settings };
+
+    const configPath = getProjectSettingsFile(cwd);
+    await fs.writeFile(configPath, JSON.stringify(merged, null, 2) + '\n');
+  }
+
+  /**
+   * Load project-local settings (overrides everything)
+   */
+  static async loadLocalSettings(cwd: string = process.cwd()): Promise<RuntimeChoices> {
+    try {
+      const configPath = getProjectLocalSettingsFile(cwd);
+      const content = await fs.readFile(configPath, 'utf-8');
+      return JSON.parse(content);
+    } catch {
+      return {};
+    }
+  }
+
+  /**
+   * Save project-local settings
+   */
+  static async saveLocalSettings(settings: RuntimeChoices, cwd: string = process.cwd()): Promise<void> {
+    // Ensure directory exists
+    const configDir = path.join(cwd, CONFIG_DIR);
+    await fs.mkdir(configDir, { recursive: true });
+
+    const configPath = getProjectLocalSettingsFile(cwd);
+    await fs.writeFile(configPath, JSON.stringify(settings, null, 2) + '\n');
+  }
+
+  /**
+   * Convenient method: save Claude configuration (API keys to home, other settings to project)
+   */
+  static async saveClaudeConfig(
+    projectConfig: ProjectSettings,
+    userConfig: UserSettings,
+    cwd: string = process.cwd()
+  ): Promise<void> {
+    // Save API keys to home directory
+    if (userConfig.claudeApiKey || userConfig.claudeProvider || userConfig.claudeProviderConfig) {
+      await this.saveHomeSettings(userConfig);
+    }
+
+    // Save other settings to project
+    await this.saveProjectSettings(projectConfig, cwd);
+
+    // Create .gitignore pattern file if it doesn't exist (excluding .local.json)
+    await this.addGitignore(cwd);
+  }
+
+  /**
+   * Add .sylphx-flow/ to .gitignore with proper patterns
+   */
+  private static async addGitignore(cwd: string): Promise<void> {
+    const gitignorePath = path.join(cwd, '.gitignore');
+    const patterns = [
+      '',
+      '# Sylphx Flow - local settings (never commit)',
+      '.sylphx-flow/*.local.json',
+    ];
+
+    try {
+      const content = await fs.readFile(gitignorePath, 'utf-8');
+
+      // Check if pattern already exists
+      if (!content.includes('.sylphx-flow/*.local.json')) {
+        await fs.appendFile(gitignorePath, patterns.join('\n') + '\n');
+      }
+    } catch {
+      // .gitignore doesn't exist - create it
+      await fs.writeFile(gitignorePath, patterns.join('\n').trim() + '\n');
+    }
+  }
+
+  /**
+   * Check if config directory exists
+   */
+  static async isInitialized(cwd: string = process.cwd()): Promise<boolean> {
+    try {
+      const configDir = path.join(cwd, CONFIG_DIR);
+      await fs.access(configDir);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+}

package/src/services/evaluation-service.ts

@@ -0,0 +1,271 @@
+import { spawn } from 'node:child_process';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import type { InkMonitor } from '../components/benchmark-monitor.js';
+import { DEFAULT_AGENTS, PERFORMANCE_SCORE_RANGES } from '../constants/benchmark-constants.js';
+import type { AgentTimings, AgentWork } from '../types/benchmark.js';
+import { ProcessManager } from '../utils/process-manager.js';
+
+/**
+ * Evaluate agent results by running Claude to analyze agent work
+ * Pure function (with side effects: file I/O, process spawning)
+ */
+export async function evaluateResults(
+  outputDir: string,
+  reportDir: string | undefined,
+  monitor?: InkMonitor
+): Promise<void> {
+  // First, collect actual timing information for each agent
+  const agentTimings: AgentTimings = {};
+  const agentDirs = DEFAULT_AGENTS.map((agent) => path.join(outputDir, agent));
+
+  for (const agentDir of agentDirs) {
+    const agentName = path.basename(agentDir);
+    try {
+      // Try to read the execution-time.txt file first
+      const timingFile = path.join(agentDir, 'execution-time.txt');
+      const timingContent = await fs.readFile(timingFile, 'utf-8');
+
+      // Parse the timing information
+      const durationMatch = timingContent.match(/Duration:\s*(\d+)\s*seconds/);
+      const duration = durationMatch ? Number.parseInt(durationMatch[1], 10) : 0;
+
+      agentTimings[agentName] = { duration };
+    } catch (_error) {
+      // Fallback: try to read from timing.json
+      try {
+        const timingJsonFile = path.join(agentDir, 'timing.json');
+        const timingContent = await fs.readFile(timingJsonFile, 'utf-8');
+        const _timingData = JSON.parse(timingContent);
+
+        // If we have timing data but no duration, estimate it
+        agentTimings[agentName] = { duration: 0 }; // Unknown duration
+      } catch (_fallbackError) {
+        agentTimings[agentName] = { duration: 0 }; // No timing data available
+      }
+    }
+  }
+
+  const evaluatorPrompt = await buildEvaluationPrompt(agentTimings);
+
+  // Collect all agent work by reading their created files
+  const agentWork: AgentWork = {};
+
+  for (const agentDir of agentDirs) {
+    const agentName = path.basename(agentDir);
+    try {
+      // Read all files in agent directory
+      const files = await fs.readdir(agentDir);
+
+      // FUNCTIONAL: Build file content array instead of string accumulation
+      const fileContents: string[] = [];
+      for (const file of files) {
+        const filePath = path.join(agentDir, file);
+        const stat = await fs.stat(filePath);
+
+        if (stat.isFile()) {
+          const content = await fs.readFile(filePath, 'utf-8');
+          fileContents.push(`\n--- File: ${file} ---\n${content}\n`);
+        }
+      }
+
+      // Join at the end
+      agentWork[agentName] = `=== ${agentName} WORK ===\n\n${fileContents.join('')}`;
+    } catch (error) {
+      agentWork[agentName] = `=== ${agentName} WORK ===\n\nERROR: Could not read files - ${error}`;
+    }
+  }
+
+  // Combine all agent work into input for evaluator
+  const allWork = Object.values(agentWork).join(`\n${'='.repeat(80)}\n`);
+  const fullInput = `${evaluatorPrompt}\n\nAGENT WORK TO EVALUATE:\n${allWork}`;
+
+  // Write evaluation prompt to temp file
+  const tempEvalFile = path.join(outputDir, '.evaluation-prompt.md');
+  await fs.writeFile(tempEvalFile, fullInput);
+
+  // Add evaluation agent to monitor if available
+  if (monitor) {
+    monitor.addAgent('evaluator');
+    monitor.updateAgentStatus('evaluator', 'running');
+  }
+
+  // Run evaluation with Claude
+  const evaluationProcess = spawn(
+    'claude',
+    [
+      '--system-prompt',
+      `@${tempEvalFile}`,
+      '--dangerously-skip-permissions',
+      '--output-format',
+      'stream-json',
+      '--verbose',
+      'Please evaluate the agent work as described in the system prompt.',
+    ],
+    {
+      cwd: outputDir,
+      stdio: ['inherit', 'pipe', 'pipe'],
+      env: {
+        ...process.env,
+        FORCE_NO_PROGRESS: '1',
+        CI: '1',
+        PYTHONUNBUFFERED: '1',
+      },
+    }
+  );
+
+  // Track evaluation process for cleanup
+  ProcessManager.getInstance().trackChildProcess(evaluationProcess);
+
+  // FUNCTIONAL: Use arrays for immutable buffer accumulation
+  const evaluationOutputChunks: string[] = [];
+  let incompleteStdoutLine = '';
+
+  evaluationProcess.stdout?.on('data', (data) => {
+    const output = data.toString();
+
+    // Process complete lines only - keep incomplete data in buffer
+    const combined = incompleteStdoutLine + output;
+    const lines = combined.split('\n');
+    incompleteStdoutLine = lines.pop() || ''; // Keep last incomplete line
+
+    for (const line of lines) {
+      if (!line.trim()) {
+        continue;
+      }
+
+      try {
+        const jsonData = JSON.parse(line);
+
+        if (jsonData.type === 'assistant' && jsonData.message?.content) {
+          // Extract text content from assistant message
+          for (const content of jsonData.message.content) {
+            if (content.type === 'text') {
+              const textContent = content.text.trim();
+              if (textContent) {
+                evaluationOutputChunks.push(`${textContent}\n`);
+                // Add to monitor if available
+                monitor?.addAgentOutput('evaluator', textContent);
+              }
+            }
+          }
+        }
+      } catch (_e) {
+        // Skip invalid JSON (shouldn't happen with stream-json)
+        // For non-JSON output, add to evaluation output
+        evaluationOutputChunks.push(`${line}\n`);
+        monitor?.addAgentOutput('evaluator', line);
+      }
+    }
+  });
+
+  return new Promise((resolve, reject) => {
+    evaluationProcess.on('close', async (code) => {
+      // Update evaluator status
+      monitor?.updateAgentStatus('evaluator', code === 0 ? 'completed' : 'error');
+
+      if (code === 0) {
+        // Save report to both temp directory and optionally to project directory
+        // FUNCTIONAL: Join output chunks at the end
+        const evaluationOutput = evaluationOutputChunks.join('');
+        const tempReportPath = path.join(outputDir, 'evaluation-report.md');
+        await fs.writeFile(tempReportPath, evaluationOutput);
+
+        // Save summary of what each agent created
+        const summary = Object.entries(agentWork)
+          .map(([agent, content]) => {
+            const fileCount = (content.match(/--- File: /g) || []).length;
+            return `${agent}: ${fileCount} files created`;
+          })
+          .join('\n');
+
+        const tempSummaryPath = path.join(outputDir, 'summary.txt');
+        await fs.writeFile(tempSummaryPath, summary);
+
+        // Show completion message and display the full LLM output
+        if (monitor) {
+          monitor?.addAgentOutput('evaluator', '📊 Evaluation completed!');
+          monitor?.addAgentOutput('evaluator', `📁 Report saved to: ${tempReportPath}`);
+          monitor?.addAgentOutput('evaluator', '');
+          monitor?.addAgentOutput('evaluator', '🏆 EVALUATION RESULTS:');
+          monitor?.addAgentOutput('evaluator', '');
+
+          // Display the complete LLM evaluation output directly
+          const lines = evaluationOutput.split('\n');
+          lines.forEach((line, _index) => {
+            if (line.trim()) {
+              monitor?.addAgentOutput('evaluator', line);
+            }
+          });
+
+          monitor?.addAgentOutput('evaluator', '');
+          monitor?.addAgentOutput('evaluator', '✅ End of evaluation report');
+        }
+
+        // Clean up evaluation temp file
+        try {
+          await fs.unlink(tempEvalFile);
+        } catch (_error) {
+          // Ignore cleanup errors
+        }
+
+        // Also save to project directory if report option is provided
+        if (reportDir) {
+          const projectReportPath = path.join(process.cwd(), reportDir, 'evaluation-report.md');
+          const projectSummaryPath = path.join(process.cwd(), reportDir, 'summary.txt');
+
+          // Ensure report directory exists
+          await fs.mkdir(path.dirname(projectReportPath), { recursive: true });
+
+          await fs.writeFile(projectReportPath, evaluationOutput);
+          await fs.writeFile(projectSummaryPath, summary);
+
+          if (monitor) {
+            monitor?.addAgentOutput(
+              'evaluator',
+              `📁 Project report saved to: ${projectReportPath}`
+            );
+          }
+        }
+
+        resolve();
+      } else {
+        monitor?.addAgentOutput('evaluator', `❌ Evaluation failed with exit code ${code}`);
+        reject(new Error(`Evaluation failed with code ${code}`));
+      }
+    });
+
+    evaluationProcess.on('error', (error) => {
+      reject(error);
+    });
+  });
+}
+
+/**
+ * Internal helper: Build evaluation prompt from agent timings
+ */
+async function buildEvaluationPrompt(agentTimings: AgentTimings): Promise<string> {
+  // Load template from file - required file, no fallback
+  const templatePath = path.join(process.cwd(), 'templates', 'evaluation-prompt.md');
+
+  try {
+    const template = await fs.readFile(templatePath, 'utf-8');
+
+    // Generate agent performance data section
+    const performanceData = Object.entries(agentTimings)
+      .map(([agent, timing]) => {
+        const duration = timing.duration || 0;
+        const scoreRange = PERFORMANCE_SCORE_RANGES.find((range) => duration <= range.max)!;
+        return `- ${agent}: ${duration}s execution time (Performance: ${scoreRange.score}/10)`;
+      })
+      .join('\n');
+
+    // Replace template variables
+    return template.replace('{{AGENT_PERFORMANCE_DATA}}', performanceData);
+  } catch (error) {
+    const errorMessage = error instanceof Error ? error.message : String(error);
+    throw new Error(
+      `Failed to load evaluation template from ${templatePath}. Error: ${errorMessage}\n\nPlease ensure:\n1. The file exists at: ${templatePath}\n2. The file is readable (check permissions)\n3. The file contains valid markdown content`
+    );
+  }
+}