outcome-cli 1.0.0

package/src/utils/logger.ts

@@ -0,0 +1,419 @@
+/**
+ * Logger Utility - Structured logging for agent attempts
+ *
+ * Provides mandatory logging for all agent attempts including:
+ * - Agent and outcome identification
+ * - Token usage tracking
+ * - Result status and failure reasons
+ * - Secret redaction and log capping for security and performance
+ *
+ * @module utils/logger
+ * @see Requirements 6.1, 6.2, 6.3
+ */
+
+import { readFileSync, writeFileSync, existsSync } from 'fs';
+import { join } from 'path';
+
+/**
+ * Maximum number of log entries per outcome to prevent memory exhaustion.
+ */
+const MAX_LOGS_PER_OUTCOME = 1000;
+
+/**
+ * Patterns for detecting and redacting secrets in log messages.
+ * Add more patterns as needed for different secret types.
+ */
+const SECRET_PATTERNS = [
+  // API keys (common formats)
+  /\b(sk-[a-zA-Z0-9]{48})\b/g,  // OpenAI
+  /\b(xoxb-[0-9]+-[0-9]+-[a-zA-Z0-9]+)\b/g,  // Slack
+  /\b([a-zA-Z0-9]{32})\b/g,  // Generic 32-char keys
+  /\b([a-zA-Z0-9]{40})\b/g,  // GitHub tokens, etc.
+  // Password patterns
+  /\bpassword["\s]*:[\s"]*([^\s,"}]+)\b/gi,
+  /\btoken["\s]*:[\s"]*([^\s,"}]+)\b/gi,
+  /\bsecret["\s]*:[\s"]*([^\s,"}]+)\b/gi,
+  /\bkey["\s]*:[\s"]*([^\s,"}]+)\b/gi,
+  // Authorization headers
+  /\bAuthorization["\s]*:[\s"]*([^\s,"}]+)\b/gi,
+  /\bBearer\s+([a-zA-Z0-9._-]+)\b/gi,
+];
+
+/**
+ * Redacts secrets from a string using predefined patterns.
+ *
+ * @param text - The text to redact secrets from
+ * @returns The text with secrets replaced with [REDACTED]
+ */
+function redactSecrets(text: string): string {
+  let redacted = text;
+  for (const pattern of SECRET_PATTERNS) {
+    redacted = redacted.replace(pattern, '[REDACTED]');
+  }
+  return redacted;
+}
+
+/**
+ * Redacts secrets from log entry metadata recursively.
+ *
+ * @param obj - The object to redact secrets from
+ * @returns A new object with secrets redacted
+ */
+function redactSecretsInObject(obj: any): any {
+  if (typeof obj === 'string') {
+    return redactSecrets(obj);
+  }
+  if (Array.isArray(obj)) {
+    return obj.map(redactSecretsInObject);
+  }
+  if (obj !== null && typeof obj === 'object') {
+    const result: any = {};
+    for (const [key, value] of Object.entries(obj)) {
+      result[key] = redactSecretsInObject(value);
+    }
+    return result;
+  }
+  return obj;
+}
+
+/**
+ * Result status for a log entry.
+ */
+export type LogResult = 'SUCCESS' | 'FAILURE' | 'PENDING';
+
+/**
+ * Structured log entry for agent attempts.
+ *
+ * Every agent attempt is logged with this structure for auditing.
+ *
+ * @see Requirements 6.1 - Record agent ID, outcome ID, prompt version, tokens spent, result, failure reason
+ */
+export interface LogEntry {
+  /** ISO timestamp when the log entry was created */
+  timestamp: string;
+  /** Unique identifier for the agent */
+  agentId: string;
+  /** Unique identifier for the outcome being attempted */
+  outcomeId: string;
+  /** Version identifier for the prompt used */
+  promptVersion: string;
+  /** Number of tokens spent in this attempt */
+  tokensSpent: number;
+  /** Result status of the attempt */
+  result: LogResult;
+  /** Reason for failure (required when result is FAILURE) */
+  failureReason?: string;
+  /** Additional metadata for the log entry */
+  metadata?: Record<string, unknown>;
+}
+
+/**
+ * Input for creating a log entry.
+ * Timestamp is auto-generated if not provided.
+ */
+export interface LogEntryInput {
+  agentId: string;
+  outcomeId: string;
+  promptVersion: string;
+  tokensSpent: number;
+  result: LogResult;
+  failureReason?: string;
+  metadata?: Record<string, unknown>;
+}
+
+/**
+ * Log file path for local persistence.
+ * In production, this would be replaced with Durable Objects storage.
+ */
+const LOG_FILE = join(process.cwd(), 'logs.json');
+
+/**
+ * In-memory log storage with file persistence for CLI.
+ */
+const logStore: Map<string, LogEntry[]> = new Map();
+
+/**
+ * Load logs from file on startup.
+ */
+function loadLogsFromFile(): void {
+  try {
+    if (existsSync(LOG_FILE)) {
+      const data = readFileSync(LOG_FILE, 'utf-8');
+      const parsed = JSON.parse(data) as Record<string, LogEntry[]>;
+      for (const [key, entries] of Object.entries(parsed)) {
+        logStore.set(key, entries);
+      }
+    }
+  } catch {
+    // Ignore errors, start fresh
+  }
+}
+
+/**
+ * Save logs to file for persistence.
+ */
+function saveLogsToFile(): void {
+  try {
+    const obj: Record<string, LogEntry[]> = {};
+    for (const [key, entries] of logStore.entries()) {
+      obj[key] = entries;
+    }
+    writeFileSync(LOG_FILE, JSON.stringify(obj, null, 2));
+  } catch {
+    // Ignore errors
+  }
+}
+
+// Load logs on module initialization
+loadLogsFromFile();
+
+/**
+ * Generates a storage key for logs based on outcome ID.
+ *
+ * @param outcomeId - The outcome ID to generate key for
+ * @returns Storage key string
+ */
+function getLogKey(outcomeId: string): string {
+  return `logs:${outcomeId}`;
+}
+
+/**
+ * Logs an agent attempt with structured data.
+ *
+ * @param entry - The log entry data
+ * @returns The created LogEntry with timestamp
+ *
+ * @example
+ * log({
+ *   agentId: 'agent-001',
+ *   outcomeId: 'qualified_sales_interest',
+ *   promptVersion: 'v1.0.0',
+ *   tokensSpent: 500,
+ *   result: 'SUCCESS'
+ * });
+ *
+ * @see Requirements 6.1
+ */
+export function log(entry: LogEntryInput): LogEntry {
+  const logEntry: LogEntry = {
+    timestamp: new Date().toISOString(),
+    agentId: entry.agentId,
+    outcomeId: entry.outcomeId,
+    promptVersion: entry.promptVersion,
+    tokensSpent: entry.tokensSpent,
+    result: entry.result,
+    ...(entry.failureReason && { failureReason: redactSecrets(entry.failureReason) }),
+    ...(entry.metadata && { metadata: redactSecretsInObject(entry.metadata) }),
+  };
+
+  // Store the log entry
+  const key = getLogKey(entry.outcomeId);
+  const existingLogs = logStore.get(key) ?? [];
+
+  // Implement log capping - keep only the most recent entries
+  if (existingLogs.length >= MAX_LOGS_PER_OUTCOME) {
+    existingLogs.splice(0, existingLogs.length - MAX_LOGS_PER_OUTCOME + 1);
+  }
+
+  existingLogs.push(logEntry);
+  logStore.set(key, existingLogs);
+
+  // Persist to file for CLI usage
+  saveLogsToFile();
+
+  // Also output to console for CLI visibility (with redaction)
+  const formattedEntry = formatLogEntry(logEntry);
+  if (logEntry.result === 'FAILURE') {
+    console.error(formattedEntry);
+  } else {
+    console.log(formattedEntry);
+  }
+
+  return logEntry;
+}
+
+/**
+ * Retrieves all logs for a specific outcome.
+ *
+ * @param outcomeId - The outcome ID to retrieve logs for
+ * @returns Array of log entries for the outcome
+ *
+ * @example
+ * const logs = getLogs('qualified_sales_interest');
+ * console.log(`Found ${logs.length} log entries`);
+ *
+ * @see Requirements 6.2, 6.3
+ */
+export function getLogs(outcomeId: string): LogEntry[] {
+  const key = getLogKey(outcomeId);
+  return logStore.get(key) ?? [];
+}
+
+/**
+ * Retrieves logs for a specific agent within an outcome.
+ *
+ * @param outcomeId - The outcome ID
+ * @param agentId - The agent ID to filter by
+ * @returns Array of log entries for the agent
+ */
+export function getAgentLogs(outcomeId: string, agentId: string): LogEntry[] {
+  const logs = getLogs(outcomeId);
+  return logs.filter((entry) => entry.agentId === agentId);
+}
+
+/**
+ * Clears all logs for a specific outcome.
+ * Useful for testing and cleanup.
+ *
+ * @param outcomeId - The outcome ID to clear logs for
+ */
+export function clearLogs(outcomeId: string): void {
+  const key = getLogKey(outcomeId);
+  logStore.delete(key);
+}
+
+/**
+ * Clears all logs from the store.
+ * Useful for testing.
+ */
+export function clearAllLogs(): void {
+  logStore.clear();
+}
+
+/**
+ * Gets the total number of log entries across all outcomes.
+ *
+ * @returns Total log count
+ */
+export function getTotalLogCount(): number {
+  let count = 0;
+  for (const logs of logStore.values()) {
+    count += logs.length;
+  }
+  return count;
+}
+
+/**
+ * Formats a log entry for console output.
+ *
+ * @param entry - The log entry to format
+ * @returns Formatted string for display
+ */
+function formatLogEntry(entry: LogEntry): string {
+  const statusIcon = entry.result === 'SUCCESS' ? '✅' : entry.result === 'FAILURE' ? '❌' : '⏳';
+  const timestamp = entry.timestamp.split('T')[1]?.split('.')[0] ?? entry.timestamp;
+  
+  let message = `${statusIcon} [${timestamp}] Agent:${entry.agentId} | Outcome:${entry.outcomeId} | Tokens:${entry.tokensSpent} | ${entry.result}`;
+  
+  if (entry.failureReason) {
+    message += ` | Reason: ${entry.failureReason}`;
+  }
+  
+  return message;
+}
+
+/**
+ * Formats all logs for an outcome for CLI display.
+ *
+ * @param outcomeId - The outcome ID to format logs for
+ * @returns Formatted string for CLI output
+ *
+ * @see Requirements 6.2 - Logs viewable via CLI
+ */
+export function formatLogsForCli(outcomeId: string): string {
+  const logs = getLogs(outcomeId);
+  
+  if (logs.length === 0) {
+    return `No logs found for outcome: ${outcomeId}`;
+  }
+
+  const header = `\n📋 Logs for outcome: ${outcomeId} (${logs.length} entries)\n${'─'.repeat(60)}`;
+  const formattedLogs = logs.map(formatLogEntry).join('\n');
+  const footer = `${'─'.repeat(60)}\n`;
+
+  return `${header}\n${formattedLogs}\n${footer}`;
+}
+
+/**
+ * Creates a log entry for a successful attempt.
+ *
+ * @param agentId - Agent ID
+ * @param outcomeId - Outcome ID
+ * @param promptVersion - Prompt version
+ * @param tokensSpent - Tokens spent
+ * @param metadata - Optional metadata
+ * @returns The created log entry
+ */
+export function logSuccess(
+  agentId: string,
+  outcomeId: string,
+  promptVersion: string,
+  tokensSpent: number,
+  metadata?: Record<string, unknown>
+): LogEntry {
+  return log({
+    agentId,
+    outcomeId,
+    promptVersion,
+    tokensSpent,
+    result: 'SUCCESS',
+    metadata,
+  });
+}
+
+/**
+ * Creates a log entry for a failed attempt.
+ *
+ * @param agentId - Agent ID
+ * @param outcomeId - Outcome ID
+ * @param promptVersion - Prompt version
+ * @param tokensSpent - Tokens spent
+ * @param failureReason - Reason for failure
+ * @param metadata - Optional metadata
+ * @returns The created log entry
+ */
+export function logFailure(
+  agentId: string,
+  outcomeId: string,
+  promptVersion: string,
+  tokensSpent: number,
+  failureReason: string,
+  metadata?: Record<string, unknown>
+): LogEntry {
+  return log({
+    agentId,
+    outcomeId,
+    promptVersion,
+    tokensSpent,
+    result: 'FAILURE',
+    failureReason,
+    metadata,
+  });
+}
+
+/**
+ * Creates a log entry for a pending attempt.
+ *
+ * @param agentId - Agent ID
+ * @param outcomeId - Outcome ID
+ * @param promptVersion - Prompt version
+ * @param tokensSpent - Tokens spent so far
+ * @param metadata - Optional metadata
+ * @returns The created log entry
+ */
+export function logPending(
+  agentId: string,
+  outcomeId: string,
+  promptVersion: string,
+  tokensSpent: number,
+  metadata?: Record<string, unknown>
+): LogEntry {
+  return log({
+    agentId,
+    outcomeId,
+    promptVersion,
+    tokensSpent,
+    result: 'PENDING',
+    metadata,
+  });
+}

package/src/utils/output-parsers.ts

@@ -0,0 +1,216 @@
+/**
+ * Structured Output Parsers for Command Results
+ *
+ * Parses outputs from tests, linting, benchmarks, and security scans
+ * into structured data for evaluation and scoring.
+ */
+
+import { CommandResult } from './command-runner';
+
+export interface TestResult {
+  totalTests: number;
+  passedTests: number;
+  failedTests: number;
+  skippedTests: number;
+  testErrors: string[];
+  testNames: string[];
+  success: boolean;
+}
+
+export interface LintResult {
+  totalIssues: number;
+  errorCount: number;
+  warningCount: number;
+  issues: Array<{
+    file: string;
+    line: number;
+    column: number;
+    severity: 'error' | 'warning' | 'info';
+    message: string;
+    rule?: string;
+  }>;
+  success: boolean;
+}
+
+export interface BenchmarkResult {
+  totalBenchmarks: number;
+  metrics: Array<{
+    name: string;
+    timeMs?: number;
+    memoryMb?: number;
+    opsPerSec?: number;
+    score?: number;
+  }>;
+  success: boolean;
+}
+
+export interface SecurityScanResult {
+  totalVulnerabilities: number;
+  criticalCount: number;
+  highCount: number;
+  mediumCount: number;
+  lowCount: number;
+  vulnerabilities: Array<{
+    severity: 'critical' | 'high' | 'medium' | 'low';
+    package?: string;
+    description: string;
+    cve?: string;
+  }>;
+  success: boolean;
+}
+
+/**
+ * Parse test output (supports Jest, Vitest, Mocha, etc.)
+ */
+export function parseTestOutput(result: CommandResult): TestResult {
+  const output = result.stdout + '\n' + result.stderr;
+  const testResult: TestResult = {
+    totalTests: 0,
+    passedTests: 0,
+    failedTests: 0,
+    skippedTests: 0,
+    testErrors: [],
+    testNames: [],
+    success: result.exitCode === 0,
+  };
+
+  // Jest/Vitest patterns
+  const jestPassedMatch = output.match(/Tests?:\s*(\d+)\s*passed/i);
+  const jestFailedMatch = output.match(/Tests?:\s*(\d+)\s*failed/i);
+  const jestSkippedMatch = output.match(/Tests?:\s*(\d+)\s*skipped/i);
+
+  if (jestPassedMatch) testResult.passedTests = parseInt(jestPassedMatch[1]);
+  if (jestFailedMatch) testResult.failedTests = parseInt(jestFailedMatch[1]);
+  if (jestSkippedMatch) testResult.skippedTests = parseInt(jestSkippedMatch[1]);
+
+  testResult.totalTests = testResult.passedTests + testResult.failedTests + testResult.skippedTests;
+
+  // Extract test names from passed/failed sections
+  const testNameRegex = /^\s*(✓|✗|✕|❌|PASS|FAIL)\s+(.+)$/gm;
+  let match;
+  while ((match = testNameRegex.exec(output)) !== null) {
+    testResult.testNames.push(match[2].trim());
+  }
+
+  // Extract error messages
+  const errorRegex = /(Error:|FAIL|Failed to compile)/gi;
+  const errorSections = output.split(errorRegex).slice(1);
+  testResult.testErrors = errorSections.filter((_, i) => i % 2 === 0).slice(0, 10); // Limit to 10 errors
+
+  return testResult;
+}
+
+/**
+ * Parse lint output (supports ESLint, TSLint, Prettier, etc.)
+ */
+export function parseLintOutput(result: CommandResult): LintResult {
+  const output = result.stdout + '\n' + result.stderr;
+  const lintResult: LintResult = {
+    totalIssues: 0,
+    errorCount: 0,
+    warningCount: 0,
+    issues: [],
+    success: result.exitCode === 0,
+  };
+
+  // ESLint patterns
+  const eslintIssueRegex = /^(.+): line (\d+), col (\d+), (Error|Warning) - (.+?)(?:\s+\((.+?)\))?\s*$/gm;
+  let match;
+  while ((match = eslintIssueRegex.exec(output)) !== null) {
+    const [, file, line, col, severity, message, rule] = match;
+    lintResult.issues.push({
+      file,
+      line: parseInt(line),
+      column: parseInt(col),
+      severity: severity.toLowerCase() as 'error' | 'warning',
+      message,
+      rule,
+    });
+  }
+
+  // Count issues
+  lintResult.errorCount = lintResult.issues.filter(i => i.severity === 'error').length;
+  lintResult.warningCount = lintResult.issues.filter(i => i.severity === 'warning').length;
+  lintResult.totalIssues = lintResult.issues.length;
+
+  return lintResult;
+}
+
+/**
+ * Parse benchmark output (supports Benchmark.js, custom benchmarks)
+ */
+export function parseBenchmarkOutput(result: CommandResult): BenchmarkResult {
+  const output = result.stdout + '\n' + result.stderr;
+  const benchResult: BenchmarkResult = {
+    totalBenchmarks: 0,
+    metrics: [],
+    success: result.exitCode === 0,
+  };
+
+  // Benchmark.js patterns
+  const benchRegex = /^(.+?)\s+x\s+([\d,]+)\s+ops\/sec\s+±([\d.]+)%\s+\(([\d]+)\s+runs\s+sampled\)$/gm;
+  let match;
+  while ((match = benchRegex.exec(output)) !== null) {
+    const [, name, opsPerSec] = match;
+    benchResult.metrics.push({
+      name: name.trim(),
+      opsPerSec: parseFloat(opsPerSec.replace(/,/g, '')),
+    });
+  }
+
+  // Custom benchmark patterns (time-based)
+  const timeRegex = /^(.+?):\s*([\d.]+)\s*(ms|milliseconds?|s|seconds?)$/gm;
+  while ((match = timeRegex.exec(output)) !== null) {
+    const [, name, time, unit] = match;
+    benchResult.metrics.push({
+      name: name.trim(),
+      timeMs: unit.startsWith('ms') ? parseFloat(time) : parseFloat(time) * 1000,
+    });
+  }
+
+  benchResult.totalBenchmarks = benchResult.metrics.length;
+
+  return benchResult;
+}
+
+/**
+ * Parse security scan output (supports npm audit, Snyk, etc.)
+ */
+export function parseSecurityScanOutput(result: CommandResult): SecurityScanResult {
+  const output = result.stdout + '\n' + result.stderr;
+  const scanResult: SecurityScanResult = {
+    totalVulnerabilities: 0,
+    criticalCount: 0,
+    highCount: 0,
+    mediumCount: 0,
+    lowCount: 0,
+    vulnerabilities: [],
+    success: result.exitCode === 0,
+  };
+
+  // npm audit patterns
+  const auditRegex = /^(\d+)\s+(low|moderate|high|critical)\s+severity\s+vulnerability\s+in\s+(.+?):\s*(.+?)\s*$/gm;
+  let match;
+  while ((match = auditRegex.exec(output)) !== null) {
+    const [, count, severity, package, description] = match;
+    const vulnCount = parseInt(count);
+    scanResult.vulnerabilities.push({
+      severity: severity === 'moderate' ? 'medium' : severity as 'critical' | 'high' | 'medium' | 'low',
+      package,
+      description,
+    });
+
+    // Update counts
+    switch (severity) {
+      case 'critical': scanResult.criticalCount += vulnCount; break;
+      case 'high': scanResult.highCount += vulnCount; break;
+      case 'moderate': scanResult.mediumCount += vulnCount; break;
+      case 'low': scanResult.lowCount += vulnCount; break;
+    }
+  }
+
+  scanResult.totalVulnerabilities = scanResult.criticalCount + scanResult.highCount +
+                                    scanResult.mediumCount + scanResult.lowCount;
+
+  return scanResult;
+}