claude-mycelium 2.0.0 → 2.1.0

package/src/gc/runner.ts

@@ -0,0 +1,314 @@
+/**
+ * Garbage Collection Runner
+ *
+ * Per Phase 7 specification §8 & §14, this module implements:
+ * - Delete traces older than retention period
+ * - Compact remaining traces
+ * - Delete weak inhibitors (strength <0.05)
+ * - Generate GC report
+ * - Auto-run every 100 spawns
+ *
+ * Usage:
+ * ```typescript
+ * import { runGC } from './runner.js';
+ * const report = await runGC();
+ * ```
+ */
+
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { logDebug, logWarn } from '../utils/logger.js';
+import { fileExists } from '../utils/file-utils.js';
+import { compactTraces } from './trace-compactor.js';
+import type { TraceEvent, GCReport, Inhibitor } from '../types/index.js';
+
+/**
+ * Garbage collection configuration (per spec §13)
+ */
+export const GC_CONFIG = {
+  lowConfidenceRetentionDays: 7,    // Keep recent traces for 7 days
+  traceRetentionDays: 30,            // Keep high-efficiency traces for 30 days
+  compactionThreshold: 50,           // Compact if >50 events per file
+  inhibitorStrengthThreshold: 0.05,  // Delete inhibitors with strength <0.05
+};
+
+/**
+ * Run garbage collection
+ * - Delete old traces
+ * - Compact remaining traces
+ * - Clean weak inhibitors
+ * - Generate report
+ */
+export async function runGC(): Promise<GCReport> {
+  const startTime = Date.now();
+  const report: GCReport = {
+    timestamp: new Date().toISOString(),
+    deleted: 0,
+    compacted: 0,
+    size_before_mb: 0,
+    size_after_mb: 0,
+    errors: [],
+    duration_ms: 0,
+  };
+
+  const tracesDir = '.agent-meta/traces';
+
+  // Ensure traces directory exists
+  if (!fileExists(tracesDir)) {
+    logDebug('Traces directory does not exist, skipping GC');
+    report.duration_ms = Date.now() - startTime;
+    return report;
+  }
+
+  try {
+    // Step 1: Calculate size before
+    report.size_before_mb = await getDirectorySize(tracesDir);
+
+    // Step 2: Clean old traces
+    const cleanReport = await cleanOldTraces(tracesDir);
+    report.deleted += cleanReport.deleted;
+    report.compacted += cleanReport.compacted;
+    report.errors.push(...cleanReport.errors);
+
+    // Step 3: Clean weak inhibitors
+    const inhibitorReport = await cleanWeakInhibitors();
+    report.deleted += inhibitorReport.deleted;
+    report.errors.push(...inhibitorReport.errors);
+
+    // Step 4: Calculate size after
+    report.size_before_mb = await getDirectorySize(tracesDir);
+
+    // Step 5: Log report
+    await logGCReport(report);
+
+    logDebug('GC complete', {
+      deleted: report.deleted,
+      compacted: report.compacted,
+      size_before_mb: report.size_before_mb.toFixed(2),
+      size_after_mb: report.size_after_mb.toFixed(2),
+      duration_ms: report.duration_ms,
+    });
+  } catch (error) {
+    const message = error instanceof Error ? error.message : 'Unknown error';
+    report.errors.push(`GC runner error: ${message}`);
+    logWarn('GC runner failed', { error: message });
+  }
+
+  report.duration_ms = Date.now() - startTime;
+  return report;
+}
+
+/**
+ * Delete traces older than retention period
+ * Compact remaining traces
+ */
+async function cleanOldTraces(tracesDir: string): Promise<{
+  deleted: number;
+  compacted: number;
+  errors: string[];
+}> {
+  const result = { deleted: 0, compacted: 0, errors: [] as string[] };
+  const now = Date.now();
+
+  let traceFiles: string[];
+  try {
+    const entries = await fs.readdir(tracesDir);
+    traceFiles = entries.filter(f => f.endsWith('.ndjson'));
+  } catch (error) {
+    const message = error instanceof Error ? error.message : 'Unknown error';
+    result.errors.push(`Failed to list trace files: ${message}`);
+    return result;
+  }
+
+  for (const file of traceFiles) {
+    const filePath = path.join(tracesDir, file);
+
+    try {
+      // Read trace events
+      const content = await fs.readFile(filePath, 'utf-8');
+      const lines = content.split('\n').filter(Boolean);
+      const events: TraceEvent[] = [];
+
+      for (const line of lines) {
+        try {
+          events.push(JSON.parse(line) as TraceEvent);
+        } catch {
+          // Skip malformed lines
+        }
+      }
+
+      // Filter old events
+      const filtered = events.filter((event) => {
+        const ageMs = now - new Date(event.timestamp).getTime();
+        const ageDays = ageMs / (1000 * 60 * 60 * 24);
+
+        // Keep recent events (< 7 days)
+        if (ageDays < GC_CONFIG.lowConfidenceRetentionDays) {
+          return true;
+        }
+
+        // Keep high-efficiency events longer (< 30 days + efficiency > 0.2)
+        if (ageDays < GC_CONFIG.traceRetentionDays && event.efficiency > 0.2) {
+          return true;
+        }
+
+        result.deleted++;
+        return false;
+      });
+
+      // Compact if too many events
+      let output: TraceEvent[];
+      if (filtered.length > GC_CONFIG.compactionThreshold) {
+        output = compactTraces(filtered);
+        result.compacted++;
+      } else {
+        output = filtered;
+      }
+
+      // Write back (or delete if empty)
+      if (output.length === 0) {
+        await fs.unlink(filePath);
+      } else {
+        const outputContent = output.map(e => JSON.stringify(e)).join('\n') + '\n';
+        await fs.writeFile(filePath, outputContent);
+      }
+    } catch (error) {
+      const message = error instanceof Error ? error.message : 'Unknown error';
+      result.errors.push(`${file}: ${message}`);
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Delete weak inhibitors (strength < 0.05)
+ */
+async function cleanWeakInhibitors(): Promise<{
+  deleted: number;
+  errors: string[];
+}> {
+  const result = { deleted: 0, errors: [] as string[] };
+  const inhibitorsPath = '.agent-meta/_inhibitors.ndjson';
+
+  if (!fileExists(inhibitorsPath)) {
+    return result;
+  }
+
+  try {
+    const content = await fs.readFile(inhibitorsPath, 'utf-8');
+    const lines = content.split('\n').filter(Boolean);
+    const inhibitors: Inhibitor[] = [];
+
+    for (const line of lines) {
+      try {
+        const inhibitor = JSON.parse(line) as Inhibitor;
+        inhibitors.push(inhibitor);
+      } catch {
+        // Skip malformed lines
+      }
+    }
+
+    // Filter out weak inhibitors
+    const strong = inhibitors.filter((i) => {
+      const isWeak = i.strength < GC_CONFIG.inhibitorStrengthThreshold;
+      if (isWeak) {
+        result.deleted++;
+      }
+      return !isWeak;
+    });
+
+    // Write back
+    const outputContent = strong.map(i => JSON.stringify(i)).join('\n');
+    if (strong.length > 0) {
+      await fs.writeFile(inhibitorsPath, outputContent + '\n');
+    } else {
+      await fs.writeFile(inhibitorsPath, '');
+    }
+
+    logDebug('Inhibitor cleanup', {
+      total: inhibitors.length,
+      removed: result.deleted,
+      remaining: strong.length,
+    });
+  } catch (error) {
+    const message = error instanceof Error ? error.message : 'Unknown error';
+    result.errors.push(`Inhibitor cleanup failed: ${message}`);
+  }
+
+  return result;
+}
+
+/**
+ * Calculate total size of a directory in MB
+ */
+async function getDirectorySize(dirPath: string): Promise<number> {
+  let totalBytes = 0;
+
+  try {
+    const entries = await fs.readdir(dirPath);
+
+    for (const entry of entries) {
+      const fullPath = path.join(dirPath, entry);
+      const stat = await fs.stat(fullPath);
+
+      if (stat.isFile()) {
+        totalBytes += stat.size;
+      } else if (stat.isDirectory()) {
+        totalBytes += await getDirectorySize(fullPath);
+      }
+    }
+  } catch {
+    // Directory doesn't exist or can't be read
+  }
+
+  return totalBytes / (1024 * 1024); // Convert to MB
+}
+
+/**
+ * Append GC report to GC log
+ */
+async function logGCReport(report: GCReport): Promise<void> {
+  const gcLogPath = '.agent-meta/_gc.log';
+
+  try {
+    const logEntry = JSON.stringify(report);
+    await fs.appendFile(gcLogPath, logEntry + '\n');
+  } catch (error) {
+    logWarn('Failed to log GC report', {
+      error: error instanceof Error ? error.message : 'Unknown error',
+    });
+  }
+}
+
+/**
+ * Read GC log
+ */
+export async function readGCLog(): Promise<GCReport[]> {
+  const gcLogPath = '.agent-meta/_gc.log';
+
+  if (!fileExists(gcLogPath)) {
+    return [];
+  }
+
+  try {
+    const content = await fs.readFile(gcLogPath, 'utf-8');
+    const lines = content.split('\n').filter(Boolean);
+    const reports: GCReport[] = [];
+
+    for (const line of lines) {
+      try {
+        reports.push(JSON.parse(line) as GCReport);
+      } catch {
+        // Skip malformed lines
+      }
+    }
+
+    return reports;
+  } catch (error) {
+    logWarn('Failed to read GC log', {
+      error: error instanceof Error ? error.message : 'Unknown error',
+    });
+    return [];
+  }
+}

package/src/gc/trace-compactor.ts

@@ -0,0 +1,187 @@
+/**
+ * Trace Compaction Module
+ *
+ * Per Phase 7 specification §14, this module implements:
+ * - Keep last 10 samples per file
+ * - Keep all samples from last 7 days
+ * - Summarize older samples (avg efficiency, date range)
+ * - Write summaries with __summary__: true flag
+ *
+ * Usage:
+ * ```typescript
+ * import { compactTraces, summarizeOld, keepRecent } from './trace-compactor.js';
+ * const compacted = compactTraces(events);
+ * ```
+ */
+
+import type { TraceEvent, Mode } from '../types/index.js';
+import { logDebug } from '../utils/logger.js';
+
+/**
+ * Keep last 10 samples per file + all samples from last 7 days
+ * Summarize older samples by mode
+ */
+export function compactTraces(events: TraceEvent[]): TraceEvent[] {
+  if (events.length === 0) return [];
+
+  const now = Date.now();
+  const sevenDaysMs = 7 * 24 * 60 * 60 * 1000;
+
+  // Separate events into recent (last 7 days) and old
+  const recent: TraceEvent[] = [];
+  const old: TraceEvent[] = [];
+
+  for (const event of events) {
+    const ageMs = now - new Date(event.timestamp).getTime();
+    if (ageMs < sevenDaysMs) {
+      recent.push(event);
+    } else {
+      old.push(event);
+    }
+  }
+
+  // Keep last 10 from recent, summarize the rest of old
+  const recentToKeep = recent.slice(-10);
+  const oldToSummarize = [
+    ...recent.slice(0, -10),
+    ...old,
+  ];
+
+  logDebug('Trace compaction', {
+    total_events: events.length,
+    recent_kept: recentToKeep.length,
+    old_to_summarize: oldToSummarize.length,
+  });
+
+  if (oldToSummarize.length === 0) {
+    return recentToKeep;
+  }
+
+  // Summarize old events by mode
+  const summaries = summarizeOld(oldToSummarize);
+
+  return [...summaries, ...recentToKeep];
+}
+
+/**
+ * Summarize old events by mode
+ * Creates summary events with aggregated metrics
+ */
+export function summarizeOld(events: TraceEvent[]): TraceEvent[] {
+  if (events.length === 0) return [];
+
+  const byMode = new Map<Mode, TraceEvent[]>();
+
+  // Group by mode
+  for (const event of events) {
+    if (!byMode.has(event.mode)) {
+      byMode.set(event.mode, []);
+    }
+    byMode.get(event.mode)!.push(event);
+  }
+
+  const summaries: TraceEvent[] = [];
+
+  for (const [mode, traces] of byMode) {
+    if (traces.length === 0) continue;
+
+    const summary = createSummaryEvent(mode, traces);
+    summaries.push(summary);
+  }
+
+  logDebug('Summarized old traces', {
+    modes: summaries.length,
+    total_compressed: events.length,
+  });
+
+  return summaries;
+}
+
+/**
+ * Keep recent samples (last N days) verbatim
+ */
+export function keepRecent(events: TraceEvent[], days: number = 7): TraceEvent[] {
+  const now = Date.now();
+  const msPerDay = 24 * 60 * 60 * 1000;
+  const cutoffMs = days * msPerDay;
+
+  return events.filter((event) => {
+    const ageMs = now - new Date(event.timestamp).getTime();
+    return ageMs < cutoffMs;
+  });
+}
+
+/**
+ * Create a summary event from multiple trace events
+ */
+function createSummaryEvent(mode: Mode, traces: TraceEvent[]): TraceEvent {
+  if (traces.length === 0) {
+    throw new Error('Cannot create summary from empty trace list');
+  }
+
+  const timestamps = traces.map(t => new Date(t.timestamp).getTime());
+  const minTime = Math.min(...timestamps);
+  const maxTime = Math.max(...timestamps);
+
+  const efficiencies = traces.map(t => t.efficiency);
+  const avgEfficiency = efficiencies.reduce((a, b) => a + b, 0) / efficiencies.length;
+  const minEfficiency = Math.min(...efficiencies);
+  const maxEfficiency = Math.max(...efficiencies);
+
+  const ciPassed = traces.filter(t => t.ci_passed).length;
+  const successRate = ciPassed / traces.length;
+
+  // Create summary event (inherits from first event for structure)
+  const firstEvent = traces[0];
+  const lastEvent = traces[traces.length - 1];
+
+  const summaryEvent: TraceEvent = {
+    id: `summary-${mode}-${Date.now()}`,
+    timestamp: lastEvent.timestamp,
+    file_path: firstEvent.file_path,
+    mode,
+    gradient_before: average(traces.map(t => t.gradient_before)),
+    gradient_after: average(traces.map(t => t.gradient_after)),
+    gradient_delta: average(traces.map(t => t.gradient_delta)),
+    metabolic_cost: sum(traces.map(t => t.metabolic_cost)),
+    efficiency: avgEfficiency,
+    ci_passed: traces.every(t => t.ci_passed),
+    changes: {
+      additions: sum(traces.map(t => t.changes.additions)),
+      deletions: sum(traces.map(t => t.changes.deletions)),
+      files_touched: [...new Set(traces.flatMap(t => t.changes.files_touched))],
+    },
+    notes: [
+      `Compacted ${traces.length} ${mode} events`,
+      `Date range: ${new Date(minTime).toISOString().split('T')[0]} to ${new Date(maxTime).toISOString().split('T')[0]}`,
+      `Efficiency: avg=${avgEfficiency.toFixed(3)}, range=[${minEfficiency.toFixed(3)}, ${maxEfficiency.toFixed(3)}]`,
+      `CI success rate: ${(successRate * 100).toFixed(1)}%`,
+    ],
+    cost: {
+      tokens_in: sum(traces.map(t => t.cost.tokens_in)),
+      tokens_out: sum(traces.map(t => t.cost.tokens_out)),
+      model: firstEvent.cost.model,
+      estimated_usd: sum(traces.map(t => t.cost.estimated_usd)),
+    },
+  };
+
+  // Add metadata flag for queries
+  (summaryEvent as any).__summary__ = true;
+
+  return summaryEvent;
+}
+
+/**
+ * Helper: calculate average of numbers
+ */
+function average(nums: number[]): number {
+  if (nums.length === 0) return 0;
+  return sum(nums) / nums.length;
+}
+
+/**
+ * Helper: sum array of numbers
+ */
+function sum(nums: number[]): number {
+  return nums.reduce((a, b) => a + b, 0);
+}

package/src/index.ts

@@ -9,10 +9,11 @@
  * - LLM: Anthropic integration
  * - Trace: Event tracking
  * - Cost: Token and cost tracking
+ * - Task: LLM-based task planning and execution (Phase 5)
  * - Utils: Logging, file I/O, configuration
  * - Types: Complete TypeScript interfaces
  *
- * Per initial-spec and Phase 2 spec.
+ * Per initial-spec, Phase 2 spec, and Phase 5 (Task System).
  *
  * Usage:
  * ```typescript
@@ -22,6 +23,8 @@
  *   createAnthropicClient,
  *   recordTraceEvent,
  *   calculateCost,
+ *   planTask,
+ *   executeTask,
  * } from 'claude-mycelium';
  * ```
  */
@@ -36,6 +39,9 @@ export { createAnthropicClient, type LLMRequest, type LLMResponse, LLMError } fr
 export * from './trace/index.js';
 export * from './cost/index.js';
 
+// Phase 5: Task System
+export * from './task/index.js';
+
 // Utilities
 export * from './utils/index.js';
 

package/src/prompts/index.ts

@@ -7,7 +7,8 @@
  */
 
 import { readFile } from 'fs/promises';
-import type { Mode, GradientScore } from '../types/index.js';
+import type { Mode } from '../types/index.js';
+import type { GradientScore } from '../core/gradient.js';
 import type { PromptContext } from './types.js';
 
 export { generatePrompt as generateErrorReducerPrompt } from './error-reducer.js';