autonomous-flow-daemon 1.0.0 → 1.6.0

package/src/core/evolution.ts

@@ -0,0 +1,215 @@
+/**
+ * Self-Evolution Engine
+ *
+ * Analyzes quarantined (corrupted) files against their restored originals,
+ * generates failure lessons, and writes them to afd-lessons.md so AI agents
+ * learn from their own mistakes.
+ */
+
+import { readdirSync, readFileSync, writeFileSync, renameSync, existsSync } from "fs";
+import { resolve, basename } from "path";
+import { QUARANTINE_DIR } from "../constants";
+import { lineDiff } from "./log-utils";
+
+const LESSONS_FILE = "afd-lessons.md";
+const LEARNED_SUFFIX = ".learned";
+
+export interface QuarantineEntry {
+  /** Full path inside .afd/quarantine/ */
+  quarantinePath: string;
+  /** Original file path (reconstructed from quarantine filename) */
+  originalPath: string;
+  /** Timestamp string extracted from filename */
+  timestamp: string;
+  /** Whether this entry has already been learned */
+  learned: boolean;
+}
+
+export interface EvolutionLesson {
+  entry: QuarantineEntry;
+  diff: string[];
+  corruptedContent: string;
+  restoredContent: string | null;
+  failureType: "corruption" | "deletion";
+  suggestion: string;
+}
+
+export interface EvolutionStats {
+  totalQuarantined: number;
+  totalLearned: number;
+  pending: number;
+  lessons: EvolutionLesson[];
+}
+
+/** Parse quarantine filename: YYYYMMDD_HHMMSS_originalname → { timestamp, originalPath } */
+function parseQuarantineName(filename: string): { timestamp: string; originalPath: string } | null {
+  // e.g. 20260401_021028_.claude_hooks.json or 20260401_020741_.claudeignore
+  const match = filename.replace(LEARNED_SUFFIX, "").match(/^(\d{8}_\d{6})_(.+)$/);
+  if (!match) return null;
+  const ts = match[1];
+  // Reconstruct path: underscores that were separators become path separators
+  // Heuristic: first underscore-delimited segment starting with "." is likely a directory
+  const rawName = match[2];
+  // Reverse the flatten: `.claude_hooks.json` → `.claude/hooks.json`
+  const originalPath = rawName.replace(/^\.([^.]+)_/, ".$1/");
+  return { timestamp: ts, originalPath };
+}
+
+/** List all quarantined entries */
+export function listQuarantine(): QuarantineEntry[] {
+  if (!existsSync(QUARANTINE_DIR)) return [];
+  const files = readdirSync(QUARANTINE_DIR);
+  const entries: QuarantineEntry[] = [];
+
+  for (const file of files) {
+    const isLearned = file.endsWith(LEARNED_SUFFIX);
+    const cleanName = file.replace(LEARNED_SUFFIX, "");
+    const parsed = parseQuarantineName(cleanName);
+    if (!parsed) continue;
+    entries.push({
+      quarantinePath: resolve(QUARANTINE_DIR, file),
+      originalPath: parsed.originalPath,
+      timestamp: parsed.timestamp,
+      learned: isLearned,
+    });
+  }
+
+  return entries.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
+}
+
+/** Detect what type of corruption occurred */
+function detectFailureType(content: string): "corruption" | "deletion" {
+  return content.trim() === "DELETED" ? "deletion" : "corruption";
+}
+
+/** Generate a human-readable suggestion based on the failure */
+function generateSuggestion(entry: QuarantineEntry, failureType: string, corruptedContent: string, restoredContent: string | null): string {
+  const file = entry.originalPath;
+
+  if (failureType === "deletion") {
+    return `Do NOT delete \`${file}\`. This is an immune-critical file protected by afd. ` +
+      `If you need to modify it, edit the content instead of removing the file.`;
+  }
+
+  // Corruption analysis
+  if (file.endsWith(".json")) {
+    try {
+      JSON.parse(corruptedContent);
+    } catch (e) {
+      const msg = e instanceof Error ? e.message : String(e);
+      return `When editing \`${file}\`, ensure valid JSON syntax. ` +
+        `Common mistake: ${msg}. Always validate JSON structure after editing.`;
+    }
+  }
+
+  if (corruptedContent.trim().length === 0) {
+    return `Do NOT empty \`${file}\`. The file was overwritten with blank content. ` +
+      `Preserve existing content when making changes.`;
+  }
+
+  if (restoredContent && corruptedContent.length < restoredContent.length * 0.1) {
+    return `Do NOT truncate \`${file}\`. Content was reduced by >90%. ` +
+      `When editing, preserve the existing structure and only modify specific sections.`;
+  }
+
+  return `When editing \`${file}\`, be careful not to corrupt its structure. ` +
+    `Always verify the file is still valid after changes.`;
+}
+
+/** Analyze pending (unlearned) quarantine entries and produce lessons */
+export function analyzeQuarantine(): EvolutionStats {
+  const entries = listQuarantine();
+  const totalQuarantined = entries.length;
+  const totalLearned = entries.filter(e => e.learned).length;
+  const pending = entries.filter(e => !e.learned);
+
+  const lessons: EvolutionLesson[] = [];
+
+  for (const entry of pending) {
+    const corruptedContent = readFileSync(entry.quarantinePath, "utf-8");
+    const failureType = detectFailureType(corruptedContent);
+
+    let restoredContent: string | null = null;
+    let diff: string[] = [];
+
+    if (existsSync(entry.originalPath)) {
+      restoredContent = readFileSync(entry.originalPath, "utf-8");
+      if (failureType === "corruption") {
+        diff = lineDiff(corruptedContent, restoredContent, 20);
+      } else {
+        diff = [`  (file was deleted — restored from antibody snapshot)`];
+      }
+    } else {
+      diff = [`  (original file not found — may still be deleted)`];
+    }
+
+    const suggestion = generateSuggestion(entry, failureType, corruptedContent, restoredContent);
+
+    lessons.push({ entry, diff, corruptedContent, restoredContent, failureType, suggestion });
+  }
+
+  return { totalQuarantined, totalLearned, pending: pending.length, lessons };
+}
+
+/** Mark a quarantine entry as learned by renaming with .learned suffix */
+export function markLearned(entry: QuarantineEntry): void {
+  if (entry.learned) return;
+  const newPath = entry.quarantinePath + LEARNED_SUFFIX;
+  renameSync(entry.quarantinePath, newPath);
+}
+
+/** Build the lesson block for afd-lessons.md */
+function buildLessonBlock(lesson: EvolutionLesson): string {
+  const ts = lesson.entry.timestamp.replace("_", "T");
+  const formattedTs = `${ts.slice(0, 4)}-${ts.slice(4, 6)}-${ts.slice(6, 8)} ${ts.slice(9, 11)}:${ts.slice(11, 13)}:${ts.slice(13, 15)}`;
+  const lines: string[] = [
+    `### ${lesson.entry.originalPath} (${formattedTs})`,
+    `- **Type**: ${lesson.failureType === "deletion" ? "Unauthorized Deletion" : "Content Corruption"}`,
+    `- **Rule**: ${lesson.suggestion}`,
+  ];
+  if (lesson.diff.length > 0) {
+    lines.push(`- **Diff**:`);
+    lines.push("```");
+    lines.push(...lesson.diff);
+    lines.push("```");
+  }
+  return lines.join("\n");
+}
+
+/**
+ * Write lessons to afd-lessons.md and mark entries as learned.
+ * Returns the number of new lessons written.
+ */
+export function evolve(): { lessonsWritten: number; totalLessons: number } {
+  const stats = analyzeQuarantine();
+  if (stats.lessons.length === 0) {
+    return { lessonsWritten: 0, totalLessons: stats.totalLearned };
+  }
+
+  // Build new lesson blocks
+  const newBlocks = stats.lessons.map(buildLessonBlock);
+
+  // Read or initialize afd-lessons.md
+  let existing = "";
+  if (existsSync(LESSONS_FILE)) {
+    existing = readFileSync(LESSONS_FILE, "utf-8");
+  }
+
+  if (!existing.includes("# Failure Lessons")) {
+    existing = `# Failure Lessons\n\n` +
+      `> Auto-generated by afd Self-Evolution engine.\n` +
+      `> AI agents: read these rules to avoid repeating past mistakes.\n\n` +
+      existing;
+  }
+
+  // Append new lessons
+  const updated = existing.trimEnd() + "\n\n" + newBlocks.join("\n\n") + "\n";
+  writeFileSync(LESSONS_FILE, updated, "utf-8");
+
+  // Mark all analyzed entries as learned
+  for (const lesson of stats.lessons) {
+    markLearned(lesson.entry);
+  }
+
+  return { lessonsWritten: stats.lessons.length, totalLessons: stats.totalLearned + stats.lessons.length };
+}

package/src/core/hologram/engine.ts

@@ -0,0 +1,71 @@
+import { Parser, Language, type Tree } from "web-tree-sitter";
+import { resolve, dirname } from "path";
+
+/**
+ * Singleton Tree-sitter engine with grammar caching.
+ * Parser.init() runs once; grammar WASMs are lazy-loaded and cached in-memory.
+ */
+export class TreeSitterEngine {
+  private static instance: TreeSitterEngine | null = null;
+  private parser: Parser | null = null;
+  private grammarCache = new Map<string, Language>();
+  private initPromise: Promise<void> | null = null;
+
+  static async getInstance(): Promise<TreeSitterEngine> {
+    if (!this.instance) {
+      this.instance = new TreeSitterEngine();
+      await this.instance.init();
+    }
+    return this.instance;
+  }
+
+  /** Reset singleton — for testing only */
+  static resetForTest(): void {
+    if (this.instance?.parser) {
+      this.instance.parser.delete();
+    }
+    this.instance = null;
+  }
+
+  private async init(): Promise<void> {
+    if (this.initPromise) return this.initPromise;
+    this.initPromise = (async () => {
+      await Parser.init();
+      this.parser = new Parser();
+    })();
+    return this.initPromise;
+  }
+
+  async parse(source: string, grammarName: string): Promise<Tree> {
+    const grammar = await this.loadGrammar(grammarName);
+    this.parser!.setLanguage(grammar);
+    const tree = this.parser!.parse(source);
+    if (!tree) throw new Error(`Failed to parse with grammar: ${grammarName}`);
+    return tree;
+  }
+
+  private async loadGrammar(grammarName: string): Promise<Language> {
+    const cached = this.grammarCache.get(grammarName);
+    if (cached) return cached;
+
+    const wasmPath = resolveGrammarWasm(grammarName);
+    const lang = await Language.load(wasmPath);
+    this.grammarCache.set(grammarName, lang);
+    return lang;
+  }
+}
+
+/** Resolve WASM file path from installed npm grammar package */
+function resolveGrammarWasm(grammarName: string): string {
+  // Grammar packages: tree-sitter-typescript, tree-sitter-python, etc.
+  // WASM file is at package root: node_modules/tree-sitter-{name}/tree-sitter-{name}.wasm
+  const packageName = `tree-sitter-${grammarName}`;
+  try {
+    // require.resolve returns bindings/node/index.js — walk up to package root
+    const pkgJson = require.resolve(`${packageName}/package.json`);
+    return resolve(dirname(pkgJson), `${packageName}.wasm`);
+  } catch {
+    // Fallback: try direct node_modules path
+    return resolve(process.cwd(), "node_modules", packageName, `${packageName}.wasm`);
+  }
+}

package/src/core/hologram/fallback.ts

@@ -0,0 +1,11 @@
+import type { HologramResult } from "./types";
+
+/** L0 fallback: return full source when tree-sitter cannot parse */
+export function fallbackL0(filePath: string, source: string): HologramResult {
+  return {
+    hologram: source,
+    originalLength: source.length,
+    hologramLength: source.length,
+    savings: 0,
+  };
+}

package/src/core/hologram/incremental.ts

@@ -0,0 +1,227 @@
+/**
+ * Incremental Hologram — diff-only mode.
+ *
+ * Compares previous and current hologram extractions,
+ * returns only changed nodes with surrounding context in unified-diff style.
+ */
+
+import type { Tree } from "web-tree-sitter";
+import type { HologramResult, LanguageExtractor, HologramOptions } from "./types";
+import { TreeSitterEngine } from "./engine";
+
+/** In-memory cache for previous hologram lines (per file path) */
+const hologramCache = new Map<string, string[]>();
+
+/** Maximum cache entries */
+const MAX_CACHE_SIZE = 200;
+
+export function clearHologramCache(): void {
+  hologramCache.clear();
+}
+
+export function setCachedHologram(filePath: string, lines: string[]): void {
+  // True LRU: delete existing entry first so re-insert moves it to end
+  hologramCache.delete(filePath);
+  if (hologramCache.size >= MAX_CACHE_SIZE) {
+    const oldestKey = hologramCache.keys().next().value;
+    if (oldestKey) hologramCache.delete(oldestKey);
+  }
+  hologramCache.set(filePath, lines);
+}
+
+export function getCachedHologram(filePath: string): string[] | undefined {
+  const value = hologramCache.get(filePath);
+  if (value !== undefined) {
+    // Promote to most-recently-used position
+    hologramCache.delete(filePath);
+    hologramCache.set(filePath, value);
+  }
+  return value;
+}
+
+/**
+ * Generate an incremental (diff-only) hologram.
+ * Compares current extraction with cached previous result.
+ * Returns unified-diff style output showing only changed nodes.
+ */
+export async function generateIncrementalHologram(
+  filePath: string,
+  source: string,
+  extractor: LanguageExtractor,
+  options?: HologramOptions,
+): Promise<HologramResult> {
+  const engine = await TreeSitterEngine.getInstance();
+  const tree = await engine.parse(source, extractor.grammarName);
+  const currentLines = extractor.extract(tree, source, options);
+  tree.delete();
+
+  const previousLines = getCachedHologram(filePath);
+
+  // Cache current result for next diff
+  setCachedHologram(filePath, currentLines);
+
+  // No previous → return full hologram with diff header
+  if (!previousLines) {
+    const hologram = currentLines.join("\n");
+    return {
+      hologram,
+      originalLength: source.length,
+      hologramLength: hologram.length,
+      savings: source.length > 0
+        ? Math.round((source.length - hologram.length) / source.length * 1000) / 10
+        : 0,
+      language: extractor.grammarName,
+      isDiff: false,
+      changedNodes: currentLines.length,
+    };
+  }
+
+  // Diff previous vs current lines
+  const diffOutput = buildUnifiedDiff(filePath, previousLines, currentLines);
+
+  return {
+    hologram: diffOutput.text,
+    originalLength: source.length,
+    hologramLength: diffOutput.text.length,
+    savings: source.length > 0
+      ? Math.round((source.length - diffOutput.text.length) / source.length * 1000) / 10
+      : 0,
+    language: extractor.grammarName,
+    isDiff: true,
+    changedNodes: diffOutput.changedCount,
+  };
+}
+
+interface DiffOutput {
+  text: string;
+  changedCount: number;
+}
+
+/**
+ * Build a unified-diff style output comparing old and new hologram lines.
+ * Groups unchanged lines into summary markers, shows changed lines with +/- prefixes.
+ */
+function buildUnifiedDiff(filePath: string, oldLines: string[], newLines: string[]): DiffOutput {
+  const header = `--- a/${filePath} (previous)\n+++ b/${filePath} (current)\n`;
+
+  // Simple line-by-line diff using LCS approach
+  const hunks = computeHunks(oldLines, newLines);
+
+  if (hunks.length === 0) {
+    return {
+      text: header + "@@ no changes @@",
+      changedCount: 0,
+    };
+  }
+
+  const parts: string[] = [header];
+  let changedCount = 0;
+  let oldIdx = 0;
+
+  for (const hunk of hunks) {
+    // Show unchanged lines before this hunk as a summary
+    const unchangedBefore = hunk.oldStart - oldIdx;
+    if (unchangedBefore > 0) {
+      const summaryLines = oldLines.slice(oldIdx, hunk.oldStart);
+      const firstLine = summaryLines[0]?.split("{")[0]?.trim() ?? "...";
+      parts.push(`@@ unchanged: ${firstLine} (${unchangedBefore} ${unchangedBefore === 1 ? "declaration" : "declarations"}) @@`);
+    }
+
+    // Show removed lines
+    for (let i = hunk.oldStart; i < hunk.oldStart + hunk.oldCount; i++) {
+      parts.push(`- ${oldLines[i]}`);
+      changedCount++;
+    }
+
+    // Show added lines
+    for (let i = hunk.newStart; i < hunk.newStart + hunk.newCount; i++) {
+      parts.push(`+ ${newLines[i]}`);
+    }
+
+    oldIdx = hunk.oldStart + hunk.oldCount;
+  }
+
+  // Trailing unchanged
+  const trailingCount = oldLines.length - oldIdx;
+  if (trailingCount > 0) {
+    parts.push(`@@ unchanged: ${trailingCount} more ${trailingCount === 1 ? "declaration" : "declarations"} @@`);
+  }
+
+  return { text: parts.join("\n"), changedCount };
+}
+
+interface Hunk {
+  oldStart: number;
+  oldCount: number;
+  newStart: number;
+  newCount: number;
+}
+
+/**
+ * Compute diff hunks between old and new line arrays.
+ * Uses a simple O(n*m) LCS-based diff suitable for small arrays (hologram lines are typically < 100).
+ */
+function computeHunks(oldLines: string[], newLines: string[]): Hunk[] {
+  const n = oldLines.length;
+  const m = newLines.length;
+
+  // Guard: for very large inputs, fall back to full diff to stay within SEAM budget
+  if (n * m > 50_000) {
+    return [{ oldStart: 0, oldCount: n, newStart: 0, newCount: m }];
+  }
+
+  // Build LCS table
+  const dp: number[][] = Array.from({ length: n + 1 }, () => Array(m + 1).fill(0));
+  for (let i = 1; i <= n; i++) {
+    for (let j = 1; j <= m; j++) {
+      if (oldLines[i - 1] === newLines[j - 1]) {
+        dp[i][j] = dp[i - 1][j - 1] + 1;
+      } else {
+        dp[i][j] = Math.max(dp[i - 1][j], dp[i][j - 1]);
+      }
+    }
+  }
+
+  // Backtrack to find matching lines
+  const matches: Array<[number, number]> = [];
+  let i = n, j = m;
+  while (i > 0 && j > 0) {
+    if (oldLines[i - 1] === newLines[j - 1]) {
+      matches.unshift([i - 1, j - 1]);
+      i--; j--;
+    } else if (dp[i - 1][j] > dp[i][j - 1]) {
+      i--;
+    } else {
+      j--;
+    }
+  }
+
+  // Convert matches to hunks (gaps between matches)
+  const hunks: Hunk[] = [];
+  let oi = 0, ni = 0;
+
+  for (const [mi, mj] of matches) {
+    if (oi < mi || ni < mj) {
+      hunks.push({
+        oldStart: oi,
+        oldCount: mi - oi,
+        newStart: ni,
+        newCount: mj - ni,
+      });
+    }
+    oi = mi + 1;
+    ni = mj + 1;
+  }
+
+  // Trailing diff
+  if (oi < n || ni < m) {
+    hunks.push({
+      oldStart: oi,
+      oldCount: n - oi,
+      newStart: ni,
+      newCount: m - ni,
+    });
+  }
+
+  return hunks;
+}

package/src/core/hologram/py-extractor.ts

@@ -0,0 +1,132 @@
+import type { Node, Tree } from "web-tree-sitter";
+import type { LanguageExtractor, HologramOptions } from "./types";
+
+/** Stub a Python function/method body — keep signature + "..." */
+function stubPythonBody(node: Node, source: string): string {
+  const body = node.childForFieldName("body");
+  if (!body) return node.text;
+  return source.slice(node.startIndex, body.startIndex).trimEnd() + " ...";
+}
+
+/** Extract a Python function definition */
+function extractFunction(node: Node, source: string): string {
+  // Decorators
+  const decorators = collectDecorators(node, source);
+  const sig = stubPythonBody(node, source);
+  return decorators + sig;
+}
+
+/** Extract a Python class with method signatures */
+function extractClass(node: Node, source: string): string {
+  const decorators = collectDecorators(node, source);
+  const nameNode = node.childForFieldName("name");
+  const name = nameNode?.text ?? "?";
+  const superclasses = node.childForFieldName("superclasses");
+  const sup = superclasses ? superclasses.text : "";
+  const body = node.childForFieldName("body");
+
+  const header = `class ${name}${sup ? `(${sup.replace(/^\(|\)$/g, "")})` : ""}:`;
+
+  if (!body) return decorators + header;
+
+  const members: string[] = [];
+  for (const child of body.namedChildren) {
+    switch (child.type) {
+      case "function_definition": {
+        const memberDecorators = collectDecorators(child, source);
+        members.push(indent(memberDecorators + stubPythonBody(child, source)));
+        break;
+      }
+      case "expression_statement": {
+        // Type-annotated assignments: x: int = 42
+        const expr = child.namedChildren[0];
+        if (expr?.type === "assignment" || expr?.type === "type") {
+          members.push(indent(child.text.split("\n")[0]));
+        }
+        break;
+      }
+      case "class_definition": {
+        // Nested class — just show header
+        const nestedName = child.childForFieldName("name")?.text ?? "?";
+        members.push(indent(`class ${nestedName}: ...`));
+        break;
+      }
+    }
+  }
+
+  if (members.length === 0) {
+    return decorators + header + "\n    ...";
+  }
+
+  return decorators + header + "\n" + members.join("\n");
+}
+
+/** Collect decorator lines above a node */
+function collectDecorators(node: Node, source: string): string {
+  const decorators: string[] = [];
+  // In tree-sitter-python, decorators are children of the decorated_definition
+  // or are 'decorator' type children of the function/class
+  const parent = node.parent;
+  if (parent?.type === "decorated_definition") {
+    for (const child of parent.namedChildren) {
+      if (child.type === "decorator") {
+        decorators.push(child.text);
+      }
+    }
+  }
+  return decorators.length > 0 ? decorators.join("\n") + "\n" : "";
+}
+
+function indent(s: string): string {
+  return s.split("\n").map(line => "    " + line).join("\n");
+}
+
+/** Process a single top-level statement */
+function extractTopLevel(node: Node, source: string): string | null {
+  switch (node.type) {
+    case "import_statement":
+    case "import_from_statement":
+      return node.text;
+    case "function_definition":
+      return extractFunction(node, source);
+    case "class_definition":
+      return extractClass(node, source);
+    case "decorated_definition": {
+      // Unwrap to get the inner function/class
+      const inner = node.namedChildren.find(c =>
+        c.type === "function_definition" || c.type === "class_definition");
+      if (inner) return extractTopLevel(inner, source);
+      return null;
+    }
+    case "expression_statement": {
+      // Module-level type annotations or assignments
+      const expr = node.namedChildren[0];
+      if (expr?.type === "assignment" || expr?.type === "type") {
+        return node.text.split("\n")[0];
+      }
+      // __all__ = [...]
+      if (node.text.startsWith("__all__")) {
+        return node.text;
+      }
+      return null;
+    }
+    default:
+      return null;
+  }
+}
+
+export const pyExtractor: LanguageExtractor = {
+  extensions: ["py", "pyi"],
+  grammarName: "python",
+
+  extract(tree: Tree, source: string, _options?: HologramOptions): string[] {
+    const lines: string[] = [];
+
+    for (const stmt of tree.rootNode.namedChildren) {
+      const line = extractTopLevel(stmt, source);
+      if (line) lines.push(line);
+    }
+
+    return lines;
+  },
+};