clawmatrix 0.4.1 → 0.5.0

package/src/knowledge-sync.ts

@@ -49,7 +49,9 @@ const TAG = "knowledge";
 const REGISTRY_DOC_ID = "";
 const SYNC_CONFIG_FILE = ".clawmatrix.sync";
 /** TTL for writtenByExport entries (ms). Stale entries are cleaned up to prevent leaks. */
-const EXPORT_MARKER_TTL = 30_000;
+const EXPORT_MARKER_TTL = 60_000;
+/** Interval for periodic writtenByExport cleanup (ms). */
+const EXPORT_CLEANUP_INTERVAL = 60_000;
 /** Max concurrent file I/O operations during walkDir / export. */
 const MAX_IO_CONCURRENCY = 32;
 /** Delay before batched git commit (ms). */
@@ -73,14 +75,20 @@ function legacyDocFileName(relPath: string): string {
   return relPath.replaceAll("/", "--") + ".automerge";
 }
 
-/** Run async tasks with bounded concurrency. */
-async function pMap<T, R>(items: T[], fn: (item: T) => Promise<R>, concurrency: number): Promise<R[]> {
-  const results: R[] = new Array(items.length);
+/** Run async tasks with bounded concurrency. Individual errors are caught
+ *  and logged so that one failing item does not abort the entire batch. */
+async function pMap<T, R>(items: T[], fn: (item: T) => Promise<R>, concurrency: number): Promise<(R | undefined)[]> {
+  const results: (R | undefined)[] = new Array(items.length);
   let i = 0;
   async function worker() {
     while (i < items.length) {
       const idx = i++;
-      results[idx] = await fn(items[idx]);
+      try {
+        results[idx] = await fn(items[idx]);
+      } catch (err) {
+        debug(TAG, `pMap item ${idx} failed: ${err}`);
+        results[idx] = undefined;
+      }
     }
   }
   await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => worker()));
@@ -114,16 +122,27 @@ async function streamToString(stream: ReadableStream | null): Promise<string> {
 }
 
 /** Update FileDoc content using Automerge.updateText for character-level CRDT merging. */
-function changeFileContent(doc: Automerge.Doc<FileDoc>, content: string): Automerge.Doc<FileDoc> {
+function changeFileContent(doc: Automerge.Doc<FileDoc>, content: string, attribution?: string): Automerge.Doc<FileDoc> {
+  const changeOpts = attribution ? { message: attribution } : undefined;
   const currentContent = doc.content;
   // If content field doesn't exist yet (new doc), initialize it first
   if (currentContent === undefined) {
+    if (changeOpts) {
+      return Automerge.change(doc, changeOpts, (d) => {
+        (d as FileDoc).content = content;
+      });
+    }
     return Automerge.change(doc, (d) => {
       (d as FileDoc).content = content;
     });
   }
   // Use updateText for minimal diff — enables proper concurrent merge
   if (currentContent === content) return doc;
+  if (changeOpts) {
+    return Automerge.change(doc, changeOpts, (d) => {
+      Automerge.updateText(d, ["content"], content);
+    });
+  }
   return Automerge.change(doc, (d) => {
     Automerge.updateText(d, ["content"], content);
   });
@@ -149,6 +168,10 @@ export class KnowledgeSync {
   /** Deferred git commit timer — batches multiple remote syncs into one commit. */
   private gitCommitTimer: ReturnType<typeof setTimeout> | null = null;
   private pendingGitSources = new Set<string>();
+  /** Periodic cleanup interval for stale writtenByExport entries. */
+  private exportCleanupTimer: ReturnType<typeof setInterval> | null = null;
+  /** Pending file attribution from tool hooks — consumed by handleLocalChangesInner. */
+  private pendingAttribution = new Map<string, { nodeId: string; agentId: string; agentType?: string; sessionKey?: string }>();
 
   // ── Paths ──────────────────────────────────────────────────────
   private registryPath: string;
@@ -251,6 +274,12 @@ export class KnowledgeSync {
       this.scheduleSync();
     });
 
+    // Periodic cleanup of stale export markers to prevent memory leaks
+    // when watcher events are missed or debounced away.
+    this.exportCleanupTimer = setInterval(() => {
+      this.cleanupStaleExportMarkers();
+    }, EXPORT_CLEANUP_INTERVAL);
+
     debug(TAG, "file watcher started");
   }
 
@@ -260,6 +289,10 @@ export class KnowledgeSync {
       clearTimeout(this.debounceTimer);
       this.debounceTimer = null;
     }
+    if (this.exportCleanupTimer) {
+      clearInterval(this.exportCleanupTimer);
+      this.exportCleanupTimer = null;
+    }
     this.watcher?.close();
     this.watcher = null;
     await this.flushPendingGitCommit();
@@ -418,6 +451,117 @@ export class KnowledgeSync {
     return this.matcher(relPath);
   }
 
+  /** Get change history for a synced file, with attribution. */
+  getFileHistory(relPath: string): Array<{ timestamp: number; message: string; actor: string }> {
+    const doc = this.fileDocs.get(relPath);
+    if (!doc) return [];
+
+    try {
+      const history = Automerge.getHistory(doc);
+      return history.map((state) => {
+        // Automerge time is in seconds; convert to ms if it looks like seconds
+        let ts = state.change.time ?? 0;
+        if (ts > 0 && ts < 1e12) ts *= 1000;
+        return {
+          timestamp: ts,
+          message: state.change.message ?? "",
+          actor: state.change.actor ?? "",
+        };
+      });
+    } catch {
+      return [];
+    }
+  }
+
+  /**
+   * Get per-line blame for a synced file.
+   * Returns an attribution entry for each line of the current content,
+   * indicating which change (and thus which node/agent) last modified it.
+   */
+  getFileBlame(relPath: string): Array<{ line: number; timestamp: number; message: string; actor: string }> | null {
+    const doc = this.fileDocs.get(relPath);
+    if (!doc || !doc.content) return null;
+
+    try {
+      const history = Automerge.getHistory(doc);
+      if (history.length === 0) return null;
+
+      const currentLines = doc.content.split("\n");
+
+      // Build snapshots array with metadata (reverse order for backward walk)
+      const snapshots: Array<{ lines: Set<string>; ts: number; message: string; actor: string }> = [];
+      for (const state of history) {
+        let ts = state.change.time ?? 0;
+        if (ts > 0 && ts < 1e12) ts *= 1000;
+        const content = state.snapshot.content ?? "";
+        // Store unique line contents present in each snapshot
+        const lineSet = new Set<string>();
+        for (const line of content.split("\n")) lineSet.add(line);
+        snapshots.push({ lines: lineSet, ts, message: state.change.message ?? "", actor: state.change.actor ?? "" });
+      }
+
+      // For each line in current content, walk backward through history.
+      // The line is attributed to the LATEST change whose snapshot contains it,
+      // but the PREVIOUS snapshot does NOT. This handles insertions correctly.
+      const blame: Array<{ line: number; timestamp: number; message: string; actor: string }> = [];
+
+      for (let i = 0; i < currentLines.length; i++) {
+        const lineContent = currentLines[i];
+        let attributed = false;
+
+        // Walk backward through history
+        for (let h = snapshots.length - 1; h >= 0; h--) {
+          const snap = snapshots[h];
+          const prevSnap = h > 0 ? snapshots[h - 1] : null;
+
+          // This snapshot has the line, but previous doesn't → this change introduced it
+          if (snap.lines.has(lineContent) && (!prevSnap || !prevSnap.lines.has(lineContent))) {
+            blame.push({ line: i + 1, timestamp: snap.ts, message: snap.message, actor: snap.actor });
+            attributed = true;
+            break;
+          }
+        }
+
+        if (!attributed) {
+          // Fallback: attribute to the first change
+          const first = snapshots[0];
+          blame.push({ line: i + 1, timestamp: first?.ts ?? 0, message: first?.message ?? "", actor: first?.actor ?? "" });
+        }
+      }
+
+      return blame;
+    } catch {
+      return null;
+    }
+  }
+
+  /** List all synced files with metadata. */
+  listSyncedFiles(): Array<{ path: string; version: number; updatedAt: number; deleted: boolean }> {
+    const files = this.registry.files;
+    if (!files) return [];
+    return Object.entries(files).map(([path, meta]) => ({
+      path,
+      version: meta.version,
+      updatedAt: meta.updatedAt,
+      deleted: meta.deleted,
+    })).filter(f => !f.deleted);
+  }
+
+  /**
+   * Set pending attribution for a file path.
+   * Called by tool hooks (after_tool_call, ACP stream) before fsWatcher picks up the change.
+   * The attribution is consumed by handleLocalChangesInner when the file change is processed.
+   */
+  setPendingAttribution(absOrRelPath: string, attribution: { nodeId: string; agentId: string; agentType?: string; sessionKey?: string }) {
+    // Convert absolute path to relative
+    const relPath = absOrRelPath.startsWith(this.opts.workspacePath)
+      ? absOrRelPath.slice(this.opts.workspacePath.length + 1)
+      : absOrRelPath;
+    this.pendingAttribution.set(relPath, attribution);
+    // Auto-cleanup after 30s in case fsWatcher never fires (file outside sync scope)
+    setTimeout(() => this.pendingAttribution.delete(relPath), 30_000);
+  }
+
   // ── Private: sync helpers ──────────────────────────────────────
 
   private syncDocWithPeer(peerId: string, docId: string) {
@@ -613,8 +757,15 @@ export class KnowledgeSync {
     for (const relPath of [...added, ...modified]) {
       const content = contentCache.get(relPath)!;
 
+      // Use pending attribution from tool hooks (precise), fallback to nodeId (human edit)
+      const pending = this.pendingAttribution.get(relPath);
+      const attribution = pending
+        ? JSON.stringify(pending)
+        : JSON.stringify({ nodeId: this.opts.nodeId });
+      if (pending) this.pendingAttribution.delete(relPath);
+
       let doc = this.fileDocs.get(relPath) ?? Automerge.init<FileDoc>();
-      doc = changeFileContent(doc, content);
+      doc = changeFileContent(doc, content, attribution);
       this.fileDocs.set(relPath, doc);
 
       await this.saveFileDoc(relPath);

package/src/local-tools.ts

@@ -11,6 +11,9 @@
  */
 
 import { spawn } from "node:child_process";
+import { readdir, lstat, readlink } from "node:fs/promises";
+import { join } from "node:path";
+import { nanoid } from "nanoid";
 import {
   createReadTool,
   createWriteTool,
@@ -26,11 +29,15 @@ interface ExecParams {
   timeout?: number;
 }
 
+interface ListParams {
+  path: string;
+}
+
 type ToolResult = Record<string, unknown>;
 
 // ── Constants ──────────────────────────────────────────────────────
 
-const LOCAL_TOOLS = new Set(["exec", "read", "write", "edit"]);
+const LOCAL_TOOLS = new Set(["exec", "read", "write", "edit", "list"]);
 const DEFAULT_EXEC_TIMEOUT = 300; // seconds
 const MAX_OUTPUT_BYTES = 512 * 1024; // 512KB
 
@@ -53,6 +60,8 @@ export async function executeLocally(
       return executePiTool("write", params);
     case "edit":
       return executePiTool("edit", params);
+    case "list":
+      return executeList(params as unknown as ListParams);
     default:
       throw new Error(`Unknown local tool: ${tool}`);
   }
@@ -124,6 +133,60 @@ async function executeExec(params: ExecParams): Promise<ToolResult> {
   });
 }
 
+// ── list: directory listing via node:fs ────────────────────────────
+
+function formatPermissions(mode: number): string {
+  const perms = ["---", "--x", "-w-", "-wx", "r--", "r-x", "rw-", "rwx"];
+  const owner = perms[(mode >> 6) & 7];
+  const group = perms[(mode >> 3) & 7];
+  const other = perms[mode & 7];
+  return `${owner}${group}${other}`;
+}
+
+async function executeList(params: ListParams): Promise<ToolResult> {
+  const { path: dirPath } = params;
+  if (!dirPath) throw new Error("list: path is required");
+
+  const entries = await readdir(dirPath);
+  const items = await Promise.all(
+    entries.map(async (name) => {
+      const fullPath = join(dirPath, name);
+      try {
+        const st = await lstat(fullPath);
+        let type: "file" | "directory" | "symlink" = "file";
+        let target: string | undefined;
+        if (st.isSymbolicLink()) {
+          type = "symlink";
+          try {
+            target = await readlink(fullPath);
+          } catch {}
+        } else if (st.isDirectory()) {
+          type = "directory";
+        }
+        return {
+          name,
+          type,
+          size: st.size,
+          mtime: st.mtime.toISOString(),
+          permissions: formatPermissions(st.mode),
+          ...(target !== undefined && { target }),
+        };
+      } catch {
+        return { name, type: "file" as const, size: 0, mtime: "", permissions: "" };
+      }
+    }),
+  );
+
+  // Sort: directories first, then alphabetically
+  items.sort((a, b) => {
+    if (a.type === "directory" && b.type !== "directory") return -1;
+    if (a.type !== "directory" && b.type === "directory") return 1;
+    return a.name.localeCompare(b.name);
+  });
+
+  return { path: dirPath, items };
+}
+
 // ── read/write/edit: reuse pi-coding-agent factories ───────────────
 
 /** Cache key includes cwd so tools are recreated if the working directory changes. */
@@ -163,7 +226,7 @@ async function executePiTool(
   params: Record<string, unknown>,
 ): Promise<ToolResult> {
   const tool = getPiTool(name);
-  const toolCallId = crypto.randomUUID();
+  const toolCallId = nanoid();
 
   const result = (await tool.execute(toolCallId, params)) as {
     content: Array<{ type: string; text?: string }>;

package/src/log-replication.ts

@@ -0,0 +1,198 @@
+/**
+ * Sequence-based log replication protocol for ClawMatrix.
+ *
+ * Replicates append-only tables (audit_log, health_events, handoff_history)
+ * across mesh peers using vector clocks and delta sync.
+ *
+ * Each replicated row has (node_id, source_seq) — the node that created the
+ * event and its original autoincrement sequence number. The combination is
+ * globally unique (UNIQUE index in SQLite).
+ *
+ * Protocol:
+ *   1. On peer connect → both sides send log_sync { request: true, vector }
+ *   2. Receiver computes delta using vector clock and sends missing rows
+ *   3. Rows inserted via INSERT OR IGNORE (dedup on node_id + source_seq)
+ *   4. If more rows remain, sender marks hasMore and continues
+ *
+ * This replaces Automerge-based sync for append-only data (no conflicts).
+ */
+
+import type { Store, ReplicatedTable } from "./store.ts";
+import type { LogSyncFrame } from "./types.ts";
+import { debug } from "./debug.ts";
+
+const TAG = "log-repl";
+const REPLICATED_TABLES: ReplicatedTable[] = ["audit_log", "health_events", "handoff_history"];
+const BATCH_SIZE = 500;
+
+export interface LogReplicatorOptions {
+  store: Store;
+  nodeId: string;
+  /** Send a frame to a specific peer. */
+  sendTo: (peerId: string, frame: LogSyncFrame) => void;
+}
+
+export class LogReplicator {
+  private store: Store;
+  private nodeId: string;
+  private sendTo: (peerId: string, frame: LogSyncFrame) => void;
+  /** Tracks connected peers for broadcasting local inserts. */
+  private connectedPeers = new Set<string>();
+
+  constructor(opts: LogReplicatorOptions) {
+    this.store = opts.store;
+    this.nodeId = opts.nodeId;
+    this.sendTo = opts.sendTo;
+  }
+
+  // ── Peer lifecycle ──────────────────────────────────────────
+
+  /** Called when a peer connects. Sends sync requests for all tables. */
+  initPeerSync(peerId: string) {
+    this.connectedPeers.add(peerId);
+    for (const table of REPLICATED_TABLES) {
+      this.sendSyncRequest(peerId, table);
+    }
+  }
+
+  /** Called when a peer disconnects. */
+  removePeerSync(peerId: string) {
+    this.connectedPeers.delete(peerId);
+  }
+
+  // ── Handle incoming frames ──────────────────────────────────
+
+  /** Handle a log_sync frame from a peer. */
+  handleSyncMessage(frame: LogSyncFrame) {
+    const peerId = frame.from;
+    const { table, vector, rows, request } = frame.payload;
+
+    if (!REPLICATED_TABLES.includes(table)) {
+      debug(TAG, `ignoring unknown table "${table}" from ${peerId}`);
+      return;
+    }
+
+    // If peer is requesting our delta, compute and send it
+    if (request && vector) {
+      this.sendDelta(peerId, table, vector);
+    }
+
+    // If peer sent rows, apply them
+    if (rows && rows.length > 0) {
+      const inserted = this.store.insertReplicatedRows(table, rows);
+      debug(TAG, `applied ${inserted}/${rows.length} rows to ${table} from ${peerId}`);
+
+      // Update persisted replication state
+      this.updateReplicationState(table, rows);
+    }
+  }
+
+  // ── Notify on local insert ──────────────────────────────────
+
+  /** Called after a local insert to push the new row to all connected peers. */
+  notifyLocalInsert(table: ReplicatedTable) {
+    if (this.connectedPeers.size === 0) return;
+
+    // Get the latest row from our node
+    const rows = this.store.getRowsSince(table, this.nodeId, this.getLocalMaxSeq(table) - 1, 1);
+    if (rows.length === 0) return;
+    this.updateReplicationState(table, rows);
+
+    // Broadcast to all connected peers
+    for (const peerId of this.connectedPeers) {
+      const frame: LogSyncFrame = {
+        type: "log_sync",
+        from: this.nodeId,
+        to: peerId,
+        timestamp: Date.now(),
+        payload: { table, rows },
+      };
+      this.sendTo(peerId, frame);
+    }
+  }
+
+  // ── Internal ────────────────────────────────────────────────
+
+  /** Send a sync request with our vector clock for a table. */
+  private sendSyncRequest(peerId: string, table: ReplicatedTable) {
+    const vector = this.store.getResumeVector(table);
+    const frame: LogSyncFrame = {
+      type: "log_sync",
+      from: this.nodeId,
+      to: peerId,
+      timestamp: Date.now(),
+      payload: { table, vector, request: true },
+    };
+    debug(TAG, `requesting sync for ${table} from ${peerId} (vector: ${JSON.stringify(vector)})`);
+    this.sendTo(peerId, frame);
+  }
+
+  /** Compute and send delta rows that the peer is missing. */
+  private sendDelta(peerId: string, table: ReplicatedTable, peerVector: Record<string, number>) {
+    const localVector = this.store.getVectorClock(table);
+
+    // For each node_id we know about, check if peer is behind
+    let totalSent = 0;
+    for (const nodeId of Object.keys(localVector)) {
+      const localMax = localVector[nodeId] ?? 0;
+      const peerMax = peerVector[nodeId] ?? 0;
+
+      if (localMax <= peerMax) continue; // peer is up to date for this node
+
+      // Fetch missing rows in batches
+      let afterSeq = peerMax;
+      let hasMore = true;
+
+      while (hasMore) {
+        const rows = this.store.getRowsSince(table, nodeId, afterSeq, BATCH_SIZE);
+        if (rows.length === 0) break;
+
+        hasMore = rows.length === BATCH_SIZE;
+        afterSeq = (rows[rows.length - 1] as Record<string, number>).source_seq;
+
+        const frame: LogSyncFrame = {
+          type: "log_sync",
+          from: this.nodeId,
+          to: peerId,
+          timestamp: Date.now(),
+          payload: { table, rows, hasMore },
+        };
+        this.sendTo(peerId, frame);
+        totalSent += rows.length;
+      }
+    }
+
+    if (totalSent > 0) {
+      debug(TAG, `sent ${totalSent} delta rows for ${table} to ${peerId}`);
+    }
+  }
+
+  /** Update persisted replication state after applying rows. */
+  private updateReplicationState(table: ReplicatedTable, rows: Record<string, unknown>[]) {
+    // Group by node_id and find max source_seq per node
+    const maxSeqs = new Map<string, number>();
+    for (const row of rows) {
+      const nodeId = row.node_id as string;
+      const sourceSeq = row.source_seq as number;
+      const current = maxSeqs.get(nodeId) ?? 0;
+      if (sourceSeq > current) maxSeqs.set(nodeId, sourceSeq);
+    }
+
+    for (const [nodeId, maxSeq] of maxSeqs) {
+      const existing = this.store.getReplicationState(table, nodeId);
+      if (maxSeq > existing) {
+        this.store.setReplicationState(table, nodeId, maxSeq);
+      }
+    }
+  }
+
+  /** Get local max source_seq for our own node. */
+  private getLocalMaxSeq(table: ReplicatedTable): number {
+    return this.store.getMaxSeq(table, this.nodeId);
+  }
+
+  /** Destroy: clean up state. */
+  destroy() {
+    this.connectedPeers.clear();
+  }
+}