clawmatrix 0.1.23 → 0.2.1

package/src/knowledge-sync.ts

@@ -1,26 +1,42 @@
 import * as Automerge from "@automerge/automerge";
 import { watch, type FSWatcher } from "node:fs";
-import { readdir, readFile, stat as fsStat, writeFile, mkdir, unlink } from "node:fs/promises";
+import { readdir, readFile, stat as fsStat, writeFile, mkdir, rename } from "node:fs/promises";
 import path from "node:path";
 import ignore, { type Ignore } from "ignore";
+import picomatch from "picomatch";
 
 import { spawnProcess } from "./compat.ts";
 import { debug } from "./debug.ts";
 import type { PeerManager } from "./peer-manager.ts";
 import type { KnowledgeSyncFrame } from "./types.ts";
 
-/** Automerge document schema: map of relative paths to file contents. */
-interface KnowledgeDoc {
-  files: Record<string, string>;
+// ── Document schemas ──────────────────────────────────────────────
+
+/** Registry CRDT: tracks which files exist in the sync set. */
+interface RegistryDoc {
+  files: Record<string, {
+    deleted: boolean;
+    version: number;
+    updatedAt: number;
+  }>;
+}
+
+/** Per-file CRDT: just the content. */
+interface FileDoc {
+  content: string;
 }
 
+// ── Options ───────────────────────────────────────────────────────
+
 export interface KnowledgeSyncOptions {
   /** Workspace directory to watch and sync. */
   workspacePath: string;
-  /** Path to persist the Automerge document (binary). */
-  storePath: string;
+  /** Directory to persist state (.clawmatrix/). */
+  stateDir: string;
   /** Local node ID. */
   nodeId: string;
+  /** Glob whitelist patterns. Empty = sync nothing. */
+  paths: string[];
   /** Debounce interval in ms for fs changes. */
   debounce: number;
   /** Max file size in bytes. Files larger than this are skipped. */
@@ -30,6 +46,46 @@ export interface KnowledgeSyncOptions {
 }
 
 const TAG = "knowledge";
+const REGISTRY_DOC_ID = "";
+const SYNC_CONFIG_FILE = ".clawmatrix.sync";
+/** TTL for writtenByExport entries (ms). Stale entries are cleaned up to prevent leaks. */
+const EXPORT_MARKER_TTL = 30_000;
+/** Max concurrent file I/O operations during walkDir / export. */
+const MAX_IO_CONCURRENCY = 32;
+/** Delay before batched git commit (ms). */
+const GIT_COMMIT_DELAY = 1000;
+
+/** Parse .clawmatrix.sync content into glob patterns (one per line, # comments, blank lines ignored). */
+function parseSyncConfig(content: string): string[] {
+  return content
+    .split("\n")
+    .map((line) => line.trim())
+    .filter((line) => line.length > 0 && !line.startsWith("#"));
+}
+
+/** Convert relPath to safe filename using percent-encoding for path separators and existing percent signs. */
+function docFileName(relPath: string): string {
+  return relPath.replaceAll("%", "%25").replaceAll("/", "%2F") + ".automerge";
+}
+
+/** Legacy doc filename format (v0.1.x): replaced / with -- */
+function legacyDocFileName(relPath: string): string {
+  return relPath.replaceAll("/", "--") + ".automerge";
+}
+
+/** Run async tasks with bounded concurrency. */
+async function pMap<T, R>(items: T[], fn: (item: T) => Promise<R>, concurrency: number): Promise<R[]> {
+  const results: R[] = new Array(items.length);
+  let i = 0;
+  async function worker() {
+    while (i < items.length) {
+      const idx = i++;
+      results[idx] = await fn(items[idx]);
+    }
+  }
+  await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => worker()));
+  return results;
+}
 
 async function streamToString(stream: ReadableStream | null): Promise<string> {
   if (!stream) return "";
@@ -43,59 +99,139 @@ async function streamToString(stream: ReadableStream | null): Promise<string> {
   return Buffer.concat(chunks).toString("utf-8");
 }
 
+/** Update FileDoc content using Automerge.updateText for character-level CRDT merging. */
+function changeFileContent(doc: Automerge.Doc<FileDoc>, content: string): Automerge.Doc<FileDoc> {
+  const currentContent = doc.content;
+  // If content field doesn't exist yet (new doc), initialize it first
+  if (currentContent === undefined) {
+    return Automerge.change(doc, (d) => {
+      (d as FileDoc).content = content;
+    });
+  }
+  // Use updateText for minimal diff — enables proper concurrent merge
+  if (currentContent === content) return doc;
+  return Automerge.change(doc, (d) => {
+    Automerge.updateText(d, ["content"], content);
+  });
+}
+
 export class KnowledgeSync {
-  private doc: Automerge.Doc<KnowledgeDoc>;
-  private syncStates = new Map<string, Automerge.SyncState>();
+  // ── State ──────────────────────────────────────────────────────
+  private registry: Automerge.Doc<RegistryDoc>;
+  private fileDocs = new Map<string, Automerge.Doc<FileDoc>>();
+  private syncStates = new Map<string, Automerge.SyncState>(); // "peerId:docId"
+  private pendingChanges = new Set<string>();
+  private activePaths: string[] = [];  // resolved from .clawmatrix.sync or config fallback
+  private matcher: (path: string) => boolean;
+  private ig: Ignore = ignore();
+
+  // ── FS / Watcher ───────────────────────────────────────────────
   private watcher: FSWatcher | null = null;
   private debounceTimer: ReturnType<typeof setTimeout> | null = null;
-  /** Paths recently written by exportToFs with their expected content — used to
-   *  suppress watcher-triggered syncs for our own writes. Entries are cleared
-   *  once handleLocalChanges confirms the file content matches. */
-  private writtenByExport = new Map<string, string>();
+  /** Paths recently written by exportFileToFs — suppress watcher re-trigger. Stores {content, timestamp}. */
+  private writtenByExport = new Map<string, { content: string; ts: number }>();
+  /** Deferred git commit timer — batches multiple remote syncs into one commit. */
+  private gitCommitTimer: ReturnType<typeof setTimeout> | null = null;
+  private pendingGitSources = new Set<string>();
+
+  // ── Paths ──────────────────────────────────────────────────────
+  private registryPath: string;
+  private docsDir: string;
+
   private opts: KnowledgeSyncOptions;
-  private ig: Ignore = ignore();
 
   constructor(opts: KnowledgeSyncOptions) {
     this.opts = opts;
-    this.doc = Automerge.init<KnowledgeDoc>();
+    this.registry = Automerge.init<RegistryDoc>();
+    this.registryPath = path.join(opts.stateDir, "registry.automerge");
+    this.docsDir = path.join(opts.stateDir, "docs");
+    // Initialize matcher from config paths as default (loadSyncConfig may override)
+    this.activePaths = opts.paths;
+    this.matcher = opts.paths.length > 0 ? picomatch(opts.paths) : () => false;
   }
 
+  // ── Public API ─────────────────────────────────────────────────
+
   async start() {
     debug(TAG, `starting knowledge sync: workspace=${this.opts.workspacePath}`);
 
-    // Ensure workspace directory exists
     await mkdir(this.opts.workspacePath, { recursive: true });
+    await mkdir(this.docsDir, { recursive: true });
 
-    // Load .gitignore rules
     await this.loadGitignore();
 
-    // Load persisted doc or initialize from existing files
-    const loaded = await this.loadDoc();
-    if (loaded) {
-      this.doc = loaded;
-      const fileCount = Object.keys(this.doc.files ?? {}).length;
-      debug(TAG, `loaded persisted doc: ${fileCount} files`);
-      // Export doc state to filesystem (in case doc has newer data)
-      await this.exportToFs();
-    } else {
-      // First run: import existing files into doc
-      debug(TAG, "no persisted doc found, importing from filesystem");
-      await this.importFromFs();
-      const fileCount = Object.keys(this.doc.files ?? {}).length;
-      debug(TAG, `imported ${fileCount} files from workspace`);
-      await this.saveDoc();
+    // Load .clawmatrix.sync (bootstrap config file) or fall back to config.paths
+    await this.loadSyncConfig();
+
+    // Migrate from legacy single-doc format if needed
+    await this.migrateFromLegacy();
+
+    // Load registry
+    const loadedRegistry = await this.loadAutomergeDoc<RegistryDoc>(this.registryPath);
+    if (loadedRegistry) {
+      this.registry = loadedRegistry;
+      const entries = Object.keys(this.registry.files ?? {}).filter(
+        (k) => !this.registry.files[k].deleted,
+      );
+      debug(TAG, `loaded registry: ${entries.length} active files`);
+    }
+
+    // Load all per-file docs referenced in registry (with fallback to legacy filename format)
+    const registryFiles = this.registry.files ?? {};
+    for (const [relPath, meta] of Object.entries(registryFiles)) {
+      if (meta.deleted) continue;
+      const newName = docFileName(relPath);
+      const docPath = path.join(this.docsDir, newName);
+      let doc = await this.loadAutomergeDoc<FileDoc>(docPath);
+      // Fallback: try legacy "--" filename format and migrate if found
+      if (!doc) {
+        const legacyName = legacyDocFileName(relPath);
+        if (legacyName !== newName) {
+          const legacyPath = path.join(this.docsDir, legacyName);
+          doc = await this.loadAutomergeDoc<FileDoc>(legacyPath);
+          if (doc) {
+            await this.saveAutomergeDoc(docPath, doc);
+            await rename(legacyPath, legacyPath + ".migrated").catch(() => {});
+            debug(TAG, `migrated doc filename: ${legacyName} → ${newName}`);
+          }
+        }
+      }
+      if (doc) {
+        this.fileDocs.set(relPath, doc);
+      }
     }
 
-    // Initialize git repo if not already
+    // Bootstrap: ensure .clawmatrix.sync itself is a synced CRDT doc
+    await this.ensureSyncConfigDoc();
+
+    // Import current workspace files that match whitelist
+    await this.importFromFs();
+
+    // Export any doc state to filesystem
+    await this.exportAllToFs();
+
+    // Initialize git repo
     await this.gitInit();
 
     // Start watching for file changes
     this.watcher = watch(this.opts.workspacePath, { recursive: true }, (_event, filename) => {
       if (!filename) return;
-      // Ignore hidden files
+      // .clawmatrix.sync is special — always watch it
+      if (filename === SYNC_CONFIG_FILE) {
+        this.pendingChanges.add(filename);
+        this.scheduleSync();
+        return;
+      }
+      // .gitignore change triggers reload on next handleLocalChanges
+      if (filename === ".gitignore") {
+        this.pendingChanges.add(filename);
+        this.scheduleSync();
+        return;
+      }
       if (filename.startsWith(".")) return;
-      // Ignore gitignored files
       if (this.ig.ignores(filename)) return;
+      if (!this.matchesWhitelist(filename)) return;
+      this.pendingChanges.add(filename);
       this.scheduleSync();
     });
 
@@ -110,75 +246,223 @@ export class KnowledgeSync {
     }
     this.watcher?.close();
     this.watcher = null;
-    await this.saveDoc();
-    debug(TAG, "knowledge sync stopped, doc saved");
+    await this.flushPendingGitCommit();
+    await this.saveAll();
+    debug(TAG, "knowledge sync stopped");
   }
 
   /** Handle incoming knowledge_sync frame from a peer. */
   async handleSyncMessage(frame: KnowledgeSyncFrame) {
     const peerId = frame.from;
-    const msgSize = frame.payload.data.length;
-    debug(TAG, `received sync message from ${peerId} (${msgSize} bytes)`);
-
-    const syncState = this.syncStates.get(peerId) ?? Automerge.initSyncState();
-    const message = new Uint8Array(Buffer.from(frame.payload.data, "base64"));
+    const { docId, data } = frame.payload;
 
-    const [newDoc, newSyncState, _patch] = Automerge.receiveSyncMessage(
-      this.doc,
-      syncState,
-      message,
-    );
-
-    const oldFileCount = Object.keys(this.doc.files ?? {}).length;
-    const newFileCount = Object.keys(newDoc.files ?? {}).length;
-    this.doc = newDoc;
-    this.syncStates.set(peerId, newSyncState);
-
-    if (oldFileCount !== newFileCount) {
-      debug(TAG, `doc updated: ${oldFileCount} → ${newFileCount} files`);
+    // Backward compat: ignore frames without docId (from old nodes)
+    if (docId === undefined || docId === null) {
+      debug(TAG, `ignoring legacy sync frame from ${peerId} (no docId)`);
+      return;
     }
 
-    // Export changes to filesystem and git commit
-    await this.exportToFs();
-    await this.saveDoc();
-    await this.gitCommit(`sync from ${peerId}`);
+    const message = new Uint8Array(Buffer.from(data, "base64"));
+    const syncKey = `${peerId}:${docId}`;
 
-    // Continue sync protocol (may need to send response)
-    this.sendSyncMessage(peerId);
+    try {
+      if (docId === REGISTRY_DOC_ID) {
+        await this.handleRegistrySync(peerId, syncKey, message);
+      } else {
+        await this.handleFileSync(peerId, docId, syncKey, message);
+      }
+    } catch (err) {
+      debug(TAG, `error handling sync message from ${peerId} docId=${docId || "registry"}: ${err}`);
+    }
   }
 
   /** Called when a new peer connects — initiate sync. */
   initPeerSync(peerId: string) {
+    if (peerId === this.opts.nodeId) return; // skip self
     debug(TAG, `initiating sync with peer ${peerId}`);
-    this.syncStates.set(peerId, Automerge.initSyncState());
-    this.sendSyncMessage(peerId);
+    // Start with registry sync — file syncs follow after registry exchange
+    const syncKey = `${peerId}:${REGISTRY_DOC_ID}`;
+    this.syncStates.set(syncKey, Automerge.initSyncState());
+    this.sendSyncMessage(peerId, REGISTRY_DOC_ID);
+
+    // Also initiate sync for all known file docs
+    for (const relPath of this.fileDocs.keys()) {
+      this.syncDocWithPeer(peerId, relPath);
+    }
   }
 
   /** Called when a peer disconnects — clean up sync state. */
   removePeerSync(peerId: string) {
     debug(TAG, `removing sync state for peer ${peerId}`);
-    this.syncStates.delete(peerId);
+    const prefix = `${peerId}:`;
+    for (const key of this.syncStates.keys()) {
+      if (key.startsWith(prefix)) {
+        this.syncStates.delete(key);
+      }
+    }
   }
 
-  // ── Private methods ─────────────────────────────────────────────
+  // ── Private: frame handlers ─────────────────────────────────────
 
-  private async loadGitignore() {
-    this.ig = ignore();
-    // Always ignore .git and automerge store
-    this.ig.add([".git", ".clawmatrix"]);
-    try {
-      const content = await readFile(path.join(this.opts.workspacePath, ".gitignore"), "utf-8");
-      this.ig.add(content);
-      debug(TAG, "loaded .gitignore rules");
-    } catch {
-      // No .gitignore — that's fine
+  private async handleRegistrySync(peerId: string, syncKey: string, message: Uint8Array) {
+    const syncState = this.syncStates.get(syncKey) ?? Automerge.initSyncState();
+    const [newDoc, newSyncState] = Automerge.receiveSyncMessage(
+      this.registry,
+      syncState,
+      message,
+    );
+
+    const oldFiles = Object.keys(this.registry.files ?? {});
+    this.registry = newDoc;
+    this.syncStates.set(syncKey, newSyncState);
+    const newFiles = Object.keys(newDoc.files ?? {});
+
+    if (oldFiles.length !== newFiles.length) {
+      debug(TAG, `registry updated from ${peerId}: ${oldFiles.length} → ${newFiles.length} entries`);
+    }
+
+    await this.saveAutomergeDoc(this.registryPath, this.registry);
+
+    // Discover new files from registry and initiate their sync
+    for (const [relPath, meta] of Object.entries(newDoc.files ?? {})) {
+      if (meta.deleted) {
+        // Clean up sync states for deleted files
+        this.cleanupDeletedFileSyncStates(relPath);
+        continue;
+      }
+      if (!this.fileDocs.has(relPath)) {
+        this.fileDocs.set(relPath, Automerge.init<FileDoc>());
+        debug(TAG, `discovered new file from registry: ${relPath}`);
+      }
+      this.syncDocWithPeer(peerId, relPath);
     }
+
+    this.sendSyncMessage(peerId, REGISTRY_DOC_ID);
   }
 
-  private isIgnored(relPath: string): boolean {
-    return this.ig.ignores(relPath);
+  private async handleFileSync(peerId: string, docId: string, syncKey: string, message: Uint8Array) {
+    let doc = this.fileDocs.get(docId) ?? Automerge.init<FileDoc>();
+    const syncState = this.syncStates.get(syncKey) ?? Automerge.initSyncState();
+
+    const [newDoc, newSyncState] = Automerge.receiveSyncMessage(
+      doc,
+      syncState,
+      message,
+    );
+
+    this.fileDocs.set(docId, newDoc);
+    this.syncStates.set(syncKey, newSyncState);
+
+    await this.saveFileDoc(docId);
+    await this.exportFileToFs(docId);
+
+    // Reload sync config only when content is non-empty (partial syncs may yield empty content)
+    if (docId === SYNC_CONFIG_FILE && newDoc.content) {
+      await this.loadSyncConfig();
+      debug(TAG, `sync config updated from ${peerId}, reloaded patterns: ${JSON.stringify(this.activePaths)}`);
+    }
+
+    // Defer git commit — schedule instead of immediate
+    this.schedulePendingGitCommit(peerId);
+
+    this.sendSyncMessage(peerId, docId);
+  }
+
+  /** Remove sync states for a deleted file across all peers. */
+  private cleanupDeletedFileSyncStates(docId: string) {
+    const suffix = `:${docId}`;
+    for (const key of this.syncStates.keys()) {
+      if (key.endsWith(suffix)) {
+        this.syncStates.delete(key);
+      }
+    }
+  }
+
+  // ── Whitelist matching ─────────────────────────────────────────
+
+  matchesWhitelist(relPath: string): boolean {
+    if (relPath === SYNC_CONFIG_FILE) return true; // always synced
+    if (this.activePaths.length === 0) return false;
+    return this.matcher(relPath);
+  }
+
+  // ── Private: sync helpers ──────────────────────────────────────
+
+  private syncDocWithPeer(peerId: string, docId: string) {
+    const syncKey = `${peerId}:${docId}`;
+    if (!this.syncStates.has(syncKey)) {
+      this.syncStates.set(syncKey, Automerge.initSyncState());
+    }
+    this.sendSyncMessage(peerId, docId);
+  }
+
+  private sendSyncMessage(peerId: string, docId: string) {
+    const syncKey = `${peerId}:${docId}`;
+    const syncState = this.syncStates.get(syncKey) ?? Automerge.initSyncState();
+    const doc = docId === REGISTRY_DOC_ID ? this.registry : this.fileDocs.get(docId);
+    if (!doc) return;
+
+    const [newSyncState, message] = Automerge.generateSyncMessage(doc, syncState);
+    this.syncStates.set(syncKey, newSyncState);
+
+    if (!message) return;
+
+    debug(TAG, `sending sync for ${docId || "registry"} to ${peerId} (${message.byteLength} bytes)`);
+
+    const frame: KnowledgeSyncFrame = {
+      type: "knowledge_sync",
+      from: this.opts.nodeId,
+      to: peerId,
+      timestamp: Date.now(),
+      payload: {
+        docId,
+        data: Buffer.from(message).toString("base64"),
+      },
+    };
+
+    this.opts.peerManager.router.sendTo(peerId, frame);
+  }
+
+  private broadcastSync(docId: string) {
+    const peers = this.opts.peerManager.router.getAllPeers();
+    for (const peer of peers) {
+      this.syncDocWithPeer(peer.nodeId, docId);
+    }
+  }
+
+  // ── Private: deferred git commit ────────────────────────────────
+
+  /** Batch multiple remote syncs into a single git commit after a short delay. */
+  private schedulePendingGitCommit(source: string) {
+    this.pendingGitSources.add(source);
+    if (this.gitCommitTimer) return; // already scheduled
+    this.gitCommitTimer = setTimeout(() => {
+      this.gitCommitTimer = null;
+      const sources = [...this.pendingGitSources];
+      this.pendingGitSources.clear();
+      if (sources.length > 0) {
+        const msg = `sync from ${sources.join(", ")}`;
+        this.gitCommit(msg).catch((err) => {
+          debug(TAG, `deferred git commit error: ${err}`);
+        });
+      }
+    }, GIT_COMMIT_DELAY);
+  }
+
+  private async flushPendingGitCommit() {
+    if (this.gitCommitTimer) {
+      clearTimeout(this.gitCommitTimer);
+      this.gitCommitTimer = null;
+    }
+    if (this.pendingGitSources.size > 0) {
+      const sources = [...this.pendingGitSources];
+      this.pendingGitSources.clear();
+      await this.gitCommit(`sync from ${sources.join(", ")}`);
+    }
   }
 
+  // ── Private: local change handling ─────────────────────────────
+
   private scheduleSync() {
     if (this.debounceTimer) clearTimeout(this.debounceTimer);
     this.debounceTimer = setTimeout(() => {
@@ -190,41 +474,75 @@ export class KnowledgeSync {
   }
 
   private async handleLocalChanges() {
-    // Reload .gitignore in case it changed
-    await this.loadGitignore();
+    // Only process files in pendingChanges (incremental)
+    const changesToProcess = new Set(this.pendingChanges);
+    this.pendingChanges.clear();
 
-    const currentFiles = await this.readWorkspaceFiles();
-    const docFiles = this.doc.files ?? {};
+    if (changesToProcess.size === 0) return;
 
-    // Clear export markers whose content matches the current file (our write landed).
-    // If the content differs, a real local edit happened after export — treat it as modified.
-    for (const [relPath, expectedContent] of this.writtenByExport) {
-      if (currentFiles[relPath] === expectedContent) {
-        this.writtenByExport.delete(relPath);
-      }
+    // Reload .gitignore only when it actually changed
+    if (changesToProcess.has(".gitignore")) {
+      await this.loadGitignore();
+      changesToProcess.delete(".gitignore");
+    }
+
+    // If .clawmatrix.sync changed, reload patterns before processing other files
+    if (changesToProcess.has(SYNC_CONFIG_FILE)) {
+      await this.loadSyncConfig();
     }
 
-    // Collect changed files for logging
+    // Clean up stale export markers
+    this.cleanupStaleExportMarkers();
+
     const added: string[] = [];
     const modified: string[] = [];
     const deleted: string[] = [];
+    // Cache file contents from detection phase to avoid re-reading
+    const contentCache = new Map<string, string>();
 
-    for (const [relPath, content] of Object.entries(currentFiles)) {
-      // Skip files that were just written by exportToFs and haven't been edited since
-      if (this.writtenByExport.has(relPath)) continue;
-      if (!(relPath in docFiles)) {
-        added.push(relPath);
-      } else if (docFiles[relPath] !== content) {
-        modified.push(relPath);
+    for (const relPath of changesToProcess) {
+      if (!this.matchesWhitelist(relPath)) continue;
+      if (relPath !== SYNC_CONFIG_FILE && this.isIgnored(relPath)) continue;
+
+      // Clear export marker if content matches
+      const exportMarker = this.writtenByExport.get(relPath);
+
+      const absPath = path.join(this.opts.workspacePath, relPath);
+      let currentContent: string | null = null;
+      try {
+        const st = await fsStat(absPath);
+        if (st.size > this.opts.maxFileSize) continue;
+        currentContent = await readFile(absPath, "utf-8");
+      } catch {
+        // File doesn't exist — deletion
       }
-    }
 
-    for (const relPath of Object.keys(docFiles)) {
-      // Only treat as deleted if the file is genuinely gone, not just skipped by size limit
-      if (!(relPath in currentFiles)) {
-        const absPath = path.join(this.opts.workspacePath, relPath);
-        const exists = await fsStat(absPath).then(() => true, () => false);
-        if (!exists) deleted.push(relPath);
+      if (exportMarker !== undefined) {
+        if (currentContent === exportMarker.content) {
+          this.writtenByExport.delete(relPath);
+          continue; // Our own write, skip
+        }
+        this.writtenByExport.delete(relPath);
+      }
+
+      const existingDoc = this.fileDocs.get(relPath);
+      const registryEntry = this.registry.files?.[relPath];
+
+      if (currentContent === null) {
+        // File deleted
+        if (existingDoc || (registryEntry && !registryEntry.deleted)) {
+          deleted.push(relPath);
+        }
+      } else if (!existingDoc || !registryEntry || registryEntry.deleted) {
+        added.push(relPath);
+        contentCache.set(relPath, currentContent);
+      } else {
+        // Check if content changed
+        const docContent = existingDoc.content ?? "";
+        if (docContent !== currentContent) {
+          modified.push(relPath);
+          contentCache.set(relPath, currentContent);
+        }
       }
     }
 
@@ -232,68 +550,275 @@ export class KnowledgeSync {
 
     debug(
       TAG,
-      `local changes detected: +${added.length} ~${modified.length} -${deleted.length}` +
+      `local changes: +${added.length} ~${modified.length} -${deleted.length}` +
         (added.length > 0 ? ` added=[${added.join(",")}]` : "") +
         (modified.length > 0 ? ` modified=[${modified.join(",")}]` : "") +
         (deleted.length > 0 ? ` deleted=[${deleted.join(",")}]` : ""),
     );
 
-    // Update automerge doc — only write dirty files to minimize change history
-    this.doc = Automerge.change(this.doc, (d) => {
-      if (!d.files) {
-        (d as KnowledgeDoc).files = {};
+    // Process additions and modifications — reuse cached content
+    for (const relPath of [...added, ...modified]) {
+      const content = contentCache.get(relPath)!;
+
+      let doc = this.fileDocs.get(relPath) ?? Automerge.init<FileDoc>();
+      doc = changeFileContent(doc, content);
+      this.fileDocs.set(relPath, doc);
+
+      await this.saveFileDoc(relPath);
+      this.broadcastSync(relPath);
+    }
+
+    // Process deletions (tombstone in registry)
+    for (const relPath of deleted) {
+      this.fileDocs.delete(relPath);
+      this.cleanupDeletedFileSyncStates(relPath);
+      // Remove persisted file doc
+      try {
+        const docPath = path.join(this.docsDir, docFileName(relPath));
+        await rename(docPath, docPath + ".deleted");
+      } catch { /* ignore */ }
+    }
+
+    // Update registry
+    this.registry = Automerge.change(this.registry, (d) => {
+      if (!d.files) (d as RegistryDoc).files = {};
+      for (const relPath of added) {
+        d.files[relPath] = {
+          deleted: false,
+          version: 1,
+          updatedAt: Date.now(),
+        };
+      }
+      for (const relPath of modified) {
+        const existing = d.files[relPath];
+        if (existing) {
+          existing.version = (existing.version ?? 0) + 1;
+          existing.updatedAt = Date.now();
+        }
+      }
+      for (const relPath of deleted) {
+        if (d.files[relPath]) {
+          d.files[relPath].deleted = true;
+          d.files[relPath].updatedAt = Date.now();
+        }
       }
-      for (const p of added)    d.files[p] = currentFiles[p];
-      for (const p of modified) d.files[p] = currentFiles[p];
-      for (const p of deleted)  delete d.files[p];
     });
 
-    await this.saveDoc();
+    await this.saveAutomergeDoc(this.registryPath, this.registry);
     await this.gitCommit("local changes");
 
-    // Send sync messages to all peers
-    const peerCount = this.opts.peerManager.router.getAllPeers().length;
-    debug(TAG, `broadcasting changes to ${peerCount} peer(s)`);
-    this.broadcastSync();
+    // Broadcast registry + changed files
+    this.broadcastSync(REGISTRY_DOC_ID);
   }
 
-  private broadcastSync() {
-    const peers = this.opts.peerManager.router.getAllPeers();
-    for (const peer of peers) {
-      if (!this.syncStates.has(peer.nodeId)) {
-        this.syncStates.set(peer.nodeId, Automerge.initSyncState());
+  // ── Private: sync config (.clawmatrix.sync) ─────────────────────
+
+  /** Load .clawmatrix.sync from filesystem or CRDT doc, fall back to config.paths. */
+  private async loadSyncConfig() {
+    const syncConfigPath = path.join(this.opts.workspacePath, SYNC_CONFIG_FILE);
+    let content: string | null = null;
+
+    // Try filesystem first
+    try {
+      content = await readFile(syncConfigPath, "utf-8");
+    } catch { /* doesn't exist */ }
+
+    // If not on filesystem, try the CRDT doc (e.g. just synced from remote, not yet exported)
+    if (content === null) {
+      const doc = this.fileDocs.get(SYNC_CONFIG_FILE);
+      if (doc?.content) {
+        content = doc.content;
+      }
+    }
+
+    if (content !== null) {
+      const patterns = parseSyncConfig(content);
+      this.activePaths = patterns;
+      this.matcher = patterns.length > 0 ? picomatch(patterns) : () => false;
+      debug(TAG, `loaded sync config: ${patterns.length} patterns from ${SYNC_CONFIG_FILE}`);
+    } else {
+      // Fall back to config.paths
+      this.activePaths = this.opts.paths;
+      this.matcher = this.opts.paths.length > 0 ? picomatch(this.opts.paths) : () => false;
+      if (this.opts.paths.length > 0) {
+        debug(TAG, `no ${SYNC_CONFIG_FILE}, using config fallback: ${JSON.stringify(this.opts.paths)}`);
       }
-      this.sendSyncMessage(peer.nodeId);
     }
   }
 
-  private sendSyncMessage(peerId: string) {
-    const syncState = this.syncStates.get(peerId) ?? Automerge.initSyncState();
-    const [newSyncState, message] = Automerge.generateSyncMessage(this.doc, syncState);
+  /** Ensure .clawmatrix.sync exists as a CRDT doc if it exists on filesystem. */
+  private async ensureSyncConfigDoc() {
+    const syncConfigPath = path.join(this.opts.workspacePath, SYNC_CONFIG_FILE);
+    let content: string | null = null;
+    try {
+      content = await readFile(syncConfigPath, "utf-8");
+    } catch { /* doesn't exist */ }
 
-    this.syncStates.set(peerId, newSyncState);
+    if (content === null) return;
 
-    if (!message) {
-      debug(TAG, `no sync message to send to ${peerId} (already in sync)`);
-      return;
+    const existingDoc = this.fileDocs.get(SYNC_CONFIG_FILE);
+    if (existingDoc && existingDoc.content === content) return;
+
+    // Create or update the CRDT doc
+    let doc = existingDoc ?? Automerge.init<FileDoc>();
+    doc = changeFileContent(doc, content);
+    this.fileDocs.set(SYNC_CONFIG_FILE, doc);
+
+    // Add to registry
+    this.registry = Automerge.change(this.registry, (d) => {
+      if (!d.files) (d as RegistryDoc).files = {};
+      const existing = d.files[SYNC_CONFIG_FILE];
+      if (!existing || existing.deleted) {
+        d.files[SYNC_CONFIG_FILE] = { deleted: false, version: 1, updatedAt: Date.now() };
+      } else {
+        existing.version = (existing.version ?? 0) + 1;
+        existing.updatedAt = Date.now();
+      }
+    });
+
+    await this.saveFileDoc(SYNC_CONFIG_FILE);
+    await this.saveAutomergeDoc(this.registryPath, this.registry);
+    debug(TAG, `ensured ${SYNC_CONFIG_FILE} as CRDT doc`);
+  }
+
+  // ── Private: filesystem I/O ────────────────────────────────────
+
+  private async loadGitignore() {
+    this.ig = ignore();
+    this.ig.add([".git", ".clawmatrix"]);
+    try {
+      const content = await readFile(path.join(this.opts.workspacePath, ".gitignore"), "utf-8");
+      this.ig.add(content);
+    } catch { /* no .gitignore */ }
+  }
+
+  private isIgnored(relPath: string): boolean {
+    return this.ig.ignores(relPath);
+  }
+
+  /** Clean up stale export markers that were never consumed by the watcher. */
+  private cleanupStaleExportMarkers() {
+    const now = Date.now();
+    for (const [relPath, marker] of this.writtenByExport) {
+      if (now - marker.ts > EXPORT_MARKER_TTL) {
+        this.writtenByExport.delete(relPath);
+      }
+    }
+  }
+
+  /** Import workspace files matching whitelist into file docs. */
+  private async importFromFs() {
+    const files = await this.readWhitelistedFiles();
+
+    let imported = 0;
+    const changedPaths: string[] = [];
+    for (const [relPath, content] of Object.entries(files)) {
+      const existingDoc = this.fileDocs.get(relPath);
+      const existingContent = existingDoc?.content ?? "";
+
+      if (!existingDoc) {
+        // New file — create doc
+        let doc = Automerge.init<FileDoc>();
+        doc = changeFileContent(doc, content);
+        this.fileDocs.set(relPath, doc);
+
+        // Add to registry
+        this.registry = Automerge.change(this.registry, (d) => {
+          if (!d.files) (d as RegistryDoc).files = {};
+          if (!d.files[relPath] || d.files[relPath].deleted) {
+            d.files[relPath] = {
+              deleted: false,
+              version: 1,
+              updatedAt: Date.now(),
+            };
+          }
+        });
+        changedPaths.push(relPath);
+        imported++;
+      } else if (existingContent !== content) {
+        // Content changed — update doc
+        const newDoc = changeFileContent(existingDoc, content);
+        this.fileDocs.set(relPath, newDoc);
+
+        this.registry = Automerge.change(this.registry, (d) => {
+          if (!d.files) (d as RegistryDoc).files = {};
+          const entry = d.files[relPath];
+          if (entry) {
+            entry.version = (entry.version ?? 0) + 1;
+            entry.updatedAt = Date.now();
+          }
+        });
+        changedPaths.push(relPath);
+        imported++;
+      }
     }
 
-    debug(TAG, `sending sync message to ${peerId} (${message.byteLength} bytes)`);
+    if (imported > 0) {
+      debug(TAG, `imported ${imported} files from workspace`);
+      // Only save changed docs + registry (not all docs)
+      await this.saveAutomergeDoc(this.registryPath, this.registry);
+      await Promise.all(changedPaths.map((relPath) => this.saveFileDoc(relPath)));
+    }
+  }
 
-    const frame: KnowledgeSyncFrame = {
-      type: "knowledge_sync",
-      from: this.opts.nodeId,
-      to: peerId,
-      timestamp: Date.now(),
-      payload: {
-        data: Buffer.from(message).toString("base64"),
-      },
-    };
+  /** Export all file docs to filesystem. */
+  private async exportAllToFs() {
+    const writeOps: Array<{ relPath: string; content: string; absPath: string }> = [];
+    for (const [relPath, doc] of this.fileDocs) {
+      const meta = this.registry.files?.[relPath];
+      if (meta?.deleted) continue;
+      if (this.isIgnored(relPath)) continue;
 
-    this.opts.peerManager.router.sendTo(peerId, frame);
+      const content = doc.content ?? "";
+      const absPath = path.join(this.opts.workspacePath, relPath);
+
+      let currentContent: string | null = null;
+      try {
+        currentContent = await readFile(absPath, "utf-8");
+      } catch { /* file doesn't exist */ }
+
+      if (currentContent !== content) {
+        writeOps.push({ relPath, content, absPath });
+      }
+    }
+
+    if (writeOps.length > 0) {
+      await pMap(writeOps, async ({ relPath, content, absPath }) => {
+        this.writtenByExport.set(relPath, { content, ts: Date.now() });
+        await mkdir(path.dirname(absPath), { recursive: true });
+        await writeFile(absPath, content, "utf-8");
+      }, MAX_IO_CONCURRENCY);
+      debug(TAG, `exported ${writeOps.length} files to filesystem`);
+    }
   }
 
-  private async readWorkspaceFiles(): Promise<Record<string, string>> {
+  /** Export a single file doc to filesystem. */
+  private async exportFileToFs(relPath: string) {
+    const doc = this.fileDocs.get(relPath);
+    if (!doc) return;
+
+    const meta = this.registry.files?.[relPath];
+    if (meta?.deleted) return;
+    if (this.isIgnored(relPath)) return;
+
+    const content = doc.content ?? "";
+    const absPath = path.join(this.opts.workspacePath, relPath);
+
+    let currentContent: string | null = null;
+    try {
+      currentContent = await readFile(absPath, "utf-8");
+    } catch { /* file doesn't exist */ }
+
+    if (currentContent !== content) {
+      this.writtenByExport.set(relPath, { content, ts: Date.now() });
+      await mkdir(path.dirname(absPath), { recursive: true });
+      await writeFile(absPath, content, "utf-8");
+      debug(TAG, `exported file: ${relPath}`);
+    }
+  }
+
+  /** Read all workspace files matching whitelist. */
+  private async readWhitelistedFiles(): Promise<Record<string, string>> {
     const files: Record<string, string> = {};
     await this.walkDir(this.opts.workspacePath, "", files);
     return files;
@@ -301,58 +826,138 @@ export class KnowledgeSync {
 
   private async walkDir(base: string, rel: string, result: Record<string, string>) {
     const entries = await readdir(path.join(base, rel), { withFileTypes: true });
+
+    // Separate files and directories for parallel processing
+    const fileEntries: Array<{ relPath: string; filePath: string }> = [];
+    const dirEntries: string[] = [];
+
     for (const entry of entries) {
       const relPath = rel ? `${rel}/${entry.name}` : entry.name;
-      // Skip hidden files/dirs
-      if (entry.name.startsWith(".")) continue;
+      if (entry.name.startsWith(".")) {
+        // Allow .clawmatrix.sync at workspace root
+        if (relPath === SYNC_CONFIG_FILE && entry.isFile()) {
+          fileEntries.push({ relPath, filePath: path.join(base, relPath) });
+        }
+        continue;
+      }
       if (entry.isDirectory()) {
-        // For directories, check with trailing slash so negation rules like !memory/ work
         if (this.isIgnored(relPath + "/")) continue;
-        await this.walkDir(base, relPath, result);
+        dirEntries.push(relPath);
       } else if (entry.isFile()) {
         if (this.isIgnored(relPath)) continue;
-        const filePath = path.join(base, relPath);
-        const st = await fsStat(filePath);
-        if (st.size > this.opts.maxFileSize) continue;
-        const content = await readFile(filePath, "utf-8");
-        result[relPath] = content;
+        if (!this.matchesWhitelist(relPath)) continue;
+        fileEntries.push({ relPath, filePath: path.join(base, relPath) });
       }
     }
+
+    // Read files with bounded concurrency
+    await pMap(fileEntries, async ({ relPath, filePath }) => {
+      try {
+        const st = await fsStat(filePath);
+        if (st.size > this.opts.maxFileSize) return;
+        result[relPath] = await readFile(filePath, "utf-8");
+      } catch { /* file disappeared between readdir and read */ }
+    }, MAX_IO_CONCURRENCY);
+
+    // Recurse into directories (each walkDir internally bounds its own I/O)
+    await Promise.all(dirEntries.map((dirRel) => this.walkDir(base, dirRel, result)));
   }
 
-  private async exportToFs() {
-    const docFiles = this.doc.files ?? {};
-    const currentFiles = await this.readWorkspaceFiles();
+  // ── Private: persistence ───────────────────────────────────────
 
-    let written = 0;
+  private async loadAutomergeDoc<T>(filePath: string): Promise<Automerge.Doc<T> | null> {
+    try {
+      const data = await readFile(filePath);
+      return Automerge.load<T>(new Uint8Array(data));
+    } catch {
+      return null;
+    }
+  }
 
-    for (const [relPath, content] of Object.entries(docFiles)) {
-      // Don't export files that would be gitignored
-      if (this.isIgnored(relPath)) continue;
-      if (currentFiles[relPath] !== content) {
-        this.writtenByExport.set(relPath, content);
-        const absPath = path.join(this.opts.workspacePath, relPath);
-        await mkdir(path.dirname(absPath), { recursive: true });
-        await writeFile(absPath, content, "utf-8");
-        written++;
+  private async saveAutomergeDoc<T>(filePath: string, doc: Automerge.Doc<T>) {
+    const data = Automerge.save(doc);
+    await mkdir(path.dirname(filePath), { recursive: true });
+    await writeFile(filePath, Buffer.from(data));
+  }
+
+  private async saveFileDoc(relPath: string) {
+    const doc = this.fileDocs.get(relPath);
+    if (!doc) return;
+    const docPath = path.join(this.docsDir, docFileName(relPath));
+    await this.saveAutomergeDoc(docPath, doc);
+  }
+
+  private async saveAll() {
+    await this.saveAutomergeDoc(this.registryPath, this.registry);
+    await Promise.all(
+      [...this.fileDocs].map(([relPath]) => this.saveFileDoc(relPath)),
+    );
+  }
+
+  // ── Private: migration from legacy format ──────────────────────
+
+  private async migrateFromLegacy() {
+    const legacyPath = path.join(this.opts.stateDir, "knowledge.automerge");
+    const registryExists = await fsStat(this.registryPath).then(() => true, () => false);
+
+    if (registryExists) return; // Already migrated or fresh install
+
+    let legacyDoc: Automerge.Doc<{ files: Record<string, string> }> | null = null;
+    try {
+      const data = await readFile(legacyPath);
+      legacyDoc = Automerge.load<{ files: Record<string, string> }>(new Uint8Array(data));
+    } catch {
+      return; // No legacy file
+    }
+
+    const legacyFiles = legacyDoc.files ?? {};
+    const fileCount = Object.keys(legacyFiles).length;
+    if (fileCount === 0) return;
+
+    debug(TAG, `migrating ${fileCount} files from legacy single-doc format`);
+
+    // Create per-file docs + registry entries
+    for (const [relPath, content] of Object.entries(legacyFiles)) {
+      // Only migrate files matching current whitelist (or all if no whitelist)
+      if (this.activePaths.length > 0 && !this.matchesWhitelist(relPath)) {
+        debug(TAG, `migration: skipping ${relPath} (not in whitelist)`);
+        continue;
       }
+
+      let doc = Automerge.init<FileDoc>();
+      doc = Automerge.change(doc, (d) => {
+        (d as FileDoc).content = content;
+      });
+      this.fileDocs.set(relPath, doc);
+
+      this.registry = Automerge.change(this.registry, (d) => {
+        if (!d.files) (d as RegistryDoc).files = {};
+        d.files[relPath] = {
+          deleted: false,
+          version: 1,
+          updatedAt: Date.now(),
+        };
+      });
     }
 
-    // Note: we intentionally do NOT delete local files that are absent from
-    // the doc. A locally created file that hasn't been synced yet would be
-    // lost if we deleted it here. Deletions propagate through the doc via
-    // handleLocalChanges() → Automerge change → broadcastSync() instead.
+    // Save all new docs
+    await this.saveAll();
 
-    if (written > 0) {
-      debug(TAG, `exported to filesystem: ${written} written`);
+    // Rename legacy file
+    try {
+      await rename(legacyPath, legacyPath + ".migrated");
+      debug(TAG, "legacy file renamed to knowledge.automerge.migrated");
+    } catch {
+      debug(TAG, "warning: could not rename legacy file");
     }
+
+    debug(TAG, `migration complete: ${this.fileDocs.size} files migrated`);
   }
 
-  // ── Git operations ──────────────────────────────────────────────
+  // ── Git operations ─────────────────────────────────────────────
 
   private async gitInit() {
     try {
-      // Check if already a git repo
       const proc = spawnProcess(["git", "rev-parse", "--git-dir"], {
         cwd: this.opts.workspacePath,
         stdout: "pipe",
@@ -360,7 +965,6 @@ export class KnowledgeSync {
       });
       const code = await proc.exited;
       if (code !== 0) {
-        // Not a git repo — initialize
         const init = spawnProcess(["git", "init"], {
           cwd: this.opts.workspacePath,
           stdout: "pipe",
@@ -368,11 +972,9 @@ export class KnowledgeSync {
         });
         await init.exited;
         debug(TAG, `initialized git repo at ${this.opts.workspacePath}`);
-      } else {
-        debug(TAG, "git repo already initialized");
       }
 
-      // Ensure local git user config exists (so commits don't fail on unconfigured machines)
+      // Ensure git user config
       const nameCheck = spawnProcess(["git", "config", "user.name"], {
         cwd: this.opts.workspacePath,
         stdout: "pipe",
@@ -406,7 +1008,6 @@ export class KnowledgeSync {
 
   private async gitCommit(message: string) {
     try {
-      // git add -A (respects .gitignore)
       const add = spawnProcess(["git", "add", "-A"], {
         cwd: this.opts.workspacePath,
         stdout: "pipe",
@@ -414,16 +1015,14 @@ export class KnowledgeSync {
       });
       await add.exited;
 
-      // Check if there are staged changes
       const diff = spawnProcess(["git", "diff", "--cached", "--quiet"], {
         cwd: this.opts.workspacePath,
         stdout: "pipe",
         stderr: "pipe",
       });
       const diffCode = await diff.exited;
-      if (diffCode === 0) return; // Nothing to commit
+      if (diffCode === 0) return;
 
-      // Commit
       const commit = spawnProcess(
         ["git", "commit", "-m", `[clawmatrix] ${message}`],
         {
@@ -443,34 +1042,4 @@ export class KnowledgeSync {
       debug(TAG, `git commit failed: ${err}`);
     }
   }
-
-  // ── Persistence ─────────────────────────────────────────────────
-
-  private async loadDoc(): Promise<Automerge.Doc<KnowledgeDoc> | null> {
-    try {
-      const data = await readFile(this.opts.storePath);
-      debug(TAG, `loaded automerge doc from ${this.opts.storePath} (${data.byteLength} bytes)`);
-      return Automerge.load<KnowledgeDoc>(new Uint8Array(data));
-    } catch {
-      return null;
-    }
-  }
-
-  private async saveDoc() {
-    const data = Automerge.save(this.doc);
-    await mkdir(path.dirname(this.opts.storePath), { recursive: true });
-    await writeFile(this.opts.storePath, Buffer.from(data));
-  }
-
-  private async importFromFs() {
-    const files = await this.readWorkspaceFiles();
-    if (Object.keys(files).length === 0) return;
-
-    this.doc = Automerge.change(this.doc, (d) => {
-      (d as KnowledgeDoc).files = {};
-      for (const [relPath, content] of Object.entries(files)) {
-        d.files[relPath] = content;
-      }
-    });
-  }
 }