clawmatrix 0.1.16 → 0.1.18

package/src/knowledge-sync.ts

@@ -0,0 +1,426 @@
+import * as Automerge from "@automerge/automerge";
+import { watch, type FSWatcher } from "node:fs";
+import { readdir, readFile, writeFile, mkdir, unlink } from "node:fs/promises";
+import path from "node:path";
+import ignore, { type Ignore } from "ignore";
+
+import { spawnProcess } from "./compat.ts";
+import { debug } from "./debug.ts";
+import type { PeerManager } from "./peer-manager.ts";
+import type { KnowledgeSyncFrame } from "./types.ts";
+
+/** Automerge document schema: map of relative paths to file contents. */
+interface KnowledgeDoc {
+  files: Record<string, string>;
+}
+
+export interface KnowledgeSyncOptions {
+  /** Workspace directory to watch and sync. */
+  workspacePath: string;
+  /** Path to persist the Automerge document (binary). */
+  storePath: string;
+  /** Local node ID. */
+  nodeId: string;
+  /** Debounce interval in ms for fs changes. */
+  debounce: number;
+  /** PeerManager for sending frames. */
+  peerManager: PeerManager;
+}
+
+const TAG = "knowledge";
+
+export class KnowledgeSync {
+  private doc: Automerge.Doc<KnowledgeDoc>;
+  private syncStates = new Map<string, Automerge.SyncState>();
+  private watcher: FSWatcher | null = null;
+  private debounceTimer: ReturnType<typeof setTimeout> | null = null;
+  private suppressWatch = false;
+  private opts: KnowledgeSyncOptions;
+  private ig: Ignore = ignore();
+
+  constructor(opts: KnowledgeSyncOptions) {
+    this.opts = opts;
+    this.doc = Automerge.init<KnowledgeDoc>();
+  }
+
+  async start() {
+    debug(TAG, `starting knowledge sync: workspace=${this.opts.workspacePath}`);
+
+    // Ensure workspace directory exists
+    await mkdir(this.opts.workspacePath, { recursive: true });
+
+    // Load .gitignore rules
+    await this.loadGitignore();
+
+    // Load persisted doc or initialize from existing files
+    const loaded = await this.loadDoc();
+    if (loaded) {
+      this.doc = loaded;
+      const fileCount = Object.keys(this.doc.files ?? {}).length;
+      debug(TAG, `loaded persisted doc: ${fileCount} files`);
+      // Export doc state to filesystem (in case doc has newer data)
+      await this.exportToFs();
+    } else {
+      // First run: import existing files into doc
+      debug(TAG, "no persisted doc found, importing from filesystem");
+      await this.importFromFs();
+      const fileCount = Object.keys(this.doc.files ?? {}).length;
+      debug(TAG, `imported ${fileCount} files from workspace`);
+      await this.saveDoc();
+    }
+
+    // Initialize git repo if not already
+    await this.gitInit();
+
+    // Start watching for file changes
+    this.watcher = watch(this.opts.workspacePath, { recursive: true }, (_event, filename) => {
+      if (this.suppressWatch || !filename) return;
+      // Ignore hidden files
+      if (filename.startsWith(".")) return;
+      // Ignore gitignored files
+      if (this.ig.ignores(filename)) return;
+      this.scheduleSync();
+    });
+
+    debug(TAG, "file watcher started");
+  }
+
+  async stop() {
+    debug(TAG, "stopping knowledge sync");
+    if (this.debounceTimer) {
+      clearTimeout(this.debounceTimer);
+      this.debounceTimer = null;
+    }
+    this.watcher?.close();
+    this.watcher = null;
+    await this.saveDoc();
+    debug(TAG, "knowledge sync stopped, doc saved");
+  }
+
+  /** Handle incoming knowledge_sync frame from a peer. */
+  async handleSyncMessage(frame: KnowledgeSyncFrame) {
+    const peerId = frame.from;
+    const msgSize = frame.payload.data.length;
+    debug(TAG, `received sync message from ${peerId} (${msgSize} bytes)`);
+
+    const syncState = this.syncStates.get(peerId) ?? Automerge.initSyncState();
+    const message = new Uint8Array(Buffer.from(frame.payload.data, "base64"));
+
+    const [newDoc, newSyncState, _patch] = Automerge.receiveSyncMessage(
+      this.doc,
+      syncState,
+      message,
+    );
+
+    const oldFileCount = Object.keys(this.doc.files ?? {}).length;
+    const newFileCount = Object.keys(newDoc.files ?? {}).length;
+    this.doc = newDoc;
+    this.syncStates.set(peerId, newSyncState);
+
+    if (oldFileCount !== newFileCount) {
+      debug(TAG, `doc updated: ${oldFileCount} → ${newFileCount} files`);
+    }
+
+    // Export changes to filesystem and git commit
+    await this.exportToFs();
+    await this.saveDoc();
+    await this.gitCommit(`sync from ${peerId}`);
+
+    // Continue sync protocol (may need to send response)
+    this.sendSyncMessage(peerId);
+  }
+
+  /** Called when a new peer connects — initiate sync. */
+  initPeerSync(peerId: string) {
+    debug(TAG, `initiating sync with peer ${peerId}`);
+    this.syncStates.set(peerId, Automerge.initSyncState());
+    this.sendSyncMessage(peerId);
+  }
+
+  /** Called when a peer disconnects — clean up sync state. */
+  removePeerSync(peerId: string) {
+    debug(TAG, `removing sync state for peer ${peerId}`);
+    this.syncStates.delete(peerId);
+  }
+
+  // ── Private methods ─────────────────────────────────────────────
+
+  private async loadGitignore() {
+    this.ig = ignore();
+    // Always ignore .git and automerge store
+    this.ig.add([".git", ".clawmatrix"]);
+    try {
+      const content = await readFile(path.join(this.opts.workspacePath, ".gitignore"), "utf-8");
+      this.ig.add(content);
+      debug(TAG, "loaded .gitignore rules");
+    } catch {
+      // No .gitignore — that's fine
+    }
+  }
+
+  private isIgnored(relPath: string): boolean {
+    return this.ig.ignores(relPath);
+  }
+
+  private scheduleSync() {
+    if (this.debounceTimer) clearTimeout(this.debounceTimer);
+    this.debounceTimer = setTimeout(() => {
+      this.debounceTimer = null;
+      this.handleLocalChanges().catch((err) => {
+        debug(TAG, `local change handling error: ${err}`);
+      });
+    }, this.opts.debounce);
+  }
+
+  private async handleLocalChanges() {
+    // Reload .gitignore in case it changed
+    await this.loadGitignore();
+
+    const currentFiles = await this.readWorkspaceFiles();
+    const docFiles = this.doc.files ?? {};
+
+    // Collect changed files for logging
+    const added: string[] = [];
+    const modified: string[] = [];
+    const deleted: string[] = [];
+
+    for (const [relPath, content] of Object.entries(currentFiles)) {
+      if (!(relPath in docFiles)) {
+        added.push(relPath);
+      } else if (docFiles[relPath] !== content) {
+        modified.push(relPath);
+      }
+    }
+
+    for (const relPath of Object.keys(docFiles)) {
+      if (!(relPath in currentFiles)) {
+        deleted.push(relPath);
+      }
+    }
+
+    if (added.length === 0 && modified.length === 0 && deleted.length === 0) return;
+
+    debug(
+      TAG,
+      `local changes detected: +${added.length} ~${modified.length} -${deleted.length}` +
+        (added.length > 0 ? ` added=[${added.join(",")}]` : "") +
+        (modified.length > 0 ? ` modified=[${modified.join(",")}]` : "") +
+        (deleted.length > 0 ? ` deleted=[${deleted.join(",")}]` : ""),
+    );
+
+    // Update automerge doc
+    this.doc = Automerge.change(this.doc, (d) => {
+      if (!d.files) {
+        (d as KnowledgeDoc).files = {};
+      }
+      for (const [relPath, content] of Object.entries(currentFiles)) {
+        d.files[relPath] = content;
+      }
+      for (const relPath of Object.keys(d.files)) {
+        if (!(relPath in currentFiles)) {
+          delete d.files[relPath];
+        }
+      }
+    });
+
+    await this.saveDoc();
+    await this.gitCommit("local changes");
+
+    // Send sync messages to all peers
+    const peerCount = this.opts.peerManager.router.getAllPeers().length;
+    debug(TAG, `broadcasting changes to ${peerCount} peer(s)`);
+    this.broadcastSync();
+  }
+
+  private broadcastSync() {
+    const peers = this.opts.peerManager.router.getAllPeers();
+    for (const peer of peers) {
+      if (!this.syncStates.has(peer.nodeId)) {
+        this.syncStates.set(peer.nodeId, Automerge.initSyncState());
+      }
+      this.sendSyncMessage(peer.nodeId);
+    }
+  }
+
+  private sendSyncMessage(peerId: string) {
+    const syncState = this.syncStates.get(peerId) ?? Automerge.initSyncState();
+    const [newSyncState, message] = Automerge.generateSyncMessage(this.doc, syncState);
+
+    this.syncStates.set(peerId, newSyncState);
+
+    if (!message) {
+      debug(TAG, `no sync message to send to ${peerId} (already in sync)`);
+      return;
+    }
+
+    debug(TAG, `sending sync message to ${peerId} (${message.byteLength} bytes)`);
+
+    const frame: KnowledgeSyncFrame = {
+      type: "knowledge_sync",
+      from: this.opts.nodeId,
+      to: peerId,
+      timestamp: Date.now(),
+      payload: {
+        data: Buffer.from(message).toString("base64"),
+      },
+    };
+
+    this.opts.peerManager.router.sendTo(peerId, frame);
+  }
+
+  private async readWorkspaceFiles(): Promise<Record<string, string>> {
+    const files: Record<string, string> = {};
+    await this.walkDir(this.opts.workspacePath, "", files);
+    return files;
+  }
+
+  private async walkDir(base: string, rel: string, result: Record<string, string>) {
+    const entries = await readdir(path.join(base, rel), { withFileTypes: true });
+    for (const entry of entries) {
+      const relPath = rel ? `${rel}/${entry.name}` : entry.name;
+      // Skip hidden files/dirs
+      if (entry.name.startsWith(".")) continue;
+      // Skip gitignored paths
+      if (this.isIgnored(relPath)) continue;
+      if (entry.isDirectory()) {
+        // For directories, check with trailing slash
+        if (this.isIgnored(relPath + "/")) continue;
+        await this.walkDir(base, relPath, result);
+      } else if (entry.isFile()) {
+        const content = await readFile(path.join(base, relPath), "utf-8");
+        result[relPath] = content;
+      }
+    }
+  }
+
+  private async exportToFs() {
+    this.suppressWatch = true;
+    try {
+      const docFiles = this.doc.files ?? {};
+      const currentFiles = await this.readWorkspaceFiles();
+
+      let written = 0;
+      let removed = 0;
+
+      for (const [relPath, content] of Object.entries(docFiles)) {
+        // Don't export files that would be gitignored
+        if (this.isIgnored(relPath)) continue;
+        if (currentFiles[relPath] !== content) {
+          const absPath = path.join(this.opts.workspacePath, relPath);
+          await mkdir(path.dirname(absPath), { recursive: true });
+          await writeFile(absPath, content, "utf-8");
+          written++;
+        }
+      }
+
+      for (const relPath of Object.keys(currentFiles)) {
+        if (!(relPath in docFiles)) {
+          const absPath = path.join(this.opts.workspacePath, relPath);
+          await unlink(absPath).catch(() => {});
+          removed++;
+        }
+      }
+
+      if (written > 0 || removed > 0) {
+        debug(TAG, `exported to filesystem: ${written} written, ${removed} removed`);
+      }
+    } finally {
+      setTimeout(() => {
+        this.suppressWatch = false;
+      }, 500);
+    }
+  }
+
+  // ── Git operations ──────────────────────────────────────────────
+
+  private async gitInit() {
+    try {
+      // Check if already a git repo
+      const proc = spawnProcess(["git", "rev-parse", "--git-dir"], {
+        cwd: this.opts.workspacePath,
+        stdout: "pipe",
+        stderr: "pipe",
+      });
+      const code = await proc.exited;
+      if (code !== 0) {
+        // Not a git repo — initialize
+        const init = spawnProcess(["git", "init"], {
+          cwd: this.opts.workspacePath,
+          stdout: "pipe",
+          stderr: "pipe",
+        });
+        await init.exited;
+        debug(TAG, `initialized git repo at ${this.opts.workspacePath}`);
+      } else {
+        debug(TAG, "git repo already initialized");
+      }
+    } catch {
+      debug(TAG, "git not available, skipping git integration");
+    }
+  }
+
+  private async gitCommit(message: string) {
+    try {
+      // git add -A (respects .gitignore)
+      const add = spawnProcess(["git", "add", "-A"], {
+        cwd: this.opts.workspacePath,
+        stdout: "pipe",
+        stderr: "pipe",
+      });
+      await add.exited;
+
+      // Check if there are staged changes
+      const diff = spawnProcess(["git", "diff", "--cached", "--quiet"], {
+        cwd: this.opts.workspacePath,
+        stdout: "pipe",
+        stderr: "pipe",
+      });
+      const diffCode = await diff.exited;
+      if (diffCode === 0) return; // Nothing to commit
+
+      // Commit
+      const commit = spawnProcess(
+        ["git", "commit", "-m", `[clawmatrix] ${message}`],
+        {
+          cwd: this.opts.workspacePath,
+          stdout: "pipe",
+          stderr: "pipe",
+        },
+      );
+      await commit.exited;
+      debug(TAG, `git commit: ${message}`);
+    } catch (err) {
+      debug(TAG, `git commit failed: ${err}`);
+    }
+  }
+
+  // ── Persistence ─────────────────────────────────────────────────
+
+  private async loadDoc(): Promise<Automerge.Doc<KnowledgeDoc> | null> {
+    try {
+      const data = await readFile(this.opts.storePath);
+      debug(TAG, `loaded automerge doc from ${this.opts.storePath} (${data.byteLength} bytes)`);
+      return Automerge.load<KnowledgeDoc>(new Uint8Array(data));
+    } catch {
+      return null;
+    }
+  }
+
+  private async saveDoc() {
+    const data = Automerge.save(this.doc);
+    await mkdir(path.dirname(this.opts.storePath), { recursive: true });
+    await writeFile(this.opts.storePath, Buffer.from(data));
+  }
+
+  private async importFromFs() {
+    const files = await this.readWorkspaceFiles();
+    if (Object.keys(files).length === 0) return;
+
+    this.doc = Automerge.change(this.doc, (d) => {
+      (d as KnowledgeDoc).files = {};
+      for (const [relPath, content] of Object.entries(files)) {
+        d.files[relPath] = content;
+      }
+    });
+  }
+}

package/src/tools/cluster-events.ts

@@ -13,32 +13,32 @@ export function createClusterEventsTool(): AnyAgentTool {
         action: {
           type: "string",
           enum: ["query", "consume"],
-          description: '"query" to list events, "consume" to mark events as processed',
+          description: '"query" to list events (type/source/unconsumed/since/limit filters apply), "consume" to mark as processed (requires ids)',
         },
         type: {
           type: "string",
-          description: 'Filter by event type (e.g. "message_received", "call_missed"). Only for action=query.',
+          description: 'Filter by event type (e.g. "message_received")',
         },
         source: {
           type: "string",
-          description: 'Filter by source (e.g. "shortcuts", "iphone"). Only for action=query.',
+          description: 'Filter by source (e.g. "shortcuts")',
         },
         unconsumed: {
           type: "boolean",
-          description: "Only return unconsumed events. Default true. Only for action=query.",
+          description: "Only unconsumed events (default true)",
         },
         since: {
           type: "number",
-          description: "Only return events after this unix timestamp (ms). Only for action=query.",
+          description: "Events after this unix timestamp (ms)",
         },
         limit: {
           type: "number",
-          description: "Max events to return. Default 20. Only for action=query.",
+          description: "Max events to return (default 20)",
         },
         ids: {
           type: "array",
           items: { type: "string" },
-          description: "Event IDs to mark as consumed. Required for action=consume.",
+          description: "Event IDs to consume",
         },
       },
       required: ["action"],
@@ -105,7 +105,7 @@ export function createClusterEventsTool(): AnyAgentTool {
         }));
 
         return {
-          content: [{ type: "text" as const, text: JSON.stringify(summary, null, 2) }],
+          content: [{ type: "text" as const, text: JSON.stringify(summary) }],
           details: { events: summary },
         };
       } catch (err) {

package/src/tools/cluster-exec.ts

@@ -53,7 +53,7 @@ export function createClusterExecTool(): AnyAgentTool {
           content: [
             {
               type: "text" as const,
-              text: JSON.stringify(result, null, 2),
+              text: JSON.stringify(result),
             },
           ],
           details: result,

package/src/tools/cluster-handoff-reply.ts

@@ -34,7 +34,7 @@ export function createClusterHandoffReplyTool(): AnyAgentTool {
             content: [
               {
                 type: "text" as const,
-                text: `Remote agent needs more information.\n\nHandoff ID: ${result.handoffId}\nQuestion: ${result.result}\n\nUse cluster_handoff_reply again to respond.`,
+                text: `Input required (handoff_id:${result.handoffId}): ${result.result}`,
               },
             ],
             details: result,
@@ -52,11 +52,7 @@ export function createClusterHandoffReplyTool(): AnyAgentTool {
           content: [
             {
               type: "text" as const,
-              text: JSON.stringify(
-                { nodeId: result.nodeId, agent: result.agent, result: result.result },
-                null,
-                2,
-              ),
+              text: JSON.stringify({ nodeId: result.nodeId, agent: result.agent, result: result.result }),
             },
           ],
           details: result,

package/src/tools/cluster-handoff.ts

@@ -42,7 +42,7 @@ export function createClusterHandoffTool(): AnyAgentTool {
             content: [
               {
                 type: "text" as const,
-                text: `Remote agent needs more information before continuing.\n\nHandoff ID: ${result.handoffId}\nQuestion: ${result.result}\n\nUse cluster_handoff_reply tool to respond.`,
+                text: `Input required (handoff_id:${result.handoffId}): ${result.result}`,
               },
             ],
             details: result,
@@ -65,15 +65,7 @@ export function createClusterHandoffTool(): AnyAgentTool {
           content: [
             {
               type: "text" as const,
-              text: JSON.stringify(
-                {
-                  nodeId: result.nodeId,
-                  agent: result.agent,
-                  result: result.result,
-                },
-                null,
-                2,
-              ),
+              text: JSON.stringify({ nodeId: result.nodeId, agent: result.agent, result: result.result }),
             },
           ],
           details: result,

package/src/tools/cluster-peers.ts

@@ -21,36 +21,29 @@ export function createClusterPeersTool(): AnyAgentTool {
             description: a.description,
             tags: a.tags,
           })),
-          models: entry.models.map((m) => ({
-            id: m.id,
-            provider: m.provider,
-          })),
+          models: entry.models.map((m) => m.id),
           tags: entry.tags,
           tools: entry.toolProxy?.enabled ? (entry.toolProxy.allow ?? []) : [],
           status: entry.connection?.isOpen ? "connected" : "unreachable",
           latencyMs: entry.latencyMs,
         }));
 
-        // Include satellite nodes
+        // Include satellite nodes (minimal fields — no agents/models)
         const satellites = runtime.webHandler?.getSatelliteContexts() ?? runtime.peerManager.satelliteContexts;
         for (const sat of satellites) {
           if (Date.now() - sat.ts >= 600_000) continue;
           peers.push({
             nodeId: sat.nodeId,
-            agents: [],
-            models: [],
-            tags: [],
             tools: sat.tools ?? [],
             status: "satellite",
-            latencyMs: undefined,
-          });
+          } as (typeof peers)[number]);
         }
 
         return {
           content: [
             {
               type: "text" as const,
-              text: JSON.stringify(peers, null, 2),
+              text: JSON.stringify(peers),
             },
           ],
           details: peers,

package/src/tools/cluster-read.ts

@@ -30,7 +30,7 @@ export function createClusterReadTool(): AnyAgentTool {
           content: [
             {
               type: "text" as const,
-              text: JSON.stringify(result, null, 2),
+              text: JSON.stringify(result),
             },
           ],
           details: result,

package/src/tools/cluster-tool.ts

@@ -39,7 +39,7 @@ export function createClusterToolTool(): AnyAgentTool {
           content: [
             {
               type: "text" as const,
-              text: JSON.stringify(result, null, 2),
+              text: JSON.stringify(result),
             },
           ],
           details: result,

package/src/tools/cluster-write.ts

@@ -42,7 +42,7 @@ export function createClusterWriteTool(): AnyAgentTool {
           content: [
             {
               type: "text" as const,
-              text: JSON.stringify(result, null, 2),
+              text: JSON.stringify(result),
             },
           ],
           details: result,

package/src/types.ts

@@ -319,6 +319,14 @@ export interface IngestedEvent {
   consumed: boolean;   // whether an agent has consumed this event
 }
 
+// ── Knowledge sync ────────────────────────────────────────────────
+export interface KnowledgeSyncFrame extends ClusterFrame {
+  type: "knowledge_sync";
+  payload: {
+    data: string; // base64-encoded Automerge sync message
+  };
+}
+
 // ── Union of all frame types ───────────────────────────────────────
 export type AnyClusterFrame =
   | AuthChallenge
@@ -343,4 +351,5 @@ export type AnyClusterFrame =
   | HandoffInputRequired
   | HandoffInput
   | ToolProxyRequest
-  | ToolProxyResponse;
+  | ToolProxyResponse
+  | KnowledgeSyncFrame;