clawmatrix 0.2.11 → 0.4.0

package/src/knowledge-sync.ts

@@ -1,6 +1,6 @@
 import * as Automerge from "@automerge/automerge";
 import { watch, type FSWatcher } from "node:fs";
-import { readdir, readFile, stat as fsStat, writeFile, mkdir, rename } from "node:fs/promises";
+import { readdir, readFile, stat as fsStat, writeFile, mkdir, rename, unlink } from "node:fs/promises";
 import path from "node:path";
 import ignore, { type Ignore } from "ignore";
 import picomatch from "picomatch";
@@ -87,6 +87,20 @@ async function pMap<T, R>(items: T[], fn: (item: T) => Promise<R>, concurrency:
   return results;
 }
 
+/** Race a promise against a timeout; calls onTimeout before rejecting. */
+function withTimeout<T>(promise: Promise<T>, ms: number, onTimeout?: () => void): Promise<T> {
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      onTimeout?.();
+      reject(new Error(`Timed out after ${ms}ms`));
+    }, ms);
+    promise.then(
+      (v) => { clearTimeout(timer); resolve(v); },
+      (e) => { clearTimeout(timer); reject(e); },
+    );
+  });
+}
+
 async function streamToString(stream: ReadableStream | null): Promise<string> {
   if (!stream) return "";
   const reader = stream.getReader();
@@ -131,7 +145,7 @@ export class KnowledgeSync {
   private localChangesRunning = false;
   private localChangesQueued = false;
   /** Paths recently written by exportFileToFs — suppress watcher re-trigger. Stores {content, timestamp}. */
-  private writtenByExport = new Map<string, { content: string; ts: number }>();
+  private writtenByExport = new Map<string, { content: string | null; ts: number }>();
   /** Deferred git commit timer — batches multiple remote syncs into one commit. */
   private gitCommitTimer: ReturnType<typeof setTimeout> | null = null;
   private pendingGitSources = new Set<string>();
@@ -326,10 +340,20 @@ export class KnowledgeSync {
     await this.saveAutomergeDoc(this.registryPath, this.registry);
 
     // Discover new files from registry and initiate their sync
+    const deletedPaths: string[] = [];
     for (const [relPath, meta] of Object.entries(newDoc.files ?? {})) {
       if (meta.deleted) {
-        // Clean up sync states for deleted files
+        // Clean up sync states, in-memory doc, persisted doc, and local file
         this.cleanupDeletedFileSyncStates(relPath);
+        if (this.fileDocs.has(relPath)) {
+          this.fileDocs.delete(relPath);
+          deletedPaths.push(relPath);
+          // Remove persisted automerge doc
+          const docPath = path.join(this.docsDir, docFileName(relPath));
+          await rename(docPath, docPath + ".deleted").catch(() => {});
+          // Delete local file from workspace
+          await this.deleteLocalFile(relPath);
+        }
         continue;
       }
       if (!this.fileDocs.has(relPath)) {
@@ -339,6 +363,12 @@ export class KnowledgeSync {
       this.syncDocWithPeer(peerId, relPath);
     }
 
+    // Commit remote deletions to git
+    if (deletedPaths.length > 0) {
+      debug(TAG, `remote deletion from ${peerId}: ${deletedPaths.join(", ")}`);
+      this.schedulePendingGitCommit(peerId);
+    }
+
     this.sendSyncMessage(peerId, REGISTRY_DOC_ID);
   }
 
@@ -846,6 +876,30 @@ export class KnowledgeSync {
     }
   }
 
+  /** Delete a local file from workspace (triggered by remote deletion). */
+  private async deleteLocalFile(relPath: string) {
+    const absPath = path.resolve(this.opts.workspacePath, relPath);
+
+    // Prevent path traversal
+    if (!absPath.startsWith(this.opts.workspacePath + path.sep) && absPath !== this.opts.workspacePath) {
+      debug(TAG, `blocked path traversal on delete: ${relPath}`);
+      return;
+    }
+
+    // Mark as our own deletion so the watcher doesn't re-process it.
+    // handleLocalChangesInner sees currentContent === null === marker.content and skips.
+    this.writtenByExport.set(relPath, { content: null, ts: Date.now() });
+    try {
+      await unlink(absPath);
+      debug(TAG, `deleted local file: ${relPath}`);
+    } catch (err) {
+      // File may not exist locally — that's fine
+      if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
+        debug(TAG, `failed to delete local file ${relPath}: ${err}`);
+      }
+    }
+  }
+
   /** Read all workspace files matching whitelist. */
   private async readWhitelistedFiles(): Promise<Record<string, string>> {
     const files: Record<string, string> = {};
@@ -954,9 +1008,7 @@ export class KnowledgeSync {
       }
 
       let doc = Automerge.init<FileDoc>();
-      doc = Automerge.change(doc, (d) => {
-        (d as FileDoc).content = content;
-      });
+      doc = changeFileContent(doc, content);
       this.fileDocs.set(relPath, doc);
 
       this.registry = Automerge.change(this.registry, (d) => {
@@ -1036,20 +1088,20 @@ export class KnowledgeSync {
   }
 
   private async gitCommit(message: string) {
+    const GIT_TIMEOUT_MS = 3000;
     try {
       const add = spawnProcess(["git", "add", "-A"], {
         cwd: this.opts.workspacePath,
         stdout: "pipe",
         stderr: "pipe",
       });
-      await add.exited;
-
+      await withTimeout(add.exited, GIT_TIMEOUT_MS, () => add.kill());
       const diff = spawnProcess(["git", "diff", "--cached", "--quiet"], {
         cwd: this.opts.workspacePath,
         stdout: "pipe",
         stderr: "pipe",
       });
-      const diffCode = await diff.exited;
+      const diffCode = await withTimeout(diff.exited, GIT_TIMEOUT_MS, () => diff.kill());
       if (diffCode === 0) return;
 
       const commit = spawnProcess(
@@ -1060,7 +1112,7 @@ export class KnowledgeSync {
           stderr: "pipe",
         },
       );
-      const commitCode = await commit.exited;
+      const commitCode = await withTimeout(commit.exited, GIT_TIMEOUT_MS, () => commit.kill());
       if (commitCode !== 0) {
         const stderr = await streamToString(commit.stderr);
         debug(TAG, `git commit failed (exit ${commitCode}): ${stderr}`);

package/src/model-proxy.ts

@@ -362,6 +362,11 @@ export class ModelProxy {
       this.cacheCleanupTimer = null;
     }
     if (this.httpServer) {
+      // Force-close all keep-alive connections so the port is released immediately
+      const server = this.httpServer as typeof this.httpServer & { closeAllConnections?: () => void };
+      if (typeof server.closeAllConnections === "function") {
+        server.closeAllConnections();
+      }
       this.httpServer.close();
       this.httpServer = null;
     }
@@ -754,7 +759,7 @@ export class ModelProxy {
     let currentId = requestId;
     let currentTarget = targetNodeId;
     let currentFrame = frame;
-    let remaining = failoverCandidates;
+    let failoverIdx = 0; // index into failoverCandidates (avoids slice allocations)
     const maxAttempts = failoverCandidates.length + 1;
 
     for (let attempt = 0; attempt < maxAttempts; attempt++) {
@@ -763,13 +768,13 @@ export class ModelProxy {
 
         if (!result.success) {
           // Upstream error — try failover if available
-          if (remaining.length > 0 && buildFrame) {
-            const next = remaining[0]!;
-            debug("proxy", `failover: remote error "${result.error}" → trying ${next.routeNodeId} (${remaining.length - 1} left)`);
+          if (failoverIdx < failoverCandidates.length && buildFrame) {
+            const next = failoverCandidates[failoverIdx]!;
+            debug("proxy", `failover: remote error "${result.error}" → trying ${next.routeNodeId} (${failoverCandidates.length - failoverIdx - 1} left)`);
+            failoverIdx++;
             currentId = crypto.randomUUID();
             currentFrame = buildFrame(next, currentId);
             currentTarget = next.routeNodeId;
-            remaining = remaining.slice(1);
             continue;
           }
           return {
@@ -782,13 +787,13 @@ export class ModelProxy {
         return this.formatNonStreamResult(result, currentId, currentFrame, responseFormat);
       } catch (err) {
         // Timeout or send failure — try failover
-        if (remaining.length > 0 && buildFrame) {
-          const next = remaining[0]!;
-          debug("proxy", `failover: ${err instanceof Error ? err.message : String(err)} → trying ${next.routeNodeId} (${remaining.length - 1} left)`);
+        if (failoverIdx < failoverCandidates.length && buildFrame) {
+          const next = failoverCandidates[failoverIdx]!;
+          debug("proxy", `failover: ${err instanceof Error ? err.message : String(err)} → trying ${next.routeNodeId} (${failoverCandidates.length - failoverIdx - 1} left)`);
+          failoverIdx++;
           currentId = crypto.randomUUID();
           currentFrame = buildFrame(next, currentId);
           currentTarget = next.routeNodeId;
-          remaining = remaining.slice(1);
           continue;
         }
         return {
@@ -980,6 +985,24 @@ export class ModelProxy {
     const pending = this.pending.get(frame.id);
     if (!pending?.stream || !pending.controller || !pending.encoder) return;
 
+    // Reset activity timer — keeps long-running streams alive and detects
+    // stalled connections within modelTimeout of the last received chunk.
+    clearTimeout(pending.timer);
+    if (!frame.payload.done) {
+      pending.timer = setTimeout(() => {
+        // Capture references before cleanup removes pending from the map
+        const { stableStreamId, responseFormat, controller, encoder, model, failoverCandidates, buildFrame } = pending;
+        this.cleanupRequest(frame.id);
+        this.peerManager.router.markFailed(frame.id);
+        this.tryStreamFailover(
+          stableStreamId ?? frame.id, responseFormat,
+          controller!, encoder!, model ?? "",
+          failoverCandidates ?? [], buildFrame,
+          `stream stalled (no data for ${this.modelTimeout / 1000}s)`,
+        );
+      }, this.modelTimeout);
+    }
+
     try {
       if (pending.responseFormat === "responses") {
         this.handleModelStreamResponses(frame, pending);
@@ -1305,7 +1328,14 @@ export class ModelProxy {
         let chatFallbackResult: Awaited<ReturnType<ModelProxy["retryWithChatCompletions"]>> = null;
         try {
           result = JSON.parse(responseText);
-        } catch {
+          // Detect error objects in 200 OK responses (some APIs return HTTP 200 with error body)
+          if (result.error && typeof result.error === "object" && !result.choices && !result.output) {
+            const errMsg = (result.error as { message?: string }).message ?? JSON.stringify(result.error);
+            throw new Error(`Upstream error (200 OK): ${String(errMsg).slice(0, 200)}`);
+          }
+        } catch (parseErr) {
+          // Re-throw non-parse errors (e.g. upstream error detection above)
+          if (!(parseErr instanceof SyntaxError)) throw parseErr;
           // Upstream returned non-JSON (e.g. SSE in non-stream mode) — try chat completions fallback
           if (!cachedApi && isResponsesApi) {
             debug("model_req", `responses API returned non-JSON for "${model.id}", retrying with chat completions`);

package/src/peer-manager.ts

@@ -93,6 +93,8 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
   private wss: WebSocketServer | null = null;
   private reconnectTimers = new Map<string, ReturnType<typeof setTimeout>>();
   private reconnectAttempts = new Map<string, number>();
+  /** Deferred disconnect timers — grace period before broadcasting peer_leave. */
+  private disconnectGraceTimers = new Map<string, ReturnType<typeof setTimeout>>();
   private stopped = false;
   /** Map from ws WebSocket to Connection for inbound connections. */
   private inboundConnections = new Map<WsWebSocket, Connection>();
@@ -165,6 +167,17 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
     }
   }
 
+  /** Update the local tool proxy catalog and re-broadcast to all peers. */
+  updateToolCatalog(catalog: import("./types.ts").ToolCatalogEntry[]) {
+    if (this.localCapabilities.toolProxy) {
+      this.localCapabilities.toolProxy = { ...this.localCapabilities.toolProxy, catalog };
+    }
+    this.router.updateLocalToolCatalog(catalog);
+    for (const conn of this.router.getDirectConnections()) {
+      this.sendPeerSync(conn);
+    }
+  }
+
   // ── Lifecycle ──────────────────────────────────────────────────
   async start() {
     await this.approvalManager.load();
@@ -190,6 +203,12 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
       clearTimeout(timer);
     }
     this.reconnectTimers.clear();
+    // Flush all disconnect grace timers (execute leave immediately on shutdown)
+    for (const [nodeId, timer] of this.disconnectGraceTimers) {
+      clearTimeout(timer);
+      this.executePeerLeave(nodeId);
+    }
+    this.disconnectGraceTimers.clear();
 
     this.router.broadcast({
       type: "peer_leave",
@@ -202,18 +221,47 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
       conn.close(1000, "shutdown");
     }
 
+    this.closeServers();
+
+    this.rateLimiter.destroy();
+    this.approvalManager.destroy();
+    this.router.destroy();
+  }
+
+  /** Force-stop without broadcasting or waiting — used when graceful stop times out. */
+  forceStop() {
+    this.stopped = true;
+    for (const timer of this.reconnectTimers.values()) clearTimeout(timer);
+    this.reconnectTimers.clear();
+    for (const [, timer] of this.disconnectGraceTimers) clearTimeout(timer);
+    this.disconnectGraceTimers.clear();
+    if (this.gossipDebounceTimer) {
+      clearTimeout(this.gossipDebounceTimer);
+      this.gossipDebounceTimer = null;
+    }
+    for (const conn of this.router.getDirectConnections()) {
+      try { conn.close(1000, "shutdown"); } catch { /* best effort */ }
+    }
+    this.closeServers();
+    this.rateLimiter.destroy();
+    this.approvalManager.destroy();
+    this.router.destroy();
+  }
+
+  private closeServers() {
     if (this.wss) {
       this.wss.close();
       this.wss = null;
     }
     if (this.httpServer) {
+      // Force-close all keep-alive connections so the port is released immediately
+      const server = this.httpServer as typeof this.httpServer & { closeAllConnections?: () => void };
+      if (typeof server.closeAllConnections === "function") {
+        server.closeAllConnections();
+      }
       this.httpServer.close();
       this.httpServer = null;
     }
-
-    this.rateLimiter.destroy();
-    this.approvalManager.destroy();
-    this.router.destroy();
   }
 
   /** Set an HTTP request handler for non-WebSocket requests (e.g. web dashboard). */
@@ -461,9 +509,6 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
         if (this.pendingApprovalConns.has(nodeId)) {
           debug("approval", `reusing pending approval for ${nodeId}, updating conn ref`);
           this.pendingApprovalConns.set(nodeId, { conn, caps });
-          if (this.config.peerApproval?.mode === "required") {
-            conn.on("close", () => this.onPeerDisconnected(conn));
-          }
           return;
         }
 
@@ -495,10 +540,12 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
             );
           }
         });
-        // In required mode, don't complete the join yet
+        // In required mode, don't complete the join yet.
+        // No close handler needed here: the peer was never added to the router,
+        // so onPeerDisconnected would broadcast a spurious peer_leave.
+        // If the conn drops before approval resolves, the .then() handler sees
+        // activeConn.isOpen === false and skips all actions.
         if (this.config.peerApproval?.mode === "required") {
-          // Wire up close handler to clean up if connection drops while pending
-          conn.on("close", () => this.onPeerDisconnected(conn));
           return;
         }
         // In notify mode, requestApproval resolves immediately, but
@@ -515,6 +562,9 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
   private completePeerJoin(conn: Connection, caps: NodeCapabilities) {
     const nodeId = conn.remoteNodeId!;
 
+    // Cancel disconnect grace timer if the peer is reconnecting
+    const wasInGrace = this.cancelDisconnectGrace(nodeId);
+
     // If there's an existing connection for this nodeId (e.g. peer reconnected
     // while old TCP hadn't closed yet), close it AFTER overwriting the route so
     // the stale-close guard in onPeerDisconnected correctly skips cleanup.
@@ -585,15 +635,58 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
       return;
     }
 
+    // Grace period: defer peer_leave broadcast to allow quick reconnection
+    // (e.g. iOS WiFi ↔ cellular handoff, brief audio interruption).
+    // If the peer reconnects within the grace window, completePeerJoin
+    // will cancel this timer via cancelDisconnectGrace.
+    const graceMs = this.config.disconnectGrace ?? 30_000;
+    if (graceMs <= 0) {
+      this.executePeerLeave(nodeId, conn);
+      return;
+    }
+    debug("peer", `onPeerDisconnected(${nodeId}): starting ${graceMs / 1000}s grace period`);
+
+    // Clear any existing grace timer for this node (shouldn't happen, but be safe)
+    this.cancelDisconnectGrace(nodeId);
+
+    this.disconnectGraceTimers.set(nodeId, setTimeout(() => {
+      this.disconnectGraceTimers.delete(nodeId);
+      this.executePeerLeave(nodeId, conn);
+    }, graceMs));
+  }
+
+  /** Cancel a pending disconnect grace timer (called when peer reconnects quickly). */
+  private cancelDisconnectGrace(nodeId: string): boolean {
+    const timer = this.disconnectGraceTimers.get(nodeId);
+    if (timer) {
+      clearTimeout(timer);
+      this.disconnectGraceTimers.delete(nodeId);
+      debug("peer", `cancelDisconnectGrace(${nodeId}): peer reconnected within grace period`);
+      return true;
+    }
+    return false;
+  }
+
+  /** Execute the actual peer leave (after grace period expires or immediate for shutdown). */
+  private executePeerLeave(nodeId: string, conn?: Connection) {
+    // Double-check the route hasn't been replaced by a new connection during grace
+    if (conn) {
+      const currentRoute = this.router.getRoute(nodeId);
+      if (currentRoute?.connection && currentRoute.connection !== conn) {
+        debug("peer", `executePeerLeave(${nodeId}): route replaced during grace — skipping`);
+        return;
+      }
+    }
+
     audit("peer_leave", { nodeId });
     this.router.removePeer(nodeId);
 
     // Remove satellite contexts that were only reachable via this peer
-    this.satelliteContexts = this.satelliteContexts.filter(s => {
-      // Keep satellites that are not associated with the disconnected peer
-      // (satellite nodeIds typically differ from mesh peer nodeIds)
-      return s.nodeId !== nodeId;
-    });
+    for (let i = this.satelliteContexts.length - 1; i >= 0; i--) {
+      if (this.satelliteContexts[i].nodeId === nodeId) {
+        this.satelliteContexts.splice(i, 1);
+      }
+    }
 
     this.router.broadcast({
       type: "peer_leave",
@@ -748,13 +841,17 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
         const prev = this.router.getRoute(peer.nodeId);
         const hadAgents = prev?.agents.length ?? 0;
         const hadDirectPeers = prev?.directPeers.length ?? 0;
-        const hadToolProxy = JSON.stringify(prev?.toolProxy);
         const hadDeviceInfo = prev?.deviceInfo?.hostname;
         const hadAcpAgents = prev?.acpAgents?.length ?? 0;
+        const hadToolProxyEnabled = prev?.toolProxy?.enabled;
+        const hadToolProxyCatalogLen = prev?.toolProxy?.catalog?.length ?? 0;
+        const hadToolProxyAllowLen = prev?.toolProxy?.allow?.length ?? 0;
         this.router.updatePeerCapabilities(peer.nodeId, peer);
         if (peer.agents.length !== hadAgents || peer.models.length !== (prev?.models.length ?? 0)
             || (peer.directPeers?.length ?? 0) !== hadDirectPeers
-            || JSON.stringify(peer.toolProxy) !== hadToolProxy
+            || peer.toolProxy?.enabled !== hadToolProxyEnabled
+            || (peer.toolProxy?.catalog?.length ?? 0) !== hadToolProxyCatalogLen
+            || (peer.toolProxy?.allow?.length ?? 0) !== hadToolProxyAllowLen
             || peer.deviceInfo?.hostname !== hadDeviceInfo
             || (peer.acpAgents?.length ?? 0) !== hadAcpAgents) {
           changed = true;

package/src/rate-limiter.ts

@@ -33,19 +33,20 @@ export class RateLimiter {
 
     let timestamps = this.attempts.get(ip);
     if (timestamps) {
-      // Remove expired entries
-      timestamps = timestamps.filter((t) => t > cutoff);
+      // In-place pruning: find first non-expired index and splice
+      let firstValid = 0;
+      while (firstValid < timestamps.length && timestamps[firstValid] <= cutoff) firstValid++;
+      if (firstValid > 0) timestamps.splice(0, firstValid);
     } else {
       timestamps = [];
+      this.attempts.set(ip, timestamps);
     }
 
     if (timestamps.length >= this.config.maxAttempts) {
-      this.attempts.set(ip, timestamps);
       return false;
     }
 
     timestamps.push(now);
-    this.attempts.set(ip, timestamps);
     return true;
   }
 
@@ -61,19 +62,24 @@ export class RateLimiter {
   /** Get remaining attempts for an IP. */
   remaining(ip: string): number {
     const cutoff = Date.now() - this.config.windowMs;
-    const timestamps = this.attempts.get(ip) ?? [];
-    const active = timestamps.filter((t) => t > cutoff).length;
+    const timestamps = this.attempts.get(ip);
+    if (!timestamps) return this.config.maxAttempts;
+    let active = 0;
+    for (let i = timestamps.length - 1; i >= 0; i--) {
+      if (timestamps[i] > cutoff) active++; else break;
+    }
     return Math.max(0, this.config.maxAttempts - active);
   }
 
   private gc() {
     const cutoff = Date.now() - this.config.windowMs;
     for (const [ip, timestamps] of this.attempts) {
-      const active = timestamps.filter((t) => t > cutoff);
-      if (active.length === 0) {
+      let firstValid = 0;
+      while (firstValid < timestamps.length && timestamps[firstValid] <= cutoff) firstValid++;
+      if (firstValid === timestamps.length) {
         this.attempts.delete(ip);
-      } else {
-        this.attempts.set(ip, active);
+      } else if (firstValid > 0) {
+        timestamps.splice(0, firstValid);
       }
     }
   }