clawmatrix 0.4.1 → 0.5.0

package/src/connection.ts

@@ -1,4 +1,4 @@
-import { EventEmitter } from "node:events";
+import { EventEmitter } from "eventemitter3";
 import type {
   AgentInfo,
   AnyClusterFrame,
@@ -18,6 +18,8 @@ import {
   derivePerPeerSecret,
   encryptBinary,
   decryptBinary,
+  encryptBinaryRaw,
+  decryptBinaryRaw,
   isBinaryEncrypted,
   publicKeyToBase64,
   base64ToPublicKey,
@@ -36,7 +38,7 @@ export type ConnectionRole = "inbound" | "outbound";
 
 /** Minimal transport interface that works with both standard WebSocket and Bun's ServerWebSocket. */
 export interface WsTransport {
-  send(data: string): void;
+  send(data: string | Buffer): void;
   close(code?: number, reason?: string): void;
   readonly readyState: number;
 }
@@ -99,6 +101,14 @@ export class Connection extends EventEmitter<ConnectionEvents> {
   private _remotePublicKey: Buffer | null = null;
   private sessionKey: Buffer | null = null;
 
+  // ── Frame batching (coalesce small frames within a window) ─────
+  private pendingBatch: Buffer[] = [];
+  private batchTimer: ReturnType<typeof setTimeout> | null = null;
+  private batchBytes = 0;
+  static readonly BATCH_WINDOW = 5;        // ms
+  static readonly BATCH_MAX_FRAMES = 10;
+  static readonly BATCH_MAX_BYTES = 65536;  // 64KB
+
   constructor(
     transport: WsTransport,
     role: ConnectionRole,
@@ -174,69 +184,210 @@ export class Connection extends EventEmitter<ConnectionEvents> {
   send(frame: ClusterFrame | AnyClusterFrame) {
     if (this.closed) return;
     if (this.sessionKey) {
-      // Full frame encryption → binary base64 envelope (no JSON structure on wire)
-      this.sendRaw(encryptBinary(this.sessionKey, JSON.stringify(frame), this.compressionEnabled));
+      this.enqueueBatch(encryptBinaryRaw(this.sessionKey, JSON.stringify(frame), this.compressionEnabled));
     } else {
       this.sendRaw(frame);
     }
   }
 
-  /** Send raw data. Strings sent as-is (for binary envelopes); objects JSON-encoded. */
+  /** Send a frame immediately, bypassing the batch queue. */
+  sendDirect(frame: ClusterFrame | AnyClusterFrame) {
+    if (this.closed) return;
+    if (this.sessionKey) {
+      this.sendRaw(encryptBinaryRaw(this.sessionKey, JSON.stringify(frame), false));
+    } else {
+      this.sendRaw(frame);
+    }
+  }
+
+  /** Send dummy traffic directly (bypasses batch queue). */
+  private sendDummy() {
+    if (this.closed || !this.sessionKey) return;
+    this.sendRaw(encryptBinaryRaw(this.sessionKey, JSON.stringify({ type: "_d", from: "", timestamp: 0 }), false));
+  }
+
+  /** Send raw data. Buffers sent as binary frames; strings as-is; objects JSON-encoded. */
   private sendRaw(data: unknown) {
     if (this.transport.readyState === WebSocket.OPEN) {
-      this.transport.send(typeof data === "string" ? data : JSON.stringify(data));
+      if (Buffer.isBuffer(data)) {
+        this.transport.send(data);
+      } else {
+        this.transport.send(typeof data === "string" ? data : JSON.stringify(data));
+      }
+    }
+  }
+
+  // ── Batch coalescing ──────────────────────────────────────────
+  private enqueueBatch(buf: Buffer) {
+    this.pendingBatch.push(buf);
+    this.batchBytes += buf.length;
+
+    if (this.pendingBatch.length >= Connection.BATCH_MAX_FRAMES ||
+        this.batchBytes >= Connection.BATCH_MAX_BYTES) {
+      this.flushBatch();
+      return;
     }
+
+    if (!this.batchTimer) {
+      this.batchTimer = setTimeout(() => this.flushBatch(), Connection.BATCH_WINDOW);
+    }
+  }
+
+  private flushBatch() {
+    if (this.batchTimer) {
+      clearTimeout(this.batchTimer);
+      this.batchTimer = null;
+    }
+
+    const frames = this.pendingBatch;
+    this.pendingBatch = [];
+    this.batchBytes = 0;
+
+    if (frames.length === 0) return;
+
+    if (frames.length === 1) {
+      this.sendRaw(frames[0]);
+      return;
+    }
+
+    // Multi-frame batch: [uint16 count][uint32 len1][frame1][uint32 len2][frame2]...
+    let totalSize = 2; // count header
+    for (const f of frames) totalSize += 4 + f.length;
+    const batch = Buffer.alloc(totalSize);
+    batch.writeUInt16BE(frames.length, 0);
+    let offset = 2;
+    for (const f of frames) {
+      batch.writeUInt32BE(f.length, offset);
+      offset += 4;
+      f.copy(batch, offset);
+      offset += f.length;
+    }
+    this.sendRaw(batch);
   }
 
   // ── Message dispatch ───────────────────────────────────────────
   private async onRawMessage(data: unknown) {
-    const str = typeof data === "string" ? data : String(data);
-    if (!str.length) return;
     this.lastReceivedAt = Date.now();
 
     let frame: AnyClusterFrame | undefined;
 
-    // Binary encrypted envelope (base64, not JSON)
-    if (this.sessionKey && isBinaryEncrypted(str)) {
-      try {
-        frame = JSON.parse(decryptBinary(this.sessionKey, str));
-      } catch (err) {
-        debug("e2ee", `Frame decryption failed: ${err}`);
+    // Binary frame (Buffer) — decrypt directly without base64
+    if (Buffer.isBuffer(data)) {
+      if (data.length === 0) return;
+
+      // Check for batch format: first 2 bytes = frame count (2..BATCH_MAX_FRAMES).
+      // Validate by pre-scanning all length headers to confirm total matches data.length.
+      // This prevents false positives when a single encrypted frame's flags+IV
+      // happen to look like a valid batch header (e.g. flags=0x00, IV[0]=0x03).
+      if (data.length >= 6 && this.sessionKey) {
+        const count = data.readUInt16BE(0);
+        if (count >= 2 && count <= Connection.BATCH_MAX_FRAMES) {
+          // Pre-scan: verify all frame lengths add up exactly
+          let scanOffset = 2;
+          let valid = true;
+          for (let i = 0; i < count; i++) {
+            if (scanOffset + 4 > data.length) { valid = false; break; }
+            const len = data.readUInt32BE(scanOffset);
+            scanOffset += 4 + len;
+            if (scanOffset > data.length) { valid = false; break; }
+          }
+          if (valid && scanOffset === data.length) {
+            // Confirmed batch — unpack frames
+            let offset = 2;
+            for (let i = 0; i < count; i++) {
+              const len = data.readUInt32BE(offset);
+              offset += 4;
+              const frameBuf = data.subarray(offset, offset + len);
+              offset += len;
+              this.processSingleBinaryFrame(frameBuf);
+            }
+            return;
+          }
+        }
+      }
+
+      // Single binary frame
+      if (this.sessionKey) {
+        this.processSingleBinaryFrame(data);
         return;
+      } else {
+        // Binary frame without session key — try as UTF-8 JSON
+        try {
+          frame = JSON.parse(data.toString());
+        } catch {
+          return;
+        }
       }
-      // Discard dummy traffic
-      if ((frame as any).type === "_d") return;
     }
 
-    // JSON: handshake messages or plaintext frames
     if (!frame) {
-      try {
-        frame = JSON.parse(str);
-      } catch {
-        return;
+      const str = typeof data === "string" ? data : String(data);
+      if (!str.length) return;
+
+      // Binary encrypted envelope (base64 text, for backward compat with old nodes)
+      if (this.sessionKey && isBinaryEncrypted(str)) {
+        try {
+          // Legacy base64 path always used JSON strings
+          frame = JSON.parse(decryptBinary(this.sessionKey, str));
+        } catch (err) {
+          debug("e2ee", `Frame decryption failed: ${err}`);
+          return;
+        }
+        // Discard dummy traffic
+        if ((frame as any).type === "_d") return;
+      }
+
+      // JSON: handshake messages or plaintext frames
+      if (!frame) {
+        try {
+          frame = JSON.parse(str);
+        } catch {
+          return;
+        }
       }
     }
 
+    this.dispatchFrame(frame!);
+  }
+
+  /** Decrypt and dispatch a single binary encrypted frame. */
+  private processSingleBinaryFrame(buf: Buffer) {
+    try {
+      const frame = JSON.parse(decryptBinaryRaw(this.sessionKey!, buf));
+      // Discard dummy traffic
+      if ((frame as any).type === "_d") return;
+      this.dispatchFrame(frame);
+    } catch {
+      // Fallback: Buffer may contain base64 text (e.g. ws package delivered a text frame as Buffer)
+      try {
+        const str = buf.toString();
+        if (isBinaryEncrypted(str)) {
+          const frame = JSON.parse(decryptBinary(this.sessionKey!, str));
+          if ((frame as any).type === "_d") return;
+          this.dispatchFrame(frame);
+          return;
+        }
+      } catch { /* ignore */ }
+      debug("e2ee", `Frame decryption failed (tried both binary and base64)`);
+    }
+  }
+
+  /** Dispatch a fully parsed frame (post-decryption). */
+  private dispatchFrame(frame: AnyClusterFrame) {
     if (!this.authenticated) {
-      await this.handleAuthMessage(frame!);
+      this.handleAuthMessage(frame);
       return;
     }
 
-    // Approval pending: HMAC passed but auth_ok not yet sent.
-    // Only allow ping/pong — drop all business frames.
     if (!this.authOkSent) {
-      if (frame!.type !== "ping" && frame!.type !== "pong") return;
+      if (frame.type !== "ping" && frame.type !== "pong") return;
     }
 
-    if (frame!.type === "ping") {
-      this.send({
-        type: "pong",
-        from: this.nodeId,
-        timestamp: Date.now(),
-      } as AnyClusterFrame);
+    if (frame.type === "ping") {
+      this.send({ type: "pong", from: this.nodeId, timestamp: Date.now() } as AnyClusterFrame);
       return;
     }
-    if (frame!.type === "pong") {
+    if (frame.type === "pong") {
       this.missedPongs = 0;
       if (this.lastPingSentAt > 0) {
         const rtt = Date.now() - this.lastPingSentAt;
@@ -246,7 +397,7 @@ export class Connection extends EventEmitter<ConnectionEvents> {
       return;
     }
 
-    this.emit("message", frame!);
+    this.emit("message", frame);
   }
 
   private async handleAuthMessage(frame: any) {
@@ -296,7 +447,9 @@ export class Connection extends EventEmitter<ConnectionEvents> {
           // Signal the outbound that auth is pending so it extends its timer
           // instead of timing out after AUTH_TIMEOUT (10s).
           if (this.sessionKey) {
-            this.send({ type: "auth_pending", from: this.nodeId, timestamp: Date.now() } as AnyClusterFrame);
+            // Send immediately (bypass batch) so client gets it as a standalone frame
+            const pending = { type: "auth_pending", from: this.nodeId, timestamp: Date.now() };
+            this.sendRaw(encryptBinaryRaw(this.sessionKey, JSON.stringify(pending), false));
           } else {
             this.sendRaw({ p: 1 });
           }
@@ -453,7 +606,9 @@ export class Connection extends EventEmitter<ConnectionEvents> {
     } as AuthOk;
 
     if (this.sessionKey) {
-      this.send(authOk);
+      // Send auth_ok immediately (bypass batch queue) so the client
+      // receives it as a standalone binary frame, not mixed into a batch.
+      this.sendRaw(encryptBinaryRaw(this.sessionKey, JSON.stringify(authOk), false));
     } else {
       this.sendRaw(authOk);
     }
@@ -488,8 +643,19 @@ export class Connection extends EventEmitter<ConnectionEvents> {
     this.scheduleHeartbeatTick();
   }
 
+  /** Compute adaptive heartbeat interval based on measured latency. */
+  private adaptiveHeartbeatInterval(): number {
+    if (this.latencyMs > 0) {
+      // Low-latency (50ms) → 6s, high-latency (3s) → 20s
+      const base = Math.max(6_000, Math.min(this.latencyMs * 8, 20_000));
+      return base + Math.random() * 3_000;
+    }
+    // No latency measured yet — use default
+    return HEARTBEAT_BASE + Math.random() * HEARTBEAT_JITTER;
+  }
+
   private scheduleHeartbeatTick() {
-    const interval = HEARTBEAT_BASE + Math.random() * HEARTBEAT_JITTER;
+    const interval = this.adaptiveHeartbeatInterval();
     this.heartbeatTimer = setTimeout(this.heartbeatTick, interval);
   }
 
@@ -505,6 +671,15 @@ export class Connection extends EventEmitter<ConnectionEvents> {
       return;
     }
 
+    // Skip ping if recent traffic proves the connection is alive
+    const currentInterval = this.adaptiveHeartbeatInterval();
+    const timeSinceReceive = Date.now() - this.lastReceivedAt;
+    if (this.lastReceivedAt > 0 && timeSinceReceive < currentInterval * 0.5) {
+      this.missedPongs = 0;
+      this.scheduleHeartbeatTick();
+      return;
+    }
+
     // Increment before checking: this ping is about to be sent and
     // counts as outstanding until a pong arrives.
     this.missedPongs++;
@@ -551,13 +726,16 @@ export class Connection extends EventEmitter<ConnectionEvents> {
 
   private dummyTick = () => {
     if (this.closed || !this.sessionKey) return;
-    this.send({ type: "_d", from: "", timestamp: 0 } as unknown as AnyClusterFrame);
+    // Dummy traffic bypasses batch queue — sent directly
+    this.sendDummy();
     this.scheduleDummyTick();
   };
 
   // ── Cleanup ────────────────────────────────────────────────────
   close(code = 1000, reason = "normal") {
     if (this.closed) return;
+    // Flush any pending batch before closing
+    this.flushBatch();
     this.closed = true;
     this.clearAuthTimer();
     if (this.heartbeatTimer) {

package/src/crypto.ts

@@ -15,7 +15,7 @@ import {
   hkdfSync,
   randomBytes,
 } from "node:crypto";
-import { deflateSync, inflateSync } from "node:zlib";
+import { deflateRawSync, inflateRawSync } from "node:zlib";
 
 // ── Key exchange ──────────────────────────────────────────────────
 
@@ -93,13 +93,13 @@ export function deriveSessionKey(
 export function encryptBinary(
   sessionKey: Buffer,
   plaintext: string,
-  compress: boolean = false,
+  compress: boolean = true,
 ): string {
   let data: Buffer;
   let flags = 0;
 
-  if (compress && plaintext.length > 256) {
-    const deflated = deflateSync(Buffer.from(plaintext));
+  if (compress && plaintext.length > 1024) {
+    const deflated = deflateRawSync(Buffer.from(plaintext));
     if (deflated.length < plaintext.length * 0.9) {
       data = deflated;
       flags |= 0x01;
@@ -147,7 +147,7 @@ export function decryptBinary(
   const data = decrypted.subarray(4, 4 + actualLen);
 
   if (flags & 0x01) {
-    return inflateSync(data).toString();
+    return inflateRawSync(data).toString();
   }
   return data.toString();
 }
@@ -157,6 +157,73 @@ export function isBinaryEncrypted(s: string): boolean {
   return s.length > 0 && s[0] !== "{" && s[0] !== "[";
 }
 
+// ── Binary raw (Buffer) variants — skip base64 for WS binary frames ──
+
+/** Encrypt plaintext to a raw Buffer (no base64). For sending as WS binary frame.
+ *  Same logic as encryptBinary but returns Buffer directly → ~33% smaller on wire. */
+export function encryptBinaryRaw(
+  sessionKey: Buffer,
+  plaintext: string,
+  compress: boolean = true,
+): Buffer {
+  let data: Buffer;
+  let flags = 0;
+
+  if (compress && plaintext.length > 1024) {
+    const deflated = deflateRawSync(Buffer.from(plaintext));
+    if (deflated.length < plaintext.length * 0.9) {
+      data = deflated;
+      flags |= 0x01;
+    } else {
+      data = Buffer.from(plaintext);
+    }
+  } else {
+    data = Buffer.from(plaintext);
+  }
+
+  // Random padding: 4-byte length prefix + data + random padding
+  const padLen = 16 + (randomBytes(1)[0]! % 48);
+  const lenBuf = Buffer.alloc(4);
+  lenBuf.writeUInt32BE(data.length, 0);
+  const padded = Buffer.concat([lenBuf, data, randomBytes(padLen)]);
+
+  const iv = randomBytes(12);
+  const cipher = createCipheriv("aes-256-gcm", sessionKey, iv);
+  const encrypted = Buffer.concat([cipher.update(padded), cipher.final()]);
+  const tag = cipher.getAuthTag();
+
+  return Buffer.concat([Buffer.from([flags]), iv, encrypted, tag]);
+}
+
+/** Decrypt a raw Buffer envelope (no base64). Throws on auth failure. */
+export function decryptBinaryRaw(
+  sessionKey: Buffer,
+  buf: Buffer,
+): string {
+  if (buf.length < 1 + 12 + 16) throw new Error("Encrypted data too short");
+
+  const flags = buf[0]!;
+  const iv = buf.subarray(1, 13);
+  const rest = buf.subarray(13);
+  const ciphertext = rest.subarray(0, rest.length - 16);
+  const tag = rest.subarray(rest.length - 16);
+
+  const decipher = createDecipheriv("aes-256-gcm", sessionKey, iv);
+  decipher.setAuthTag(tag);
+  const decrypted = Buffer.concat([decipher.update(ciphertext), decipher.final()]);
+
+  // Strip length-prefixed padding
+  const actualLen = decrypted.readUInt32BE(0);
+  const data = decrypted.subarray(4, 4 + actualLen);
+
+  if (flags & 0x01) {
+    return inflateRawSync(data).toString();
+  }
+  return data.toString();
+}
+
+
+
 // ── Helpers ───────────────────────────────────────────────────────
 
 /** Encode a raw public key to base64 for wire transmission. */

package/src/device-info.ts

@@ -1,7 +1,7 @@
 /**
  * Collect local device/system information using Node.js os module.
  */
-import { arch, cpus, hostname, totalmem, type, release } from "node:os";
+import { arch, cpus, hostname, totalmem, type, release, homedir } from "node:os";
 import { dirname, join } from "node:path";
 import { readFileSync } from "node:fs";
 import type { DeviceInfo } from "./types.ts";
@@ -33,8 +33,25 @@ function resolveOpenclawVersion(hint?: string): string | undefined {
   return undefined;
 }
 
+/** Try to read agents.defaults.workspace from OpenClaw config file (~/.openclaw/openclaw.json). */
+function resolveWorkspace(openclawConfig?: Record<string, unknown>): string | undefined {
+  // 1. Try from the passed-in config object
+  const agentsDefaults = (openclawConfig?.agents as Record<string, unknown> | undefined)?.defaults as Record<string, unknown> | undefined;
+  if (typeof agentsDefaults?.workspace === "string") return agentsDefaults.workspace;
+
+  // 2. Fallback: read ~/.openclaw/openclaw.json directly
+  try {
+    const configPath = join(homedir(), ".openclaw", "openclaw.json");
+    const raw = JSON.parse(readFileSync(configPath, "utf-8"));
+    const ws = raw?.agents?.defaults?.workspace;
+    if (typeof ws === "string") return ws;
+  } catch { /* config not found or invalid */ }
+
+  return undefined;
+}
+
 /** Collect device info once at startup. */
-export function collectDeviceInfo(openclawVersion?: string): DeviceInfo {
+export function collectDeviceInfo(openclawVersion?: string, openclawConfig?: Record<string, unknown>): DeviceInfo {
   const cpuList = cpus();
   return {
     os: `${type()} ${release()}`,
@@ -44,5 +61,7 @@ export function collectDeviceInfo(openclawVersion?: string): DeviceInfo {
     totalMemoryMB: Math.round(totalmem() / (1024 * 1024)),
     hostname: hostname(),
     openclawVersion: resolveOpenclawVersion(openclawVersion),
+    cwd: process.cwd(),
+    workspace: resolveWorkspace(openclawConfig),
   };
 }

package/src/file-transfer.ts

@@ -1,6 +1,7 @@
 import { createHash } from "node:crypto";
 import { readFile, writeFile, stat, mkdir, lstat, realpath } from "node:fs/promises";
 import path from "node:path";
+import { nanoid } from "nanoid";
 import { debug } from "./debug.ts";
 import type { PeerManager } from "./peer-manager.ts";
 import type { ClawMatrixConfig } from "./config.ts";
@@ -114,7 +115,7 @@ export class FileTransferManager {
 
     const checksum = createHash("sha256").update(fileData).digest("hex");
     const totalChunks = Math.ceil(fileData.length / this.config.chunkSize) || 1;
-    const sessionId = crypto.randomUUID();
+    const sessionId = nanoid();
 
     return new Promise<TransferResult>((resolve, reject) => {
       const timer = this.createTimer(sessionId, "pending");
@@ -160,7 +161,7 @@ export class FileTransferManager {
     const resolvedPath = path.resolve(localPath);
     await this.validatePath(resolvedPath);
 
-    const sessionId = crypto.randomUUID();
+    const sessionId = nanoid();
 
     return new Promise<TransferResult>((resolve, reject) => {
       const timer = this.createTimer(sessionId, "pending");