clawmatrix 0.1.22 → 0.2.0

package/src/crypto.ts

@@ -0,0 +1,179 @@
+/**
+ * E2EE cryptographic primitives: X25519 ECDH key exchange + AES-256-GCM payload encryption.
+ *
+ * Uses node:crypto for X25519 (not available in Web Crypto API) and
+ * synchronous AES-256-GCM to keep Connection.send() non-async.
+ */
+
+import {
+  generateKeyPairSync,
+  diffieHellman,
+  createPublicKey,
+  createPrivateKey,
+  createCipheriv,
+  createDecipheriv,
+  hkdfSync,
+  randomBytes,
+} from "node:crypto";
+import { deflateSync, inflateSync } from "node:zlib";
+
+// ── Key exchange ──────────────────────────────────────────────────
+
+export interface KeyPair {
+  publicKey: Buffer;   // 32 bytes raw X25519 public key
+  privateKey: Buffer;  // 32 bytes raw X25519 private key (PKCS8-wrapped internally)
+  /** PKCS8-encoded private key for node:crypto diffieHellman. */
+  _privateKeyObject: ReturnType<typeof createPrivateKey>;
+}
+
+/** Generate an ephemeral X25519 keypair. */
+export function generateX25519KeyPair(): KeyPair {
+  const { publicKey, privateKey } = generateKeyPairSync("x25519", {
+    publicKeyEncoding: { type: "spki", format: "der" },
+    privateKeyEncoding: { type: "pkcs8", format: "der" },
+  });
+
+  // Extract raw 32-byte public key from SPKI DER (last 32 bytes)
+  const rawPublic = Buffer.from(publicKey.subarray(publicKey.length - 32));
+
+  return {
+    publicKey: rawPublic,
+    privateKey: Buffer.from(privateKey),
+    _privateKeyObject: createPrivateKey({
+      key: privateKey,
+      format: "der",
+      type: "pkcs8",
+    }),
+  };
+}
+
+/** Derive a 256-bit AES session key from X25519 ECDH shared secret.
+ *  Both sides produce the same key regardless of who is client/server. */
+export function deriveSessionKey(
+  localKeyPair: KeyPair,
+  remotePublicKeyRaw: Buffer,
+  salt?: Buffer,
+): Buffer {
+  // Wrap raw 32-byte public key back into SPKI DER for node:crypto
+  const spkiPrefix = Buffer.from([
+    0x30, 0x2a, 0x30, 0x05, 0x06, 0x03, 0x2b, 0x65,
+    0x6e, 0x03, 0x21, 0x00,
+  ]);
+  const spkiDer = Buffer.concat([spkiPrefix, remotePublicKeyRaw]);
+  const remoteKeyObject = createPublicKey({
+    key: spkiDer,
+    format: "der",
+    type: "spki",
+  });
+
+  const sharedSecret = diffieHellman({
+    privateKey: localKeyPair._privateKeyObject,
+    publicKey: remoteKeyObject,
+  });
+
+  // HKDF-SHA256: derive 32-byte AES key with fixed info string
+  const derived = hkdfSync(
+    "sha256",
+    sharedSecret,
+    salt ?? Buffer.alloc(0), // salt should include connection-specific nonce for forward secrecy
+    "clawmatrix-e2ee-v1",
+    32,
+  );
+
+  return Buffer.from(derived);
+}
+
+// ── Binary encrypted wire format ────────────────────────────────────
+// Wire: base64([flags:1][iv:12][ciphertext][tag:16])
+// flags: bit 0 = compressed, bits 1-7 reserved
+// The result is a single base64 string — no JSON structure to fingerprint.
+
+/** Encrypt a plaintext string to a base64 binary envelope.
+ *  Applies random padding (16-64 bytes) to resist traffic analysis. */
+export function encryptBinary(
+  sessionKey: Buffer,
+  plaintext: string,
+  compress: boolean = false,
+): string {
+  let data: Buffer;
+  let flags = 0;
+
+  if (compress && plaintext.length > 256) {
+    const deflated = deflateSync(Buffer.from(plaintext));
+    if (deflated.length < plaintext.length * 0.9) {
+      data = deflated;
+      flags |= 0x01;
+    } else {
+      data = Buffer.from(plaintext);
+    }
+  } else {
+    data = Buffer.from(plaintext);
+  }
+
+  // Random padding: 4-byte length prefix + data + random padding
+  const padLen = 16 + (randomBytes(1)[0]! % 48);
+  const lenBuf = Buffer.alloc(4);
+  lenBuf.writeUInt32BE(data.length, 0);
+  const padded = Buffer.concat([lenBuf, data, randomBytes(padLen)]);
+
+  const iv = randomBytes(12);
+  const cipher = createCipheriv("aes-256-gcm", sessionKey, iv);
+  const encrypted = Buffer.concat([cipher.update(padded), cipher.final()]);
+  const tag = cipher.getAuthTag();
+
+  return Buffer.concat([Buffer.from([flags]), iv, encrypted, tag]).toString("base64");
+}
+
+/** Decrypt a base64 binary envelope. Throws on auth failure. */
+export function decryptBinary(
+  sessionKey: Buffer,
+  b64: string,
+): string {
+  const buf = Buffer.from(b64, "base64");
+  if (buf.length < 1 + 12 + 16) throw new Error("Encrypted data too short");
+
+  const flags = buf[0]!;
+  const iv = buf.subarray(1, 13);
+  const rest = buf.subarray(13);
+  const ciphertext = rest.subarray(0, rest.length - 16);
+  const tag = rest.subarray(rest.length - 16);
+
+  const decipher = createDecipheriv("aes-256-gcm", sessionKey, iv);
+  decipher.setAuthTag(tag);
+  const decrypted = Buffer.concat([decipher.update(ciphertext), decipher.final()]);
+
+  // Strip length-prefixed padding
+  const actualLen = decrypted.readUInt32BE(0);
+  const data = decrypted.subarray(4, 4 + actualLen);
+
+  if (flags & 0x01) {
+    return inflateSync(data).toString();
+  }
+  return data.toString();
+}
+
+/** Check if a raw WS message string is a binary encrypted envelope (not JSON). */
+export function isBinaryEncrypted(s: string): boolean {
+  return s.length > 0 && s[0] !== "{" && s[0] !== "[";
+}
+
+// ── Helpers ───────────────────────────────────────────────────────
+
+/** Encode a raw public key to base64 for wire transmission. */
+export function publicKeyToBase64(key: Buffer): string {
+  return key.toString("base64");
+}
+
+/** Decode a base64 public key from wire format. */
+export function base64ToPublicKey(b64: string): Buffer {
+  return Buffer.from(b64, "base64");
+}
+
+/** Derive a per-peer authentication secret from the master secret and both node IDs.
+ *  Sorted node IDs ensure both sides derive the same value regardless of who initiates. */
+export function derivePerPeerSecret(masterSecret: string, nodeIdA: string, nodeIdB: string): string {
+  const sorted = [nodeIdA, nodeIdB].sort();
+  const salt = Buffer.from(sorted.join(":"));
+  const derived = hkdfSync("sha256", Buffer.from(masterSecret), salt, "clawmatrix-peer-auth-v1", 32);
+  return Buffer.from(derived).toString("hex");
+}

package/src/debug.ts

@@ -1,5 +1,18 @@
-const enabled = process.env.CLAWMATRIX_DEBUG !== "0";
+type Logger = {
+  info: (message: string) => void;
+};
+
+let _logger: Logger | undefined;
+
+export function initDebugLogger(logger: Logger) {
+  _logger = logger;
+}
 
 export function debug(tag: string, msg: string) {
-  if (enabled) console.debug(`[clawmatrix:${tag}] ${msg}`);
+  const message = `[clawmatrix:${tag}] ${msg}`;
+  if (_logger) {
+    _logger.info(message);
+  } else {
+    console.log(message);
+  }
 }

package/src/e2e/helpers.ts

@@ -0,0 +1,318 @@
+/**
+ * Shared test infrastructure for E2E tests.
+ *
+ * Provides TestNode (wraps PeerManager + HandoffManager + ModelProxy + ToolProxy
+ * with real WebSocket connections) and helpers for port allocation, mesh readiness,
+ * and mock HTTP servers.
+ */
+
+import { PeerManager } from "../peer-manager.ts";
+import { HandoffManager } from "../handoff.ts";
+import { ModelProxy } from "../model-proxy.ts";
+import { ToolProxy, type GatewayInfo } from "../tool-proxy.ts";
+import type { ClawMatrixConfig } from "../config.ts";
+import type { AnyClusterFrame } from "../types.ts";
+import type { PluginLogger } from "openclaw/plugin-sdk";
+
+// ── Port allocation ─────────────────────────────────────────────────
+// Range 19500-19999, avoiding mesh.test.ts range (19100-19400).
+let nextPort = 19500;
+
+/** Allocate a unique port for a test node. Not concurrency-safe across processes. */
+export function allocPort(): number {
+  return nextPort++;
+}
+
+/** Reset port counter (call in afterAll if tests run in a loop). */
+export function resetPorts(start = 19500): void {
+  nextPort = start;
+}
+
+// ── Logger ──────────────────────────────────────────────────────────
+
+/** Minimal no-op logger satisfying PluginLogger. */
+export function createTestLogger(): PluginLogger {
+  return {
+    info: () => {},
+    warn: () => {},
+    error: () => {},
+    debug: () => {},
+  } as unknown as PluginLogger;
+}
+
+// ── Config builder ──────────────────────────────────────────────────
+
+export interface TestNodeOptions {
+  nodeId: string;
+  listen?: boolean;
+  listenPort?: number;
+  peers?: Array<{ nodeId: string; url: string }>;
+  agents?: Array<{ id: string; description: string; tags: string[] }>;
+  models?: Array<{
+    id: string;
+    provider: string;
+    description?: string;
+    baseUrl?: string;
+    apiKey?: string;
+    api?: string;
+  }>;
+  tags?: string[];
+  proxyPort?: number;
+  toolProxy?: {
+    enabled: boolean;
+    allow?: string[];
+    deny?: string[];
+    maxOutputBytes?: number;
+  };
+  proxyModels?: Array<{
+    id: string;
+    nodeId: string;
+    provider?: string;
+    description?: string;
+    api?: string;
+  }>;
+  handoffTimeout?: number;
+  toolTimeout?: number;
+  modelTimeout?: number;
+  /** Override gatewayInfo (default: port 0, no auth). */
+  gatewayInfo?: GatewayInfo;
+  /** Enable E2E encryption (default: false). */
+  e2ee?: boolean;
+}
+
+function buildConfig(options: TestNodeOptions): ClawMatrixConfig {
+  return {
+    nodeId: options.nodeId,
+    secret: "test-secret-long-enough",
+    listen: options.listen ?? false,
+    listenHost: "0.0.0.0",
+    listenPort: options.listenPort ?? 0,
+    peers: options.peers ?? [],
+    agents: options.agents ?? [],
+    models: options.models ?? [],
+    proxyModels: (options.proxyModels ?? []).map((m) => ({
+      id: m.id,
+      nodeId: m.nodeId,
+      provider: m.provider,
+      description: m.description,
+      api: m.api,
+    })),
+    tags: options.tags ?? [],
+    proxyPort: options.proxyPort ?? 0,
+    toolProxy: options.toolProxy
+      ? {
+          enabled: options.toolProxy.enabled,
+          allow: options.toolProxy.allow ?? [],
+          deny: options.toolProxy.deny ?? [],
+          maxOutputBytes: options.toolProxy.maxOutputBytes ?? 1_048_576,
+        }
+      : undefined,
+    handoffTimeout: options.handoffTimeout ?? 600_000,
+    modelTimeout: options.modelTimeout ?? 120_000,
+    toolTimeout: options.toolTimeout ?? 30_000,
+    peerApproval: {
+      enabled: false,
+      mode: "notify",
+      timeout: 1_200_000,
+      allowList: [],
+      persistPath: "approved-peers.json",
+    },
+    e2ee: options.e2ee ?? false,
+    compression: false,
+  } as ClawMatrixConfig;
+}
+
+// ── TestNode ────────────────────────────────────────────────────────
+
+export class TestNode {
+  readonly config: ClawMatrixConfig;
+  readonly peerManager: PeerManager;
+  readonly handoffManager: HandoffManager;
+  readonly toolProxy: ToolProxy;
+  readonly modelProxy: ModelProxy;
+  /** Frames received via the dispatch wiring (for assertions). */
+  readonly receivedFrames: AnyClusterFrame[] = [];
+
+  constructor(options: TestNodeOptions) {
+    this.config = buildConfig(options);
+
+    const gatewayInfo: GatewayInfo = options.gatewayInfo ?? { port: 0 };
+    const logger = createTestLogger();
+
+    this.peerManager = new PeerManager(this.config);
+    this.handoffManager = new HandoffManager(this.config, this.peerManager, gatewayInfo);
+    this.toolProxy = new ToolProxy(this.config, this.peerManager, gatewayInfo, logger);
+    this.modelProxy = new ModelProxy(this.config, this.peerManager, gatewayInfo, {} as any);
+
+    // Wire frame dispatch (mirrors ClusterRuntime.dispatchFrame)
+    this.peerManager.on("frame", (frame) => {
+      this.receivedFrames.push(frame);
+      this.dispatchFrame(frame);
+    });
+  }
+
+  private dispatchFrame(frame: AnyClusterFrame): void {
+    switch (frame.type) {
+      case "handoff_req":
+        this.handoffManager.handleRequest(frame as any).catch(() => {});
+        break;
+      case "handoff_stream":
+        this.handoffManager.handleStream(frame as any);
+        break;
+      case "handoff_res":
+        this.handoffManager.handleResponse(frame as any);
+        break;
+      case "handoff_cancel":
+        this.handoffManager.handleCancel(frame as any);
+        break;
+      case "handoff_input_required":
+        this.handoffManager.handleInputRequired(frame as any);
+        break;
+      case "handoff_input":
+        this.handoffManager.handleInput(frame as any).catch(() => {});
+        break;
+      case "model_req":
+        this.modelProxy.handleModelRequest(frame as any).catch(() => {});
+        break;
+      case "model_res":
+        this.modelProxy.handleModelResponse(frame as any);
+        break;
+      case "model_stream":
+        this.modelProxy.handleModelStream(frame as any);
+        break;
+      case "tool_req":
+        this.toolProxy.handleRequest(frame as any).catch(() => {});
+        break;
+      case "tool_res":
+        this.toolProxy.handleResponse(frame as any);
+        break;
+      case "tool_batch_req":
+        this.toolProxy.handleBatchRequest(frame as any).catch(() => {});
+        break;
+      case "tool_batch_res":
+        this.toolProxy.handleBatchResponse(frame as any);
+        break;
+      default:
+        // Unhandled frame types are silently ignored in tests
+        break;
+    }
+  }
+
+  /** Start the node (model proxy HTTP + peer manager WebSocket). */
+  async start(): Promise<void> {
+    this.modelProxy.start();
+    await this.peerManager.start();
+  }
+
+  /** Gracefully shut down all subsystems. */
+  async stop(): Promise<void> {
+    this.handoffManager.destroy();
+    this.toolProxy.destroy();
+    this.modelProxy.stop();
+    await this.peerManager.stop();
+  }
+}
+
+// ── Mesh helpers ────────────────────────────────────────────────────
+
+/**
+ * Wait until two nodes have established a WebSocket connection to each other.
+ * Resolves when both sides have fired `peerConnected`.
+ */
+export async function waitForConnection(a: TestNode, b: TestNode, timeout = 5000): Promise<void> {
+  const aKnowsB = a.peerManager.router.getRoute(b.config.nodeId) !== undefined;
+  const bKnowsA = b.peerManager.router.getRoute(a.config.nodeId) !== undefined;
+  if (aKnowsB && bKnowsA) return;
+
+  await Promise.all([
+    aKnowsB
+      ? Promise.resolve()
+      : new Promise<void>((resolve, reject) => {
+          const timer = setTimeout(() => reject(new Error(`waitForConnection: ${a.config.nodeId} did not see ${b.config.nodeId} within ${timeout}ms`)), timeout);
+          const handler = (nodeId: string) => {
+            if (nodeId === b.config.nodeId) {
+              clearTimeout(timer);
+              a.peerManager.off("peerConnected", handler);
+              resolve();
+            }
+          };
+          a.peerManager.on("peerConnected", handler);
+        }),
+    bKnowsA
+      ? Promise.resolve()
+      : new Promise<void>((resolve, reject) => {
+          const timer = setTimeout(() => reject(new Error(`waitForConnection: ${b.config.nodeId} did not see ${a.config.nodeId} within ${timeout}ms`)), timeout);
+          const handler = (nodeId: string) => {
+            if (nodeId === a.config.nodeId) {
+              clearTimeout(timer);
+              b.peerManager.off("peerConnected", handler);
+              resolve();
+            }
+          };
+          b.peerManager.on("peerConnected", handler);
+        }),
+  ]);
+}
+
+/**
+ * Wait until every node in the list knows about every other node via gossip.
+ * Polls the routing table at short intervals.
+ */
+export async function waitForMesh(nodes: TestNode[], timeout = 5000): Promise<void> {
+  const deadline = Date.now() + timeout;
+
+  while (Date.now() < deadline) {
+    let complete = true;
+    for (const node of nodes) {
+      for (const other of nodes) {
+        if (node === other) continue;
+        if (!node.peerManager.router.getRoute(other.config.nodeId)) {
+          complete = false;
+          break;
+        }
+      }
+      if (!complete) break;
+    }
+    if (complete) return;
+    await new Promise((r) => setTimeout(r, 50));
+  }
+
+  // Build a diagnostic message
+  const missing: string[] = [];
+  for (const node of nodes) {
+    for (const other of nodes) {
+      if (node === other) continue;
+      if (!node.peerManager.router.getRoute(other.config.nodeId)) {
+        missing.push(`${node.config.nodeId} -> ${other.config.nodeId}`);
+      }
+    }
+  }
+  throw new Error(`waitForMesh: timed out after ${timeout}ms. Missing routes: ${missing.join(", ")}`);
+}
+
+// ── Mock HTTP server ────────────────────────────────────────────────
+
+export interface MockHttpServer {
+  url: string;
+  port: number;
+  close: () => void;
+}
+
+/**
+ * Create a mock HTTP server using Bun.serve on a random port.
+ * Useful for faking gateway API or model API responses in E2E tests.
+ */
+export async function createMockHttpServer(
+  handler: (req: Request) => Response | Promise<Response>,
+): Promise<MockHttpServer> {
+  const server = Bun.serve({
+    port: 0, // random available port
+    fetch: handler,
+  });
+
+  return {
+    url: `http://localhost:${server.port}`,
+    port: server.port,
+    close: () => server.stop(true),
+  };
+}