clawmatrix 0.2.8 → 0.2.11

package/src/file-transfer.ts

@@ -0,0 +1,671 @@
+import { createHash } from "node:crypto";
+import { readFile, writeFile, stat, mkdir } from "node:fs/promises";
+import path from "node:path";
+import { debug } from "./debug.ts";
+import type { PeerManager } from "./peer-manager.ts";
+import type { ClawMatrixConfig } from "./config.ts";
+import type {
+  FileTransferInit,
+  FileTransferAck,
+  FileTransferChunk,
+  FileTransferChunkAck,
+  FileTransferComplete,
+} from "./types.ts";
+
+interface FileTransferConfig {
+  enabled: boolean;
+  chunkSize: number;
+  maxFileSize: number;
+  timeout: number;
+  allowedPaths: string[];
+}
+
+interface PendingTransfer {
+  sessionId: string;
+  direction: "push" | "pull";
+  filePath: string;
+  targetPath: string;
+  remoteNode: string;
+  resolve: (result: TransferResult) => void;
+  reject: (err: Error) => void;
+  timer: ReturnType<typeof setTimeout>;
+  // Push state
+  fileData?: Buffer;
+  chunkSize?: number;
+  totalChunks?: number;
+  sentChunks?: number;
+  // Pull state
+  chunks?: Map<number, Buffer>;
+  expectedSize?: number;
+  expectedChunks?: number;
+  expectedChecksum?: string;
+  receivedChunks?: number;
+}
+
+interface ReceivingTransfer {
+  sessionId: string;
+  direction: "push" | "pull";
+  filePath: string;
+  targetPath: string;
+  fromNode: string;
+  fileSize: number;
+  totalChunks: number;
+  chunkSize: number;
+  checksum: string;
+  chunks: Map<number, Buffer>;
+  receivedChunks: number;
+  timer: ReturnType<typeof setTimeout>;
+  /** Cached file data for pull-mode serving to avoid re-reading per chunk */
+  cachedData?: Buffer;
+}
+
+export interface TransferResult {
+  success: boolean;
+  bytesTransferred: number;
+  error?: string;
+}
+
+export class FileTransferManager {
+  private config: FileTransferConfig;
+  private nodeId: string;
+  private peerManager: PeerManager;
+  private pending = new Map<string, PendingTransfer>();
+  private receiving = new Map<string, ReceivingTransfer>();
+
+  constructor(config: ClawMatrixConfig, peerManager: PeerManager) {
+    this.nodeId = config.nodeId;
+    this.peerManager = peerManager;
+    const ft = config.fileTransfer;
+    this.config = {
+      enabled: ft?.enabled ?? false,
+      chunkSize: ft?.chunkSize ?? 262_144,
+      maxFileSize: ft?.maxFileSize ?? 104_857_600,
+      timeout: ft?.timeout ?? 300_000,
+      allowedPaths: ft?.allowedPaths ?? [],
+    };
+  }
+
+  // ── Public API ──────────────────────────────────────────────────
+
+  async pushFile(remoteNode: string, localPath: string, remotePath: string): Promise<TransferResult> {
+    this.ensureEnabled();
+    const resolvedPath = path.resolve(localPath);
+    this.validatePath(resolvedPath);
+
+    const fileData = await readFile(resolvedPath);
+    if (fileData.length > this.config.maxFileSize) {
+      throw new Error(`File too large: ${fileData.length} bytes (max ${this.config.maxFileSize})`);
+    }
+
+    const checksum = createHash("sha256").update(fileData).digest("hex");
+    const totalChunks = Math.ceil(fileData.length / this.config.chunkSize) || 1;
+    const sessionId = crypto.randomUUID();
+
+    return new Promise<TransferResult>((resolve, reject) => {
+      const timer = this.createTimer(sessionId, "pending");
+
+      this.pending.set(sessionId, {
+        sessionId,
+        direction: "push",
+        filePath: resolvedPath,
+        targetPath: remotePath,
+        remoteNode,
+        resolve,
+        reject,
+        timer,
+        fileData,
+        chunkSize: this.config.chunkSize,
+        totalChunks,
+        sentChunks: 0,
+      });
+
+      this.peerManager.sendTo(remoteNode, {
+        type: "file_transfer_init",
+        id: sessionId,
+        from: this.nodeId,
+        to: remoteNode,
+        timestamp: Date.now(),
+        payload: {
+          sessionId,
+          direction: "push",
+          filePath: resolvedPath,
+          targetPath: remotePath,
+          fileSize: fileData.length,
+          totalChunks,
+          chunkSize: this.config.chunkSize,
+          checksum,
+        },
+      } as FileTransferInit);
+    });
+  }
+
+  async pullFile(remoteNode: string, remotePath: string, localPath: string): Promise<TransferResult> {
+    this.ensureEnabled();
+    const resolvedPath = path.resolve(localPath);
+    this.validatePath(resolvedPath);
+
+    const sessionId = crypto.randomUUID();
+
+    return new Promise<TransferResult>((resolve, reject) => {
+      const timer = this.createTimer(sessionId, "pending");
+
+      this.pending.set(sessionId, {
+        sessionId,
+        direction: "pull",
+        filePath: remotePath,
+        targetPath: resolvedPath,
+        remoteNode,
+        resolve,
+        reject,
+        timer,
+        chunks: new Map(),
+        receivedChunks: 0,
+      });
+
+      this.peerManager.sendTo(remoteNode, {
+        type: "file_transfer_init",
+        id: sessionId,
+        from: this.nodeId,
+        to: remoteNode,
+        timestamp: Date.now(),
+        payload: {
+          sessionId,
+          direction: "pull",
+          filePath: remotePath,
+          targetPath: resolvedPath,
+          fileSize: 0,
+          totalChunks: 0,
+          chunkSize: this.config.chunkSize,
+          checksum: "",
+        },
+      } as FileTransferInit);
+    });
+  }
+
+  // ── Frame handlers (called by cluster-service dispatch) ─────────
+
+  async handleInit(frame: FileTransferInit): Promise<void> {
+    const { sessionId, direction, filePath, targetPath, fileSize, totalChunks, chunkSize, checksum } = frame.payload;
+
+    // Validate enabled
+    if (!this.config.enabled) {
+      this.sendAck(frame.from, sessionId, frame.id, false, "File transfer not enabled on this node");
+      return;
+    }
+
+    if (direction === "push") {
+      // Remote wants to push a file to us
+      const resolvedTarget = path.resolve(targetPath);
+      if (!this.isPathAllowed(resolvedTarget)) {
+        this.sendAck(frame.from, sessionId, frame.id, false, "Target path not allowed");
+        return;
+      }
+      if (fileSize > this.config.maxFileSize) {
+        this.sendAck(frame.from, sessionId, frame.id, false, `File too large: ${fileSize} bytes (max ${this.config.maxFileSize})`);
+        return;
+      }
+
+      // Accept and prepare to receive chunks
+      const timer = this.createTimer(sessionId, "receiving");
+      this.receiving.set(sessionId, {
+        sessionId,
+        direction: "push",
+        filePath,
+        targetPath: resolvedTarget,
+        fromNode: frame.from,
+        fileSize,
+        totalChunks,
+        chunkSize,
+        checksum,
+        chunks: new Map(),
+        receivedChunks: 0,
+        timer,
+      });
+
+      this.sendAck(frame.from, sessionId, frame.id, true);
+    } else {
+      // Remote wants to pull a file from us
+      const resolvedSource = path.resolve(filePath);
+      if (!this.isPathAllowed(resolvedSource)) {
+        this.sendAck(frame.from, sessionId, frame.id, false, "Source path not allowed");
+        return;
+      }
+
+      let fileData: Buffer;
+      try {
+        fileData = await readFile(resolvedSource);
+      } catch (err) {
+        this.sendAck(frame.from, sessionId, frame.id, false, `Cannot read file: ${err instanceof Error ? err.message : String(err)}`);
+        return;
+      }
+
+      if (fileData.length > this.config.maxFileSize) {
+        this.sendAck(frame.from, sessionId, frame.id, false, `File too large: ${fileData.length} bytes`);
+        return;
+      }
+
+      const fileChecksum = createHash("sha256").update(fileData).digest("hex");
+      const fileTotalChunks = Math.ceil(fileData.length / this.config.chunkSize) || 1;
+
+      // Send ack with file metadata
+      this.peerManager.sendTo(frame.from, {
+        type: "file_transfer_ack",
+        id: frame.id,
+        from: this.nodeId,
+        to: frame.from,
+        timestamp: Date.now(),
+        payload: {
+          sessionId,
+          accepted: true,
+          fileSize: fileData.length,
+          totalChunks: fileTotalChunks,
+          checksum: fileChecksum,
+        },
+      } as FileTransferAck);
+
+      // Store as a "receiving" entry for tracking (we are the sender in pull mode)
+      const timer = this.createTimer(sessionId, "receiving");
+      this.receiving.set(sessionId, {
+        sessionId,
+        direction: "pull",
+        filePath: resolvedSource,
+        targetPath,
+        fromNode: frame.from,
+        fileSize: fileData.length,
+        totalChunks: fileTotalChunks,
+        chunkSize: this.config.chunkSize,
+        checksum: fileChecksum,
+        chunks: new Map(),
+        receivedChunks: 0,
+        timer,
+        cachedData: fileData,
+      });
+
+      // Start sending chunks (stop-and-wait: send first, wait for ack)
+      this.sendChunkFromReceiving(sessionId, 0, fileData);
+    }
+  }
+
+  handleAck(frame: FileTransferAck): void {
+    const { sessionId, accepted, error, fileSize, totalChunks, checksum } = frame.payload;
+    const transfer = this.pending.get(sessionId);
+    if (!transfer) return;
+
+    this.resetTimer(sessionId, "pending");
+
+    if (!accepted) {
+      this.completePending(sessionId, { success: false, bytesTransferred: 0, error: error || "Transfer rejected" });
+      return;
+    }
+
+    if (transfer.direction === "push") {
+      // Start sending chunks
+      this.sendNextChunk(sessionId);
+    } else {
+      // Pull mode: store expected metadata
+      transfer.expectedSize = fileSize;
+      transfer.expectedChunks = totalChunks;
+      transfer.expectedChecksum = checksum;
+      // Wait for chunks from remote
+    }
+  }
+
+  handleChunk(frame: FileTransferChunk): void {
+    const { sessionId, chunkIndex, data } = frame.payload;
+
+    // Check if this is a pull we initiated (we're receiving)
+    const pending = this.pending.get(sessionId);
+    if (pending && pending.direction === "pull") {
+      this.resetTimer(sessionId, "pending");
+      const buf = Buffer.from(data, "base64");
+      pending.chunks!.set(chunkIndex, buf);
+      pending.receivedChunks = (pending.receivedChunks ?? 0) + 1;
+
+      // Send chunk ack
+      this.peerManager.sendTo(pending.remoteNode, {
+        type: "file_transfer_chunk_ack",
+        id: frame.id,
+        from: this.nodeId,
+        to: pending.remoteNode,
+        timestamp: Date.now(),
+        payload: { sessionId, chunkIndex, success: true },
+      } as FileTransferChunkAck);
+
+      // Check if all chunks received
+      if (pending.receivedChunks === pending.expectedChunks) {
+        this.finalizePull(sessionId).catch((err) => {
+          this.completePending(sessionId, {
+            success: false,
+            bytesTransferred: 0,
+            error: err instanceof Error ? err.message : String(err),
+          });
+        });
+      }
+      return;
+    }
+
+    // Otherwise, this is a push from remote (we're receiving)
+    const receiving = this.receiving.get(sessionId);
+    if (!receiving || receiving.direction !== "push") return;
+
+    this.resetTimer(sessionId, "receiving");
+    const buf = Buffer.from(data, "base64");
+    receiving.chunks.set(chunkIndex, buf);
+    receiving.receivedChunks++;
+
+    // Send chunk ack
+    this.peerManager.sendTo(receiving.fromNode, {
+      type: "file_transfer_chunk_ack",
+      id: frame.id,
+      from: this.nodeId,
+      to: receiving.fromNode,
+      timestamp: Date.now(),
+      payload: { sessionId, chunkIndex, success: true },
+    } as FileTransferChunkAck);
+
+    // Check if all chunks received
+    if (receiving.receivedChunks === receiving.totalChunks) {
+      this.finalizePushReceive(sessionId).catch((err) => {
+        debug("file-transfer", `finalizePushReceive error: ${err}`);
+      });
+    }
+  }
+
+  handleChunkAck(frame: FileTransferChunkAck): void {
+    const { sessionId, chunkIndex, success, error } = frame.payload;
+
+    // Push mode: we sent a chunk, got ack, send next
+    const pending = this.pending.get(sessionId);
+    if (pending && pending.direction === "push") {
+      this.resetTimer(sessionId, "pending");
+      if (!success) {
+        this.completePending(sessionId, {
+          success: false,
+          bytesTransferred: chunkIndex * (pending.chunkSize ?? this.config.chunkSize),
+          error: error || `Chunk ${chunkIndex} rejected`,
+        });
+        return;
+      }
+      pending.sentChunks = chunkIndex + 1;
+      if (pending.sentChunks! < pending.totalChunks!) {
+        this.sendNextChunk(sessionId);
+      }
+      // If all sent, wait for file_transfer_complete from receiver
+      return;
+    }
+
+    // Pull mode (we are serving): got chunk ack, send next
+    const receiving = this.receiving.get(sessionId);
+    if (receiving && receiving.direction === "pull") {
+      this.resetTimer(sessionId, "receiving");
+      if (!success) {
+        this.cleanupReceiving(sessionId);
+        return;
+      }
+      const nextIndex = chunkIndex + 1;
+      if (nextIndex < receiving.totalChunks) {
+        if (receiving.cachedData) {
+          this.sendChunkFromReceiving(sessionId, nextIndex, receiving.cachedData);
+        } else {
+          // Fallback: read file if cached data is missing
+          readFile(receiving.filePath).then((data) => {
+            receiving.cachedData = data;
+            this.sendChunkFromReceiving(sessionId, nextIndex, data);
+          }).catch(() => {
+            this.cleanupReceiving(sessionId);
+          });
+        }
+      }
+      // If all chunks sent, wait for file_transfer_complete from puller
+    }
+  }
+
+  handleComplete(frame: FileTransferComplete): void {
+    const { sessionId, success, error, bytesTransferred } = frame.payload;
+
+    // Pending transfer completed
+    const pending = this.pending.get(sessionId);
+    if (pending) {
+      this.completePending(sessionId, {
+        success,
+        bytesTransferred: bytesTransferred ?? 0,
+        error: error || undefined,
+      });
+      return;
+    }
+
+    // Receiving transfer completed (pull mode serving — remote confirms)
+    const receiving = this.receiving.get(sessionId);
+    if (receiving) {
+      this.cleanupReceiving(sessionId);
+    }
+  }
+
+  destroy(): void {
+    for (const [id, transfer] of this.pending) {
+      clearTimeout(transfer.timer);
+      transfer.reject(new Error("FileTransferManager destroyed"));
+    }
+    this.pending.clear();
+
+    for (const [id, transfer] of this.receiving) {
+      clearTimeout(transfer.timer);
+    }
+    this.receiving.clear();
+  }
+
+  // ── Internal helpers ────────────────────────────────────────────
+
+  private ensureEnabled(): void {
+    if (!this.config.enabled) {
+      throw new Error("File transfer is not enabled");
+    }
+  }
+
+  private validatePath(resolvedPath: string): void {
+    if (!this.isPathAllowed(resolvedPath)) {
+      throw new Error(`Path not allowed: ${resolvedPath}`);
+    }
+  }
+
+  private isPathAllowed(resolvedPath: string): boolean {
+    // TODO(security): allowedPaths 为空时默认允许所有路径，当前仅用于受信任网络。
+    // 开放到非受信环境前需改为默认拒绝，或要求显式配置 allowedPaths。
+    if (this.config.allowedPaths.length === 0) return true;
+    const normalized = path.resolve(resolvedPath);
+    return this.config.allowedPaths.some((allowed) => {
+      const resolvedAllowed = path.resolve(allowed);
+      return normalized === resolvedAllowed || normalized.startsWith(resolvedAllowed + path.sep);
+    });
+  }
+
+  private sendAck(to: string, sessionId: string, frameId: string, accepted: boolean, error?: string): void {
+    this.peerManager.sendTo(to, {
+      type: "file_transfer_ack",
+      id: frameId,
+      from: this.nodeId,
+      to,
+      timestamp: Date.now(),
+      payload: { sessionId, accepted, error },
+    } as FileTransferAck);
+  }
+
+  private sendNextChunk(sessionId: string): void {
+    const transfer = this.pending.get(sessionId);
+    if (!transfer || !transfer.fileData) return;
+
+    const chunkIndex = transfer.sentChunks!;
+    const start = chunkIndex * transfer.chunkSize!;
+    const end = Math.min(start + transfer.chunkSize!, transfer.fileData.length);
+    const chunk = transfer.fileData.subarray(start, end);
+
+    this.peerManager.sendTo(transfer.remoteNode, {
+      type: "file_transfer_chunk",
+      id: transfer.sessionId,
+      from: this.nodeId,
+      to: transfer.remoteNode,
+      timestamp: Date.now(),
+      payload: {
+        sessionId,
+        chunkIndex,
+        data: chunk.toString("base64"),
+      },
+    } as FileTransferChunk);
+  }
+
+  private sendChunkFromReceiving(sessionId: string, chunkIndex: number, fileData: Buffer): void {
+    const receiving = this.receiving.get(sessionId);
+    if (!receiving) return;
+
+    const start = chunkIndex * receiving.chunkSize;
+    const end = Math.min(start + receiving.chunkSize, fileData.length);
+    const chunk = fileData.subarray(start, end);
+
+    this.peerManager.sendTo(receiving.fromNode, {
+      type: "file_transfer_chunk",
+      id: sessionId,
+      from: this.nodeId,
+      to: receiving.fromNode,
+      timestamp: Date.now(),
+      payload: {
+        sessionId,
+        chunkIndex,
+        data: chunk.toString("base64"),
+      },
+    } as FileTransferChunk);
+  }
+
+  private async finalizePushReceive(sessionId: string): Promise<void> {
+    const receiving = this.receiving.get(sessionId);
+    if (!receiving) return;
+
+    // Reassemble file
+    const buffers: Buffer[] = [];
+    for (let i = 0; i < receiving.totalChunks; i++) {
+      const chunk = receiving.chunks.get(i);
+      if (!chunk) {
+        this.sendComplete(receiving.fromNode, sessionId, false, "Missing chunk " + i);
+        this.cleanupReceiving(sessionId);
+        return;
+      }
+      buffers.push(chunk);
+    }
+
+    const assembled = Buffer.concat(buffers);
+
+    // Verify checksum
+    const actualChecksum = createHash("sha256").update(assembled).digest("hex");
+    if (actualChecksum !== receiving.checksum) {
+      this.sendComplete(receiving.fromNode, sessionId, false, `Checksum mismatch: expected ${receiving.checksum}, got ${actualChecksum}`);
+      this.cleanupReceiving(sessionId);
+      return;
+    }
+
+    // Write file
+    try {
+      await mkdir(path.dirname(receiving.targetPath), { recursive: true });
+      await writeFile(receiving.targetPath, assembled);
+    } catch (err) {
+      this.sendComplete(receiving.fromNode, sessionId, false, `Write error: ${err instanceof Error ? err.message : String(err)}`);
+      this.cleanupReceiving(sessionId);
+      return;
+    }
+
+    this.sendComplete(receiving.fromNode, sessionId, true, undefined, assembled.length);
+    this.cleanupReceiving(sessionId);
+  }
+
+  private async finalizePull(sessionId: string): Promise<void> {
+    const transfer = this.pending.get(sessionId);
+    if (!transfer) return;
+
+    // Reassemble file
+    const buffers: Buffer[] = [];
+    for (let i = 0; i < transfer.expectedChunks!; i++) {
+      const chunk = transfer.chunks!.get(i);
+      if (!chunk) {
+        throw new Error(`Missing chunk ${i}`);
+      }
+      buffers.push(chunk);
+    }
+
+    const assembled = Buffer.concat(buffers);
+
+    // Verify checksum
+    const actualChecksum = createHash("sha256").update(assembled).digest("hex");
+    if (actualChecksum !== transfer.expectedChecksum) {
+      // Notify remote of failure
+      this.sendComplete(transfer.remoteNode, sessionId, false, `Checksum mismatch: expected ${transfer.expectedChecksum}, got ${actualChecksum}`);
+      this.completePending(sessionId, {
+        success: false,
+        bytesTransferred: 0,
+        error: `Checksum mismatch`,
+      });
+      return;
+    }
+
+    // Write file
+    await mkdir(path.dirname(transfer.targetPath), { recursive: true });
+    await writeFile(transfer.targetPath, assembled);
+
+    // Notify remote of success
+    this.sendComplete(transfer.remoteNode, sessionId, true, undefined, assembled.length);
+    this.completePending(sessionId, { success: true, bytesTransferred: assembled.length });
+  }
+
+  private sendComplete(to: string, sessionId: string, success: boolean, error?: string, bytesTransferred?: number): void {
+    this.peerManager.sendTo(to, {
+      type: "file_transfer_complete",
+      id: sessionId,
+      from: this.nodeId,
+      to,
+      timestamp: Date.now(),
+      payload: { sessionId, success, error, bytesTransferred },
+    } as FileTransferComplete);
+  }
+
+  private completePending(sessionId: string, result: TransferResult): void {
+    const transfer = this.pending.get(sessionId);
+    if (!transfer) return;
+    clearTimeout(transfer.timer);
+    this.pending.delete(sessionId);
+    transfer.resolve(result);
+  }
+
+  private cleanupReceiving(sessionId: string): void {
+    const receiving = this.receiving.get(sessionId);
+    if (!receiving) return;
+    clearTimeout(receiving.timer);
+    this.receiving.delete(sessionId);
+  }
+
+  private createTimer(sessionId: string, mapType: "pending" | "receiving"): ReturnType<typeof setTimeout> {
+    return setTimeout(() => {
+      if (mapType === "pending") {
+        const transfer = this.pending.get(sessionId);
+        if (transfer) {
+          this.pending.delete(sessionId);
+          transfer.reject(new Error("File transfer timeout"));
+        }
+      } else {
+        this.receiving.delete(sessionId);
+      }
+    }, this.config.timeout);
+  }
+
+  private resetTimer(sessionId: string, mapType: "pending" | "receiving"): void {
+    if (mapType === "pending") {
+      const transfer = this.pending.get(sessionId);
+      if (transfer) {
+        clearTimeout(transfer.timer);
+        transfer.timer = this.createTimer(sessionId, "pending");
+      }
+    } else {
+      const receiving = this.receiving.get(sessionId);
+      if (receiving) {
+        clearTimeout(receiving.timer);
+        receiving.timer = this.createTimer(sessionId, "receiving");
+      }
+    }
+  }
+}