@hashtree/worker 0.1.26 → 0.2.1

package/src/p2p/webrtcProxy.ts

@@ -0,0 +1,519 @@
+/**
+ * WebRTC Proxy
+ *
+ * Thin transport layer that manages RTCPeerConnection in main thread.
+ * Worker controls all logic - this just executes commands and reports events.
+ *
+ * Main thread owns RTCPeerConnection because it's not available in workers.
+ * See: https://github.com/w3c/webrtc-extensions/issues/77
+ */
+
+import type { WebRTCCommand, WebRTCEvent } from './protocol.js';
+import { BoundedQueue } from './boundedQueue.js';
+import { getErrorMessage } from './errorMessage.js';
+import { UploadRateLimiter } from './uploadRateLimiter.js';
+
+const REQUEST_MESSAGE_TYPE = 0x00;
+
+const isTestMode = typeof globalThis !== 'undefined' &&
+  Boolean((globalThis as { __HTREE_P2P_TEST_MODE__?: boolean }).__HTREE_P2P_TEST_MODE__);
+const ICE_SERVERS: RTCIceServer[] = isTestMode
+  ? []
+  : [
+      { urls: 'stun:stun.l.google.com:19302' },
+      { urls: 'stun:stun.cloudflare.com:3478' },
+    ];
+
+interface PeerConnection {
+  pc: RTCPeerConnection;
+  dataChannel: RTCDataChannel | null;
+  pubkey: string;
+  pendingCandidates: RTCIceCandidateInit[];
+  sendQueue: BoundedQueue<Uint8Array>;
+  sending: boolean;
+  bufferHighSignaled: boolean;  // Track if we've signaled high buffer to worker
+  drainTimeoutId: ReturnType<typeof setTimeout> | null;
+}
+
+type EventCallback = (event: WebRTCEvent) => void;
+type WebRTCProxyConfig = {
+  maxUploadBytesPerSecond?: number | null;
+};
+
+export class WebRTCProxy {
+  private peers = new Map<string, PeerConnection>();
+  private onEvent: EventCallback;
+  private readonly uploadRateLimiter: UploadRateLimiter;
+
+  // Queue limits to prevent memory blowup on slow/stalled connections
+  private static readonly MAX_QUEUE_BYTES = 8 * 1024 * 1024;  // 8MB per peer
+  private static readonly MAX_QUEUE_ITEMS = 100;
+
+  constructor(onEvent: EventCallback, config: WebRTCProxyConfig = {}) {
+    this.onEvent = onEvent;
+    this.uploadRateLimiter = new UploadRateLimiter({
+      bytesPerSecond: config.maxUploadBytesPerSecond,
+    });
+  }
+
+  private createSendQueue(peerId: string): BoundedQueue<Uint8Array> {
+    return new BoundedQueue<Uint8Array>({
+      maxItems: WebRTCProxy.MAX_QUEUE_ITEMS,
+      maxBytes: WebRTCProxy.MAX_QUEUE_BYTES,
+      getBytes: (item) => item.byteLength,
+      onDrop: (item) => {
+        console.warn(`[WebRTCProxy] Queue overflow for ${peerId.slice(0, 8)}, dropped ${item.byteLength}B`);
+      },
+    });
+  }
+
+  /**
+   * Handle command from worker
+   */
+  handleCommand(cmd: WebRTCCommand): void {
+    switch (cmd.type) {
+      case 'rtc:createPeer':
+        this.createPeer(cmd.peerId, cmd.pubkey);
+        break;
+      case 'rtc:closePeer':
+        this.closePeer(cmd.peerId);
+        break;
+      case 'rtc:createOffer':
+        this.createOffer(cmd.peerId);
+        break;
+      case 'rtc:createAnswer':
+        this.createAnswer(cmd.peerId);
+        break;
+      case 'rtc:setLocalDescription':
+        this.setLocalDescription(cmd.peerId, cmd.sdp);
+        break;
+      case 'rtc:setRemoteDescription':
+        this.setRemoteDescription(cmd.peerId, cmd.sdp);
+        break;
+      case 'rtc:addIceCandidate':
+        this.addIceCandidate(cmd.peerId, cmd.candidate);
+        break;
+      case 'rtc:sendData':
+        this.sendData(cmd.peerId, cmd.data);
+        break;
+    }
+  }
+
+  private createPeer(peerId: string, pubkey: string): void {
+    // Clean up existing if present
+    if (this.peers.has(peerId)) {
+      this.closePeer(peerId);
+    }
+
+    const pc = new RTCPeerConnection({ iceServers: ICE_SERVERS });
+
+    const peer: PeerConnection = {
+      pc,
+      dataChannel: null,
+      pubkey,
+      pendingCandidates: [],
+      sendQueue: this.createSendQueue(peerId),
+      sending: false,
+      bufferHighSignaled: false,
+      drainTimeoutId: null,
+    };
+
+    // Create data channel (offerer creates, answerer receives via ondatachannel)
+    const dc = pc.createDataChannel('hashtree', {
+      ordered: true,
+    });
+    this.setupDataChannel(peerId, dc);
+    peer.dataChannel = dc;
+
+    // Handle incoming data channel (for answerer)
+    pc.ondatachannel = (event) => {
+      this.setupDataChannel(peerId, event.channel);
+      peer.dataChannel = event.channel;
+    };
+
+    // ICE candidate gathering
+    pc.onicecandidate = (event) => {
+      this.onEvent({
+        type: 'rtc:iceCandidate',
+        peerId,
+        candidate: event.candidate?.toJSON() ?? null,
+      });
+
+      if (!event.candidate) {
+        this.onEvent({ type: 'rtc:iceGatheringComplete', peerId });
+      }
+    };
+
+    // Connection state changes
+    pc.onconnectionstatechange = () => {
+      this.onEvent({
+        type: 'rtc:peerStateChange',
+        peerId,
+        state: pc.connectionState,
+      });
+
+      if (pc.connectionState === 'closed' || pc.connectionState === 'failed') {
+        this.cleanupPeer(peerId);
+      }
+    };
+
+    this.peers.set(peerId, peer);
+    this.onEvent({ type: 'rtc:peerCreated', peerId });
+  }
+
+  private setupDataChannel(peerId: string, dc: RTCDataChannel): void {
+    dc.binaryType = 'arraybuffer';
+
+    dc.onopen = () => {
+      this.onEvent({ type: 'rtc:dataChannelOpen', peerId });
+    };
+
+    // If channel is already open (can happen with ondatachannel), fire event immediately
+    if (dc.readyState === 'open') {
+      this.onEvent({ type: 'rtc:dataChannelOpen', peerId });
+    }
+
+    dc.onclose = () => {
+      this.onEvent({ type: 'rtc:dataChannelClose', peerId });
+    };
+
+    dc.onerror = (event) => {
+      const errorEvent = event as RTCErrorEvent;
+      this.onEvent({
+        type: 'rtc:dataChannelError',
+        peerId,
+        error: errorEvent.error?.message || 'Unknown error',
+      });
+    };
+
+    dc.onmessage = (event) => {
+      const data = event.data instanceof ArrayBuffer
+        ? new Uint8Array(event.data)
+        : new Uint8Array(0);
+
+      this.onEvent({
+        type: 'rtc:dataChannelMessage',
+        peerId,
+        data,
+      });
+    };
+  }
+
+  private async createOffer(peerId: string): Promise<void> {
+    const peer = this.peers.get(peerId);
+    if (!peer) return;
+
+    try {
+      const offer = await peer.pc.createOffer();
+      this.onEvent({
+        type: 'rtc:offerCreated',
+        peerId,
+        sdp: offer,
+      });
+    } catch (err) {
+      console.error('[WebRTCProxy] Failed to create offer:', err);
+    }
+  }
+
+  private async createAnswer(peerId: string): Promise<void> {
+    const peer = this.peers.get(peerId);
+    if (!peer) return;
+
+    try {
+      const answer = await peer.pc.createAnswer();
+      this.onEvent({
+        type: 'rtc:answerCreated',
+        peerId,
+        sdp: answer,
+      });
+    } catch (err) {
+      console.error('[WebRTCProxy] Failed to create answer:', err);
+    }
+  }
+
+  private async setLocalDescription(peerId: string, sdp: RTCSessionDescriptionInit): Promise<void> {
+    const peer = this.peers.get(peerId);
+    if (!peer) return;
+
+    try {
+      await peer.pc.setLocalDescription(sdp);
+      this.onEvent({ type: 'rtc:descriptionSet', peerId });
+    } catch (err) {
+      this.onEvent({
+        type: 'rtc:descriptionSet',
+        peerId,
+        error: getErrorMessage(err),
+      });
+    }
+  }
+
+  private async setRemoteDescription(peerId: string, sdp: RTCSessionDescriptionInit): Promise<void> {
+    const peer = this.peers.get(peerId);
+    if (!peer) return;
+
+    try {
+      if (sdp.type === 'offer'
+        && peer.pc.signalingState !== 'stable'
+        && peer.pc.signalingState !== 'closed') {
+        await peer.pc.setLocalDescription({ type: 'rollback' });
+      }
+
+      await peer.pc.setRemoteDescription(sdp);
+
+      // Apply any pending ICE candidates
+      for (const candidate of peer.pendingCandidates) {
+        await peer.pc.addIceCandidate(candidate);
+      }
+      peer.pendingCandidates = [];
+
+      this.onEvent({ type: 'rtc:descriptionSet', peerId });
+    } catch (err) {
+      this.onEvent({
+        type: 'rtc:descriptionSet',
+        peerId,
+        error: getErrorMessage(err),
+      });
+    }
+  }
+
+  private async addIceCandidate(peerId: string, candidate: RTCIceCandidateInit): Promise<void> {
+    const peer = this.peers.get(peerId);
+    if (!peer) return;
+
+    // Queue if remote description not set yet
+    if (!peer.pc.remoteDescription) {
+      peer.pendingCandidates.push(candidate);
+      return;
+    }
+
+    try {
+      await peer.pc.addIceCandidate(candidate);
+    } catch (err) {
+      console.error('[WebRTCProxy] Failed to add ICE candidate:', err);
+    }
+  }
+
+  // 256KB threshold - pause sending when buffer exceeds this
+  private static readonly BUFFER_THRESHOLD = 256 * 1024;
+  // 4MB threshold for sendQueue - signal worker to pause when exceeded
+  private static readonly QUEUE_HIGH_THRESHOLD = 4 * 1024 * 1024;
+  // 1MB threshold for sendQueue - signal worker to resume when below
+  private static readonly QUEUE_LOW_THRESHOLD = 1 * 1024 * 1024;
+
+  private getQueueSize(peer: PeerConnection): number {
+    return peer.sendQueue.bytes;
+  }
+
+  private isPriorityDataMessage(data: Uint8Array): boolean {
+    return data.byteLength > 0 && data[0] === REQUEST_MESSAGE_TYPE;
+  }
+
+  private sendData(peerId: string, data: Uint8Array): void {
+    const peer = this.peers.get(peerId);
+    if (!peer?.dataChannel || peer.dataChannel.readyState !== 'open') {
+      return;
+    }
+
+    // Small request frames should overtake bulky response traffic so cache misses
+    // are not starved by background uploads on the same peer connection.
+    if (this.isPriorityDataMessage(data)) {
+      peer.sendQueue.unshift(data);
+    } else {
+      peer.sendQueue.push(data);
+    }
+
+    // Check if queue is getting too large - signal worker to slow down
+    const queueSize = this.getQueueSize(peer);
+    if (!peer.bufferHighSignaled && queueSize > WebRTCProxy.QUEUE_HIGH_THRESHOLD) {
+      peer.bufferHighSignaled = true;
+      this.onEvent({ type: 'rtc:bufferHigh', peerId });
+    }
+
+    // Start draining if not already
+    if (!peer.sending) {
+      this.drainQueue(peerId);
+    }
+  }
+
+  private drainQueue(peerId: string): void {
+    const peer = this.peers.get(peerId);
+    if (!peer?.dataChannel || peer.dataChannel.readyState !== 'open') {
+      return;
+    }
+
+    if (peer.drainTimeoutId) {
+      clearTimeout(peer.drainTimeoutId);
+      peer.drainTimeoutId = null;
+    }
+
+    peer.sending = true;
+    const dc = peer.dataChannel;
+
+    // Send as much as we can without overflowing the buffer
+    while (!peer.sendQueue.isEmpty && dc.bufferedAmount < WebRTCProxy.BUFFER_THRESHOLD) {
+      const nextData = peer.sendQueue.peek();
+      if (!nextData) {
+        break;
+      }
+
+      const reservation = this.uploadRateLimiter.reserve(nextData.byteLength);
+      if (!reservation.allowed) {
+        peer.sending = false;
+        this.scheduleRateLimitedDrain(peerId, reservation.delayMs);
+        return;
+      }
+
+      const data = peer.sendQueue.shift()!;
+      try {
+        const payload = data.byteOffset === 0 && data.byteLength === data.buffer.byteLength
+          ? data.buffer
+          : data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength);
+        dc.send(payload as ArrayBuffer);
+      } catch {
+        // Drop on failure instead of infinite re-queue (prevents memory blowup)
+        console.warn(`[WebRTCProxy] Send failed for ${peerId.slice(0, 8)}, dropped ${data.byteLength}B`);
+        break;
+      }
+    }
+
+    // Check if queue has drained enough to signal worker to resume
+    if (peer.bufferHighSignaled) {
+      const queueSize = this.getQueueSize(peer);
+      if (queueSize < WebRTCProxy.QUEUE_LOW_THRESHOLD) {
+        peer.bufferHighSignaled = false;
+        this.onEvent({ type: 'rtc:bufferLow', peerId });
+      }
+    }
+
+    // If there's more to send, wait for buffer to drain
+    if (!peer.sendQueue.isEmpty) {
+      dc.bufferedAmountLowThreshold = WebRTCProxy.BUFFER_THRESHOLD / 2;
+      dc.onbufferedamountlow = () => {
+        dc.onbufferedamountlow = null;
+        this.drainQueue(peerId);
+      };
+    } else {
+      peer.sending = false;
+    }
+  }
+
+  private closePeer(peerId: string): void {
+    const peer = this.peers.get(peerId);
+    if (!peer) return;
+
+    this.cleanupPeer(peerId);
+    this.onEvent({ type: 'rtc:peerClosed', peerId });
+  }
+
+  private cleanupPeer(peerId: string): void {
+    const peer = this.peers.get(peerId);
+    if (!peer) return;
+
+    // Clear send queue
+    peer.sendQueue.clear();
+    peer.sending = false;
+    if (peer.drainTimeoutId) {
+      clearTimeout(peer.drainTimeoutId);
+      peer.drainTimeoutId = null;
+    }
+
+    // Close data channel
+    if (peer.dataChannel) {
+      peer.dataChannel.onopen = null;
+      peer.dataChannel.onclose = null;
+      peer.dataChannel.onerror = null;
+      peer.dataChannel.onmessage = null;
+      peer.dataChannel.onbufferedamountlow = null;
+      peer.dataChannel.close();
+    }
+
+    // Close peer connection
+    peer.pc.onicecandidate = null;
+    peer.pc.ondatachannel = null;
+    peer.pc.onconnectionstatechange = null;
+    peer.pc.close();
+
+    this.peers.delete(peerId);
+  }
+
+  /**
+   * Close all connections
+   */
+  close(): void {
+    for (const peerId of this.peers.keys()) {
+      this.closePeer(peerId);
+    }
+  }
+
+  /**
+   * Get connected peer count
+   */
+  getConnectedCount(): number {
+    let count = 0;
+    for (const peer of this.peers.values()) {
+      if (peer.pc.connectionState === 'connected' &&
+          peer.dataChannel?.readyState === 'open') {
+        count++;
+      }
+    }
+    return count;
+  }
+
+  /**
+   * Get all peer IDs
+   */
+  getPeerIds(): string[] {
+    return Array.from(this.peers.keys());
+  }
+
+  setUploadLimitBytesPerSecond(maxUploadBytesPerSecond?: number | null): void {
+    this.uploadRateLimiter.setBytesPerSecond(maxUploadBytesPerSecond);
+    for (const peerId of this.peers.keys()) {
+      const peer = this.peers.get(peerId);
+      if (!peer || peer.sendQueue.isEmpty) {
+        continue;
+      }
+      if (!peer.sending) {
+        this.drainQueue(peerId);
+      }
+    }
+  }
+
+  private scheduleRateLimitedDrain(peerId: string, delayMs: number): void {
+    const peer = this.peers.get(peerId);
+    if (!peer || peer.drainTimeoutId) {
+      return;
+    }
+
+    peer.drainTimeoutId = setTimeout(() => {
+      const activePeer = this.peers.get(peerId);
+      if (!activePeer) {
+        return;
+      }
+      activePeer.drainTimeoutId = null;
+      this.drainQueue(peerId);
+    }, delayMs);
+  }
+}
+
+// Singleton instance
+let instance: WebRTCProxy | null = null;
+
+export function initWebRTCProxy(onEvent: EventCallback): WebRTCProxy {
+  if (instance) {
+    instance.close();
+  }
+  instance = new WebRTCProxy(onEvent);
+  return instance;
+}
+
+export function getWebRTCProxy(): WebRTCProxy | null {
+  return instance;
+}
+
+export function closeWebRTCProxy(): void {
+  if (instance) {
+    instance.close();
+    instance = null;
+  }
+}

package/src/privacyGuards.ts

@@ -0,0 +1,31 @@
+import type { CID } from '@hashtree/core';
+
+const ENCRYPTION_KEY_BYTES = 32;
+
+/**
+ * Uploads must always point to encrypted content CIDs.
+ */
+export function assertEncryptedUploadCid(cid: CID): void {
+  if (!cid.key) {
+    throw new Error('Refusing to upload unencrypted CID');
+  }
+  if (cid.key.length !== ENCRYPTION_KEY_BYTES) {
+    throw new Error('Refusing to upload CID with invalid encryption key');
+  }
+}
+
+/**
+ * Mark known-encrypted block hashes as safe for peer serving.
+ */
+export function markEncryptedHashes(hashes: Iterable<string>, allowlist: Set<string>): void {
+  for (const hashHex of hashes) {
+    allowlist.add(hashHex.toLowerCase());
+  }
+}
+
+/**
+ * Peer responses are restricted to hashes explicitly marked as encrypted.
+ */
+export function shouldServeHashToPeer(hashHex: string, allowlist: ReadonlySet<string>): boolean {
+  return allowlist.has(hashHex.toLowerCase());
+}

package/src/protocol.ts

@@ -0,0 +1,124 @@
+import type { CID } from '@hashtree/core';
+
+export interface BlossomServerConfig {
+  url: string;
+  read?: boolean;
+  write?: boolean;
+}
+
+export interface WorkerConfig {
+  storeName?: string;
+  blossomServers?: BlossomServerConfig[];
+  relays?: string[];
+  storageMaxBytes?: number;
+  connectivityProbeIntervalMs?: number;
+  diagnosticsEnabled?: boolean;
+  diagnosticsMirrorToConsole?: boolean;
+}
+
+export type WorkerDiagnosticLevel = 'debug' | 'info' | 'warn' | 'error';
+export type WorkerDiagnosticDataValue = string | number | boolean | null;
+
+export interface WorkerDiagnosticEvent {
+  scope: string;
+  code: string;
+  level: WorkerDiagnosticLevel;
+  message: string;
+  timestamp: number;
+  data?: Record<string, WorkerDiagnosticDataValue>;
+}
+
+export interface ConnectivityState {
+  online: boolean;
+  reachableReadServers: number;
+  totalReadServers: number;
+  reachableWriteServers: number;
+  totalWriteServers: number;
+  updatedAt: number;
+}
+
+export type BlobSource = 'idb' | 'blossom' | 'p2p';
+
+export interface UploadServerStatus {
+  url: string;
+  uploaded: number;
+  skipped: number;
+  failed: number;
+}
+
+export interface UploadProgressState {
+  hashHex: string;
+  nhash: string;
+  totalServers: number;
+  processedServers: number;
+  uploadedServers: number;
+  skippedServers: number;
+  failedServers: number;
+  totalChunks?: number;
+  processedChunks?: number;
+  /** 0..1 normalized progress for chunk upload traversal */
+  progressRatio?: number;
+  serverStatuses?: UploadServerStatus[];
+  complete: boolean;
+  error?: string;
+}
+
+export interface BlossomBandwidthServerStats {
+  url: string;
+  bytesSent: number;
+  bytesReceived: number;
+}
+
+export interface BlossomBandwidthState {
+  totalBytesSent: number;
+  totalBytesReceived: number;
+  updatedAt: number;
+  servers: BlossomBandwidthServerStats[];
+}
+
+export interface BlobStreamStarted {
+  id: string;
+  streamId: string;
+}
+
+export interface RootResolveOptions {
+  timeoutMs?: number;
+  settleMs?: number;
+}
+
+export type WorkerRequest =
+  | { type: 'init'; id: string; config: WorkerConfig }
+  | { type: 'close'; id: string }
+  | { type: 'putBlob'; id: string; data: Uint8Array; mimeType?: string; upload?: boolean }
+  | { type: 'beginPutBlobStream'; id: string; mimeType?: string; upload?: boolean }
+  | { type: 'appendPutBlobStream'; id: string; streamId: string; chunk: Uint8Array }
+  | { type: 'finishPutBlobStream'; id: string; streamId: string }
+  | { type: 'cancelPutBlobStream'; id: string; streamId: string }
+  | { type: 'p2pFetchResult'; id: string; requestId: string; data?: Uint8Array; error?: string }
+  | { type: 'getBlob'; id: string; hashHex: string; forPeer?: boolean }
+  | { type: 'registerMediaPort'; id: string; port: MessagePort }
+  | { type: 'setBlossomServers'; id: string; servers: BlossomServerConfig[] }
+  | { type: 'setStorageMaxBytes'; id: string; maxBytes: number }
+  | { type: 'getStorageStats'; id: string }
+  | { type: 'probeConnectivity'; id: string }
+  | { type: 'resolveRoot'; id: string; npub: string; path?: string; timeoutMs?: number; settleMs?: number }
+  | { type: 'watchRoot'; id: string; npub: string; path?: string; timeoutMs?: number; settleMs?: number }
+  | { type: 'unwatchRoot'; id: string; watchId: string };
+
+export type WorkerResponse =
+  | { type: 'ready'; id: string }
+  | { type: 'error'; id?: string; error: string }
+  | { type: 'diagnostic'; event: WorkerDiagnosticEvent }
+  | { type: 'p2pFetch'; requestId: string; hashHex: string }
+  | { type: 'blobStreamStarted'; id: string; streamId: string }
+  | { type: 'blobStored'; id: string; hashHex: string; nhash: string }
+  | { type: 'blob'; id: string; data?: Uint8Array; source?: BlobSource; error?: string }
+  | { type: 'cid'; id: string; cid?: CID; error?: string }
+  | { type: 'void'; id: string; error?: string }
+  | { type: 'storageStats'; id: string; items: number; bytes: number; maxBytes: number; error?: string }
+  | { type: 'connectivity'; id: string; state?: ConnectivityState; error?: string }
+  | { type: 'rootWatchStarted'; id: string; watchId: string; cid?: CID; error?: string }
+  | { type: 'rootUpdate'; watchId: string; cid?: CID }
+  | { type: 'connectivityUpdate'; state: ConnectivityState }
+  | { type: 'blossomBandwidth'; stats: BlossomBandwidthState }
+  | { type: 'uploadProgress'; progress: UploadProgressState };