@waku/sds 0.0.8-d637425.0 → 0.0.8

package/src/message_channel/repair/buffers.ts

@@ -0,0 +1,277 @@
+import { Logger } from "@waku/utils";
+
+import type { HistoryEntry, MessageId } from "../message.js";
+
+const log = new Logger("sds:repair:buffers");
+
+/**
+ * Entry in the outgoing repair buffer with request timing
+ */
+interface OutgoingBufferEntry {
+  entry: HistoryEntry;
+  tReq: number; // Timestamp when this repair request should be sent
+  requested: boolean; // Whether this repair has already been requested by the local node
+}
+
+/**
+ * Entry in the incoming repair buffer with response timing
+ */
+interface IncomingBufferEntry {
+  entry: HistoryEntry;
+  tResp: number; // Timestamp when we should respond with this repair
+}
+
+/**
+ * Buffer for outgoing repair requests (messages we need)
+ * Maintains a sorted array by T_req for efficient retrieval of eligible entries
+ */
+export class OutgoingRepairBuffer {
+  // Sorted array by T_req (ascending - earliest first)
+  private items: OutgoingBufferEntry[] = [];
+  private readonly maxSize: number;
+
+  public constructor(maxSize = 1000) {
+    this.maxSize = maxSize;
+  }
+
+  /**
+   * Add a missing message to the outgoing repair request buffer
+   * If message already exists, it is not updated (keeps original T_req)
+   * @returns true if the entry was added, false if it already existed
+   */
+  public add(entry: HistoryEntry, tReq: number): boolean {
+    const messageId = entry.messageId;
+
+    // Check if already exists - do NOT update T_req per spec
+    if (this.has(messageId)) {
+      log.info(
+        `Message ${messageId} already in outgoing buffer, keeping original T_req`
+      );
+      return false;
+    }
+
+    // Check buffer size limit
+    if (this.items.length >= this.maxSize) {
+      // Evict furthest T_req entry (last in sorted array) to preserve repairs that need to be sent the soonest
+      const evicted = this.items.pop()!;
+      log.warn(
+        `Buffer full, evicted furthest entry ${evicted.entry.messageId} with T_req ${evicted.tReq}`
+      );
+    }
+
+    // Add new entry and re-sort
+    const newEntry: OutgoingBufferEntry = { entry, tReq, requested: false };
+    const combined = [...this.items, newEntry];
+
+    // Sort by T_req (ascending)
+    combined.sort((a, b) => a.tReq - b.tReq);
+
+    this.items = combined;
+    log.info(`Added ${messageId} to outgoing buffer with T_req: ${tReq}`);
+    return true;
+  }
+
+  /**
+   * Remove a message from the buffer (e.g., when received)
+   */
+  public remove(messageId: MessageId): void {
+    this.items = this.items.filter(
+      (item) => item.entry.messageId !== messageId
+    );
+  }
+
+  /**
+   * Get eligible repair requests (where T_req <= currentTime)
+   * Returns up to maxRequests entries from the front of the sorted array
+   * Marks returned entries as requested but keeps them in buffer until received
+   */
+  public getEligible(
+    currentTime: number = Date.now(),
+    maxRequests = 3
+  ): HistoryEntry[] {
+    const eligible: HistoryEntry[] = [];
+
+    // Iterate from front of sorted array (earliest T_req first)
+    for (const item of this.items) {
+      // Since array is sorted, once we hit an item with tReq > currentTime,
+      // all remaining items also have tReq > currentTime
+      if (item.tReq > currentTime) {
+        break;
+      }
+
+      // Only return items that haven't been requested yet
+      if (!item.requested && eligible.length < maxRequests) {
+        eligible.push(item.entry);
+        // Mark as requested so we don't request it again
+        item.requested = true;
+        log.info(
+          `Repair request for ${item.entry.messageId} is eligible and marked as requested`
+        );
+      }
+
+      // If we've found enough eligible items, exit early
+      if (eligible.length >= maxRequests) {
+        break;
+      }
+    }
+
+    return eligible;
+  }
+
+  /**
+   * Check if a message is in the buffer
+   */
+  public has(messageId: MessageId): boolean {
+    return this.items.some((item) => item.entry.messageId === messageId);
+  }
+
+  /**
+   * Get the current buffer size
+   */
+  public get size(): number {
+    return this.items.length;
+  }
+
+  /**
+   * Clear all entries
+   */
+  public clear(): void {
+    this.items = [];
+  }
+
+  /**
+   * Get all entries (for testing/debugging)
+   */
+  public getAll(): HistoryEntry[] {
+    return this.items.map((item) => item.entry);
+  }
+
+  /**
+   * Get items array directly (for testing)
+   */
+  public getItems(): OutgoingBufferEntry[] {
+    return [...this.items];
+  }
+}
+
+/**
+ * Buffer for incoming repair requests (repairs we need to send)
+ * Maintains a sorted array by T_resp for efficient retrieval of ready entries
+ */
+export class IncomingRepairBuffer {
+  // Sorted array by T_resp (ascending - earliest first)
+  private items: IncomingBufferEntry[] = [];
+  private readonly maxSize: number;
+
+  public constructor(maxSize = 1000) {
+    this.maxSize = maxSize;
+  }
+
+  /**
+   * Add a repair request that we can fulfill
+   * If message already exists, it is ignored (not updated)
+   * @returns true if the entry was added, false if it already existed
+   */
+  public add(entry: HistoryEntry, tResp: number): boolean {
+    const messageId = entry.messageId;
+
+    // Check if already exists - ignore per spec
+    if (this.has(messageId)) {
+      log.info(`Message ${messageId} already in incoming buffer, ignoring`);
+      return false;
+    }
+
+    // Check buffer size limit
+    if (this.items.length >= this.maxSize) {
+      // Evict furthest T_resp entry (last in sorted array)
+      const evicted = this.items.pop()!;
+      log.warn(
+        `Buffer full, evicted furthest entry ${evicted.entry.messageId} with T_resp ${evicted.tResp}`
+      );
+    }
+
+    // Add new entry and re-sort
+    const newEntry: IncomingBufferEntry = { entry, tResp };
+    const combined = [...this.items, newEntry];
+
+    // Sort by T_resp (ascending)
+    combined.sort((a, b) => a.tResp - b.tResp);
+
+    this.items = combined;
+    log.info(`Added ${messageId} to incoming buffer with T_resp: ${tResp}`);
+    return true;
+  }
+
+  /**
+   * Remove a message from the buffer
+   */
+  public remove(messageId: MessageId): void {
+    this.items = this.items.filter(
+      (item) => item.entry.messageId !== messageId
+    );
+  }
+
+  /**
+   * Get repairs ready to be sent (where T_resp <= currentTime)
+   * Removes and returns ready entries
+   */
+  public getReady(currentTime: number): HistoryEntry[] {
+    // Find cutoff point - first item with tResp > currentTime
+    // Since array is sorted, all items before this are ready
+    let cutoff = 0;
+    for (let i = 0; i < this.items.length; i++) {
+      if (this.items[i].tResp > currentTime) {
+        cutoff = i;
+        break;
+      }
+      // If we reach the end, all items are ready
+      cutoff = i + 1;
+    }
+
+    // Extract ready items and log them
+    const ready = this.items.slice(0, cutoff).map((item) => {
+      log.info(`Repair for ${item.entry.messageId} is ready to be sent`);
+      return item.entry;
+    });
+
+    // Keep only items after cutoff
+    this.items = this.items.slice(cutoff);
+
+    return ready;
+  }
+
+  /**
+   * Check if a message is in the buffer
+   */
+  public has(messageId: MessageId): boolean {
+    return this.items.some((item) => item.entry.messageId === messageId);
+  }
+
+  /**
+   * Get the current buffer size
+   */
+  public get size(): number {
+    return this.items.length;
+  }
+
+  /**
+   * Clear all entries
+   */
+  public clear(): void {
+    this.items = [];
+  }
+
+  /**
+   * Get all entries (for testing/debugging)
+   */
+  public getAll(): HistoryEntry[] {
+    return this.items.map((item) => item.entry);
+  }
+
+  /**
+   * Get items array directly (for testing)
+   */
+  public getItems(): IncomingBufferEntry[] {
+    return [...this.items];
+  }
+}

package/src/message_channel/repair/repair.ts

@@ -0,0 +1,331 @@
+import { Logger } from "@waku/utils";
+
+import type { HistoryEntry, MessageId } from "../message.js";
+import { Message } from "../message.js";
+import type { ILocalHistory } from "../message_channel.js";
+
+import { IncomingRepairBuffer, OutgoingRepairBuffer } from "./buffers.js";
+import {
+  bigintToNumber,
+  calculateXorDistance,
+  combinedHash,
+  hashString,
+  ParticipantId
+} from "./utils.js";
+
+const log = new Logger("sds:repair:manager");
+
+/**
+ * Per SDS-R spec: One response group per 128 participants
+ */
+const PARTICIPANTS_PER_RESPONSE_GROUP = 128;
+
+/**
+ * Event emitter callback for repair events
+ */
+export type RepairEventEmitter = (event: string, detail: unknown) => void;
+
+/**
+ * Configuration for SDS-R repair protocol
+ */
+export interface RepairConfig {
+  /** Minimum wait time before requesting repair (milliseconds) */
+  tMin?: number;
+  /** Maximum wait time for repair window (milliseconds) */
+  tMax?: number;
+  /** Number of response groups for load distribution */
+  numResponseGroups?: number;
+  /** Maximum buffer size for repair requests */
+  bufferSize?: number;
+}
+
+/**
+ * Default configuration values based on spec recommendations
+ */
+export const DEFAULT_REPAIR_CONFIG: Required<RepairConfig> = {
+  tMin: 30000, // 30 seconds
+  tMax: 120000, // 120 seconds
+  numResponseGroups: 1, // Recommendation is 1 group per PARTICIPANTS_PER_RESPONSE_GROUP participants
+  bufferSize: 1000
+};
+
+/**
+ * Manager for SDS-R repair protocol
+ * Handles repair request/response timing and coordination
+ */
+export class RepairManager {
+  private readonly participantId: ParticipantId;
+  private readonly config: Required<RepairConfig>;
+  private readonly outgoingBuffer: OutgoingRepairBuffer;
+  private readonly incomingBuffer: IncomingRepairBuffer;
+  private readonly eventEmitter?: RepairEventEmitter;
+
+  public constructor(
+    participantId: ParticipantId,
+    config: RepairConfig = {},
+    eventEmitter?: RepairEventEmitter
+  ) {
+    this.participantId = participantId;
+    this.config = { ...DEFAULT_REPAIR_CONFIG, ...config };
+    this.eventEmitter = eventEmitter;
+
+    this.outgoingBuffer = new OutgoingRepairBuffer(this.config.bufferSize);
+    this.incomingBuffer = new IncomingRepairBuffer(this.config.bufferSize);
+
+    log.info(`RepairManager initialized for participant ${participantId}`);
+  }
+
+  /**
+   * Calculate T_req - when to request repair for a missing message
+   * Per spec: T_req = current_time + hash(participant_id, message_id) % (T_max - T_min) + T_min
+   */
+  public calculateTReq(messageId: MessageId, currentTime = Date.now()): number {
+    const hash = combinedHash(this.participantId, messageId);
+    const range = BigInt(this.config.tMax - this.config.tMin);
+    const offset = bigintToNumber(hash % range) + this.config.tMin;
+    return currentTime + offset;
+  }
+
+  /**
+   * Calculate T_resp - when to respond with a repair
+   * Per spec: T_resp = current_time + (distance * hash(message_id)) % T_max
+   * where distance = participant_id XOR sender_id
+   */
+  public calculateTResp(
+    senderId: ParticipantId,
+    messageId: MessageId,
+    currentTime = Date.now()
+  ): number {
+    const distance = calculateXorDistance(this.participantId, senderId);
+    const messageHash = hashString(messageId);
+    const product = distance * messageHash;
+    const offset = bigintToNumber(product % BigInt(this.config.tMax));
+    return currentTime + offset;
+  }
+
+  /**
+   * Determine if this participant is in the response group for a message
+   * Per spec: (hash(participant_id, message_id) % num_response_groups) ==
+   *           (hash(sender_id, message_id) % num_response_groups)
+   */
+  public isInResponseGroup(
+    senderId: ParticipantId,
+    messageId: MessageId
+  ): boolean {
+    if (!senderId) {
+      // Cannot determine response group without sender_id
+      return false;
+    }
+
+    const numGroups = BigInt(this.config.numResponseGroups);
+    if (numGroups <= BigInt(1)) {
+      // Single group, everyone is in it
+      return true;
+    }
+
+    const participantGroup =
+      combinedHash(this.participantId, messageId) % numGroups;
+    const senderGroup = combinedHash(senderId, messageId) % numGroups;
+
+    return participantGroup === senderGroup;
+  }
+
+  /**
+   * Handle missing dependencies by adding them to outgoing repair buffer
+   * Called when causal dependencies are detected as missing
+   */
+  public markDependenciesMissing(
+    missingEntries: HistoryEntry[],
+    currentTime = Date.now()
+  ): void {
+    for (const entry of missingEntries) {
+      // Calculate when to request this repair
+      const tReq = this.calculateTReq(entry.messageId, currentTime);
+
+      // Add to outgoing buffer - only log and emit event if actually added
+      const wasAdded = this.outgoingBuffer.add(entry, tReq);
+
+      if (wasAdded) {
+        log.info(
+          `Added missing dependency ${entry.messageId} to repair buffer with T_req=${tReq}`
+        );
+
+        // Emit event
+        this.eventEmitter?.("RepairRequestQueued", {
+          messageId: entry.messageId,
+          tReq
+        });
+      }
+    }
+  }
+
+  /**
+   * Handle receipt of a message - remove from repair buffers
+   * Called when a message is successfully received
+   */
+  public markMessageReceived(messageId: MessageId): void {
+    // Remove from both buffers as we no longer need to request or respond
+    const wasInOutgoing = this.outgoingBuffer.has(messageId);
+    const wasInIncoming = this.incomingBuffer.has(messageId);
+
+    if (wasInOutgoing) {
+      this.outgoingBuffer.remove(messageId);
+      log.info(
+        `Removed ${messageId} from outgoing repair buffer after receipt`
+      );
+    }
+
+    if (wasInIncoming) {
+      this.incomingBuffer.remove(messageId);
+      log.info(
+        `Removed ${messageId} from incoming repair buffer after receipt`
+      );
+    }
+  }
+
+  /**
+   * Get repair requests that are eligible to be sent
+   * Returns up to maxRequests entries where T_req <= currentTime
+   */
+  public getRepairRequests(
+    maxRequests = 3,
+    currentTime = Date.now()
+  ): HistoryEntry[] {
+    return this.outgoingBuffer.getEligible(currentTime, maxRequests);
+  }
+
+  /**
+   * Process incoming repair requests from other participants
+   * Adds to incoming buffer if we can fulfill and are in response group
+   */
+  public processIncomingRepairRequests(
+    requests: HistoryEntry[],
+    localHistory: ILocalHistory,
+    currentTime = Date.now()
+  ): void {
+    for (const request of requests) {
+      // Remove from our own outgoing buffer (someone else is requesting it)
+      this.outgoingBuffer.remove(request.messageId);
+
+      // Check if we have this message
+      const message = localHistory.find(
+        (m) => m.messageId === request.messageId
+      );
+      if (!message) {
+        log.info(
+          `Cannot fulfill repair for ${request.messageId} - not in local history`
+        );
+        continue;
+      }
+
+      // Check if we're in the response group
+      if (!request.senderId) {
+        log.warn(
+          `Cannot determine response group for ${request.messageId} - missing sender_id`
+        );
+        continue;
+      }
+
+      if (!this.isInResponseGroup(request.senderId, request.messageId)) {
+        log.info(`Not in response group for ${request.messageId}`);
+        continue;
+      }
+
+      // Calculate when to respond
+      const tResp = this.calculateTResp(
+        request.senderId,
+        request.messageId,
+        currentTime
+      );
+
+      // Add to incoming buffer - only log and emit event if actually added
+      const wasAdded = this.incomingBuffer.add(request, tResp);
+
+      if (wasAdded) {
+        log.info(
+          `Will respond to repair request for ${request.messageId} at T_resp=${tResp}`
+        );
+
+        // Emit event
+        this.eventEmitter?.("RepairResponseQueued", {
+          messageId: request.messageId,
+          tResp
+        });
+      }
+    }
+  }
+
+  /**
+   * Sweep outgoing buffer for repairs that should be requested
+   * Returns entries where T_req <= currentTime
+   */
+  public sweepOutgoingBuffer(
+    maxRequests = 3,
+    currentTime = Date.now()
+  ): HistoryEntry[] {
+    return this.getRepairRequests(maxRequests, currentTime);
+  }
+
+  /**
+   * Sweep incoming buffer for repairs ready to be sent
+   * Returns messages that should be rebroadcast
+   */
+  public sweepIncomingBuffer(
+    localHistory: ILocalHistory,
+    currentTime = Date.now()
+  ): Message[] {
+    const ready = this.incomingBuffer.getReady(currentTime);
+    const messages: Message[] = [];
+
+    for (const entry of ready) {
+      const message = localHistory.find((m) => m.messageId === entry.messageId);
+      if (message) {
+        messages.push(message);
+        log.info(`Sending repair for ${entry.messageId}`);
+      } else {
+        log.warn(`Message ${entry.messageId} no longer in local history`);
+      }
+    }
+
+    return messages;
+  }
+
+  /**
+   * Clear all buffers
+   */
+  public clear(): void {
+    this.outgoingBuffer.clear();
+    this.incomingBuffer.clear();
+  }
+
+  /**
+   * Update number of response groups (e.g., when participants change)
+   */
+  public updateResponseGroups(numParticipants: number): void {
+    if (
+      numParticipants < 0 ||
+      !Number.isFinite(numParticipants) ||
+      !Number.isInteger(numParticipants)
+    ) {
+      throw new Error(
+        `Invalid numParticipants: ${numParticipants}. Must be a positive integer.`
+      );
+    }
+
+    if (numParticipants > Number.MAX_SAFE_INTEGER) {
+      log.warn(
+        `numParticipants ${numParticipants} exceeds MAX_SAFE_INTEGER, using MAX_SAFE_INTEGER`
+      );
+      numParticipants = Number.MAX_SAFE_INTEGER;
+    }
+
+    // Per spec: num_response_groups = max(1, num_participants / PARTICIPANTS_PER_RESPONSE_GROUP)
+    this.config.numResponseGroups = Math.max(
+      1,
+      Math.floor(numParticipants / PARTICIPANTS_PER_RESPONSE_GROUP)
+    );
+    log.info(
+      `Updated response groups to ${this.config.numResponseGroups} for ${numParticipants} participants`
+    );
+  }
+}

package/src/message_channel/repair/utils.ts

@@ -0,0 +1,80 @@
+import { sha256 } from "@noble/hashes/sha2";
+import { bytesToHex } from "@waku/utils/bytes";
+
+import type { MessageId } from "../message.js";
+
+/**
+ * ParticipantId can be a string or converted to a numeric representation for XOR operations
+ */
+export type ParticipantId = string;
+
+/**
+ * Compute SHA256 hash and convert to integer for modulo operations
+ * Uses first 8 bytes of hash for the integer conversion
+ */
+export function hashToInteger(input: string): bigint {
+  const hashBytes = sha256(new TextEncoder().encode(input));
+  // Use first 8 bytes for a 64-bit integer
+  const view = new DataView(hashBytes.buffer, 0, 8);
+  return view.getBigUint64(0, false); // big-endian
+}
+
+/**
+ * Compute combined hash for (participantId, messageId) and convert to integer
+ * This is used for T_req calculations and response group membership
+ */
+export function combinedHash(
+  participantId: ParticipantId,
+  messageId: MessageId
+): bigint {
+  const combined = `${participantId}${messageId}`;
+  return hashToInteger(combined);
+}
+
+/**
+ * Convert ParticipantId to numeric representation for XOR operations
+ * TODO: Not per spec, further review needed
+ * The spec assumes participant IDs support XOR natively, but we're using
+ * SHA256 hash to ensure consistent numeric representation for string IDs
+ */
+export function participantIdToNumeric(participantId: ParticipantId): bigint {
+  return hashToInteger(participantId);
+}
+
+/**
+ * Calculate XOR distance between two participant IDs
+ * Used for T_resp calculations where distance affects response timing
+ */
+export function calculateXorDistance(
+  participantId1: ParticipantId,
+  participantId2: ParticipantId
+): bigint {
+  const numeric1 = participantIdToNumeric(participantId1);
+  const numeric2 = participantIdToNumeric(participantId2);
+  return numeric1 ^ numeric2;
+}
+
+/**
+ * Helper to convert bigint to number for timing calculations
+ * Ensures the result fits in JavaScript's number range
+ */
+export function bigintToNumber(value: bigint): number {
+  // For timing calculations, we modulo by MAX_SAFE_INTEGER to ensure it fits
+  const maxSafe = BigInt(Number.MAX_SAFE_INTEGER);
+  return Number(value % maxSafe);
+}
+
+/**
+ * Calculate hash for a single string (used for message_id in T_resp)
+ */
+export function hashString(input: string): bigint {
+  return hashToInteger(input);
+}
+
+/**
+ * Convert a hash result to hex string for debugging/logging
+ */
+export function hashToHex(input: string): string {
+  const hashBytes = sha256(new TextEncoder().encode(input));
+  return bytesToHex(hashBytes);
+}