@waku/sds 0.0.8-d637425.0 → 0.0.8

package/package.json

@@ -1 +1,93 @@
-{"name":"@waku/sds","version":"0.0.8-d637425.0","description":"Scalable Data Sync implementation for the browser. Based on https://github.com/vacp2p/rfc-index/blob/main/vac/raw/sds.md","types":"./dist/index.d.ts","module":"./dist/index.js","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js"}},"typesVersions":{"*":{"*":["*","dist/*","dist/*/index"]}},"type":"module","author":"Waku Team","homepage":"https://github.com/waku-org/js-waku/tree/master/packages/scalable-data-sync#readme","repository":{"type":"git","url":"https://github.com/waku-org/js-waku.git"},"bugs":{"url":"https://github.com/waku-org/js-waku/issues"},"license":"MIT OR Apache-2.0","keywords":["waku","decentralized","secure","communication","web3","ethereum","dapps","privacy"],"scripts":{"build":"run-s build:**","build:esm":"tsc","build:bundle":"rollup --config rollup.config.js","fix":"run-s fix:*","fix:lint":"eslint src *.js --fix","check":"run-s check:*","check:lint":"eslint src *.js","check:spelling":"cspell \"{README.md,src/**/*.ts}\"","check:tsc":"tsc -p tsconfig.dev.json","prepublish":"npm run build","reset-hard":"git clean -dfx -e .idea && git reset --hard && npm i && npm run build","test":"NODE_ENV=test run-s test:*","test:node":"NODE_ENV=test TS_NODE_PROJECT=./tsconfig.dev.json mocha","test:browser":"NODE_ENV=test karma start karma.conf.cjs"},"engines":{"node":">=22"},"dependencies":{"@libp2p/interface":"2.10.4","@noble/hashes":"^1.7.1","@waku/proto":"0.0.15-d637425.0","@waku/utils":"0.0.28-d637425.0","chai":"^5.1.2","lodash":"^4.17.21"},"devDependencies":{"@rollup/plugin-commonjs":"^25.0.7","@rollup/plugin-json":"^6.0.0","@rollup/plugin-node-resolve":"^15.2.3","@waku/build-utils":"*","allure-commandline":"^2.27.0","allure-mocha":"^2.9.2","cspell":"^8.6.1","fast-check":"^3.19.0","mocha-multi-reporters":"^1.5.1","npm-run-all":"^4.1.5","rollup":"^4.12.0"},"files":["dist","bundle","src/**/*.ts","!**/*.spec.*","!**/*.json","CHANGELOG.md","LICENSE","README.md"]}
+{
+    "name": "@waku/sds",
+    "version": "0.0.8",
+    "description": "Scalable Data Sync implementation for the browser. Based on https://github.com/vacp2p/rfc-index/blob/main/vac/raw/sds.md",
+    "types": "./dist/index.d.ts",
+    "module": "./dist/index.js",
+    "exports": {
+        ".": {
+            "types": "./dist/index.d.ts",
+            "import": "./dist/index.js"
+        }
+    },
+    "typesVersions": {
+        "*": {
+            "*": [
+                "*",
+                "dist/*",
+                "dist/*/index"
+            ]
+        }
+    },
+    "type": "module",
+    "author": "Waku Team",
+    "homepage": "https://github.com/waku-org/js-waku/tree/master/packages/scalable-data-sync#readme",
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/waku-org/js-waku.git"
+    },
+    "bugs": {
+        "url": "https://github.com/waku-org/js-waku/issues"
+    },
+    "license": "MIT OR Apache-2.0",
+    "keywords": [
+        "waku",
+        "decentralized",
+        "secure",
+        "communication",
+        "web3",
+        "ethereum",
+        "dapps",
+        "privacy"
+    ],
+    "scripts": {
+        "build": "run-s build:**",
+        "build:esm": "tsc",
+        "build:bundle": "rollup --config rollup.config.js",
+        "fix": "run-s fix:*",
+        "fix:lint": "eslint src *.js --fix",
+        "check": "run-s check:*",
+        "check:lint": "eslint src *.js",
+        "check:spelling": "cspell \"{README.md,src/**/*.ts}\"",
+        "check:tsc": "tsc -p tsconfig.dev.json",
+        "prepublish": "npm run build",
+        "reset-hard": "git clean -dfx -e .idea && git reset --hard && npm i && npm run build",
+        "test": "NODE_ENV=test run-s test:*",
+        "test:node": "NODE_ENV=test TS_NODE_PROJECT=./tsconfig.dev.json mocha",
+        "test:browser": "NODE_ENV=test karma start karma.conf.cjs"
+    },
+    "engines": {
+        "node": ">=22"
+    },
+    "dependencies": {
+        "@libp2p/interface": "2.10.4",
+        "@noble/hashes": "^1.7.1",
+        "@waku/proto": "^0.0.15",
+        "@waku/utils": "^0.0.27",
+        "chai": "^5.1.2",
+        "lodash": "^4.17.21"
+    },
+    "devDependencies": {
+        "@rollup/plugin-commonjs": "^25.0.7",
+        "@rollup/plugin-json": "^6.0.0",
+        "@rollup/plugin-node-resolve": "^15.2.3",
+        "@waku/build-utils": "*",
+        "allure-commandline": "^2.27.0",
+        "allure-mocha": "^2.9.2",
+        "cspell": "^8.6.1",
+        "fast-check": "^3.19.0",
+        "mocha-multi-reporters": "^1.5.1",
+        "npm-run-all": "^4.1.5",
+        "rollup": "^4.12.0"
+    },
+    "files": [
+        "dist",
+        "bundle",
+        "src/**/*.ts",
+        "!**/*.spec.*",
+        "!**/*.json",
+        "CHANGELOG.md",
+        "LICENSE",
+        "README.md"
+    ]
+}

package/src/index.ts

@@ -14,8 +14,14 @@ export {
   type HistoryEntry,
   type ChannelId,
   type MessageChannelEvents,
-  type SenderId,
+  type ParticipantId,
   type MessageId
 } from "./message_channel/index.js";
 
+/**
+ * @deprecated Use ParticipantId instead. SenderId has been renamed to ParticipantId
+ * to better reflect that it represents a channel participant, not just a message sender.
+ */
+export type { ParticipantId as SenderId } from "./message_channel/index.js";
+
 export { BloomFilter };

package/src/message_channel/events.ts

@@ -1,4 +1,4 @@
-import { HistoryEntry, Message, MessageId } from "./message.js";
+import { HistoryEntry, Message, MessageId, ParticipantId } from "./message.js";
 
 export enum MessageChannelEvent {
   OutMessageSent = "sds:out:message-sent",
@@ -10,7 +10,13 @@ export enum MessageChannelEvent {
   OutSyncSent = "sds:out:sync-sent",
   InSyncReceived = "sds:in:sync-received",
   InMessageLost = "sds:in:message-irretrievably-lost",
-  ErrorTask = "sds:error-task"
+  ErrorTask = "sds:error-task",
+  // SDS-R Repair Events
+  RepairRequestQueued = "sds:repair:request-queued",
+  RepairRequestSent = "sds:repair:request-sent",
+  RepairRequestReceived = "sds:repair:request-received",
+  RepairResponseQueued = "sds:repair:response-queued",
+  RepairResponseSent = "sds:repair:response-sent"
 }
 
 export type MessageChannelEvents = {
@@ -26,5 +32,24 @@ export type MessageChannelEvents = {
   [MessageChannelEvent.InMessageLost]: CustomEvent<HistoryEntry[]>;
   [MessageChannelEvent.OutSyncSent]: CustomEvent<Message>;
   [MessageChannelEvent.InSyncReceived]: CustomEvent<Message>;
-  [MessageChannelEvent.ErrorTask]: CustomEvent<any>;
+  [MessageChannelEvent.ErrorTask]: CustomEvent<unknown>;
+  [MessageChannelEvent.RepairRequestQueued]: CustomEvent<{
+    messageId: MessageId;
+    tReq: number;
+  }>;
+  [MessageChannelEvent.RepairRequestSent]: CustomEvent<{
+    messageIds: MessageId[];
+    carrierMessageId: MessageId;
+  }>;
+  [MessageChannelEvent.RepairRequestReceived]: CustomEvent<{
+    messageIds: MessageId[];
+    fromSenderId?: ParticipantId;
+  }>;
+  [MessageChannelEvent.RepairResponseQueued]: CustomEvent<{
+    messageId: MessageId;
+    tResp: number;
+  }>;
+  [MessageChannelEvent.RepairResponseSent]: CustomEvent<{
+    messageId: MessageId;
+  }>;
 };

package/src/message_channel/index.ts

@@ -8,7 +8,7 @@ export {
   HistoryEntry,
   Message,
   MessageId,
-  SenderId,
+  ParticipantId,
   SyncMessage,
   isContentMessage,
   isEphemeralMessage,

package/src/message_channel/message.ts

@@ -4,19 +4,20 @@ import { Logger } from "@waku/utils";
 export type MessageId = string;
 export type HistoryEntry = proto_sds_message.HistoryEntry;
 export type ChannelId = string;
-export type SenderId = string;
+export type ParticipantId = string;
 
 const log = new Logger("sds:message");
 
 export class Message implements proto_sds_message.SdsMessage {
   public constructor(
-    public messageId: string,
+    public messageId: MessageId,
     public channelId: string,
-    public senderId: string,
+    public senderId: ParticipantId,
     public causalHistory: proto_sds_message.HistoryEntry[],
     public lamportTimestamp?: bigint | undefined,
     public bloomFilter?: Uint8Array<ArrayBufferLike> | undefined,
     public content?: Uint8Array<ArrayBufferLike> | undefined,
+    public repairRequest: proto_sds_message.HistoryEntry[] = [],
     /**
      * Not encoded, set after it is sent, used to include in follow-up messages
      */
@@ -38,7 +39,8 @@ export class Message implements proto_sds_message.SdsMessage {
         causalHistory,
         lamportTimestamp,
         bloomFilter,
-        content
+        content,
+        repairRequest
       } = proto_sds_message.SdsMessage.decode(data);
 
       if (testContentMessage({ lamportTimestamp, content })) {
@@ -49,7 +51,8 @@ export class Message implements proto_sds_message.SdsMessage {
           causalHistory,
           lamportTimestamp!,
           bloomFilter,
-          content!
+          content!,
+          repairRequest
         );
       }
 
@@ -61,7 +64,8 @@ export class Message implements proto_sds_message.SdsMessage {
           causalHistory,
           undefined,
           bloomFilter,
-          content!
+          content!,
+          repairRequest
         );
       }
 
@@ -73,7 +77,8 @@ export class Message implements proto_sds_message.SdsMessage {
           causalHistory,
           lamportTimestamp!,
           bloomFilter,
-          undefined
+          undefined,
+          repairRequest
         );
       }
       log.error(
@@ -90,13 +95,14 @@ export class Message implements proto_sds_message.SdsMessage {
 
 export class SyncMessage extends Message {
   public constructor(
-    public messageId: string,
+    public messageId: MessageId,
     public channelId: string,
-    public senderId: string,
+    public senderId: ParticipantId,
     public causalHistory: proto_sds_message.HistoryEntry[],
     public lamportTimestamp: bigint,
     public bloomFilter: Uint8Array<ArrayBufferLike> | undefined,
     public content: undefined,
+    public repairRequest: proto_sds_message.HistoryEntry[] = [],
     /**
      * Not encoded, set after it is sent, used to include in follow-up messages
      */
@@ -110,6 +116,7 @@ export class SyncMessage extends Message {
       lamportTimestamp,
       bloomFilter,
       content,
+      repairRequest,
       retrievalHint
     );
   }
@@ -134,13 +141,14 @@ export function isSyncMessage(
 
 export class EphemeralMessage extends Message {
   public constructor(
-    public messageId: string,
+    public messageId: MessageId,
     public channelId: string,
-    public senderId: string,
+    public senderId: ParticipantId,
     public causalHistory: proto_sds_message.HistoryEntry[],
     public lamportTimestamp: undefined,
     public bloomFilter: Uint8Array<ArrayBufferLike> | undefined,
     public content: Uint8Array<ArrayBufferLike>,
+    public repairRequest: proto_sds_message.HistoryEntry[] = [],
     /**
      * Not encoded, set after it is sent, used to include in follow-up messages
      */
@@ -157,6 +165,7 @@ export class EphemeralMessage extends Message {
       lamportTimestamp,
       bloomFilter,
       content,
+      repairRequest,
       retrievalHint
     );
   }
@@ -182,13 +191,14 @@ function testEphemeralMessage(message: {
 
 export class ContentMessage extends Message {
   public constructor(
-    public messageId: string,
+    public messageId: MessageId,
     public channelId: string,
-    public senderId: string,
+    public senderId: ParticipantId,
     public causalHistory: proto_sds_message.HistoryEntry[],
     public lamportTimestamp: bigint,
     public bloomFilter: Uint8Array<ArrayBufferLike> | undefined,
     public content: Uint8Array<ArrayBufferLike>,
+    public repairRequest: proto_sds_message.HistoryEntry[] = [],
     /**
      * Not encoded, set after it is sent, used to include in follow-up messages
      */
@@ -205,6 +215,7 @@ export class ContentMessage extends Message {
       lamportTimestamp,
       bloomFilter,
       content,
+      repairRequest,
       retrievalHint
     );
   }

package/src/message_channel/message_channel.ts

@@ -18,15 +18,21 @@ import {
   isSyncMessage,
   Message,
   MessageId,
-  SenderId,
+  ParticipantId,
   SyncMessage
 } from "./message.js";
+import { RepairConfig, RepairManager } from "./repair/repair.js";
 
 export const DEFAULT_BLOOM_FILTER_OPTIONS = {
   capacity: 10000,
   errorRate: 0.001
 };
 
+/**
+ * Maximum number of repair requests to include in a single message
+ */
+const MAX_REPAIR_REQUESTS_PER_MESSAGE = 3;
+
 const DEFAULT_CAUSAL_HISTORY_SIZE = 200;
 const DEFAULT_POSSIBLE_ACKS_THRESHOLD = 2;
 
@@ -46,6 +52,15 @@ export interface MessageChannelOptions {
    * How many possible acks does it take to consider it a definitive ack.
    */
   possibleAcksThreshold?: number;
+  /**
+   * Whether to enable SDS-R repair protocol.
+   * @default true
+   */
+  enableRepair?: boolean;
+  /**
+   * SDS-R repair configuration. Only used if enableRepair is true.
+   */
+  repairConfig?: RepairConfig;
 }
 
 export type ILocalHistory = Pick<
@@ -55,7 +70,7 @@ export type ILocalHistory = Pick<
 
 export class MessageChannel extends TypedEventEmitter<MessageChannelEvents> {
   public readonly channelId: ChannelId;
-  public readonly senderId: SenderId;
+  public readonly senderId: ParticipantId;
   private lamportTimestamp: bigint;
   private filter: DefaultBloomFilter;
   private outgoingBuffer: ContentMessage[];
@@ -66,6 +81,7 @@ export class MessageChannel extends TypedEventEmitter<MessageChannelEvents> {
   private readonly causalHistorySize: number;
   private readonly possibleAcksThreshold: number;
   private readonly timeoutForLostMessagesMs?: number;
+  private readonly repairManager?: RepairManager;
 
   private tasks: Task[] = [];
   private handlers: Handlers = {
@@ -88,7 +104,7 @@ export class MessageChannel extends TypedEventEmitter<MessageChannelEvents> {
 
   public constructor(
     channelId: ChannelId,
-    senderId: SenderId,
+    senderId: ParticipantId,
     options: MessageChannelOptions = {},
     localHistory: ILocalHistory = new MemLocalHistory()
   ) {
@@ -109,6 +125,17 @@ export class MessageChannel extends TypedEventEmitter<MessageChannelEvents> {
       options.possibleAcksThreshold ?? DEFAULT_POSSIBLE_ACKS_THRESHOLD;
     this.timeReceived = new Map();
     this.timeoutForLostMessagesMs = options.timeoutForLostMessagesMs;
+
+    // Only construct RepairManager if repair is enabled (default: true)
+    if (options.enableRepair ?? true) {
+      this.repairManager = new RepairManager(
+        senderId,
+        options.repairConfig,
+        (event: string, detail: unknown) => {
+          this.safeSendEvent(event as MessageChannelEvent, { detail });
+        }
+      );
+    }
   }
 
   public static getMessageId(payload: Uint8Array): MessageId {
@@ -272,9 +299,7 @@ export class MessageChannel extends TypedEventEmitter<MessageChannelEvents> {
         );
         const missingDependencies = message.causalHistory.filter(
           (messageHistoryEntry) =>
-            !this.localHistory.some(
-              ({ messageId }) => messageId === messageHistoryEntry.messageId
-            )
+            !this.isMessageAvailable(messageHistoryEntry.messageId)
         );
         if (missingDependencies.length === 0) {
           if (isContentMessage(message) && this.deliverMessage(message)) {
@@ -355,6 +380,44 @@ export class MessageChannel extends TypedEventEmitter<MessageChannelEvents> {
     );
   }
 
+  /**
+   * Sweep repair incoming buffer and rebroadcast messages ready for repair.
+   * Per SDS-R spec: periodically check for repair responses that are due.
+   *
+   * @param callback - callback to rebroadcast the message
+   * @returns Promise that resolves when all ready repairs have been sent
+   */
+  public async sweepRepairIncomingBuffer(
+    callback?: (message: Message) => Promise<boolean>
+  ): Promise<Message[]> {
+    const repairsToSend =
+      this.repairManager?.sweepIncomingBuffer(this.localHistory) ?? [];
+
+    if (callback) {
+      for (const message of repairsToSend) {
+        try {
+          await callback(message);
+          log.info(
+            this.senderId,
+            "repair message rebroadcast",
+            message.messageId
+          );
+
+          // Emit RepairResponseSent event
+          this.safeSendEvent(MessageChannelEvent.RepairResponseSent, {
+            detail: {
+              messageId: message.messageId
+            }
+          });
+        } catch (error) {
+          log.error("Failed to rebroadcast repair message:", error);
+        }
+      }
+    }
+
+    return repairsToSend;
+  }
+
   /**
    * Send a sync message to the SDS channel.
    *
@@ -369,6 +432,12 @@ export class MessageChannel extends TypedEventEmitter<MessageChannelEvents> {
     callback?: (message: SyncMessage) => Promise<boolean>
   ): Promise<boolean> {
     this.lamportTimestamp = lamportTimestampIncrement(this.lamportTimestamp);
+
+    // Get repair requests to include in sync message (SDS-R)
+    const repairRequests =
+      this.repairManager?.getRepairRequests(MAX_REPAIR_REQUESTS_PER_MESSAGE) ??
+      [];
+
     const message = new SyncMessage(
       // does not need to be secure randomness
       `sync-${Math.random().toString(36).substring(2)}`,
@@ -376,18 +445,22 @@ export class MessageChannel extends TypedEventEmitter<MessageChannelEvents> {
       this.senderId,
       this.localHistory
         .slice(-this.causalHistorySize)
-        .map(({ messageId, retrievalHint }) => {
-          return { messageId, retrievalHint };
+        .map(({ messageId, retrievalHint, senderId }) => {
+          return { messageId, retrievalHint, senderId };
         }),
       this.lamportTimestamp,
       this.filter.toBytes(),
-      undefined
+      undefined,
+      repairRequests
     );
 
-    if (!message.causalHistory || message.causalHistory.length === 0) {
+    if (
+      (!message.causalHistory || message.causalHistory.length === 0) &&
+      repairRequests.length === 0
+    ) {
       log.info(
         this.senderId,
-        "no causal history in sync message, aborting sending"
+        "no causal history and no repair requests in sync message, aborting sending"
       );
       return false;
     }
@@ -399,6 +472,17 @@ export class MessageChannel extends TypedEventEmitter<MessageChannelEvents> {
         this.safeSendEvent(MessageChannelEvent.OutSyncSent, {
           detail: message
         });
+
+        // Emit RepairRequestSent event if repair requests were included
+        if (repairRequests.length > 0) {
+          this.safeSendEvent(MessageChannelEvent.RepairRequestSent, {
+            detail: {
+              messageIds: repairRequests.map((r) => r.messageId),
+              carrierMessageId: message.messageId
+            }
+          });
+        }
+
         return true;
       } catch (error) {
         log.error(
@@ -464,6 +548,26 @@ export class MessageChannel extends TypedEventEmitter<MessageChannelEvents> {
         detail: message
       });
     }
+
+    // SDS-R: Handle received message in repair manager
+    this.repairManager?.markMessageReceived(message.messageId);
+
+    // SDS-R: Process incoming repair requests
+    if (message.repairRequest && message.repairRequest.length > 0) {
+      // Emit RepairRequestReceived event
+      this.safeSendEvent(MessageChannelEvent.RepairRequestReceived, {
+        detail: {
+          messageIds: message.repairRequest.map((r) => r.messageId),
+          fromSenderId: message.senderId
+        }
+      });
+
+      this.repairManager?.processIncomingRepairRequests(
+        message.repairRequest,
+        this.localHistory
+      );
+    }
+
     this.reviewAckStatus(message);
     if (isContentMessage(message)) {
       this.filter.insert(message.messageId);
@@ -471,9 +575,7 @@ export class MessageChannel extends TypedEventEmitter<MessageChannelEvents> {
 
     const missingDependencies = message.causalHistory.filter(
       (messageHistoryEntry) =>
-        !this.localHistory.some(
-          ({ messageId }) => messageId === messageHistoryEntry.messageId
-        )
+        !this.isMessageAvailable(messageHistoryEntry.messageId)
     );
 
     if (missingDependencies.length > 0) {
@@ -487,6 +589,9 @@ export class MessageChannel extends TypedEventEmitter<MessageChannelEvents> {
         missingDependencies.map((ch) => ch.messageId)
       );
 
+      // SDS-R: Track missing dependencies in repair manager
+      this.repairManager?.markDependenciesMissing(missingDependencies);
+
       this.safeSendEvent(MessageChannelEvent.InMessageMissing, {
         detail: Array.from(missingDependencies)
       });
@@ -549,18 +654,26 @@ export class MessageChannel extends TypedEventEmitter<MessageChannelEvents> {
     // It's a new message
     if (!message) {
       log.info(this.senderId, "sending new message", messageId);
+
+      // Get repair requests to include in the message (SDS-R)
+      const repairRequests =
+        this.repairManager?.getRepairRequests(
+          MAX_REPAIR_REQUESTS_PER_MESSAGE
+        ) ?? [];
+
       message = new ContentMessage(
         messageId,
         this.channelId,
         this.senderId,
         this.localHistory
           .slice(-this.causalHistorySize)
-          .map(({ messageId, retrievalHint }) => {
-            return { messageId, retrievalHint };
+          .map(({ messageId, retrievalHint, senderId }) => {
+            return { messageId, retrievalHint, senderId };
           }),
         this.lamportTimestamp,
         this.filter.toBytes(),
-        payload
+        payload,
+        repairRequests
       );
 
       this.outgoingBuffer.push(message);
@@ -616,6 +729,26 @@ export class MessageChannel extends TypedEventEmitter<MessageChannelEvents> {
     }
   }
 
+  /**
+   * Check if a message is available (either in localHistory or incomingBuffer)
+   * This prevents treating messages as "missing" when they've already been received
+   * but are waiting in the incoming buffer for their dependencies.
+   *
+   * @param messageId - The ID of the message to check
+   * @private
+   */
+  private isMessageAvailable(messageId: MessageId): boolean {
+    // Check if in local history
+    if (this.localHistory.some((m) => m.messageId === messageId)) {
+      return true;
+    }
+    // Check if in incoming buffer (already received, waiting for dependencies)
+    if (this.incomingBuffer.some((m) => m.messageId === messageId)) {
+      return true;
+    }
+    return false;
+  }
+
   /**
    * Return true if the message was "delivered"
    *
@@ -657,6 +790,7 @@ export class MessageChannel extends TypedEventEmitter<MessageChannelEvents> {
     }
 
     this.localHistory.push(message);
+
     return true;
   }