@enbox/agent 0.5.0 → 0.5.2

package/src/sync-engine-level.ts

@@ -1,5 +1,5 @@
 import type { AbstractLevel } from 'abstract-level';
-import type { GenericMessage, MessageEvent, MessagesSubscribeReply, MessagesSyncReply, SubscriptionMessage } from '@enbox/dwn-sdk-js';
+import type { GenericMessage, MessageEvent, MessagesSubscribeReply, MessagesSyncDiffEntry, MessagesSyncReply, StateIndex, SubscriptionMessage } from '@enbox/dwn-sdk-js';
 
 import ms from 'ms';
 
@@ -23,12 +23,71 @@ export type SyncEngineLevelParams = {
 };
 
 /**
- * Maximum bit prefix depth before falling back to leaf enumeration.
- * At depth 16, each subtree covers ~1/65536 of the key space, which is a good
- * balance between round-trip count and leaf-set size.
+ * Maximum bit prefix depth for the per-node tree walk (legacy fallback).
+ * At depth 16, each subtree covers ~1/65536 of the key space.
  */
 const MAX_DIFF_DEPTH = 16;
 
+/**
+ * Bit depth for the batched diff protocol.
+ * Lower than MAX_DIFF_DEPTH because the batched diff sends all subtree hashes
+ * in a single request — fine granularity comes from the server-side leaf
+ * enumeration, not from deeper prefixes. Depth 8 = 256 buckets, which is
+ * a good balance between hash map size and leaf-set resolution.
+ */
+const BATCHED_DIFF_DEPTH = 8;
+
+/**
+ * Maximum number of concurrent remote HTTP requests during a tree diff.
+ * The binary tree walk fans out in parallel — without a limit, depth N
+ * produces 2^N concurrent requests, which can exhaust server rate limits.
+ */
+const REMOTE_CONCURRENCY = 4;
+
+/**
+ * Counting semaphore for bounding concurrent async operations.
+ * Used by the tree walk to limit in-flight remote HTTP requests.
+ */
+class Semaphore {
+  private _permits: number;
+  private readonly _waiting: (() => void)[] = [];
+
+  constructor(permits: number) {
+    this._permits = permits;
+  }
+
+  /** Wait until a permit is available, then consume one. */
+  async acquire(): Promise<void> {
+    if (this._permits > 0) {
+      this._permits--;
+      return;
+    }
+    return new Promise<void>((resolve) => {
+      this._waiting.push(resolve);
+    });
+  }
+
+  /** Release a permit, waking the next waiter if any. */
+  release(): void {
+    const next = this._waiting.shift();
+    if (next) {
+      next();
+    } else {
+      this._permits++;
+    }
+  }
+
+  /** Acquire a permit, run the task, then release regardless of outcome. */
+  async run<T>(fn: () => Promise<T>): Promise<T> {
+    await this.acquire();
+    try {
+      return await fn();
+    } finally {
+      this.release();
+    }
+  }
+}
+
 /**
  * Key for the subscription cursor sublevel. Cursors are keyed by
  * `{did}^{dwnUrl}[^{protocol}]` and store an opaque EventLog cursor string.
@@ -238,15 +297,48 @@ export class SyncEngineLevel implements SyncEngine {
             continue;
           }
 
-          // Phase 2: Walk the tree to find differing subtrees.
-          const diff = await this.walkTreeDiff({
+          // Phase 2: Compute the diff in a single round-trip using the
+          // batched 'diff' action.  This replaces the per-node tree walk
+          // that previously required dozens of HTTP requests.
+          const diff = await this.diffWithRemote({
             did, dwnUrl, delegateDid, protocol,
           });
 
           // Phase 3: Pull missing messages (remote has, local doesn't).
+          // The diff response may include inline message data — use it
+          // directly instead of re-fetching via individual MessagesRead calls.
           if (!direction || direction === 'pull') {
             if (diff.onlyRemote.length > 0) {
-              await this.pullMessages({ did, dwnUrl, delegateDid, protocol, messageCids: diff.onlyRemote });
+              // Separate entries into three categories:
+              // 1. Fully prefetched: have message + inline data (or no data needed)
+              // 2. Need data fetch: have message but missing data for RecordsWrite
+              // 3. Need full fetch: no message at all
+              const prefetched: (MessagesSyncDiffEntry & { message: GenericMessage })[] = [];
+              const needsFetchCids: string[] = [];
+
+              for (const entry of diff.onlyRemote) {
+                if (!entry.message) {
+                  // No message at all — need full fetch.
+                  needsFetchCids.push(entry.messageCid);
+                } else if (
+                  entry.message.descriptor.interface === 'Records' &&
+                  entry.message.descriptor.method === 'Write' &&
+                  (entry.message.descriptor as any).dataCid &&
+                  !entry.encodedData
+                ) {
+                  // RecordsWrite with data but data wasn't inlined (too large).
+                  // Need to fetch individually to get the data stream.
+                  needsFetchCids.push(entry.messageCid);
+                } else {
+                  // Fully prefetched (message + data or no data needed).
+                  prefetched.push(entry as MessagesSyncDiffEntry & { message: GenericMessage });
+                }
+              }
+              await this.pullMessages({
+                did, dwnUrl, delegateDid, protocol,
+                messageCids: needsFetchCids,
+                prefetched,
+              });
             }
           }
 
@@ -756,17 +848,55 @@ export class SyncEngineLevel implements SyncEngine {
     return this._defaultHashHex.get(depth) ?? '';
   }
 
+  /**
+   * Parse a bit prefix string (e.g. "0110101") into a boolean array
+   * for the StateIndex API. Each '1' maps to `true` (right child),
+   * each '0' maps to `false` (left child).
+   */
+  private static parseBitPrefix(prefix: string): boolean[] {
+    return Array.from(prefix, (ch): boolean => ch === '1');
+  }
+
   // ---------------------------------------------------------------------------
   // SMT Root Comparison
   // ---------------------------------------------------------------------------
 
   /**
-   * Get the SMT root hash from the local DWN via a MessagesSync 'root' action.
+   * Access the local DWN's StateIndex directly, bypassing the `processMessage`
+   * pipeline. The sync engine runs in the same process as the local DWN, so
+   * there is no need for message signing, schema validation, or authentication
+   * when querying our own state.
+   *
+   * Returns `undefined` in remote mode (no in-process DWN). The local methods
+   * fall back to `processRequest` in that case, routing through RPC to the
+   * local DWN server.
+   */
+  private get stateIndex(): StateIndex | undefined {
+    if (this.agent.dwn.isRemoteMode) {
+      return undefined;
+    }
+    return this.agent.dwn.node.storage.stateIndex;
+  }
+
+  /**
+   * Get the SMT root hash from the local DWN.
+   *
+   * In local mode: queries the StateIndex directly (fast, no processMessage overhead).
+   * In remote mode: constructs a signed MessagesSync message and routes through RPC.
+   *
    * Returns a hex-encoded root hash string.
    */
   private async getLocalRoot(did: string, delegateDid?: string, protocol?: string): Promise<string> {
-    const permissionGrantId = await this.getSyncPermissionGrantId(did, delegateDid, protocol);
+    const si = this.stateIndex;
+    if (si) {
+      const rootHash = protocol !== undefined
+        ? await si.getProtocolRoot(did, protocol)
+        : await si.getRoot(did);
+      return hashToHex(rootHash);
+    }
 
+    // Remote mode fallback: go through processRequest → RPC.
+    const permissionGrantId = await this.getSyncPermissionGrantId(did, delegateDid, protocol);
     const response = await this.agent.dwn.processRequest({
       author        : did,
       target        : did,
@@ -778,7 +908,6 @@ export class SyncEngineLevel implements SyncEngine {
         permissionGrantId
       }
     });
-
     const reply = response.reply as MessagesSyncReply;
     return reply.root ?? '';
   }
@@ -834,11 +963,17 @@ export class SyncEngineLevel implements SyncEngine {
     // Hoist permission grant lookup — resolved once and reused for all subtree/leaf requests.
     const permissionGrantId = await this.getSyncPermissionGrantId(did, delegateDid, protocol);
 
+    // Gate remote HTTP requests through a semaphore so the binary tree walk
+    // doesn't produce an exponential burst of concurrent requests.  Local
+    // DWN requests (in-process) are not gated.
+    const remoteSemaphore = new Semaphore(REMOTE_CONCURRENCY);
+
     const walk = async (prefix: string): Promise<void> => {
       // Get subtree hashes for this prefix from local and remote.
+      // Only the remote request is gated by the semaphore.
       const [localHash, remoteHash] = await Promise.all([
         this.getLocalSubtreeHash(did, prefix, delegateDid, protocol, permissionGrantId),
-        this.getRemoteSubtreeHash(did, dwnUrl, prefix, delegateDid, protocol, permissionGrantId),
+        remoteSemaphore.run(() => this.getRemoteSubtreeHash(did, dwnUrl, prefix, delegateDid, protocol, permissionGrantId)),
       ]);
 
       // If hashes match, this subtree is identical — skip.
@@ -857,7 +992,9 @@ export class SyncEngineLevel implements SyncEngine {
         return;
       }
       if (localHash === emptyHash && remoteHash !== emptyHash) {
-        const remoteLeaves = await this.getRemoteLeaves(did, dwnUrl, prefix, delegateDid, protocol, permissionGrantId);
+        const remoteLeaves = await remoteSemaphore.run(
+          () => this.getRemoteLeaves(did, dwnUrl, prefix, delegateDid, protocol, permissionGrantId),
+        );
         onlyRemote.push(...remoteLeaves);
         return;
       }
@@ -866,7 +1003,7 @@ export class SyncEngineLevel implements SyncEngine {
       if (prefix.length >= MAX_DIFF_DEPTH) {
         const [localLeaves, remoteLeaves] = await Promise.all([
           this.getLocalLeaves(did, prefix, delegateDid, protocol, permissionGrantId),
-          this.getRemoteLeaves(did, dwnUrl, prefix, delegateDid, protocol, permissionGrantId),
+          remoteSemaphore.run(() => this.getRemoteLeaves(did, dwnUrl, prefix, delegateDid, protocol, permissionGrantId)),
         ]);
 
         const localSet = new Set(localLeaves);
@@ -896,9 +1033,146 @@ export class SyncEngineLevel implements SyncEngine {
     return { onlyLocal, onlyRemote };
   }
 
+  // ---------------------------------------------------------------------------
+  // Batched Diff — single round-trip set reconciliation
+  // ---------------------------------------------------------------------------
+
+  /**
+   * Compute the diff between local and remote in a single HTTP round-trip.
+   *
+   * 1. Walk the local SMT directly (no processMessage) to collect subtree
+   *    hashes at `MAX_DIFF_DEPTH`.
+   * 2. Send a single `MessagesSync action:'diff'` to the remote with all
+   *    non-empty subtree hashes.
+   * 3. The remote compares and returns `onlyRemote` (with inline messages)
+   *    and `onlyLocal` prefixes.
+   * 4. Enumerate local leaves for the `onlyLocal` prefixes directly.
+   *
+   * This replaces `walkTreeDiff()` which required one HTTP call per tree node.
+   */
+  private async diffWithRemote({ did, dwnUrl, delegateDid, protocol }: {
+    did: string;
+    dwnUrl: string;
+    delegateDid?: string;
+    protocol?: string;
+  }): Promise<{ onlyRemote: MessagesSyncDiffEntry[]; onlyLocal: string[] }> {
+    // Step 1: Collect local subtree hashes at BATCHED_DIFF_DEPTH directly from StateIndex.
+    const localHashes = await this.collectLocalSubtreeHashes(did, protocol, BATCHED_DIFF_DEPTH);
+
+    // Step 2: Send a single 'diff' request to the remote with our hashes.
+    const permissionGrantId = await this.getSyncPermissionGrantId(did, delegateDid, protocol);
+
+    const syncMessage = await this.agent.dwn.processRequest({
+      store         : false,
+      author        : did,
+      target        : did,
+      messageType   : DwnInterface.MessagesSync,
+      granteeDid    : delegateDid,
+      messageParams : {
+        action : 'diff',
+        protocol,
+        hashes : localHashes,
+        depth  : BATCHED_DIFF_DEPTH,
+        permissionGrantId,
+      }
+    });
+
+    const reply = await this.agent.rpc.sendDwnRequest({
+      dwnUrl,
+      targetDid : did,
+      message   : syncMessage.message,
+    }) as MessagesSyncReply;
+
+    if (reply.status.code !== 200) {
+      throw new Error(`SyncEngineLevel: diff failed with ${reply.status.code}: ${reply.status.detail}`);
+    }
+
+    // Step 3: Enumerate local leaves for prefixes the remote reported as onlyLocal.
+    const permissionGrantIdForLeaves = await this.getSyncPermissionGrantId(did, delegateDid, protocol);
+    const onlyLocalCids: string[] = [];
+    for (const prefix of reply.onlyLocal ?? []) {
+      const leaves = await this.getLocalLeaves(did, prefix, delegateDid, protocol, permissionGrantIdForLeaves);
+      onlyLocalCids.push(...leaves);
+    }
+
+    return {
+      onlyRemote : reply.onlyRemote ?? [],
+      onlyLocal  : onlyLocalCids,
+    };
+  }
+
+  /**
+   * Walk the local SMT to a given depth and collect non-empty subtree hashes.
+   * Returns a `{ prefix: hexHash }` map. Empty subtrees (matching the default
+   * hash) are omitted.
+   *
+   * Uses direct StateIndex access in local mode. In remote mode, falls back
+   * to `getLocalSubtreeHash` which routes through RPC.
+   */
+  private async collectLocalSubtreeHashes(
+    did: string,
+    protocol: string | undefined,
+    depth: number,
+  ): Promise<Record<string, string>> {
+    const result: Record<string, string> = {};
+    const defaultHash = await this.getDefaultHashHex(depth);
+    const si = this.stateIndex;
+
+    const walk = async (prefix: string, currentDepth: number): Promise<void> => {
+      let hexHash: string;
+
+      if (si) {
+        // Fast path: direct StateIndex access (local mode).
+        const bitPath = SyncEngineLevel.parseBitPrefix(prefix);
+        const hash = protocol !== undefined
+          ? await si.getProtocolSubtreeHash(did, protocol, bitPath)
+          : await si.getSubtreeHash(did, bitPath);
+        hexHash = hashToHex(hash);
+      } else {
+        // Remote mode fallback.
+        hexHash = await this.getLocalSubtreeHash(did, prefix, undefined, protocol);
+      }
+
+      if (hexHash === defaultHash) {
+        // Empty subtree — omit from the map.
+        return;
+      }
+
+      if (currentDepth >= depth) {
+        result[prefix] = hexHash;
+        return;
+      }
+
+      // Recurse into children.
+      await Promise.all([
+        walk(prefix + '0', currentDepth + 1),
+        walk(prefix + '1', currentDepth + 1),
+      ]);
+    };
+
+    await walk('', 0);
+    return result;
+  }
+
+  /**
+   * Get the subtree hash at a given bit prefix from the local DWN.
+   *
+   * In local mode: queries the StateIndex directly.
+   * In remote mode: constructs a signed MessagesSync message and routes through RPC.
+   */
   private async getLocalSubtreeHash(
     did: string, prefix: string, delegateDid?: string, protocol?: string, permissionGrantId?: string
   ): Promise<string> {
+    const si = this.stateIndex;
+    if (si) {
+      const bitPath = SyncEngineLevel.parseBitPrefix(prefix);
+      const hash = protocol !== undefined
+        ? await si.getProtocolSubtreeHash(did, protocol, bitPath)
+        : await si.getSubtreeHash(did, bitPath);
+      return hashToHex(hash);
+    }
+
+    // Remote mode fallback.
     const response = await this.agent.dwn.processRequest({
       author        : did,
       target        : did,
@@ -911,7 +1185,6 @@ export class SyncEngineLevel implements SyncEngine {
         permissionGrantId
       }
     });
-
     const reply = response.reply as MessagesSyncReply;
     return reply.hash ?? '';
   }
@@ -942,9 +1215,24 @@ export class SyncEngineLevel implements SyncEngine {
     return reply.hash ?? '';
   }
 
+  /**
+   * Get all leaf messageCids under a given prefix from the local DWN.
+   *
+   * In local mode: queries the StateIndex directly.
+   * In remote mode: constructs a signed MessagesSync message and routes through RPC.
+   */
   private async getLocalLeaves(
     did: string, prefix: string, delegateDid?: string, protocol?: string, permissionGrantId?: string
   ): Promise<string[]> {
+    const si = this.stateIndex;
+    if (si) {
+      const bitPath = SyncEngineLevel.parseBitPrefix(prefix);
+      return protocol !== undefined
+        ? await si.getProtocolLeaves(did, protocol, bitPath)
+        : await si.getLeaves(did, bitPath);
+    }
+
+    // Remote mode fallback.
     const response = await this.agent.dwn.processRequest({
       author        : did,
       target        : did,
@@ -957,7 +1245,6 @@ export class SyncEngineLevel implements SyncEngine {
         permissionGrantId
       }
     });
-
     const reply = response.reply as MessagesSyncReply;
     return reply.entries ?? [];
   }
@@ -995,16 +1282,21 @@ export class SyncEngineLevel implements SyncEngine {
   /**
    * Fetches missing messages from the remote DWN and processes them locally
    * in dependency order (topological sort).
+   *
+   * When prefetched entries are provided (from the batched diff response),
+   * they are processed directly without additional HTTP round-trips.
+   * Only `messageCids` that were NOT prefetched are fetched individually.
    */
-  private async pullMessages({ did, dwnUrl, delegateDid, protocol, messageCids }: {
+  private async pullMessages({ did, dwnUrl, delegateDid, protocol, messageCids, prefetched }: {
     did: string;
     dwnUrl: string;
     delegateDid?: string;
     protocol?: string;
     messageCids: string[];
+    prefetched?: MessagesSyncDiffEntry[];
   }): Promise<void> {
     return pullMessages({
-      did, dwnUrl, delegateDid, protocol, messageCids,
+      did, dwnUrl, delegateDid, protocol, messageCids, prefetched,
       agent          : this.agent,
       permissionsApi : this._permissionsApi,
     });

package/src/sync-messages.ts

@@ -1,15 +1,23 @@
 import type { EnboxPlatformAgent } from './types/agent.js';
 import type { PermissionsApi } from './types/permissions.js';
-import type { GenericMessage, MessagesReadReply, UnionMessageReply } from '@enbox/dwn-sdk-js';
+import type { GenericMessage, MessagesReadReply, MessagesSyncDiffEntry, UnionMessageReply } from '@enbox/dwn-sdk-js';
 
-import { DwnInterfaceName, DwnMethodName, Message } from '@enbox/dwn-sdk-js';
+import { DwnInterfaceName, DwnMethodName, Encoder, Message } from '@enbox/dwn-sdk-js';
 
 import { DwnInterface } from './types/dwn.js';
 import { isRecordsWrite } from './utils.js';
 import { topologicalSort } from './sync-topological-sort.js';
 
-/** Entry type for fetched messages with optional data stream. */
-export type SyncMessageEntry = { message: GenericMessage; dataStream?: ReadableStream<Uint8Array> };
+/** Maximum data size (in bytes) to buffer in memory for retry. Larger payloads are re-fetched. */
+const MAX_BUFFER_SIZE = 1_048_576; // 1 MB
+
+/** Entry type for fetched messages with optional data stream and retry buffer. */
+export type SyncMessageEntry = {
+  message: GenericMessage;
+  dataStream?: ReadableStream<Uint8Array>;
+  /** Buffered data bytes for retry — avoids re-fetching from remote when stream is consumed. */
+  bufferedData?: Uint8Array;
+};
 
 /**
  * 202: message was successfully written to the remote DWN
@@ -43,55 +51,165 @@ export async function getMessageCid(message: GenericMessage): Promise<string> {
  * Fetches missing messages from the remote DWN and processes them on the local DWN
  * in dependency order (topological sort).
  *
- * Messages that fail processing are re-fetched from the remote before each retry
- * pass rather than buffered in memory. ReadableStream is single-use, so a failed
- * message's data stream is consumed on the first attempt. Re-fetching provides a
- * fresh stream without holding all record data in memory simultaneously.
+ * Small data payloads (≤ 1 MB) are buffered during the initial fetch so that
+ * retries can replay the data from memory instead of re-fetching from remote.
+ * Large payloads are re-fetched on retry since buffering them would consume
+ * too much memory.
  */
-export async function pullMessages({ did, dwnUrl, delegateDid, protocol, messageCids, agent, permissionsApi }: {
+export async function pullMessages({ did, dwnUrl, delegateDid, protocol, messageCids, prefetched, agent, permissionsApi }: {
   did: string;
   dwnUrl: string;
   delegateDid?: string;
   protocol?: string;
   messageCids: string[];
+  /** Pre-fetched message entries from the batched diff response (already have message + data). */
+  prefetched?: MessagesSyncDiffEntry[];
   agent: EnboxPlatformAgent;
   permissionsApi: PermissionsApi;
 }): Promise<void> {
-  // Step 1: Fetch all missing messages from the remote in parallel.
-  const fetched = await fetchRemoteMessages({ did, dwnUrl, delegateDid, protocol, messageCids, agent, permissionsApi });
+  // Convert prefetched diff entries into SyncMessageEntry format.
+  const prefetchedEntries: SyncMessageEntry[] = [];
+  if (prefetched) {
+    for (const entry of prefetched) {
+      if (!entry.message) { continue; }
+      const syncEntry: SyncMessageEntry = { message: entry.message };
+      if (entry.encodedData) {
+        // Convert base64url-encoded data to a ReadableStream.
+        const bytes = Encoder.base64UrlToBytes(entry.encodedData);
+        syncEntry.bufferedData = bytes;
+        syncEntry.dataStream = new ReadableStream<Uint8Array>({
+          start(controller): void {
+            controller.enqueue(bytes);
+            controller.close();
+          }
+        });
+      }
+      prefetchedEntries.push(syncEntry);
+    }
+  }
+
+  // Step 1: Fetch remaining messages (not prefetched) from the remote.
+  const fetched = messageCids.length > 0
+    ? await fetchRemoteMessages({ did, dwnUrl, delegateDid, protocol, messageCids, agent, permissionsApi })
+    : [];
+
+  // Merge prefetched entries with remotely fetched ones.
+  const allFetched = [...prefetchedEntries, ...fetched];
 
   // Step 2: Build dependency graph and topological sort.
-  const sorted = topologicalSort(fetched);
+  const sorted = topologicalSort(allFetched);
+
+  // Step 3: Buffer small data streams so they can be replayed on retry.
+  await bufferSmallStreams(sorted);
 
-  // Step 3: Process messages in dependency order with multi-pass retry.
-  // Retry up to MAX_RETRY_PASSES times for messages that fail due to
-  // dependency ordering issues (e.g., a RecordsWrite whose ProtocolsConfigure
-  // hasn't committed yet). Failed messages are re-fetched from the remote
-  // to obtain a fresh data stream, since ReadableStream is single-use.
+  // Step 4: Process messages in dependency order with multi-pass retry.
   const MAX_RETRY_PASSES = 3;
   let pending = sorted;
 
   for (let pass = 0; pass <= MAX_RETRY_PASSES && pending.length > 0; pass++) {
-    const failedCids: string[] = [];
+    const failed: SyncMessageEntry[] = [];
 
     for (const entry of pending) {
-      const pullReply = await agent.dwn.processRawMessage(did, entry.message, { dataStream: entry.dataStream });
+      // Create a fresh ReadableStream from the buffer if available (stream is single-use).
+      const dataStream = entry.bufferedData
+        ? new ReadableStream<Uint8Array>({ start(c): void { c.enqueue(entry.bufferedData!); c.close(); } })
+        : entry.dataStream;
+
+      const pullReply = await agent.dwn.processRawMessage(did, entry.message, { dataStream });
+
       if (!syncMessageReplyIsSuccessful(pullReply)) {
-        const cid = await getMessageCid(entry.message);
-        failedCids.push(cid);
+        failed.push(entry);
       }
     }
 
-    // Re-fetch failed messages from the remote to get fresh data streams.
-    if (failedCids.length > 0) {
-      const reFetched = await fetchRemoteMessages({ did, dwnUrl, delegateDid, protocol, messageCids: failedCids, agent, permissionsApi });
-      pending = topologicalSort(reFetched);
+    if (failed.length > 0) {
+      // Separate entries that have a buffer (can retry locally) from those
+      // that need a fresh fetch (large payloads whose stream was consumed).
+      const needsRefetch: string[] = [];
+      const canRetry: SyncMessageEntry[] = [];
+
+      for (const entry of failed) {
+        if (entry.bufferedData || !entry.dataStream) {
+          // Has a buffer or has no data — can retry without re-fetching.
+          canRetry.push(entry);
+        } else {
+          // Large payload whose stream was consumed — must re-fetch.
+          const cid = await getMessageCid(entry.message);
+          needsRefetch.push(cid);
+        }
+      }
+
+      // Re-fetch only the large-payload messages that we couldn't buffer.
+      if (needsRefetch.length > 0) {
+        const reFetched = await fetchRemoteMessages({ did, dwnUrl, delegateDid, protocol, messageCids: needsRefetch, agent, permissionsApi });
+        canRetry.push(...reFetched);
+      }
+
+      pending = topologicalSort(canRetry);
     } else {
       pending = [];
     }
   }
 }
 
+/**
+ * Buffers small data streams into `Uint8Array` so they can be replayed on retry.
+ * Streams larger than `MAX_BUFFER_SIZE` are left as-is (will be re-fetched on retry).
+ */
+async function bufferSmallStreams(entries: SyncMessageEntry[]): Promise<void> {
+  for (const entry of entries) {
+    if (!entry.dataStream) {
+      continue;
+    }
+
+    // Read the stream into memory. If it exceeds the threshold, stop and
+    // leave the entry without a buffer (it will be re-fetched on retry).
+    const chunks: Uint8Array[] = [];
+    let totalSize = 0;
+    let exceededThreshold = false;
+    const reader = entry.dataStream.getReader();
+
+    try {
+      for (;;) {
+        const { done, value } = await reader.read();
+        if (done) { break; }
+        totalSize += value.byteLength;
+        if (totalSize > MAX_BUFFER_SIZE) {
+          exceededThreshold = true;
+          break;
+        }
+        chunks.push(value);
+      }
+    } finally {
+      reader.releaseLock();
+    }
+
+    if (exceededThreshold) {
+      // Stream exceeded the buffer threshold. Leave dataStream consumed —
+      // the retry path will re-fetch from remote.
+      entry.dataStream = undefined;
+      continue;
+    }
+
+    // Combine chunks into a single Uint8Array buffer.
+    const buffer = new Uint8Array(totalSize);
+    let offset = 0;
+    for (const chunk of chunks) {
+      buffer.set(chunk, offset);
+      offset += chunk.byteLength;
+    }
+
+    entry.bufferedData = buffer;
+    // Create a fresh ReadableStream from the buffer for the first processing attempt.
+    entry.dataStream = new ReadableStream<Uint8Array>({
+      start(controller): void {
+        controller.enqueue(buffer);
+        controller.close();
+      }
+    });
+  }
+}
+
 /**
  * Fetches messages from a remote DWN by their CIDs using MessagesRead.
  */
@@ -123,8 +241,9 @@ export async function fetchRemoteMessages({ did, dwnUrl, delegateDid, protocol,
     }
   }
 
-  // Fetch messages in parallel with bounded concurrency.
-  const CONCURRENCY = 10;
+  // Fetch messages in parallel with bounded concurrency.  Keep this low
+  // to avoid bursting through the remote server's rate limits during sync.
+  const CONCURRENCY = 4;
   let cursor = 0;
 
   while (cursor < messageCids.length) {

package/src/utils.ts

@@ -168,4 +168,4 @@ export function concatenateUrl(baseUrl: string, path: string): string {
   }
 
   return `${baseUrl}/${path}`;
-}
+}