@enbox/agent 0.7.6 → 0.7.8

package/src/sync-messages.ts

@@ -1,13 +1,13 @@
 import type { GenericMessage, MessagesReadReply, MessagesSyncDiffEntry, UnionMessageReply } from '@enbox/dwn-sdk-js';
 
 import type { EnboxPlatformAgent } from './types/agent.js';
-import type { PermissionsApi } from './types/permissions.js';
 import type { PermanentPushFailure, PushResult } from './types/sync.js';
 
 import { DwnInterfaceName, DwnMethodName, Encoder, Message } from '@enbox/dwn-sdk-js';
 
 import { DwnInterface } from './types/dwn.js';
 import { isRecordsWrite } from './utils.js';
+import { toMessagesPermissionGrantIds } from './sync-permission-grants.js';
 import { topologicalSort } from './sync-topological-sort.js';
 
 /** Maximum data size (in bytes) to buffer in memory for retry. Larger payloads are re-fetched. */
@@ -21,6 +21,26 @@ export type SyncMessageEntry = {
   bufferedData?: Uint8Array;
 };
 
+/** Optional pre-apply gate for inbound sync entries. */
+export type PullMessageAcceptance = (
+  entry: SyncMessageEntry,
+  entries: SyncMessageEntry[],
+) => Promise<boolean>;
+
+/** Raised when an in-flight pull is cancelled before local apply can continue. */
+export class SyncPullAbortedError extends Error {
+  constructor() {
+    super('Sync pull aborted because the sync target is no longer current.');
+    this.name = 'SyncPullAbortedError';
+  }
+}
+
+function assertShouldContinue(shouldContinue: (() => boolean) | undefined): void {
+  if (shouldContinue?.() === false) {
+    throw new SyncPullAbortedError();
+  }
+}
+
 /**
  * 202: message was successfully written to the remote DWN
  * 204: an initial write message was written without any data
@@ -103,116 +123,185 @@ export async function getMessageCid(message: GenericMessage): Promise<string> {
  * Large payloads are re-fetched on retry since buffering them would consume
  * too much memory.
  */
-export async function pullMessages({ did, dwnUrl, delegateDid, protocol, messageCids, prefetched, agent, permissionsApi }: {
+export async function pullMessages({ did, dwnUrl, delegateDid, permissionGrantIds, messageCids, prefetched, acceptEntry, shouldContinue, agent }: {
   did: string;
   dwnUrl: string;
   delegateDid?: string;
-  protocol?: string;
+  permissionGrantIds?: string[];
   messageCids: string[];
   /** Pre-fetched message entries from the batched diff response (already have message + data). */
   prefetched?: MessagesSyncDiffEntry[];
+  acceptEntry?: PullMessageAcceptance;
+  shouldContinue?: () => boolean;
   agent: EnboxPlatformAgent;
-  permissionsApi: PermissionsApi;
 }): Promise<string[]> {
-  // Convert prefetched diff entries into SyncMessageEntry format.
-  const prefetchedEntries: SyncMessageEntry[] = [];
-  if (prefetched) {
-    for (const entry of prefetched) {
-      if (!entry.message) { continue; }
-      const syncEntry: SyncMessageEntry = { message: entry.message };
-      if (entry.encodedData) {
-        // Convert base64url-encoded data to a ReadableStream.
-        const bytes = Encoder.base64UrlToBytes(entry.encodedData);
-        syncEntry.bufferedData = bytes;
-        syncEntry.dataStream = new ReadableStream<Uint8Array>({
-          start(controller): void {
-            controller.enqueue(bytes);
-            controller.close();
-          }
-        });
-      }
-      prefetchedEntries.push(syncEntry);
-    }
-  }
+  assertShouldContinue(shouldContinue);
 
   // Step 1: Fetch remaining messages (not prefetched) from the remote.
   const fetched = messageCids.length > 0
-    ? await fetchRemoteMessages({ did, dwnUrl, delegateDid, protocol, messageCids, agent, permissionsApi })
+    ? await fetchRemoteMessages({ did, dwnUrl, delegateDid, permissionGrantIds, messageCids, agent })
     : [];
+  assertShouldContinue(shouldContinue);
 
   // Merge prefetched entries with remotely fetched ones.
-  const allFetched = [...prefetchedEntries, ...fetched];
+  const allFetched = [...syncEntriesFromPrefetchedDiff(prefetched), ...fetched];
+
+  const { accepted, rejectedCids } = await applyPullAcceptanceGate(allFetched, acceptEntry, shouldContinue);
+  assertShouldContinue(shouldContinue);
 
   // Step 2: Build dependency graph and topological sort.
-  const sorted = topologicalSort(allFetched);
+  const sorted = topologicalSort(accepted);
 
   // Step 3: Buffer small data streams so they can be replayed on retry.
-  await bufferSmallStreams(sorted);
+  await bufferSmallStreams(sorted, shouldContinue);
+  assertShouldContinue(shouldContinue);
 
   // Step 4: Process messages in dependency order with multi-pass retry.
   const MAX_RETRY_PASSES = 3;
   let pending = sorted;
 
   for (let pass = 0; pass <= MAX_RETRY_PASSES && pending.length > 0; pass++) {
-    const failed: SyncMessageEntry[] = [];
+    assertShouldContinue(shouldContinue);
+    const failed = await processPullPass({ did, entries: pending, shouldContinue, agent });
+    assertShouldContinue(shouldContinue);
+    pending = failed.length === 0
+      ? []
+      : await preparePullRetry({ did, dwnUrl, delegateDid, permissionGrantIds, failed, shouldContinue, agent });
+  }
 
-    for (const entry of pending) {
-      // Create a fresh ReadableStream from the buffer if available (stream is single-use).
-      const dataStream = entry.bufferedData
-        ? new ReadableStream<Uint8Array>({ start(c): void { c.enqueue(entry.bufferedData!); c.close(); } })
-        : entry.dataStream;
+  // Return CIDs that permanently failed after all retry passes.
+  return [...rejectedCids, ...await getMessageCids(pending)];
+}
 
-      const pullReply = await agent.dwn.processRawMessage(did, entry.message, { dataStream });
+async function applyPullAcceptanceGate(
+  entries: SyncMessageEntry[],
+  acceptEntry: PullMessageAcceptance | undefined,
+  shouldContinue?: () => boolean,
+): Promise<{ accepted: SyncMessageEntry[]; rejectedCids: string[] }> {
+  if (!acceptEntry) {
+    return { accepted: entries, rejectedCids: [] };
+  }
 
-      if (!syncMessageReplyIsSuccessful(pullReply)) {
-        failed.push(entry);
-      }
+  const accepted: SyncMessageEntry[] = [];
+  const rejectedCids: string[] = [];
+  for (const entry of entries) {
+    assertShouldContinue(shouldContinue);
+    if (await acceptEntry(entry, entries)) {
+      accepted.push(entry);
+    } else {
+      rejectedCids.push(await getMessageCid(entry.message));
     }
+    assertShouldContinue(shouldContinue);
+  }
 
-    if (failed.length > 0) {
-      // Separate entries that have a buffer (can retry locally) from those
-      // that need a fresh fetch (large payloads whose stream was consumed).
-      const needsRefetch: string[] = [];
-      const canRetry: SyncMessageEntry[] = [];
-
-      for (const entry of failed) {
-        if (entry.bufferedData || !entry.dataStream) {
-          // Has a buffer or has no data — can retry without re-fetching.
-          canRetry.push(entry);
-        } else {
-          // Large payload whose stream was consumed — must re-fetch.
-          const cid = await getMessageCid(entry.message);
-          needsRefetch.push(cid);
-        }
-      }
+  return { accepted, rejectedCids };
+}
 
-      // Re-fetch only the large-payload messages that we couldn't buffer.
-      if (needsRefetch.length > 0) {
-        const reFetched = await fetchRemoteMessages({ did, dwnUrl, delegateDid, protocol, messageCids: needsRefetch, agent, permissionsApi });
-        canRetry.push(...reFetched);
-      }
+function syncEntriesFromPrefetchedDiff(prefetched?: MessagesSyncDiffEntry[]): SyncMessageEntry[] {
+  if (!prefetched) { return []; }
+
+  const entries: SyncMessageEntry[] = [];
+  for (const entry of prefetched) {
+    if (!entry.message) { continue; }
 
-      pending = topologicalSort(canRetry);
+    const syncEntry: SyncMessageEntry = { message: entry.message };
+    if (entry.encodedData) {
+      const bytes = Encoder.base64UrlToBytes(entry.encodedData);
+      syncEntry.bufferedData = bytes;
+      syncEntry.dataStream = dataStreamFromBytes(bytes);
+    }
+    entries.push(syncEntry);
+  }
+  return entries;
+}
+
+function dataStreamFromBytes(bytes: Uint8Array): ReadableStream<Uint8Array> {
+  return new ReadableStream<Uint8Array>({
+    start(controller): void {
+      controller.enqueue(bytes);
+      controller.close();
+    }
+  });
+}
+
+function replayableDataStream(entry: SyncMessageEntry): ReadableStream<Uint8Array> | undefined {
+  return entry.bufferedData ? dataStreamFromBytes(entry.bufferedData) : entry.dataStream;
+}
+
+async function processPullPass({ did, entries, shouldContinue, agent }: {
+  did: string;
+  entries: SyncMessageEntry[];
+  shouldContinue?: () => boolean;
+  agent: EnboxPlatformAgent;
+}): Promise<SyncMessageEntry[]> {
+  const failed: SyncMessageEntry[] = [];
+
+  for (const entry of entries) {
+    assertShouldContinue(shouldContinue);
+    const pullReply = await agent.dwn.processRawMessage(did, entry.message, {
+      dataStream: replayableDataStream(entry),
+    });
+    assertShouldContinue(shouldContinue);
+
+    if (!syncMessageReplyIsSuccessful(pullReply)) {
+      failed.push(entry);
+    }
+  }
+
+  return failed;
+}
+
+async function preparePullRetry({ did, dwnUrl, delegateDid, permissionGrantIds, failed, shouldContinue, agent }: {
+  did: string;
+  dwnUrl: string;
+  delegateDid?: string;
+  permissionGrantIds?: string[];
+  failed: SyncMessageEntry[];
+  shouldContinue?: () => boolean;
+  agent: EnboxPlatformAgent;
+}): Promise<SyncMessageEntry[]> {
+  const canRetry: SyncMessageEntry[] = [];
+  const needsRefetch: string[] = [];
+
+  for (const entry of failed) {
+    if (entry.bufferedData || !entry.dataStream) {
+      canRetry.push(entry);
     } else {
-      pending = [];
+      needsRefetch.push(await getMessageCid(entry.message));
     }
   }
 
-  // Return CIDs that permanently failed after all retry passes.
-  const permanentlyFailed: string[] = [];
-  for (const entry of pending) {
-    const cid = await getMessageCid(entry.message);
-    permanentlyFailed.push(cid);
+  if (needsRefetch.length > 0) {
+    assertShouldContinue(shouldContinue);
+    canRetry.push(...await fetchRemoteMessages({
+      did,
+      dwnUrl,
+      delegateDid,
+      permissionGrantIds,
+      messageCids: needsRefetch,
+      agent,
+    }));
+    assertShouldContinue(shouldContinue);
   }
-  return permanentlyFailed;
+
+  return topologicalSort(canRetry);
+}
+
+async function getMessageCids(entries: SyncMessageEntry[]): Promise<string[]> {
+  const cids: string[] = [];
+  for (const entry of entries) {
+    cids.push(await getMessageCid(entry.message));
+  }
+  return cids;
 }
 
 /**
  * Buffers small data streams into `Uint8Array` so they can be replayed on retry.
  * Streams larger than `MAX_BUFFER_SIZE` are left as-is (will be re-fetched on retry).
  */
-async function bufferSmallStreams(entries: SyncMessageEntry[]): Promise<void> {
+async function bufferSmallStreams(entries: SyncMessageEntry[], shouldContinue?: () => boolean): Promise<void> {
   for (const entry of entries) {
+    assertShouldContinue(shouldContinue);
     if (!entry.dataStream) {
       continue;
     }
@@ -228,6 +317,7 @@ async function bufferSmallStreams(entries: SyncMessageEntry[]): Promise<void> {
       for (;;) {
         const { done, value } = await reader.read();
         if (done) { break; }
+        assertShouldContinue(shouldContinue);
         totalSize += value.byteLength;
         if (totalSize > MAX_BUFFER_SIZE) {
           exceededThreshold = true;
@@ -256,41 +346,24 @@ async function bufferSmallStreams(entries: SyncMessageEntry[]): Promise<void> {
 
     entry.bufferedData = buffer;
     // Create a fresh ReadableStream from the buffer for the first processing attempt.
-    entry.dataStream = new ReadableStream<Uint8Array>({
-      start(controller): void {
-        controller.enqueue(buffer);
-        controller.close();
-      }
-    });
+    entry.dataStream = dataStreamFromBytes(buffer);
+    assertShouldContinue(shouldContinue);
   }
 }
 
 /**
  * Fetches messages from a remote DWN by their CIDs using MessagesRead.
  */
-export async function fetchRemoteMessages({ did, dwnUrl, delegateDid, protocol, messageCids, agent, permissionsApi }: {
+export async function fetchRemoteMessages({ did, dwnUrl, delegateDid, permissionGrantIds, messageCids, agent }: {
   did: string;
   dwnUrl: string;
   delegateDid?: string;
-  protocol?: string;
+  permissionGrantIds?: string[];
   messageCids: string[];
   agent: EnboxPlatformAgent;
-  permissionsApi: PermissionsApi;
 }): Promise<SyncMessageEntry[]> {
   const results: SyncMessageEntry[] = [];
 
-  let permissionGrantId: string | undefined;
-  if (delegateDid) {
-    const messagesReadGrant = await permissionsApi.getPermissionForRequest({
-      connectedDid : did,
-      messageType  : DwnInterface.MessagesRead,
-      delegateDid,
-      protocol,
-      cached       : true
-    });
-    permissionGrantId = messagesReadGrant.grant.id;
-  }
-
   // Fetch messages in parallel with bounded concurrency.  Keep this low
   // to avoid bursting through the remote server's rate limits during sync.
   const CONCURRENCY = 4;
@@ -308,7 +381,7 @@ export async function fetchRemoteMessages({ did, dwnUrl, delegateDid, protocol,
         target        : did,
         messageType   : DwnInterface.MessagesRead,
         granteeDid    : delegateDid,
-        messageParams : { messageCid, permissionGrantId }
+        messageParams : { messageCid, permissionGrantIds: toMessagesPermissionGrantIds(permissionGrantIds) }
       });
 
       let reply: MessagesReadReply;
@@ -356,116 +429,147 @@ export async function fetchRemoteMessages({ did, dwnUrl, delegateDid, protocol,
  * on the first failure. Callers use this to advance the push checkpoint
  * incrementally — only up to the highest contiguous success.
  */
-export async function pushMessages({ did, dwnUrl, delegateDid, protocol, messageCids, agent, permissionsApi }: {
+export async function pushMessages({ did, dwnUrl, delegateDid, permissionGrantIds, messageCids, agent }: {
   did: string;
   dwnUrl: string;
   delegateDid?: string;
-  protocol?: string;
+  permissionGrantIds?: string[];
   messageCids: string[];
   agent: EnboxPlatformAgent;
-  permissionsApi: PermissionsApi;
 }): Promise<PushResult> {
   const succeeded: string[] = [];
-  const failed: string[] = [];
   const permanentlyFailed: PermanentPushFailure[] = [];
+  const { fetched, failed } = await fetchLocalMessagesForPush({
+    did,
+    delegateDid,
+    permissionGrantIds,
+    messageCids,
+    agent,
+  });
+
+  // Step 2: Sort in dependency order using topological sort.
+  const sorted = topologicalSort(fetched);
+
+  // Step 3: Buffer data streams so they survive fetch retries.
+  // ReadableStream is single-use — if sendDwnRequest's underlying fetch
+  // retries the HTTP request, the original stream is already consumed.
+  await bufferSmallStreams(sorted);
 
-  // Step 1: Fetch all local messages (streams are pull-based, not yet consumed).
+  // Step 4: Push messages in dependency order.
+  for (const entry of sorted) {
+    const outcome = await pushSingleMessage({ did, dwnUrl, entry, agent });
+    if (outcome.status === 'succeeded') {
+      succeeded.push(outcome.cid);
+    } else if (outcome.status === 'permanent') {
+      permanentlyFailed.push(outcome.failure);
+    } else {
+      failed.push(outcome.cid);
+    }
+  }
+
+  return { succeeded, failed, permanentlyFailed };
+}
+
+async function fetchLocalMessagesForPush({ did, delegateDid, permissionGrantIds, messageCids, agent }: {
+  did: string;
+  delegateDid?: string;
+  permissionGrantIds?: string[];
+  messageCids: string[];
+  agent: EnboxPlatformAgent;
+}): Promise<{ fetched: SyncMessageEntry[]; failed: string[] }> {
   const fetched: SyncMessageEntry[] = [];
+  const failed: string[] = [];
+
   for (const messageCid of messageCids) {
-    const dwnMessage = await getLocalMessage({ author: did, messageCid, delegateDid, protocol, agent, permissionsApi });
+    const dwnMessage = await getLocalMessage({ author: did, messageCid, delegateDid, permissionGrantIds, agent });
     if (dwnMessage) {
       fetched.push(dwnMessage);
     } else {
-      // Message could not be fetched locally — mark as failed.
       failed.push(messageCid);
     }
   }
 
-  // Step 2: Sort in dependency order using topological sort.
-  const sorted = topologicalSort(fetched);
+  return { fetched, failed };
+}
 
-  // Step 3: Buffer data streams so they survive fetch retries.
-  // ReadableStream is single-use — if sendDwnRequest's underlying fetch
-  // retries the HTTP request, the original stream is already consumed.
-  await bufferSmallStreams(sorted);
+type PushSingleMessageOutcome =
+  | { status: 'succeeded'; cid: string }
+  | { status: 'failed'; cid: string }
+  | { status: 'permanent'; cid: string; failure: PermanentPushFailure };
 
-  // Step 4: Push messages in dependency order.
-  for (const entry of sorted) {
-    const cid = await getMessageCid(entry.message);
+async function pushSingleMessage({ did, dwnUrl, entry, agent }: {
+  did: string;
+  dwnUrl: string;
+  entry: SyncMessageEntry;
+  agent: EnboxPlatformAgent;
+}): Promise<PushSingleMessageOutcome> {
+  const cid = await getMessageCid(entry.message);
 
-    // Use a Blob for buffered data — unlike ReadableStream, Blob is
-    // replayable so fetchWithRetry can retry the HTTP request on failure.
-    const data = entry.bufferedData
-      ? new Blob([entry.bufferedData] as BlobPart[], { type: 'application/octet-stream' })
-      : entry.dataStream;
+  try {
+    const reply = await agent.rpc.sendDwnRequest({
+      dwnUrl,
+      targetDid : did,
+      data      : pushData(entry),
+      message   : entry.message
+    });
 
-    try {
-      const reply = await agent.rpc.sendDwnRequest({
-        dwnUrl,
-        targetDid : did,
-        data,
-        message   : entry.message
-      });
+    if (syncMessageReplyIsSuccessful(reply, entry.message)) {
+      return { status: 'succeeded', cid };
+    }
 
-      if (syncMessageReplyIsSuccessful(reply, entry.message)) {
-        succeeded.push(cid);
-      } else if (isPermanentPushFailure(reply)) {
-        // Permanent failures (400/401/403) will never succeed — do NOT retry.
-        // These include protocol violations (RecordLimitExceeded), auth errors,
-        // and schema validation failures.
-        if (reply.status.code === 400 && reply.status.detail?.includes('record limit')) {
-          // Expected for singleton convergence in multi-device scenarios:
-          // one device created a singleton record, this device has a
-          // different one, and the remote rejects the duplicate.
-          console.debug(`SyncEngineLevel: singleton already exists on remote, skipping push for ${cid}`);
-        } else {
-          console.warn(`SyncEngineLevel: push permanently failed for ${cid}: ${reply.status.code} ${reply.status.detail}`);
-        }
-        permanentlyFailed.push({ cid, statusCode: reply.status.code, detail: reply.status.detail ?? '' });
-      } else {
-        // Transient failures (5xx, etc.) — worth retrying.
-        console.error(`SyncEngineLevel: push failed for ${cid}: ${reply.status.code} ${reply.status.detail}`);
-        failed.push(cid);
-      }
-    } catch (error: any) {
-      // Network errors — transient, worth retrying.
-      console.error(`SyncEngineLevel: push error for ${cid}: ${error.message ?? error}`);
-      failed.push(cid);
+    if (isPermanentPushFailure(reply)) {
+      logPermanentPushFailure(cid, reply);
+      return {
+        status  : 'permanent',
+        cid,
+        failure : { cid, statusCode: reply.status.code, detail: reply.status.detail ?? '' },
+      };
     }
+
+    console.error(`SyncEngineLevel: push failed for ${cid}: ${reply.status.code} ${reply.status.detail}`);
+    return { status: 'failed', cid };
+  } catch (error: any) {
+    console.error(`SyncEngineLevel: push error for ${cid}: ${error.message ?? error}`);
+    return { status: 'failed', cid };
   }
+}
 
-  return { succeeded, failed, permanentlyFailed };
+function pushData(entry: SyncMessageEntry): Blob | ReadableStream<Uint8Array> | undefined {
+  // Use a Blob for buffered data: unlike ReadableStream, Blob is replayable,
+  // so fetchWithRetry can retry the HTTP request after a transport failure.
+  return entry.bufferedData
+    ? new Blob([entry.bufferedData] as BlobPart[], { type: 'application/octet-stream' })
+    : entry.dataStream;
+}
+
+function logPermanentPushFailure(cid: string, reply: UnionMessageReply): void {
+  if (reply.status.code === 400 && reply.status.detail?.includes('record limit')) {
+    // Expected for singleton convergence in multi-device scenarios: one device
+    // created a singleton record, this device has another, and the remote
+    // rejects the duplicate.
+    console.debug(`SyncEngineLevel: singleton already exists on remote, skipping push for ${cid}`);
+    return;
+  }
+
+  console.warn(`SyncEngineLevel: push permanently failed for ${cid}: ${reply.status.code} ${reply.status.detail}`);
 }
 
 /**
  * Reads a message from the local DWN by its CID using MessagesRead.
  */
-export async function getLocalMessage({ author, delegateDid, protocol, messageCid, agent, permissionsApi }: {
+export async function getLocalMessage({ author, delegateDid, permissionGrantIds, messageCid, agent }: {
   author: string;
   delegateDid?: string;
-  protocol?: string;
+  permissionGrantIds?: string[];
   messageCid: string;
   agent: EnboxPlatformAgent;
-  permissionsApi: PermissionsApi;
 }): Promise<SyncMessageEntry | undefined> {
-  let permissionGrantId: string | undefined;
-  if (delegateDid) {
-    const messagesReadGrant = await permissionsApi.getPermissionForRequest({
-      connectedDid : author,
-      messageType  : DwnInterface.MessagesRead,
-      delegateDid,
-      protocol,
-      cached       : true
-    });
-    permissionGrantId = messagesReadGrant.grant.id;
-  }
-
   const { reply } = await agent.dwn.processRequest({
     author,
     target        : author,
     messageType   : DwnInterface.MessagesRead,
     granteeDid    : delegateDid,
-    messageParams : { messageCid, permissionGrantId }
+    messageParams : { messageCid, permissionGrantIds: toMessagesPermissionGrantIds(permissionGrantIds) }
   });
 
   if (reply.status.code !== 200 || !reply.entry) {