@enbox/agent 0.1.5 → 0.1.7

package/src/sync-messages.ts

@@ -0,0 +1,279 @@
+import type { PermissionsApi } from './types/permissions.js';
+import type { Web5PlatformAgent } from './types/agent.js';
+import type { GenericMessage, MessagesReadReply, UnionMessageReply } from '@enbox/dwn-sdk-js';
+
+import { DwnInterfaceName, DwnMethodName, Message } from '@enbox/dwn-sdk-js';
+
+import { DwnInterface } from './types/dwn.js';
+import { isRecordsWrite } from './utils.js';
+import { topologicalSort } from './sync-topological-sort.js';
+
+/** Entry type for fetched messages with optional data stream. */
+export type SyncMessageEntry = { message: GenericMessage; dataStream?: ReadableStream<Uint8Array> };
+
+/**
+ * 202: message was successfully written to the remote DWN
+ * 204: an initial write message was written without any data
+ * 409: message was already present on the remote DWN
+ * RecordsDelete + 404: the initial write was not found or already deleted
+ */
+export function syncMessageReplyIsSuccessful(reply: UnionMessageReply): boolean {
+  return reply.status.code === 202 ||
+    reply.status.code === 204 ||
+    reply.status.code === 409 ||
+    (
+      reply.entry?.message.descriptor.interface === DwnInterfaceName.Records &&
+      reply.entry?.message.descriptor.method === DwnMethodName.Delete &&
+      reply.status.code === 404
+    );
+}
+
+/**
+ * Helper to get the CID of a message for logging purposes.
+ */
+export async function getMessageCid(message: GenericMessage): Promise<string> {
+  try {
+    return await Message.getCid(message);
+  } catch {
+    return 'unknown';
+  }
+}
+
+/**
+ * Fetches missing messages from the remote DWN and processes them on the local DWN
+ * in dependency order (topological sort).
+ *
+ * Messages that fail processing are re-fetched from the remote before each retry
+ * pass rather than buffered in memory. ReadableStream is single-use, so a failed
+ * message's data stream is consumed on the first attempt. Re-fetching provides a
+ * fresh stream without holding all record data in memory simultaneously.
+ */
+export async function pullMessages({ did, dwnUrl, delegateDid, protocol, messageCids, agent, permissionsApi }: {
+  did: string;
+  dwnUrl: string;
+  delegateDid?: string;
+  protocol?: string;
+  messageCids: string[];
+  agent: Web5PlatformAgent;
+  permissionsApi: PermissionsApi;
+}): Promise<void> {
+  // Step 1: Fetch all missing messages from the remote in parallel.
+  const fetched = await fetchRemoteMessages({ did, dwnUrl, delegateDid, protocol, messageCids, agent, permissionsApi });
+
+  // Step 2: Build dependency graph and topological sort.
+  const sorted = topologicalSort(fetched);
+
+  // Step 3: Process messages in dependency order with multi-pass retry.
+  // Retry up to MAX_RETRY_PASSES times for messages that fail due to
+  // dependency ordering issues (e.g., a RecordsWrite whose ProtocolsConfigure
+  // hasn't committed yet). Failed messages are re-fetched from the remote
+  // to obtain a fresh data stream, since ReadableStream is single-use.
+  const MAX_RETRY_PASSES = 3;
+  let pending = sorted;
+
+  for (let pass = 0; pass <= MAX_RETRY_PASSES && pending.length > 0; pass++) {
+    const failedCids: string[] = [];
+
+    for (const entry of pending) {
+      const pullReply = await agent.dwn.node.processMessage(did, entry.message, { dataStream: entry.dataStream });
+      if (!syncMessageReplyIsSuccessful(pullReply)) {
+        const cid = await getMessageCid(entry.message);
+        failedCids.push(cid);
+      }
+    }
+
+    // Re-fetch failed messages from the remote to get fresh data streams.
+    if (failedCids.length > 0) {
+      const reFetched = await fetchRemoteMessages({ did, dwnUrl, delegateDid, protocol, messageCids: failedCids, agent, permissionsApi });
+      pending = topologicalSort(reFetched);
+    } else {
+      pending = [];
+    }
+  }
+}
+
+/**
+ * Fetches messages from a remote DWN by their CIDs using MessagesRead.
+ */
+export async function fetchRemoteMessages({ did, dwnUrl, delegateDid, protocol, messageCids, agent, permissionsApi }: {
+  did: string;
+  dwnUrl: string;
+  delegateDid?: string;
+  protocol?: string;
+  messageCids: string[];
+  agent: Web5PlatformAgent;
+  permissionsApi: PermissionsApi;
+}): Promise<SyncMessageEntry[]> {
+  const results: SyncMessageEntry[] = [];
+
+  let permissionGrantId: string | undefined;
+  if (delegateDid) {
+    try {
+      const messagesReadGrant = await permissionsApi.getPermissionForRequest({
+        connectedDid : did,
+        messageType  : DwnInterface.MessagesRead,
+        delegateDid,
+        protocol,
+        cached       : true
+      });
+      permissionGrantId = messagesReadGrant.grant.id;
+    } catch (error: any) {
+      console.error('SyncEngineLevel: pull - Error fetching MessagesRead permission grant for delegate DID', error);
+      return results;
+    }
+  }
+
+  // Fetch messages in parallel with bounded concurrency.
+  const CONCURRENCY = 10;
+  let cursor = 0;
+
+  while (cursor < messageCids.length) {
+    const batch = messageCids.slice(cursor, cursor + CONCURRENCY);
+    cursor += CONCURRENCY;
+
+    type FetchResult = SyncMessageEntry | undefined;
+    const batchResults = await Promise.all(batch.map(async (messageCid): Promise<FetchResult> => {
+      const messagesRead = await agent.processDwnRequest({
+        store         : false,
+        author        : did,
+        target        : did,
+        messageType   : DwnInterface.MessagesRead,
+        granteeDid    : delegateDid,
+        messageParams : { messageCid, permissionGrantId }
+      });
+
+      let reply: MessagesReadReply;
+      try {
+        reply = await agent.rpc.sendDwnRequest({
+          dwnUrl,
+          targetDid : did,
+          message   : messagesRead.message,
+        }) as MessagesReadReply;
+      } catch (error: any) {
+        console.error(`SyncEngineLevel: pull - failed to read ${messageCid} from ${dwnUrl}:`, error.message ?? error);
+        return undefined;
+      }
+
+      if (reply.status.code !== 200 || !reply.entry?.message) {
+        return undefined;
+      }
+
+      const replyEntry = reply.entry;
+      let dataStream: ReadableStream<Uint8Array> | undefined;
+      if (isRecordsWrite(replyEntry) && replyEntry.data) {
+        dataStream = replyEntry.data;
+      }
+
+      return { message: replyEntry.message, dataStream };
+    }));
+
+    for (const result of batchResults) {
+      if (result) {
+        results.push(result);
+      }
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Reads missing messages from the local DWN and pushes them to the remote DWN.
+ * Messages are fetched first, then sorted in dependency order (topological sort)
+ * so that initial writes come before updates, and ProtocolsConfigures come before
+ * records that reference those protocols.
+ */
+export async function pushMessages({ did, dwnUrl, delegateDid, protocol, messageCids, agent, permissionsApi }: {
+  did: string;
+  dwnUrl: string;
+  delegateDid?: string;
+  protocol?: string;
+  messageCids: string[];
+  agent: Web5PlatformAgent;
+  permissionsApi: PermissionsApi;
+}): Promise<void> {
+  // Step 1: Fetch all local messages (streams are pull-based, not yet consumed).
+  const fetched: SyncMessageEntry[] = [];
+  for (const messageCid of messageCids) {
+    const dwnMessage = await getLocalMessage({ author: did, messageCid, delegateDid, protocol, agent, permissionsApi });
+    if (dwnMessage) {
+      fetched.push(dwnMessage);
+    }
+  }
+
+  // Step 2: Sort in dependency order using topological sort.
+  const sorted = topologicalSort(fetched);
+
+  // Step 3: Push messages in dependency order, consuming each stream as we go.
+  for (const entry of sorted) {
+    try {
+      const reply = await agent.rpc.sendDwnRequest({
+        dwnUrl,
+        targetDid : did,
+        data      : entry.dataStream,
+        message   : entry.message
+      });
+
+      if (!syncMessageReplyIsSuccessful(reply)) {
+        const cid = await getMessageCid(entry.message);
+        console.error(`SyncEngineLevel: push failed for ${cid}: ${reply.status.code} ${reply.status.detail}`);
+      }
+    } catch (error: any) {
+      // Preserve the original error so callers can diagnose the root cause.
+      const detail = error.message ?? error;
+      throw new Error(`SyncEngineLevel: push to ${dwnUrl} failed: ${detail}`);
+    }
+  }
+}
+
+/**
+ * Reads a message from the local DWN by its CID using MessagesRead.
+ */
+export async function getLocalMessage({ author, delegateDid, protocol, messageCid, agent, permissionsApi }: {
+  author: string;
+  delegateDid?: string;
+  protocol?: string;
+  messageCid: string;
+  agent: Web5PlatformAgent;
+  permissionsApi: PermissionsApi;
+}): Promise<SyncMessageEntry | undefined> {
+  let permissionGrantId: string | undefined;
+  if (delegateDid) {
+    try {
+      const messagesReadGrant = await permissionsApi.getPermissionForRequest({
+        connectedDid : author,
+        messageType  : DwnInterface.MessagesRead,
+        delegateDid,
+        protocol,
+        cached       : true
+      });
+      permissionGrantId = messagesReadGrant.grant.id;
+    } catch (error: any) {
+      console.error('SyncEngineLevel: push - Error fetching MessagesRead permission grant for delegate DID', error);
+      return;
+    }
+  }
+
+  const { reply } = await agent.dwn.processRequest({
+    author,
+    target        : author,
+    messageType   : DwnInterface.MessagesRead,
+    granteeDid    : delegateDid,
+    messageParams : { messageCid, permissionGrantId }
+  });
+
+  if (reply.status.code !== 200 || !reply.entry) {
+    return undefined;
+  }
+  const messageEntry = reply.entry!;
+
+  const result: SyncMessageEntry = {
+    message: messageEntry.message
+  };
+
+  if (isRecordsWrite(messageEntry) && messageEntry.data) {
+    result.dataStream = messageEntry.data;
+  }
+
+  return result;
+}

package/src/sync-topological-sort.ts

@@ -0,0 +1,167 @@
+import type { GenericMessage } from '@enbox/dwn-sdk-js';
+
+import { DwnInterfaceName, DwnMethodName, PermissionsProtocol } from '@enbox/dwn-sdk-js';
+
+/**
+ * Checks whether a message is an initial RecordsWrite (not an update).
+ * An initial write has dateCreated === messageTimestamp (first write for this recordId).
+ */
+function isInitialWrite(message: GenericMessage): boolean {
+  const desc = message.descriptor as any;
+  if (desc.interface !== DwnInterfaceName.Records || desc.method !== DwnMethodName.Write) {
+    return false;
+  }
+  // A RecordsWrite is initial if dateCreated === messageTimestamp (first write for this recordId).
+  return desc.dateCreated === desc.messageTimestamp;
+}
+
+/**
+ * Builds a dependency graph from the fetched messages and returns them in
+ * topological order so that dependencies are processed before dependents.
+ *
+ * Dependencies:
+ * - ProtocolsConfigure must come before any RecordsWrite using that protocol
+ * - Parent record must come before child record (via parentId)
+ * - Initial write must come before update writes (same recordId, not initial)
+ * - Permission grant must come before records using that permissionGrantId
+ */
+export function topologicalSort<T extends { message: GenericMessage }>(
+  messages: T[]
+): T[] {
+  if (messages.length <= 1) {
+    return messages;
+  }
+
+  // Index messages by various keys for dependency resolution.
+  const byIndex = new Map<number, T>();
+  const protocolConfigureIndex = new Map<string, number>(); // protocol URL -> index
+  const initialWriteIndex = new Map<string, number>(); // recordId -> index of initial write
+  const grantIndex = new Map<string, number>(); // grant recordId -> index
+
+  for (let i = 0; i < messages.length; i++) {
+    const entry = messages[i];
+    byIndex.set(i, entry);
+    const desc = entry.message.descriptor;
+
+    if (desc.interface === DwnInterfaceName.Protocols && desc.method === DwnMethodName.Configure) {
+      const protocolUrl = (desc as any).definition?.protocol;
+      if (protocolUrl) {
+        protocolConfigureIndex.set(protocolUrl, i);
+      }
+    }
+
+    if (desc.interface === DwnInterfaceName.Records && desc.method === DwnMethodName.Write) {
+      const recordId = (entry.message as any).recordId;
+      const initial = isInitialWrite(entry.message);
+      if (initial && recordId) {
+        initialWriteIndex.set(recordId, i);
+      }
+
+      // Index permission grants by recordId so dependents can reference them.
+      if (
+        (desc as any).protocol === PermissionsProtocol.uri &&
+        (desc as any).protocolPath === PermissionsProtocol.grantPath &&
+        recordId
+      ) {
+        grantIndex.set(recordId, i);
+      }
+    }
+  }
+
+  // Build adjacency list (edges: dependency -> dependent).
+  const edges = new Map<number, Set<number>>();
+  const inDegree = new Array(messages.length).fill(0) as number[];
+
+  const addEdge = (from: number, to: number): void => {
+    if (from === to) {
+      return;
+    }
+    if (!edges.has(from)) {
+      edges.set(from, new Set());
+    }
+    const edgeSet = edges.get(from)!;
+    if (!edgeSet.has(to)) {
+      edgeSet.add(to);
+      inDegree[to]++;
+    }
+  };
+
+  for (let i = 0; i < messages.length; i++) {
+    const desc = messages[i].message.descriptor;
+
+    // Protocol dependency: RecordsWrite depends on ProtocolsConfigure for its protocol.
+    if (desc.interface === DwnInterfaceName.Records) {
+      const protocol = (desc as any).protocol;
+      if (protocol && protocolConfigureIndex.has(protocol)) {
+        addEdge(protocolConfigureIndex.get(protocol)!, i);
+      }
+    }
+
+    // Parent dependency: child record depends on parent record.
+    if (desc.interface === DwnInterfaceName.Records && (desc as any).parentId) {
+      const parentId = (desc as any).parentId;
+      if (initialWriteIndex.has(parentId)) {
+        addEdge(initialWriteIndex.get(parentId)!, i);
+      }
+    }
+
+    // Initial write dependency: update depends on initial write.
+    if (desc.interface === DwnInterfaceName.Records && desc.method === DwnMethodName.Write) {
+      const recordId = (messages[i].message as any).recordId;
+      if (recordId && !isInitialWrite(messages[i].message) && initialWriteIndex.has(recordId)) {
+        addEdge(initialWriteIndex.get(recordId)!, i);
+      }
+    }
+
+    // Delete depends on initial write.
+    if (desc.interface === DwnInterfaceName.Records && desc.method === DwnMethodName.Delete) {
+      const recordId = (desc as any).recordId;
+      if (recordId && initialWriteIndex.has(recordId)) {
+        addEdge(initialWriteIndex.get(recordId)!, i);
+      }
+    }
+
+    // Permission grant dependency: message depends on the grant it references.
+    const permissionGrantId = (desc as any).permissionGrantId;
+    if (permissionGrantId && grantIndex.has(permissionGrantId)) {
+      addEdge(grantIndex.get(permissionGrantId)!, i);
+    }
+  }
+
+  // Kahn's algorithm for topological sort.
+  const queue: number[] = [];
+  for (let i = 0; i < messages.length; i++) {
+    if (inDegree[i] === 0) {
+      queue.push(i);
+    }
+  }
+
+  const sorted: T[] = [];
+  while (queue.length > 0) {
+    const node = queue.shift()!;
+    sorted.push(byIndex.get(node)!);
+
+    const neighbors = edges.get(node);
+    if (neighbors) {
+      for (const neighbor of neighbors) {
+        inDegree[neighbor]--;
+        if (inDegree[neighbor] === 0) {
+          queue.push(neighbor);
+        }
+      }
+    }
+  }
+
+  // If there are nodes not in sorted (cycle), append them at the end.
+  if (sorted.length < messages.length) {
+    const sortedSet = new Set(sorted);
+    for (let i = 0; i < messages.length; i++) {
+      const entry = byIndex.get(i)!;
+      if (!sortedSet.has(entry)) {
+        sorted.push(entry);
+      }
+    }
+  }
+
+  return sorted;
+}

package/src/test-harness.ts

@@ -128,6 +128,25 @@ export class PlatformAgentTestHarness {
     }
   }
 
+  /**
+   * Clear only DWN-level stores (data, messages, state index, resumable tasks,
+   * sync, permissions) and the DWN-backed store caches — but preserve the
+   * agent DID, vault, and all key/DID/identity material.
+   *
+   * Use this in `beforeEach` when `createAgentDid()` (and optionally
+   * `createIdentity()`) was called once in `beforeAll` to avoid expensive
+   * DID re-creation on every test.
+   */
+  public async clearDwnStores(): Promise<void> {
+    await this.syncStore.clear();
+    await this.dwnDataStore.clear();
+    await this.dwnStateIndex.clear();
+    await this.dwnMessageStore.clear();
+    await this.dwnResumableTaskStore.clear();
+    await this.agent.permissions.clear();
+    this.dwnStores.clear();
+  }
+
   public async closeStorage(): Promise<void> {
     await this.didResolverCache.close();
     await this.dwnDataStore.close();

package/src/types/permissions.ts

@@ -7,6 +7,8 @@ export type FetchPermissionsParams = {
   grantor?: string;
   protocol?: string;
   remote?: boolean;
+  /** Whether to filter out revoked grants. Defaults to `true`. */
+  checkRevoked?: boolean;
 };
 
 export type FetchPermissionRequestParams = {