@enbox/dwn-sdk-js 0.2.0 → 0.2.2

package/src/handlers/messages-sync.ts

@@ -1,8 +1,10 @@
+import type { GenericMessage } from '../types/message-types.js';
 import type { MessageStore } from '../types/message-store.js';
 import type { HandlerDependencies, MethodHandler } from '../types/method-handler.js';
-import type { MessagesSyncMessage, MessagesSyncReply } from '../types/messages-types.js';
+import type { MessagesSyncDiffEntry, MessagesSyncMessage, MessagesSyncReply } from '../types/messages-types.js';
 
 import { authenticate } from '../core/auth.js';
+import { Encoder } from '../utils/encoder.js';
 import { hashToHex } from '../smt/smt-utils.js';
 import { messageReplyFromError } from '../core/message-reply.js';
 import { MessagesGrantAuthorization } from '../core/messages-grant-authorization.js';
@@ -10,6 +12,13 @@ import { MessagesSync } from '../interfaces/messages-sync.js';
 import { PermissionsProtocol } from '../protocols/permissions.js';
 import { DwnError, DwnErrorCode } from '../core/dwn-error.js';
 
+/**
+ * Default maximum inline data size for diff responses (256 KB).
+ * RecordsWrite data payloads smaller than this are base64url-encoded and
+ * included directly in the diff reply, avoiding a separate MessagesRead round-trip.
+ */
+const DEFAULT_MAX_INLINE_DATA_SIZE = 262_144;
+
 
 export class MessagesSyncHandler implements MethodHandler {
 
@@ -70,6 +79,10 @@ export class MessagesSyncHandler implements MethodHandler {
         };
       }
 
+      case 'diff': {
+        return this.handleDiff(tenant, message);
+      }
+
       default: {
         return {
           status: { code: 400, detail: `Unknown action: ${action as string}` },
@@ -81,6 +94,272 @@ export class MessagesSyncHandler implements MethodHandler {
     }
   }
 
+  /**
+   * Handle the 'diff' action: the client sends its subtree hashes at a given
+   * depth, and the server compares them against its own tree to compute the
+   * set difference in a single round-trip.
+   *
+   * Response includes:
+   * - `onlyRemote`: messages the server has that the client doesn't, with
+   *   inline data for small payloads.
+   * - `onlyLocal`: bit prefixes where the client has entries the server
+   *   doesn't (client can enumerate its own leaves for these prefixes).
+   */
+  private async handleDiff(
+    tenant: string,
+    message: MessagesSyncMessage,
+  ): Promise<MessagesSyncReply> {
+    const { protocol, hashes: clientHashes, depth } = message.descriptor;
+
+    if (!clientHashes || depth === undefined) {
+      return {
+        status: { code: 400, detail: 'diff action requires hashes and depth' },
+      };
+    }
+
+    const stateIndex = this.deps.stateIndex!;
+    const onlyRemoteCids: string[] = [];
+    const onlyLocalPrefixes: string[] = [];
+
+    // Get the default (empty subtree) hash at the given depth so we can
+    // filter out client entries that represent empty subtrees.
+    const defaultHashHex = await this.getDefaultHashHex(depth);
+
+    // Build the set of all prefixes at the given depth that either side has.
+    // Filter out client prefixes whose hash equals the default (empty subtree)
+    // hash — these represent empty subtrees and should be treated the same as
+    // omitted prefixes.
+    const allPrefixes = new Set<string>();
+    for (const [pfx, hash] of Object.entries(clientHashes)) {
+      if (hash !== defaultHashHex) {
+        allPrefixes.add(pfx);
+      }
+    }
+
+    // Enumerate server-side non-empty prefixes by walking the server's
+    // tree to the given depth and collecting leaf prefixes.
+    const serverHashes = await this.collectSubtreeHashes(tenant, protocol, depth);
+    for (const prefix of Object.keys(serverHashes)) {
+      allPrefixes.add(prefix);
+    }
+
+    // Compare each prefix's hash between client and server.
+    for (const pfx of allPrefixes) {
+      const clientHash = clientHashes[pfx]; // undefined if client has empty subtree
+      const serverHash = serverHashes[pfx]; // undefined if server has empty subtree
+
+      if (clientHash === serverHash) {
+        // Identical subtree — skip.
+        continue;
+      }
+
+      if (serverHash === undefined) {
+        // Client has entries the server doesn't.
+        onlyLocalPrefixes.push(pfx);
+        continue;
+      }
+
+      if (clientHash === undefined) {
+        // Server has entries the client doesn't — enumerate server leaves.
+        const bitPath = MessagesSyncHandler.parseBitPrefix(pfx);
+        const leaves = protocol !== undefined
+          ? await stateIndex.getProtocolLeaves(tenant, protocol, bitPath)
+          : await stateIndex.getLeaves(tenant, bitPath);
+        onlyRemoteCids.push(...leaves);
+        continue;
+      }
+
+      // Both sides have entries but they differ — enumerate both and set-diff.
+      const bitPath = MessagesSyncHandler.parseBitPrefix(pfx);
+      const serverLeaves = protocol !== undefined
+        ? await stateIndex.getProtocolLeaves(tenant, protocol, bitPath)
+        : await stateIndex.getLeaves(tenant, bitPath);
+
+      // We don't have the client's leaves, so we report all server leaves
+      // as onlyRemote (the client will de-duplicate locally). We also
+      // report this prefix as onlyLocal so the client can check its own leaves.
+      onlyRemoteCids.push(...serverLeaves);
+      onlyLocalPrefixes.push(pfx);
+    }
+
+    // Build response entries with inline message data where possible.
+    const onlyRemote = await this.buildDiffEntries(tenant, onlyRemoteCids);
+
+    return {
+      status    : { code: 200, detail: 'OK' },
+      onlyRemote,
+      onlyLocal : onlyLocalPrefixes,
+    };
+  }
+
+  /**
+   * Walk the server's SMT to the given depth and collect all non-empty
+   * subtree hashes as a `{ prefix: hexHash }` map.
+   */
+  private async collectSubtreeHashes(
+    tenant: string,
+    protocol: string | undefined,
+    depth: number,
+  ): Promise<Record<string, string>> {
+    const stateIndex = this.deps.stateIndex!;
+    const result: Record<string, string> = {};
+    const defaultHashHex = await this.getDefaultHashHex(depth);
+
+    const walk = async (prefix: string, currentDepth: number): Promise<void> => {
+      const bitPath = MessagesSyncHandler.parseBitPrefix(prefix);
+      const hash = protocol !== undefined
+        ? await stateIndex.getProtocolSubtreeHash(tenant, protocol, bitPath)
+        : await stateIndex.getSubtreeHash(tenant, bitPath);
+      const hexHash = hashToHex(hash);
+
+      if (hexHash === defaultHashHex) {
+        // Empty subtree — don't include in the result.
+        return;
+      }
+
+      if (currentDepth >= depth) {
+        // Reached target depth with a non-empty subtree.
+        result[prefix] = hexHash;
+        return;
+      }
+
+      // Recurse into children.
+      await Promise.all([
+        walk(prefix + '0', currentDepth + 1),
+        walk(prefix + '1', currentDepth + 1),
+      ]);
+    };
+
+    await walk('', 0);
+    return result;
+  }
+
+  /**
+   * Get the hex-encoded default hash for a given depth. Lazily cached.
+   */
+  private _defaultHashHexCache?: Map<number, string>;
+  private async getDefaultHashHex(depth: number): Promise<string> {
+    if (this._defaultHashHexCache === undefined) {
+      const { initDefaultHashes } = await import('../smt/smt-utils.js');
+      const defaults = await initDefaultHashes();
+      this._defaultHashHexCache = new Map<number, string>();
+      for (let d = 0; d <= 256; d++) {
+        this._defaultHashHexCache.set(d, hashToHex(defaults[d]));
+      }
+    }
+    return this._defaultHashHexCache.get(depth) ?? '';
+  }
+
+  /**
+   * Build diff response entries for the given messageCids.
+   * Reads each message from the MessageStore and, for small RecordsWrite
+   * payloads, inlines the data as base64url.
+   */
+  private async buildDiffEntries(
+    tenant: string,
+    messageCids: string[],
+  ): Promise<MessagesSyncDiffEntry[]> {
+    const entries: MessagesSyncDiffEntry[] = [];
+
+    for (const messageCid of messageCids) {
+      const { message, encodedData: inlineData, data } = await this.readMessageByCid(tenant, messageCid);
+      if (!message) {
+        // Message was deleted between diff computation and read — skip.
+        continue;
+      }
+
+      const entry: MessagesSyncDiffEntry = { messageCid, message };
+
+      // Use inline data from the MessageStore if available (small payloads).
+      if (inlineData) {
+        entry.encodedData = inlineData;
+      } else if (data) {
+        // Data is in the DataStore — inline it if small enough.
+        const bytes = await MessagesSyncHandler.streamToBytes(data);
+        if (bytes.byteLength <= DEFAULT_MAX_INLINE_DATA_SIZE) {
+          entry.encodedData = Encoder.bytesToBase64Url(bytes);
+        }
+        // Large payloads are NOT inlined — client fetches via MessagesRead.
+      }
+
+      entries.push(entry);
+    }
+
+    return entries;
+  }
+
+  /**
+   * Read a message and its data from the MessageStore + DataStore by CID.
+   */
+  private async readMessageByCid(
+    tenant: string,
+    messageCid: string,
+  ): Promise<{ message?: GenericMessage; encodedData?: string; data?: ReadableStream<Uint8Array> }> {
+    const storedMessage = await this.deps.messageStore.get(tenant, messageCid);
+    if (!storedMessage) {
+      return {};
+    }
+
+    // Extract and strip `encodedData` from the stored message.
+    // `encodedData` is an internal storage optimization for small payloads
+    // that are stored inline in the MessageStore rather than in the DataStore.
+    // It must not be included in the wire-format message (the recipient's
+    // DWN would reject it as an unexpected top-level property).
+    let inlineEncodedData: string | undefined;
+    if ('encodedData' in storedMessage) {
+      inlineEncodedData = (storedMessage as any).encodedData as string;
+      delete (storedMessage as any).encodedData;
+    }
+
+    let data: ReadableStream<Uint8Array> | undefined;
+
+    // Check if this is a RecordsWrite with data.
+    if (storedMessage.descriptor.interface === 'Records' &&
+        storedMessage.descriptor.method === 'Write') {
+      const dataCid = (storedMessage as any).descriptor?.dataCid as string | undefined;
+      if (dataCid) {
+        // Try DataStore first (for large payloads stored externally).
+        if (this.deps.dataStore) {
+          // DataStore uses recordId, not messageCid. For RecordsWrite, the
+          // recordId is in the descriptor.
+          const recordId = (storedMessage as any).descriptor?.recordId as string | undefined;
+          if (recordId) {
+            const dataResult = await this.deps.dataStore.get(tenant, recordId, dataCid);
+            if (dataResult?.dataStream) {
+              data = dataResult.dataStream;
+            }
+          }
+        }
+      }
+    }
+
+    return { message: storedMessage, encodedData: inlineEncodedData, data };
+  }
+
+  /**
+   * Read a ReadableStream to completion and return the bytes.
+   */
+  private static async streamToBytes(stream: ReadableStream<Uint8Array>): Promise<Uint8Array> {
+    const reader = stream.getReader();
+    const chunks: Uint8Array[] = [];
+    let totalSize = 0;
+
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) { break; }
+      chunks.push(value);
+      totalSize += value.byteLength;
+    }
+
+    const result = new Uint8Array(totalSize);
+    let offset = 0;
+    for (const chunk of chunks) {
+      result.set(chunk, offset);
+      offset += chunk.byteLength;
+    }
+    return result;
+  }
+
   /**
    * Parse a bit prefix string (e.g. "0110101") into a boolean array.
    */

package/src/handlers/protocols-configure.ts

@@ -54,6 +54,16 @@ export class ProtocolsConfigureHandler implements MethodHandler {
     };
     const { messages: existingMessages } = await this.deps.messageStore.query(tenant, [ query ]);
 
+    // If the exact same message already exists, return 409 immediately.
+    // This prevents duplicate key violations in the MessageStore and StateIndex
+    // when sync pushes a message that the remote already has.
+    const incomingCid = await Message.getCid(message);
+    for (const existing of existingMessages) {
+      if (await Message.getCid(existing) === incomingCid) {
+        return { status: { code: 409, detail: 'Conflict' } };
+      }
+    }
+
     // find newest message, and if the incoming message is the newest
     let newestMessage = await Message.getNewestMessage(existingMessages);
     let incomingMessageIsNewest = false;

package/src/index.ts

@@ -2,7 +2,7 @@
 export type { DwnConfig } from './dwn.js';
 export type { EventListener, EventLog, EventLogEntry, EventLogReadOptions, EventLogReadResult, EventLogSubscribeOptions, EventSubscription, MessageEvent, SubscriptionEose, SubscriptionEvent, SubscriptionListener, SubscriptionMessage, SubscriptionReply } from './types/subscriptions.js';
 export type { AuthorizationModel, Descriptor, DelegatedGrantRecordsWriteMessage, GenericMessage, GenericMessageReply, GenericSignaturePayload, MessageSort, MessageSubscription, Pagination, QueryResultEntry, Status } from './types/message-types.js';
-export type { MessagesFilter, MessagesReadMessage as MessagesReadMessage, MessagesReadReply as MessagesReadReply, MessagesReadReplyEntry as MessagesReadReplyEntry, MessagesReadDescriptor, MessagesSubscribeDescriptor, MessagesSubscribeMessage, MessagesSubscribeReply, MessagesSubscribeMessageOptions, MessagesSyncAction, MessagesSyncDescriptor, MessagesSyncMessage, MessagesSyncReply } from './types/messages-types.js';
+export type { MessagesFilter, MessagesReadMessage as MessagesReadMessage, MessagesReadReply as MessagesReadReply, MessagesReadReplyEntry as MessagesReadReplyEntry, MessagesReadDescriptor, MessagesSubscribeDescriptor, MessagesSubscribeMessage, MessagesSubscribeReply, MessagesSubscribeMessageOptions, MessagesSyncAction, MessagesSyncDescriptor, MessagesSyncDiffEntry, MessagesSyncMessage, MessagesSyncReply } from './types/messages-types.js';
 export type { GT, LT, Filter, FilterValue, KeyValues, EqualFilter, OneOfFilter, RangeFilter, RangeCriterion, PaginationCursor, QueryOptions, RangeValue, StartsWithFilter } from './types/query-types.js';
 export type { ProtocolsConfigureDescriptor, ProtocolDefinition, ProtocolTypes, ProtocolRuleSet, ProtocolsQueryFilter, ProtocolsConfigureMessage, ProtocolsQueryMessage, ProtocolsQueryReply, ProtocolActionRule, ProtocolDeliveryStrategy, ProtocolPathEncryption, ProtocolsQueryDescriptor, ProtocolRecordLimitDefinition, ProtocolSizeDefinition, ProtocolTagsDefinition, ProtocolTagSchema, ProtocolType, ProtocolUses } from './types/protocols-types.js';
 export { ProtocolRecordLimitStrategy } from './types/protocols-types.js';

package/src/interfaces/messages-sync.ts

@@ -15,6 +15,10 @@ export type MessagesSyncOptions = {
   prefix? : string;
   messageTimestamp? : string;
   permissionGrantId? : string;
+  /** For `action: 'diff'`: client's subtree hashes at `depth`. */
+  hashes? : Record<string, string>;
+  /** For `action: 'diff'`: bit depth at which hashes were computed. */
+  depth? : number;
 };
 
 export class MessagesSync extends AbstractMessage<MessagesSyncMessage> {
@@ -39,6 +43,8 @@ export class MessagesSync extends AbstractMessage<MessagesSyncMessage> {
       protocol          : options.protocol,
       prefix            : options.prefix,
       permissionGrantId : options.permissionGrantId,
+      hashes            : options.hashes,
+      depth             : options.depth,
     };
 
     removeUndefinedProperties(descriptor);

package/src/types/messages-types.ts

@@ -36,7 +36,7 @@ export type MessagesReadReply = GenericMessageReply & {
   entry?: MessagesReadReplyEntry;
 };
 
-export type MessagesSyncAction = 'root' | 'subtree' | 'leaves';
+export type MessagesSyncAction = 'root' | 'subtree' | 'leaves' | 'diff';
 
 export type MessagesSyncDescriptor = {
   interface : DwnInterfaceName.Messages;
@@ -46,6 +46,20 @@ export type MessagesSyncDescriptor = {
   protocol? : string; // optional protocol scope
   prefix? : string; // bit path for subtree/leaves (e.g. "0110101...")
   permissionGrantId? : string;
+  /**
+   * For `action: 'diff'`: a map of `{ bitPrefix: hexHash }` representing the client's
+   * subtree hashes at `depth`. The server compares each hash against its own tree
+   * and returns the set difference in a single round-trip.
+   *
+   * Only non-default (non-empty) subtree hashes need to be included — prefixes
+   * whose hash equals the default empty-subtree hash may be omitted.
+   */
+  hashes? : Record<string, string>;
+  /**
+   * For `action: 'diff'`: the bit depth at which the client computed its subtree
+   * hashes. Required when `action` is `'diff'`.
+   */
+  depth? : number;
 };
 
 export type MessagesSyncMessage = GenericMessage & {
@@ -53,10 +67,26 @@ export type MessagesSyncMessage = GenericMessage & {
   descriptor : MessagesSyncDescriptor;
 };
 
+/**
+ * Entry in a diff response representing a message the server has that the
+ * client does not (or vice versa). Optionally includes the full message
+ * and inline data for small payloads.
+ */
+export type MessagesSyncDiffEntry = {
+  messageCid : string;
+  message? : GenericMessage;
+  /** Base64url-encoded data for small RecordsWrite payloads (≤ maxInlineDataSize). */
+  encodedData? : string;
+};
+
 export type MessagesSyncReply = GenericMessageReply & {
   root? : string; // hex-encoded root hash (for 'root' action)
   hash? : string; // hex-encoded subtree hash (for 'subtree' action)
   entries? : string[]; // messageCid[] (for 'leaves' action)
+  /** For 'diff' action: messageCids (or full messages) that the server has but the client doesn't. */
+  onlyRemote? : MessagesSyncDiffEntry[];
+  /** For 'diff' action: bit prefixes where the client has entries the server doesn't. */
+  onlyLocal? : string[];
 };
 
 export type MessagesSubscribeMessageOptions = {