@enbox/dwn-sdk-js 0.3.7 → 0.3.8

package/src/handlers/messages-sync.ts

@@ -1,17 +1,24 @@
 import type { GenericMessage } from '../types/message-types.js';
 import type { MessageStore } from '../types/message-store.js';
+import type { StateIndex } from '../types/state-index.js';
 import type { HandlerDependencies, MethodHandler } from '../types/method-handler.js';
-import type { MessagesSyncDiffEntry, MessagesSyncMessage, MessagesSyncReply } from '../types/messages-types.js';
+import type { MessagesSyncDependencyEntry, MessagesSyncDiffEntry, MessagesSyncMessage, MessagesSyncReply } from '../types/messages-types.js';
+import type { RecordsProjectionScope, RecordsProjectionSnapshot } from '../sync/records-projection.js';
 
 import { authenticate } from '../core/auth.js';
 import { DwnConstant } from '../core/dwn-constant.js';
 import { Encoder } from '../utils/encoder.js';
 import { hashToHex } from '../smt/smt-utils.js';
+import { Message } from '../core/message.js';
 import { messageReplyFromError } from '../core/message-reply.js';
 import { MessagesGrantAuthorization } from '../core/messages-grant-authorization.js';
 import { MessagesSync } from '../interfaces/messages-sync.js';
-import { PermissionsProtocol } from '../protocols/permissions.js';
+import { Records } from '../utils/records.js';
+import { RecordsProjection } from '../sync/records-projection.js';
+import { RecordsWrite } from '../interfaces/records-write.js';
+import { SortDirection } from '../types/query-types.js';
 import { DwnError, DwnErrorCode } from '../core/dwn-error.js';
+import { DwnInterfaceName, DwnMethodName } from '../enums/dwn-interface-method.js';
 
 /**
  * Maximum inline data size for diff responses — aligned with the
@@ -22,6 +29,21 @@ import { DwnError, DwnErrorCode } from '../core/dwn-error.js';
  */
 const DEFAULT_MAX_INLINE_DATA_SIZE = DwnConstant.maxDataSizeAllowedToBeEncoded;
 
+type StoredMessageWithEncodedData = GenericMessage & { encodedData?: string };
+type StoredMessageWithInternalFields = GenericMessage & {
+  encodedData?: unknown;
+  initialWrite?: unknown;
+};
+type RecordsWriteProtocolDescriptor = GenericMessage['descriptor'] & { protocol?: unknown };
+type RecordsWriteProtocolMetadata = { protocol: string; messageTimestamp: string };
+type ProtocolsConfigureDefinitionDescriptor = GenericMessage['descriptor'] & {
+  definition?: {
+    protocol?: unknown;
+    uses?: Record<string, unknown>;
+  };
+};
+type ProjectionScopes = readonly [RecordsProjectionScope, ...RecordsProjectionScope[]];
+
 
 export class MessagesSyncHandler implements MethodHandler {
 
@@ -46,44 +68,25 @@ export class MessagesSyncHandler implements MethodHandler {
       return messageReplyFromError(e, 401);
     }
 
-    const { action, protocol, prefix } = message.descriptor;
+    const { action } = message.descriptor;
+    const projectionScopes = MessagesSyncHandler.getProjectionScopes(message);
 
     try {
       switch (action) {
       case 'root': {
-        const rootHash = protocol === undefined
-          ? await this.deps.stateIndex!.getRoot(tenant)
-          : await this.deps.stateIndex!.getProtocolRoot(tenant, protocol);
-        return {
-          status : { code: 200, detail: 'OK' },
-          root   : hashToHex(rootHash),
-        };
+        return await this.handleRoot(tenant, message, projectionScopes);
       }
 
       case 'subtree': {
-        const bitPath = MessagesSyncHandler.parseBitPrefix(prefix!);
-        const hash = protocol === undefined
-          ? await this.deps.stateIndex!.getSubtreeHash(tenant, bitPath)
-          : await this.deps.stateIndex!.getProtocolSubtreeHash(tenant, protocol, bitPath);
-        return {
-          status : { code: 200, detail: 'OK' },
-          hash   : hashToHex(hash),
-        };
+        return await this.handleSubtree(tenant, message, projectionScopes);
       }
 
       case 'leaves': {
-        const bitPath = MessagesSyncHandler.parseBitPrefix(prefix!);
-        const leaves = protocol === undefined
-          ? await this.deps.stateIndex!.getLeaves(tenant, bitPath)
-          : await this.deps.stateIndex!.getProtocolLeaves(tenant, protocol, bitPath);
-        return {
-          status  : { code: 200, detail: 'OK' },
-          entries : leaves,
-        };
+        return await this.handleLeaves(tenant, message, projectionScopes);
       }
 
       case 'diff': {
-        return this.handleDiff(tenant, message);
+        return await this.handleDiff(tenant, message);
       }
 
       default: {
@@ -97,6 +100,112 @@ export class MessagesSyncHandler implements MethodHandler {
     }
   }
 
+  private async handleRoot(
+    tenant: string,
+    message: MessagesSyncMessage,
+    projectionScopes: ProjectionScopes | undefined
+  ): Promise<MessagesSyncReply> {
+    const root = await this.getRootHex(tenant, message.descriptor.protocol, projectionScopes);
+    return {
+      status : { code: 200, detail: 'OK' },
+      root   : root,
+    };
+  }
+
+  private async getRootHex(
+    tenant: string,
+    protocol: string | undefined,
+    projectionScopes: ProjectionScopes | undefined
+  ): Promise<string> {
+    if (projectionScopes === undefined) {
+      const rootHash = await this.getIndexedRootHash(tenant, protocol);
+      return hashToHex(rootHash);
+    }
+
+    return this.withProjectionSnapshot(tenant, projectionScopes, snapshot => snapshot.getRootHex());
+  }
+
+  private async getIndexedRootHash(
+    tenant: string,
+    protocol: string | undefined
+  ): Promise<Uint8Array> {
+    if (protocol === undefined) {
+      return this.deps.stateIndex!.getRoot(tenant);
+    }
+
+    return this.deps.stateIndex!.getProtocolRoot(tenant, protocol);
+  }
+
+  private async handleSubtree(
+    tenant: string,
+    message: MessagesSyncMessage,
+    projectionScopes: ProjectionScopes | undefined
+  ): Promise<MessagesSyncReply> {
+    const bitPath = MessagesSyncHandler.parseBitPrefix(message.descriptor.prefix!);
+    const hash = await this.getSubtreeHash(tenant, message.descriptor.protocol, projectionScopes, bitPath);
+
+    return {
+      status : { code: 200, detail: 'OK' },
+      hash   : hashToHex(hash),
+    };
+  }
+
+  private async handleLeaves(
+    tenant: string,
+    message: MessagesSyncMessage,
+    projectionScopes: ProjectionScopes | undefined
+  ): Promise<MessagesSyncReply> {
+    const bitPath = MessagesSyncHandler.parseBitPrefix(message.descriptor.prefix!);
+    const leaves = await this.getLeaves(tenant, message.descriptor.protocol, projectionScopes, bitPath);
+
+    return {
+      status  : { code: 200, detail: 'OK' },
+      entries : leaves,
+    };
+  }
+
+  private async getSubtreeHash(
+    tenant: string,
+    protocol: string | undefined,
+    projectionScopes: ProjectionScopes | undefined,
+    bitPath: boolean[]
+  ): Promise<Uint8Array> {
+    if (projectionScopes === undefined) {
+      return this.getIndexedSubtreeHash(tenant, protocol, bitPath);
+    }
+
+    return this.withProjectionSnapshot(tenant, projectionScopes, snapshot => snapshot.getSubtreeHash(bitPath));
+  }
+
+  private async getLeaves(
+    tenant: string,
+    protocol: string | undefined,
+    projectionScopes: ProjectionScopes | undefined,
+    bitPath: boolean[]
+  ): Promise<string[]> {
+    if (projectionScopes === undefined) {
+      return this.getIndexedLeaves(tenant, protocol, bitPath);
+    }
+
+    return this.withProjectionSnapshot(tenant, projectionScopes, snapshot => snapshot.getLeaves(bitPath));
+  }
+
+  private async getIndexedSubtreeHash(
+    tenant: string,
+    protocol: string | undefined,
+    bitPath: boolean[]
+  ): Promise<Uint8Array> {
+    return MessagesSyncHandler.getIndexedSubtreeHashFromStateIndex(this.deps.stateIndex!, tenant, protocol, bitPath);
+  }
+
+  private async getIndexedLeaves(
+    tenant: string,
+    protocol: string | undefined,
+    bitPath: boolean[]
+  ): Promise<string[]> {
+    return MessagesSyncHandler.getIndexedLeavesFromStateIndex(this.deps.stateIndex!, tenant, protocol, bitPath);
+  }
+
   /**
    * Handle the 'diff' action: the client sends its subtree hashes at a given
    * depth, and the server compares them against its own tree to compute the
@@ -113,6 +222,7 @@ export class MessagesSyncHandler implements MethodHandler {
     message: MessagesSyncMessage,
   ): Promise<MessagesSyncReply> {
     const { protocol, hashes: clientHashes, depth } = message.descriptor;
+    const projectionScopes = MessagesSyncHandler.getProjectionScopes(message);
 
     if (!clientHashes || depth === undefined) {
       return {
@@ -121,77 +231,80 @@ export class MessagesSyncHandler implements MethodHandler {
     }
 
     const stateIndex = this.deps.stateIndex!;
+    const projectionSnapshot = await this.createProjectionSnapshot(tenant, projectionScopes);
     const onlyRemoteCids: string[] = [];
     const onlyLocalPrefixes: string[] = [];
 
-    // Get the default (empty subtree) hash at the given depth so we can
-    // filter out client entries that represent empty subtrees.
-    const defaultHashHex = await this.getDefaultHashHex(depth);
-
-    // Build the set of all prefixes at the given depth that either side has.
-    // Filter out client prefixes whose hash equals the default (empty subtree)
-    // hash — these represent empty subtrees and should be treated the same as
-    // omitted prefixes.
-    const allPrefixes = new Set<string>();
-    for (const [pfx, hash] of Object.entries(clientHashes)) {
-      if (hash !== defaultHashHex) {
-        allPrefixes.add(pfx);
+    try {
+      // Get the default (empty subtree) hash at the given depth so we can
+      // filter out client entries that represent empty subtrees.
+      const defaultHashHex = await this.getDefaultHashHex(depth);
+
+      // Build the set of all prefixes at the given depth that either side has.
+      // Filter out client prefixes whose hash equals the default (empty subtree)
+      // hash — these represent empty subtrees and should be treated the same as
+      // omitted prefixes.
+      const allPrefixes = new Set<string>();
+      for (const [pfx, hash] of Object.entries(clientHashes)) {
+        if (hash !== defaultHashHex) {
+          allPrefixes.add(pfx);
+        }
       }
-    }
 
-    // Enumerate server-side non-empty prefixes by walking the server's
-    // tree to the given depth and collecting leaf prefixes.
-    const serverHashes = await this.collectSubtreeHashes(tenant, protocol, depth);
-    for (const prefix of Object.keys(serverHashes)) {
-      allPrefixes.add(prefix);
-    }
+      // Enumerate server-side non-empty prefixes by walking the server's
+      // tree to the given depth and collecting leaf prefixes.
+      const serverHashes = await this.collectSubtreeHashes(tenant, protocol, projectionSnapshot, depth);
+      for (const prefix of Object.keys(serverHashes)) {
+        allPrefixes.add(prefix);
+      }
 
-    // Compare each prefix's hash between client and server.
-    for (const pfx of allPrefixes) {
-      const clientHash = clientHashes[pfx]; // undefined if client has empty subtree
-      const serverHash = serverHashes[pfx]; // undefined if server has empty subtree
+      // Compare each prefix's hash between client and server.
+      for (const pfx of allPrefixes) {
+        const clientHash = clientHashes[pfx]; // undefined if client has empty subtree
+        const serverHash = serverHashes[pfx]; // undefined if server has empty subtree
 
-      if (clientHash === serverHash) {
-        // Identical subtree — skip.
-        continue;
-      }
+        if (clientHash === serverHash) {
+          // Identical subtree — skip.
+          continue;
+        }
 
-      if (serverHash === undefined) {
-        // Client has entries the server doesn't.
-        onlyLocalPrefixes.push(pfx);
-        continue;
-      }
+        if (serverHash === undefined) {
+          // Client has entries the server doesn't.
+          onlyLocalPrefixes.push(pfx);
+          continue;
+        }
 
-      if (clientHash === undefined) {
-        // Server has entries the client doesn't — enumerate server leaves.
         const bitPath = MessagesSyncHandler.parseBitPrefix(pfx);
-        const leaves = protocol === undefined
-          ? await stateIndex.getLeaves(tenant, bitPath)
-          : await stateIndex.getProtocolLeaves(tenant, protocol, bitPath);
-        onlyRemoteCids.push(...leaves);
-        continue;
+        const serverLeaves = await MessagesSyncHandler.getServerLeaves(
+          stateIndex,
+          tenant,
+          protocol,
+          projectionSnapshot,
+          bitPath
+        );
+
+        onlyRemoteCids.push(...serverLeaves);
+        if (clientHash !== undefined) {
+          // Both sides have entries but they differ. The client will enumerate
+          // its own leaves for this prefix and de-duplicate server leaves.
+          onlyLocalPrefixes.push(pfx);
+        }
       }
-
-      // Both sides have entries but they differ — enumerate both and set-diff.
-      const bitPath = MessagesSyncHandler.parseBitPrefix(pfx);
-      const serverLeaves = protocol === undefined
-        ? await stateIndex.getLeaves(tenant, bitPath)
-        : await stateIndex.getProtocolLeaves(tenant, protocol, bitPath);
-
-      // We don't have the client's leaves, so we report all server leaves
-      // as onlyRemote (the client will de-duplicate locally). We also
-      // report this prefix as onlyLocal so the client can check its own leaves.
-      onlyRemoteCids.push(...serverLeaves);
-      onlyLocalPrefixes.push(pfx);
+    } finally {
+      await projectionSnapshot?.close();
     }
 
     // Build response entries with inline message data where possible.
     const onlyRemote = await this.buildDiffEntries(tenant, onlyRemoteCids);
+    const dependencies = projectionScopes === undefined
+      ? []
+      : await this.buildProjectedDependencyEntries(tenant, onlyRemote);
 
     return {
       status    : { code: 200, detail: 'OK' },
       onlyRemote,
       onlyLocal : onlyLocalPrefixes,
+      ...(dependencies.length > 0 ? { dependencies } : {}),
     };
   }
 
@@ -202,18 +315,23 @@ export class MessagesSyncHandler implements MethodHandler {
   private async collectSubtreeHashes(
     tenant: string,
     protocol: string | undefined,
+    projectionSnapshot: RecordsProjectionSnapshot | undefined,
     depth: number,
   ): Promise<Record<string, string>> {
     const stateIndex = this.deps.stateIndex!;
     const result: Record<string, string> = {};
-    const defaultHashHex = await this.getDefaultHashHex(depth);
 
     const walk = async (prefix: string, currentDepth: number): Promise<void> => {
       const bitPath = MessagesSyncHandler.parseBitPrefix(prefix);
-      const hash = protocol === undefined
-        ? await stateIndex.getSubtreeHash(tenant, bitPath)
-        : await stateIndex.getProtocolSubtreeHash(tenant, protocol, bitPath);
+      const hash = await MessagesSyncHandler.getServerSubtreeHash(
+        stateIndex,
+        tenant,
+        protocol,
+        projectionSnapshot,
+        bitPath
+      );
       const hexHash = hashToHex(hash);
+      const defaultHashHex = await this.getDefaultHashHex(currentDepth);
 
       if (hexHash === defaultHashHex) {
         // Empty subtree — don't include in the result.
@@ -237,6 +355,75 @@ export class MessagesSyncHandler implements MethodHandler {
     return result;
   }
 
+  private async createProjectionSnapshot(
+    tenant: string,
+    projectionScopes: ProjectionScopes | undefined
+  ): Promise<RecordsProjectionSnapshot | undefined> {
+    if (projectionScopes === undefined) {
+      return undefined;
+    }
+
+    return RecordsProjection.createSnapshot({
+      tenant,
+      messageStore : this.deps.messageStore,
+      scopes       : projectionScopes,
+    });
+  }
+
+  private static async getServerLeaves(
+    stateIndex: StateIndex,
+    tenant: string,
+    protocol: string | undefined,
+    projectionSnapshot: RecordsProjectionSnapshot | undefined,
+    bitPath: boolean[]
+  ): Promise<string[]> {
+    if (projectionSnapshot === undefined) {
+      return MessagesSyncHandler.getIndexedLeavesFromStateIndex(stateIndex, tenant, protocol, bitPath);
+    }
+
+    return projectionSnapshot.getLeaves(bitPath);
+  }
+
+  private static async getServerSubtreeHash(
+    stateIndex: StateIndex,
+    tenant: string,
+    protocol: string | undefined,
+    projectionSnapshot: RecordsProjectionSnapshot | undefined,
+    bitPath: boolean[]
+  ): Promise<Uint8Array> {
+    if (projectionSnapshot === undefined) {
+      return MessagesSyncHandler.getIndexedSubtreeHashFromStateIndex(stateIndex, tenant, protocol, bitPath);
+    }
+
+    return projectionSnapshot.getSubtreeHash(bitPath);
+  }
+
+  private static async getIndexedLeavesFromStateIndex(
+    stateIndex: StateIndex,
+    tenant: string,
+    protocol: string | undefined,
+    bitPath: boolean[]
+  ): Promise<string[]> {
+    if (protocol === undefined) {
+      return stateIndex.getLeaves(tenant, bitPath);
+    }
+
+    return stateIndex.getProtocolLeaves(tenant, protocol, bitPath);
+  }
+
+  private static async getIndexedSubtreeHashFromStateIndex(
+    stateIndex: StateIndex,
+    tenant: string,
+    protocol: string | undefined,
+    bitPath: boolean[]
+  ): Promise<Uint8Array> {
+    if (protocol === undefined) {
+      return stateIndex.getSubtreeHash(tenant, bitPath);
+    }
+
+    return stateIndex.getProtocolSubtreeHash(tenant, protocol, bitPath);
+  }
+
   /**
    * Get the hex-encoded default hash for a given depth. Lazily cached.
    */
@@ -291,6 +478,278 @@ export class MessagesSyncHandler implements MethodHandler {
     return entries;
   }
 
+  private async buildProjectedDependencyEntries(
+    tenant: string,
+    primaryEntries: MessagesSyncDiffEntry[],
+  ): Promise<MessagesSyncDependencyEntry[]> {
+    const dependenciesByCid = new Map<string, MessagesSyncDependencyEntry>();
+    const configsByProtocol = new Map<string, GenericMessage[]>();
+
+    for (const primaryEntry of primaryEntries) {
+      const protocolMetadata = MessagesSyncHandler.recordsWriteProtocolMetadata(primaryEntry.message);
+      if (protocolMetadata !== undefined) {
+        await this.addProtocolConfigClosureDependencies(
+          tenant,
+          protocolMetadata.protocol,
+          protocolMetadata.messageTimestamp,
+          primaryEntry.messageCid,
+          configsByProtocol,
+          dependenciesByCid,
+        );
+        continue;
+      }
+
+      const initialWrite = await this.readRecordsDeleteInitialWrite(tenant, primaryEntry.message);
+      if (initialWrite === undefined) {
+        continue;
+      }
+
+      await MessagesSyncHandler.addRecordsInitialWriteDependency(
+        primaryEntry.messageCid,
+        initialWrite,
+        dependenciesByCid,
+      );
+
+      const initialWriteMetadata = MessagesSyncHandler.recordsWriteProtocolMetadata(initialWrite);
+      if (initialWriteMetadata === undefined) {
+        continue;
+      }
+
+      await this.addProtocolConfigClosureDependencies(
+        tenant,
+        initialWriteMetadata.protocol,
+        initialWriteMetadata.messageTimestamp,
+        primaryEntry.messageCid,
+        configsByProtocol,
+        dependenciesByCid,
+      );
+    }
+
+    return [...dependenciesByCid.values()];
+  }
+
+  private async readRecordsDeleteInitialWrite(
+    tenant: string,
+    message: GenericMessage | undefined,
+  ): Promise<GenericMessage | undefined> {
+    const recordId = MessagesSyncHandler.recordsDeleteRecordId(message);
+    if (recordId === undefined) {
+      return undefined;
+    }
+
+    const { messages } = await this.deps.messageStore.query(tenant, [{
+      interface : DwnInterfaceName.Records,
+      method    : DwnMethodName.Write,
+      recordId,
+    }]);
+    const initialWrite = await RecordsWrite.getInitialWrite(messages);
+    return initialWrite === undefined ? undefined : MessagesSyncHandler.toWireMessage(initialWrite);
+  }
+
+  private static async addRecordsInitialWriteDependency(
+    rootMessageCid: string,
+    dependency: GenericMessage,
+    dependenciesByCid: Map<string, MessagesSyncDependencyEntry>,
+  ): Promise<void> {
+    const dependencyCid = await Message.getCid(dependency);
+    if (dependenciesByCid.has(dependencyCid)) {
+      return;
+    }
+
+    dependenciesByCid.set(dependencyCid, {
+      dependencyClass : 'recordsInitialWrite',
+      messageCid      : dependencyCid,
+      message         : dependency,
+      rootMessageCid,
+    });
+  }
+
+  private async addProtocolConfigClosureDependencies(
+    tenant: string,
+    rootProtocol: string,
+    rootMessageTimestamp: string,
+    rootMessageCid: string,
+    configsByProtocol: Map<string, GenericMessage[]>,
+    dependenciesByCid: Map<string, MessagesSyncDependencyEntry>,
+  ): Promise<void> {
+    // Dependency hints are not part of the projected root; they are advisory
+    // bootstrap data for the receiver. Bound each closure to the primary
+    // RecordsWrite timestamp because protocol authorization uses the definition
+    // active when that record was created, not whatever config is newest today.
+    const visitedProtocols = new Set<string>();
+    const pendingProtocols = [rootProtocol];
+
+    for (
+      let protocol = MessagesSyncHandler.takeNextUnvisitedProtocol(pendingProtocols, visitedProtocols);
+      protocol !== undefined;
+      protocol = MessagesSyncHandler.takeNextUnvisitedProtocol(pendingProtocols, visitedProtocols)
+    ) {
+      const configs = await this.getCachedGoverningProtocolsConfigure(
+        tenant,
+        protocol,
+        rootMessageTimestamp,
+        configsByProtocol,
+      );
+      await MessagesSyncHandler.addProtocolConfigDependencies({
+        configs,
+        rootMessageCid,
+        visitedProtocols,
+        pendingProtocols,
+        dependenciesByCid,
+      });
+    }
+  }
+
+  private async getCachedGoverningProtocolsConfigure(
+    tenant: string,
+    protocol: string,
+    messageTimestamp: string,
+    configsByProtocol: Map<string, GenericMessage[]>,
+  ): Promise<GenericMessage[]> {
+    const configCacheKey = JSON.stringify([protocol, messageTimestamp]);
+    const cachedConfigs = configsByProtocol.get(configCacheKey);
+    if (cachedConfigs !== undefined) {
+      return cachedConfigs;
+    }
+
+    const configs = await this.readGoverningProtocolsConfigure(tenant, protocol, messageTimestamp);
+    configsByProtocol.set(configCacheKey, configs);
+    return configs;
+  }
+
+  private static async addProtocolConfigDependencies({
+    configs,
+    rootMessageCid,
+    visitedProtocols,
+    pendingProtocols,
+    dependenciesByCid,
+  }: {
+    configs: GenericMessage[];
+    rootMessageCid: string;
+    visitedProtocols: Set<string>;
+    pendingProtocols: string[];
+    dependenciesByCid: Map<string, MessagesSyncDependencyEntry>;
+  }): Promise<void> {
+    for (const dependency of configs) {
+      await MessagesSyncHandler.addProtocolConfigDependency(rootMessageCid, dependency, dependenciesByCid);
+      MessagesSyncHandler.queueUnvisitedProtocols(
+        MessagesSyncHandler.protocolsConfigureUses(dependency),
+        visitedProtocols,
+        pendingProtocols,
+      );
+    }
+  }
+
+  private static async addProtocolConfigDependency(
+    rootMessageCid: string,
+    dependency: GenericMessage,
+    dependenciesByCid: Map<string, MessagesSyncDependencyEntry>,
+  ): Promise<void> {
+    const dependencyCid = await Message.getCid(dependency);
+    if (dependenciesByCid.has(dependencyCid)) {
+      return;
+    }
+
+    dependenciesByCid.set(dependencyCid, {
+      dependencyClass : 'protocolsConfigure',
+      messageCid      : dependencyCid,
+      message         : dependency,
+      rootMessageCid,
+    });
+  }
+
+  private static takeNextUnvisitedProtocol(
+    pendingProtocols: string[],
+    visitedProtocols: Set<string>,
+  ): string | undefined {
+    while (pendingProtocols.length > 0) {
+      const protocol = pendingProtocols.shift()!;
+      if (visitedProtocols.has(protocol)) {
+        continue;
+      }
+      visitedProtocols.add(protocol);
+      return protocol;
+    }
+    return undefined;
+  }
+
+  private static queueUnvisitedProtocols(
+    protocols: string[],
+    visitedProtocols: Set<string>,
+    pendingProtocols: string[],
+  ): void {
+    for (const protocol of protocols) {
+      if (!visitedProtocols.has(protocol)) {
+        pendingProtocols.push(protocol);
+      }
+    }
+  }
+
+  private static protocolsConfigureUses(message: GenericMessage): string[] {
+    if (
+      message.descriptor.interface !== DwnInterfaceName.Protocols ||
+      message.descriptor.method !== DwnMethodName.Configure
+    ) {
+      return [];
+    }
+
+    const uses = (message.descriptor as ProtocolsConfigureDefinitionDescriptor).definition?.uses;
+    return uses === undefined
+      ? []
+      : Object.values(uses).filter((protocol): protocol is string => typeof protocol === 'string');
+  }
+
+  private async readGoverningProtocolsConfigure(
+    tenant: string,
+    protocol: string,
+    messageTimestamp: string,
+  ): Promise<GenericMessage[]> {
+    const { messages } = await this.deps.messageStore.query(
+      tenant,
+      [{
+        interface        : DwnInterfaceName.Protocols,
+        method           : DwnMethodName.Configure,
+        protocol,
+        messageTimestamp : { lte: messageTimestamp },
+      }],
+      { messageTimestamp: SortDirection.Descending },
+    );
+
+    const governingMessage = await Message.getNewestMessage(messages);
+    return governingMessage === undefined ? [] : [governingMessage];
+  }
+
+  private static recordsWriteProtocolMetadata(message: GenericMessage | undefined): RecordsWriteProtocolMetadata | undefined {
+    if (
+      message?.descriptor.interface !== DwnInterfaceName.Records ||
+      message.descriptor.method !== DwnMethodName.Write
+    ) {
+      return undefined;
+    }
+
+    const protocol = (message.descriptor as RecordsWriteProtocolDescriptor).protocol;
+    return typeof protocol === 'string'
+      ? { protocol, messageTimestamp: message.descriptor.messageTimestamp }
+      : undefined;
+  }
+
+  private static recordsDeleteRecordId(message: GenericMessage | undefined): string | undefined {
+    if (
+      message?.descriptor.interface !== DwnInterfaceName.Records ||
+      message.descriptor.method !== DwnMethodName.Delete
+    ) {
+      return undefined;
+    }
+
+    const recordId = (message.descriptor as Record<string, unknown>).recordId;
+    return typeof recordId === 'string' ? recordId : undefined;
+  }
+
+  private static toWireMessage(message: GenericMessage): GenericMessage {
+    const { encodedData: _encodedData, initialWrite: _initialWrite, ...wireMessage } = message as StoredMessageWithInternalFields;
+    return wireMessage;
+  }
+
   /**
    * Read a message and its data from the MessageStore + DataStore by CID.
    */
@@ -309,28 +768,24 @@ export class MessagesSyncHandler implements MethodHandler {
     // It must not be included in the wire-format message (the recipient's
     // DWN would reject it as an unexpected top-level property).
     let inlineEncodedData: string | undefined;
-    if ('encodedData' in storedMessage) {
-      inlineEncodedData = (storedMessage as any).encodedData as string;
-      delete (storedMessage as any).encodedData;
+    if (MessagesSyncHandler.hasEncodedData(storedMessage)) {
+      inlineEncodedData = storedMessage.encodedData;
+      delete storedMessage.encodedData;
     }
 
     let data: ReadableStream<Uint8Array> | undefined;
 
-    // Check if this is a RecordsWrite with data.
-    if (storedMessage.descriptor.interface === 'Records' &&
-        storedMessage.descriptor.method === 'Write') {
-      const dataCid = (storedMessage as any).descriptor?.dataCid as string | undefined;
-      if (dataCid) {
-        // Try DataStore first (for large payloads stored externally).
+    // Check if this is a RecordsWrite with data that is small enough to inline.
+    if (inlineEncodedData === undefined && Records.isRecordsWrite(storedMessage)) {
+      const { dataCid, dataSize } = storedMessage.descriptor;
+      if (dataSize <= DEFAULT_MAX_INLINE_DATA_SIZE) {
+        // Some stores may have small payload bytes in DataStore instead of encodedData.
         if (this.deps.dataStore) {
-          // DataStore uses recordId, not messageCid. For RecordsWrite, the
-          // recordId is in the descriptor.
-          const recordId = (storedMessage as any).descriptor?.recordId as string | undefined;
-          if (recordId) {
-            const dataResult = await this.deps.dataStore.get(tenant, recordId, dataCid);
-            if (dataResult?.dataStream) {
-              data = dataResult.dataStream;
-            }
+          // DataStore uses recordId, not messageCid. For RecordsWrite,
+          // recordId is a top-level message property.
+          const dataResult = await this.deps.dataStore.get(tenant, storedMessage.recordId, dataCid);
+          if (dataResult?.dataStream) {
+            data = dataResult.dataStream;
           }
         }
       }
@@ -339,6 +794,39 @@ export class MessagesSyncHandler implements MethodHandler {
     return { message: storedMessage, encodedData: inlineEncodedData, data };
   }
 
+  private static hasEncodedData(message: GenericMessage): message is StoredMessageWithEncodedData {
+    return 'encodedData' in message && typeof message.encodedData === 'string';
+  }
+
+  private static getProjectionScopes(
+    message: MessagesSyncMessage,
+  ): ProjectionScopes | undefined {
+    const { projectionScopes } = message.descriptor;
+    if (projectionScopes === undefined) {
+      return undefined;
+    }
+
+    return projectionScopes as [RecordsProjectionScope, ...RecordsProjectionScope[]];
+  }
+
+  private async withProjectionSnapshot<T>(
+    tenant: string,
+    scopes: readonly [RecordsProjectionScope, ...RecordsProjectionScope[]],
+    fn: (snapshot: RecordsProjectionSnapshot) => Promise<T>,
+  ): Promise<T> {
+    const snapshot = await RecordsProjection.createSnapshot({
+      tenant       : tenant,
+      messageStore : this.deps.messageStore,
+      scopes       : scopes,
+    });
+
+    try {
+      return await fn(snapshot);
+    } finally {
+      await snapshot.close();
+    }
+  }
+
   /**
    * Read a ReadableStream to completion and return the bytes.
    */
@@ -389,13 +877,16 @@ export class MessagesSyncHandler implements MethodHandler {
   ): Promise<void> {
     if (messagesSync.author === tenant) {
       return;
-    } else if (messagesSync.author !== undefined && messagesSync.signaturePayload!.permissionGrantId !== undefined) {
-      const permissionGrant = await PermissionsProtocol.fetchGrant(tenant, messageStore, messagesSync.signaturePayload!.permissionGrantId);
+    }
+
+    const permissionGrantIds = Message.getPermissionGrantIds(messagesSync.signaturePayload!);
+    if (messagesSync.author !== undefined && permissionGrantIds.length > 0) {
+      const permissionGrants = await MessagesGrantAuthorization.fetchPermissionGrants(tenant, messageStore, permissionGrantIds);
       await MessagesGrantAuthorization.authorizeSubscribeOrSync({
         incomingMessage : messagesSync.message,
         expectedGrantor : tenant,
         expectedGrantee : messagesSync.author,
-        permissionGrant,
+        permissionGrants,
         messageStore
       });
     } else {