@enbox/dwn-sdk-js 0.0.7 → 0.1.0

package/src/protocols/permissions.ts

@@ -1,14 +1,21 @@
+import type { Filter } from '../types/query-types.js';
 import type { GenericMessage } from '../types/message-types.js';
+import type { MessagesFilter } from '../types/messages-types.js';
 import type { MessageSigner } from '../types/signer.js';
 import type { MessageStore } from '../types/message-store.js';
 import type { ProtocolDefinition } from '../types/protocols-types.js';
+import type { CoreProtocol, CoreProtocolStores } from '../core/core-protocol.js';
 import type { DataEncodedRecordsWriteMessage, RecordsWriteMessage } from '../types/records-types.js';
 import type { PermissionConditions, PermissionGrantData, PermissionRequestData, PermissionRevocationData, PermissionScope, RecordsPermissionScope } from '../types/permission-types.js';
 
+import { DwnConstant } from '../core/dwn-constant.js';
 import { Encoder } from '../utils/encoder.js';
+import { FilterUtility } from '../utils/filter.js';
+import { Message } from '../core/message.js';
 import { PermissionGrant } from './permission-grant.js';
 import { PermissionRequest } from './permission-request.js';
-import { RecordsWrite } from '../../src/interfaces/records-write.js';
+import { Records } from '../utils/records.js';
+import { RecordsWrite } from '../interfaces/records-write.js';
 import { Time } from '../utils/time.js';
 import { validateJsonSchema } from '../schema-validator.js';
 import { DwnError, DwnErrorCode } from '../core/dwn-error.js';
@@ -79,12 +86,16 @@ export type PermissionRevocationCreateOptions = {
 
 /**
  * This is a first-class DWN protocol for managing permission grants of a given DWN.
+ *
+ * It implements the `CoreProtocol` interface so that its lifecycle hooks
+ * (validation, pre-processing, post-processing) are dispatched generically
+ * by the `CoreProtocolRegistry` rather than being hardcoded in handlers.
  */
-export class PermissionsProtocol {
+export class PermissionsProtocol implements CoreProtocol {
   /**
    * The URI of the DWN Permissions protocol.
    */
-  public static readonly uri = 'https://tbd.website/dwn/permissions';
+  public static readonly uri = 'https://identity.foundation/dwn/permissions';
 
   /**
    * The protocol path of the `request` record.
@@ -156,6 +167,137 @@ export class PermissionsProtocol {
     }
   };
 
+  // ---------- CoreProtocol instance accessors ----------
+
+  /** @inheritdoc */
+  public get uri(): string {
+    return PermissionsProtocol.uri;
+  }
+
+  /** @inheritdoc */
+  public get definition(): ProtocolDefinition {
+    return PermissionsProtocol.definition;
+  }
+
+  // ---------- CoreProtocol lifecycle hooks ----------
+
+  /** @inheritdoc */
+  public validateRecord(message: RecordsWriteMessage, dataBytes: Uint8Array): void {
+    PermissionsProtocol.validateSchema(message, dataBytes);
+  }
+
+  /**
+   * Pre-processing hook for permission revocation records.
+   * Validates that the revocation's `tags.protocol` matches the grant's scoped protocol.
+   */
+  public async preProcessWrite(
+    tenant: string,
+    message: RecordsWriteMessage,
+    messageStore: MessageStore,
+  ): Promise<void> {
+    if (message.descriptor.protocolPath !== PermissionsProtocol.revocationPath) {
+      return;
+    }
+
+    // fetch the parent grant to compare the scoped protocol against the revocation tag
+    const permissionGrantId = message.descriptor.parentId!;
+    const grant = await PermissionsProtocol.fetchGrant(tenant, messageStore, permissionGrantId);
+
+    const revokeTagProtocol = message.descriptor.tags?.protocol;
+    const grantProtocol = 'protocol' in grant.scope ? grant.scope.protocol : undefined;
+    if (grantProtocol !== revokeTagProtocol) {
+      throw new DwnError(
+        DwnErrorCode.PermissionsProtocolValidateRevocationProtocolTagMismatch,
+        `Revocation protocol ${revokeTagProtocol} does not match grant protocol ${grantProtocol}`
+      );
+    }
+  }
+
+  /**
+   * Post-processing hook for permission revocation records.
+   * When a grant is revoked, all messages authorized by that grant and created
+   * after the revocation timestamp are deleted from all stores.
+   *
+   * Deletion order is deliberate to avoid orphaned data in case of crash:
+   *   1. data store  (large blobs first)
+   *   2. state index (SMT entries)
+   *   3. message store
+   */
+  public async postProcessWrite(
+    tenant: string,
+    recordsWrite: RecordsWrite,
+    stores: CoreProtocolStores,
+  ): Promise<void> {
+    if (recordsWrite.message.descriptor.protocolPath !== PermissionsProtocol.revocationPath) {
+      return;
+    }
+
+    const permissionGrantId = recordsWrite.message.descriptor.parentId!;
+    const grantAuthorizedMessagesQuery = {
+      permissionGrantId,
+      dateCreated: { gte: recordsWrite.message.descriptor.messageTimestamp },
+    };
+    const { messages: grantAuthorizedMessages } = await stores.messageStore.query(tenant, [grantAuthorizedMessagesQuery]);
+
+    if (grantAuthorizedMessages.length === 0) {
+      return;
+    }
+
+    // 1. Delete data from the data store first to avoid orphaned data blobs in case of crash.
+    //    Only RecordsWrite messages with data larger than maxDataSizeAllowedToBeEncoded have data in the data store.
+    for (const message of grantAuthorizedMessages) {
+      if (message.descriptor.method === DwnMethodName.Write) {
+        const recordsWriteMessage = message as RecordsWriteMessage;
+        if (recordsWriteMessage.descriptor.dataSize > DwnConstant.maxDataSizeAllowedToBeEncoded) {
+          await stores.dataStore.delete(tenant, recordsWriteMessage.recordId, recordsWriteMessage.descriptor.dataCid);
+        }
+      }
+    }
+
+    // 2. Compute CIDs and delete from state index before message store to avoid orphaned state entries.
+    const messageCids = await Promise.all(grantAuthorizedMessages.map((message): Promise<string> => Message.getCid(message)));
+    await stores.stateIndex.delete(tenant, messageCids);
+
+    // 3. Finally delete all messages from the message store.
+    await Promise.all(messageCids.map((cid): Promise<void> => stores.messageStore.delete(tenant, cid)));
+  }
+
+  /** @inheritdoc */
+  public mapErrorToStatusCode(errorCode: string): number | undefined {
+    if (errorCode.startsWith('PermissionsProtocolValidate')) {
+      return 400;
+    }
+    return undefined;
+  }
+
+  /**
+   * Constructs an additional filter for protocol-scoped message queries so that
+   * permission records (grants, requests, revocations) tagged with the target
+   * protocol appear alongside that protocol's own records in sync/subscribe/read results.
+   */
+  public constructAdditionalMessageFilter(filter: MessagesFilter): Filter | undefined {
+    const { protocol, messageTimestamp } = filter;
+    if (protocol === undefined) {
+      return undefined;
+    }
+
+    const taggedFilter = {
+      protocol: PermissionsProtocol.uri,
+      ...Records.convertTagsFilter({ protocol }),
+    } as Filter;
+
+    if (messageTimestamp !== undefined) {
+      const messageTimestampFilter = FilterUtility.convertRangeCriterion(messageTimestamp);
+      if (messageTimestampFilter) {
+        taggedFilter.messageTimestamp = messageTimestampFilter;
+      }
+    }
+
+    return taggedFilter;
+  }
+
+  // ---------- Static utility methods ----------
+
   public static parseRequest(base64UrlEncodedRequest: string): PermissionRequestData {
     return Encoder.base64UrlToObject(base64UrlEncodedRequest);
   }
@@ -394,7 +536,7 @@ export class PermissionsProtocol {
     }
 
     const permissionGrantMessage = possibleGrantMessage as DataEncodedRecordsWriteMessage;
-    const permissionGrant = await PermissionGrant.parse(permissionGrantMessage);
+    const permissionGrant = PermissionGrant.parse(permissionGrantMessage);
 
     return permissionGrant;
   }
@@ -421,11 +563,11 @@ export class PermissionsProtocol {
       const grant = await PermissionsProtocol.fetchGrant(tenant, messageStore, incomingMessage.descriptor.parentId!);
       return grant.scope;
     } else if (incomingMessage.descriptor.protocolPath === PermissionsProtocol.grantPath) {
-      const grant = await PermissionGrant.parse(incomingMessage);
+      const grant = PermissionGrant.parse(incomingMessage);
       return grant.scope;
     } else {
       // if the record is not a grant or revocation, it must be a request
-      const request = await PermissionRequest.parse(incomingMessage);
+      const request = PermissionRequest.parse(incomingMessage);
       return request.scope;
     }
   }

package/src/state-index/state-index-level.ts

@@ -83,7 +83,8 @@ export class StateIndexLevel implements StateIndex {
     const globalSmt = await this.getGlobalTree(tenant);
     await globalSmt.insert(messageCid);
 
-    // If the message is associated with a protocol, insert into the protocol-scoped tree
+    // Insert into the protocol-scoped tree if the message has a protocol (e.g. RecordsWrite).
+    // Non-record messages like ProtocolsConfigure do not have a protocol.
     const protocol = indexes.protocol as string | undefined;
     if (protocol !== undefined) {
       const protoSmt = await this.getProtocolTree(tenant, protocol);
@@ -104,7 +105,7 @@ export class StateIndexLevel implements StateIndex {
       // Delete from global tree
       await globalSmt.delete(messageCid);
 
-      // Delete from protocol tree if applicable
+      // Delete from protocol tree if the message had a protocol
       if (indexes !== undefined) {
         const protocol = indexes.protocol as string | undefined;
         if (protocol !== undefined) {
@@ -204,11 +205,8 @@ export class StateIndexLevel implements StateIndex {
    */
   private async storeIndexes(tenant: string, messageCid: string, indexes: KeyValues): Promise<void> {
     const metaPartition = await this.getMetaPartition(tenant);
-    // Only store the minimal set of indexes needed for deletion (protocol)
-    const minimalIndexes: KeyValues = {};
-    if (indexes.protocol !== undefined) {
-      minimalIndexes.protocol = indexes.protocol;
-    }
+    // Store the protocol index needed for deletion
+    const minimalIndexes: KeyValues = { protocol: indexes.protocol };
     await metaPartition.put(messageCid, JSON.stringify(minimalIndexes));
   }
 

package/src/store/data-store-level.ts

@@ -1,4 +1,5 @@
 import type { ImportResult } from 'ipfs-unixfs-importer';
+import type { LevelWrapper } from './level-wrapper.js';
 import type { DataStore, DataStoreGetResult, DataStorePutResult } from '../types/data-store.js';
 
 import { BlockstoreLevel } from './blockstore-level.js';
@@ -7,12 +8,20 @@ import { DataStream } from '../utils/data-stream.js';
 import { exporter } from 'ipfs-unixfs-exporter';
 import { importer } from 'ipfs-unixfs-importer';
 
+const textEncoder = new TextEncoder();
+const textDecoder = new TextDecoder();
+
 /**
  * A simple implementation of {@link DataStore} that works in both the browser and server-side.
  * Leverages LevelDB under the hood.
  *
- * It has the following structure (`+` represents an additional sublevel/partition):
- *   'data' + <tenant> + <recordId> + <dataCid> -> <data>
+ * It has the following sublevel structure (`+` represents an additional sublevel):
+ *   'refs'      + <tenant> + <recordId>  : key <dataCid>  → JSON { dataSize }
+ *   'blocks'    + <dataCid>              : key <blockCid>  → block data (shared)
+ *   'refcounts' : key <dataCid>                            → JSON { count, dataSize }
+ *
+ * Identical data (same dataCid) is stored only once in the blocks sublevel.
+ * Multiple (tenant, recordId) pairs can reference the same blocks.
  */
 export class DataStoreLevel implements DataStore {
   config: DataStoreLevelConfig;
@@ -41,29 +50,74 @@ export class DataStoreLevel implements DataStore {
   }
 
   async put(tenant: string, recordId: string, dataCid: string, dataStream: ReadableStream<Uint8Array>): Promise<DataStorePutResult> {
-    const blockstoreForData = await this.getBlockstoreForStoringData(tenant, recordId, dataCid);
+    // Check if this exact ref already exists (idempotent re-put).
+    const refsPartition = await this.getRefsPartition(tenant, recordId);
+    const existingRef = await refsPartition.get(dataCid);
+    if (existingRef) {
+      await dataStream.cancel();
+      const { dataSize } = JSON.parse(textDecoder.decode(existingRef)) as { dataSize: number };
+      return { dataSize };
+    }
 
-    const asyncDataBlocks = importer([{ content: DataStream.asAsyncIterable(dataStream) }], blockstoreForData, { cidVersion: 1 });
+    // Check refcount — if > 0, blocks already exist for this dataCid.
+    const refcountsPartition = await this.getRefcountsPartition();
+    const rawRefcount = await refcountsPartition.get(dataCid);
+    const refcountData = rawRefcount
+      ? JSON.parse(textDecoder.decode(rawRefcount)) as { count: number; dataSize: number }
+      : { count: 0, dataSize: 0 };
+
+    let dataSize: number;
+
+    if (refcountData.count > 0) {
+      // Blocks already exist — skip import.
+      await dataStream.cancel();
+      dataSize = refcountData.dataSize;
+    } else {
+      // First write — import blocks into the shared blocks partition.
+      const blocksPartition = await this.getBlocksPartition(dataCid);
+      const asyncDataBlocks = importer(
+        [{ content: DataStream.asAsyncIterable(dataStream) }],
+        blocksPartition,
+        { cidVersion: 1 }
+      );
+
+      // NOTE: the last block contains the root CID as well as info to derive the data size.
+      let dataDagRoot!: ImportResult;
+      for await (dataDagRoot of asyncDataBlocks) { ; }
+      dataSize = Number(dataDagRoot.unixfs?.fileSize() ?? dataDagRoot.size);
+    }
 
-    // NOTE: the last block contains the root CID as well as info to derive the data size
-    let dataDagRoot!: ImportResult;
-    for await (dataDagRoot of asyncDataBlocks) { ; }
+    // Write ref entry.
+    await refsPartition.put(dataCid, textEncoder.encode(JSON.stringify({ dataSize })));
 
-    return {
-      dataSize: Number(dataDagRoot.unixfs?.fileSize() ?? dataDagRoot.size)
-    };
+    // Increment refcount.
+    await refcountsPartition.put(dataCid, textEncoder.encode(JSON.stringify({
+      count: refcountData.count + 1,
+      dataSize,
+    })));
+
+    return { dataSize };
   }
 
   public async get(tenant: string, recordId: string, dataCid: string): Promise<DataStoreGetResult | undefined> {
-    const blockstoreForData = await this.getBlockstoreForStoringData(tenant, recordId, dataCid);
+    // Check ref exists.
+    const refsPartition = await this.getRefsPartition(tenant, recordId);
+    const rawRef = await refsPartition.get(dataCid);
+    if (!rawRef) {
+      return undefined;
+    }
+
+    const { dataSize } = JSON.parse(textDecoder.decode(rawRef)) as { dataSize: number };
 
-    const exists = await blockstoreForData.has(dataCid);
+    // Export from the shared blocks partition.
+    const blocksPartition = await this.getBlocksPartition(dataCid);
+    const exists = await blocksPartition.has(dataCid);
     if (!exists) {
       return undefined;
     }
 
-    // data is chunked into dag-pb unixfs blocks. re-inflate the chunks.
-    const dataDagRoot = await exporter(dataCid, blockstoreForData);
+    // Data is chunked into DAG-PB UnixFS blocks. Re-inflate the chunks.
+    const dataDagRoot = await exporter(dataCid, blocksPartition);
     const contentIterator = dataDagRoot.content();
 
     const dataStream = new ReadableStream<Uint8Array>({
@@ -77,21 +131,42 @@ export class DataStoreLevel implements DataStore {
       }
     });
 
-    let dataSize = dataDagRoot.size;
+    return { dataSize, dataStream };
+  }
 
-    if (dataDagRoot.type === 'file' || dataDagRoot.type === 'directory') {
-      dataSize = dataDagRoot.unixfs.fileSize();
+  public async delete(tenant: string, recordId: string, dataCid: string): Promise<void> {
+    // Check ref exists.
+    const refsPartition = await this.getRefsPartition(tenant, recordId);
+    const rawRef = await refsPartition.get(dataCid);
+    if (!rawRef) {
+      return;
     }
 
-    return {
-      dataSize: Number(dataSize),
-      dataStream,
-    };
-  }
+    // Remove ref.
+    await refsPartition.delete(dataCid);
 
-  public async delete(tenant: string, recordId: string, dataCid: string): Promise<void> {
-    const blockstoreForData = await this.getBlockstoreForStoringData(tenant, recordId, dataCid);
-    await blockstoreForData.clear();
+    // Decrement refcount and GC blocks if this was the last ref.
+    const refcountsPartition = await this.getRefcountsPartition();
+    const rawRefcount = await refcountsPartition.get(dataCid);
+    if (!rawRefcount) {
+      return;
+    }
+
+    const refcountData = JSON.parse(textDecoder.decode(rawRefcount)) as { count: number; dataSize: number };
+    const newCount = refcountData.count - 1;
+
+    if (newCount <= 0) {
+      // Last reference removed — garbage-collect blocks and refcount entry.
+      const blocksPartition = await this.getBlocksPartition(dataCid);
+      await blocksPartition.clear();
+      await refcountsPartition.delete(dataCid);
+    } else {
+      // Other references remain — update the count.
+      await refcountsPartition.put(dataCid, textEncoder.encode(JSON.stringify({
+        count    : newCount,
+        dataSize : refcountData.dataSize,
+      })));
+    }
   }
 
   /**
@@ -102,15 +177,30 @@ export class DataStoreLevel implements DataStore {
   }
 
   /**
-   * Gets the blockstore used for storing data for the given `tenant -> `recordId` -> `dataCid`.
+   * Gets the refs sublevel for the given tenant and recordId.
+   * Caller uses `dataCid` as the key within the returned partition.
+   */
+  private async getRefsPartition(tenant: string, recordId: string): Promise<LevelWrapper<Uint8Array>> {
+    const refsRoot = await this.blockstore.db.partition('refs');
+    const tenantPartition = await refsRoot.partition(tenant);
+    return tenantPartition.partition(recordId);
+  }
+
+  /**
+   * Gets the shared blocks sublevel for the given dataCid.
+   * Used as a Blockstore for ipfs-unixfs-importer/exporter.
+   */
+  private async getBlocksPartition(dataCid: string): Promise<BlockstoreLevel> {
+    const blocksRoot = await this.blockstore.partition('blocks');
+    return blocksRoot.partition(dataCid);
+  }
+
+  /**
+   * Gets the refcounts sublevel.
+   * Key: `dataCid` → JSON `{ count, dataSize }`.
    */
-  private async getBlockstoreForStoringData(tenant: string, recordId: string, dataCid: string): Promise<BlockstoreLevel> {
-    const dataPartitionName = 'data';
-    const blockstoreForData = await this.blockstore.partition(dataPartitionName);
-    const blockstoreOfGivenTenant = await blockstoreForData.partition(tenant);
-    const blockstoreOfGivenRecordId = await blockstoreOfGivenTenant.partition(recordId);
-    const blockstoreOfGivenDataCidOfRecordId = await blockstoreOfGivenRecordId.partition(dataCid);
-    return blockstoreOfGivenDataCidOfRecordId;
+  private async getRefcountsPartition(): Promise<LevelWrapper<Uint8Array>> {
+    return this.blockstore.db.partition('refcounts');
   }
 
 }
@@ -118,4 +208,4 @@ export class DataStoreLevel implements DataStore {
 export type DataStoreLevelConfig = {
   blockstoreLocation?: string,
   createLevelDatabase?: typeof createLevelDatabase,
-};
+};

package/src/store/index-level.ts

@@ -477,9 +477,18 @@ export class IndexLevel {
     }
 
     try {
-      await Promise.all(filters.map(filter => {
-        return this.executeSingleFilterQuery(tenant, filter, sortProperty, matches, options );
-      }));
+      // Execute filters sequentially rather than with Promise.all.
+      // Firefox's IndexedDB implementation has two related async issues that cause flaky failures:
+      //   1. Concurrent cursors/transactions can silently miss recently-committed data
+      //   2. A read transaction opened immediately after a write transaction's oncomplete event
+      //      can fail to see the written data (write-read race)
+      // Serializing eliminates both races. Performance impact is negligible: only 3 call sites
+      // (non-owner RecordsQuery/Subscribe/Count) ever pass multiple filters, and each filter
+      // reduces to a single bounded LevelDB range scan completing in single-digit ms.
+      // See: https://github.com/enboxorg/enbox/issues/264
+      for (const filter of filters) {
+        await this.executeSingleFilterQuery(tenant, filter, sortProperty, matches, options);
+      }
     } catch (error) {
       if ((error as DwnError).code === DwnErrorCode.IndexInvalidSortPropertyInMemory) {
         // return empty results if the sort property is invalid.
@@ -501,6 +510,10 @@ export class IndexLevel {
 
   /**
    * Execute a filtered query against a single filter and return all results.
+   *
+   * Sub-queries (exact match, range, OneOf) are executed sequentially to avoid opening
+   * multiple concurrent IndexedDB cursors. Firefox's IDB implementation intermittently
+   * drops results when cursors overlap — see https://github.com/enboxorg/enbox/issues/264
    */
   private async executeSingleFilterQuery(
     tenant: string,
@@ -510,13 +523,30 @@ export class IndexLevel {
     levelOptions?: IndexLevelOptions
   ): Promise<void> {
 
-    // Note: We have an array of Promises in order to support OR (anyOf) matches when given a list of accepted values for a property
-    const filterPromises: Promise<IndexedItem[]>[] = [];
+    // Collects results from each sub-query sequentially to avoid concurrent IndexedDB cursor races in Firefox.
+    const processResults = (indexItems: IndexedItem[]): void => {
+      for (const indexedItem of indexItems) {
+        // short circuit: if a data is already included to the final matched key set (by a different `Filter`),
+        // no need to evaluate if the data satisfies this current filter being evaluated
+        // otherwise check that the item is a match.
+        if (matches.has(indexedItem.messageCid) || !FilterUtility.matchFilter(indexedItem.indexes, filter)) {
+          continue;
+        }
+
+        // ensure that each matched item has the sortProperty, otherwise fail the entire query.
+        if (indexedItem.indexes[sortProperty] === undefined) {
+          throw new DwnError(DwnErrorCode.IndexInvalidSortPropertyInMemory, `invalid sort property ${sortProperty}`);
+        }
+
+        matches.set(indexedItem.messageCid, indexedItem);
+      }
+    };
 
     // If the filter is empty, then we just iterate over one of the indexes that contains all the records and return all items.
     if (isEmptyObject(filter)) {
-      const getAllItemsPromise = this.getAllItems(tenant, sortProperty);
-      filterPromises.push(getAllItemsPromise);
+      const allItems = await this.getAllItems(tenant, sortProperty);
+      processResults(allItems);
+      return;
     }
 
     // else the filter is not empty
@@ -526,40 +556,19 @@ export class IndexLevel {
       // We will find the union of these many individual queries later.
       if (FilterUtility.isEqualFilter(propertyFilter)) {
         // propertyFilter is an EqualFilter, meaning it is a non-object primitive type
-        const exactMatchesPromise = this.filterExactMatches(tenant, propertyName, propertyFilter, levelOptions);
-        filterPromises.push(exactMatchesPromise);
+        const exactMatches = await this.filterExactMatches(tenant, propertyName, propertyFilter, levelOptions);
+        processResults(exactMatches);
       } else if (FilterUtility.isOneOfFilter(propertyFilter)) {
         // `propertyFilter` is a OneOfFilter
-        // Support OR matches by querying for each values separately, then adding them to the promises array.
+        // Support OR matches by querying for each value separately and sequentially.
         for (const propertyValue of new Set(propertyFilter)) {
-          const exactMatchesPromise = this.filterExactMatches(tenant, propertyName, propertyValue, levelOptions);
-          filterPromises.push(exactMatchesPromise);
+          const exactMatches = await this.filterExactMatches(tenant, propertyName, propertyValue, levelOptions);
+          processResults(exactMatches);
         }
       } else if (FilterUtility.isRangeFilter(propertyFilter)) {
         // `propertyFilter` is a `RangeFilter`
-        const rangeMatchesPromise = this.filterRangeMatches(tenant, propertyName, propertyFilter, levelOptions);
-        filterPromises.push(rangeMatchesPromise);
-      }
-    }
-
-    // acting as an OR match for the property, any of the promises returning a match will be treated as a property match
-    for (const promise of filterPromises) {
-      const indexItems = await promise;
-      // reminder: the promise returns a list of IndexedItem satisfying a particular property match
-      for (const indexedItem of indexItems) {
-        // short circuit: if a data is already included to the final matched key set (by a different `Filter`),
-        // no need to evaluate if the data satisfies this current filter being evaluated
-        // otherwise check that the item is a match.
-        if (matches.has(indexedItem.messageCid) || !FilterUtility.matchFilter(indexedItem.indexes, filter)) {
-          continue;
-        }
-
-        // ensure that each matched item has the sortProperty, otherwise fail the entire query.
-        if (indexedItem.indexes[sortProperty] === undefined) {
-          throw new DwnError(DwnErrorCode.IndexInvalidSortPropertyInMemory, `invalid sort property ${sortProperty}`);
-        }
-
-        matches.set(indexedItem.messageCid, indexedItem);
+        const rangeMatches = await this.filterRangeMatches(tenant, propertyName, propertyFilter, levelOptions);
+        processResults(rangeMatches);
       }
     }
   }