@enbox/dwn-sdk-js 0.4.16 → 0.4.18

package/src/interfaces/records-query.ts

@@ -52,7 +52,7 @@ export class RecordsQuery extends AbstractMessage<RecordsQueryMessage> {
     }
 
     await Records.validateDelegatedGrantReferentialIntegrity(message, signaturePayload);
-    Records.validateNestedProtocolPathQueryScope(
+    Records.validateNestedProtocolPathScope(
       message.descriptor.filter,
       DwnErrorCode.RecordsQueryNestedProtocolPathContextIdInvalid,
       'RecordsQuery'

package/src/interfaces/records-subscribe.ts

@@ -48,10 +48,13 @@ export class RecordsSubscribe extends AbstractMessage<RecordsSubscribeMessage> {
     }
 
     await Records.validateDelegatedGrantReferentialIntegrity(message, signaturePayload);
-    Records.validateNestedProtocolPathQueryScope(
+    Records.validateNestedProtocolPathScope(
       message.descriptor.filter,
       DwnErrorCode.RecordsSubscribeNestedProtocolPathContextIdInvalid,
-      'RecordsSubscribe'
+      'RecordsSubscribe',
+      message.descriptor.cursor === undefined
+        && message.descriptor.pagination?.limit !== undefined
+        && !signaturePayload?.protocolRole?.includes('/'),
     );
 
     if (signaturePayload?.protocolRole !== undefined) {

package/src/store/index-level.ts

@@ -4,6 +4,7 @@ import type { LevelWrapperBatchOperation, LevelWrapperIteratorOptions, } from '.
 import { isEmptyObject } from '@enbox/common';
 import { lexicographicalCompare } from '../utils/string.js';
 import { SortDirection } from '../types/query-types.js';
+import { assertValidSubtreeFilters, FilterSelector, FilterUtility, isSubtreeFilter } from '../utils/filter.js';
 import {
   countWithCompoundIndex,
   createCompoundIndexDeleteOperation,
@@ -13,7 +14,6 @@ import {
 } from './index-level-compound.js';
 import { createLevelDatabase, LevelWrapper } from './level-wrapper.js';
 import { DwnError, DwnErrorCode } from '../core/dwn-error.js';
-import { FilterSelector, FilterUtility } from '../utils/filter.js';
 
 export type IndexLevelConfig = {
   location: string,
@@ -23,6 +23,13 @@ export type IndexLevelConfig = {
 
 export type IndexedItem = { messageCid: string, indexes: KeyValues };
 
+type RecordLimitParentGroupsInput = {
+  candidateFilter: Filter;
+  filters: Filter[];
+  max: number;
+  parentIds: readonly string[];
+};
+
 /** A value that can be encoded into a lexicographically sortable index key. */
 type IndexableValue = string | number | boolean;
 
@@ -318,17 +325,12 @@ export class IndexLevel {
    * @returns {IndexedItem[]} an array of `IndexedItem` that match the given filters.
    */
   async query(tenant: string, filters: Filter[], queryOptions: QueryOptions, options?: IndexLevelOptions): Promise<IndexedItem[]> {
+    assertValidSubtreeFilters(filters);
 
     // Strategy 1: try compound index for single-filter queries
-    if (filters.length === 1 && !isEmptyObject(filters[0])) {
-      const compoundResult = selectCompoundIndex(filters[0], queryOptions, this._compoundIndexes);
-      if (compoundResult !== undefined) {
-        return queryWithCompoundIndex(
-          this.db, tenant, filters[0], queryOptions, compoundResult,
-          IndexLevel.encodeValue, IndexLevel.delimiter,
-          this.queryWithIteratorPaging.bind(this), options
-        );
-      }
+    const compoundQuery = this.queryUsingCompoundIndex(tenant, filters, queryOptions, options);
+    if (compoundQuery !== undefined) {
+      return compoundQuery;
     }
 
     // Strategy 2: in-memory paging for concise filters
@@ -340,6 +342,47 @@ export class IndexLevel {
     return this.queryWithIteratorPaging(tenant, filters, queryOptions, options);
   }
 
+  /**
+   * Queries one bounded page without selecting the in-memory strategy.
+   *
+   * Projection callers use this when they must close the outer iterator before
+   * opening related index ranges. A matching compound index is preferred;
+   * otherwise the requested-sort index is streamed directly.
+   */
+  async queryWithBoundedPaging(
+    tenant: string,
+    filters: Filter[],
+    queryOptions: QueryOptions,
+    options?: IndexLevelOptions,
+  ): Promise<IndexedItem[]> {
+    assertValidSubtreeFilters(filters);
+
+    const compoundQuery = this.queryUsingCompoundIndex(tenant, filters, queryOptions, options);
+    return compoundQuery ?? this.queryWithIteratorPaging(tenant, filters, queryOptions, options);
+  }
+
+  private queryUsingCompoundIndex(
+    tenant: string,
+    filters: Filter[],
+    queryOptions: QueryOptions,
+    options?: IndexLevelOptions,
+  ): Promise<IndexedItem[]> | undefined {
+    if (filters.length !== 1 || isEmptyObject(filters[0])) {
+      return undefined;
+    }
+
+    const compoundIndex = selectCompoundIndex(filters[0], queryOptions, this._compoundIndexes);
+    if (compoundIndex === undefined) {
+      return undefined;
+    }
+
+    return queryWithCompoundIndex(
+      this.db, tenant, filters[0], queryOptions, compoundIndex,
+      IndexLevel.encodeValue, IndexLevel.delimiter,
+      this.queryWithIteratorPaging.bind(this), options
+    );
+  }
+
   /**
    * Counts the number of items that match the given filters without loading full records.
    *
@@ -348,6 +391,7 @@ export class IndexLevel {
    */
   async count(tenant: string, filters: Filter[], queryOptions: Omit<QueryOptions, 'limit' | 'cursor'>,
     options?: IndexLevelOptions): Promise<number> {
+    assertValidSubtreeFilters(filters);
 
     // try compound index for single-filter queries
     if (filters.length === 1 && !isEmptyObject(filters[0])) {
@@ -365,6 +409,207 @@ export class IndexLevel {
     return results.length;
   }
 
+  /**
+   * Finds the inclusive rank boundary for one `$recordLimit` parent group.
+   *
+   * The existing `parentId` index keeps the scan isolated to one nested-record
+   * group. Root records have no `parentId`, so they are read from the exact
+   * `protocolPath` range and explicitly fenced to rows without a parent. The
+   * bounded max-heap retains only the earliest `max` rank keys while the range
+   * is streamed; no group-sized array is materialized.
+   */
+  public async findRecordLimitRankCutoff(
+    tenant: string,
+    candidateFilter: Filter,
+    parentId: string | undefined,
+    max: number,
+    options?: IndexLevelOptions,
+  ): Promise<string | undefined> {
+    if (!Number.isInteger(max) || max < 1) {
+      throw new TypeError(`IndexLevel: record-limit max must be a positive integer.`);
+    }
+
+    const protocolPath = candidateFilter.protocolPath;
+    if (typeof protocolPath !== 'string') {
+      throw new TypeError(`IndexLevel: record-limit candidate filter must contain one protocolPath.`);
+    }
+
+    const indexName = parentId === undefined ? 'protocolPath' : 'parentId';
+    const indexValue = parentId ?? protocolPath;
+    const rankHeap: string[] = [];
+
+    for await (const item of this.iterateExactMatches(tenant, indexName, indexValue, options)) {
+      if ((parentId === undefined && item.indexes.parentId !== undefined) ||
+          !FilterUtility.matchFilter(item.indexes, candidateFilter)) {
+        continue;
+      }
+
+      IndexLevel.addToRecordLimitRankHeap(
+        rankHeap,
+        IndexLevel.createRecordLimitRankKey(item.indexes),
+        max,
+      );
+    }
+
+    return rankHeap[0];
+  }
+
+  /**
+   * Queries the bounded occupants of explicitly selected direct-parent groups.
+   * Occupancy is resolved before caller filters, then the survivors are globally
+   * sorted and paginated using ordinary IndexLevel cursor semantics.
+   */
+  public async queryRecordLimitParentGroups(
+    tenant: string,
+    input: RecordLimitParentGroupsInput & { queryOptions: QueryOptions },
+    options?: IndexLevelOptions,
+  ): Promise<IndexedItem[]> {
+    assertValidSubtreeFilters([input.candidateFilter, ...input.filters]);
+    const { filters, queryOptions } = input;
+    const { cursor, limit, sortDirection = SortDirection.Ascending, sortProperty } = queryOptions;
+    if (limit !== undefined && limit <= 0) {
+      return [];
+    }
+
+    const cursorStartingKey = cursor === undefined ? undefined : this.createStartingKeyFromCursor(cursor);
+    const compareItems = (left: IndexedItem, right: IndexedItem): number =>
+      this.sortItems(left, right, sortProperty, sortDirection);
+    const matches: IndexedItem[] = [];
+
+    for await (const item of this.iterateRecordLimitParentGroupOccupants(tenant, input, options)) {
+      if (item.indexes[sortProperty] === undefined ||
+          !FilterUtility.matchAnyFilter(item.indexes, filters) ||
+          (cursorStartingKey !== undefined &&
+            !this.isItemAfterCursor(item, sortDirection, sortProperty, cursorStartingKey))) {
+        continue;
+      }
+
+      matches.push(item);
+    }
+
+    matches.sort(compareItems);
+    return limit === undefined ? matches : matches.slice(0, limit);
+  }
+
+  /** Counts the bounded occupants of explicitly selected direct-parent groups. */
+  public async countRecordLimitParentGroups(
+    tenant: string,
+    input: RecordLimitParentGroupsInput & { sortProperty: string },
+    options?: IndexLevelOptions,
+  ): Promise<number> {
+    assertValidSubtreeFilters([input.candidateFilter, ...input.filters]);
+    let count = 0;
+    for await (const item of this.iterateRecordLimitParentGroupOccupants(tenant, input, options)) {
+      if (item.indexes[input.sortProperty] !== undefined &&
+          FilterUtility.matchAnyFilter(item.indexes, input.filters)) {
+        count++;
+      }
+    }
+    return count;
+  }
+
+  /** Streams occupants from exact parentId ranges without opening the requested-sort index. */
+  private async * iterateRecordLimitParentGroupOccupants(
+    tenant: string,
+    input: RecordLimitParentGroupsInput,
+    options?: IndexLevelOptions,
+  ): AsyncGenerator<IndexedItem> {
+    for (const parentId of new Set(input.parentIds)) {
+      const candidateFilter: Filter = { ...input.candidateFilter, parentId };
+      const cutoff = await this.findRecordLimitRankCutoff(
+        tenant,
+        candidateFilter,
+        parentId,
+        input.max,
+        options,
+      );
+      if (cutoff === undefined) {
+        continue;
+      }
+
+      for await (const item of this.iterateExactMatches(tenant, 'parentId', parentId, options)) {
+        if (FilterUtility.matchFilter(item.indexes, candidateFilter) &&
+            lexicographicalCompare(IndexLevel.createRecordLimitRankKey(item.indexes), cutoff) <= 0) {
+          yield item;
+        }
+      }
+    }
+  }
+
+  /**
+   * Builds the deterministic `$recordLimit` rank key: oldest creation time,
+   * then record ID. Message CID deliberately does not participate because
+   * occupancy belongs to the logical record, not a particular update.
+   */
+  public static createRecordLimitRankKey(indexes: KeyValues): string {
+    const { dateCreated, recordId } = indexes;
+    if (typeof dateCreated !== 'string' || typeof recordId !== 'string') {
+      throw new TypeError(`IndexLevel: record-limit candidates require string dateCreated and recordId indexes.`);
+    }
+
+    return IndexLevel.keySegmentJoin(IndexLevel.encodeValue(dateCreated), IndexLevel.encodeValue(recordId));
+  }
+
+  private static addToRecordLimitRankHeap(
+    heap: string[],
+    value: string,
+    max: number,
+  ): void {
+    if (heap.length < max) {
+      heap.push(value);
+      IndexLevel.siftRecordLimitRankHeapUp(heap, heap.length - 1);
+      return;
+    }
+
+    if (lexicographicalCompare(value, heap[0]) >= 0) {
+      return;
+    }
+
+    heap[0] = value;
+    IndexLevel.siftRecordLimitRankHeapDown(heap, 0);
+  }
+
+  private static siftRecordLimitRankHeapUp(
+    heap: string[],
+    startIndex: number,
+  ): void {
+    let index = startIndex;
+    while (index > 0) {
+      const parentIndex = (index - 1) >>> 1;
+      if (lexicographicalCompare(heap[parentIndex], heap[index]) >= 0) {
+        return;
+      }
+
+      [heap[parentIndex], heap[index]] = [heap[index], heap[parentIndex]];
+      index = parentIndex;
+    }
+  }
+
+  private static siftRecordLimitRankHeapDown(
+    heap: string[],
+    startIndex: number,
+  ): void {
+    let index = startIndex;
+    while (true) {
+      const leftIndex = index * 2 + 1;
+      if (leftIndex >= heap.length) {
+        return;
+      }
+
+      const rightIndex = leftIndex + 1;
+      const largerChildIndex = rightIndex < heap.length &&
+        lexicographicalCompare(heap[rightIndex], heap[leftIndex]) > 0
+        ? rightIndex
+        : leftIndex;
+      if (lexicographicalCompare(heap[index], heap[largerChildIndex]) >= 0) {
+        return;
+      }
+
+      [heap[index], heap[largerChildIndex]] = [heap[largerChildIndex], heap[index]];
+      index = largerChildIndex;
+    }
+  }
+
   /**
    * Queries the sort property index for items that match the filters. If no filters are provided, all items are returned.
    * This query is a linear iterator over the sorted index, checking each item for a match.
@@ -589,6 +834,17 @@ export class IndexLevel {
           const exactMatches = await this.filterExactMatches(tenant, propertyName, propertyValue, levelOptions);
           processResults(exactMatches);
         }
+      } else if (isSubtreeFilter(propertyFilter)) {
+        const exactMatches = await this.filterExactMatches(tenant, propertyName, propertyFilter.subtree, levelOptions);
+        processResults(exactMatches);
+
+        const descendantMatches = await this.filterSubtreeDescendantMatches(
+          tenant,
+          propertyName,
+          propertyFilter.subtree,
+          levelOptions,
+        );
+        processResults(descendantMatches);
       } else if (FilterUtility.isRangeFilter(propertyFilter)) {
         // `propertyFilter` is a `RangeFilter`
         const rangeMatches = await this.filterRangeMatches(tenant, propertyName, propertyFilter, levelOptions);
@@ -615,19 +871,54 @@ export class IndexLevel {
     propertyValue: EqualFilter,
     options?: IndexLevelOptions
   ): Promise<IndexedItem[]> {
+    const matches: IndexedItem[] = [];
+    for await (const item of this.iterateExactMatches(tenant, propertyName, propertyValue, options)) {
+      matches.push(item);
+    }
+    return matches;
+  }
 
+  /** Streams one exact property-value range without materializing the range. */
+  private async * iterateExactMatches(
+    tenant: string,
+    propertyName: string,
+    propertyValue: EqualFilter,
+    options?: IndexLevelOptions,
+  ): AsyncGenerator<IndexedItem> {
     const matchPrefix = IndexLevel.keySegmentJoin(IndexLevel.encodeValue(propertyValue));
+    const filterPartition = await this.getIndexPartition(tenant, propertyName);
+
+    for await (const [key, value] of filterPartition.iterator({ gt: matchPrefix }, options)) {
+      if (!key.startsWith(matchPrefix)) {
+        return;
+      }
+      yield JSON.parse(value) as IndexedItem;
+    }
+  }
+
+  /**
+   * Returns every slash-delimited descendant of one hierarchical path.
+   *
+   * The lower bound removes JSON's closing quote so it is the encoded key
+   * prefix itself. ASCII `/` and `0` are adjacent in the store's lexicographic
+   * ordering, making `path0` the first value outside that prefix without
+   * relying on a finite Unicode sentinel.
+   */
+  private async filterSubtreeDescendantMatches(
+    tenant: string,
+    propertyName: string,
+    subtree: string,
+    options?: IndexLevelOptions,
+  ): Promise<IndexedItem[]> {
+    const encodedPrefix = IndexLevel.encodeValue(`${subtree}/`).slice(0, -1);
     const iteratorOptions: LevelWrapperIteratorOptions<string> = {
-      gt: matchPrefix
+      gte : encodedPrefix,
+      lt  : IndexLevel.encodeValue(`${subtree}0`),
     };
 
     const filterPartition = await this.getIndexPartition(tenant, propertyName);
     const matches: IndexedItem[] = [];
-    for await (const [ key, value ] of filterPartition.iterator(iteratorOptions, options)) {
-      // immediately stop if we arrive at an index that contains a different property value
-      if (!key.startsWith(matchPrefix)) {
-        break;
-      }
+    for await (const [ _key, value ] of filterPartition.iterator(iteratorOptions, options)) {
       matches.push(JSON.parse(value) as IndexedItem);
     }
     return matches;
@@ -704,24 +995,13 @@ export class IndexLevel {
    * Since the array is already sorted, binary search provides O(log n) performance instead of O(n).
    */
   private findCursorStartingIndex(items: IndexedItem[], sortDirection: SortDirection, sortProperty: string, cursorStartingKey: string): number {
-
-    const isAfterCursor = (item: IndexedItem): boolean => {
-      const { messageCid, indexes } = item;
-      const sortValue = indexes[sortProperty] as string | number;
-      const itemCompareValue = IndexLevel.keySegmentJoin(IndexLevel.encodeValue(sortValue), messageCid);
-
-      return sortDirection === SortDirection.Ascending ?
-        itemCompareValue > cursorStartingKey :
-        itemCompareValue < cursorStartingKey;
-    };
-
     // binary search for the first item after the cursor
     let low = 0;
     let high = items.length;
 
     while (low < high) {
       const mid = (low + high) >>> 1;
-      if (isAfterCursor(items[mid])) {
+      if (this.isItemAfterCursor(items[mid], sortDirection, sortProperty, cursorStartingKey)) {
         high = mid;
       } else {
         low = mid + 1;
@@ -731,6 +1011,21 @@ export class IndexLevel {
     return low < items.length ? low : -1;
   }
 
+  private isItemAfterCursor(
+    item: IndexedItem,
+    sortDirection: SortDirection,
+    sortProperty: string,
+    cursorStartingKey: string,
+  ): boolean {
+    const { messageCid, indexes } = item;
+    const sortValue = indexes[sortProperty] as string | number;
+    const itemCompareValue = IndexLevel.keySegmentJoin(IndexLevel.encodeValue(sortValue), messageCid);
+
+    return sortDirection === SortDirection.Ascending ?
+      itemCompareValue > cursorStartingKey :
+      itemCompareValue < cursorStartingKey;
+  }
+
   /**
    * Gets the indexes given an messageCid. This is a reverse lookup to construct starting keys, as well as deleting indexed items.
    */