@enbox/dwn-sdk-js 0.4.17 → 0.4.18

package/src/store/index-level.ts

@@ -23,6 +23,13 @@ export type IndexLevelConfig = {
 
 export type IndexedItem = { messageCid: string, indexes: KeyValues };
 
+type RecordLimitParentGroupsInput = {
+  candidateFilter: Filter;
+  filters: Filter[];
+  max: number;
+  parentIds: readonly string[];
+};
+
 /** A value that can be encoded into a lexicographically sortable index key. */
 type IndexableValue = string | number | boolean;
 
@@ -429,27 +436,106 @@ export class IndexLevel {
 
     const indexName = parentId === undefined ? 'protocolPath' : 'parentId';
     const indexValue = parentId ?? protocolPath;
-    const matchPrefix = IndexLevel.keySegmentJoin(IndexLevel.encodeValue(indexValue), '');
-    const partition = await this.getIndexPartition(tenant, indexName);
     const rankHeap: string[] = [];
 
-    for await (const [key, value] of partition.iterator({ gt: matchPrefix }, options)) {
-      if (!key.startsWith(matchPrefix)) {
-        break;
-      }
-
-      const item = JSON.parse(value) as IndexedItem;
+    for await (const item of this.iterateExactMatches(tenant, indexName, indexValue, options)) {
       if ((parentId === undefined && item.indexes.parentId !== undefined) ||
           !FilterUtility.matchFilter(item.indexes, candidateFilter)) {
         continue;
       }
 
-      IndexLevel.addToBoundedMaxHeap(rankHeap, IndexLevel.createRecordLimitRankKey(item.indexes), max);
+      IndexLevel.addToRecordLimitRankHeap(
+        rankHeap,
+        IndexLevel.createRecordLimitRankKey(item.indexes),
+        max,
+      );
     }
 
     return rankHeap[0];
   }
 
+  /**
+   * Queries the bounded occupants of explicitly selected direct-parent groups.
+   * Occupancy is resolved before caller filters, then the survivors are globally
+   * sorted and paginated using ordinary IndexLevel cursor semantics.
+   */
+  public async queryRecordLimitParentGroups(
+    tenant: string,
+    input: RecordLimitParentGroupsInput & { queryOptions: QueryOptions },
+    options?: IndexLevelOptions,
+  ): Promise<IndexedItem[]> {
+    assertValidSubtreeFilters([input.candidateFilter, ...input.filters]);
+    const { filters, queryOptions } = input;
+    const { cursor, limit, sortDirection = SortDirection.Ascending, sortProperty } = queryOptions;
+    if (limit !== undefined && limit <= 0) {
+      return [];
+    }
+
+    const cursorStartingKey = cursor === undefined ? undefined : this.createStartingKeyFromCursor(cursor);
+    const compareItems = (left: IndexedItem, right: IndexedItem): number =>
+      this.sortItems(left, right, sortProperty, sortDirection);
+    const matches: IndexedItem[] = [];
+
+    for await (const item of this.iterateRecordLimitParentGroupOccupants(tenant, input, options)) {
+      if (item.indexes[sortProperty] === undefined ||
+          !FilterUtility.matchAnyFilter(item.indexes, filters) ||
+          (cursorStartingKey !== undefined &&
+            !this.isItemAfterCursor(item, sortDirection, sortProperty, cursorStartingKey))) {
+        continue;
+      }
+
+      matches.push(item);
+    }
+
+    matches.sort(compareItems);
+    return limit === undefined ? matches : matches.slice(0, limit);
+  }
+
+  /** Counts the bounded occupants of explicitly selected direct-parent groups. */
+  public async countRecordLimitParentGroups(
+    tenant: string,
+    input: RecordLimitParentGroupsInput & { sortProperty: string },
+    options?: IndexLevelOptions,
+  ): Promise<number> {
+    assertValidSubtreeFilters([input.candidateFilter, ...input.filters]);
+    let count = 0;
+    for await (const item of this.iterateRecordLimitParentGroupOccupants(tenant, input, options)) {
+      if (item.indexes[input.sortProperty] !== undefined &&
+          FilterUtility.matchAnyFilter(item.indexes, input.filters)) {
+        count++;
+      }
+    }
+    return count;
+  }
+
+  /** Streams occupants from exact parentId ranges without opening the requested-sort index. */
+  private async * iterateRecordLimitParentGroupOccupants(
+    tenant: string,
+    input: RecordLimitParentGroupsInput,
+    options?: IndexLevelOptions,
+  ): AsyncGenerator<IndexedItem> {
+    for (const parentId of new Set(input.parentIds)) {
+      const candidateFilter: Filter = { ...input.candidateFilter, parentId };
+      const cutoff = await this.findRecordLimitRankCutoff(
+        tenant,
+        candidateFilter,
+        parentId,
+        input.max,
+        options,
+      );
+      if (cutoff === undefined) {
+        continue;
+      }
+
+      for await (const item of this.iterateExactMatches(tenant, 'parentId', parentId, options)) {
+        if (FilterUtility.matchFilter(item.indexes, candidateFilter) &&
+            lexicographicalCompare(IndexLevel.createRecordLimitRankKey(item.indexes), cutoff) <= 0) {
+          yield item;
+        }
+      }
+    }
+  }
+
   /**
    * Builds the deterministic `$recordLimit` rank key: oldest creation time,
    * then record ID. Message CID deliberately does not participate because
@@ -464,10 +550,14 @@ export class IndexLevel {
     return IndexLevel.keySegmentJoin(IndexLevel.encodeValue(dateCreated), IndexLevel.encodeValue(recordId));
   }
 
-  private static addToBoundedMaxHeap(heap: string[], value: string, max: number): void {
+  private static addToRecordLimitRankHeap(
+    heap: string[],
+    value: string,
+    max: number,
+  ): void {
     if (heap.length < max) {
       heap.push(value);
-      IndexLevel.siftRecordLimitRankUp(heap, heap.length - 1);
+      IndexLevel.siftRecordLimitRankHeapUp(heap, heap.length - 1);
       return;
     }
 
@@ -476,10 +566,13 @@ export class IndexLevel {
     }
 
     heap[0] = value;
-    IndexLevel.siftRecordLimitRankDown(heap, 0);
+    IndexLevel.siftRecordLimitRankHeapDown(heap, 0);
   }
 
-  private static siftRecordLimitRankUp(heap: string[], startIndex: number): void {
+  private static siftRecordLimitRankHeapUp(
+    heap: string[],
+    startIndex: number,
+  ): void {
     let index = startIndex;
     while (index > 0) {
       const parentIndex = (index - 1) >>> 1;
@@ -492,7 +585,10 @@ export class IndexLevel {
     }
   }
 
-  private static siftRecordLimitRankDown(heap: string[], startIndex: number): void {
+  private static siftRecordLimitRankHeapDown(
+    heap: string[],
+    startIndex: number,
+  ): void {
     let index = startIndex;
     while (true) {
       const leftIndex = index * 2 + 1;
@@ -501,7 +597,8 @@ export class IndexLevel {
       }
 
       const rightIndex = leftIndex + 1;
-      const largerChildIndex = rightIndex < heap.length && lexicographicalCompare(heap[rightIndex], heap[leftIndex]) > 0
+      const largerChildIndex = rightIndex < heap.length &&
+        lexicographicalCompare(heap[rightIndex], heap[leftIndex]) > 0
         ? rightIndex
         : leftIndex;
       if (lexicographicalCompare(heap[index], heap[largerChildIndex]) >= 0) {
@@ -774,22 +871,29 @@ export class IndexLevel {
     propertyValue: EqualFilter,
     options?: IndexLevelOptions
   ): Promise<IndexedItem[]> {
+    const matches: IndexedItem[] = [];
+    for await (const item of this.iterateExactMatches(tenant, propertyName, propertyValue, options)) {
+      matches.push(item);
+    }
+    return matches;
+  }
 
+  /** Streams one exact property-value range without materializing the range. */
+  private async * iterateExactMatches(
+    tenant: string,
+    propertyName: string,
+    propertyValue: EqualFilter,
+    options?: IndexLevelOptions,
+  ): AsyncGenerator<IndexedItem> {
     const matchPrefix = IndexLevel.keySegmentJoin(IndexLevel.encodeValue(propertyValue));
-    const iteratorOptions: LevelWrapperIteratorOptions<string> = {
-      gt: matchPrefix
-    };
-
     const filterPartition = await this.getIndexPartition(tenant, propertyName);
-    const matches: IndexedItem[] = [];
-    for await (const [ key, value ] of filterPartition.iterator(iteratorOptions, options)) {
-      // immediately stop if we arrive at an index that contains a different property value
+
+    for await (const [key, value] of filterPartition.iterator({ gt: matchPrefix }, options)) {
       if (!key.startsWith(matchPrefix)) {
-        break;
+        return;
       }
-      matches.push(JSON.parse(value) as IndexedItem);
+      yield JSON.parse(value) as IndexedItem;
     }
-    return matches;
   }
 
   /**
@@ -891,24 +995,13 @@ export class IndexLevel {
    * Since the array is already sorted, binary search provides O(log n) performance instead of O(n).
    */
   private findCursorStartingIndex(items: IndexedItem[], sortDirection: SortDirection, sortProperty: string, cursorStartingKey: string): number {
-
-    const isAfterCursor = (item: IndexedItem): boolean => {
-      const { messageCid, indexes } = item;
-      const sortValue = indexes[sortProperty] as string | number;
-      const itemCompareValue = IndexLevel.keySegmentJoin(IndexLevel.encodeValue(sortValue), messageCid);
-
-      return sortDirection === SortDirection.Ascending ?
-        itemCompareValue > cursorStartingKey :
-        itemCompareValue < cursorStartingKey;
-    };
-
     // binary search for the first item after the cursor
     let low = 0;
     let high = items.length;
 
     while (low < high) {
       const mid = (low + high) >>> 1;
-      if (isAfterCursor(items[mid])) {
+      if (this.isItemAfterCursor(items[mid], sortDirection, sortProperty, cursorStartingKey)) {
         high = mid;
       } else {
         low = mid + 1;
@@ -918,6 +1011,21 @@ export class IndexLevel {
     return low < items.length ? low : -1;
   }
 
+  private isItemAfterCursor(
+    item: IndexedItem,
+    sortDirection: SortDirection,
+    sortProperty: string,
+    cursorStartingKey: string,
+  ): boolean {
+    const { messageCid, indexes } = item;
+    const sortValue = indexes[sortProperty] as string | number;
+    const itemCompareValue = IndexLevel.keySegmentJoin(IndexLevel.encodeValue(sortValue), messageCid);
+
+    return sortDirection === SortDirection.Ascending ?
+      itemCompareValue > cursorStartingKey :
+      itemCompareValue < cursorStartingKey;
+  }
+
   /**
    * Gets the indexes given an messageCid. This is a reverse lookup to construct starting keys, as well as deleting indexed items.
    */

package/src/store/message-store-level.ts

@@ -347,15 +347,28 @@ export class MessageStoreLevel implements MessageStore, ReplicationFeedReader {
     // this adds 1 to the limit if provided, that way we can check to see if there are additional results and provide a return cursor.
     const queryOptions = MessageStoreLevel.buildQueryOptions(messageSort, pagination);
     const index = await this.index();
-    const results = options?.recordLimit === undefined
-      ? await index.query(tenant, filters, queryOptions, options)
-      : await this.collectRecordLimitPopulation({
-        tenant,
-        filters,
-        queryOptions,
-        recordLimit: options.recordLimit,
-        options,
-      });
+    const recordLimit = options?.recordLimit;
+    let results: IndexedItem[];
+    if (recordLimit === undefined) {
+      results = await index.query(tenant, filters, queryOptions, options);
+    } else {
+      const parentIds = MessageStoreLevel.getRecordLimitParentIds(recordLimit);
+      results = parentIds === undefined
+        ? await this.collectRecordLimitPopulation({
+          tenant,
+          filters,
+          queryOptions,
+          recordLimit,
+          options,
+        })
+        : await index.queryRecordLimitParentGroups(tenant, {
+          candidateFilter : MessageStoreLevel.buildRecordLimitCandidateFilter(recordLimit),
+          filters,
+          max             : recordLimit.max,
+          parentIds,
+          queryOptions,
+        }, options);
+    }
 
     let cursor: PaginationCursor | undefined;
     // checks to see if the returned results are greater than the limit, which would indicate additional results.
@@ -387,16 +400,28 @@ export class MessageStoreLevel implements MessageStore, ReplicationFeedReader {
 
     const queryOptions = MessageStoreLevel.buildQueryOptions(messageSort);
     const index = await this.index();
-    if (options?.recordLimit === undefined) {
+    const recordLimit = options?.recordLimit;
+    if (recordLimit === undefined) {
       return index.count(tenant, filters, queryOptions, options);
     }
 
+    const parentIds = MessageStoreLevel.getRecordLimitParentIds(recordLimit);
+    if (parentIds !== undefined) {
+      return index.countRecordLimitParentGroups(tenant, {
+        candidateFilter : MessageStoreLevel.buildRecordLimitCandidateFilter(recordLimit),
+        filters,
+        max             : recordLimit.max,
+        parentIds,
+        sortProperty    : queryOptions.sortProperty,
+      }, options);
+    }
+
     let count = 0;
     for await (const _item of this.iterateRecordLimitPopulation({
       tenant,
       filters,
       queryOptions,
-      recordLimit: options.recordLimit,
+      recordLimit,
       options,
     })) {
       count++;
@@ -544,9 +569,19 @@ export class MessageStoreLevel implements MessageStore, ReplicationFeedReader {
     if (recordLimit.contextId !== undefined) {
       filter.contextId = { subtree: recordLimit.contextId };
     }
+    if (recordLimit.parentId !== undefined) {
+      filter.parentId = recordLimit.parentId;
+    }
     return filter;
   }
 
+  private static getRecordLimitParentIds(recordLimit: RecordLimitOccupancy): readonly string[] | undefined {
+    if (recordLimit.parentId === undefined) {
+      return undefined;
+    }
+    return Array.isArray(recordLimit.parentId) ? recordLimit.parentId : [recordLimit.parentId];
+  }
+
   /**
    * Builds the IndexLevel QueryOptions object given MessageStore sort and pagination parameters.
    */

package/src/types/message-store.ts

@@ -19,9 +19,14 @@ export type RecordLimitOccupancy = {
   /**
    * Root of the candidate context subtree. An exact-record request supplies
    * its direct-parent context here so sibling candidates still consume slots.
-   * Omitted for root protocol paths.
+   * Omitted for root protocol paths and selections scoped only by direct parent.
    */
   contextId?: string;
+  /**
+   * Direct-parent groups selected by the Records filter. Including this in
+   * the candidate population keeps each selected group's ranking complete.
+   */
+  parentId?: string | string[];
   max: number;
 };
 

package/src/types/records-types.ts

@@ -161,7 +161,7 @@ export type RecordsFilter = {
   schema?: string;
   tags?: { [property:string]: RecordsWriteTagsFilter }
   recordId?: string;
-  parentId?: string;
+  parentId?: string | string[];
   dataFormat?: string;
   dataSize?: RangeFilter;
   dataCid?: string;

package/src/utils/record-limit-occupancy.ts

@@ -110,7 +110,7 @@ export async function resolveRecordLimitOccupancy(input: RecordLimitPolicyDepend
   recordsFilter: RecordsFilter;
   messageTimestamp: string;
 }): Promise<RecordLimitOccupancy | undefined> {
-  const { contextId, protocol, protocolPath } = input.recordsFilter;
+  const { contextId, parentId, protocol, protocolPath } = input.recordsFilter;
   if (protocol === undefined || protocolPath === undefined) {
     return undefined;
   }
@@ -134,8 +134,11 @@ export async function resolveRecordLimitOccupancy(input: RecordLimitPolicyDepend
   if (!protocolPath.includes('/')) {
     return recordLimit;
   }
+  if (parentId !== undefined) {
+    recordLimit.parentId = parentId;
+  }
   if (contextId === undefined) {
-    return undefined;
+    return recordLimit;
   }
 
   const contextDepth = contextId.split('/').length;

package/src/utils/records.ts

@@ -280,16 +280,19 @@ export class Records {
   }
 
   /**
-   * Nested protocol-path queries must select the target path itself or one of
-   * its ancestor contexts. An omitted, malformed, or deeper context cannot
-   * identify a bounded subtree of the selected protocol path.
+   * Nested protocol-path collection requests must select one or more direct
+   * parents, the target path itself, or one of its ancestor contexts. A
+   * bounded path-wide subscription may omit that scope because its initial
+   * page is explicitly capped; record-limited results still project each
+   * direct-parent population independently.
    */
-  public static validateNestedProtocolPathQueryScope(
+  public static validateNestedProtocolPathScope(
     filter: RecordsFilter,
     errorCode: DwnErrorCode,
     operationName: string,
+    allowBoundedPathWideSelection = false,
   ): void {
-    const { contextId, protocolPath } = filter;
+    const { contextId, parentId, protocolPath } = filter;
     if (!protocolPath?.includes('/')) {
       return;
     }
@@ -298,6 +301,14 @@ export class Records {
       return;
     }
 
+    if (parentId !== undefined && contextId === undefined) {
+      return;
+    }
+
+    if (contextId === undefined && allowBoundedPathWideSelection) {
+      return;
+    }
+
     const protocolPathDepth = protocolPath.split('/').length;
     const contextIdSegments = contextId?.split('/');
     if (
@@ -310,7 +321,7 @@ export class Records {
 
     throw new DwnError(
       errorCode,
-      `${operationName} for nested protocol path '${protocolPath}' must include a contextId selecting that path or one of its ancestors`
+      `${operationName} for nested protocol path '${protocolPath}' must include parentId or a contextId selecting that path or one of its ancestors`
     );
   }
 
@@ -405,16 +416,6 @@ export class Records {
     return filterCopy;
   }
 
-  public static buildUnpublishedControlRecordsFilter(filter: RecordsFilter, dateSort?: DateSort): Filter {
-    return {
-      ...Records.convertFilter(filter, dateSort),
-      interface         : DwnInterfaceName.Records,
-      method            : DwnMethodName.Write,
-      isLatestBaseState : true,
-      published         : false,
-    };
-  }
-
   public static buildControlRecordsFilters(filters: Filter[]): Filter[] {
     const controlFilters: Filter[] = [];
     for (const filter of filters) {
@@ -584,13 +585,6 @@ export class Records {
     }
   }
 
-  /**
-   * Determines if signature payload contains a protocolRole and should be authorized as such.
-   */
-  public static shouldProtocolAuthorize(signaturePayload: GenericSignaturePayload): boolean {
-    return signaturePayload.protocolRole !== undefined;
-  }
-
   /**
    * Checks if the filter supports returning published records.
    */
@@ -639,33 +633,4 @@ export class Records {
     return !incomingDeleteIsNewest;
   }
 
-  /**
-   * Checks whether or not the incoming records query filter should build an unpublished recipient MessageStore filter.
-   *
-   * @param filter The incoming RecordsFilter to evaluate against.
-   * @param recipient The recipient to check against the filter, typically the query/subscribe message author.
-   * @returns {boolean} True if the filter contains the recipient, or if the recipient filter is undefined/empty.
-   */
-  static shouldBuildUnpublishedRecipientFilter(filter: RecordsFilter, recipient: string): boolean {
-    const { recipient: recipientFilter } = filter;
-
-    return Array.isArray(recipientFilter) ?
-      recipientFilter.length === 0 || recipientFilter.includes(recipient) :
-      recipientFilter === undefined || recipientFilter === recipient;
-  }
-
-  /**
-   * Checks whether or not the incoming records query filter should build an unpublished author MessageStore filter.
-   *
-   * @param filter The incoming RecordsFilter to evaluate against.
-   * @param author The author to check against the filter, typically the query/subscribe message author.
-   * @returns {boolean} True if the filter contains the author, or if the author filter is undefined/empty.
-   */
-  static shouldBuildUnpublishedAuthorFilter(filter: RecordsFilter, author: string): boolean {
-    const { author: authorFilter } = filter;
-
-    return Array.isArray(authorFilter) ?
-      authorFilter.length === 0 || authorFilter.includes(author) :
-      authorFilter === undefined || authorFilter === author;
-  }
 }