@travetto/model-indexed 8.0.0-alpha.11 → 8.0.0-alpha.12

package/README.md

@@ -205,6 +205,9 @@ export interface ModelIndexedSupport extends ModelBasicSupport {
 
   /**
    * Page through entities by ranged index as defined by fields of idx
+   *
+   * Note: Limit is generally honored, but can vary depending on the underlying storage implementation.
+   *
    * @param cls The type to search by
    * @param idx The index to search against
    * @param body The payload of fields needed to search
@@ -214,10 +217,14 @@ export interface ModelIndexedSupport extends ModelBasicSupport {
     T extends ModelType,
     S extends SortedIndexSelection<T>,
     K extends KeyedIndexSelection<T>
-  >(cls: Class<T>, idx: SortedIndex<T, K, S>, body: KeyedIndexBody<T, K>, options?: ListPageOptions): Promise<ListPageResult<T>>;
+  >(cls: Class<T>, idx: SortedIndex<T, K, S>, body: KeyedIndexBody<T, K>, options?: ModelPageOptions): Promise<ModelPageResult<T>>;
 
   /**
    * List all entities by ranged index as defined by fields of idx
+   *
+   * Note: Limit is generally honored, but can vary depending on the underlying storage implementation.
+   * Batch size hint can be used to optimize batch size, but is not guaranteed.
+   *
    * @param cls The type to search by
    * @param idx The index to search against
    * @param body The payload of fields needed to search
@@ -226,7 +233,7 @@ export interface ModelIndexedSupport extends ModelBasicSupport {
     T extends ModelType,
     S extends SortedIndexSelection<T>,
     K extends KeyedIndexSelection<T>
-  >(cls: Class<T>, idx: SortedIndex<T, K, S>, body: KeyedIndexBody<T, K>,): AsyncIterable<T>;
+  >(cls: Class<T>, idx: SortedIndex<T, K, S>, body: KeyedIndexBody<T, K>, options?: ModelListOptions): AsyncIterable<T[]>;
 }
 ```
 
@@ -237,7 +244,7 @@ The service provides these operations:
    *  `updateByIndex` — Update an existing item by index
    *  `updatePartialByIndex` — Partially update an item by index
    *  `pageByIndex` — Fetch a page of items with pagination metadata
-   *  `listByIndex` — Stream all matching items from a sorted index
+   *  `listByIndex` — Stream matching items from a sorted index in batches, optionally capped by `limit`
 
 ### Getting Items
 Use `getByIndex` to fetch a single item by providing all required key fields.
@@ -345,15 +352,15 @@ export async function listExample(modelService: ModelIndexedSupport) {
 }
 ```
 
-Use `listByIndex` when you want to iterate through every matching item as an async stream.
+Use `listByIndex` when you want to iterate through matching items as an async stream of batches.  The same list options used by `list` are supported here, including `limit` when you want to stop after a fixed number of records.
 
 **Code: Streaming by Sorted Index**
 ```typescript
 export async function listStreamExample(modelService: ModelIndexedSupport) {
   const items: User[] = [];
 
-  for await (const user of modelService.listByIndex(User, recentUsers, {})) {
-    items.push(user);
+  for await (const batch of modelService.listByIndex(User, recentUsers, {}, { limit: 25 })) {
+    items.push(...batch);
   }
 
   return items;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model-indexed",
-  "version": "8.0.0-alpha.11",
+  "version": "8.0.0-alpha.12",
   "type": "module",
   "description": "Basic indexing support for model sources that support it.",
   "keywords": [
@@ -26,13 +26,13 @@
     "directory": "module/model-indexed"
   },
   "dependencies": {
-    "@travetto/model": "^8.0.0-alpha.10",
-    "@travetto/registry": "^8.0.0-alpha.10",
-    "@travetto/schema": "^8.0.0-alpha.10"
+    "@travetto/model": "^8.0.0-alpha.11",
+    "@travetto/registry": "^8.0.0-alpha.11",
+    "@travetto/schema": "^8.0.0-alpha.11"
   },
   "peerDependencies": {
-    "@travetto/cli": "^8.0.0-alpha.15",
-    "@travetto/test": "^8.0.0-alpha.10"
+    "@travetto/cli": "^8.0.0-alpha.16",
+    "@travetto/test": "^8.0.0-alpha.11"
   },
   "peerDependenciesMeta": {
     "@travetto/cli": {

package/src/types/list.ts

@@ -1,11 +1,12 @@
 import type { ModelType } from '@travetto/model';
 
-export type ListPageOptions<O = string> = {
+export interface ModelPageOptions<O = string> {
+  batchSizeHint?: number;
   limit?: number;
   offset?: O;
-};
+}
 
-export type ListPageResult<T extends ModelType> = {
+export interface ModelPageResult<T extends ModelType> {
   items: T[];
   nextOffset?: string;
-};
+}

package/src/types/service.ts

@@ -1,11 +1,11 @@
-import type { ModelType, ModelBasicSupport, OptionalId } from '@travetto/model';
+import type { ModelType, ModelBasicSupport, OptionalId, ModelListOptions } from '@travetto/model';
 import type { Class } from '@travetto/runtime';
 
 import type {
   KeyedIndexSelection, KeyedIndexBody, SortedIndexSelection, SortedIndex,
   SingleItemIndex, FullKeyedIndexBody, FullKeyedIndexWithPartialBody
 } from './indexes.ts';
-import type { ListPageOptions, ListPageResult } from './list.ts';
+import type { ModelPageOptions, ModelPageResult } from './list.ts';
 
 /**
  * Support for simple indexed activity
@@ -75,6 +75,9 @@ export interface ModelIndexedSupport extends ModelBasicSupport {
 
   /**
    * Page through entities by ranged index as defined by fields of idx
+   *
+   * Note: Limit is generally honored, but can vary depending on the underlying storage implementation.
+   *
    * @param cls The type to search by
    * @param idx The index to search against
    * @param body The payload of fields needed to search
@@ -84,10 +87,14 @@ export interface ModelIndexedSupport extends ModelBasicSupport {
     T extends ModelType,
     S extends SortedIndexSelection<T>,
     K extends KeyedIndexSelection<T>
-  >(cls: Class<T>, idx: SortedIndex<T, K, S>, body: KeyedIndexBody<T, K>, options?: ListPageOptions): Promise<ListPageResult<T>>;
+  >(cls: Class<T>, idx: SortedIndex<T, K, S>, body: KeyedIndexBody<T, K>, options?: ModelPageOptions): Promise<ModelPageResult<T>>;
 
   /**
    * List all entities by ranged index as defined by fields of idx
+   *
+   * Note: Limit is generally honored, but can vary depending on the underlying storage implementation.
+   * Batch size hint can be used to optimize batch size, but is not guaranteed.
+   *
    * @param cls The type to search by
    * @param idx The index to search against
    * @param body The payload of fields needed to search
@@ -96,5 +103,5 @@ export interface ModelIndexedSupport extends ModelBasicSupport {
     T extends ModelType,
     S extends SortedIndexSelection<T>,
     K extends KeyedIndexSelection<T>
-  >(cls: Class<T>, idx: SortedIndex<T, K, S>, body: KeyedIndexBody<T, K>,): AsyncIterable<T>;
+  >(cls: Class<T>, idx: SortedIndex<T, K, S>, body: KeyedIndexBody<T, K>, options?: ModelListOptions): AsyncIterable<T[]>;
 }

package/support/test/indexed.ts

@@ -1,4 +1,5 @@
 import assert from 'node:assert';
+import timers from 'node:timers/promises';
 
 import { Suite, Test } from '@travetto/test';
 import { Schema } from '@travetto/schema';
@@ -8,7 +9,7 @@ import { BaseModelSuite } from '@travetto/model/support/test/base.ts';
 
 import type { ModelIndexedSupport } from '../../src/types/service.ts';
 import { keyedIndex, sortedIndex } from '../../src/indexes.ts';
-import { IndexedFieldError } from '../../__index__.ts';
+import { IndexedFieldError } from '../../src/types/error.ts';
 
 @Model('index_user')
 class User {
@@ -371,4 +372,28 @@ export abstract class ModelIndexedSuite extends BaseModelSuite<ModelIndexedSuppo
     assert(items.length === allColors.length);
     assert.deepEqual(items, allColors.toReversed());
   }
+
+  @Test()
+  async listByIndexAbortSignal() {
+    const service = await this.service;
+
+    await Promise.all(
+      [20, 30, 40].map(age => service.create(User3, User3.from({ name: 'page', age, color: `${age}` })))
+    );
+
+    const controller = new AbortController();
+    const found: User3[] = [];
+
+    for await (const items of service.listByIndex(User3, userAgeIndex, { name: 'page' }, { abort: controller.signal, batchSizeHint: 1 })) {
+      found.push(...items);
+      controller.abort();
+      await timers.setTimeout(10);
+    }
+
+    if (this.indexLimitSkew) {
+      assert(found.length < this.indexLimitSkew && found.length > 0);
+    } else {
+      assert(found.length === 1);
+    }
+  }
 }