@travetto/model 8.0.0-alpha.10 → 8.0.0-alpha.12

package/README.md

@@ -110,12 +110,19 @@ export interface ModelCrudSupport extends ModelBasicSupport {
   updatePartial<T extends ModelType>(cls: Class<T>, item: Partial<T> & { id: string }, view?: string): Promise<T>;
 
   /**
-   * List all items
+   * List all items of a collection, results returned in batches of items.
+   *
+   * Note: Batch size hint can be used to optimize batch size, but is not guaranteed.
+   *
+   * @param cls The class to list
+   * @param options Options for listing
    */
-  list<T extends ModelType>(cls: Class<T>): AsyncIterable<T>;
+  list<T extends ModelType>(cls: Class<T>, options?: ModelListOptions): AsyncIterable<T[]>;
 }
 ```
 
+The `list` operation returns batches of model records as an async stream.  It also accepts listing options such as `limit` to cap how many records are produced, alongside other runtime controls such as abort signals and batch size hints.
+
 ### Expiry
 Certain implementations will also provide support for automatic [Expiry](https://github.com/travetto/travetto/tree/main/module/model/src/types/expiry.ts#L10) of data at runtime.  This is extremely useful for temporary data as, and is used in the [Caching](https://github.com/travetto/travetto/tree/main/module/cache#readme "Caching functionality with decorators for declarative use.") module for expiring data accordingly.
 
@@ -262,8 +269,8 @@ export abstract class BaseModelSuite<T> {
     const svc = (await this.service);
     if (ModelCrudUtil.isSupported(svc)) {
       let i = 0;
-      for await (const __el of svc.list(cls)) {
-        i += 1;
+      for await (const batch of svc.list(cls)) {
+        i += batch.length;
       }
       return i;
     } else {
@@ -292,12 +299,12 @@ export abstract class BaseModelSuite<T> {
     return DependencyRegistryIndex.getInstance(this.serviceClass);
   }
 
-  async toArray<U>(src: AsyncIterable<U> | AsyncGenerator<U>): Promise<U[]> {
-    const out: U[] = [];
+  async toArray<U>(src: AsyncIterable<U | U[]> | AsyncGenerator<U | U[]>): Promise<U[]> {
+    const out: (U | U[])[] = [];
     for await (const el of src) {
       out.push(el);
     }
-    return out;
+    return castTo(out.flat());
   }
 }
 ```

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model",
-  "version": "8.0.0-alpha.10",
+  "version": "8.0.0-alpha.12",
   "type": "module",
   "description": "Datastore abstraction for core operations.",
   "keywords": [
@@ -27,14 +27,14 @@
     "directory": "module/model"
   },
   "dependencies": {
-    "@travetto/config": "^8.0.0-alpha.10",
-    "@travetto/di": "^8.0.0-alpha.10",
-    "@travetto/registry": "^8.0.0-alpha.10",
-    "@travetto/schema": "^8.0.0-alpha.10"
+    "@travetto/config": "^8.0.0-alpha.12",
+    "@travetto/di": "^8.0.0-alpha.11",
+    "@travetto/registry": "^8.0.0-alpha.11",
+    "@travetto/schema": "^8.0.0-alpha.12"
   },
   "peerDependencies": {
-    "@travetto/cli": "^8.0.0-alpha.15",
-    "@travetto/test": "^8.0.0-alpha.10"
+    "@travetto/cli": "^8.0.0-alpha.17",
+    "@travetto/test": "^8.0.0-alpha.11"
   },
   "peerDependenciesMeta": {
     "@travetto/cli": {

package/src/types/crud.ts

@@ -1,6 +1,6 @@
 import type { Class } from '@travetto/runtime';
 
-import type { ModelType, OptionalId } from '../types/model.ts';
+import type { ModelListOptions, ModelType, OptionalId } from '../types/model.ts';
 
 import type { ModelBasicSupport } from './basic.ts';
 
@@ -38,7 +38,12 @@ export interface ModelCrudSupport extends ModelBasicSupport {
   updatePartial<T extends ModelType>(cls: Class<T>, item: Partial<T> & { id: string }, view?: string): Promise<T>;
 
   /**
-   * List all items
+   * List all items of a collection, results returned in batches of items.
+   *
+   * Note: Batch size hint can be used to optimize batch size, but is not guaranteed.
+   *
+   * @param cls The class to list
+   * @param options Options for listing
    */
-  list<T extends ModelType>(cls: Class<T>): AsyncIterable<T>;
+  list<T extends ModelType>(cls: Class<T>, options?: ModelListOptions): AsyncIterable<T[]>;
 }

package/src/types/model.ts

@@ -16,4 +16,13 @@ export interface ModelType {
   id: string;
 }
 
-export type OptionalId<T extends ModelType> = Omit<T, 'id'> & { id?: string };
+export type OptionalId<T extends ModelType> = Omit<T, 'id'> & { id?: string };
+
+/**
+ * Options for listing items
+ */
+export interface ModelListOptions {
+  abort?: AbortSignal;
+  limit?: number;
+  batchSizeHint?: number;
+}

package/src/util/crud.ts

@@ -34,6 +34,21 @@ export class ModelCrudUtil {
     return { create, valid };
   }
 
+  static async filterOutNotFound<T extends ModelType>(actions: Promise<T>[] | undefined): Promise<T[]> {
+    if (!actions) {
+      return [];
+    }
+    return (await Promise.allSettled(actions)).map(p => {
+      if (p.status === 'fulfilled') {
+        return p.value;
+      } else if (p.reason instanceof NotFoundError) {
+        return undefined!;
+      } else {
+        throw p.reason;
+      }
+    }).filter(item => !!item);
+  }
+
   /**
    * Load model
    * @param cls Class to load model for
@@ -117,9 +132,6 @@ export class ModelCrudUtil {
         item = await handler(item) ?? item;
       }
     }
-    if (typeof item === 'object' && item && 'prePersist' in item && typeof item['prePersist'] === 'function') {
-      item = await item.prePersist() ?? item;
-    }
     return item;
   }
 
@@ -131,9 +143,6 @@ export class ModelCrudUtil {
     for (const handler of castTo<DataHandler<T>[]>(config.postLoad ?? [])) {
       item = await handler(item) ?? item;
     }
-    if (typeof item === 'object' && item && 'postLoad' in item && typeof item['postLoad'] === 'function') {
-      item = await item.postLoad() ?? item;
-    }
     return item;
   }
 

package/support/test/base.ts

@@ -22,8 +22,8 @@ export abstract class BaseModelSuite<T> {
     const svc = (await this.service);
     if (ModelCrudUtil.isSupported(svc)) {
       let i = 0;
-      for await (const __el of svc.list(cls)) {
-        i += 1;
+      for await (const batch of svc.list(cls)) {
+        i += batch.length;
       }
       return i;
     } else {
@@ -52,11 +52,11 @@ export abstract class BaseModelSuite<T> {
     return DependencyRegistryIndex.getInstance(this.serviceClass);
   }
 
-  async toArray<U>(src: AsyncIterable<U> | AsyncGenerator<U>): Promise<U[]> {
-    const out: U[] = [];
+  async toArray<U>(src: AsyncIterable<U | U[]> | AsyncGenerator<U | U[]>): Promise<U[]> {
+    const out: (U | U[])[] = [];
     for await (const el of src) {
       out.push(el);
     }
-    return out;
+    return castTo(out.flat());
   }
 }

package/support/test/crud.ts

@@ -3,7 +3,7 @@ import timers from 'node:timers/promises';
 
 import { Suite, Test } from '@travetto/test';
 import { Schema, Text, Precision, Required, } from '@travetto/schema';
-import { type ModelCrudSupport, Model, NotFoundError, PersistValue } from '@travetto/model';
+import { type ModelCrudSupport, Model, NotFoundError, PersistValue, PrePersist } from '@travetto/model';
 
 import { BaseModelSuite } from './base.ts';
 
@@ -42,14 +42,13 @@ class SimpleList {
 }
 
 @Model()
+@PrePersist((item) => {
+  item.name = `${item.name}-suffix`;
+})
 class User2 {
   id: string;
   address?: Address;
   name: string;
-
-  prePersist() {
-    this.name = `${this.name}-suffix`;
-  }
 }
 
 @Model()
@@ -77,6 +76,8 @@ class BigIntModel {
 @Suite()
 export abstract class ModelCrudSuite extends BaseModelSuite<ModelCrudSupport> {
 
+  indexLimitSkew = 0;
+
   @Test('save it')
   async save() {
     const service = await this.service;
@@ -268,6 +269,39 @@ export abstract class ModelCrudSuite extends BaseModelSuite<ModelCrudSupport> {
     assert(found[2].age === people[2].age);
   }
 
+  @Test('verify list abort signal')
+  async listAbortSignal() {
+    const service = await this.service;
+
+    await Promise.all(
+      [1, 2, 3, 4, 5, 6, 7, 8, 9, 10].map(x => service.upsert(Person, Person.from({
+        id: service.idSource.create(),
+        name: 'Bob',
+        age: 20 + x,
+        gender: 'm',
+        address: {
+          street1: 'a',
+          ...(x === 1 ? { street2: 'b' } : {})
+        }
+      })))
+    );
+
+    const controller = new AbortController();
+    const found: Person[] = [];
+
+    for await (const items of service.list(Person, { abort: controller.signal, batchSizeHint: 1 })) {
+      found.push(...items);
+      controller.abort();
+      await timers.setTimeout(10);
+    }
+
+    if (this.indexLimitSkew) {
+      assert(found.length > 0 && found.length < this.indexLimitSkew);
+    } else {
+      assert(found.length === 1);
+    }
+  }
+
   @Test('save it')
   async verifyRaw() {
     const service = await this.service;

package/support/test/polymorphism.ts

@@ -71,7 +71,7 @@ export abstract class ModelPolymorphismSuite extends BaseModelSuite<ModelCrudSup
     const fire2 = await service.get(Worker, fire.id);
     assert(fire2 instanceof Firefighter);
 
-    const all = await Array.fromAsync(service.list(Worker));
+    const all = await this.toArray(service.list(Worker));
     assert(all.length === 3);
 
     const doc3 = all.find(x => x instanceof Doctor);
@@ -89,7 +89,7 @@ export abstract class ModelPolymorphismSuite extends BaseModelSuite<ModelCrudSup
     assert(eng3.major === 'oranges');
     assert(eng3.name === 'cob');
 
-    const engineers = await Array.fromAsync(service.list(Engineer));
+    const engineers = await this.toArray(service.list(Engineer));
     assert(engineers.length === 1);
 
     await service.create(Engineer, Engineer.from({
@@ -97,10 +97,10 @@ export abstract class ModelPolymorphismSuite extends BaseModelSuite<ModelCrudSup
       name: 'bob2'
     }));
 
-    const all2 = await Array.fromAsync(service.list(Worker));
+    const all2 = await this.toArray(service.list(Worker));
     assert(all2.length === 4);
 
-    const engineers2 = await Array.fromAsync(service.list(Engineer));
+    const engineers2 = await this.toArray(service.list(Engineer));
     assert(engineers2.length === 2);
   }