@travetto/model 8.0.0-alpha.1 → 8.0.0-alpha.11

package/README.md

@@ -16,7 +16,7 @@ yarn add @travetto/model
 This module provides a set of contracts/interfaces to data model persistence, modification and retrieval.  This module builds heavily upon the [Schema](https://github.com/travetto/travetto/tree/main/module/schema#readme "Data type registry for runtime validation, reflection and binding."), which is used for data model validation.
 
 ## A Simple Model
-A model can be simply defined by usage of the [@Model](https://github.com/travetto/travetto/tree/main/module/model/src/registry/decorator.ts#L14) decorator, which opts it into the [Schema](https://github.com/travetto/travetto/tree/main/module/schema#readme "Data type registry for runtime validation, reflection and binding.") contracts, as well as making it available to the [ModelRegistryIndex](https://github.com/travetto/travetto/tree/main/module/model/src/registry/registry-index.ts#L16).
+A model can be simply defined by usage of the [@Model](https://github.com/travetto/travetto/tree/main/module/model/src/registry/decorator.ts#L14) decorator, which opts it into the [Schema](https://github.com/travetto/travetto/tree/main/module/schema#readme "Data type registry for runtime validation, reflection and binding.") contracts, as well as making it available to the [ModelRegistryIndex](https://github.com/travetto/travetto/tree/main/module/model/src/registry/registry-index.ts#L12).
 
 **Code: Basic Structure**
 ```typescript
@@ -33,7 +33,7 @@ export class SampleModel {
 Once the model is defined, it can be leveraged with any of the services that implement the various model storage contracts.  These contracts allow for persisting and fetching of the associated model object.
 
 ## Contracts
-The module is mainly composed of contracts.  The contracts define the expected interface for various model patterns. The primary contracts are [Basic](https://github.com/travetto/travetto/tree/main/module/model/src/types/basic.ts#L8), [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/types/crud.ts#L11), [Indexed](https://github.com/travetto/travetto/tree/main/module/model/src/types/indexed.ts#L11), [Expiry](https://github.com/travetto/travetto/tree/main/module/model/src/types/expiry.ts#L10), [Blob](https://github.com/travetto/travetto/tree/main/module/model/src/types/blob.ts#L8) and [Bulk](https://github.com/travetto/travetto/tree/main/module/model/src/types/bulk.ts#L64).
+The module is mainly composed of contracts.  The contracts define the expected interface for various model patterns. The primary contracts are [Basic](https://github.com/travetto/travetto/tree/main/module/model/src/types/basic.ts#L8), [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/types/crud.ts#L11), [Expiry](https://github.com/travetto/travetto/tree/main/module/model/src/types/expiry.ts#L10), [Blob](https://github.com/travetto/travetto/tree/main/module/model/src/types/blob.ts#L8) and [Bulk](https://github.com/travetto/travetto/tree/main/module/model/src/types/bulk.ts#L64).
 
 ### Basic
 All [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.") implementations, must honor the [Basic](https://github.com/travetto/travetto/tree/main/module/model/src/types/basic.ts#L8) contract to be able to participate in the model ecosystem.  This contract represents the bare minimum for a model service.
@@ -110,51 +110,18 @@ export interface ModelCrudSupport extends ModelBasicSupport {
   updatePartial<T extends ModelType>(cls: Class<T>, item: Partial<T> & { id: string }, view?: string): Promise<T>;
 
   /**
-   * List all items
+   * List all items of a collection, results returned in batches of items.
+   *
+   * Note: Batch size hint can be used to optimize batch size, but is not guaranteed.
+   *
+   * @param cls The class to list
+   * @param options Options for listing
    */
-  list<T extends ModelType>(cls: Class<T>): AsyncIterable<T>;
+  list<T extends ModelType>(cls: Class<T>, options?: ModelListOptions): AsyncIterable<T[]>;
 }
 ```
 
-### Indexed
-Additionally, an implementation may support the ability for basic [Indexed](https://github.com/travetto/travetto/tree/main/module/model/src/types/indexed.ts#L11) queries. This is not the full featured query support of [Data Model Querying](https://github.com/travetto/travetto/tree/main/module/model-query#readme "Datastore abstraction for advanced query support."), but allowing for indexed lookups.  This does not support listing by index, but may be added at a later date.
-
-**Code: Indexed Contract**
-```typescript
-export interface ModelIndexedSupport extends ModelBasicSupport {
-  /**
-   * Get entity by index as defined by fields of idx and the body fields
-   * @param cls The type to search by
-   * @param idx The index name to search against
-   * @param body The payload of fields needed to search
-   */
-  getByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<T>;
-
-  /**
-   * Delete entity by index as defined by fields of idx and the body fields
-   * @param cls The type to search by
-   * @param idx The index name to search against
-   * @param body The payload of fields needed to search
-   */
-  deleteByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<void>;
-
-  /**
-   * List entity by ranged index as defined by fields of idx and the body fields
-   * @param cls The type to search by
-   * @param idx The index name to search against
-   * @param body The payload of fields needed to search
-   */
-  listByIndex<T extends ModelType>(cls: Class<T>, idx: string, body?: DeepPartial<T>): AsyncIterable<T>;
-
-  /**
-   * Upsert by index, allowing the index to act as a primary key
-   * @param cls The type to create for
-   * @param idx The index name to use
-   * @param body The document to potentially store
-   */
-  upsertByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: OptionalId<T>): Promise<T>;
-}
-```
+The `list` operation returns batches of model records as an async stream.  It also accepts listing options such as `limit` to cap how many records are produced, alongside other runtime controls such as abort signals and batch size hints.
 
 ### Expiry
 Certain implementations will also provide support for automatic [Expiry](https://github.com/travetto/travetto/tree/main/module/model/src/types/expiry.ts#L10) of data at runtime.  This is extremely useful for temporary data as, and is used in the [Caching](https://github.com/travetto/travetto/tree/main/module/cache#readme "Caching functionality with decorators for declarative use.") module for expiring data accordingly.
@@ -302,8 +269,8 @@ export abstract class BaseModelSuite<T> {
     const svc = (await this.service);
     if (ModelCrudUtil.isSupported(svc)) {
       let i = 0;
-      for await (const __el of svc.list(cls)) {
-        i += 1;
+      for await (const batch of svc.list(cls)) {
+        i += batch.length;
       }
       return i;
     } else {
@@ -332,12 +299,12 @@ export abstract class BaseModelSuite<T> {
     return DependencyRegistryIndex.getInstance(this.serviceClass);
   }
 
-  async toArray<U>(src: AsyncIterable<U> | AsyncGenerator<U>): Promise<U[]> {
-    const out: U[] = [];
+  async toArray<U>(src: AsyncIterable<U | U[]> | AsyncGenerator<U | U[]>): Promise<U[]> {
+    const out: (U | U[])[] = [];
     for await (const el of src) {
       out.push(el);
     }
-    return out;
+    return castTo(out.flat());
   }
 }
 ```
@@ -354,7 +321,7 @@ Usage: model:export [options] <provider:string> <models...:string>
 Options:
   -p, --profile <string>  Application profiles
   -m, --module <module>   Module to run for
-  -h, --help              display help for command
+  --help                  display help for command
 
 Providers
 --------------------
@@ -377,7 +344,7 @@ Usage: model:install [options] <provider:string> <models...:string>
 Options:
   -p, --profile <string>  Application profiles
   -m, --module <module>   Module to run for
-  -h, --help              display help for command
+  --help                  display help for command
 
 Providers
 --------------------

package/__index__.ts

@@ -8,7 +8,6 @@ export * from './src/types/basic.ts';
 export * from './src/types/blob.ts';
 export * from './src/types/bulk.ts';
 export * from './src/types/crud.ts';
-export * from './src/types/indexed.ts';
 export * from './src/types/expiry.ts';
 export * from './src/types/storage.ts';
 
@@ -16,7 +15,6 @@ export * from './src/util/blob.ts';
 export * from './src/util/bulk.ts';
 export * from './src/util/crud.ts';
 export * from './src/util/expiry.ts';
-export * from './src/util/indexed.ts';
 export * from './src/util/storage.ts';
 
 export * from './src/error/exists.ts';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model",
-  "version": "8.0.0-alpha.1",
+  "version": "8.0.0-alpha.11",
   "type": "module",
   "description": "Datastore abstraction for core operations.",
   "keywords": [
@@ -27,14 +27,14 @@
     "directory": "module/model"
   },
   "dependencies": {
-    "@travetto/config": "^8.0.0-alpha.1",
-    "@travetto/di": "^8.0.0-alpha.1",
-    "@travetto/registry": "^8.0.0-alpha.1",
-    "@travetto/schema": "^8.0.0-alpha.1"
+    "@travetto/config": "^8.0.0-alpha.11",
+    "@travetto/di": "^8.0.0-alpha.11",
+    "@travetto/registry": "^8.0.0-alpha.11",
+    "@travetto/schema": "^8.0.0-alpha.11"
   },
   "peerDependencies": {
-    "@travetto/cli": "^8.0.0-alpha.1",
-    "@travetto/test": "^8.0.0-alpha.1"
+    "@travetto/cli": "^8.0.0-alpha.16",
+    "@travetto/test": "^8.0.0-alpha.11"
   },
   "peerDependenciesMeta": {
     "@travetto/cli": {

package/src/error/invalid-index.ts

@@ -1,13 +1,13 @@
 import { type Class, RuntimeError } from '@travetto/runtime';
 
-import type { IndexConfig } from '../registry/types.ts';
 import type { ModelType } from '../types/model.ts';
+import type { IndexConfig } from '../registry/types.ts';
 
 /**
  * Represents when an index is invalid
  */
 export class IndexNotSupported<T extends ModelType> extends RuntimeError {
-  constructor(cls: Class<T>, idx: IndexConfig<T>, message: string = '') {
+  constructor(cls: Class<T>, idx: IndexConfig, message: string = '') {
     super(`${typeof cls === 'string' ? cls : cls.name} and index ${idx.name} of type ${idx.type} is not supported. ${message}`.trim(), { category: 'data' });
   }
 }

package/src/registry/decorator.ts

@@ -1,8 +1,8 @@
-import { RuntimeError, castTo, type Class, getClass } from '@travetto/runtime';
+import { castTo, type Class, getClass } from '@travetto/runtime';
 import { SchemaRegistryIndex } from '@travetto/schema';
 
 import type { ModelType } from '../types/model.ts';
-import type { DataHandler, IndexConfig, ModelConfig, PrePersistScope } from './types.ts';
+import type { DataHandler, ModelConfig, PrePersistScope } from './types.ts';
 import { ModelRegistryIndex } from './registry-index.ts';
 
 /**
@@ -24,19 +24,6 @@ export function Model(config: Partial<ModelConfig<ModelType>> | string = {}) {
   };
 }
 
-/**
- * Defines an index on a model
- * @kind decorator
- */
-export function Index<T extends ModelType>(...indices: IndexConfig<T>[]) {
-  if (indices.some(config => config.fields.some(field => field === 'id'))) {
-    throw new RuntimeError('Cannot create an index with the id field');
-  }
-  return function (cls: Class<T>): void {
-    ModelRegistryIndex.getForRegister(cls).register({ indices });
-  };
-}
-
 /**
  * Model field decorator for denoting expiry date/time
  * @augments `@travetto/schema:Field`

package/src/registry/registry-adapter.ts

@@ -7,7 +7,7 @@ import type { ModelConfig } from './types.ts';
 function combineClasses(target: ModelConfig, sources: Partial<ModelConfig>[]): ModelConfig {
   for (const source of sources) {
     Object.assign(target, source, {
-      indices: [...(target.indices || []), ...(source.indices || [])],
+      indices: { ...(target.indices || {}), ...(source.indices || {}) },
       postLoad: [...(target.postLoad || []), ...(source.postLoad || [])],
       prePersist: [...(target.prePersist || []), ...(source.prePersist || [])],
     });
@@ -29,7 +29,7 @@ export class ModelRegistryAdapter implements RegistryAdapter<ModelConfig> {
   register(...data: Partial<ModelConfig>[]): ModelConfig {
     const config = this.#config ??= {
       class: this.#cls,
-      indices: [],
+      indices: {},
       autoCreate: 'development',
       store: this.#cls.name,
       postLoad: [],

package/src/registry/registry-index.ts

@@ -2,13 +2,9 @@ import { type RegistryIndex, RegistryIndexStore, Registry } from '@travetto/regi
 import { RuntimeError, castTo, type Class } from '@travetto/runtime';
 import { SchemaRegistryIndex } from '@travetto/schema';
 
-import type { IndexConfig, IndexType, ModelConfig } from './types.ts';
+import type { IndexConfig, ModelConfig } from './types.ts';
 import type { ModelType } from '../types/model.ts';
 import { ModelRegistryAdapter } from './registry-adapter.ts';
-import { IndexNotSupported } from '../error/invalid-index.ts';
-import { NotFoundError } from '../error/not-found.ts';
-
-type IndexResult<T extends ModelType, K extends IndexType[]> = IndexConfig<T> & { type: K[number] };
 
 /**
  * Model registry index for managing model configurations across classes
@@ -33,12 +29,8 @@ export class ModelRegistryIndex implements RegistryIndex {
     return this.#instance.getStoreName(cls);
   }
 
-  static getIndices<T extends ModelType, K extends IndexType[]>(cls: Class<T>, supportedTypes?: K): IndexResult<T, K>[] {
-    return this.#instance.getIndices(cls, supportedTypes);
-  }
-
-  static getIndex<T extends ModelType, K extends IndexType[]>(cls: Class<T>, name: string, supportedTypes?: K): IndexResult<T, K> {
-    return this.#instance.getIndex(cls, name, supportedTypes);
+  static getIndices<T extends ModelType>(cls: Class<T>): IndexConfig[] {
+    return this.#instance.getIndices(cls);
   }
 
   static getExpiryFieldName<T extends ModelType>(cls: Class<T>): keyof T {
@@ -94,25 +86,11 @@ export class ModelRegistryIndex implements RegistryIndex {
     return this.store.get(cls).get().store;
   }
 
-  /**
-   * Get Index
-   */
-  getIndex<T extends ModelType, K extends IndexType[]>(cls: Class<T>, name: string, supportedTypes?: K): IndexResult<T, K> {
-    const config = this.getConfig(cls).indices?.find((idx): idx is IndexConfig<T> => idx.name === name);
-    if (!config) {
-      throw new NotFoundError(`${cls.name} Index`, `${name}`);
-    }
-    if (supportedTypes && !supportedTypes.includes(config.type)) {
-      throw new IndexNotSupported(cls, config, `${config.type} indices are not supported.`);
-    }
-    return config;
-  }
-
   /**
    * Get Indices
    */
-  getIndices<T extends ModelType, K extends IndexType[]>(cls: Class<T>, supportedTypes?: K): IndexResult<T, K>[] {
-    return (this.getConfig(cls).indices ?? []).filter((idx): idx is IndexConfig<T> => !supportedTypes || supportedTypes.includes(idx.type));
+  getIndices<T extends ModelType>(cls: Class<T>): IndexConfig[] {
+    return Object.values(this.getConfig(cls).indices ?? []);
   }
 
   /**

package/src/registry/types.ts

@@ -1,23 +1,29 @@
-import type { Class, Primitive, ValidFields } from '@travetto/runtime';
+import type { Class } from '@travetto/runtime';
 
 import type { ModelType } from '../types/model.ts';
 
-type RetainPrimitiveFields<T> = Pick<T, ValidFields<T, Primitive | Date>>;
-
-export type SortClauseRaw<T> = {
-  [P in keyof T]?:
-  T[P] extends object ? SortClauseRaw<RetainPrimitiveFields<T[P]>> : 1 | -1;
-};
-
-type IndexClauseRaw<T> = {
-  [P in keyof T]?:
-  T[P] extends object ? IndexClauseRaw<RetainPrimitiveFields<T[P]>> : 1 | -1 | true;
-};
-
 export type DataHandler<T = unknown> = (inst: T) => (Promise<T | void> | T | void);
 
 export type PrePersistScope = 'full' | 'partial' | 'all';
 
+/**
+ * Index options
+ */
+export type IndexConfig<V extends string = string> = {
+  /**
+   * Index name
+   */
+  name: string;
+  /**
+   * Type
+   */
+  type: V;
+  /**
+   * Class the index belongs to
+   */
+  class: Class<ModelType>;
+};
+
 /**
  * Model config
  */
@@ -33,7 +39,7 @@ export class ModelConfig<T extends ModelType = ModelType> {
   /**
    * Indices
    */
-  indices?: IndexConfig<T>[];
+  indices?: Record<string, IndexConfig>;
   /**
    * Vendor specific extras
    */
@@ -54,29 +60,4 @@ export class ModelConfig<T extends ModelType = ModelType> {
    * Post-load handlers
    */
   postLoad?: DataHandler<unknown>[];
-}
-
-/**
- * Supported index types
- */
-export type IndexType = 'unique' | 'unsorted' | 'sorted';
-
-/**
- * Index options
- */
-export type IndexConfig<T extends ModelType> = {
-  /**
-   * Index name
-   */
-  name: string;
-  /**
-   * Fields and sort order
-   */
-  fields: IndexClauseRaw<RetainPrimitiveFields<T>>[];
-  /**
-   * Type
-   */
-  type: IndexType;
-};
-
-export type IndexField<T extends ModelType> = IndexClauseRaw<RetainPrimitiveFields<T>>;
+}

package/src/types/crud.ts

@@ -1,6 +1,6 @@
 import type { Class } from '@travetto/runtime';
 
-import type { ModelType, OptionalId } from '../types/model.ts';
+import type { ModelListOptions, ModelType, OptionalId } from '../types/model.ts';
 
 import type { ModelBasicSupport } from './basic.ts';
 
@@ -38,7 +38,12 @@ export interface ModelCrudSupport extends ModelBasicSupport {
   updatePartial<T extends ModelType>(cls: Class<T>, item: Partial<T> & { id: string }, view?: string): Promise<T>;
 
   /**
-   * List all items
+   * List all items of a collection, results returned in batches of items.
+   *
+   * Note: Batch size hint can be used to optimize batch size, but is not guaranteed.
+   *
+   * @param cls The class to list
+   * @param options Options for listing
    */
-  list<T extends ModelType>(cls: Class<T>): AsyncIterable<T>;
+  list<T extends ModelType>(cls: Class<T>, options?: ModelListOptions): AsyncIterable<T[]>;
 }

package/src/types/model.ts

@@ -16,4 +16,13 @@ export interface ModelType {
   id: string;
 }
 
-export type OptionalId<T extends ModelType> = Omit<T, 'id'> & { id?: string };
+export type OptionalId<T extends ModelType> = Omit<T, 'id'> & { id?: string };
+
+/**
+ * Options for listing items
+ */
+export interface ModelListOptions {
+  abort?: AbortSignal;
+  limit?: number;
+  batchSizeHint?: number;
+}

package/src/util/crud.ts

@@ -34,6 +34,21 @@ export class ModelCrudUtil {
     return { create, valid };
   }
 
+  static async filterOutNotFound<T extends ModelType>(actions: Promise<T>[] | undefined): Promise<T[]> {
+    if (!actions) {
+      return [];
+    }
+    return (await Promise.allSettled(actions)).map(p => {
+      if (p.status === 'fulfilled') {
+        return p.value;
+      } else if (p.reason instanceof NotFoundError) {
+        return undefined!;
+      } else {
+        throw p.reason;
+      }
+    }).filter(item => !!item);
+  }
+
   /**
    * Load model
    * @param cls Class to load model for
@@ -117,9 +132,6 @@ export class ModelCrudUtil {
         item = await handler(item) ?? item;
       }
     }
-    if (typeof item === 'object' && item && 'prePersist' in item && typeof item['prePersist'] === 'function') {
-      item = await item.prePersist() ?? item;
-    }
     return item;
   }
 
@@ -131,9 +143,6 @@ export class ModelCrudUtil {
     for (const handler of castTo<DataHandler<T>[]>(config.postLoad ?? [])) {
       item = await handler(item) ?? item;
     }
-    if (typeof item === 'object' && item && 'postLoad' in item && typeof item['postLoad'] === 'function') {
-      item = await item.postLoad() ?? item;
-    }
     return item;
   }
 

package/support/doc.support.tsx

@@ -8,7 +8,6 @@ import type { ModelBlobSupport } from '../src/types/blob.ts';
 import type { ModelBulkSupport } from '../src/types/bulk.ts';
 import type { ModelCrudSupport } from '../src/types/crud.ts';
 import type { ModelExpirySupport } from '../src/types/expiry.ts';
-import type { ModelIndexedSupport } from '../src/types/indexed.ts';
 
 const toLink = (title: string, target: Function): DocJSXElementByFn<'CodeLink'> =>
   d.codeLink(title, Runtime.getSourceFile(target), new RegExp(`\\binterface\\s+${target.name}`));
@@ -17,7 +16,6 @@ export const Links = {
   Basic: toLink('Basic', toConcrete<ModelBasicSupport>()),
   Crud: toLink('CRUD', toConcrete<ModelCrudSupport>()),
   Expiry: toLink('Expiry', toConcrete<ModelExpirySupport>()),
-  Indexed: toLink('Indexed', toConcrete<ModelIndexedSupport>()),
   Bulk: toLink('Bulk', toConcrete<ModelBulkSupport>()),
   Blob: toLink('Blob', toConcrete<ModelBlobSupport>()),
 };

package/support/test/base.ts

@@ -22,8 +22,8 @@ export abstract class BaseModelSuite<T> {
     const svc = (await this.service);
     if (ModelCrudUtil.isSupported(svc)) {
       let i = 0;
-      for await (const __el of svc.list(cls)) {
-        i += 1;
+      for await (const batch of svc.list(cls)) {
+        i += batch.length;
       }
       return i;
     } else {
@@ -52,11 +52,11 @@ export abstract class BaseModelSuite<T> {
     return DependencyRegistryIndex.getInstance(this.serviceClass);
   }
 
-  async toArray<U>(src: AsyncIterable<U> | AsyncGenerator<U>): Promise<U[]> {
-    const out: U[] = [];
+  async toArray<U>(src: AsyncIterable<U | U[]> | AsyncGenerator<U | U[]>): Promise<U[]> {
+    const out: (U | U[])[] = [];
     for await (const el of src) {
       out.push(el);
     }
-    return out;
+    return castTo(out.flat());
   }
 }

package/support/test/blob.ts

@@ -92,7 +92,7 @@ export abstract class ModelBlobSuite extends BaseModelSuite<ModelBlobSupport> {
     const range = BinaryMetadataUtil.enforceRange({ start: 10, end: 20 }, partialMeta);
     assert(subContent.length === (range.end - range.start) + 1);
 
-    const og = await this.fixture.readText('/text.txt');
+    const og = await this.fixture.readUTF8('/text.txt');
 
     assert(subContent === og.substring(10, 21));
 

package/support/test/crud.ts

@@ -3,7 +3,7 @@ import timers from 'node:timers/promises';
 
 import { Suite, Test } from '@travetto/test';
 import { Schema, Text, Precision, Required, } from '@travetto/schema';
-import { type ModelCrudSupport, Model, NotFoundError, PersistValue } from '@travetto/model';
+import { type ModelCrudSupport, Model, NotFoundError, PersistValue, PrePersist } from '@travetto/model';
 
 import { BaseModelSuite } from './base.ts';
 
@@ -42,14 +42,13 @@ class SimpleList {
 }
 
 @Model()
+@PrePersist((item) => {
+  item.name = `${item.name}-suffix`;
+})
 class User2 {
   id: string;
   address?: Address;
   name: string;
-
-  prePersist() {
-    this.name = `${this.name}-suffix`;
-  }
 }
 
 @Model()
@@ -77,6 +76,8 @@ class BigIntModel {
 @Suite()
 export abstract class ModelCrudSuite extends BaseModelSuite<ModelCrudSupport> {
 
+  indexLimitSkew = 0;
+
   @Test('save it')
   async save() {
     const service = await this.service;
@@ -268,6 +269,39 @@ export abstract class ModelCrudSuite extends BaseModelSuite<ModelCrudSupport> {
     assert(found[2].age === people[2].age);
   }
 
+  @Test('verify list abort signal')
+  async listAbortSignal() {
+    const service = await this.service;
+
+    await Promise.all(
+      [1, 2, 3, 4, 5, 6, 7, 8, 9, 10].map(x => service.upsert(Person, Person.from({
+        id: service.idSource.create(),
+        name: 'Bob',
+        age: 20 + x,
+        gender: 'm',
+        address: {
+          street1: 'a',
+          ...(x === 1 ? { street2: 'b' } : {})
+        }
+      })))
+    );
+
+    const controller = new AbortController();
+    const found: Person[] = [];
+
+    for await (const items of service.list(Person, { abort: controller.signal, batchSizeHint: 1 })) {
+      found.push(...items);
+      controller.abort();
+      await timers.setTimeout(10);
+    }
+
+    if (this.indexLimitSkew) {
+      assert(found.length > 0 && found.length < this.indexLimitSkew);
+    } else {
+      assert(found.length === 1);
+    }
+  }
+
   @Test('save it')
   async verifyRaw() {
     const service = await this.service;

package/support/test/polymorphism.ts

@@ -3,13 +3,12 @@ import timers from 'node:timers/promises';
 
 import { Suite, Test } from '@travetto/test';
 import { castTo } from '@travetto/runtime';
-import { Schema, DiscriminatorField, Text, TypeMismatchError, Discriminated } from '@travetto/schema';
+import { Schema, DiscriminatorField, Text, TypeMismatchError } from '@travetto/schema';
 import {
-  type ModelIndexedSupport, Index, type ModelCrudSupport, Model,
+  type ModelCrudSupport, Model,
   NotFoundError, SubTypeNotSupportedError, PersistValue
 } from '@travetto/model';
 
-import { ModelIndexedUtil } from '../../src/util/indexed.ts';
 import { ExistsError } from '../../src/error/exists.ts';
 
 import { BaseModelSuite } from './base.ts';
@@ -43,42 +42,6 @@ export class Engineer extends Worker {
   major: string;
 }
 
-@Model()
-@Index({
-  name: 'worker-name',
-  type: 'sorted',
-  fields: [{ name: 1 }, { age: 1 }]
-})
-@Discriminated('type')
-export class IndexedWorker {
-  id: string;
-  type: string;
-  name: string;
-  age?: number;
-}
-@Model()
-export class IndexedDoctor extends IndexedWorker {
-  specialty: string;
-}
-
-@Model()
-export class IndexedFirefighter extends IndexedWorker {
-  firehouse: number;
-}
-
-@Model()
-export class IndexedEngineer extends IndexedWorker {
-  major: string;
-}
-
-async function collect<T>(iterable: AsyncIterable<T>): Promise<T[]> {
-  const out: T[] = [];
-  for await (const el of iterable) {
-    out.push(el);
-  }
-  return out;
-}
-
 @Suite()
 export abstract class ModelPolymorphismSuite extends BaseModelSuite<ModelCrudSupport> {
 
@@ -108,7 +71,7 @@ export abstract class ModelPolymorphismSuite extends BaseModelSuite<ModelCrudSup
     const fire2 = await service.get(Worker, fire.id);
     assert(fire2 instanceof Firefighter);
 
-    const all = await collect(service.list(Worker));
+    const all = await this.toArray(service.list(Worker));
     assert(all.length === 3);
 
     const doc3 = all.find(x => x instanceof Doctor);
@@ -126,7 +89,7 @@ export abstract class ModelPolymorphismSuite extends BaseModelSuite<ModelCrudSup
     assert(eng3.major === 'oranges');
     assert(eng3.name === 'cob');
 
-    const engineers = await collect(service.list(Engineer));
+    const engineers = await this.toArray(service.list(Engineer));
     assert(engineers.length === 1);
 
     await service.create(Engineer, Engineer.from({
@@ -134,10 +97,10 @@ export abstract class ModelPolymorphismSuite extends BaseModelSuite<ModelCrudSup
       name: 'bob2'
     }));
 
-    const all2 = await collect(service.list(Worker));
+    const all2 = await this.toArray(service.list(Worker));
     assert(all2.length === 4);
 
-    const engineers2 = await collect(service.list(Engineer));
+    const engineers2 = await this.toArray(service.list(Engineer));
     assert(engineers2.length === 2);
   }
 
@@ -207,75 +170,4 @@ export abstract class ModelPolymorphismSuite extends BaseModelSuite<ModelCrudSup
       e => e instanceof SubTypeNotSupportedError || e instanceof NotFoundError
     );
   }
-
-  @Test('Polymorphic index', { skip: BaseModelSuite.ifNot(ModelIndexedUtil.isSupported) })
-  async polymorphicIndexGet() {
-    const service: ModelIndexedSupport = castTo(await this.service);
-    const now = 30;
-    const [doc, fire, eng] = [
-      IndexedDoctor.from({ name: 'bob', specialty: 'feet', age: now }),
-      IndexedFirefighter.from({ name: 'rob', firehouse: 20, age: now }),
-      IndexedEngineer.from({ name: 'cob', major: 'oranges', age: now })
-    ];
-
-    await this.saveAll(IndexedWorker, [doc, fire, eng]);
-
-    const result = await service.getByIndex(IndexedWorker, 'worker-name', {
-      age: now,
-      name: 'rob'
-    });
-
-    assert(result instanceof IndexedFirefighter);
-
-    try {
-      const res2 = await service.getByIndex(IndexedFirefighter, 'worker-name', {
-        age: now,
-        name: 'rob'
-      });
-      assert(res2 instanceof IndexedFirefighter); // If service allows for get by subtype
-    } catch (err) {
-      assert(err instanceof SubTypeNotSupportedError || err instanceof NotFoundError); // If it does not
-    }
-  }
-
-  @Test('Polymorphic index', { skip: BaseModelSuite.ifNot(ModelIndexedUtil.isSupported) })
-  async polymorphicIndexDelete() {
-    const service: ModelIndexedSupport = castTo(await this.service);
-    const now = 30;
-    const [doc, fire, eng] = [
-      IndexedDoctor.from({ name: 'bob', specialty: 'feet', age: now }),
-      IndexedFirefighter.from({ name: 'rob', firehouse: 20, age: now }),
-      IndexedEngineer.from({ name: 'cob', major: 'oranges', age: now })
-    ];
-
-    await this.saveAll(IndexedWorker, [doc, fire, eng]);
-
-    assert(await this.getSize(IndexedWorker) === 3);
-
-    await service.deleteByIndex(IndexedWorker, 'worker-name', {
-      age: now,
-      name: 'bob'
-    });
-
-    assert(await this.getSize(IndexedWorker) === 2);
-    assert(await this.getSize(IndexedDoctor) === 0);
-
-    try {
-      await service.deleteByIndex(IndexedFirefighter, 'worker-name', {
-        age: now,
-        name: 'rob'
-      });
-    } catch (err) {
-      assert(err instanceof SubTypeNotSupportedError || err instanceof NotFoundError);
-    }
-
-    try {
-      await service.deleteByIndex(IndexedEngineer, 'worker-name', {
-        age: now,
-        name: 'bob'
-      });
-    } catch (err) {
-      assert(err instanceof SubTypeNotSupportedError || err instanceof NotFoundError);
-    }
-  }
 }

package/src/types/indexed.ts

@@ -1,43 +0,0 @@
-import type { Class, DeepPartial } from '@travetto/runtime';
-
-import type { ModelType, OptionalId } from '../types/model.ts';
-import type { ModelBasicSupport } from './basic.ts';
-
-/**
- * Support for simple indexed activity
- *
- * @concrete
- */
-export interface ModelIndexedSupport extends ModelBasicSupport {
-  /**
-   * Get entity by index as defined by fields of idx and the body fields
-   * @param cls The type to search by
-   * @param idx The index name to search against
-   * @param body The payload of fields needed to search
-   */
-  getByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<T>;
-
-  /**
-   * Delete entity by index as defined by fields of idx and the body fields
-   * @param cls The type to search by
-   * @param idx The index name to search against
-   * @param body The payload of fields needed to search
-   */
-  deleteByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<void>;
-
-  /**
-   * List entity by ranged index as defined by fields of idx and the body fields
-   * @param cls The type to search by
-   * @param idx The index name to search against
-   * @param body The payload of fields needed to search
-   */
-  listByIndex<T extends ModelType>(cls: Class<T>, idx: string, body?: DeepPartial<T>): AsyncIterable<T>;
-
-  /**
-   * Upsert by index, allowing the index to act as a primary key
-   * @param cls The type to create for
-   * @param idx The index name to use
-   * @param body The document to potentially store
-   */
-  upsertByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: OptionalId<T>): Promise<T>;
-}

package/src/util/indexed.ts

@@ -1,144 +0,0 @@
-import { castTo, type Class, type DeepPartial, hasFunction, TypedObject } from '@travetto/runtime';
-
-import { IndexNotSupported } from '../error/invalid-index.ts';
-import { NotFoundError } from '../error/not-found.ts';
-import type { IndexConfig } from '../registry/types.ts';
-import type { ModelCrudSupport } from '../types/crud.ts';
-import type { ModelIndexedSupport } from '../types/indexed.ts';
-import type { ModelType, OptionalId } from '../types/model.ts';
-import { ModelRegistryIndex } from '../registry/registry-index.ts';
-
-type ComputeConfig = {
-  includeSortInFields?: boolean;
-  emptyValue?: unknown;
-  emptySortValue?: unknown;
-};
-
-type IndexFieldPart = { path: string[], value: (string | boolean | Date | number) };
-type IndexSortPart = { path: string[], dir: number, value: number | Date };
-
-const DEFAULT_SEP = '\u8203';
-
-/**
- * Utils for working with indexed model services
- */
-export class ModelIndexedUtil {
-
-  /**
-   * Type guard for determining if service supports indexed operation
-   */
-  static isSupported = hasFunction<ModelIndexedSupport>('getByIndex');
-
-  /**
-   * Compute flattened field to value mappings
-   * @param cls Class to get info for
-   * @param idx Index config
-   * @param item Item to read values from
-   */
-  static computeIndexParts<T extends ModelType>(
-    cls: Class<T>, idx: IndexConfig<T> | string, item: DeepPartial<T>, opts: ComputeConfig = {}
-  ): { fields: IndexFieldPart[], sorted: IndexSortPart | undefined } {
-    const config = typeof idx === 'string' ? ModelRegistryIndex.getIndex(cls, idx) : idx;
-    const sortField = config.type === 'sorted' ? config.fields.at(-1) : undefined;
-
-    const fields: IndexFieldPart[] = [];
-    let sortDirection: number = 0;
-    let sorted: IndexSortPart | undefined;
-
-    for (const field of config.fields) {
-      let fieldRef: Record<string, unknown> = field;
-      let itemRef: Record<string, unknown> = item;
-      const parts = [];
-
-      while (itemRef !== undefined && itemRef !== null) {
-        const key = TypedObject.keys(fieldRef)[0];
-        itemRef = castTo(itemRef[key]);
-        parts.push(key);
-        if (typeof fieldRef[key] === 'boolean' || typeof fieldRef[key] === 'number') {
-          if (config.type === 'sorted') {
-            sortDirection = fieldRef[key] === true ? 1 : fieldRef[key] === false ? 0 : fieldRef[key];
-          }
-          break; // At the bottom
-        } else {
-          fieldRef = castTo(fieldRef[key]);
-        }
-      }
-      if (field === sortField) {
-        sorted = { path: parts, dir: sortDirection, value: castTo(itemRef) };
-      }
-      if (itemRef === undefined || itemRef === null) {
-        const empty = field === sortField ? opts.emptySortValue : opts.emptyValue;
-        if (empty === undefined || empty === Error) {
-          throw new IndexNotSupported(cls, config, `Missing field value for ${parts.join('.')}`);
-        }
-      } else {
-        if (field !== sortField || (opts.includeSortInFields ?? true)) {
-          fields.push({ path: parts, value: castTo(itemRef) });
-        }
-      }
-    }
-
-    return { fields, sorted };
-  }
-
-  /**
-   * Project item via index
-   * @param cls Type to get index for
-   * @param idx Index config
-   */
-  static projectIndex<T extends ModelType>(cls: Class<T>, idx: IndexConfig<T> | string, item?: DeepPartial<T>, config?: ComputeConfig): Record<string, unknown> {
-    const response: Record<string, unknown> = {};
-    for (const { path, value } of this.computeIndexParts(cls, idx, item ?? {}, config).fields) {
-      let sub: Record<string, unknown> = response;
-      const all = path.slice(0);
-      const last = all.pop()!;
-      for (const part of all) {
-        sub = castTo(sub[part] ??= {});
-      }
-      sub[last] = value;
-    }
-    return response;
-  }
-
-  /**
-   * Compute index key as a single value
-   * @param cls Class to get index for
-   * @param idx Index config
-   * @param item item to process
-   */
-  static computeIndexKey<T extends ModelType>(
-    cls: Class<T>,
-    idx: IndexConfig<T> | string,
-    item: DeepPartial<T> = {},
-    config?: ComputeConfig & { separator?: string }
-  ): { type: string, key: string, sort?: number | Date } {
-    const { fields, sorted } = this.computeIndexParts(cls, idx, item, { ...(config ?? {}), includeSortInFields: false });
-    const key = fields.map(({ value }) => value).map(value => `${value}`).join(config?.separator ?? DEFAULT_SEP);
-    const indexConfig = typeof idx === 'string' ? ModelRegistryIndex.getIndex(cls, idx) : idx;
-    return !sorted ? { type: indexConfig.type, key } : { type: indexConfig.type, key, sort: sorted.value };
-  }
-
-  /**
-   * Naive upsert by index
-   * @param service
-   * @param cls
-   * @param idx
-   * @param body
-   */
-  static async naiveUpsert<T extends ModelType>(
-    service: ModelIndexedSupport & ModelCrudSupport,
-    cls: Class<T>, idx: string, body: OptionalId<T>
-  ): Promise<T> {
-    try {
-      const { id } = await service.getByIndex(cls, idx, castTo(body));
-      body.id = id;
-      return await service.update(cls, castTo(body));
-    } catch (error) {
-      if (error instanceof NotFoundError) {
-        return await service.create(cls, body);
-      } else {
-        throw error;
-      }
-    }
-  }
-}

package/support/test/indexed.ts

@@ -1,190 +0,0 @@
-import assert from 'node:assert';
-
-import { Suite, Test } from '@travetto/test';
-import { Schema } from '@travetto/schema';
-import { TimeUtil } from '@travetto/runtime';
-
-import { Index, Model } from '../../src/registry/decorator.ts';
-import type { ModelIndexedSupport } from '../../src/types/indexed.ts';
-import { NotFoundError } from '../../src/error/not-found.ts';
-import { IndexNotSupported } from '../../src/error/invalid-index.ts';
-
-import { BaseModelSuite } from './base.ts';
-
-@Model('index_user')
-@Index({
-  name: 'userName',
-  type: 'unsorted',
-  fields: [{ name: 1 }]
-})
-class User {
-  id: string;
-  name: string;
-}
-
-@Model('index_user_2')
-class User2 {
-  id: string;
-  name: string;
-}
-
-@Model()
-@Index({ type: 'sorted', name: 'userAge', fields: [{ name: 1 }, { age: 1 }] })
-class User3 {
-  id: string;
-  name: string;
-  age: number;
-  color?: string;
-}
-
-@Schema()
-class Child {
-  name: string;
-  age: number;
-}
-
-@Model()
-@Index({ type: 'sorted', name: 'childAge', fields: [{ child: { name: 1 } }, { child: { age: 1 } }] })
-@Index({ type: 'sorted', name: 'nameCreated', fields: [{ child: { name: 1 } }, { createdDate: 1 }] })
-class User4 {
-  id: string;
-  createdDate?: Date = new Date();
-  color: string;
-  child: Child;
-}
-
-@Suite()
-export abstract class ModelIndexedSuite extends BaseModelSuite<ModelIndexedSupport> {
-  @Test()
-  async writeAndRead() {
-    const service = await this.service;
-
-    await service.create(User, User.from({ name: 'bob1' }));
-    await service.create(User, User.from({ name: 'bob2' }));
-
-    const found1 = await service.getByIndex(User, 'userName', {
-      name: 'bob1'
-    });
-
-    assert(found1.name === 'bob1');
-
-    const found2 = await service.getByIndex(User, 'userName', {
-      name: 'bob2'
-    });
-
-    assert(found2.name === 'bob2');
-  }
-
-  @Test()
-  async readMissingIndex() {
-    const service = await this.service;
-    await assert.rejects(() => service.getByIndex(User, 'missing', {}), NotFoundError);
-  }
-
-  @Test()
-  async readMissingValue() {
-    const service = await this.service;
-    await assert.rejects(() => service.getByIndex(User, 'userName', { name: 'jim' }), NotFoundError);
-  }
-
-  @Test()
-  async readDifferentType() {
-    const service = await this.service;
-    await assert.rejects(() => service.getByIndex(User2, 'userName', { name: 'jim' }), NotFoundError);
-  }
-
-  @Test()
-  async queryMultiple() {
-    const service = await this.service;
-
-    await service.create(User3, User3.from({ name: 'bob', age: 20 }));
-    await service.create(User3, User3.from({ name: 'bob', age: 30, color: 'green' }));
-
-    const found = await service.getByIndex(User3, 'userAge', { name: 'bob', age: 30 });
-
-    assert(found.color === 'green');
-
-    const found2 = await service.getByIndex(User3, 'userAge', { name: 'bob', age: 20 });
-
-    assert(!found2.color);
-
-    await assert.rejects(() => service.getByIndex(User3, 'userAge', { name: 'bob' }));
-  }
-
-  @Test()
-  async queryList() {
-    const service = await this.service;
-
-    await service.create(User3, User3.from({ name: 'bob', age: 40, color: 'blue' }));
-    await service.create(User3, User3.from({ name: 'bob', age: 30, color: 'red' }));
-    await service.create(User3, User3.from({ name: 'bob', age: 50, color: 'green' }));
-
-    const arr = await this.toArray(service.listByIndex(User3, 'userAge', { name: 'bob' }));
-
-    assert(arr[0].color === 'red' && arr[0].name === 'bob');
-    assert(arr[1].color === 'blue' && arr[1].name === 'bob');
-    assert(arr[2].color === 'green' && arr[2].name === 'bob');
-
-    await assert.rejects(() => this.toArray(service.listByIndex(User3, 'userAge', {})), IndexNotSupported);
-  }
-
-  @Test()
-  async queryDeepList() {
-    const service = await this.service;
-
-    await service.create(User4, User4.from({ child: { name: 'bob', age: 40 }, color: 'blue' }));
-    await service.create(User4, User4.from({ child: { name: 'bob', age: 30 }, color: 'red' }));
-    await service.create(User4, User4.from({ child: { name: 'bob', age: 50 }, color: 'green' }));
-
-    const arr = await this.toArray(service.listByIndex(User4, 'childAge', User4.from({ child: { name: 'bob' } })));
-    assert(arr[0].color === 'red' && arr[0].child.name === 'bob' && arr[0].child.age === 30);
-    assert(arr[1].color === 'blue' && arr[1].child.name === 'bob' && arr[1].child.age === 40);
-    assert(arr[2].color === 'green' && arr[2].child.name === 'bob' && arr[2].child.age === 50);
-
-    await assert.rejects(() => this.toArray(service.listByIndex(User4, 'childAge', {})), IndexNotSupported);
-  }
-
-  @Test()
-  async queryComplexDateList() {
-    const service = await this.service;
-
-    await service.create(User4, User4.from({ child: { name: 'bob', age: 40 }, createdDate: TimeUtil.fromNow('3d'), color: 'blue' }));
-    await service.create(User4, User4.from({ child: { name: 'bob', age: 30 }, createdDate: TimeUtil.fromNow('2d'), color: 'red' }));
-    await service.create(User4, User4.from({ child: { name: 'bob', age: 50 }, createdDate: TimeUtil.fromNow('-1d'), color: 'green' }));
-
-    const arr = await this.toArray(service.listByIndex(User4, 'nameCreated', { child: { name: 'bob' } }));
-
-    assert(arr[0].color === 'green' && arr[0].child.name === 'bob' && arr[0].child.age === 50);
-    assert(arr[1].color === 'red' && arr[1].child.name === 'bob' && arr[1].child.age === 30);
-    assert(arr[2].color === 'blue' && arr[2].child.name === 'bob' && arr[2].child.age === 40);
-
-    await assert.rejects(() => this.toArray(service.listByIndex(User4, 'nameCreated', {})), IndexNotSupported);
-  }
-
-  @Test()
-  async upsertByIndex() {
-    const service = await this.service;
-
-    const user1 = await service.upsertByIndex(User4, 'childAge', { child: { name: 'bob', age: 40 }, color: 'blue' });
-    const user2 = await service.upsertByIndex(User4, 'childAge', { child: { name: 'bob', age: 40 }, color: 'green' });
-    const user3 = await service.upsertByIndex(User4, 'childAge', { child: { name: 'bob', age: 40 }, color: 'red' });
-
-    const arr = await this.toArray(service.listByIndex(User4, 'childAge', { child: { name: 'bob' } }));
-    assert(arr.length === 1);
-
-    assert(user1.id === user2.id);
-    assert(user2.id === user3.id);
-    assert(user1.color === 'blue');
-    assert(user3.color === 'red');
-
-    const user4 = await service.upsertByIndex(User4, 'childAge', { child: { name: 'bob', age: 30 }, color: 'red' });
-    const arr2 = await this.toArray(service.listByIndex(User4, 'childAge', { child: { name: 'bob' } }));
-    assert(arr2.length === 2);
-
-    await service.deleteByIndex(User4, 'childAge', user1);
-
-    const arr3 = await this.toArray(service.listByIndex(User4, 'childAge', { child: { name: 'bob' } }));
-    assert(arr3.length === 1);
-    assert(arr3[0].id === user4.id);
-  }
-}