@travetto/model-mongo 8.0.0-alpha.3 → 8.0.0-alpha.31

package/README.md

@@ -13,25 +13,25 @@ npm install @travetto/model-mongo
 yarn add @travetto/model-mongo
 ```
 
-This module provides an [mongodb](https://mongodb.com)-based implementation for the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.").  This source allows the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.") module to read, write and query against [mongodb](https://mongodb.com).. Given the dynamic nature of [mongodb](https://mongodb.com), during development when models are modified, nothing needs to be done to adapt to the latest schema. 
+This module provides an [mongodb](https://mongodb.com)-based implementation for the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations."). This source allows the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.") module to read, write and query against [mongodb](https://mongodb.com).. Given the dynamic nature of [mongodb](https://mongodb.com), during development when models are modified, nothing needs to be done to adapt to the latest schema. 
 
 Supported features:
-   *  [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/types/crud.ts#L11)
-   *  [Expiry](https://github.com/travetto/travetto/tree/main/module/model/src/types/expiry.ts#L10)
-   *  [Bulk](https://github.com/travetto/travetto/tree/main/module/model/src/types/bulk.ts#L64)
-   *  [Indexed](https://github.com/travetto/travetto/tree/main/module/model/src/types/indexed.ts#L11)
    *  [Blob](https://github.com/travetto/travetto/tree/main/module/model/src/types/blob.ts#L8)
+   *  [Bulk](https://github.com/travetto/travetto/tree/main/module/model/src/types/bulk.ts#L60)
+   *  [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/types/crud.ts#L10)
+   *  [Expiry](https://github.com/travetto/travetto/tree/main/module/model/src/types/expiry.ts#L10)
+   *  [Indexed](https://github.com/travetto/travetto/tree/main/module/model-indexed/src/types/service.ts#L21)
    *  [Query Crud](https://github.com/travetto/travetto/tree/main/module/model-query/src/types/crud.ts#L11)
    *  [Facet](https://github.com/travetto/travetto/tree/main/module/model-query/src/types/facet.ts#L14)
-   *  [Query](https://github.com/travetto/travetto/tree/main/module/model-query/src/types/query.ts#L10)
    *  [Suggest](https://github.com/travetto/travetto/tree/main/module/model-query/src/types/suggest.ts#L12)
+   *  [Query](https://github.com/travetto/travetto/tree/main/module/model-query/src/types/query.ts#L10)
 
 Out of the box, by installing the module, everything should be wired up by default.If you need to customize any aspect of the source or config, you can override and register it with the [Dependency Injection](https://github.com/travetto/travetto/tree/main/module/di#readme "Dependency registration/management and injection support.") module.
 
 **Code: Wiring up a custom Model Source**
 ```typescript
 import { InjectableFactory } from '@travetto/di';
-import { MongoModelService, type MongoModelConfig } from '@travetto/model-mongo';
+import { type MongoModelConfig, MongoModelService } from '@travetto/model-mongo';
 
 export class Init {
   @InjectableFactory({
@@ -82,7 +82,7 @@ export class MongoModelConfig {
    */
   @Field({ type: Object })
   options: Omit<mongo.MongoClientOptions, 'cert'> & {
-    cert?: | Buffer | string | BinaryType | (BinaryType | Buffer | string)[];
+    cert?: Buffer | string | BinaryType | (BinaryType | Buffer | string)[];
   } = {};
   /**
    * Allow storage modification at runtime
@@ -124,15 +124,14 @@ export class MongoModelConfig {
     if (!this.port || Number.isNaN(this.port)) {
       this.port = 27017;
     }
-    if (!this.hosts || !this.hosts.length) {
+    if (!this.hosts?.length) {
       this.hosts = ['localhost'];
     }
 
     const options = this.options;
     if (options.ssl) {
       if (options.cert) {
-        options.cert = (await Promise.all([options.cert].flat(2).map(readCert)))
-          .map(BinaryUtil.binaryArrayToUint8Array);
+        options.cert = (await Promise.all([options.cert].flat(2).map(readCert))).map(BinaryUtil.binaryArrayToUint8Array);
       }
       if (options.tlsCertificateKeyFile) {
         options.tlsCertificateKeyFile = await RuntimeResources.resolve(options.tlsCertificateKeyFile);
@@ -155,23 +154,20 @@ export class MongoModelConfig {
    * Build connection URLs
    */
   get url(): string {
-    const hosts = this.hosts!
-      .map(host => (this.srvRecord || host.includes(':')) ? host : `${host}:${this.port ?? 27017}`)
-      .join(',');
+    const hosts = this.hosts!.map(host => (this.srvRecord || host.includes(':') ? host : `${host}:${this.port ?? 27017}`)).join(',');
     const optionString = new URLSearchParams(
       Object.entries(this.options)
         .filter((pair): pair is [string, string | number | boolean] => ['string', 'number', 'boolean'].includes(typeof pair[1]))
         .map(([k, v]) => [k, `${v}`])
-    )
-      .toString();
-    let creds = '';
+    ).toString();
+    let credentials = '';
     if (this.username) {
-      creds = `${[this.username, this.password].filter(part => !!part).join(':')}@`;
+      credentials = `${[this.username, this.password].filter(part => !!part).join(':')}@`;
     }
-    const url = `mongodb${this.srvRecord ? '+srv' : ''}://${creds}${hosts}/${this.namespace}?${optionString}`;
+    const url = `mongodb${this.srvRecord ? '+srv' : ''}://${credentials}${hosts}/${this.namespace}?${optionString}`;
     return url;
   }
 }
 ```
 
-Additionally, you can see that the class is registered with the [@Config](https://github.com/travetto/travetto/tree/main/module/config/src/decorator.ts#L13) annotation, and so these values can be overridden using the standard [Configuration](https://github.com/travetto/travetto/tree/main/module/config#readme "Configuration support") resolution paths.The SSL file options in `clientOptions` will automatically be resolved to files when given a path.  This path can be a resource path (will attempt to lookup using [RuntimeResources](https://github.com/travetto/travetto/tree/main/module/runtime/src/resources.ts#L8)) or just a standard file path.
+Additionally, you can see that the class is registered with the [@Config](https://github.com/travetto/travetto/tree/main/module/config/src/decorator.ts#L13) annotation, and so these values can be overridden using the standard [Configuration](https://github.com/travetto/travetto/tree/main/module/config#readme "Configuration support") resolution paths.The SSL file options in `clientOptions` will automatically be resolved to files when given a path. This path can be a resource path (will attempt to lookup using [RuntimeResources](https://github.com/travetto/travetto/tree/main/module/runtime/src/resources.ts#L8)) or just a standard file path.

package/__index__.ts

@@ -1,2 +1,2 @@
+export * from './src/config.ts';
 export * from './src/service.ts';
-export * from './src/config.ts';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model-mongo",
-  "version": "8.0.0-alpha.3",
+  "version": "8.0.0-alpha.31",
   "type": "module",
   "description": "Mongo backing for the travetto model module.",
   "keywords": [
@@ -26,13 +26,14 @@
     "directory": "module/model-mongo"
   },
   "dependencies": {
-    "@travetto/config": "^8.0.0-alpha.3",
-    "@travetto/model": "^8.0.0-alpha.3",
-    "@travetto/model-query": "^8.0.0-alpha.3",
-    "mongodb": "^7.1.0"
+    "@travetto/config": "^8.0.0-alpha.26",
+    "@travetto/model": "^8.0.0-alpha.27",
+    "@travetto/model-indexed": "^8.0.0-alpha.29",
+    "@travetto/model-query": "^8.0.0-alpha.30",
+    "mongodb": "^7.5.0"
   },
   "peerDependencies": {
-    "@travetto/cli": "^8.0.0-alpha.4"
+    "@travetto/cli": "^8.0.0-alpha.32"
   },
   "peerDependenciesMeta": {
     "@travetto/cli": {

package/src/config.ts

@@ -1,9 +1,9 @@
 import type mongo from 'mongodb';
 
-import { type TimeSpan, Runtime, RuntimeResources, BinaryUtil, CodecUtil, type BinaryType, type BinaryArray } from '@travetto/runtime';
 import { Config } from '@travetto/config';
-import { Field } from '@travetto/schema';
 import { PostConstruct } from '@travetto/di';
+import { type BinaryArray, type BinaryType, BinaryUtil, CodecUtil, Runtime, RuntimeResources, type TimeSpan } from '@travetto/runtime';
+import { Field } from '@travetto/schema';
 
 const readCert = async (input: BinaryType | string): Promise<BinaryArray> => {
   if (BinaryUtil.isBinaryType(input)) {
@@ -55,7 +55,7 @@ export class MongoModelConfig {
    */
   @Field({ type: Object })
   options: Omit<mongo.MongoClientOptions, 'cert'> & {
-    cert?: | Buffer | string | BinaryType | (BinaryType | Buffer | string)[];
+    cert?: Buffer | string | BinaryType | (BinaryType | Buffer | string)[];
   } = {};
   /**
    * Allow storage modification at runtime
@@ -97,15 +97,14 @@ export class MongoModelConfig {
     if (!this.port || Number.isNaN(this.port)) {
       this.port = 27017;
     }
-    if (!this.hosts || !this.hosts.length) {
+    if (!this.hosts?.length) {
       this.hosts = ['localhost'];
     }
 
     const options = this.options;
     if (options.ssl) {
       if (options.cert) {
-        options.cert = (await Promise.all([options.cert].flat(2).map(readCert)))
-          .map(BinaryUtil.binaryArrayToUint8Array);
+        options.cert = (await Promise.all([options.cert].flat(2).map(readCert))).map(BinaryUtil.binaryArrayToUint8Array);
       }
       if (options.tlsCertificateKeyFile) {
         options.tlsCertificateKeyFile = await RuntimeResources.resolve(options.tlsCertificateKeyFile);
@@ -128,20 +127,17 @@ export class MongoModelConfig {
    * Build connection URLs
    */
   get url(): string {
-    const hosts = this.hosts!
-      .map(host => (this.srvRecord || host.includes(':')) ? host : `${host}:${this.port ?? 27017}`)
-      .join(',');
+    const hosts = this.hosts!.map(host => (this.srvRecord || host.includes(':') ? host : `${host}:${this.port ?? 27017}`)).join(',');
     const optionString = new URLSearchParams(
       Object.entries(this.options)
         .filter((pair): pair is [string, string | number | boolean] => ['string', 'number', 'boolean'].includes(typeof pair[1]))
         .map(([k, v]) => [k, `${v}`])
-    )
-      .toString();
-    let creds = '';
+    ).toString();
+    let credentials = '';
     if (this.username) {
-      creds = `${[this.username, this.password].filter(part => !!part).join(':')}@`;
+      credentials = `${[this.username, this.password].filter(part => !!part).join(':')}@`;
     }
-    const url = `mongodb${this.srvRecord ? '+srv' : ''}://${creds}${hosts}/${this.namespace}?${optionString}`;
+    const url = `mongodb${this.srvRecord ? '+srv' : ''}://${credentials}${hosts}/${this.namespace}?${optionString}`;
     return url;
   }
-}
+}

package/src/internal/util.ts

@@ -1,17 +1,22 @@
 import {
-  Binary, type CreateIndexesOptions, type Filter, type FindCursor, type IndexDirection, ObjectId, type WithId as MongoWithId,
-  type IndexDescriptionInfo
+  Binary,
+  type CreateIndexesOptions,
+  type Filter,
+  type FindCursor,
+  type IndexDescriptionInfo,
+  type IndexDirection,
+  type WithId as MongoWithId,
+  ObjectId
 } from 'mongodb';
 
-import { RuntimeError, CodecUtil, castTo, type Class, toConcrete, TypedObject, BinaryUtil } from '@travetto/runtime';
-import { type DistanceUnit, type PageableModelQuery, type WhereClause, ModelQueryUtil } from '@travetto/model-query';
-import type { ModelType, IndexField, IndexConfig } from '@travetto/model';
-import { DataUtil, SchemaRegistryIndex, type Point } from '@travetto/schema';
+import { type IndexConfig, IndexNotSupported, type ModelType } from '@travetto/model';
+import { isModelIndexedIndex } from '@travetto/model-indexed';
+import { type DistanceUnit, isModelQueryIndex, ModelQueryUtil, type PageableModelQuery, type WhereClause } from '@travetto/model-query';
+import { BinaryUtil, type Class, CodecUtil, castTo, RuntimeError, toConcrete } from '@travetto/runtime';
+import { DataUtil, type Point, SchemaRegistryIndex } from '@travetto/schema';
 
 const PointConcrete = toConcrete<Point>();
 
-type IdxConfig = CreateIndexesOptions;
-
 /**
  * Converting units to various radians
  */
@@ -25,38 +30,31 @@ const RADIANS_TO: Record<DistanceUnit, number> = {
 
 export type WithId<T, I = unknown> = T & { _id?: I };
 export type BasicIdx = Record<string, IndexDirection>;
-export type PlainIdx = Record<string, -1 | 0 | 1>;
+
+function flattenKeys(obj: Record<string, unknown>, prefix = ''): Record<string, 1 | -1> {
+  const out: Record<string, 1 | -1> = {};
+  for (const [key, value] of Object.entries(obj)) {
+    const path = prefix ? `${prefix}.${key}` : key;
+    if (typeof value === 'object' && value !== null) {
+      Object.assign(out, flattenKeys(castTo(value), path));
+    } else {
+      out[path] = typeof value === 'boolean' ? (value ? 1 : -1) : castTo<-1 | 1>(value);
+    }
+  }
+  return out;
+}
 
 /**
  * Basic mongo utils for conforming to the model module
  */
 export class MongoUtil {
-
   static namespaceIndex(cls: Class, name: string): string {
     return `${cls.Ⲑid}__${name}`.replace(/[^a-zA-Z0-9_]+/g, '_');
   }
 
-  static toIndex<T extends ModelType>(field: IndexField<T>): PlainIdx {
-    const keys = [];
-    while (typeof field !== 'number' && typeof field !== 'boolean' && Object.keys(field)) {
-      const key = TypedObject.keys(field)[0];
-      field = castTo(field[key]);
-      keys.push(key);
-    }
-    const rf: number | boolean = castTo(field);
-    return {
-      [keys.join('.')]: typeof rf === 'boolean' ? (rf ? 1 : 0) : castTo<-1 | 1 | 0>(rf)
-    };
-  }
-
   static uuid(value: string): Binary {
     try {
-      return new Binary(
-        BinaryUtil.binaryArrayToUint8Array(
-          CodecUtil.fromHexString(value.replaceAll('-', ''))
-        ),
-        Binary.SUBTYPE_UUID
-      );
+      return new Binary(BinaryUtil.binaryArrayToUint8Array(CodecUtil.fromHexString(value.replaceAll('-', ''))), Binary.SUBTYPE_UUID);
     } catch (err) {
       if (err instanceof RuntimeError && err.message === 'Invalid hex string') {
         return null!;
@@ -96,7 +94,12 @@ export class MongoUtil {
   }
 
   /**/
-  static extractSimple<T>(base: Class<T> | undefined, item: Record<string, unknown>, path: string = '', recursive: boolean = true): Record<string, unknown> {
+  static extractSimple<T>(
+    base: Class<T> | undefined,
+    item: Record<string, unknown>,
+    path: string = '',
+    recursive: boolean = true
+  ): Record<string, unknown> {
     const fields = base ? SchemaRegistryIndex.getOptional(base)?.getFields() : undefined;
     const out: Record<string, unknown> = {};
     const sub = item;
@@ -142,7 +145,7 @@ export class MongoUtil {
           value.$regex = DataUtil.toRegex(castTo(value.$regex));
         } else if (firstKey && '$near' in value) {
           const dist: number = castTo(value.$maxDistance);
-          const distance = dist / RADIANS_TO[(castTo<DistanceUnit>(value.$unit) ?? 'km')];
+          const distance = dist / RADIANS_TO[castTo<DistanceUnit>(value.$unit) ?? 'km'];
           value.$maxDistance = distance;
           delete value.$unit;
         } else if (firstKey && '$geoWithin' in value) {
@@ -166,8 +169,8 @@ export class MongoUtil {
     return out;
   }
 
-  static getExtraIndices<T extends ModelType>(cls: Class<T>): [BasicIdx, IdxConfig][] {
-    const out: [BasicIdx, IdxConfig][] = [];
+  static getExtraIndices<T extends ModelType>(cls: Class<T>): [BasicIdx, CreateIndexesOptions][] {
+    const out: [BasicIdx, CreateIndexesOptions][] = [];
     const textFields: string[] = [];
     SchemaRegistryIndex.visitFields(cls, (field, path) => {
       if (field.type === PointConcrete) {
@@ -185,30 +188,44 @@ export class MongoUtil {
     return out;
   }
 
-  static getPlainIndex(idx: IndexConfig<ModelType>): PlainIdx {
-    let out: PlainIdx = {};
-    for (const config of idx.fields.map(value => this.toIndex(value))) {
-      out = Object.assign(out, config);
-    }
-    return out;
-  }
+  static getIndex(cls: Class, idx: IndexConfig): [BasicIdx, CreateIndexesOptions] {
+    const name = this.namespaceIndex(cls, idx.name);
+    if (isModelQueryIndex(idx)) {
+      const out = idx.fields.reduce(
+        (acc, field) => Object.assign(acc, { ...flattenKeys(castTo(field)) }),
+        castTo<Record<string, -1 | 0 | 1>>({})
+      );
 
-  static getIndices<T extends ModelType>(cls: Class<T>, indices: IndexConfig<ModelType>[] = []): [BasicIdx, IdxConfig][] {
-    return [
-      ...indices.map(idx => [this.getPlainIndex(idx), { ...(idx.type === 'unique' ? { unique: true } : {}), name: this.namespaceIndex(cls, idx.name) }] as const),
-      ...this.getExtraIndices(cls)
-    ].map(idx => [...idx]);
+      return [out, { name, unique: !!idx.unique }];
+    } else if (isModelIndexedIndex(idx)) {
+      const filter = Object.fromEntries([
+        ...idx.keyTemplate.map(({ path }) => [path.join('.'), 1]),
+        ...idx.sortTemplate.map(({ path, value }) => [path.join('.'), value === -1 ? -1 : 1])
+      ]);
+      switch (idx.type) {
+        case 'indexed:keyed':
+          return [filter, { name, unique: idx.unique }];
+        case 'indexed:sorted':
+          return [filter, { name }];
+      }
+    } else {
+      throw new IndexNotSupported(cls, idx);
+    }
   }
 
-  static prepareCursor<T extends ModelType>(cls: Class<T>, cursor: FindCursor<T | MongoWithId<T>>, query: PageableModelQuery<T>): FindCursor<T> {
+  static prepareCursor<T extends ModelType>(
+    cls: Class<T>,
+    cursor: FindCursor<T | MongoWithId<T>>,
+    query: PageableModelQuery<T>
+  ): FindCursor<T> {
     if (query.select) {
       const selectKey = Object.keys(query.select)[0];
       const select = typeof selectKey === 'string' && selectKey.startsWith('$') ? query.select : this.extractSimple(cls, query.select);
       // Remove id if not explicitly defined, and selecting fields directly
-      if (!select['_id']) {
+      if (!select._id) {
         const values = new Set([...Object.values(select)]);
         if (values.has(1) || values.has(true)) {
-          select['_id'] = false;
+          select._id = false;
         }
       }
       cursor.project(select);
@@ -236,9 +253,9 @@ export class MongoUtil {
       existing.expireAfterSeconds !== pendingOptions.expireAfterSeconds ||
       existing.bucketSize !== pendingOptions.bucketSize;
 
-    const existingFields = existing.textIndexVersion ?
-      Object.fromEntries(Object.entries(existing.weights ?? {}).map(([key]) => [key, 'text'])) :
-      existing.key;
+    const existingFields = existing.textIndexVersion
+      ? Object.fromEntries(Object.entries(existing.weights ?? {}).map(([key]) => [key, 'text']))
+      : existing.key;
 
     const pendingKeySet = new Set(Object.keys(pendingKey));
     const existingKeySet = new Set(Object.keys(existingFields));
@@ -248,10 +265,10 @@ export class MongoUtil {
     const overlap = [...pendingKeySet.intersection(existingKeySet)];
     changed ||= overlap.length !== pendingKeySet.size;
 
-    for (let i = 0; i < overlap.length && !changed; i++) {
+    for (let i = 0; i < overlap.length && !changed; i += 1) {
       changed ||= existingFields[overlap[i]] !== pendingKey[overlap[i]];
     }
 
     return changed;
   }
-}
+}

package/src/service.ts

@@ -1,60 +1,183 @@
 import {
   type Binary,
-  type Db, GridFSBucket, MongoClient, type GridFSFile, type Collection,
-  type ObjectId, type RootFilterOperators, type Filter,
+  type Collection,
+  type Db,
+  type Filter,
+  type FindCursor,
+  GridFSBucket,
+  type GridFSFile,
+  MongoClient,
+  MongoServerError,
   type WithId as MongoWithId,
+  type ObjectId,
+  type RootFilterOperators
 } from 'mongodb';
 
+import { Injectable, PostConstruct } from '@travetto/di';
 import {
-  ModelRegistryIndex, type ModelType, type OptionalId, type ModelCrudSupport, type ModelStorageSupport,
-  type ModelExpirySupport, type ModelBulkSupport, type ModelIndexedSupport, type BulkOperation, type BulkResponse,
-  NotFoundError, ExistsError, type ModelBlobSupport,
-  ModelCrudUtil, ModelIndexedUtil, ModelStorageUtil, ModelExpiryUtil, ModelBulkUtil
+  type BulkOperation,
+  type BulkResponse,
+  ExistsError,
+  type ModelBlobSupport,
+  type ModelBulkSupport,
+  ModelBulkUtil,
+  type ModelCrudSupport,
+  ModelCrudUtil,
+  type ModelExpirySupport,
+  ModelExpiryUtil,
+  type ModelListOptions,
+  ModelRegistryIndex,
+  type ModelStorageSupport,
+  ModelStorageUtil,
+  type ModelType,
+  NotFoundError,
+  type OptionalId,
+  UniqueError
 } from '@travetto/model';
 import {
-  type ModelQuery, type ModelQueryCrudSupport, type ModelQueryFacetSupport, type ModelQuerySupport,
-  type PageableModelQuery, type ValidStringFields, type WhereClause, type ModelQuerySuggestSupport,
-  QueryVerifier, ModelQueryUtil, ModelQuerySuggestUtil, ModelQueryCrudUtil,
+  type FullKeyedIndexBody,
+  type FullKeyedIndexWithPartialBody,
+  type KeyedIndexBody,
+  type KeyedIndexSelection,
+  ModelIndexedComputedIndex,
+  type ModelIndexedSearchOptions,
+  type ModelIndexedSupport,
+  ModelIndexedUtil,
+  type ModelPageOptions,
+  type ModelPageResult,
+  type SingleItemIndex,
+  type SortedIndex,
+  type SortedIndexSelection,
+  type SortedIndexSelectionType
+} from '@travetto/model-indexed';
+import {
+  type ModelQuery,
+  type ModelQueryCrudSupport,
+  ModelQueryCrudUtil,
   type ModelQueryFacet,
+  type ModelQueryFacetSupport,
+  type ModelQuerySuggestSupport,
+  ModelQuerySuggestUtil,
+  type ModelQuerySupport,
+  ModelQueryUtil,
+  type PageableModelQuery,
+  QueryVerifier,
+  type ValidStringFields,
+  type WhereClause
 } from '@travetto/model-query';
-
 import {
-  ShutdownManager, type Class, type DeepPartial, TypedObject,
-  castTo, asFull, type BinaryMetadata, type ByteRange, type BinaryType, BinaryUtil, BinaryMetadataUtil,
+  asFull,
+  type BinaryMetadata,
+  BinaryMetadataUtil,
+  type BinaryType,
+  BinaryUtil,
+  type ByteRange,
+  type Class,
+  castTo,
+  JSONUtil,
+  ShutdownManager,
+  TypedObject
 } from '@travetto/runtime';
-import { Injectable, PostConstruct } from '@travetto/di';
 
-import { MongoUtil, type PlainIdx, type WithId } from './internal/util.ts';
 import type { MongoModelConfig } from './config.ts';
-
-const ListIndexSymbol = Symbol();
+import { MongoUtil, type WithId } from './internal/util.ts';
 
 type BlobRaw = GridFSFile & { metadata?: BinaryMetadata };
 
 type MongoTextSearch = RootFilterOperators<unknown>['$text'];
 
+const handleDuplicateKeyError = (cls: Class, id: string, error: unknown): unknown => {
+  if (error instanceof MongoServerError && error.message.includes('duplicate key error')) {
+    if (error.message.includes('_id_') || (error.keyPattern && '_id' in error.keyPattern)) {
+      return new ExistsError(cls, id);
+    }
+    const match = error.message.match(/index: ([\w$]+)/);
+    const constraint = match ? match[1] : error.keyPattern ? Object.keys(error.keyPattern).join(',') : 'unique';
+    return new UniqueError(cls, constraint, { detail: error.message });
+  }
+  return error;
+};
+
 export const ModelBlobNamespace = '__blobs';
 
 /**
  * Mongo-based model source
  */
 @Injectable()
-export class MongoModelService implements
-  ModelCrudSupport, ModelStorageSupport,
-  ModelBulkSupport, ModelBlobSupport,
-  ModelIndexedSupport, ModelQuerySupport,
-  ModelQueryCrudSupport, ModelQueryFacetSupport,
-  ModelQuerySuggestSupport, ModelExpirySupport {
-
+export class MongoModelService
+  implements
+    ModelCrudSupport,
+    ModelStorageSupport,
+    ModelBulkSupport,
+    ModelBlobSupport,
+    ModelIndexedSupport,
+    ModelQuerySupport,
+    ModelQueryCrudSupport,
+    ModelQueryFacetSupport,
+    ModelQuerySuggestSupport,
+    ModelExpirySupport
+{
   #db: Db;
   #bucket: GridFSBucket;
   idSource = ModelCrudUtil.uuidSource();
   client: MongoClient;
   config: MongoModelConfig;
 
-  constructor(config: MongoModelConfig) { this.config = config; }
+  constructor(config: MongoModelConfig) {
+    this.config = config;
+  }
+
+  async *#iterateCursor<T extends ModelType>(
+    cls: Class<T>,
+    cursor: FindCursor,
+    options?: ModelListOptions & ModelPageOptions<number>
+  ): AsyncGenerator<T[]> {
+    const batchSize = options?.batchSizeHint ?? 100;
+    let batch: T[] = [];
+    const maxCount = options?.limit ?? Number.MAX_SAFE_INTEGER;
+    for await (const item of cursor
+      .batchSize(batchSize)
+      .limit(maxCount)
+      .skip(options?.offset ?? 0)) {
+      if (options?.abort?.aborted) {
+        cursor.close();
+        break;
+      }
+      batch.push(item);
+      if (batch.length >= batchSize) {
+        yield await ModelCrudUtil.filterOutNotFound(batch.map(i => this.postLoad(cls, i)));
+        batch = [];
+      }
+    }
+    if (batch.length) {
+      yield await ModelCrudUtil.filterOutNotFound(batch.map(i => this.postLoad(cls, i)));
+    }
+  }
 
-  restoreId(item: { id?: string, _id?: unknown }): void {
+  async #buildIndexQuery<T extends ModelType>(
+    cls: Class<T>,
+    idx: SortedIndex<T>,
+    body: KeyedIndexBody<T>,
+    transformWhere?: (where: WhereClause<T>) => WhereClause<T>
+  ): Promise<FindCursor> {
+    const store = await this.getStore(cls);
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate();
+    let whereClause: WhereClause<T> = castTo(computed.project());
+    if (transformWhere) {
+      whereClause = transformWhere(whereClause);
+    }
+
+    const where = this.getWhereFilter(cls, whereClause);
+    let q = store.find(where, { timeout: true }).batchSize(100);
+
+    // TODO: We could cache this
+    if ('sort' in idx) {
+      q = q.sort(idx.sortTemplate.map(({ path, value }) => [path.join('.'), value] as const));
+    }
+    return q;
+  }
+
+  restoreId(item: { id?: string; _id?: unknown }): void {
     if (item._id) {
       item.id ??= MongoUtil.idToString(castTo(item._id));
       delete item._id;
@@ -74,11 +197,11 @@ export class MongoModelService implements
     return item;
   }
 
-  preUpdate<T extends OptionalId<ModelType>>(item: T & { _id?: Binary, id: string }): string;
+  preUpdate<T extends OptionalId<ModelType>>(item: T & { _id?: Binary; id: string }): string;
   preUpdate<T extends OptionalId<ModelType>>(item: Omit<T, 'id'> & { _id?: Binary }): undefined;
-  preUpdate<T extends OptionalId<ModelType>>(item: T & { _id?: Binary, id: undefined }): undefined;
-  preUpdate<T extends OptionalId<ModelType>>(item: T & { _id?: Binary, id?: string }): string | undefined {
-    if (item && item.id) {
+  preUpdate<T extends OptionalId<ModelType>>(item: T & { _id?: Binary; id: undefined }): undefined;
+  preUpdate<T extends OptionalId<ModelType>>(item: T & { _id?: Binary; id?: string }): string | undefined {
+    if (item?.id) {
       const id = item.id;
       item._id = MongoUtil.uuid(id);
       if (!this.config.storeId) {
@@ -102,7 +225,7 @@ export class MongoModelService implements
   async initializeClient(): Promise<void> {
     this.client = await MongoClient.connect(this.config.url, {
       ...this.config.connectionOptions,
-      useBigInt64: true,
+      useBigInt64: true
     });
     this.#db = this.client.db(this.config.namespace);
     this.#bucket = new GridFSBucket(this.#db, {
@@ -123,7 +246,7 @@ export class MongoModelService implements
   }
 
   // Storage
-  async createStorage(): Promise<void> { }
+  async createStorage(): Promise<void> {}
 
   async deleteStorage(): Promise<void> {
     await this.#db.dropDatabase();
@@ -131,7 +254,7 @@ export class MongoModelService implements
 
   async upsertModel(cls: Class): Promise<void> {
     const col = await this.getStore(cls);
-    const indices = MongoUtil.getIndices(cls, ModelRegistryIndex.getConfig(cls).indices);
+    const indices = [...ModelRegistryIndex.getIndices(cls).map(idx => MongoUtil.getIndex(cls, idx)), ...MongoUtil.getExtraIndices(cls)];
     const existingIndices = (await col.indexes().catch(() => [])).filter(idx => idx.name !== '_id_');
 
     const pendingMap = Object.fromEntries(indices.map(pair => [pair[1].name!, pair]));
@@ -165,7 +288,7 @@ export class MongoModelService implements
   }
 
   async truncateBlob(): Promise<void> {
-    await this.#bucket.drop().catch(() => { });
+    await this.#bucket.drop().catch(() => {});
   }
 
   /**
@@ -193,11 +316,15 @@ export class MongoModelService implements
     const id = this.preUpdate(cleaned);
 
     const store = await this.getStore(cls);
-    const result = await store.insertOne(castTo(cleaned));
-    if (!result.insertedId) {
-      throw new ExistsError(cls, id);
+    try {
+      const result = await store.insertOne(castTo(cleaned));
+      if (!result.insertedId) {
+        throw new ExistsError(cls, id);
+      }
+      return this.postUpdate(cleaned, id);
+    } catch (error) {
+      throw handleDuplicateKeyError(cls, id, error);
     }
-    return this.postUpdate(cleaned, id);
   }
 
   async update<T extends ModelType>(cls: Class<T>, item: T): Promise<T> {
@@ -217,17 +344,9 @@ export class MongoModelService implements
     const store = await this.getStore(cls);
 
     try {
-      await store.updateOne(
-        this.getIdFilter(cls, id, false),
-        { $set: cleaned },
-        { upsert: true }
-      );
+      await store.updateOne(this.getIdFilter(cls, id, false), { $set: cleaned }, { upsert: true });
     } catch (error) {
-      if (error instanceof Error && error.message.includes('duplicate key error')) {
-        throw new ExistsError(cls, id);
-      } else {
-        throw error;
-      }
+      throw handleDuplicateKeyError(cls, id, error);
     }
     return this.postUpdate(cleaned, id);
   }
@@ -238,24 +357,23 @@ export class MongoModelService implements
     const final = await ModelCrudUtil.prePartialUpdate(cls, item, view);
     const simple = MongoUtil.extractSimple(cls, final, undefined, false);
 
-    const operation: Partial<T> = castTo(Object
-      .entries(simple)
-      .reduce<Partial<Record<'$unset' | '$set', Record<string, unknown>>>>((document, [key, value]) => {
+    const operation: Partial<T> = castTo(
+      Object.entries(simple).reduce<Partial<Record<'$unset' | '$set', Record<string, unknown>>>>((document, [key, value]) => {
         if (value === null || value === undefined) {
           (document.$unset ??= {})[key] = value;
         } else {
           (document.$set ??= {})[key] = value;
         }
         return document;
-      }, {}));
+      }, {})
+    );
 
     const id = item.id;
 
-    const result = await store.findOneAndUpdate(
-      this.getIdFilter(cls, id),
-      operation,
-      { returnDocument: 'after', includeResultMetadata: true }
-    );
+    const result = await store.findOneAndUpdate(this.getIdFilter(cls, id), operation, {
+      returnDocument: 'after',
+      includeResultMetadata: true
+    });
 
     if (!result.value) {
       throw new NotFoundError(cls, id);
@@ -272,23 +390,18 @@ export class MongoModelService implements
     }
   }
 
-  async * list<T extends ModelType>(cls: Class<T>): AsyncIterable<T> {
+  async *list<T extends ModelType>(cls: Class<T>, options?: ModelListOptions): AsyncIterable<T[]> {
     const store = await this.getStore(cls);
-    const cursor = store.find(this.getWhereFilter(cls, {}), { timeout: true }).batchSize(100);
-    for await (const item of cursor) {
-      try {
-        yield await this.postLoad(cls, item);
-      } catch (error) {
-        if (!(error instanceof NotFoundError)) {
-          throw error;
-        }
-      }
-    }
+    const cursor = store.find(this.getWhereFilter(cls, {}), { timeout: true });
+    yield* this.#iterateCursor(cls, cursor, options);
   }
 
   // Blob
   async upsertBlob(location: string, input: BinaryType, metadata?: BinaryMetadata, overwrite = true): Promise<void> {
-    const existing = await this.getBlobMetadata(location).then(() => true, () => false);
+    const existing = await this.getBlobMetadata(location).then(
+      () => true,
+      () => false
+    );
     if (!overwrite && existing) {
       return;
     }
@@ -320,10 +433,9 @@ export class MongoModelService implements
   }
 
   async updateBlobMetadata(location: string, metadata: BinaryMetadata): Promise<void> {
-    await this.#db.collection<{ metadata: BinaryMetadata }>(`${ModelBlobNamespace}.files`).findOneAndUpdate(
-      { filename: location },
-      { $set: { metadata, contentType: metadata.contentType! } },
-    );
+    await this.#db
+      .collection<{ metadata: BinaryMetadata }>(`${ModelBlobNamespace}.files`)
+      .findOneAndUpdate({ filename: location }, { $set: { metadata, contentType: metadata.contentType! } });
   }
 
   // Bulk
@@ -356,7 +468,10 @@ export class MongoModelService implements
         bulk.insert(operation.insert);
       } else if (operation.upsert) {
         const id = this.preUpdate(operation.upsert);
-        bulk.find({ _id: MongoUtil.uuid(id!) }).upsert().updateOne({ $set: operation.upsert });
+        bulk
+          .find({ _id: MongoUtil.uuid(id!) })
+          .upsert()
+          .updateOne({ $set: operation.upsert });
       } else if (operation.update) {
         const id = this.preUpdate(operation.update);
         bulk.find({ _id: MongoUtil.uuid(id) }).update({ $set: operation.update });
@@ -405,55 +520,105 @@ export class MongoModelService implements
   }
 
   // Indexed
-  async getByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<T> {
-    const { key } = ModelIndexedUtil.computeIndexKey(cls, idx, body);
+  async getByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SingleItemIndex<T, K, S>,
+    body: FullKeyedIndexBody<T, K, S>
+  ): Promise<T> {
     const store = await this.getStore(cls);
-    const result = await store.findOne(
-      this.getWhereFilter(
-        cls,
-        castTo(ModelIndexedUtil.projectIndex(cls, idx, body))
-      )
-    );
+
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate({ sort: true });
+
+    const result = await store.findOne(this.getWhereFilter(cls, castTo(computed.project({ sort: true, includeId: true }))));
     if (!result) {
-      throw new NotFoundError(`${cls.name}: ${idx}`, key);
+      throw new NotFoundError(`${cls.name}: ${idx}`, computed.getKey({ sort: true }));
     }
     return await this.postLoad(cls, result);
   }
 
-  async deleteByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<void> {
-    const { key } = ModelIndexedUtil.computeIndexKey(cls, idx, body);
+  async deleteByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SingleItemIndex<T, K, S>,
+    body: FullKeyedIndexBody<T, K, S>
+  ): Promise<void> {
     const store = await this.getStore(cls);
-    const result = await store.deleteOne(
-      this.getWhereFilter(
-        cls,
-        castTo(ModelIndexedUtil.projectIndex(cls, idx, body))
-      )
-    );
-    if (result.deletedCount) {
-      return;
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate({ sort: true });
+
+    const result = await store.deleteOne(this.getWhereFilter(cls, castTo(computed.project({ sort: true, includeId: true }))));
+    if (!result.deletedCount) {
+      throw new NotFoundError(`${cls.name}: ${idx}`, computed.getKey({ sort: true }));
     }
-    throw new NotFoundError(`${cls.name}: ${idx}`, key);
   }
 
-  async upsertByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: OptionalId<T>): Promise<T> {
+  upsertByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SingleItemIndex<T, K, S>,
+    body: OptionalId<T>
+  ): Promise<T> {
     return ModelIndexedUtil.naiveUpsert(this, cls, idx, body);
   }
 
-  async * listByIndex<T extends ModelType>(cls: Class<T>, idx: string, body?: DeepPartial<T>): AsyncIterable<T> {
-    const store = await this.getStore(cls);
-    const idxConfig = ModelRegistryIndex.getIndex(cls, idx, ['sorted', 'unsorted']);
+  updateByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SingleItemIndex<T, K, S>,
+    body: T
+  ): Promise<T> {
+    return ModelIndexedUtil.naiveUpdate(this, cls, idx, body);
+  }
+
+  async updatePartialByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SingleItemIndex<T, K>,
+    body: FullKeyedIndexWithPartialBody<T, K, S>
+  ): Promise<T> {
+    const item = await ModelCrudUtil.naivePartialUpdate(cls, () => this.getByIndex(cls, idx, castTo(body)), castTo(body));
+    return this.update(cls, item);
+  }
+
+  async pageByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+    options?: ModelPageOptions
+  ): Promise<ModelPageResult<T>> {
+    {
+      const offset = options?.offset ? JSONUtil.fromBase64<number>(options.offset) : 0;
+      const cursor = await this.#buildIndexQuery(cls, idx, body);
+      const batches = await Array.fromAsync(this.#iterateCursor(cls, cursor, { limit: 100, ...options, offset }));
+      const items = batches.flat();
+      return { items, nextOffset: items.length ? JSONUtil.toBase64(offset + items.length) : undefined };
+    }
+  }
 
-    const where = this.getWhereFilter(
-      cls,
-      castTo(ModelIndexedUtil.projectIndex(cls, idx, body, { emptySortValue: { $exists: true } }))
+  async *listByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+    options?: ModelListOptions
+  ): AsyncIterable<T[]> {
+    const cursor = await this.#buildIndexQuery(cls, idx, body);
+    yield* this.#iterateCursor(cls, cursor, options);
+  }
+
+  async suggestByIndex<
+    T extends ModelType,
+    S extends SortedIndexSelection<T>,
+    K extends KeyedIndexSelection<T>,
+    B extends SortedIndexSelectionType<T, S> & string
+  >(cls: Class<T>, idx: SortedIndex<T, K, S>, body: KeyedIndexBody<T, K>, prefix: B, options?: ModelIndexedSearchOptions): Promise<T[]> {
+    const cursor = await this.#buildIndexQuery(cls, idx, body, where =>
+      castTo({
+        $and: [
+          where,
+          {
+            [idx.sortTemplate[0].path.join('.')]: ModelIndexedUtil.getSuggestRegex(prefix)
+          }
+        ]
+      })
     );
+    const batches = await Array.fromAsync(this.#iterateCursor(cls, cursor, { limit: 10, ...options }));
 
-    const sort = castTo<{ [ListIndexSymbol]: PlainIdx }>(idxConfig)[ListIndexSymbol] ??= MongoUtil.getPlainIndex(idxConfig);
-    const cursor = store.find(where, { timeout: true }).batchSize(100).sort(castTo(sort));
-
-    for await (const item of cursor) {
-      yield await this.postLoad(cls, item);
-    }
+    return batches.flat();
   }
 
   // Query
@@ -513,15 +678,14 @@ export class MongoModelService implements
     const item = await ModelCrudUtil.prePartialUpdate(cls, data);
     const col = await this.getStore(cls);
     const items = MongoUtil.extractSimple(cls, item);
-    const final = Object.entries(items).reduce<Partial<Record<'$unset' | '$set', Record<string, unknown>>>>(
-      (document, [key, value]) => {
-        if (value === null || value === undefined) {
-          (document.$unset ??= {})[key] = value;
-        } else {
-          (document.$set ??= {})[key] = value;
-        }
-        return document;
-      }, {});
+    const final = Object.entries(items).reduce<Partial<Record<'$unset' | '$set', Record<string, unknown>>>>((document, [key, value]) => {
+      if (value === null || value === undefined) {
+        (document.$unset ??= {})[key] = value;
+      } else {
+        (document.$set ??= {})[key] = value;
+      }
+      return document;
+    }, {});
 
     const filter = MongoUtil.extractWhereFilter(cls, query.where);
     const result = await col.updateMany(filter, castTo(final));
@@ -529,7 +693,7 @@ export class MongoModelService implements
   }
 
   // Facet
-  async facet<T extends ModelType>(cls: Class<T>, field: ValidStringFields<T>, query?: ModelQuery<T>): Promise<ModelQueryFacet[]> {
+  async facetByQuery<T extends ModelType>(cls: Class<T>, field: ValidStringFields<T>, query?: ModelQuery<T>): Promise<ModelQueryFacet[]> {
     await QueryVerifier.verify(cls, query);
 
     const col = await this.getStore(cls);
@@ -555,7 +719,7 @@ export class MongoModelService implements
       }
     ];
 
-    const result = await col.aggregate<{ _id: ObjectId, count: number }>(aggregations).toArray();
+    const result = await col.aggregate<{ _id: ObjectId; count: number }>(aggregations).toArray();
 
     return result
       .map(item => ({
@@ -566,18 +730,28 @@ export class MongoModelService implements
   }
 
   // Suggest
-  async suggestValues<T extends ModelType>(cls: Class<T>, field: ValidStringFields<T>, prefix?: string, query?: PageableModelQuery<T>): Promise<string[]> {
+  async suggestValuesByQuery<T extends ModelType>(
+    cls: Class<T>,
+    field: ValidStringFields<T>,
+    prefix?: string,
+    query?: PageableModelQuery<T>
+  ): Promise<string[]> {
     await QueryVerifier.verify(cls, query);
     const resolvedQuery = ModelQuerySuggestUtil.getSuggestFieldQuery<T>(cls, field, prefix, query);
     const results = await this.query<T>(cls, resolvedQuery);
-    return ModelQuerySuggestUtil.combineSuggestResults<T, string>(cls, field, prefix, results, (a) => a, query && query.limit);
+    return ModelQuerySuggestUtil.combineSuggestResults<T, string>(cls, field, prefix, results, a => a, query?.limit);
   }
 
-  async suggest<T extends ModelType>(cls: Class<T>, field: ValidStringFields<T>, prefix?: string, query?: PageableModelQuery<T>): Promise<T[]> {
+  async suggestByQuery<T extends ModelType>(
+    cls: Class<T>,
+    field: ValidStringFields<T>,
+    prefix?: string,
+    query?: PageableModelQuery<T>
+  ): Promise<T[]> {
     await QueryVerifier.verify(cls, query);
     const resolvedQuery = ModelQuerySuggestUtil.getSuggestQuery<T>(cls, field, prefix, query);
     const results = await this.query<T>(cls, resolvedQuery);
-    return ModelQuerySuggestUtil.combineSuggestResults(cls, field, prefix, results, (_, b) => b, query && query.limit);
+    return ModelQuerySuggestUtil.combineSuggestResults(cls, field, prefix, results, (_, b) => b, query?.limit);
   }
 
   // Other
@@ -590,12 +764,14 @@ export class MongoModelService implements
       search = { $search: search, $language: 'en' };
     }
 
-    (query.sort ??= []).unshift(castTo<(typeof query.sort[0])>({
-      score: { $meta: 'textScore' }
-    }));
+    (query.sort ??= []).unshift(
+      castTo<(typeof query.sort)[0]>({
+        score: { $meta: 'textScore' }
+      })
+    );
 
     const cursor = col.find(castTo({ $and: [{ $text: search }, filter] }), {});
     const items = await MongoUtil.prepareCursor(cls, cursor, query).toArray();
     return await Promise.all(items.map(item => this.postLoad(cls, item)));
   }
-}
+}

package/support/service.mongo.ts

@@ -1,10 +1,16 @@
 import type { ServiceDescriptor } from '@travetto/cli';
 
-const version = process.env.MONGO_VERSION || '8.2';
+const version = process.env.MONGO_VERSION || '8.3';
+
+/* cspell:words orbstack pthread rseq glibc TUNABLES */
 
 export const service: ServiceDescriptor = {
   name: 'mongodb',
   version,
   port: 27017,
-  image: `mongo:${version}`
-};
+  image: `mongo:${version}`,
+  env: {
+    // Temp until mongo image fixes orbstack issue
+    GLIBC_TUNABLES: 'glibc.pthread.rseq=1'
+  }
+};