@travetto/model-mongo 8.0.0-alpha.1 → 8.0.0-alpha.11

package/README.md

@@ -19,8 +19,8 @@ Supported features:
    *  [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/types/crud.ts#L11)
    *  [Expiry](https://github.com/travetto/travetto/tree/main/module/model/src/types/expiry.ts#L10)
    *  [Bulk](https://github.com/travetto/travetto/tree/main/module/model/src/types/bulk.ts#L64)
-   *  [Indexed](https://github.com/travetto/travetto/tree/main/module/model/src/types/indexed.ts#L11)
    *  [Blob](https://github.com/travetto/travetto/tree/main/module/model/src/types/blob.ts#L8)
+   *  [Indexed](https://github.com/travetto/travetto/tree/main/module/model-indexed/src/types/service.ts#L15)
    *  [Query Crud](https://github.com/travetto/travetto/tree/main/module/model-query/src/types/crud.ts#L11)
    *  [Facet](https://github.com/travetto/travetto/tree/main/module/model-query/src/types/facet.ts#L14)
    *  [Query](https://github.com/travetto/travetto/tree/main/module/model-query/src/types/query.ts#L10)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model-mongo",
-  "version": "8.0.0-alpha.1",
+  "version": "8.0.0-alpha.11",
   "type": "module",
   "description": "Mongo backing for the travetto model module.",
   "keywords": [
@@ -26,13 +26,14 @@
     "directory": "module/model-mongo"
   },
   "dependencies": {
-    "@travetto/config": "^8.0.0-alpha.1",
-    "@travetto/model": "^8.0.0-alpha.1",
-    "@travetto/model-query": "^8.0.0-alpha.1",
-    "mongodb": "^7.1.0"
+    "@travetto/config": "^8.0.0-alpha.10",
+    "@travetto/model": "^8.0.0-alpha.10",
+    "@travetto/model-indexed": "^8.0.0-alpha.11",
+    "@travetto/model-query": "^8.0.0-alpha.10",
+    "mongodb": "^7.1.1"
   },
   "peerDependencies": {
-    "@travetto/cli": "^8.0.0-alpha.1"
+    "@travetto/cli": "^8.0.0-alpha.15"
   },
   "peerDependenciesMeta": {
     "@travetto/cli": {

package/src/internal/util.ts

@@ -3,15 +3,14 @@ import {
   type IndexDescriptionInfo
 } from 'mongodb';
 
-import { RuntimeError, CodecUtil, castTo, type Class, toConcrete, TypedObject, BinaryUtil } from '@travetto/runtime';
-import { type DistanceUnit, type PageableModelQuery, type WhereClause, ModelQueryUtil } from '@travetto/model-query';
-import type { ModelType, IndexField, IndexConfig } from '@travetto/model';
+import { RuntimeError, CodecUtil, castTo, type Class, toConcrete, BinaryUtil } from '@travetto/runtime';
+import { type DistanceUnit, type PageableModelQuery, type WhereClause, isModelQueryIndex, ModelQueryUtil } from '@travetto/model-query';
+import { type ModelType, type IndexConfig, IndexNotSupported } from '@travetto/model';
 import { DataUtil, SchemaRegistryIndex, type Point } from '@travetto/schema';
+import { isModelIndexedIndex } from '@travetto/model-indexed';
 
 const PointConcrete = toConcrete<Point>();
 
-type IdxConfig = CreateIndexesOptions;
-
 /**
  * Converting units to various radians
  */
@@ -25,7 +24,19 @@ const RADIANS_TO: Record<DistanceUnit, number> = {
 
 export type WithId<T, I = unknown> = T & { _id?: I };
 export type BasicIdx = Record<string, IndexDirection>;
-export type PlainIdx = Record<string, -1 | 0 | 1>;
+
+function flattenKeys(obj: Record<string, unknown>, prefix = ''): Record<string, 1 | -1> {
+  const out: Record<string, 1 | -1> = {};
+  for (const [key, value] of Object.entries(obj)) {
+    const path = prefix ? `${prefix}.${key}` : key;
+    if (typeof value === 'object' && value !== null) {
+      Object.assign(out, flattenKeys(castTo(value), path));
+    } else {
+      out[path] = typeof value === 'boolean' ? (value ? 1 : -1) : castTo<-1 | 1>(value);
+    }
+  }
+  return out;
+}
 
 /**
  * Basic mongo utils for conforming to the model module
@@ -36,19 +47,6 @@ export class MongoUtil {
     return `${cls.Ⲑid}__${name}`.replace(/[^a-zA-Z0-9_]+/g, '_');
   }
 
-  static toIndex<T extends ModelType>(field: IndexField<T>): PlainIdx {
-    const keys = [];
-    while (typeof field !== 'number' && typeof field !== 'boolean' && Object.keys(field)) {
-      const key = TypedObject.keys(field)[0];
-      field = castTo(field[key]);
-      keys.push(key);
-    }
-    const rf: number | boolean = castTo(field);
-    return {
-      [keys.join('.')]: typeof rf === 'boolean' ? (rf ? 1 : 0) : castTo<-1 | 1 | 0>(rf)
-    };
-  }
-
   static uuid(value: string): Binary {
     try {
       return new Binary(
@@ -166,8 +164,8 @@ export class MongoUtil {
     return out;
   }
 
-  static getExtraIndices<T extends ModelType>(cls: Class<T>): [BasicIdx, IdxConfig][] {
-    const out: [BasicIdx, IdxConfig][] = [];
+  static getExtraIndices<T extends ModelType>(cls: Class<T>): [BasicIdx, CreateIndexesOptions][] {
+    const out: [BasicIdx, CreateIndexesOptions][] = [];
     const textFields: string[] = [];
     SchemaRegistryIndex.visitFields(cls, (field, path) => {
       if (field.type === PointConcrete) {
@@ -185,19 +183,26 @@ export class MongoUtil {
     return out;
   }
 
-  static getPlainIndex(idx: IndexConfig<ModelType>): PlainIdx {
-    let out: PlainIdx = {};
-    for (const config of idx.fields.map(value => this.toIndex(value))) {
-      out = Object.assign(out, config);
-    }
-    return out;
-  }
+  static getIndex(cls: Class, idx: IndexConfig): [BasicIdx, CreateIndexesOptions] {
+    const name = this.namespaceIndex(cls, idx.name);
+    if (isModelQueryIndex(idx)) {
+      const out = idx.fields.reduce(
+        (acc, field) => ({ ...acc, ...flattenKeys(castTo(field)) }),
+        castTo<Record<string, -1 | 0 | 1>>({}));
 
-  static getIndices<T extends ModelType>(cls: Class<T>, indices: IndexConfig<ModelType>[] = []): [BasicIdx, IdxConfig][] {
-    return [
-      ...indices.map(idx => [this.getPlainIndex(idx), { ...(idx.type === 'unique' ? { unique: true } : {}), name: this.namespaceIndex(cls, idx.name) }] as const),
-      ...this.getExtraIndices(cls)
-    ].map(idx => [...idx]);
+      return [out, { name, unique: idx.type === 'query:unique', }];
+    } else if (isModelIndexedIndex(idx)) {
+      const filter = Object.fromEntries([
+        ...idx.keyTemplate.map(({ path }) => [path.join('.'), 1]),
+        ...idx.sortTemplate.map(({ path, value }) => [path.join('.'), value === -1 ? -1 : 1])
+      ]);
+      switch (idx.type) {
+        case 'indexed:keyed': return [filter, { name, unique: idx.unique }];
+        case 'indexed:sorted': return [filter, { name }];
+      }
+    } else {
+      throw new IndexNotSupported(cls, idx);
+    }
   }
 
   static prepareCursor<T extends ModelType>(cls: Class<T>, cursor: FindCursor<T | MongoWithId<T>>, query: PageableModelQuery<T>): FindCursor<T> {

package/src/service.ts

@@ -3,13 +3,13 @@ import {
   type Db, GridFSBucket, MongoClient, type GridFSFile, type Collection,
   type ObjectId, type RootFilterOperators, type Filter,
   type WithId as MongoWithId,
+  type FindCursor,
 } from 'mongodb';
 
 import {
-  ModelRegistryIndex, type ModelType, type OptionalId, type ModelCrudSupport, type ModelStorageSupport,
-  type ModelExpirySupport, type ModelBulkSupport, type ModelIndexedSupport, type BulkOperation, type BulkResponse,
-  NotFoundError, ExistsError, type ModelBlobSupport,
-  ModelCrudUtil, ModelIndexedUtil, ModelStorageUtil, ModelExpiryUtil, ModelBulkUtil
+  ModelRegistryIndex, type ModelType, type OptionalId, type ModelCrudSupport, type ModelStorageSupport, type ModelExpirySupport,
+  type ModelBulkSupport, type BulkOperation, type BulkResponse, NotFoundError, ExistsError, type ModelBlobSupport,
+  ModelCrudUtil, ModelStorageUtil, ModelExpiryUtil, ModelBulkUtil,
 } from '@travetto/model';
 import {
   type ModelQuery, type ModelQueryCrudSupport, type ModelQueryFacetSupport, type ModelQuerySupport,
@@ -17,18 +17,22 @@ import {
   QueryVerifier, ModelQueryUtil, ModelQuerySuggestUtil, ModelQueryCrudUtil,
   type ModelQueryFacet,
 } from '@travetto/model-query';
+import {
+  type ModelIndexedSupport, type KeyedIndexSelection, type KeyedIndexBody, type ListPageOptions, ModelIndexedUtil,
+  type SingleItemIndex, type SortedIndexSelection, type ListPageResult, type SortedIndex, type FullKeyedIndexBody,
+  type FullKeyedIndexWithPartialBody, ModelIndexedComputedIndex,
+} from '@travetto/model-indexed';
 
 import {
-  ShutdownManager, type Class, type DeepPartial, TypedObject,
+  ShutdownManager, type Class, TypedObject,
   castTo, asFull, type BinaryMetadata, type ByteRange, type BinaryType, BinaryUtil, BinaryMetadataUtil,
+  JSONUtil,
 } from '@travetto/runtime';
 import { Injectable, PostConstruct } from '@travetto/di';
 
-import { MongoUtil, type PlainIdx, type WithId } from './internal/util.ts';
+import { MongoUtil, type WithId } from './internal/util.ts';
 import type { MongoModelConfig } from './config.ts';
 
-const ListIndexSymbol = Symbol();
-
 type BlobRaw = GridFSFile & { metadata?: BinaryMetadata };
 
 type MongoTextSearch = RootFilterOperators<unknown>['$text'];
@@ -54,6 +58,30 @@ export class MongoModelService implements
 
   constructor(config: MongoModelConfig) { this.config = config; }
 
+  async #buildIndexQuery<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>
+  ): Promise<FindCursor> {
+    const store = await this.getStore(cls);
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate();
+
+    const where = this.getWhereFilter(cls, castTo(computed.project()));
+
+    let q = store.find(where, { timeout: true })
+      .batchSize(100);
+
+    // TODO: We could cache this
+    if ('sort' in idx) {
+      q = q.sort(idx.sortTemplate.map(({ path, value }) => [path.join('.'), value] as const));
+    }
+    return q;
+  }
+
   restoreId(item: { id?: string, _id?: unknown }): void {
     if (item._id) {
       item.id ??= MongoUtil.idToString(castTo(item._id));
@@ -131,7 +159,10 @@ export class MongoModelService implements
 
   async upsertModel(cls: Class): Promise<void> {
     const col = await this.getStore(cls);
-    const indices = MongoUtil.getIndices(cls, ModelRegistryIndex.getConfig(cls).indices);
+    const indices = [
+      ...ModelRegistryIndex.getIndices(cls).map(idx => MongoUtil.getIndex(cls, idx)),
+      ...MongoUtil.getExtraIndices(cls)
+    ];
     const existingIndices = (await col.indexes().catch(() => [])).filter(idx => idx.name !== '_id_');
 
     const pendingMap = Object.fromEntries(indices.map(pair => [pair[1].name!, pair]));
@@ -405,54 +436,116 @@ export class MongoModelService implements
   }
 
   // Indexed
-  async getByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<T> {
-    const { key } = ModelIndexedUtil.computeIndexKey(cls, idx, body);
+  async getByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: FullKeyedIndexBody<T, K, S>): Promise<T> {
     const store = await this.getStore(cls);
+
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate({ sort: true });
+
     const result = await store.findOne(
-      this.getWhereFilter(
-        cls,
-        castTo(ModelIndexedUtil.projectIndex(cls, idx, body))
-      )
+      this.getWhereFilter(cls, castTo(computed.project({ sort: true, includeId: true })))
     );
     if (!result) {
-      throw new NotFoundError(`${cls.name}: ${idx}`, key);
+      throw new NotFoundError(`${cls.name}: ${idx}`, computed.getKey({ sort: true }));
     }
     return await this.postLoad(cls, result);
+
   }
 
-  async deleteByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<void> {
-    const { key } = ModelIndexedUtil.computeIndexKey(cls, idx, body);
+  async deleteByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: FullKeyedIndexBody<T, K, S>): Promise<void> {
     const store = await this.getStore(cls);
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate({ sort: true });
+
     const result = await store.deleteOne(
-      this.getWhereFilter(
-        cls,
-        castTo(ModelIndexedUtil.projectIndex(cls, idx, body))
-      )
+      this.getWhereFilter(cls, castTo(computed.project({ sort: true, includeId: true })))
     );
-    if (result.deletedCount) {
-      return;
+    if (!result.deletedCount) {
+      throw new NotFoundError(`${cls.name}: ${idx}`, computed.getKey({ sort: true }));
     }
-    throw new NotFoundError(`${cls.name}: ${idx}`, key);
   }
 
-  async upsertByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: OptionalId<T>): Promise<T> {
+  upsertByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: OptionalId<T>): Promise<T> {
     return ModelIndexedUtil.naiveUpsert(this, cls, idx, body);
   }
 
-  async * listByIndex<T extends ModelType>(cls: Class<T>, idx: string, body?: DeepPartial<T>): AsyncIterable<T> {
-    const store = await this.getStore(cls);
-    const idxConfig = ModelRegistryIndex.getIndex(cls, idx, ['sorted', 'unsorted']);
-
-    const where = this.getWhereFilter(
-      cls,
-      castTo(ModelIndexedUtil.projectIndex(cls, idx, body, { emptySortValue: { $exists: true } }))
-    );
+  updateByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: T): Promise<T> {
+    return ModelIndexedUtil.naiveUpdate(this, cls, idx, body);
+  }
+
+  async updatePartialByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K>, body: FullKeyedIndexWithPartialBody<T, K, S>): Promise<T> {
+    const item = await ModelCrudUtil.naivePartialUpdate(cls, () => this.getByIndex(cls, idx, castTo(body)), castTo(body));
+    return this.update(cls, item);
+  }
+
+  async pageByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+    options?: ListPageOptions,
+  ): Promise<ListPageResult<T>> {
+    {
+      const offset = options?.offset ? JSONUtil.fromBase64<number>(options.offset) : 0;
+      const limit = options?.limit ?? 100;
+      const cursor = (await this.#buildIndexQuery(cls, idx, body))
+        .limit(limit)
+        .skip(offset);
+
+      const items: T[] = [];
+      for await (const item of cursor) {
+        items.push(await this.postLoad(cls, item));
+      }
+      return { items, nextOffset: items.length ? JSONUtil.toBase64(offset + items.length) : undefined };
 
-    const sort = castTo<{ [ListIndexSymbol]: PlainIdx }>(idxConfig)[ListIndexSymbol] ??= MongoUtil.getPlainIndex(idxConfig);
-    const cursor = store.find(where, { timeout: true }).batchSize(100).sort(castTo(sort));
+    }
+  }
 
-    for await (const item of cursor) {
-      yield await this.postLoad(cls, item);
+  async *  listByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+  ): AsyncIterable<T> {
+    let offset = 0;
+    while (offset >= 0) {
+      const cursor = (await this.#buildIndexQuery(cls, idx, body))
+        .limit(100)
+        .skip(offset);
+
+      const items = await cursor.toArray();
+      if (items.length === 0) {
+        offset = -1;
+      } else {
+        offset += items.length;
+        for (const item of items) {
+          yield await this.postLoad(cls, item);
+        }
+      }
     }
   }