@travetto/model-mongo 5.0.0-rc.9 → 5.0.1

package/README.md

@@ -17,10 +17,10 @@ This module provides an [mongodb](https://mongodb.com)-based implementation for
 
 Supported features:
    *  [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/service/crud.ts#L11)
-   *  [Streaming](https://github.com/travetto/travetto/tree/main/module/model/src/service/stream.ts#L3)
    *  [Expiry](https://github.com/travetto/travetto/tree/main/module/model/src/service/expiry.ts#L11)
    *  [Bulk](https://github.com/travetto/travetto/tree/main/module/model/src/service/bulk.ts#L19)
    *  [Indexed](https://github.com/travetto/travetto/tree/main/module/model/src/service/indexed.ts#L12)
+   *  [Blob](https://github.com/travetto/travetto/tree/main/module/model/src/service/blob.ts#L8)
    *  [Query Crud](https://github.com/travetto/travetto/tree/main/module/model-query/src/service/crud.ts#L11)
    *  [Facet](https://github.com/travetto/travetto/tree/main/module/model-query/src/service/facet.ts#L12)
    *  [Query](https://github.com/travetto/travetto/tree/main/module/model-query/src/service/query.ts#L10)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model-mongo",
-  "version": "5.0.0-rc.9",
+  "version": "5.0.1",
   "description": "Mongo backing for the travetto model module.",
   "keywords": [
     "mongo",
@@ -25,13 +25,13 @@
     "directory": "module/model-mongo"
   },
   "dependencies": {
-    "@travetto/config": "^5.0.0-rc.9",
-    "@travetto/model": "^5.0.0-rc.9",
-    "@travetto/model-query": "^5.0.0-rc.9",
+    "@travetto/config": "^5.0.1",
+    "@travetto/model": "^5.0.1",
+    "@travetto/model-query": "^5.0.1",
     "mongodb": "^6.8.0"
   },
   "peerDependencies": {
-    "@travetto/command": "^5.0.0-rc.9"
+    "@travetto/command": "^5.0.1"
   },
   "peerDependenciesMeta": {
     "@travetto/command": {

package/src/internal/util.ts

@@ -1,11 +1,14 @@
-import { Binary, ObjectId } from 'mongodb';
+import { Binary, CreateIndexesOptions, FindCursor, IndexDirection, ObjectId } from 'mongodb';
 
 import { castTo, Class, TypedObject } from '@travetto/runtime';
-import { DistanceUnit, WhereClause } from '@travetto/model-query';
-import type { ModelType, IndexField } from '@travetto/model';
+import { DistanceUnit, PageableModelQuery, WhereClause } from '@travetto/model-query';
+import type { ModelType, IndexField, IndexConfig } from '@travetto/model';
 import { DataUtil, SchemaRegistry } from '@travetto/schema';
 import { ModelQueryUtil } from '@travetto/model-query/src/internal/service/query';
 import { AllViewⲐ } from '@travetto/schema/src/internal/types';
+import { PointImpl } from '@travetto/model-query/src/internal/model/point';
+
+type IdxCfg = CreateIndexesOptions;
 
 /**
  * Converting units to various radians
@@ -21,12 +24,15 @@ const RADIANS_TO: Record<DistanceUnit, number> = {
 export type WithId<T, I = unknown> = T & { _id?: I };
 const isWithId = <T extends ModelType, I = unknown>(o: T): o is WithId<T, I> => o && '_id' in o;
 
+export type BasicIdx = Record<string, IndexDirection>;
+export type PlainIdx = Record<string, -1 | 0 | 1>;
+
 /**
  * Basic mongo utils for conforming to the model module
  */
 export class MongoUtil {
 
-  static toIndex<T extends ModelType>(f: IndexField<T>): Record<string, number> {
+  static toIndex<T extends ModelType>(f: IndexField<T>): PlainIdx {
     const keys = [];
     while (typeof f !== 'number' && typeof f !== 'boolean' && Object.keys(f)) {
       const key = TypedObject.keys(f)[0];
@@ -35,7 +41,7 @@ export class MongoUtil {
     }
     const rf: number | boolean = castTo(f);
     return {
-      [keys.join('.')]: typeof rf === 'boolean' ? (rf ? 1 : 0) : rf
+      [keys.join('.')]: typeof rf === 'boolean' ? (rf ? 1 : 0) : castTo<-1 | 1 | 0>(rf)
     };
   }
 
@@ -153,4 +159,68 @@ export class MongoUtil {
     }
     return out;
   }
+
+  static getExtraIndices<T extends ModelType>(cls: Class<T>): BasicIdx[] {
+    const out: BasicIdx[] = [];
+    const textFields: string[] = [];
+    SchemaRegistry.visitFields(cls, (field, path) => {
+      if (field.type === PointImpl) {
+        const name = [...path, field].map(x => x.name).join('.');
+        out.push({ [name]: '2d' });
+      } else if (field.specifiers?.includes('text') && (field.specifiers?.includes('long') || field.specifiers.includes('search'))) {
+        const name = [...path, field].map(x => x.name).join('.');
+        textFields.push(name);
+      }
+    });
+    if (textFields.length) {
+      const text: BasicIdx = {};
+      for (const field of textFields) {
+        text[field] = 'text';
+      }
+      out.push(text);
+    }
+    return out;
+  }
+
+  static getPlainIndex(idx: IndexConfig<ModelType>): PlainIdx {
+    let out: PlainIdx = {};
+    for (const cfg of idx.fields.map(x => this.toIndex(x))) {
+      out = Object.assign(out, cfg);
+    }
+    return out;
+  }
+
+  static getIndices<T extends ModelType>(cls: Class<T>, indices: IndexConfig<ModelType>[] = []): [BasicIdx, IdxCfg][] {
+    return [
+      ...indices.map(idx => [this.getPlainIndex(idx), (idx.type === 'unique' ? { unique: true } : {})] as const),
+      ...this.getExtraIndices(cls).map((x) => [x, {}] as const)
+    ].map(x => [...x]);
+  }
+
+  static prepareCursor<T extends ModelType>(cls: Class<T>, cursor: FindCursor<T>, query: PageableModelQuery<T>): FindCursor<T> {
+    if (query.select) {
+      const selectKey = Object.keys(query.select)[0];
+      const select = typeof selectKey === 'string' && selectKey.startsWith('$') ? query.select : this.extractSimple(cls, query.select);
+      // Remove id if not explicitly defined, and selecting fields directly
+      if (!select['_id']) {
+        const values = new Set([...Object.values(select)]);
+        if (values.has(1) || values.has(true)) {
+          select['_id'] = false;
+        }
+      }
+      cursor.project(select);
+    }
+
+    if (query.sort) {
+      cursor = cursor.sort(Object.assign({}, ...query.sort.map(x => this.extractSimple(cls, x))));
+    }
+
+    cursor = cursor.limit(Math.trunc(query.limit ?? 200));
+
+    if (query.offset && typeof query.offset === 'number') {
+      cursor = cursor.skip(Math.trunc(query.offset ?? 0));
+    }
+
+    return cursor;
+  }
 }

package/src/service.ts

@@ -1,19 +1,11 @@
-// Wildcard import needed here due to packaging issues
-import {
-  type Db, GridFSBucket, MongoClient, type Sort, type CreateIndexesOptions,
-  type GridFSFile, type IndexSpecification, type Collection, ObjectId,
-  Binary
-} from 'mongodb';
-import { Readable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 
+import { type Db, GridFSBucket, MongoClient, type GridFSFile, type Collection, type ObjectId, type Binary, type RootFilterOperators } from 'mongodb';
+
 import {
-  ModelRegistry, ModelType, OptionalId,
-  ModelCrudSupport, ModelStorageSupport, ModelStreamSupport,
-  ModelExpirySupport, ModelBulkSupport, ModelIndexedSupport,
-  StreamMeta, BulkOp, BulkResponse,
-  NotFoundError, ExistsError, IndexConfig,
-  StreamRange
+  ModelRegistry, ModelType, OptionalId, ModelCrudSupport, ModelStorageSupport,
+  ModelExpirySupport, ModelBulkSupport, ModelIndexedSupport, BulkOp, BulkResponse,
+  NotFoundError, ExistsError, ModelBlobSupport
 } from '@travetto/model';
 import {
   ModelQuery, ModelQueryCrudSupport, ModelQueryFacetSupport, ModelQuerySupport,
@@ -21,32 +13,30 @@ import {
   QueryVerifier
 } from '@travetto/model-query';
 
-import { ShutdownManager, type Class, type DeepPartial, AppError, TypedObject, castTo, asFull } from '@travetto/runtime';
+import {
+  ShutdownManager, type Class, type DeepPartial, TypedObject,
+  castTo, asFull, BlobMeta, ByteRange, BinaryInput, BinaryUtil
+} from '@travetto/runtime';
 import { Injectable } from '@travetto/di';
-import { FieldConfig, SchemaRegistry, SchemaValidator } from '@travetto/schema';
 
 import { ModelCrudUtil } from '@travetto/model/src/internal/service/crud';
 import { ModelIndexedUtil } from '@travetto/model/src/internal/service/indexed';
 import { ModelStorageUtil } from '@travetto/model/src/internal/service/storage';
 import { ModelQueryUtil } from '@travetto/model-query/src/internal/service/query';
 import { ModelQuerySuggestUtil } from '@travetto/model-query/src/internal/service/suggest';
-import { PointImpl } from '@travetto/model-query/src/internal/model/point';
 import { ModelQueryExpiryUtil } from '@travetto/model-query/src/internal/service/expiry';
 import { ModelExpiryUtil } from '@travetto/model/src/internal/service/expiry';
-import { enforceRange, StreamModel, STREAMS } from '@travetto/model/src/internal/service/stream';
-import { AllViewⲐ } from '@travetto/schema/src/internal/types';
 import { ModelBulkUtil } from '@travetto/model/src/internal/service/bulk';
+import { MODEL_BLOB, ModelBlobNamespace, ModelBlobUtil } from '@travetto/model/src/internal/service/blob';
 
-import { MongoUtil, WithId } from './internal/util';
+import { MongoUtil, PlainIdx, WithId } from './internal/util';
 import { MongoModelConfig } from './config';
 
-const IdxFieldsⲐ = Symbol.for('@travetto/model-mongo:idx');
+const ListIndexⲐ = Symbol.for('@travetto/mongo-model:list-index');
 
-const asFielded = (cfg: IndexConfig<ModelType>): { [IdxFieldsⲐ]: Sort } => castTo(cfg);
+type BlobRaw = GridFSFile & { metadata?: BlobMeta };
 
-type IdxCfg = CreateIndexesOptions;
-
-type StreamRaw = GridFSFile & { metadata: StreamMeta };
+type MongoTextSearch = RootFilterOperators<unknown>['$text'];
 
 /**
  * Mongo-based model source
@@ -54,7 +44,7 @@ type StreamRaw = GridFSFile & { metadata: StreamMeta };
 @Injectable()
 export class MongoModelService implements
   ModelCrudSupport, ModelStorageSupport,
-  ModelBulkSupport, ModelStreamSupport,
+  ModelBulkSupport, ModelBlobSupport,
   ModelIndexedSupport, ModelQuerySupport,
   ModelQueryCrudSupport, ModelQueryFacetSupport,
   ModelQuerySuggestSupport, ModelExpirySupport {
@@ -66,11 +56,11 @@ export class MongoModelService implements
 
   constructor(public readonly config: MongoModelConfig) { }
 
-  async #describeStreamRaw(location: string): Promise<StreamRaw> {
-    const files: StreamRaw[] = castTo(await this.#bucket.find({ filename: location }, { limit: 1 }).toArray());
+  async #describeBlobRaw(location: string): Promise<BlobRaw> {
+    const files: BlobRaw[] = await this.#bucket.find({ filename: location }, { limit: 1 }).toArray();
 
     if (!files?.length) {
-      throw new NotFoundError(STREAMS, location);
+      throw new NotFoundError(ModelBlobNamespace, location);
     }
 
     return files[0];
@@ -80,7 +70,7 @@ export class MongoModelService implements
     this.client = await MongoClient.connect(this.config.url, this.config.options);
     this.#db = this.client.db(this.config.namespace);
     this.#bucket = new GridFSBucket(this.#db, {
-      bucketName: STREAMS,
+      bucketName: ModelBlobNamespace,
       writeConcern: { w: 1 }
     });
     await ModelStorageUtil.registerModelChangeListener(this);
@@ -99,45 +89,13 @@ export class MongoModelService implements
     await this.#db.dropDatabase();
   }
 
-  getGeoIndices<T extends ModelType>(cls: Class<T>, path: FieldConfig[] = [], root = cls): IndexSpecification[] {
-    const fields = SchemaRegistry.has(cls) ?
-      Object.values(SchemaRegistry.get(cls).views[AllViewⲐ].schema) :
-      [];
-    const out: IndexSpecification[] = [];
-    for (const field of fields) {
-      if (SchemaRegistry.has(field.type)) {
-        // Recurse
-        out.push(...this.getGeoIndices(field.type, [...path, field], root));
-      } else if (field.type === PointImpl) {
-        const name = [...path, field].map(x => x.name).join('.');
-        console.debug('Preparing geo-index', { cls: root.Ⲑid, name });
-        out.push({ [name]: '2d' });
-      }
-    }
-    return out;
-  }
-
-  getIndices<T extends ModelType>(cls: Class<T>): ([IndexSpecification] | [IndexSpecification, IdxCfg])[] {
-    const indices = ModelRegistry.get(cls).indices ?? [];
-    return [
-      ...indices.map((idx): [IndexSpecification, IdxCfg] => {
-        const combined = asFielded(idx)[IdxFieldsⲐ] ??= Object.assign({}, ...idx.fields.map(x => MongoUtil.toIndex(x)));
-        return [
-          castTo(combined),
-          (idx.type === 'unique' ? { unique: true } : {})
-        ];
-      }),
-      ...this.getGeoIndices(cls).map((x): [IndexSpecification] => [x])
-    ];
-  }
-
   async establishIndices<T extends ModelType>(cls: Class<T>): Promise<void> {
     const col = await this.getStore(cls);
-    const creating = this.getIndices(cls);
+    const creating = MongoUtil.getIndices(cls, ModelRegistry.get(cls).indices);
     if (creating.length) {
       console.debug('Creating indexes', { indices: creating });
       for (const el of creating) {
-        await col.createIndex(el[0], el[1] ?? {});
+        await col.createIndex(...el);
       }
     }
   }
@@ -151,10 +109,8 @@ export class MongoModelService implements
   }
 
   async truncateModel<T extends ModelType>(cls: Class<T>): Promise<void> {
-    if (cls === StreamModel) {
-      try {
-        await this.#bucket.drop();
-      } catch { }
+    if (cls === MODEL_BLOB) {
+      await this.#bucket.drop().catch(() => { });
     } else {
       const col = await this.getStore(cls);
       await col.deleteMany({});
@@ -226,13 +182,7 @@ export class MongoModelService implements
   async updatePartial<T extends ModelType>(cls: Class<T>, item: Partial<T> & { id: string }, view?: string): Promise<T> {
     const store = await this.getStore(cls);
 
-    if (view) {
-      await SchemaValidator.validate(cls, item, view);
-    }
-
-    item = await ModelCrudUtil.prePersist(cls, item, 'partial');
-
-    let final: Record<string, unknown> = item;
+    let final: Record<string, unknown> = await ModelCrudUtil.prePartialUpdate(cls, item, view);
 
     const items = MongoUtil.extractSimple(cls, final, undefined, false);
     final = Object
@@ -283,37 +233,38 @@ export class MongoModelService implements
     }
   }
 
-  // Stream
-  async upsertStream(location: string, input: Readable, meta: StreamMeta): Promise<void> {
+  // Blob
+  async upsertBlob(location: string, input: BinaryInput, meta?: BlobMeta, overwrite = true): Promise<void> {
+    const existing = await this.describeBlob(location).then(() => true, () => false);
+    if (!overwrite && existing) {
+      return;
+    }
+    const [stream, blobMeta] = await ModelBlobUtil.getInput(input, meta);
     const writeStream = this.#bucket.openUploadStream(location, {
-      contentType: meta.contentType,
-      metadata: meta
+      contentType: blobMeta.contentType,
+      metadata: blobMeta,
     });
+    await pipeline(stream, writeStream);
 
-    await pipeline(input, writeStream);
-  }
-
-  async getStream(location: string, range?: StreamRange): Promise<Readable> {
-    const meta = await this.describeStream(location);
-
-    if (range) {
-      range = enforceRange(range, meta.size);
-      range.end! += 1; // range is exclusive
+    if (existing) {
+      const [read] = await this.#bucket.find({ filename: location, _id: { $ne: writeStream.id } }).toArray();
+      await this.#bucket.delete(read._id);
     }
+  }
 
-    const res = await this.#bucket.openDownloadStreamByName(location, range);
-    if (!res) {
-      throw new NotFoundError(STREAMS, location);
-    }
-    return res;
+  async getBlob(location: string, range?: ByteRange): Promise<Blob> {
+    const meta = await this.describeBlob(location);
+    const final = range ? ModelBlobUtil.enforceRange(range, meta.size!) : undefined;
+    const mongoRange = final ? { start: final.start, end: final.end + 1 } : undefined;
+    return BinaryUtil.readableBlob(() => this.#bucket.openDownloadStreamByName(location, mongoRange), { ...meta, range: final });
   }
 
-  async describeStream(location: string): Promise<StreamMeta> {
-    return (await this.#describeStreamRaw(location)).metadata;
+  async describeBlob(location: string): Promise<BlobMeta> {
+    return (await this.#describeBlobRaw(location)).metadata ?? {};
   }
 
-  async deleteStream(location: string): Promise<void> {
-    const fileId = (await this.#describeStreamRaw(location))._id;
+  async deleteBlob(location: string): Promise<void> {
+    const fileId = (await this.#describeBlobRaw(location))._id;
     await this.#bucket.delete(fileId);
   }
 
@@ -426,18 +377,15 @@ export class MongoModelService implements
 
   async * listByIndex<T extends ModelType>(cls: Class<T>, idx: string, body?: DeepPartial<T>): AsyncIterable<T> {
     const store = await this.getStore(cls);
-    const idxCfg = ModelRegistry.getIndex(cls, idx);
-
-    if (idxCfg.type === 'unique') {
-      throw new AppError('Cannot list on unique indices', 'data');
-    }
+    const idxCfg = ModelRegistry.getIndex(cls, idx, ['sorted', 'unsorted']);
 
     const where = this.getWhereFilter(
       cls,
       castTo(ModelIndexedUtil.projectIndex(cls, idx, body, { emptySortValue: { $exists: true } }))
     );
 
-    const cursor = store.find(where, { timeout: true }).batchSize(100).sort(asFielded(idxCfg)[IdxFieldsⲐ]);
+    const sort = castTo<{ [ListIndexⲐ]: PlainIdx }>(idxCfg)[ListIndexⲐ] ??= MongoUtil.getPlainIndex(idxCfg);
+    const cursor = store.find(where, { timeout: true }).batchSize(100).sort(castTo(sort));
 
     for await (const el of cursor) {
       yield (await MongoUtil.postLoadId(await ModelCrudUtil.load(cls, el)));
@@ -450,31 +398,8 @@ export class MongoModelService implements
 
     const col = await this.getStore(cls);
     const filter = MongoUtil.extractWhereFilter(cls, query.where);
-    let cursor = col.find<T>(filter, {});
-    if (query.select) {
-      const selectKey = Object.keys(query.select)[0];
-      const select = typeof selectKey === 'string' && selectKey.startsWith('$') ? query.select : MongoUtil.extractSimple(cls, query.select);
-      // Remove id if not explicitly defined, and selecting fields directly
-      if (!select['_id']) {
-        const values = new Set([...Object.values(select)]);
-        if (values.has(1) || values.has(true)) {
-          select['_id'] = false;
-        }
-      }
-      cursor.project(select);
-    }
-
-    if (query.sort) {
-      cursor = cursor.sort(Object.assign({}, ...query.sort.map(x => MongoUtil.extractSimple(cls, x))));
-    }
-
-    cursor = cursor.limit(Math.trunc(query.limit ?? 200));
-
-    if (query.offset && typeof query.offset === 'number') {
-      cursor = cursor.skip(Math.trunc(query.offset ?? 0));
-    }
-
-    const items = await cursor.toArray();
+    const cursor = col.find<T>(filter, {});
+    const items = await MongoUtil.prepareCursor(cls, cursor, query).toArray();
     return await Promise.all(items.map(r => ModelCrudUtil.load(cls, r).then(MongoUtil.postLoadId)));
   }
 
@@ -492,7 +417,7 @@ export class MongoModelService implements
   }
 
   // Query Crud
-  async updateOneWithQuery<T extends ModelType>(cls: Class<T>, data: T, query: ModelQuery<T>): Promise<T> {
+  async updateByQuery<T extends ModelType>(cls: Class<T>, data: T, query: ModelQuery<T>): Promise<T> {
     await QueryVerifier.verify(cls, query);
 
     const col = await this.getStore(cls);
@@ -517,11 +442,13 @@ export class MongoModelService implements
     return res.deletedCount ?? 0;
   }
 
-  async updateByQuery<T extends ModelType>(cls: Class<T>, query: ModelQuery<T>, data: Partial<T>): Promise<number> {
+  async updatePartialByQuery<T extends ModelType>(cls: Class<T>, query: ModelQuery<T>, data: Partial<T>): Promise<number> {
     await QueryVerifier.verify(cls, query);
 
+    const item = await ModelCrudUtil.prePartialUpdate(cls, data);
+
     const col = await this.getStore(cls);
-    const items = MongoUtil.extractSimple(cls, data);
+    const items = MongoUtil.extractSimple(cls, item);
     const final = Object.entries(items).reduce<Partial<Record<'$unset' | '$set', Record<string, unknown>>>>(
       (acc, [k, v]) => {
         if (v === null || v === undefined) {
@@ -586,4 +513,24 @@ export class MongoModelService implements
     const results = await this.query<T>(cls, q);
     return ModelQuerySuggestUtil.combineSuggestResults(cls, field, prefix, results, (_, b) => b, query && query.limit);
   }
+
+  // Other
+  async queryText<T extends ModelType>(cls: Class<T>, search: string | MongoTextSearch, query: PageableModelQuery<T> = {}): Promise<T[]> {
+    await QueryVerifier.verify(cls, query);
+
+    const col = await this.getStore(cls);
+    const filter = MongoUtil.extractWhereFilter(cls, query.where);
+    if (typeof search === 'string') {
+      search = { $search: search, $language: 'en' };
+    }
+
+    (query.sort ??= []).unshift({
+      // @ts-expect-error
+      score: { $meta: 'textScore' }
+    });
+
+    const cursor = col.find<T>({ $and: [{ $text: search }, filter] }, {});
+    const items = await MongoUtil.prepareCursor(cls, cursor, query).toArray();
+    return await Promise.all(items.map(r => ModelCrudUtil.load(cls, r).then(MongoUtil.postLoadId)));
+  }
 }