@travetto/model-firestore 8.0.0-alpha.2 → 8.0.0-alpha.21

package/README.md

@@ -17,7 +17,7 @@ This module provides an [Firestore](https://firebase.google.com/docs/firestore)-
 
 Supported features:
    *  [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/types/crud.ts#L11)
-   *  [Indexed](https://github.com/travetto/travetto/tree/main/module/model/src/types/indexed.ts#L11)
+   *  [Indexed](https://github.com/travetto/travetto/tree/main/module/model-indexed/src/types/service.ts#L16)
 
 Out of the box, by installing the module, everything should be wired up by default.If you need to customize any aspect of the source or config, you can override and register it with the [Dependency Injection](https://github.com/travetto/travetto/tree/main/module/di#readme "Dependency registration/management and injection support.") module.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model-firestore",
-  "version": "8.0.0-alpha.2",
+  "version": "8.0.0-alpha.21",
   "description": "Firestore backing for the travetto model module.",
   "keywords": [
     "typescript",
@@ -25,13 +25,14 @@
     "directory": "module/model-firestore"
   },
   "dependencies": {
-    "@google-cloud/firestore": "^8.3.0",
-    "@travetto/config": "^8.0.0-alpha.2",
-    "@travetto/model": "^8.0.0-alpha.2",
-    "@travetto/runtime": "^8.0.0-alpha.2"
+    "@google-cloud/firestore": "^8.6.0",
+    "@travetto/config": "^8.0.0-alpha.18",
+    "@travetto/model": "^8.0.0-alpha.19",
+    "@travetto/model-indexed": "^8.0.0-alpha.21",
+    "@travetto/runtime": "^8.0.0-alpha.17"
   },
   "peerDependencies": {
-    "@travetto/cli": "^8.0.0-alpha.3"
+    "@travetto/cli": "^8.0.0-alpha.24"
   },
   "peerDependenciesMeta": {
     "@travetto/cli": {
@@ -39,7 +40,12 @@
     }
   },
   "travetto": {
-    "displayName": "Firestore Model Support"
+    "displayName": "Firestore Model Support",
+    "build": {
+      "externalDependencies": [
+        "google-gax"
+      ]
+    }
   },
   "private": false,
   "publishConfig": {

package/src/service.ts

@@ -1,12 +1,17 @@
 import { type DocumentData, FieldValue, Firestore, type Query } from '@google-cloud/firestore';
 
-import { JSONUtil, ShutdownManager, type Class, type DeepPartial } from '@travetto/runtime';
+import { castTo, JSONUtil, ShutdownManager, type Class, RuntimeError } from '@travetto/runtime';
 import { Injectable, PostConstruct } from '@travetto/di';
 import {
-  type ModelCrudSupport, ModelRegistryIndex, type ModelStorageSupport,
-  type ModelIndexedSupport, type ModelType, NotFoundError, type OptionalId,
-  ModelCrudUtil, ModelIndexedUtil,
+  type ModelCrudSupport, ModelRegistryIndex, type ModelStorageSupport, type ModelType, NotFoundError, type OptionalId, ModelCrudUtil,
+  type ModelListOptions,
 } from '@travetto/model';
+import {
+  type ModelIndexedSupport, type KeyedIndexSelection, type KeyedIndexBody, type ModelPageOptions, ModelIndexedUtil,
+  type SingleItemIndex, type SortedIndexSelection, type ModelPageResult, type SortedIndex, type FullKeyedIndexBody,
+  type FullKeyedIndexWithPartialBody, ModelIndexedComputedIndex, warnIfIndexedUniqueIndex, warnIfNonIndexedIndex,
+  type ModelIndexedSearchOptions, type SortedIndexSelectionType
+} from '@travetto/model-indexed';
 
 import type { FirestoreModelConfig } from './config.ts';
 
@@ -38,6 +43,72 @@ export class FirestoreModelService implements ModelCrudSupport, ModelStorageSupp
     return this.client.collection(this.#resolveTable(cls));
   }
 
+  async #getIdByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: FullKeyedIndexBody<T, K, S>): Promise<string> {
+    ModelCrudUtil.ensureNotSubType(cls);
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate({ sort: true });
+    const query = [...computed.allParts, ...(computed.idPart ? [computed.idPart] : [])].reduce<Query>(
+      (result, { path, value }) => result.where(path.join('.'), '==', value),
+      this.#getCollection(cls)
+    );
+
+    const item = await query.get();
+    if (!item || item.empty) {
+      throw new NotFoundError(`${cls.name} Index=${idx}`, computed.getKey());
+    }
+    if (item.size > 1) {
+      throw new RuntimeError(`Multiple items found for ${cls.name} Index=${idx}`);
+    }
+    return item.docs[0].data().id;
+  }
+
+  #buildIndexQuery<T extends ModelType>(cls: Class<T>, idx: SortedIndex<T>, body: KeyedIndexBody<T>): Query {
+    ModelCrudUtil.ensureNotSubType(cls);
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate();
+
+    let query = computed.keyedParts.reduce<Query>((result, { path, value, state }) =>
+      result.where(path.join('.'), '==', (state === 'empty' ? null : value)), this.#getCollection(cls));
+
+    for (const { path, value } of idx.sortTemplate) {
+      query = query.orderBy(path.join('.'), value === 1 ? 'asc' : 'desc');
+    }
+    return query;
+  }
+
+  async * #scanCollection<T extends ModelType>(
+    cls: Class<T>,
+    queryBuilder: () => Query,
+    options?: ModelListOptions & ModelPageOptions<number>
+  ): AsyncIterable<{ items: T[], nextOffset?: number }> {
+    const limit = options?.limit ?? Number.MAX_SAFE_INTEGER;
+    const batchSize = Math.min(options?.batchSizeHint ?? 100, limit);
+
+    let offset = options?.offset ?? 0;
+    let produced = 0;
+
+    while (!(options?.abort?.aborted) && produced < limit) {
+      const query = queryBuilder().limit(batchSize).offset(offset);
+
+      const { docs, size } = await query.get();
+      if (size === 0) {
+        break;
+      }
+
+      const remaining = (produced + size > limit) ? docs.slice(0, limit - produced) : docs;
+
+      offset += size;
+
+      const items = await ModelCrudUtil.filterOutNotFound(
+        remaining.map(item => ModelCrudUtil.load(cls, item.data()!)));
+      produced += items.length;
+
+      yield { items, nextOffset: offset };
+    }
+  }
+
   @PostConstruct()
   async initializeClient(): Promise<void> {
     this.client = new Firestore({ ...this.config, useBigInt: true });
@@ -45,13 +116,16 @@ export class FirestoreModelService implements ModelCrudSupport, ModelStorageSupp
   }
 
   // Storage
-  async createStorage(): Promise<void> { }
+  async createStorage(): Promise<void> {
+    for (const cls of ModelRegistryIndex.getClasses()) {
+      warnIfIndexedUniqueIndex(this, cls, ModelRegistryIndex.getIndices(cls));
+      warnIfNonIndexedIndex(this, cls, ModelRegistryIndex.getIndices(cls));
+    }
+  }
   async deleteStorage(): Promise<void> { }
 
-  async deleteModel(cls: Class): Promise<void> {
-    for await (const item of this.list(cls)) {
-      await this.delete(cls, item.id);
-    }
+  async deleteModel<T extends ModelType>(cls: Class<T>): Promise<void> {
+    await this.client.recursiveDelete(this.#getCollection(cls));
   }
 
   // Crud
@@ -105,63 +179,113 @@ export class FirestoreModelService implements ModelCrudSupport, ModelStorageSupp
     }
   }
 
-  async * list<T extends ModelType>(cls: Class<T>): AsyncIterable<T> {
-    const batch = await this.#getCollection(cls).select().get();
-    for (const item of batch.docs) {
-      try {
-        yield await this.get(cls, item.id);
-      } catch (error) {
-        if (!(error instanceof NotFoundError)) {
-          throw error;
-        }
-      }
+  async * list<T extends ModelType>(cls: Class<T>, options?: ModelListOptions): AsyncIterable<T[]> {
+    for await (const { items } of this.#scanCollection(cls, () => this.#getCollection(cls), options)) {
+      yield items;
     }
   }
 
-  // Indexed
-  async #getIdByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<string> {
-    ModelCrudUtil.ensureNotSubType(cls);
-
-    const { fields } = ModelIndexedUtil.computeIndexParts(cls, idx, body);
-    const query = fields.reduce<Query>(
-      (result, { path, value }) => result.where(path.join('.'), '==', value),
-      this.#getCollection(cls)
-    );
-
-    const item = await query.get();
-
-    if (item && !item.empty) {
-      return item.docs[0].id;
-    }
-    throw new NotFoundError(`${cls.name} Index=${idx}`, ModelIndexedUtil.computeIndexKey(cls, idx, body, { separator: '; ' })?.key);
-  }
+  // Indexed contract
 
-  async getByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<T> {
+  async getByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: FullKeyedIndexBody<T, K, S>): Promise<T> {
     return this.get(cls, await this.#getIdByIndex(cls, idx, body));
   }
 
-  async deleteByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<void> {
+  async deleteByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: FullKeyedIndexBody<T, K, S>): Promise<void> {
     return this.delete(cls, await this.#getIdByIndex(cls, idx, body));
   }
 
-  upsertByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: OptionalId<T>): Promise<T> {
+  upsertByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: OptionalId<T>): Promise<T> {
     return ModelIndexedUtil.naiveUpsert(this, cls, idx, body);
   }
 
-  async * listByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): AsyncIterable<T> {
-    ModelCrudUtil.ensureNotSubType(cls);
+  updateByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: T): Promise<T> {
+    return ModelIndexedUtil.naiveUpdate(this, cls, idx, body);
+  }
 
-    const config = ModelRegistryIndex.getIndex(cls, idx, ['sorted', 'unsorted']);
-    const { fields, sorted } = ModelIndexedUtil.computeIndexParts(cls, config, body, { emptySortValue: null });
-    let query = fields.reduce<Query>((result, { path, value }) =>
-      result.where(path.join('.'), '==', value), this.#getCollection(cls));
+  async updatePartialByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: FullKeyedIndexWithPartialBody<T, K, S>): Promise<T> {
+    const item = await ModelCrudUtil.naivePartialUpdate(cls, () => this.getByIndex(cls, idx, castTo(body)), castTo(body));
+    return this.update(cls, item);
+  }
 
-    if (sorted) {
-      query = query.orderBy(sorted.path.join('.'), sorted.dir === 1 ? 'asc' : 'desc');
+  async pageByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+    options?: ModelPageOptions,
+  ): Promise<ModelPageResult<T>> {
+    const items: T[] = [];
+    let nextOffset: number | undefined;
+    for await (const batch of this.#scanCollection(cls, () => this.#buildIndexQuery(cls, idx, body), {
+      limit: 100,
+      ...options,
+      offset: options?.offset ? JSONUtil.fromBase64<number>(options.offset) : 0
+    })) {
+      items.push(...batch.items);
+      nextOffset = batch.nextOffset;
     }
 
-    for (const item of (await query.get()).docs) {
-      yield await ModelCrudUtil.load(cls, item.data()!);
+    return { items, nextOffset: nextOffset ? JSONUtil.toBase64(nextOffset) : undefined };
+  }
+
+  async * listByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+    options?: ModelListOptions
+  ): AsyncIterable<T[]> {
+    for await (const { items } of this.#scanCollection(cls, () => this.#buildIndexQuery(cls, idx, body), options)) {
+      yield items;
     }
   }
+
+  async suggestByIndex<
+    T extends ModelType,
+    S extends SortedIndexSelection<T>,
+    K extends KeyedIndexSelection<T>,
+    B extends SortedIndexSelectionType<T, S> & string
+  >(cls: Class<T>, idx: SortedIndex<T, K, S>, body: KeyedIndexBody<T, K>, prefix: B, options?: ModelIndexedSearchOptions): Promise<T[]> {
+    const results: T[] = [];
+
+    const field = idx.sortTemplate[0].path.join('.');
+
+    for await (const { items } of this.#scanCollection(cls,
+      () => this.#buildIndexQuery(cls, idx, body)
+        .where(field, '>=', prefix)
+        .where(field, '<', `${prefix}\uf8ff`),
+      { limit: 10, ...options })
+    ) {
+      results.push(...items);
+    }
+
+    return results;
+  }
 }