@travetto/model-firestore 8.0.0-alpha.3 → 8.0.0-alpha.31

package/README.md

@@ -13,11 +13,11 @@ npm install @travetto/model-firestore
 yarn add @travetto/model-firestore
 ```
 
-This module provides an [Firestore](https://firebase.google.com/docs/firestore)-based implementation of the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.").  This source allows the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.") module to read, write and query against [Firestore](https://firebase.google.com/docs/firestore). 
+This module provides an [Firestore](https://firebase.google.com/docs/firestore)-based implementation of the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations."). This source allows the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.") module to read, write and query against [Firestore](https://firebase.google.com/docs/firestore). 
 
 Supported features:
-   *  [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/types/crud.ts#L11)
-   *  [Indexed](https://github.com/travetto/travetto/tree/main/module/model/src/types/indexed.ts#L11)
+   *  [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/types/crud.ts#L10)
+   *  [Indexed](https://github.com/travetto/travetto/tree/main/module/model-indexed/src/types/service.ts#L21)
 
 Out of the box, by installing the module, everything should be wired up by default.If you need to customize any aspect of the source or config, you can override and register it with the [Dependency Injection](https://github.com/travetto/travetto/tree/main/module/di#readme "Dependency registration/management and injection support.") module.
 
@@ -42,7 +42,7 @@ where the [FirestoreModelConfig](https://github.com/travetto/travetto/tree/main/
 ```typescript
 @Config('model.firestore')
 export class FirestoreModelConfig {
-  databaseURL?: string;
+  databaseId?: string;
   credentialsFile?: string;
   emulator?: string;
   projectId?: string;
@@ -52,8 +52,8 @@ export class FirestoreModelConfig {
 
   @PostConstruct()
   async finalizeConfig(): Promise<void> {
-    if (!this.databaseURL && !Runtime.production) {
-      this.projectId ??= 'trv-local-dev';
+    if (!this.projectId && !Runtime.production) {
+      this.projectId = 'trv-local-dev';
       this.emulator ??= 'localhost:7000'; // From docker
     }
     if (this.emulator) {

package/__index__.ts

@@ -1,2 +1,2 @@
+export * from './src/config.ts';
 export * from './src/service.ts';
-export * from './src/config.ts';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model-firestore",
-  "version": "8.0.0-alpha.3",
+  "version": "8.0.0-alpha.31",
   "description": "Firestore backing for the travetto model module.",
   "keywords": [
     "typescript",
@@ -25,13 +25,14 @@
     "directory": "module/model-firestore"
   },
   "dependencies": {
-    "@google-cloud/firestore": "^8.3.0",
-    "@travetto/config": "^8.0.0-alpha.3",
-    "@travetto/model": "^8.0.0-alpha.3",
-    "@travetto/runtime": "^8.0.0-alpha.3"
+    "@google-cloud/firestore": "^8.7.0",
+    "@travetto/config": "^8.0.0-alpha.23",
+    "@travetto/model": "^8.0.0-alpha.24",
+    "@travetto/model-indexed": "^8.0.0-alpha.26",
+    "@travetto/runtime": "^8.0.0-alpha.21"
   },
   "peerDependencies": {
-    "@travetto/cli": "^8.0.0-alpha.4"
+    "@travetto/cli": "^8.0.0-alpha.29"
   },
   "peerDependenciesMeta": {
     "@travetto/cli": {
@@ -39,7 +40,12 @@
     }
   },
   "travetto": {
-    "displayName": "Firestore Model Support"
+    "displayName": "Firestore Model Support",
+    "build": {
+      "externalDependencies": [
+        "google-gax"
+      ]
+    }
   },
   "private": false,
   "publishConfig": {

package/src/config.ts

@@ -1,18 +1,17 @@
-import { JSONUtil, Runtime, RuntimeResources } from '@travetto/runtime';
 import { Config } from '@travetto/config';
-import { Schema, SchemaValidator } from '@travetto/schema';
 import { PostConstruct } from '@travetto/di';
+import { JSONUtil, Runtime, RuntimeResources } from '@travetto/runtime';
+import { Schema, SchemaValidator } from '@travetto/schema';
 
 @Schema()
 class FirestoreModelConfigCredentials {
   client_email: string;
-  project_id: string;
   private_key: string;
 }
 
 @Config('model.firestore')
 export class FirestoreModelConfig {
-  databaseURL?: string;
+  databaseId?: string;
   credentialsFile?: string;
   emulator?: string;
   projectId?: string;
@@ -22,8 +21,8 @@ export class FirestoreModelConfig {
 
   @PostConstruct()
   async finalizeConfig(): Promise<void> {
-    if (!this.databaseURL && !Runtime.production) {
-      this.projectId ??= 'trv-local-dev';
+    if (!this.projectId && !Runtime.production) {
+      this.projectId = 'trv-local-dev';
       this.emulator ??= 'localhost:7000'; // From docker
     }
     if (this.emulator) {
@@ -35,4 +34,4 @@ export class FirestoreModelConfig {
       await SchemaValidator.validate(FirestoreModelConfigCredentials, this.credentials);
     }
   }
-}
+}

package/src/service.ts

@@ -1,12 +1,35 @@
 import { type DocumentData, FieldValue, Firestore, type Query } from '@google-cloud/firestore';
 
-import { JSONUtil, ShutdownManager, type Class, type DeepPartial } from '@travetto/runtime';
 import { Injectable, PostConstruct } from '@travetto/di';
 import {
-  type ModelCrudSupport, ModelRegistryIndex, type ModelStorageSupport,
-  type ModelIndexedSupport, type ModelType, NotFoundError, type OptionalId,
-  ModelCrudUtil, ModelIndexedUtil,
+  type ModelCrudSupport,
+  ModelCrudUtil,
+  type ModelListOptions,
+  ModelRegistryIndex,
+  type ModelStorageSupport,
+  type ModelType,
+  NotFoundError,
+  type OptionalId
 } from '@travetto/model';
+import {
+  type FullKeyedIndexBody,
+  type FullKeyedIndexWithPartialBody,
+  type KeyedIndexBody,
+  type KeyedIndexSelection,
+  ModelIndexedComputedIndex,
+  type ModelIndexedSearchOptions,
+  type ModelIndexedSupport,
+  ModelIndexedUtil,
+  type ModelPageOptions,
+  type ModelPageResult,
+  type SingleItemIndex,
+  type SortedIndex,
+  type SortedIndexSelection,
+  type SortedIndexSelectionType,
+  warnIfIndexedUniqueIndex,
+  warnIfNonIndexedIndex
+} from '@travetto/model-indexed';
+import { type Class, castTo, JSONUtil, RuntimeError, ShutdownManager } from '@travetto/runtime';
 
 import type { FirestoreModelConfig } from './config.ts';
 
@@ -19,25 +42,97 @@ const setMissingValues = <T>(input: T, missingValue: unknown): T =>
  */
 @Injectable()
 export class FirestoreModelService implements ModelCrudSupport, ModelStorageSupport, ModelIndexedSupport {
+  static resolveTable(cls: Class, namespace?: string): string {
+    let table = ModelRegistryIndex.getStoreName(cls);
+    if (namespace) {
+      table = `${namespace}_${table}`;
+    }
+    return table;
+  }
 
   idSource = ModelCrudUtil.uuidSource();
   client: Firestore;
   config: FirestoreModelConfig;
 
-  constructor(config: FirestoreModelConfig) { this.config = config; }
+  constructor(config: FirestoreModelConfig) {
+    this.config = config;
+  }
 
   #resolveTable(cls: Class): string {
-    let table = ModelRegistryIndex.getStoreName(cls);
-    if (this.config.namespace) {
-      table = `${this.config.namespace}_${table}`;
-    }
-    return table;
+    return FirestoreModelService.resolveTable(cls, this.config.namespace);
   }
 
   #getCollection(cls: Class): FirebaseFirestore.CollectionReference<DocumentData> {
     return this.client.collection(this.#resolveTable(cls));
   }
 
+  async #getIdByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SingleItemIndex<T, K, S>,
+    body: FullKeyedIndexBody<T, K, S>
+  ): Promise<string> {
+    ModelCrudUtil.ensureNotSubType(cls);
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate({ sort: true });
+    const query = [...computed.allParts, ...(computed.idPart ? [computed.idPart] : [])].reduce<Query>(
+      (result, { path, value }) => result.where(path.join('.'), '==', value),
+      this.#getCollection(cls)
+    );
+
+    const item = await query.get();
+    if (!item || item.empty) {
+      throw new NotFoundError(`${cls.name} Index=${idx}`, computed.getKey());
+    }
+    if (item.size > 1) {
+      throw new RuntimeError(`Multiple items found for ${cls.name} Index=${idx}`);
+    }
+    return item.docs[0].data().id;
+  }
+
+  #buildIndexQuery<T extends ModelType>(cls: Class<T>, idx: SortedIndex<T>, body: KeyedIndexBody<T>): Query {
+    ModelCrudUtil.ensureNotSubType(cls);
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate();
+
+    let query = computed.keyedParts.reduce<Query>(
+      (result, { path, value, state }) => result.where(path.join('.'), '==', state === 'empty' ? null : value),
+      this.#getCollection(cls)
+    );
+
+    for (const { path, value } of idx.sortTemplate) {
+      query = query.orderBy(path.join('.'), value === 1 ? 'asc' : 'desc');
+    }
+    return query;
+  }
+
+  async *#scanCollection<T extends ModelType>(
+    cls: Class<T>,
+    queryBuilder: () => Query,
+    options?: ModelListOptions & ModelPageOptions<number>
+  ): AsyncIterable<{ items: T[]; nextOffset?: number }> {
+    const limit = options?.limit ?? Number.MAX_SAFE_INTEGER;
+    const batchSize = Math.min(options?.batchSizeHint ?? 100, limit);
+
+    let offset = options?.offset ?? 0;
+    let produced = 0;
+
+    while (!options?.abort?.aborted && produced < limit) {
+      const query = queryBuilder().limit(batchSize).offset(offset);
+
+      const { docs, size } = await query.get();
+      if (size === 0) {
+        break;
+      }
+
+      const remaining = produced + size > limit ? docs.slice(0, limit - produced) : docs;
+
+      offset += size;
+
+      const items = await ModelCrudUtil.filterOutNotFound(remaining.map(item => ModelCrudUtil.load(cls, item.data()!)));
+      produced += items.length;
+
+      yield { items, nextOffset: offset };
+    }
+  }
+
   @PostConstruct()
   async initializeClient(): Promise<void> {
     this.client = new Firestore({ ...this.config, useBigInt: true });
@@ -45,20 +140,23 @@ export class FirestoreModelService implements ModelCrudSupport, ModelStorageSupp
   }
 
   // Storage
-  async createStorage(): Promise<void> { }
-  async deleteStorage(): Promise<void> { }
-
-  async deleteModel(cls: Class): Promise<void> {
-    for await (const item of this.list(cls)) {
-      await this.delete(cls, item.id);
+  async createStorage(): Promise<void> {
+    for (const cls of ModelRegistryIndex.getClasses()) {
+      warnIfIndexedUniqueIndex(this, cls, ModelRegistryIndex.getIndices(cls));
+      warnIfNonIndexedIndex(this, cls, ModelRegistryIndex.getIndices(cls));
     }
   }
+  async deleteStorage(): Promise<void> {}
+
+  async deleteModel<T extends ModelType>(cls: Class<T>): Promise<void> {
+    await this.client.recursiveDelete(this.#getCollection(cls));
+  }
 
   // Crud
   async get<T extends ModelType>(cls: Class<T>, id: string): Promise<T> {
     const result = await this.#getCollection(cls).doc(id).get();
 
-    if (result && result.exists) {
+    if (result?.exists) {
       return await ModelCrudUtil.load(cls, result.data()!);
     }
     throw new NotFoundError(cls, id);
@@ -89,7 +187,9 @@ export class FirestoreModelService implements ModelCrudSupport, ModelStorageSupp
     const id = item.id;
     const full = await ModelCrudUtil.naivePartialUpdate(cls, () => this.get(cls, id), item, view);
     const cleaned = setMissingValues(full, FieldValue.delete());
-    await this.#getCollection(cls).doc(id).set(cleaned, { mergeFields: Object.keys(full) });
+    await this.#getCollection(cls)
+      .doc(id)
+      .set(cleaned, { mergeFields: Object.keys(full) });
     return this.get(cls, id);
   }
 
@@ -105,63 +205,104 @@ export class FirestoreModelService implements ModelCrudSupport, ModelStorageSupp
     }
   }
 
-  async * list<T extends ModelType>(cls: Class<T>): AsyncIterable<T> {
-    const batch = await this.#getCollection(cls).select().get();
-    for (const item of batch.docs) {
-      try {
-        yield await this.get(cls, item.id);
-      } catch (error) {
-        if (!(error instanceof NotFoundError)) {
-          throw error;
-        }
-      }
+  async *list<T extends ModelType>(cls: Class<T>, options?: ModelListOptions): AsyncIterable<T[]> {
+    for await (const { items } of this.#scanCollection(cls, () => this.#getCollection(cls), options)) {
+      yield items;
     }
   }
 
-  // Indexed
-  async #getIdByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<string> {
-    ModelCrudUtil.ensureNotSubType(cls);
-
-    const { fields } = ModelIndexedUtil.computeIndexParts(cls, idx, body);
-    const query = fields.reduce<Query>(
-      (result, { path, value }) => result.where(path.join('.'), '==', value),
-      this.#getCollection(cls)
-    );
+  // Indexed contract
 
-    const item = await query.get();
-
-    if (item && !item.empty) {
-      return item.docs[0].id;
-    }
-    throw new NotFoundError(`${cls.name} Index=${idx}`, ModelIndexedUtil.computeIndexKey(cls, idx, body, { separator: '; ' })?.key);
-  }
-
-  async getByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<T> {
+  async getByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SingleItemIndex<T, K, S>,
+    body: FullKeyedIndexBody<T, K, S>
+  ): Promise<T> {
     return this.get(cls, await this.#getIdByIndex(cls, idx, body));
   }
 
-  async deleteByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<void> {
+  async deleteByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SingleItemIndex<T, K, S>,
+    body: FullKeyedIndexBody<T, K, S>
+  ): Promise<void> {
     return this.delete(cls, await this.#getIdByIndex(cls, idx, body));
   }
 
-  upsertByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: OptionalId<T>): Promise<T> {
+  upsertByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SingleItemIndex<T, K, S>,
+    body: OptionalId<T>
+  ): Promise<T> {
     return ModelIndexedUtil.naiveUpsert(this, cls, idx, body);
   }
 
-  async * listByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): AsyncIterable<T> {
-    ModelCrudUtil.ensureNotSubType(cls);
+  updateByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SingleItemIndex<T, K, S>,
+    body: T
+  ): Promise<T> {
+    return ModelIndexedUtil.naiveUpdate(this, cls, idx, body);
+  }
 
-    const config = ModelRegistryIndex.getIndex(cls, idx, ['sorted', 'unsorted']);
-    const { fields, sorted } = ModelIndexedUtil.computeIndexParts(cls, config, body, { emptySortValue: null });
-    let query = fields.reduce<Query>((result, { path, value }) =>
-      result.where(path.join('.'), '==', value), this.#getCollection(cls));
+  async updatePartialByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SingleItemIndex<T, K, S>,
+    body: FullKeyedIndexWithPartialBody<T, K, S>
+  ): Promise<T> {
+    const item = await ModelCrudUtil.naivePartialUpdate(cls, () => this.getByIndex(cls, idx, castTo(body)), castTo(body));
+    return this.update(cls, item);
+  }
 
-    if (sorted) {
-      query = query.orderBy(sorted.path.join('.'), sorted.dir === 1 ? 'asc' : 'desc');
+  async pageByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+    options?: ModelPageOptions
+  ): Promise<ModelPageResult<T>> {
+    const items: T[] = [];
+    let nextOffset: number | undefined;
+    for await (const batch of this.#scanCollection(cls, () => this.#buildIndexQuery(cls, idx, body), {
+      limit: 100,
+      ...options,
+      offset: options?.offset ? JSONUtil.fromBase64<number>(options.offset) : 0
+    })) {
+      items.push(...batch.items);
+      nextOffset = batch.nextOffset;
     }
 
-    for (const item of (await query.get()).docs) {
-      yield await ModelCrudUtil.load(cls, item.data()!);
+    return { items, nextOffset: nextOffset ? JSONUtil.toBase64(nextOffset) : undefined };
+  }
+
+  async *listByIndex<T extends ModelType, K extends KeyedIndexSelection<T>, S extends SortedIndexSelection<T>>(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+    options?: ModelListOptions
+  ): AsyncIterable<T[]> {
+    for await (const { items } of this.#scanCollection(cls, () => this.#buildIndexQuery(cls, idx, body), options)) {
+      yield items;
     }
   }
-}
+
+  async suggestByIndex<
+    T extends ModelType,
+    S extends SortedIndexSelection<T>,
+    K extends KeyedIndexSelection<T>,
+    B extends SortedIndexSelectionType<T, S> & string
+  >(cls: Class<T>, idx: SortedIndex<T, K, S>, body: KeyedIndexBody<T, K>, prefix: B, options?: ModelIndexedSearchOptions): Promise<T[]> {
+    const results: T[] = [];
+
+    const field = idx.sortTemplate[0].path.join('.');
+
+    for await (const { items } of this.#scanCollection(
+      cls,
+      () => this.#buildIndexQuery(cls, idx, body).where(field, '>=', prefix).where(field, '<', `${prefix}\uf8ff`),
+      { limit: 10, ...options }
+    )) {
+      results.push(...items);
+    }
+
+    return results;
+  }
+}

package/support/cli.firestore_indexes.ts

@@ -0,0 +1,120 @@
+import cp from 'node:child_process';
+import fs from 'node:fs/promises';
+
+import { CliCommand, type CliCommandShape, CliModuleFlag } from '@travetto/cli';
+import { DependencyRegistryIndex } from '@travetto/di';
+import { ManifestFileUtil } from '@travetto/manifest';
+import { ModelRegistryIndex } from '@travetto/model';
+import { isModelIndexedIndex } from '@travetto/model-indexed';
+import { Registry } from '@travetto/registry';
+import { Env, ExecUtil, JSONUtil, Runtime } from '@travetto/runtime';
+
+import { FirestoreModelConfig } from '../src/config.ts';
+import { FirestoreModelService } from '../src/service.ts';
+
+type FirestoreIndexField = {
+  fieldPath: string;
+  order: 'ASCENDING' | 'DESCENDING';
+};
+
+type FirestoreIndexDefinition = {
+  collectionGroup: string;
+  queryScope: 'COLLECTION' | 'COLLECTION_GROUP';
+  fields: FirestoreIndexField[];
+};
+
+type FirestoreIndexSet = {
+  indexes: FirestoreIndexDefinition[];
+  fieldOverrides: unknown[];
+};
+
+/**
+ * Generate the Firestore composite indexes JSON for all registered models.
+ *
+ * The resulting JSON can be written to stdout or to a file path for use in
+ * firebase-cli deployments.
+ */
+@CliCommand()
+export class FirestoreIndexesCommand implements CliCommandShape {
+  indexFile = 'firestore.indexes.json';
+  firebaseFile = 'firebase.json';
+
+  @CliModuleFlag({ short: 'm' })
+  module: string;
+
+  /** Should we deploy after writing the json file? */
+  deploy?: boolean;
+
+  finalize(): void {
+    Env.DEBUG.set(false);
+  }
+
+  async main(): Promise<void> {
+    await Registry.init();
+
+    const config = await DependencyRegistryIndex.getInstance(FirestoreModelConfig);
+
+    const indexesList: FirestoreIndexDefinition[] = [];
+
+    for (const cls of ModelRegistryIndex.getClasses()) {
+      const indices = ModelRegistryIndex.getIndices(cls)
+        .filter(isModelIndexedIndex)
+        // We need at least 2 fields.  All 1 field indices are already handled
+        .filter(idx => (idx.keyTemplate?.length ?? 0) + (idx.sortTemplate?.length ?? 0) >= 2);
+
+      for (const idx of indices) {
+        indexesList.push({
+          collectionGroup: FirestoreModelService.resolveTable(cls, config.namespace),
+          queryScope: 'COLLECTION',
+          fields: [...idx.keyTemplate, ...idx.sortTemplate].map(part => ({
+            fieldPath: part.path.join('.'),
+            order: part.value === -1 ? 'DESCENDING' : 'ASCENDING'
+          }))
+        });
+      }
+    }
+
+    const outputData: FirestoreIndexSet = {
+      indexes: indexesList,
+      fieldOverrides: []
+    };
+
+    const text = JSONUtil.toUTF8Pretty(outputData);
+
+    await ManifestFileUtil.bufferedFileWrite(this.indexFile, text);
+
+    const firebaseLocation = Runtime.workspaceRelative(this.firebaseFile);
+    if (!(await fs.stat(firebaseLocation, { throwIfNoEntry: false }))) {
+      await ManifestFileUtil.bufferedFileWrite(firebaseLocation, '{}');
+    }
+
+    const firebaseContext = JSONUtil.fromBinaryArray<{ firestore?: { database?: string; indexes?: string }[] }>(
+      await fs.readFile(firebaseLocation)
+    );
+    const existing = (firebaseContext.firestore ??= []);
+    const found = existing.find(x => x.database === config.databaseId);
+
+    let changed = true;
+    if (!found) {
+      existing.push({ indexes: this.indexFile, database: config.databaseId });
+    } else if (found.indexes !== this.indexFile) {
+      found.indexes = this.indexFile;
+    } else {
+      changed = false;
+    }
+
+    if (changed) {
+      await ManifestFileUtil.bufferedFileWrite(firebaseLocation, JSONUtil.toUTF8Pretty(firebaseContext));
+    }
+
+    if (this.deploy) {
+      const child = cp.spawn(
+        'firebase',
+        ['deploy', '--only', 'firestore:indexes'],
+        // Complete take over
+        { stdio: 'inherit' }
+      );
+      await ExecUtil.getResult(child);
+    }
+  }
+}

package/support/service.firestore.ts

@@ -10,4 +10,4 @@ export const service: ServiceDescriptor = {
     FIRESTORE_PROJECT_ID: 'trv-local-dev'
   },
   image: `ridedott/firestore-emulator:${version}`
-};
+};