@travetto/model-elasticsearch 8.0.0-alpha.1 → 8.0.0-alpha.11

package/README.md

@@ -13,13 +13,13 @@ npm install @travetto/model-elasticsearch
 yarn add @travetto/model-elasticsearch
 ```
 
-This module provides an [elasticsearch](https://elastic.co)-based implementation of the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.").  This source allows the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.") module to read, write and query against [elasticsearch](https://elastic.co). In development mode, [ElasticsearchModelService](https://github.com/travetto/travetto/tree/main/module/model-elasticsearch/src/service.ts#L36) will also modify the [elasticsearch](https://elastic.co) schema in real time to minimize impact to development. 
+This module provides an [elasticsearch](https://elastic.co)-based implementation of the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.").  This source allows the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.") module to read, write and query against [elasticsearch](https://elastic.co). In development mode, [ElasticsearchModelService](https://github.com/travetto/travetto/tree/main/module/model-elasticsearch/src/service.ts#L39) will also modify the [elasticsearch](https://elastic.co) schema in real time to minimize impact to development. 
 
 Supported features:
    *  [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/types/crud.ts#L11)
    *  [Bulk](https://github.com/travetto/travetto/tree/main/module/model/src/types/bulk.ts#L64)
    *  [Expiry](https://github.com/travetto/travetto/tree/main/module/model/src/types/expiry.ts#L10)
-   *  [Indexed](https://github.com/travetto/travetto/tree/main/module/model/src/types/indexed.ts#L11)
+   *  [Indexed](https://github.com/travetto/travetto/tree/main/module/model-indexed/src/types/service.ts#L15)
    *  [Query Crud](https://github.com/travetto/travetto/tree/main/module/model-query/src/types/crud.ts#L11)
    *  [Facet](https://github.com/travetto/travetto/tree/main/module/model-query/src/types/facet.ts#L14)
    *  [Query](https://github.com/travetto/travetto/tree/main/module/model-query/src/types/query.ts#L10)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model-elasticsearch",
-  "version": "8.0.0-alpha.1",
+  "version": "8.0.0-alpha.11",
   "type": "module",
   "description": "Elasticsearch backing for the travetto model module, with real-time modeling support for Elasticsearch mappings.",
   "keywords": [
@@ -29,12 +29,13 @@
   },
   "dependencies": {
     "@elastic/elasticsearch": "^9.3.4",
-    "@travetto/config": "^8.0.0-alpha.1",
-    "@travetto/model": "^8.0.0-alpha.1",
-    "@travetto/model-query": "^8.0.0-alpha.1"
+    "@travetto/config": "^8.0.0-alpha.10",
+    "@travetto/model": "^8.0.0-alpha.10",
+    "@travetto/model-indexed": "^8.0.0-alpha.11",
+    "@travetto/model-query": "^8.0.0-alpha.10"
   },
   "peerDependencies": {
-    "@travetto/cli": "^8.0.0-alpha.1"
+    "@travetto/cli": "^8.0.0-alpha.15"
   },
   "peerDependenciesMeta": {
     "@travetto/cli": {

package/src/internal/query.ts

@@ -1,9 +1,10 @@
 import type * as estypes from '@elastic/elasticsearch/api/types';
 
-import { castTo, type Class, TypedObject } from '@travetto/runtime';
+import { type Any, castTo, type Class, TypedObject } from '@travetto/runtime';
 import { type WhereClause, type SelectClause, type SortClause, type Query, ModelQueryUtil } from '@travetto/model-query';
-import { type IndexConfig, type ModelType, ModelRegistryIndex } from '@travetto/model';
+import { type ModelType, ModelRegistryIndex } from '@travetto/model';
 import { DataUtil, SchemaRegistryIndex } from '@travetto/schema';
+import type { SortedIndex } from '@travetto/model-indexed';
 
 import { type EsSchemaConfig } from './types.ts';
 
@@ -13,7 +14,7 @@ import { type EsSchemaConfig } from './types.ts';
 export class ElasticsearchQueryUtil {
 
   /**
-   * Convert `a.b.c` to `a : { b : { c : ... }}`
+   * Convert `a : { b : { c : ... }}` to `a.b.c`
    */
   static extractSimple<T>(input: T, path: string = ''): Record<string, unknown> {
     const out: Record<string, unknown> = {};
@@ -51,13 +52,19 @@ export class ElasticsearchQueryUtil {
   /**
    * Build sort mechanism
    */
-  static getSort<T extends ModelType>(sort: SortClause<T>[] | IndexConfig<T>['fields']): estypes.Sort {
-    return sort.map<estypes.SortOptions>(option => {
-      const item = this.extractSimple(option);
-      const key = Object.keys(item)[0];
-      const value: boolean | -1 | 1 = castTo(item[key]);
-      return { [key]: { order: value === 1 || value === true ? 'asc' : 'desc' } };
-    });
+  static getSort<T extends ModelType>(sort: SortClause<T>[] | SortedIndex<T, Any, Any>): estypes.Sort {
+    if (Array.isArray(sort)) {
+      return sort.map<estypes.SortOptions>(option => {
+        const item = this.extractSimple(option);
+        const key = Object.keys(item)[0];
+        const value: boolean | -1 | 1 = castTo(item[key]);
+        return { [key]: { order: value === 1 || value === true ? 'asc' : 'desc' } };
+      });
+    } else {
+      return sort.sortTemplate.map<estypes.SortOptions>(field =>
+        ({ [field.path.join('.')]: { order: field.value === 1 ? 'asc' : 'desc' } })
+      );
+    }
   }
 
   /**

package/src/service.ts

@@ -3,19 +3,22 @@ import type * as estypes from '@elastic/elasticsearch/api/types';
 
 import {
   type ModelCrudSupport, type BulkOperation, type BulkResponse, type ModelBulkSupport, type ModelExpirySupport,
-  type ModelIndexedSupport, type ModelType, type ModelStorageSupport, NotFoundError, ModelRegistryIndex, type OptionalId,
-  ModelCrudUtil, ModelIndexedUtil, ModelStorageUtil, ModelExpiryUtil, ModelBulkUtil,
+  type ModelType, type ModelStorageSupport, NotFoundError, ModelRegistryIndex, type OptionalId,
+  ModelCrudUtil, ModelStorageUtil, ModelExpiryUtil, ModelBulkUtil
 } from '@travetto/model';
-import { ShutdownManager, type DeepPartial, type Class, castTo, asFull, TypedObject, asConstructable, JSONUtil } from '@travetto/runtime';
+import { ShutdownManager, type Class, castTo, asFull, TypedObject, asConstructable, JSONUtil } from '@travetto/runtime';
 import { BindUtil } from '@travetto/schema';
 import { Injectable, PostConstruct } from '@travetto/di';
 import {
-  type ModelQuery, type ModelQueryCrudSupport, type ModelQueryFacetSupport,
-  type ModelQuerySupport, type PageableModelQuery, type Query, type ValidStringFields,
-  QueryVerifier, type ModelQuerySuggestSupport,
-  ModelQueryUtil, ModelQuerySuggestUtil, ModelQueryCrudUtil,
-  type ModelQueryFacet,
+  type ModelQuery, type ModelQueryCrudSupport, type ModelQueryFacetSupport, type ModelQuerySupport, type PageableModelQuery,
+  type Query, type ValidStringFields, QueryVerifier, type ModelQuerySuggestSupport, ModelQueryUtil, ModelQuerySuggestUtil,
+  ModelQueryCrudUtil, type ModelQueryFacet,
 } from '@travetto/model-query';
+import {
+  type ModelIndexedSupport, type KeyedIndexSelection, type KeyedIndexBody, type ListPageOptions, ModelIndexedUtil,
+  type SingleItemIndex, type SortedIndexSelection, type ListPageResult, type SortedIndex, type FullKeyedIndexBody,
+  type FullKeyedIndexWithPartialBody, ModelIndexedComputedIndex
+} from '@travetto/model-indexed';
 
 import type { ElasticsearchModelConfig } from './config.ts';
 import type { EsBulkError } from './internal/types.ts';
@@ -48,6 +51,57 @@ export class ElasticsearchModelService implements
 
   constructor(config: ElasticsearchModelConfig) { this.config = config; }
 
+  async * #scrollIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+    options?: ListPageOptions<estypes.SortResults>
+  ): AsyncIterable<{
+    hits: estypes.SearchResponse<T>['hits']['hits'];
+    nextOffset?: estypes.SortResults | undefined;
+  }> {
+    const limit = options?.limit ?? 100;
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate();
+
+    let search = await this.execSearch<T>(cls, {
+      ...(options?.offset ?
+        { size: limit, search_after: options.offset } :
+        { scroll: '2m', size: 100 }
+      ),
+      query: ElasticsearchQueryUtil.getSearchQuery(cls,
+        ElasticsearchQueryUtil.extractWhereTermQuery(cls, computed.project())
+      ),
+      sort: ElasticsearchQueryUtil.getSort(idx)
+    });
+
+    let hits = search.hits.hits.slice(0, limit);
+    let produced = hits.length;
+
+    if (hits.length) {
+      yield { hits, nextOffset: hits.at(-1)?.sort };
+    }
+
+    while (produced < limit && search._scroll_id && hits.length) {
+      search = await this.client.scroll({ scroll_id: search._scroll_id, scroll: '2m' });
+      hits = search.hits.hits;
+      produced += hits.length;
+      if (produced > limit) {
+        hits = hits.slice(0, limit - (produced - hits.length));
+      }
+      if (hits.length) {
+        yield { hits, nextOffset: hits.at(-1)?.sort };
+      }
+    }
+
+    if (search._scroll_id) {
+      await this.client.clearScroll({ scroll_id: search._scroll_id }).catch(() => { });
+    }
+  }
+
   @PostConstruct()
   async initializeClient(this: ElasticsearchModelService): Promise<void> {
     this.client = new Client({
@@ -354,69 +408,128 @@ export class ElasticsearchModelService implements
   }
 
   // Indexed
-  async getByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<T> {
-    const { key } = ModelIndexedUtil.computeIndexKey(cls, idx, body);
+  async getByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: FullKeyedIndexBody<T, K, S>): Promise<T> {
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate({ sort: true });
+
     const result = await this.execSearch<T>(cls, {
       query: ElasticsearchQueryUtil.getSearchQuery(cls,
-        ElasticsearchQueryUtil.extractWhereTermQuery(cls,
-          ModelIndexedUtil.projectIndex(cls, idx, body))
+        ElasticsearchQueryUtil.extractWhereTermQuery(cls, computed.project({
+          sort: true,
+          includeId: true,
+          emptyValue: { $exists: true }
+        }))
+
       )
     });
     if (!result.hits.hits.length) {
-      throw new NotFoundError(`${cls.name}: ${idx}`, key);
+      throw new NotFoundError(`${cls.name}: ${idx}`, computed.getKey({ sort: true }));
     }
     return this.postLoad(cls, result.hits.hits[0]);
+
   }
 
-  async deleteByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<void> {
-    const { key } = ModelIndexedUtil.computeIndexKey(cls, idx, body);
+  async deleteByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: FullKeyedIndexBody<T, K, S>): Promise<void> {
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate({ sort: true });
     const result = await this.client.deleteByQuery({
       index: this.manager.getIdentity(cls).index,
       query: ElasticsearchQueryUtil.getSearchQuery(cls,
-        ElasticsearchQueryUtil.extractWhereTermQuery(cls,
-          ModelIndexedUtil.projectIndex(cls, idx, body))
+        ElasticsearchQueryUtil.extractWhereTermQuery(cls, computed.project({
+          sort: true,
+          includeId: true,
+          emptyValue: { $exists: true }
+        }))
       ),
       refresh: true
     });
-    if (result.deleted) {
-      return;
+    if (!result.deleted) {
+      throw new NotFoundError(`${cls.name}: ${idx}`, computed.getKey({ sort: true }));
     }
-    throw new NotFoundError(`${cls.name}: ${idx}`, key);
   }
 
-  async upsertByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: OptionalId<T>): Promise<T> {
+  upsertByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: OptionalId<T>): Promise<T> {
     return ModelIndexedUtil.naiveUpsert(this, cls, idx, body);
   }
 
-  async * listByIndex<T extends ModelType>(cls: Class<T>, idx: string, body?: DeepPartial<T>): AsyncIterable<T> {
-    const config = ModelRegistryIndex.getIndex(cls, idx, ['sorted', 'unsorted']);
-    let search = await this.execSearch<T>(cls, {
-      scroll: '2m',
-      size: 100,
-      query: ElasticsearchQueryUtil.getSearchQuery(cls,
-        ElasticsearchQueryUtil.extractWhereTermQuery(cls,
-          ModelIndexedUtil.projectIndex(cls, idx, body, { emptySortValue: { $exists: true } }))
-      ),
-      sort: ElasticsearchQueryUtil.getSort(config.fields)
-    });
+  updateByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: T): Promise<T> {
+    return ModelIndexedUtil.naiveUpdate(this, cls, idx, body);
+  }
 
-    while (search.hits.hits.length > 0) {
-      for (const hit of search.hits.hits) {
+  async updatePartialByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: FullKeyedIndexWithPartialBody<T, K, S>): Promise<T> {
+    const item = await ModelCrudUtil.naivePartialUpdate(cls, () => this.getByIndex(cls, idx, castTo(body)), castTo(body));
+    return this.update(cls, item);
+  }
+
+  async pageByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+    options?: ListPageOptions,
+  ): Promise<ListPageResult<T>> {
+    const offset = options?.offset ? JSONUtil.fromBase64<estypes.SortResults>(options.offset) : undefined;
+    const items: T[] = [];
+    let lastNextOffset: estypes.SortResults | undefined;
+    for await (const { hits, nextOffset } of this.#scrollIndex(cls, idx, body, { ...options, offset })) {
+      lastNextOffset = nextOffset;
+      for (const hit of hits) {
         try {
-          yield this.postLoad(cls, hit);
+          items.push(await this.postLoad(cls, hit));
+        } catch (error) {
+          if (!(error instanceof NotFoundError)) {
+            throw error;
+          }
+        }
+      }
+    }
+    return { items, nextOffset: lastNextOffset ? JSONUtil.toBase64(lastNextOffset) : undefined };
+  }
+
+  async * listByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+  ): AsyncIterable<T> {
+    for await (const { hits } of this.#scrollIndex(cls, idx, body, { limit: Number.MAX_SAFE_INTEGER })) {
+      for (const hit of hits) {
+        try {
+          yield await this.postLoad(cls, hit);
         } catch (error) {
           if (!(error instanceof NotFoundError)) {
             throw error;
           }
         }
-        search = await this.client.scroll({
-          scroll_id: search._scroll_id,
-          scroll: '2m'
-        });
       }
     }
   }
 
+
   // Query
   async query<T extends ModelType>(cls: Class<T>, query: PageableModelQuery<T>): Promise<T[]> {
     await QueryVerifier.verify(cls, query);