@travetto/model-elasticsearch 8.0.0-alpha.10 → 8.0.0-alpha.12

package/README.md

@@ -13,13 +13,13 @@ npm install @travetto/model-elasticsearch
 yarn add @travetto/model-elasticsearch
 ```
 
-This module provides an [elasticsearch](https://elastic.co)-based implementation of the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.").  This source allows the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.") module to read, write and query against [elasticsearch](https://elastic.co). In development mode, [ElasticsearchModelService](https://github.com/travetto/travetto/tree/main/module/model-elasticsearch/src/service.ts#L39) will also modify the [elasticsearch](https://elastic.co) schema in real time to minimize impact to development. 
+This module provides an [elasticsearch](https://elastic.co)-based implementation of the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.").  This source allows the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.") module to read, write and query against [elasticsearch](https://elastic.co). In development mode, [ElasticsearchModelService](https://github.com/travetto/travetto/tree/main/module/model-elasticsearch/src/service.ts#L40) will also modify the [elasticsearch](https://elastic.co) schema in real time to minimize impact to development. 
 
 Supported features:
    *  [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/types/crud.ts#L11)
    *  [Bulk](https://github.com/travetto/travetto/tree/main/module/model/src/types/bulk.ts#L64)
    *  [Expiry](https://github.com/travetto/travetto/tree/main/module/model/src/types/expiry.ts#L10)
-   *  [Indexed](https://github.com/travetto/travetto/tree/main/module/model-indexed/src/types/service.ts#L23)
+   *  [Indexed](https://github.com/travetto/travetto/tree/main/module/model-indexed/src/types/service.ts#L15)
    *  [Query Crud](https://github.com/travetto/travetto/tree/main/module/model-query/src/types/crud.ts#L11)
    *  [Facet](https://github.com/travetto/travetto/tree/main/module/model-query/src/types/facet.ts#L14)
    *  [Query](https://github.com/travetto/travetto/tree/main/module/model-query/src/types/query.ts#L10)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model-elasticsearch",
-  "version": "8.0.0-alpha.10",
+  "version": "8.0.0-alpha.12",
   "type": "module",
   "description": "Elasticsearch backing for the travetto model module, with real-time modeling support for Elasticsearch mappings.",
   "keywords": [
@@ -29,13 +29,13 @@
   },
   "dependencies": {
     "@elastic/elasticsearch": "^9.3.4",
-    "@travetto/config": "^8.0.0-alpha.10",
-    "@travetto/model": "^8.0.0-alpha.10",
-    "@travetto/model-indexed": "^8.0.0-alpha.10",
-    "@travetto/model-query": "^8.0.0-alpha.10"
+    "@travetto/config": "^8.0.0-alpha.11",
+    "@travetto/model": "^8.0.0-alpha.11",
+    "@travetto/model-indexed": "^8.0.0-alpha.12",
+    "@travetto/model-query": "^8.0.0-alpha.11"
   },
   "peerDependencies": {
-    "@travetto/cli": "^8.0.0-alpha.15"
+    "@travetto/cli": "^8.0.0-alpha.16"
   },
   "peerDependenciesMeta": {
     "@travetto/cli": {

package/src/service.ts

@@ -4,7 +4,8 @@ import type * as estypes from '@elastic/elasticsearch/api/types';
 import {
   type ModelCrudSupport, type BulkOperation, type BulkResponse, type ModelBulkSupport, type ModelExpirySupport,
   type ModelType, type ModelStorageSupport, NotFoundError, ModelRegistryIndex, type OptionalId,
-  ModelCrudUtil, ModelStorageUtil, ModelExpiryUtil, ModelBulkUtil
+  ModelCrudUtil, ModelStorageUtil, ModelExpiryUtil, ModelBulkUtil,
+  type ModelListOptions
 } from '@travetto/model';
 import { ShutdownManager, type Class, castTo, asFull, TypedObject, asConstructable, JSONUtil } from '@travetto/runtime';
 import { BindUtil } from '@travetto/schema';
@@ -15,8 +16,8 @@ import {
   ModelQueryCrudUtil, type ModelQueryFacet,
 } from '@travetto/model-query';
 import {
-  type ModelIndexedSupport, type KeyedIndexSelection, type KeyedIndexBody, type ListPageOptions, ModelIndexedUtil,
-  type SingleItemIndex, type SortedIndexSelection, type ListPageResult, type SortedIndex, type FullKeyedIndexBody,
+  type ModelIndexedSupport, type KeyedIndexSelection, type KeyedIndexBody, type ModelPageOptions, ModelIndexedUtil,
+  type SingleItemIndex, type SortedIndexSelection, type ModelPageResult, type SortedIndex, type FullKeyedIndexBody,
   type FullKeyedIndexWithPartialBody, ModelIndexedComputedIndex
 } from '@travetto/model-indexed';
 
@@ -51,6 +52,73 @@ export class ElasticsearchModelService implements
 
   constructor(config: ElasticsearchModelConfig) { this.config = config; }
 
+  /**
+ * Convert _id to id
+ */
+  async #postLoad<T extends ModelType>(cls: Class<T>, input: estypes.SearchHit<T> | estypes.GetGetResult<T>): Promise<T> {
+    let item = {
+      ...(input._id ? { id: input._id } : {}),
+      ...input._source!,
+    };
+
+    item = await ModelCrudUtil.load(cls, item);
+
+    const { expiresAt } = ModelRegistryIndex.getConfig(cls);
+
+    if (expiresAt) {
+      const expiry = ModelExpiryUtil.getExpiryState(cls, item);
+      if (!expiry.expired) {
+        return item;
+      }
+      throw new NotFoundError(cls, item.id);
+    } else {
+      return item;
+    }
+  }
+
+  async * #scrollCollection<T extends ModelType>(
+    cls: Class<T>,
+    buildSearch: (offset?: estypes.SortResults) => estypes.SearchRequest,
+    options?: ModelPageOptions<estypes.SortResults> & ModelListOptions
+  ): AsyncIterable<{ items: T[], nextOffset?: estypes.SortResults | undefined }> {
+    const limit = options?.limit ?? Number.MAX_SAFE_INTEGER;
+    const batchSize = Math.min(limit, options?.batchSizeHint ?? 100);
+
+    let search: estypes.SearchResponse<T> = await this.execSearch<T>(cls, {
+      size: batchSize,
+      ...buildSearch(options?.offset),
+    });
+
+    let hits = search.hits.hits;
+    let produced = 0;
+
+    while (produced < limit && hits.length && !(options?.abort?.aborted)) {
+      hits = search.hits.hits;
+      produced += hits.length;
+      if (produced > limit) {
+        hits = hits.slice(0, limit - (produced - hits.length));
+      }
+      const items = await ModelCrudUtil.filterOutNotFound(
+        hits.map(hit => this.#postLoad(cls, hit))
+      );
+      const nextOffset = hits.at(-1)?.sort;
+      yield { items, nextOffset };
+      if (search._scroll_id) {
+        search = await this.client.scroll({ scroll_id: search._scroll_id });
+      } else if (nextOffset) {
+        search = await this.execSearch<T>(cls, {
+          size: batchSize,
+          ...buildSearch(nextOffset),
+        });
+      } else {
+        hits = [];
+      }
+    }
+    if (search._scroll_id) {
+      await this.client.clearScroll({ scroll_id: search._scroll_id }).catch(() => { });
+    }
+  }
+
   async * #scrollIndex<
     T extends ModelType,
     K extends KeyedIndexSelection<T>,
@@ -59,47 +127,19 @@ export class ElasticsearchModelService implements
     cls: Class<T>,
     idx: SortedIndex<T, K, S>,
     body: KeyedIndexBody<T, K>,
-    options?: ListPageOptions<estypes.SortResults>
+    options?: ModelPageOptions<estypes.SortResults> & ModelListOptions
   ): AsyncIterable<{
-    hits: estypes.SearchResponse<T>['hits']['hits'];
+    items: T[];
     nextOffset?: estypes.SortResults | undefined;
   }> {
-    const limit = options?.limit ?? 100;
     const computed = ModelIndexedComputedIndex.get(idx, body).validate();
-
-    let search = await this.execSearch<T>(cls, {
-      ...(options?.offset ?
-        { size: limit, search_after: options.offset } :
-        { scroll: '2m', size: 100 }
-      ),
+    yield* this.#scrollCollection(cls, (offset) => ({
       query: ElasticsearchQueryUtil.getSearchQuery(cls,
         ElasticsearchQueryUtil.extractWhereTermQuery(cls, computed.project())
       ),
+      search_after: offset,
       sort: ElasticsearchQueryUtil.getSort(idx)
-    });
-
-    let hits = search.hits.hits.slice(0, limit);
-    let produced = hits.length;
-
-    if (hits.length) {
-      yield { hits, nextOffset: hits.at(-1)?.sort };
-    }
-
-    while (produced < limit && search._scroll_id && hits.length) {
-      search = await this.client.scroll({ scroll_id: search._scroll_id, scroll: '2m' });
-      hits = search.hits.hits;
-      produced += hits.length;
-      if (produced > limit) {
-        hits = hits.slice(0, limit - (produced - hits.length));
-      }
-      if (hits.length) {
-        yield { hits, nextOffset: hits.at(-1)?.sort };
-      }
-    }
-
-    if (search._scroll_id) {
-      await this.client.clearScroll({ scroll_id: search._scroll_id }).catch(() => { });
-    }
+    }), options);
   }
 
   @PostConstruct()
@@ -163,30 +203,6 @@ export class ElasticsearchModelService implements
     return item;
   }
 
-  /**
-   * Convert _id to id
-   */
-  async postLoad<T extends ModelType>(cls: Class<T>, input: estypes.SearchHit<T> | estypes.GetGetResult<T>): Promise<T> {
-    let item = {
-      ...(input._id ? { id: input._id } : {}),
-      ...input._source!,
-    };
-
-    item = await ModelCrudUtil.load(cls, item);
-
-    const { expiresAt } = ModelRegistryIndex.getConfig(cls);
-
-    if (expiresAt) {
-      const expiry = ModelExpiryUtil.getExpiryState(cls, item);
-      if (!expiry.expired) {
-        return item;
-      }
-      throw new NotFoundError(cls, item.id);
-    } else {
-      return item;
-    }
-  }
-
   createStorage(): Promise<void> { return this.manager.createStorage(); }
   deleteStorage(): Promise<void> { return this.manager.deleteStorage(); }
   upsertModel(cls: Class): Promise<void> { return this.manager.upsertModel(cls); }
@@ -197,7 +213,7 @@ export class ElasticsearchModelService implements
   async get<T extends ModelType>(cls: Class<T>, id: string): Promise<T> {
     try {
       const result = await this.client.get<T>({ ...this.manager.getIdentity(cls), id });
-      return this.postLoad(cls, result);
+      return this.#postLoad(cls, result);
     } catch {
       throw new NotFoundError(cls, id);
     }
@@ -300,27 +316,12 @@ export class ElasticsearchModelService implements
     return this.get(cls, id);
   }
 
-  async * list<T extends ModelType>(cls: Class<T>): AsyncIterable<T> {
-    let search: estypes.SearchResponse<T> = await this.execSearch<T>(cls, {
-      scroll: '2m',
-      size: 100,
-      query: ElasticsearchQueryUtil.getSearchQuery(cls, {})
-    });
-
-    while (search.hits.hits.length > 0) {
-      for (const hit of search.hits.hits) {
-        try {
-          yield this.postLoad(cls, hit);
-        } catch (error) {
-          if (!(error instanceof NotFoundError)) {
-            throw error;
-          }
-        }
-        search = await this.client.scroll({
-          scroll_id: search._scroll_id,
-          scroll: '2m'
-        });
-      }
+  async * list<T extends ModelType>(cls: Class<T>, options?: ModelListOptions): AsyncIterable<T[]> {
+    for await (const batch of this.#scrollCollection(cls, () => ({
+      query: ElasticsearchQueryUtil.getSearchQuery(cls, {}),
+      scroll: '2m'
+    }), options)) {
+      yield batch.items;
     }
   }
 
@@ -419,6 +420,7 @@ export class ElasticsearchModelService implements
       query: ElasticsearchQueryUtil.getSearchQuery(cls,
         ElasticsearchQueryUtil.extractWhereTermQuery(cls, computed.project({
           sort: true,
+          includeId: true,
           emptyValue: { $exists: true }
         }))
 
@@ -427,7 +429,7 @@ export class ElasticsearchModelService implements
     if (!result.hits.hits.length) {
       throw new NotFoundError(`${cls.name}: ${idx}`, computed.getKey({ sort: true }));
     }
-    return this.postLoad(cls, result.hits.hits[0]);
+    return this.#postLoad(cls, result.hits.hits[0]);
 
   }
 
@@ -442,6 +444,7 @@ export class ElasticsearchModelService implements
       query: ElasticsearchQueryUtil.getSearchQuery(cls,
         ElasticsearchQueryUtil.extractWhereTermQuery(cls, computed.project({
           sort: true,
+          includeId: true,
           emptyValue: { $exists: true }
         }))
       ),
@@ -477,7 +480,7 @@ export class ElasticsearchModelService implements
     return this.update(cls, item);
   }
 
-  async listByIndex<
+  async pageByIndex<
     T extends ModelType,
     K extends KeyedIndexSelection<T>,
     S extends SortedIndexSelection<T>
@@ -485,26 +488,32 @@ export class ElasticsearchModelService implements
     cls: Class<T>,
     idx: SortedIndex<T, K, S>,
     body: KeyedIndexBody<T, K>,
-    options?: ListPageOptions,
-  ): Promise<ListPageResult<T>> {
+    options?: ModelPageOptions,
+  ): Promise<ModelPageResult<T>> {
     const offset = options?.offset ? JSONUtil.fromBase64<estypes.SortResults>(options.offset) : undefined;
     const items: T[] = [];
     let lastNextOffset: estypes.SortResults | undefined;
-    for await (const { hits, nextOffset } of this.#scrollIndex(cls, idx, body, { ...options, offset })) {
+    for await (const { items: fetched, nextOffset } of this.#scrollIndex(cls, idx, body, { limit: 100, ...options, offset })) {
       lastNextOffset = nextOffset;
-      for (const hit of hits) {
-        try {
-          items.push(await this.postLoad(cls, hit));
-        } catch (error) {
-          if (!(error instanceof NotFoundError)) {
-            throw error;
-          }
-        }
-      }
+      items.push(...fetched);
     }
     return { items, nextOffset: lastNextOffset ? JSONUtil.toBase64(lastNextOffset) : undefined };
   }
 
+  async * listByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+    options?: ModelListOptions
+  ): AsyncIterable<T[]> {
+    for await (const { items } of this.#scrollIndex(cls, idx, body, options)) {
+      yield items;
+    }
+  }
 
   // Query
   async query<T extends ModelType>(cls: Class<T>, query: PageableModelQuery<T>): Promise<T[]> {
@@ -513,7 +522,7 @@ export class ElasticsearchModelService implements
     const search = ElasticsearchQueryUtil.getSearchObject(cls, query, this.config.schemaConfig);
     const results = await this.execSearch(cls, search);
     const shouldRemoveIds = query.select && 'id' in query.select && !query.select.id;
-    return Promise.all(results.hits.hits.map(hit => this.postLoad(cls, hit).then(item => {
+    return Promise.all(results.hits.hits.map(hit => this.#postLoad(cls, hit).then(item => {
       if (shouldRemoveIds) {
         delete castTo<OptionalId<T>>(item).id;
       }
@@ -615,7 +624,7 @@ export class ElasticsearchModelService implements
     const resolvedQuery = ModelQuerySuggestUtil.getSuggestQuery<T>(cls, field, prefix, query);
     const search = ElasticsearchQueryUtil.getSearchObject(cls, resolvedQuery);
     const result = await this.execSearch(cls, search);
-    const all = await Promise.all(result.hits.hits.map(hit => this.postLoad(cls, hit)));
+    const all = await Promise.all(result.hits.hits.map(hit => this.#postLoad(cls, hit)));
     return ModelQuerySuggestUtil.combineSuggestResults(cls, field, prefix, all, (_, value) => value, query && query.limit);
   }