@travetto/model-elasticsearch 5.0.18 → 5.0.20

package/README.md

@@ -13,7 +13,7 @@ npm install @travetto/model-elasticsearch
 yarn add @travetto/model-elasticsearch
 ```
 
-This module provides an [elasticsearch](https://elastic.co)-based implementation of the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.").  This source allows the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.") module to read, write and query against [elasticsearch](https://elastic.co). In development mode, [ElasticsearchModelService](https://github.com/travetto/travetto/tree/main/module/model-elasticsearch/src/service.ts#L42) will also modify the [elasticsearch](https://elastic.co) schema in real time to minimize impact to development. 
+This module provides an [elasticsearch](https://elastic.co)-based implementation of the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.").  This source allows the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.") module to read, write and query against [elasticsearch](https://elastic.co). In development mode, [ElasticsearchModelService](https://github.com/travetto/travetto/tree/main/module/model-elasticsearch/src/service.ts#L37) will also modify the [elasticsearch](https://elastic.co) schema in real time to minimize impact to development. 
 
 Supported features:
    *  [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/service/crud.ts#L11)
@@ -76,6 +76,11 @@ export class ElasticsearchModelConfig {
    * Auto-create, disabled in prod by default
    */
   autoCreate?: boolean;
+  /**
+   * Should we store the id as a string in the document
+   */
+  storeId?: boolean;
+
   /**
    * Base schema config for elasticsearch
    */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model-elasticsearch",
-  "version": "5.0.18",
+  "version": "5.0.20",
   "description": "Elasticsearch backing for the travetto model module, with real-time modeling support for Elasticsearch mappings.",
   "keywords": [
     "elasticsearch",
@@ -28,10 +28,10 @@
   },
   "dependencies": {
     "@elastic/elasticsearch": "^8.17.0",
-    "@travetto/cli": "^5.0.17",
-    "@travetto/config": "^5.0.14",
-    "@travetto/model": "^5.0.15",
-    "@travetto/model-query": "^5.0.15"
+    "@travetto/cli": "^5.0.19",
+    "@travetto/config": "^5.0.16",
+    "@travetto/model": "^5.0.17",
+    "@travetto/model-query": "^5.0.17"
   },
   "travetto": {
     "displayName": "Elasticsearch Model Source"

package/src/config.ts

@@ -29,6 +29,11 @@ export class ElasticsearchModelConfig {
    * Auto-create, disabled in prod by default
    */
   autoCreate?: boolean;
+  /**
+   * Should we store the id as a string in the document
+   */
+  storeId?: boolean;
+
   /**
    * Base schema config for elasticsearch
    */

package/src/index-manager.ts

@@ -1,5 +1,4 @@
-import { Client } from '@elastic/elasticsearch';
-import { ReindexRequest } from '@elastic/elasticsearch/lib/api/types';
+import { Client, estypes } from '@elastic/elasticsearch';
 
 import { Class } from '@travetto/runtime';
 import { ModelRegistry, ModelType } from '@travetto/model';
@@ -164,7 +163,7 @@ export class IndexManager implements ModelStorageSupport {
 
       const allChange = removes.concat(fieldChanges);
 
-      const reindexBody: ReindexRequest = {
+      const reindexBody: estypes.ReindexRequest = {
         source: { index: curr },
         dest: { index: next },
         script: {

package/src/internal/query.ts

@@ -78,6 +78,10 @@ export class ElasticsearchQueryUtil {
         ((key === 'id' && !path) ? '_id' : `${path}${key}`) :
         `${path}${key}`;
 
+      const sPathQuery = (val: unknown): {} => (key === 'id' && !path) ?
+        { ids: { values: Array.isArray(val) ? val : [val] } } :
+        { [Array.isArray(val) ? 'terms' : 'term']: { [sPath]: val } };
+
       if (DataUtil.isPlainObject(top)) {
         const subKey = Object.keys(top)[0];
         if (!subKey.startsWith('$')) {
@@ -100,29 +104,19 @@ export class ElasticsearchQueryUtil {
               break;
             }
             case '$in': {
-              items.push({ terms: { [sPath]: Array.isArray(v) ? v : [v] } });
+              items.push(sPathQuery(Array.isArray(v) ? v : [v]));
               break;
             }
             case '$nin': {
-              items.push({
-                bool: {
-                  ['must_not']: [{
-                    terms: {
-                      [sPath]: Array.isArray(v) ? v : [v]
-                    }
-                  }]
-                }
-              });
+              items.push({ bool: { ['must_not']: [sPathQuery(Array.isArray(v) ? v : [v])] } });
               break;
             }
             case '$eq': {
-              items.push({ term: { [sPath]: v } });
+              items.push(sPathQuery(v));
               break;
             }
             case '$ne': {
-              items.push({
-                bool: { ['must_not']: [{ term: { [sPath]: v } }] }
-              });
+              items.push({ bool: { ['must_not']: [sPathQuery(v)] } });
               break;
             }
             case '$exists': {
@@ -183,11 +177,7 @@ export class ElasticsearchQueryUtil {
         }
         // Handle operations
       } else {
-        items.push({
-          [Array.isArray(top) ? 'terms' : 'term']: {
-            [(key === 'id' && !path) ? '_id' : `${path}${key}`]: top
-          }
-        });
+        items.push(sPathQuery(top));
       }
     }
     if (items.length === 1) {
@@ -281,31 +271,4 @@ export class ElasticsearchQueryUtil {
 
     return search;
   }
-
-
-  /**
-   * Safely load the data, excluding ids if needed
-   */
-  static cleanIdRemoval<T>(req: estypes.SearchRequest, results: estypes.SearchResponse<T>): T[] {
-    const out: T[] = [];
-
-    const toArr = <V>(x: V | V[] | undefined): V[] => (x ? (Array.isArray(x) ? x : [x]) : []);
-
-    // determine if id
-    const select = [
-      toArr(req._source_includes),
-      toArr(req._source_excludes)
-    ];
-    const includeId = select[0].includes('_id') || (select[0].length === 0 && !select[1].includes('_id'));
-
-    for (const r of results.hits.hits) {
-      const obj = r._source!;
-      if (includeId) {
-        castTo<{ _id: string }>(obj)._id = r._id!;
-      }
-      out.push(obj);
-    }
-
-    return out;
-  }
 }

package/src/internal/schema.ts

@@ -1,4 +1,4 @@
-import { InlineScript, MappingProperty, MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types';
+import { estypes } from '@elastic/elasticsearch';
 
 import { Class } from '@travetto/runtime';
 import { ModelRegistry } from '@travetto/model';
@@ -15,8 +15,8 @@ export class ElasticsearchSchemaUtil {
   /**
    * Build the update script for a given object
    */
-  static generateUpdateScript(o: Record<string, unknown>): InlineScript {
-    const out: InlineScript = {
+  static generateUpdateScript(o: Record<string, unknown>): estypes.Script {
+    const out: estypes.Script = {
       lang: 'painless',
       source: `
  for (entry in params.body.entrySet()) {
@@ -42,7 +42,7 @@ export class ElasticsearchSchemaUtil {
    * @param o
    * @returns
    */
-  static generateReplaceScript(o: Record<string, unknown>): InlineScript {
+  static generateReplaceScript(o: Record<string, unknown>): estypes.Script {
     return {
       lang: 'painless',
       source: 'ctx._source.clear(); ctx._source.putAll(params.body)',
@@ -53,7 +53,7 @@ export class ElasticsearchSchemaUtil {
   /**
    * Build one or more mappings depending on the polymorphic state
    */
-  static generateSchemaMapping(cls: Class, config?: EsSchemaConfig): MappingTypeMapping {
+  static generateSchemaMapping(cls: Class, config?: EsSchemaConfig): estypes.MappingTypeMapping {
     return ModelRegistry.get(cls).baseType ?
       this.generateAllMapping(cls, config) :
       this.generateSingleMapping(cls, config);
@@ -62,9 +62,9 @@ export class ElasticsearchSchemaUtil {
   /**
    * Generate all mappings
    */
-  static generateAllMapping(cls: Class, config?: EsSchemaConfig): MappingTypeMapping {
+  static generateAllMapping(cls: Class, config?: EsSchemaConfig): estypes.MappingTypeMapping {
     const allTypes = ModelRegistry.getClassesByBaseType(cls);
-    return allTypes.reduce<MappingTypeMapping>((acc, schemaCls) => {
+    return allTypes.reduce<estypes.MappingTypeMapping>((acc, schemaCls) => {
       DataUtil.deepAssign(acc, this.generateSingleMapping(schemaCls, config));
       return acc;
     }, { properties: {}, dynamic: false });
@@ -73,10 +73,10 @@ export class ElasticsearchSchemaUtil {
   /**
    * Build a mapping for a given class
    */
-  static generateSingleMapping<T>(cls: Class<T>, config?: EsSchemaConfig): MappingTypeMapping {
+  static generateSingleMapping<T>(cls: Class<T>, config?: EsSchemaConfig): estypes.MappingTypeMapping {
     const schema = SchemaRegistry.getViewSchema(cls);
 
-    const props: Record<string, MappingProperty> = {};
+    const props: Record<string, estypes.MappingProperty> = {};
 
     for (const field of schema.fields) {
       const conf = schema.schema[field];

package/src/service.ts

@@ -30,10 +30,6 @@ import { ElasticsearchQueryUtil } from './internal/query';
 import { ElasticsearchSchemaUtil } from './internal/schema';
 import { IndexManager } from './index-manager';
 
-type WithId<T> = T & { _id?: string };
-
-const isWithId = <T extends ModelType>(o: T): o is WithId<T> => !o && '_id' in o;
-
 /**
  * Elasticsearch model source.
  */
@@ -74,13 +70,34 @@ export class ElasticsearchModelService implements
     }
   }
 
+  preUpdate(o: { id: string }): string;
+  preUpdate(o: {}): undefined;
+  preUpdate(o: { id?: string }): string | undefined {
+    if ('id' in o && typeof o.id === 'string') {
+      const id = o.id;
+      if (!this.config.storeId) {
+        delete o.id;
+      }
+      return id;
+    }
+    return;
+  }
+
+  postUpdate<T extends ModelType>(o: T, id?: string): T {
+    if (!this.config.storeId) {
+      o.id = id!;
+    }
+    return o;
+  }
+
   /**
    * Convert _id to id
    */
-  async postLoad<T extends ModelType>(cls: Class<T>, item: T): Promise<T> {
-    if (isWithId(item)) {
-      delete item._id;
-    }
+  async postLoad<T extends ModelType>(cls: Class<T>, inp: estypes.SearchHit<T> | estypes.GetGetResult<T>): Promise<T> {
+    let item = {
+      ...(inp._id ? { id: inp._id } : {}),
+      ...inp._source!,
+    };
 
     item = await ModelCrudUtil.load(cls, item);
 
@@ -120,8 +137,8 @@ export class ElasticsearchModelService implements
 
   async get<T extends ModelType>(cls: Class<T>, id: string): Promise<T> {
     try {
-      const res = await this.client.get({ ...this.manager.getIdentity(cls), id });
-      return this.postLoad(cls, castTo(res._source));
+      const res = await this.client.get<T>({ ...this.manager.getIdentity(cls), id });
+      return this.postLoad(cls, res);
     } catch {
       throw new NotFoundError(cls, id);
     }
@@ -150,7 +167,7 @@ export class ElasticsearchModelService implements
   async create<T extends ModelType>(cls: Class<T>, o: OptionalId<T>): Promise<T> {
     try {
       const clean = await ModelCrudUtil.preStore(cls, o, this);
-      const id = clean.id;
+      const id = this.preUpdate(clean);
 
       await this.client.index({
         ...this.manager.getIdentity(cls),
@@ -159,7 +176,7 @@ export class ElasticsearchModelService implements
         body: clean
       });
 
-      return clean;
+      return this.postUpdate(clean, id);
     } catch (err) {
       console.error(err);
       throw err;
@@ -171,7 +188,7 @@ export class ElasticsearchModelService implements
 
     o = await ModelCrudUtil.preStore(cls, o, this);
 
-    const id = o.id;
+    const id = this.preUpdate(o);
 
     if (ModelRegistry.get(cls).expiresAt) {
       await this.get(cls, id);
@@ -185,18 +202,18 @@ export class ElasticsearchModelService implements
       body: o
     });
 
-    o.id = id;
-    return o;
+    return this.postUpdate(o, id);
   }
 
   async upsert<T extends ModelType>(cls: Class<T>, o: OptionalId<T>): Promise<T> {
     ModelCrudUtil.ensureNotSubType(cls);
 
     const item = await ModelCrudUtil.preStore(cls, o, this);
+    const id = this.preUpdate(item);
 
     await this.client.update({
       ...this.manager.getIdentity(cls),
-      id: item.id,
+      id,
       refresh: true,
       body: {
         doc: item,
@@ -204,18 +221,15 @@ export class ElasticsearchModelService implements
       }
     });
 
-    return item;
+    return this.postUpdate(item, id);
   }
 
   async updatePartial<T extends ModelType>(cls: Class<T>, data: Partial<T> & { id: string }, view?: string): Promise<T> {
     ModelCrudUtil.ensureNotSubType(cls);
 
-    const item = await ModelCrudUtil.prePartialUpdate(cls, data, view);
-
+    const id = data.id;
+    const item = castTo<typeof data>(await ModelCrudUtil.prePartialUpdate(cls, data, view));
     const script = ElasticsearchSchemaUtil.generateUpdateScript(item);
-    const id = item.id!;
-
-    console.debug('Partial Script', { script });
 
     try {
       await this.client.update({
@@ -246,7 +260,7 @@ export class ElasticsearchModelService implements
     while (search.hits.hits.length > 0) {
       for (const el of search.hits.hits) {
         try {
-          yield this.postLoad(cls, el._source!);
+          yield this.postLoad(cls, el);
         } catch (err) {
           if (!(err instanceof NotFoundError)) {
             throw err;
@@ -272,11 +286,14 @@ export class ElasticsearchModelService implements
       if (op.delete) {
         acc.push({ delete: { ...ident, _id: op.delete.id } });
       } else if (op.insert) {
-        acc.push({ create: { ...ident, _id: op.insert.id } }, castTo(op.insert));
+        const id = this.preUpdate(op.insert);
+        acc.push({ create: { ...ident, _id: id } }, castTo(op.insert));
       } else if (op.upsert) {
-        acc.push({ index: { ...ident, _id: op.upsert.id } }, castTo(op.upsert));
+        const id = this.preUpdate(op.upsert);
+        acc.push({ index: { ...ident, _id: id } }, castTo(op.upsert));
       } else if (op.update) {
-        acc.push({ update: { ...ident, _id: op.update.id } }, { doc: op.update });
+        const id = this.preUpdate(op.update);
+        acc.push({ update: { ...ident, _id: id } }, { doc: op.update });
       }
       return acc;
     }, []);
@@ -350,7 +367,7 @@ export class ElasticsearchModelService implements
     if (!res.hits.hits.length) {
       throw new NotFoundError(`${cls.name}: ${idx}`, key);
     }
-    return this.postLoad(cls, res.hits.hits[0]._source!);
+    return this.postLoad(cls, res.hits.hits[0]);
   }
 
   async deleteByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<void> {
@@ -388,7 +405,7 @@ export class ElasticsearchModelService implements
     while (search.hits.hits.length > 0) {
       for (const el of search.hits.hits) {
         try {
-          yield this.postLoad(cls, el._source!);
+          yield this.postLoad(cls, el);
         } catch (err) {
           if (!(err instanceof NotFoundError)) {
             throw err;
@@ -408,8 +425,13 @@ export class ElasticsearchModelService implements
 
     const req = ElasticsearchQueryUtil.getSearchObject(cls, query, this.config.schemaConfig);
     const results = await this.execSearch(cls, req);
-    const items = ElasticsearchQueryUtil.cleanIdRemoval(req, results);
-    return Promise.all(items.map(m => this.postLoad(cls, m)));
+    const shouldRemoveIds = query.select && 'id' in query.select && !query.select.id;
+    return Promise.all(results.hits.hits.map(m => this.postLoad(cls, m).then(v => {
+      if (shouldRemoveIds) {
+        delete castTo<OptionalId<T>>(v).id;
+      }
+      return v;
+    })));
   }
 
   async queryOne<T extends ModelType>(cls: Class<T>, query: ModelQuery<T>, failOnMany: boolean = true): Promise<T> {
@@ -431,10 +453,12 @@ export class ElasticsearchModelService implements
     await QueryVerifier.verify(cls, query);
 
     const item = await ModelCrudUtil.preStore(cls, data, this);
-    const id = item.id;
+    const id = this.preUpdate(item);
 
     const where = ModelQueryUtil.getWhereClause(cls, query.where);
-    where.id = item.id;
+    if (id) {
+      where.id = id;
+    }
     query.where = where;
 
     if (ModelRegistry.get(cls).expiresAt) {
@@ -465,7 +489,7 @@ export class ElasticsearchModelService implements
       }
     }
 
-    return item;
+    return this.postUpdate(item, id);
   }
 
   async deleteByQuery<T extends ModelType>(cls: Class<T>, query: ModelQuery<T> = {}): Promise<number> {
@@ -484,7 +508,6 @@ export class ElasticsearchModelService implements
     await QueryVerifier.verify(cls, query);
 
     const item = await ModelCrudUtil.prePartialUpdate(cls, data);
-
     const script = ElasticsearchSchemaUtil.generateUpdateScript(item);
 
     const search = ElasticsearchQueryUtil.getSearchObject(cls, query, this.config.schemaConfig);
@@ -505,9 +528,8 @@ export class ElasticsearchModelService implements
     const q = ModelQuerySuggestUtil.getSuggestQuery<T>(cls, field, prefix, query);
     const search = ElasticsearchQueryUtil.getSearchObject(cls, q);
     const res = await this.execSearch(cls, search);
-    const safe = ElasticsearchQueryUtil.cleanIdRemoval<T>(search, res);
-    const combined = ModelQuerySuggestUtil.combineSuggestResults(cls, field, prefix, safe, (x, v) => v, query && query.limit);
-    return Promise.all(combined.map(m => this.postLoad(cls, m)));
+    const all = await Promise.all(res.hits.hits.map(x => this.postLoad(cls, x)));
+    return ModelQuerySuggestUtil.combineSuggestResults(cls, field, prefix, all, (x, v) => v, query && query.limit);
   }
 
   async suggestValues<T extends ModelType>(cls: Class<T>, field: ValidStringFields<T>, prefix?: string, query?: PageableModelQuery<T>): Promise<string[]> {
@@ -519,8 +541,8 @@ export class ElasticsearchModelService implements
     });
     const search = ElasticsearchQueryUtil.getSearchObject(cls, q);
     const res = await this.execSearch(cls, search);
-    const safe = ElasticsearchQueryUtil.cleanIdRemoval(search, res);
-    return ModelQuerySuggestUtil.combineSuggestResults(cls, field, prefix, safe, x => x, query && query.limit);
+    const all = await Promise.all(res.hits.hits.map(x => castTo<T>(({ [field]: field === 'id' ? x._id : x._source![field] }))));
+    return ModelQuerySuggestUtil.combineSuggestResults(cls, field, prefix, all, x => x, query && query.limit);
   }
 
   // Facet