@travetto/model-elasticsearch 7.0.0-rc.2 → 7.0.0-rc.4

package/README.md

@@ -65,9 +65,9 @@ export class ElasticsearchModelConfig {
    */
   namespace = 'app';
   /**
-   * Auto-create, disabled in prod by default
+   * Allow storage modifification
    */
-  autoCreate?: boolean;
+  modifyStorage?: boolean;
   /**
    * Should we store the id as a string in the document
    */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model-elasticsearch",
-  "version": "7.0.0-rc.2",
+  "version": "7.0.0-rc.4",
   "description": "Elasticsearch backing for the travetto model module, with real-time modeling support for Elasticsearch mappings.",
   "keywords": [
     "elasticsearch",
@@ -28,10 +28,10 @@
   },
   "dependencies": {
     "@elastic/elasticsearch": "^9.2.0",
-    "@travetto/cli": "^7.0.0-rc.2",
-    "@travetto/config": "^7.0.0-rc.2",
-    "@travetto/model": "^7.0.0-rc.2",
-    "@travetto/model-query": "^7.0.0-rc.2"
+    "@travetto/cli": "^7.0.0-rc.4",
+    "@travetto/config": "^7.0.0-rc.4",
+    "@travetto/model": "^7.0.0-rc.4",
+    "@travetto/model-query": "^7.0.0-rc.4"
   },
   "travetto": {
     "displayName": "Elasticsearch Model Source"

package/src/config.ts

@@ -25,9 +25,9 @@ export class ElasticsearchModelConfig {
    */
   namespace = 'app';
   /**
-   * Auto-create, disabled in prod by default
+   * Allow storage modifification
    */
-  autoCreate?: boolean;
+  modifyStorage?: boolean;
   /**
    * Should we store the id as a string in the document
    */

package/src/index-manager.ts

@@ -2,7 +2,6 @@ import { Client, estypes } from '@elastic/elasticsearch';
 
 import { Class } from '@travetto/runtime';
 import { ModelRegistryIndex, ModelType, ModelStorageSupport } from '@travetto/model';
-import { SchemaChange, SchemaRegistryIndex } from '@travetto/schema';
 
 import { ElasticsearchModelConfig } from './config.ts';
 import { ElasticsearchSchemaUtil } from './internal/schema.ts';
@@ -12,8 +11,6 @@ import { ElasticsearchSchemaUtil } from './internal/schema.ts';
  */
 export class IndexManager implements ModelStorageSupport {
 
-  #indexToAlias = new Map<string, string>();
-  #aliasToIndex = new Map<string, string>();
   #identities = new Map<Class, { index: string }>();
   #client: Client;
   config: ElasticsearchModelConfig;
@@ -24,7 +21,7 @@ export class IndexManager implements ModelStorageSupport {
   }
 
   getStore(cls: Class): string {
-    return ModelRegistryIndex.getStoreName(cls).toLowerCase().replace(/[^A-Za-z0-9_]+/g, '_');
+    return ModelRegistryIndex.getStoreName(cls);
   }
 
   /**
@@ -51,22 +48,6 @@ export class IndexManager implements ModelStorageSupport {
     return { ...this.#identities.get(cls)! };
   }
 
-  /**
-   * Build alias mappings from the current state in the database
-   */
-  async computeAliasMappings(force = false): Promise<void> {
-    if (force || !this.#indexToAlias.size) {
-      const aliases = await this.#client.cat.aliases({ format: 'json' });
-
-      this.#indexToAlias = new Map();
-      this.#aliasToIndex = new Map();
-      for (const al of aliases) {
-        this.#indexToAlias.set(al.index!, al.alias!);
-        this.#aliasToIndex.set(al.alias!, al.index!);
-      }
-    }
-  }
-
   /**
    * Create index for type
    * @param cls
@@ -94,24 +75,6 @@ export class IndexManager implements ModelStorageSupport {
     return concreteIndex;
   }
 
-  /**
-   * Build an index if missing
-   */
-  async createIndexIfMissing(cls: Class): Promise<void> {
-    const baseCls = SchemaRegistryIndex.getBaseClass(cls);
-    const identity = this.getIdentity(baseCls);
-    try {
-      await this.#client.search(identity);
-    } catch {
-      await this.createIndex(baseCls);
-    }
-  }
-
-  async createModel(cls: Class<ModelType>): Promise<void> {
-    await this.createIndexIfMissing(cls);
-    await this.computeAliasMappings(true);
-  }
-
   async exportModel(cls: Class<ModelType>): Promise<string> {
     const schema = ElasticsearchSchemaUtil.generateSchemaMapping(cls, this.config.schemaConfig);
     const { index } = this.getIdentity(cls); // Already namespaced
@@ -122,78 +85,57 @@ export class IndexManager implements ModelStorageSupport {
   }
 
   async deleteModel(cls: Class<ModelType>): Promise<void> {
-    const alias = this.getNamespacedIndex(this.getStore(cls));
-    if (this.#aliasToIndex.get(alias)) {
-      await this.#client.indices.delete({
-        index: this.#aliasToIndex.get(alias)!
-      });
-      await this.computeAliasMappings(true);
-    }
+    const { index } = this.getIdentity(cls);
+    const aliasedIndices = await this.#client.indices.getAlias();
+
+    const toDelete = Object.keys(aliasedIndices[index]?.aliases ?? {})
+      .filter(item => index in (aliasedIndices[item]?.aliases ?? {}));
+
+    console.debug('Deleting Model', { index, toDelete });
+    await Promise.all(toDelete.map(target => this.#client.indices.delete({ index: target })));
   }
 
   /**
-   * When the schema changes
+   * Create or update schema as necessary
    */
-  async changeSchema(cls: Class, change: SchemaChange): Promise<void> {
-    // Find which fields are gone
-    const removes = change.subs.reduce<string[]>((toRemove, subChange) => {
-      toRemove.push(...subChange.fields
-        .filter(event => event.type === 'removing')
-        .map(event => [...subChange.path.map(field => field.name), event.previous!.name].join('.')));
-      return toRemove;
-    }, []);
-
-    // Find which types have changed
-    const fieldChanges = change.subs.reduce<string[]>((toChange, subChange) => {
-      toChange.push(...subChange.fields
-        .filter(event => event.type === 'changed')
-        .filter(event => event.previous?.type !== event.current?.type)
-        .map(event => [...subChange.path.map(field => field.name), event.previous!.name].join('.')));
-      return toChange;
-    }, []);
-
+  async upsertModel(cls: Class<ModelType>): Promise<void> {
     const { index } = this.getIdentity(cls);
-
-    // If removing fields or changing types, run as script to update data
-    if (removes.length || fieldChanges.length) { // Removing and adding
-      const next = await this.createIndex(cls, false);
-
-      const aliases = (await this.#client.indices.getAlias({ index })).body;
-      const current = Object.keys(aliases)[0];
-
-      const allChange = removes.concat(fieldChanges);
-
-      const reindexBody: estypes.ReindexRequest = {
-        source: { index: current },
-        dest: { index: next },
-        script: {
-          lang: 'painless',
-          source: allChange.map(part => `ctx._source.remove("${part}");`).join(' ') // Removing
-        },
-        wait_for_completion: true
-      };
-
-      // Reindex
-      await this.#client.reindex(reindexBody);
-
-      await Promise.all(Object.keys(aliases)
-        .map(alias => this.#client.indices.delete({ index: alias })));
-
-      await this.#client.indices.putAlias({ index: next, name: index });
-    } else { // Only update the schema
-      const schema = ElasticsearchSchemaUtil.generateSchemaMapping(cls, this.config.schemaConfig);
-
-      await this.#client.indices.putMapping({
-        index,
-        ...schema,
-      });
+    const resolvedAlias = await this.#client.indices.getMapping({ index }).catch(() => undefined);
+
+    if (resolvedAlias) {
+      const [currentIndex] = Object.keys(resolvedAlias ?? {});
+      const pendingMapping = ElasticsearchSchemaUtil.generateSchemaMapping(cls, this.config.schemaConfig);
+      const changedFields = ElasticsearchSchemaUtil.getChangedFields(resolvedAlias[currentIndex].mappings, pendingMapping);
+
+      if (changedFields.length) { // If any fields changed, reindex
+        console.debug('Updated Model', { index, currentIndex, changedFields });
+        const pendingIndex = await this.createIndex(cls, false);
+
+        const reindexBody: estypes.ReindexRequest = {
+          source: { index: currentIndex },
+          dest: { index: pendingIndex },
+          script: {
+            lang: 'painless',
+            source: changedFields.map(change => `ctx._source.remove("${change}");`).join(' ') // Removing
+          },
+          wait_for_completion: true
+        };
+
+        await this.#client.reindex(reindexBody);
+
+        // Update aliases
+        await this.#client.indices.putAlias({ index: pendingIndex, name: index });
+        const toDelete = Object.keys(resolvedAlias).filter(item => item !== pendingIndex);
+        await Promise.all(toDelete.map(alias => this.#client.indices.delete({ index: alias })));
+      }
+    } else { // Create if non-existent
+      console.debug('Creating Model', { index });
+      await this.createIndex(cls);
     }
   }
 
   async createStorage(): Promise<void> {
-    // Pre-create indexes if missing
     console.debug('Create Storage', { idx: this.getNamespacedIndex('*') });
-    await this.computeAliasMappings(true);
   }
 
   async deleteStorage(): Promise<void> {
@@ -201,6 +143,5 @@ export class IndexManager implements ModelStorageSupport {
     await this.#client.indices.delete({
       index: this.getNamespacedIndex('*')
     });
-    await this.computeAliasMappings(true);
   }
 }

package/src/internal/schema.ts

@@ -7,6 +7,9 @@ import { EsSchemaConfig } from './types.ts';
 
 const PointConcrete = toConcrete<Point>();
 
+const isMappingType = (input: estypes.MappingProperty): input is estypes.MappingTypeMapping =>
+  (input.type === 'object' || input.type === 'nested') && 'properties' in input && !!input.properties;
+
 /**
  * Utils for ES Schema management
  */
@@ -143,4 +146,31 @@ export class ElasticsearchSchemaUtil {
 
     return { properties, dynamic: false };
   }
+
+  /**
+   * Gets list of all changed fields between two mappings
+   */
+  static getChangedFields(current: estypes.MappingTypeMapping, needed: estypes.MappingTypeMapping, prefix = ''): string[] {
+    const currentProperties = (current.properties ?? {});
+    const neededProperties = (needed.properties ?? {});
+    const allKeys = new Set([...Object.keys(currentProperties), ...Object.keys(neededProperties)]);
+    const changed: string[] = [];
+
+    for (const key of allKeys) {
+      const path = prefix ? `${prefix}.${key}` : key;
+      const currentProperty = currentProperties[key];
+      const neededProperty = neededProperties[key];
+
+      if (!currentProperty || !neededProperty || currentProperty.type !== neededProperty.type) {
+        changed.push(path);
+      } else if (isMappingType(currentProperty) || isMappingType(neededProperty)) {
+        changed.push(...this.getChangedFields(
+          'properties' in currentProperty ? currentProperty : { properties: {} },
+          'properties' in neededProperty ? neededProperty : { properties: {} },
+          path
+        ));
+      }
+    }
+    return changed;
+  }
 }

package/src/service.ts

@@ -6,7 +6,7 @@ import {
   ModelCrudUtil, ModelIndexedUtil, ModelStorageUtil, ModelExpiryUtil, ModelBulkUtil,
 } from '@travetto/model';
 import { ShutdownManager, type DeepPartial, type Class, castTo, asFull, TypedObject, asConstructable } from '@travetto/runtime';
-import { SchemaChange, BindUtil } from '@travetto/schema';
+import { BindUtil } from '@travetto/schema';
 import { Injectable } from '@travetto/di';
 import {
   ModelQuery, ModelQueryCrudSupport, ModelQueryFacetSupport,
@@ -115,17 +115,16 @@ export class ElasticsearchModelService implements
     await this.client.cluster.health({});
     this.manager = new IndexManager(this.config, this.client);
 
-    await ModelStorageUtil.registerModelChangeListener(this.manager);
+    await ModelStorageUtil.storageInitialization(this.manager);
     ShutdownManager.onGracefulShutdown(() => this.client.close());
     ModelExpiryUtil.registerCull(this);
   }
 
   createStorage(): Promise<void> { return this.manager.createStorage(); }
   deleteStorage(): Promise<void> { return this.manager.deleteStorage(); }
-  createModel(cls: Class): Promise<void> { return this.manager.createModel(cls); }
+  upsertModel(cls: Class): Promise<void> { return this.manager.upsertModel(cls); }
   exportModel(cls: Class): Promise<string> { return this.manager.exportModel(cls); }
   deleteModel(cls: Class): Promise<void> { return this.manager.deleteModel(cls); }
-  changeSchema(cls: Class, change: SchemaChange): Promise<void> { return this.manager.changeSchema(cls, change); }
   truncateModel(cls: Class): Promise<void> { return this.deleteByQuery(cls, {}).then(() => { }); }
 
   async get<T extends ModelType>(cls: Class<T>, id: string): Promise<T> {