@travetto/model-sql 7.0.0-rc.1 → 7.0.0-rc.3

package/src/service.ts

@@ -1,11 +1,11 @@
 import {
   ModelType,
-  BulkOp, BulkResponse, ModelCrudSupport, ModelStorageSupport, ModelBulkSupport,
+  BulkOperation, BulkResponse, ModelCrudSupport, ModelStorageSupport, ModelBulkSupport,
   NotFoundError, ModelRegistryIndex, ExistsError, OptionalId, ModelIdSource,
   ModelExpiryUtil, ModelCrudUtil, ModelStorageUtil, ModelBulkUtil,
 } from '@travetto/model';
 import { castTo, Class } from '@travetto/runtime';
-import { DataUtil, SchemaChange } from '@travetto/schema';
+import { DataUtil } from '@travetto/schema';
 import { AsyncContext } from '@travetto/context';
 import { Injectable } from '@travetto/di';
 import {
@@ -72,11 +72,11 @@ export class SQLModelService implements
         )
       )).records : [];
 
-    const allIds = new Set(all.map(el => el.id));
+    const allIds = new Set(all.map(type => type.id));
 
-    for (const [el, idx] of toCheck.entries()) {
-      if (!allIds.has(el)) { // If not found
-        addedIds.set(idx, el);
+    for (const [id, idx] of toCheck.entries()) {
+      if (!allIds.has(id)) { // If not found
+        addedIds.set(idx, id);
       }
     }
 
@@ -99,31 +99,23 @@ export class SQLModelService implements
   }
 
   async postConstruct(): Promise<void> {
-    if (this.#dialect) {
-      if (this.#dialect.conn.init) {
-        await this.#dialect.conn.init();
-      }
-      this.idSource = ModelCrudUtil.uuidSource(this.#dialect.ID_LEN);
-      this.#manager = new TableManager(this.#context, this.#dialect);
-      await ModelStorageUtil.registerModelChangeListener(this);
-      ModelExpiryUtil.registerCull(this);
-    }
-  }
-
-  get conn(): Connection {
-    return this.#dialect.conn;
+    await this.#dialect.connection.init?.();
+    this.idSource = ModelCrudUtil.uuidSource(this.#dialect.ID_LENGTH);
+    this.#manager = new TableManager(this.#context, this.#dialect);
+    await ModelStorageUtil.storageInitialization(this);
+    ModelExpiryUtil.registerCull(this);
   }
 
-  async exportModel<T extends ModelType>(e: Class<T>): Promise<string> {
-    return (await this.#manager.exportTables(e)).join('\n');
+  get connection(): Connection {
+    return this.#dialect.connection;
   }
 
-  async changeSchema(cls: Class, change: SchemaChange): Promise<void> {
-    await this.#manager.changeSchema(cls, change);
+  async exportModel<T extends ModelType>(cls: Class<T>): Promise<string> {
+    return (await this.#manager.exportTables(cls)).join('\n');
   }
 
-  async createModel(cls: Class): Promise<void> {
-    await this.#manager.createTables(cls);
+  async upsertModel(cls: Class): Promise<void> {
+    await this.#manager.upsertTables(cls);
   }
 
   async deleteModel(cls: Class): Promise<void> {
@@ -144,11 +136,11 @@ export class SQLModelService implements
       for (const ins of this.#dialect.getAllInsertSQL(cls, prepped)) {
         await this.#exec(ins);
       }
-    } catch (err) {
-      if (err instanceof ExistsError) {
+    } catch (error) {
+      if (error instanceof ExistsError) {
         throw new ExistsError(cls, prepped.id);
       } else {
-        throw err;
+        throw error;
       }
     }
     return prepped;
@@ -166,9 +158,9 @@ export class SQLModelService implements
       if (item.id) {
         await this.#deleteRaw(cls, item.id, {}, false);
       }
-    } catch (err) {
-      if (!(err instanceof NotFoundError)) {
-        throw err;
+    } catch (error) {
+      if (!(error instanceof NotFoundError)) {
+        throw error;
       }
     }
     return await this.create(cls, item);
@@ -203,7 +195,7 @@ export class SQLModelService implements
   }
 
   @Transactional()
-  async processBulk<T extends ModelType>(cls: Class<T>, operations: BulkOp<T>[]): Promise<BulkResponse> {
+  async processBulk<T extends ModelType>(cls: Class<T>, operations: BulkOperation<T>[]): Promise<BulkResponse> {
 
     const { insertedIds, upsertedIds, existingUpsertedIds } = await ModelBulkUtil.preStore(cls, operations, this);
 
@@ -211,16 +203,17 @@ export class SQLModelService implements
 
     await this.#checkUpsertedIds(cls,
       addedIds,
-      new Map([...existingUpsertedIds.entries()].map(([k, v]) => [v, k]))
+      new Map([...existingUpsertedIds.entries()].map(([key, value]) => [value, key]))
     );
 
-    const get = <K extends keyof BulkOp<T>>(k: K): Required<BulkOp<T>>[K][] =>
-      operations.map(x => x[k]).filter((x): x is Required<BulkOp<T>>[K] => !!x);
+    const get = <K extends keyof BulkOperation<T>>(key: K): Required<BulkOperation<T>>[K][] =>
+      operations.map(item => item[key]).filter((item): item is Required<BulkOperation<T>>[K] => !!item);
 
-    const getStatements = async (k: keyof BulkOp<T>): Promise<InsertWrapper[]> =>
-      (await SQLModelUtil.getInserts(cls, get(k))).filter(x => !!x.records.length);
+    const getStatements = async (key: keyof BulkOperation<T>): Promise<InsertWrapper[]> =>
+      (await SQLModelUtil.getInserts(cls, get(key))).filter(wrapper => !!wrapper.records.length);
 
-    const deletes = [{ stack: SQLModelUtil.classToStack(cls), ids: get('delete').map(x => x.id) }].filter(x => !!x.ids.length);
+    const deletes = [{ stack: SQLModelUtil.classToStack(cls), ids: get('delete').map(wrapper => wrapper.id) }]
+      .filter(wrapper => !!wrapper.ids.length);
 
     const [inserts, upserts, updates] = await Promise.all([
       getStatements('insert'),
@@ -242,19 +235,19 @@ export class SQLModelService implements
   @Connected()
   async query<T extends ModelType>(cls: Class<T>, query: PageableModelQuery<T>): Promise<T[]> {
     await QueryVerifier.verify(cls, query);
-    const { records: res } = await this.#exec<T>(this.#dialect.getQuerySQL(cls, query, ModelQueryUtil.getWhereClause(cls, query.where)));
+    const { records } = await this.#exec<T>(this.#dialect.getQuerySQL(cls, query, ModelQueryUtil.getWhereClause(cls, query.where)));
     if (ModelRegistryIndex.has(cls)) {
-      await this.#dialect.fetchDependents(cls, res, query && query.select);
+      await this.#dialect.fetchDependents(cls, records, query && query.select);
     }
 
-    const cleaned = SQLModelUtil.cleanResults<T>(this.#dialect, res);
-    return await Promise.all(cleaned.map(m => ModelCrudUtil.load(cls, m)));
+    const cleaned = SQLModelUtil.cleanResults<T>(this.#dialect, records);
+    return await Promise.all(cleaned.map(item => ModelCrudUtil.load(cls, item)));
   }
 
   @Connected()
   async queryOne<T extends ModelType>(cls: Class<T>, builder: ModelQuery<T>, failOnMany = true): Promise<T> {
-    const res = await this.query<T>(cls, { ...builder, limit: failOnMany ? 2 : 1 });
-    return ModelQueryUtil.verifyGetSingleCounts<T>(cls, failOnMany, res, builder.where);
+    const results = await this.query<T>(cls, { ...builder, limit: failOnMany ? 2 : 1 });
+    return ModelQueryUtil.verifyGetSingleCounts<T>(cls, failOnMany, results, builder.where);
   }
 
   @Connected()
@@ -294,43 +287,43 @@ export class SQLModelService implements
   @Connected()
   async suggest<T extends ModelType>(cls: Class<T>, field: ValidStringFields<T>, prefix?: string, query?: PageableModelQuery<T>): Promise<T[]> {
     await QueryVerifier.verify(cls, query);
-    const q = ModelQuerySuggestUtil.getSuggestQuery<T>(cls, field, prefix, query);
-    const results = await this.query<T>(cls, q);
+    const resolvedQuery = ModelQuerySuggestUtil.getSuggestQuery<T>(cls, field, prefix, query);
+    const results = await this.query<T>(cls, resolvedQuery);
     return ModelQuerySuggestUtil.combineSuggestResults(cls, field, prefix, results, (a, b) => b, query?.limit);
   }
 
   @Connected()
   async suggestValues<T extends ModelType>(cls: Class<T>, field: ValidStringFields<T>, prefix?: string, query?: PageableModelQuery<T>): Promise<string[]> {
     await QueryVerifier.verify(cls, query);
-    const q = ModelQuerySuggestUtil.getSuggestFieldQuery(cls, field, prefix, query);
-    const results = await this.query(cls, q);
+    const resolvedQuery = ModelQuerySuggestUtil.getSuggestFieldQuery(cls, field, prefix, query);
+    const results = await this.query(cls, resolvedQuery);
 
     const modelTypeField: ValidStringFields<ModelType> = castTo(field);
-    return ModelQuerySuggestUtil.combineSuggestResults(cls, modelTypeField, prefix, results, x => x, query?.limit);
+    return ModelQuerySuggestUtil.combineSuggestResults(cls, modelTypeField, prefix, results, result => result, query?.limit);
   }
 
   @Connected()
   async facet<T extends ModelType>(cls: Class<T>, field: ValidStringFields<T>, query?: ModelQuery<T>): Promise<ModelQueryFacet[]> {
     await QueryVerifier.verify(cls, query);
-    const col = this.#dialect.ident(field);
-    const ttl = this.#dialect.ident('count');
-    const key = this.#dialect.ident('key');
-    const q = [
+    const col = this.#dialect.identifier(field);
+    const ttl = this.#dialect.identifier('count');
+    const key = this.#dialect.identifier('key');
+    const sql = [
       `SELECT ${col} as ${key}, COUNT(${col}) as ${ttl}`,
       this.#dialect.getFromSQL(cls),
     ];
-    q.push(
+    sql.push(
       this.#dialect.getWhereSQL(cls, ModelQueryUtil.getWhereClause(cls, query?.where))
     );
-    q.push(
+    sql.push(
       `GROUP BY ${col}`,
       `ORDER BY ${ttl} DESC`
     );
 
-    const results = await this.#exec<{ key: string, count: number }>(q.join('\n'));
-    return results.records.map(x => {
-      x.count = DataUtil.coerceType(x.count, Number);
-      return x;
+    const results = await this.#exec<{ key: string, count: number }>(sql.join('\n'));
+    return results.records.map(result => {
+      result.count = DataUtil.coerceType(result.count, Number);
+      return result;
     });
   }
 }

package/src/table-manager.ts

@@ -1,8 +1,7 @@
 import { AsyncContext, WithAsyncContext } from '@travetto/context';
 import { ModelRegistryIndex } from '@travetto/model';
 import { Class } from '@travetto/runtime';
-import { ChangeEvent } from '@travetto/registry';
-import { SchemaChange } from '@travetto/schema';
+import { SchemaRegistryIndex, type SchemaFieldConfig } from '@travetto/schema';
 
 import { Connected, Transactional } from './connection/decorator.ts';
 import { SQLDialect } from './dialect/base.ts';
@@ -10,6 +9,10 @@ import { SQLModelUtil } from './util.ts';
 import { Connection } from './connection/base.ts';
 import { VisitStack } from './types.ts';
 
+type UpsertStructure = { dropIndex: string[], createIndex: string[], table: string[] };
+const isSimpleField = (input: VisitStack | undefined): input is SchemaFieldConfig =>
+  !!input && (!('type' in input) || (input.type && !SchemaRegistryIndex.has(input.type)));
+
 /**
  * Manage creation/updating of all tables
  */
@@ -27,59 +30,108 @@ export class TableManager {
     return this.#dialect.executeSQL<T>(sql);
   }
 
+  /**
+   * Get a valid connection
+   */
+  get connection(): Connection {
+    return this.#dialect.connection;
+  }
+
   /**
    * Create all needed tables for a given class
    */
   async exportTables(cls: Class): Promise<string[]> {
     const out: string[] = [];
-    for (const op of this.#dialect.getCreateAllTablesSQL(cls)) {
-      out.push(op);
+    for (const command of this.#dialect.getCreateAllTablesSQL(cls)) {
+      out.push(command);
     }
     const indices = ModelRegistryIndex.getConfig(cls).indices;
     if (indices) {
-      for (const op of this.#dialect.getCreateAllIndicesSQL(cls, indices)) {
-        out.push(op);
+      for (const command of this.#dialect.getCreateAllIndicesSQL(cls, indices)) {
+        out.push(command);
       }
     }
     return out;
   }
 
-  /**
-   * Create all needed tables for a given class
-   */
   @WithAsyncContext()
   @Connected()
-  @Transactional()
-  async createTables(cls: Class): Promise<void> {
-    for (const op of this.#dialect.getCreateAllTablesSQL(cls)) {
-      await this.#exec(op);
-    }
-    const indices = ModelRegistryIndex.getConfig(cls).indices;
-    if (indices) {
-      for (const op of this.#dialect.getCreateAllIndicesSQL(cls, indices)) {
-        try {
-          await this.#exec(op);
-        } catch (err) {
-          if (!(err instanceof Error)) {
-            throw err;
+  async getUpsertTablesSQL(cls: Class): Promise<UpsertStructure> {
+    const sqlCommands: UpsertStructure = { dropIndex: [], createIndex: [], table: [] };
+
+    const onVisit = async (type: Class, fields: SchemaFieldConfig[], path: VisitStack[]): Promise<void> => {
+      const found = await this.#dialect.describeTable(this.#dialect.namespace(path));
+      const existingFields = new Map(found?.columns.map(column => [column.name, column]) ?? []);
+      const existingIndices = new Map(found?.indices.map(index => [index.name, index]) ?? []);
+      const model = path.length === 1 ? ModelRegistryIndex.getConfig(type) : undefined;
+      const requestedIndices = new Map((model?.indices ?? []).map(index => [this.#dialect.getIndexName(type, index), index]) ?? []);
+
+
+      // Manage fields
+      if (!existingFields.size) {
+        sqlCommands.table.push(this.#dialect.getCreateTableSQL(path));
+      } else { // Existing
+        // Fields
+        const requestedFields = new Map(fields.map(field => [field.name, field]));
+        const top = path.at(-1);
+
+        if (isSimpleField(top)) {
+          requestedFields.set(top.name, top);
+        }
+
+        for (const [column, field] of requestedFields.entries()) {
+          if (!existingFields.has(column)) {
+            sqlCommands.table.push(this.#dialect.getAddColumnSQL([...path, field]));
+          } else if (this.#dialect.isColumnChanged(field, existingFields.get(column)!)) {
+            sqlCommands.table.push(this.#dialect.getModifyColumnSQL([...path, field]));
           }
-          if (!/\bexists|duplicate\b/i.test(err.message)) {
-            throw err;
+        }
+
+        // TODO: Handle dropping tables that are FK'd when no longer in use
+
+        for (const column of existingFields.keys()) {
+          if (!requestedFields.has(column)) {
+            sqlCommands.table.push(this.#dialect.getDropColumnSQL([...path, { name: column, type: undefined!, array: false }]));
           }
         }
       }
-    }
+
+      // Manage indices
+      for (const index of requestedIndices.keys()) {
+        if (!existingIndices.has(index)) {
+          sqlCommands.createIndex.push(this.#dialect.getCreateIndexSQL(type, requestedIndices.get(index)!));
+        } else if (this.#dialect.isIndexChanged(requestedIndices.get(index)!, existingIndices.get(index)!)) {
+          sqlCommands.dropIndex.push(this.#dialect.getDropIndexSQL(type, existingIndices.get(index)!.name));
+          sqlCommands.createIndex.push(this.#dialect.getCreateIndexSQL(type, requestedIndices.get(index)!));
+        }
+      }
+
+      for (const index of existingIndices.keys()) {
+        if (!requestedIndices.has(index)) {
+          sqlCommands.dropIndex.push(this.#dialect.getDropIndexSQL(type, existingIndices.get(index)!.name));
+        }
+      }
+    };
+
+    const schema = SchemaRegistryIndex.getConfig(cls);
+    await SQLModelUtil.visitSchema(schema, {
+      onRoot: async ({ config, path, fields, descend }) => { await onVisit(config.class, fields, path); return descend(); },
+      onSub: async ({ config, path, fields, descend }) => { await onVisit(config.type, fields, path); return descend(); },
+      onSimple: async ({ config, path, fields }) => { await onVisit(config.type, fields, path); }
+    });
+    return sqlCommands;
   }
 
-  /**
-   * Drop all tables for a given class
-   */
   @WithAsyncContext()
   @Connected()
   @Transactional()
-  async dropTables(cls: Class): Promise<void> {
-    for (const op of this.#dialect.getDropAllTablesSQL(cls)) {
-      await this.#exec(op);
+  async upsertTables(cls: Class): Promise<void> {
+    // Enforce id length
+    this.#dialect.enforceIdLength(cls);
+
+    const sqlCommands = await this.getUpsertTablesSQL(cls);
+    for (const key of ['dropIndex', 'table', 'createIndex'] as const) {
+      await Promise.all(sqlCommands[key].map(command => this.#exec(command)));
     }
   }
 
@@ -89,43 +141,21 @@ export class TableManager {
   @WithAsyncContext()
   @Connected()
   @Transactional()
-  async truncateTables(cls: Class): Promise<void> {
-    for (const op of this.#dialect.getTruncateAllTablesSQL(cls)) {
-      await this.#exec(op);
+  async dropTables(cls: Class): Promise<void> {
+    for (const command of this.#dialect.getDropAllTablesSQL(cls)) {
+      await this.#exec(command);
     }
   }
 
   /**
-   * Get a valid connection
-   */
-  get conn(): Connection {
-    return this.#dialect.conn;
-  }
-
-  /**
-   * When the schema changes, update SQL
+   * Drop all tables for a given class
    */
   @WithAsyncContext()
-  @Transactional()
   @Connected()
-  async changeSchema(cls: Class, change: SchemaChange): Promise<void> {
-    try {
-      const rootStack = SQLModelUtil.classToStack(cls);
-
-      const changes = change.subs.reduce<Record<ChangeEvent<unknown>['type'], VisitStack[][]>>((acc, v) => {
-        const path = v.path.map(f => ({ ...f }));
-        for (const ev of v.fields) {
-          acc[ev.type].push([...rootStack, ...path, { ...(ev.type === 'removing' ? ev.prev : ev.curr)! }]);
-        }
-        return acc;
-      }, { added: [], changed: [], removing: [] });
-
-      await Promise.all(changes.added.map(v => this.#dialect.executeSQL(this.#dialect.getAddColumnSQL(v))));
-      await Promise.all(changes.changed.map(v => this.#dialect.executeSQL(this.#dialect.getModifyColumnSQL(v))));
-      await Promise.all(changes.removing.map(v => this.#dialect.executeSQL(this.#dialect.getDropColumnSQL(v))));
-    } catch (err) {
-      // Failed to change
-      console.error('Unable to change field', { error: err });
+  @Transactional()
+  async truncateTables(cls: Class): Promise<void> {
+    for (const command of this.#dialect.getTruncateAllTablesSQL(cls)) {
+      await this.#exec(command);
     }
   }
 }

package/src/types.ts

@@ -6,6 +6,6 @@ export type VisitStack = {
   [TableSymbol]?: string;
   array?: boolean;
   type: Class;
-  name: string | symbol;
+  name: string;
   index?: number;
 };