@travetto/model-sql 7.0.0-rc.2 → 7.0.0-rc.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model-sql",
-  "version": "7.0.0-rc.2",
+  "version": "7.0.0-rc.3",
   "description": "SQL backing for the travetto model module, with real-time modeling support for SQL schemas.",
   "keywords": [
     "sql",
@@ -27,14 +27,14 @@
     "directory": "module/model-sql"
   },
   "dependencies": {
-    "@travetto/cli": "^7.0.0-rc.2",
-    "@travetto/config": "^7.0.0-rc.2",
-    "@travetto/context": "^7.0.0-rc.2",
-    "@travetto/model": "^7.0.0-rc.2",
-    "@travetto/model-query": "^7.0.0-rc.2"
+    "@travetto/cli": "^7.0.0-rc.3",
+    "@travetto/config": "^7.0.0-rc.3",
+    "@travetto/context": "^7.0.0-rc.3",
+    "@travetto/model": "^7.0.0-rc.3",
+    "@travetto/model-query": "^7.0.0-rc.3"
   },
   "peerDependencies": {
-    "@travetto/test": "^7.0.0-rc.2"
+    "@travetto/test": "^7.0.0-rc.3"
   },
   "peerDependenciesMeta": {
     "@travetto/test": {

package/src/config.ts

@@ -1,5 +1,5 @@
 import { Config } from '@travetto/config';
-import { asFull } from '@travetto/runtime';
+import { asFull, Runtime } from '@travetto/runtime';
 
 /**
  * SQL Model Config
@@ -17,11 +17,11 @@ export class SQLModelConfig<T extends {} = {}> {
   /**
    * Username
    */
-  user = '';
+  user = Runtime.production ? '' : 'travetto';
   /**
    * Password
    */
-  password = '';
+  password = Runtime.production ? '' : 'travetto';
   /**
    * Table prefix
    */
@@ -31,9 +31,9 @@ export class SQLModelConfig<T extends {} = {}> {
    */
   database = 'app';
   /**
-   * Auto schema creation
+   * Allow storage modification at runtime
    */
-  autoCreate?: boolean;
+  modifyStorage?: boolean;
   /**
    * Db version
    */

package/src/dialect/base.ts

@@ -16,6 +16,12 @@ interface Alias {
   path: VisitStack[];
 }
 
+export type SQLTableDescription = {
+  columns: { name: string, type: string, is_notnull: boolean }[];
+  foreignKeys: { name: string, from_column: string, to_column: string, to_table: string }[];
+  indices: { name: string, columns: { name: string, desc: boolean }[], is_unique: boolean }[];
+};
+
 @Schema()
 class Total {
   total: number;
@@ -24,7 +30,7 @@ class Total {
 function makeField(name: string, type: Class, required: boolean, extra: Partial<SchemaFieldConfig>): SchemaFieldConfig {
   return {
     name,
-    owner: null!,
+    class: null!,
     type,
     array: false,
     ...(required ? { required: { active: true } } : {}),
@@ -105,8 +111,8 @@ export abstract class SQLDialect implements DialectState {
    * Generate an id field
    */
   idField = makeField('id', String, true, {
-    maxlength: { n: this.ID_LENGTH },
-    minlength: { n: this.ID_LENGTH }
+    maxlength: { limit: this.ID_LENGTH },
+    minlength: { limit: this.ID_LENGTH }
   });
 
   /**
@@ -118,8 +124,8 @@ export abstract class SQLDialect implements DialectState {
    * Parent path reference
    */
   parentPathField = makeField('__parent_path', String, true, {
-    maxlength: { n: this.HASH_LENGTH },
-    minlength: { n: this.HASH_LENGTH },
+    maxlength: { limit: this.HASH_LENGTH },
+    minlength: { limit: this.HASH_LENGTH },
     required: { active: true }
   });
 
@@ -127,8 +133,8 @@ export abstract class SQLDialect implements DialectState {
    * Path reference
    */
   pathField = makeField('__path', String, true, {
-    maxlength: { n: this.HASH_LENGTH },
-    minlength: { n: this.HASH_LENGTH },
+    maxlength: { limit: this.HASH_LENGTH },
+    minlength: { limit: this.HASH_LENGTH },
     required: { active: true }
   });
 
@@ -156,6 +162,11 @@ export abstract class SQLDialect implements DialectState {
    */
   abstract hash(input: string): string;
 
+  /**
+   * Describe a table structure
+   */
+  abstract describeTable(table: string): Promise<SQLTableDescription | undefined>;
+
   executeSQL<T>(sql: string): Promise<{ records: T[], count: number }> {
     return this.connection.execute<T>(this.connection.active, sql);
   }
@@ -253,7 +264,7 @@ export abstract class SQLDialect implements DialectState {
       if (config.specifiers?.includes('text')) {
         type = this.COLUMN_TYPES.TEXT;
       } else {
-        type = this.PARAMETERIZED_COLUMN_TYPES.VARCHAR(config.maxlength ? config.maxlength.n : this.DEFAULT_STRING_LENGTH);
+        type = this.PARAMETERIZED_COLUMN_TYPES.VARCHAR(config.maxlength?.limit ?? this.DEFAULT_STRING_LENGTH);
       }
     } else if (config.type === PointConcrete) {
       type = this.COLUMN_TYPES.POINT;
@@ -267,12 +278,13 @@ export abstract class SQLDialect implements DialectState {
   /**
    * FieldConfig to Column definition
    */
-  getColumnDefinition(config: SchemaFieldConfig): string | undefined {
+  getColumnDefinition(config: SchemaFieldConfig, overrideRequired?: boolean): string | undefined {
     const type = this.getColumnType(config);
     if (!type) {
       return;
     }
-    return `${this.identifier(config)} ${type} ${(config.required?.active !== false) ? 'NOT NULL' : 'DEFAULT NULL'}`;
+    const required = overrideRequired ? true : (config.required?.active ?? false);
+    return `${this.identifier(config)} ${type} ${required ? 'NOT NULL' : ''}`;
   }
 
   /**
@@ -420,7 +432,7 @@ export abstract class SQLDialect implements DialectState {
         // If dealing with simple external
         sStack.push({
           name: field.name,
-          owner: null!,
+          class: null!,
           type: field.type
         });
       }
@@ -652,17 +664,11 @@ ${this.getLimitSQL(cls, query)}`;
       let idField = fields.find(field => field.name === this.idField.name);
       if (!idField) {
         fields.push(idField = this.idField);
-      } else {
-        idField.maxlength = { n: this.ID_LENGTH };
       }
     }
 
     const fieldSql = fields
-      .map(field => {
-        const def = this.getColumnDefinition(field) || '';
-        return field.name === this.idField.name && !parent ?
-          def.replace('DEFAULT NULL', 'NOT NULL') : def;
-      })
+      .map(field => this.getColumnDefinition(field, field.name === this.idField.name && !parent) || '')
       .filter(line => !!line.trim())
       .join(',\n  ');
 
@@ -714,6 +720,14 @@ CREATE TABLE IF NOT EXISTS ${this.table(stack)} (
     return indices.map(idx => this.getCreateIndexSQL(cls, idx));
   }
 
+  /**
+   * Get index name
+   */
+  getIndexName<T extends ModelType>(cls: Class<T>, idx: IndexConfig<ModelType>): string {
+    const table = this.namespace(SQLModelUtil.classToStack(cls));
+    return ['idx', table, idx.name.toLowerCase().replaceAll('-', '_')].join('_');
+  }
+
   /**
    * Get CREATE INDEX sql
    */
@@ -727,12 +741,20 @@ CREATE TABLE IF NOT EXISTS ${this.table(stack)} (
       }
       return [castTo(key), typeof value === 'number' ? value === 1 : (!!value)];
     });
-    const constraint = `idx_${table}_${fields.map(([field]) => field).join('_')}`;
+    const constraint = this.getIndexName(cls, idx);
     return `CREATE ${idx.type === 'unique' ? 'UNIQUE ' : ''}INDEX ${constraint} ON ${this.identifier(table)} (${fields
       .map(([name, sel]) => `${this.identifier(name)} ${sel ? 'ASC' : 'DESC'}`)
       .join(', ')});`;
   }
 
+  /**
+   * Get DROP INDEX sql
+   */
+  getDropIndexSQL<T extends ModelType>(cls: Class<T>, idx: IndexConfig<T> | string): string {
+    const constraint = typeof idx === 'string' ? idx : this.getIndexName(cls, idx);
+    return `DROP INDEX ${this.identifier(constraint)} ;`;
+  }
+
   /**
    * Drop all tables for a given class
    */
@@ -869,8 +891,8 @@ ${this.getWhereSQL(type, where)};`;
   }
 
   /**
-  * Get elements by ids
-  */
+   * Get elements by ids
+   */
   getSelectRowsByIdsSQL(stack: VisitStack[], ids: string[], select: SchemaFieldConfig[] = []): string {
     const config = stack.at(-1)!;
     const orderBy = !config.array ?
@@ -1035,4 +1057,45 @@ ${this.getWhereSQL(cls, where!)}`;
 
     return out;
   }
+
+  /**
+   * Determine if a column has changed
+   */
+  isColumnChanged(requested: SchemaFieldConfig, existing: SQLTableDescription['columns'][number],): boolean {
+    const requestedColumnType = this.getColumnType(requested);
+    const result =
+      (requested.name !== this.idField.name && !!requested.required?.active !== !!existing.is_notnull)
+      || (requestedColumnType.toUpperCase() !== existing.type.toUpperCase());
+
+    return result;
+  }
+
+  /**
+   * Determine if an index has changed
+   */
+  isIndexChanged(requested: IndexConfig<ModelType>, existing: SQLTableDescription['indices'][number]): boolean {
+    let result =
+      (existing.is_unique && requested.type !== 'unique')
+      || requested.fields.length !== existing.columns.length;
+
+    for (let i = 0; i < requested.fields.length && !result; i++) {
+      const [[key, value]] = Object.entries(requested.fields[i]);
+      const desc = value === -1;
+      result ||= key !== existing.columns[i].name && desc !== existing.columns[i].desc;
+    }
+
+    return result;
+  }
+
+  /**
+   * Enforce the dialect specific id length
+   */
+  enforceIdLength(cls: Class<ModelType>): void {
+    const config = SchemaRegistryIndex.getConfig(cls);
+    const idField = config.fields[this.idField.name];
+    if (idField) {
+      idField.maxlength = { limit: this.ID_LENGTH };
+      idField.minlength = { limit: this.ID_LENGTH };
+    }
+  }
 }

package/src/service.ts

@@ -5,7 +5,7 @@ import {
   ModelExpiryUtil, ModelCrudUtil, ModelStorageUtil, ModelBulkUtil,
 } from '@travetto/model';
 import { castTo, Class } from '@travetto/runtime';
-import { DataUtil, SchemaChange } from '@travetto/schema';
+import { DataUtil } from '@travetto/schema';
 import { AsyncContext } from '@travetto/context';
 import { Injectable } from '@travetto/di';
 import {
@@ -99,15 +99,11 @@ export class SQLModelService implements
   }
 
   async postConstruct(): Promise<void> {
-    if (this.#dialect) {
-      if (this.#dialect.connection.init) {
-        await this.#dialect.connection.init();
-      }
-      this.idSource = ModelCrudUtil.uuidSource(this.#dialect.ID_LENGTH);
-      this.#manager = new TableManager(this.#context, this.#dialect);
-      await ModelStorageUtil.registerModelChangeListener(this);
-      ModelExpiryUtil.registerCull(this);
-    }
+    await this.#dialect.connection.init?.();
+    this.idSource = ModelCrudUtil.uuidSource(this.#dialect.ID_LENGTH);
+    this.#manager = new TableManager(this.#context, this.#dialect);
+    await ModelStorageUtil.storageInitialization(this);
+    ModelExpiryUtil.registerCull(this);
   }
 
   get connection(): Connection {
@@ -118,12 +114,8 @@ export class SQLModelService implements
     return (await this.#manager.exportTables(cls)).join('\n');
   }
 
-  async changeSchema(cls: Class, change: SchemaChange): Promise<void> {
-    await this.#manager.changeSchema(cls, change);
-  }
-
-  async createModel(cls: Class): Promise<void> {
-    await this.#manager.createTables(cls);
+  async upsertModel(cls: Class): Promise<void> {
+    await this.#manager.upsertTables(cls);
   }
 
   async deleteModel(cls: Class): Promise<void> {

package/src/table-manager.ts

@@ -1,8 +1,7 @@
 import { AsyncContext, WithAsyncContext } from '@travetto/context';
 import { ModelRegistryIndex } from '@travetto/model';
 import { Class } from '@travetto/runtime';
-import { ChangeEvent } from '@travetto/registry';
-import { SchemaChange } from '@travetto/schema';
+import { SchemaRegistryIndex, type SchemaFieldConfig } from '@travetto/schema';
 
 import { Connected, Transactional } from './connection/decorator.ts';
 import { SQLDialect } from './dialect/base.ts';
@@ -10,6 +9,10 @@ import { SQLModelUtil } from './util.ts';
 import { Connection } from './connection/base.ts';
 import { VisitStack } from './types.ts';
 
+type UpsertStructure = { dropIndex: string[], createIndex: string[], table: string[] };
+const isSimpleField = (input: VisitStack | undefined): input is SchemaFieldConfig =>
+  !!input && (!('type' in input) || (input.type && !SchemaRegistryIndex.has(input.type)));
+
 /**
  * Manage creation/updating of all tables
  */
@@ -27,6 +30,13 @@ export class TableManager {
     return this.#dialect.executeSQL<T>(sql);
   }
 
+  /**
+   * Get a valid connection
+   */
+  get connection(): Connection {
+    return this.#dialect.connection;
+  }
+
   /**
    * Create all needed tables for a given class
    */
@@ -44,30 +54,84 @@ export class TableManager {
     return out;
   }
 
-  /**
-   * Create all needed tables for a given class
-   */
   @WithAsyncContext()
   @Connected()
-  @Transactional()
-  async createTables(cls: Class): Promise<void> {
-    for (const command of this.#dialect.getCreateAllTablesSQL(cls)) {
-      await this.#exec(command);
-    }
-    const indices = ModelRegistryIndex.getConfig(cls).indices;
-    if (indices) {
-      for (const command of this.#dialect.getCreateAllIndicesSQL(cls, indices)) {
-        try {
-          await this.#exec(command);
-        } catch (error) {
-          if (!(error instanceof Error)) {
-            throw error;
+  async getUpsertTablesSQL(cls: Class): Promise<UpsertStructure> {
+    const sqlCommands: UpsertStructure = { dropIndex: [], createIndex: [], table: [] };
+
+    const onVisit = async (type: Class, fields: SchemaFieldConfig[], path: VisitStack[]): Promise<void> => {
+      const found = await this.#dialect.describeTable(this.#dialect.namespace(path));
+      const existingFields = new Map(found?.columns.map(column => [column.name, column]) ?? []);
+      const existingIndices = new Map(found?.indices.map(index => [index.name, index]) ?? []);
+      const model = path.length === 1 ? ModelRegistryIndex.getConfig(type) : undefined;
+      const requestedIndices = new Map((model?.indices ?? []).map(index => [this.#dialect.getIndexName(type, index), index]) ?? []);
+
+
+      // Manage fields
+      if (!existingFields.size) {
+        sqlCommands.table.push(this.#dialect.getCreateTableSQL(path));
+      } else { // Existing
+        // Fields
+        const requestedFields = new Map(fields.map(field => [field.name, field]));
+        const top = path.at(-1);
+
+        if (isSimpleField(top)) {
+          requestedFields.set(top.name, top);
+        }
+
+        for (const [column, field] of requestedFields.entries()) {
+          if (!existingFields.has(column)) {
+            sqlCommands.table.push(this.#dialect.getAddColumnSQL([...path, field]));
+          } else if (this.#dialect.isColumnChanged(field, existingFields.get(column)!)) {
+            sqlCommands.table.push(this.#dialect.getModifyColumnSQL([...path, field]));
           }
-          if (!/\bexists|duplicate\b/i.test(error.message)) {
-            throw error;
+        }
+
+        // TODO: Handle dropping tables that are FK'd when no longer in use
+
+        for (const column of existingFields.keys()) {
+          if (!requestedFields.has(column)) {
+            sqlCommands.table.push(this.#dialect.getDropColumnSQL([...path, { name: column, type: undefined!, array: false }]));
           }
         }
       }
+
+      // Manage indices
+      for (const index of requestedIndices.keys()) {
+        if (!existingIndices.has(index)) {
+          sqlCommands.createIndex.push(this.#dialect.getCreateIndexSQL(type, requestedIndices.get(index)!));
+        } else if (this.#dialect.isIndexChanged(requestedIndices.get(index)!, existingIndices.get(index)!)) {
+          sqlCommands.dropIndex.push(this.#dialect.getDropIndexSQL(type, existingIndices.get(index)!.name));
+          sqlCommands.createIndex.push(this.#dialect.getCreateIndexSQL(type, requestedIndices.get(index)!));
+        }
+      }
+
+      for (const index of existingIndices.keys()) {
+        if (!requestedIndices.has(index)) {
+          sqlCommands.dropIndex.push(this.#dialect.getDropIndexSQL(type, existingIndices.get(index)!.name));
+        }
+      }
+    };
+
+    const schema = SchemaRegistryIndex.getConfig(cls);
+    await SQLModelUtil.visitSchema(schema, {
+      onRoot: async ({ config, path, fields, descend }) => { await onVisit(config.class, fields, path); return descend(); },
+      onSub: async ({ config, path, fields, descend }) => { await onVisit(config.type, fields, path); return descend(); },
+      onSimple: async ({ config, path, fields }) => { await onVisit(config.type, fields, path); }
+    });
+    return sqlCommands;
+  }
+
+  @WithAsyncContext()
+  @Connected()
+  @Transactional()
+  async upsertTables(cls: Class): Promise<void> {
+    // Enforce id length
+    this.#dialect.enforceIdLength(cls);
+
+    const sqlCommands = await this.getUpsertTablesSQL(cls);
+    for (const key of ['dropIndex', 'table', 'createIndex'] as const) {
+      await Promise.all(sqlCommands[key].map(command => this.#exec(command)));
     }
   }
 
@@ -94,38 +158,4 @@ export class TableManager {
       await this.#exec(command);
     }
   }
-
-  /**
-   * Get a valid connection
-   */
-  get connection(): Connection {
-    return this.#dialect.connection;
-  }
-
-  /**
-   * When the schema changes, update SQL
-   */
-  @WithAsyncContext()
-  @Transactional()
-  @Connected()
-  async changeSchema(cls: Class, change: SchemaChange): Promise<void> {
-    try {
-      const rootStack = SQLModelUtil.classToStack(cls);
-
-      const changes = change.subs.reduce<Record<ChangeEvent<unknown>['type'], VisitStack[][]>>((result, value) => {
-        const path = value.path.map(field => ({ ...field }));
-        for (const event of value.fields) {
-          result[event.type].push([...rootStack, ...path, { ...(event.type === 'removing' ? event.previous : event.current)! }]);
-        }
-        return result;
-      }, { added: [], changed: [], removing: [] });
-
-      await Promise.all(changes.added.map(value => this.#dialect.executeSQL(this.#dialect.getAddColumnSQL(value))));
-      await Promise.all(changes.changed.map(value => this.#dialect.executeSQL(this.#dialect.getModifyColumnSQL(value))));
-      await Promise.all(changes.removing.map(value => this.#dialect.executeSQL(this.#dialect.getDropColumnSQL(value))));
-    } catch (error) {
-      // Failed to change
-      console.error('Unable to change field', { error });
-    }
-  }
 }

package/src/util.ts

@@ -104,7 +104,7 @@ export class SQLModelUtil {
    * Process a schema structure, synchronously
    */
   static visitSchemaSync(config: SchemaClassConfig | SchemaFieldConfig, handler: VisitHandler<void>, state: VisitState = { path: [] }): void {
-    const path = 'class' in config ? this.classToStack(config.class) : [...state.path, config];
+    const path = 'fields' in config ? this.classToStack(config.class) : [...state.path, config];
     const { local: fields, foreign } = this.getFieldsByLocation(path);
 
     const descend = (): void => {
@@ -121,7 +121,7 @@ export class SQLModelUtil {
       }
     };
 
-    if ('class' in config) {
+    if ('fields' in config) {
       return handler.onRoot({ config, fields, descend, path });
     } else {
       return handler.onSub({ config, fields, descend, path });
@@ -132,7 +132,7 @@ export class SQLModelUtil {
    * Visit a Schema structure
    */
   static async visitSchema(config: SchemaClassConfig | SchemaFieldConfig, handler: VisitHandler<Promise<void>>, state: VisitState = { path: [] }): Promise<void> {
-    const path = 'class' in config ? this.classToStack(config.class) : [...state.path, config];
+    const path = 'fields' in config ? this.classToStack(config.class) : [...state.path, config];
     const { local: fields, foreign } = this.getFieldsByLocation(path);
 
     const descend = async (): Promise<void> => {
@@ -149,7 +149,7 @@ export class SQLModelUtil {
       }
     };
 
-    if ('class' in config) {
+    if ('fields' in config) {
       return handler.onRoot({ config, fields, descend, path });
     } else {
       return handler.onSub({ config, fields, descend, path });
@@ -210,7 +210,7 @@ export class SQLModelUtil {
    */
   static select<T>(cls: Class<T>, select?: SelectClause<T>): SchemaFieldConfig[] {
     if (!select || Object.keys(select).length === 0) {
-      return [{ type: cls, name: '*', owner: cls, array: false }];
+      return [{ type: cls, name: '*', class: cls, array: false }];
     }
 
     const { localMap } = this.getFieldsByLocation(this.classToStack(cls));

package/support/test/query.ts

@@ -69,7 +69,7 @@ export abstract class BaseSQLTest extends BaseModelSuite<SQLModelService> {
     dct.resolveName = (stack: VisitStack[]) => {
       const field: SchemaFieldConfig = castTo(stack.at(-1));
       const parent: SchemaFieldConfig = castTo(stack.at(-2));
-      return `${field.owner ? field.owner.name.toString() : parent.name.toString()}.${field.name.toString()}`;
+      return `${field.class ? field.class.name.toString() : parent.name.toString()}.${field.name.toString()}`;
     };
 
     const qryStr = dct.getWhereGroupingSQL(WhereType, qry);
@@ -81,7 +81,7 @@ export abstract class BaseSQLTest extends BaseModelSuite<SQLModelService> {
     const dct = await this.dialect;
     dct.resolveName = (stack: VisitStack[]) => {
       const field: SchemaFieldConfig = castTo(stack.at(-1));
-      return `${field.owner?.name}.${field.name.toString()}`;
+      return `${field.class?.name}.${field.name.toString()}`;
     };
 
     const out = dct.getWhereGroupingSQL(User, {