oak-db 3.3.14 → 4.0.1

package/lib/MySQL/store.js

@@ -37,6 +37,9 @@ class MysqlStore extends CascadeStore_1.CascadeStore {
     async exec(script, txnId) {
         await this.connector.exec(script, txnId);
     }
+    supportsTransactionalDdl() {
+        return false;
+    }
     connector;
     translator;
     constructor(storageSchema, configuration) {
@@ -304,26 +307,164 @@ class MysqlStore extends CascadeStore_1.CascadeStore {
     async disconnect() {
         await this.connector.disconnect();
     }
-    async initialize(option) {
+    shouldResetEntity(option, entity) {
+        const ifExists = option?.ifExists || 'drop';
+        const entityDef = this.translator.schema[entity];
+        if (ifExists === 'drop') {
+            return true;
+        }
+        if (ifExists === 'dropIfNotStatic') {
+            return !entityDef.static;
+        }
+        return false;
+    }
+    quoteIdentifier(identifier) {
+        return `\`${identifier.replace(/`/g, '``')}\``;
+    }
+    buildResetDropSql(tableNames) {
+        return `DROP TABLE IF EXISTS ${tableNames.map((tableName) => this.quoteIdentifier(tableName)).join(', ')};`;
+    }
+    buildBatchAddForeignKeySql(sqls) {
+        const grouped = new Map();
+        const orderedTables = [];
+        const passthroughSqls = [];
+        sqls.forEach((sql) => {
+            const normalized = sql.trim();
+            const match = /^ALTER TABLE\s+(`[^`]+`)\s+(ADD CONSTRAINT[\s\S]+);$/i.exec(normalized);
+            if (!match) {
+                passthroughSqls.push(normalized);
+                return;
+            }
+            const [, tableName, clause] = match;
+            if (!grouped.has(tableName)) {
+                grouped.set(tableName, []);
+                orderedTables.push(tableName);
+            }
+            grouped.get(tableName).push(clause);
+        });
+        return [
+            ...orderedTables.map((tableName) => {
+                const clauses = grouped.get(tableName);
+                return `ALTER TABLE ${tableName} ${clauses.join(', ')};`;
+            }),
+            ...passthroughSqls,
+        ];
+    }
+    shouldLogInitializeProgress(current, total, step) {
+        return total <= step || current === 1 || current === total || current % step === 0;
+    }
+    isFullManagedReset(option) {
+        return (option?.ifExists || 'drop') === 'drop';
+    }
+    getCreateOptionForInitialize() {
+        return {
+            ifExists: 'omit',
+        };
+    }
+    async resetManagedSchema(option) {
+        const tableNames = Object.keys(this.translator.schema).filter((entity) => this.shouldResetEntity(option, entity)).map((entity) => this.translator.schema[entity].storageName || entity);
+        if (tableNames.length === 0) {
+            return;
+        }
+        await this.connector.withConnection(async (connection) => {
+            // MySQL 的 reset 必须在同一个 session 里关闭 FK 检查。
+            // 而且要把同一批表放进尽量少的 DROP TABLE 语句里；
+            // 逐表 DROP 在复杂 FK 图上会触发 metadata lock deadlock。
+            await connection.query('SET FOREIGN_KEY_CHECKS = 0;');
+            try {
+                await connection.query(this.buildResetDropSql(tableNames));
+            }
+            finally {
+                try {
+                    await connection.query('SET FOREIGN_KEY_CHECKS = 1;');
+                }
+                catch (err) {
+                    connection.destroy();
+                    throw err;
+                }
+            }
+        });
+    }
+    async createManagedSchema(option) {
         const schema = this.getSchema();
-        for (const entity in schema) {
+        const entities = Object.keys(schema);
+        // drop/reset 阶段已经独立完成，这里统一走 omit create，
+        // 避免 translator 再带入“每个实体自带 drop SQL”的旧流程。
+        for (let idx = 0; idx < entities.length; idx++) {
+            const entity = entities[idx];
+            if (this.shouldLogInitializeProgress(idx + 1, entities.length, 10)) {
+                console.log(`Initializing MySQL tables (${idx + 1}/${entities.length}): ${entity}`);
+            }
             const sqls = this.translator.translateCreateEntity(entity, option);
             for (const sql of sqls) {
                 await this.connector.exec(sql);
             }
         }
     }
+    async applyForeignKeysFromPlan(plan) {
+        const sqls = [
+            ...plan.forwardSql.filter((stmt) => /foreign key/i.test(stmt)),
+            ...plan.manualSql.filter((stmt) => /add constraint .*foreign key/i.test(stmt)),
+        ];
+        const batchedSqls = this.buildBatchAddForeignKeySql(sqls);
+        // initialize 阶段只执行“新增 FK”动作：
+        // 全量 reset 直接使用目标 schema 里的声明；半保留场景则先 inspect 再只补缺失项。
+        for (let idx = 0; idx < batchedSqls.length; idx++) {
+            if (this.shouldLogInitializeProgress(idx + 1, batchedSqls.length, 20)) {
+                console.log(`Initializing MySQL foreign keys (${idx + 1}/${batchedSqls.length})`);
+            }
+            const sql = batchedSqls[idx];
+            await this.connector.exec(sql);
+        }
+    }
+    async applyDeclaredForeignKeys() {
+        const plan = this.connector.diffSchema({}, this.translator.schema, this.translator, {
+            compareForeignKeys: true,
+        });
+        await this.applyForeignKeysFromPlan(plan);
+    }
+    async applyMissingForeignKeys() {
+        const inspection = await this.connector.inspectSchema(this.translator);
+        const plan = this.connector.diffSchema(inspection.schema, this.translator.schema, this.translator, {
+            compareForeignKeys: true,
+            currentTableStats: inspection.tableStats,
+        });
+        await this.applyForeignKeysFromPlan(plan);
+    }
+    async initialize(option) {
+        console.log('Initializing MySQL schema reset phase...');
+        await this.resetManagedSchema(option);
+        console.log('Initializing MySQL schema create phase...');
+        await this.createManagedSchema(this.getCreateOptionForInitialize());
+        console.log('Initializing MySQL foreign key phase...');
+        if (this.isFullManagedReset(option)) {
+            await this.applyDeclaredForeignKeys();
+        }
+        else {
+            await this.applyMissingForeignKeys();
+        }
+    }
     // 从数据库中读取当前schema
     readSchema() {
-        return this.translator.readSchema((sql) => this.connector.exec(sql));
+        return this.connector.readSchema(this.translator);
     }
     /**
      * 根据载入的dataSchema，和数据库中原来的schema，决定如何来upgrade
      * 制订出来的plan分为两阶段：增加阶段和削减阶段，在两个阶段之间，由用户来修正数据
      */
-    async makeUpgradePlan() {
-        const originSchema = await this.readSchema();
-        const plan = this.diffSchema(originSchema, this.translator.schema);
+    async makeUpgradePlan(options) {
+        const inspection = await this.connector.inspectSchema(this.translator);
+        // 自动探测“历史库完全没有物理 FK”的情况：
+        // 这类旧库先关闭 FK 对比，避免 upgrade 持续产出噪音计划。
+        const hasPhysicalForeignKeys = Object.values(inspection.schema).some((tableDef) => Object.values(tableDef.attributes || {}).some((attr) => {
+            const attribute = attr;
+            return attribute.type === 'ref' && typeof attribute.ref === 'string';
+        }));
+        const plan = this.diffSchema(inspection.schema, this.translator.schema, {
+            ...options,
+            currentTableStats: options?.currentTableStats || inspection.tableStats,
+            compareForeignKeys: options?.compareForeignKeys ?? hasPhysicalForeignKeys,
+        });
         return plan;
     }
     /**
@@ -331,109 +472,8 @@ class MysqlStore extends CascadeStore_1.CascadeStore {
      * @param schemaOld
      * @param SchemaNew
      */
-    diffSchema(schemaOld, schemaNew) {
-        const plan = {
-            newTables: {},
-            newIndexes: {},
-            updatedIndexes: {},
-            updatedTables: {},
-        };
-        for (const table in schemaNew) {
-            // mysql数据字典不分大小写的
-            if (schemaOld[table] || schemaOld[table.toLowerCase()]) {
-                const { attributes, indexes } = schemaOld[table] || schemaOld[table.toLowerCase()];
-                const { attributes: attributesNew, indexes: indexesNew } = schemaNew[table];
-                const assignToUpdateTables = (attr, isNew) => {
-                    if (!plan.updatedTables[table]) {
-                        plan.updatedTables[table] = {
-                            attributes: {
-                                [attr]: {
-                                    ...attributesNew[attr],
-                                    isNew,
-                                }
-                            }
-                        };
-                    }
-                    else {
-                        plan.updatedTables[table].attributes[attr] = {
-                            ...attributesNew[attr],
-                            isNew,
-                        };
-                    }
-                };
-                for (const attr in attributesNew) {
-                    if (attributes[attr]) {
-                        // 因为反向无法复原原来定义的attribute类型，这里就比较两次创建的sql是不是一致，不是太好的设计。
-                        const sql1 = this.translator.translateAttributeDef(attr, attributesNew[attr]);
-                        const sql2 = this.translator.translateAttributeDef(attr, attributes[attr]);
-                        if (!this.translator.compareSql(sql1, sql2)) {
-                            assignToUpdateTables(attr, false);
-                        }
-                    }
-                    else {
-                        assignToUpdateTables(attr, true);
-                    }
-                }
-                if (indexesNew) {
-                    const assignToIndexes = (index, isNew) => {
-                        if (isNew) {
-                            if (plan.newIndexes[table]) {
-                                plan.newIndexes[table].push(index);
-                            }
-                            else {
-                                plan.newIndexes[table] = [index];
-                            }
-                        }
-                        else {
-                            if (plan.updatedIndexes[table]) {
-                                plan.updatedIndexes[table].push(index);
-                            }
-                            else {
-                                plan.updatedIndexes[table] = [index];
-                            }
-                        }
-                    };
-                    const compareConfig = (config1, config2) => {
-                        const unique1 = config1?.unique || false;
-                        const unique2 = config2?.unique || false;
-                        if (unique1 !== unique2) {
-                            return false;
-                        }
-                        const type1 = config1?.type || 'btree';
-                        const type2 = config2?.type || 'btree';
-                        // parser目前无法从mysql中读出来，所以不比了
-                        return type1 === type2;
-                    };
-                    for (const index of indexesNew) {
-                        const { name, config, attributes } = index;
-                        const origin = indexes?.find(ele => ele.name === name);
-                        if (origin) {
-                            if (JSON.stringify(attributes) !== JSON.stringify(origin.attributes)) {
-                                // todo，这里要细致比较，不能用json.stringify
-                                assignToIndexes(index, false);
-                            }
-                            else {
-                                if (!compareConfig(config, origin.config)) {
-                                    assignToIndexes(index, false);
-                                }
-                            }
-                        }
-                        else {
-                            assignToIndexes(index, true);
-                        }
-                    }
-                }
-            }
-            else {
-                plan.newTables[table] = {
-                    attributes: schemaNew[table].attributes,
-                };
-                if (schemaNew[table].indexes) {
-                    plan.newIndexes[table] = schemaNew[table].indexes;
-                }
-            }
-        }
-        return plan;
+    diffSchema(schemaOld, schemaNew, options) {
+        return this.connector.diffSchema(schemaOld, schemaNew, this.translator, options);
     }
 }
 exports.MysqlStore = MysqlStore;

package/lib/MySQL/translator.d.ts

@@ -8,6 +8,9 @@ export interface MySqlSelectOption extends SqlSelectOption {
 export interface MysqlOperateOption extends SqlOperateOption {
 }
 export declare class MySqlTranslator<ED extends EntityDict & BaseEntityDict> extends SqlTranslator<ED> {
+    readonly maxIndexNameLength = 64;
+    getPhysicalIndexName(entityName: string, tableName: string, logicalName: string, suffix?: string): string;
+    getLegacyPhysicalIndexNames(entityName: string, tableName: string, logicalName: string, suffix?: string): string[];
     protected getDefaultSelectFilter(alias: string, option?: MySqlSelectOption): string;
     private makeUpSchema;
     constructor(schema: StorageSchema<ED>);
@@ -98,6 +101,7 @@ export declare class MySqlTranslator<ED extends EntityDict & BaseEntityDict> ext
     protected translateAttrValue(dataType: DataType | Ref, value: any): string;
     protected translateFullTextSearch<T extends keyof ED>(value: Q_FullTextValue, entity: T, alias: string): string;
     translateAttributeDef(attr: string, attrDef: Attribute): string;
+    translateColumnDefinition(entity: string, attr: string, attrDef: Attribute): string;
     translateCreateEntity<T extends keyof ED>(entity: T, options?: CreateEntityOption): string[];
     private translateFnName;
     private translateAttrInExpression;
@@ -109,6 +113,6 @@ export declare class MySqlTranslator<ED extends EntityDict & BaseEntityDict> ext
      * 将MySQL返回的Type回译成oak的类型，是 populateDataTypeDef 的反函数
      * @param type
      */
-    private reTranslateToAttribute;
+    reTranslateToAttribute(type: string): Attribute;
     readSchema(execFn: (sql: string) => Promise<any>): Promise<StorageSchema<ED>>;
 }

package/lib/MySQL/translator.js

@@ -7,6 +7,7 @@ const util_1 = require("util");
 const lodash_1 = require("lodash");
 const types_1 = require("oak-domain/lib/types");
 const sqlTranslator_1 = require("../sqlTranslator");
+const indexName_1 = require("../utils/indexName");
 const GeoTypes = [
     {
         type: 'point',
@@ -72,6 +73,26 @@ function transformGeoData(data) {
     }
 }
 class MySqlTranslator extends sqlTranslator_1.SqlTranslator {
+    maxIndexNameLength = 64;
+    getPhysicalIndexName(entityName, tableName, logicalName, suffix = '') {
+        return (0, indexName_1.buildCompactPhysicalIndexName)({
+            entityName,
+            tableName,
+            logicalName,
+            suffix,
+            maxLength: this.maxIndexNameLength,
+        });
+    }
+    getLegacyPhysicalIndexNames(entityName, tableName, logicalName, suffix = '') {
+        return (0, indexName_1.buildLegacyPhysicalIndexNames)({
+            entityName,
+            tableName,
+            logicalName,
+            suffix,
+            maxLength: this.maxIndexNameLength,
+            serverTruncatesWhenOverflow: false,
+        });
+    }
     getDefaultSelectFilter(alias, option) {
         if (option?.includedDeleted) {
             return '';
@@ -579,6 +600,9 @@ class MySqlTranslator extends sqlTranslator_1.SqlTranslator {
         }
         return sql;
     }
+    translateColumnDefinition(entity, attr, attrDef) {
+        return this.translateAttributeDef(attr, attrDef);
+    }
     translateCreateEntity(entity, options) {
         const ifExists = options?.ifExists || 'drop';
         const { schema } = this;
@@ -619,17 +643,17 @@ class MySqlTranslator extends sqlTranslator_1.SqlTranslator {
                 sql += ',\n';
                 indexes.forEach(({ name, attributes, config }, idx) => {
                     const { unique, type, parser } = config || {};
-                    // 因为有deleteAt的存在，这里的unique没意义，只能框架自己去建立checker来处理
-                    /* if (unique) {
+                    const physicalName = this.getPhysicalIndexName(String(entity), storageName || entity, name);
+                    if (unique) {
                         sql += ' unique ';
                     }
-                    else */ if (type === 'fulltext') {
+                    else if (type === 'fulltext') {
                         sql += ' fulltext ';
                     }
                     else if (type === 'spatial') {
                         sql += ' spatial ';
                     }
-                    sql += `index \`${name}\` `;
+                    sql += `index \`${physicalName}\` `;
                     if (type === 'hash') {
                         sql += ` using hash `;
                     }
@@ -1136,7 +1160,7 @@ class MySqlTranslator extends sqlTranslator_1.SqlTranslator {
             };
         }
         const withPrecisionDataTypes = MySqlTranslator.withPrecisionDataTypes.join('|');
-        result = (new RegExp(`^(${withPrecisionDataTypes})\\((\\d+),(d+)\\)$`)).exec(type);
+        result = (new RegExp(`^(${withPrecisionDataTypes})\\((\\d+),(\\d+)\\)$`)).exec(type);
         if (result) {
             return {
                 type: result[1],

package/lib/PostgreSQL/connector.d.ts

@@ -1,10 +1,15 @@
 import { Pool, PoolClient, QueryResult, QueryResultRow } from 'pg';
 import { TxnOption } from 'oak-domain/lib/types';
+import { EntityDict, StorageSchema } from 'oak-domain/lib/types';
+import { EntityDict as BaseEntityDict } from 'oak-domain/lib/base-app-domain';
 import { PostgreSQLConfiguration } from './types/Configuration';
+import { PostgreSQLTranslator } from './translator';
+import { MigrationPlanningOptions, Plan, SchemaInspectionResult } from '../types/migration';
 export declare class PostgreSQLConnector {
     pool: Pool;
     configuration: PostgreSQLConfiguration;
     txnDict: Record<string, PoolClient>;
+    txnExecQueueDict: Record<string, Promise<void>>;
     constructor(configuration: PostgreSQLConfiguration);
     connect(): void;
     disconnect(): Promise<void>;
@@ -14,6 +19,11 @@ export declare class PostgreSQLConnector {
      */
     private mapIsolationLevel;
     exec(sql: string, txn?: string): Promise<[QueryResultRow[], QueryResult]>;
+    private enqueueTxnQuery;
+    private enqueueTxnTask;
+    readSchema<ED extends EntityDict & BaseEntityDict>(translator: PostgreSQLTranslator<ED>): Promise<StorageSchema<ED>>;
+    inspectSchema<ED extends EntityDict & BaseEntityDict>(translator: PostgreSQLTranslator<ED>): Promise<SchemaInspectionResult<ED>>;
+    diffSchema<ED extends EntityDict & BaseEntityDict>(currentSchema: StorageSchema<ED>, targetSchema: StorageSchema<ED>, translator: PostgreSQLTranslator<ED>, options?: MigrationPlanningOptions): Plan<ED>;
     commitTransaction(txn: string): Promise<void>;
     rollbackTransaction(txn: string): Promise<void>;
     /**

package/lib/PostgreSQL/connector.js

@@ -5,13 +5,16 @@ const tslib_1 = require("tslib");
 const pg_1 = require("pg");
 const uuid_1 = require("uuid");
 const assert_1 = tslib_1.__importDefault(require("assert"));
+const migration_1 = require("./migration");
 class PostgreSQLConnector {
     pool;
     configuration;
     txnDict;
+    txnExecQueueDict;
     constructor(configuration) {
         this.configuration = configuration;
         this.txnDict = {};
+        this.txnExecQueueDict = {};
         this.pool = new pg_1.Pool(configuration);
         // 错误处理
         this.pool.on('error', (err) => {
@@ -49,12 +52,14 @@ class PostgreSQLConnector {
             }
             await connection.query(beginStmt);
             this.txnDict[id] = connection;
+            this.txnExecQueueDict[id] = Promise.resolve();
             return id;
         }
         catch (error) {
             // 如果启动事务失败，释放连接
             connection.release();
             delete this.txnDict[id];
+            delete this.txnExecQueueDict[id];
             throw error;
         }
     }
@@ -80,37 +85,58 @@ class PostgreSQLConnector {
         if (process.env.NODE_ENV === 'development') {
             // console.log(`SQL: ${sql}; \n`);
         }
-        let result;
         if (txn) {
-            const connection = this.txnDict[txn];
-            (0, assert_1.default)(connection, `Transaction ${txn} not found`);
-            result = await connection.query(sql);
-        }
-        else {
-            result = await this.pool.query(sql);
+            const result = await this.enqueueTxnQuery(txn, (connection) => connection.query(sql));
+            // 返回格式与 mysql2 兼容: [rows, fields/result]
+            return [result.rows, result];
         }
+        const result = await this.pool.query(sql);
         // 返回格式与 mysql2 兼容: [rows, fields/result]
         return [result.rows, result];
     }
+    async enqueueTxnQuery(txn, executor) {
+        const connection = this.txnDict[txn];
+        (0, assert_1.default)(connection, `Transaction ${txn} not found`);
+        return this.enqueueTxnTask(txn, () => executor(connection));
+    }
+    async enqueueTxnTask(txn, executor) {
+        const queued = this.txnExecQueueDict[txn] || Promise.resolve();
+        const task = queued
+            .catch(() => undefined)
+            .then(() => executor());
+        this.txnExecQueueDict[txn] = task.then(() => undefined, () => undefined);
+        return task;
+    }
+    async readSchema(translator) {
+        return (0, migration_1.readPostgreSqlSchema)(this, translator);
+    }
+    async inspectSchema(translator) {
+        return (0, migration_1.inspectPostgreSqlSchema)(this, translator);
+    }
+    diffSchema(currentSchema, targetSchema, translator, options) {
+        return (0, migration_1.buildPostgreSqlMigrationPlan)(currentSchema, targetSchema, translator, options);
+    }
     async commitTransaction(txn) {
         const connection = this.txnDict[txn];
         (0, assert_1.default)(connection, `Transaction ${txn} not found`);
+        delete this.txnDict[txn];
         try {
-            await connection.query('COMMIT');
+            await this.enqueueTxnTask(txn, () => connection.query('COMMIT'));
         }
         finally {
-            delete this.txnDict[txn];
+            delete this.txnExecQueueDict[txn];
             connection.release();
         }
     }
     async rollbackTransaction(txn) {
         const connection = this.txnDict[txn];
         (0, assert_1.default)(connection, `Transaction ${txn} not found`);
+        delete this.txnDict[txn];
         try {
-            await connection.query('ROLLBACK');
+            await this.enqueueTxnTask(txn, () => connection.query('ROLLBACK'));
         }
         finally {
-            delete this.txnDict[txn];
+            delete this.txnExecQueueDict[txn];
             connection.release();
         }
     }

package/lib/PostgreSQL/migration.d.ts

@@ -0,0 +1,10 @@
+import { EntityDict } from 'oak-domain/lib/types';
+import { EntityDict as BaseEntityDict } from 'oak-domain/lib/base-app-domain';
+import { MigrationSchema, MigrationPlanningOptions, Plan, SchemaInspectionResult } from '../types/migration';
+import { PostgreSQLConnector } from './connector';
+import { PostgreSQLTranslator } from './translator';
+type PostgreSqlMigrationEntityDict = EntityDict & BaseEntityDict;
+export declare function readPostgreSqlSchema<ED extends PostgreSqlMigrationEntityDict>(connector: PostgreSQLConnector, translator: PostgreSQLTranslator<ED>): Promise<MigrationSchema<ED>>;
+export declare function inspectPostgreSqlSchema<ED extends PostgreSqlMigrationEntityDict>(connector: PostgreSQLConnector, translator: PostgreSQLTranslator<ED>): Promise<SchemaInspectionResult<ED>>;
+export declare function buildPostgreSqlMigrationPlan<ED extends PostgreSqlMigrationEntityDict>(currentSchema: MigrationSchema<ED>, targetSchema: MigrationSchema<ED>, translator: PostgreSQLTranslator<ED>, options?: MigrationPlanningOptions): Plan<ED>;
+export {};