dbgate-tools 6.7.2 → 6.8.0

package/lib/DatabaseAnalyser.js

@@ -151,6 +151,11 @@ class DatabaseAnalyser {
         }
         const res = {};
         for (const field of STRUCTURE_FIELDS) {
+            const isAll = this.modifications.some(x => x.action == 'all' && x.objectTypeField == field);
+            if (isAll) {
+                res[field] = newlyAnalysed[field] || [];
+                continue;
+            }
             const removedIds = this.modifications
                 .filter(x => x.action == 'remove' && x.objectTypeField == field)
                 .map(x => x.objectId);

package/lib/SqlDumper.d.ts

@@ -116,6 +116,7 @@ export declare class SqlDumper implements AlterProcessor {
     selectTableIntoNewTable(sourceName: NamedObjectInfo, targetName: NamedObjectInfo): void;
     truncateTable(name: NamedObjectInfo): void;
     dropConstraints(table: TableInfo, dropReferences?: boolean): void;
+    sanitizeTableConstraints(table: TableInfo): TableInfo;
     recreateTable(oldTable: TableInfo, newTable: TableInfo): void;
     createSqlObject(obj: SqlObjectInfo): void;
     getSqlObjectSqlName(ojectTypeField: string): "PROCEDURE" | "VIEW" | "FUNCTION" | "TRIGGER" | "MATERIALIZED VIEW" | "EVENT";

package/lib/SqlDumper.js

@@ -10,6 +10,7 @@ const isDate_1 = __importDefault(require("lodash/isDate"));
 const isArray_1 = __importDefault(require("lodash/isArray"));
 const isPlainObject_1 = __importDefault(require("lodash/isPlainObject"));
 const keys_1 = __importDefault(require("lodash/keys"));
+const cloneDeep_1 = __importDefault(require("lodash/cloneDeep"));
 const v1_1 = __importDefault(require("uuid/v1"));
 class SqlDumper {
     static convertKeywordCase(keyword) {
@@ -76,6 +77,8 @@ class SqlDumper {
         }
         else if (value === null || value === void 0 ? void 0 : value.$bigint)
             this.putRaw(value === null || value === void 0 ? void 0 : value.$bigint);
+        else if (value === null || value === void 0 ? void 0 : value.$decimal)
+            this.putRaw(value === null || value === void 0 ? void 0 : value.$decimal);
         else if ((0, isPlainObject_1.default)(value) || (0, isArray_1.default)(value))
             this.putStringValue(JSON.stringify(value));
         else
@@ -598,6 +601,55 @@ class SqlDumper {
             this.dropPrimaryKey(table.primaryKey);
         }
     }
+    sanitizeTableConstraints(table) {
+        // Create a deep copy of the table
+        const sanitized = (0, cloneDeep_1.default)(table);
+        // Get the set of existing column names
+        const existingColumns = new Set(sanitized.columns.map(col => col.columnName));
+        // Filter primary key columns to only include existing columns
+        if (sanitized.primaryKey) {
+            const validPkColumns = sanitized.primaryKey.columns.filter(col => existingColumns.has(col.columnName));
+            if (validPkColumns.length === 0) {
+                // If no valid columns remain, remove the primary key entirely
+                sanitized.primaryKey = null;
+            }
+            else if (validPkColumns.length < sanitized.primaryKey.columns.length) {
+                // Update primary key with only valid columns
+                sanitized.primaryKey = {
+                    ...sanitized.primaryKey,
+                    columns: validPkColumns
+                };
+            }
+        }
+        // Filter sorting key columns to only include existing columns
+        if (sanitized.sortingKey) {
+            const validSkColumns = sanitized.sortingKey.columns.filter(col => existingColumns.has(col.columnName));
+            if (validSkColumns.length === 0) {
+                sanitized.sortingKey = null;
+            }
+            else if (validSkColumns.length < sanitized.sortingKey.columns.length) {
+                sanitized.sortingKey = {
+                    ...sanitized.sortingKey,
+                    columns: validSkColumns
+                };
+            }
+        }
+        // Filter foreign keys to only include those with all columns present
+        if (sanitized.foreignKeys) {
+            sanitized.foreignKeys = sanitized.foreignKeys.filter(fk => fk.columns.every(col => existingColumns.has(col.columnName)));
+        }
+        // Filter indexes to only include those with all columns present
+        if (sanitized.indexes) {
+            sanitized.indexes = sanitized.indexes.filter(idx => idx.columns.every(col => existingColumns.has(col.columnName)));
+        }
+        // Filter unique constraints to only include those with all columns present
+        if (sanitized.uniques) {
+            sanitized.uniques = sanitized.uniques.filter(uq => uq.columns.every(col => existingColumns.has(col.columnName)));
+        }
+        // Filter dependencies (references from other tables) - these should remain as-is
+        // since they don't affect the CREATE TABLE statement for this table
+        return sanitized;
+    }
     recreateTable(oldTable, newTable) {
         if (!oldTable.pairingId || !newTable.pairingId || oldTable.pairingId != newTable.pairingId) {
             throw new Error('Recreate is not possible: oldTable.paringId != newTable.paringId');
@@ -609,29 +661,31 @@ class SqlDumper {
             newcol: newTable.columns.find(x => x.pairingId == oldcol.pairingId),
         }))
             .filter(x => x.newcol);
+        // Create a sanitized version of newTable with constraints that only reference existing columns
+        const sanitizedNewTable = this.sanitizeTableConstraints(newTable);
         if (this.driver.supportsTransactions) {
             this.dropConstraints(oldTable, true);
             this.renameTable(oldTable, tmpTable);
-            this.createTable(newTable);
-            const autoinc = newTable.columns.find(x => x.autoIncrement);
+            this.createTable(sanitizedNewTable);
+            const autoinc = sanitizedNewTable.columns.find(x => x.autoIncrement);
             if (autoinc) {
-                this.allowIdentityInsert(newTable, true);
+                this.allowIdentityInsert(sanitizedNewTable, true);
             }
-            this.putCmd('^insert ^into %f (%,i) select %,i ^from %f', newTable, columnPairs.map(x => x.newcol.columnName), columnPairs.map(x => x.oldcol.columnName), { ...oldTable, pureName: tmpTable });
+            this.putCmd('^insert ^into %f (%,i) select %,i ^from %f', sanitizedNewTable, columnPairs.map(x => x.newcol.columnName), columnPairs.map(x => x.oldcol.columnName), { ...oldTable, pureName: tmpTable });
             if (autoinc) {
-                this.allowIdentityInsert(newTable, false);
+                this.allowIdentityInsert(sanitizedNewTable, false);
             }
             if (this.dialect.dropForeignKey) {
-                newTable.dependencies.forEach(cnt => this.createConstraint(cnt));
+                sanitizedNewTable.dependencies.forEach(cnt => this.createConstraint(cnt));
             }
             this.dropTable({ ...oldTable, pureName: tmpTable });
         }
         else {
             // we have to preserve old table as long as possible
-            this.createTable({ ...newTable, pureName: tmpTable });
-            this.putCmd('^insert ^into %f (%,i) select %,s ^from %f', { ...newTable, pureName: tmpTable }, columnPairs.map(x => x.newcol.columnName), columnPairs.map(x => x.oldcol.columnName), oldTable);
+            this.createTable({ ...sanitizedNewTable, pureName: tmpTable });
+            this.putCmd('^insert ^into %f (%,i) select %,s ^from %f', { ...sanitizedNewTable, pureName: tmpTable }, columnPairs.map(x => x.newcol.columnName), columnPairs.map(x => x.oldcol.columnName), oldTable);
             this.dropTable(oldTable);
-            this.renameTable({ ...newTable, pureName: tmpTable }, newTable.pureName);
+            this.renameTable({ ...sanitizedNewTable, pureName: tmpTable }, newTable.pureName);
         }
     }
     createSqlObject(obj) {

package/lib/alterPlan.d.ts

@@ -65,8 +65,8 @@ interface AlterOperation_RenameConstraint {
 }
 interface AlterOperation_RecreateTable {
     operationType: 'recreateTable';
-    table: TableInfo;
-    operations: AlterOperation[];
+    oldTable: TableInfo;
+    newTable: TableInfo;
 }
 interface AlterOperation_FillPreloadedRows {
     operationType: 'fillPreloadedRows';
@@ -110,7 +110,7 @@ export declare class AlterPlan {
     renameSqlObject(table: TableInfo, newName: string): void;
     renameColumn(column: ColumnInfo, newName: string): void;
     renameConstraint(constraint: ConstraintInfo, newName: string): void;
-    recreateTable(table: TableInfo, operations: AlterOperation[]): void;
+    recreateTable(oldTable: TableInfo, newTable: TableInfo): void;
     fillPreloadedRows(table: NamedObjectInfo, oldRows: any[], newRows: any[], key: string[], insertOnly: string[], autoIncrementColumn: string): void;
     setTableOption(table: TableInfo, optionName: string, optionValue: string): void;
     run(processor: AlterProcessor): void;
@@ -121,6 +121,7 @@ export declare class AlterPlan {
     _canCreateConstraint(cnt: ConstraintInfo): boolean;
     _canDropConstraint(cnt: ConstraintInfo): boolean;
     _testTableRecreate(op: AlterOperation, operationType: string, isAllowed: boolean | Function, objectField: string): AlterOperation[] | null;
+    _removeRecreatedTableAlters(): AlterOperation[];
     _groupTableRecreations(): AlterOperation[];
     _moveForeignKeysToLast(): AlterOperation[];
     _filterAllowedOperations(): AlterOperation[];

package/lib/alterPlan.js

@@ -5,9 +5,6 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.runAlterOperation = exports.AlterPlan = void 0;
 const lodash_1 = __importDefault(require("lodash"));
-const diffTools_1 = require("./diffTools");
-const database_info_alter_processor_1 = require("./database-info-alter-processor");
-const DatabaseAnalyser_1 = require("./DatabaseAnalyser");
 class AlterPlan {
     constructor(wholeOldDb, wholeNewDb, dialect, opts) {
         this.wholeOldDb = wholeOldDb;
@@ -111,11 +108,11 @@ class AlterPlan {
             newName,
         });
     }
-    recreateTable(table, operations) {
+    recreateTable(oldTable, newTable) {
         this.operations.push({
             operationType: 'recreateTable',
-            table,
-            operations,
+            oldTable,
+            newTable,
         });
         this.recreates.tables += 1;
     }
@@ -181,7 +178,11 @@ class AlterPlan {
                         return opRes;
                     }),
                     op,
-                ];
+                ].filter(op => {
+                    // filter duplicated drops
+                    const existingDrop = this.operations.find(o => o.operationType == 'dropConstraint' && o.oldObject === op['oldObject']);
+                    return existingDrop == null;
+                });
                 return res;
             }
             for (const [testedOperationType, testedDependencies, testedObject] of [
@@ -325,50 +326,110 @@ class AlterPlan {
                 // skip this operation, as it cannot be achieved
                 return [];
             }
-            const table = this.wholeNewDb.tables.find(x => x.pureName == op[objectField].pureName && x.schemaName == op[objectField].schemaName);
+            const oldTable = this.wholeOldDb.tables.find(x => x.pureName == op[objectField].pureName && x.schemaName == op[objectField].schemaName);
+            const newTable = this.wholeNewDb.tables.find(x => x.pureName == op[objectField].pureName && x.schemaName == op[objectField].schemaName);
             this.recreates.tables += 1;
             return [
                 {
                     operationType: 'recreateTable',
-                    table,
-                    operations: [op],
+                    oldTable,
+                    newTable,
+                    // operations: [op],
                 },
             ];
         }
         return null;
     }
-    _groupTableRecreations() {
+    _removeRecreatedTableAlters() {
         const res = [];
-        const recreates = {};
+        const recreates = new Set();
         for (const op of this.operations) {
-            if (op.operationType == 'recreateTable' && op.table) {
-                const existingRecreate = recreates[`${op.table.schemaName}||${op.table.pureName}`];
-                if (existingRecreate) {
-                    existingRecreate.operations.push(...op.operations);
-                }
-                else {
-                    const recreate = {
-                        ...op,
-                        operations: [...op.operations],
-                    };
-                    res.push(recreate);
-                    recreates[`${op.table.schemaName}||${op.table.pureName}`] = recreate;
-                }
+            if (op.operationType == 'recreateTable' && op.oldTable && op.newTable) {
+                const key = `${op.oldTable.schemaName}||${op.oldTable.pureName}`;
+                recreates.add(key);
             }
-            else {
-                // @ts-ignore
-                const oldObject = op.oldObject || op.object;
-                if (oldObject) {
-                    const recreated = recreates[`${oldObject.schemaName}||${oldObject.pureName}`];
-                    if (recreated) {
-                        recreated.operations.push(op);
-                        continue;
+        }
+        for (const op of this.operations) {
+            switch (op.operationType) {
+                case 'createColumn':
+                case 'createConstraint':
+                    {
+                        const key = `${op.newObject.schemaName}||${op.newObject.pureName}`;
+                        if (recreates.has(key)) {
+                            // skip create inside recreated table
+                            continue;
+                        }
+                    }
+                    break;
+                case 'dropColumn':
+                case 'dropConstraint':
+                case 'changeColumn':
+                    {
+                        const key = `${op.oldObject.schemaName}||${op.oldObject.pureName}`;
+                        if (recreates.has(key)) {
+                            // skip drop/change inside recreated table
+                            continue;
+                        }
                     }
+                    break;
+                case 'renameColumn':
+                    {
+                        const key = `${op.object.schemaName}||${op.object.pureName}`;
+                        if (recreates.has(key)) {
+                            // skip rename inside recreated table
+                            continue;
+                        }
+                    }
+                    break;
+            }
+            res.push(op);
+        }
+        return res;
+    }
+    _groupTableRecreations() {
+        const res = [];
+        const recreates = new Set();
+        for (const op of this.operations) {
+            if (op.operationType == 'recreateTable' && op.oldTable && op.newTable) {
+                const key = `${op.oldTable.schemaName}||${op.oldTable.pureName}`;
+                if (recreates.has(key)) {
+                    // prevent duplicate recreates
+                    continue;
                 }
-                res.push(op);
+                recreates.add(key);
             }
+            res.push(op);
         }
         return res;
+        // const res = [];
+        // const recreates = {};
+        // for (const op of this.operations) {
+        //   if (op.operationType == 'recreateTable' && op.table) {
+        //     const existingRecreate = recreates[`${op.table.schemaName}||${op.table.pureName}`];
+        //     if (existingRecreate) {
+        //       existingRecreate.operations.push(...op.operations);
+        //     } else {
+        //       const recreate = {
+        //         ...op,
+        //         operations: [...op.operations],
+        //       };
+        //       res.push(recreate);
+        //       recreates[`${op.table.schemaName}||${op.table.pureName}`] = recreate;
+        //     }
+        //   } else {
+        //     // @ts-ignore
+        //     const oldObject: TableInfo = op.oldObject || op.object;
+        //     if (oldObject) {
+        //       const recreated = recreates[`${oldObject.schemaName}||${oldObject.pureName}`];
+        //       if (recreated) {
+        //         recreated.operations.push(op);
+        //         continue;
+        //       }
+        //     }
+        //     res.push(op);
+        //   }
+        // }
+        // return res;
     }
     _moveForeignKeysToLast() {
         if (!this.dialect.createForeignKey) {
@@ -427,6 +488,7 @@ class AlterPlan {
         // console.log('*****************OPERATIONS2', this.operations);
         this.operations = this._groupTableRecreations();
         // console.log('*****************OPERATIONS3', this.operations);
+        this.operations = this._removeRecreatedTableAlters();
         this.operations = this._moveForeignKeysToLast();
         // console.log('*****************OPERATIONS4', this.operations);
         this.operations = this._filterAllowedOperations();
@@ -486,16 +548,16 @@ function runAlterOperation(op, processor) {
             break;
         case 'recreateTable':
             {
-                const oldTable = (0, diffTools_1.generateTablePairingId)(op.table);
-                const newTable = lodash_1.default.cloneDeep(oldTable);
-                const newDb = DatabaseAnalyser_1.DatabaseAnalyser.createEmptyStructure();
-                newDb.tables.push(newTable);
-                // console.log('////////////////////////////newTable1', newTable);
-                op.operations.forEach(child => runAlterOperation(child, new database_info_alter_processor_1.DatabaseInfoAlterProcessor(newDb)));
-                // console.log('////////////////////////////op.operations', op.operations);
-                // console.log('////////////////////////////op.table', op.table);
-                // console.log('////////////////////////////newTable2', newTable);
-                processor.recreateTable(oldTable, newTable);
+                // const oldTable = generateTablePairingId(op.table);
+                // const newTable = _.cloneDeep(oldTable);
+                // const newDb = DatabaseAnalyser.createEmptyStructure();
+                // newDb.tables.push(newTable);
+                // // console.log('////////////////////////////newTable1', newTable);
+                // op.operations.forEach(child => runAlterOperation(child, new DatabaseInfoAlterProcessor(newDb)));
+                // // console.log('////////////////////////////op.operations', op.operations);
+                // // console.log('////////////////////////////op.table', op.table);
+                // // console.log('////////////////////////////newTable2', newTable);
+                processor.recreateTable(op.oldTable, op.newTable);
             }
             break;
     }

package/lib/stringTools.d.ts

@@ -7,7 +7,7 @@ export declare function base64ToHex(base64String: any): string;
 export declare function hexToBase64(hexString: any): string;
 export declare function parseCellValue(value: any, editorTypes?: DataEditorTypesBehaviour): any;
 export declare function stringifyCellValue(value: any, intent: 'gridCellIntent' | 'inlineEditorIntent' | 'multilineEditorIntent' | 'stringConversionIntent' | 'exportIntent' | 'clipboardIntent', editorTypes?: DataEditorTypesBehaviour, gridFormattingOptions?: {
-    useThousandsSeparator?: boolean;
+    thousandsSeparator?: string;
 }, jsonParsedValue?: any): {
     value: string;
     gridStyle?: 'textCellStyle' | 'valueCellStyle' | 'nullCellStyle';

package/lib/stringTools.js

@@ -37,9 +37,11 @@ function base64ToHex(base64String) {
     return '0x' + hexString.toUpperCase();
 }
 exports.base64ToHex = base64ToHex;
-;
 function hexToBase64(hexString) {
-    const binaryString = hexString.match(/.{1,2}/g).map(byte => String.fromCharCode(parseInt(byte, 16))).join('');
+    const binaryString = hexString
+        .match(/.{1,2}/g)
+        .map(byte => String.fromCharCode(parseInt(byte, 16)))
+        .join('');
     return btoa(binaryString);
 }
 exports.hexToBase64 = hexToBase64;
@@ -55,8 +57,8 @@ function parseCellValue(value, editorTypes) {
         if (mHex) {
             return {
                 $binary: {
-                    base64: hexToBase64(value.substring(2))
-                }
+                    base64: hexToBase64(value.substring(2)),
+                },
             };
         }
     }
@@ -182,6 +184,30 @@ function stringifyJsonToGrid(value) {
     }
     return { value: '(JSON)', gridStyle: 'nullCellStyle' };
 }
+function formatNumberCustomSeparator(value, thousandsSeparator) {
+    const [intPart, decPart] = value.split('.');
+    const intPartWithSeparator = intPart.replace(/\B(?=(\d{3})+(?!\d))/g, thousandsSeparator);
+    return decPart ? `${intPartWithSeparator}.${decPart}` : intPartWithSeparator;
+}
+function formatCellNumber(value, gridFormattingOptions) {
+    const separator = gridFormattingOptions === null || gridFormattingOptions === void 0 ? void 0 : gridFormattingOptions.thousandsSeparator;
+    if ((0, isNumber_1.default)(value)) {
+        if (separator === 'none' || (value < 1000 && value > -1000))
+            return value.toString();
+        if (separator === 'system')
+            return value.toLocaleString();
+    }
+    // fallback for system locale
+    if (separator === 'space' || separator === 'system')
+        return formatNumberCustomSeparator(value.toString(), ' ');
+    if (separator === 'narrowspace')
+        return formatNumberCustomSeparator(value.toString(), '\u202F');
+    if (separator === 'comma')
+        return formatNumberCustomSeparator(value.toString(), ',');
+    if (separator === 'dot')
+        return formatNumberCustomSeparator(value.toString(), '.');
+    return value.toString();
+}
 function stringifyCellValue(value, intent, editorTypes, gridFormattingOptions, jsonParsedValue) {
     var _a, _b;
     if (editorTypes === null || editorTypes === void 0 ? void 0 : editorTypes.parseSqlNull) {
@@ -217,6 +243,12 @@ function stringifyCellValue(value, intent, editorTypes, gridFormattingOptions, j
             gridStyle: 'valueCellStyle',
         };
     }
+    if (value === null || value === void 0 ? void 0 : value.$decimal) {
+        return {
+            value: formatCellNumber(value.$decimal, gridFormattingOptions),
+            gridStyle: 'valueCellStyle',
+        };
+    }
     if (editorTypes === null || editorTypes === void 0 ? void 0 : editorTypes.parseHexAsBuffer) {
         // if (value?.type == 'Buffer' && _isArray(value.data)) {
         //   return { value: '0x' + arrayToHexString(value.data), gridStyle: 'valueCellStyle' };
@@ -235,13 +267,13 @@ function stringifyCellValue(value, intent, editorTypes, gridFormattingOptions, j
     }
     if (value === null || value === void 0 ? void 0 : value.$bigint) {
         return {
-            value: value.$bigint,
+            value: formatCellNumber(value.$bigint, gridFormattingOptions),
             gridStyle: 'valueCellStyle',
         };
     }
     if (typeof value === 'bigint') {
         return {
-            value: value.toString(),
+            value: formatCellNumber(value.toString(), gridFormattingOptions),
             gridStyle: 'valueCellStyle',
         };
     }
@@ -310,12 +342,8 @@ function stringifyCellValue(value, intent, editorTypes, gridFormattingOptions, j
     if ((0, isNumber_1.default)(value)) {
         switch (intent) {
             case 'gridCellIntent':
-                return {
-                    value: (gridFormattingOptions === null || gridFormattingOptions === void 0 ? void 0 : gridFormattingOptions.useThousandsSeparator) && (value >= 10000 || value <= -10000)
-                        ? value.toLocaleString()
-                        : value.toString(),
-                    gridStyle: 'valueCellStyle',
-                };
+                const separator = gridFormattingOptions === null || gridFormattingOptions === void 0 ? void 0 : gridFormattingOptions.thousandsSeparator;
+                return { value: formatCellNumber(value, gridFormattingOptions), gridStyle: 'valueCellStyle' };
             default:
                 return { value: value.toString() };
         }
@@ -406,6 +434,9 @@ function shouldOpenMultilineDialog(value) {
     if (value === null || value === void 0 ? void 0 : value.$bigint) {
         return false;
     }
+    if (value === null || value === void 0 ? void 0 : value.$decimal) {
+        return false;
+    }
     if ((0, isPlainObject_1.default)(value) || (0, isArray_1.default)(value)) {
         return true;
     }
@@ -669,6 +700,9 @@ function deserializeJsTypesFromJsonParse(obj) {
         if (value === null || value === void 0 ? void 0 : value.$bigint) {
             return BigInt(value.$bigint);
         }
+        if (value === null || value === void 0 ? void 0 : value.$decimal) {
+            return value.$decimal;
+        }
     });
 }
 exports.deserializeJsTypesFromJsonParse = deserializeJsTypesFromJsonParse;
@@ -683,6 +717,9 @@ function deserializeJsTypesReviver(key, value) {
     if (value === null || value === void 0 ? void 0 : value.$bigint) {
         return BigInt(value.$bigint);
     }
+    if (value === null || value === void 0 ? void 0 : value.$decimal) {
+        return value.$decimal;
+    }
     return value;
 }
 exports.deserializeJsTypesReviver = deserializeJsTypesReviver;

package/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "6.7.2",
+  "version": "6.8.0",
   "name": "dbgate-tools",
   "main": "lib/index.js",
   "typings": "lib/index.d.ts",
@@ -26,7 +26,7 @@
   ],
   "devDependencies": {
     "@types/node": "^13.7.0",
-    "dbgate-types": "^6.7.2",
+    "dbgate-types": "^6.8.0",
     "jest": "^28.1.3",
     "ts-jest": "^28.0.7",
     "typescript": "^4.4.3"
@@ -34,7 +34,7 @@
   "dependencies": {
     "blueimp-md5": "^2.19.0",
     "dbgate-query-splitter": "^4.11.9",
-    "dbgate-sqltree": "^6.7.2",
+    "dbgate-sqltree": "^6.8.0",
     "debug": "^4.3.4",
     "json-stable-stringify": "^1.0.1",
     "lodash": "^4.17.21",