dbgate-tools 4.6.0 → 4.7.0

package/lib/DatabaseAnalyser.d.ts

@@ -21,6 +21,7 @@ export declare class DatabaseAnalyser {
     getDeletedObjectsForField(snapshot: any, objectTypeField: any): any;
     getDeletedObjects(snapshot: any): any[];
     getModifications(): Promise<any[]>;
+    safeQuery(sql: any): Promise<import("dbgate-types").QueryResult>;
     static createEmptyStructure(): DatabaseInfo;
     static byTableFilter(table: any): (x: any) => boolean;
     static extractPrimaryKeys(table: any, pkColumns: any): {

package/lib/DatabaseAnalyser.js

@@ -19,6 +19,21 @@ const pick_1 = __importDefault(require("lodash/pick"));
 const compact_1 = __importDefault(require("lodash/compact"));
 const STRUCTURE_FIELDS = ['tables', 'collections', 'views', 'matviews', 'functions', 'procedures', 'triggers'];
 const fp_pick = arg => array => (0, pick_1.default)(array, arg);
+function mergeTableRowCounts(info, rowCounts) {
+    return Object.assign(Object.assign({}, info), { tables: (info.tables || []).map(table => {
+            var _a, _b;
+            return (Object.assign(Object.assign({}, table), { tableRowCount: (_b = (_a = rowCounts.find(x => x.objectId == table.objectId)) === null || _a === void 0 ? void 0 : _a.tableRowCount) !== null && _b !== void 0 ? _b : table.tableRowCount }));
+        }) });
+}
+function areDifferentRowCounts(db1, db2) {
+    for (const t1 of db1.tables || []) {
+        const t2 = (db2.tables || []).find(x => x.objectId == t1.objectId);
+        if ((t1 === null || t1 === void 0 ? void 0 : t1.tableRowCount) !== (t2 === null || t2 === void 0 ? void 0 : t2.tableRowCount)) {
+            return true;
+        }
+    }
+    return false;
+}
 class DatabaseAnalyser {
     constructor(pool, driver, version) {
         this.pool = pool;
@@ -78,14 +93,27 @@ class DatabaseAnalyser {
     incrementalAnalysis(structure) {
         return __awaiter(this, void 0, void 0, function* () {
             this.structure = structure;
-            this.modifications = yield this.getModifications();
-            if (this.modifications == null) {
+            const modifications = yield this.getModifications();
+            if (modifications == null) {
                 // modifications not implemented, perform full analysis
                 this.structure = null;
                 return this.addEngineField(yield this._runAnalysis());
             }
-            if (this.modifications.length == 0)
-                return null;
+            const structureModifications = modifications.filter(x => x.action != 'setTableRowCounts');
+            const setTableRowCounts = modifications.find(x => x.action == 'setTableRowCounts');
+            let structureWithRowCounts = null;
+            if (setTableRowCounts) {
+                const newStructure = mergeTableRowCounts(structure, setTableRowCounts.rowCounts);
+                if (areDifferentRowCounts(structure, newStructure)) {
+                    structureWithRowCounts = newStructure;
+                }
+            }
+            if (structureModifications.length == 0) {
+                return structureWithRowCounts ? this.addEngineField(structureWithRowCounts) : null;
+            }
+            this.modifications = structureModifications;
+            if (structureWithRowCounts)
+                this.structure = structureWithRowCounts;
             console.log('DB modifications detected:', this.modifications);
             return this.addEngineField(this.mergeAnalyseResult(yield this._runAnalysis()));
         });
@@ -228,9 +256,34 @@ class DatabaseAnalyser {
                     res.push(action);
                 }
             }
+            const rowCounts = (snapshot.tables || [])
+                .filter(x => x.tableRowCount != null)
+                .map(x => ({
+                objectId: x.objectId,
+                tableRowCount: x.tableRowCount,
+            }));
+            if (rowCounts.length > 0) {
+                res.push({
+                    action: 'setTableRowCounts',
+                    rowCounts,
+                });
+            }
             return [...(0, compact_1.default)(res), ...this.getDeletedObjects(snapshot)];
         });
     }
+    safeQuery(sql) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                return yield this.driver.query(this.pool, sql);
+            }
+            catch (err) {
+                console.log('Error running analyser query', err.message);
+                return {
+                    rows: [],
+                };
+            }
+        });
+    }
     static createEmptyStructure() {
         return {
             tables: [],

package/lib/SqlDumper.d.ts

@@ -9,12 +9,14 @@ export declare class SqlDumper implements AlterProcessor {
     putRaw(text: any): void;
     escapeString(value: any): string;
     putStringValue(value: any): void;
+    putByteArrayValue(value: any): void;
     putValue(value: any): void;
     putCmd(format: any, ...args: any[]): void;
     putFormattedValue(c: any, value: any): void;
     putFormattedList(c: any, collection: any): void;
     put(format: string, ...args: any[]): void;
     autoIncrement(): void;
+    specialColumnOptions(column: any): void;
     columnDefinition(column: ColumnInfo, { includeDefault, includeNullable, includeCollate }?: {
         includeDefault?: boolean;
         includeNullable?: boolean;

package/lib/SqlDumper.js

@@ -8,6 +8,8 @@ const lodash_1 = __importDefault(require("lodash"));
 const isString_1 = __importDefault(require("lodash/isString"));
 const isNumber_1 = __importDefault(require("lodash/isNumber"));
 const isDate_1 = __importDefault(require("lodash/isDate"));
+const isArray_1 = __importDefault(require("lodash/isArray"));
+const isPlainObject_1 = __importDefault(require("lodash/isPlainObject"));
 const v1_1 = __importDefault(require("uuid/v1"));
 class SqlDumper {
     constructor(driver) {
@@ -39,6 +41,9 @@ class SqlDumper {
         this.putRaw(this.escapeString(value));
         this.putRaw("'");
     }
+    putByteArrayValue(value) {
+        this.putRaw('NULL');
+    }
     putValue(value) {
         if (value === null)
             this.putRaw('NULL');
@@ -52,6 +57,10 @@ class SqlDumper {
             this.putRaw(value.toString());
         else if ((0, isDate_1.default)(value))
             this.putStringValue(new Date(value).toISOString());
+        else if ((value === null || value === void 0 ? void 0 : value.type) == 'Buffer' && (0, isArray_1.default)(value === null || value === void 0 ? void 0 : value.data))
+            this.putByteArrayValue(value === null || value === void 0 ? void 0 : value.data);
+        else if ((0, isPlainObject_1.default)(value) || (0, isArray_1.default)(value))
+            this.putStringValue(JSON.stringify(value));
         else
             this.putRaw('NULL');
     }
@@ -161,6 +170,7 @@ class SqlDumper {
     autoIncrement() {
         this.put(' ^auto_increment');
     }
+    specialColumnOptions(column) { }
     columnDefinition(column, { includeDefault = true, includeNullable = true, includeCollate = true } = {}) {
         var _a;
         if (column.computedExpression) {
@@ -174,9 +184,7 @@ class SqlDumper {
             this.autoIncrement();
         }
         this.putRaw(' ');
-        if (column.isSparse) {
-            this.put(' ^sparse ');
-        }
+        this.specialColumnOptions(column);
         if (includeNullable) {
             this.put(column.notNull ? '^not ^null' : '^null');
         }

package/lib/createBulkInsertStreamBase.js

@@ -25,6 +25,7 @@ function createBulkInsertStreamBase(driver, stream, pool, name, options) {
     writable.buffer = [];
     writable.structure = null;
     writable.columnNames = null;
+    writable.requireFixedStructure = !driver.dialect.nosql;
     writable.addRow = (row) => __awaiter(this, void 0, void 0, function* () {
         if (writable.structure) {
             writable.buffer.push(row);
@@ -39,18 +40,18 @@ function createBulkInsertStreamBase(driver, stream, pool, name, options) {
         // console.log('ANALYSING', name, structure);
         if (structure && options.dropIfExists) {
             console.log(`Dropping table ${fullNameQuoted}`);
-            yield driver.query(pool, `DROP TABLE ${fullNameQuoted}`);
+            yield driver.script(pool, `DROP TABLE ${fullNameQuoted}`);
         }
         if (options.createIfNotExists && (!structure || options.dropIfExists)) {
             console.log(`Creating table ${fullNameQuoted}`);
             const dmp = driver.createDumper();
             dmp.createTable((0, tableTransforms_1.prepareTableForImport)(Object.assign(Object.assign({}, writable.structure), name)));
             console.log(dmp.s);
-            yield driver.query(pool, dmp.s);
+            yield driver.script(pool, dmp.s);
             structure = yield driver.analyseSingleTable(pool, name);
         }
         if (options.truncate) {
-            yield driver.query(pool, `TRUNCATE TABLE ${fullNameQuoted}`);
+            yield driver.script(pool, `TRUNCATE TABLE ${fullNameQuoted}`);
         }
         writable.columnNames = (0, intersection_1.default)(structure.columns.map(x => x.columnName), writable.structure.columns.map(x => x.columnName));
     });

package/lib/diffTools.js

@@ -134,6 +134,18 @@ function testEqualColumns(a, b, checkName, checkDefault, opts = {}) {
         // opts.DiffLogger.Trace('Column {0}, {1}: different is_sparse: {2}; {3}', a, b, a.IsSparse, b.IsSparse);
         return false;
     }
+    if ((a.isUnsigned || false) != (b.isUnsigned || false)) {
+        console.debug(`Column ${a.pureName}.${a.columnName}, ${b.pureName}.${b.columnName}: different unsigned: ${a.isUnsigned}, ${b.isUnsigned}`);
+        return false;
+    }
+    if ((a.isZerofill || false) != (b.isZerofill || false)) {
+        console.debug(`Column ${a.pureName}.${a.columnName}, ${b.pureName}.${b.columnName}: different zerofill: ${a.isZerofill}, ${b.isZerofill}`);
+        return false;
+    }
+    if ((a.columnComment || '') != (b.columnComment || '')) {
+        console.debug(`Column ${a.pureName}.${a.columnName}, ${b.pureName}.${b.columnName}: different comment: ${a.columnComment}, ${b.columnComment}`);
+        return false;
+    }
     if (!testEqualTypes(a, b, opts)) {
         return false;
     }

package/lib/driverBase.d.ts

@@ -9,6 +9,10 @@ export declare const driverBase: {
         stringEscapeChar: string;
         fallbackDataType: string;
         quoteIdentifier(s: any): any;
+        columnProperties: {
+            isSparse: boolean;
+            isPersisted: boolean;
+        };
     };
     analyseFull(pool: any, version: any): Promise<any>;
     analyseSingleObject(pool: any, name: any, typeField?: string): Promise<any>;

package/lib/driverBase.js

@@ -21,6 +21,10 @@ const dialect = {
     quoteIdentifier(s) {
         return s;
     },
+    columnProperties: {
+        isSparse: false,
+        isPersisted: false,
+    },
 };
 exports.driverBase = {
     analyserClass: null,

package/lib/stringTools.d.ts

@@ -1,2 +1,6 @@
 export declare function arrayToHexString(byteArray: any): any;
 export declare function hexStringToArray(inputString: any): any[];
+export declare function parseCellValue(value: any): any;
+export declare function stringifyCellValue(value: any): any;
+export declare function safeJsonParse(json: any, defaultValue?: any, logError?: boolean): any;
+export declare function isJsonLikeLongString(value: any): RegExpMatchArray;

package/lib/stringTools.js

@@ -1,8 +1,14 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.hexStringToArray = exports.arrayToHexString = void 0;
+exports.isJsonLikeLongString = exports.safeJsonParse = exports.stringifyCellValue = exports.parseCellValue = exports.hexStringToArray = exports.arrayToHexString = void 0;
+const isString_1 = __importDefault(require("lodash/isString"));
+const isArray_1 = __importDefault(require("lodash/isArray"));
+const isPlainObject_1 = __importDefault(require("lodash/isPlainObject"));
 function arrayToHexString(byteArray) {
-    return byteArray.reduce((output, elem) => output + ('0' + elem.toString(16)).slice(-2), '');
+    return byteArray.reduce((output, elem) => output + ('0' + elem.toString(16)).slice(-2), '').toUpperCase();
 }
 exports.arrayToHexString = arrayToHexString;
 function hexStringToArray(inputString) {
@@ -14,3 +20,52 @@ function hexStringToArray(inputString) {
     return res;
 }
 exports.hexStringToArray = hexStringToArray;
+function parseCellValue(value) {
+    if (!(0, isString_1.default)(value))
+        return value;
+    if (value == '(NULL)')
+        return null;
+    const mHex = value.match(/^0x([0-9a-fA-F][0-9a-fA-F])+$/);
+    if (mHex) {
+        return {
+            type: 'Buffer',
+            data: hexStringToArray(value.substring(2)),
+        };
+    }
+    const mOid = value.match(/^ObjectId\("([0-9a-f]{24})"\)$/);
+    if (mOid) {
+        return { $oid: mOid[1] };
+    }
+    return value;
+}
+exports.parseCellValue = parseCellValue;
+function stringifyCellValue(value) {
+    if (value === null)
+        return '(NULL)';
+    if (value === undefined)
+        return '(NoField)';
+    if ((value === null || value === void 0 ? void 0 : value.type) == 'Buffer' && (0, isArray_1.default)(value.data))
+        return '0x' + arrayToHexString(value.data);
+    if (value === null || value === void 0 ? void 0 : value.$oid)
+        return `ObjectId("${value === null || value === void 0 ? void 0 : value.$oid}")`;
+    if ((0, isPlainObject_1.default)(value) || (0, isArray_1.default)(value))
+        return JSON.stringify(value);
+    return value;
+}
+exports.stringifyCellValue = stringifyCellValue;
+function safeJsonParse(json, defaultValue, logError = false) {
+    try {
+        return JSON.parse(json);
+    }
+    catch (err) {
+        if (logError) {
+            console.error(`Error parsing JSON value "${json}"`, err);
+        }
+        return defaultValue;
+    }
+}
+exports.safeJsonParse = safeJsonParse;
+function isJsonLikeLongString(value) {
+    return (0, isString_1.default)(value) && value.length > 100 && value.match(/^\s*\{.*\}\s*$|^\s*\[.*\]\s*$/);
+}
+exports.isJsonLikeLongString = isJsonLikeLongString;

package/lib/structureTools.d.ts

@@ -1,4 +1,6 @@
-import { DatabaseInfo, TableInfo } from 'dbgate-types';
+import { DatabaseInfo, TableInfo, ApplicationDefinition } from 'dbgate-types';
 export declare function addTableDependencies(db: DatabaseInfo): DatabaseInfo;
 export declare function extendTableInfo(table: TableInfo): TableInfo;
 export declare function extendDatabaseInfo(db: DatabaseInfo): DatabaseInfo;
+export declare function extendDatabaseInfoFromApps(db: DatabaseInfo, apps: ApplicationDefinition[]): DatabaseInfo;
+export declare function isTableColumnUnique(table: TableInfo, column: string): boolean;

package/lib/structureTools.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.extendDatabaseInfo = exports.extendTableInfo = exports.addTableDependencies = void 0;
+exports.isTableColumnUnique = exports.extendDatabaseInfoFromApps = exports.extendDatabaseInfo = exports.extendTableInfo = exports.addTableDependencies = void 0;
 const flatten_1 = __importDefault(require("lodash/flatten"));
 function addTableDependencies(db) {
     const allForeignKeys = (0, flatten_1.default)(db.tables.map(x => x.foreignKeys || []));
@@ -22,3 +22,26 @@ function extendDatabaseInfo(db) {
     return fillDatabaseExtendedInfo(addTableDependencies(db));
 }
 exports.extendDatabaseInfo = extendDatabaseInfo;
+function extendDatabaseInfoFromApps(db, apps) {
+    if (!db || !apps)
+        return db;
+    const dbExt = Object.assign(Object.assign({}, db), { tables: db.tables.map(table => (Object.assign(Object.assign({}, table), { foreignKeys: [
+                ...(table.foreignKeys || []),
+                ...(0, flatten_1.default)(apps.map(app => app.virtualReferences || []))
+                    .filter(fk => fk.pureName == table.pureName && fk.schemaName == table.schemaName)
+                    .map(fk => (Object.assign(Object.assign({}, fk), { constraintType: 'foreignKey', isVirtual: true }))),
+            ] }))) });
+    return addTableDependencies(dbExt);
+}
+exports.extendDatabaseInfoFromApps = extendDatabaseInfoFromApps;
+function isTableColumnUnique(table, column) {
+    if (table.primaryKey && table.primaryKey.columns.length == 1 && table.primaryKey.columns[0].columnName == column) {
+        return true;
+    }
+    const uqs = [...(table.uniques || []), ...(table.indexes || []).filter(x => x.isUnique)];
+    if (uqs.find(uq => uq.columns.length == 1 && uq.columns[0].columnName == column)) {
+        return true;
+    }
+    return false;
+}
+exports.isTableColumnUnique = isTableColumnUnique;

package/lib/tableTransforms.js

@@ -8,6 +8,9 @@ const cloneDeep_1 = __importDefault(require("lodash/cloneDeep"));
 function prepareTableForImport(table) {
     const res = (0, cloneDeep_1.default)(table);
     res.foreignKeys = [];
+    res.indexes = [];
+    res.uniques = [];
+    res.checks = [];
     if (res.primaryKey)
         res.primaryKey.constraintName = null;
     return res;

package/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "4.6.0",
+  "version": "4.7.0",
   "name": "dbgate-tools",
   "main": "lib/index.js",
   "typings": "lib/index.d.ts",
@@ -25,14 +25,14 @@
   ],
   "devDependencies": {
     "@types/node": "^13.7.0",
-    "dbgate-types": "^4.6.0",
+    "dbgate-types": "^4.7.0",
     "jest": "^24.9.0",
     "ts-jest": "^25.2.1",
     "typescript": "^4.4.3"
   },
   "dependencies": {
     "lodash": "^4.17.21",
-    "dbgate-query-splitter": "^4.6.0",
+    "dbgate-query-splitter": "^4.7.0",
     "uuid": "^3.4.0"
   }
 }