dbgate-tools 6.2.1 → 6.3.0

package/lib/ScriptWriter.d.ts

@@ -10,8 +10,7 @@ export declare class ScriptWriter {
     assign(variableName: any, functionName: any, props: any): void;
     assignValue(variableName: any, jsonValue: any): void;
     requirePackage(packageName: any): void;
-    copyStream(sourceVar: any, targetVar: any, colmapVar?: any): void;
-    dumpDatabase(options: any): void;
+    copyStream(sourceVar: any, targetVar: any, colmapVar?: any, progressName?: string): void;
     importDatabase(options: any): void;
     dataDuplicator(options: any): void;
     comment(s: any): void;
@@ -27,9 +26,8 @@ export declare class ScriptWriterJson {
     endLine(): void;
     assign(variableName: any, functionName: any, props: any): void;
     assignValue(variableName: any, jsonValue: any): void;
-    copyStream(sourceVar: any, targetVar: any, colmapVar?: any): void;
+    copyStream(sourceVar: any, targetVar: any, colmapVar?: any, progressName?: string): void;
     comment(text: any): void;
-    dumpDatabase(options: any): void;
     importDatabase(options: any): void;
     dataDuplicator(options: any): void;
     getScript(schedule?: any): {

package/lib/ScriptWriter.js

@@ -37,16 +37,14 @@ class ScriptWriter {
     requirePackage(packageName) {
         this.packageNames.push(packageName);
     }
-    copyStream(sourceVar, targetVar, colmapVar = null) {
-        if (colmapVar) {
-            this._put(`await dbgateApi.copyStream(${sourceVar}, ${targetVar}, {columns: ${colmapVar}});`);
-        }
-        else {
-            this._put(`await dbgateApi.copyStream(${sourceVar}, ${targetVar});`);
-        }
-    }
-    dumpDatabase(options) {
-        this._put(`await dbgateApi.dumpDatabase(${JSON.stringify(options)});`);
+    copyStream(sourceVar, targetVar, colmapVar = null, progressName) {
+        let opts = '{';
+        if (colmapVar)
+            opts += `columns: ${colmapVar}, `;
+        if (progressName)
+            opts += `progressName: "${progressName}", `;
+        opts += '}';
+        this._put(`await dbgateApi.copyStream(${sourceVar}, ${targetVar}, ${opts});`);
     }
     importDatabase(options) {
         this._put(`await dbgateApi.importDatabase(${JSON.stringify(options)});`);
@@ -103,12 +101,13 @@ class ScriptWriterJson {
             jsonValue,
         });
     }
-    copyStream(sourceVar, targetVar, colmapVar = null) {
+    copyStream(sourceVar, targetVar, colmapVar = null, progressName) {
         this.commands.push({
             type: 'copyStream',
             sourceVar,
             targetVar,
             colmapVar,
+            progressName,
         });
     }
     comment(text) {
@@ -117,12 +116,6 @@ class ScriptWriterJson {
             text,
         });
     }
-    dumpDatabase(options) {
-        this.commands.push({
-            type: 'dumpDatabase',
-            options,
-        });
-    }
     importDatabase(options) {
         this.commands.push({
             type: 'importDatabase',
@@ -163,7 +156,7 @@ function jsonScriptToJavascript(json) {
                 script.assignValue(cmd.variableName, cmd.jsonValue);
                 break;
             case 'copyStream':
-                script.copyStream(cmd.sourceVar, cmd.targetVar, cmd.colmapVar);
+                script.copyStream(cmd.sourceVar, cmd.targetVar, cmd.colmapVar, cmd.progressName);
                 break;
             case 'endLine':
                 script.endLine();
@@ -171,9 +164,6 @@ function jsonScriptToJavascript(json) {
             case 'comment':
                 script.comment(cmd.text);
                 break;
-            case 'dumpDatabase':
-                script.dumpDatabase(cmd.options);
-                break;
             case 'importDatabase':
                 script.importDatabase(cmd.options);
                 break;

package/lib/createBulkInsertStreamBase.js

@@ -17,6 +17,8 @@ const intersection_1 = __importDefault(require("lodash/intersection"));
 const fromPairs_1 = __importDefault(require("lodash/fromPairs"));
 const getLogger_1 = require("./getLogger");
 const tableTransforms_1 = require("./tableTransforms");
+const rowProgressReporter_1 = require("./rowProgressReporter");
+const stringTools_1 = require("./stringTools");
 const logger = (0, getLogger_1.getLogger)('bulkStreamBase');
 function createBulkInsertStreamBase(driver, stream, dbhan, name, options) {
     const fullNameQuoted = name.schemaName
@@ -31,6 +33,7 @@ function createBulkInsertStreamBase(driver, stream, dbhan, name, options) {
     writable.columnNames = null;
     writable.columnDataTypes = null;
     writable.requireFixedStructure = driver.databaseEngineTypes.includes('sql');
+    writable.rowsReporter = new rowProgressReporter_1.RowProgressReporter(options.progressName);
     writable.addRow = (row) => __awaiter(this, void 0, void 0, function* () {
         if (writable.structure) {
             writable.buffer.push(row);
@@ -42,77 +45,91 @@ function createBulkInsertStreamBase(driver, stream, dbhan, name, options) {
     });
     writable.checkStructure = () => __awaiter(this, void 0, void 0, function* () {
         var _a;
-        let structure = (_a = options.targetTableStructure) !== null && _a !== void 0 ? _a : (yield driver.analyseSingleTable(dbhan, name));
-        if (structure) {
-            writable.structure = structure;
-        }
-        if (structure && options.dropIfExists) {
-            logger.info(`Dropping table ${fullNameQuoted}`);
-            yield driver.script(dbhan, `DROP TABLE ${fullNameQuoted}`);
-        }
-        if (options.createIfNotExists && (!structure || options.dropIfExists)) {
-            const dmp = driver.createDumper();
-            const createdTableInfo = driver.adaptTableInfo((0, tableTransforms_1.prepareTableForImport)(Object.assign(Object.assign({}, writable.structure), name)));
-            dmp.createTable(createdTableInfo);
-            logger.info({ sql: dmp.s }, `Creating table ${fullNameQuoted}`);
-            yield driver.script(dbhan, dmp.s);
-            structure = yield driver.analyseSingleTable(dbhan, name);
-            writable.structure = structure;
-        }
-        if (!writable.structure) {
-            throw new Error(`Error importing table - ${fullNameQuoted} not found`);
+        try {
+            let structure = (_a = options.targetTableStructure) !== null && _a !== void 0 ? _a : (yield driver.analyseSingleTable(dbhan, name));
+            if (structure) {
+                writable.structure = structure;
+            }
+            if (structure && options.dropIfExists) {
+                logger.info(`Dropping table ${fullNameQuoted}`);
+                yield driver.script(dbhan, `DROP TABLE ${fullNameQuoted}`);
+            }
+            if (options.createIfNotExists && (!structure || options.dropIfExists)) {
+                const dmp = driver.createDumper();
+                const createdTableInfo = driver.adaptTableInfo((0, tableTransforms_1.prepareTableForImport)(Object.assign(Object.assign({}, writable.structure), name)));
+                dmp.createTable(createdTableInfo);
+                logger.info({ sql: dmp.s }, `Creating table ${fullNameQuoted}`);
+                yield driver.script(dbhan, dmp.s);
+                structure = yield driver.analyseSingleTable(dbhan, name);
+                writable.structure = structure;
+            }
+            if (!writable.structure) {
+                throw new Error(`Error importing table - ${fullNameQuoted} not found`);
+            }
+            if (options.truncate) {
+                yield driver.script(dbhan, `TRUNCATE TABLE ${fullNameQuoted}`);
+            }
+            writable.columnNames = (0, intersection_1.default)(structure.columns.map(x => x.columnName), writable.structure.columns.map(x => x.columnName));
+            writable.columnDataTypes = (0, fromPairs_1.default)(writable.columnNames.map(colName => {
+                var _a;
+                return [
+                    colName,
+                    (_a = writable.structure.columns.find(x => x.columnName == colName)) === null || _a === void 0 ? void 0 : _a.dataType,
+                ];
+            }));
         }
-        if (options.truncate) {
-            yield driver.script(dbhan, `TRUNCATE TABLE ${fullNameQuoted}`);
+        catch (err) {
+            logger.error((0, stringTools_1.extractErrorLogData)(err), 'Error during preparing bulk insert table, stopped');
+            writable.destroy(err);
         }
-        writable.columnNames = (0, intersection_1.default)(structure.columns.map(x => x.columnName), writable.structure.columns.map(x => x.columnName));
-        writable.columnDataTypes = (0, fromPairs_1.default)(writable.columnNames.map(colName => {
-            var _a;
-            return [
-                colName,
-                (_a = writable.structure.columns.find(x => x.columnName == colName)) === null || _a === void 0 ? void 0 : _a.dataType,
-            ];
-        }));
     });
     writable.send = () => __awaiter(this, void 0, void 0, function* () {
         const rows = writable.buffer;
         writable.buffer = [];
-        if (driver.dialect.allowMultipleValuesInsert) {
-            const dmp = driver.createDumper();
-            dmp.putRaw(`INSERT INTO ${fullNameQuoted} (`);
-            dmp.putCollection(',', writable.columnNames, col => dmp.putRaw(driver.dialect.quoteIdentifier(col)));
-            dmp.putRaw(')\n VALUES\n');
-            let wasRow = false;
-            for (const row of rows) {
-                if (wasRow)
-                    dmp.putRaw(',\n');
-                dmp.putRaw('(');
-                dmp.putCollection(',', writable.columnNames, col => { var _a; return dmp.putValue(row[col], (_a = writable.columnDataTypes) === null || _a === void 0 ? void 0 : _a[col]); });
-                dmp.putRaw(')');
-                wasRow = true;
-            }
-            dmp.putRaw(';');
-            // require('fs').writeFileSync('/home/jena/test.sql', dmp.s);
-            // console.log(dmp.s);
-            yield driver.query(dbhan, dmp.s, { discardResult: true });
-        }
-        else {
-            for (const row of rows) {
+        try {
+            if (driver.dialect.allowMultipleValuesInsert) {
                 const dmp = driver.createDumper();
                 dmp.putRaw(`INSERT INTO ${fullNameQuoted} (`);
                 dmp.putCollection(',', writable.columnNames, col => dmp.putRaw(driver.dialect.quoteIdentifier(col)));
                 dmp.putRaw(')\n VALUES\n');
-                dmp.putRaw('(');
-                dmp.putCollection(',', writable.columnNames, col => { var _a; return dmp.putValue(row[col], (_a = writable.columnDataTypes) === null || _a === void 0 ? void 0 : _a[col]); });
-                dmp.putRaw(')');
+                let wasRow = false;
+                for (const row of rows) {
+                    if (wasRow)
+                        dmp.putRaw(',\n');
+                    dmp.putRaw('(');
+                    dmp.putCollection(',', writable.columnNames, col => { var _a; return dmp.putValue(row[col], (_a = writable.columnDataTypes) === null || _a === void 0 ? void 0 : _a[col]); });
+                    dmp.putRaw(')');
+                    wasRow = true;
+                }
+                dmp.putRaw(';');
+                // require('fs').writeFileSync('/home/jena/test.sql', dmp.s);
                 // console.log(dmp.s);
                 yield driver.query(dbhan, dmp.s, { discardResult: true });
+                writable.rowsReporter.add(rows.length);
+            }
+            else {
+                for (const row of rows) {
+                    const dmp = driver.createDumper();
+                    dmp.putRaw(`INSERT INTO ${fullNameQuoted} (`);
+                    dmp.putCollection(',', writable.columnNames, col => dmp.putRaw(driver.dialect.quoteIdentifier(col)));
+                    dmp.putRaw(')\n VALUES\n');
+                    dmp.putRaw('(');
+                    dmp.putCollection(',', writable.columnNames, col => { var _a; return dmp.putValue(row[col], (_a = writable.columnDataTypes) === null || _a === void 0 ? void 0 : _a[col]); });
+                    dmp.putRaw(')');
+                    // console.log(dmp.s);
+                    yield driver.query(dbhan, dmp.s, { discardResult: true });
+                    writable.rowsReporter.add(1);
+                }
+            }
+            if (options.commitAfterInsert) {
+                const dmp = driver.createDumper();
+                dmp.commitTransaction();
+                yield driver.query(dbhan, dmp.s, { discardResult: true });
             }
         }
-        if (options.commitAfterInsert) {
-            const dmp = driver.createDumper();
-            dmp.commitTransaction();
-            yield driver.query(dbhan, dmp.s, { discardResult: true });
+        catch (err) {
+            logger.error((0, stringTools_1.extractErrorLogData)(err), 'Error during base bulk insert, insert stopped');
+            writable.destroy(err);
         }
     });
     writable.sendIfFull = () => __awaiter(this, void 0, void 0, function* () {
@@ -127,6 +144,7 @@ function createBulkInsertStreamBase(driver, stream, dbhan, name, options) {
     });
     writable._final = (callback) => __awaiter(this, void 0, void 0, function* () {
         yield writable.send();
+        writable.rowsReporter.finish();
         callback();
     });
     return writable;

package/lib/getConnectionLabel.d.ts

@@ -3,3 +3,4 @@ export declare function getConnectionLabel(connection: any, { allowExplicitDatab
     allowExplicitDatabase?: boolean;
     showUnsaved?: boolean;
 }): any;
+export declare function getEngineLabel(connection: any): any;

package/lib/getConnectionLabel.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getConnectionLabel = exports.getDatabaseFileLabel = void 0;
+exports.getEngineLabel = exports.getConnectionLabel = exports.getDatabaseFileLabel = void 0;
 function getDatabaseFileLabel(databaseFile) {
     if (!databaseFile)
         return databaseFile;
@@ -45,3 +45,11 @@ function getConnectionLabel(connection, { allowExplicitDatabase = true, showUnsa
     return res;
 }
 exports.getConnectionLabel = getConnectionLabel;
+function getEngineLabel(connection) {
+    const match = ((connection === null || connection === void 0 ? void 0 : connection.engine) || '').match(/^([^@]*)@/);
+    if (match) {
+        return match[1];
+    }
+    return connection === null || connection === void 0 ? void 0 : connection.engine;
+}
+exports.getEngineLabel = getEngineLabel;

package/lib/index.d.ts

@@ -25,3 +25,4 @@ export * from './detectSqlFilterBehaviour';
 export * from './filterBehaviours';
 export * from './schemaInfoTools';
 export * from './dbKeysLoader';
+export * from './rowProgressReporter';

package/lib/index.js

@@ -41,3 +41,4 @@ __exportStar(require("./detectSqlFilterBehaviour"), exports);
 __exportStar(require("./filterBehaviours"), exports);
 __exportStar(require("./schemaInfoTools"), exports);
 __exportStar(require("./dbKeysLoader"), exports);
+__exportStar(require("./rowProgressReporter"), exports);

package/lib/rowProgressReporter.d.ts

@@ -0,0 +1,10 @@
+export declare class RowProgressReporter {
+    progressName: any;
+    field: string;
+    counter: number;
+    timeoutHandle: any;
+    constructor(progressName: any, field?: string);
+    add(count: number): void;
+    finish(): void;
+    send(): void;
+}

package/lib/rowProgressReporter.js

@@ -0,0 +1,45 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RowProgressReporter = void 0;
+class RowProgressReporter {
+    constructor(progressName, field = 'writtenRowCount') {
+        this.progressName = progressName;
+        this.field = field;
+        this.counter = 0;
+        this.timeoutHandle = null;
+    }
+    add(count) {
+        this.counter += count;
+        if (!this.progressName) {
+            return;
+        }
+        if (this.timeoutHandle) {
+            return;
+        }
+        this.timeoutHandle = setTimeout(() => {
+            this.timeoutHandle = null;
+            this.send();
+        }, 1000);
+    }
+    finish() {
+        if (!this.progressName) {
+            return;
+        }
+        if (this.timeoutHandle) {
+            clearTimeout(this.timeoutHandle);
+            this.timeoutHandle = null;
+        }
+        this.send();
+    }
+    send() {
+        if (!this.progressName) {
+            return;
+        }
+        process.send({
+            msgtype: 'progress',
+            progressName: this.progressName,
+            [this.field]: this.counter,
+        });
+    }
+}
+exports.RowProgressReporter = RowProgressReporter;

package/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "6.2.1",
+  "version": "6.3.0",
   "name": "dbgate-tools",
   "main": "lib/index.js",
   "typings": "lib/index.d.ts",
@@ -25,14 +25,14 @@
   ],
   "devDependencies": {
     "@types/node": "^13.7.0",
-    "dbgate-types": "^6.2.1",
+    "dbgate-types": "^6.3.0",
     "jest": "^24.9.0",
     "ts-jest": "^25.2.1",
     "typescript": "^4.4.3"
   },
   "dependencies": {
     "dbgate-query-splitter": "^4.11.3",
-    "dbgate-sqltree": "^6.2.1",
+    "dbgate-sqltree": "^6.3.0",
     "debug": "^4.3.4",
     "json-stable-stringify": "^1.0.1",
     "lodash": "^4.17.21",