dbgate-datalib 6.3.3 → 6.4.0

package/lib/ChangeSet.d.ts

@@ -52,6 +52,7 @@ export declare function deleteChangeSetRows(changeSet: ChangeSet, definition: Ch
 export declare function getChangeSetInsertedRows(changeSet: ChangeSet, name?: NamedObjectInfo): any[];
 export declare function changeSetInsertNewRow(changeSet: ChangeSet, name?: NamedObjectInfo): ChangeSet;
 export declare function changeSetInsertDocuments(changeSet: ChangeSet, documents: any[], name?: NamedObjectInfo, insertIfNotExistsFieldNames?: string[]): ChangeSet;
+export declare function createMergedRowsChangeSet(table: TableInfo, updatedRows: any[], insertedRows: any[], mergeKey: string[]): ChangeSet;
 export declare function changeSetContainsChanges(changeSet: ChangeSet): boolean;
 export declare function changeSetChangedCount(changeSet: ChangeSet): number;
 export declare function removeSchemaFromChangeSet(changeSet: ChangeSet): {

package/lib/ChangeSet.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.removeSchemaFromChangeSet = exports.changeSetChangedCount = exports.changeSetContainsChanges = exports.changeSetInsertDocuments = exports.changeSetInsertNewRow = exports.getChangeSetInsertedRows = exports.deleteChangeSetRows = exports.revertChangeSetRowChanges = exports.changeSetToSql = exports.extractChangeSetCondition = exports.batchUpdateChangeSet = exports.setChangeSetRowData = exports.setChangeSetValue = exports.findExistingChangeSetItem = exports.createChangeSet = void 0;
+exports.removeSchemaFromChangeSet = exports.changeSetChangedCount = exports.changeSetContainsChanges = exports.createMergedRowsChangeSet = exports.changeSetInsertDocuments = exports.changeSetInsertNewRow = exports.getChangeSetInsertedRows = exports.deleteChangeSetRows = exports.revertChangeSetRowChanges = exports.changeSetToSql = exports.extractChangeSetCondition = exports.batchUpdateChangeSet = exports.setChangeSetRowData = exports.setChangeSetValue = exports.findExistingChangeSetItem = exports.createChangeSet = void 0;
 const lodash_1 = __importDefault(require("lodash"));
 function createChangeSet() {
     return {
@@ -405,6 +405,22 @@ function changeSetInsertDocuments(changeSet, documents, name, insertIfNotExistsF
         ] });
 }
 exports.changeSetInsertDocuments = changeSetInsertDocuments;
+function createMergedRowsChangeSet(table, updatedRows, insertedRows, mergeKey) {
+    const res = createChangeSet();
+    res.updates = updatedRows.map(row => ({
+        pureName: table.pureName,
+        schemaName: table.schemaName,
+        fields: lodash_1.default.omit(row, mergeKey),
+        condition: lodash_1.default.pick(row, mergeKey),
+    }));
+    res.inserts = insertedRows.map(row => ({
+        pureName: table.pureName,
+        schemaName: table.schemaName,
+        fields: row,
+    }));
+    return res;
+}
+exports.createMergedRowsChangeSet = createMergedRowsChangeSet;
 function changeSetContainsChanges(changeSet) {
     var _a;
     if (!changeSet)

package/lib/DataReplicator.d.ts

@@ -0,0 +1,89 @@
+/// <reference types="lodash" />
+import { SqlDumper } from 'dbgate-tools';
+import { DatabaseInfo, EngineDriver, ForeignKeyInfo, NamedObjectInfo, QueryResult, TableInfo } from 'dbgate-types';
+export interface DataReplicatorItem {
+    openStream: () => Promise<ReadableStream>;
+    name: string;
+    findExisting: (row: any) => boolean;
+    createNew: (row: any) => boolean;
+    updateExisting: (row: any) => boolean;
+    deleteMissing: boolean;
+    deleteRestrictionColumns: string[];
+    matchColumns: string[];
+}
+export interface DataReplicatorOptions {
+    rollbackAfterFinish?: boolean;
+    skipRowsWithUnresolvedRefs?: boolean;
+    setNullForUnresolvedNullableRefs?: boolean;
+    generateSqlScript?: boolean;
+    runid?: string;
+}
+declare class ReplicatorReference {
+    base: ReplicatorItemHolder;
+    ref: ReplicatorItemHolder;
+    isMandatory: boolean;
+    foreignKey: ForeignKeyInfo;
+    constructor(base: ReplicatorItemHolder, ref: ReplicatorItemHolder, isMandatory: boolean, foreignKey: ForeignKeyInfo);
+    get columnName(): string;
+}
+declare class ReplicatorWeakReference {
+    base: ReplicatorItemHolder;
+    ref: TableInfo;
+    foreignKey: ForeignKeyInfo;
+    constructor(base: ReplicatorItemHolder, ref: TableInfo, foreignKey: ForeignKeyInfo);
+    get columnName(): string;
+}
+declare class ReplicatorItemHolder {
+    item: DataReplicatorItem;
+    replicator: DataReplicator;
+    references: ReplicatorReference[];
+    backReferences: ReplicatorReference[];
+    weakReferences: ReplicatorWeakReference[];
+    table: TableInfo;
+    isPlanned: boolean;
+    idMap: {};
+    autoColumn: string;
+    isManualAutoColumn: boolean;
+    refByColumn: {
+        [columnName: string]: ReplicatorReference;
+    };
+    isReferenced: boolean;
+    get name(): string;
+    constructor(item: DataReplicatorItem, replicator: DataReplicator);
+    initializeReferences(): void;
+    createInsertObject(chunk: any, weakrefcols?: string[]): import("lodash").Omit<Pick<any, string>, string>;
+    createUpdateObject(chunk: any): import("lodash").Omit<Pick<any, string>, string>;
+    getMissingWeakRefsForRow(row: any): Promise<string[]>;
+    runImport(): Promise<{
+        inserted: number;
+        mapped: number;
+        missing: number;
+        skipped: number;
+        updated: number;
+        deleted: number;
+    }>;
+}
+export declare class DataReplicator {
+    pool: any;
+    driver: EngineDriver;
+    db: DatabaseInfo;
+    items: DataReplicatorItem[];
+    stream: any;
+    copyStream: (input: any, output: any, options: any) => Promise<void>;
+    options: DataReplicatorOptions;
+    itemHolders: ReplicatorItemHolder[];
+    itemPlan: ReplicatorItemHolder[];
+    result: string;
+    dumper: SqlDumper;
+    identityValues: {
+        [fullTableName: string]: number;
+    };
+    constructor(pool: any, driver: EngineDriver, db: DatabaseInfo, items: DataReplicatorItem[], stream: any, copyStream: (input: any, output: any, options: any) => Promise<void>, options?: DataReplicatorOptions);
+    findItemToPlan(): ReplicatorItemHolder;
+    createPlan(): void;
+    runDumperCommand(cmd: (dmp: SqlDumper) => void | string): Promise<void>;
+    runDumperQuery(cmd: (dmp: SqlDumper) => void | string): Promise<QueryResult>;
+    generateIdentityValue(column: string, table: NamedObjectInfo): Promise<number>;
+    run(): Promise<void>;
+}
+export {};

package/lib/DataReplicator.js

@@ -0,0 +1,430 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DataReplicator = void 0;
+const dbgate_tools_1 = require("dbgate-tools");
+const pick_1 = __importDefault(require("lodash/pick"));
+const omit_1 = __importDefault(require("lodash/omit"));
+const json_stable_stringify_1 = __importDefault(require("json-stable-stringify"));
+const logger = (0, dbgate_tools_1.getLogger)('dataReplicator');
+class ReplicatorReference {
+    constructor(base, ref, isMandatory, foreignKey) {
+        this.base = base;
+        this.ref = ref;
+        this.isMandatory = isMandatory;
+        this.foreignKey = foreignKey;
+    }
+    get columnName() {
+        return this.foreignKey.columns[0].columnName;
+    }
+}
+class ReplicatorWeakReference {
+    constructor(base, ref, foreignKey) {
+        this.base = base;
+        this.ref = ref;
+        this.foreignKey = foreignKey;
+    }
+    get columnName() {
+        return this.foreignKey.columns[0].columnName;
+    }
+}
+class ReplicatorItemHolder {
+    get name() {
+        return this.item.name;
+    }
+    constructor(item, replicator) {
+        var _a, _b, _c, _d, _e, _f, _g, _h;
+        this.item = item;
+        this.replicator = replicator;
+        this.references = [];
+        this.backReferences = [];
+        // not mandatory references to entities out of the model
+        this.weakReferences = [];
+        this.isPlanned = false;
+        this.idMap = {};
+        this.refByColumn = {};
+        this.table = replicator.db.tables.find(x => x.pureName.toUpperCase() == item.name.toUpperCase());
+        this.autoColumn = (_a = this.table.columns.find(x => x.autoIncrement)) === null || _a === void 0 ? void 0 : _a.columnName;
+        if (((_c = (_b = this.table.primaryKey) === null || _b === void 0 ? void 0 : _b.columns) === null || _c === void 0 ? void 0 : _c.length) != 1 ||
+            ((_f = (_e = (_d = this.table.primaryKey) === null || _d === void 0 ? void 0 : _d.columns) === null || _e === void 0 ? void 0 : _e[0]) === null || _f === void 0 ? void 0 : _f.columnName) != this.autoColumn) {
+            this.autoColumn = null;
+        }
+        if (!this.autoColumn && ((_h = (_g = this.table.primaryKey) === null || _g === void 0 ? void 0 : _g.columns) === null || _h === void 0 ? void 0 : _h.length) == 1) {
+            const name = this.table.primaryKey.columns[0].columnName;
+            const column = this.table.columns.find(x => x.columnName == name);
+            if ((0, dbgate_tools_1.isTypeNumber)(column === null || column === void 0 ? void 0 : column.dataType)) {
+                this.autoColumn = name;
+                this.isManualAutoColumn = true;
+            }
+        }
+        if (this.autoColumn && this.replicator.options.generateSqlScript) {
+            this.isManualAutoColumn = true;
+        }
+    }
+    initializeReferences() {
+        var _a, _b;
+        for (const fk of this.table.foreignKeys) {
+            if (((_a = fk.columns) === null || _a === void 0 ? void 0 : _a.length) != 1)
+                continue;
+            const refHolder = this.replicator.itemHolders.find(y => y.name.toUpperCase() == fk.refTableName.toUpperCase());
+            const isMandatory = (_b = this.table.columns.find(x => { var _a; return x.columnName == ((_a = fk.columns[0]) === null || _a === void 0 ? void 0 : _a.columnName); })) === null || _b === void 0 ? void 0 : _b.notNull;
+            if (refHolder == null) {
+                if (!isMandatory) {
+                    const weakref = new ReplicatorWeakReference(this, this.replicator.db.tables.find(x => x.pureName == fk.refTableName), fk);
+                    this.weakReferences.push(weakref);
+                }
+            }
+            else {
+                const newref = new ReplicatorReference(this, refHolder, isMandatory, fk);
+                this.references.push(newref);
+                this.refByColumn[newref.columnName] = newref;
+                refHolder.isReferenced = true;
+            }
+        }
+    }
+    createInsertObject(chunk, weakrefcols) {
+        const res = (0, omit_1.default)((0, pick_1.default)(chunk, this.table.columns.map(x => x.columnName)), [this.autoColumn, ...this.backReferences.map(x => x.columnName), ...(weakrefcols ? weakrefcols : [])]);
+        for (const key in res) {
+            const ref = this.refByColumn[key];
+            if (ref) {
+                // remap id
+                const oldId = res[key];
+                res[key] = ref.ref.idMap[oldId];
+                if (ref.isMandatory && res[key] == null) {
+                    // mandatory refertence not matched
+                    if (this.replicator.options.skipRowsWithUnresolvedRefs) {
+                        return null;
+                    }
+                    throw new Error(`Unresolved reference, base=${ref.base.name}, ref=${ref.ref.name}, ${key}=${chunk[key]}`);
+                }
+            }
+        }
+        return res;
+    }
+    createUpdateObject(chunk) {
+        const res = (0, omit_1.default)((0, pick_1.default)(chunk, this.table.columns.map(x => x.columnName)), [this.autoColumn, ...this.backReferences.map(x => x.columnName), ...this.references.map(x => x.columnName)]);
+        return res;
+    }
+    // returns list of columns that are weak references and are not resolved
+    getMissingWeakRefsForRow(row) {
+        var _a;
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.replicator.options.setNullForUnresolvedNullableRefs || !((_a = this.weakReferences) === null || _a === void 0 ? void 0 : _a.length)) {
+                return [];
+            }
+            const qres = yield (0, dbgate_tools_1.runQueryOnDriver)(this.replicator.pool, this.replicator.driver, dmp => {
+                dmp.put('^select ');
+                dmp.putCollection(',', this.weakReferences, weakref => {
+                    dmp.put('(^case ^when ^exists (^select * ^from %f where %i = %v) ^then 1 ^else 0 ^end) as %i', weakref.ref, weakref.foreignKey.columns[0].refColumnName, row[weakref.foreignKey.columns[0].columnName], weakref.foreignKey.columns[0].columnName);
+                });
+                if (this.replicator.driver.dialect.requireFromDual) {
+                    dmp.put(' ^from ^dual');
+                }
+            });
+            const qrow = qres.rows[0];
+            return this.weakReferences.filter(x => qrow[x.columnName] == 0).map(x => x.columnName);
+        });
+    }
+    runImport() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const readStream = yield this.item.openStream();
+            const driver = this.replicator.driver;
+            const pool = this.replicator.pool;
+            let inserted = 0;
+            let mapped = 0;
+            let updated = 0;
+            let deleted = 0;
+            let missing = 0;
+            let skipped = 0;
+            let lastLogged = new Date();
+            const { deleteMissing, deleteRestrictionColumns } = this.item;
+            const deleteRestrictions = {};
+            const usedKeyRows = {};
+            const writeStream = (0, dbgate_tools_1.createAsyncWriteStream)(this.replicator.stream, {
+                processItem: (chunk) => __awaiter(this, void 0, void 0, function* () {
+                    if (chunk.__isStreamHeader) {
+                        return;
+                    }
+                    const doFind = () => __awaiter(this, void 0, void 0, function* () {
+                        var _a, _b, _c;
+                        let insertedObj = this.createInsertObject(chunk);
+                        const res = yield (0, dbgate_tools_1.runQueryOnDriver)(pool, driver, dmp => {
+                            dmp.put('^select %i ^from %f ^where ', this.autoColumn, this.table);
+                            dmp.putCollection(' and ', this.item.matchColumns, x => {
+                                dmp.put('%i = %v', x, insertedObj[x]);
+                            });
+                        });
+                        const resId = (_c = (_b = Object.entries(((_a = res === null || res === void 0 ? void 0 : res.rows) === null || _a === void 0 ? void 0 : _a[0]) || {})) === null || _b === void 0 ? void 0 : _b[0]) === null || _c === void 0 ? void 0 : _c[1];
+                        if (resId != null) {
+                            mapped += 1;
+                            this.idMap[chunk[this.autoColumn]] = resId;
+                        }
+                        return resId;
+                    });
+                    const doUpdate = (recordId) => __awaiter(this, void 0, void 0, function* () {
+                        const updateObj = this.createUpdateObject(chunk);
+                        if (Object.keys(updateObj).length == 0) {
+                            skipped += 1;
+                            return;
+                        }
+                        yield this.replicator.runDumperCommand(dmp => {
+                            dmp.put('^update %f ^ set ', this.table);
+                            dmp.putCollection(',', Object.keys(updateObj), x => {
+                                dmp.put('%i = %v', x, updateObj[x]);
+                            });
+                            dmp.put(' ^where %i = %v', this.autoColumn, recordId);
+                            dmp.endCommand();
+                        });
+                        updated += 1;
+                    });
+                    const doInsert = () => __awaiter(this, void 0, void 0, function* () {
+                        var _d, _e, _f;
+                        // console.log('chunk', this.name, JSON.stringify(chunk));
+                        const weakrefcols = yield this.getMissingWeakRefsForRow(chunk);
+                        let insertedObj = this.createInsertObject(chunk, weakrefcols);
+                        // console.log('insertedObj', this.name, JSON.stringify(insertedObj));
+                        if (insertedObj == null) {
+                            skipped += 1;
+                            return;
+                        }
+                        if (this.isManualAutoColumn) {
+                            const maxId = yield this.replicator.generateIdentityValue(this.autoColumn, this.table);
+                            insertedObj = Object.assign(Object.assign({}, insertedObj), { [this.autoColumn]: maxId });
+                            this.idMap[chunk[this.autoColumn]] = maxId;
+                        }
+                        let res = yield this.replicator.runDumperQuery(dmp => {
+                            dmp.put('^insert ^into %f (%,i) ^values (%,v)', this.table, Object.keys(insertedObj), Object.values(insertedObj));
+                            dmp.endCommand();
+                            if (this.autoColumn &&
+                                this.isReferenced &&
+                                !this.replicator.driver.dialect.requireStandaloneSelectForScopeIdentity &&
+                                !this.isManualAutoColumn) {
+                                dmp.selectScopeIdentity(this.table);
+                            }
+                        });
+                        inserted += 1;
+                        if (this.autoColumn && this.isReferenced && !this.isManualAutoColumn) {
+                            if (this.replicator.driver.dialect.requireStandaloneSelectForScopeIdentity) {
+                                res = yield (0, dbgate_tools_1.runQueryOnDriver)(pool, driver, dmp => dmp.selectScopeIdentity(this.table));
+                            }
+                            // console.log('IDRES', JSON.stringify(res));
+                            // console.log('*********** ENTRIES OF', res?.rows?.[0]);
+                            const resId = (_f = (_e = Object.entries((_d = res === null || res === void 0 ? void 0 : res.rows) === null || _d === void 0 ? void 0 : _d[0])) === null || _e === void 0 ? void 0 : _e[0]) === null || _f === void 0 ? void 0 : _f[1];
+                            if (resId != null) {
+                                this.idMap[chunk[this.autoColumn]] = resId;
+                            }
+                            return resId;
+                        }
+                    });
+                    const doMarkDelete = () => {
+                        const insertedObj = this.createInsertObject(chunk);
+                        if ((deleteRestrictionColumns === null || deleteRestrictionColumns === void 0 ? void 0 : deleteRestrictionColumns.length) > 0) {
+                            const restriction = (0, pick_1.default)(insertedObj, deleteRestrictionColumns);
+                            const key = (0, json_stable_stringify_1.default)(restriction);
+                            deleteRestrictions[key] = restriction;
+                        }
+                        const usedKey = (0, pick_1.default)(insertedObj, this.item.matchColumns);
+                        usedKeyRows[(0, json_stable_stringify_1.default)(usedKey)] = usedKey;
+                    };
+                    const findExisting = this.item.findExisting(chunk);
+                    const updateExisting = this.item.updateExisting(chunk);
+                    const createNew = this.item.createNew(chunk);
+                    if (deleteMissing) {
+                        doMarkDelete();
+                    }
+                    let recordId = null;
+                    if (findExisting) {
+                        recordId = yield doFind();
+                    }
+                    if (updateExisting && recordId != null) {
+                        yield doUpdate(recordId);
+                    }
+                    if (createNew && recordId == null) {
+                        recordId = yield doInsert();
+                    }
+                    if (recordId == null && findExisting) {
+                        missing += 1;
+                    }
+                    if (new Date().getTime() - lastLogged.getTime() > 5000) {
+                        logger.info(`Replicating ${this.item.name} in progress, inserted ${inserted} rows, mapped ${mapped} rows, missing ${missing} rows, skipped ${skipped} rows, updated ${updated} rows`);
+                        lastLogged = new Date();
+                    }
+                    // this.idMap[oldId] = newId;
+                }),
+            });
+            const dumpConditionArray = (dmp, array, positive) => {
+                dmp.putCollection(positive ? ' or ' : ' and ', array, x => {
+                    dmp.put('(');
+                    dmp.putCollection(positive ? ' and ' : ' or ', Object.keys(x), y => {
+                        dmp.put(positive ? '%i = %v' : 'not (%i = %v)', y, x[y]);
+                    });
+                    dmp.put(')');
+                });
+            };
+            const dumpDeleteCondition = (dmp) => {
+                const deleteRestrictionValues = Object.values(deleteRestrictions);
+                const usedKeyRowsValues = Object.values(usedKeyRows);
+                if (deleteRestrictionValues.length == 0 && usedKeyRowsValues.length == 0) {
+                    return;
+                }
+                dmp.put(' ^where ');
+                if ((deleteRestrictionColumns === null || deleteRestrictionColumns === void 0 ? void 0 : deleteRestrictionColumns.length) > 0) {
+                    dmp.put('(');
+                    dumpConditionArray(dmp, deleteRestrictionValues, true);
+                    dmp.put(')');
+                    if (usedKeyRowsValues.length > 0) {
+                        dmp.put(' ^and ');
+                    }
+                }
+                dumpConditionArray(dmp, Object.values(usedKeyRows), false);
+            };
+            const doDelete = () => __awaiter(this, void 0, void 0, function* () {
+                const countRes = yield (0, dbgate_tools_1.runQueryOnDriver)(pool, driver, dmp => {
+                    dmp.put('^select count(*) as ~cnt ^from %f', this.table);
+                    dumpDeleteCondition(dmp);
+                    dmp.endCommand();
+                });
+                const count = parseInt(countRes.rows[0].cnt);
+                if (count > 0) {
+                    yield this.replicator.runDumperCommand(dmp => {
+                        dmp.put('^delete ^from %f', this.table);
+                        dumpDeleteCondition(dmp);
+                        dmp.endCommand();
+                    });
+                    deleted += count;
+                }
+            });
+            yield this.replicator.copyStream(readStream, writeStream, {});
+            if (deleteMissing) {
+                yield doDelete();
+            }
+            // await this.replicator.driver.writeQueryStream(this.replicator.pool, {
+            //   mapResultId: (oldId, newId) => {
+            //     this.idMap[oldId] = newId;
+            //   },
+            // });
+            return { inserted, mapped, missing, skipped, updated, deleted };
+        });
+    }
+}
+class DataReplicator {
+    constructor(pool, driver, db, items, stream, copyStream, options = {}) {
+        this.pool = pool;
+        this.driver = driver;
+        this.db = db;
+        this.items = items;
+        this.stream = stream;
+        this.copyStream = copyStream;
+        this.options = options;
+        this.itemPlan = [];
+        this.result = '';
+        this.identityValues = {};
+        this.itemHolders = items.map(x => new ReplicatorItemHolder(x, this));
+        this.itemHolders.forEach(x => x.initializeReferences());
+        // @ts-ignore
+        this.dumper = driver.createDumper();
+    }
+    findItemToPlan() {
+        for (const item of this.itemHolders) {
+            if (item.isPlanned)
+                continue;
+            if (item.references.every(x => x.ref.isPlanned)) {
+                return item;
+            }
+        }
+        for (const item of this.itemHolders) {
+            if (item.isPlanned)
+                continue;
+            if (item.references.every(x => x.ref.isPlanned || !x.isMandatory)) {
+                const backReferences = item.references.filter(x => !x.ref.isPlanned);
+                item.backReferences = backReferences;
+                return item;
+            }
+        }
+        throw new Error('Cycle in mandatory references');
+    }
+    createPlan() {
+        while (this.itemPlan.length < this.itemHolders.length) {
+            const item = this.findItemToPlan();
+            item.isPlanned = true;
+            this.itemPlan.push(item);
+        }
+    }
+    runDumperCommand(cmd) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.options.generateSqlScript) {
+                cmd(this.dumper);
+            }
+            else {
+                yield (0, dbgate_tools_1.runCommandOnDriver)(this.pool, this.driver, cmd);
+            }
+        });
+    }
+    runDumperQuery(cmd) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.options.generateSqlScript) {
+                cmd(this.dumper);
+                return {
+                    rows: [],
+                };
+            }
+            else {
+                return yield (0, dbgate_tools_1.runQueryOnDriver)(this.pool, this.driver, cmd);
+            }
+        });
+    }
+    generateIdentityValue(column, table) {
+        var _a;
+        return __awaiter(this, void 0, void 0, function* () {
+            const tableKey = `${table.schemaName}.${table.pureName}`;
+            if (!(tableKey in this.identityValues)) {
+                const max = yield (0, dbgate_tools_1.runQueryOnDriver)(this.pool, this.driver, dmp => {
+                    dmp.put('^select max(%i) as ~maxid ^from %f', column, table);
+                });
+                const maxId = Math.max((_a = max.rows[0]['maxid']) !== null && _a !== void 0 ? _a : 0, 0) + 1;
+                this.identityValues[tableKey] = maxId;
+                return maxId;
+            }
+            this.identityValues[tableKey] += 1;
+            return this.identityValues[tableKey];
+        });
+    }
+    run() {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.createPlan();
+            yield this.runDumperCommand(dmp => dmp.beginTransaction());
+            try {
+                for (const item of this.itemPlan) {
+                    const stats = yield item.runImport();
+                    logger.info(`Replicated ${item.name}, inserted ${stats.inserted} rows, mapped ${stats.mapped} rows, missing ${stats.missing} rows, skipped ${stats.skipped} rows, updated ${stats.updated} rows, deleted ${stats.deleted} rows`);
+                }
+            }
+            catch (err) {
+                logger.error((0, dbgate_tools_1.extractErrorLogData)(err), `Failed replicator job, rollbacking. ${err.message}`);
+                yield this.runDumperCommand(dmp => dmp.rollbackTransaction());
+                return;
+            }
+            if (this.options.rollbackAfterFinish) {
+                logger.info('Rollbacking transaction, nothing was changed');
+                yield this.runDumperCommand(dmp => dmp.rollbackTransaction());
+            }
+            else {
+                logger.info('Committing replicator transaction');
+                yield this.runDumperCommand(dmp => dmp.commitTransaction());
+            }
+            this.result = this.dumper.s;
+        });
+    }
+}
+exports.DataReplicator = DataReplicator;

package/lib/index.d.ts

@@ -18,7 +18,7 @@ export * from './processPerspectiveDefaultColunns';
 export * from './PerspectiveDataPattern';
 export * from './PerspectiveDataLoader';
 export * from './perspectiveTools';
-export * from './DataDuplicator';
+export * from './DataReplicator';
 export * from './FreeTableGridDisplay';
 export * from './FreeTableModel';
 export * from './CustomGridDisplay';

package/lib/index.js

@@ -34,7 +34,7 @@ __exportStar(require("./processPerspectiveDefaultColunns"), exports);
 __exportStar(require("./PerspectiveDataPattern"), exports);
 __exportStar(require("./PerspectiveDataLoader"), exports);
 __exportStar(require("./perspectiveTools"), exports);
-__exportStar(require("./DataDuplicator"), exports);
+__exportStar(require("./DataReplicator"), exports);
 __exportStar(require("./FreeTableGridDisplay"), exports);
 __exportStar(require("./FreeTableModel"), exports);
 __exportStar(require("./CustomGridDisplay"), exports);

package/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "6.3.3",
+  "version": "6.4.0",
   "name": "dbgate-datalib",
   "main": "lib/index.js",
   "typings": "lib/index.d.ts",
@@ -13,13 +13,13 @@
     "lib"
   ],
   "dependencies": {
-    "dbgate-sqltree": "^6.3.3",
-    "dbgate-tools": "^6.3.3",
-    "dbgate-filterparser": "^6.3.3",
+    "dbgate-sqltree": "^6.4.0",
+    "dbgate-tools": "^6.4.0",
+    "dbgate-filterparser": "^6.4.0",
     "uuid": "^3.4.0"
   },
   "devDependencies": {
-    "dbgate-types": "^6.3.3",
+    "dbgate-types": "^6.4.0",
     "@types/node": "^13.7.0",
     "jest": "^28.1.3",
     "ts-jest": "^28.0.7",

package/lib/DataDuplicator.d.ts

@@ -1,70 +0,0 @@
-/// <reference types="lodash" />
-import { DatabaseInfo, EngineDriver, ForeignKeyInfo, TableInfo } from 'dbgate-types';
-export interface DataDuplicatorItem {
-    openStream: () => Promise<ReadableStream>;
-    name: string;
-    operation: 'copy' | 'lookup' | 'insertMissing';
-    matchColumns: string[];
-}
-export interface DataDuplicatorOptions {
-    rollbackAfterFinish?: boolean;
-    skipRowsWithUnresolvedRefs?: boolean;
-    setNullForUnresolvedNullableRefs?: boolean;
-}
-declare class DuplicatorReference {
-    base: DuplicatorItemHolder;
-    ref: DuplicatorItemHolder;
-    isMandatory: boolean;
-    foreignKey: ForeignKeyInfo;
-    constructor(base: DuplicatorItemHolder, ref: DuplicatorItemHolder, isMandatory: boolean, foreignKey: ForeignKeyInfo);
-    get columnName(): string;
-}
-declare class DuplicatorWeakReference {
-    base: DuplicatorItemHolder;
-    ref: TableInfo;
-    foreignKey: ForeignKeyInfo;
-    constructor(base: DuplicatorItemHolder, ref: TableInfo, foreignKey: ForeignKeyInfo);
-    get columnName(): string;
-}
-declare class DuplicatorItemHolder {
-    item: DataDuplicatorItem;
-    duplicator: DataDuplicator;
-    references: DuplicatorReference[];
-    backReferences: DuplicatorReference[];
-    weakReferences: DuplicatorWeakReference[];
-    table: TableInfo;
-    isPlanned: boolean;
-    idMap: {};
-    autoColumn: string;
-    refByColumn: {
-        [columnName: string]: DuplicatorReference;
-    };
-    isReferenced: boolean;
-    get name(): string;
-    constructor(item: DataDuplicatorItem, duplicator: DataDuplicator);
-    initializeReferences(): void;
-    createInsertObject(chunk: any, weakrefcols: string[]): import("lodash").Omit<Pick<any, string>, string>;
-    getMissingWeakRefsForRow(row: any): Promise<string[]>;
-    runImport(): Promise<{
-        inserted: number;
-        mapped: number;
-        missing: number;
-        skipped: number;
-    }>;
-}
-export declare class DataDuplicator {
-    pool: any;
-    driver: EngineDriver;
-    db: DatabaseInfo;
-    items: DataDuplicatorItem[];
-    stream: any;
-    copyStream: (input: any, output: any) => Promise<void>;
-    options: DataDuplicatorOptions;
-    itemHolders: DuplicatorItemHolder[];
-    itemPlan: DuplicatorItemHolder[];
-    constructor(pool: any, driver: EngineDriver, db: DatabaseInfo, items: DataDuplicatorItem[], stream: any, copyStream: (input: any, output: any) => Promise<void>, options?: DataDuplicatorOptions);
-    findItemToPlan(): DuplicatorItemHolder;
-    createPlan(): void;
-    run(): Promise<void>;
-}
-export {};

package/lib/DataDuplicator.js

@@ -1,274 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DataDuplicator = void 0;
-const dbgate_tools_1 = require("dbgate-tools");
-const pick_1 = __importDefault(require("lodash/pick"));
-const omit_1 = __importDefault(require("lodash/omit"));
-const logger = (0, dbgate_tools_1.getLogger)('dataDuplicator');
-class DuplicatorReference {
-    constructor(base, ref, isMandatory, foreignKey) {
-        this.base = base;
-        this.ref = ref;
-        this.isMandatory = isMandatory;
-        this.foreignKey = foreignKey;
-    }
-    get columnName() {
-        return this.foreignKey.columns[0].columnName;
-    }
-}
-class DuplicatorWeakReference {
-    constructor(base, ref, foreignKey) {
-        this.base = base;
-        this.ref = ref;
-        this.foreignKey = foreignKey;
-    }
-    get columnName() {
-        return this.foreignKey.columns[0].columnName;
-    }
-}
-class DuplicatorItemHolder {
-    get name() {
-        return this.item.name;
-    }
-    constructor(item, duplicator) {
-        var _a, _b, _c, _d, _e, _f;
-        this.item = item;
-        this.duplicator = duplicator;
-        this.references = [];
-        this.backReferences = [];
-        // not mandatory references to entities out of the model
-        this.weakReferences = [];
-        this.isPlanned = false;
-        this.idMap = {};
-        this.refByColumn = {};
-        this.table = duplicator.db.tables.find(x => x.pureName.toUpperCase() == item.name.toUpperCase());
-        this.autoColumn = (_a = this.table.columns.find(x => x.autoIncrement)) === null || _a === void 0 ? void 0 : _a.columnName;
-        if (((_c = (_b = this.table.primaryKey) === null || _b === void 0 ? void 0 : _b.columns) === null || _c === void 0 ? void 0 : _c.length) != 1 ||
-            ((_f = (_e = (_d = this.table.primaryKey) === null || _d === void 0 ? void 0 : _d.columns) === null || _e === void 0 ? void 0 : _e[0]) === null || _f === void 0 ? void 0 : _f.columnName) != this.autoColumn) {
-            this.autoColumn = null;
-        }
-    }
-    initializeReferences() {
-        var _a, _b;
-        for (const fk of this.table.foreignKeys) {
-            if (((_a = fk.columns) === null || _a === void 0 ? void 0 : _a.length) != 1)
-                continue;
-            const refHolder = this.duplicator.itemHolders.find(y => y.name.toUpperCase() == fk.refTableName.toUpperCase());
-            const isMandatory = (_b = this.table.columns.find(x => { var _a; return x.columnName == ((_a = fk.columns[0]) === null || _a === void 0 ? void 0 : _a.columnName); })) === null || _b === void 0 ? void 0 : _b.notNull;
-            if (refHolder == null) {
-                if (!isMandatory) {
-                    const weakref = new DuplicatorWeakReference(this, this.duplicator.db.tables.find(x => x.pureName == fk.refTableName), fk);
-                    this.weakReferences.push(weakref);
-                }
-            }
-            else {
-                const newref = new DuplicatorReference(this, refHolder, isMandatory, fk);
-                this.references.push(newref);
-                this.refByColumn[newref.columnName] = newref;
-                refHolder.isReferenced = true;
-            }
-        }
-    }
-    createInsertObject(chunk, weakrefcols) {
-        const res = (0, omit_1.default)((0, pick_1.default)(chunk, this.table.columns.map(x => x.columnName)), [this.autoColumn, ...this.backReferences.map(x => x.columnName), ...weakrefcols]);
-        for (const key in res) {
-            const ref = this.refByColumn[key];
-            if (ref) {
-                // remap id
-                res[key] = ref.ref.idMap[res[key]];
-                if (ref.isMandatory && res[key] == null) {
-                    // mandatory refertence not matched
-                    if (this.duplicator.options.skipRowsWithUnresolvedRefs) {
-                        return null;
-                    }
-                    throw new Error(`Unresolved reference, base=${ref.base.name}, ref=${ref.ref.name}, ${key}=${chunk[key]}`);
-                }
-            }
-        }
-        return res;
-    }
-    // returns list of columns that are weak references and are not resolved
-    getMissingWeakRefsForRow(row) {
-        var _a;
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.duplicator.options.setNullForUnresolvedNullableRefs || !((_a = this.weakReferences) === null || _a === void 0 ? void 0 : _a.length)) {
-                return [];
-            }
-            const qres = yield (0, dbgate_tools_1.runQueryOnDriver)(this.duplicator.pool, this.duplicator.driver, dmp => {
-                dmp.put('^select ');
-                dmp.putCollection(',', this.weakReferences, weakref => {
-                    dmp.put('(^case ^when ^exists (^select * ^from %f where %i = %v) ^then 1 ^else 0 ^end) as %i', weakref.ref, weakref.foreignKey.columns[0].refColumnName, row[weakref.foreignKey.columns[0].columnName], weakref.foreignKey.columns[0].columnName);
-                });
-                if (this.duplicator.driver.dialect.requireFromDual) {
-                    dmp.put(' ^from ^dual');
-                }
-            });
-            const qrow = qres.rows[0];
-            return this.weakReferences.filter(x => qrow[x.columnName] == 0).map(x => x.columnName);
-        });
-    }
-    runImport() {
-        return __awaiter(this, void 0, void 0, function* () {
-            const readStream = yield this.item.openStream();
-            const driver = this.duplicator.driver;
-            const pool = this.duplicator.pool;
-            let inserted = 0;
-            let mapped = 0;
-            let missing = 0;
-            let skipped = 0;
-            let lastLogged = new Date();
-            const existingWeakRefs = {};
-            const writeStream = (0, dbgate_tools_1.createAsyncWriteStream)(this.duplicator.stream, {
-                processItem: (chunk) => __awaiter(this, void 0, void 0, function* () {
-                    var _a, _b, _c;
-                    if (chunk.__isStreamHeader) {
-                        return;
-                    }
-                    const doCopy = () => __awaiter(this, void 0, void 0, function* () {
-                        var _d, _e, _f;
-                        // console.log('chunk', this.name, JSON.stringify(chunk));
-                        const weakrefcols = yield this.getMissingWeakRefsForRow(chunk);
-                        const insertedObj = this.createInsertObject(chunk, weakrefcols);
-                        // console.log('insertedObj', this.name, JSON.stringify(insertedObj));
-                        if (insertedObj == null) {
-                            skipped += 1;
-                            return;
-                        }
-                        let res = yield (0, dbgate_tools_1.runQueryOnDriver)(pool, driver, dmp => {
-                            dmp.put('^insert ^into %f (%,i) ^values (%,v)', this.table, Object.keys(insertedObj), Object.values(insertedObj));
-                            if (this.autoColumn &&
-                                this.isReferenced &&
-                                !this.duplicator.driver.dialect.requireStandaloneSelectForScopeIdentity) {
-                                dmp.selectScopeIdentity(this.table);
-                            }
-                        });
-                        inserted += 1;
-                        if (this.autoColumn && this.isReferenced) {
-                            if (this.duplicator.driver.dialect.requireStandaloneSelectForScopeIdentity) {
-                                res = yield (0, dbgate_tools_1.runQueryOnDriver)(pool, driver, dmp => dmp.selectScopeIdentity(this.table));
-                            }
-                            // console.log('IDRES', JSON.stringify(res));
-                            // console.log('*********** ENTRIES OF', res?.rows?.[0]);
-                            const resId = (_f = (_e = Object.entries((_d = res === null || res === void 0 ? void 0 : res.rows) === null || _d === void 0 ? void 0 : _d[0])) === null || _e === void 0 ? void 0 : _e[0]) === null || _f === void 0 ? void 0 : _f[1];
-                            if (resId != null) {
-                                this.idMap[chunk[this.autoColumn]] = resId;
-                            }
-                        }
-                    });
-                    switch (this.item.operation) {
-                        case 'copy': {
-                            yield doCopy();
-                            break;
-                        }
-                        case 'insertMissing':
-                        case 'lookup': {
-                            const res = yield (0, dbgate_tools_1.runQueryOnDriver)(pool, driver, dmp => dmp.put('^select %i ^from %f ^where %i = %v', this.autoColumn, this.table, this.item.matchColumns[0], chunk[this.item.matchColumns[0]]));
-                            const resId = (_c = (_b = Object.entries((_a = res === null || res === void 0 ? void 0 : res.rows) === null || _a === void 0 ? void 0 : _a[0])) === null || _b === void 0 ? void 0 : _b[0]) === null || _c === void 0 ? void 0 : _c[1];
-                            if (resId != null) {
-                                mapped += 1;
-                                this.idMap[chunk[this.autoColumn]] = resId;
-                            }
-                            else if (this.item.operation == 'insertMissing') {
-                                yield doCopy();
-                            }
-                            else {
-                                missing += 1;
-                            }
-                            break;
-                        }
-                    }
-                    if (new Date().getTime() - lastLogged.getTime() > 5000) {
-                        logger.info(`Duplicating ${this.item.name} in progress, inserted ${inserted} rows, mapped ${mapped} rows, missing ${missing} rows, skipped ${skipped} rows`);
-                        lastLogged = new Date();
-                    }
-                    // this.idMap[oldId] = newId;
-                }),
-            });
-            yield this.duplicator.copyStream(readStream, writeStream);
-            // await this.duplicator.driver.writeQueryStream(this.duplicator.pool, {
-            //   mapResultId: (oldId, newId) => {
-            //     this.idMap[oldId] = newId;
-            //   },
-            // });
-            return { inserted, mapped, missing, skipped };
-        });
-    }
-}
-class DataDuplicator {
-    constructor(pool, driver, db, items, stream, copyStream, options = {}) {
-        this.pool = pool;
-        this.driver = driver;
-        this.db = db;
-        this.items = items;
-        this.stream = stream;
-        this.copyStream = copyStream;
-        this.options = options;
-        this.itemPlan = [];
-        this.itemHolders = items.map(x => new DuplicatorItemHolder(x, this));
-        this.itemHolders.forEach(x => x.initializeReferences());
-    }
-    findItemToPlan() {
-        for (const item of this.itemHolders) {
-            if (item.isPlanned)
-                continue;
-            if (item.references.every(x => x.ref.isPlanned)) {
-                return item;
-            }
-        }
-        for (const item of this.itemHolders) {
-            if (item.isPlanned)
-                continue;
-            if (item.references.every(x => x.ref.isPlanned || !x.isMandatory)) {
-                const backReferences = item.references.filter(x => !x.ref.isPlanned);
-                item.backReferences = backReferences;
-                return item;
-            }
-        }
-        throw new Error('Cycle in mandatory references');
-    }
-    createPlan() {
-        while (this.itemPlan.length < this.itemHolders.length) {
-            const item = this.findItemToPlan();
-            item.isPlanned = true;
-            this.itemPlan.push(item);
-        }
-    }
-    run() {
-        return __awaiter(this, void 0, void 0, function* () {
-            this.createPlan();
-            yield (0, dbgate_tools_1.runCommandOnDriver)(this.pool, this.driver, dmp => dmp.beginTransaction());
-            try {
-                for (const item of this.itemPlan) {
-                    const stats = yield item.runImport();
-                    logger.info(`Duplicated ${item.name}, inserted ${stats.inserted} rows, mapped ${stats.mapped} rows, missing ${stats.missing} rows, skipped ${stats.skipped} rows`);
-                }
-            }
-            catch (err) {
-                logger.error((0, dbgate_tools_1.extractErrorLogData)(err), `Failed duplicator job, rollbacking. ${err.message}`);
-                yield (0, dbgate_tools_1.runCommandOnDriver)(this.pool, this.driver, dmp => dmp.rollbackTransaction());
-                return;
-            }
-            if (this.options.rollbackAfterFinish) {
-                logger.info('Rollbacking transaction, nothing was changed');
-                yield (0, dbgate_tools_1.runCommandOnDriver)(this.pool, this.driver, dmp => dmp.rollbackTransaction());
-            }
-            else {
-                logger.info('Committing duplicator transaction');
-                yield (0, dbgate_tools_1.runCommandOnDriver)(this.pool, this.driver, dmp => dmp.commitTransaction());
-            }
-        });
-    }
-}
-exports.DataDuplicator = DataDuplicator;