dbgate-datalib 5.5.7-alpha.16 → 5.5.7-alpha.26

package/lib/DataDuplicator.d.ts

@@ -9,6 +9,7 @@ export interface DataDuplicatorItem {
 export interface DataDuplicatorOptions {
     rollbackAfterFinish?: boolean;
     skipRowsWithUnresolvedRefs?: boolean;
+    setNullForUnresolvedNullableRefs?: boolean;
 }
 declare class DuplicatorReference {
     base: DuplicatorItemHolder;
@@ -18,11 +19,19 @@ declare class DuplicatorReference {
     constructor(base: DuplicatorItemHolder, ref: DuplicatorItemHolder, isMandatory: boolean, foreignKey: ForeignKeyInfo);
     get columnName(): string;
 }
+declare class DuplicatorWeakReference {
+    base: DuplicatorItemHolder;
+    ref: TableInfo;
+    foreignKey: ForeignKeyInfo;
+    constructor(base: DuplicatorItemHolder, ref: TableInfo, foreignKey: ForeignKeyInfo);
+    get columnName(): string;
+}
 declare class DuplicatorItemHolder {
     item: DataDuplicatorItem;
     duplicator: DataDuplicator;
     references: DuplicatorReference[];
     backReferences: DuplicatorReference[];
+    weakReferences: DuplicatorWeakReference[];
     table: TableInfo;
     isPlanned: boolean;
     idMap: {};
@@ -34,7 +43,8 @@ declare class DuplicatorItemHolder {
     get name(): string;
     constructor(item: DataDuplicatorItem, duplicator: DataDuplicator);
     initializeReferences(): void;
-    createInsertObject(chunk: any): import("lodash").Omit<Pick<any, string>, string>;
+    createInsertObject(chunk: any, weakrefcols: string[]): import("lodash").Omit<Pick<any, string>, string>;
+    getMissingWeakRefsForRow(row: any): Promise<string[]>;
     runImport(): Promise<{
         inserted: number;
         mapped: number;

package/lib/DataDuplicator.js

@@ -28,6 +28,16 @@ class DuplicatorReference {
         return this.foreignKey.columns[0].columnName;
     }
 }
+class DuplicatorWeakReference {
+    constructor(base, ref, foreignKey) {
+        this.base = base;
+        this.ref = ref;
+        this.foreignKey = foreignKey;
+    }
+    get columnName() {
+        return this.foreignKey.columns[0].columnName;
+    }
+}
 class DuplicatorItemHolder {
     get name() {
         return this.item.name;
@@ -38,6 +48,8 @@ class DuplicatorItemHolder {
         this.duplicator = duplicator;
         this.references = [];
         this.backReferences = [];
+        // not mandatory references to entities out of the model
+        this.weakReferences = [];
         this.isPlanned = false;
         this.idMap = {};
         this.refByColumn = {};
@@ -54,17 +66,23 @@ class DuplicatorItemHolder {
             if (((_a = fk.columns) === null || _a === void 0 ? void 0 : _a.length) != 1)
                 continue;
             const refHolder = this.duplicator.itemHolders.find(y => y.name.toUpperCase() == fk.refTableName.toUpperCase());
-            if (refHolder == null)
-                continue;
             const isMandatory = (_b = this.table.columns.find(x => { var _a; return x.columnName == ((_a = fk.columns[0]) === null || _a === void 0 ? void 0 : _a.columnName); })) === null || _b === void 0 ? void 0 : _b.notNull;
-            const newref = new DuplicatorReference(this, refHolder, isMandatory, fk);
-            this.references.push(newref);
-            this.refByColumn[newref.columnName] = newref;
-            refHolder.isReferenced = true;
+            if (refHolder == null) {
+                if (!isMandatory) {
+                    const weakref = new DuplicatorWeakReference(this, this.duplicator.db.tables.find(x => x.pureName == fk.refTableName), fk);
+                    this.weakReferences.push(weakref);
+                }
+            }
+            else {
+                const newref = new DuplicatorReference(this, refHolder, isMandatory, fk);
+                this.references.push(newref);
+                this.refByColumn[newref.columnName] = newref;
+                refHolder.isReferenced = true;
+            }
         }
     }
-    createInsertObject(chunk) {
-        const res = (0, omit_1.default)((0, pick_1.default)(chunk, this.table.columns.map(x => x.columnName)), [this.autoColumn, ...this.backReferences.map(x => x.columnName)]);
+    createInsertObject(chunk, weakrefcols) {
+        const res = (0, omit_1.default)((0, pick_1.default)(chunk, this.table.columns.map(x => x.columnName)), [this.autoColumn, ...this.backReferences.map(x => x.columnName), ...weakrefcols]);
         for (const key in res) {
             const ref = this.refByColumn[key];
             if (ref) {
@@ -81,6 +99,23 @@ class DuplicatorItemHolder {
         }
         return res;
     }
+    // returns list of columns that are weak references and are not resolved
+    getMissingWeakRefsForRow(row) {
+        var _a;
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.duplicator.options.setNullForUnresolvedNullableRefs || !((_a = this.weakReferences) === null || _a === void 0 ? void 0 : _a.length)) {
+                return [];
+            }
+            const qres = yield (0, dbgate_tools_1.runQueryOnDriver)(this.duplicator.pool, this.duplicator.driver, dmp => {
+                dmp.put('^select ');
+                dmp.putCollection(',', this.weakReferences, weakref => {
+                    dmp.put('(^case ^when ^exists (^select * ^from %f where %i = %v) ^then 1 ^else 0 ^end) as %i', weakref.ref, weakref.foreignKey.columns[0].refColumnName, row[weakref.foreignKey.columns[0].columnName], weakref.foreignKey.columns[0].columnName);
+                });
+            });
+            const qrow = qres.rows[0];
+            return this.weakReferences.filter(x => qrow[x.columnName] == 0).map(x => x.columnName);
+        });
+    }
     runImport() {
         return __awaiter(this, void 0, void 0, function* () {
             const readStream = yield this.item.openStream();
@@ -91,6 +126,7 @@ class DuplicatorItemHolder {
             let missing = 0;
             let skipped = 0;
             let lastLogged = new Date();
+            const existingWeakRefs = {};
             const writeStream = (0, dbgate_tools_1.createAsyncWriteStream)(this.duplicator.stream, {
                 processItem: (chunk) => __awaiter(this, void 0, void 0, function* () {
                     var _a, _b, _c;
@@ -100,7 +136,8 @@ class DuplicatorItemHolder {
                     const doCopy = () => __awaiter(this, void 0, void 0, function* () {
                         var _d, _e, _f;
                         // console.log('chunk', this.name, JSON.stringify(chunk));
-                        const insertedObj = this.createInsertObject(chunk);
+                        const weakrefcols = yield this.getMissingWeakRefsForRow(chunk);
+                        const insertedObj = this.createInsertObject(chunk, weakrefcols);
                         // console.log('insertedObj', this.name, JSON.stringify(insertedObj));
                         if (insertedObj == null) {
                             skipped += 1;

package/lib/FreeTableGridDisplay.d.ts

@@ -34,7 +34,6 @@ export declare class FreeTableGridDisplay extends GridDisplay {
         undropColumnName?: string;
         contentHash?: string;
         engine?: string;
-        undropPureName?: string;
     }[];
     getDisplayColumn(col: ColumnInfo): {
         pureName: string;
@@ -64,6 +63,5 @@ export declare class FreeTableGridDisplay extends GridDisplay {
         undropColumnName?: string;
         contentHash?: string;
         engine?: string;
-        undropPureName?: string;
     };
 }

package/lib/ScriptDrivedDeployer.d.ts

@@ -0,0 +1,32 @@
+import { DatabaseModelFile } from 'dbgate-tools';
+import { EngineDriver } from 'dbgate-types';
+interface DeployScriptJournalItem {
+    id: number;
+    name: string;
+    category: string;
+    first_run_date: string;
+    last_run_date: string;
+    script_hash: string;
+}
+export declare class ScriptDrivedDeployer {
+    dbhan: any;
+    driver: EngineDriver;
+    files: DatabaseModelFile[];
+    predeploy: DatabaseModelFile[];
+    uninstall: DatabaseModelFile[];
+    install: DatabaseModelFile[];
+    once: DatabaseModelFile[];
+    postdeploy: DatabaseModelFile[];
+    isEmpty: boolean;
+    journalItems: DeployScriptJournalItem[];
+    constructor(dbhan: any, driver: EngineDriver, files: DatabaseModelFile[]);
+    loadJournalItems(): Promise<void>;
+    runPre(): Promise<void>;
+    runPost(): Promise<void>;
+    run(): Promise<void>;
+    runFiles(files: DatabaseModelFile[], category: string): Promise<void>;
+    saveToJournal(file: DatabaseModelFile, category: string, hash: string): Promise<void>;
+    runFileCore(file: DatabaseModelFile, category: string, hash: string): Promise<void>;
+    runFile(file: DatabaseModelFile, category: string): Promise<void>;
+}
+export {};

package/lib/ScriptDrivedDeployer.js

@@ -0,0 +1,171 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ScriptDrivedDeployer = void 0;
+const dbgate_tools_1 = require("dbgate-tools");
+const crypto_1 = __importDefault(require("crypto"));
+const sortBy_1 = __importDefault(require("lodash/sortBy"));
+const logger = (0, dbgate_tools_1.getLogger)('ScriptDrivedDeployer');
+class ScriptDrivedDeployer {
+    constructor(dbhan, driver, files) {
+        this.dbhan = dbhan;
+        this.driver = driver;
+        this.files = files;
+        this.predeploy = [];
+        this.uninstall = [];
+        this.install = [];
+        this.once = [];
+        this.postdeploy = [];
+        this.isEmpty = false;
+        this.journalItems = [];
+        this.predeploy = files.filter(x => x.name.endsWith('.predeploy.sql'));
+        this.uninstall = files.filter(x => x.name.endsWith('.uninstall.sql'));
+        this.install = files.filter(x => x.name.endsWith('.install.sql'));
+        this.once = files.filter(x => x.name.endsWith('.once.sql'));
+        this.postdeploy = files.filter(x => x.name.endsWith('.postdeploy.sql'));
+        this.isEmpty =
+            this.predeploy.length === 0 &&
+                this.uninstall.length === 0 &&
+                this.install.length === 0 &&
+                this.once.length === 0 &&
+                this.postdeploy.length === 0;
+    }
+    loadJournalItems() {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const { rows } = yield (0, dbgate_tools_1.runQueryOnDriver)(this.dbhan, this.driver, dmp => dmp.put('select * from ~dbgate_deploy_journal'));
+                this.journalItems = rows;
+                logger.debug(`Loaded ${rows.length} items from DbGate deploy journal`);
+            }
+            catch (err) {
+                logger.warn((0, dbgate_tools_1.extractErrorLogData)(err), 'Error loading DbGate deploy journal, creating table dbgate_deploy_journal');
+                const dmp = this.driver.createDumper();
+                dmp.createTable({
+                    pureName: 'dbgate_deploy_journal',
+                    columns: [
+                        { columnName: 'id', dataType: 'int', autoIncrement: true, notNull: true, pureName: 'dbgate_deploy_journal' },
+                        { columnName: 'name', dataType: 'varchar(100)', notNull: true, pureName: 'dbgate_deploy_journal' },
+                        { columnName: 'category', dataType: 'varchar(100)', notNull: true, pureName: 'dbgate_deploy_journal' },
+                        { columnName: 'script_hash', dataType: 'varchar(100)', notNull: true, pureName: 'dbgate_deploy_journal' },
+                        { columnName: 'first_run_date', dataType: 'varchar(100)', notNull: true, pureName: 'dbgate_deploy_journal' },
+                        { columnName: 'last_run_date', dataType: 'varchar(100)', notNull: true, pureName: 'dbgate_deploy_journal' },
+                        { columnName: 'run_count', dataType: 'int', notNull: true, pureName: 'dbgate_deploy_journal' },
+                    ],
+                    foreignKeys: [],
+                    primaryKey: {
+                        columns: [{ columnName: 'id' }],
+                        constraintType: 'primaryKey',
+                        pureName: 'dbgate_deploy_journal',
+                    },
+                });
+                yield this.driver.query(this.dbhan, dmp.s, { discardResult: true });
+            }
+        });
+    }
+    runPre() {
+        return __awaiter(this, void 0, void 0, function* () {
+            // don't create journal table if no scripts are present
+            if (this.isEmpty)
+                return;
+            yield this.loadJournalItems();
+            yield this.runFiles(this.predeploy, 'predeploy');
+        });
+    }
+    runPost() {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.runFiles(this.install, 'install');
+            yield this.runFiles(this.once, 'once');
+            yield this.runFiles(this.postdeploy, 'postdeploy');
+        });
+    }
+    run() {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.runPre();
+            yield this.runPost();
+        });
+    }
+    runFiles(files, category) {
+        return __awaiter(this, void 0, void 0, function* () {
+            for (const file of (0, sortBy_1.default)(files, x => x.name)) {
+                yield this.runFile(file, category);
+            }
+        });
+    }
+    saveToJournal(file, category, hash) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const existing = this.journalItems.find(x => x.name == file.name);
+            if (existing) {
+                yield (0, dbgate_tools_1.runCommandOnDriver)(this.dbhan, this.driver, dmp => {
+                    dmp.put('update ~dbgate_deploy_journal set ~last_run_date = %v, ~script_hash = %v, ~run_count = ~run_count + 1 where ~id = %v', new Date().toISOString(), hash, existing.id);
+                });
+            }
+            else {
+                yield (0, dbgate_tools_1.runCommandOnDriver)(this.dbhan, this.driver, dmp => {
+                    dmp.put('insert into ~dbgate_deploy_journal (~name, ~category, ~first_run_date, ~last_run_date, ~script_hash, ~run_count) values (%v, %v, %v, %v, %v, 1)', file.name, category, new Date().toISOString(), new Date().toISOString(), hash);
+                });
+            }
+        });
+    }
+    runFileCore(file, category, hash) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.driver.supportsTransactions) {
+                (0, dbgate_tools_1.runCommandOnDriver)(this.dbhan, this.driver, dmp => dmp.beginTransaction());
+            }
+            logger.debug(`Running ${category} script ${file.name}`);
+            try {
+                yield this.driver.script(this.dbhan, file.text);
+                yield this.saveToJournal(file, category, hash);
+            }
+            catch (err) {
+                logger.error((0, dbgate_tools_1.extractErrorLogData)(err), `Error running ${category} script ${file.name}`);
+                if (this.driver.supportsTransactions) {
+                    (0, dbgate_tools_1.runCommandOnDriver)(this.dbhan, this.driver, dmp => dmp.rollbackTransaction());
+                    return;
+                }
+            }
+            if (this.driver.supportsTransactions) {
+                (0, dbgate_tools_1.runCommandOnDriver)(this.dbhan, this.driver, dmp => dmp.commitTransaction());
+            }
+        });
+    }
+    runFile(file, category) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const hash = crypto_1.default.createHash('md5').update(file.text.trim()).digest('hex');
+            const journalItem = this.journalItems.find(x => x.name == file.name);
+            const isEqual = journalItem && journalItem.script_hash == hash;
+            switch (category) {
+                case 'predeploy':
+                case 'postdeploy':
+                    yield this.runFileCore(file, category, hash);
+                    break;
+                case 'once':
+                    if (journalItem)
+                        return;
+                    yield this.runFileCore(file, category, hash);
+                    break;
+                case 'install':
+                    if (isEqual)
+                        return;
+                    const uninstallFile = this.uninstall.find(x => x.name == file.name.replace('.install.sql', '.uninstall.sql'));
+                    if (uninstallFile && journalItem) {
+                        // file was previously installed, uninstall first
+                        yield this.runFileCore(uninstallFile, 'uninstall', crypto_1.default.createHash('md5').update(uninstallFile.text.trim()).digest('hex'));
+                    }
+                    yield this.runFileCore(file, category, hash);
+                    break;
+            }
+        });
+    }
+}
+exports.ScriptDrivedDeployer = ScriptDrivedDeployer;

package/lib/TableGridDisplay.d.ts

@@ -56,7 +56,6 @@ export declare class TableGridDisplay extends GridDisplay {
         undropColumnName?: string;
         contentHash?: string;
         engine?: string;
-        undropPureName?: string;
     }[];
     addJoinsFromExpandedColumns(select: Select, columns: DisplayColumn[], parentAlias: string, columnSources: any): void;
     addReferenceToSelect(select: Select, parentAlias: string, column: DisplayColumn): void;
@@ -99,7 +98,6 @@ export declare class TableGridDisplay extends GridDisplay {
         undropColumnName?: string;
         contentHash?: string;
         engine?: string;
-        undropPureName?: string;
     };
     addAddedColumnsToSelect(select: Select, columns: DisplayColumn[], parentAlias: string, displayedColumnInfo: DisplayedColumnInfo): void;
     get hasReferences(): boolean;

package/lib/ViewGridDisplay.d.ts

@@ -33,7 +33,6 @@ export declare class ViewGridDisplay extends GridDisplay {
         undropColumnName?: string;
         contentHash?: string;
         engine?: string;
-        undropPureName?: string;
     }[];
     getDisplayColumn(view: ViewInfo, col: ColumnInfo): {
         pureName: string;
@@ -63,7 +62,6 @@ export declare class ViewGridDisplay extends GridDisplay {
         undropColumnName?: string;
         contentHash?: string;
         engine?: string;
-        undropPureName?: string;
     };
     createSelect(options?: {}): import("dbgate-sqltree").Select;
 }

package/lib/index.d.ts

@@ -22,3 +22,4 @@ export * from './DataDuplicator';
 export * from './FreeTableGridDisplay';
 export * from './FreeTableModel';
 export * from './CustomGridDisplay';
+export * from './ScriptDrivedDeployer';

package/lib/index.js

@@ -38,3 +38,4 @@ __exportStar(require("./DataDuplicator"), exports);
 __exportStar(require("./FreeTableGridDisplay"), exports);
 __exportStar(require("./FreeTableModel"), exports);
 __exportStar(require("./CustomGridDisplay"), exports);
+__exportStar(require("./ScriptDrivedDeployer"), exports);

package/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "5.5.7-alpha.16",
+  "version": "5.5.7-alpha.26",
   "name": "dbgate-datalib",
   "main": "lib/index.js",
   "typings": "lib/index.d.ts",
@@ -13,13 +13,13 @@
     "lib"
   ],
   "dependencies": {
-    "dbgate-sqltree": "^5.5.7-alpha.16",
-    "dbgate-tools": "^5.5.7-alpha.16",
-    "dbgate-filterparser": "^5.5.7-alpha.16",
+    "dbgate-sqltree": "^5.5.7-alpha.26",
+    "dbgate-tools": "^5.5.7-alpha.26",
+    "dbgate-filterparser": "^5.5.7-alpha.26",
     "uuid": "^3.4.0"
   },
   "devDependencies": {
-    "dbgate-types": "^5.5.7-alpha.16",
+    "dbgate-types": "^5.5.7-alpha.26",
     "@types/node": "^13.7.0",
     "jest": "^28.1.3",
     "ts-jest": "^28.0.7",