@sqb/migrator 4.1.5 → 4.5.0

package/README.md

@@ -0,0 +1,69 @@
+
+<center>
+<p>
+  <img src="https://user-images.githubusercontent.com/3836517/32965280-1a2b63ce-cbe7-11e7-8ee1-ba47313503c5.png" width="500px" alt="SQB Logo"/>
+</p>
+</center>
+
+<br>
+  
+[![NPM Version][npm-image]][npm-url]
+[![NPM Downloads][downloads-image]][downloads-url]
+[![Build Status][travis-image]][travis-url]
+[![Test Coverage][coveralls-image]][coveralls-url]
+[![Dependencies][dependencies-image]][dependencies-url]
+[![DevDependencies][devdependencies-image]][devdependencies-url]
+[![Package Quality][quality-image]][quality-url]
+
+
+## About SQB
+
+SQB is an extensible, multi-dialect SQL query builder and Database connection wrapper for NodeJS.
+
+## Main goals
+
+- Single code base for any sql based database
+- Powerful and simplified query coding scheme
+- Fast applications with low memory requirements
+- Let applications work with large data tables efficiently
+- Support latest JavaScript language standards
+- Lightweight and extensible framework. 
+
+
+You can report bugs and discuss features on the [GitHub issues](https://github.com/sqbjs/sqb/issues) page
+
+Thanks to all of the great [contributions](https://github.com/sqbjs/sqb/graphs/contributors) to the project.
+
+You may want to check detailed [DOCUMENTATION](https://sqbjs.github.io/sqb/)
+
+
+## Installation
+
+```bash
+$ npm install @sqb/migrator --save
+```
+
+## Node Compatibility
+
+  - node >= 16.x
+  
+   
+### License
+SQB is available under [MIT](LICENSE) license.
+
+[npm-image]: https://img.shields.io/npm/v/@sqb/migrator.svg
+[npm-url]: https://npmjs.org/package/@sqb/migrator
+[travis-image]: https://img.shields.io/travis/sqbjs/@sqb/migrator/master.svg
+[travis-url]: https://travis-ci.org/sqbjs/@sqb/migrator
+[coveralls-image]: https://img.shields.io/coveralls/sqbjs/@sqb/migrator/master.svg
+[coveralls-url]: https://coveralls.io/r/sqbjs/@sqb/migrator
+[downloads-image]: https://img.shields.io/npm/dm/@sqb/migrator.svg
+[downloads-url]: https://npmjs.org/package/@sqb/migrator
+[gitter-image]: https://badges.gitter.im/sqbjs/@sqb/migrator.svg
+[gitter-url]: https://gitter.im/sqbjs/@sqb/migrator?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
+[dependencies-image]: https://david-dm.org/sqbjs/@sqb/migrator/status.svg
+[dependencies-url]:https://david-dm.org/sqbjs/@sqb/migrator
+[devdependencies-image]: https://david-dm.org/sqbjs/@sqb/migrator/dev-status.svg
+[devdependencies-url]:https://david-dm.org/sqbjs/@sqb/migrator?type=dev
+[quality-image]: http://npm.packagequality.com/shield/@sqb/migrator.png
+[quality-url]: http://packagequality.com/#?package=@sqb/migrator

package/{dist → cjs}/db-migrator.js

@@ -5,7 +5,7 @@ const tslib_1 = require("tslib");
 const fs_1 = tslib_1.__importDefault(require("fs"));
 const path_1 = tslib_1.__importDefault(require("path"));
 const postgresql_client_1 = require("postgresql-client");
-const load_task_files_1 = require("./load-task-files");
+const load_task_files_js_1 = require("./load-task-files.js");
 const ignoreEventListener = () => 0;
 class DbMigrator {
     async execute(options) {
@@ -108,7 +108,7 @@ class DbMigrator {
             maxVersion = Math.max(minVersion, m.version);
             for (const t of m.tasks) {
                 if (typeof t === 'string') {
-                    m.tasks = (0, load_task_files_1.loadTaskFiles)(t);
+                    m.tasks = (0, load_task_files_js_1.loadTaskFiles)(t);
                 }
             }
         }

package/cjs/index.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const tslib_1 = require("tslib");
+tslib_1.__exportStar(require("./db-migrator.js"), exports);
+tslib_1.__exportStar(require("./types.js"), exports);

package/cjs/package.json

@@ -0,0 +1,3 @@
+{
+  "type": "commonjs"
+}

package/{dist → esm}/db-migrator.d.ts

@@ -1,5 +1,5 @@
 import { Connection } from 'postgresql-client';
-import { InsertDataMigrationTask, Migration, MigrationPackage } from './types';
+import { InsertDataMigrationTask, Migration, MigrationPackage } from './types.js';
 export interface MigrationExecuteOptions {
     connection: Connection;
     migrationPackage: MigrationPackage;

package/esm/db-migrator.js

@@ -0,0 +1,210 @@
+import fs from 'fs';
+import path from 'path';
+import { stringifyValueForSQL } from 'postgresql-client';
+import { loadTaskFiles } from './load-task-files.js';
+const ignoreEventListener = () => 0;
+export class DbMigrator {
+    async execute(options) {
+        const { connection, schema } = options;
+        const migrationPackage = DbMigrator.loadMigrationPackage(options.migrationPackage);
+        const targetVersion = Math.min(options.targetVersion || Number.MAX_SAFE_INTEGER, migrationPackage.maxVersion);
+        if (targetVersion && targetVersion < migrationPackage.minVersion)
+            throw new Error(`Version mismatch. Target schema version (${targetVersion}) is lower than ` +
+                `migration package min version (${migrationPackage.minVersion})`);
+        const emit = options.eventListener || ignoreEventListener;
+        const schemaInfo = await this._getSchemaInfo(options);
+        if (schemaInfo.version && schemaInfo.version < migrationPackage.minVersion - 1)
+            throw new Error(`This package can migrate starting from ${migrationPackage.minVersion - 1} but current version is ${schemaInfo.version}`);
+        const { migrations } = migrationPackage;
+        // calculate total scripts;
+        const total = migrations
+            .reduce((i, x) => i + x.tasks.length, 0);
+        const needBackup = !!migrations.find(x => !!x.backup);
+        emit('start');
+        let task;
+        try {
+            await this._lockSchema();
+            if (needBackup) {
+                emit('backup');
+                await this._backup();
+            }
+            await connection.execute(`update ${schema}.${schemaInfo.infTable} set status = 'migration'`);
+            // Execute migration tasks
+            for (const migration of migrations) {
+                if (migration.version > targetVersion || schemaInfo.version >= migration.version)
+                    continue;
+                for (let index = 0; index < migration.tasks.length; index++) {
+                    task = migration.tasks[index];
+                    emit('processing', { task: task.title, total, index });
+                    if ('fn' in task && typeof task.fn === 'function') {
+                        await task.fn(connection);
+                    }
+                    else if (task.script) {
+                        const script = task.script.replace(/(\${(\w+)})/gi, (s, ...args) => {
+                            switch (args[1].toLowerCase()) {
+                                case 'schema':
+                                    return schema;
+                                case 'owner':
+                                    return schemaInfo.owner;
+                                case 'tablespace':
+                                    return schemaInfo.tablespace;
+                            }
+                            return s;
+                        });
+                        try {
+                            await connection.execute(script);
+                        }
+                        catch (e) {
+                            e.message += `\n${task.title}`;
+                            if (e.lineNr)
+                                e.message += ` (${e.lineNr},${e.colNr}):\n` + e.line;
+                            if (e.colNr)
+                                e.message += '\n' + ' '.repeat(e.colNr - 1) + '^';
+                            // noinspection ExceptionCaughtLocallyJS
+                            throw e;
+                        }
+                    }
+                    else if ('tableName' in task) {
+                        const script = DbMigrator.objectDataToSql({ ...task, schema });
+                        await connection.execute(script);
+                    }
+                    emit('process', { task: task.title, total, index });
+                }
+            }
+            await connection.query(`update ${schema}.${schemaInfo.infTable} set status = $1, schema_version = $2, ` +
+                'updated_at = current_timestamp', { params: ['ready', targetVersion] });
+        }
+        catch (e) {
+            if (needBackup) {
+                emit('backup');
+                await this._restore();
+            }
+            throw e;
+        }
+        finally {
+            await this._unlockSchema();
+        }
+        emit('finish');
+        return true;
+    }
+    static loadMigrationPackage(pkg) {
+        const migarr = typeof pkg.migrations === 'function' ? pkg.migrations() : pkg.migrations;
+        const migrations = [];
+        for (const x of migarr) {
+            if (typeof x === 'string')
+                locateMigrations(migrations, x);
+            else
+                migrations.push(x);
+        }
+        let minVersion = 0;
+        let maxVersion = 0;
+        migrations.sort((a, b) => a.version - b.version);
+        for (const m of migrations) {
+            minVersion = !minVersion ? m.version : Math.min(minVersion, m.version);
+            maxVersion = Math.max(minVersion, m.version);
+            for (const t of m.tasks) {
+                if (typeof t === 'string') {
+                    m.tasks = loadTaskFiles(t);
+                }
+            }
+        }
+        return {
+            minVersion,
+            maxVersion,
+            migrations
+        };
+    }
+    async _lockSchema() {
+        // todo
+    }
+    async _unlockSchema() {
+        // todo
+    }
+    async _backup() {
+        // todo
+    }
+    async _restore() {
+        // todo
+    }
+    async _getSchemaInfo(options) {
+        const { connection, schema } = options;
+        let owner = options.owner || 'postgres';
+        let tablespace = options.tablespace || 'pg_default';
+        let version = 0;
+        let status = '';
+        let r = await connection.query('select schema_owner FROM information_schema.schemata WHERE schema_name = $1', { params: [schema], objectRows: true });
+        if (r.rows && r.rows[0]) {
+            owner = r.rows[0].schema_owner;
+        }
+        else {
+            await connection.execute(`CREATE SCHEMA ${schema} AUTHORIZATION ${owner};`);
+        }
+        // Determine tablespace of schema
+        const infTable = options.migrationPackage.informationTableName || '__schema_info';
+        r = await connection.query('SELECT tablespace FROM pg_tables WHERE schemaname = $1 AND tablename = $2', { params: [schema, infTable], objectRows: true });
+        if (r.rows && r.rows[0]) {
+            tablespace = r.rows[0].tablespace || tablespace;
+            r = await connection.query(`SELECT schema_version, status FROM ${schema}.${infTable}`, { objectRows: true });
+            if (r.rows && r.rows[0]) {
+                version = r.rows[0].schema_version;
+                status = r.rows[0].status;
+            }
+        }
+        else {
+            // Create if not exists
+            await connection.execute(`
+CREATE TABLE ${schema}.${infTable}
+(
+    status varchar(16) not null,
+    schema_version integer not null default 0,
+    created_at timestamp without time zone not null default current_timestamp,
+    updated_at timestamp without time zone
+) TABLESPACE ${tablespace};
+insert into ${schema}.${infTable} (status) values ('init');
+      `);
+            status = 'init';
+        }
+        return {
+            infTable,
+            tablespace,
+            owner,
+            version,
+            status
+        };
+    }
+    static objectDataToSql(task) {
+        let sql = '';
+        for (const rows of task.rows) {
+            const keys = Object.keys(rows);
+            sql += `insert into ${task.schema}.${task.tableName} (${keys}) values (`;
+            for (let i = 0; i < keys.length; i++) {
+                sql += (i ? ', ' : '') + stringifyValueForSQL(rows[keys[i]]);
+            }
+            sql += ');\n';
+        }
+        return sql;
+    }
+}
+function locateMigrations(trg, dir) {
+    for (const f of ['migration.ts', 'migration.js', 'migration.json']) {
+        const filename = path.join(dir, f);
+        if (fs.existsSync(filename)) {
+            const x = require(filename);
+            const fileDir = path.dirname(filename);
+            const obj = x.default || x;
+            if (obj.version && obj.tasks) {
+                const migration = { ...obj };
+                migration.tasks = obj.tasks.map(k => (typeof k === 'string' ? path.resolve(fileDir, k) : k));
+                trg.push(migration);
+            }
+            return;
+        }
+    }
+    const files = fs.readdirSync(dir);
+    for (const f of files) {
+        const dirname = path.join(dir, f);
+        if (fs.statSync(dirname).isDirectory()) {
+            locateMigrations(trg, dirname);
+        }
+    }
+}

package/esm/index.d.ts

@@ -0,0 +1,2 @@
+export * from './db-migrator.js';
+export * from './types.js';

package/esm/index.js

@@ -0,0 +1,2 @@
+export * from './db-migrator.js';
+export * from './types.js';

package/{dist → esm}/load-task-files.d.ts

@@ -1,2 +1,2 @@
-import { MigrationTask } from './types';
+import { MigrationTask } from './types.js';
 export declare function loadTaskFiles(pattern: string): MigrationTask[];

package/esm/load-task-files.js

@@ -0,0 +1,34 @@
+import glob from 'fast-glob';
+import fs from 'fs';
+import path from 'path';
+export function loadTaskFiles(pattern) {
+    const out = [];
+    const files = glob.sync(pattern.replace(/\\/g, '/'), { absolute: true, onlyFiles: true });
+    for (const filename of files) {
+        const ext = path.extname(filename).toLowerCase();
+        if (ext === '.sql') {
+            const script = fs.readFileSync(filename, 'utf-8');
+            out.push({
+                title: path.basename(filename),
+                script
+            });
+        }
+        else if (ext === '.json') {
+            try {
+                const json = JSON.parse(fs.readFileSync(filename, 'utf-8'));
+                if (typeof json === 'object' && json.tableName && json.rows) {
+                    if (!json.tableName) { // noinspection ExceptionCaughtLocallyJS
+                        throw new Error('"tableName" property does not exists');
+                    }
+                    json.title = json.title || 'Migrate data into ' + json.tableName;
+                    out.push(json);
+                }
+            }
+            catch (e) {
+                e.message = e.message + '\n' + filename;
+                throw e;
+            }
+        }
+    }
+    return out;
+}

package/{dist → esm}/types.d.ts

@@ -1,4 +1,3 @@
-import { Connection } from 'postgresql-client';
 export declare type MigrationsThunk = () => (string | Migration)[];
 export interface MigrationPackage {
     description: string;
@@ -17,7 +16,7 @@ export interface Migration {
     version: number;
     tasks: string[] | MigrationTask[];
 }
-export declare type MigrationTaskFunction = (connection: Connection) => void | Promise<void>;
+export declare type MigrationTaskFunction = (connection: any) => void | Promise<void>;
 export declare type MigrationTask = SqlScriptMigrationTask | CustomMigrationTask | InsertDataMigrationTask;
 export interface BaseMigrationTask {
     title: string;

package/esm/types.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,52 +1,67 @@
 {
-  "name": "@sqb/migrator",
-  "description": "Database migrator for SQB",
-  "version": "4.1.5",
-  "author": "Panates",
-  "contributors": [
-    "Eray Hanoglu <e.hanoglu@panates.com>"
+  "name" : "@sqb/migrator",
+  "description" : "Database migrator for SQB",
+  "version" : "4.5.0",
+  "author" : "Panates",
+  "contributors" : [
+    "Eray Hanoglu <e.hanoglu@panates.com>",
+    "Ilker Gurelli <i.gurelli@panates.com>"
   ],
-  "license": "Apache-2.0",
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/sqbjs/sqb.git",
-    "directory": "packages/migrator"
+  "license" : "Apache-2.0",
+  "repository" : {
+    "type" : "git",
+    "url" : "https://github.com/sqbjs/sqb.git",
+    "directory" : "packages/migrator"
   },
-  "keywords": [
-    "javascript",
-    "typescript",
+  "scripts" : {
+    "compile" : "tsc",
+    "prebuild" : "npm run lint && npm run clean",
+    "build" : "npm run build:cjs && npm run build:esm",
+    "build:cjs" : "tsc -b tsconfig-build-cjs.json",
+    "build:esm" : "tsc -b tsconfig-build-esm.json",
+    "postbuild" : "cp README.md package.json ../../LICENSE ../../build/migrator && cp ../../package.cjs.json ../../build/migrator/cjs/package.json",
+    "lint" : "eslint .",
+    "test" : "jest",
+    "cover" : "jest --collect-coverage",
+    "clean" : "npm run clean:src | npm run clean:dist | npm run clean:cover",
+    "clean:src" : "ts-cleanup -s src --all",
+    "clean:dist" : "rimraf ../../build/migrator",
+    "clean:cover" : "rimraf ../../coverage/migrator"
+  },
+  "peerDependencies" : {
+    "@sqb/builder" : "^4.5.0",
+    "@sqb/connect" : "^4.5.0"
+  },
+  "type" : "module",
+  "main" : "cjs/index.js",
+  "module" : "esm/index.js",
+  "types" : "esm/index.d.ts",
+  "exports" : {
+    "." : {
+      "require" : "./cjs/index.js",
+      "default" : "./esm/index.js"
+    },
+    "./cjs" : "./cjs/index.js",
+    "./esm" : "./esm/index.js"
+  },
+  "engines" : {
+    "node" : ">=16.0",
+    "npm" : ">=7.0.0"
+  },
+  "files" : [
+    "bin/",
+    "cjs/",
+    "esm/",
+    "LICENSE",
+    "README.md"
+  ],
+  "keywords" : [
+    "sqb",
     "sql",
     "query",
     "database",
     "db",
     "migrate",
     "migrator"
-  ],
-  "peerDependencies": {
-    "@sqb/builder": "^4.1.4"
-  },
-  "main": "dist/index.js",
-  "types": "dist/index.d.ts",
-  "engines": {
-    "node": ">= 16.0"
-  },
-  "directories": {
-    "lib": "dist",
-    "test": "test"
-  },
-  "files": [
-    "LICENSE",
-    "README",
-    "dist/"
-  ],
-  "scripts": {
-    "build": "tsc -b tsconfig-build.json",
-    "prebuild": "npm run lint && npm run clean:dist",
-    "postbuild": "copyfiles package.json README.md LICENSE ../../build/migrator",
-    "clean": "ts-cleanup -s src --all & ts-cleanup -s test",
-    "clean:dist": "rimraf ../../build/migrator",
-    "compile": "tsc",
-    "lint": "eslint src/** --no-error-on-unmatched-pattern",
-    "test": "TS_NODE_PROJECT='./test/tsconfig.json' mocha -r ts-node/register -r tsconfig-paths/register --reporter spec test/**/*.spec.ts"
-  }
-}
+  ]
+}

package/dist/index.d.ts

@@ -1,2 +0,0 @@
-export * from './db-migrator';
-export * from './types';

package/dist/index.js

@@ -1,5 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const tslib_1 = require("tslib");
-tslib_1.__exportStar(require("./db-migrator"), exports);
-tslib_1.__exportStar(require("./types"), exports);