@sqb/migrator 4.9.0 → 4.10.0

package/cjs/utils/get-calling-filename.js

@@ -0,0 +1,23 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getCallingFilename = void 0;
+const PATH_PATTERN = /^(?:file:\/\/)?(.+)$/;
+function getCallingFilename(position = 0) {
+    position++;
+    if (position >= Error.stackTraceLimit)
+        return '';
+    const oldPrepareStackTrace = Error.prepareStackTrace;
+    Error.prepareStackTrace = (_, stack) => stack;
+    const stack = new Error().stack;
+    Error.prepareStackTrace = oldPrepareStackTrace;
+    if (stack !== null && typeof stack === 'object') {
+        // stack[0] holds this file
+        // stack[1] holds where this function was called
+        const s = stack[position] ?
+            stack[position].getFileName() : undefined;
+        const m = s ? PATH_PATTERN.exec(s) : undefined;
+        return m ? m[1] : '';
+    }
+    return '';
+}
+exports.getCallingFilename = getCallingFilename;

package/esm/adapters/pg-migration-adapter.js

@@ -0,0 +1,194 @@
+import { stringifyValueForSQL } from 'postgresql-client';
+import { PgAdapter } from '@sqb/postgres';
+import { MigrationAdapter } from '../migration-adapter.js';
+import { isCustomMigrationTask, isInsertDataMigrationTask, isSqlScriptMigrationTask } from '../migration-package.js';
+import { MigrationStatus } from '../types.js';
+const pgAdapter = new PgAdapter();
+export class PgMigrationAdapter extends MigrationAdapter {
+    constructor() {
+        super(...arguments);
+        this._infoSchema = 'public';
+        this._packageName = '';
+        this._version = 0;
+        this._status = MigrationStatus.idle;
+        this.defaultVariables = {
+            tablespace: 'pg_default',
+            schema: 'public',
+            owner: 'postgres'
+        };
+        this.summaryTable = 'migration_summary';
+        this.eventTable = 'migration_events';
+    }
+    get packageName() {
+        return this._packageName;
+    }
+    get version() {
+        return this._version;
+    }
+    get status() {
+        return this._status;
+    }
+    get infoSchema() {
+        return this._infoSchema;
+    }
+    get summaryTableFull() {
+        return this.infoSchema + '.' + this.summaryTable;
+    }
+    get eventTableFull() {
+        return this.infoSchema + '.' + this.eventTable;
+    }
+    static async create(options) {
+        // Create connection
+        const connection = (await pgAdapter.connect(options.connection)).intlcon;
+        try {
+            const adapter = new PgMigrationAdapter();
+            adapter._connection = connection;
+            adapter._infoSchema = options.infoSchema || '__migration';
+            adapter.defaultVariables.schema = options.connection.schema || '';
+            if (!adapter.defaultVariables.schema) {
+                const r = await connection.query('SELECT CURRENT_SCHEMA ', { objectRows: true });
+                adapter.defaultVariables.schema = (r.rows?.[0]?.current_schema) || 'public';
+            }
+            // Check if migration schema
+            await connection.query(`CREATE SCHEMA IF NOT EXISTS ${adapter.infoSchema} AUTHORIZATION postgres;`);
+            // Create summary table if not exists
+            await connection.execute(`
+CREATE TABLE IF NOT EXISTS ${adapter.summaryTableFull}
+(
+    package_name varchar not null,
+    status varchar(16) not null,
+    current_version integer not null default 0,
+    created_at timestamp without time zone not null default current_timestamp,
+    updated_at timestamp without time zone,
+    CONSTRAINT pk_${adapter.summaryTable} PRIMARY KEY (package_name)
+)`);
+            // Create events table if not exists
+            await connection.execute(`
+CREATE TABLE IF NOT EXISTS ${adapter.eventTableFull}
+(
+    id serial not null,
+    package_name varchar not null,           
+    version integer not null default 0,
+    event varchar(16) not null,
+    event_time timestamp without time zone not null,
+    message text not null,          
+    details text,
+    CONSTRAINT pk_${adapter.eventTable} PRIMARY KEY (id)
+)`);
+            // Insert summary record if not exists
+            const r = await connection.query(`SELECT status FROM ${adapter.summaryTableFull} WHERE package_name = $1`, { params: [adapter.packageName], objectRows: true });
+            if (!(r && r.rows?.length)) {
+                await connection.query(`insert into ${adapter.summaryTableFull} ` +
+                    '(package_name, status) values ($1, $2)', { params: [adapter.packageName, MigrationStatus.idle] });
+            }
+            await adapter.refresh();
+            return adapter;
+        }
+        catch (e) {
+            await connection.close(0);
+            throw e;
+        }
+    }
+    async close() {
+        await this._connection.close();
+    }
+    async refresh() {
+        const r = await this._connection.query(`SELECT * FROM ${this.summaryTableFull} WHERE package_name = $1`, { params: [this.packageName], objectRows: true });
+        const row = r.rows && r.rows[0];
+        if (!row)
+            throw new Error('Summary record did not created');
+        this._version = row.current_version;
+        this._status = row.status;
+    }
+    async update(info) {
+        let sql = '';
+        const params = [];
+        if (info.status && info.status !== this.status) {
+            params.push(info.status);
+            sql += ', status = $' + (params.length);
+        }
+        if (info.version && info.version !== this.version) {
+            params.push(info.version);
+            sql += ', current_version = $' + (params.length);
+        }
+        if (sql) {
+            sql = `update ${this.summaryTableFull} set updated_at = current_timestamp` + sql;
+            await this._connection.query(sql, { params });
+            if (info.status)
+                this._status = info.status;
+            if (info.version)
+                this._version = info.version;
+        }
+    }
+    async writeEvent(event) {
+        const sql = `insert into ${this.eventTableFull} ` +
+            '(package_name, version, event, event_time, message, details) ' +
+            'values ($1, $2, $3, CURRENT_TIMESTAMP, $4, $5)';
+        await this._connection.query(sql, {
+            params: [
+                this.packageName, event.version, event.event,
+                event.message, event.details
+            ]
+        });
+    }
+    async executeTask(task, variables) {
+        variables = {
+            ...this.defaultVariables,
+            ...variables
+        };
+        if (isSqlScriptMigrationTask(task)) {
+            try {
+                const script = task.script
+                    .replace(/(\${(\w+)})/g, (s, ...args) => variables[args[1]] || s);
+                await this._connection.execute(script);
+            }
+            catch (e) {
+                let msg = `Error in task "${task.title}"`;
+                if (task.filename)
+                    msg += '\n at ' + task.filename;
+                if (e.lineNr) {
+                    if (!task.filename)
+                        e.message += '\n at';
+                    msg += ` (${e.lineNr},${e.colNr}):\n` + e.line;
+                    if (e.colNr)
+                        msg += '\n' + ' '.repeat(e.colNr - 1) + '^';
+                }
+                e.message = msg + '\n\n' + e.message;
+                throw e;
+            }
+            return;
+        }
+        if (isCustomMigrationTask(task)) {
+            await task.fn(this._connection, this);
+            return;
+        }
+        if (isInsertDataMigrationTask(task)) {
+            const script = task.rows
+                .map(row => this.rowToSql(variables.schema + '.' + task.tableName, row))
+                .join('\n');
+            await this._connection.execute(script);
+        }
+    }
+    backupDatabase() {
+        return Promise.resolve(undefined);
+    }
+    lockSchema() {
+        return Promise.resolve(undefined);
+    }
+    restoreDatabase() {
+        return Promise.resolve(undefined);
+    }
+    unlockSchema() {
+        return Promise.resolve(undefined);
+    }
+    rowToSql(tableName, row) {
+        let sql = '';
+        const keys = Object.keys(row);
+        sql += `insert into ${tableName} (${keys}) values (`;
+        for (let i = 0; i < keys.length; i++) {
+            sql += (i ? ', ' : '') + stringifyValueForSQL(row[keys[i]]);
+        }
+        sql += ');\n';
+        return sql;
+    }
+}

package/esm/db-migrator.js

@@ -1,210 +1,104 @@
-import fs from 'node:fs';
-import path from 'node:path';
-import { stringifyValueForSQL } from 'postgresql-client';
-import { loadTaskFiles } from './load-task-files.js';
-const ignoreEventListener = () => 0;
-export class DbMigrator {
+import { AsyncEventEmitter } from 'strict-typed-events';
+import { PgMigrationAdapter } from './adapters/pg-migration-adapter.js';
+import { MigrationAdapter } from './migration-adapter.js';
+import { MigrationPackage } from './migration-package.js';
+import { MigrationStatus } from './types.js';
+export class DbMigrator extends AsyncEventEmitter {
     async execute(options) {
-        const { connection, schema } = options;
-        const migrationPackage = await DbMigrator.loadMigrationPackage(options.migrationPackage);
-        const targetVersion = Math.min(options.targetVersion || Number.MAX_SAFE_INTEGER, migrationPackage.maxVersion);
-        if (targetVersion && targetVersion < migrationPackage.minVersion)
+        if (!options.connection.dialect)
+            throw new TypeError(`You must provide connection.dialect`);
+        const migrationPackage = await MigrationPackage.load(options.migrationPackage);
+        let minVersion = migrationPackage.migrations
+            .reduce((a, m) => Math.min(a, m.version), Number.MAX_SAFE_INTEGER);
+        if (minVersion === Number.MAX_SAFE_INTEGER)
+            minVersion = 0;
+        const maxVersion = migrationPackage.migrations
+            .reduce((a, m) => Math.max(a, m.version), 0);
+        const targetVersion = Math.min(options?.targetVersion || Number.MAX_SAFE_INTEGER, maxVersion);
+        if (targetVersion && targetVersion < minVersion) { // noinspection ExceptionCaughtLocallyJS
             throw new Error(`Version mismatch. Target schema version (${targetVersion}) is lower than ` +
-                `migration package min version (${migrationPackage.minVersion})`);
-        const emit = options.eventListener || ignoreEventListener;
-        const schemaInfo = await this._getSchemaInfo(options);
-        if (schemaInfo.version && schemaInfo.version < migrationPackage.minVersion - 1)
-            throw new Error(`This package can migrate starting from ${migrationPackage.minVersion - 1} but current version is ${schemaInfo.version}`);
-        const { migrations } = migrationPackage;
-        // calculate total scripts;
-        const total = migrations
-            .reduce((i, x) => i + x.tasks.length, 0);
-        const needBackup = !!migrations.find(x => !!x.backup);
-        emit('start');
-        let task;
+                `migration package min version (${minVersion})`);
+        }
+        let migrationAdapter;
+        switch (options.connection.dialect) {
+            case 'postgres': {
+                migrationAdapter = await PgMigrationAdapter.create(options);
+                break;
+            }
+            default:
+                throw new TypeError(`Migration adapter for "${options.connection.dialect}" dialect is not implemented yet`);
+        }
+        let needBackup = false;
         try {
-            await this._lockSchema();
+            if (migrationAdapter.version && migrationAdapter.version < minVersion - 1) { // noinspection ExceptionCaughtLocallyJS
+                throw new Error(`This package can migrate starting from ${minVersion - 1} but current version is ${migrationAdapter.version}`);
+            }
+            const { migrations } = migrationPackage;
+            // calculate total scripts;
+            const total = migrations
+                .reduce((i, x) => i + x.tasks.length, 0);
+            needBackup = !!migrations.find(x => !!x.backup);
+            await this.emitAsync('start');
+            let task;
+            await migrationAdapter.lockSchema();
             if (needBackup) {
-                emit('backup');
-                await this._backup();
+                await this.emitAsync('backup');
+                await migrationAdapter.backupDatabase();
             }
-            await connection.execute(`update ${schema}.${schemaInfo.infTable} set status = 'migration'`);
             // Execute migration tasks
             for (const migration of migrations) {
-                if (migration.version > targetVersion || schemaInfo.version >= migration.version)
+                if (migration.version > targetVersion || migrationAdapter.version >= migration.version)
                     continue;
                 for (let index = 0; index < migration.tasks.length; index++) {
                     task = migration.tasks[index];
-                    emit('processing', { task: task.title, total, index });
-                    if ('fn' in task && typeof task.fn === 'function') {
-                        await task.fn(connection);
-                    }
-                    else if (task.script) {
-                        const script = task.script.replace(/(\${(\w+)})/gi, (s, ...args) => {
-                            switch (args[1].toLowerCase()) {
-                                case 'schema':
-                                    return schema;
-                                case 'owner':
-                                    return schemaInfo.owner;
-                                case 'tablespace':
-                                    return schemaInfo.tablespace;
-                            }
-                            return s;
+                    await this.emitAsync('task-start', { task: task.title, total, index });
+                    await migrationAdapter.update({ status: MigrationStatus.busy });
+                    await migrationAdapter.writeEvent({
+                        event: MigrationAdapter.EventKind.started,
+                        version: migration.version,
+                        message: `Task "${task.title}" started`
+                    });
+                    try {
+                        await migrationAdapter.executeTask(task, {
+                            schema: options.connection.schema,
+                            ...options.scriptVariables,
+                        });
+                        await migrationAdapter.writeEvent({
+                            event: MigrationAdapter.EventKind.success,
+                            version: migration.version,
+                            message: `Task "${task.title}" completed`
                         });
-                        try {
-                            await connection.execute(script);
-                        }
-                        catch (e) {
-                            e.message += `\n${task.title}`;
-                            if (e.lineNr)
-                                e.message += ` (${e.lineNr},${e.colNr}):\n` + e.line;
-                            if (e.colNr)
-                                e.message += '\n' + ' '.repeat(e.colNr - 1) + '^';
-                            // noinspection ExceptionCaughtLocallyJS
-                            throw e;
-                        }
                     }
-                    else if ('tableName' in task) {
-                        const script = DbMigrator.objectDataToSql({ ...task, schema });
-                        await connection.execute(script);
+                    catch (e) {
+                        await migrationAdapter.writeEvent({
+                            event: MigrationAdapter.EventKind.error,
+                            version: migration.version,
+                            message: String(e)
+                        });
+                        // noinspection ExceptionCaughtLocallyJS
+                        throw e;
                     }
-                    emit('process', { task: task.title, total, index });
+                    await this.emitAsync('task-finish', { task: task.title, total, index });
                 }
+                await migrationAdapter.update({ version: migration.version });
             }
-            await connection.query(`update ${schema}.${schemaInfo.infTable} set status = $1, schema_version = $2, ` +
-                'updated_at = current_timestamp', { params: ['ready', targetVersion] });
         }
         catch (e) {
             if (needBackup) {
-                emit('backup');
-                await this._restore();
+                await this.emitAsync('restore');
+                await migrationAdapter.restoreDatabase();
             }
             throw e;
         }
         finally {
-            await this._unlockSchema();
-        }
-        emit('finish');
-        return true;
-    }
-    static async loadMigrationPackage(pkg) {
-        const migarr = typeof pkg.migrations === 'function' ? await pkg.migrations() : pkg.migrations;
-        const migrations = [];
-        for (const x of migarr) {
-            if (typeof x === 'string')
-                await locateMigrations(migrations, x);
-            else
-                migrations.push(x);
-        }
-        let minVersion = 0;
-        let maxVersion = 0;
-        migrations.sort((a, b) => a.version - b.version);
-        for (const m of migrations) {
-            minVersion = !minVersion ? m.version : Math.min(minVersion, m.version);
-            maxVersion = Math.max(minVersion, m.version);
-            for (const t of m.tasks) {
-                if (typeof t === 'string') {
-                    m.tasks = loadTaskFiles(t);
-                }
+            try {
+                await migrationAdapter.unlockSchema();
             }
-        }
-        return {
-            minVersion,
-            maxVersion,
-            migrations
-        };
-    }
-    async _lockSchema() {
-        // todo
-    }
-    async _unlockSchema() {
-        // todo
-    }
-    async _backup() {
-        // todo
-    }
-    async _restore() {
-        // todo
-    }
-    async _getSchemaInfo(options) {
-        const { connection, schema } = options;
-        let owner = options.owner || 'postgres';
-        let tablespace = options.tablespace || 'pg_default';
-        let version = 0;
-        let status = '';
-        let r = await connection.query('select schema_owner FROM information_schema.schemata WHERE schema_name = $1', { params: [schema], objectRows: true });
-        if (r.rows && r.rows[0]) {
-            owner = r.rows[0].schema_owner;
-        }
-        else {
-            await connection.execute(`CREATE SCHEMA ${schema} AUTHORIZATION ${owner};`);
-        }
-        // Determine tablespace of schema
-        const infTable = options.migrationPackage.informationTableName || '__schema_info';
-        r = await connection.query('SELECT tablespace FROM pg_tables WHERE schemaname = $1 AND tablename = $2', { params: [schema, infTable], objectRows: true });
-        if (r.rows && r.rows[0]) {
-            tablespace = r.rows[0].tablespace || tablespace;
-            r = await connection.query(`SELECT schema_version, status FROM ${schema}.${infTable}`, { objectRows: true });
-            if (r.rows && r.rows[0]) {
-                version = r.rows[0].schema_version;
-                status = r.rows[0].status;
-            }
-        }
-        else {
-            // Create if not exists
-            await connection.execute(`
-CREATE TABLE ${schema}.${infTable}
-(
-    status varchar(16) not null,
-    schema_version integer not null default 0,
-    created_at timestamp without time zone not null default current_timestamp,
-    updated_at timestamp without time zone
-) TABLESPACE ${tablespace};
-insert into ${schema}.${infTable} (status) values ('init');
-      `);
-            status = 'init';
-        }
-        return {
-            infTable,
-            tablespace,
-            owner,
-            version,
-            status
-        };
-    }
-    static objectDataToSql(task) {
-        let sql = '';
-        for (const rows of task.rows) {
-            const keys = Object.keys(rows);
-            sql += `insert into ${task.schema}.${task.tableName} (${keys}) values (`;
-            for (let i = 0; i < keys.length; i++) {
-                sql += (i ? ', ' : '') + stringifyValueForSQL(rows[keys[i]]);
+            finally {
+                await migrationAdapter.close();
             }
-            sql += ');\n';
-        }
-        return sql;
-    }
-}
-async function locateMigrations(trg, dir) {
-    for (const f of ['migration.ts', 'migration.js', 'migration.json']) {
-        const filename = path.join(dir, f);
-        if (fs.existsSync(filename)) {
-            const x = await import(filename);
-            const fileDir = path.dirname(filename);
-            const obj = x.default || x;
-            if (obj.version && obj.tasks) {
-                const migration = { ...obj };
-                migration.tasks = obj.tasks.map(k => (typeof k === 'string' ? path.resolve(fileDir, k) : k));
-                trg.push(migration);
-            }
-            return;
-        }
-    }
-    const files = fs.readdirSync(dir);
-    for (const f of files) {
-        const dirname = path.join(dir, f);
-        if (fs.statSync(dirname).isDirectory()) {
-            await locateMigrations(trg, dirname);
         }
+        await this.emitAsync('finish');
+        return true;
     }
 }

package/esm/index.js

@@ -1,2 +1,4 @@
 export * from './db-migrator.js';
+export * from './migration-package.js';
+export * from './migration-adapter.js';
 export * from './types.js';

package/esm/migration-adapter.js

@@ -0,0 +1,10 @@
+export class MigrationAdapter {
+}
+(function (MigrationAdapter) {
+    let EventKind;
+    (function (EventKind) {
+        EventKind["started"] = "started";
+        EventKind["success"] = "success";
+        EventKind["error"] = "error";
+    })(EventKind = MigrationAdapter.EventKind || (MigrationAdapter.EventKind = {}));
+})(MigrationAdapter || (MigrationAdapter = {}));

package/esm/migration-package.js

@@ -0,0 +1,131 @@
+import glob from 'fast-glob';
+import fs from 'fs/promises';
+import path from 'path';
+import { getCallingFilename } from './utils/get-calling-filename.js';
+export function isSqlScriptMigrationTask(x) {
+    return typeof x === 'object' && typeof x.script === 'string';
+}
+export function isInsertDataMigrationTask(x) {
+    return typeof x === 'object' &&
+        typeof x.tableName === 'string' &&
+        Array.isArray(x.rows);
+}
+export function isCustomMigrationTask(x) {
+    return typeof x === 'object' &&
+        typeof x.fn === 'function';
+}
+export var MigrationPackage;
+(function (MigrationPackage) {
+    async function load(asyncConfig) {
+        const out = {
+            ...asyncConfig,
+            migrations: []
+        };
+        if (!Array.isArray(asyncConfig.migrations))
+            throw new TypeError('You must provide array of MigrationConfig in "migrations" property');
+        const baseDir = path.dirname(getCallingFilename(1));
+        if (asyncConfig.migrations?.length) {
+            const srcMigrations = [];
+            const trgMigrations = [];
+            out.migrations = trgMigrations;
+            let x;
+            for (x of asyncConfig.migrations) {
+                x = typeof x === 'function' ? await x() : x;
+                if (typeof x === 'object' && x.tasks)
+                    srcMigrations.push(x);
+                else if (typeof x === 'string') {
+                    srcMigrations.push(...await loadMigrations(path.resolve(baseDir, x.replace(/\\/g, '/'))));
+                }
+            }
+            srcMigrations.sort((a, b) => a.version - b.version);
+            for (const migration of srcMigrations) {
+                const trgMigration = { ...migration, tasks: [] };
+                trgMigrations.push(trgMigration);
+                const srcTasks = migration.tasks;
+                trgMigration.tasks = [];
+                for (const t of srcTasks) {
+                    if (typeof t === 'object') {
+                        trgMigration.tasks.push(t);
+                    }
+                    else if (typeof t === 'string') {
+                        let pattern = t.replace(/\\/g, '/');
+                        pattern = path.resolve(migration.dirname || baseDir, pattern);
+                        const files = await glob(pattern, { absolute: true, onlyFiles: true });
+                        for (const filename of files) {
+                            const ext = path.extname(filename).toLowerCase();
+                            if (!path.basename(filename, ext).endsWith('.task'))
+                                continue;
+                            if (ext === '.sql') {
+                                const script = await fs.readFile(filename, 'utf-8');
+                                trgMigration.tasks.push({
+                                    title: path.basename(filename),
+                                    filename,
+                                    script
+                                });
+                            }
+                            else if (ext === '.json') {
+                                try {
+                                    const json = JSON.parse(await fs.readFile(filename, 'utf-8'));
+                                    if (typeof json !== 'object')
+                                        continue;
+                                    if (json.script) {
+                                        json.title = json.title || 'Run sql script';
+                                        json.filename = filename;
+                                        trgMigration.tasks.push(json);
+                                        continue;
+                                    }
+                                    if (json.tableName && json.rows) {
+                                        json.title = json.title || 'Migrate data into ' + json.tableName;
+                                        json.filename = filename;
+                                        trgMigration.tasks.push(json);
+                                        continue;
+                                    }
+                                    if (typeof json.fn === 'function') {
+                                        json.title = json.title || 'Run custom function';
+                                        json.filename = filename;
+                                        trgMigration.tasks.push(json);
+                                    }
+                                }
+                                catch (e) {
+                                    e.message = `Error in ${filename}\n` + e.message;
+                                    throw e;
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        return out;
+    }
+    MigrationPackage.load = load;
+    async function loadMigrations(pattern) {
+        const out = [];
+        const files = await glob(pattern, { absolute: true, onlyFiles: true });
+        for (const filename of files) {
+            const ext = path.extname(filename).toLowerCase();
+            if (path.basename(filename, ext) !== 'migration')
+                continue;
+            let json;
+            if (['.js', '.ts', '.cjs', '.mjs'].includes(ext)) {
+                json = await import(filename);
+                if (json.__esModule)
+                    json = json.default;
+            }
+            else if (ext === '.json') {
+                try {
+                    json = JSON.parse(await fs.readFile(filename, 'utf-8'));
+                }
+                catch (e) {
+                    e.message = `Error in ${filename}\n` + e.message;
+                    throw e;
+                }
+            }
+            if (json && typeof json === 'object' && json.version && Array.isArray(json.tasks)) {
+                json.dirname = path.dirname(filename);
+                out.push(json);
+            }
+        }
+        return out;
+    }
+})(MigrationPackage || (MigrationPackage = {}));

package/esm/types.js

@@ -1 +1,5 @@
-export {};
+export var MigrationStatus;
+(function (MigrationStatus) {
+    MigrationStatus["idle"] = "idle";
+    MigrationStatus["busy"] = "busy";
+})(MigrationStatus || (MigrationStatus = {}));