@bitblit/ratchet-rdbms 4.0.115-alpha

package/lib/rds-mysql-style-connection-provider.d.ts

@@ -0,0 +1,23 @@
+import { Connection, ConnectionOptions } from 'mysql2/promise';
+import { SshTunnelService } from './ssh-tunnel-service.js';
+import { MysqlStyleConnectionProvider } from './model/mysql/mysql-style-connection-provider.js';
+import { ConnectionConfig } from './model/connection-config.js';
+import { QueryDefaults } from './model/query-defaults.js';
+export declare class RdsMysqlStyleConnectionProvider implements MysqlStyleConnectionProvider {
+    private inConfigPromise;
+    private additionalConfig;
+    private ssh?;
+    static DEFAULT_CONNECTION_OPTIONS: ConnectionOptions;
+    private tunnels;
+    private dbPromise;
+    private readonly configPromise;
+    constructor(inConfigPromise: Promise<ConnectionConfig>, additionalConfig?: ConnectionOptions, ssh?: SshTunnelService);
+    get usingSshTunnel(): boolean;
+    private addShutdownHandlerToProcess;
+    clearConnectionCache(): Promise<boolean>;
+    getConnection(name: string): Promise<Connection | undefined>;
+    createNonPooledDatabaseConnection(queryDefaults: QueryDefaults, additionalConfig?: ConnectionOptions): Promise<Connection | undefined>;
+    private getDbConfig;
+    private createDatabaseConnection;
+    private prepareConnectionConfig;
+}

package/lib/rds-mysql-style-connection-provider.js

@@ -0,0 +1,159 @@
+import maria from 'mysql2/promise';
+import { Logger } from '@bitblit/ratchet-common/lib/logger/logger.js';
+import getPort from 'get-port';
+import _ from 'lodash';
+import { ErrorRatchet } from '@bitblit/ratchet-common/lib/lang/error-ratchet.js';
+import { RequireRatchet } from '@bitblit/ratchet-common/lib/lang/require-ratchet.js';
+export class RdsMysqlStyleConnectionProvider {
+    inConfigPromise;
+    additionalConfig;
+    ssh;
+    static DEFAULT_CONNECTION_OPTIONS = { multipleStatements: true };
+    tunnels = new Map();
+    dbPromise = new Map();
+    configPromise;
+    constructor(inConfigPromise, additionalConfig = RdsMysqlStyleConnectionProvider.DEFAULT_CONNECTION_OPTIONS, ssh) {
+        this.inConfigPromise = inConfigPromise;
+        this.additionalConfig = additionalConfig;
+        this.ssh = ssh;
+        this.configPromise = this.prepareConnectionConfig(inConfigPromise);
+        Logger.info('Added shutdown handler to the process (Only once per instantiation)');
+        this.addShutdownHandlerToProcess();
+    }
+    get usingSshTunnel() {
+        return !!this.ssh;
+    }
+    addShutdownHandlerToProcess() {
+        process.on('exit', () => {
+            Logger.info('Process is shutting down, closing connections');
+            this.clearConnectionCache().catch((err) => {
+                Logger.error('Shutdown connection failed : %s', err);
+            });
+        });
+    }
+    async clearConnectionCache() {
+        const rval = false;
+        Logger.info('Clearing connection cache for RdsMysqlConnectionProvider');
+        const oldDbHooks = this.dbPromise;
+        const oldSshTunnels = this.tunnels;
+        this.dbPromise = new Map();
+        this.tunnels = new Map();
+        if (oldDbHooks.size > 0) {
+            const hookNames = Array.from(oldDbHooks.keys());
+            for (const hookName of hookNames) {
+                Logger.info('Shutting down connection : %s', hookName);
+                const oldDbHook = oldDbHooks.get(hookName);
+                try {
+                    const oldConn = await oldDbHook;
+                    if (oldConn) {
+                        await oldConn.destroy();
+                        Logger.info('Finished destroying old connection');
+                    }
+                    else {
+                        Logger.warn('Could not get old connection, so not destroying it');
+                    }
+                }
+                catch (err) {
+                    if (ErrorRatchet.asErr(err).message.includes('closed state')) {
+                    }
+                    else {
+                        Logger.error('Something went wrong closing the connection : %s', err);
+                        throw err;
+                    }
+                }
+            }
+        }
+        if (oldSshTunnels.size > 0) {
+            const tunnelNames = Array.from(oldSshTunnels.keys());
+            for (const tunnelName of tunnelNames) {
+                try {
+                    Logger.info('Shutting down SSH tunnel: %s', tunnelName);
+                    const tunnel = oldSshTunnels.get(tunnelName);
+                    if (tunnel) {
+                        await this.ssh.shutdown(tunnel);
+                    }
+                    else {
+                        Logger.warn('Could not get old tunnel, so not destroying it');
+                    }
+                    Logger.info('SSH Tunnel closed');
+                }
+                catch (err) {
+                    Logger.error('Failure closing old tunnel : %s', err);
+                }
+            }
+        }
+        Logger.info('Old db and tunnel removed');
+        return rval;
+    }
+    async getConnection(name) {
+        if (!this.dbPromise.has(name)) {
+            const dbConfig = await this.getDbConfig(name);
+            const connection = this.createDatabaseConnection(dbConfig, this.additionalConfig, true);
+            this.dbPromise.set(name, connection);
+        }
+        return this.dbPromise.get(name);
+    }
+    async createNonPooledDatabaseConnection(queryDefaults, additionalConfig = RdsMysqlStyleConnectionProvider.DEFAULT_CONNECTION_OPTIONS) {
+        Logger.info('Creating non-pooled connection for %s', queryDefaults.databaseName);
+        const dbConfig = await this.getDbConfig(queryDefaults.databaseName);
+        const rval = await this.createDatabaseConnection(dbConfig, additionalConfig, false);
+        return rval;
+    }
+    async getDbConfig(name) {
+        Logger.info('RdsMysqlStyleConnectionProvider:getDbConfig:Initiating promise for %s', name);
+        const cfgs = await this.configPromise;
+        const dbConfig = cfgs.dbList.find((s) => s.label === name.toLowerCase());
+        if (!dbConfig) {
+            throw new Error(`Cannot find any connection config named ${name}`);
+        }
+        return dbConfig;
+    }
+    async createDatabaseConnection(dbCfg, additionalConfig = RdsMysqlStyleConnectionProvider.DEFAULT_CONNECTION_OPTIONS, clearCacheOnConnectionFailure, sshConfigPromise) {
+        Logger.debug('In RdsMysqlStyleConnectionProvider:createDatabaseConnection : %s', dbCfg.label);
+        const cfgCopy = _.omit(_.clone(dbCfg), 'label', 'tunnelPort');
+        if (this.usingSshTunnel && dbCfg.tunnelPort) {
+            let tunnel = this.tunnels.get(dbCfg.label);
+            if (!tunnel) {
+                Logger.debug('Creating SSH tunnel from local port %d to remote host %s and port %d', dbCfg.tunnelPort, dbCfg.host, dbCfg.port);
+                const sshConfig = await sshConfigPromise;
+                tunnel = await this.ssh.createSSHTunnel(sshConfig, dbCfg.host, dbCfg.port, dbCfg.tunnelPort);
+                this.tunnels.set(dbCfg.label, tunnel);
+            }
+            cfgCopy.port = tunnel.serverOptions.port ?? -1;
+            cfgCopy.host = 'localhost';
+        }
+        Logger.debug('Opening connection for RdsMysqlStyleConnectionProvider');
+        let connection;
+        try {
+            connection = await maria.createConnection({ ...additionalConfig, ...cfgCopy });
+        }
+        catch (err) {
+            Logger.info('Failed trying to create connection : %s : clearing for retry', err);
+            if (clearCacheOnConnectionFailure) {
+                this.dbPromise = new Map();
+            }
+            return undefined;
+        }
+        connection.on('error', (err) => {
+            Logger.info('An error was detected on the connection : %s : Clearing', err);
+            this.clearConnectionCache()
+                .then((cleared) => {
+                Logger.info('Connection cleared: %s', cleared);
+            })
+                .catch((err) => Logger.error('Failed to clear RDS connection cache: %j', err));
+        });
+        Logger.info('Added error handler to db, there are now %d error handlers and %d shutdown handlers', connection.rawListeners('error').length, process.rawListeners('exit').length);
+        return connection;
+    }
+    async prepareConnectionConfig(inputPromise) {
+        RequireRatchet.notNullOrUndefined(inputPromise, 'input');
+        const cfg = await inputPromise;
+        RequireRatchet.true(cfg.dbList.length > 0, 'input.dbList');
+        if (this.usingSshTunnel) {
+            for (const dbConfig of cfg.dbList) {
+                dbConfig.tunnelPort = await getPort();
+            }
+        }
+        return cfg;
+    }
+}

package/lib/ssh-tunnel-service.d.ts

@@ -0,0 +1,6 @@
+import { SshTunnelContainer } from './model/ssh/ssh-tunnel-container.js';
+import { SshTunnelConfig } from './model/ssh/ssh-tunnel-config.js';
+export declare class SshTunnelService {
+    shutdown(ssh: SshTunnelContainer): Promise<boolean>;
+    createSSHTunnel(sshOptions: SshTunnelConfig, dstHost: string, dstPort: number, localPort: number): Promise<SshTunnelContainer>;
+}

package/lib/ssh-tunnel-service.js

@@ -0,0 +1,47 @@
+import * as TunnelSsh from 'tunnel-ssh';
+import { Logger } from '@bitblit/ratchet-common/lib/logger/logger.js';
+export class SshTunnelService {
+    async shutdown(ssh) {
+        if (!ssh.connection) {
+            Logger.info('Not shutting down tunnel - non-tunnel passed');
+            return false;
+        }
+        try {
+            Logger.info('Shutting down SSH Tunnel');
+            ssh.connection.end();
+            ssh.server.close();
+            return true;
+        }
+        catch (err) {
+            Logger.error('Error closing ssh tunnel : %s', err);
+            return false;
+        }
+    }
+    async createSSHTunnel(sshOptions, dstHost, dstPort, localPort) {
+        const tunnelOptions = {
+            autoClose: true,
+        };
+        const serverOptions = {
+            port: localPort,
+        };
+        const forwardOptions = {
+            srcAddr: 'localhost',
+            srcPort: localPort,
+            dstAddr: dstHost,
+            dstPort: dstPort,
+        };
+        const [server, connection] = await TunnelSsh.createTunnel(tunnelOptions, serverOptions, sshOptions, forwardOptions);
+        server.on('error', (err) => {
+            Logger.warn('SSH Server Error : %s', err);
+        });
+        const rval = {
+            tunnelOptions: tunnelOptions,
+            serverOptions: serverOptions,
+            sshOptions: sshOptions,
+            forwardOptions: forwardOptions,
+            server: server,
+            connection: connection,
+        };
+        return rval;
+    }
+}

package/lib/transactional-named-parameter-maria-db-service.d.ts

@@ -0,0 +1,23 @@
+import { NamedParameterMariaDbService } from './named-parameter-maria-db-service.js';
+import { MysqlStyleConnectionProvider } from './model/mysql/mysql-style-connection-provider.js';
+import { ConnectionOptions } from 'mysql2/promise';
+import { QueryBuilder } from './query-builder/query-builder.js';
+import { MysqlUpdateResults } from './model/mysql/mysql-update-results.js';
+import { QueryDefaults } from './model/query-defaults.js';
+import { QueryTextProvider } from './model/query-text-provider.js';
+export declare class TransactionalNamedParameterMariaDbService extends NamedParameterMariaDbService {
+    private myQueryProvider;
+    private myConnectionProvider;
+    private myQueryDefaults;
+    private currentTxFlag?;
+    static create(src: NamedParameterMariaDbService, queryDefaults: QueryDefaults, additionalConfig?: ConnectionOptions): Promise<TransactionalNamedParameterMariaDbService>;
+    constructor(myQueryProvider: QueryTextProvider, myConnectionProvider: MysqlStyleConnectionProvider, myQueryDefaults: QueryDefaults);
+    cleanShutdown(): Promise<void>;
+    startTransaction(): Promise<void>;
+    commitTransaction(): Promise<void>;
+    rollBackTransaction(): Promise<void>;
+    buildAndExecuteUpdateOrInsertInTransaction(queryBuilder: QueryBuilder, timeoutMS?: number): Promise<MysqlUpdateResults | null>;
+    buildAndExecuteInTransaction<T>(queryBuilder: QueryBuilder, timeoutMS?: number): Promise<T[] | null>;
+    static oneStepBuildAndExecuteUpdateOrInsertInTransaction(src: NamedParameterMariaDbService, queryBuilder: QueryBuilder, timeoutMS?: number, additionalConfig?: ConnectionOptions): Promise<MysqlUpdateResults | null>;
+    static oneStepBuildAndExecuteInTransaction<T>(src: NamedParameterMariaDbService, queryBuilder: QueryBuilder, timeoutMS?: number, additionalConfig?: ConnectionOptions): Promise<T[] | null>;
+}

package/lib/transactional-named-parameter-maria-db-service.js

@@ -0,0 +1,129 @@
+import { NamedParameterMariaDbService } from './named-parameter-maria-db-service.js';
+import { ErrorRatchet } from '@bitblit/ratchet-common/lib/lang/error-ratchet.js';
+import { Logger } from '@bitblit/ratchet-common/lib/logger/logger.js';
+import { StringRatchet } from '@bitblit/ratchet-common/lib/lang/string-ratchet.js';
+export class TransactionalNamedParameterMariaDbService extends NamedParameterMariaDbService {
+    myQueryProvider;
+    myConnectionProvider;
+    myQueryDefaults;
+    currentTxFlag;
+    static async create(src, queryDefaults, additionalConfig) {
+        Logger.info('createTransactionalNamedParameterMariaDbService :  %j : %j', queryDefaults, additionalConfig);
+        const connProv = await src.createNonPooledMysqlStyleConnectionProvider(queryDefaults, additionalConfig);
+        return new TransactionalNamedParameterMariaDbService(src.getQueryProvider(), connProv, queryDefaults);
+    }
+    constructor(myQueryProvider, myConnectionProvider, myQueryDefaults) {
+        super(myQueryProvider, myConnectionProvider, myQueryDefaults);
+        this.myQueryProvider = myQueryProvider;
+        this.myConnectionProvider = myConnectionProvider;
+        this.myQueryDefaults = myQueryDefaults;
+    }
+    async cleanShutdown() {
+        Logger.info('cleanShutdown');
+        try {
+            const conn = await this.myConnectionProvider.getConnection();
+            if (conn) {
+                Logger.info('Shutting down connection');
+                conn.destroy();
+            }
+        }
+        catch (err) {
+            Logger.info('Failure shutting down single-use connection : %s', err);
+        }
+    }
+    async startTransaction() {
+        if (!this.currentTxFlag) {
+            this.currentTxFlag = StringRatchet.createRandomHexString(10);
+            Logger.info('Starting a transaction : %s', this.currentTxFlag);
+            const conn = await this.myConnectionProvider.getConnection();
+            await conn?.beginTransaction();
+        }
+        else {
+            ErrorRatchet.throwFormattedErr('Tried to start a new transaction while one is already in progress : %s', this.currentTxFlag);
+        }
+    }
+    async commitTransaction() {
+        if (this.currentTxFlag) {
+            Logger.info('commit a transaction : %s', this.currentTxFlag);
+            const conn = await this.myConnectionProvider.getConnection();
+            await conn?.commit();
+            this.currentTxFlag = undefined;
+        }
+        else {
+            ErrorRatchet.throwFormattedErr('Cannot commit transaction - none in process');
+        }
+    }
+    async rollBackTransaction() {
+        if (this.currentTxFlag) {
+            Logger.info('rollBack a transaction : %s', this.currentTxFlag);
+            const conn = await this.myConnectionProvider.getConnection();
+            await conn?.rollback();
+            this.currentTxFlag = undefined;
+        }
+        else {
+            ErrorRatchet.throwFormattedErr('Cannot rollBack transaction - none in process');
+        }
+    }
+    async buildAndExecuteUpdateOrInsertInTransaction(queryBuilder, timeoutMS = this.myQueryDefaults.timeoutMS) {
+        Logger.info('buildAndExecuteUpdateOrInsertInTransaction');
+        await this.startTransaction();
+        try {
+            const rval = await this.buildAndExecuteUpdateOrInsert(queryBuilder, timeoutMS);
+            await this.commitTransaction();
+            return rval;
+        }
+        catch (err) {
+            Logger.error('Failed - rolling back transaction : %s', err, err);
+            await this.rollBackTransaction();
+            return null;
+        }
+    }
+    async buildAndExecuteInTransaction(queryBuilder, timeoutMS = this.myQueryDefaults.timeoutMS) {
+        Logger.info('buildAndExecuteInTransaction');
+        await this.startTransaction();
+        try {
+            const rval = await this.buildAndExecute(queryBuilder, timeoutMS);
+            await this.commitTransaction();
+            return rval;
+        }
+        catch (err) {
+            Logger.error('Failed - rolling back transaction : %s', err, err);
+            await this.rollBackTransaction();
+            return null;
+        }
+    }
+    static async oneStepBuildAndExecuteUpdateOrInsertInTransaction(src, queryBuilder, timeoutMS = src.getQueryDefaults().timeoutMS, additionalConfig) {
+        let handler;
+        let rval = null;
+        try {
+            handler = await TransactionalNamedParameterMariaDbService.create(src, src.getQueryDefaults(), additionalConfig);
+            rval = await handler.buildAndExecuteUpdateOrInsertInTransaction(queryBuilder, timeoutMS);
+        }
+        catch (err) {
+            Logger.error('Failure in oneStepBuildAndExecuteUpdateOrInsertInTransaction : %j : %s', queryBuilder, err, err);
+        }
+        finally {
+            if (handler) {
+                await handler.cleanShutdown();
+            }
+        }
+        return rval;
+    }
+    static async oneStepBuildAndExecuteInTransaction(src, queryBuilder, timeoutMS = src.getQueryDefaults().timeoutMS, additionalConfig) {
+        let handler;
+        let rval = null;
+        try {
+            handler = await TransactionalNamedParameterMariaDbService.create(src, src.getQueryDefaults(), additionalConfig);
+            rval = await handler.buildAndExecuteInTransaction(queryBuilder, timeoutMS);
+        }
+        catch (err) {
+            Logger.error('Failure in oneStepbuildAndExecuteInTransaction : %j : %s', queryBuilder, err, err);
+        }
+        finally {
+            if (handler) {
+                await handler.cleanShutdown();
+            }
+        }
+        return rval;
+    }
+}

package/lib/util/relational-database-utils.d.ts

@@ -0,0 +1,4 @@
+import { NamedParameterMariaDbService } from '../named-parameter-maria-db-service.js';
+export declare class RelationalDatabaseUtils {
+    static uploadObjectArrayToTable<Item extends object>(db: NamedParameterMariaDbService, tableName: string, data: Item[], clearTableFirst?: boolean): Promise<number>;
+}

package/lib/util/relational-database-utils.js

@@ -0,0 +1,29 @@
+import { RequireRatchet } from '@bitblit/ratchet-common/lib/lang/require-ratchet.js';
+import { Logger } from '@bitblit/ratchet-common/lib/logger/logger.js';
+import { TransactionIsolationLevel } from '../model/transaction-isolation-level.js';
+export class RelationalDatabaseUtils {
+    static async uploadObjectArrayToTable(db, tableName, data, clearTableFirst = false) {
+        RequireRatchet.notNullOrUndefined(db, 'db');
+        RequireRatchet.notNullOrUndefined(tableName, 'tableName');
+        RequireRatchet.notNullOrUndefined(data, 'data');
+        Logger.info('Writing %d items to %s (clear: %s)', data.length, tableName, clearTableFirst);
+        if (clearTableFirst) {
+            Logger.info('Clearing table %s', tableName);
+            const { results: deleteResult } = await db.executeQueryWithMeta(TransactionIsolationLevel.Default, 'DELETE FROM ' + tableName, {});
+            Logger.info('Removed %d rows', deleteResult.affectedRows);
+        }
+        const columns = new Set(data.flatMap((row) => Object.keys(row)));
+        const colArr = Array.from(columns);
+        Logger.info('Found columns : %j', colArr);
+        const sql = 'INSERT INTO ' + tableName + ' (' + colArr.join(',') + ') VALUES :multiValue';
+        const { results: insertResult } = await db.executeQueryWithMeta(TransactionIsolationLevel.Default, sql, {
+            multiValue: data,
+        });
+        const inserted = insertResult.affectedRows;
+        Logger.info('Wrote %d rows', inserted);
+        if (inserted !== data.length) {
+            Logger.warn('Should have written %d but wrote %d', data.length, inserted);
+        }
+        return inserted;
+    }
+}

package/package.json

@@ -0,0 +1,68 @@
+{
+  "name": "@bitblit/ratchet-rdbms",
+  "version": "4.0.115-alpha",
+  "description": "Ratchet tooling for working with relational databases",
+  "sideEffects": false,
+  "type": "module",
+  "module": "index.js",
+  "files": [
+    "lib/*",
+    "bin/*"
+  ],
+  "contributors": [
+    "Christopher Weiss <bitblit@gmail.com>",
+    "Zach Herridge"
+  ],
+  "husky": {
+    "hooks": {
+      "pre-commit": "pretty-quick --staged"
+    }
+  },
+  "prettier": {
+    "printWidth": 140,
+    "singleQuote": true,
+    "arrowParens": "always"
+  },
+  "config": {},
+  "scripts": {
+    "watch": "tsc-watch",
+    "clean": "shx rm -Rf lib",
+    "test": "cross-env NODE_OPTIONS=--experimental-vm-modules jest",
+    "docs": "typedoc",
+    "lint": "eslint src/**/*.ts",
+    "lint-fix": "eslint --fix src/**/*.ts",
+    "__generate-barrels": "barrelsby -q --delete -d src -e .*\\.spec\\.ts",
+    "__build": "yarn run generate-barrels && tsc",
+    "build": "tsc",
+    "force-build": "tsc --build --force"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/bitblit/Ratchet"
+  },
+  "keywords": [
+    "wrench",
+    "utility"
+  ],
+  "bugs": {
+    "url": "https://github.com/bitblit/Ratchet/issues"
+  },
+  "homepage": "https://github.com/bitblit/Ratchet#readme",
+  "engines": {
+    "node": ">=14.18"
+  },
+  "license": "Apache-2.0",
+  "dependencies": {
+    "@bitblit/ratchet-common": "4.0.115-alpha",
+    "mysql2": "3.3.0",
+    "tunnel-ssh": "5.0.5",
+    "get-port": "4.0.0"
+  },
+  "peerDependencies": {
+    "mysql2": "3.3.0",
+    "tunnel-ssh": "5.0.5",
+    "get-port": "4.0.0"
+  },
+  "resolutions": {},
+  "devDependencies": {}
+}