@squiz/db-lib 1.2.1-alpha.100

package/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "@squiz/db-lib",
+  "version": "1.2.1-alpha.100",
+  "description": "",
+  "main": "lib/index.js",
+  "scripts": {
+    "start": "node ./lib/index.js",
+    "compile": "node build.js && npx tsc",
+    "lint": "eslint ./src --ext .ts",
+    "test": "jest -c jest.config.ts --passWithNoTests",
+    "test:update-snapshots": "jest -c jest.config.ts --updateSnapshot",
+    "clean": "rimraf \".tsbuildinfo\" \"./lib\""
+  },
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "@types/jest": "28.1.6",
+    "@types/node": "17.0.27",
+    "@types/pg": "8.6.5",
+    "esbuild": "0.15.3",
+    "eslint": "8.22.0",
+    "fs-extra": "10.1.0",
+    "jest": "28.1.3",
+    "rimraf": "3.0.2",
+    "ts-jest": "28.0.7",
+    "ts-loader": "9.3.1",
+    "ts-node": "10.9.1",
+    "typescript": "4.7.4"
+  },
+  "dependencies": {
+    "@aws-sdk/client-secrets-manager": "3.121.0",
+    "@squiz/dx-logger-lib": "^1.2.1-alpha.100",
+    "dotenv": "16.0.1",
+    "pg": "8.7.3"
+  },
+  "gitHead": "909ab48128b72579f5f1390b7f045a3a41958795"
+}

package/src/AbstractRepository.ts

@@ -0,0 +1,179 @@
+import { PoolClient, Pool } from 'pg';
+import { Repositories } from './Repositories';
+
+export interface Reader<T> {
+  find(item: Partial<T>): Promise<T[]>;
+  findOne(id: string | Partial<T>): Promise<T | undefined>;
+}
+
+export interface Writer<T> {
+  create(value: Partial<T>): Promise<T>;
+  update(where: Partial<T>, newValue: Partial<T>): Promise<T[]>;
+  delete(where: Partial<T>): Promise<number>;
+}
+
+export type Repository<T> = Reader<T> & Writer<T>;
+
+export abstract class AbstractRepository<T> implements Reader<T>, Writer<T> {
+  protected tableName: string;
+
+  /** object where the key is the model property name amd the value is sql column name */
+  protected modelPropertyToSqlColumn: { [key in keyof T]: string };
+  /** object where the key is the sql column name and the value is the model property name */
+  protected sqlColumnToModelProperty: { [key: string]: string };
+
+  constructor(
+    protected repositories: Repositories,
+    protected pool: Pool,
+    tableName: string,
+    mapping: { [key in keyof T]: string },
+    protected classRef: { new (data?: Record<string, unknown>): T },
+  ) {
+    this.tableName = `"${tableName}"`;
+
+    this.modelPropertyToSqlColumn = mapping;
+
+    this.sqlColumnToModelProperty = Object.entries(mapping).reduce((prev, curr) => {
+      const [modelProp, columnName] = curr as [string, string];
+      prev[columnName] = modelProp;
+      return prev;
+    }, {} as { [key: string]: string });
+  }
+
+  protected async getConnection(): Promise<PoolClient> {
+    return await this.pool.connect();
+  }
+
+  async create(value: Partial<T>, transactionClient: PoolClient | null = null): Promise<T> {
+    const columns = Object.keys(value)
+      .map((a) => `"${this.modelPropertyToSqlColumn[a as keyof T]}"`)
+      .join(', ');
+
+    const values = Object.values(value)
+      .map((a, index) => `$${index + 1}`)
+      .join(', ');
+
+    const result = await this.executeQuery(
+      `INSERT INTO ${this.tableName} (${columns}) VALUES (${values}) RETURNING *`,
+      Object.values(value),
+      transactionClient,
+    );
+
+    return result[0];
+  }
+
+  async update(where: Partial<T>, newValue: Partial<T>, transactionClient: PoolClient | null = null): Promise<T[]> {
+    const whereValues = Object.values(where);
+    const newValues = Object.values(newValue);
+
+    const setString = Object.keys(newValue)
+      .map((a, index) => `"${this.modelPropertyToSqlColumn[a as keyof T]}" = $${index + 1}`)
+      .join(', ');
+
+    const whereString = this.createWhereStringFromPartialModel(where, newValues.length);
+
+    const result = await this.executeQuery(
+      `UPDATE ${this.tableName} 
+        SET ${setString} 
+        WHERE ${whereString}
+        RETURNING *`,
+      [...Object.values(newValues), ...Object.values(whereValues)],
+      transactionClient,
+    );
+
+    return result;
+  }
+
+  async delete(where: Partial<T>, transactionClient: PoolClient | null = null): Promise<number> {
+    const client = transactionClient ?? (await this.getConnection());
+
+    try {
+      const whereString = this.createWhereStringFromPartialModel(where);
+
+      const result = await client.query(`DELETE FROM ${this.tableName} WHERE ${whereString}`, Object.values(where));
+
+      return result.rowCount;
+    } finally {
+      if (client && !transactionClient) {
+        client.release();
+      }
+    }
+  }
+
+  protected createWhereStringFromPartialModel(values: Partial<T>, initialIndex: number = 0) {
+    const keys = Object.keys(values);
+
+    if (keys.length == 0) {
+      throw new Error(`Values cannot be an empty object. It must have at least one property`);
+    }
+
+    const sql = keys.reduce((acc, key, index) => {
+      const condition = `"${this.modelPropertyToSqlColumn[key as keyof T]}" = $${1 + index + initialIndex}`;
+
+      return acc === '' ? `${acc} ${condition}` : `${acc} AND ${condition}`;
+    }, '');
+
+    return sql;
+  }
+
+  protected async executeQuery(
+    query: string,
+    values: any[],
+    transactionClient: PoolClient | null = null,
+  ): Promise<T[]> {
+    const client = transactionClient ?? (await this.getConnection());
+    try {
+      const result = await client.query(query, values);
+
+      return result.rows.map((a) => this.createAndHydrateModel(a));
+    } finally {
+      if (client && !transactionClient) {
+        client.release();
+      }
+    }
+  }
+
+  protected createAndHydrateModel(row: any): T {
+    const inputData: Record<string, unknown> = {};
+
+    for (const key of Object.keys(row)) {
+      const translatedKey = this.sqlColumnToModelProperty[key];
+      inputData[translatedKey] = row[key];
+    }
+
+    return new this.classRef(inputData);
+  }
+
+  async findOne(item: Partial<T>): Promise<T | undefined> {
+    const result = await this.executeQuery(
+      `SELECT *
+      FROM ${this.tableName}
+      WHERE ${this.createWhereStringFromPartialModel(item)}
+      LIMIT 1`,
+      Object.values(item),
+    );
+
+    return result[0];
+  }
+
+  async find(item: Partial<T>): Promise<T[]> {
+    const result = await this.executeQuery(
+      `SELECT *
+      FROM ${this.tableName}
+      WHERE ${this.createWhereStringFromPartialModel(item)}`,
+      Object.values(item),
+    );
+
+    return result;
+  }
+
+  async findAll(): Promise<T[]> {
+    const result = await this.executeQuery(
+      `SELECT *
+      FROM ${this.tableName}`,
+      [],
+    );
+
+    return result;
+  }
+}

package/src/ConnectionManager.ts

@@ -0,0 +1,86 @@
+import { Pool } from 'pg';
+
+import os from 'os';
+import { Migrator } from './Migrator';
+
+import { PoolClient } from 'pg';
+import { Repositories } from './Repositories';
+
+export interface DbConnection {
+  user: string;
+  password: string;
+  host: string;
+  port: number;
+  database: string;
+}
+
+export interface ConnectionStringObj {
+  connectionString: string;
+}
+
+export type TransactionClient = PoolClient;
+
+export class ConnectionManager<T extends Repositories> {
+  public readonly pool: Pool;
+  public readonly repositories: T;
+
+  constructor(
+    protected applicationName: string,
+    connection: string | DbConnection,
+    protected migrationDirectory: string,
+    protected migrationList: string[],
+    repositoryCreator: (dbManager: ConnectionManager<T>) => T,
+  ) {
+    let connectionInfo: ConnectionStringObj | DbConnection;
+
+    if (typeof connection === 'string') {
+      connectionInfo = { connectionString: connection };
+    } else {
+      connectionInfo = connection;
+    }
+
+    this.pool = new Pool({
+      ...connectionInfo,
+
+      application_name: applicationName,
+
+      query_timeout: 5000, // TODO consider
+      idleTimeoutMillis: 0, // TODO consider
+      connectionTimeoutMillis: 2000,
+
+      max: os.cpus().length * 2,
+    });
+
+    this.repositories = repositoryCreator(this);
+  }
+
+  public async applyMigrations() {
+    const connection = await this.pool.connect();
+
+    const migrator = new Migrator(this.migrationDirectory, this.migrationList, connection);
+    await migrator.migrate();
+  }
+
+  public async close() {
+    await this.pool.end();
+    this.pool.removeAllListeners();
+  }
+
+  public async executeInTransaction<T>(func: (client: TransactionClient) => Promise<T>): Promise<T> {
+    const client = await this.pool.connect();
+
+    try {
+      await client.query('BEGIN');
+      const value = await func(client);
+      await client.query('COMMIT');
+      return value;
+    } catch (e) {
+      await client.query('ROLLBACK');
+      throw e;
+    } finally {
+      if (client) {
+        client.release();
+      }
+    }
+  }
+}

package/src/Migrator.ts

@@ -0,0 +1,143 @@
+import fs from 'fs/promises';
+import { PoolClient } from 'pg';
+import path from 'path';
+import os from 'os';
+import { getLogger } from '@squiz/dx-logger-lib';
+
+// Be carful about changing this value. This needs to be consistent across deployments
+// this ID is used to signal other running instances that a migration is executing.
+const MIGRATION_ADVISORY_LOCK = 4569465;
+
+const logger = getLogger({ name: 'db-migrator', meta: { pid: process.pid, hostname: os.hostname() } });
+
+export class Migrator {
+  constructor(protected migrationDir: string, protected migrationList: string[], protected pool: PoolClient) {}
+
+  protected async ensureMigrationTableExists() {
+    return this.pool.query('create table if not exists "__migrations__" (id varchar(128) NOT NULL)');
+  }
+
+  protected async getAppliedMigrations() {
+    await this.ensureMigrationTableExists();
+    const result = await this.pool.query('select * from __migrations__');
+
+    return result.rows.map(function (row) {
+      return row.id;
+    });
+  }
+
+  protected async applyMigration(migration: string, sql: string) {
+    try {
+      const result = await this.pool.query(sql);
+      logger.info('Applying ' + migration);
+
+      if (result.rowCount !== undefined) {
+        logger.info('affected rows', result.rowCount);
+      }
+
+      await this.pool.query('insert into __migrations__ (id) values ($1)', [migration]);
+    } catch (e) {
+      logger.info('error occurred running migration', migration, e);
+      throw e;
+    }
+  }
+
+  protected async getPending(migrationsList: string[], appliedMigrations: string[]) {
+    const pending: string[] = [];
+
+    // get all migrations
+    for (let i = 0; i < migrationsList.length; i++) {
+      if (migrationsList[i] !== appliedMigrations[i]) {
+        pending.push(migrationsList[i]);
+      }
+    }
+
+    // validate order
+    for (let i = 0; i < pending.length; i++) {
+      if (appliedMigrations.includes(pending[i])) {
+        throw new Error(`${pending[i]} has already run. Are you sure your migrations are in the correct order.`);
+      }
+    }
+
+    return pending;
+  }
+
+  protected async getSql(migration: string) {
+    const sql = await fs.readFile(path.join(this.migrationDir, migration), { encoding: 'utf-8' });
+
+    return sql;
+  }
+
+  protected async tryToObtainLock(): Promise<boolean> {
+    const result = await this.pool.query(`SELECT pg_try_advisory_lock(${MIGRATION_ADVISORY_LOCK}) as lockobtained`);
+
+    return result.rows[0].lockobtained;
+  }
+
+  protected async releaseLock(): Promise<void> {
+    await this.pool.query(`SELECT pg_advisory_unlock(${MIGRATION_ADVISORY_LOCK}) lockreleased`);
+  }
+
+  public async migrate(): Promise<any> {
+    try {
+      const lockObtained = await this.tryToObtainLock();
+
+      if (lockObtained === false) {
+        logger.info('migration already running');
+        await sleep(500);
+        return await this.migrate();
+      }
+
+      await this.runMigrations();
+      logger.info('completed migration');
+      await this.releaseLock();
+
+      this.dispose();
+    } catch (e) {
+      logger.info('migration failed releasing lock');
+      await this.releaseLock();
+      throw e;
+    }
+  }
+
+  protected async runMigrations() {
+    const appliedMigrations = await this.getAppliedMigrations();
+    const pending = await this.getPending(this.migrationList, appliedMigrations);
+
+    if (pending.length === 0) {
+      logger.info('No pending migrations');
+      return;
+    }
+
+    logger.info('Pending migrations:\n\t', pending.join('\n\t'));
+
+    for (const migration of pending) {
+      await this.runMigration(migration);
+    }
+  }
+
+  protected async runMigration(migration: string) {
+    try {
+      await this.pool.query('BEGIN');
+
+      const sql = await this.getSql(migration);
+      await this.applyMigration(migration, sql);
+
+      await this.pool.query('COMMIT');
+    } catch (e) {
+      logger.error('migration failed', migration, e);
+      await this.pool.query('ROLLBACK');
+      throw e;
+    }
+  }
+
+  protected dispose() {
+    return this.pool.release();
+  }
+}
+
+async function sleep(timeMs: number): Promise<void> {
+  return new Promise((resolve) => {
+    setTimeout(resolve, timeMs);
+  });
+}