@voxpelli/pg-utils 1.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Pelle Wessman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,96 @@
+# @voxpelli/pg-utils
+
+My personal database utils / helpers for Postgres
+
+[![npm version](https://img.shields.io/npm/v/@voxpelli/pg-utils.svg?style=flat)](https://www.npmjs.com/package/@voxpelli/pg-utils)
+[![npm downloads](https://img.shields.io/npm/dm/@voxpelli/pg-utils.svg?style=flat)](https://www.npmjs.com/package/@voxpelli/pg-utils)
+[![neostandard javascript style](https://img.shields.io/badge/code_style-neostandard-7fffff?style=flat&labelColor=ff80ff)](https://github.com/neostandard/neostandard)
+[![Module type: ESM](https://img.shields.io/badge/module%20type-esm-brightgreen)](https://github.com/voxpelli/badges-cjs-esm)
+[![Types in JS](https://img.shields.io/badge/types_in_js-yes-brightgreen)](https://github.com/voxpelli/types-in-js)
+[![Follow @voxpelli@mastodon.social](https://img.shields.io/mastodon/follow/109247025527949675?domain=https%3A%2F%2Fmastodon.social&style=social)](https://mastodon.social/@voxpelli)
+
+## Usage
+
+```javascript
+import {
+  csvFromFolderToDb,
+  PgTestHelpers,
+} from '@voxpelli/pg-utils';
+
+import pg from 'pg';
+
+const pgHelpers = new PgTestHelpers({
+  connectionString: 'postgres://user:pass@localhost/example',
+  fixtureFolder: new URL('./fixtures', ),
+  pg,
+  umzug: new Umzug({
+    // ...
+  }),
+});
+```
+
+## PgTestHelpers
+
+Class that creates a helpers instance
+
+### Syntax
+
+```ts
+new PgTestHelpers({
+  connectionString: 'postgres://user:pass@localhost/example',
+  fixtureFolder: new URL('./fixtures', ),
+  umzugInstall: new Umzug({
+    // ...
+  }),
+});
+```
+
+### Arguments
+
+* `options` – _[`PgTestHelpersOptions`](#pgtesthelpersoptions)_
+
+### PgTestHelpersOptions
+
+* `connectionString` – _`string`_ – a connection string for the postgres database
+* `fixtureFolder` – _`[string]`_ – _optional_ – the path to a folder of `.csv`-file fixtures named by their respective table
+* `tablesWithDependencies` – _`[string[][] | string[]]`_ – _optional_ – names of tables that depend on other tables. If some of these tables depend on each other, then use nested arrays to ensure that within the same array no two tables depend on each other
+* `umzugInstall` – _`Umzug`_ – an umzug instance that can be used to initialize tables
+
+### Methods
+
+* `initTables() => Promise<void>` – sets up all of the tables
+* `insertFixtures() => Promise<void>` – inserts all the fixtures data into the tables (only usable if `fixtureFolder` has been set)
+* `removeTables() => Promise<void>` – removes all of the tables (starting with `tablesWithDependencies`)
+
+## csvFromFolderToDb()
+
+Imports data into tables from a folder of CSV files. All files will be imported and they should named by their table names + `.csv`.
+
+### Syntax
+
+```ts
+csvFromFolderToDb(pool, path, [tablesWithDependencies]) => Promise<void>
+```
+
+### Arguments
+
+* `pool` – _`string | pg.Pool`_ – a postgres pool to use for the queries or a connection string that will be used to create one
+* `path` – _`string | URL`_ – the path to the folder that contains the CSV:s named by their table names
+* `tablesWithDependencies` – _`[string[]]`_ – _optional_ – names of tables that depend on other tables. The first name in this list will have its fixtures inserted last
+
+### Returns
+
+`Promise` that resolves on completion
+
+<!-- ## Used by
+
+* [`example`](https://example.com/) – used by this one to do X and Y
+
+## Similar modules
+
+* [`example`](https://example.com/) – is similar in this way
+
+## See also
+
+* [Announcement blog post](#)
+* [Announcement tweet](#) -->

package/index.d.ts

@@ -0,0 +1,3 @@
+export { csvFromFolderToDb } from "./lib/csv-folder-to-db.js";
+export { PgTestHelpers } from "./lib/test-helpers.js";
+//# sourceMappingURL=index.d.ts.map

package/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["index.js"],"names":[],"mappings":""}

package/index.js

@@ -0,0 +1,2 @@
+export { csvFromFolderToDb } from './lib/csv-folder-to-db.js';
+export { PgTestHelpers } from './lib/test-helpers.js';

package/lib/csv-folder-to-db.d.ts

@@ -0,0 +1,4 @@
+export function csvFromFolderToDb(connection: ConnectionConfig | Pool, path: string | URL, tablesWithDependencies?: string[] | undefined): Promise<void>;
+import type { ConnectionConfig } from './utils.js';
+import type { Pool } from 'pg';
+//# sourceMappingURL=csv-folder-to-db.d.ts.map

package/lib/csv-folder-to-db.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv-folder-to-db.d.ts","sourceRoot":"","sources":["csv-folder-to-db.js"],"names":[],"mappings":"AAmEA,8CALW,gBAAgB,GAAG,IAAI,QACvB,MAAM,GAAG,GAAG,2BACZ,MAAM,EAAE,GAAG,SAAS,GAClB,OAAO,CAAC,IAAI,CAAC,CAwBzB;sCA7EqC,YAAY;0BAFxB,IAAI"}

package/lib/csv-folder-to-db.js

@@ -0,0 +1,90 @@
+import { createReadStream } from 'node:fs';
+import { opendir } from 'node:fs/promises';
+import pathModule from 'node:path';
+import { pipeline as promisedPipeline } from 'node:stream/promises';
+import { fileURLToPath } from 'node:url';
+
+import { from as copyFrom } from 'pg-copy-streams';
+
+import { createPgPool } from './utils.js';
+
+/** @import { Pool } from 'pg' */
+
+/** @import { ConnectionConfig } from './utils.js' */
+
+/**
+ * @param {string | URL} path
+ * @param {string} [extension]
+ * @returns {AsyncIterable<string>}
+ */
+async function * filesInFolder (path, extension) {
+  const dir = await opendir(path);
+  const stringPath = typeof path === 'string' ? path : fileURLToPath(path);
+
+  for await (const dirent of dir) {
+    if (!dirent.isFile()) continue;
+    if (extension && pathModule.extname(dirent.name) !== extension) continue;
+    yield pathModule.join(stringPath, dirent.name);
+  }
+}
+
+/**
+ * @param {string | URL} path
+ * @param {string[]} tablesWithDependencies
+ * @returns {Promise<string[]>}
+ */
+async function getFilesOrderedByDependencies (path, tablesWithDependencies) {
+  /** @type {Map<string, string>} */
+  const fileMap = new Map();
+
+  for await (const file of filesInFolder(path, '.csv')) {
+    fileMap.set(pathModule.basename(file, '.csv'), file);
+  }
+
+  /** @type {string[]} */
+  const files = [];
+
+  for (const table of tablesWithDependencies) {
+    const file = fileMap.get(table);
+
+    if (file) {
+      files.unshift(file);
+      fileMap.delete(table);
+    }
+  }
+
+  return [
+    ...fileMap.values(),
+    ...files,
+  ];
+}
+
+/**
+ * @param {ConnectionConfig | Pool} connection
+ * @param {string | URL} path
+ * @param {string[] | undefined} [tablesWithDependencies]
+ * @returns {Promise<void>}
+ */
+export async function csvFromFolderToDb (connection, path, tablesWithDependencies = []) {
+  const files = await getFilesOrderedByDependencies(path, tablesWithDependencies);
+
+  const pool = (typeof connection === 'object' && 'connect' in connection) ? connection : createPgPool(connection);
+
+  const client = await pool.connect();
+
+  for (const file of files) {
+    const name = pathModule.basename(file, '.csv');
+    const dbCopy = client.query(copyFrom(`COPY ${name} FROM STDIN DELIMITER ',' CSV HEADER`));
+
+    // eslint-disable-next-line security/detect-non-literal-fs-filename
+    const csvContent = createReadStream(file);
+
+    try {
+      await promisedPipeline(csvContent, dbCopy);
+    } catch (cause) {
+      throw new Error(`Failed inserting data into "${name}"`, { cause });
+    }
+  }
+
+  client.release();
+}

package/lib/test-helpers.d.ts

@@ -0,0 +1,17 @@
+export class PgTestHelpers {
+    constructor(options: PgTestHelpersOptions);
+    queryPromise: Pool["query"];
+    initTables(): Promise<void>;
+    insertFixtures(): Promise<void>;
+    removeTables(): Promise<void>;
+    #private;
+}
+export type PgTestHelpersOptions = {
+    connectionConfig: ConnectionConfig;
+    fixtureFolder?: string | URL;
+    schema: string | URL | import("umzug").Umzug;
+    tablesWithDependencies?: Array<string[] | string>;
+};
+import type { Pool } from 'pg';
+import type { ConnectionConfig } from './utils.js';
+//# sourceMappingURL=test-helpers.d.ts.map

package/lib/test-helpers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-helpers.d.ts","sourceRoot":"","sources":["test-helpers.js"],"names":[],"mappings":"AAsBA;IAaE,qBADY,oBAAoB,EAiC/B;IAnCD,cADW,IAAI,CAAC,OAAO,CAAC,CACX;IAoEb,cADc,OAAO,CAAC,IAAI,CAAC,CAiB1B;IAGD,kBADc,OAAO,CAAC,IAAI,CAAC,CAM1B;IAGD,gBADc,OAAO,CAAC,IAAI,CAAC,CAM1B;;CACF;;sBArHa,gBAAgB;oBAChB,MAAM,GAAG,GAAG;YACZ,MAAM,GAAG,GAAG,GAAG,OAAO,OAAO,EAAE,KAAK;6BACpC,KAAK,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC;;0BATZ,IAAI;sCAEQ,YAAY"}

package/lib/test-helpers.js

@@ -0,0 +1,134 @@
+import { readFile } from 'node:fs/promises';
+
+import {
+  createUmzeptionPgContext,
+  pgInstallSchemaFromString,
+} from 'umzeption';
+
+import { csvFromFolderToDb } from './csv-folder-to-db.js';
+import { createPgPool, isStringArray, TypeNeverError } from './utils.js';
+
+/** @import { Pool } from 'pg' */
+
+/** @import { ConnectionConfig } from './utils.js' */
+
+/**
+ * @typedef PgTestHelpersOptions
+ * @property {ConnectionConfig} connectionConfig
+ * @property {string | URL} [fixtureFolder]
+ * @property {string | URL | import('umzug').Umzug} schema
+ * @property {Array<string[] | string>} [tablesWithDependencies]
+ */
+
+export class PgTestHelpers {
+  /** @type {string | URL | undefined} */
+  #fixtureFolder;
+  /** @type {Pool} */
+  #pool;
+  /** @type {PgTestHelpersOptions['schema']} */
+  #schema;
+  /** @type {Array<string[] | string> | undefined} */
+  #tablesWithDependencies;
+  /** @type {Pool['query']} */
+  queryPromise;
+
+  /** @param {PgTestHelpersOptions} options */
+  constructor (options) {
+    if (!options || typeof options !== 'object') {
+      throw new TypeNeverError(options, 'Expected an options object');
+    }
+
+    const {
+      connectionConfig,
+      fixtureFolder,
+      schema,
+      tablesWithDependencies,
+    } = options;
+
+    if (typeof connectionConfig !== 'string' && typeof connectionConfig !== 'object') {
+      throw new TypeNeverError(connectionConfig, 'Invalid connectionConfig, expected a string or an object');
+    }
+    if (fixtureFolder && typeof fixtureFolder !== 'string' && !(fixtureFolder instanceof URL)) {
+      throw new TypeNeverError(fixtureFolder, 'Invalid fixtureFolder, expected a string');
+    }
+    if (typeof schema !== 'string' && typeof schema !== 'object') {
+      throw new TypeNeverError(schema, 'Invalid schema, expected a string or an object');
+    }
+    if (tablesWithDependencies && !Array.isArray(tablesWithDependencies)) {
+      throw new TypeNeverError(tablesWithDependencies, 'Invalid tablesWithDependencies, expected an array');
+    }
+
+    const pool = createPgPool(connectionConfig);
+
+    this.#fixtureFolder = fixtureFolder;
+    this.#tablesWithDependencies = tablesWithDependencies;
+    this.#pool = pool;
+    this.#schema = schema;
+    this.queryPromise = pool.query.bind(pool);
+  }
+
+  /** @returns {Promise<string[]>} */
+  async #getTableNames () {
+    const { rows } = await this.queryPromise('SELECT tablename FROM pg_tables WHERE schemaname = \'public\'');
+    return rows.map(row => row.tablename);
+  }
+
+  /**
+   * @param {Array<string[] | string>} tables
+   * @returns {Promise<void>}
+   */
+  async #removeTablesByName (tables) {
+    if (isStringArray(tables)) {
+      await Promise.all(
+        tables.map(name => this.queryPromise('DROP TABLE IF EXISTS ' + name + ' CASCADE'))
+      ).catch(cause => {
+        throw new Error(`Failed to drop tables: ${tables}`, { cause });
+      });
+    } else {
+      for (const name of tables) {
+        await (
+          Array.isArray(name)
+            ? this.#removeTablesByName(name)
+            : this.queryPromise('DROP TABLE IF EXISTS ' + name + ' CASCADE').catch(cause => {
+              throw new Error(`Failed to drop table: ${name}`, { cause });
+            })
+        );
+      }
+    }
+  }
+
+  /** @returns {Promise<void>} */
+  async initTables () {
+    try {
+      if (typeof this.#schema === 'object' && 'up' in this.#schema) {
+        await this.#schema.up();
+        return;
+      }
+
+      const schema = this.#schema instanceof URL
+        // eslint-disable-next-line security/detect-non-literal-fs-filename
+        ? await readFile(this.#schema, 'utf8')
+        : this.#schema;
+
+      return pgInstallSchemaFromString(createUmzeptionPgContext(this.#pool), schema);
+    } catch (cause) {
+      throw new Error('Failed to create tables', { cause });
+    }
+  }
+
+  /** @returns {Promise<void>} */
+  async insertFixtures () {
+    if (!this.#fixtureFolder) {
+      throw new Error('No fixture folder defined');
+    }
+    return csvFromFolderToDb(this.#pool, this.#fixtureFolder, this.#tablesWithDependencies?.flat());
+  }
+
+  /** @returns {Promise<void>} */
+  async removeTables () {
+    if (this.#tablesWithDependencies) {
+      await this.#removeTablesByName(this.#tablesWithDependencies);
+    }
+    await this.#removeTablesByName(await this.#getTableNames());
+  }
+}

package/lib/utils.d.ts

@@ -0,0 +1,8 @@
+export function createPgPool(connectionString: ConnectionConfig): import("pg").Pool;
+export function isStringArray(value: unknown): value is string[];
+export class TypeNeverError extends TypeError {
+    constructor(value: never, message: string, options?: ErrorOptions);
+}
+export type ConnectionConfigObject = Pick<import("pg").ClientConfig, "host" | "port" | "user" | "password" | "stream">;
+export type ConnectionConfig = string | ConnectionConfigObject;
+//# sourceMappingURL=utils.d.ts.map

package/lib/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["utils.js"],"names":[],"mappings":"AASA,+CAHW,gBAAgB,GACd,OAAO,IAAI,EAAE,IAAI,CAO7B;AA2BD,qCAHW,OAAO,GACL,KAAK,IAAI,MAAM,EAAE,CAK7B;AA5BD;IAME,mBAJW,KAAK,WACL,MAAM,YACN,YAAY,EAItB;CACF;qCAvBa,IAAI,CAAC,OAAO,IAAI,EAAE,YAAY,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,UAAU,GAAG,QAAQ,CAAC;+BACjF,MAAM,GAAG,sBAAsB"}

package/lib/utils.js

@@ -0,0 +1,45 @@
+import pg from 'pg';
+
+/** @typedef {Pick<import('pg').ClientConfig, 'host' | 'port' | 'user' | 'password' | 'stream'>} ConnectionConfigObject */
+/** @typedef {string | ConnectionConfigObject} ConnectionConfig */
+
+/**
+ * @param {ConnectionConfig} connectionString
+ * @returns {import('pg').Pool}
+ */
+export function createPgPool (connectionString) {
+  return new pg.Pool({
+    ...(typeof connectionString === 'object' ? connectionString : { connectionString }),
+    allowExitOnIdle: true,
+  });
+}
+
+export class TypeNeverError extends TypeError {
+  /**
+   * @param {never} value
+   * @param {string} message
+   * @param {ErrorOptions} [options]
+   */
+  constructor (value, message, options) {
+    super(`${message}. Got: ${typeof value}`, options);
+  }
+}
+
+/**
+ * Array.isArray() on its own give type any[]
+ *
+ * @param {unknown} value
+ * @returns {value is unknown[]}
+ */
+function typesafeIsArray (value) {
+  return Array.isArray(value);
+}
+
+/**
+ * @param {unknown} value
+ * @returns {value is string[]}
+ */
+export function isStringArray (value) {
+  if (!typesafeIsArray(value)) return false;
+  return value.every(item => typeof item === 'string');
+}

package/package.json

@@ -0,0 +1,74 @@
+{
+  "name": "@voxpelli/pg-utils",
+  "version": "1.0.0",
+  "description": " My personal database utils / helpers for Postgres",
+  "homepage": "http://github.com/voxpelli/pg-utils",
+  "repository": {
+    "type": "git",
+    "url": "git://github.com/voxpelli/pg-utils.git"
+  },
+  "keywords": [],
+  "author": "Pelle Wessman <pelle@kodfabrik.se> (http://kodfabrik.se/)",
+  "license": "MIT",
+  "engines": {
+    "node": "^20.9.0 || >=22.0.0"
+  },
+  "type": "module",
+  "exports": "./index.js",
+  "types": "index.d.ts",
+  "files": [
+    "index.js",
+    "index.d.ts",
+    "index.d.ts.map",
+    "lib/**/*.js",
+    "lib/**/*.d.ts",
+    "lib/**/*.d.ts.map"
+  ],
+  "scripts": {
+    "build:0": "run-s clean",
+    "build:1-declaration": "tsc -p declaration.tsconfig.json",
+    "build": "run-s build:*",
+    "check:installed-check": "installed-check",
+    "check:knip": "knip",
+    "check:lint": "eslint",
+    "check:tsc": "tsc",
+    "check:type-coverage": "type-coverage --detail --strict --at-least 95 --ignore-files 'test/*'",
+    "check": "run-s clean && run-p check:*",
+    "clean:declarations-top": "rm -rf $(find . -maxdepth 1 -type f -name '*.d.ts*' ! -name 'index.d.ts')",
+    "clean:declarations-lib": "rm -rf $(find lib -type f -name '*.d.ts*' ! -name '*-types.d.ts')",
+    "clean": "run-p clean:*",
+    "prepare": "husky",
+    "prepublishOnly": "run-s build",
+    "test:mocha": "c8 --reporter=lcov --reporter=text mocha 'test/**/*.spec.js'",
+    "test-ci": "run-s test:*",
+    "test": "run-s check test:*"
+  },
+  "devDependencies": {
+    "@types/chai": "^4.3.20",
+    "@types/chai-as-promised": "^7.1.8",
+    "@types/mocha": "^10.0.10",
+    "@types/node": "^20.17.9",
+    "@types/pg-copy-streams": "^1.2.5",
+    "@voxpelli/eslint-config": "^22.2.0",
+    "@voxpelli/tsconfig": "^15.1.0",
+    "c8": "^10.1.2",
+    "chai": "^4.5.0",
+    "chai-as-promised": "^7.1.2",
+    "dotenv": "^16.4.6",
+    "eslint": "^9.16.0",
+    "husky": "^9.1.7",
+    "installed-check": "^9.3.0",
+    "knip": "^5.38.4",
+    "mocha": "^10.8.2",
+    "npm-run-all2": "^7.0.1",
+    "type-coverage": "^2.29.7",
+    "typescript": "~5.7.2"
+  },
+  "dependencies": {
+    "@types/pg": "^8.11.10",
+    "pg": "^8.13.1",
+    "pg-copy-streams": "^6.0.6",
+    "umzeption": "^0.4.1",
+    "umzug": "^3.8.2"
+  }
+}