@voxpelli/pg-utils 3.0.1 → 3.1.0

package/README.md

@@ -7,6 +7,7 @@ My personal database utils / helpers for Postgres
 [![neostandard javascript style](https://img.shields.io/badge/code_style-neostandard-7fffff?style=flat&labelColor=ff80ff)](https://github.com/neostandard/neostandard)
 [![Module type: ESM](https://img.shields.io/badge/module%20type-esm-brightgreen)](https://github.com/voxpelli/badges-cjs-esm)
 [![Types in JS](https://img.shields.io/badge/types_in_js-yes-brightgreen)](https://github.com/voxpelli/types-in-js)
+[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/voxpelli/pg-utils)
 [![Follow @voxpelli@mastodon.social](https://img.shields.io/mastodon/follow/109247025527949675?domain=https%3A%2F%2Fmastodon.social&style=social)](https://mastodon.social/@voxpelli)
 
 ## Usage
@@ -14,6 +15,7 @@ My personal database utils / helpers for Postgres
 ```javascript
 import {
   csvFromFolderToDb,
+  dbToCsvFolder,
   PgTestHelpers,
 } from '@voxpelli/pg-utils';
 
@@ -22,9 +24,9 @@ const pgHelpers = new PgTestHelpers({
   fixtureFolder: new URL('./fixtures', import.meta.url),
   ignoreTables: ['xyz'],
   schema: new URL('./create-tables.sql', import.meta.url),
-  tablesWithDependencies: [
-    'abc',
+  tableLoadOrder: [
     ['foo', 'bar'],
+    'abc',
   ]
 });
 
@@ -55,7 +57,7 @@ new PgTestHelpers({
     // ...
   ],
   schema: new URL('./create-tables.sql', import.meta.url),
-  tablesWithDependencies: [
+  tableLoadOrder: [
     // ...
   ]
 });
@@ -71,13 +73,14 @@ new PgTestHelpers({
 * `fixtureFolder` – _`[string | URL]`_ – _optional_ – the path to a folder of `.csv`-file fixtures named by their respective table
 * `ignoreTables` – _`[string[]]`_ – _optional_ – names of tables to ignore when dropping
 * `schema` – _`string | URL | Umzug`_ – an umzug instance that can be used to initialize tables or the schema itself or a `URL` to a text file containing the schema
-* `tablesWithDependencies` – _`[Array<string[] | string>]`_ – _optional_ – names of tables that depend on other tables. If some of these tables depend on each other, then use nested arrays to ensure that within the same array no two tables depend on each other
+* `tableLoadOrder` – _`[Array<string[] | string>]`_ – _optional_ – tables in parent-first insertion order: the first item is loaded first and dropped last. Use nested arrays to group tables that can be dropped in parallel. Mutually exclusive with `tablesWithDependencies`.
+* `tablesWithDependencies` – _`[Array<string[] | string>]`_ – _optional_ – **Deprecated:** use `tableLoadOrder` instead. Tables in leaf-first deletion order: the first item is dropped first and loaded last.
 
 ### Methods
 
 * `initTables() => Promise<void>` – sets up all of the tables. Automatically acquires an exclusive database lock on first call.
 * `insertFixtures() => Promise<void>` – inserts all the fixtures data into the tables (only usable if `fixtureFolder` has been set). Automatically acquires an exclusive database lock on first call.
-* `removeTables() => Promise<void>` – removes all of the tables (starting with `tablesWithDependencies`). Automatically acquires an exclusive database lock on first call.
+* `removeTables() => Promise<void>` – removes all of the tables (respecting `tableLoadOrder` / `tablesWithDependencies` ordering). Automatically acquires an exclusive database lock on first call.
 * `end() => Promise<void>` – releases the database lock (if acquired) and closes all database connections. **Always call this when done** to properly clean up resources.
 
 #### Database Locking
@@ -91,14 +94,38 @@ Imports data into tables from a folder of CSV files. All files will be imported
 ### Syntax
 
 ```ts
-csvFromFolderToDb(pool, path, [tablesWithDependencies]) => Promise<void>
+csvFromFolderToDb(pool, path, [options]) => Promise<void>
 ```
 
 ### Arguments
 
 * `pool` – _`string | pg.Pool`_ – a postgres pool to use for the queries or a connection string that will be used to create one
 * `path` – _`string | URL`_ – the path to the folder that contains the CSV:s named by their table names
-* `tablesWithDependencies` – _`[string[]]`_ – _optional_ – names of tables that depend on other tables. The first name in this list will have its fixtures inserted last
+* `options` – _`object`_ – _optional_ – ordering options (also accepts a `string[]` for backwards compatibility, treated as `tablesWithDependencies`)
+  * `tableLoadOrder` – _`string[]`_ – tables in parent-first insertion order: the first table is loaded first. Mutually exclusive with `tablesWithDependencies`.
+  * `tablesWithDependencies` – _`string[]`_ – **Deprecated:** use `tableLoadOrder` instead. Tables in leaf-first deletion order: the first table is loaded last.
+
+### Returns
+
+`Promise` that resolves on completion
+
+## dbToCsvFolder()
+
+Exports database tables to CSV files in a folder. Each table is written as `<table>.csv` with a header row.
+
+### Syntax
+
+```ts
+dbToCsvFolder(connection, outputPath, tables, [options]) => Promise<void>
+```
+
+### Arguments
+
+* `connection` – _`string | pg.Pool`_ – a postgres pool to use for the queries or a connection string that will be used to create one
+* `outputPath` – _`string | URL`_ – the directory to write CSV files into (created if it does not exist)
+* `tables` – _`string[]`_ – explicit list of table names to export
+* `options` – _`object`_ – _optional_
+  * `orderBy` – _`string`_ – SQL `ORDER BY` expression for deterministic output (default: `'1'`, i.e. the first column)
 
 ### Returns
 

package/index.d.ts

@@ -1,3 +1,4 @@
 export { csvFromFolderToDb } from "./lib/csv-folder-to-db.js";
+export { dbToCsvFolder } from "./lib/db-to-csv-folder.js";
 export { PgTestHelpers } from "./lib/test-helpers.js";
 //# sourceMappingURL=index.d.ts.map

package/index.js

@@ -1,2 +1,3 @@
 export { csvFromFolderToDb } from './lib/csv-folder-to-db.js';
+export { dbToCsvFolder } from './lib/db-to-csv-folder.js';
 export { PgTestHelpers } from './lib/test-helpers.js';

package/lib/csv-folder-to-db.d.ts

@@ -1,3 +1,7 @@
-export function csvFromFolderToDb(connection: string | Pool, path: string | URL, tablesWithDependencies?: string[] | undefined): Promise<void>;
+export function csvFromFolderToDb(connection: string | Pool, path: string | URL, options?: string[] | CsvFromFolderToDbOptions): Promise<void>;
+export type CsvFromFolderToDbOptions = {
+    tableLoadOrder?: string[];
+    tablesWithDependencies?: string[];
+};
 import type { Pool } from 'pg';
 //# sourceMappingURL=csv-folder-to-db.d.ts.map

package/lib/csv-folder-to-db.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"csv-folder-to-db.d.ts","sourceRoot":"","sources":["csv-folder-to-db.js"],"names":[],"mappings":"AAiEA,8CALW,MAAM,GAAG,IAAI,QACb,MAAM,GAAG,GAAG,2BACZ,MAAM,EAAE,GAAG,SAAS,GAClB,OAAO,CAAC,IAAI,CAAC,CA0BzB;0BA/EyB,IAAI"}
+{"version":3,"file":"csv-folder-to-db.d.ts","sourceRoot":"","sources":["csv-folder-to-db.js"],"names":[],"mappings":"AAkHA,8CALW,MAAM,GAAG,IAAI,QACb,MAAM,GAAG,GAAG,YACZ,MAAM,EAAE,GAAG,wBAAwB,GACjC,OAAO,CAAC,IAAI,CAAC,CA2BzB;;qBA1Ea,MAAM,EAAE;6BACR,MAAM,EAAE;;0BAvDI,IAAI"}

package/lib/csv-folder-to-db.js

@@ -5,6 +5,7 @@ import { pipeline as promisedPipeline } from 'node:stream/promises';
 import { fileURLToPath } from 'node:url';
 
 import { from as copyFrom } from 'pg-copy-streams';
+import pg from 'pg';
 
 import { createPgPool } from './utils.js';
 
@@ -57,13 +58,62 @@ async function getFilesOrderedByDependencies (path, tablesWithDependencies) {
   ];
 }
 
+/** @type {boolean} */
+let hasWarnedDeprecation = false;
+
+/**
+ * @typedef CsvFromFolderToDbOptions
+ * @property {string[]} [tableLoadOrder] Tables in parent-first insertion order (first item loaded first, dropped last)
+ * @property {string[]} [tablesWithDependencies] Deprecated: use `tableLoadOrder` instead. Tables in leaf-first deletion order (first item dropped first, loaded last)
+ */
+
+/**
+ * @param {string[] | CsvFromFolderToDbOptions | undefined} options
+ * @returns {string[]}
+ */
+function resolveTableOrder (options) {
+  if (!options) return [];
+  if (Array.isArray(options)) {
+    warnDeprecation();
+    return options;
+  }
+
+  const { tableLoadOrder, tablesWithDependencies } = options;
+
+  if (tableLoadOrder && tablesWithDependencies) {
+    throw new Error('Cannot specify both tableLoadOrder and tablesWithDependencies');
+  }
+
+  if (tablesWithDependencies) {
+    warnDeprecation();
+    return tablesWithDependencies;
+  }
+
+  if (tableLoadOrder) {
+    // tableLoadOrder is parent-first; reverse to get the leaf-first
+    // order that getFilesOrderedByDependencies expects via unshift()
+    return [...tableLoadOrder].reverse();
+  }
+
+  return [];
+}
+
+function warnDeprecation () {
+  if (!hasWarnedDeprecation) {
+    hasWarnedDeprecation = true;
+    // eslint-disable-next-line no-console
+    console.warn('@voxpelli/pg-utils: tablesWithDependencies is deprecated, use tableLoadOrder instead (parent-first insertion order)');
+  }
+}
+
 /**
  * @param {string | Pool} connection
  * @param {string | URL} path
- * @param {string[] | undefined} [tablesWithDependencies]
+ * @param {string[] | CsvFromFolderToDbOptions} [options]
  * @returns {Promise<void>}
  */
-export async function csvFromFolderToDb (connection, path, tablesWithDependencies = []) {
+export async function csvFromFolderToDb (connection, path, options) {
+  const tablesWithDependencies = resolveTableOrder(options);
   const files = await getFilesOrderedByDependencies(path, tablesWithDependencies);
 
   const pool = (typeof connection === 'object' && 'connect' in connection) ? connection : createPgPool(connection);
@@ -73,7 +123,7 @@ export async function csvFromFolderToDb (connection, path, tablesWithDependencie
   try {
     for (const file of files) {
       const name = pathModule.basename(file, '.csv');
-      const dbCopy = client.query(copyFrom(`COPY ${name} FROM STDIN WITH (FORMAT csv, HEADER MATCH)`));
+      const dbCopy = client.query(copyFrom(`COPY ${pg.escapeIdentifier(name)} FROM STDIN WITH (FORMAT csv, HEADER MATCH)`));
 
       // eslint-disable-next-line security/detect-non-literal-fs-filename
       const csvContent = createReadStream(file);

package/lib/db-to-csv-folder.d.ts

@@ -0,0 +1,6 @@
+export function dbToCsvFolder(connection: string | Pool, outputPath: string | URL, tables: string[], options?: DbToCsvFolderOptions): Promise<void>;
+export type DbToCsvFolderOptions = {
+    orderBy?: string;
+};
+import type { Pool } from 'pg';
+//# sourceMappingURL=db-to-csv-folder.d.ts.map

package/lib/db-to-csv-folder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"db-to-csv-folder.d.ts","sourceRoot":"","sources":["db-to-csv-folder.js"],"names":[],"mappings":"AA8BA,0CANW,MAAM,GAAG,IAAI,cACb,MAAM,GAAG,GAAG,UACZ,MAAM,EAAE,YACR,oBAAoB,GAClB,OAAO,CAAC,IAAI,CAAC,CA0BzB;;cAvCa,MAAM;;0BAJM,IAAI"}

package/lib/db-to-csv-folder.js

@@ -0,0 +1,55 @@
+import { createWriteStream } from 'node:fs';
+import { mkdir } from 'node:fs/promises';
+import pathModule from 'node:path';
+import { pipeline } from 'node:stream/promises';
+import { fileURLToPath } from 'node:url';
+
+import pg from 'pg';
+import { to as copyTo } from 'pg-copy-streams';
+
+import { createPgPool } from './utils.js';
+
+/** @import { Pool } from 'pg' */
+
+/**
+ * @typedef DbToCsvFolderOptions
+ * @property {string} [orderBy]
+ */
+
+/**
+ * Exports database tables to CSV files in a folder.
+ *
+ * Each table is exported as `<table>.csv` with a header row, ordered by
+ * the specified column (default: first column) for deterministic output.
+ *
+ * @param {string | Pool} connection
+ * @param {string | URL} outputPath
+ * @param {string[]} tables
+ * @param {DbToCsvFolderOptions} [options]
+ * @returns {Promise<void>}
+ */
+export async function dbToCsvFolder (connection, outputPath, tables, options = {}) {
+  const { orderBy = '1' } = options;
+
+  const dirPath = typeof outputPath === 'string' ? outputPath : fileURLToPath(outputPath);
+
+  // eslint-disable-next-line security/detect-non-literal-fs-filename
+  await mkdir(dirPath, { recursive: true });
+
+  const pool = (typeof connection === 'object' && 'connect' in connection) ? connection : createPgPool(connection);
+
+  const client = await pool.connect();
+
+  try {
+    for (const table of tables) {
+      const sql = `COPY (SELECT * FROM ${pg.escapeIdentifier(table)} ORDER BY ${orderBy}) TO STDOUT WITH (FORMAT csv, HEADER true)`;
+      const copyToStream = client.query(copyTo(sql));
+      const filePath = pathModule.join(dirPath, `${table}.csv`);
+
+      // eslint-disable-next-line security/detect-non-literal-fs-filename
+      await pipeline(copyToStream, createWriteStream(filePath));
+    }
+  } finally {
+    client.release();
+  }
+}

package/lib/test-helpers.d.ts

@@ -12,6 +12,7 @@ export type PgTestHelpersOptions = {
     fixtureFolder?: string | URL;
     ignoreTables?: string[];
     schema: string | URL | ((pool: Pool) => import("umzug").Umzug<import("umzeption").UmzeptionContext<"pg", import("umzeption").FastifyPostgresStyleDb>>);
+    tableLoadOrder?: Array<string[] | string>;
     tablesWithDependencies?: Array<string[] | string>;
 };
 import type { Pool } from 'pg';

package/lib/test-helpers.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"test-helpers.d.ts","sourceRoot":"","sources":["test-helpers.js"],"names":[],"mappings":"AAqBA;IAqBE,qBADY,oBAAoB,EAuC/B;IAzCD,cADW,IAAI,CAAC,OAAO,CAAC,CACX;IA8Gb,cADc,OAAO,CAAC,IAAI,CAAC,CAmB1B;IAGD,kBADc,OAAO,CAAC,IAAI,CAAC,CAS1B;IAGD,gBADc,OAAO,CAAC,IAAI,CAAC,CAgB1B;IAGD,OADc,OAAO,CAAC,IAAI,CAAC,CAO1B;;CACF;;sBAhMa,MAAM;oBACN,MAAM,GAAG,GAAG;mBACZ,MAAM,EAAE;YACR,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,KAAK,OAAO,OAAO,EAAE,KAAK,CAAC,OAAO,WAAW,EAAE,gBAAgB,CAAC,IAAI,EAAE,OAAO,WAAW,EAAE,sBAAsB,CAAC,CAAC,CAAC;6BAC9I,KAAK,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC;;0BARJ,IAAI"}
+{"version":3,"file":"test-helpers.d.ts","sourceRoot":"","sources":["test-helpers.js"],"names":[],"mappings":"AAuBA;IAuBE,qBADY,oBAAoB,EAoD/B;IAtDD,cADW,IAAI,CAAC,OAAO,CAAC,CACX;IA8Hb,cADc,OAAO,CAAC,IAAI,CAAC,CAmB1B;IAGD,kBADc,OAAO,CAAC,IAAI,CAAC,CAkB1B;IAGD,gBADc,OAAO,CAAC,IAAI,CAAC,CAgB1B;IAGD,OADc,OAAO,CAAC,IAAI,CAAC,CAO1B;;CACF;;sBA5Na,MAAM;oBACN,MAAM,GAAG,GAAG;mBACZ,MAAM,EAAE;YACR,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,KAAK,OAAO,OAAO,EAAE,KAAK,CAAC,OAAO,WAAW,EAAE,gBAAgB,CAAC,IAAI,EAAE,OAAO,WAAW,EAAE,sBAAsB,CAAC,CAAC,CAAC;qBAC9I,KAAK,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC;6BACxB,KAAK,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC;;0BATJ,IAAI"}

package/lib/test-helpers.js

@@ -1,5 +1,6 @@
 import { readFile } from 'node:fs/promises';
 
+import pg from 'pg';
 import {
   createUmzeptionPgContext,
   pgInstallSchemaFromString,
@@ -16,7 +17,8 @@ import { createPgPool, createAndLockConnection, releaseLock, isStringArray, Type
  * @property {string | URL} [fixtureFolder]
  * @property {string[]} [ignoreTables]
  * @property {string | URL | ((pool: Pool) => import('umzug').Umzug<import('umzeption').UmzeptionContext<'pg', import('umzeption').FastifyPostgresStyleDb>>)} schema
- * @property {Array<string[] | string>} [tablesWithDependencies]
+ * @property {Array<string[] | string>} [tableLoadOrder] Tables in parent-first insertion order. First item loaded first, dropped last. Mutually exclusive with `tablesWithDependencies`.
+ * @property {Array<string[] | string>} [tablesWithDependencies] Deprecated: use `tableLoadOrder` instead. Tables in leaf-first deletion order. First item dropped first, loaded last.
  */
 
 export class PgTestHelpers {
@@ -36,6 +38,8 @@ export class PgTestHelpers {
   #schema;
   /** @type {Array<string[] | string> | undefined} */
   #tablesWithDependencies;
+  /** @type {string[] | undefined} */
+  #tableLoadOrder;
   /** @type {Pool['query']} */
   queryPromise;
 
@@ -50,6 +54,7 @@ export class PgTestHelpers {
       fixtureFolder,
       ignoreTables,
       schema,
+      tableLoadOrder,
       tablesWithDependencies,
     } = options;
 
@@ -65,16 +70,28 @@ export class PgTestHelpers {
     if (typeof schema !== 'string' && typeof schema !== 'object' && typeof schema !== 'function') {
       throw new TypeNeverError(schema, 'Invalid schema, expected a string, object or function');
     }
+    if (tableLoadOrder && tablesWithDependencies) {
+      throw new Error('Cannot specify both tableLoadOrder and tablesWithDependencies');
+    }
+    if (tableLoadOrder && !Array.isArray(tableLoadOrder)) {
+      throw new TypeNeverError(tableLoadOrder, 'Invalid tableLoadOrder, expected an array');
+    }
     if (tablesWithDependencies && !Array.isArray(tablesWithDependencies)) {
       throw new TypeNeverError(tablesWithDependencies, 'Invalid tablesWithDependencies, expected an array');
     }
 
     const pool = createPgPool(connectionString);
 
+    // tableLoadOrder is parent-first; reverse to get the leaf-first
+    // order that internal methods expect (drop first item first, load last)
+    this.#tablesWithDependencies = tableLoadOrder
+      ? [...tableLoadOrder].reverse()
+      : tablesWithDependencies;
+    this.#tableLoadOrder = tableLoadOrder?.flat();
+
     this.#connectionString = connectionString;
     this.#fixtureFolder = fixtureFolder;
     this.#ignoreTables = ignoreTables;
-    this.#tablesWithDependencies = tablesWithDependencies;
     this.#pool = pool;
     this.#schema = schema;
     this.queryPromise = pool.query.bind(pool);
@@ -94,7 +111,7 @@ export class PgTestHelpers {
   async #removeTablesByName (tables, allowParallelRemoval = false) {
     if (allowParallelRemoval && isStringArray(tables)) {
       await Promise.all(
-        tables.map(name => this.queryPromise('DROP TABLE IF EXISTS ' + name + ' CASCADE'))
+        tables.map(name => this.queryPromise('DROP TABLE IF EXISTS ' + pg.escapeIdentifier(/** @type {string} */ (name)) + ' CASCADE'))
       ).catch(cause => {
         throw new Error(`Failed to drop tables: ${tables}`, { cause });
       });
@@ -103,7 +120,7 @@ export class PgTestHelpers {
         await (
           Array.isArray(name)
             ? this.#removeTablesByName(name, true)
-            : this.queryPromise('DROP TABLE IF EXISTS ' + name + ' CASCADE').catch(cause => {
+            : this.queryPromise('DROP TABLE IF EXISTS ' + pg.escapeIdentifier(/** @type {string} */ (name)) + ' CASCADE').catch(cause => {
               throw new Error(`Failed to drop table: ${name}`, { cause });
             })
         );
@@ -132,6 +149,9 @@ export class PgTestHelpers {
       this.#lockClient = createAndLockConnection(this.#connectionString).then(result => {
         this.#lockClient = result;
         return result;
+      }, (err) => {
+        this.#lockClient = undefined;
+        throw err;
       });
     }
 
@@ -175,7 +195,16 @@ export class PgTestHelpers {
 
     await this.#ensureLocked();
 
-    return csvFromFolderToDb(this.#pool, this.#fixtureFolder, this.#tablesWithDependencies?.flat());
+    /** @type {import('./csv-folder-to-db.js').CsvFromFolderToDbOptions | undefined} */
+    let options;
+
+    if (this.#tableLoadOrder) {
+      options = { tableLoadOrder: this.#tableLoadOrder };
+    } else if (this.#tablesWithDependencies) {
+      options = { tablesWithDependencies: this.#tablesWithDependencies.flat() };
+    }
+
+    return csvFromFolderToDb(this.#pool, this.#fixtureFolder, options);
   }
 
   /** @returns {Promise<void>} */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@voxpelli/pg-utils",
-  "version": "3.0.1",
+  "version": "3.1.0",
   "description": " My personal database utils / helpers for Postgres",
   "homepage": "http://github.com/voxpelli/pg-utils",
   "repository": {