@devopsplaybook.io/common-utils 1.10.1 → 1.11.0-beta.24.d8f9aef

package/dist/src/DbUtilsNoTelemetry.js

@@ -6,6 +6,33 @@ exports.DbUtilsNoTelemetryExecSQL = DbUtilsNoTelemetryExecSQL;
 exports.DbUtilsNoTelemetryQuerySQL = DbUtilsNoTelemetryQuerySQL;
 const DbUtils_1 = require("./DbUtils");
 let logger;
+/**
+ * Compiled SQLite statements are cached per database handle: `better-sqlite3`
+ * has no internal cache and `prepare()` dominates the ingestion hot path.
+ */
+const PREPARED_STATEMENT_CACHE_MAX = 100;
+const statementCaches = new WeakMap();
+function prepareCached(db, sql) {
+    let cache = statementCaches.get(db);
+    if (!cache) {
+        cache = new Map();
+        statementCaches.set(db, cache);
+    }
+    let statement = cache.get(sql);
+    if (!statement) {
+        if (cache.size >= PREPARED_STATEMENT_CACHE_MAX) {
+            const oldest = cache.keys().next().value;
+            if (oldest !== undefined) {
+                cache.delete(oldest);
+            }
+        }
+        statement = db.prepare(sql);
+        cache.set(sql, statement);
+    }
+    return statement;
+}
+/** Maximum number of bound parameters per statement on each backend. */
+const PARAMS_PER_STATEMENT = { postgres: 65535, sqlite: 32766 };
 /**
  * Injects the OTel logger instance used by no-telemetry DB operations.
  * Must be called once at startup.
@@ -17,15 +44,43 @@ function DbUtilsNoTelemetrySetLogger(loggerIn) {
  * Execute a multi-row INSERT with a flat parameter array.
  * Builds: INSERT INTO <tableCols> VALUES (?,?...),(?,?...),...
  *
- * @returns Number of rows inserted.
+ * Large inputs are chunked so the statement never exceeds the backend's
+ * bound-parameter limit (65535 on Postgres, 32766 on SQLite).
+ *
+ * @returns Number of rows inserted (summed across chunks).
  */
 function DbUtilsNoTelemetryBatchInsert(tableCols, numCols, rows) {
     if (rows.length === 0)
         return 0;
+    const dbType = (0, DbUtils_1.DbUtilsGetType)();
+    const maxRowsPerChunk = Math.max(1, Math.floor(PARAMS_PER_STATEMENT[dbType] / Math.max(1, numCols)));
+    if (rows.length <= maxRowsPerChunk) {
+        return DbUtilsNoTelemetryExecSQL(buildBatchInsertSQL(tableCols, numCols, rows.length), rows.flat());
+    }
+    const chunks = [];
+    for (let i = 0; i < rows.length; i += maxRowsPerChunk) {
+        chunks.push(rows.slice(i, i + maxRowsPerChunk));
+    }
+    if (dbType === "postgres") {
+        return execChunksSequentially(chunks, tableCols, numCols);
+    }
+    let total = 0;
+    for (const chunk of chunks) {
+        total += DbUtilsNoTelemetryExecSQL(buildBatchInsertSQL(tableCols, numCols, chunk.length), chunk.flat());
+    }
+    return total;
+}
+function buildBatchInsertSQL(tableCols, numCols, rowCount) {
     const rowSQL = `(${Array.from({ length: numCols }, () => "?").join(",")})`;
-    const multiValues = Array.from({ length: rows.length }, () => rowSQL).join(",");
-    const sql = `INSERT ${tableCols} VALUES ${multiValues}`;
-    return DbUtilsNoTelemetryExecSQL(sql, rows.flat());
+    const multiValues = Array.from({ length: rowCount }, () => rowSQL).join(",");
+    return `INSERT ${tableCols} VALUES ${multiValues}`;
+}
+async function execChunksSequentially(chunks, tableCols, numCols) {
+    let total = 0;
+    for (const chunk of chunks) {
+        total += (await DbUtilsNoTelemetryExecSQL(buildBatchInsertSQL(tableCols, numCols, chunk.length), chunk.flat()));
+    }
+    return total;
 }
 /**
  * Execute a write SQL statement **without** creating an OTel span.
@@ -50,7 +105,8 @@ function DbUtilsNoTelemetryExecSQL(sql, params = []) {
         });
     }
     // SQLite (better-sqlite3) – synchronous
-    const stmt = (0, DbUtils_1.DbUtilsGetDatabase)().prepare(sql);
+    const db = (0, DbUtils_1.DbUtilsGetDatabase)();
+    const stmt = prepareCached(db, sql);
     const result = stmt.run(params);
     return result.changes;
 }
@@ -80,6 +136,6 @@ function DbUtilsNoTelemetryQuerySQL(sql, params = [], debug = false) {
         });
     }
     // SQLite (better-sqlite3) – synchronous
-    const stmt = (0, DbUtils_1.DbUtilsGetDatabase)().prepare(sql);
+    const stmt = prepareCached((0, DbUtils_1.DbUtilsGetDatabase)(), sql);
     return stmt.all(params);
 }

package/dist/src/PostgresDbUtils.d.ts

@@ -10,7 +10,24 @@ export interface PostgresDbConfig {
     DATABASE_POSTGRES_USER: string;
     DATABASE_POSTGRES_PASSWORD: string;
     DATABASE_POSTGRES_DATABASE: string;
+    /**
+     * Optional per-session `statement_timeout` (milliseconds) applied to every
+     * pool. Disabled when absent or 0 (backward-compatible default).
+     */
+    DATABASE_POSTGRES_STATEMENT_TIMEOUT_MS?: number;
+    /**
+     * Optional per-session `idle_in_transaction_session_timeout` (milliseconds)
+     * applied to every pool. Disabled when absent or 0.
+     */
+    DATABASE_POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT_MS?: number;
 }
+/**
+ * Named advisory-lock purposes. Each name maps to a fixed 32-bit key (the
+ * first four ASCII characters of the purpose) so that every replica of every
+ * service booting against the same database serialises the same operation.
+ */
+export type PostgresLockName = "migration" | "auth_token" | "users_bootstrap";
+export declare const POSTGRES_LOCK_KEYS: Record<PostgresLockName, number>;
 /**
  * Class-based PostgreSQL utility that manages a schema-specific pool (used
  * during migrations) and an optional shared runtime pool (used for
@@ -45,7 +62,10 @@ export declare class PostgresSchemaDbUtils {
      * @returns Number of rows changed.
      */
     execSQL(context: Span, sql: string, params?: any[], useSchemaPool?: boolean): Promise<number>;
-    /** Execute an entire SQL file (used for migrations). */
+    /**
+     * Execute an entire SQL file (used for migrations).
+     * Migration files must not contain their own transaction control statements.
+     */
     execSQLFile(context: Span, filename: string, useSchemaPool?: boolean): Promise<void>;
     /**
      * Execute a read SQL query with OTel tracing.
@@ -58,9 +78,6 @@ export declare class PostgresSchemaDbUtils {
     transaction(context: Span, callback: (client: any) => Promise<void>, useSchemaPool?: boolean): Promise<void>;
     /** Close both the schema pool and the runtime pool. */
     closeAll(): Promise<void>;
-    private execSQLForSchema;
-    private execSQLFileForSchema;
-    private querySQLForSchema;
 }
 /**
  * Injects the OTel tracer and logger instances used by all Postgres operations.
@@ -70,23 +87,37 @@ export declare function PostgresDbUtilsSetOTel(tracerIn: StandardTracer, loggerI
 /**
  * Creates the Postgres connection pool and applies pending migration files
  * from `sqlDir`.
+ *
+ * Migrations are applied under a session-level advisory lock (concurrently
+ * booting replicas serialise instead of double-applying) and each file plus
+ * its `db_version` row runs in its own transaction: a failing migration is
+ * rolled back and never recorded.
  */
 export declare function PostgresDbUtilsInit(context: Span, config: PostgresDbConfig, sqlDir: string): Promise<void>;
+/**
+ * Run a callback while holding a Postgres advisory lock, serialising the
+ * callback across every replica of every service booting against the same
+ * database (used for the `auth_token` and first-user bootstrap races).
+ */
+export declare function PostgresDbUtilsWithAdvisoryLock<T>(lock: PostgresLockName, callback: () => Promise<T>): Promise<T>;
 /** Returns the underlying `pg.Pool` instance. */
 export declare function PostgresDbUtilsGetPool(): Pool;
 /**
  * Execute a write SQL statement with OTel tracing.
  * @returns Number of rows changed.
  */
-export declare function PostgresDbUtilsExecSQL(context: Span, sql: string, params?: unknown[]): Promise<number>;
-/** Execute an entire SQL file (used for migrations). */
-export declare function PostgresDbUtilsExecSQLFile(context: Span, filename: string): Promise<void>;
+export declare function PostgresDbUtilsExecSQL(context: Span | undefined, sql: string, params?: unknown[]): Promise<number>;
+/**
+ * Execute an entire SQL file (used for migrations).
+ * Migration files must not contain their own transaction control statements.
+ */
+export declare function PostgresDbUtilsExecSQLFile(context: Span | undefined, filename: string): Promise<void>;
 /**
  * Execute a read SQL query with OTel tracing.
  * @returns Array of row objects.
  */
-export declare function PostgresDbUtilsQuerySQL(context: Span, sql: string, params?: unknown[], debug?: boolean): Promise<any[]>;
+export declare function PostgresDbUtilsQuerySQL(context: Span | undefined, sql: string, params?: unknown[], debug?: boolean): Promise<any[]>;
 /** Start a transaction. */
-export declare function PostgresDbUtilsTransactionStart(context: Span): Promise<void>;
+export declare function PostgresDbUtilsTransactionStart(context: Span | undefined): Promise<void>;
 /** Commit a transaction. */
-export declare function PostgresDbUtilsTransactionCommit(context: Span): Promise<void>;
+export declare function PostgresDbUtilsTransactionCommit(context: Span | undefined): Promise<void>;