@keetanetwork/anchor 0.0.58 → 0.0.60

package/lib/queue/drivers/queue_postgres.d.ts

@@ -1,18 +1,34 @@
 import type { KeetaAnchorQueueStorageDriver, KeetaAnchorQueueStorageDriverConstructor, KeetaAnchorQueueRequest, KeetaAnchorQueueRequestID, KeetaAnchorQueueEntry, KeetaAnchorQueueEntryExtra, KeetaAnchorQueueEntryAncillaryData, KeetaAnchorQueueStatus, KeetaAnchorQueueFilter } from '../index.ts';
 import type { JSONSerializable } from '../../utils/json.js';
 import type * as pg from 'pg';
+type KeetaAnchorQueueStorageDriverPostgresOptions = {
+    /**
+     * Function that returns a Postgres connection pool.
+     */
+    pool: () => Promise<pg.Pool>;
+    /**
+     * The prefix to use for the database tables. Defaults to 'queue',
+     * resulting in table names 'queue_entries', 'queue_idempotent_keys',
+     * and `queue_schema_version`.
+     */
+    tablePrefix?: string | undefined;
+};
 export default class KeetaAnchorQueueStorageDriverPostgres<QueueRequest extends JSONSerializable = JSONSerializable, QueueResult extends JSONSerializable = JSONSerializable> implements KeetaAnchorQueueStorageDriver<QueueRequest, QueueResult> {
     private readonly logger;
     private poolInternal;
+    private tablePrefix;
+    private tableNameEntries;
+    private tableNameSchemaVersion;
+    private tableNameIdempotentKeys;
     private dbInitializationPromise;
+    private serializationRetryCount;
+    private debugForceIndexScan;
     readonly name = "KeetaAnchorQueueStorageDriverPostgres";
     readonly id: string;
     readonly path: string[];
     private readonly pathStr;
     private toctouDelay;
-    constructor(options: NonNullable<ConstructorParameters<KeetaAnchorQueueStorageDriverConstructor<QueueRequest, QueueResult>>[0]> & {
-        pool: () => Promise<pg.Pool>;
-    });
+    constructor(options: NonNullable<ConstructorParameters<KeetaAnchorQueueStorageDriverConstructor<QueueRequest, QueueResult>>[0]> & KeetaAnchorQueueStorageDriverPostgresOptions);
     private initializeDBConnection;
     private methodLogger;
     private runWithRetry;
@@ -26,4 +42,5 @@ export default class KeetaAnchorQueueStorageDriverPostgres<QueueRequest extends
     destroy(): Promise<void>;
     [Symbol.asyncDispose](): Promise<void>;
 }
+export {};
 //# sourceMappingURL=queue_postgres.d.ts.map

package/lib/queue/drivers/queue_postgres.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"queue_postgres.d.ts","sourceRoot":"","sources":["../../../../src/lib/queue/drivers/queue_postgres.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,6BAA6B,EAC7B,wCAAwC,EACxC,uBAAuB,EACvB,yBAAyB,EACzB,qBAAqB,EACrB,0BAA0B,EAC1B,kCAAkC,EAClC,sBAAsB,EACtB,sBAAsB,EAEtB,MAAM,aAAa,CAAC;AAWrB,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAE5D,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI,CAAC;AAkB9B,MAAM,CAAC,OAAO,OAAO,qCAAqC,CAAC,YAAY,SAAS,gBAAgB,GAAG,gBAAgB,EAAE,WAAW,SAAS,gBAAgB,GAAG,gBAAgB,CAAE,YAAW,6BAA6B,CAAC,YAAY,EAAE,WAAW,CAAC;IAChP,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;IAC5C,OAAO,CAAC,YAAY,CAAyC;IAC7D,OAAO,CAAC,uBAAuB,CAAiC;IAEhE,QAAQ,CAAC,IAAI,2CAA2C;IACxD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAM;IAC7B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,WAAW,CAAgD;gBAEvD,OAAO,EAAE,WAAW,CAAC,qBAAqB,CAAC,wCAAwC,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;QAAE,IAAI,EAAE,MAAM,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;KAAE;YAWrJ,sBAAsB;IAyDpC,OAAO,CAAC,YAAY;YASN,YAAY;YA2CZ,eAAe;YAiBf,aAAa;IAoCrB,GAAG,CAAC,OAAO,EAAE,uBAAuB,CAAC,YAAY,CAAC,EAAE,IAAI,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,yBAAyB,CAAC;IA6D1H,SAAS,CAAC,EAAE,EAAE,yBAAyB,EAAE,MAAM,EAAE,sBAAsB,EAAE,SAAS,CAAC,EAAE,kCAAkC,CAAC,WAAW,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAyDpJ,GAAG,CAAC,EAAE,EAAE,yBAAyB,GAAG,OAAO,CAAC,qBAAqB,CAAC,YAAY,EAAE,WAAW,CAAC,GAAG,IAAI,CAAC;IA8CpG,KAAK,CAAC,MAAM,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,qBAAqB,CAAC,YAAY,EAAE,WAAW,CAAC,EAAE,CAAC;IAuEnG,SAAS,CAAC,IAAI,EAAE,MAAM,GAAI,OAAO,CAAC,6BAA6B,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;IAiB3F,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAMxB,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC;CAwB5C"}
+{"version":3,"file":"queue_postgres.d.ts","sourceRoot":"","sources":["../../../../src/lib/queue/drivers/queue_postgres.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACX,6BAA6B,EAC7B,wCAAwC,EACxC,uBAAuB,EACvB,yBAAyB,EACzB,qBAAqB,EACrB,0BAA0B,EAC1B,kCAAkC,EAClC,sBAAsB,EACtB,sBAAsB,EAEtB,MAAM,aAAa,CAAC;AAWrB,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAE5D,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI,CAAC;AAkB9B,KAAK,4CAA4C,GAAG;IACnD;;OAEG;IACH,IAAI,EAAE,MAAM,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;IAE7B;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACjC,CAAC;AAEF,MAAM,CAAC,OAAO,OAAO,qCAAqC,CAAC,YAAY,SAAS,gBAAgB,GAAG,gBAAgB,EAAE,WAAW,SAAS,gBAAgB,GAAG,gBAAgB,CAAE,YAAW,6BAA6B,CAAC,YAAY,EAAE,WAAW,CAAC;IAChP,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;IAC5C,OAAO,CAAC,YAAY,CAAyC;IAC7D,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,sBAAsB,CAAS;IACvC,OAAO,CAAC,uBAAuB,CAAS;IACxC,OAAO,CAAC,uBAAuB,CAAiC;IAChE,OAAO,CAAC,uBAAuB,CAAK;IACpC,OAAO,CAAC,mBAAmB,CAAS;IAEpC,QAAQ,CAAC,IAAI,2CAA2C;IACxD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAM;IAC7B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,WAAW,CAAgD;gBAEvD,OAAO,EAAE,WAAW,CAAC,qBAAqB,CAAC,wCAAwC,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,4CAA4C;YAgBhK,sBAAsB;IAgJpC,OAAO,CAAC,YAAY;YASN,YAAY;YA8CZ,eAAe;YAiBf,aAAa;IA0CrB,GAAG,CAAC,OAAO,EAAE,uBAAuB,CAAC,YAAY,CAAC,EAAE,IAAI,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,yBAAyB,CAAC;IA6D1H,SAAS,CAAC,EAAE,EAAE,yBAAyB,EAAE,MAAM,EAAE,sBAAsB,EAAE,SAAS,CAAC,EAAE,kCAAkC,CAAC,WAAW,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAyDpJ,GAAG,CAAC,EAAE,EAAE,yBAAyB,GAAG,OAAO,CAAC,qBAAqB,CAAC,YAAY,EAAE,WAAW,CAAC,GAAG,IAAI,CAAC;IA8CpG,KAAK,CAAC,MAAM,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,qBAAqB,CAAC,YAAY,EAAE,WAAW,CAAC,EAAE,CAAC;IA2EnG,SAAS,CAAC,IAAI,EAAE,MAAM,GAAI,OAAO,CAAC,6BAA6B,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;IAkB3F,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAMxB,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC;CAwC5C"}

package/lib/queue/drivers/queue_postgres.js

@@ -4,7 +4,13 @@ import { asleep } from '../../utils/asleep.js';
 export default class KeetaAnchorQueueStorageDriverPostgres {
     logger;
     poolInternal = null;
+    tablePrefix;
+    tableNameEntries;
+    tableNameSchemaVersion;
+    tableNameIdempotentKeys;
     dbInitializationPromise = null;
+    serializationRetryCount = 0;
+    debugForceIndexScan = false;
     name = 'KeetaAnchorQueueStorageDriverPostgres';
     id;
     path = [];
@@ -13,10 +19,14 @@ export default class KeetaAnchorQueueStorageDriverPostgres {
     constructor(options) {
         this.id = options?.id ?? crypto.randomUUID();
         this.logger = options?.logger;
+        this.tablePrefix = options?.tablePrefix;
         this.poolInternal = options.pool;
         this.path = options.path ?? [];
         this.pathStr = ['root', ...this.path].join('.');
         Object.freeze(this.path);
+        this.tableNameEntries = `${this.tablePrefix ?? 'queue'}_entries`;
+        this.tableNameIdempotentKeys = `${this.tablePrefix ?? 'queue'}_idempotent_keys`;
+        this.tableNameSchemaVersion = `${this.tablePrefix ?? 'queue'}_schema_version`;
         this.methodLogger('new')?.debug('Initialized Postgres queue storage driver');
     }
     async initializeDBConnection(pool) {
@@ -27,35 +37,115 @@ export default class KeetaAnchorQueueStorageDriverPostgres {
             return (pool);
         }
         this.dbInitializationPromise = (async () => {
-            logger?.debug('Initializing DB schema for queue storage driver');
+            logger?.debug('Initializing DB schema for queue storage driver on tables', [this.tableNameEntries, this.tableNameIdempotentKeys]);
             const client = await pool.connect();
             try {
-                await client.query(`
-					CREATE TABLE IF NOT EXISTS queue_entries (
-						id TEXT NOT NULL,
-						path TEXT NOT NULL,
-						request TEXT NOT NULL,
-						output TEXT,
-						last_error TEXT,
-						status TEXT NOT NULL,
-						created BIGINT NOT NULL,
-						updated BIGINT NOT NULL,
-						worker BIGINT,
-						failures INTEGER NOT NULL DEFAULT 0,
-						PRIMARY KEY (id, path)
-					)`);
-                await client.query(`
-					CREATE TABLE IF NOT EXISTS queue_idempotent_keys (
-						entry_id TEXT NOT NULL,
-						idempotent_id TEXT NOT NULL,
-						path TEXT NOT NULL,
-						UNIQUE (idempotent_id, path),
-						PRIMARY KEY (entry_id, idempotent_id, path),
-						FOREIGN KEY (entry_id, path) REFERENCES queue_entries(id, path)
-					)`);
-                await client.query('CREATE INDEX IF NOT EXISTS idx_queue_entries_status ON queue_entries(status)');
-                await client.query('CREATE INDEX IF NOT EXISTS idx_queue_entries_updated ON queue_entries(updated)');
-                await client.query('CREATE INDEX IF NOT EXISTS idx_queue_idempotent_keys_idempotent_id ON queue_idempotent_keys(idempotent_id)');
+                /*
+                 * Random lock key (32-bit integer), to ensure
+                 * that multiple instances of this driver
+                 * (potentially across different
+                 * instances/process) will run the migration
+                 * sequentially to avoid multiple concurrent
+                 * migrations from competing with each other
+                 * and causing delays
+                 */
+                const lockId = 0x24995E48;
+                logger?.debug('Acquiring advisory lock for schema migration');
+                await client.query('SELECT pg_advisory_lock($1)', [lockId]);
+                try {
+                    // Create schema version table if it doesn't exist
+                    await client.query(`
+						CREATE TABLE IF NOT EXISTS ${this.tableNameSchemaVersion} (
+							version INTEGER NOT NULL,
+							applied_at BIGINT NOT NULL,
+							PRIMARY KEY (version)
+						)`);
+                    // Check current schema version
+                    const versionResult = await client.query(`SELECT MAX(version) as version FROM ${this.tableNameSchemaVersion}`);
+                    const currentVersion = versionResult.rows[0]?.version ?? 0;
+                    logger?.debug(`Current queue schema version: ${currentVersion}`);
+                    // Version 1: Initial schema
+                    if (currentVersion < 1) {
+                        logger?.debug('Applying schema version 1: Initial tables and indexes');
+                        await client.query('BEGIN');
+                        try {
+                            await client.query(`
+								CREATE TABLE IF NOT EXISTS ${this.tableNameEntries} (
+									id TEXT NOT NULL,
+									path TEXT NOT NULL,
+									request TEXT NOT NULL,
+									output TEXT,
+									last_error TEXT,
+									status TEXT NOT NULL,
+									created BIGINT NOT NULL,
+									updated BIGINT NOT NULL,
+									worker BIGINT,
+									failures INTEGER NOT NULL DEFAULT 0,
+									PRIMARY KEY (id, path)
+								)`);
+                            await client.query(`
+								CREATE TABLE IF NOT EXISTS ${this.tableNameIdempotentKeys} (
+									entry_id TEXT NOT NULL,
+									idempotent_id TEXT NOT NULL,
+									path TEXT NOT NULL,
+									UNIQUE (idempotent_id, path),
+									PRIMARY KEY (entry_id, idempotent_id, path),
+									FOREIGN KEY (entry_id, path) REFERENCES ${this.tableNameEntries}(id, path)
+								)`);
+                            // Old single-column indexes (for pre-version-2 schemas)
+                            await client.query(`CREATE INDEX IF NOT EXISTS idx_${this.tableNameEntries}_status ON ${this.tableNameEntries}(status)`);
+                            await client.query(`CREATE INDEX IF NOT EXISTS idx_${this.tableNameEntries}_updated ON ${this.tableNameEntries}(updated)`);
+                            await client.query(`CREATE INDEX IF NOT EXISTS idx_${this.tableNameIdempotentKeys}_idempotent_id ON ${this.tableNameIdempotentKeys}(idempotent_id)`);
+                            await client.query(`INSERT INTO ${this.tableNameSchemaVersion} (version, applied_at) VALUES (1, $1)`, [Date.now()]);
+                            await client.query('COMMIT');
+                            logger?.debug('Applied schema version 1');
+                        }
+                        catch (error) {
+                            await client.query('ROLLBACK');
+                            throw (error);
+                        }
+                    }
+                    // Version 2: Partition-aware composite indexes
+                    if (currentVersion < 2) {
+                        logger?.debug('Applying schema version 2: Partition-aware composite indexes');
+                        // Create new partition-aware indexes (CONCURRENTLY must be outside transaction)
+                        logger?.debug('Creating partition-aware indexes...');
+                        await client.query(`CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_${this.tableNameEntries}_path_status ON ${this.tableNameEntries}(path, status)`);
+                        await client.query(`CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_${this.tableNameEntries}_path_updated ON ${this.tableNameEntries}(path, updated)`);
+                        await client.query(`CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_${this.tableNameEntries}_path_status_updated ON ${this.tableNameEntries}(path, status, updated)`);
+                        await client.query(`CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_${this.tableNameIdempotentKeys}_path_idempotent_id ON ${this.tableNameIdempotentKeys}(path, idempotent_id)`);
+                        // Now drop old indexes and record version
+                        await client.query('BEGIN');
+                        try {
+                            // Drop old indexes that are now redundant (these will fail gracefully if indexes don't exist)
+                            logger?.debug('Dropping old single-column indexes...');
+                            await client.query(`DROP INDEX IF EXISTS idx_${this.tableNameEntries}_status`);
+                            await client.query(`DROP INDEX IF EXISTS idx_${this.tableNameEntries}_updated`);
+                            await client.query(`DROP INDEX IF EXISTS idx_${this.tableNameIdempotentKeys}_idempotent_id`);
+                            await client.query(`INSERT INTO ${this.tableNameSchemaVersion} (version, applied_at) VALUES (2, $1)`, [Date.now()]);
+                            await client.query('COMMIT');
+                            logger?.debug('Applied schema version 2');
+                        }
+                        catch (error) {
+                            await client.query('ROLLBACK');
+                            throw (error);
+                        }
+                    }
+                    logger?.debug('Schema is up to date');
+                }
+                finally {
+                    // Always release the advisory lock
+                    // Note: Advisory locks are session-based and auto-release on disconnect,
+                    // but we explicitly unlock for clarity and to avoid holding locks longer than needed
+                    try {
+                        logger?.debug('Releasing advisory lock');
+                        await client.query('SELECT pg_advisory_unlock($1)', [lockId]);
+                    }
+                    catch (unlockError) {
+                        // Log but don't throw - the lock will be auto-released when connection closes
+                        logger?.debug('Failed to explicitly release advisory lock (will auto-release on disconnect):', unlockError);
+                    }
+                }
             }
             finally {
                 client.release();
@@ -95,6 +185,8 @@ export default class KeetaAnchorQueueStorageDriverPostgres {
                     const errorCode = 'code' in error ? error.code : null;
                     if (errorCode === '40001' || errorCode === '40P01') {
                         logger?.debug('Serialization failure or deadlock detected');
+                        // Track serialization retries for instrumentation
+                        this.serializationRetryCount++;
                         const minBackoff = 100;
                         const maxBackoff = 30_000;
                         const backoffIntervalSize = Math.min(maxBackoff - minBackoff, (retry + 50) ** 2);
@@ -125,12 +217,16 @@ export default class KeetaAnchorQueueStorageDriverPostgres {
     async dbTransaction(className, fn) {
         const pool = await this.newDBConnection();
         const logger = this.methodLogger(className);
+        const debugForceIndexScan = this.debugForceIndexScan;
         const result = await this.runWithRetry(async function () {
             const client = await pool.connect();
             try {
                 logger?.debug('Starting DB transaction');
                 await client.query('BEGIN TRANSACTION ISOLATION LEVEL SERIALIZABLE');
                 logger?.debug('DB transaction started');
+                if (debugForceIndexScan) {
+                    await client.query('SET LOCAL enable_seqscan TO off');
+                }
                 const retval = await fn(client, logger);
                 logger?.debug('Committing DB transaction');
                 await client.query('COMMIT');
@@ -158,7 +254,7 @@ export default class KeetaAnchorQueueStorageDriverPostgres {
         return (await this.dbTransaction('add', async (client, logger) => {
             let entryID = ConvertStringToRequestID(info?.id);
             if (entryID) {
-                const existingEntry = await client.query('SELECT id FROM queue_entries WHERE id = $1 AND path = $2', [entryID, this.pathStr]);
+                const existingEntry = await client.query(`SELECT id FROM ${this.tableNameEntries} WHERE id = $1 AND path = $2`, [entryID, this.pathStr]);
                 if (existingEntry.rows.length > 0) {
                     logger?.debug(`Request with id ${String(entryID)} already exists, ignoring`);
                     return (entryID);
@@ -169,7 +265,7 @@ export default class KeetaAnchorQueueStorageDriverPostgres {
             if (idempotentIDs) {
                 const matchingIdempotentEntries = new Set();
                 for (const idempotentID of idempotentIDs) {
-                    const idempotentEntryExists = await client.query('SELECT idempotent_id FROM queue_idempotent_keys WHERE idempotent_id = $1 AND path = $2', [idempotentID, this.pathStr]);
+                    const idempotentEntryExists = await client.query(`SELECT idempotent_id FROM ${this.tableNameIdempotentKeys} WHERE idempotent_id = $1 AND path = $2`, [idempotentID, this.pathStr]);
                     if (idempotentEntryExists.rows.length > 0) {
                         matchingIdempotentEntries.add(idempotentID);
                     }
@@ -187,11 +283,11 @@ export default class KeetaAnchorQueueStorageDriverPostgres {
              * The status to use for the new entry
              */
             const status = info?.status ?? 'pending';
-            await client.query(`INSERT INTO queue_entries (id, path, request, output, last_error, status, created, updated, worker, failures)
+            await client.query(`INSERT INTO ${this.tableNameEntries} (id, path, request, output, last_error, status, created, updated, worker, failures)
 				 VALUES ($1, $2, $3, NULL, NULL, $4, $5, $6, NULL, 0)`, [entryID, this.pathStr, requestJSON, status, currentTime, currentTime]);
             if (idempotentIDs && idempotentIDs.size > 0) {
                 for (const idempotentID of idempotentIDs) {
-                    await client.query('INSERT INTO queue_idempotent_keys (entry_id, path, idempotent_id) VALUES ($1, $2, $3)', [entryID, this.pathStr, idempotentID]);
+                    await client.query(`INSERT INTO ${this.tableNameIdempotentKeys} (entry_id, path, idempotent_id) VALUES ($1, $2, $3)`, [entryID, this.pathStr, idempotentID]);
                 }
             }
             return (entryID);
@@ -200,7 +296,7 @@ export default class KeetaAnchorQueueStorageDriverPostgres {
     async setStatus(id, status, ancillary) {
         const { oldStatus } = ancillary ?? {};
         return (await this.dbTransaction('setStatus', async (client, logger) => {
-            const existingEntry = await client.query('SELECT status, failures, last_error, output FROM queue_entries WHERE id = $1 AND path = $2', [id, this.pathStr]);
+            const existingEntry = await client.query(`SELECT status, failures, last_error, output FROM ${this.tableNameEntries} WHERE id = $1 AND path = $2 FOR UPDATE`, [id, this.pathStr]);
             if (existingEntry.rows.length === 0) {
                 throw (new Error(`Request with ID ${String(id)} not found`));
             }
@@ -218,13 +314,13 @@ export default class KeetaAnchorQueueStorageDriverPostgres {
             let updateQuery;
             let updateParams;
             if (oldStatus) {
-                updateQuery = `UPDATE queue_entries
+                updateQuery = `UPDATE ${this.tableNameEntries}
 				               SET status = $1, updated = $2, worker = $3, failures = $4, last_error = $5, output = $6
 				               WHERE id = $7 AND path = $8 AND status = $9`;
                 updateParams = [status, currentTime, workerValue, newFailures, newLastError, newOutput, id, this.pathStr, oldStatus];
             }
             else {
-                updateQuery = `UPDATE queue_entries
+                updateQuery = `UPDATE ${this.tableNameEntries}
 				               SET status = $1, updated = $2, worker = $3, failures = $4, last_error = $5, output = $6
 				               WHERE id = $7 AND path = $8`;
                 updateParams = [status, currentTime, workerValue, newFailures, newLastError, newOutput, id, this.pathStr];
@@ -232,7 +328,7 @@ export default class KeetaAnchorQueueStorageDriverPostgres {
             const result = await client.query(updateQuery, updateParams);
             await this.toctouDelay?.();
             if (oldStatus && result.rowCount === 0) {
-                const currentEntry = await client.query('SELECT status FROM queue_entries WHERE id = $1 AND path = $2', [id, this.pathStr]);
+                const currentEntry = await client.query(`SELECT status FROM ${this.tableNameEntries} WHERE id = $1 AND path = $2`, [id, this.pathStr]);
                 const currentStatus = currentEntry.rows[0]?.status;
                 if (currentEntry.rows.length > 0) {
                     if (currentStatus === undefined) {
@@ -249,7 +345,7 @@ export default class KeetaAnchorQueueStorageDriverPostgres {
     async get(id) {
         return (await this.dbTransaction('get', async (client) => {
             const row = await client.query(`SELECT id, request, output, last_error, status, created, updated, worker, failures
-				 FROM queue_entries WHERE id = $1 AND path = $2`, [id, this.pathStr]);
+				 FROM ${this.tableNameEntries} WHERE id = $1 AND path = $2`, [id, this.pathStr]);
             if (row.rows.length === 0) {
                 return (null);
             }
@@ -257,7 +353,7 @@ export default class KeetaAnchorQueueStorageDriverPostgres {
             if (!entry) {
                 return (null);
             }
-            const idempotentRows = await client.query('SELECT idempotent_id FROM queue_idempotent_keys WHERE entry_id = $1 AND path = $2', [id, this.pathStr]);
+            const idempotentRows = await client.query(`SELECT idempotent_id FROM ${this.tableNameIdempotentKeys} WHERE entry_id = $1 AND path = $2`, [id, this.pathStr]);
             const idempotentKeys = idempotentRows.rows.length > 0
                 ? new Set(idempotentRows.rows.map(function (idempotentRow) {
                     return (ConvertStringToRequestID(idempotentRow.idempotent_id));
@@ -296,10 +392,13 @@ export default class KeetaAnchorQueueStorageDriverPostgres {
                 conditions.push(`updated < $${paramIndex++}`);
                 params.push(filter.updatedBefore.getTime());
             }
-            let query = 'SELECT id, request, output, last_error, status, created, updated, worker, failures FROM queue_entries';
+            let query = `SELECT id, request, output, last_error, status, created, updated, worker, failures FROM ${this.tableNameEntries}`;
             if (conditions.length > 0) {
                 query += ' WHERE ' + conditions.join(' AND ');
             }
+            // Use random ordering to prevent multiple workers from contending for the same rows
+            // This spreads the load when multiple workers query simultaneously
+            query += ' ORDER BY RANDOM()';
             if (filter?.limit !== undefined) {
                 query += ` LIMIT $${paramIndex++}`;
                 params.push(filter.limit);
@@ -307,7 +406,7 @@ export default class KeetaAnchorQueueStorageDriverPostgres {
             const rows = await client.query(query, params);
             const entries = [];
             for (const row of rows.rows) {
-                const idempotentRows = await client.query('SELECT idempotent_id FROM queue_idempotent_keys WHERE entry_id = $1 AND path = $2', [row.id, this.pathStr]);
+                const idempotentRows = await client.query(`SELECT idempotent_id FROM ${this.tableNameIdempotentKeys} WHERE entry_id = $1 AND path = $2`, [row.id, this.pathStr]);
                 const idempotentKeys = idempotentRows.rows.length > 0
                     ? new Set(idempotentRows.rows.map(function (idempotentRow) {
                         return (ConvertStringToRequestID(idempotentRow.idempotent_id));
@@ -342,7 +441,8 @@ export default class KeetaAnchorQueueStorageDriverPostgres {
             id: `${this.id}::${path}`,
             logger: this.logger,
             pool: this.poolInternal,
-            path: [...this.path, path]
+            path: [...this.path, path],
+            tablePrefix: this.tablePrefix
         });
         return (retval);
     }
@@ -366,6 +466,18 @@ export default class KeetaAnchorQueueStorageDriverPostgres {
             },
             unsetToctouDelay: () => {
                 this.toctouDelay = undefined;
+            },
+            getSerializationRetryCount: () => {
+                return (this.serializationRetryCount);
+            },
+            resetSerializationRetryCount: () => {
+                this.serializationRetryCount = 0;
+            },
+            enableDebugForceIndexScan: () => {
+                this.debugForceIndexScan = true;
+            },
+            disableDebugForceIndexScan: () => {
+                this.debugForceIndexScan = false;
             }
         });
     }

package/lib/queue/drivers/queue_postgres.js.map

@@ -1 +1 @@
-{"version":3,"file":"queue_postgres.js","sourceRoot":"","sources":["../../../../src/lib/queue/drivers/queue_postgres.ts"],"names":[],"mappings":"AAYA,OAAO,EACN,YAAY,EACZ,mBAAmB,EACnB,wBAAwB,EACxB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAuB/C,MAAM,CAAC,OAAO,OAAO,qCAAqC;IACxC,MAAM,CAAqB;IACpC,YAAY,GAAoC,IAAI,CAAC;IACrD,uBAAuB,GAA4B,IAAI,CAAC;IAEvD,IAAI,GAAG,uCAAuC,CAAC;IAC/C,EAAE,CAAS;IACX,IAAI,GAAa,EAAE,CAAC;IACZ,OAAO,CAAS;IACzB,WAAW,GAAsC,SAAS,CAAC;IAEnE,YAAY,OAAuJ;QAClK,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,EAAE,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAC7C,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,CAAA;QAC7B,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;QACjC,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEzB,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC9E,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAAC,IAAa;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,wBAAwB,CAAC,CAAC;QAE3D,IAAI,IAAI,CAAC,uBAAuB,KAAK,IAAI,EAAE,CAAC;YAC3C,MAAM,EAAE,KAAK,CAAC,qFAAqF,CAAC,CAAC;YAErG,MAAM,IAAI,CAAC,uBAAuB,CAAC;YACnC,OAAM,CAAC,IAAI,CAAC,CAAC;QACd,CAAC;QAED,IAAI,CAAC,uBAAuB,GAAG,CAAC,KAAK,IAAI,EAAE;YAC1C,MAAM,EAAE,KAAK,CAAC,iDAAiD,CAAC,CAAC;YAEjE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACpC,IAAI,CAAC;gBACJ,MAAM,MAAM,CAAC,KAAK,CAAC;;;;;;;;;;;;;OAahB,CAAC,CAAC;gBAEL,MAAM,MAAM,CAAC,KAAK,CAAC;;;;;;;;OAQhB,CAAC,CAAC;gBAEL,MAAM,MAAM,CAAC,KAAK,CAAC,8EAA8E,CAAC,CAAC;gBACnG,MAAM,MAAM,CAAC,KAAK,CAAC,gFAAgF,CAAC,CAAC;gBACrG,MAAM,MAAM,CAAC,KAAK,CAAC,4GAA4G,CAAC,CAAC;YAClI,CAAC;oBAAS,CAAC;gBACV,MAAM,CAAC,OAAO,EAAE,CAAC;YAClB,CAAC;YAED,MAAM,EAAE,KAAK,CAAC,6DAA6D,CAAC,CAAC;YAE7E,OAAM,CAAC,IAAI,CAAC,CAAC;QACd,CAAC,CAAC,EAAE,CAAC;QAEL,MAAM,IAAI,CAAC,uBAAuB,CAAC;QAEnC,OAAM,CAAC,IAAI,CAAC,CAAC;IACd,CAAC;IAEO,YAAY,CAAC,MAAc;QAClC,OAAM,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE;YAChC,KAAK,EAAE,uCAAuC;YAC9C,IAAI,EAAE,yCAAyC;YAC/C,MAAM,EAAE,MAAM;YACd,UAAU,EAAE,IAAI,CAAC,EAAE;SACnB,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,YAAY,CAAI,EAAoB;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC;QAEjD,IAAI,SAAkB,CAAC;QACvB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC;YACzC,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;gBAChC,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,EAAE,KAAK,CAAC,kEAAkE,CAAC,CAAC;gBAE7G,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;oBAC7B,+DAA+D;oBAC/D,MAAK,CAAC,SAAS,CAAC,CAAC;gBAClB,CAAC;gBACD,MAAK,CAAC,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC,CAAC;YACjE,CAAC;YAED,IAAI,CAAC;gBACJ,OAAM,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACpB,CAAC;YAAC,OAAO,KAAc,EAAE,CAAC;gBACzB,SAAS,GAAG,KAAK,CAAC;gBAElB,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;oBAC5B,MAAM,SAAS,GAAG,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;oBACtD,IAAI,SAAS,KAAK,OAAO,IAAI,SAAS,KAAK,OAAO,EAAE,CAAC;wBACpD,MAAM,EAAE,KAAK,CAAC,4CAA4C,CAAC,CAAC;wBAE5D,MAAM,UAAU,GAAG,GAAG,CAAC;wBACvB,MAAM,UAAU,GAAG,MAAM,CAAC;wBAC1B,MAAM,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,UAAU,EAAE,CAAC,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;wBACjF,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,mBAAmB,CAAC,CAAC,GAAG,UAAU,CAAC;wBAE/E,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,EAAE,KAAK,CAAC,4BAA4B,OAAO,cAAc,KAAK,QAAQ,EAAE,IAAI,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC;wBAC5H,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC;wBAEtB,SAAS;oBACV,CAAC;gBACF,CAAC;gBAED,MAAK,CAAC,KAAK,CAAC,CAAC;YACd,CAAC;QACF,CAAC;QACD,MAAK,CAAC,SAAS,CAAC,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,eAAe;QAC5B,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;YAChC,MAAK,CAAC,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC,CAAC;QAC1D,CAAC;QAED,OAAM,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE;YACzC,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;gBAChC,MAAK,CAAC,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC,CAAC;YAC1D,CAAC;YAED,IAAI,CAAC,YAAY,CAAC,iBAAiB,CAAC,EAAE,KAAK,CAAC,iBAAiB,CAAC,CAAC;YAC/D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAEvC,OAAM,CAAC,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,aAAa,CAAI,SAAiB,EAAE,EAAqE;QACtH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAE5C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK;YAC3C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YAEpC,IAAI,CAAC;gBACJ,MAAM,EAAE,KAAK,CAAC,yBAAyB,CAAC,CAAC;gBACzC,MAAM,MAAM,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;gBACrE,MAAM,EAAE,KAAK,CAAC,wBAAwB,CAAC,CAAC;gBAExC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBAExC,MAAM,EAAE,KAAK,CAAC,2BAA2B,CAAC,CAAC;gBAC3C,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBAC7B,MAAM,EAAE,KAAK,CAAC,0BAA0B,CAAC,CAAC;gBAE1C,OAAM,CAAC,MAAM,CAAC,CAAC;YAChB,CAAC;YAAC,OAAO,KAAc,EAAE,CAAC;gBACzB,IAAI,CAAC;oBACJ,MAAM,EAAE,KAAK,CAAC,2CAA2C,EAAE,KAAK,CAAC,CAAC;oBAClE,MAAM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;oBAC/B,MAAM,EAAE,KAAK,CAAC,4BAA4B,CAAC,CAAC;gBAC7C,CAAC;gBAAC,MAAM,CAAC;oBACR,MAAM,EAAE,KAAK,CAAC,sCAAsC,CAAC,CAAC;gBACvD,CAAC;gBACD,MAAK,CAAC,KAAK,CAAC,CAAC;YACd,CAAC;oBAAS,CAAC;gBACV,MAAM,CAAC,OAAO,EAAE,CAAC;YAClB,CAAC;QACF,CAAC,CAAC,CAAC;QAEH,OAAM,CAAC,MAAM,CAAC,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,OAA8C,EAAE,IAAiC;QAC1F,OAAM,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAsC,EAAE;YACnG,IAAI,OAAO,GAAG,wBAAwB,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACjD,IAAI,OAAO,EAAE,CAAC;gBACb,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,KAAK,CAAiB,0DAA0D,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC9I,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACnC,MAAM,EAAE,KAAK,CAAC,mBAAmB,MAAM,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;oBAC7E,OAAM,CAAC,OAAO,CAAC,CAAC;gBACjB,CAAC;gBAED,MAAM,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YAC5B,CAAC;YAED,MAAM,aAAa,GAAG,IAAI,EAAE,cAAc,CAAC;YAC3C,IAAI,aAAa,EAAE,CAAC;gBACnB,MAAM,yBAAyB,GAAG,IAAI,GAAG,EAA6B,CAAC;gBACvE,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;oBAC1C,MAAM,qBAAqB,GAAG,MAAM,MAAM,CAAC,KAAK,CAC/C,wFAAwF,EACxF,CAAC,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,CAC5B,CAAC;oBACF,IAAI,qBAAqB,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC3C,yBAAyB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;oBAC7C,CAAC;gBACF,CAAC;gBAED,IAAI,yBAAyB,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;oBAC1C,MAAK,CAAC,IAAI,MAAM,CAAC,qBAAqB,CAAC,2DAA2D,EAAE,yBAAyB,CAAC,CAAC,CAAC;gBACjI,CAAC;gBAED,MAAM,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YAC5B,CAAC;YAED,OAAO,KAAK,wBAAwB,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;YAE1D,MAAM,EAAE,KAAK,CAAC,6BAA6B,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAE9D,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC/B,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YAE5C;;eAEG;YACH,MAAM,MAAM,GAAG,IAAI,EAAE,MAAM,IAAI,SAAS,CAAC;YAEzC,MAAM,MAAM,CAAC,KAAK,CACjB;0DACsD,EACtD,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,CAAC,CACtE,CAAC;YAEF,IAAI,aAAa,IAAI,aAAa,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBAC7C,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;oBAC1C,MAAM,MAAM,CAAC,KAAK,CAAC,uFAAuF,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;gBACpJ,CAAC;YACF,CAAC;YAED,OAAM,CAAC,OAAO,CAAC,CAAC;QACjB,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,EAA6B,EAAE,MAA8B,EAAE,SAA2D;QACzI,MAAM,EAAE,SAAS,EAAE,GAAG,SAAS,IAAI,EAAE,CAAC;QAEtC,OAAM,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAiB,EAAE;YACpF,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,KAAK,CAAyG,4FAA4F,EAAE,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;YACnQ,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrC,MAAK,CAAC,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC;YAC7D,CAAC;YAED,MAAM,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YAE3B,MAAM,YAAY,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC3C,IAAI,CAAC,YAAY,EAAE,CAAC;gBACnB,MAAK,CAAC,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC;YAC7D,CAAC;YAED,MAAM,QAAQ,GAAG,mBAAmB,CAAc,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YAC/F,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YAC/C,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC;YACpC,MAAM,WAAW,GAAG,QAAQ,CAAC,QAAQ,IAAI,YAAY,CAAC,QAAQ,CAAC;YAC/D,MAAM,YAAY,GAAG,QAAQ,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,UAAU,CAAC;YACrG,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC;YAExG,IAAI,WAAmB,CAAC;YACxB,IAAI,YAAoE,CAAC;YAEzE,IAAI,SAAS,EAAE,CAAC;gBACf,WAAW,GAAG;;+DAE6C,CAAC;gBAC5D,YAAY,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,EAAE,EAAE,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;YACtH,CAAC;iBAAM,CAAC;gBACP,WAAW,GAAG;;+CAE6B,CAAC;gBAC5C,YAAY,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YAC3G,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;YAE7D,MAAM,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YAE3B,IAAI,SAAS,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;gBACxC,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,KAAK,CAAqC,8DAA8D,EAAE,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;gBAChK,MAAM,aAAa,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC;gBACnD,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAClC,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;wBACjC,MAAK,CAAC,IAAI,KAAK,CAAC,yEAAyE,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;oBACzG,CAAC;oBACD,MAAK,CAAC,IAAI,MAAM,CAAC,2BAA2B,CAAC,EAAE,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC,CAAC;gBAC7E,CAAC;qBAAM,CAAC;oBACP,MAAK,CAAC,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC;gBAC7D,CAAC;YACF,CAAC;QACF,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EAA6B;QACtC,OAAM,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAoE,EAAE;YACzH,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,KAAK,CAC7B;oDACgD,EAChD,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAClB,CAAC;YAEF,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3B,OAAM,CAAC,IAAI,CAAC,CAAC;YACd,CAAC;YAED,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC1B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACZ,OAAM,CAAC,IAAI,CAAC,CAAC;YACd,CAAC;YAED,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,KAAK,CACxC,mFAAmF,EACnF,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAClB,CAAC;YAEF,MAAM,cAAc,GAAG,cAAc,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;gBACpD,CAAC,CAAC,IAAI,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,UAAS,aAA4B;oBACtE,OAAM,CAAC,wBAAwB,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC;gBAC/D,CAAC,CAAC,CAAC;gBACH,CAAC,CAAC,SAAS,CAAC;YAEb,OAAM,CAAC;gBACN,EAAE,EAAE,wBAAwB,CAAC,KAAK,CAAC,EAAE,CAAC;gBACtC,yEAAyE;gBACzE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAiB;gBAClD,yEAAyE;gBACzE,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAgB,CAAC,CAAC,CAAC,IAAI;gBACrE,SAAS,EAAE,KAAK,CAAC,UAAU;gBAC3B,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,OAAO,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBACxC,OAAO,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBACxC,yEAAyE;gBACzE,MAAM,EAAE,KAAK,CAAC,MAAoD;gBAClE,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,cAAc,EAAE,cAAc;aAC9B,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,MAA+B;QAC1C,OAAM,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAA+D,EAAE;YAC9H,MAAM,EAAE,KAAK,CAAC,0BAA0B,IAAI,CAAC,EAAE,eAAe,EAAE,MAAM,CAAC,CAAC;YAExE,MAAM,UAAU,GAAa,EAAE,CAAC;YAChC,MAAM,MAAM,GAAwB,EAAE,CAAC;YACvC,IAAI,UAAU,GAAG,CAAC,CAAC;YAEnB,UAAU,CAAC,IAAI,CAAC,WAAW,UAAU,EAAE,EAAE,CAAC,CAAC;YAC3C,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAE1B,IAAI,MAAM,EAAE,MAAM,EAAE,CAAC;gBACpB,UAAU,CAAC,IAAI,CAAC,aAAa,UAAU,EAAE,EAAE,CAAC,CAAC;gBAC7C,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5B,CAAC;YAED,IAAI,MAAM,EAAE,aAAa,EAAE,CAAC;gBAC3B,UAAU,CAAC,IAAI,CAAC,cAAc,UAAU,EAAE,EAAE,CAAC,CAAC;gBAC9C,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC;YAC7C,CAAC;YAED,IAAI,KAAK,GAAG,uGAAuG,CAAC;YAEpH,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,KAAK,IAAI,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC/C,CAAC;YAED,IAAI,MAAM,EAAE,KAAK,KAAK,SAAS,EAAE,CAAC;gBACjC,KAAK,IAAI,WAAW,UAAU,EAAE,EAAE,CAAC;gBACnC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC3B,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,KAAK,CAAgB,KAAK,EAAE,MAAM,CAAC,CAAC;YAE9D,MAAM,OAAO,GAAuD,EAAE,CAAC;YAEvE,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBAC7B,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,KAAK,CACxC,mFAAmF,EACnF,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CACtB,CAAC;gBAEF,MAAM,cAAc,GAAG,cAAc,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;oBACpD,CAAC,CAAC,IAAI,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,UAAS,aAA4B;wBACtE,OAAM,CAAC,wBAAwB,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC;oBAC/D,CAAC,CAAC,CAAC;oBACH,CAAC,CAAC,SAAS,CAAC;gBAEb,OAAO,CAAC,IAAI,CAAC;oBACZ,EAAE,EAAE,wBAAwB,CAAC,GAAG,CAAC,EAAE,CAAC;oBACpC,yEAAyE;oBACzE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAiB;oBAChD,yEAAyE;oBACzE,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAgB,CAAC,CAAC,CAAC,IAAI;oBACjE,SAAS,EAAE,GAAG,CAAC,UAAU;oBACzB,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,OAAO,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBACtC,OAAO,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBACtC,yEAAyE;oBACzE,MAAM,EAAE,GAAG,CAAC,MAAoD;oBAChE,QAAQ,EAAE,GAAG,CAAC,QAAQ;oBACtB,cAAc,EAAE,cAAc;iBAC9B,CAAC,CAAC;YACJ,CAAC;YAED,MAAM,EAAE,KAAK,CAAC,yBAAyB,IAAI,CAAC,EAAE,eAAe,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAE9G,OAAM,CAAC,OAAO,CAAC,CAAC;QACjB,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAY;QAC3B,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,EAAE,KAAK,CAAC,uDAAuD,IAAI,EAAE,CAAC,CAAC;QAErG,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;YAChC,MAAK,CAAC,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC,CAAC;QAC5E,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,qCAAqC,CAA4B;YACnF,EAAE,EAAE,GAAG,IAAI,CAAC,EAAE,KAAK,IAAI,EAAE;YACzB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,IAAI,EAAE,IAAI,CAAC,YAAY;YACvB,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC;SAC1B,CAAC,CAAC;QAEH,OAAM,CAAC,MAAM,CAAC,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,OAAO;QACZ,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,qBAAqB,CAAC,CAAC;QAE3D,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QAC1B,OAAM,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IAC9B,CAAC;IAED,gBAAgB;IAChB,QAAQ,CAAC,GAAW;QAInB,IAAI,GAAG,KAAK,sCAAsC,EAAE,CAAC;YACpD,MAAK,CAAC,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC,CAAC;QACnD,CAAC;QAED,OAAM,CAAC;YACN,cAAc,EAAE,CAAC,KAAa,EAAQ,EAAE;gBACvC,IAAI,CAAC,WAAW,GAAG,KAAK,IAAmB,EAAE;oBAC5C,OAAM,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC7B,CAAC,CAAC;YACH,CAAC;YACD,gBAAgB,EAAE,GAAS,EAAE;gBAC5B,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC;YAC9B,CAAC;SACD,CAAC,CAAC;IACJ,CAAC;CACD","sourcesContent":["import type {\n\tKeetaAnchorQueueStorageDriver,\n\tKeetaAnchorQueueStorageDriverConstructor,\n\tKeetaAnchorQueueRequest,\n\tKeetaAnchorQueueRequestID,\n\tKeetaAnchorQueueEntry,\n\tKeetaAnchorQueueEntryExtra,\n\tKeetaAnchorQueueEntryAncillaryData,\n\tKeetaAnchorQueueStatus,\n\tKeetaAnchorQueueFilter,\n\tKeetaAnchorQueueWorkerID\n} from '../index.ts';\nimport {\n\tMethodLogger,\n\tManageStatusUpdates,\n\tConvertStringToRequestID\n} from '../internal.js';\nimport { Errors } from '../common.js';\n\nimport { asleep } from '../../utils/asleep.js';\n\nimport type { Logger } from '../../log/index.ts';\nimport type { JSONSerializable } from '../../utils/json.js';\n\nimport type * as pg from 'pg';\n\ntype QueueEntryRow = {\n\tid: string;\n\trequest: string;\n\toutput: string | null;\n\tlast_error: string | null;\n\tstatus: KeetaAnchorQueueStatus;\n\tcreated: Date;\n\tupdated: Date;\n\tworker: number | null;\n\tfailures: number;\n};\n\ntype IdempotentRow = {\n\tidempotent_id: string;\n};\n\nexport default class KeetaAnchorQueueStorageDriverPostgres<QueueRequest extends JSONSerializable = JSONSerializable, QueueResult extends JSONSerializable = JSONSerializable> implements KeetaAnchorQueueStorageDriver<QueueRequest, QueueResult> {\n\tprivate readonly logger: Logger | undefined;\n\tprivate poolInternal: (() => Promise<pg.Pool>) | null = null;\n\tprivate dbInitializationPromise: Promise<boolean> | null = null;\n\n\treadonly name = 'KeetaAnchorQueueStorageDriverPostgres';\n\treadonly id: string;\n\treadonly path: string[] = [];\n\tprivate readonly pathStr: string;\n\tprivate toctouDelay: (() => Promise<void>) | undefined = undefined;\n\n\tconstructor(options: NonNullable<ConstructorParameters<KeetaAnchorQueueStorageDriverConstructor<QueueRequest, QueueResult>>[0]> & { pool: () => Promise<pg.Pool>; }) {\n\t\tthis.id = options?.id ?? crypto.randomUUID();\n\t\tthis.logger = options?.logger\n\t\tthis.poolInternal = options.pool;\n\t\tthis.path = options.path ?? [];\n\t\tthis.pathStr = ['root', ...this.path].join('.');\n\t\tObject.freeze(this.path);\n\n\t\tthis.methodLogger('new')?.debug('Initialized Postgres queue storage driver');\n\t}\n\n\tprivate async initializeDBConnection(pool: pg.Pool): Promise<pg.Pool> {\n\t\tconst logger = this.methodLogger('initializeDBConnection');\n\n\t\tif (this.dbInitializationPromise !== null) {\n\t\t\tlogger?.debug('DB schema initialization already in progress or completed, waiting for it to finish');\n\n\t\t\tawait this.dbInitializationPromise;\n\t\t\treturn(pool);\n\t\t}\n\n\t\tthis.dbInitializationPromise = (async () => {\n\t\t\tlogger?.debug('Initializing DB schema for queue storage driver');\n\n\t\t\tconst client = await pool.connect();\n\t\t\ttry {\n\t\t\t\tawait client.query(`\n\t\t\t\t\tCREATE TABLE IF NOT EXISTS queue_entries (\n\t\t\t\t\t\tid TEXT NOT NULL,\n\t\t\t\t\t\tpath TEXT NOT NULL,\n\t\t\t\t\t\trequest TEXT NOT NULL,\n\t\t\t\t\t\toutput TEXT,\n\t\t\t\t\t\tlast_error TEXT,\n\t\t\t\t\t\tstatus TEXT NOT NULL,\n\t\t\t\t\t\tcreated BIGINT NOT NULL,\n\t\t\t\t\t\tupdated BIGINT NOT NULL,\n\t\t\t\t\t\tworker BIGINT,\n\t\t\t\t\t\tfailures INTEGER NOT NULL DEFAULT 0,\n\t\t\t\t\t\tPRIMARY KEY (id, path)\n\t\t\t\t\t)`);\n\n\t\t\t\tawait client.query(`\n\t\t\t\t\tCREATE TABLE IF NOT EXISTS queue_idempotent_keys (\n\t\t\t\t\t\tentry_id TEXT NOT NULL,\n\t\t\t\t\t\tidempotent_id TEXT NOT NULL,\n\t\t\t\t\t\tpath TEXT NOT NULL,\n\t\t\t\t\t\tUNIQUE (idempotent_id, path),\n\t\t\t\t\t\tPRIMARY KEY (entry_id, idempotent_id, path),\n\t\t\t\t\t\tFOREIGN KEY (entry_id, path) REFERENCES queue_entries(id, path)\n\t\t\t\t\t)`);\n\n\t\t\t\tawait client.query('CREATE INDEX IF NOT EXISTS idx_queue_entries_status ON queue_entries(status)');\n\t\t\t\tawait client.query('CREATE INDEX IF NOT EXISTS idx_queue_entries_updated ON queue_entries(updated)');\n\t\t\t\tawait client.query('CREATE INDEX IF NOT EXISTS idx_queue_idempotent_keys_idempotent_id ON queue_idempotent_keys(idempotent_id)');\n\t\t\t} finally {\n\t\t\t\tclient.release();\n\t\t\t}\n\n\t\t\tlogger?.debug('Completed DB schema initialization for queue storage driver');\n\n\t\t\treturn(true);\n\t\t})();\n\n\t\tawait this.dbInitializationPromise;\n\n\t\treturn(pool);\n\t}\n\n\tprivate methodLogger(method: string): Logger | undefined {\n\t\treturn(MethodLogger(this.logger, {\n\t\t\tclass: 'KeetaAnchorQueueStorageDriverPostgres',\n\t\t\tfile: 'src/lib/queue/drivers/queue_postgres.ts',\n\t\t\tmethod: method,\n\t\t\tinstanceID: this.id\n\t\t}));\n\t}\n\n\tprivate async runWithRetry<T>(fn: () => Promise<T>): Promise<T> {\n\t\tconst logger = this.methodLogger('runWithRetry');\n\n\t\tlet lastError: unknown;\n\t\tfor (let retry = 0; retry < 16; retry++) {\n\t\t\tif (this.poolInternal === null) {\n\t\t\t\tthis.methodLogger('runWithRetry')?.debug('Aborting DB operation retries because the instance was destroyed');\n\n\t\t\t\tif (lastError !== undefined) {\n\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/only-throw-error\n\t\t\t\t\tthrow(lastError);\n\t\t\t\t}\n\t\t\t\tthrow(new Error('Aborting because the instance was destroyed'));\n\t\t\t}\n\n\t\t\ttry {\n\t\t\t\treturn(await fn());\n\t\t\t} catch (error: unknown) {\n\t\t\t\tlastError = error;\n\n\t\t\t\tif (error instanceof Error) {\n\t\t\t\t\tconst errorCode = 'code' in error ? error.code : null;\n\t\t\t\t\tif (errorCode === '40001' || errorCode === '40P01') {\n\t\t\t\t\t\tlogger?.debug('Serialization failure or deadlock detected');\n\n\t\t\t\t\t\tconst minBackoff = 100;\n\t\t\t\t\t\tconst maxBackoff = 30_000;\n\t\t\t\t\t\tconst backoffIntervalSize = Math.min(maxBackoff - minBackoff, (retry + 50) ** 2);\n\t\t\t\t\t\tconst backoff = Math.round((Math.random() * backoffIntervalSize)) + minBackoff;\n\n\t\t\t\t\t\tthis.methodLogger('runWithRetry')?.debug(`Retrying DB operation in ${backoff}ms (retry #${retry}) from`, new Error().stack);\n\t\t\t\t\t\tawait asleep(backoff);\n\n\t\t\t\t\t\tcontinue;\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tthrow(error);\n\t\t\t}\n\t\t}\n\t\tthrow(lastError);\n\t}\n\n\tprivate async newDBConnection(): Promise<pg.Pool> {\n\t\tif (this.poolInternal === null) {\n\t\t\tthrow(new Error('Database connection is not available'));\n\t\t}\n\n\t\treturn(await this.runWithRetry(async () => {\n\t\t\tif (this.poolInternal === null) {\n\t\t\t\tthrow(new Error('Database connection is not available'));\n\t\t\t}\n\n\t\t\tthis.methodLogger('newDBConnection')?.debug('Getting DB pool');\n\t\t\tconst pool = await this.poolInternal();\n\n\t\t\treturn(await this.initializeDBConnection(pool));\n\t\t}));\n\t}\n\n\tprivate async dbTransaction<T>(className: string, fn: (client: pg.PoolClient, logger: Logger | undefined) => Promise<T>): Promise<T> {\n\t\tconst pool = await this.newDBConnection();\n\t\tconst logger = this.methodLogger(className);\n\n\t\tconst result = await this.runWithRetry(async function() {\n\t\t\tconst client = await pool.connect();\n\n\t\t\ttry {\n\t\t\t\tlogger?.debug('Starting DB transaction');\n\t\t\t\tawait client.query('BEGIN TRANSACTION ISOLATION LEVEL SERIALIZABLE');\n\t\t\t\tlogger?.debug('DB transaction started');\n\n\t\t\t\tconst retval = await fn(client, logger);\n\n\t\t\t\tlogger?.debug('Committing DB transaction');\n\t\t\t\tawait client.query('COMMIT');\n\t\t\t\tlogger?.debug('DB transaction committed');\n\n\t\t\t\treturn(retval);\n\t\t\t} catch (error: unknown) {\n\t\t\t\ttry {\n\t\t\t\t\tlogger?.debug('Rolling back DB transaction due to error:', error);\n\t\t\t\t\tawait client.query('ROLLBACK');\n\t\t\t\t\tlogger?.debug('DB transaction rolled back');\n\t\t\t\t} catch {\n\t\t\t\t\tlogger?.debug('Error rolling back DB transaction !!');\n\t\t\t\t}\n\t\t\t\tthrow(error);\n\t\t\t} finally {\n\t\t\t\tclient.release();\n\t\t\t}\n\t\t});\n\n\t\treturn(result);\n\t}\n\n\tasync add(request: KeetaAnchorQueueRequest<QueueRequest>, info?: KeetaAnchorQueueEntryExtra): Promise<KeetaAnchorQueueRequestID> {\n\t\treturn(await this.dbTransaction('add', async (client, logger): Promise<KeetaAnchorQueueRequestID> => {\n\t\t\tlet entryID = ConvertStringToRequestID(info?.id);\n\t\t\tif (entryID) {\n\t\t\t\tconst existingEntry = await client.query<{ id: string }>('SELECT id FROM queue_entries WHERE id = $1 AND path = $2', [entryID, this.pathStr]);\n\t\t\t\tif (existingEntry.rows.length > 0) {\n\t\t\t\t\tlogger?.debug(`Request with id ${String(entryID)} already exists, ignoring`);\n\t\t\t\t\treturn(entryID);\n\t\t\t\t}\n\n\t\t\t\tawait this.toctouDelay?.();\n\t\t\t}\n\n\t\t\tconst idempotentIDs = info?.idempotentKeys;\n\t\t\tif (idempotentIDs) {\n\t\t\t\tconst matchingIdempotentEntries = new Set<KeetaAnchorQueueRequestID>();\n\t\t\t\tfor (const idempotentID of idempotentIDs) {\n\t\t\t\t\tconst idempotentEntryExists = await client.query<IdempotentRow>(\n\t\t\t\t\t\t'SELECT idempotent_id FROM queue_idempotent_keys WHERE idempotent_id = $1 AND path = $2',\n\t\t\t\t\t\t[idempotentID, this.pathStr]\n\t\t\t\t\t);\n\t\t\t\t\tif (idempotentEntryExists.rows.length > 0) {\n\t\t\t\t\t\tmatchingIdempotentEntries.add(idempotentID);\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tif (matchingIdempotentEntries.size !== 0) {\n\t\t\t\t\tthrow(new Errors.IdempotentExistsError('One or more idempotent entries already exist in the queue', matchingIdempotentEntries));\n\t\t\t\t}\n\n\t\t\t\tawait this.toctouDelay?.();\n\t\t\t}\n\n\t\t\tentryID ??= ConvertStringToRequestID(crypto.randomUUID());\n\n\t\t\tlogger?.debug(`Enqueuing request with id ${String(entryID)}`);\n\n\t\t\tconst currentTime = Date.now();\n\t\t\tconst requestJSON = JSON.stringify(request);\n\n\t\t\t/**\n\t\t\t * The status to use for the new entry\n\t\t\t */\n\t\t\tconst status = info?.status ?? 'pending';\n\n\t\t\tawait client.query(\n\t\t\t\t`INSERT INTO queue_entries (id, path, request, output, last_error, status, created, updated, worker, failures)\n\t\t\t\t VALUES ($1, $2, $3, NULL, NULL, $4, $5, $6, NULL, 0)`,\n\t\t\t\t[entryID, this.pathStr, requestJSON, status, currentTime, currentTime]\n\t\t\t);\n\n\t\t\tif (idempotentIDs && idempotentIDs.size > 0) {\n\t\t\t\tfor (const idempotentID of idempotentIDs) {\n\t\t\t\t\tawait client.query('INSERT INTO queue_idempotent_keys (entry_id, path, idempotent_id) VALUES ($1, $2, $3)', [entryID, this.pathStr, idempotentID]);\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn(entryID);\n\t\t}));\n\t}\n\n\tasync setStatus(id: KeetaAnchorQueueRequestID, status: KeetaAnchorQueueStatus, ancillary?: KeetaAnchorQueueEntryAncillaryData<QueueResult>): Promise<void> {\n\t\tconst { oldStatus } = ancillary ?? {};\n\n\t\treturn(await this.dbTransaction('setStatus', async (client, logger): Promise<void> => {\n\t\t\tconst existingEntry = await client.query<{ status: KeetaAnchorQueueStatus; failures: number; last_error: string | null; output: string | null }>('SELECT status, failures, last_error, output FROM queue_entries WHERE id = $1 AND path = $2', [id, this.pathStr]);\n\t\t\tif (existingEntry.rows.length === 0) {\n\t\t\t\tthrow(new Error(`Request with ID ${String(id)} not found`));\n\t\t\t}\n\n\t\t\tawait this.toctouDelay?.();\n\n\t\t\tconst currentEntry = existingEntry.rows[0];\n\t\t\tif (!currentEntry) {\n\t\t\t\tthrow(new Error(`Request with ID ${String(id)} not found`));\n\t\t\t}\n\n\t\t\tconst newEntry = ManageStatusUpdates<QueueResult>(id, currentEntry, status, ancillary, logger);\n\t\t\tconst currentTime = newEntry.updated.getTime();\n\t\t\tconst workerValue = newEntry.worker;\n\t\t\tconst newFailures = newEntry.failures ?? currentEntry.failures;\n\t\t\tconst newLastError = newEntry.lastError !== undefined ? newEntry.lastError : currentEntry.last_error;\n\t\t\tconst newOutput = newEntry.output !== undefined ? JSON.stringify(newEntry.output) : currentEntry.output;\n\n\t\t\tlet updateQuery: string;\n\t\t\tlet updateParams: (KeetaAnchorQueueRequestID | string | number | null)[];\n\n\t\t\tif (oldStatus) {\n\t\t\t\tupdateQuery = `UPDATE queue_entries\n\t\t\t\t               SET status = $1, updated = $2, worker = $3, failures = $4, last_error = $5, output = $6\n\t\t\t\t               WHERE id = $7 AND path = $8 AND status = $9`;\n\t\t\t\tupdateParams = [status, currentTime, workerValue, newFailures, newLastError, newOutput, id, this.pathStr, oldStatus];\n\t\t\t} else {\n\t\t\t\tupdateQuery = `UPDATE queue_entries\n\t\t\t\t               SET status = $1, updated = $2, worker = $3, failures = $4, last_error = $5, output = $6\n\t\t\t\t               WHERE id = $7 AND path = $8`;\n\t\t\t\tupdateParams = [status, currentTime, workerValue, newFailures, newLastError, newOutput, id, this.pathStr];\n\t\t\t}\n\n\t\t\tconst result = await client.query(updateQuery, updateParams);\n\n\t\t\tawait this.toctouDelay?.();\n\n\t\t\tif (oldStatus && result.rowCount === 0) {\n\t\t\t\tconst currentEntry = await client.query<{ status: KeetaAnchorQueueStatus }>('SELECT status FROM queue_entries WHERE id = $1 AND path = $2', [id, this.pathStr]);\n\t\t\t\tconst currentStatus = currentEntry.rows[0]?.status;\n\t\t\t\tif (currentEntry.rows.length > 0) {\n\t\t\t\t\tif (currentStatus === undefined) {\n\t\t\t\t\t\tthrow(new Error(`internal error: could not retrieve current status for request with ID ${String(id)}`));\n\t\t\t\t\t}\n\t\t\t\t\tthrow(new Errors.IncorrectStateAssertedError(id, oldStatus, currentStatus));\n\t\t\t\t} else {\n\t\t\t\t\tthrow(new Error(`Request with ID ${String(id)} not found`));\n\t\t\t\t}\n\t\t\t}\n\t\t}));\n\t}\n\n\tasync get(id: KeetaAnchorQueueRequestID): Promise<KeetaAnchorQueueEntry<QueueRequest, QueueResult> | null> {\n\t\treturn(await this.dbTransaction('get', async (client): Promise<KeetaAnchorQueueEntry<QueueRequest, QueueResult> | null> => {\n\t\t\tconst row = await client.query<QueueEntryRow>(\n\t\t\t\t`SELECT id, request, output, last_error, status, created, updated, worker, failures\n\t\t\t\t FROM queue_entries WHERE id = $1 AND path = $2`,\n\t\t\t\t[id, this.pathStr]\n\t\t\t);\n\n\t\t\tif (row.rows.length === 0) {\n\t\t\t\treturn(null);\n\t\t\t}\n\n\t\t\tconst entry = row.rows[0];\n\t\t\tif (!entry) {\n\t\t\t\treturn(null);\n\t\t\t}\n\n\t\t\tconst idempotentRows = await client.query<IdempotentRow>(\n\t\t\t\t'SELECT idempotent_id FROM queue_idempotent_keys WHERE entry_id = $1 AND path = $2',\n\t\t\t\t[id, this.pathStr]\n\t\t\t);\n\n\t\t\tconst idempotentKeys = idempotentRows.rows.length > 0\n\t\t\t\t? new Set(idempotentRows.rows.map(function(idempotentRow: IdempotentRow) {\n\t\t\t\t\treturn(ConvertStringToRequestID(idempotentRow.idempotent_id));\n\t\t\t\t}))\n\t\t\t\t: undefined;\n\n\t\t\treturn({\n\t\t\t\tid: ConvertStringToRequestID(entry.id),\n\t\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\t\trequest: JSON.parse(entry.request) as QueueRequest,\n\t\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\t\toutput: entry.output ? JSON.parse(entry.output) as QueueResult : null,\n\t\t\t\tlastError: entry.last_error,\n\t\t\t\tstatus: entry.status,\n\t\t\t\tcreated: new Date(Number(entry.created)),\n\t\t\t\tupdated: new Date(Number(entry.updated)),\n\t\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\t\tworker: entry.worker as unknown as KeetaAnchorQueueWorkerID | null,\n\t\t\t\tfailures: entry.failures,\n\t\t\t\tidempotentKeys: idempotentKeys\n\t\t\t});\n\t\t}));\n\t}\n\n\tasync query(filter?: KeetaAnchorQueueFilter): Promise<KeetaAnchorQueueEntry<QueueRequest, QueueResult>[]> {\n\t\treturn(await this.dbTransaction('query', async (client, logger): Promise<KeetaAnchorQueueEntry<QueueRequest, QueueResult>[]> => {\n\t\t\tlogger?.debug(`Querying queue with id ${this.id} with filter:`, filter);\n\n\t\t\tconst conditions: string[] = [];\n\t\t\tconst params: (string | number)[] = [];\n\t\t\tlet paramIndex = 1;\n\n\t\t\tconditions.push(`path = $${paramIndex++}`);\n\t\t\tparams.push(this.pathStr);\n\n\t\t\tif (filter?.status) {\n\t\t\t\tconditions.push(`status = $${paramIndex++}`);\n\t\t\t\tparams.push(filter.status);\n\t\t\t}\n\n\t\t\tif (filter?.updatedBefore) {\n\t\t\t\tconditions.push(`updated < $${paramIndex++}`);\n\t\t\t\tparams.push(filter.updatedBefore.getTime());\n\t\t\t}\n\n\t\t\tlet query = 'SELECT id, request, output, last_error, status, created, updated, worker, failures FROM queue_entries';\n\n\t\t\tif (conditions.length > 0) {\n\t\t\t\tquery += ' WHERE ' + conditions.join(' AND ');\n\t\t\t}\n\n\t\t\tif (filter?.limit !== undefined) {\n\t\t\t\tquery += ` LIMIT $${paramIndex++}`;\n\t\t\t\tparams.push(filter.limit);\n\t\t\t}\n\n\t\t\tconst rows = await client.query<QueueEntryRow>(query, params);\n\n\t\t\tconst entries: KeetaAnchorQueueEntry<QueueRequest, QueueResult>[] = [];\n\n\t\t\tfor (const row of rows.rows) {\n\t\t\t\tconst idempotentRows = await client.query<IdempotentRow>(\n\t\t\t\t\t'SELECT idempotent_id FROM queue_idempotent_keys WHERE entry_id = $1 AND path = $2',\n\t\t\t\t\t[row.id, this.pathStr]\n\t\t\t\t);\n\n\t\t\t\tconst idempotentKeys = idempotentRows.rows.length > 0\n\t\t\t\t\t? new Set(idempotentRows.rows.map(function(idempotentRow: IdempotentRow) {\n\t\t\t\t\t\treturn(ConvertStringToRequestID(idempotentRow.idempotent_id));\n\t\t\t\t\t}))\n\t\t\t\t\t: undefined;\n\n\t\t\t\tentries.push({\n\t\t\t\t\tid: ConvertStringToRequestID(row.id),\n\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\t\t\trequest: JSON.parse(row.request) as QueueRequest,\n\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\t\t\toutput: row.output ? JSON.parse(row.output) as QueueResult : null,\n\t\t\t\t\tlastError: row.last_error,\n\t\t\t\t\tstatus: row.status,\n\t\t\t\t\tcreated: new Date(Number(row.created)),\n\t\t\t\t\tupdated: new Date(Number(row.updated)),\n\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\t\t\tworker: row.worker as unknown as KeetaAnchorQueueWorkerID | null,\n\t\t\t\t\tfailures: row.failures,\n\t\t\t\t\tidempotentKeys: idempotentKeys\n\t\t\t\t});\n\t\t\t}\n\n\t\t\tlogger?.debug(`Queried queue with id ${this.id} with filter:`, filter, '-- found', entries.length, 'entries');\n\n\t\t\treturn(entries);\n\t\t}));\n\t}\n\n\tasync partition(path: string) : Promise<KeetaAnchorQueueStorageDriver<QueueRequest, QueueResult>> {\n\t\tthis.methodLogger('partition')?.debug(`Creating partitioned queue storage driver for path: ${path}`);\n\n\t\tif (this.poolInternal === null) {\n\t\t\tthrow(new Error('Asked to partition but the instance has been destroyed'));\n\t\t}\n\n\t\tconst retval = new KeetaAnchorQueueStorageDriverPostgres<QueueRequest, QueueResult>({\n\t\t\tid: `${this.id}::${path}`,\n\t\t\tlogger: this.logger,\n\t\t\tpool: this.poolInternal,\n\t\t\tpath: [...this.path, path]\n\t\t});\n\n\t\treturn(retval);\n\t}\n\n\tasync destroy(): Promise<void> {\n\t\tthis.methodLogger('destroy')?.debug('Destroying instance');\n\n\t\tthis.poolInternal = null;\n\t}\n\n\tasync [Symbol.asyncDispose](): Promise<void> {\n\t\treturn(await this.destroy());\n\t}\n\n\t/** @internal */\n\t_Testing(key: string): {\n\t\tsetToctouDelay(delay: number): void;\n\t\tunsetToctouDelay(): void;\n\t} {\n\t\tif (key !== 'bc81abf8-e43b-490b-b486-744fb49a5082') {\n\t\t\tthrow(new Error('This is a testing only method'));\n\t\t}\n\n\t\treturn({\n\t\t\tsetToctouDelay: (delay: number): void => {\n\t\t\t\tthis.toctouDelay = async (): Promise<void> => {\n\t\t\t\t\treturn(await asleep(delay));\n\t\t\t\t};\n\t\t\t},\n\t\t\tunsetToctouDelay: (): void => {\n\t\t\t\tthis.toctouDelay = undefined;\n\t\t\t}\n\t\t});\n\t}\n}\n"]}
+{"version":3,"file":"queue_postgres.js","sourceRoot":"","sources":["../../../../src/lib/queue/drivers/queue_postgres.ts"],"names":[],"mappings":"AAcA,OAAO,EACN,YAAY,EACZ,mBAAmB,EACnB,wBAAwB,EACxB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAqC/C,MAAM,CAAC,OAAO,OAAO,qCAAqC;IACxC,MAAM,CAAqB;IACpC,YAAY,GAAoC,IAAI,CAAC;IACrD,WAAW,CAAqB;IAChC,gBAAgB,CAAS;IACzB,sBAAsB,CAAS;IAC/B,uBAAuB,CAAS;IAChC,uBAAuB,GAA4B,IAAI,CAAC;IACxD,uBAAuB,GAAG,CAAC,CAAC;IAC5B,mBAAmB,GAAG,KAAK,CAAC;IAE3B,IAAI,GAAG,uCAAuC,CAAC;IAC/C,EAAE,CAAS;IACX,IAAI,GAAa,EAAE,CAAC;IACZ,OAAO,CAAS;IACzB,WAAW,GAAsC,SAAS,CAAC;IAEnE,YAAY,OAAkK;QAC7K,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,EAAE,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAC7C,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;QAC9B,IAAI,CAAC,WAAW,GAAG,OAAO,EAAE,WAAW,CAAC;QACxC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;QACjC,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEzB,IAAI,CAAC,gBAAgB,GAAG,GAAG,IAAI,CAAC,WAAW,IAAI,OAAO,UAAU,CAAC;QACjE,IAAI,CAAC,uBAAuB,GAAG,GAAG,IAAI,CAAC,WAAW,IAAI,OAAO,kBAAkB,CAAC;QAChF,IAAI,CAAC,sBAAsB,GAAG,GAAG,IAAI,CAAC,WAAW,IAAI,OAAO,iBAAiB,CAAC;QAE9E,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC9E,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAAC,IAAa;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,wBAAwB,CAAC,CAAC;QAE3D,IAAI,IAAI,CAAC,uBAAuB,KAAK,IAAI,EAAE,CAAC;YAC3C,MAAM,EAAE,KAAK,CAAC,qFAAqF,CAAC,CAAC;YAErG,MAAM,IAAI,CAAC,uBAAuB,CAAC;YACnC,OAAM,CAAC,IAAI,CAAC,CAAC;QACd,CAAC;QAED,IAAI,CAAC,uBAAuB,GAAG,CAAC,KAAK,IAAI,EAAE;YAC1C,MAAM,EAAE,KAAK,CAAC,2DAA2D,EAAE,CAAC,IAAI,CAAC,gBAAgB,EAAE,IAAI,CAAC,uBAAuB,CAAC,CAAC,CAAC;YAElI,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACpC,IAAI,CAAC;gBACJ;;;;;;;;mBAQG;gBACH,MAAM,MAAM,GAAG,UAAU,CAAC;gBAC1B,MAAM,EAAE,KAAK,CAAC,8CAA8C,CAAC,CAAC;gBAC9D,MAAM,MAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;gBAE5D,IAAI,CAAC;oBACJ,kDAAkD;oBAClD,MAAM,MAAM,CAAC,KAAK,CAAC;mCACW,IAAI,CAAC,sBAAsB;;;;QAItD,CAAC,CAAC;oBAEL,+BAA+B;oBAC/B,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,KAAK,CAAsB,uCAAuC,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;oBACpI,MAAM,cAAc,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,CAAC,CAAC;oBAE3D,MAAM,EAAE,KAAK,CAAC,iCAAiC,cAAc,EAAE,CAAC,CAAC;oBAEjE,4BAA4B;oBAC5B,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;wBACxB,MAAM,EAAE,KAAK,CAAC,uDAAuD,CAAC,CAAC;wBAEvE,MAAM,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;wBAC5B,IAAI,CAAC;4BACJ,MAAM,MAAM,CAAC,KAAK,CAAC;qCACW,IAAI,CAAC,gBAAgB;;;;;;;;;;;;UAYhD,CAAC,CAAC;4BAEL,MAAM,MAAM,CAAC,KAAK,CAAC;qCACW,IAAI,CAAC,uBAAuB;;;;;;mDAMd,IAAI,CAAC,gBAAgB;UAC9D,CAAC,CAAC;4BAEL,wDAAwD;4BACxD,MAAM,MAAM,CAAC,KAAK,CAAC,kCAAkC,IAAI,CAAC,gBAAgB,cAAc,IAAI,CAAC,gBAAgB,UAAU,CAAC,CAAC;4BACzH,MAAM,MAAM,CAAC,KAAK,CAAC,kCAAkC,IAAI,CAAC,gBAAgB,eAAe,IAAI,CAAC,gBAAgB,WAAW,CAAC,CAAC;4BAC3H,MAAM,MAAM,CAAC,KAAK,CAAC,kCAAkC,IAAI,CAAC,uBAAuB,qBAAqB,IAAI,CAAC,uBAAuB,iBAAiB,CAAC,CAAC;4BAErJ,MAAM,MAAM,CAAC,KAAK,CAAC,eAAe,IAAI,CAAC,sBAAsB,uCAAuC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;4BACpH,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;4BAC7B,MAAM,EAAE,KAAK,CAAC,0BAA0B,CAAC,CAAC;wBAC3C,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BAChB,MAAM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;4BAC/B,MAAK,CAAC,KAAK,CAAC,CAAC;wBACd,CAAC;oBACF,CAAC;oBAED,+CAA+C;oBAC/C,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;wBACxB,MAAM,EAAE,KAAK,CAAC,8DAA8D,CAAC,CAAC;wBAE9E,gFAAgF;wBAChF,MAAM,EAAE,KAAK,CAAC,qCAAqC,CAAC,CAAC;wBACrD,MAAM,MAAM,CAAC,KAAK,CAAC,+CAA+C,IAAI,CAAC,gBAAgB,mBAAmB,IAAI,CAAC,gBAAgB,gBAAgB,CAAC,CAAC;wBACjJ,MAAM,MAAM,CAAC,KAAK,CAAC,+CAA+C,IAAI,CAAC,gBAAgB,oBAAoB,IAAI,CAAC,gBAAgB,iBAAiB,CAAC,CAAC;wBACnJ,MAAM,MAAM,CAAC,KAAK,CAAC,+CAA+C,IAAI,CAAC,gBAAgB,2BAA2B,IAAI,CAAC,gBAAgB,yBAAyB,CAAC,CAAC;wBAClK,MAAM,MAAM,CAAC,KAAK,CAAC,+CAA+C,IAAI,CAAC,uBAAuB,0BAA0B,IAAI,CAAC,uBAAuB,uBAAuB,CAAC,CAAC;wBAE7K,0CAA0C;wBAC1C,MAAM,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;wBAC5B,IAAI,CAAC;4BACJ,8FAA8F;4BAC9F,MAAM,EAAE,KAAK,CAAC,uCAAuC,CAAC,CAAC;4BACvD,MAAM,MAAM,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,gBAAgB,SAAS,CAAC,CAAC;4BAC/E,MAAM,MAAM,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,gBAAgB,UAAU,CAAC,CAAC;4BAChF,MAAM,MAAM,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,uBAAuB,gBAAgB,CAAC,CAAC;4BAE7F,MAAM,MAAM,CAAC,KAAK,CAAC,eAAe,IAAI,CAAC,sBAAsB,uCAAuC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;4BACpH,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;4BAC7B,MAAM,EAAE,KAAK,CAAC,0BAA0B,CAAC,CAAC;wBAC3C,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BAChB,MAAM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;4BAC/B,MAAK,CAAC,KAAK,CAAC,CAAC;wBACd,CAAC;oBACF,CAAC;oBAED,MAAM,EAAE,KAAK,CAAC,sBAAsB,CAAC,CAAC;gBACvC,CAAC;wBAAS,CAAC;oBACV,mCAAmC;oBACnC,yEAAyE;oBACzE,qFAAqF;oBACrF,IAAI,CAAC;wBACJ,MAAM,EAAE,KAAK,CAAC,yBAAyB,CAAC,CAAC;wBACzC,MAAM,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;oBAC/D,CAAC;oBAAC,OAAO,WAAW,EAAE,CAAC;wBACtB,8EAA8E;wBAC9E,MAAM,EAAE,KAAK,CAAC,+EAA+E,EAAE,WAAW,CAAC,CAAC;oBAC7G,CAAC;gBACF,CAAC;YACF,CAAC;oBAAS,CAAC;gBACV,MAAM,CAAC,OAAO,EAAE,CAAC;YAClB,CAAC;YAED,MAAM,EAAE,KAAK,CAAC,6DAA6D,CAAC,CAAC;YAE7E,OAAM,CAAC,IAAI,CAAC,CAAC;QACd,CAAC,CAAC,EAAE,CAAC;QAEL,MAAM,IAAI,CAAC,uBAAuB,CAAC;QAEnC,OAAM,CAAC,IAAI,CAAC,CAAC;IACd,CAAC;IAEO,YAAY,CAAC,MAAc;QAClC,OAAM,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE;YAChC,KAAK,EAAE,uCAAuC;YAC9C,IAAI,EAAE,yCAAyC;YAC/C,MAAM,EAAE,MAAM;YACd,UAAU,EAAE,IAAI,CAAC,EAAE;SACnB,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,YAAY,CAAI,EAAoB;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC;QAEjD,IAAI,SAAkB,CAAC;QACvB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC;YACzC,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;gBAChC,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,EAAE,KAAK,CAAC,kEAAkE,CAAC,CAAC;gBAE7G,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;oBAC7B,+DAA+D;oBAC/D,MAAK,CAAC,SAAS,CAAC,CAAC;gBAClB,CAAC;gBACD,MAAK,CAAC,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC,CAAC;YACjE,CAAC;YAED,IAAI,CAAC;gBACJ,OAAM,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACpB,CAAC;YAAC,OAAO,KAAc,EAAE,CAAC;gBACzB,SAAS,GAAG,KAAK,CAAC;gBAElB,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;oBAC5B,MAAM,SAAS,GAAG,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;oBACtD,IAAI,SAAS,KAAK,OAAO,IAAI,SAAS,KAAK,OAAO,EAAE,CAAC;wBACpD,MAAM,EAAE,KAAK,CAAC,4CAA4C,CAAC,CAAC;wBAE5D,kDAAkD;wBAClD,IAAI,CAAC,uBAAuB,EAAE,CAAC;wBAE/B,MAAM,UAAU,GAAG,GAAG,CAAC;wBACvB,MAAM,UAAU,GAAG,MAAM,CAAC;wBAC1B,MAAM,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,UAAU,EAAE,CAAC,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;wBACjF,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,mBAAmB,CAAC,CAAC,GAAG,UAAU,CAAC;wBAE/E,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,EAAE,KAAK,CAAC,4BAA4B,OAAO,cAAc,KAAK,QAAQ,EAAE,IAAI,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC;wBAC5H,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC;wBAEtB,SAAS;oBACV,CAAC;gBACF,CAAC;gBAED,MAAK,CAAC,KAAK,CAAC,CAAC;YACd,CAAC;QACF,CAAC;QACD,MAAK,CAAC,SAAS,CAAC,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,eAAe;QAC5B,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;YAChC,MAAK,CAAC,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC,CAAC;QAC1D,CAAC;QAED,OAAM,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE;YACzC,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;gBAChC,MAAK,CAAC,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC,CAAC;YAC1D,CAAC;YAED,IAAI,CAAC,YAAY,CAAC,iBAAiB,CAAC,EAAE,KAAK,CAAC,iBAAiB,CAAC,CAAC;YAC/D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAEvC,OAAM,CAAC,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,aAAa,CAAI,SAAiB,EAAE,EAAqE;QACtH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAE5C,MAAM,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC;QAErD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK;YAC3C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YAEpC,IAAI,CAAC;gBACJ,MAAM,EAAE,KAAK,CAAC,yBAAyB,CAAC,CAAC;gBACzC,MAAM,MAAM,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;gBACrE,MAAM,EAAE,KAAK,CAAC,wBAAwB,CAAC,CAAC;gBAExC,IAAI,mBAAmB,EAAE,CAAC;oBACzB,MAAM,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;gBACvD,CAAC;gBAED,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBAExC,MAAM,EAAE,KAAK,CAAC,2BAA2B,CAAC,CAAC;gBAC3C,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBAC7B,MAAM,EAAE,KAAK,CAAC,0BAA0B,CAAC,CAAC;gBAE1C,OAAM,CAAC,MAAM,CAAC,CAAC;YAChB,CAAC;YAAC,OAAO,KAAc,EAAE,CAAC;gBACzB,IAAI,CAAC;oBACJ,MAAM,EAAE,KAAK,CAAC,2CAA2C,EAAE,KAAK,CAAC,CAAC;oBAClE,MAAM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;oBAC/B,MAAM,EAAE,KAAK,CAAC,4BAA4B,CAAC,CAAC;gBAC7C,CAAC;gBAAC,MAAM,CAAC;oBACR,MAAM,EAAE,KAAK,CAAC,sCAAsC,CAAC,CAAC;gBACvD,CAAC;gBACD,MAAK,CAAC,KAAK,CAAC,CAAC;YACd,CAAC;oBAAS,CAAC;gBACV,MAAM,CAAC,OAAO,EAAE,CAAC;YAClB,CAAC;QACF,CAAC,CAAC,CAAC;QAEH,OAAM,CAAC,MAAM,CAAC,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,OAA8C,EAAE,IAAiC;QAC1F,OAAM,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAsC,EAAE;YACnG,IAAI,OAAO,GAAG,wBAAwB,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACjD,IAAI,OAAO,EAAE,CAAC;gBACb,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,KAAK,CAAiB,kBAAkB,IAAI,CAAC,gBAAgB,8BAA8B,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;gBACzJ,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACnC,MAAM,EAAE,KAAK,CAAC,mBAAmB,MAAM,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;oBAC7E,OAAM,CAAC,OAAO,CAAC,CAAC;gBACjB,CAAC;gBAED,MAAM,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YAC5B,CAAC;YAED,MAAM,aAAa,GAAG,IAAI,EAAE,cAAc,CAAC;YAC3C,IAAI,aAAa,EAAE,CAAC;gBACnB,MAAM,yBAAyB,GAAG,IAAI,GAAG,EAA6B,CAAC;gBACvE,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;oBAC1C,MAAM,qBAAqB,GAAG,MAAM,MAAM,CAAC,KAAK,CAC/C,6BAA6B,IAAI,CAAC,uBAAuB,yCAAyC,EAClG,CAAC,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,CAC5B,CAAC;oBACF,IAAI,qBAAqB,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC3C,yBAAyB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;oBAC7C,CAAC;gBACF,CAAC;gBAED,IAAI,yBAAyB,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;oBAC1C,MAAK,CAAC,IAAI,MAAM,CAAC,qBAAqB,CAAC,2DAA2D,EAAE,yBAAyB,CAAC,CAAC,CAAC;gBACjI,CAAC;gBAED,MAAM,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YAC5B,CAAC;YAED,OAAO,KAAK,wBAAwB,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;YAE1D,MAAM,EAAE,KAAK,CAAC,6BAA6B,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAE9D,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC/B,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YAE5C;;eAEG;YACH,MAAM,MAAM,GAAG,IAAI,EAAE,MAAM,IAAI,SAAS,CAAC;YAEzC,MAAM,MAAM,CAAC,KAAK,CACjB,eAAe,IAAI,CAAC,gBAAgB;0DACkB,EACtD,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,CAAC,CACtE,CAAC;YAEF,IAAI,aAAa,IAAI,aAAa,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBAC7C,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;oBAC1C,MAAM,MAAM,CAAC,KAAK,CAAC,eAAe,IAAI,CAAC,uBAAuB,sDAAsD,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;gBAC9J,CAAC;YACF,CAAC;YAED,OAAM,CAAC,OAAO,CAAC,CAAC;QACjB,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,EAA6B,EAAE,MAA8B,EAAE,SAA2D;QACzI,MAAM,EAAE,SAAS,EAAE,GAAG,SAAS,IAAI,EAAE,CAAC;QAEtC,OAAM,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAiB,EAAE;YACpF,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,KAAK,CAAyG,oDAAoD,IAAI,CAAC,gBAAgB,yCAAyC,EAAE,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;YACzR,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrC,MAAK,CAAC,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC;YAC7D,CAAC;YAED,MAAM,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YAE3B,MAAM,YAAY,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC3C,IAAI,CAAC,YAAY,EAAE,CAAC;gBACnB,MAAK,CAAC,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC;YAC7D,CAAC;YAED,MAAM,QAAQ,GAAG,mBAAmB,CAAc,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YAC/F,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YAC/C,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC;YACpC,MAAM,WAAW,GAAG,QAAQ,CAAC,QAAQ,IAAI,YAAY,CAAC,QAAQ,CAAC;YAC/D,MAAM,YAAY,GAAG,QAAQ,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,UAAU,CAAC;YACrG,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC;YAExG,IAAI,WAAmB,CAAC;YACxB,IAAI,YAAoE,CAAC;YAEzE,IAAI,SAAS,EAAE,CAAC;gBACf,WAAW,GAAG,UAAU,IAAI,CAAC,gBAAgB;;+DAEc,CAAC;gBAC5D,YAAY,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,EAAE,EAAE,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;YACtH,CAAC;iBAAM,CAAC;gBACP,WAAW,GAAG,UAAU,IAAI,CAAC,gBAAgB;;+CAEF,CAAC;gBAC5C,YAAY,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YAC3G,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;YAE7D,MAAM,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YAE3B,IAAI,SAAS,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;gBACxC,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,KAAK,CAAqC,sBAAsB,IAAI,CAAC,gBAAgB,8BAA8B,EAAE,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC3K,MAAM,aAAa,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC;gBACnD,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAClC,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;wBACjC,MAAK,CAAC,IAAI,KAAK,CAAC,yEAAyE,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;oBACzG,CAAC;oBACD,MAAK,CAAC,IAAI,MAAM,CAAC,2BAA2B,CAAC,EAAE,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC,CAAC;gBAC7E,CAAC;qBAAM,CAAC;oBACP,MAAK,CAAC,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC;gBAC7D,CAAC;YACF,CAAC;QACF,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EAA6B;QACtC,OAAM,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAoE,EAAE;YACzH,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,KAAK,CAC7B;YACQ,IAAI,CAAC,gBAAgB,8BAA8B,EAC3D,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAClB,CAAC;YAEF,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3B,OAAM,CAAC,IAAI,CAAC,CAAC;YACd,CAAC;YAED,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC1B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACZ,OAAM,CAAC,IAAI,CAAC,CAAC;YACd,CAAC;YAED,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,KAAK,CACxC,6BAA6B,IAAI,CAAC,uBAAuB,oCAAoC,EAC7F,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAClB,CAAC;YAEF,MAAM,cAAc,GAAG,cAAc,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;gBACpD,CAAC,CAAC,IAAI,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,UAAS,aAA4B;oBACtE,OAAM,CAAC,wBAAwB,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC;gBAC/D,CAAC,CAAC,CAAC;gBACH,CAAC,CAAC,SAAS,CAAC;YAEb,OAAM,CAAC;gBACN,EAAE,EAAE,wBAAwB,CAAC,KAAK,CAAC,EAAE,CAAC;gBACtC,yEAAyE;gBACzE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAiB;gBAClD,yEAAyE;gBACzE,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAgB,CAAC,CAAC,CAAC,IAAI;gBACrE,SAAS,EAAE,KAAK,CAAC,UAAU;gBAC3B,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,OAAO,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBACxC,OAAO,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBACxC,yEAAyE;gBACzE,MAAM,EAAE,KAAK,CAAC,MAAoD;gBAClE,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,cAAc,EAAE,cAAc;aAC9B,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,MAA+B;QAC1C,OAAM,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAA+D,EAAE;YAC9H,MAAM,EAAE,KAAK,CAAC,0BAA0B,IAAI,CAAC,EAAE,eAAe,EAAE,MAAM,CAAC,CAAC;YAExE,MAAM,UAAU,GAAa,EAAE,CAAC;YAChC,MAAM,MAAM,GAAwB,EAAE,CAAC;YACvC,IAAI,UAAU,GAAG,CAAC,CAAC;YAEnB,UAAU,CAAC,IAAI,CAAC,WAAW,UAAU,EAAE,EAAE,CAAC,CAAC;YAC3C,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAE1B,IAAI,MAAM,EAAE,MAAM,EAAE,CAAC;gBACpB,UAAU,CAAC,IAAI,CAAC,aAAa,UAAU,EAAE,EAAE,CAAC,CAAC;gBAC7C,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5B,CAAC;YAED,IAAI,MAAM,EAAE,aAAa,EAAE,CAAC;gBAC3B,UAAU,CAAC,IAAI,CAAC,cAAc,UAAU,EAAE,EAAE,CAAC,CAAC;gBAC9C,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC;YAC7C,CAAC;YAED,IAAI,KAAK,GAAG,2FAA2F,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAE/H,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,KAAK,IAAI,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC/C,CAAC;YAED,oFAAoF;YACpF,mEAAmE;YACnE,KAAK,IAAI,oBAAoB,CAAC;YAE9B,IAAI,MAAM,EAAE,KAAK,KAAK,SAAS,EAAE,CAAC;gBACjC,KAAK,IAAI,WAAW,UAAU,EAAE,EAAE,CAAC;gBACnC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC3B,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,KAAK,CAAgB,KAAK,EAAE,MAAM,CAAC,CAAC;YAE9D,MAAM,OAAO,GAAuD,EAAE,CAAC;YAEvE,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBAC7B,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,KAAK,CACxC,6BAA6B,IAAI,CAAC,uBAAuB,oCAAoC,EAC7F,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CACtB,CAAC;gBAEF,MAAM,cAAc,GAAG,cAAc,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;oBACpD,CAAC,CAAC,IAAI,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,UAAS,aAA4B;wBACtE,OAAM,CAAC,wBAAwB,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC;oBAC/D,CAAC,CAAC,CAAC;oBACH,CAAC,CAAC,SAAS,CAAC;gBAEb,OAAO,CAAC,IAAI,CAAC;oBACZ,EAAE,EAAE,wBAAwB,CAAC,GAAG,CAAC,EAAE,CAAC;oBACpC,yEAAyE;oBACzE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAiB;oBAChD,yEAAyE;oBACzE,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAgB,CAAC,CAAC,CAAC,IAAI;oBACjE,SAAS,EAAE,GAAG,CAAC,UAAU;oBACzB,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,OAAO,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBACtC,OAAO,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBACtC,yEAAyE;oBACzE,MAAM,EAAE,GAAG,CAAC,MAAoD;oBAChE,QAAQ,EAAE,GAAG,CAAC,QAAQ;oBACtB,cAAc,EAAE,cAAc;iBAC9B,CAAC,CAAC;YACJ,CAAC;YAED,MAAM,EAAE,KAAK,CAAC,yBAAyB,IAAI,CAAC,EAAE,eAAe,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAE9G,OAAM,CAAC,OAAO,CAAC,CAAC;QACjB,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAY;QAC3B,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,EAAE,KAAK,CAAC,uDAAuD,IAAI,EAAE,CAAC,CAAC;QAErG,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;YAChC,MAAK,CAAC,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC,CAAC;QAC5E,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,qCAAqC,CAA4B;YACnF,EAAE,EAAE,GAAG,IAAI,CAAC,EAAE,KAAK,IAAI,EAAE;YACzB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,IAAI,EAAE,IAAI,CAAC,YAAY;YACvB,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC;YAC1B,WAAW,EAAE,IAAI,CAAC,WAAW;SAC7B,CAAC,CAAC;QAEH,OAAM,CAAC,MAAM,CAAC,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,OAAO;QACZ,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,qBAAqB,CAAC,CAAC;QAE3D,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QAC1B,OAAM,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IAC9B,CAAC;IAED,gBAAgB;IAChB,QAAQ,CAAC,GAAW;QAQnB,IAAI,GAAG,KAAK,sCAAsC,EAAE,CAAC;YACpD,MAAK,CAAC,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC,CAAC;QACnD,CAAC;QAED,OAAM,CAAC;YACN,cAAc,EAAE,CAAC,KAAa,EAAQ,EAAE;gBACvC,IAAI,CAAC,WAAW,GAAG,KAAK,IAAmB,EAAE;oBAC5C,OAAM,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC7B,CAAC,CAAC;YACH,CAAC;YACD,gBAAgB,EAAE,GAAS,EAAE;gBAC5B,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC;YAC9B,CAAC;YACD,0BAA0B,EAAE,GAAW,EAAE;gBACxC,OAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACtC,CAAC;YACD,4BAA4B,EAAE,GAAS,EAAE;gBACxC,IAAI,CAAC,uBAAuB,GAAG,CAAC,CAAC;YAClC,CAAC;YACD,yBAAyB,EAAE,GAAS,EAAE;gBACrC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;YACjC,CAAC;YACD,0BAA0B,EAAE,GAAS,EAAE;gBACtC,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;YAClC,CAAC;SACD,CAAC,CAAC;IACJ,CAAC;CACD","sourcesContent":["// cspell:ignore seqscan\n\nimport type {\n\tKeetaAnchorQueueStorageDriver,\n\tKeetaAnchorQueueStorageDriverConstructor,\n\tKeetaAnchorQueueRequest,\n\tKeetaAnchorQueueRequestID,\n\tKeetaAnchorQueueEntry,\n\tKeetaAnchorQueueEntryExtra,\n\tKeetaAnchorQueueEntryAncillaryData,\n\tKeetaAnchorQueueStatus,\n\tKeetaAnchorQueueFilter,\n\tKeetaAnchorQueueWorkerID\n} from '../index.ts';\nimport {\n\tMethodLogger,\n\tManageStatusUpdates,\n\tConvertStringToRequestID\n} from '../internal.js';\nimport { Errors } from '../common.js';\n\nimport { asleep } from '../../utils/asleep.js';\n\nimport type { Logger } from '../../log/index.ts';\nimport type { JSONSerializable } from '../../utils/json.js';\n\nimport type * as pg from 'pg';\n\ntype QueueEntryRow = {\n\tid: string;\n\trequest: string;\n\toutput: string | null;\n\tlast_error: string | null;\n\tstatus: KeetaAnchorQueueStatus;\n\tcreated: Date;\n\tupdated: Date;\n\tworker: number | null;\n\tfailures: number;\n};\n\ntype IdempotentRow = {\n\tidempotent_id: string;\n};\n\ntype KeetaAnchorQueueStorageDriverPostgresOptions = {\n\t/**\n\t * Function that returns a Postgres connection pool.\n\t */\n\tpool: () => Promise<pg.Pool>;\n\n\t/**\n\t * The prefix to use for the database tables. Defaults to 'queue',\n\t * resulting in table names 'queue_entries', 'queue_idempotent_keys',\n\t * and `queue_schema_version`.\n\t */\n\ttablePrefix?: string | undefined;\n};\n\nexport default class KeetaAnchorQueueStorageDriverPostgres<QueueRequest extends JSONSerializable = JSONSerializable, QueueResult extends JSONSerializable = JSONSerializable> implements KeetaAnchorQueueStorageDriver<QueueRequest, QueueResult> {\n\tprivate readonly logger: Logger | undefined;\n\tprivate poolInternal: (() => Promise<pg.Pool>) | null = null;\n\tprivate tablePrefix: string | undefined;\n\tprivate tableNameEntries: string;\n\tprivate tableNameSchemaVersion: string;\n\tprivate tableNameIdempotentKeys: string;\n\tprivate dbInitializationPromise: Promise<boolean> | null = null;\n\tprivate serializationRetryCount = 0;\n\tprivate debugForceIndexScan = false;\n\n\treadonly name = 'KeetaAnchorQueueStorageDriverPostgres';\n\treadonly id: string;\n\treadonly path: string[] = [];\n\tprivate readonly pathStr: string;\n\tprivate toctouDelay: (() => Promise<void>) | undefined = undefined;\n\n\tconstructor(options: NonNullable<ConstructorParameters<KeetaAnchorQueueStorageDriverConstructor<QueueRequest, QueueResult>>[0]> & KeetaAnchorQueueStorageDriverPostgresOptions) {\n\t\tthis.id = options?.id ?? crypto.randomUUID();\n\t\tthis.logger = options?.logger;\n\t\tthis.tablePrefix = options?.tablePrefix;\n\t\tthis.poolInternal = options.pool;\n\t\tthis.path = options.path ?? [];\n\t\tthis.pathStr = ['root', ...this.path].join('.');\n\t\tObject.freeze(this.path);\n\n\t\tthis.tableNameEntries = `${this.tablePrefix ?? 'queue'}_entries`;\n\t\tthis.tableNameIdempotentKeys = `${this.tablePrefix ?? 'queue'}_idempotent_keys`;\n\t\tthis.tableNameSchemaVersion = `${this.tablePrefix ?? 'queue'}_schema_version`;\n\n\t\tthis.methodLogger('new')?.debug('Initialized Postgres queue storage driver');\n\t}\n\n\tprivate async initializeDBConnection(pool: pg.Pool): Promise<pg.Pool> {\n\t\tconst logger = this.methodLogger('initializeDBConnection');\n\n\t\tif (this.dbInitializationPromise !== null) {\n\t\t\tlogger?.debug('DB schema initialization already in progress or completed, waiting for it to finish');\n\n\t\t\tawait this.dbInitializationPromise;\n\t\t\treturn(pool);\n\t\t}\n\n\t\tthis.dbInitializationPromise = (async () => {\n\t\t\tlogger?.debug('Initializing DB schema for queue storage driver on tables', [this.tableNameEntries, this.tableNameIdempotentKeys]);\n\n\t\t\tconst client = await pool.connect();\n\t\t\ttry {\n\t\t\t\t/*\n\t\t\t\t * Random lock key (32-bit integer), to ensure\n\t\t\t\t * that multiple instances of this driver\n\t\t\t\t * (potentially across different\n\t\t\t\t * instances/process) will run the migration\n\t\t\t\t * sequentially to avoid multiple concurrent\n\t\t\t\t * migrations from competing with each other\n\t\t\t\t * and causing delays\n\t\t\t\t */\n\t\t\t\tconst lockId = 0x24995E48;\n\t\t\t\tlogger?.debug('Acquiring advisory lock for schema migration');\n\t\t\t\tawait client.query('SELECT pg_advisory_lock($1)', [lockId]);\n\n\t\t\t\ttry {\n\t\t\t\t\t// Create schema version table if it doesn't exist\n\t\t\t\t\tawait client.query(`\n\t\t\t\t\t\tCREATE TABLE IF NOT EXISTS ${this.tableNameSchemaVersion} (\n\t\t\t\t\t\t\tversion INTEGER NOT NULL,\n\t\t\t\t\t\t\tapplied_at BIGINT NOT NULL,\n\t\t\t\t\t\t\tPRIMARY KEY (version)\n\t\t\t\t\t\t)`);\n\n\t\t\t\t\t// Check current schema version\n\t\t\t\t\tconst versionResult = await client.query<{ version: number }>(`SELECT MAX(version) as version FROM ${this.tableNameSchemaVersion}`);\n\t\t\t\t\tconst currentVersion = versionResult.rows[0]?.version ?? 0;\n\n\t\t\t\t\tlogger?.debug(`Current queue schema version: ${currentVersion}`);\n\n\t\t\t\t\t// Version 1: Initial schema\n\t\t\t\t\tif (currentVersion < 1) {\n\t\t\t\t\t\tlogger?.debug('Applying schema version 1: Initial tables and indexes');\n\n\t\t\t\t\t\tawait client.query('BEGIN');\n\t\t\t\t\t\ttry {\n\t\t\t\t\t\t\tawait client.query(`\n\t\t\t\t\t\t\t\tCREATE TABLE IF NOT EXISTS ${this.tableNameEntries} (\n\t\t\t\t\t\t\t\t\tid TEXT NOT NULL,\n\t\t\t\t\t\t\t\t\tpath TEXT NOT NULL,\n\t\t\t\t\t\t\t\t\trequest TEXT NOT NULL,\n\t\t\t\t\t\t\t\t\toutput TEXT,\n\t\t\t\t\t\t\t\t\tlast_error TEXT,\n\t\t\t\t\t\t\t\t\tstatus TEXT NOT NULL,\n\t\t\t\t\t\t\t\t\tcreated BIGINT NOT NULL,\n\t\t\t\t\t\t\t\t\tupdated BIGINT NOT NULL,\n\t\t\t\t\t\t\t\t\tworker BIGINT,\n\t\t\t\t\t\t\t\t\tfailures INTEGER NOT NULL DEFAULT 0,\n\t\t\t\t\t\t\t\t\tPRIMARY KEY (id, path)\n\t\t\t\t\t\t\t\t)`);\n\n\t\t\t\t\t\t\tawait client.query(`\n\t\t\t\t\t\t\t\tCREATE TABLE IF NOT EXISTS ${this.tableNameIdempotentKeys} (\n\t\t\t\t\t\t\t\t\tentry_id TEXT NOT NULL,\n\t\t\t\t\t\t\t\t\tidempotent_id TEXT NOT NULL,\n\t\t\t\t\t\t\t\t\tpath TEXT NOT NULL,\n\t\t\t\t\t\t\t\t\tUNIQUE (idempotent_id, path),\n\t\t\t\t\t\t\t\t\tPRIMARY KEY (entry_id, idempotent_id, path),\n\t\t\t\t\t\t\t\t\tFOREIGN KEY (entry_id, path) REFERENCES ${this.tableNameEntries}(id, path)\n\t\t\t\t\t\t\t\t)`);\n\n\t\t\t\t\t\t\t// Old single-column indexes (for pre-version-2 schemas)\n\t\t\t\t\t\t\tawait client.query(`CREATE INDEX IF NOT EXISTS idx_${this.tableNameEntries}_status ON ${this.tableNameEntries}(status)`);\n\t\t\t\t\t\t\tawait client.query(`CREATE INDEX IF NOT EXISTS idx_${this.tableNameEntries}_updated ON ${this.tableNameEntries}(updated)`);\n\t\t\t\t\t\t\tawait client.query(`CREATE INDEX IF NOT EXISTS idx_${this.tableNameIdempotentKeys}_idempotent_id ON ${this.tableNameIdempotentKeys}(idempotent_id)`);\n\n\t\t\t\t\t\t\tawait client.query(`INSERT INTO ${this.tableNameSchemaVersion} (version, applied_at) VALUES (1, $1)`, [Date.now()]);\n\t\t\t\t\t\t\tawait client.query('COMMIT');\n\t\t\t\t\t\t\tlogger?.debug('Applied schema version 1');\n\t\t\t\t\t\t} catch (error) {\n\t\t\t\t\t\t\tawait client.query('ROLLBACK');\n\t\t\t\t\t\t\tthrow(error);\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\t// Version 2: Partition-aware composite indexes\n\t\t\t\t\tif (currentVersion < 2) {\n\t\t\t\t\t\tlogger?.debug('Applying schema version 2: Partition-aware composite indexes');\n\n\t\t\t\t\t\t// Create new partition-aware indexes (CONCURRENTLY must be outside transaction)\n\t\t\t\t\t\tlogger?.debug('Creating partition-aware indexes...');\n\t\t\t\t\t\tawait client.query(`CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_${this.tableNameEntries}_path_status ON ${this.tableNameEntries}(path, status)`);\n\t\t\t\t\t\tawait client.query(`CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_${this.tableNameEntries}_path_updated ON ${this.tableNameEntries}(path, updated)`);\n\t\t\t\t\t\tawait client.query(`CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_${this.tableNameEntries}_path_status_updated ON ${this.tableNameEntries}(path, status, updated)`);\n\t\t\t\t\t\tawait client.query(`CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_${this.tableNameIdempotentKeys}_path_idempotent_id ON ${this.tableNameIdempotentKeys}(path, idempotent_id)`);\n\n\t\t\t\t\t\t// Now drop old indexes and record version\n\t\t\t\t\t\tawait client.query('BEGIN');\n\t\t\t\t\t\ttry {\n\t\t\t\t\t\t\t// Drop old indexes that are now redundant (these will fail gracefully if indexes don't exist)\n\t\t\t\t\t\t\tlogger?.debug('Dropping old single-column indexes...');\n\t\t\t\t\t\t\tawait client.query(`DROP INDEX IF EXISTS idx_${this.tableNameEntries}_status`);\n\t\t\t\t\t\t\tawait client.query(`DROP INDEX IF EXISTS idx_${this.tableNameEntries}_updated`);\n\t\t\t\t\t\t\tawait client.query(`DROP INDEX IF EXISTS idx_${this.tableNameIdempotentKeys}_idempotent_id`);\n\n\t\t\t\t\t\t\tawait client.query(`INSERT INTO ${this.tableNameSchemaVersion} (version, applied_at) VALUES (2, $1)`, [Date.now()]);\n\t\t\t\t\t\t\tawait client.query('COMMIT');\n\t\t\t\t\t\t\tlogger?.debug('Applied schema version 2');\n\t\t\t\t\t\t} catch (error) {\n\t\t\t\t\t\t\tawait client.query('ROLLBACK');\n\t\t\t\t\t\t\tthrow(error);\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\tlogger?.debug('Schema is up to date');\n\t\t\t\t} finally {\n\t\t\t\t\t// Always release the advisory lock\n\t\t\t\t\t// Note: Advisory locks are session-based and auto-release on disconnect,\n\t\t\t\t\t// but we explicitly unlock for clarity and to avoid holding locks longer than needed\n\t\t\t\t\ttry {\n\t\t\t\t\t\tlogger?.debug('Releasing advisory lock');\n\t\t\t\t\t\tawait client.query('SELECT pg_advisory_unlock($1)', [lockId]);\n\t\t\t\t\t} catch (unlockError) {\n\t\t\t\t\t\t// Log but don't throw - the lock will be auto-released when connection closes\n\t\t\t\t\t\tlogger?.debug('Failed to explicitly release advisory lock (will auto-release on disconnect):', unlockError);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} finally {\n\t\t\t\tclient.release();\n\t\t\t}\n\n\t\t\tlogger?.debug('Completed DB schema initialization for queue storage driver');\n\n\t\t\treturn(true);\n\t\t})();\n\n\t\tawait this.dbInitializationPromise;\n\n\t\treturn(pool);\n\t}\n\n\tprivate methodLogger(method: string): Logger | undefined {\n\t\treturn(MethodLogger(this.logger, {\n\t\t\tclass: 'KeetaAnchorQueueStorageDriverPostgres',\n\t\t\tfile: 'src/lib/queue/drivers/queue_postgres.ts',\n\t\t\tmethod: method,\n\t\t\tinstanceID: this.id\n\t\t}));\n\t}\n\n\tprivate async runWithRetry<T>(fn: () => Promise<T>): Promise<T> {\n\t\tconst logger = this.methodLogger('runWithRetry');\n\n\t\tlet lastError: unknown;\n\t\tfor (let retry = 0; retry < 16; retry++) {\n\t\t\tif (this.poolInternal === null) {\n\t\t\t\tthis.methodLogger('runWithRetry')?.debug('Aborting DB operation retries because the instance was destroyed');\n\n\t\t\t\tif (lastError !== undefined) {\n\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/only-throw-error\n\t\t\t\t\tthrow(lastError);\n\t\t\t\t}\n\t\t\t\tthrow(new Error('Aborting because the instance was destroyed'));\n\t\t\t}\n\n\t\t\ttry {\n\t\t\t\treturn(await fn());\n\t\t\t} catch (error: unknown) {\n\t\t\t\tlastError = error;\n\n\t\t\t\tif (error instanceof Error) {\n\t\t\t\t\tconst errorCode = 'code' in error ? error.code : null;\n\t\t\t\t\tif (errorCode === '40001' || errorCode === '40P01') {\n\t\t\t\t\t\tlogger?.debug('Serialization failure or deadlock detected');\n\n\t\t\t\t\t\t// Track serialization retries for instrumentation\n\t\t\t\t\t\tthis.serializationRetryCount++;\n\n\t\t\t\t\t\tconst minBackoff = 100;\n\t\t\t\t\t\tconst maxBackoff = 30_000;\n\t\t\t\t\t\tconst backoffIntervalSize = Math.min(maxBackoff - minBackoff, (retry + 50) ** 2);\n\t\t\t\t\t\tconst backoff = Math.round((Math.random() * backoffIntervalSize)) + minBackoff;\n\n\t\t\t\t\t\tthis.methodLogger('runWithRetry')?.debug(`Retrying DB operation in ${backoff}ms (retry #${retry}) from`, new Error().stack);\n\t\t\t\t\t\tawait asleep(backoff);\n\n\t\t\t\t\t\tcontinue;\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tthrow(error);\n\t\t\t}\n\t\t}\n\t\tthrow(lastError);\n\t}\n\n\tprivate async newDBConnection(): Promise<pg.Pool> {\n\t\tif (this.poolInternal === null) {\n\t\t\tthrow(new Error('Database connection is not available'));\n\t\t}\n\n\t\treturn(await this.runWithRetry(async () => {\n\t\t\tif (this.poolInternal === null) {\n\t\t\t\tthrow(new Error('Database connection is not available'));\n\t\t\t}\n\n\t\t\tthis.methodLogger('newDBConnection')?.debug('Getting DB pool');\n\t\t\tconst pool = await this.poolInternal();\n\n\t\t\treturn(await this.initializeDBConnection(pool));\n\t\t}));\n\t}\n\n\tprivate async dbTransaction<T>(className: string, fn: (client: pg.PoolClient, logger: Logger | undefined) => Promise<T>): Promise<T> {\n\t\tconst pool = await this.newDBConnection();\n\t\tconst logger = this.methodLogger(className);\n\n\t\tconst debugForceIndexScan = this.debugForceIndexScan;\n\n\t\tconst result = await this.runWithRetry(async function() {\n\t\t\tconst client = await pool.connect();\n\n\t\t\ttry {\n\t\t\t\tlogger?.debug('Starting DB transaction');\n\t\t\t\tawait client.query('BEGIN TRANSACTION ISOLATION LEVEL SERIALIZABLE');\n\t\t\t\tlogger?.debug('DB transaction started');\n\n\t\t\t\tif (debugForceIndexScan) {\n\t\t\t\t\tawait client.query('SET LOCAL enable_seqscan TO off');\n\t\t\t\t}\n\n\t\t\t\tconst retval = await fn(client, logger);\n\n\t\t\t\tlogger?.debug('Committing DB transaction');\n\t\t\t\tawait client.query('COMMIT');\n\t\t\t\tlogger?.debug('DB transaction committed');\n\n\t\t\t\treturn(retval);\n\t\t\t} catch (error: unknown) {\n\t\t\t\ttry {\n\t\t\t\t\tlogger?.debug('Rolling back DB transaction due to error:', error);\n\t\t\t\t\tawait client.query('ROLLBACK');\n\t\t\t\t\tlogger?.debug('DB transaction rolled back');\n\t\t\t\t} catch {\n\t\t\t\t\tlogger?.debug('Error rolling back DB transaction !!');\n\t\t\t\t}\n\t\t\t\tthrow(error);\n\t\t\t} finally {\n\t\t\t\tclient.release();\n\t\t\t}\n\t\t});\n\n\t\treturn(result);\n\t}\n\n\tasync add(request: KeetaAnchorQueueRequest<QueueRequest>, info?: KeetaAnchorQueueEntryExtra): Promise<KeetaAnchorQueueRequestID> {\n\t\treturn(await this.dbTransaction('add', async (client, logger): Promise<KeetaAnchorQueueRequestID> => {\n\t\t\tlet entryID = ConvertStringToRequestID(info?.id);\n\t\t\tif (entryID) {\n\t\t\t\tconst existingEntry = await client.query<{ id: string }>(`SELECT id FROM ${this.tableNameEntries} WHERE id = $1 AND path = $2`, [entryID, this.pathStr]);\n\t\t\t\tif (existingEntry.rows.length > 0) {\n\t\t\t\t\tlogger?.debug(`Request with id ${String(entryID)} already exists, ignoring`);\n\t\t\t\t\treturn(entryID);\n\t\t\t\t}\n\n\t\t\t\tawait this.toctouDelay?.();\n\t\t\t}\n\n\t\t\tconst idempotentIDs = info?.idempotentKeys;\n\t\t\tif (idempotentIDs) {\n\t\t\t\tconst matchingIdempotentEntries = new Set<KeetaAnchorQueueRequestID>();\n\t\t\t\tfor (const idempotentID of idempotentIDs) {\n\t\t\t\t\tconst idempotentEntryExists = await client.query<IdempotentRow>(\n\t\t\t\t\t\t`SELECT idempotent_id FROM ${this.tableNameIdempotentKeys} WHERE idempotent_id = $1 AND path = $2`,\n\t\t\t\t\t\t[idempotentID, this.pathStr]\n\t\t\t\t\t);\n\t\t\t\t\tif (idempotentEntryExists.rows.length > 0) {\n\t\t\t\t\t\tmatchingIdempotentEntries.add(idempotentID);\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tif (matchingIdempotentEntries.size !== 0) {\n\t\t\t\t\tthrow(new Errors.IdempotentExistsError('One or more idempotent entries already exist in the queue', matchingIdempotentEntries));\n\t\t\t\t}\n\n\t\t\t\tawait this.toctouDelay?.();\n\t\t\t}\n\n\t\t\tentryID ??= ConvertStringToRequestID(crypto.randomUUID());\n\n\t\t\tlogger?.debug(`Enqueuing request with id ${String(entryID)}`);\n\n\t\t\tconst currentTime = Date.now();\n\t\t\tconst requestJSON = JSON.stringify(request);\n\n\t\t\t/**\n\t\t\t * The status to use for the new entry\n\t\t\t */\n\t\t\tconst status = info?.status ?? 'pending';\n\n\t\t\tawait client.query(\n\t\t\t\t`INSERT INTO ${this.tableNameEntries} (id, path, request, output, last_error, status, created, updated, worker, failures)\n\t\t\t\t VALUES ($1, $2, $3, NULL, NULL, $4, $5, $6, NULL, 0)`,\n\t\t\t\t[entryID, this.pathStr, requestJSON, status, currentTime, currentTime]\n\t\t\t);\n\n\t\t\tif (idempotentIDs && idempotentIDs.size > 0) {\n\t\t\t\tfor (const idempotentID of idempotentIDs) {\n\t\t\t\t\tawait client.query(`INSERT INTO ${this.tableNameIdempotentKeys} (entry_id, path, idempotent_id) VALUES ($1, $2, $3)`, [entryID, this.pathStr, idempotentID]);\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn(entryID);\n\t\t}));\n\t}\n\n\tasync setStatus(id: KeetaAnchorQueueRequestID, status: KeetaAnchorQueueStatus, ancillary?: KeetaAnchorQueueEntryAncillaryData<QueueResult>): Promise<void> {\n\t\tconst { oldStatus } = ancillary ?? {};\n\n\t\treturn(await this.dbTransaction('setStatus', async (client, logger): Promise<void> => {\n\t\t\tconst existingEntry = await client.query<{ status: KeetaAnchorQueueStatus; failures: number; last_error: string | null; output: string | null }>(`SELECT status, failures, last_error, output FROM ${this.tableNameEntries} WHERE id = $1 AND path = $2 FOR UPDATE`, [id, this.pathStr]);\n\t\t\tif (existingEntry.rows.length === 0) {\n\t\t\t\tthrow(new Error(`Request with ID ${String(id)} not found`));\n\t\t\t}\n\n\t\t\tawait this.toctouDelay?.();\n\n\t\t\tconst currentEntry = existingEntry.rows[0];\n\t\t\tif (!currentEntry) {\n\t\t\t\tthrow(new Error(`Request with ID ${String(id)} not found`));\n\t\t\t}\n\n\t\t\tconst newEntry = ManageStatusUpdates<QueueResult>(id, currentEntry, status, ancillary, logger);\n\t\t\tconst currentTime = newEntry.updated.getTime();\n\t\t\tconst workerValue = newEntry.worker;\n\t\t\tconst newFailures = newEntry.failures ?? currentEntry.failures;\n\t\t\tconst newLastError = newEntry.lastError !== undefined ? newEntry.lastError : currentEntry.last_error;\n\t\t\tconst newOutput = newEntry.output !== undefined ? JSON.stringify(newEntry.output) : currentEntry.output;\n\n\t\t\tlet updateQuery: string;\n\t\t\tlet updateParams: (KeetaAnchorQueueRequestID | string | number | null)[];\n\n\t\t\tif (oldStatus) {\n\t\t\t\tupdateQuery = `UPDATE ${this.tableNameEntries}\n\t\t\t\t               SET status = $1, updated = $2, worker = $3, failures = $4, last_error = $5, output = $6\n\t\t\t\t               WHERE id = $7 AND path = $8 AND status = $9`;\n\t\t\t\tupdateParams = [status, currentTime, workerValue, newFailures, newLastError, newOutput, id, this.pathStr, oldStatus];\n\t\t\t} else {\n\t\t\t\tupdateQuery = `UPDATE ${this.tableNameEntries}\n\t\t\t\t               SET status = $1, updated = $2, worker = $3, failures = $4, last_error = $5, output = $6\n\t\t\t\t               WHERE id = $7 AND path = $8`;\n\t\t\t\tupdateParams = [status, currentTime, workerValue, newFailures, newLastError, newOutput, id, this.pathStr];\n\t\t\t}\n\n\t\t\tconst result = await client.query(updateQuery, updateParams);\n\n\t\t\tawait this.toctouDelay?.();\n\n\t\t\tif (oldStatus && result.rowCount === 0) {\n\t\t\t\tconst currentEntry = await client.query<{ status: KeetaAnchorQueueStatus }>(`SELECT status FROM ${this.tableNameEntries} WHERE id = $1 AND path = $2`, [id, this.pathStr]);\n\t\t\t\tconst currentStatus = currentEntry.rows[0]?.status;\n\t\t\t\tif (currentEntry.rows.length > 0) {\n\t\t\t\t\tif (currentStatus === undefined) {\n\t\t\t\t\t\tthrow(new Error(`internal error: could not retrieve current status for request with ID ${String(id)}`));\n\t\t\t\t\t}\n\t\t\t\t\tthrow(new Errors.IncorrectStateAssertedError(id, oldStatus, currentStatus));\n\t\t\t\t} else {\n\t\t\t\t\tthrow(new Error(`Request with ID ${String(id)} not found`));\n\t\t\t\t}\n\t\t\t}\n\t\t}));\n\t}\n\n\tasync get(id: KeetaAnchorQueueRequestID): Promise<KeetaAnchorQueueEntry<QueueRequest, QueueResult> | null> {\n\t\treturn(await this.dbTransaction('get', async (client): Promise<KeetaAnchorQueueEntry<QueueRequest, QueueResult> | null> => {\n\t\t\tconst row = await client.query<QueueEntryRow>(\n\t\t\t\t`SELECT id, request, output, last_error, status, created, updated, worker, failures\n\t\t\t\t FROM ${this.tableNameEntries} WHERE id = $1 AND path = $2`,\n\t\t\t\t[id, this.pathStr]\n\t\t\t);\n\n\t\t\tif (row.rows.length === 0) {\n\t\t\t\treturn(null);\n\t\t\t}\n\n\t\t\tconst entry = row.rows[0];\n\t\t\tif (!entry) {\n\t\t\t\treturn(null);\n\t\t\t}\n\n\t\t\tconst idempotentRows = await client.query<IdempotentRow>(\n\t\t\t\t`SELECT idempotent_id FROM ${this.tableNameIdempotentKeys} WHERE entry_id = $1 AND path = $2`,\n\t\t\t\t[id, this.pathStr]\n\t\t\t);\n\n\t\t\tconst idempotentKeys = idempotentRows.rows.length > 0\n\t\t\t\t? new Set(idempotentRows.rows.map(function(idempotentRow: IdempotentRow) {\n\t\t\t\t\treturn(ConvertStringToRequestID(idempotentRow.idempotent_id));\n\t\t\t\t}))\n\t\t\t\t: undefined;\n\n\t\t\treturn({\n\t\t\t\tid: ConvertStringToRequestID(entry.id),\n\t\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\t\trequest: JSON.parse(entry.request) as QueueRequest,\n\t\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\t\toutput: entry.output ? JSON.parse(entry.output) as QueueResult : null,\n\t\t\t\tlastError: entry.last_error,\n\t\t\t\tstatus: entry.status,\n\t\t\t\tcreated: new Date(Number(entry.created)),\n\t\t\t\tupdated: new Date(Number(entry.updated)),\n\t\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\t\tworker: entry.worker as unknown as KeetaAnchorQueueWorkerID | null,\n\t\t\t\tfailures: entry.failures,\n\t\t\t\tidempotentKeys: idempotentKeys\n\t\t\t});\n\t\t}));\n\t}\n\n\tasync query(filter?: KeetaAnchorQueueFilter): Promise<KeetaAnchorQueueEntry<QueueRequest, QueueResult>[]> {\n\t\treturn(await this.dbTransaction('query', async (client, logger): Promise<KeetaAnchorQueueEntry<QueueRequest, QueueResult>[]> => {\n\t\t\tlogger?.debug(`Querying queue with id ${this.id} with filter:`, filter);\n\n\t\t\tconst conditions: string[] = [];\n\t\t\tconst params: (string | number)[] = [];\n\t\t\tlet paramIndex = 1;\n\n\t\t\tconditions.push(`path = $${paramIndex++}`);\n\t\t\tparams.push(this.pathStr);\n\n\t\t\tif (filter?.status) {\n\t\t\t\tconditions.push(`status = $${paramIndex++}`);\n\t\t\t\tparams.push(filter.status);\n\t\t\t}\n\n\t\t\tif (filter?.updatedBefore) {\n\t\t\t\tconditions.push(`updated < $${paramIndex++}`);\n\t\t\t\tparams.push(filter.updatedBefore.getTime());\n\t\t\t}\n\n\t\t\tlet query = `SELECT id, request, output, last_error, status, created, updated, worker, failures FROM ${this.tableNameEntries}`;\n\n\t\t\tif (conditions.length > 0) {\n\t\t\t\tquery += ' WHERE ' + conditions.join(' AND ');\n\t\t\t}\n\n\t\t\t// Use random ordering to prevent multiple workers from contending for the same rows\n\t\t\t// This spreads the load when multiple workers query simultaneously\n\t\t\tquery += ' ORDER BY RANDOM()';\n\n\t\t\tif (filter?.limit !== undefined) {\n\t\t\t\tquery += ` LIMIT $${paramIndex++}`;\n\t\t\t\tparams.push(filter.limit);\n\t\t\t}\n\n\t\t\tconst rows = await client.query<QueueEntryRow>(query, params);\n\n\t\t\tconst entries: KeetaAnchorQueueEntry<QueueRequest, QueueResult>[] = [];\n\n\t\t\tfor (const row of rows.rows) {\n\t\t\t\tconst idempotentRows = await client.query<IdempotentRow>(\n\t\t\t\t\t`SELECT idempotent_id FROM ${this.tableNameIdempotentKeys} WHERE entry_id = $1 AND path = $2`,\n\t\t\t\t\t[row.id, this.pathStr]\n\t\t\t\t);\n\n\t\t\t\tconst idempotentKeys = idempotentRows.rows.length > 0\n\t\t\t\t\t? new Set(idempotentRows.rows.map(function(idempotentRow: IdempotentRow) {\n\t\t\t\t\t\treturn(ConvertStringToRequestID(idempotentRow.idempotent_id));\n\t\t\t\t\t}))\n\t\t\t\t\t: undefined;\n\n\t\t\t\tentries.push({\n\t\t\t\t\tid: ConvertStringToRequestID(row.id),\n\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\t\t\trequest: JSON.parse(row.request) as QueueRequest,\n\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\t\t\toutput: row.output ? JSON.parse(row.output) as QueueResult : null,\n\t\t\t\t\tlastError: row.last_error,\n\t\t\t\t\tstatus: row.status,\n\t\t\t\t\tcreated: new Date(Number(row.created)),\n\t\t\t\t\tupdated: new Date(Number(row.updated)),\n\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\t\t\tworker: row.worker as unknown as KeetaAnchorQueueWorkerID | null,\n\t\t\t\t\tfailures: row.failures,\n\t\t\t\t\tidempotentKeys: idempotentKeys\n\t\t\t\t});\n\t\t\t}\n\n\t\t\tlogger?.debug(`Queried queue with id ${this.id} with filter:`, filter, '-- found', entries.length, 'entries');\n\n\t\t\treturn(entries);\n\t\t}));\n\t}\n\n\tasync partition(path: string) : Promise<KeetaAnchorQueueStorageDriver<QueueRequest, QueueResult>> {\n\t\tthis.methodLogger('partition')?.debug(`Creating partitioned queue storage driver for path: ${path}`);\n\n\t\tif (this.poolInternal === null) {\n\t\t\tthrow(new Error('Asked to partition but the instance has been destroyed'));\n\t\t}\n\n\t\tconst retval = new KeetaAnchorQueueStorageDriverPostgres<QueueRequest, QueueResult>({\n\t\t\tid: `${this.id}::${path}`,\n\t\t\tlogger: this.logger,\n\t\t\tpool: this.poolInternal,\n\t\t\tpath: [...this.path, path],\n\t\t\ttablePrefix: this.tablePrefix\n\t\t});\n\n\t\treturn(retval);\n\t}\n\n\tasync destroy(): Promise<void> {\n\t\tthis.methodLogger('destroy')?.debug('Destroying instance');\n\n\t\tthis.poolInternal = null;\n\t}\n\n\tasync [Symbol.asyncDispose](): Promise<void> {\n\t\treturn(await this.destroy());\n\t}\n\n\t/** @internal */\n\t_Testing(key: string): {\n\t\tsetToctouDelay(delay: number): void;\n\t\tunsetToctouDelay(): void;\n\t\tgetSerializationRetryCount(): number;\n\t\tresetSerializationRetryCount(): void;\n\t\tenableDebugForceIndexScan(): void;\n\t\tdisableDebugForceIndexScan(): void;\n\t} {\n\t\tif (key !== 'bc81abf8-e43b-490b-b486-744fb49a5082') {\n\t\t\tthrow(new Error('This is a testing only method'));\n\t\t}\n\n\t\treturn({\n\t\t\tsetToctouDelay: (delay: number): void => {\n\t\t\t\tthis.toctouDelay = async (): Promise<void> => {\n\t\t\t\t\treturn(await asleep(delay));\n\t\t\t\t};\n\t\t\t},\n\t\t\tunsetToctouDelay: (): void => {\n\t\t\t\tthis.toctouDelay = undefined;\n\t\t\t},\n\t\t\tgetSerializationRetryCount: (): number => {\n\t\t\t\treturn(this.serializationRetryCount);\n\t\t\t},\n\t\t\tresetSerializationRetryCount: (): void => {\n\t\t\t\tthis.serializationRetryCount = 0;\n\t\t\t},\n\t\t\tenableDebugForceIndexScan: (): void => {\n\t\t\t\tthis.debugForceIndexScan = true;\n\t\t\t},\n\t\t\tdisableDebugForceIndexScan: (): void => {\n\t\t\t\tthis.debugForceIndexScan = false;\n\t\t\t}\n\t\t});\n\t}\n}\n"]}

package/lib/resolver.d.ts

@@ -192,7 +192,7 @@ type ServiceMetadata = {
         assetMovement?: {
             [id: string]: SharedAnchorMetadataLegalExtension & {
                 operations: {
-                    [Operation in ('initiateTransfer' | 'executeTransfer' | 'getTransferStatus' | 'initiatePersistentForwardingTemplate' | 'createPersistentForwardingTemplate' | 'listPersistentForwardingTemplate' | 'createPersistentForwarding' | 'listPersistentForwarding' | 'listTransactions' | 'shareKYC')]?: ServiceMetadataEndpoint;
+                    [Operation in ('initiateTransfer' | 'simulateTransfer' | 'executeTransfer' | 'getTransferStatus' | 'initiatePersistentForwardingTemplate' | 'createPersistentForwardingTemplate' | 'listPersistentForwardingTemplate' | 'createPersistentForwarding' | 'listPersistentForwarding' | 'listTransactions' | 'shareKYC')]?: ServiceMetadataEndpoint;
                 };
                 supportedAssets: SupportedAssetsMetadata[];
                 locationMetadata?: AnchorCustomLocationMetadata | undefined;