@keetanetwork/anchor 0.0.35 → 0.0.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/queue/drivers/queue_postgres.d.ts +1 -0
- package/lib/queue/drivers/queue_postgres.d.ts.map +1 -1
- package/lib/queue/drivers/queue_postgres.js +27 -14
- package/lib/queue/drivers/queue_postgres.js.map +1 -1
- package/lib/queue/drivers/queue_redis.d.ts +1 -0
- package/lib/queue/drivers/queue_redis.d.ts.map +1 -1
- package/lib/queue/drivers/queue_redis.js +95 -87
- package/lib/queue/drivers/queue_redis.js.map +1 -1
- package/lib/queue/drivers/queue_sqlite3.d.ts +1 -0
- package/lib/queue/drivers/queue_sqlite3.d.ts.map +1 -1
- package/lib/queue/drivers/queue_sqlite3.js +26 -13
- package/lib/queue/drivers/queue_sqlite3.js.map +1 -1
- package/lib/queue/index.d.ts.map +1 -1
- package/lib/queue/index.js.map +1 -1
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
|
@@ -9,6 +9,7 @@ export default class KeetaAnchorQueueStorageDriverPostgres<QueueRequest extends
|
|
|
9
9
|
readonly id: string;
|
|
10
10
|
readonly path: string[];
|
|
11
11
|
private readonly pathStr;
|
|
12
|
+
private toctouDelay;
|
|
12
13
|
constructor(options: NonNullable<ConstructorParameters<KeetaAnchorQueueStorageDriverConstructor<QueueRequest, QueueResult>>[0]> & {
|
|
13
14
|
pool: () => Promise<pg.Pool>;
|
|
14
15
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue_postgres.d.ts","sourceRoot":"","sources":["../../../../src/lib/queue/drivers/queue_postgres.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,6BAA6B,EAC7B,wCAAwC,EACxC,uBAAuB,EACvB,yBAAyB,EACzB,qBAAqB,EACrB,0BAA0B,EAC1B,kCAAkC,EAClC,sBAAsB,EACtB,sBAAsB,EAEtB,MAAM,aAAa,CAAC;AAWrB,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAE5D,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI,CAAC;AAkB9B,MAAM,CAAC,OAAO,OAAO,qCAAqC,CAAC,YAAY,SAAS,gBAAgB,GAAG,gBAAgB,EAAE,WAAW,SAAS,gBAAgB,GAAG,gBAAgB,CAAE,YAAW,6BAA6B,CAAC,YAAY,EAAE,WAAW,CAAC;IAChP,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;IAC5C,OAAO,CAAC,YAAY,CAAyC;IAC7D,OAAO,CAAC,uBAAuB,CAAiC;IAEhE,QAAQ,CAAC,IAAI,2CAA2C;IACxD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAM;IAC7B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;
|
|
1
|
+
{"version":3,"file":"queue_postgres.d.ts","sourceRoot":"","sources":["../../../../src/lib/queue/drivers/queue_postgres.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,6BAA6B,EAC7B,wCAAwC,EACxC,uBAAuB,EACvB,yBAAyB,EACzB,qBAAqB,EACrB,0BAA0B,EAC1B,kCAAkC,EAClC,sBAAsB,EACtB,sBAAsB,EAEtB,MAAM,aAAa,CAAC;AAWrB,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAE5D,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI,CAAC;AAkB9B,MAAM,CAAC,OAAO,OAAO,qCAAqC,CAAC,YAAY,SAAS,gBAAgB,GAAG,gBAAgB,EAAE,WAAW,SAAS,gBAAgB,GAAG,gBAAgB,CAAE,YAAW,6BAA6B,CAAC,YAAY,EAAE,WAAW,CAAC;IAChP,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;IAC5C,OAAO,CAAC,YAAY,CAAyC;IAC7D,OAAO,CAAC,uBAAuB,CAAiC;IAEhE,QAAQ,CAAC,IAAI,2CAA2C;IACxD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAM;IAC7B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,WAAW,CAAgD;gBAEvD,OAAO,EAAE,WAAW,CAAC,qBAAqB,CAAC,wCAAwC,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;QAAE,IAAI,EAAE,MAAM,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;KAAE;YAWrJ,sBAAsB;IAyDpC,OAAO,CAAC,YAAY;YASN,YAAY;YA2CZ,eAAe;YAiBf,aAAa;IAoCrB,GAAG,CAAC,OAAO,EAAE,uBAAuB,CAAC,YAAY,CAAC,EAAE,IAAI,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,yBAAyB,CAAC;IA6D1H,SAAS,CAAC,EAAE,EAAE,yBAAyB,EAAE,MAAM,EAAE,sBAAsB,EAAE,SAAS,CAAC,EAAE,kCAAkC,CAAC,WAAW,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAyDpJ,GAAG,CAAC,EAAE,EAAE,yBAAyB,GAAG,OAAO,CAAC,qBAAqB,CAAC,YAAY,EAAE,WAAW,CAAC,GAAG,IAAI,CAAC;IA8CpG,KAAK,CAAC,MAAM,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,qBAAqB,CAAC,YAAY,EAAE,WAAW,CAAC,EAAE,CAAC;IAuEnG,SAAS,CAAC,IAAI,EAAE,MAAM,GAAI,OAAO,CAAC,6BAA6B,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;IAiB3F,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAMxB,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC;CAwB5C"}
|
|
@@ -9,6 +9,7 @@ export default class KeetaAnchorQueueStorageDriverPostgres {
|
|
|
9
9
|
id;
|
|
10
10
|
path = [];
|
|
11
11
|
pathStr;
|
|
12
|
+
toctouDelay = undefined;
|
|
12
13
|
constructor(options) {
|
|
13
14
|
this.id = options?.id ?? crypto.randomUUID();
|
|
14
15
|
this.logger = options?.logger;
|
|
@@ -128,7 +129,7 @@ export default class KeetaAnchorQueueStorageDriverPostgres {
|
|
|
128
129
|
const client = await pool.connect();
|
|
129
130
|
try {
|
|
130
131
|
logger?.debug('Starting DB transaction');
|
|
131
|
-
await client.query('BEGIN');
|
|
132
|
+
await client.query('BEGIN TRANSACTION ISOLATION LEVEL SERIALIZABLE');
|
|
132
133
|
logger?.debug('DB transaction started');
|
|
133
134
|
const retval = await fn(client, logger);
|
|
134
135
|
logger?.debug('Committing DB transaction');
|
|
@@ -162,6 +163,7 @@ export default class KeetaAnchorQueueStorageDriverPostgres {
|
|
|
162
163
|
logger?.debug(`Request with id ${String(entryID)} already exists, ignoring`);
|
|
163
164
|
return (entryID);
|
|
164
165
|
}
|
|
166
|
+
await this.toctouDelay?.();
|
|
165
167
|
}
|
|
166
168
|
const idempotentIDs = info?.idempotentKeys;
|
|
167
169
|
if (idempotentIDs) {
|
|
@@ -175,6 +177,7 @@ export default class KeetaAnchorQueueStorageDriverPostgres {
|
|
|
175
177
|
if (matchingIdempotentEntries.size !== 0) {
|
|
176
178
|
throw (new Errors.IdempotentExistsError('One or more idempotent entries already exist in the queue', matchingIdempotentEntries));
|
|
177
179
|
}
|
|
180
|
+
await this.toctouDelay?.();
|
|
178
181
|
}
|
|
179
182
|
entryID ??= ConvertStringToRequestID(crypto.randomUUID());
|
|
180
183
|
logger?.debug(`Enqueuing request with id ${String(entryID)}`);
|
|
@@ -184,21 +187,13 @@ export default class KeetaAnchorQueueStorageDriverPostgres {
|
|
|
184
187
|
* The status to use for the new entry
|
|
185
188
|
*/
|
|
186
189
|
const status = info?.status ?? 'pending';
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
await client.query('INSERT INTO queue_idempotent_keys (entry_id, path, idempotent_id) VALUES ($1, $2, $3)', [entryID, this.pathStr, idempotentID]);
|
|
193
|
-
}
|
|
190
|
+
await client.query(`INSERT INTO queue_entries (id, path, request, output, last_error, status, created, updated, worker, failures)
|
|
191
|
+
VALUES ($1, $2, $3, NULL, NULL, $4, $5, $6, NULL, 0)`, [entryID, this.pathStr, requestJSON, status, currentTime, currentTime]);
|
|
192
|
+
if (idempotentIDs && idempotentIDs.size > 0) {
|
|
193
|
+
for (const idempotentID of idempotentIDs) {
|
|
194
|
+
await client.query('INSERT INTO queue_idempotent_keys (entry_id, path, idempotent_id) VALUES ($1, $2, $3)', [entryID, this.pathStr, idempotentID]);
|
|
194
195
|
}
|
|
195
196
|
}
|
|
196
|
-
catch (error) {
|
|
197
|
-
if (error instanceof Error && error.message.includes('duplicate key') && idempotentIDs) {
|
|
198
|
-
throw (new Errors.IdempotentExistsError('One or more idempotent entries already exist in the queue', idempotentIDs));
|
|
199
|
-
}
|
|
200
|
-
throw (error);
|
|
201
|
-
}
|
|
202
197
|
return (entryID);
|
|
203
198
|
}));
|
|
204
199
|
}
|
|
@@ -209,6 +204,7 @@ export default class KeetaAnchorQueueStorageDriverPostgres {
|
|
|
209
204
|
if (existingEntry.rows.length === 0) {
|
|
210
205
|
throw (new Error(`Request with ID ${String(id)} not found`));
|
|
211
206
|
}
|
|
207
|
+
await this.toctouDelay?.();
|
|
212
208
|
const currentEntry = existingEntry.rows[0];
|
|
213
209
|
if (!currentEntry) {
|
|
214
210
|
throw (new Error(`Request with ID ${String(id)} not found`));
|
|
@@ -234,6 +230,7 @@ export default class KeetaAnchorQueueStorageDriverPostgres {
|
|
|
234
230
|
updateParams = [status, currentTime, workerValue, newFailures, newLastError, newOutput, id, this.pathStr];
|
|
235
231
|
}
|
|
236
232
|
const result = await client.query(updateQuery, updateParams);
|
|
233
|
+
await this.toctouDelay?.();
|
|
237
234
|
if (oldStatus && result.rowCount === 0) {
|
|
238
235
|
const currentEntry = await client.query('SELECT status FROM queue_entries WHERE id = $1 AND path = $2', [id, this.pathStr]);
|
|
239
236
|
const currentStatus = currentEntry.rows[0]?.status;
|
|
@@ -356,5 +353,21 @@ export default class KeetaAnchorQueueStorageDriverPostgres {
|
|
|
356
353
|
async [Symbol.asyncDispose]() {
|
|
357
354
|
return (await this.destroy());
|
|
358
355
|
}
|
|
356
|
+
/** @internal */
|
|
357
|
+
_Testing(key) {
|
|
358
|
+
if (key !== 'bc81abf8-e43b-490b-b486-744fb49a5082') {
|
|
359
|
+
throw (new Error('This is a testing only method'));
|
|
360
|
+
}
|
|
361
|
+
return ({
|
|
362
|
+
setToctouDelay: (delay) => {
|
|
363
|
+
this.toctouDelay = async () => {
|
|
364
|
+
return (await asleep(delay));
|
|
365
|
+
};
|
|
366
|
+
},
|
|
367
|
+
unsetToctouDelay: () => {
|
|
368
|
+
this.toctouDelay = undefined;
|
|
369
|
+
}
|
|
370
|
+
});
|
|
371
|
+
}
|
|
359
372
|
}
|
|
360
373
|
//# sourceMappingURL=queue_postgres.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue_postgres.js","sourceRoot":"","sources":["../../../../src/lib/queue/drivers/queue_postgres.ts"],"names":[],"mappings":"AAYA,OAAO,EACN,YAAY,EACZ,mBAAmB,EACnB,wBAAwB,EACxB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAuB/C,MAAM,CAAC,OAAO,OAAO,qCAAqC;IACxC,MAAM,CAAqB;IACpC,YAAY,GAAoC,IAAI,CAAC;IACrD,uBAAuB,GAA4B,IAAI,CAAC;IAEvD,IAAI,GAAG,uCAAuC,CAAC;IAC/C,EAAE,CAAS;IACX,IAAI,GAAa,EAAE,CAAC;IACZ,OAAO,CAAS;IAEjC,YAAY,OAAuJ;QAClK,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,EAAE,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAC7C,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,CAAA;QAC7B,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;QACjC,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEzB,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC9E,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAAC,IAAa;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,wBAAwB,CAAC,CAAC;QAE3D,IAAI,IAAI,CAAC,uBAAuB,KAAK,IAAI,EAAE,CAAC;YAC3C,MAAM,EAAE,KAAK,CAAC,qFAAqF,CAAC,CAAC;YAErG,MAAM,IAAI,CAAC,uBAAuB,CAAC;YACnC,OAAM,CAAC,IAAI,CAAC,CAAC;QACd,CAAC;QAED,IAAI,CAAC,uBAAuB,GAAG,CAAC,KAAK,IAAI,EAAE;YAC1C,MAAM,EAAE,KAAK,CAAC,iDAAiD,CAAC,CAAC;YAEjE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACpC,IAAI,CAAC;gBACJ,MAAM,MAAM,CAAC,KAAK,CAAC;;;;;;;;;;;;;OAahB,CAAC,CAAC;gBAEL,MAAM,MAAM,CAAC,KAAK,CAAC;;;;;;;;OAQhB,CAAC,CAAC;gBAEL,MAAM,MAAM,CAAC,KAAK,CAAC,8EAA8E,CAAC,CAAC;gBACnG,MAAM,MAAM,CAAC,KAAK,CAAC,gFAAgF,CAAC,CAAC;gBACrG,MAAM,MAAM,CAAC,KAAK,CAAC,4GAA4G,CAAC,CAAC;YAClI,CAAC;oBAAS,CAAC;gBACV,MAAM,CAAC,OAAO,EAAE,CAAC;YAClB,CAAC;YAED,MAAM,EAAE,KAAK,CAAC,6DAA6D,CAAC,CAAC;YAE7E,OAAM,CAAC,IAAI,CAAC,CAAC;QACd,CAAC,CAAC,EAAE,CAAC;QAEL,MAAM,IAAI,CAAC,uBAAuB,CAAC;QAEnC,OAAM,CAAC,IAAI,CAAC,CAAC;IACd,CAAC;IAEO,YAAY,CAAC,MAAc;QAClC,OAAM,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE;YAChC,KAAK,EAAE,uCAAuC;YAC9C,IAAI,EAAE,yCAAyC;YAC/C,MAAM,EAAE,MAAM;YACd,UAAU,EAAE,IAAI,CAAC,EAAE;SACnB,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,YAAY,CAAI,EAAoB;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC;QAEjD,IAAI,SAAkB,CAAC;QACvB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC;YACzC,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;gBAChC,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,EAAE,KAAK,CAAC,kEAAkE,CAAC,CAAC;gBAE7G,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;oBAC7B,+DAA+D;oBAC/D,MAAK,CAAC,SAAS,CAAC,CAAC;gBAClB,CAAC;gBACD,MAAK,CAAC,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC,CAAC;YACjE,CAAC;YAED,IAAI,CAAC;gBACJ,OAAM,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACpB,CAAC;YAAC,OAAO,KAAc,EAAE,CAAC;gBACzB,SAAS,GAAG,KAAK,CAAC;gBAElB,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;oBAC5B,MAAM,SAAS,GAAG,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;oBACtD,IAAI,SAAS,KAAK,OAAO,IAAI,SAAS,KAAK,OAAO,EAAE,CAAC;wBACpD,MAAM,EAAE,KAAK,CAAC,4CAA4C,CAAC,CAAC;wBAE5D,MAAM,UAAU,GAAG,GAAG,CAAC;wBACvB,MAAM,UAAU,GAAG,MAAM,CAAC;wBAC1B,MAAM,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,UAAU,EAAE,CAAC,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;wBACjF,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,mBAAmB,CAAC,CAAC,GAAG,UAAU,CAAC;wBAE/E,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,EAAE,KAAK,CAAC,4BAA4B,OAAO,cAAc,KAAK,QAAQ,EAAE,IAAI,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC;wBAC5H,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC;wBAEtB,SAAS;oBACV,CAAC;gBACF,CAAC;gBAED,MAAK,CAAC,KAAK,CAAC,CAAC;YACd,CAAC;QACF,CAAC;QACD,MAAK,CAAC,SAAS,CAAC,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,eAAe;QAC5B,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;YAChC,MAAK,CAAC,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC,CAAC;QAC1D,CAAC;QAED,OAAM,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE;YACzC,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;gBAChC,MAAK,CAAC,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC,CAAC;YAC1D,CAAC;YAED,IAAI,CAAC,YAAY,CAAC,iBAAiB,CAAC,EAAE,KAAK,CAAC,iBAAiB,CAAC,CAAC;YAC/D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAEvC,OAAM,CAAC,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,aAAa,CAAI,SAAiB,EAAE,EAAqE;QACtH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAE5C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK;YAC3C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YAEpC,IAAI,CAAC;gBACJ,MAAM,EAAE,KAAK,CAAC,yBAAyB,CAAC,CAAC;gBACzC,MAAM,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBAC5B,MAAM,EAAE,KAAK,CAAC,wBAAwB,CAAC,CAAC;gBAExC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBAExC,MAAM,EAAE,KAAK,CAAC,2BAA2B,CAAC,CAAC;gBAC3C,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBAC7B,MAAM,EAAE,KAAK,CAAC,0BAA0B,CAAC,CAAC;gBAE1C,OAAM,CAAC,MAAM,CAAC,CAAC;YAChB,CAAC;YAAC,OAAO,KAAc,EAAE,CAAC;gBACzB,IAAI,CAAC;oBACJ,MAAM,EAAE,KAAK,CAAC,2CAA2C,EAAE,KAAK,CAAC,CAAC;oBAClE,MAAM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;oBAC/B,MAAM,EAAE,KAAK,CAAC,4BAA4B,CAAC,CAAC;gBAC7C,CAAC;gBAAC,MAAM,CAAC;oBACR,MAAM,EAAE,KAAK,CAAC,sCAAsC,CAAC,CAAC;gBACvD,CAAC;gBACD,MAAK,CAAC,KAAK,CAAC,CAAC;YACd,CAAC;oBAAS,CAAC;gBACV,MAAM,CAAC,OAAO,EAAE,CAAC;YAClB,CAAC;QACF,CAAC,CAAC,CAAC;QAEH,OAAM,CAAC,MAAM,CAAC,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,OAA8C,EAAE,IAAiC;QAC1F,OAAM,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAsC,EAAE;YACnG,IAAI,OAAO,GAAG,wBAAwB,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACjD,IAAI,OAAO,EAAE,CAAC;gBACb,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,KAAK,CAAiB,0DAA0D,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC9I,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACnC,MAAM,EAAE,KAAK,CAAC,mBAAmB,MAAM,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;oBAC7E,OAAM,CAAC,OAAO,CAAC,CAAC;gBACjB,CAAC;YACF,CAAC;YAED,MAAM,aAAa,GAAG,IAAI,EAAE,cAAc,CAAC;YAC3C,IAAI,aAAa,EAAE,CAAC;gBACnB,MAAM,yBAAyB,GAAG,IAAI,GAAG,EAA6B,CAAC;gBACvE,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;oBAC1C,MAAM,qBAAqB,GAAG,MAAM,MAAM,CAAC,KAAK,CAC/C,wFAAwF,EACxF,CAAC,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,CAC5B,CAAC;oBACF,IAAI,qBAAqB,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC3C,yBAAyB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;oBAC7C,CAAC;gBACF,CAAC;gBAED,IAAI,yBAAyB,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;oBAC1C,MAAK,CAAC,IAAI,MAAM,CAAC,qBAAqB,CAAC,2DAA2D,EAAE,yBAAyB,CAAC,CAAC,CAAC;gBACjI,CAAC;YACF,CAAC;YAED,OAAO,KAAK,wBAAwB,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;YAE1D,MAAM,EAAE,KAAK,CAAC,6BAA6B,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAE9D,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC/B,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YAE5C;;eAEG;YACH,MAAM,MAAM,GAAG,IAAI,EAAE,MAAM,IAAI,SAAS,CAAC;YAEzC,IAAI,CAAC;gBACJ,MAAM,MAAM,CAAC,KAAK,CACjB;2DACsD,EACtD,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,CAAC,CACtE,CAAC;gBAEF,IAAI,aAAa,IAAI,aAAa,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;oBAC7C,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;wBAC1C,MAAM,MAAM,CAAC,KAAK,CAAC,uFAAuF,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;oBACpJ,CAAC;gBACF,CAAC;YACF,CAAC;YAAC,OAAO,KAAc,EAAE,CAAC;gBACzB,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,IAAI,aAAa,EAAE,CAAC;oBACxF,MAAK,CAAC,IAAI,MAAM,CAAC,qBAAqB,CAAC,2DAA2D,EAAE,aAAa,CAAC,CAAC,CAAC;gBACrH,CAAC;gBACD,MAAK,CAAC,KAAK,CAAC,CAAC;YACd,CAAC;YAED,OAAM,CAAC,OAAO,CAAC,CAAC;QACjB,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,EAA6B,EAAE,MAA8B,EAAE,SAA2D;QACzI,MAAM,EAAE,SAAS,EAAE,GAAG,SAAS,IAAI,EAAE,CAAC;QAEtC,OAAM,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAiB,EAAE;YACpF,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,KAAK,CAAyG,4FAA4F,EAAE,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;YACnQ,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrC,MAAK,CAAC,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC;YAC7D,CAAC;YAED,MAAM,YAAY,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC3C,IAAI,CAAC,YAAY,EAAE,CAAC;gBACnB,MAAK,CAAC,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC;YAC7D,CAAC;YAED,MAAM,QAAQ,GAAG,mBAAmB,CAAc,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YAC/F,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YAC/C,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC;YACpC,MAAM,WAAW,GAAG,QAAQ,CAAC,QAAQ,IAAI,YAAY,CAAC,QAAQ,CAAC;YAC/D,MAAM,YAAY,GAAG,QAAQ,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,UAAU,CAAC;YACrG,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC;YAExG,IAAI,WAAmB,CAAC;YACxB,IAAI,YAAoE,CAAC;YAEzE,IAAI,SAAS,EAAE,CAAC;gBACf,WAAW,GAAG;;+DAE6C,CAAC;gBAC5D,YAAY,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,EAAE,EAAE,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;YACtH,CAAC;iBAAM,CAAC;gBACP,WAAW,GAAG;;+CAE6B,CAAC;gBAC5C,YAAY,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YAC3G,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;YAE7D,IAAI,SAAS,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;gBACxC,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,KAAK,CAAqC,8DAA8D,EAAE,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;gBAChK,MAAM,aAAa,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC;gBACnD,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAClC,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;wBACjC,MAAK,CAAC,IAAI,KAAK,CAAC,yEAAyE,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;oBACzG,CAAC;oBACD,MAAK,CAAC,IAAI,MAAM,CAAC,2BAA2B,CAAC,EAAE,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC,CAAC;gBAC7E,CAAC;qBAAM,CAAC;oBACP,MAAK,CAAC,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC;gBAC7D,CAAC;YACF,CAAC;QACF,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EAA6B;QACtC,OAAM,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAoE,EAAE;YACzH,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,KAAK,CAC7B;oDACgD,EAChD,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAClB,CAAC;YAEF,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3B,OAAM,CAAC,IAAI,CAAC,CAAC;YACd,CAAC;YAED,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC1B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACZ,OAAM,CAAC,IAAI,CAAC,CAAC;YACd,CAAC;YAED,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,KAAK,CACxC,mFAAmF,EACnF,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAClB,CAAC;YAEF,MAAM,cAAc,GAAG,cAAc,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;gBACpD,CAAC,CAAC,IAAI,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,UAAS,aAA4B;oBACtE,OAAM,CAAC,wBAAwB,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC;gBAC/D,CAAC,CAAC,CAAC;gBACH,CAAC,CAAC,SAAS,CAAC;YAEb,OAAM,CAAC;gBACN,EAAE,EAAE,wBAAwB,CAAC,KAAK,CAAC,EAAE,CAAC;gBACtC,yEAAyE;gBACzE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAiB;gBAClD,yEAAyE;gBACzE,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAgB,CAAC,CAAC,CAAC,IAAI;gBACrE,SAAS,EAAE,KAAK,CAAC,UAAU;gBAC3B,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,OAAO,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBACxC,OAAO,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBACxC,yEAAyE;gBACzE,MAAM,EAAE,KAAK,CAAC,MAAoD;gBAClE,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,cAAc,EAAE,cAAc;aAC9B,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,MAA+B;QAC1C,OAAM,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAA+D,EAAE;YAC9H,MAAM,EAAE,KAAK,CAAC,0BAA0B,IAAI,CAAC,EAAE,eAAe,EAAE,MAAM,CAAC,CAAC;YAExE,MAAM,UAAU,GAAa,EAAE,CAAC;YAChC,MAAM,MAAM,GAAwB,EAAE,CAAC;YACvC,IAAI,UAAU,GAAG,CAAC,CAAC;YAEnB,UAAU,CAAC,IAAI,CAAC,WAAW,UAAU,EAAE,EAAE,CAAC,CAAC;YAC3C,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAE1B,IAAI,MAAM,EAAE,MAAM,EAAE,CAAC;gBACpB,UAAU,CAAC,IAAI,CAAC,aAAa,UAAU,EAAE,EAAE,CAAC,CAAC;gBAC7C,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5B,CAAC;YAED,IAAI,MAAM,EAAE,aAAa,EAAE,CAAC;gBAC3B,UAAU,CAAC,IAAI,CAAC,cAAc,UAAU,EAAE,EAAE,CAAC,CAAC;gBAC9C,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC;YAC7C,CAAC;YAED,IAAI,KAAK,GAAG,uGAAuG,CAAC;YAEpH,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,KAAK,IAAI,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC/C,CAAC;YAED,IAAI,MAAM,EAAE,KAAK,KAAK,SAAS,EAAE,CAAC;gBACjC,KAAK,IAAI,WAAW,UAAU,EAAE,EAAE,CAAC;gBACnC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC3B,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,KAAK,CAAgB,KAAK,EAAE,MAAM,CAAC,CAAC;YAE9D,MAAM,OAAO,GAAuD,EAAE,CAAC;YAEvE,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBAC7B,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,KAAK,CACxC,mFAAmF,EACnF,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CACtB,CAAC;gBAEF,MAAM,cAAc,GAAG,cAAc,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;oBACpD,CAAC,CAAC,IAAI,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,UAAS,aAA4B;wBACtE,OAAM,CAAC,wBAAwB,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC;oBAC/D,CAAC,CAAC,CAAC;oBACH,CAAC,CAAC,SAAS,CAAC;gBAEb,OAAO,CAAC,IAAI,CAAC;oBACZ,EAAE,EAAE,wBAAwB,CAAC,GAAG,CAAC,EAAE,CAAC;oBACpC,yEAAyE;oBACzE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAiB;oBAChD,yEAAyE;oBACzE,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAgB,CAAC,CAAC,CAAC,IAAI;oBACjE,SAAS,EAAE,GAAG,CAAC,UAAU;oBACzB,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,OAAO,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBACtC,OAAO,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBACtC,yEAAyE;oBACzE,MAAM,EAAE,GAAG,CAAC,MAAoD;oBAChE,QAAQ,EAAE,GAAG,CAAC,QAAQ;oBACtB,cAAc,EAAE,cAAc;iBAC9B,CAAC,CAAC;YACJ,CAAC;YAED,MAAM,EAAE,KAAK,CAAC,yBAAyB,IAAI,CAAC,EAAE,eAAe,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAE9G,OAAM,CAAC,OAAO,CAAC,CAAC;QACjB,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAY;QAC3B,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,EAAE,KAAK,CAAC,uDAAuD,IAAI,EAAE,CAAC,CAAC;QAErG,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;YAChC,MAAK,CAAC,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC,CAAC;QAC5E,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,qCAAqC,CAA4B;YACnF,EAAE,EAAE,GAAG,IAAI,CAAC,EAAE,KAAK,IAAI,EAAE;YACzB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,IAAI,EAAE,IAAI,CAAC,YAAY;YACvB,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC;SAC1B,CAAC,CAAC;QAEH,OAAM,CAAC,MAAM,CAAC,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,OAAO;QACZ,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,qBAAqB,CAAC,CAAC;QAE3D,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QAC1B,OAAM,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IAC9B,CAAC;CACD","sourcesContent":["import type {\n\tKeetaAnchorQueueStorageDriver,\n\tKeetaAnchorQueueStorageDriverConstructor,\n\tKeetaAnchorQueueRequest,\n\tKeetaAnchorQueueRequestID,\n\tKeetaAnchorQueueEntry,\n\tKeetaAnchorQueueEntryExtra,\n\tKeetaAnchorQueueEntryAncillaryData,\n\tKeetaAnchorQueueStatus,\n\tKeetaAnchorQueueFilter,\n\tKeetaAnchorQueueWorkerID\n} from '../index.ts';\nimport {\n\tMethodLogger,\n\tManageStatusUpdates,\n\tConvertStringToRequestID\n} from '../internal.js';\nimport { Errors } from '../common.js';\n\nimport { asleep } from '../../utils/asleep.js';\n\nimport type { Logger } from '../../log/index.ts';\nimport type { JSONSerializable } from '../../utils/json.js';\n\nimport type * as pg from 'pg';\n\ntype QueueEntryRow = {\n\tid: string;\n\trequest: string;\n\toutput: string | null;\n\tlast_error: string | null;\n\tstatus: KeetaAnchorQueueStatus;\n\tcreated: Date;\n\tupdated: Date;\n\tworker: number | null;\n\tfailures: number;\n};\n\ntype IdempotentRow = {\n\tidempotent_id: string;\n};\n\nexport default class KeetaAnchorQueueStorageDriverPostgres<QueueRequest extends JSONSerializable = JSONSerializable, QueueResult extends JSONSerializable = JSONSerializable> implements KeetaAnchorQueueStorageDriver<QueueRequest, QueueResult> {\n\tprivate readonly logger: Logger | undefined;\n\tprivate poolInternal: (() => Promise<pg.Pool>) | null = null;\n\tprivate dbInitializationPromise: Promise<boolean> | null = null;\n\n\treadonly name = 'KeetaAnchorQueueStorageDriverPostgres';\n\treadonly id: string;\n\treadonly path: string[] = [];\n\tprivate readonly pathStr: string;\n\n\tconstructor(options: NonNullable<ConstructorParameters<KeetaAnchorQueueStorageDriverConstructor<QueueRequest, QueueResult>>[0]> & { pool: () => Promise<pg.Pool>; }) {\n\t\tthis.id = options?.id ?? crypto.randomUUID();\n\t\tthis.logger = options?.logger\n\t\tthis.poolInternal = options.pool;\n\t\tthis.path = options.path ?? [];\n\t\tthis.pathStr = ['root', ...this.path].join('.');\n\t\tObject.freeze(this.path);\n\n\t\tthis.methodLogger('new')?.debug('Initialized Postgres queue storage driver');\n\t}\n\n\tprivate async initializeDBConnection(pool: pg.Pool): Promise<pg.Pool> {\n\t\tconst logger = this.methodLogger('initializeDBConnection');\n\n\t\tif (this.dbInitializationPromise !== null) {\n\t\t\tlogger?.debug('DB schema initialization already in progress or completed, waiting for it to finish');\n\n\t\t\tawait this.dbInitializationPromise;\n\t\t\treturn(pool);\n\t\t}\n\n\t\tthis.dbInitializationPromise = (async () => {\n\t\t\tlogger?.debug('Initializing DB schema for queue storage driver');\n\n\t\t\tconst client = await pool.connect();\n\t\t\ttry {\n\t\t\t\tawait client.query(`\n\t\t\t\t\tCREATE TABLE IF NOT EXISTS queue_entries (\n\t\t\t\t\t\tid TEXT NOT NULL,\n\t\t\t\t\t\tpath TEXT NOT NULL,\n\t\t\t\t\t\trequest TEXT NOT NULL,\n\t\t\t\t\t\toutput TEXT,\n\t\t\t\t\t\tlast_error TEXT,\n\t\t\t\t\t\tstatus TEXT NOT NULL,\n\t\t\t\t\t\tcreated BIGINT NOT NULL,\n\t\t\t\t\t\tupdated BIGINT NOT NULL,\n\t\t\t\t\t\tworker BIGINT,\n\t\t\t\t\t\tfailures INTEGER NOT NULL DEFAULT 0,\n\t\t\t\t\t\tPRIMARY KEY (id, path)\n\t\t\t\t\t)`);\n\n\t\t\t\tawait client.query(`\n\t\t\t\t\tCREATE TABLE IF NOT EXISTS queue_idempotent_keys (\n\t\t\t\t\t\tentry_id TEXT NOT NULL,\n\t\t\t\t\t\tidempotent_id TEXT NOT NULL,\n\t\t\t\t\t\tpath TEXT NOT NULL,\n\t\t\t\t\t\tUNIQUE (idempotent_id, path),\n\t\t\t\t\t\tPRIMARY KEY (entry_id, idempotent_id, path),\n\t\t\t\t\t\tFOREIGN KEY (entry_id, path) REFERENCES queue_entries(id, path)\n\t\t\t\t\t)`);\n\n\t\t\t\tawait client.query('CREATE INDEX IF NOT EXISTS idx_queue_entries_status ON queue_entries(status)');\n\t\t\t\tawait client.query('CREATE INDEX IF NOT EXISTS idx_queue_entries_updated ON queue_entries(updated)');\n\t\t\t\tawait client.query('CREATE INDEX IF NOT EXISTS idx_queue_idempotent_keys_idempotent_id ON queue_idempotent_keys(idempotent_id)');\n\t\t\t} finally {\n\t\t\t\tclient.release();\n\t\t\t}\n\n\t\t\tlogger?.debug('Completed DB schema initialization for queue storage driver');\n\n\t\t\treturn(true);\n\t\t})();\n\n\t\tawait this.dbInitializationPromise;\n\n\t\treturn(pool);\n\t}\n\n\tprivate methodLogger(method: string): Logger | undefined {\n\t\treturn(MethodLogger(this.logger, {\n\t\t\tclass: 'KeetaAnchorQueueStorageDriverPostgres',\n\t\t\tfile: 'src/lib/queue/drivers/queue_postgres.ts',\n\t\t\tmethod: method,\n\t\t\tinstanceID: this.id\n\t\t}));\n\t}\n\n\tprivate async runWithRetry<T>(fn: () => Promise<T>): Promise<T> {\n\t\tconst logger = this.methodLogger('runWithRetry');\n\n\t\tlet lastError: unknown;\n\t\tfor (let retry = 0; retry < 16; retry++) {\n\t\t\tif (this.poolInternal === null) {\n\t\t\t\tthis.methodLogger('runWithRetry')?.debug('Aborting DB operation retries because the instance was destroyed');\n\n\t\t\t\tif (lastError !== undefined) {\n\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/only-throw-error\n\t\t\t\t\tthrow(lastError);\n\t\t\t\t}\n\t\t\t\tthrow(new Error('Aborting because the instance was destroyed'));\n\t\t\t}\n\n\t\t\ttry {\n\t\t\t\treturn(await fn());\n\t\t\t} catch (error: unknown) {\n\t\t\t\tlastError = error;\n\n\t\t\t\tif (error instanceof Error) {\n\t\t\t\t\tconst errorCode = 'code' in error ? error.code : null;\n\t\t\t\t\tif (errorCode === '40001' || errorCode === '40P01') {\n\t\t\t\t\t\tlogger?.debug('Serialization failure or deadlock detected');\n\n\t\t\t\t\t\tconst minBackoff = 100;\n\t\t\t\t\t\tconst maxBackoff = 30_000;\n\t\t\t\t\t\tconst backoffIntervalSize = Math.min(maxBackoff - minBackoff, (retry + 50) ** 2);\n\t\t\t\t\t\tconst backoff = Math.round((Math.random() * backoffIntervalSize)) + minBackoff;\n\n\t\t\t\t\t\tthis.methodLogger('runWithRetry')?.debug(`Retrying DB operation in ${backoff}ms (retry #${retry}) from`, new Error().stack);\n\t\t\t\t\t\tawait asleep(backoff);\n\n\t\t\t\t\t\tcontinue;\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tthrow(error);\n\t\t\t}\n\t\t}\n\t\tthrow(lastError);\n\t}\n\n\tprivate async newDBConnection(): Promise<pg.Pool> {\n\t\tif (this.poolInternal === null) {\n\t\t\tthrow(new Error('Database connection is not available'));\n\t\t}\n\n\t\treturn(await this.runWithRetry(async () => {\n\t\t\tif (this.poolInternal === null) {\n\t\t\t\tthrow(new Error('Database connection is not available'));\n\t\t\t}\n\n\t\t\tthis.methodLogger('newDBConnection')?.debug('Getting DB pool');\n\t\t\tconst pool = await this.poolInternal();\n\n\t\t\treturn(await this.initializeDBConnection(pool));\n\t\t}));\n\t}\n\n\tprivate async dbTransaction<T>(className: string, fn: (client: pg.PoolClient, logger: Logger | undefined) => Promise<T>): Promise<T> {\n\t\tconst pool = await this.newDBConnection();\n\t\tconst logger = this.methodLogger(className);\n\n\t\tconst result = await this.runWithRetry(async function() {\n\t\t\tconst client = await pool.connect();\n\n\t\t\ttry {\n\t\t\t\tlogger?.debug('Starting DB transaction');\n\t\t\t\tawait client.query('BEGIN');\n\t\t\t\tlogger?.debug('DB transaction started');\n\n\t\t\t\tconst retval = await fn(client, logger);\n\n\t\t\t\tlogger?.debug('Committing DB transaction');\n\t\t\t\tawait client.query('COMMIT');\n\t\t\t\tlogger?.debug('DB transaction committed');\n\n\t\t\t\treturn(retval);\n\t\t\t} catch (error: unknown) {\n\t\t\t\ttry {\n\t\t\t\t\tlogger?.debug('Rolling back DB transaction due to error:', error);\n\t\t\t\t\tawait client.query('ROLLBACK');\n\t\t\t\t\tlogger?.debug('DB transaction rolled back');\n\t\t\t\t} catch {\n\t\t\t\t\tlogger?.debug('Error rolling back DB transaction !!');\n\t\t\t\t}\n\t\t\t\tthrow(error);\n\t\t\t} finally {\n\t\t\t\tclient.release();\n\t\t\t}\n\t\t});\n\n\t\treturn(result);\n\t}\n\n\tasync add(request: KeetaAnchorQueueRequest<QueueRequest>, info?: KeetaAnchorQueueEntryExtra): Promise<KeetaAnchorQueueRequestID> {\n\t\treturn(await this.dbTransaction('add', async (client, logger): Promise<KeetaAnchorQueueRequestID> => {\n\t\t\tlet entryID = ConvertStringToRequestID(info?.id);\n\t\t\tif (entryID) {\n\t\t\t\tconst existingEntry = await client.query<{ id: string }>('SELECT id FROM queue_entries WHERE id = $1 AND path = $2', [entryID, this.pathStr]);\n\t\t\t\tif (existingEntry.rows.length > 0) {\n\t\t\t\t\tlogger?.debug(`Request with id ${String(entryID)} already exists, ignoring`);\n\t\t\t\t\treturn(entryID);\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tconst idempotentIDs = info?.idempotentKeys;\n\t\t\tif (idempotentIDs) {\n\t\t\t\tconst matchingIdempotentEntries = new Set<KeetaAnchorQueueRequestID>();\n\t\t\t\tfor (const idempotentID of idempotentIDs) {\n\t\t\t\t\tconst idempotentEntryExists = await client.query<IdempotentRow>(\n\t\t\t\t\t\t'SELECT idempotent_id FROM queue_idempotent_keys WHERE idempotent_id = $1 AND path = $2',\n\t\t\t\t\t\t[idempotentID, this.pathStr]\n\t\t\t\t\t);\n\t\t\t\t\tif (idempotentEntryExists.rows.length > 0) {\n\t\t\t\t\t\tmatchingIdempotentEntries.add(idempotentID);\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tif (matchingIdempotentEntries.size !== 0) {\n\t\t\t\t\tthrow(new Errors.IdempotentExistsError('One or more idempotent entries already exist in the queue', matchingIdempotentEntries));\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tentryID ??= ConvertStringToRequestID(crypto.randomUUID());\n\n\t\t\tlogger?.debug(`Enqueuing request with id ${String(entryID)}`);\n\n\t\t\tconst currentTime = Date.now();\n\t\t\tconst requestJSON = JSON.stringify(request);\n\n\t\t\t/**\n\t\t\t * The status to use for the new entry\n\t\t\t */\n\t\t\tconst status = info?.status ?? 'pending';\n\n\t\t\ttry {\n\t\t\t\tawait client.query(\n\t\t\t\t\t`INSERT INTO queue_entries (id, path, request, output, last_error, status, created, updated, worker, failures)\n\t\t\t\t\t VALUES ($1, $2, $3, NULL, NULL, $4, $5, $6, NULL, 0)`,\n\t\t\t\t\t[entryID, this.pathStr, requestJSON, status, currentTime, currentTime]\n\t\t\t\t);\n\n\t\t\t\tif (idempotentIDs && idempotentIDs.size > 0) {\n\t\t\t\t\tfor (const idempotentID of idempotentIDs) {\n\t\t\t\t\t\tawait client.query('INSERT INTO queue_idempotent_keys (entry_id, path, idempotent_id) VALUES ($1, $2, $3)', [entryID, this.pathStr, idempotentID]);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} catch (error: unknown) {\n\t\t\t\tif (error instanceof Error && error.message.includes('duplicate key') && idempotentIDs) {\n\t\t\t\t\tthrow(new Errors.IdempotentExistsError('One or more idempotent entries already exist in the queue', idempotentIDs));\n\t\t\t\t}\n\t\t\t\tthrow(error);\n\t\t\t}\n\n\t\t\treturn(entryID);\n\t\t}));\n\t}\n\n\tasync setStatus(id: KeetaAnchorQueueRequestID, status: KeetaAnchorQueueStatus, ancillary?: KeetaAnchorQueueEntryAncillaryData<QueueResult>): Promise<void> {\n\t\tconst { oldStatus } = ancillary ?? {};\n\n\t\treturn(await this.dbTransaction('setStatus', async (client, logger): Promise<void> => {\n\t\t\tconst existingEntry = await client.query<{ status: KeetaAnchorQueueStatus; failures: number; last_error: string | null; output: string | null }>('SELECT status, failures, last_error, output FROM queue_entries WHERE id = $1 AND path = $2', [id, this.pathStr]);\n\t\t\tif (existingEntry.rows.length === 0) {\n\t\t\t\tthrow(new Error(`Request with ID ${String(id)} not found`));\n\t\t\t}\n\n\t\t\tconst currentEntry = existingEntry.rows[0];\n\t\t\tif (!currentEntry) {\n\t\t\t\tthrow(new Error(`Request with ID ${String(id)} not found`));\n\t\t\t}\n\n\t\t\tconst newEntry = ManageStatusUpdates<QueueResult>(id, currentEntry, status, ancillary, logger);\n\t\t\tconst currentTime = newEntry.updated.getTime();\n\t\t\tconst workerValue = newEntry.worker;\n\t\t\tconst newFailures = newEntry.failures ?? currentEntry.failures;\n\t\t\tconst newLastError = newEntry.lastError !== undefined ? newEntry.lastError : currentEntry.last_error;\n\t\t\tconst newOutput = newEntry.output !== undefined ? JSON.stringify(newEntry.output) : currentEntry.output;\n\n\t\t\tlet updateQuery: string;\n\t\t\tlet updateParams: (KeetaAnchorQueueRequestID | string | number | null)[];\n\n\t\t\tif (oldStatus) {\n\t\t\t\tupdateQuery = `UPDATE queue_entries\n\t\t\t\t SET status = $1, updated = $2, worker = $3, failures = $4, last_error = $5, output = $6\n\t\t\t\t WHERE id = $7 AND path = $8 AND status = $9`;\n\t\t\t\tupdateParams = [status, currentTime, workerValue, newFailures, newLastError, newOutput, id, this.pathStr, oldStatus];\n\t\t\t} else {\n\t\t\t\tupdateQuery = `UPDATE queue_entries\n\t\t\t\t SET status = $1, updated = $2, worker = $3, failures = $4, last_error = $5, output = $6\n\t\t\t\t WHERE id = $7 AND path = $8`;\n\t\t\t\tupdateParams = [status, currentTime, workerValue, newFailures, newLastError, newOutput, id, this.pathStr];\n\t\t\t}\n\n\t\t\tconst result = await client.query(updateQuery, updateParams);\n\n\t\t\tif (oldStatus && result.rowCount === 0) {\n\t\t\t\tconst currentEntry = await client.query<{ status: KeetaAnchorQueueStatus }>('SELECT status FROM queue_entries WHERE id = $1 AND path = $2', [id, this.pathStr]);\n\t\t\t\tconst currentStatus = currentEntry.rows[0]?.status;\n\t\t\t\tif (currentEntry.rows.length > 0) {\n\t\t\t\t\tif (currentStatus === undefined) {\n\t\t\t\t\t\tthrow(new Error(`internal error: could not retrieve current status for request with ID ${String(id)}`));\n\t\t\t\t\t}\n\t\t\t\t\tthrow(new Errors.IncorrectStateAssertedError(id, oldStatus, currentStatus));\n\t\t\t\t} else {\n\t\t\t\t\tthrow(new Error(`Request with ID ${String(id)} not found`));\n\t\t\t\t}\n\t\t\t}\n\t\t}));\n\t}\n\n\tasync get(id: KeetaAnchorQueueRequestID): Promise<KeetaAnchorQueueEntry<QueueRequest, QueueResult> | null> {\n\t\treturn(await this.dbTransaction('get', async (client): Promise<KeetaAnchorQueueEntry<QueueRequest, QueueResult> | null> => {\n\t\t\tconst row = await client.query<QueueEntryRow>(\n\t\t\t\t`SELECT id, request, output, last_error, status, created, updated, worker, failures\n\t\t\t\t FROM queue_entries WHERE id = $1 AND path = $2`,\n\t\t\t\t[id, this.pathStr]\n\t\t\t);\n\n\t\t\tif (row.rows.length === 0) {\n\t\t\t\treturn(null);\n\t\t\t}\n\n\t\t\tconst entry = row.rows[0];\n\t\t\tif (!entry) {\n\t\t\t\treturn(null);\n\t\t\t}\n\n\t\t\tconst idempotentRows = await client.query<IdempotentRow>(\n\t\t\t\t'SELECT idempotent_id FROM queue_idempotent_keys WHERE entry_id = $1 AND path = $2',\n\t\t\t\t[id, this.pathStr]\n\t\t\t);\n\n\t\t\tconst idempotentKeys = idempotentRows.rows.length > 0\n\t\t\t\t? new Set(idempotentRows.rows.map(function(idempotentRow: IdempotentRow) {\n\t\t\t\t\treturn(ConvertStringToRequestID(idempotentRow.idempotent_id));\n\t\t\t\t}))\n\t\t\t\t: undefined;\n\n\t\t\treturn({\n\t\t\t\tid: ConvertStringToRequestID(entry.id),\n\t\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\t\trequest: JSON.parse(entry.request) as QueueRequest,\n\t\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\t\toutput: entry.output ? JSON.parse(entry.output) as QueueResult : null,\n\t\t\t\tlastError: entry.last_error,\n\t\t\t\tstatus: entry.status,\n\t\t\t\tcreated: new Date(Number(entry.created)),\n\t\t\t\tupdated: new Date(Number(entry.updated)),\n\t\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\t\tworker: entry.worker as unknown as KeetaAnchorQueueWorkerID | null,\n\t\t\t\tfailures: entry.failures,\n\t\t\t\tidempotentKeys: idempotentKeys\n\t\t\t});\n\t\t}));\n\t}\n\n\tasync query(filter?: KeetaAnchorQueueFilter): Promise<KeetaAnchorQueueEntry<QueueRequest, QueueResult>[]> {\n\t\treturn(await this.dbTransaction('query', async (client, logger): Promise<KeetaAnchorQueueEntry<QueueRequest, QueueResult>[]> => {\n\t\t\tlogger?.debug(`Querying queue with id ${this.id} with filter:`, filter);\n\n\t\t\tconst conditions: string[] = [];\n\t\t\tconst params: (string | number)[] = [];\n\t\t\tlet paramIndex = 1;\n\n\t\t\tconditions.push(`path = $${paramIndex++}`);\n\t\t\tparams.push(this.pathStr);\n\n\t\t\tif (filter?.status) {\n\t\t\t\tconditions.push(`status = $${paramIndex++}`);\n\t\t\t\tparams.push(filter.status);\n\t\t\t}\n\n\t\t\tif (filter?.updatedBefore) {\n\t\t\t\tconditions.push(`updated < $${paramIndex++}`);\n\t\t\t\tparams.push(filter.updatedBefore.getTime());\n\t\t\t}\n\n\t\t\tlet query = 'SELECT id, request, output, last_error, status, created, updated, worker, failures FROM queue_entries';\n\n\t\t\tif (conditions.length > 0) {\n\t\t\t\tquery += ' WHERE ' + conditions.join(' AND ');\n\t\t\t}\n\n\t\t\tif (filter?.limit !== undefined) {\n\t\t\t\tquery += ` LIMIT $${paramIndex++}`;\n\t\t\t\tparams.push(filter.limit);\n\t\t\t}\n\n\t\t\tconst rows = await client.query<QueueEntryRow>(query, params);\n\n\t\t\tconst entries: KeetaAnchorQueueEntry<QueueRequest, QueueResult>[] = [];\n\n\t\t\tfor (const row of rows.rows) {\n\t\t\t\tconst idempotentRows = await client.query<IdempotentRow>(\n\t\t\t\t\t'SELECT idempotent_id FROM queue_idempotent_keys WHERE entry_id = $1 AND path = $2',\n\t\t\t\t\t[row.id, this.pathStr]\n\t\t\t\t);\n\n\t\t\t\tconst idempotentKeys = idempotentRows.rows.length > 0\n\t\t\t\t\t? new Set(idempotentRows.rows.map(function(idempotentRow: IdempotentRow) {\n\t\t\t\t\t\treturn(ConvertStringToRequestID(idempotentRow.idempotent_id));\n\t\t\t\t\t}))\n\t\t\t\t\t: undefined;\n\n\t\t\t\tentries.push({\n\t\t\t\t\tid: ConvertStringToRequestID(row.id),\n\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\t\t\trequest: JSON.parse(row.request) as QueueRequest,\n\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\t\t\toutput: row.output ? JSON.parse(row.output) as QueueResult : null,\n\t\t\t\t\tlastError: row.last_error,\n\t\t\t\t\tstatus: row.status,\n\t\t\t\t\tcreated: new Date(Number(row.created)),\n\t\t\t\t\tupdated: new Date(Number(row.updated)),\n\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\t\t\tworker: row.worker as unknown as KeetaAnchorQueueWorkerID | null,\n\t\t\t\t\tfailures: row.failures,\n\t\t\t\t\tidempotentKeys: idempotentKeys\n\t\t\t\t});\n\t\t\t}\n\n\t\t\tlogger?.debug(`Queried queue with id ${this.id} with filter:`, filter, '-- found', entries.length, 'entries');\n\n\t\t\treturn(entries);\n\t\t}));\n\t}\n\n\tasync partition(path: string) : Promise<KeetaAnchorQueueStorageDriver<QueueRequest, QueueResult>> {\n\t\tthis.methodLogger('partition')?.debug(`Creating partitioned queue storage driver for path: ${path}`);\n\n\t\tif (this.poolInternal === null) {\n\t\t\tthrow(new Error('Asked to partition but the instance has been destroyed'));\n\t\t}\n\n\t\tconst retval = new KeetaAnchorQueueStorageDriverPostgres<QueueRequest, QueueResult>({\n\t\t\tid: `${this.id}::${path}`,\n\t\t\tlogger: this.logger,\n\t\t\tpool: this.poolInternal,\n\t\t\tpath: [...this.path, path]\n\t\t});\n\n\t\treturn(retval);\n\t}\n\n\tasync destroy(): Promise<void> {\n\t\tthis.methodLogger('destroy')?.debug('Destroying instance');\n\n\t\tthis.poolInternal = null;\n\t}\n\n\tasync [Symbol.asyncDispose](): Promise<void> {\n\t\treturn(await this.destroy());\n\t}\n}\n"]}
|
|
1
|
+
{"version":3,"file":"queue_postgres.js","sourceRoot":"","sources":["../../../../src/lib/queue/drivers/queue_postgres.ts"],"names":[],"mappings":"AAYA,OAAO,EACN,YAAY,EACZ,mBAAmB,EACnB,wBAAwB,EACxB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAuB/C,MAAM,CAAC,OAAO,OAAO,qCAAqC;IACxC,MAAM,CAAqB;IACpC,YAAY,GAAoC,IAAI,CAAC;IACrD,uBAAuB,GAA4B,IAAI,CAAC;IAEvD,IAAI,GAAG,uCAAuC,CAAC;IAC/C,EAAE,CAAS;IACX,IAAI,GAAa,EAAE,CAAC;IACZ,OAAO,CAAS;IACzB,WAAW,GAAsC,SAAS,CAAC;IAEnE,YAAY,OAAuJ;QAClK,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,EAAE,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAC7C,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,CAAA;QAC7B,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;QACjC,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEzB,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC9E,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAAC,IAAa;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,wBAAwB,CAAC,CAAC;QAE3D,IAAI,IAAI,CAAC,uBAAuB,KAAK,IAAI,EAAE,CAAC;YAC3C,MAAM,EAAE,KAAK,CAAC,qFAAqF,CAAC,CAAC;YAErG,MAAM,IAAI,CAAC,uBAAuB,CAAC;YACnC,OAAM,CAAC,IAAI,CAAC,CAAC;QACd,CAAC;QAED,IAAI,CAAC,uBAAuB,GAAG,CAAC,KAAK,IAAI,EAAE;YAC1C,MAAM,EAAE,KAAK,CAAC,iDAAiD,CAAC,CAAC;YAEjE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACpC,IAAI,CAAC;gBACJ,MAAM,MAAM,CAAC,KAAK,CAAC;;;;;;;;;;;;;OAahB,CAAC,CAAC;gBAEL,MAAM,MAAM,CAAC,KAAK,CAAC;;;;;;;;OAQhB,CAAC,CAAC;gBAEL,MAAM,MAAM,CAAC,KAAK,CAAC,8EAA8E,CAAC,CAAC;gBACnG,MAAM,MAAM,CAAC,KAAK,CAAC,gFAAgF,CAAC,CAAC;gBACrG,MAAM,MAAM,CAAC,KAAK,CAAC,4GAA4G,CAAC,CAAC;YAClI,CAAC;oBAAS,CAAC;gBACV,MAAM,CAAC,OAAO,EAAE,CAAC;YAClB,CAAC;YAED,MAAM,EAAE,KAAK,CAAC,6DAA6D,CAAC,CAAC;YAE7E,OAAM,CAAC,IAAI,CAAC,CAAC;QACd,CAAC,CAAC,EAAE,CAAC;QAEL,MAAM,IAAI,CAAC,uBAAuB,CAAC;QAEnC,OAAM,CAAC,IAAI,CAAC,CAAC;IACd,CAAC;IAEO,YAAY,CAAC,MAAc;QAClC,OAAM,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE;YAChC,KAAK,EAAE,uCAAuC;YAC9C,IAAI,EAAE,yCAAyC;YAC/C,MAAM,EAAE,MAAM;YACd,UAAU,EAAE,IAAI,CAAC,EAAE;SACnB,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,YAAY,CAAI,EAAoB;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC;QAEjD,IAAI,SAAkB,CAAC;QACvB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC;YACzC,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;gBAChC,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,EAAE,KAAK,CAAC,kEAAkE,CAAC,CAAC;gBAE7G,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;oBAC7B,+DAA+D;oBAC/D,MAAK,CAAC,SAAS,CAAC,CAAC;gBAClB,CAAC;gBACD,MAAK,CAAC,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC,CAAC;YACjE,CAAC;YAED,IAAI,CAAC;gBACJ,OAAM,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACpB,CAAC;YAAC,OAAO,KAAc,EAAE,CAAC;gBACzB,SAAS,GAAG,KAAK,CAAC;gBAElB,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;oBAC5B,MAAM,SAAS,GAAG,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;oBACtD,IAAI,SAAS,KAAK,OAAO,IAAI,SAAS,KAAK,OAAO,EAAE,CAAC;wBACpD,MAAM,EAAE,KAAK,CAAC,4CAA4C,CAAC,CAAC;wBAE5D,MAAM,UAAU,GAAG,GAAG,CAAC;wBACvB,MAAM,UAAU,GAAG,MAAM,CAAC;wBAC1B,MAAM,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,UAAU,EAAE,CAAC,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;wBACjF,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,mBAAmB,CAAC,CAAC,GAAG,UAAU,CAAC;wBAE/E,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,EAAE,KAAK,CAAC,4BAA4B,OAAO,cAAc,KAAK,QAAQ,EAAE,IAAI,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC;wBAC5H,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC;wBAEtB,SAAS;oBACV,CAAC;gBACF,CAAC;gBAED,MAAK,CAAC,KAAK,CAAC,CAAC;YACd,CAAC;QACF,CAAC;QACD,MAAK,CAAC,SAAS,CAAC,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,eAAe;QAC5B,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;YAChC,MAAK,CAAC,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC,CAAC;QAC1D,CAAC;QAED,OAAM,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE;YACzC,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;gBAChC,MAAK,CAAC,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC,CAAC;YAC1D,CAAC;YAED,IAAI,CAAC,YAAY,CAAC,iBAAiB,CAAC,EAAE,KAAK,CAAC,iBAAiB,CAAC,CAAC;YAC/D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAEvC,OAAM,CAAC,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,aAAa,CAAI,SAAiB,EAAE,EAAqE;QACtH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAE5C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK;YAC3C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YAEpC,IAAI,CAAC;gBACJ,MAAM,EAAE,KAAK,CAAC,yBAAyB,CAAC,CAAC;gBACzC,MAAM,MAAM,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;gBACrE,MAAM,EAAE,KAAK,CAAC,wBAAwB,CAAC,CAAC;gBAExC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBAExC,MAAM,EAAE,KAAK,CAAC,2BAA2B,CAAC,CAAC;gBAC3C,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBAC7B,MAAM,EAAE,KAAK,CAAC,0BAA0B,CAAC,CAAC;gBAE1C,OAAM,CAAC,MAAM,CAAC,CAAC;YAChB,CAAC;YAAC,OAAO,KAAc,EAAE,CAAC;gBACzB,IAAI,CAAC;oBACJ,MAAM,EAAE,KAAK,CAAC,2CAA2C,EAAE,KAAK,CAAC,CAAC;oBAClE,MAAM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;oBAC/B,MAAM,EAAE,KAAK,CAAC,4BAA4B,CAAC,CAAC;gBAC7C,CAAC;gBAAC,MAAM,CAAC;oBACR,MAAM,EAAE,KAAK,CAAC,sCAAsC,CAAC,CAAC;gBACvD,CAAC;gBACD,MAAK,CAAC,KAAK,CAAC,CAAC;YACd,CAAC;oBAAS,CAAC;gBACV,MAAM,CAAC,OAAO,EAAE,CAAC;YAClB,CAAC;QACF,CAAC,CAAC,CAAC;QAEH,OAAM,CAAC,MAAM,CAAC,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,OAA8C,EAAE,IAAiC;QAC1F,OAAM,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAsC,EAAE;YACnG,IAAI,OAAO,GAAG,wBAAwB,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACjD,IAAI,OAAO,EAAE,CAAC;gBACb,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,KAAK,CAAiB,0DAA0D,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC9I,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACnC,MAAM,EAAE,KAAK,CAAC,mBAAmB,MAAM,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;oBAC7E,OAAM,CAAC,OAAO,CAAC,CAAC;gBACjB,CAAC;gBAED,MAAM,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YAC5B,CAAC;YAED,MAAM,aAAa,GAAG,IAAI,EAAE,cAAc,CAAC;YAC3C,IAAI,aAAa,EAAE,CAAC;gBACnB,MAAM,yBAAyB,GAAG,IAAI,GAAG,EAA6B,CAAC;gBACvE,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;oBAC1C,MAAM,qBAAqB,GAAG,MAAM,MAAM,CAAC,KAAK,CAC/C,wFAAwF,EACxF,CAAC,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,CAC5B,CAAC;oBACF,IAAI,qBAAqB,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC3C,yBAAyB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;oBAC7C,CAAC;gBACF,CAAC;gBAED,IAAI,yBAAyB,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;oBAC1C,MAAK,CAAC,IAAI,MAAM,CAAC,qBAAqB,CAAC,2DAA2D,EAAE,yBAAyB,CAAC,CAAC,CAAC;gBACjI,CAAC;gBAED,MAAM,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YAC5B,CAAC;YAED,OAAO,KAAK,wBAAwB,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;YAE1D,MAAM,EAAE,KAAK,CAAC,6BAA6B,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAE9D,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC/B,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YAE5C;;eAEG;YACH,MAAM,MAAM,GAAG,IAAI,EAAE,MAAM,IAAI,SAAS,CAAC;YAEzC,MAAM,MAAM,CAAC,KAAK,CACjB;0DACsD,EACtD,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,CAAC,CACtE,CAAC;YAEF,IAAI,aAAa,IAAI,aAAa,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBAC7C,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;oBAC1C,MAAM,MAAM,CAAC,KAAK,CAAC,uFAAuF,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;gBACpJ,CAAC;YACF,CAAC;YAED,OAAM,CAAC,OAAO,CAAC,CAAC;QACjB,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,EAA6B,EAAE,MAA8B,EAAE,SAA2D;QACzI,MAAM,EAAE,SAAS,EAAE,GAAG,SAAS,IAAI,EAAE,CAAC;QAEtC,OAAM,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAiB,EAAE;YACpF,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,KAAK,CAAyG,4FAA4F,EAAE,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;YACnQ,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrC,MAAK,CAAC,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC;YAC7D,CAAC;YAED,MAAM,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YAE3B,MAAM,YAAY,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC3C,IAAI,CAAC,YAAY,EAAE,CAAC;gBACnB,MAAK,CAAC,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC;YAC7D,CAAC;YAED,MAAM,QAAQ,GAAG,mBAAmB,CAAc,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YAC/F,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YAC/C,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC;YACpC,MAAM,WAAW,GAAG,QAAQ,CAAC,QAAQ,IAAI,YAAY,CAAC,QAAQ,CAAC;YAC/D,MAAM,YAAY,GAAG,QAAQ,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,UAAU,CAAC;YACrG,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC;YAExG,IAAI,WAAmB,CAAC;YACxB,IAAI,YAAoE,CAAC;YAEzE,IAAI,SAAS,EAAE,CAAC;gBACf,WAAW,GAAG;;+DAE6C,CAAC;gBAC5D,YAAY,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,EAAE,EAAE,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;YACtH,CAAC;iBAAM,CAAC;gBACP,WAAW,GAAG;;+CAE6B,CAAC;gBAC5C,YAAY,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YAC3G,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;YAE7D,MAAM,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YAE3B,IAAI,SAAS,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;gBACxC,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,KAAK,CAAqC,8DAA8D,EAAE,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;gBAChK,MAAM,aAAa,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC;gBACnD,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAClC,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;wBACjC,MAAK,CAAC,IAAI,KAAK,CAAC,yEAAyE,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;oBACzG,CAAC;oBACD,MAAK,CAAC,IAAI,MAAM,CAAC,2BAA2B,CAAC,EAAE,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC,CAAC;gBAC7E,CAAC;qBAAM,CAAC;oBACP,MAAK,CAAC,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC;gBAC7D,CAAC;YACF,CAAC;QACF,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EAA6B;QACtC,OAAM,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAoE,EAAE;YACzH,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,KAAK,CAC7B;oDACgD,EAChD,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAClB,CAAC;YAEF,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3B,OAAM,CAAC,IAAI,CAAC,CAAC;YACd,CAAC;YAED,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC1B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACZ,OAAM,CAAC,IAAI,CAAC,CAAC;YACd,CAAC;YAED,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,KAAK,CACxC,mFAAmF,EACnF,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAClB,CAAC;YAEF,MAAM,cAAc,GAAG,cAAc,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;gBACpD,CAAC,CAAC,IAAI,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,UAAS,aAA4B;oBACtE,OAAM,CAAC,wBAAwB,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC;gBAC/D,CAAC,CAAC,CAAC;gBACH,CAAC,CAAC,SAAS,CAAC;YAEb,OAAM,CAAC;gBACN,EAAE,EAAE,wBAAwB,CAAC,KAAK,CAAC,EAAE,CAAC;gBACtC,yEAAyE;gBACzE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAiB;gBAClD,yEAAyE;gBACzE,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAgB,CAAC,CAAC,CAAC,IAAI;gBACrE,SAAS,EAAE,KAAK,CAAC,UAAU;gBAC3B,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,OAAO,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBACxC,OAAO,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBACxC,yEAAyE;gBACzE,MAAM,EAAE,KAAK,CAAC,MAAoD;gBAClE,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,cAAc,EAAE,cAAc;aAC9B,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,MAA+B;QAC1C,OAAM,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAA+D,EAAE;YAC9H,MAAM,EAAE,KAAK,CAAC,0BAA0B,IAAI,CAAC,EAAE,eAAe,EAAE,MAAM,CAAC,CAAC;YAExE,MAAM,UAAU,GAAa,EAAE,CAAC;YAChC,MAAM,MAAM,GAAwB,EAAE,CAAC;YACvC,IAAI,UAAU,GAAG,CAAC,CAAC;YAEnB,UAAU,CAAC,IAAI,CAAC,WAAW,UAAU,EAAE,EAAE,CAAC,CAAC;YAC3C,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAE1B,IAAI,MAAM,EAAE,MAAM,EAAE,CAAC;gBACpB,UAAU,CAAC,IAAI,CAAC,aAAa,UAAU,EAAE,EAAE,CAAC,CAAC;gBAC7C,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5B,CAAC;YAED,IAAI,MAAM,EAAE,aAAa,EAAE,CAAC;gBAC3B,UAAU,CAAC,IAAI,CAAC,cAAc,UAAU,EAAE,EAAE,CAAC,CAAC;gBAC9C,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC;YAC7C,CAAC;YAED,IAAI,KAAK,GAAG,uGAAuG,CAAC;YAEpH,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,KAAK,IAAI,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC/C,CAAC;YAED,IAAI,MAAM,EAAE,KAAK,KAAK,SAAS,EAAE,CAAC;gBACjC,KAAK,IAAI,WAAW,UAAU,EAAE,EAAE,CAAC;gBACnC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC3B,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,KAAK,CAAgB,KAAK,EAAE,MAAM,CAAC,CAAC;YAE9D,MAAM,OAAO,GAAuD,EAAE,CAAC;YAEvE,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBAC7B,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,KAAK,CACxC,mFAAmF,EACnF,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CACtB,CAAC;gBAEF,MAAM,cAAc,GAAG,cAAc,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;oBACpD,CAAC,CAAC,IAAI,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,UAAS,aAA4B;wBACtE,OAAM,CAAC,wBAAwB,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC;oBAC/D,CAAC,CAAC,CAAC;oBACH,CAAC,CAAC,SAAS,CAAC;gBAEb,OAAO,CAAC,IAAI,CAAC;oBACZ,EAAE,EAAE,wBAAwB,CAAC,GAAG,CAAC,EAAE,CAAC;oBACpC,yEAAyE;oBACzE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAiB;oBAChD,yEAAyE;oBACzE,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAgB,CAAC,CAAC,CAAC,IAAI;oBACjE,SAAS,EAAE,GAAG,CAAC,UAAU;oBACzB,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,OAAO,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBACtC,OAAO,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBACtC,yEAAyE;oBACzE,MAAM,EAAE,GAAG,CAAC,MAAoD;oBAChE,QAAQ,EAAE,GAAG,CAAC,QAAQ;oBACtB,cAAc,EAAE,cAAc;iBAC9B,CAAC,CAAC;YACJ,CAAC;YAED,MAAM,EAAE,KAAK,CAAC,yBAAyB,IAAI,CAAC,EAAE,eAAe,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAE9G,OAAM,CAAC,OAAO,CAAC,CAAC;QACjB,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAY;QAC3B,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,EAAE,KAAK,CAAC,uDAAuD,IAAI,EAAE,CAAC,CAAC;QAErG,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;YAChC,MAAK,CAAC,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC,CAAC;QAC5E,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,qCAAqC,CAA4B;YACnF,EAAE,EAAE,GAAG,IAAI,CAAC,EAAE,KAAK,IAAI,EAAE;YACzB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,IAAI,EAAE,IAAI,CAAC,YAAY;YACvB,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC;SAC1B,CAAC,CAAC;QAEH,OAAM,CAAC,MAAM,CAAC,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,OAAO;QACZ,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,qBAAqB,CAAC,CAAC;QAE3D,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QAC1B,OAAM,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IAC9B,CAAC;IAED,gBAAgB;IAChB,QAAQ,CAAC,GAAW;QAInB,IAAI,GAAG,KAAK,sCAAsC,EAAE,CAAC;YACpD,MAAK,CAAC,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC,CAAC;QACnD,CAAC;QAED,OAAM,CAAC;YACN,cAAc,EAAE,CAAC,KAAa,EAAQ,EAAE;gBACvC,IAAI,CAAC,WAAW,GAAG,KAAK,IAAmB,EAAE;oBAC5C,OAAM,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC7B,CAAC,CAAC;YACH,CAAC;YACD,gBAAgB,EAAE,GAAS,EAAE;gBAC5B,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC;YAC9B,CAAC;SACD,CAAC,CAAC;IACJ,CAAC;CACD","sourcesContent":["import type {\n\tKeetaAnchorQueueStorageDriver,\n\tKeetaAnchorQueueStorageDriverConstructor,\n\tKeetaAnchorQueueRequest,\n\tKeetaAnchorQueueRequestID,\n\tKeetaAnchorQueueEntry,\n\tKeetaAnchorQueueEntryExtra,\n\tKeetaAnchorQueueEntryAncillaryData,\n\tKeetaAnchorQueueStatus,\n\tKeetaAnchorQueueFilter,\n\tKeetaAnchorQueueWorkerID\n} from '../index.ts';\nimport {\n\tMethodLogger,\n\tManageStatusUpdates,\n\tConvertStringToRequestID\n} from '../internal.js';\nimport { Errors } from '../common.js';\n\nimport { asleep } from '../../utils/asleep.js';\n\nimport type { Logger } from '../../log/index.ts';\nimport type { JSONSerializable } from '../../utils/json.js';\n\nimport type * as pg from 'pg';\n\ntype QueueEntryRow = {\n\tid: string;\n\trequest: string;\n\toutput: string | null;\n\tlast_error: string | null;\n\tstatus: KeetaAnchorQueueStatus;\n\tcreated: Date;\n\tupdated: Date;\n\tworker: number | null;\n\tfailures: number;\n};\n\ntype IdempotentRow = {\n\tidempotent_id: string;\n};\n\nexport default class KeetaAnchorQueueStorageDriverPostgres<QueueRequest extends JSONSerializable = JSONSerializable, QueueResult extends JSONSerializable = JSONSerializable> implements KeetaAnchorQueueStorageDriver<QueueRequest, QueueResult> {\n\tprivate readonly logger: Logger | undefined;\n\tprivate poolInternal: (() => Promise<pg.Pool>) | null = null;\n\tprivate dbInitializationPromise: Promise<boolean> | null = null;\n\n\treadonly name = 'KeetaAnchorQueueStorageDriverPostgres';\n\treadonly id: string;\n\treadonly path: string[] = [];\n\tprivate readonly pathStr: string;\n\tprivate toctouDelay: (() => Promise<void>) | undefined = undefined;\n\n\tconstructor(options: NonNullable<ConstructorParameters<KeetaAnchorQueueStorageDriverConstructor<QueueRequest, QueueResult>>[0]> & { pool: () => Promise<pg.Pool>; }) {\n\t\tthis.id = options?.id ?? crypto.randomUUID();\n\t\tthis.logger = options?.logger\n\t\tthis.poolInternal = options.pool;\n\t\tthis.path = options.path ?? [];\n\t\tthis.pathStr = ['root', ...this.path].join('.');\n\t\tObject.freeze(this.path);\n\n\t\tthis.methodLogger('new')?.debug('Initialized Postgres queue storage driver');\n\t}\n\n\tprivate async initializeDBConnection(pool: pg.Pool): Promise<pg.Pool> {\n\t\tconst logger = this.methodLogger('initializeDBConnection');\n\n\t\tif (this.dbInitializationPromise !== null) {\n\t\t\tlogger?.debug('DB schema initialization already in progress or completed, waiting for it to finish');\n\n\t\t\tawait this.dbInitializationPromise;\n\t\t\treturn(pool);\n\t\t}\n\n\t\tthis.dbInitializationPromise = (async () => {\n\t\t\tlogger?.debug('Initializing DB schema for queue storage driver');\n\n\t\t\tconst client = await pool.connect();\n\t\t\ttry {\n\t\t\t\tawait client.query(`\n\t\t\t\t\tCREATE TABLE IF NOT EXISTS queue_entries (\n\t\t\t\t\t\tid TEXT NOT NULL,\n\t\t\t\t\t\tpath TEXT NOT NULL,\n\t\t\t\t\t\trequest TEXT NOT NULL,\n\t\t\t\t\t\toutput TEXT,\n\t\t\t\t\t\tlast_error TEXT,\n\t\t\t\t\t\tstatus TEXT NOT NULL,\n\t\t\t\t\t\tcreated BIGINT NOT NULL,\n\t\t\t\t\t\tupdated BIGINT NOT NULL,\n\t\t\t\t\t\tworker BIGINT,\n\t\t\t\t\t\tfailures INTEGER NOT NULL DEFAULT 0,\n\t\t\t\t\t\tPRIMARY KEY (id, path)\n\t\t\t\t\t)`);\n\n\t\t\t\tawait client.query(`\n\t\t\t\t\tCREATE TABLE IF NOT EXISTS queue_idempotent_keys (\n\t\t\t\t\t\tentry_id TEXT NOT NULL,\n\t\t\t\t\t\tidempotent_id TEXT NOT NULL,\n\t\t\t\t\t\tpath TEXT NOT NULL,\n\t\t\t\t\t\tUNIQUE (idempotent_id, path),\n\t\t\t\t\t\tPRIMARY KEY (entry_id, idempotent_id, path),\n\t\t\t\t\t\tFOREIGN KEY (entry_id, path) REFERENCES queue_entries(id, path)\n\t\t\t\t\t)`);\n\n\t\t\t\tawait client.query('CREATE INDEX IF NOT EXISTS idx_queue_entries_status ON queue_entries(status)');\n\t\t\t\tawait client.query('CREATE INDEX IF NOT EXISTS idx_queue_entries_updated ON queue_entries(updated)');\n\t\t\t\tawait client.query('CREATE INDEX IF NOT EXISTS idx_queue_idempotent_keys_idempotent_id ON queue_idempotent_keys(idempotent_id)');\n\t\t\t} finally {\n\t\t\t\tclient.release();\n\t\t\t}\n\n\t\t\tlogger?.debug('Completed DB schema initialization for queue storage driver');\n\n\t\t\treturn(true);\n\t\t})();\n\n\t\tawait this.dbInitializationPromise;\n\n\t\treturn(pool);\n\t}\n\n\tprivate methodLogger(method: string): Logger | undefined {\n\t\treturn(MethodLogger(this.logger, {\n\t\t\tclass: 'KeetaAnchorQueueStorageDriverPostgres',\n\t\t\tfile: 'src/lib/queue/drivers/queue_postgres.ts',\n\t\t\tmethod: method,\n\t\t\tinstanceID: this.id\n\t\t}));\n\t}\n\n\tprivate async runWithRetry<T>(fn: () => Promise<T>): Promise<T> {\n\t\tconst logger = this.methodLogger('runWithRetry');\n\n\t\tlet lastError: unknown;\n\t\tfor (let retry = 0; retry < 16; retry++) {\n\t\t\tif (this.poolInternal === null) {\n\t\t\t\tthis.methodLogger('runWithRetry')?.debug('Aborting DB operation retries because the instance was destroyed');\n\n\t\t\t\tif (lastError !== undefined) {\n\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/only-throw-error\n\t\t\t\t\tthrow(lastError);\n\t\t\t\t}\n\t\t\t\tthrow(new Error('Aborting because the instance was destroyed'));\n\t\t\t}\n\n\t\t\ttry {\n\t\t\t\treturn(await fn());\n\t\t\t} catch (error: unknown) {\n\t\t\t\tlastError = error;\n\n\t\t\t\tif (error instanceof Error) {\n\t\t\t\t\tconst errorCode = 'code' in error ? error.code : null;\n\t\t\t\t\tif (errorCode === '40001' || errorCode === '40P01') {\n\t\t\t\t\t\tlogger?.debug('Serialization failure or deadlock detected');\n\n\t\t\t\t\t\tconst minBackoff = 100;\n\t\t\t\t\t\tconst maxBackoff = 30_000;\n\t\t\t\t\t\tconst backoffIntervalSize = Math.min(maxBackoff - minBackoff, (retry + 50) ** 2);\n\t\t\t\t\t\tconst backoff = Math.round((Math.random() * backoffIntervalSize)) + minBackoff;\n\n\t\t\t\t\t\tthis.methodLogger('runWithRetry')?.debug(`Retrying DB operation in ${backoff}ms (retry #${retry}) from`, new Error().stack);\n\t\t\t\t\t\tawait asleep(backoff);\n\n\t\t\t\t\t\tcontinue;\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tthrow(error);\n\t\t\t}\n\t\t}\n\t\tthrow(lastError);\n\t}\n\n\tprivate async newDBConnection(): Promise<pg.Pool> {\n\t\tif (this.poolInternal === null) {\n\t\t\tthrow(new Error('Database connection is not available'));\n\t\t}\n\n\t\treturn(await this.runWithRetry(async () => {\n\t\t\tif (this.poolInternal === null) {\n\t\t\t\tthrow(new Error('Database connection is not available'));\n\t\t\t}\n\n\t\t\tthis.methodLogger('newDBConnection')?.debug('Getting DB pool');\n\t\t\tconst pool = await this.poolInternal();\n\n\t\t\treturn(await this.initializeDBConnection(pool));\n\t\t}));\n\t}\n\n\tprivate async dbTransaction<T>(className: string, fn: (client: pg.PoolClient, logger: Logger | undefined) => Promise<T>): Promise<T> {\n\t\tconst pool = await this.newDBConnection();\n\t\tconst logger = this.methodLogger(className);\n\n\t\tconst result = await this.runWithRetry(async function() {\n\t\t\tconst client = await pool.connect();\n\n\t\t\ttry {\n\t\t\t\tlogger?.debug('Starting DB transaction');\n\t\t\t\tawait client.query('BEGIN TRANSACTION ISOLATION LEVEL SERIALIZABLE');\n\t\t\t\tlogger?.debug('DB transaction started');\n\n\t\t\t\tconst retval = await fn(client, logger);\n\n\t\t\t\tlogger?.debug('Committing DB transaction');\n\t\t\t\tawait client.query('COMMIT');\n\t\t\t\tlogger?.debug('DB transaction committed');\n\n\t\t\t\treturn(retval);\n\t\t\t} catch (error: unknown) {\n\t\t\t\ttry {\n\t\t\t\t\tlogger?.debug('Rolling back DB transaction due to error:', error);\n\t\t\t\t\tawait client.query('ROLLBACK');\n\t\t\t\t\tlogger?.debug('DB transaction rolled back');\n\t\t\t\t} catch {\n\t\t\t\t\tlogger?.debug('Error rolling back DB transaction !!');\n\t\t\t\t}\n\t\t\t\tthrow(error);\n\t\t\t} finally {\n\t\t\t\tclient.release();\n\t\t\t}\n\t\t});\n\n\t\treturn(result);\n\t}\n\n\tasync add(request: KeetaAnchorQueueRequest<QueueRequest>, info?: KeetaAnchorQueueEntryExtra): Promise<KeetaAnchorQueueRequestID> {\n\t\treturn(await this.dbTransaction('add', async (client, logger): Promise<KeetaAnchorQueueRequestID> => {\n\t\t\tlet entryID = ConvertStringToRequestID(info?.id);\n\t\t\tif (entryID) {\n\t\t\t\tconst existingEntry = await client.query<{ id: string }>('SELECT id FROM queue_entries WHERE id = $1 AND path = $2', [entryID, this.pathStr]);\n\t\t\t\tif (existingEntry.rows.length > 0) {\n\t\t\t\t\tlogger?.debug(`Request with id ${String(entryID)} already exists, ignoring`);\n\t\t\t\t\treturn(entryID);\n\t\t\t\t}\n\n\t\t\t\tawait this.toctouDelay?.();\n\t\t\t}\n\n\t\t\tconst idempotentIDs = info?.idempotentKeys;\n\t\t\tif (idempotentIDs) {\n\t\t\t\tconst matchingIdempotentEntries = new Set<KeetaAnchorQueueRequestID>();\n\t\t\t\tfor (const idempotentID of idempotentIDs) {\n\t\t\t\t\tconst idempotentEntryExists = await client.query<IdempotentRow>(\n\t\t\t\t\t\t'SELECT idempotent_id FROM queue_idempotent_keys WHERE idempotent_id = $1 AND path = $2',\n\t\t\t\t\t\t[idempotentID, this.pathStr]\n\t\t\t\t\t);\n\t\t\t\t\tif (idempotentEntryExists.rows.length > 0) {\n\t\t\t\t\t\tmatchingIdempotentEntries.add(idempotentID);\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tif (matchingIdempotentEntries.size !== 0) {\n\t\t\t\t\tthrow(new Errors.IdempotentExistsError('One or more idempotent entries already exist in the queue', matchingIdempotentEntries));\n\t\t\t\t}\n\n\t\t\t\tawait this.toctouDelay?.();\n\t\t\t}\n\n\t\t\tentryID ??= ConvertStringToRequestID(crypto.randomUUID());\n\n\t\t\tlogger?.debug(`Enqueuing request with id ${String(entryID)}`);\n\n\t\t\tconst currentTime = Date.now();\n\t\t\tconst requestJSON = JSON.stringify(request);\n\n\t\t\t/**\n\t\t\t * The status to use for the new entry\n\t\t\t */\n\t\t\tconst status = info?.status ?? 'pending';\n\n\t\t\tawait client.query(\n\t\t\t\t`INSERT INTO queue_entries (id, path, request, output, last_error, status, created, updated, worker, failures)\n\t\t\t\t VALUES ($1, $2, $3, NULL, NULL, $4, $5, $6, NULL, 0)`,\n\t\t\t\t[entryID, this.pathStr, requestJSON, status, currentTime, currentTime]\n\t\t\t);\n\n\t\t\tif (idempotentIDs && idempotentIDs.size > 0) {\n\t\t\t\tfor (const idempotentID of idempotentIDs) {\n\t\t\t\t\tawait client.query('INSERT INTO queue_idempotent_keys (entry_id, path, idempotent_id) VALUES ($1, $2, $3)', [entryID, this.pathStr, idempotentID]);\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn(entryID);\n\t\t}));\n\t}\n\n\tasync setStatus(id: KeetaAnchorQueueRequestID, status: KeetaAnchorQueueStatus, ancillary?: KeetaAnchorQueueEntryAncillaryData<QueueResult>): Promise<void> {\n\t\tconst { oldStatus } = ancillary ?? {};\n\n\t\treturn(await this.dbTransaction('setStatus', async (client, logger): Promise<void> => {\n\t\t\tconst existingEntry = await client.query<{ status: KeetaAnchorQueueStatus; failures: number; last_error: string | null; output: string | null }>('SELECT status, failures, last_error, output FROM queue_entries WHERE id = $1 AND path = $2', [id, this.pathStr]);\n\t\t\tif (existingEntry.rows.length === 0) {\n\t\t\t\tthrow(new Error(`Request with ID ${String(id)} not found`));\n\t\t\t}\n\n\t\t\tawait this.toctouDelay?.();\n\n\t\t\tconst currentEntry = existingEntry.rows[0];\n\t\t\tif (!currentEntry) {\n\t\t\t\tthrow(new Error(`Request with ID ${String(id)} not found`));\n\t\t\t}\n\n\t\t\tconst newEntry = ManageStatusUpdates<QueueResult>(id, currentEntry, status, ancillary, logger);\n\t\t\tconst currentTime = newEntry.updated.getTime();\n\t\t\tconst workerValue = newEntry.worker;\n\t\t\tconst newFailures = newEntry.failures ?? currentEntry.failures;\n\t\t\tconst newLastError = newEntry.lastError !== undefined ? newEntry.lastError : currentEntry.last_error;\n\t\t\tconst newOutput = newEntry.output !== undefined ? JSON.stringify(newEntry.output) : currentEntry.output;\n\n\t\t\tlet updateQuery: string;\n\t\t\tlet updateParams: (KeetaAnchorQueueRequestID | string | number | null)[];\n\n\t\t\tif (oldStatus) {\n\t\t\t\tupdateQuery = `UPDATE queue_entries\n\t\t\t\t SET status = $1, updated = $2, worker = $3, failures = $4, last_error = $5, output = $6\n\t\t\t\t WHERE id = $7 AND path = $8 AND status = $9`;\n\t\t\t\tupdateParams = [status, currentTime, workerValue, newFailures, newLastError, newOutput, id, this.pathStr, oldStatus];\n\t\t\t} else {\n\t\t\t\tupdateQuery = `UPDATE queue_entries\n\t\t\t\t SET status = $1, updated = $2, worker = $3, failures = $4, last_error = $5, output = $6\n\t\t\t\t WHERE id = $7 AND path = $8`;\n\t\t\t\tupdateParams = [status, currentTime, workerValue, newFailures, newLastError, newOutput, id, this.pathStr];\n\t\t\t}\n\n\t\t\tconst result = await client.query(updateQuery, updateParams);\n\n\t\t\tawait this.toctouDelay?.();\n\n\t\t\tif (oldStatus && result.rowCount === 0) {\n\t\t\t\tconst currentEntry = await client.query<{ status: KeetaAnchorQueueStatus }>('SELECT status FROM queue_entries WHERE id = $1 AND path = $2', [id, this.pathStr]);\n\t\t\t\tconst currentStatus = currentEntry.rows[0]?.status;\n\t\t\t\tif (currentEntry.rows.length > 0) {\n\t\t\t\t\tif (currentStatus === undefined) {\n\t\t\t\t\t\tthrow(new Error(`internal error: could not retrieve current status for request with ID ${String(id)}`));\n\t\t\t\t\t}\n\t\t\t\t\tthrow(new Errors.IncorrectStateAssertedError(id, oldStatus, currentStatus));\n\t\t\t\t} else {\n\t\t\t\t\tthrow(new Error(`Request with ID ${String(id)} not found`));\n\t\t\t\t}\n\t\t\t}\n\t\t}));\n\t}\n\n\tasync get(id: KeetaAnchorQueueRequestID): Promise<KeetaAnchorQueueEntry<QueueRequest, QueueResult> | null> {\n\t\treturn(await this.dbTransaction('get', async (client): Promise<KeetaAnchorQueueEntry<QueueRequest, QueueResult> | null> => {\n\t\t\tconst row = await client.query<QueueEntryRow>(\n\t\t\t\t`SELECT id, request, output, last_error, status, created, updated, worker, failures\n\t\t\t\t FROM queue_entries WHERE id = $1 AND path = $2`,\n\t\t\t\t[id, this.pathStr]\n\t\t\t);\n\n\t\t\tif (row.rows.length === 0) {\n\t\t\t\treturn(null);\n\t\t\t}\n\n\t\t\tconst entry = row.rows[0];\n\t\t\tif (!entry) {\n\t\t\t\treturn(null);\n\t\t\t}\n\n\t\t\tconst idempotentRows = await client.query<IdempotentRow>(\n\t\t\t\t'SELECT idempotent_id FROM queue_idempotent_keys WHERE entry_id = $1 AND path = $2',\n\t\t\t\t[id, this.pathStr]\n\t\t\t);\n\n\t\t\tconst idempotentKeys = idempotentRows.rows.length > 0\n\t\t\t\t? new Set(idempotentRows.rows.map(function(idempotentRow: IdempotentRow) {\n\t\t\t\t\treturn(ConvertStringToRequestID(idempotentRow.idempotent_id));\n\t\t\t\t}))\n\t\t\t\t: undefined;\n\n\t\t\treturn({\n\t\t\t\tid: ConvertStringToRequestID(entry.id),\n\t\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\t\trequest: JSON.parse(entry.request) as QueueRequest,\n\t\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\t\toutput: entry.output ? JSON.parse(entry.output) as QueueResult : null,\n\t\t\t\tlastError: entry.last_error,\n\t\t\t\tstatus: entry.status,\n\t\t\t\tcreated: new Date(Number(entry.created)),\n\t\t\t\tupdated: new Date(Number(entry.updated)),\n\t\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\t\tworker: entry.worker as unknown as KeetaAnchorQueueWorkerID | null,\n\t\t\t\tfailures: entry.failures,\n\t\t\t\tidempotentKeys: idempotentKeys\n\t\t\t});\n\t\t}));\n\t}\n\n\tasync query(filter?: KeetaAnchorQueueFilter): Promise<KeetaAnchorQueueEntry<QueueRequest, QueueResult>[]> {\n\t\treturn(await this.dbTransaction('query', async (client, logger): Promise<KeetaAnchorQueueEntry<QueueRequest, QueueResult>[]> => {\n\t\t\tlogger?.debug(`Querying queue with id ${this.id} with filter:`, filter);\n\n\t\t\tconst conditions: string[] = [];\n\t\t\tconst params: (string | number)[] = [];\n\t\t\tlet paramIndex = 1;\n\n\t\t\tconditions.push(`path = $${paramIndex++}`);\n\t\t\tparams.push(this.pathStr);\n\n\t\t\tif (filter?.status) {\n\t\t\t\tconditions.push(`status = $${paramIndex++}`);\n\t\t\t\tparams.push(filter.status);\n\t\t\t}\n\n\t\t\tif (filter?.updatedBefore) {\n\t\t\t\tconditions.push(`updated < $${paramIndex++}`);\n\t\t\t\tparams.push(filter.updatedBefore.getTime());\n\t\t\t}\n\n\t\t\tlet query = 'SELECT id, request, output, last_error, status, created, updated, worker, failures FROM queue_entries';\n\n\t\t\tif (conditions.length > 0) {\n\t\t\t\tquery += ' WHERE ' + conditions.join(' AND ');\n\t\t\t}\n\n\t\t\tif (filter?.limit !== undefined) {\n\t\t\t\tquery += ` LIMIT $${paramIndex++}`;\n\t\t\t\tparams.push(filter.limit);\n\t\t\t}\n\n\t\t\tconst rows = await client.query<QueueEntryRow>(query, params);\n\n\t\t\tconst entries: KeetaAnchorQueueEntry<QueueRequest, QueueResult>[] = [];\n\n\t\t\tfor (const row of rows.rows) {\n\t\t\t\tconst idempotentRows = await client.query<IdempotentRow>(\n\t\t\t\t\t'SELECT idempotent_id FROM queue_idempotent_keys WHERE entry_id = $1 AND path = $2',\n\t\t\t\t\t[row.id, this.pathStr]\n\t\t\t\t);\n\n\t\t\t\tconst idempotentKeys = idempotentRows.rows.length > 0\n\t\t\t\t\t? new Set(idempotentRows.rows.map(function(idempotentRow: IdempotentRow) {\n\t\t\t\t\t\treturn(ConvertStringToRequestID(idempotentRow.idempotent_id));\n\t\t\t\t\t}))\n\t\t\t\t\t: undefined;\n\n\t\t\t\tentries.push({\n\t\t\t\t\tid: ConvertStringToRequestID(row.id),\n\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\t\t\trequest: JSON.parse(row.request) as QueueRequest,\n\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\t\t\toutput: row.output ? JSON.parse(row.output) as QueueResult : null,\n\t\t\t\t\tlastError: row.last_error,\n\t\t\t\t\tstatus: row.status,\n\t\t\t\t\tcreated: new Date(Number(row.created)),\n\t\t\t\t\tupdated: new Date(Number(row.updated)),\n\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\t\t\tworker: row.worker as unknown as KeetaAnchorQueueWorkerID | null,\n\t\t\t\t\tfailures: row.failures,\n\t\t\t\t\tidempotentKeys: idempotentKeys\n\t\t\t\t});\n\t\t\t}\n\n\t\t\tlogger?.debug(`Queried queue with id ${this.id} with filter:`, filter, '-- found', entries.length, 'entries');\n\n\t\t\treturn(entries);\n\t\t}));\n\t}\n\n\tasync partition(path: string) : Promise<KeetaAnchorQueueStorageDriver<QueueRequest, QueueResult>> {\n\t\tthis.methodLogger('partition')?.debug(`Creating partitioned queue storage driver for path: ${path}`);\n\n\t\tif (this.poolInternal === null) {\n\t\t\tthrow(new Error('Asked to partition but the instance has been destroyed'));\n\t\t}\n\n\t\tconst retval = new KeetaAnchorQueueStorageDriverPostgres<QueueRequest, QueueResult>({\n\t\t\tid: `${this.id}::${path}`,\n\t\t\tlogger: this.logger,\n\t\t\tpool: this.poolInternal,\n\t\t\tpath: [...this.path, path]\n\t\t});\n\n\t\treturn(retval);\n\t}\n\n\tasync destroy(): Promise<void> {\n\t\tthis.methodLogger('destroy')?.debug('Destroying instance');\n\n\t\tthis.poolInternal = null;\n\t}\n\n\tasync [Symbol.asyncDispose](): Promise<void> {\n\t\treturn(await this.destroy());\n\t}\n\n\t/** @internal */\n\t_Testing(key: string): {\n\t\tsetToctouDelay(delay: number): void;\n\t\tunsetToctouDelay(): void;\n\t} {\n\t\tif (key !== 'bc81abf8-e43b-490b-b486-744fb49a5082') {\n\t\t\tthrow(new Error('This is a testing only method'));\n\t\t}\n\n\t\treturn({\n\t\t\tsetToctouDelay: (delay: number): void => {\n\t\t\t\tthis.toctouDelay = async (): Promise<void> => {\n\t\t\t\t\treturn(await asleep(delay));\n\t\t\t\t};\n\t\t\t},\n\t\t\tunsetToctouDelay: (): void => {\n\t\t\t\tthis.toctouDelay = undefined;\n\t\t\t}\n\t\t});\n\t}\n}\n"]}
|
|
@@ -8,6 +8,7 @@ export default class KeetaAnchorQueueStorageDriverRedis<QueueRequest extends JSO
|
|
|
8
8
|
readonly id: string;
|
|
9
9
|
readonly path: string[];
|
|
10
10
|
private readonly pathStr;
|
|
11
|
+
private toctouDelay;
|
|
11
12
|
constructor(options: NonNullable<ConstructorParameters<KeetaAnchorQueueStorageDriverConstructor<QueueRequest, QueueResult>>[0]> & {
|
|
12
13
|
redis: () => Promise<RedisClientType>;
|
|
13
14
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue_redis.d.ts","sourceRoot":"","sources":["../../../../src/lib/queue/drivers/queue_redis.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,6BAA6B,EAC7B,wCAAwC,EACxC,uBAAuB,EACvB,yBAAyB,EACzB,qBAAqB,EACrB,0BAA0B,EAC1B,kCAAkC,EAClC,sBAAsB,EACtB,sBAAsB,EAEtB,MAAM,aAAa,CAAC;AASrB,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"queue_redis.d.ts","sourceRoot":"","sources":["../../../../src/lib/queue/drivers/queue_redis.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,6BAA6B,EAC7B,wCAAwC,EACxC,uBAAuB,EACvB,yBAAyB,EACzB,qBAAqB,EACrB,0BAA0B,EAC1B,kCAAkC,EAClC,sBAAsB,EACtB,sBAAsB,EAEtB,MAAM,aAAa,CAAC;AASrB,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAG5D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,OAAO,CAAC;AAe7C,MAAM,CAAC,OAAO,OAAO,kCAAkC,CAAC,YAAY,SAAS,gBAAgB,GAAG,gBAAgB,EAAE,WAAW,SAAS,gBAAgB,GAAG,gBAAgB,CAAE,YAAW,6BAA6B,CAAC,YAAY,EAAE,WAAW,CAAC;IAC7O,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;IAC5C,OAAO,CAAC,aAAa,CAAiD;IAEtE,QAAQ,CAAC,IAAI,wCAAwC;IACrD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAM;IAC7B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,WAAW,CAAgD;gBAEvD,OAAO,EAAE,WAAW,CAAC,qBAAqB,CAAC,wCAAwC,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;QAAE,KAAK,EAAE,MAAM,OAAO,CAAC,eAAe,CAAC,CAAC;KAAE;IAW5K,OAAO,CAAC,YAAY;YASN,QAAQ;IAOtB,OAAO,CAAC,QAAQ;IAIhB,OAAO,CAAC,aAAa;IAIrB,OAAO,CAAC,QAAQ;IAOV,GAAG,CAAC,OAAO,EAAE,uBAAuB,CAAC,YAAY,CAAC,EAAE,IAAI,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,yBAAyB,CAAC;IAqH1H,SAAS,CAAC,EAAE,EAAE,yBAAyB,EAAE,MAAM,EAAE,sBAAsB,EAAE,SAAS,CAAC,EAAE,kCAAkC,CAAC,WAAW,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAuGpJ,GAAG,CAAC,EAAE,EAAE,yBAAyB,GAAG,OAAO,CAAC,qBAAqB,CAAC,YAAY,EAAE,WAAW,CAAC,GAAG,IAAI,CAAC;IAkCpG,KAAK,CAAC,MAAM,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,qBAAqB,CAAC,YAAY,EAAE,WAAW,CAAC,EAAE,CAAC;IAsEnG,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,6BAA6B,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;IAiB1F,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAMxB,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC;CAwB5C"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { MethodLogger, ManageStatusUpdates, ConvertStringToRequestID } from '../internal.js';
|
|
2
2
|
import { Errors } from '../common.js';
|
|
3
|
+
import { asleep } from '../../utils/asleep.js';
|
|
3
4
|
export default class KeetaAnchorQueueStorageDriverRedis {
|
|
4
5
|
logger;
|
|
5
6
|
redisInternal = null;
|
|
@@ -7,6 +8,7 @@ export default class KeetaAnchorQueueStorageDriverRedis {
|
|
|
7
8
|
id;
|
|
8
9
|
path = [];
|
|
9
10
|
pathStr;
|
|
11
|
+
toctouDelay = undefined;
|
|
10
12
|
constructor(options) {
|
|
11
13
|
this.id = options?.id ?? crypto.randomUUID();
|
|
12
14
|
this.logger = options?.logger;
|
|
@@ -46,26 +48,6 @@ export default class KeetaAnchorQueueStorageDriverRedis {
|
|
|
46
48
|
const redis = await this.getRedis();
|
|
47
49
|
const logger = this.methodLogger('add');
|
|
48
50
|
let entryID = ConvertStringToRequestID(info?.id);
|
|
49
|
-
if (entryID) {
|
|
50
|
-
const exists = await redis.exists(this.queueKey(entryID));
|
|
51
|
-
if (exists) {
|
|
52
|
-
logger?.debug(`Request with id ${String(entryID)} already exists, ignoring`);
|
|
53
|
-
return (entryID);
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
const idempotentIDs = info?.idempotentKeys;
|
|
57
|
-
if (idempotentIDs) {
|
|
58
|
-
const matchingIdempotentEntries = new Set();
|
|
59
|
-
for (const idempotentID of idempotentIDs) {
|
|
60
|
-
const existingEntryID = await redis.get(this.idempotentKey(idempotentID));
|
|
61
|
-
if (existingEntryID) {
|
|
62
|
-
matchingIdempotentEntries.add(idempotentID);
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
if (matchingIdempotentEntries.size !== 0) {
|
|
66
|
-
throw (new Errors.IdempotentExistsError('One or more idempotent entries already exist in the queue', matchingIdempotentEntries));
|
|
67
|
-
}
|
|
68
|
-
}
|
|
69
51
|
entryID ??= ConvertStringToRequestID(crypto.randomUUID());
|
|
70
52
|
logger?.debug(`Enqueuing request with id ${String(entryID)}`);
|
|
71
53
|
const currentTime = Date.now();
|
|
@@ -85,74 +67,83 @@ export default class KeetaAnchorQueueStorageDriverRedis {
|
|
|
85
67
|
worker: null,
|
|
86
68
|
failures: 0
|
|
87
69
|
};
|
|
70
|
+
const idempotentIDs = info?.idempotentKeys;
|
|
88
71
|
if (idempotentIDs && idempotentIDs.size > 0) {
|
|
89
72
|
entryData.idempotentKeys = Array.from(idempotentIDs).map(String);
|
|
90
73
|
}
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
74
|
+
await this.toctouDelay?.();
|
|
75
|
+
const idempotentKeysArr = idempotentIDs && idempotentIDs.size > 0 ? Array.from(idempotentIDs).map(String) : [];
|
|
76
|
+
const luaScript = `
|
|
77
|
+
local entryKey = KEYS[1]
|
|
78
|
+
local statusIndexKey = KEYS[2]
|
|
79
|
+
local allIndexKey = KEYS[3]
|
|
80
|
+
|
|
81
|
+
local entryData = ARGV[1]
|
|
82
|
+
local score = ARGV[2]
|
|
83
|
+
local entryId = ARGV[3]
|
|
84
|
+
local numIdempotentKeys = tonumber(ARGV[4])
|
|
85
|
+
|
|
86
|
+
-- Check if entry already exists
|
|
87
|
+
if redis.call('EXISTS', entryKey) == 1 then
|
|
88
|
+
return 'EXISTS'
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
-- Check if any idempotent keys already exist and collect which ones
|
|
92
|
+
local conflictingKeys = {}
|
|
93
|
+
for i = 1, numIdempotentKeys do
|
|
94
|
+
local idempotentKeyIndex = 4 + numIdempotentKeys + i
|
|
95
|
+
local idempotentKey = ARGV[4 + i]
|
|
96
|
+
if redis.call('EXISTS', idempotentKey) == 1 then
|
|
97
|
+
table.insert(conflictingKeys, ARGV[idempotentKeyIndex])
|
|
109
98
|
end
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
}
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
if #conflictingKeys > 0 then
|
|
102
|
+
return {'IDEMPOTENT_EXISTS', unpack(conflictingKeys)}
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
-- Add the entry
|
|
106
|
+
redis.call('SET', entryKey, entryData)
|
|
107
|
+
redis.call('ZADD', statusIndexKey, score, entryId)
|
|
108
|
+
redis.call('ZADD', allIndexKey, score, entryId)
|
|
109
|
+
|
|
110
|
+
-- Set idempotent keys
|
|
111
|
+
for i = 1, numIdempotentKeys do
|
|
112
|
+
local idempotentKey = ARGV[4 + i]
|
|
113
|
+
redis.call('SET', idempotentKey, entryId)
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
return 'OK'
|
|
117
|
+
`;
|
|
118
|
+
const idempotentKeyPairs = idempotentKeysArr.map((idKey) => {
|
|
119
|
+
return (this.idempotentKey(ConvertStringToRequestID(idKey)));
|
|
120
|
+
});
|
|
121
|
+
const result = await redis.eval(luaScript, {
|
|
122
|
+
keys: [
|
|
123
|
+
this.queueKey(entryID),
|
|
124
|
+
this.indexKey(status),
|
|
125
|
+
this.indexKey()
|
|
126
|
+
],
|
|
127
|
+
arguments: [
|
|
128
|
+
JSON.stringify(entryData),
|
|
129
|
+
String(currentTime),
|
|
130
|
+
String(entryID),
|
|
131
|
+
String(idempotentKeysArr.length),
|
|
132
|
+
...idempotentKeyPairs,
|
|
133
|
+
...idempotentKeysArr
|
|
134
|
+
]
|
|
135
|
+
});
|
|
136
|
+
if (result === 'EXISTS') {
|
|
137
|
+
logger?.debug(`Request with id ${String(entryID)} already exists, ignoring`);
|
|
138
|
+
return (entryID);
|
|
149
139
|
}
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
140
|
+
if (Array.isArray(result) && result[0] === 'IDEMPOTENT_EXISTS') {
|
|
141
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
142
|
+
const conflictingIDsStr = result.slice(1);
|
|
143
|
+
const conflictingIDs = new Set(conflictingIDsStr.map(function (conflictingID) {
|
|
144
|
+
return (ConvertStringToRequestID(conflictingID));
|
|
145
|
+
}));
|
|
146
|
+
throw (new Errors.IdempotentExistsError('One or more idempotent entries already exist in the queue', conflictingIDs));
|
|
156
147
|
}
|
|
157
148
|
return (entryID);
|
|
158
149
|
}
|
|
@@ -161,6 +152,7 @@ export default class KeetaAnchorQueueStorageDriverRedis {
|
|
|
161
152
|
const redis = await this.getRedis();
|
|
162
153
|
const logger = this.methodLogger('setStatus');
|
|
163
154
|
const entryJSON = await redis.get(this.queueKey(id));
|
|
155
|
+
await this.toctouDelay?.();
|
|
164
156
|
if (!entryJSON) {
|
|
165
157
|
throw (new Error(`Request with ID ${String(id)} not found`));
|
|
166
158
|
}
|
|
@@ -193,24 +185,24 @@ export default class KeetaAnchorQueueStorageDriverRedis {
|
|
|
193
185
|
local allIndexKey = ARGV[5]
|
|
194
186
|
local entryId = ARGV[6]
|
|
195
187
|
local score = ARGV[7]
|
|
196
|
-
|
|
188
|
+
|
|
197
189
|
local current = redis.call('GET', key)
|
|
198
190
|
if not current then
|
|
199
191
|
return {err = 'NOT_FOUND'}
|
|
200
192
|
end
|
|
201
|
-
|
|
193
|
+
|
|
202
194
|
local currentData = cjson.decode(current)
|
|
203
195
|
if currentData.status ~= expectedStatus then
|
|
204
196
|
return {err = 'STATUS_MISMATCH'}
|
|
205
197
|
end
|
|
206
|
-
|
|
198
|
+
|
|
207
199
|
redis.call('SET', key, newData)
|
|
208
200
|
if oldIndexKey ~= newIndexKey then
|
|
209
201
|
redis.call('ZREM', oldIndexKey, entryId)
|
|
210
202
|
end
|
|
211
203
|
redis.call('ZADD', newIndexKey, score, entryId)
|
|
212
204
|
redis.call('ZADD', allIndexKey, score, entryId)
|
|
213
|
-
|
|
205
|
+
|
|
214
206
|
return {ok = 'OK'}
|
|
215
207
|
`;
|
|
216
208
|
const result = await redis.eval(luaScript, {
|
|
@@ -355,5 +347,21 @@ export default class KeetaAnchorQueueStorageDriverRedis {
|
|
|
355
347
|
async [Symbol.asyncDispose]() {
|
|
356
348
|
return (await this.destroy());
|
|
357
349
|
}
|
|
350
|
+
/** @internal */
|
|
351
|
+
_Testing(key) {
|
|
352
|
+
if (key !== 'bc81abf8-e43b-490b-b486-744fb49a5082') {
|
|
353
|
+
throw (new Error('This is a testing only method'));
|
|
354
|
+
}
|
|
355
|
+
return ({
|
|
356
|
+
setToctouDelay: (delay) => {
|
|
357
|
+
this.toctouDelay = async () => {
|
|
358
|
+
return (await asleep(delay));
|
|
359
|
+
};
|
|
360
|
+
},
|
|
361
|
+
unsetToctouDelay: () => {
|
|
362
|
+
this.toctouDelay = undefined;
|
|
363
|
+
}
|
|
364
|
+
});
|
|
365
|
+
}
|
|
358
366
|
}
|
|
359
367
|
//# sourceMappingURL=queue_redis.js.map
|