@workglow/postgres 0.2.35 → 0.2.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/job-queue/PostgresJobStore.d.ts +29 -0
- package/dist/job-queue/PostgresJobStore.d.ts.map +1 -0
- package/dist/job-queue/PostgresMessageQueue.d.ts +38 -0
- package/dist/job-queue/PostgresMessageQueue.d.ts.map +1 -0
- package/dist/job-queue/PostgresQueueStorage.d.ts +41 -10
- package/dist/job-queue/PostgresQueueStorage.d.ts.map +1 -1
- package/dist/job-queue/PostgresRateLimiterStorage.d.ts +1 -2
- package/dist/job-queue/PostgresRateLimiterStorage.d.ts.map +1 -1
- package/dist/job-queue/browser.js +401 -53
- package/dist/job-queue/browser.js.map +11 -8
- package/dist/job-queue/common.d.ts +3 -0
- package/dist/job-queue/common.d.ts.map +1 -1
- package/dist/job-queue/createPostgresQueue.d.ts +22 -0
- package/dist/job-queue/createPostgresQueue.d.ts.map +1 -0
- package/dist/job-queue/node.js +401 -53
- package/dist/job-queue/node.js.map +11 -8
- package/dist/migrations/PostgresMigrationRunner.d.ts +1 -1
- package/dist/migrations/PostgresMigrationRunner.d.ts.map +1 -1
- package/dist/migrations/postgresQueueMigrations.d.ts +9 -1
- package/dist/migrations/postgresQueueMigrations.d.ts.map +1 -1
- package/dist/migrations/postgresRateLimiterMigrations.d.ts +1 -1
- package/dist/migrations/postgresRateLimiterMigrations.d.ts.map +1 -1
- package/dist/storage/PostgresKvStorage.d.ts +1 -1
- package/dist/storage/PostgresKvStorage.d.ts.map +1 -1
- package/dist/storage/PostgresTabularStorage.d.ts +1 -1
- package/dist/storage/PostgresTabularStorage.d.ts.map +1 -1
- package/dist/storage/PostgresVectorStorage.d.ts +1 -1
- package/dist/storage/PostgresVectorStorage.d.ts.map +1 -1
- package/dist/storage/browser.js +28 -12
- package/dist/storage/browser.js.map +5 -5
- package/dist/storage/node.js +28 -12
- package/dist/storage/node.js.map +6 -6
- package/dist/text/PostgresFtsTextIndex.d.ts.map +1 -1
- package/dist/text/browser.js.map +2 -2
- package/dist/text/node.js.map +2 -2
- package/package.json +7 -7
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
// src/job-queue/PostgresQueueStorage.ts
|
|
2
|
-
import {
|
|
3
|
-
import { createServiceToken, getLogger, makeFingerprint, uuid4 } from "@workglow/util";
|
|
4
|
-
import { JobStatus as JobStatus2 } from "@workglow/job-queue";
|
|
2
|
+
import { JobStatus as JobStatus2, validateLeaseMs } from "@workglow/job-queue";
|
|
5
3
|
import {
|
|
6
4
|
assertPrefixesSafe,
|
|
7
5
|
buildPrefixInsertFragments,
|
|
@@ -10,6 +8,8 @@ import {
|
|
|
10
8
|
getPrefixParamValues,
|
|
11
9
|
PostgresDialect as PostgresDialect2
|
|
12
10
|
} from "@workglow/storage";
|
|
11
|
+
import { createServiceToken, getLogger, makeFingerprint, uuid4 } from "@workglow/util";
|
|
12
|
+
import { createHash } from "node:crypto";
|
|
13
13
|
|
|
14
14
|
// src/migrations/PostgresMigrationRunner.ts
|
|
15
15
|
import {
|
|
@@ -149,11 +149,6 @@ var JOB_STATUS_V1 = [
|
|
|
149
149
|
];
|
|
150
150
|
function assertJobStatusMatchesV1() {
|
|
151
151
|
const current = new Set(Object.values(JobStatus));
|
|
152
|
-
for (const v of JOB_STATUS_V1) {
|
|
153
|
-
if (!current.has(v)) {
|
|
154
|
-
throw new Error(`JobStatus const is missing v1 enum value "${v}"; v1 migration values are frozen.`);
|
|
155
|
-
}
|
|
156
|
-
}
|
|
157
152
|
for (const v of current) {
|
|
158
153
|
if (!JOB_STATUS_V1.includes(v)) {
|
|
159
154
|
throw new Error(`JobStatus contains "${v}" which is not in JOB_STATUS_V1. ` + `Add a new migration that runs "ALTER TYPE job_status ADD VALUE IF NOT EXISTS '${v}'" ` + `instead of mutating the v1 enum literal.`);
|
|
@@ -254,6 +249,56 @@ function postgresQueueMigrations(tableName, prefixes) {
|
|
|
254
249
|
});
|
|
255
250
|
}
|
|
256
251
|
}
|
|
252
|
+
},
|
|
253
|
+
{
|
|
254
|
+
component,
|
|
255
|
+
version: 2,
|
|
256
|
+
description: "Add abort_requested_at and lease_expires_at columns",
|
|
257
|
+
async up(db) {
|
|
258
|
+
await db.query(`
|
|
259
|
+
ALTER TABLE ${tableName}
|
|
260
|
+
ADD COLUMN IF NOT EXISTS abort_requested_at timestamp with time zone,
|
|
261
|
+
ADD COLUMN IF NOT EXISTS lease_expires_at timestamp with time zone
|
|
262
|
+
`);
|
|
263
|
+
}
|
|
264
|
+
},
|
|
265
|
+
{
|
|
266
|
+
component,
|
|
267
|
+
version: 3,
|
|
268
|
+
description: "Rename run_after→visible_at, last_ran_at→last_attempted_at, run_attempts→attempts, max_retries→max_attempts, worker_id→lease_owner; drop run_after-keyed indexes and recreate visible_at-keyed",
|
|
269
|
+
async up(db) {
|
|
270
|
+
await db.query(`
|
|
271
|
+
DO $$
|
|
272
|
+
BEGIN
|
|
273
|
+
IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='${tableName}' AND column_name='run_after' AND table_schema=current_schema()) THEN
|
|
274
|
+
EXECUTE 'ALTER TABLE ${tableName} RENAME COLUMN run_after TO visible_at';
|
|
275
|
+
END IF;
|
|
276
|
+
IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='${tableName}' AND column_name='last_ran_at' AND table_schema=current_schema()) THEN
|
|
277
|
+
EXECUTE 'ALTER TABLE ${tableName} RENAME COLUMN last_ran_at TO last_attempted_at';
|
|
278
|
+
END IF;
|
|
279
|
+
IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='${tableName}' AND column_name='run_attempts' AND table_schema=current_schema()) THEN
|
|
280
|
+
EXECUTE 'ALTER TABLE ${tableName} RENAME COLUMN run_attempts TO attempts';
|
|
281
|
+
END IF;
|
|
282
|
+
IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='${tableName}' AND column_name='max_retries' AND table_schema=current_schema()) THEN
|
|
283
|
+
EXECUTE 'ALTER TABLE ${tableName} RENAME COLUMN max_retries TO max_attempts';
|
|
284
|
+
EXECUTE 'ALTER TABLE ${tableName} ALTER COLUMN max_attempts SET DEFAULT 10';
|
|
285
|
+
END IF;
|
|
286
|
+
IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='${tableName}' AND column_name='worker_id' AND table_schema=current_schema()) THEN
|
|
287
|
+
EXECUTE 'ALTER TABLE ${tableName} RENAME COLUMN worker_id TO lease_owner';
|
|
288
|
+
END IF;
|
|
289
|
+
END $$
|
|
290
|
+
`);
|
|
291
|
+
await db.query(`DROP INDEX IF EXISTS job_fetcher${indexSuffix}_idx`);
|
|
292
|
+
await db.query(`DROP INDEX IF EXISTS job_queue_fetcher${indexSuffix}_idx`);
|
|
293
|
+
await db.query(`
|
|
294
|
+
CREATE INDEX IF NOT EXISTS job_fetcher${indexSuffix}_idx
|
|
295
|
+
ON ${tableName} (${prefixIndexPrefix}id, status, visible_at)
|
|
296
|
+
`);
|
|
297
|
+
await db.query(`
|
|
298
|
+
CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx
|
|
299
|
+
ON ${tableName} (${prefixIndexPrefix}queue, status, visible_at)
|
|
300
|
+
`);
|
|
301
|
+
}
|
|
257
302
|
}
|
|
258
303
|
];
|
|
259
304
|
}
|
|
@@ -308,26 +353,26 @@ class PostgresQueueStorage {
|
|
|
308
353
|
job.progress_message = "";
|
|
309
354
|
job.progress_details = null;
|
|
310
355
|
job.created_at = now;
|
|
311
|
-
job.
|
|
356
|
+
job.visible_at = now;
|
|
312
357
|
const prefixColumnNames = getPrefixColumnNames(this.prefixes);
|
|
313
358
|
const { columns: prefixColumnsInsert, placeholders: prefixParamPlaceholders } = buildPrefixInsertFragments(PostgresDialect2, this.prefixes, 1);
|
|
314
359
|
const prefixParamValues = this.getPrefixParamValues();
|
|
315
360
|
const baseParamStart = prefixColumnNames.length + 1;
|
|
316
361
|
const sql = `
|
|
317
362
|
INSERT INTO ${this.tableName}(
|
|
318
|
-
${prefixColumnsInsert}queue,
|
|
319
|
-
fingerprint,
|
|
320
|
-
input,
|
|
321
|
-
|
|
363
|
+
${prefixColumnsInsert}queue,
|
|
364
|
+
fingerprint,
|
|
365
|
+
input,
|
|
366
|
+
visible_at,
|
|
322
367
|
created_at,
|
|
323
368
|
deadline_at,
|
|
324
|
-
|
|
325
|
-
job_run_id,
|
|
326
|
-
progress,
|
|
327
|
-
progress_message,
|
|
369
|
+
max_attempts,
|
|
370
|
+
job_run_id,
|
|
371
|
+
progress,
|
|
372
|
+
progress_message,
|
|
328
373
|
progress_details
|
|
329
374
|
)
|
|
330
|
-
VALUES
|
|
375
|
+
VALUES
|
|
331
376
|
(${prefixParamPlaceholders}$${baseParamStart},$${baseParamStart + 1},$${baseParamStart + 2},$${baseParamStart + 3},$${baseParamStart + 4},$${baseParamStart + 5},$${baseParamStart + 6},$${baseParamStart + 7},$${baseParamStart + 8},$${baseParamStart + 9},$${baseParamStart + 10})
|
|
332
377
|
RETURNING id`;
|
|
333
378
|
const params = [
|
|
@@ -335,10 +380,10 @@ class PostgresQueueStorage {
|
|
|
335
380
|
job.queue,
|
|
336
381
|
job.fingerprint,
|
|
337
382
|
JSON.stringify(job.input),
|
|
338
|
-
job.
|
|
383
|
+
job.visible_at,
|
|
339
384
|
job.created_at,
|
|
340
385
|
job.deadline_at,
|
|
341
|
-
job.
|
|
386
|
+
job.max_attempts,
|
|
342
387
|
job.job_run_id,
|
|
343
388
|
job.progress,
|
|
344
389
|
job.progress_message,
|
|
@@ -369,32 +414,68 @@ class PostgresQueueStorage {
|
|
|
369
414
|
FROM ${this.tableName}
|
|
370
415
|
WHERE queue = $1
|
|
371
416
|
AND status = $2${prefixConditions}
|
|
372
|
-
ORDER BY
|
|
417
|
+
ORDER BY visible_at ASC
|
|
373
418
|
LIMIT $3
|
|
374
419
|
FOR UPDATE SKIP LOCKED`, [this.queueName, status, num, ...prefixParams]);
|
|
375
420
|
if (!result)
|
|
376
421
|
return [];
|
|
377
422
|
return result.rows;
|
|
378
423
|
}
|
|
379
|
-
async next(workerId) {
|
|
380
|
-
const
|
|
424
|
+
async next(workerId, opts) {
|
|
425
|
+
const leaseMs = opts?.leaseMs ?? 30000;
|
|
426
|
+
validateLeaseMs(leaseMs, "leaseMs");
|
|
427
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(7);
|
|
381
428
|
const result = await this.db.query(`
|
|
382
|
-
UPDATE ${this.tableName}
|
|
383
|
-
SET status = $1,
|
|
429
|
+
UPDATE ${this.tableName}
|
|
430
|
+
SET status = $1,
|
|
431
|
+
last_attempted_at = NOW() AT TIME ZONE 'UTC',
|
|
432
|
+
lease_owner = $4,
|
|
433
|
+
lease_expires_at = NOW() AT TIME ZONE 'UTC' + ($2 * INTERVAL '1 millisecond'),
|
|
434
|
+
-- A reclaimed PROCESSING row was claimed by a now-crashed worker;
|
|
435
|
+
-- that constitutes one used-up attempt against max_attempts.
|
|
436
|
+
-- PENDING claims must not be charged here — JobQueueWorker's
|
|
437
|
+
-- existing validateJobState() will FAIL the job in the next-step
|
|
438
|
+
-- branch when attempts >= max_attempts.
|
|
439
|
+
attempts = CASE WHEN status = $6 THEN attempts + 1 ELSE attempts END,
|
|
440
|
+
-- Always clear any stale abort_requested_at on (re)claim. A PROCESSING
|
|
441
|
+
-- row may have had abort_requested_at set before the worker crashed;
|
|
442
|
+
-- the new owner must start with a clean slate or the worker will see
|
|
443
|
+
-- the abort flag immediately and never run user code.
|
|
444
|
+
abort_requested_at = NULL
|
|
384
445
|
WHERE id = (
|
|
385
|
-
SELECT id
|
|
386
|
-
FROM ${this.tableName}
|
|
387
|
-
WHERE queue = $
|
|
388
|
-
AND
|
|
446
|
+
SELECT id
|
|
447
|
+
FROM ${this.tableName}
|
|
448
|
+
WHERE queue = $3
|
|
449
|
+
AND (
|
|
450
|
+
(status = $5 AND visible_at <= NOW() AT TIME ZONE 'UTC')
|
|
451
|
+
OR (status = $6 AND (lease_expires_at IS NULL OR lease_expires_at < NOW() AT TIME ZONE 'UTC'))
|
|
452
|
+
)
|
|
389
453
|
${prefixConditions}
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
FOR UPDATE SKIP LOCKED
|
|
454
|
+
ORDER BY visible_at ASC
|
|
455
|
+
FOR UPDATE SKIP LOCKED
|
|
393
456
|
LIMIT 1
|
|
394
457
|
)
|
|
395
|
-
RETURNING *`, [
|
|
458
|
+
RETURNING *`, [
|
|
459
|
+
JobStatus2.PROCESSING,
|
|
460
|
+
leaseMs,
|
|
461
|
+
this.queueName,
|
|
462
|
+
workerId,
|
|
463
|
+
JobStatus2.PENDING,
|
|
464
|
+
JobStatus2.PROCESSING,
|
|
465
|
+
...prefixParams
|
|
466
|
+
]);
|
|
396
467
|
return result?.rows?.[0] ?? undefined;
|
|
397
468
|
}
|
|
469
|
+
async extendLease(id, workerId, ms) {
|
|
470
|
+
validateLeaseMs(ms, "ms");
|
|
471
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(5);
|
|
472
|
+
const result = await this.db.query(`UPDATE ${this.tableName}
|
|
473
|
+
SET lease_expires_at = NOW() AT TIME ZONE 'UTC' + ($1 * INTERVAL '1 millisecond')
|
|
474
|
+
WHERE id = $2 AND queue = $3 AND lease_owner = $4 AND status = 'PROCESSING'${prefixConditions}`, [ms, id, this.queueName, workerId, ...prefixParams]);
|
|
475
|
+
if (!result || result.rowCount === 0) {
|
|
476
|
+
throw new Error(`extendLease failed: job ${String(id)} is not PROCESSING or lease is not owned by worker ${workerId}`);
|
|
477
|
+
}
|
|
478
|
+
}
|
|
398
479
|
async size(status = JobStatus2.PENDING) {
|
|
399
480
|
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
400
481
|
const result = await this.db.query(`
|
|
@@ -420,22 +501,23 @@ class PostgresQueueStorage {
|
|
|
420
501
|
WHERE id = $2 AND queue = $3${prefixConditions}`, [jobDetails.status, jobDetails.id, this.queueName, ...prefixParams]);
|
|
421
502
|
} else if (jobDetails.status === JobStatus2.PENDING) {
|
|
422
503
|
const { conditions: prefixConditions } = this.buildPrefixWhereClause(7);
|
|
423
|
-
await this.db.query(`UPDATE ${this.tableName}
|
|
424
|
-
SET
|
|
425
|
-
error = $1,
|
|
504
|
+
await this.db.query(`UPDATE ${this.tableName}
|
|
505
|
+
SET
|
|
506
|
+
error = $1,
|
|
426
507
|
error_code = $2,
|
|
427
|
-
status = $3,
|
|
428
|
-
|
|
508
|
+
status = $3,
|
|
509
|
+
visible_at = $4,
|
|
429
510
|
progress = 0,
|
|
430
511
|
progress_message = '',
|
|
431
512
|
progress_details = NULL,
|
|
432
|
-
|
|
433
|
-
|
|
513
|
+
attempts = attempts + 1,
|
|
514
|
+
last_attempted_at = NOW() AT TIME ZONE 'UTC',
|
|
515
|
+
abort_requested_at = NULL
|
|
434
516
|
WHERE id = $5 AND queue = $6${prefixConditions}`, [
|
|
435
517
|
jobDetails.error,
|
|
436
518
|
jobDetails.error_code,
|
|
437
519
|
jobDetails.status,
|
|
438
|
-
jobDetails.
|
|
520
|
+
jobDetails.visible_at,
|
|
439
521
|
jobDetails.id,
|
|
440
522
|
this.queueName,
|
|
441
523
|
...prefixParams
|
|
@@ -452,9 +534,9 @@ class PostgresQueueStorage {
|
|
|
452
534
|
progress = 100,
|
|
453
535
|
progress_message = '',
|
|
454
536
|
progress_details = NULL,
|
|
455
|
-
|
|
537
|
+
attempts = attempts + 1,
|
|
456
538
|
completed_at = NOW() AT TIME ZONE 'UTC',
|
|
457
|
-
|
|
539
|
+
last_attempted_at = NOW() AT TIME ZONE 'UTC'
|
|
458
540
|
WHERE id = $5 AND queue = $6${prefixConditions}`, [
|
|
459
541
|
jobDetails.output ? JSON.stringify(jobDetails.output) : null,
|
|
460
542
|
jobDetails.error ?? null,
|
|
@@ -466,6 +548,49 @@ class PostgresQueueStorage {
|
|
|
466
548
|
]);
|
|
467
549
|
}
|
|
468
550
|
}
|
|
551
|
+
async finalize(id, fields) {
|
|
552
|
+
const sets = [];
|
|
553
|
+
const params = [];
|
|
554
|
+
let nextParam = 1;
|
|
555
|
+
const push = (col, value) => {
|
|
556
|
+
sets.push(`${col} = $${nextParam}`);
|
|
557
|
+
params.push(value);
|
|
558
|
+
nextParam += 1;
|
|
559
|
+
};
|
|
560
|
+
if ("output" in fields) {
|
|
561
|
+
push("output", fields.output != null ? JSON.stringify(fields.output) : null);
|
|
562
|
+
}
|
|
563
|
+
if ("error" in fields)
|
|
564
|
+
push("error", fields.error ?? null);
|
|
565
|
+
if ("error_code" in fields)
|
|
566
|
+
push("error_code", fields.error_code ?? null);
|
|
567
|
+
if ("status" in fields)
|
|
568
|
+
push("status", fields.status);
|
|
569
|
+
if ("completed_at" in fields)
|
|
570
|
+
push("completed_at", fields.completed_at ?? null);
|
|
571
|
+
if ("abort_requested_at" in fields) {
|
|
572
|
+
push("abort_requested_at", fields.abort_requested_at ?? null);
|
|
573
|
+
}
|
|
574
|
+
if ("lease_owner" in fields)
|
|
575
|
+
push("lease_owner", fields.lease_owner ?? null);
|
|
576
|
+
if ("progress" in fields)
|
|
577
|
+
push("progress", fields.progress ?? 0);
|
|
578
|
+
if ("progress_message" in fields)
|
|
579
|
+
push("progress_message", fields.progress_message ?? "");
|
|
580
|
+
if ("progress_details" in fields) {
|
|
581
|
+
push("progress_details", fields.progress_details != null ? JSON.stringify(fields.progress_details) : null);
|
|
582
|
+
}
|
|
583
|
+
if (sets.length === 0)
|
|
584
|
+
return;
|
|
585
|
+
const idParam = nextParam;
|
|
586
|
+
nextParam += 1;
|
|
587
|
+
const queueParam = nextParam;
|
|
588
|
+
nextParam += 1;
|
|
589
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(nextParam);
|
|
590
|
+
await this.db.query(`UPDATE ${this.tableName}
|
|
591
|
+
SET ${sets.join(", ")}
|
|
592
|
+
WHERE id = $${idParam} AND queue = $${queueParam}${prefixConditions}`, [...params, id, this.queueName, ...prefixParams]);
|
|
593
|
+
}
|
|
469
594
|
async deleteAll() {
|
|
470
595
|
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
|
|
471
596
|
await this.db.query(`
|
|
@@ -484,23 +609,37 @@ class PostgresQueueStorage {
|
|
|
484
609
|
return result.rows[0].output;
|
|
485
610
|
}
|
|
486
611
|
async abort(jobId) {
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
UPDATE ${this.tableName}
|
|
490
|
-
|
|
491
|
-
|
|
612
|
+
{
|
|
613
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(4);
|
|
614
|
+
await this.db.query(`UPDATE ${this.tableName}
|
|
615
|
+
SET status = 'FAILED',
|
|
616
|
+
abort_requested_at = NOW() AT TIME ZONE 'UTC',
|
|
617
|
+
completed_at = NOW() AT TIME ZONE 'UTC'
|
|
618
|
+
WHERE id = $1 AND queue = $2 AND status = $3${prefixConditions}`, [jobId, this.queueName, JobStatus2.PENDING, ...prefixParams]);
|
|
619
|
+
}
|
|
620
|
+
{
|
|
621
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(4);
|
|
622
|
+
await this.db.query(`UPDATE ${this.tableName}
|
|
623
|
+
SET abort_requested_at = NOW() AT TIME ZONE 'UTC'
|
|
624
|
+
WHERE id = $1 AND queue = $2 AND status = $3${prefixConditions}`, [jobId, this.queueName, JobStatus2.PROCESSING, ...prefixParams]);
|
|
625
|
+
}
|
|
492
626
|
}
|
|
493
|
-
async
|
|
627
|
+
async releaseClaim(jobId) {
|
|
494
628
|
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
495
629
|
await this.db.query(`
|
|
496
630
|
UPDATE ${this.tableName}
|
|
497
631
|
SET status = 'PENDING',
|
|
498
|
-
|
|
632
|
+
lease_owner = NULL,
|
|
499
633
|
progress = 0,
|
|
500
634
|
progress_message = '',
|
|
501
|
-
progress_details = NULL
|
|
635
|
+
progress_details = NULL,
|
|
636
|
+
abort_requested_at = NULL
|
|
502
637
|
WHERE id = $1 AND queue = $2${prefixConditions}`, [jobId, this.queueName, ...prefixParams]);
|
|
503
638
|
}
|
|
639
|
+
async saveStatus(jobId, status) {
|
|
640
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
641
|
+
await this.db.query(`UPDATE ${this.tableName} SET status = $1 WHERE id = $2 AND queue = $3${prefixConditions}`, [status, jobId, this.queueName, ...prefixParams]);
|
|
642
|
+
}
|
|
504
643
|
async getByRunId(job_run_id) {
|
|
505
644
|
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
506
645
|
const result = await this.db.query(`
|
|
@@ -663,7 +802,6 @@ class PostgresQueueStorage {
|
|
|
663
802
|
}
|
|
664
803
|
}
|
|
665
804
|
// src/job-queue/PostgresRateLimiterStorage.ts
|
|
666
|
-
import { createServiceToken as createServiceToken2 } from "@workglow/util";
|
|
667
805
|
import {
|
|
668
806
|
assertPrefixesSafe as assertPrefixesSafe2,
|
|
669
807
|
buildPrefixWhereClause as buildPrefixWhereClause2,
|
|
@@ -671,6 +809,7 @@ import {
|
|
|
671
809
|
getPrefixParamValues as getPrefixParamValues2,
|
|
672
810
|
PostgresDialect as PostgresDialect4
|
|
673
811
|
} from "@workglow/storage";
|
|
812
|
+
import { createServiceToken as createServiceToken2 } from "@workglow/util";
|
|
674
813
|
|
|
675
814
|
// src/migrations/postgresRateLimiterMigrations.ts
|
|
676
815
|
import {
|
|
@@ -895,14 +1034,223 @@ class PostgresRateLimiterStorage {
|
|
|
895
1034
|
await this.db.query(`DELETE FROM ${this.nextAvailableTableName} WHERE queue_name = $1${prefixConditions}`, [queueName, ...prefixParams]);
|
|
896
1035
|
}
|
|
897
1036
|
}
|
|
1037
|
+
// src/job-queue/PostgresMessageQueue.ts
|
|
1038
|
+
class PostgresClaim {
|
|
1039
|
+
core;
|
|
1040
|
+
pending;
|
|
1041
|
+
id;
|
|
1042
|
+
body;
|
|
1043
|
+
attempts;
|
|
1044
|
+
workerId;
|
|
1045
|
+
constructor(core, pending, id, body, attempts, workerId) {
|
|
1046
|
+
this.core = core;
|
|
1047
|
+
this.pending = pending;
|
|
1048
|
+
this.id = id;
|
|
1049
|
+
this.body = body;
|
|
1050
|
+
this.attempts = attempts;
|
|
1051
|
+
this.workerId = workerId;
|
|
1052
|
+
}
|
|
1053
|
+
async ack(result) {
|
|
1054
|
+
const buf = this.pending.get(this.id);
|
|
1055
|
+
this.pending.delete(this.id);
|
|
1056
|
+
const current = await this.core.get(this.id) ?? this.body;
|
|
1057
|
+
const output = result !== undefined ? result : buf?.output !== undefined ? buf.output : current.output ?? null;
|
|
1058
|
+
await this.core.finalize(this.id, {
|
|
1059
|
+
output,
|
|
1060
|
+
error: null,
|
|
1061
|
+
error_code: null,
|
|
1062
|
+
status: "COMPLETED",
|
|
1063
|
+
completed_at: current.completed_at ?? new Date().toISOString()
|
|
1064
|
+
});
|
|
1065
|
+
}
|
|
1066
|
+
async retry(opts) {
|
|
1067
|
+
this.pending.delete(this.id);
|
|
1068
|
+
const delay = opts?.delaySeconds ?? 0;
|
|
1069
|
+
const current = await this.core.get(this.id) ?? this.body;
|
|
1070
|
+
await this.core.complete({
|
|
1071
|
+
...current,
|
|
1072
|
+
status: "PENDING",
|
|
1073
|
+
lease_owner: null,
|
|
1074
|
+
lease_expires_at: null,
|
|
1075
|
+
visible_at: new Date(Date.now() + delay * 1000).toISOString(),
|
|
1076
|
+
progress: 0,
|
|
1077
|
+
progress_message: "",
|
|
1078
|
+
progress_details: null
|
|
1079
|
+
});
|
|
1080
|
+
}
|
|
1081
|
+
async fail(opts) {
|
|
1082
|
+
opts?.permanent;
|
|
1083
|
+
const buf = this.pending.get(this.id);
|
|
1084
|
+
this.pending.delete(this.id);
|
|
1085
|
+
const current = await this.core.get(this.id) ?? this.body;
|
|
1086
|
+
const error = opts?.error !== undefined ? opts.error : buf?.error !== undefined ? buf.error : current.error ?? null;
|
|
1087
|
+
const errorCode = opts?.errorCode !== undefined ? opts.errorCode : buf?.errorCode !== undefined ? buf.errorCode : current.error_code ?? null;
|
|
1088
|
+
const abortRequested = opts?.abortRequested !== undefined ? opts.abortRequested : buf?.abortRequested ?? false;
|
|
1089
|
+
await this.core.finalize(this.id, {
|
|
1090
|
+
error,
|
|
1091
|
+
error_code: errorCode,
|
|
1092
|
+
abort_requested_at: abortRequested ? current.abort_requested_at ?? new Date().toISOString() : current.abort_requested_at ?? null,
|
|
1093
|
+
status: "FAILED",
|
|
1094
|
+
completed_at: current.completed_at ?? new Date().toISOString()
|
|
1095
|
+
});
|
|
1096
|
+
}
|
|
1097
|
+
async extendLease(ms) {
|
|
1098
|
+
await this.core.extendLease(this.id, this.workerId, ms);
|
|
1099
|
+
}
|
|
1100
|
+
async disable() {
|
|
1101
|
+
this.pending.delete(this.id);
|
|
1102
|
+
const current = await this.core.get(this.id);
|
|
1103
|
+
const completedAt = current?.completed_at ?? new Date().toISOString();
|
|
1104
|
+
await this.core.finalize(this.id, {
|
|
1105
|
+
status: "DISABLED",
|
|
1106
|
+
completed_at: completedAt,
|
|
1107
|
+
lease_owner: null,
|
|
1108
|
+
progress: 0,
|
|
1109
|
+
progress_message: "",
|
|
1110
|
+
progress_details: null
|
|
1111
|
+
});
|
|
1112
|
+
}
|
|
1113
|
+
}
|
|
1114
|
+
|
|
1115
|
+
class PostgresMessageQueue {
|
|
1116
|
+
scope;
|
|
1117
|
+
core;
|
|
1118
|
+
pending;
|
|
1119
|
+
constructor(core, pending) {
|
|
1120
|
+
this.core = core;
|
|
1121
|
+
this.pending = pending;
|
|
1122
|
+
this.scope = core.scope;
|
|
1123
|
+
}
|
|
1124
|
+
async send(body, opts) {
|
|
1125
|
+
return this.core.add(applySendOptions(body, opts));
|
|
1126
|
+
}
|
|
1127
|
+
async sendBatch(bodies, opts) {
|
|
1128
|
+
const ids = [];
|
|
1129
|
+
for (const body of bodies) {
|
|
1130
|
+
ids.push(await this.send(body, opts));
|
|
1131
|
+
}
|
|
1132
|
+
return ids;
|
|
1133
|
+
}
|
|
1134
|
+
async receive(opts) {
|
|
1135
|
+
const max = Math.max(1, opts.max ?? 1);
|
|
1136
|
+
const claims = [];
|
|
1137
|
+
while (claims.length < max) {
|
|
1138
|
+
const job = await this.core.next(opts.workerId, { leaseMs: opts.leaseMs });
|
|
1139
|
+
if (!job)
|
|
1140
|
+
break;
|
|
1141
|
+
claims.push(new PostgresClaim(this.core, this.pending, job.id, job, job.attempts ?? 0, opts.workerId));
|
|
1142
|
+
}
|
|
1143
|
+
return claims;
|
|
1144
|
+
}
|
|
1145
|
+
async releaseClaim(id) {
|
|
1146
|
+
this.pending.delete(id);
|
|
1147
|
+
await this.core.releaseClaim(id);
|
|
1148
|
+
}
|
|
1149
|
+
async migrate() {
|
|
1150
|
+
await this.core.migrate();
|
|
1151
|
+
}
|
|
1152
|
+
getMigrations() {
|
|
1153
|
+
return this.core.getMigrations();
|
|
1154
|
+
}
|
|
1155
|
+
subscribeToChanges(callback, options) {
|
|
1156
|
+
return this.core.subscribeToChanges(callback, options);
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
function applySendOptions(body, opts) {
|
|
1160
|
+
if (!opts)
|
|
1161
|
+
return body;
|
|
1162
|
+
const out = { ...body };
|
|
1163
|
+
if (opts.delaySeconds != null) {
|
|
1164
|
+
out.visible_at = new Date(Date.now() + opts.delaySeconds * 1000).toISOString();
|
|
1165
|
+
}
|
|
1166
|
+
if (opts.timeoutSeconds != null) {
|
|
1167
|
+
out.deadline_at = new Date(Date.now() + opts.timeoutSeconds * 1000).toISOString();
|
|
1168
|
+
}
|
|
1169
|
+
if (opts.fingerprint != null)
|
|
1170
|
+
out.fingerprint = opts.fingerprint;
|
|
1171
|
+
if (opts.jobRunId != null)
|
|
1172
|
+
out.job_run_id = opts.jobRunId;
|
|
1173
|
+
if (opts.maxAttempts != null)
|
|
1174
|
+
out.max_attempts = opts.maxAttempts;
|
|
1175
|
+
return out;
|
|
1176
|
+
}
|
|
1177
|
+
// src/job-queue/PostgresJobStore.ts
|
|
1178
|
+
class PostgresJobStore {
|
|
1179
|
+
core;
|
|
1180
|
+
pending;
|
|
1181
|
+
constructor(core, pending) {
|
|
1182
|
+
this.core = core;
|
|
1183
|
+
this.pending = pending;
|
|
1184
|
+
}
|
|
1185
|
+
get(id) {
|
|
1186
|
+
return this.core.get(id);
|
|
1187
|
+
}
|
|
1188
|
+
async peek(status, num) {
|
|
1189
|
+
return this.core.peek(status, num);
|
|
1190
|
+
}
|
|
1191
|
+
size(status) {
|
|
1192
|
+
return this.core.size(status);
|
|
1193
|
+
}
|
|
1194
|
+
async getByRunId(runId) {
|
|
1195
|
+
return this.core.getByRunId(runId);
|
|
1196
|
+
}
|
|
1197
|
+
outputForInput(input) {
|
|
1198
|
+
return this.core.outputForInput(input);
|
|
1199
|
+
}
|
|
1200
|
+
async saveProgress(id, progress, message, details) {
|
|
1201
|
+
await this.core.saveProgress(id, progress, message, details);
|
|
1202
|
+
}
|
|
1203
|
+
async saveResult(id, output) {
|
|
1204
|
+
const buf = this.pending.get(id) ?? {};
|
|
1205
|
+
buf.output = output ?? null;
|
|
1206
|
+
this.pending.set(id, buf);
|
|
1207
|
+
}
|
|
1208
|
+
async saveError(id, error, errorCode, abortRequested) {
|
|
1209
|
+
const buf = this.pending.get(id) ?? {};
|
|
1210
|
+
buf.error = error;
|
|
1211
|
+
buf.errorCode = errorCode;
|
|
1212
|
+
buf.abortRequested = abortRequested;
|
|
1213
|
+
this.pending.set(id, buf);
|
|
1214
|
+
}
|
|
1215
|
+
async deleteByStatusAndAge(status, olderThanMs) {
|
|
1216
|
+
await this.core.deleteJobsByStatusAndAge(status, olderThanMs);
|
|
1217
|
+
}
|
|
1218
|
+
async delete(id) {
|
|
1219
|
+
this.pending.delete(id);
|
|
1220
|
+
await this.core.delete(id);
|
|
1221
|
+
}
|
|
1222
|
+
async deleteAll() {
|
|
1223
|
+
this.pending.clear();
|
|
1224
|
+
await this.core.deleteAll();
|
|
1225
|
+
}
|
|
1226
|
+
async abort(id) {
|
|
1227
|
+
await this.core.abort(id);
|
|
1228
|
+
}
|
|
1229
|
+
async saveStatus(id, status) {
|
|
1230
|
+
await this.core.saveStatus(id, status);
|
|
1231
|
+
}
|
|
1232
|
+
}
|
|
1233
|
+
// src/job-queue/createPostgresQueue.ts
|
|
1234
|
+
function createPostgresQueue(queueName, pool, opts) {
|
|
1235
|
+
const core = new PostgresQueueStorage(pool, queueName, opts);
|
|
1236
|
+
const pending = new Map;
|
|
1237
|
+
return {
|
|
1238
|
+
messageQueue: new PostgresMessageQueue(core, pending),
|
|
1239
|
+
jobStore: new PostgresJobStore(core, pending),
|
|
1240
|
+
core
|
|
1241
|
+
};
|
|
1242
|
+
}
|
|
898
1243
|
export {
|
|
899
1244
|
postgresRateLimiterMigrations,
|
|
900
1245
|
postgresQueueMigrations,
|
|
1246
|
+
createPostgresQueue,
|
|
901
1247
|
PostgresRateLimiterStorage,
|
|
902
1248
|
PostgresQueueStorage,
|
|
903
1249
|
PostgresMigrationRunner,
|
|
1250
|
+
PostgresMessageQueue,
|
|
1251
|
+
PostgresJobStore,
|
|
904
1252
|
POSTGRES_RATE_LIMITER_STORAGE,
|
|
905
1253
|
POSTGRES_QUEUE_STORAGE
|
|
906
1254
|
};
|
|
907
1255
|
|
|
908
|
-
//# debugId=
|
|
1256
|
+
//# debugId=C4C2B7F208947DCA64756E2164756E21
|