@workglow/postgres 0.2.36 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/dist/job-queue/PostgresJobStore.d.ts +42 -0
  2. package/dist/job-queue/PostgresJobStore.d.ts.map +1 -0
  3. package/dist/job-queue/PostgresMessageQueue.d.ts +38 -0
  4. package/dist/job-queue/PostgresMessageQueue.d.ts.map +1 -0
  5. package/dist/job-queue/PostgresQueueStorage.d.ts +65 -10
  6. package/dist/job-queue/PostgresQueueStorage.d.ts.map +1 -1
  7. package/dist/job-queue/PostgresRateLimiterStorage.d.ts +1 -2
  8. package/dist/job-queue/PostgresRateLimiterStorage.d.ts.map +1 -1
  9. package/dist/job-queue/browser.js +546 -55
  10. package/dist/job-queue/browser.js.map +11 -8
  11. package/dist/job-queue/common.d.ts +3 -0
  12. package/dist/job-queue/common.d.ts.map +1 -1
  13. package/dist/job-queue/createPostgresQueue.d.ts +22 -0
  14. package/dist/job-queue/createPostgresQueue.d.ts.map +1 -0
  15. package/dist/job-queue/node.js +546 -55
  16. package/dist/job-queue/node.js.map +11 -8
  17. package/dist/migrations/PostgresMigrationRunner.d.ts +6 -1
  18. package/dist/migrations/PostgresMigrationRunner.d.ts.map +1 -1
  19. package/dist/migrations/postgresQueueMigrations.d.ts +9 -1
  20. package/dist/migrations/postgresQueueMigrations.d.ts.map +1 -1
  21. package/dist/migrations/postgresRateLimiterMigrations.d.ts +1 -1
  22. package/dist/migrations/postgresRateLimiterMigrations.d.ts.map +1 -1
  23. package/dist/storage/PostgresKvStorage.d.ts +1 -1
  24. package/dist/storage/PostgresKvStorage.d.ts.map +1 -1
  25. package/dist/storage/PostgresTabularStorage.d.ts +30 -1
  26. package/dist/storage/PostgresTabularStorage.d.ts.map +1 -1
  27. package/dist/storage/PostgresVectorStorage.d.ts +5 -1
  28. package/dist/storage/PostgresVectorStorage.d.ts.map +1 -1
  29. package/dist/storage/browser.js +28 -12
  30. package/dist/storage/browser.js.map +5 -5
  31. package/dist/storage/node.js +28 -12
  32. package/dist/storage/node.js.map +6 -6
  33. package/dist/text/PostgresFtsTextIndex.d.ts +10 -0
  34. package/dist/text/PostgresFtsTextIndex.d.ts.map +1 -1
  35. package/dist/text/browser.js.map +2 -2
  36. package/dist/text/node.js.map +2 -2
  37. package/package.json +8 -8
@@ -1,7 +1,5 @@
1
1
  // src/job-queue/PostgresQueueStorage.ts
2
- import { createHash } from "node:crypto";
3
- import { createServiceToken, getLogger, makeFingerprint, uuid4 } from "@workglow/util";
4
- import { JobStatus as JobStatus2 } from "@workglow/job-queue";
2
+ import { JobStatus as JobStatus2, validateLeaseMs } from "@workglow/job-queue";
5
3
  import {
6
4
  assertPrefixesSafe,
7
5
  buildPrefixInsertFragments,
@@ -10,6 +8,8 @@ import {
10
8
  getPrefixParamValues,
11
9
  PostgresDialect as PostgresDialect2
12
10
  } from "@workglow/storage";
11
+ import { createServiceToken, getLogger, makeFingerprint, uuid4 } from "@workglow/util";
12
+ import { createHash } from "node:crypto";
13
13
 
14
14
  // src/migrations/PostgresMigrationRunner.ts
15
15
  import {
@@ -149,11 +149,6 @@ var JOB_STATUS_V1 = [
149
149
  ];
150
150
  function assertJobStatusMatchesV1() {
151
151
  const current = new Set(Object.values(JobStatus));
152
- for (const v of JOB_STATUS_V1) {
153
- if (!current.has(v)) {
154
- throw new Error(`JobStatus const is missing v1 enum value "${v}"; v1 migration values are frozen.`);
155
- }
156
- }
157
152
  for (const v of current) {
158
153
  if (!JOB_STATUS_V1.includes(v)) {
159
154
  throw new Error(`JobStatus contains "${v}" which is not in JOB_STATUS_V1. ` + `Add a new migration that runs "ALTER TYPE job_status ADD VALUE IF NOT EXISTS '${v}'" ` + `instead of mutating the v1 enum literal.`);
@@ -254,6 +249,92 @@ function postgresQueueMigrations(tableName, prefixes) {
254
249
  });
255
250
  }
256
251
  }
252
+ },
253
+ {
254
+ component,
255
+ version: 2,
256
+ description: "Add abort_requested_at and lease_expires_at columns",
257
+ async up(db) {
258
+ await db.query(`
259
+ ALTER TABLE ${tableName}
260
+ ADD COLUMN IF NOT EXISTS abort_requested_at timestamp with time zone,
261
+ ADD COLUMN IF NOT EXISTS lease_expires_at timestamp with time zone
262
+ `);
263
+ }
264
+ },
265
+ {
266
+ component,
267
+ version: 3,
268
+ description: "Rename run_after→visible_at, last_ran_at→last_attempted_at, run_attempts→attempts, max_retries→max_attempts, worker_id→lease_owner; drop run_after-keyed indexes and recreate visible_at-keyed",
269
+ async up(db) {
270
+ await db.query(`
271
+ DO $$
272
+ BEGIN
273
+ IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='${tableName}' AND column_name='run_after' AND table_schema=current_schema()) THEN
274
+ EXECUTE 'ALTER TABLE ${tableName} RENAME COLUMN run_after TO visible_at';
275
+ END IF;
276
+ IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='${tableName}' AND column_name='last_ran_at' AND table_schema=current_schema()) THEN
277
+ EXECUTE 'ALTER TABLE ${tableName} RENAME COLUMN last_ran_at TO last_attempted_at';
278
+ END IF;
279
+ IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='${tableName}' AND column_name='run_attempts' AND table_schema=current_schema()) THEN
280
+ EXECUTE 'ALTER TABLE ${tableName} RENAME COLUMN run_attempts TO attempts';
281
+ END IF;
282
+ IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='${tableName}' AND column_name='max_retries' AND table_schema=current_schema()) THEN
283
+ EXECUTE 'ALTER TABLE ${tableName} RENAME COLUMN max_retries TO max_attempts';
284
+ EXECUTE 'ALTER TABLE ${tableName} ALTER COLUMN max_attempts SET DEFAULT 10';
285
+ END IF;
286
+ IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='${tableName}' AND column_name='worker_id' AND table_schema=current_schema()) THEN
287
+ EXECUTE 'ALTER TABLE ${tableName} RENAME COLUMN worker_id TO lease_owner';
288
+ END IF;
289
+ END $$
290
+ `);
291
+ await db.query(`DROP INDEX IF EXISTS job_fetcher${indexSuffix}_idx`);
292
+ await db.query(`DROP INDEX IF EXISTS job_queue_fetcher${indexSuffix}_idx`);
293
+ await db.query(`
294
+ CREATE INDEX IF NOT EXISTS job_fetcher${indexSuffix}_idx
295
+ ON ${tableName} (${prefixIndexPrefix}id, status, visible_at)
296
+ `);
297
+ await db.query(`
298
+ CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx
299
+ ON ${tableName} (${prefixIndexPrefix}queue, status, visible_at)
300
+ `);
301
+ }
302
+ },
303
+ {
304
+ component,
305
+ version: 4,
306
+ description: "Add UNIQUE partial index for findActiveByFingerprint O(1) lookup + fingerprint dedup at the DB layer (H2)",
307
+ async up(db) {
308
+ await db.query(`
309
+ CREATE UNIQUE INDEX IF NOT EXISTS idx_${tableName}_fingerprint_active
310
+ ON ${tableName}(${prefixIndexPrefix}queue, fingerprint)
311
+ WHERE status IN ('PENDING','PROCESSING')
312
+ `);
313
+ }
314
+ },
315
+ {
316
+ component,
317
+ version: 5,
318
+ description: "Converge idx_<table>_fingerprint_active to UNIQUE for DBs that applied the pre-edit v4 (non-unique) variant",
319
+ async up(db) {
320
+ const indexName = `idx_${tableName}_fingerprint_active`;
321
+ const result = await db.query(`SELECT i.indisunique
322
+ FROM pg_class c
323
+ JOIN pg_index i ON i.indexrelid = c.oid
324
+ JOIN pg_namespace n ON n.oid = c.relnamespace
325
+ WHERE c.relname = $1
326
+ AND n.nspname = current_schema()`, [indexName]);
327
+ const existing = result.rows[0];
328
+ if (existing && existing.indisunique) {
329
+ return;
330
+ }
331
+ await db.query(`DROP INDEX IF EXISTS ${indexName}`);
332
+ await db.query(`
333
+ CREATE UNIQUE INDEX IF NOT EXISTS ${indexName}
334
+ ON ${tableName}(${prefixIndexPrefix}queue, fingerprint)
335
+ WHERE status IN ('PENDING','PROCESSING')
336
+ `);
337
+ }
257
338
  }
258
339
  ];
259
340
  }
@@ -302,32 +383,32 @@ class PostgresQueueStorage {
302
383
  const now = new Date().toISOString();
303
384
  job.queue = this.queueName;
304
385
  job.job_run_id = job.job_run_id ?? uuid4();
305
- job.fingerprint = await makeFingerprint(job.input);
386
+ job.fingerprint = job.fingerprint ?? await makeFingerprint(job.input);
306
387
  job.status = JobStatus2.PENDING;
307
388
  job.progress = 0;
308
389
  job.progress_message = "";
309
390
  job.progress_details = null;
310
391
  job.created_at = now;
311
- job.run_after = now;
392
+ job.visible_at = now;
312
393
  const prefixColumnNames = getPrefixColumnNames(this.prefixes);
313
394
  const { columns: prefixColumnsInsert, placeholders: prefixParamPlaceholders } = buildPrefixInsertFragments(PostgresDialect2, this.prefixes, 1);
314
395
  const prefixParamValues = this.getPrefixParamValues();
315
396
  const baseParamStart = prefixColumnNames.length + 1;
316
397
  const sql = `
317
398
  INSERT INTO ${this.tableName}(
318
- ${prefixColumnsInsert}queue,
319
- fingerprint,
320
- input,
321
- run_after,
399
+ ${prefixColumnsInsert}queue,
400
+ fingerprint,
401
+ input,
402
+ visible_at,
322
403
  created_at,
323
404
  deadline_at,
324
- max_retries,
325
- job_run_id,
326
- progress,
327
- progress_message,
405
+ max_attempts,
406
+ job_run_id,
407
+ progress,
408
+ progress_message,
328
409
  progress_details
329
410
  )
330
- VALUES
411
+ VALUES
331
412
  (${prefixParamPlaceholders}$${baseParamStart},$${baseParamStart + 1},$${baseParamStart + 2},$${baseParamStart + 3},$${baseParamStart + 4},$${baseParamStart + 5},$${baseParamStart + 6},$${baseParamStart + 7},$${baseParamStart + 8},$${baseParamStart + 9},$${baseParamStart + 10})
332
413
  RETURNING id`;
333
414
  const params = [
@@ -335,16 +416,31 @@ class PostgresQueueStorage {
335
416
  job.queue,
336
417
  job.fingerprint,
337
418
  JSON.stringify(job.input),
338
- job.run_after,
419
+ job.visible_at,
339
420
  job.created_at,
340
421
  job.deadline_at,
341
- job.max_retries,
422
+ job.max_attempts,
342
423
  job.job_run_id,
343
424
  job.progress,
344
425
  job.progress_message,
345
426
  job.progress_details ? JSON.stringify(job.progress_details) : null
346
427
  ];
347
- const result = await this.db.query(sql, params);
428
+ let result;
429
+ try {
430
+ result = await this.db.query(sql, params);
431
+ } catch (err) {
432
+ const e = err;
433
+ const isUniqueViolation = e?.code === "23505";
434
+ const involvesFingerprint = typeof e?.constraint === "string" ? e.constraint.includes("fingerprint") : typeof e?.message === "string" ? e.message.includes("fingerprint") : false;
435
+ if (isUniqueViolation && involvesFingerprint && job.fingerprint) {
436
+ const winner = await this.findActiveByFingerprint(job.fingerprint, this.queueName);
437
+ if (winner?.id != null) {
438
+ job.id = winner.id;
439
+ return winner.id;
440
+ }
441
+ }
442
+ throw err;
443
+ }
348
444
  if (!result)
349
445
  throw new Error("Failed to add to queue");
350
446
  job.id = result.rows[0].id;
@@ -369,32 +465,68 @@ class PostgresQueueStorage {
369
465
  FROM ${this.tableName}
370
466
  WHERE queue = $1
371
467
  AND status = $2${prefixConditions}
372
- ORDER BY run_after ASC
468
+ ORDER BY visible_at ASC
373
469
  LIMIT $3
374
470
  FOR UPDATE SKIP LOCKED`, [this.queueName, status, num, ...prefixParams]);
375
471
  if (!result)
376
472
  return [];
377
473
  return result.rows;
378
474
  }
379
- async next(workerId) {
380
- const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(5);
475
+ async next(workerId, opts) {
476
+ const leaseMs = opts?.leaseMs ?? 30000;
477
+ validateLeaseMs(leaseMs, "leaseMs");
478
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(7);
381
479
  const result = await this.db.query(`
382
- UPDATE ${this.tableName}
383
- SET status = $1, last_ran_at = NOW() AT TIME ZONE 'UTC', worker_id = $4
480
+ UPDATE ${this.tableName}
481
+ SET status = $1,
482
+ last_attempted_at = NOW() AT TIME ZONE 'UTC',
483
+ lease_owner = $4,
484
+ lease_expires_at = NOW() AT TIME ZONE 'UTC' + ($2 * INTERVAL '1 millisecond'),
485
+ -- A reclaimed PROCESSING row was claimed by a now-crashed worker;
486
+ -- that constitutes one used-up attempt against max_attempts.
487
+ -- PENDING claims must not be charged here — JobQueueWorker's
488
+ -- existing validateJobState() will FAIL the job in the next-step
489
+ -- branch when attempts >= max_attempts.
490
+ attempts = CASE WHEN status = $6 THEN attempts + 1 ELSE attempts END,
491
+ -- Always clear any stale abort_requested_at on (re)claim. A PROCESSING
492
+ -- row may have had abort_requested_at set before the worker crashed;
493
+ -- the new owner must start with a clean slate or the worker will see
494
+ -- the abort flag immediately and never run user code.
495
+ abort_requested_at = NULL
384
496
  WHERE id = (
385
- SELECT id
386
- FROM ${this.tableName}
387
- WHERE queue = $2
388
- AND status = $3
497
+ SELECT id
498
+ FROM ${this.tableName}
499
+ WHERE queue = $3
500
+ AND (
501
+ (status = $5 AND visible_at <= NOW() AT TIME ZONE 'UTC')
502
+ OR (status = $6 AND (lease_expires_at IS NULL OR lease_expires_at < NOW() AT TIME ZONE 'UTC'))
503
+ )
389
504
  ${prefixConditions}
390
- AND run_after <= NOW() AT TIME ZONE 'UTC'
391
- ORDER BY run_after ASC
392
- FOR UPDATE SKIP LOCKED
505
+ ORDER BY visible_at ASC
506
+ FOR UPDATE SKIP LOCKED
393
507
  LIMIT 1
394
508
  )
395
- RETURNING *`, [JobStatus2.PROCESSING, this.queueName, JobStatus2.PENDING, workerId, ...prefixParams]);
509
+ RETURNING *`, [
510
+ JobStatus2.PROCESSING,
511
+ leaseMs,
512
+ this.queueName,
513
+ workerId,
514
+ JobStatus2.PENDING,
515
+ JobStatus2.PROCESSING,
516
+ ...prefixParams
517
+ ]);
396
518
  return result?.rows?.[0] ?? undefined;
397
519
  }
520
+ async extendLease(id, workerId, ms) {
521
+ validateLeaseMs(ms, "ms");
522
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(5);
523
+ const result = await this.db.query(`UPDATE ${this.tableName}
524
+ SET lease_expires_at = NOW() AT TIME ZONE 'UTC' + ($1 * INTERVAL '1 millisecond')
525
+ WHERE id = $2 AND queue = $3 AND lease_owner = $4 AND status = 'PROCESSING'${prefixConditions}`, [ms, id, this.queueName, workerId, ...prefixParams]);
526
+ if (!result || result.rowCount === 0) {
527
+ throw new Error(`extendLease failed: job ${String(id)} is not PROCESSING or lease is not owned by worker ${workerId}`);
528
+ }
529
+ }
398
530
  async size(status = JobStatus2.PENDING) {
399
531
  const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
400
532
  const result = await this.db.query(`
@@ -420,22 +552,23 @@ class PostgresQueueStorage {
420
552
  WHERE id = $2 AND queue = $3${prefixConditions}`, [jobDetails.status, jobDetails.id, this.queueName, ...prefixParams]);
421
553
  } else if (jobDetails.status === JobStatus2.PENDING) {
422
554
  const { conditions: prefixConditions } = this.buildPrefixWhereClause(7);
423
- await this.db.query(`UPDATE ${this.tableName}
424
- SET
425
- error = $1,
555
+ await this.db.query(`UPDATE ${this.tableName}
556
+ SET
557
+ error = $1,
426
558
  error_code = $2,
427
- status = $3,
428
- run_after = $4,
559
+ status = $3,
560
+ visible_at = $4,
429
561
  progress = 0,
430
562
  progress_message = '',
431
563
  progress_details = NULL,
432
- run_attempts = run_attempts + 1,
433
- last_ran_at = NOW() AT TIME ZONE 'UTC'
564
+ attempts = attempts + 1,
565
+ last_attempted_at = NOW() AT TIME ZONE 'UTC',
566
+ abort_requested_at = NULL
434
567
  WHERE id = $5 AND queue = $6${prefixConditions}`, [
435
568
  jobDetails.error,
436
569
  jobDetails.error_code,
437
570
  jobDetails.status,
438
- jobDetails.run_after,
571
+ jobDetails.visible_at,
439
572
  jobDetails.id,
440
573
  this.queueName,
441
574
  ...prefixParams
@@ -452,9 +585,9 @@ class PostgresQueueStorage {
452
585
  progress = 100,
453
586
  progress_message = '',
454
587
  progress_details = NULL,
455
- run_attempts = run_attempts + 1,
588
+ attempts = attempts + 1,
456
589
  completed_at = NOW() AT TIME ZONE 'UTC',
457
- last_ran_at = NOW() AT TIME ZONE 'UTC'
590
+ last_attempted_at = NOW() AT TIME ZONE 'UTC'
458
591
  WHERE id = $5 AND queue = $6${prefixConditions}`, [
459
592
  jobDetails.output ? JSON.stringify(jobDetails.output) : null,
460
593
  jobDetails.error ?? null,
@@ -466,6 +599,51 @@ class PostgresQueueStorage {
466
599
  ]);
467
600
  }
468
601
  }
602
+ async finalize(id, fields) {
603
+ const sets = [];
604
+ const params = [];
605
+ let nextParam = 1;
606
+ const push = (col, value) => {
607
+ sets.push(`${col} = $${nextParam}`);
608
+ params.push(value);
609
+ nextParam += 1;
610
+ };
611
+ if ("output" in fields) {
612
+ push("output", fields.output != null ? JSON.stringify(fields.output) : null);
613
+ }
614
+ if ("error" in fields)
615
+ push("error", fields.error ?? null);
616
+ if ("error_code" in fields)
617
+ push("error_code", fields.error_code ?? null);
618
+ if ("status" in fields)
619
+ push("status", fields.status);
620
+ if ("completed_at" in fields)
621
+ push("completed_at", fields.completed_at ?? null);
622
+ if ("abort_requested_at" in fields) {
623
+ push("abort_requested_at", fields.abort_requested_at ?? null);
624
+ }
625
+ if ("lease_owner" in fields)
626
+ push("lease_owner", fields.lease_owner ?? null);
627
+ if ("progress" in fields)
628
+ push("progress", fields.progress ?? 0);
629
+ if ("progress_message" in fields)
630
+ push("progress_message", fields.progress_message ?? "");
631
+ if ("progress_details" in fields) {
632
+ push("progress_details", fields.progress_details != null ? JSON.stringify(fields.progress_details) : null);
633
+ }
634
+ if ("visible_at" in fields)
635
+ push("visible_at", fields.visible_at ?? null);
636
+ if (sets.length === 0)
637
+ return;
638
+ const idParam = nextParam;
639
+ nextParam += 1;
640
+ const queueParam = nextParam;
641
+ nextParam += 1;
642
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(nextParam);
643
+ await this.db.query(`UPDATE ${this.tableName}
644
+ SET ${sets.join(", ")}
645
+ WHERE id = $${idParam} AND queue = $${queueParam}${prefixConditions}`, [...params, id, this.queueName, ...prefixParams]);
646
+ }
469
647
  async deleteAll() {
470
648
  const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
471
649
  await this.db.query(`
@@ -484,23 +662,37 @@ class PostgresQueueStorage {
484
662
  return result.rows[0].output;
485
663
  }
486
664
  async abort(jobId) {
487
- const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
488
- await this.db.query(`
489
- UPDATE ${this.tableName}
490
- SET status = 'ABORTING'
491
- WHERE id = $1 AND queue = $2${prefixConditions}`, [jobId, this.queueName, ...prefixParams]);
665
+ {
666
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(4);
667
+ await this.db.query(`UPDATE ${this.tableName}
668
+ SET status = 'FAILED',
669
+ abort_requested_at = NOW() AT TIME ZONE 'UTC',
670
+ completed_at = NOW() AT TIME ZONE 'UTC'
671
+ WHERE id = $1 AND queue = $2 AND status = $3${prefixConditions}`, [jobId, this.queueName, JobStatus2.PENDING, ...prefixParams]);
672
+ }
673
+ {
674
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(4);
675
+ await this.db.query(`UPDATE ${this.tableName}
676
+ SET abort_requested_at = NOW() AT TIME ZONE 'UTC'
677
+ WHERE id = $1 AND queue = $2 AND status = $3${prefixConditions}`, [jobId, this.queueName, JobStatus2.PROCESSING, ...prefixParams]);
678
+ }
492
679
  }
493
- async release(jobId) {
680
+ async releaseClaim(jobId) {
494
681
  const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
495
682
  await this.db.query(`
496
683
  UPDATE ${this.tableName}
497
684
  SET status = 'PENDING',
498
- worker_id = NULL,
685
+ lease_owner = NULL,
499
686
  progress = 0,
500
687
  progress_message = '',
501
- progress_details = NULL
688
+ progress_details = NULL,
689
+ abort_requested_at = NULL
502
690
  WHERE id = $1 AND queue = $2${prefixConditions}`, [jobId, this.queueName, ...prefixParams]);
503
691
  }
692
+ async saveStatus(jobId, status) {
693
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
694
+ await this.db.query(`UPDATE ${this.tableName} SET status = $1 WHERE id = $2 AND queue = $3${prefixConditions}`, [status, jobId, this.queueName, ...prefixParams]);
695
+ }
504
696
  async getByRunId(job_run_id) {
505
697
  const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
506
698
  const result = await this.db.query(`
@@ -529,6 +721,66 @@ class PostgresQueueStorage {
529
721
  const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
530
722
  await this.db.query(`DELETE FROM ${this.tableName} WHERE id = $1 AND queue = $2${prefixConditions}`, [jobId, this.queueName, ...prefixParams]);
531
723
  }
724
+ async findActiveByFingerprint(fingerprint, queueName) {
725
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
726
+ const result = await this.db.query(`SELECT * FROM ${this.tableName}
727
+ WHERE fingerprint = $1 AND queue = $2
728
+ AND status IN ('PENDING','PROCESSING')${prefixConditions}
729
+ ORDER BY created_at DESC
730
+ LIMIT 1`, [fingerprint, queueName, ...prefixParams]);
731
+ if (!result || result.rows.length === 0)
732
+ return;
733
+ return result.rows[0];
734
+ }
735
+ async getMany(ids) {
736
+ if (ids.length === 0)
737
+ return [];
738
+ const numericIds = ids.map((id) => Number(id)).filter((n) => Number.isInteger(n) && n > 0);
739
+ if (numericIds.length === 0)
740
+ return ids.map(() => {
741
+ return;
742
+ });
743
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
744
+ const result = await this.db.query(`SELECT * FROM ${this.tableName}
745
+ WHERE id = ANY($1::int[]) AND queue = $2${prefixConditions}`, [numericIds, this.queueName, ...prefixParams]);
746
+ if (!result)
747
+ return ids.map(() => {
748
+ return;
749
+ });
750
+ const map = new Map;
751
+ for (const row of result.rows) {
752
+ map.set(Number(row.id), row);
753
+ }
754
+ return ids.map((id) => map.get(Number(id)));
755
+ }
756
+ async completeWithResult(id, result) {
757
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
758
+ await this.db.query(`UPDATE ${this.tableName}
759
+ SET output = $1,
760
+ status = 'COMPLETED',
761
+ progress = 100,
762
+ progress_message = '',
763
+ progress_details = NULL,
764
+ completed_at = NOW() AT TIME ZONE 'UTC'
765
+ WHERE id = $2 AND queue = $3${prefixConditions}`, [result != null ? JSON.stringify(result) : null, id, this.queueName, ...prefixParams]);
766
+ }
767
+ async failWithError(id, opts) {
768
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(5);
769
+ await this.db.query(`UPDATE ${this.tableName}
770
+ SET error = COALESCE($1, error),
771
+ error_code = COALESCE($2, error_code),
772
+ abort_requested_at = CASE WHEN $3 THEN NOW() AT TIME ZONE 'UTC' ELSE abort_requested_at END,
773
+ status = 'FAILED',
774
+ completed_at = COALESCE(completed_at, NOW() AT TIME ZONE 'UTC')
775
+ WHERE id = $4 AND queue = $5${prefixConditions}`, [
776
+ "error" in opts ? opts.error ?? null : null,
777
+ "errorCode" in opts ? opts.errorCode ?? null : null,
778
+ opts.abortRequested === true,
779
+ id,
780
+ this.queueName,
781
+ ...prefixParams
782
+ ]);
783
+ }
532
784
  async deleteJobsByStatusAndAge(status, olderThanMs) {
533
785
  const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
534
786
  const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(4);
@@ -663,7 +915,6 @@ class PostgresQueueStorage {
663
915
  }
664
916
  }
665
917
  // src/job-queue/PostgresRateLimiterStorage.ts
666
- import { createServiceToken as createServiceToken2 } from "@workglow/util";
667
918
  import {
668
919
  assertPrefixesSafe as assertPrefixesSafe2,
669
920
  buildPrefixWhereClause as buildPrefixWhereClause2,
@@ -671,6 +922,7 @@ import {
671
922
  getPrefixParamValues as getPrefixParamValues2,
672
923
  PostgresDialect as PostgresDialect4
673
924
  } from "@workglow/storage";
925
+ import { createServiceToken as createServiceToken2 } from "@workglow/util";
674
926
 
675
927
  // src/migrations/postgresRateLimiterMigrations.ts
676
928
  import {
@@ -895,14 +1147,253 @@ class PostgresRateLimiterStorage {
895
1147
  await this.db.query(`DELETE FROM ${this.nextAvailableTableName} WHERE queue_name = $1${prefixConditions}`, [queueName, ...prefixParams]);
896
1148
  }
897
1149
  }
1150
+ // src/job-queue/PostgresMessageQueue.ts
1151
+ class PostgresClaim {
1152
+ core;
1153
+ pending;
1154
+ id;
1155
+ body;
1156
+ attempts;
1157
+ workerId;
1158
+ constructor(core, pending, id, body, attempts, workerId) {
1159
+ this.core = core;
1160
+ this.pending = pending;
1161
+ this.id = id;
1162
+ this.body = body;
1163
+ this.attempts = attempts;
1164
+ this.workerId = workerId;
1165
+ }
1166
+ async ack(result) {
1167
+ const buf = this.pending.get(this.id);
1168
+ this.pending.delete(this.id);
1169
+ const current = await this.core.get(this.id) ?? this.body;
1170
+ const output = result !== undefined ? result : buf?.output !== undefined ? buf.output : current.output ?? null;
1171
+ await this.core.finalize(this.id, {
1172
+ output,
1173
+ error: null,
1174
+ error_code: null,
1175
+ status: "COMPLETED",
1176
+ completed_at: current.completed_at ?? new Date().toISOString()
1177
+ });
1178
+ }
1179
+ async retry(opts) {
1180
+ this.pending.delete(this.id);
1181
+ const delay = opts?.delaySeconds ?? 0;
1182
+ const current = await this.core.get(this.id) ?? this.body;
1183
+ await this.core.complete({
1184
+ ...current,
1185
+ status: "PENDING",
1186
+ lease_owner: null,
1187
+ lease_expires_at: null,
1188
+ visible_at: new Date(Date.now() + delay * 1000).toISOString(),
1189
+ progress: 0,
1190
+ progress_message: "",
1191
+ progress_details: null
1192
+ });
1193
+ }
1194
+ async fail(opts) {
1195
+ opts?.permanent;
1196
+ const buf = this.pending.get(this.id);
1197
+ this.pending.delete(this.id);
1198
+ const current = await this.core.get(this.id) ?? this.body;
1199
+ const error = opts?.error !== undefined ? opts.error : buf?.error !== undefined ? buf.error : current.error ?? null;
1200
+ const errorCode = opts?.errorCode !== undefined ? opts.errorCode : buf?.errorCode !== undefined ? buf.errorCode : current.error_code ?? null;
1201
+ const abortRequested = opts?.abortRequested !== undefined ? opts.abortRequested : buf?.abortRequested ?? false;
1202
+ await this.core.finalize(this.id, {
1203
+ error,
1204
+ error_code: errorCode,
1205
+ abort_requested_at: abortRequested ? current.abort_requested_at ?? new Date().toISOString() : current.abort_requested_at ?? null,
1206
+ status: "FAILED",
1207
+ completed_at: current.completed_at ?? new Date().toISOString()
1208
+ });
1209
+ }
1210
+ async extendLease(ms) {
1211
+ await this.core.extendLease(this.id, this.workerId, ms);
1212
+ }
1213
+ async disable() {
1214
+ this.pending.delete(this.id);
1215
+ const current = await this.core.get(this.id);
1216
+ const completedAt = current?.completed_at ?? new Date().toISOString();
1217
+ await this.core.finalize(this.id, {
1218
+ status: "DISABLED",
1219
+ completed_at: completedAt,
1220
+ lease_owner: null,
1221
+ progress: 0,
1222
+ progress_message: "",
1223
+ progress_details: null
1224
+ });
1225
+ }
1226
+ }
1227
+
1228
+ class PostgresMessageQueue {
1229
+ scope;
1230
+ core;
1231
+ pending;
1232
+ constructor(core, pending) {
1233
+ this.core = core;
1234
+ this.pending = pending;
1235
+ this.scope = core.scope;
1236
+ }
1237
+ async send(body, opts) {
1238
+ return this.core.add(applySendOptions(body, opts));
1239
+ }
1240
+ async sendBatch(bodies, opts) {
1241
+ const ids = [];
1242
+ for (const body of bodies) {
1243
+ ids.push(await this.send(body, opts));
1244
+ }
1245
+ return ids;
1246
+ }
1247
+ async receive(opts) {
1248
+ const max = Math.max(1, opts.max ?? 1);
1249
+ const claims = [];
1250
+ while (claims.length < max) {
1251
+ const job = await this.core.next(opts.workerId, { leaseMs: opts.leaseMs });
1252
+ if (!job)
1253
+ break;
1254
+ claims.push(new PostgresClaim(this.core, this.pending, job.id, job, job.attempts ?? 0, opts.workerId));
1255
+ }
1256
+ return claims;
1257
+ }
1258
+ async releaseClaim(id) {
1259
+ this.pending.delete(id);
1260
+ await this.core.releaseClaim(id);
1261
+ }
1262
+ async migrate() {
1263
+ await this.core.migrate();
1264
+ }
1265
+ getMigrations() {
1266
+ return this.core.getMigrations();
1267
+ }
1268
+ subscribeToChanges(callback, options) {
1269
+ return this.core.subscribeToChanges(callback, options);
1270
+ }
1271
+ }
1272
+ function applySendOptions(body, opts) {
1273
+ if (!opts)
1274
+ return body;
1275
+ const out = { ...body };
1276
+ if (opts.delaySeconds != null) {
1277
+ out.visible_at = new Date(Date.now() + opts.delaySeconds * 1000).toISOString();
1278
+ }
1279
+ if (opts.timeoutSeconds != null) {
1280
+ out.deadline_at = new Date(Date.now() + opts.timeoutSeconds * 1000).toISOString();
1281
+ }
1282
+ if (opts.fingerprint != null)
1283
+ out.fingerprint = opts.fingerprint;
1284
+ if (opts.jobRunId != null)
1285
+ out.job_run_id = opts.jobRunId;
1286
+ if (opts.maxAttempts != null)
1287
+ out.max_attempts = opts.maxAttempts;
1288
+ return out;
1289
+ }
1290
+ // src/job-queue/PostgresJobStore.ts
1291
+ class PostgresJobStore {
1292
+ core;
1293
+ pending;
1294
+ constructor(core, pending) {
1295
+ this.core = core;
1296
+ this.pending = pending;
1297
+ }
1298
+ get(id) {
1299
+ return this.core.get(id);
1300
+ }
1301
+ async peek(status, num) {
1302
+ return this.core.peek(status, num);
1303
+ }
1304
+ size(status) {
1305
+ return this.core.size(status);
1306
+ }
1307
+ async getByRunId(runId) {
1308
+ return this.core.getByRunId(runId);
1309
+ }
1310
+ outputForInput(input) {
1311
+ return this.core.outputForInput(input);
1312
+ }
1313
+ async saveProgress(id, progress, message, details) {
1314
+ await this.core.saveProgress(id, progress, message, details);
1315
+ }
1316
+ async saveResult(id, output) {
1317
+ const buf = this.pending.get(id) ?? {};
1318
+ buf.output = output ?? null;
1319
+ this.pending.set(id, buf);
1320
+ }
1321
+ async saveError(id, error, errorCode, abortRequested) {
1322
+ const buf = this.pending.get(id) ?? {};
1323
+ buf.error = error;
1324
+ buf.errorCode = errorCode;
1325
+ buf.abortRequested = abortRequested;
1326
+ this.pending.set(id, buf);
1327
+ }
1328
+ async deleteByStatusAndAge(status, olderThanMs) {
1329
+ await this.core.deleteJobsByStatusAndAge(status, olderThanMs);
1330
+ }
1331
+ async delete(id) {
1332
+ this.pending.delete(id);
1333
+ await this.core.delete(id);
1334
+ }
1335
+ async deleteAll() {
1336
+ this.pending.clear();
1337
+ await this.core.deleteAll();
1338
+ }
1339
+ async abort(id) {
1340
+ await this.core.abort(id);
1341
+ }
1342
+ async saveStatus(id, status) {
1343
+ await this.core.saveStatus(id, status);
1344
+ }
1345
+ async create(body, opts) {
1346
+ const enriched = {
1347
+ ...body,
1348
+ fingerprint: opts.fingerprint ?? body.fingerprint,
1349
+ job_run_id: opts.jobRunId ?? body.job_run_id,
1350
+ max_attempts: opts.maxAttempts ?? body.max_attempts,
1351
+ deadline_at: opts.timeoutSeconds != null ? new Date(Date.now() + opts.timeoutSeconds * 1000).toISOString() : body.deadline_at
1352
+ };
1353
+ return this.core.add(enriched);
1354
+ }
1355
+ async findActiveByFingerprint(fingerprint, queueName) {
1356
+ return this.core.findActiveByFingerprint(fingerprint, queueName);
1357
+ }
1358
+ async getMany(ids) {
1359
+ return this.core.getMany(ids);
1360
+ }
1361
+ async completeWithResult(id, result) {
1362
+ this.pending.delete(id);
1363
+ await this.core.completeWithResult(id, result);
1364
+ }
1365
+ async failWithError(id, opts) {
1366
+ this.pending.delete(id);
1367
+ await this.core.failWithError(id, opts);
1368
+ }
1369
+ async markEnqueueDeferred(id, opts) {
1370
+ await this.core.finalize(id, {
1371
+ visible_at: opts.visible_at.toISOString(),
1372
+ error_code: opts.errorCode
1373
+ });
1374
+ }
1375
+ }
1376
+ // src/job-queue/createPostgresQueue.ts
1377
+ function createPostgresQueue(queueName, pool, opts) {
1378
+ const core = new PostgresQueueStorage(pool, queueName, opts);
1379
+ const pending = new Map;
1380
+ return {
1381
+ messageQueue: new PostgresMessageQueue(core, pending),
1382
+ jobStore: new PostgresJobStore(core, pending),
1383
+ core
1384
+ };
1385
+ }
898
1386
  export {
899
1387
  postgresRateLimiterMigrations,
900
1388
  postgresQueueMigrations,
1389
+ createPostgresQueue,
901
1390
  PostgresRateLimiterStorage,
902
1391
  PostgresQueueStorage,
903
1392
  PostgresMigrationRunner,
1393
+ PostgresMessageQueue,
1394
+ PostgresJobStore,
904
1395
  POSTGRES_RATE_LIMITER_STORAGE,
905
1396
  POSTGRES_QUEUE_STORAGE
906
1397
  };
907
1398
 
908
- //# debugId=A07A5231BDAE0A9B64756E2164756E21
1399
+ //# debugId=161830453993645E64756E2164756E21