bunqueue 2.9.0 → 2.9.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/README.md +7 -3
  2. package/dist/application/background/recovery/active.d.ts +1 -0
  3. package/dist/application/background/recovery/active.js +12 -0
  4. package/dist/application/background/recovery/index.js +3 -1
  5. package/dist/application/background/recovery/pending.js +1 -0
  6. package/dist/application/cleanupTasks.js +4 -16
  7. package/dist/application/contextFactory.js +3 -0
  8. package/dist/application/dlqManager.js +1 -0
  9. package/dist/application/dlqRetry.js +1 -0
  10. package/dist/application/eventsManager.d.ts +8 -1
  11. package/dist/application/eventsManager.js +76 -29
  12. package/dist/application/operations/ackHelpers.d.ts +1 -4
  13. package/dist/application/operations/ackHelpers.js +39 -14
  14. package/dist/application/operations/flowPush.js +1 -0
  15. package/dist/application/operations/flowValidation.js +4 -4
  16. package/dist/application/operations/pull.d.ts +7 -2
  17. package/dist/application/operations/pull.js +15 -57
  18. package/dist/application/operations/pullFinalization.d.ts +6 -0
  19. package/dist/application/operations/pullFinalization.js +69 -0
  20. package/dist/application/operations/pullStateTransition.d.ts +8 -7
  21. package/dist/application/operations/pullStateTransition.js +25 -18
  22. package/dist/application/operations/push.js +3 -0
  23. package/dist/application/operations/pushBatch.js +14 -7
  24. package/dist/application/operations/pushContext.d.ts +2 -1
  25. package/dist/application/postgres-queue-manager/control.d.ts +4 -0
  26. package/dist/application/postgres-queue-manager/control.js +24 -0
  27. package/dist/application/postgres-queue-manager/delivery.d.ts +5 -4
  28. package/dist/application/postgres-queue-manager/delivery.js +13 -10
  29. package/dist/application/postgres-queue-manager/groupQueries.d.ts +10 -0
  30. package/dist/application/postgres-queue-manager/groupQueries.js +40 -0
  31. package/dist/application/postgres-queue-manager/queries.d.ts +2 -2
  32. package/dist/application/postgres-queue-manager/queries.js +2 -2
  33. package/dist/application/queue-manager/control.js +1 -0
  34. package/dist/application/queue-manager/delivery.d.ts +5 -4
  35. package/dist/application/queue-manager/delivery.js +15 -8
  36. package/dist/application/queue-manager/limits.d.ts +11 -0
  37. package/dist/application/queue-manager/limits.js +61 -0
  38. package/dist/application/queue-manager/state.js +1 -1
  39. package/dist/application/queueTelemetryJournal.d.ts +2 -0
  40. package/dist/application/queueTelemetryJournal.js +23 -0
  41. package/dist/application/types/ack.d.ts +3 -1
  42. package/dist/cli/commandRouter.d.ts +1 -1
  43. package/dist/client/flowPush.js +6 -0
  44. package/dist/client/groupId.d.ts +1 -0
  45. package/dist/client/groupId.js +1 -0
  46. package/dist/client/index.d.ts +1 -1
  47. package/dist/client/jobHelpers.js +1 -0
  48. package/dist/client/queue/operations/add/bulk.js +4 -4
  49. package/dist/client/queue/operations/add/payload.d.ts +1 -0
  50. package/dist/client/queue/operations/add/payload.js +7 -1
  51. package/dist/client/queue/operations/add/single.js +3 -3
  52. package/dist/client/queue/operations/groups.d.ts +21 -0
  53. package/dist/client/queue/operations/groups.js +88 -0
  54. package/dist/client/queue/runtime/queries.d.ts +13 -0
  55. package/dist/client/queue/runtime/queries.js +31 -0
  56. package/dist/client/queue/runtime/state.d.ts +4 -0
  57. package/dist/client/queue/runtime/state.js +1 -0
  58. package/dist/client/types/index.d.ts +2 -2
  59. package/dist/client/types/options.d.ts +4 -0
  60. package/dist/client/types/worker.d.ts +8 -0
  61. package/dist/client/worker/runtime/execution.js +1 -0
  62. package/dist/client/worker/runtime/options.js +3 -0
  63. package/dist/client/worker/types/options.d.ts +2 -1
  64. package/dist/client/worker/workerPull.d.ts +2 -0
  65. package/dist/client/worker/workerPull.js +6 -4
  66. package/dist/domain/job/groupFifoOrder.d.ts +9 -0
  67. package/dist/domain/job/groupFifoOrder.js +26 -0
  68. package/dist/domain/queue/groupLimiterManager.d.ts +23 -0
  69. package/dist/domain/queue/groupLimiterManager.js +107 -0
  70. package/dist/domain/queue/groupScheduler.d.ts +31 -0
  71. package/dist/domain/queue/groupScheduler.js +256 -0
  72. package/dist/domain/queue/groupSchedulerOrder.d.ts +6 -0
  73. package/dist/domain/queue/groupSchedulerOrder.js +30 -0
  74. package/dist/domain/queue/priorityQueue.d.ts +13 -23
  75. package/dist/domain/queue/priorityQueue.js +53 -42
  76. package/dist/domain/queue/shard/keys.d.ts +1 -0
  77. package/dist/domain/queue/shard/keys.js +25 -2
  78. package/dist/domain/queue/shard/lifecycle.js +4 -0
  79. package/dist/domain/queue/shard/limits.d.ts +20 -1
  80. package/dist/domain/queue/shard/limits.js +56 -0
  81. package/dist/domain/queue/shard/state.d.ts +5 -0
  82. package/dist/domain/queue/shard/state.js +9 -1
  83. package/dist/domain/types/commands/core.d.ts +3 -0
  84. package/dist/domain/types/commands/limits.d.ts +41 -0
  85. package/dist/domain/types/commands/union.d.ts +1 -1
  86. package/dist/domain/types/group.d.ts +21 -0
  87. package/dist/domain/types/group.js +78 -0
  88. package/dist/domain/types/priorityQueue.d.ts +1 -0
  89. package/dist/infrastructure/persistence/jobOptionsBlob.d.ts +1 -0
  90. package/dist/infrastructure/persistence/jobOptionsBlob.js +10 -2
  91. package/dist/infrastructure/persistence/migrations.js +11 -0
  92. package/dist/infrastructure/persistence/postgres/admission.js +5 -2
  93. package/dist/infrastructure/persistence/postgres/batchAdmission.js +22 -4
  94. package/dist/infrastructure/persistence/postgres/claimBatch.d.ts +3 -0
  95. package/dist/infrastructure/persistence/postgres/claimBatch.js +16 -3
  96. package/dist/infrastructure/persistence/postgres/claimSelection.d.ts +8 -1
  97. package/dist/infrastructure/persistence/postgres/claimSelection.js +73 -131
  98. package/dist/infrastructure/persistence/postgres/claims.d.ts +8 -1
  99. package/dist/infrastructure/persistence/postgres/claims.js +19 -3
  100. package/dist/infrastructure/persistence/postgres/eventCatchupCursors.d.ts +32 -0
  101. package/dist/infrastructure/persistence/postgres/eventCatchupCursors.js +66 -0
  102. package/dist/infrastructure/persistence/postgres/eventJournalSchema.d.ts +2 -2
  103. package/dist/infrastructure/persistence/postgres/eventJournalSchema.js +16 -2
  104. package/dist/infrastructure/persistence/postgres/eventPruneWatermarks.d.ts +2 -0
  105. package/dist/infrastructure/persistence/postgres/eventPruneWatermarks.js +21 -3
  106. package/dist/infrastructure/persistence/postgres/events.d.ts +3 -3
  107. package/dist/infrastructure/persistence/postgres/events.js +44 -17
  108. package/dist/infrastructure/persistence/postgres/groupClaims.d.ts +15 -0
  109. package/dist/infrastructure/persistence/postgres/groupClaims.js +95 -0
  110. package/dist/infrastructure/persistence/postgres/groupSchema.d.ts +2 -0
  111. package/dist/infrastructure/persistence/postgres/groupSchema.js +155 -0
  112. package/dist/infrastructure/persistence/postgres/groupSchemaFingerprint.d.ts +3 -0
  113. package/dist/infrastructure/persistence/postgres/groupSchemaFingerprint.js +127 -0
  114. package/dist/infrastructure/persistence/postgres/groupStateRetention.d.ts +3 -0
  115. package/dist/infrastructure/persistence/postgres/groupStateRetention.js +93 -0
  116. package/dist/infrastructure/persistence/postgres/groups.d.ts +11 -0
  117. package/dist/infrastructure/persistence/postgres/groups.js +117 -0
  118. package/dist/infrastructure/persistence/postgres/queueDestruction.js +4 -0
  119. package/dist/infrastructure/persistence/postgres/runtime.d.ts +0 -1
  120. package/dist/infrastructure/persistence/postgres/runtime.js +3 -1
  121. package/dist/infrastructure/persistence/postgres/schema.d.ts +2 -2
  122. package/dist/infrastructure/persistence/postgres/schema.js +8 -4
  123. package/dist/infrastructure/persistence/postgres/schemaInitialization.js +6 -1
  124. package/dist/infrastructure/persistence/postgres/store.d.ts +11 -1
  125. package/dist/infrastructure/persistence/postgres/store.js +47 -3
  126. package/dist/infrastructure/persistence/postgres/telemetry.d.ts +2 -0
  127. package/dist/infrastructure/persistence/postgres/telemetry.js +33 -21
  128. package/dist/infrastructure/persistence/postgres/types.d.ts +1 -0
  129. package/dist/infrastructure/persistence/schema.d.ts +2 -2
  130. package/dist/infrastructure/persistence/schema.js +12 -1
  131. package/dist/infrastructure/persistence/sqlite/control.d.ts +14 -0
  132. package/dist/infrastructure/persistence/sqlite/control.js +69 -0
  133. package/dist/infrastructure/persistence/sqlite/jobs.d.ts +12 -0
  134. package/dist/infrastructure/persistence/sqlite/jobs.js +36 -0
  135. package/dist/infrastructure/persistence/sqlite/telemetry.d.ts +2 -1
  136. package/dist/infrastructure/persistence/sqlite/telemetry.js +5 -64
  137. package/dist/infrastructure/persistence/sqlite/telemetryWrites.d.ts +4 -0
  138. package/dist/infrastructure/persistence/sqlite/telemetryWrites.js +123 -0
  139. package/dist/infrastructure/persistence/sqliteSerializer.js +2 -0
  140. package/dist/infrastructure/server/handler-routes/control.js +13 -0
  141. package/dist/infrastructure/server/handlers/core.d.ts +0 -5
  142. package/dist/infrastructure/server/handlers/core.js +15 -18
  143. package/dist/infrastructure/server/handlers/groups.d.ts +11 -0
  144. package/dist/infrastructure/server/handlers/groups.js +55 -0
  145. package/dist/infrastructure/server/handlers/pushBatchValidation.js +4 -1
  146. package/dist/infrastructure/server/protocol/validation.d.ts +1 -0
  147. package/dist/infrastructure/server/protocol/validation.js +1 -0
  148. package/dist/infrastructure/server/protocol.d.ts +1 -1
  149. package/dist/infrastructure/server/protocol.js +1 -1
  150. package/package.json +2 -1
@@ -1,3 +1,4 @@
1
+ import { getGroupFifoOrder } from '../../domain/job/groupFifoOrder';
1
2
  import { storageLog } from '../../shared/logger';
2
3
  import { decodeMessagePack, encodeMessagePack } from '../../shared/msgpack';
3
4
  const DEFAULTS = {
@@ -12,10 +13,14 @@ const DEFAULTS = {
12
13
  debounceId: null,
13
14
  debounceTtl: null,
14
15
  durable: false,
16
+ groupFifoOrder: null,
15
17
  };
18
+ function validStoredOrder(value) {
19
+ return typeof value === 'string' && /^[1-9]\d*$/.test(value) ? value : null;
20
+ }
16
21
  /** Persist generation policies that do not have dedicated legacy columns. */
17
22
  export function encodeJobOptions(job) {
18
- return encodeMessagePack({
23
+ const options = {
19
24
  backoffConfig: job.backoffConfig,
20
25
  repeat: job.repeat,
21
26
  stackTraceLimit: job.stackTraceLimit,
@@ -27,7 +32,9 @@ export function encodeJobOptions(job) {
27
32
  debounceId: job.debounceId,
28
33
  debounceTtl: job.debounceTtl,
29
34
  durable: job.durable ?? false,
30
- });
35
+ };
36
+ const order = getGroupFifoOrder(job);
37
+ return encodeMessagePack(order === null ? options : { ...options, groupFifoOrder: String(order) });
31
38
  }
32
39
  /** Decode current blobs while giving pre-v34 rows their historical defaults. */
33
40
  export function decodeJobOptions(blob, context) {
@@ -47,6 +54,7 @@ export function decodeJobOptions(blob, context) {
47
54
  debounceId: decoded.debounceId ?? null,
48
55
  debounceTtl: decoded.debounceTtl ?? null,
49
56
  durable: decoded.durable ?? false,
57
+ groupFifoOrder: validStoredOrder(decoded.groupFifoOrder),
50
58
  };
51
59
  }
52
60
  catch (error) {
@@ -117,4 +117,15 @@ CREATE INDEX IF NOT EXISTS idx_queue_metric_buckets_window
117
117
  `,
118
118
  // Persist repeat-chain and advanced per-generation job policies.
119
119
  34: 'ALTER TABLE jobs ADD COLUMN extended_options BLOB;',
120
+ // Persist per-group rate and concurrency overrides.
121
+ 35: `
122
+ CREATE TABLE IF NOT EXISTS group_state (
123
+ queue TEXT NOT NULL,
124
+ group_id TEXT NOT NULL,
125
+ rate_limit INTEGER,
126
+ rate_duration INTEGER,
127
+ concurrency_limit INTEGER,
128
+ PRIMARY KEY (queue, group_id)
129
+ );
130
+ `,
120
131
  };
@@ -75,14 +75,17 @@ async function insertRow(tx, ctx, job, state, now) {
75
75
  INSERT INTO bunqueue_jobs (
76
76
  namespace, id, queue, payload, state, priority, lifo, run_at, created_at,
77
77
  started_at, completed_at, attempts, max_attempts, ttl, timeout, unique_key,
78
- unique_expires_at, custom_id, group_id, parent_id
78
+ unique_expires_at, custom_id, group_id, group_order, parent_id
79
79
  ) VALUES (
80
80
  ${ctx.config.namespace}, ${String(job.id)}, ${job.queue}, ${encodePostgresValue(job)},
81
81
  ${state}, ${job.priority}, ${job.lifo}, ${job.runAt}, ${job.createdAt},
82
82
  ${job.startedAt}, ${job.completedAt}, ${job.attempts}, ${job.maxAttempts}, ${job.ttl},
83
83
  ${job.timeout}, ${job.uniqueKey},
84
84
  ${job.uniqueKey && job.deduplicationTtl !== null ? now + job.deduplicationTtl : null},
85
- ${job.customId}, ${job.groupId}, ${job.parentId}
85
+ ${job.customId}, ${job.groupId},
86
+ CASE WHEN CAST(${job.groupId} AS TEXT) IS NULL
87
+ THEN NULL ELSE nextval('bunqueue_group_order_seq') END,
88
+ ${job.parentId}
86
89
  )
87
90
  `;
88
91
  await insertDependencies(tx, ctx, job);
@@ -57,8 +57,8 @@ function prepareAdmissions(jobs, now) {
57
57
  return { job: preparedJob, state };
58
58
  });
59
59
  }
60
- function toRecords(prepared, namespace, now) {
61
- return prepared.map(({ job, state }) => ({
60
+ function toRecords(prepared, namespace, now, groupOrders) {
61
+ return prepared.map(({ job, state }, position) => ({
62
62
  namespace,
63
63
  id: String(job.id),
64
64
  queue: job.queue,
@@ -78,15 +78,33 @@ function toRecords(prepared, namespace, now) {
78
78
  unique_expires_at: job.uniqueKey && job.deduplicationTtl !== null ? now + job.deduplicationTtl : null,
79
79
  custom_id: job.customId,
80
80
  group_id: job.groupId,
81
+ group_order: groupOrders.get(position) ?? null,
81
82
  }));
82
83
  }
84
+ async function allocateGroupOrders(tx, prepared) {
85
+ const positions = prepared.flatMap(({ job }, position) => job.groupId === null ? [] : [position]);
86
+ if (positions.length === 0)
87
+ return new Map();
88
+ const rows = await tx `
89
+ WITH ordered AS MATERIALIZED (
90
+ SELECT position
91
+ FROM unnest(${tx.array(positions, 'INTEGER')}) AS requested(position)
92
+ ORDER BY position
93
+ )
94
+ SELECT position, nextval('bunqueue_group_order_seq') AS group_order
95
+ FROM ordered
96
+ ORDER BY position
97
+ `;
98
+ return new Map(rows.map(({ position, group_order }) => [position, group_order]));
99
+ }
83
100
  async function insertRows(tx, ctx, prepared, now) {
84
- const records = toRecords(prepared, ctx.config.namespace, now);
101
+ const groupOrders = await allocateGroupOrders(tx, prepared);
102
+ const records = toRecords(prepared, ctx.config.namespace, now, groupOrders);
85
103
  let insertedCount = 0;
86
104
  for (let offset = 0; offset < records.length; offset += 1000) {
87
105
  const batch = records.slice(offset, offset + 1000);
88
106
  const inserted = await tx `
89
- INSERT INTO bunqueue_jobs ${tx(batch, 'namespace', 'id', 'queue', 'payload', 'state', 'priority', 'lifo', 'run_at', 'created_at', 'started_at', 'completed_at', 'attempts', 'max_attempts', 'ttl', 'timeout', 'unique_key', 'unique_expires_at', 'custom_id', 'group_id')}
107
+ INSERT INTO bunqueue_jobs ${tx(batch, 'namespace', 'id', 'queue', 'payload', 'state', 'priority', 'lifo', 'run_at', 'created_at', 'started_at', 'completed_at', 'attempts', 'max_attempts', 'ttl', 'timeout', 'unique_key', 'unique_expires_at', 'custom_id', 'group_id', 'group_order')}
90
108
  ON CONFLICT DO NOTHING
91
109
  RETURNING id
92
110
  `;
@@ -1,4 +1,5 @@
1
1
  import type { TransactionSQL } from 'bun';
2
+ import type { GroupPullOptions } from '../../../domain/types/group';
2
3
  import type { PostgresContext } from './context';
3
4
  import type { ClaimedPostgresJob } from './types';
4
5
  /** Claim one queue batch after its queue-state row has been locked. */
@@ -8,4 +9,6 @@ export declare function claimPostgresCandidates(tx: TransactionSQL, ctx: Postgre
8
9
  owner: string;
9
10
  requestedLeaseMs: number;
10
11
  now: number;
12
+ groupOptions?: GroupPullOptions;
13
+ groupSequence: number;
11
14
  }): Promise<ClaimedPostgresJob[]>;
@@ -2,6 +2,7 @@ import { generateLockToken, MAX_TIMELINE_ENTRIES } from '../../../domain/types/j
2
2
  import { recordPostgresJobEvents } from './batchEvents';
3
3
  import { selectPostgresClaimCandidates } from './claimSelection';
4
4
  import { decodePostgresJob, encodePostgresValue, postgresByteaBase64 } from './codec';
5
+ import { commitPostgresGroupClaims } from './groupClaims';
5
6
  function prepareClaims(rows, input) {
6
7
  const { owner, requestedLeaseMs, now, brokerId } = input;
7
8
  return rows.map((row) => {
@@ -42,15 +43,27 @@ async function persistClaims(tx, ctx, claimed, now) {
42
43
  }
43
44
  /** Claim one queue batch after its queue-state row has been locked. */
44
45
  export async function claimPostgresCandidates(tx, ctx, input) {
45
- const { queue, capacity, owner, requestedLeaseMs, now } = input;
46
- const rows = await selectPostgresClaimCandidates(tx, ctx, queue, now, capacity);
46
+ const { queue, capacity, owner, requestedLeaseMs, now, groupOptions, groupSequence } = input;
47
+ const rows = await selectPostgresClaimCandidates(tx, ctx, {
48
+ queue,
49
+ now,
50
+ capacity,
51
+ options: groupOptions,
52
+ });
47
53
  const claimed = prepareClaims(rows, {
48
54
  owner,
49
55
  requestedLeaseMs,
50
56
  now,
51
57
  brokerId: ctx.config.brokerId,
52
58
  });
53
- if (claimed.length > 0)
59
+ if (claimed.length > 0) {
60
+ await commitPostgresGroupClaims(tx, ctx, {
61
+ queue,
62
+ rows,
63
+ options: groupOptions,
64
+ sequence: groupSequence,
65
+ });
54
66
  await persistClaims(tx, ctx, claimed, now);
67
+ }
55
68
  return claimed;
56
69
  }
@@ -1,4 +1,11 @@
1
1
  import type { TransactionSQL } from 'bun';
2
+ import type { GroupPullOptions } from '../../../domain/types/group';
2
3
  import type { PostgresContext } from './context';
3
4
  import type { PostgresJobRow } from './types';
4
- export declare function selectPostgresClaimCandidates(tx: TransactionSQL, ctx: PostgresContext, queue: string, now: number, capacity: number): Promise<PostgresJobRow[]>;
5
+ export declare function hasClaimablePostgresGroup(tx: TransactionSQL, ctx: PostgresContext, queue: string, now: number): Promise<boolean>;
6
+ export declare function selectPostgresClaimCandidates(tx: TransactionSQL, ctx: PostgresContext, input: {
7
+ queue: string;
8
+ now: number;
9
+ capacity: number;
10
+ options?: GroupPullOptions;
11
+ }): Promise<PostgresJobRow[]>;
@@ -1,143 +1,75 @@
1
- async function hasClaimableGroupedJob(tx, ctx, queue, now) {
1
+ export async function hasClaimablePostgresGroup(tx, ctx, queue, now) {
2
2
  const [row] = await tx `
3
3
  SELECT EXISTS (
4
- SELECT 1
5
- FROM bunqueue_jobs AS job
6
- WHERE job.namespace = ${ctx.config.namespace}
7
- AND job.queue = ${queue}
4
+ SELECT 1 FROM bunqueue_jobs AS job
5
+ WHERE job.namespace = ${ctx.config.namespace} AND job.queue = ${queue}
8
6
  AND job.group_id IS NOT NULL
9
7
  AND job.state IN ('waiting', 'prioritized', 'delayed')
10
8
  AND job.run_at <= ${now}
11
9
  AND (job.ttl IS NULL OR job.created_at + job.ttl >= ${now})
12
10
  AND NOT EXISTS (
13
- SELECT 1
14
- FROM bunqueue_dependencies AS dependency
15
- WHERE dependency.namespace = job.namespace
16
- AND dependency.job_id = job.id
11
+ SELECT 1 FROM bunqueue_dependencies AS dependency
12
+ WHERE dependency.namespace = job.namespace AND dependency.job_id = job.id
17
13
  AND NOT EXISTS (
18
14
  SELECT 1 FROM bunqueue_completions AS completion
19
15
  WHERE completion.namespace = dependency.namespace
20
16
  AND completion.job_id = dependency.dependency_id
21
17
  )
22
18
  )
23
- AND NOT EXISTS (
24
- SELECT 1 FROM bunqueue_jobs AS active_group
25
- WHERE active_group.namespace = job.namespace
26
- AND active_group.queue = job.queue
27
- AND active_group.group_id = job.group_id
28
- AND active_group.state = 'active'
29
- AND active_group.lease_until > ${now}
30
- )
31
- ) AS exists
32
- `;
33
- return row.exists;
34
- }
35
- async function hasClaimableLifoJob(tx, ctx, queue, now) {
36
- const [row] = await tx `
37
- SELECT EXISTS (
38
- SELECT 1 FROM bunqueue_jobs AS job
39
- WHERE job.namespace = ${ctx.config.namespace}
40
- AND job.queue = ${queue}
41
- AND job.group_id IS NULL
42
- AND job.lifo
43
- AND job.state IN ('waiting', 'prioritized', 'delayed')
44
- AND job.run_at <= ${now}
45
- AND (job.ttl IS NULL OR job.created_at + job.ttl >= ${now})
46
19
  ) AS exists
47
20
  `;
48
21
  return row.exists;
49
22
  }
50
- async function selectFifoCandidates(tx, ctx, queue, now, capacity) {
51
- return await tx `
52
- WITH selected AS (
53
- SELECT job.id, job.priority, job.run_at
54
- FROM bunqueue_jobs AS job
55
- WHERE job.namespace = ${ctx.config.namespace}
56
- AND job.queue = ${queue}
57
- AND job.group_id IS NULL
58
- AND NOT job.lifo
59
- AND job.state IN ('waiting', 'prioritized', 'delayed')
60
- AND job.run_at <= ${now}
61
- AND (job.ttl IS NULL OR job.created_at + job.ttl >= ${now})
62
- AND NOT EXISTS (
63
- SELECT 1 FROM bunqueue_dependencies AS dependency
64
- WHERE dependency.namespace = job.namespace
65
- AND dependency.job_id = job.id
66
- AND NOT EXISTS (
67
- SELECT 1 FROM bunqueue_completions AS completion
68
- WHERE completion.namespace = dependency.namespace
69
- AND completion.job_id = dependency.dependency_id
70
- )
71
- )
72
- ORDER BY job.priority DESC, job.run_at ASC, job.id ASC
73
- LIMIT ${capacity}
74
- FOR UPDATE OF job SKIP LOCKED
75
- )
76
- SELECT job.*
77
- FROM bunqueue_jobs AS job
78
- JOIN selected ON selected.id = job.id
79
- WHERE job.namespace = ${ctx.config.namespace}
80
- ORDER BY selected.priority DESC, selected.run_at ASC, selected.id ASC
81
- `;
82
- }
83
23
  async function selectUngroupedCandidates(tx, ctx, queue, now, capacity) {
84
24
  return await tx `
85
25
  WITH selected AS (
86
26
  SELECT job.id, job.priority, job.lifo, job.run_at
87
27
  FROM bunqueue_jobs AS job
88
- WHERE job.namespace = ${ctx.config.namespace}
89
- AND job.queue = ${queue}
28
+ WHERE job.namespace = ${ctx.config.namespace} AND job.queue = ${queue}
90
29
  AND job.group_id IS NULL
91
30
  AND job.state IN ('waiting', 'prioritized', 'delayed')
92
31
  AND job.run_at <= ${now}
93
32
  AND (job.ttl IS NULL OR job.created_at + job.ttl >= ${now})
94
33
  AND NOT EXISTS (
95
34
  SELECT 1 FROM bunqueue_dependencies AS dependency
96
- WHERE dependency.namespace = job.namespace
97
- AND dependency.job_id = job.id
35
+ WHERE dependency.namespace = job.namespace AND dependency.job_id = job.id
98
36
  AND NOT EXISTS (
99
37
  SELECT 1 FROM bunqueue_completions AS completion
100
38
  WHERE completion.namespace = dependency.namespace
101
39
  AND completion.job_id = dependency.dependency_id
102
40
  )
103
41
  )
104
- ORDER BY
105
- job.priority DESC,
106
- CASE WHEN job.lifo THEN 0 ELSE 1 END ASC,
42
+ ORDER BY job.priority DESC,
43
+ CASE WHEN job.lifo THEN 0 ELSE 1 END,
107
44
  CASE WHEN job.lifo THEN job.id END DESC,
108
- CASE WHEN NOT job.lifo THEN job.run_at END ASC,
109
- job.id ASC
45
+ CASE WHEN NOT job.lifo THEN job.run_at END,
46
+ job.id
110
47
  LIMIT ${capacity}
111
48
  FOR UPDATE OF job SKIP LOCKED
112
49
  )
113
- SELECT job.*
114
- FROM bunqueue_jobs AS job
50
+ SELECT job.* FROM bunqueue_jobs AS job
115
51
  JOIN selected ON selected.id = job.id
116
52
  WHERE job.namespace = ${ctx.config.namespace}
117
- ORDER BY
118
- selected.priority DESC,
119
- CASE WHEN selected.lifo THEN 0 ELSE 1 END ASC,
53
+ ORDER BY selected.priority DESC,
54
+ CASE WHEN selected.lifo THEN 0 ELSE 1 END,
120
55
  CASE WHEN selected.lifo THEN selected.id END DESC,
121
- CASE WHEN NOT selected.lifo THEN selected.run_at END ASC,
122
- selected.id ASC
56
+ CASE WHEN NOT selected.lifo THEN selected.run_at END,
57
+ selected.id
123
58
  `;
124
59
  }
125
- async function selectGroupedCandidates(tx, ctx, queue, now, capacity) {
60
+ async function selectGroupedCandidates(tx, ctx, input) {
61
+ const { queue, now, capacity, options } = input;
62
+ const concurrency = options?.concurrency ?? null;
63
+ const rateMax = options?.limit?.max ?? null;
126
64
  return await tx `
127
- WITH eligible AS (
128
- SELECT job.id, job.priority, job.lifo, job.run_at,
65
+ WITH ready AS (
66
+ SELECT job.id, job.group_id,
129
67
  row_number() OVER (
130
- PARTITION BY CASE
131
- WHEN job.group_id IS NULL THEN 'job:' || job.id ELSE 'group:' || job.group_id
132
- END
133
- ORDER BY job.priority DESC,
134
- CASE WHEN job.lifo THEN 0 ELSE 1 END ASC,
135
- CASE WHEN job.lifo THEN job.id END DESC,
136
- CASE WHEN NOT job.lifo THEN job.run_at END ASC, job.id ASC
68
+ PARTITION BY job.group_id ORDER BY job.run_at, job.group_order, job.id
137
69
  ) AS group_position
138
70
  FROM bunqueue_jobs AS job
139
- WHERE job.namespace = ${ctx.config.namespace}
140
- AND job.queue = ${queue}
71
+ WHERE job.namespace = ${ctx.config.namespace} AND job.queue = ${queue}
72
+ AND job.group_id IS NOT NULL
141
73
  AND job.state IN ('waiting', 'prioritized', 'delayed')
142
74
  AND job.run_at <= ${now}
143
75
  AND (job.ttl IS NULL OR job.created_at + job.ttl >= ${now})
@@ -150,49 +82,59 @@ async function selectGroupedCandidates(tx, ctx, queue, now, capacity) {
150
82
  AND completion.job_id = dependency.dependency_id
151
83
  )
152
84
  )
153
- AND (job.group_id IS NULL OR NOT EXISTS (
154
- SELECT 1 FROM bunqueue_jobs AS active_group
155
- WHERE active_group.namespace = job.namespace AND active_group.queue = job.queue
156
- AND active_group.group_id = job.group_id AND active_group.state = 'active'
157
- AND active_group.lease_until > ${now}
158
- ))
85
+ ), active AS (
86
+ SELECT group_id, COUNT(*)::integer AS count
87
+ FROM bunqueue_jobs
88
+ WHERE namespace = ${ctx.config.namespace} AND queue = ${queue}
89
+ AND group_id IS NOT NULL AND state = 'active' AND lease_until > ${now}
90
+ GROUP BY group_id
91
+ ), capacities AS (
92
+ SELECT groups.group_id, groups.last_served,
93
+ LEAST(
94
+ ${capacity},
95
+ CASE WHEN CAST(${concurrency} AS BIGINT) IS NULL THEN ${capacity}
96
+ ELSE GREATEST(
97
+ 0,
98
+ COALESCE(groups.concurrency_limit, CAST(${concurrency} AS BIGINT))
99
+ - COALESCE(active.count, 0)
100
+ )
101
+ END,
102
+ CASE WHEN CAST(${rateMax} AS BIGINT) IS NULL THEN ${capacity}
103
+ ELSE GREATEST(
104
+ 0,
105
+ COALESCE(groups.rate_limit, CAST(${rateMax} AS BIGINT)) - groups.rate_count
106
+ )
107
+ END
108
+ ) AS available
109
+ FROM bunqueue_group_state AS groups
110
+ LEFT JOIN active ON active.group_id = groups.group_id
111
+ WHERE groups.namespace = ${ctx.config.namespace} AND groups.queue = ${queue}
112
+ ), selected AS (
113
+ SELECT ready.id, ready.group_id, ready.group_position, capacities.last_served
114
+ FROM ready JOIN capacities ON capacities.group_id = ready.group_id
115
+ WHERE ready.group_position <= capacities.available
116
+ ORDER BY ready.group_position, capacities.last_served NULLS FIRST, ready.group_id
117
+ LIMIT ${capacity}
159
118
  )
160
- SELECT job.*
161
- FROM bunqueue_jobs AS job
162
- JOIN eligible ON eligible.id = job.id
119
+ SELECT job.* FROM bunqueue_jobs AS job
120
+ JOIN selected ON selected.id = job.id
163
121
  WHERE job.namespace = ${ctx.config.namespace}
164
- AND eligible.group_position = 1
165
122
  AND job.state IN ('waiting', 'prioritized', 'delayed')
166
123
  AND job.run_at <= ${now}
167
- AND (job.ttl IS NULL OR job.created_at + job.ttl >= ${now})
168
- AND NOT EXISTS (
169
- SELECT 1 FROM bunqueue_dependencies AS dependency
170
- WHERE dependency.namespace = job.namespace AND dependency.job_id = job.id
171
- AND NOT EXISTS (
172
- SELECT 1 FROM bunqueue_completions AS completion
173
- WHERE completion.namespace = dependency.namespace
174
- AND completion.job_id = dependency.dependency_id
175
- )
176
- )
177
- AND (job.group_id IS NULL OR NOT EXISTS (
178
- SELECT 1 FROM bunqueue_jobs AS active_group
179
- WHERE active_group.namespace = job.namespace AND active_group.queue = job.queue
180
- AND active_group.group_id = job.group_id AND active_group.state = 'active'
181
- AND active_group.lease_until > ${now}
182
- ))
183
- ORDER BY job.priority DESC,
184
- CASE WHEN job.lifo THEN 0 ELSE 1 END ASC,
185
- CASE WHEN job.lifo THEN job.id END DESC,
186
- CASE WHEN NOT job.lifo THEN job.run_at END ASC, job.id ASC
187
- LIMIT ${capacity}
124
+ ORDER BY selected.group_position, selected.last_served NULLS FIRST, selected.group_id
188
125
  FOR UPDATE OF job SKIP LOCKED
189
126
  `;
190
127
  }
191
- export async function selectPostgresClaimCandidates(tx, ctx, queue, now, capacity) {
192
- if (await hasClaimableGroupedJob(tx, ctx, queue, now)) {
193
- return await selectGroupedCandidates(tx, ctx, queue, now, capacity);
194
- }
195
- return (await hasClaimableLifoJob(tx, ctx, queue, now))
196
- ? await selectUngroupedCandidates(tx, ctx, queue, now, capacity)
197
- : await selectFifoCandidates(tx, ctx, queue, now, capacity);
128
+ export async function selectPostgresClaimCandidates(tx, ctx, input) {
129
+ const { queue, now, capacity, options } = input;
130
+ const ungrouped = await selectUngroupedCandidates(tx, ctx, queue, now, capacity);
131
+ if (ungrouped.length >= capacity)
132
+ return ungrouped;
133
+ const grouped = await selectGroupedCandidates(tx, ctx, {
134
+ queue,
135
+ now,
136
+ capacity: capacity - ungrouped.length,
137
+ options,
138
+ });
139
+ return [...ungrouped, ...grouped];
198
140
  }
@@ -1,3 +1,10 @@
1
+ import type { GroupPullOptions } from '../../../domain/types/group';
1
2
  import { type PostgresContext } from './context';
2
3
  import type { ClaimedPostgresJob } from './types';
3
- export declare function claimPostgresJobs(ctx: PostgresContext, queue: string, count: number, owner: string, leaseDurationMs: number): Promise<ClaimedPostgresJob[]>;
4
+ export declare function claimPostgresJobs(ctx: PostgresContext, input: {
5
+ queue: string;
6
+ count: number;
7
+ owner: string;
8
+ leaseDurationMs: number;
9
+ groupOptions?: GroupPullOptions;
10
+ }): Promise<ClaimedPostgresJob[]>;
@@ -7,6 +7,8 @@ import { expirePendingPostgresJobs } from './expiry';
7
7
  import { repairResolvedPostgresDependencyConsumers } from './dependencyPromotion';
8
8
  import { lockPostgresBrokerSession } from './brokerSessions';
9
9
  import { runPostgresTransactionWithRetry } from './transactionRetry';
10
+ import { hasClaimablePostgresGroup } from './claimSelection';
11
+ import { ensurePostgresGroupStates, refreshPostgresGroupRateWindows } from './groupClaims';
10
12
  function toQueueState(row) {
11
13
  return {
12
14
  queue: row.queue,
@@ -19,20 +21,23 @@ function toQueueState(row) {
19
21
  concurrencyLimit: row.concurrency_limit,
20
22
  stallConfig: decodePostgresValue(row.stall_config, DEFAULT_STALL_CONFIG, 'stallConfig'),
21
23
  dlqConfig: decodePostgresValue(row.dlq_config, DEFAULT_DLQ_CONFIG, 'dlqConfig'),
24
+ groupSequence: Number(row.group_sequence),
22
25
  };
23
26
  }
24
27
  async function lockQueueState(tx, ctx, queue, exclusive) {
25
28
  const select = async () => exclusive
26
29
  ? await tx `
27
30
  SELECT queue, paused, rate_limit, rate_duration_ms, rate_window_started_at,
28
- rate_expires_at, rate_count, concurrency_limit, stall_config, dlq_config
31
+ rate_expires_at, rate_count, concurrency_limit, stall_config, dlq_config,
32
+ group_sequence
29
33
  FROM bunqueue_queue_state
30
34
  WHERE namespace = ${ctx.config.namespace} AND queue = ${queue}
31
35
  FOR UPDATE
32
36
  `
33
37
  : await tx `
34
38
  SELECT queue, paused, rate_limit, rate_duration_ms, rate_window_started_at,
35
- rate_expires_at, rate_count, concurrency_limit, stall_config, dlq_config
39
+ rate_expires_at, rate_count, concurrency_limit, stall_config, dlq_config,
40
+ group_sequence
36
41
  FROM bunqueue_queue_state
37
42
  WHERE namespace = ${ctx.config.namespace} AND queue = ${queue}
38
43
  FOR SHARE
@@ -92,7 +97,8 @@ async function activeCount(tx, ctx, queue, now) {
92
97
  `;
93
98
  return Number(row.count);
94
99
  }
95
- export async function claimPostgresJobs(ctx, queue, count, owner, leaseDurationMs) {
100
+ export async function claimPostgresJobs(ctx, input) {
101
+ const { queue, count, owner, leaseDurationMs, groupOptions } = input;
96
102
  if (count <= 0)
97
103
  return [];
98
104
  await expirePendingPostgresJobs(ctx, queue);
@@ -114,12 +120,22 @@ export async function claimPostgresJobs(ctx, queue, count, owner, leaseDurationM
114
120
  if (capacity <= 0)
115
121
  return { claimed: [], retryExclusive: false };
116
122
  await repairResolvedPostgresDependencyConsumers(tx, ctx, queue, now);
123
+ const hasGroups = await hasClaimablePostgresGroup(tx, ctx, queue, now);
124
+ if (!exclusive && hasGroups)
125
+ return { claimed: [], retryExclusive: true };
126
+ let groupSequence = state.groupSequence;
127
+ if (hasGroups) {
128
+ groupSequence = await ensurePostgresGroupStates(tx, ctx, queue, groupSequence);
129
+ await refreshPostgresGroupRateWindows(tx, ctx, queue, groupOptions, now);
130
+ }
117
131
  const claimed = await claimPostgresCandidates(tx, ctx, {
118
132
  queue,
119
133
  capacity,
120
134
  owner,
121
135
  requestedLeaseMs: leaseDurationMs,
122
136
  now,
137
+ groupOptions,
138
+ groupSequence,
123
139
  });
124
140
  if (state.rateLimit !== null && claimed.length > 0) {
125
141
  await tx `
@@ -0,0 +1,32 @@
1
+ import type { PostgresEventPruneWatermark } from './eventPruneWatermarks';
2
+ /** Per-queue catch-up bookkeeping for commit-ordered durable event delivery. */
3
+ export declare class PostgresEventCatchupCursors {
4
+ private readonly appliedCommitSeqs;
5
+ private readonly handledPruneWatermarks;
6
+ private baselineCommitSeq;
7
+ reset(baselineCommitSeq: number): void;
8
+ appliedThrough(queue: string): number;
9
+ applied(queue: string, commitSeq: number): void;
10
+ /** Record a prune watermark as accounted for and report whether it was new. */
11
+ remember(watermark: PostgresEventPruneWatermark): boolean;
12
+ /** Report whether a commit pruned its own events without this reader knowing. */
13
+ private hasUnhandledSelfPrune;
14
+ /**
15
+ * Report whether a prune watermark requires an authoritative queue refresh.
16
+ *
17
+ * Two cases need one. A reader at or behind the pruned frontier may have
18
+ * missed discarded history. And a commit that writes more queue events than
19
+ * the retained window prunes its own older events before any reader can
20
+ * observe them, so every reader must reload once per such commit, even one
21
+ * whose applied position has already moved past it. `self_pruned_commit_seq`
22
+ * carries that evidence forward per queue, so a superseding watermark does
23
+ * not drop it; queue destruction erases it with the queue itself, which is
24
+ * safe because the same transaction also invalidates the queue and commit
25
+ * sequence numbers never rewind. A watermark already accounted for never
26
+ * refreshes again. The `<=` comparison is deliberately wider than the
27
+ * observed cases: it also covers a commit split across drain batches, where
28
+ * the applied cursor already equals the pruned frontier while the rest of
29
+ * that commit is still unread.
30
+ */
31
+ requiresRefresh(watermark: PostgresEventPruneWatermark): boolean;
32
+ }
@@ -0,0 +1,66 @@
1
+ /** Per-queue catch-up bookkeeping for commit-ordered durable event delivery. */
2
+ export class PostgresEventCatchupCursors {
3
+ appliedCommitSeqs = new Map();
4
+ handledPruneWatermarks = new Map();
5
+ baselineCommitSeq = 0;
6
+ reset(baselineCommitSeq) {
7
+ this.baselineCommitSeq = baselineCommitSeq;
8
+ this.appliedCommitSeqs.clear();
9
+ this.handledPruneWatermarks.clear();
10
+ }
11
+ appliedThrough(queue) {
12
+ return this.appliedCommitSeqs.get(queue) ?? this.baselineCommitSeq;
13
+ }
14
+ applied(queue, commitSeq) {
15
+ this.appliedCommitSeqs.set(queue, Math.max(this.appliedThrough(queue), commitSeq));
16
+ }
17
+ /** Record a prune watermark as accounted for and report whether it was new. */
18
+ remember(watermark) {
19
+ const { queue, commitSeq, prunedThrough, selfPrunedCommitSeq } = watermark;
20
+ const handled = this.handledPruneWatermarks.get(queue);
21
+ if (handled !== undefined &&
22
+ handled.commitSeq >= commitSeq &&
23
+ handled.prunedThrough >= prunedThrough &&
24
+ handled.selfPrunedCommitSeq >= selfPrunedCommitSeq) {
25
+ return false;
26
+ }
27
+ this.handledPruneWatermarks.set(queue, {
28
+ commitSeq: Math.max(handled?.commitSeq ?? 0, commitSeq),
29
+ prunedThrough: Math.max(handled?.prunedThrough ?? 0, prunedThrough),
30
+ selfPrunedCommitSeq: Math.max(handled?.selfPrunedCommitSeq ?? 0, selfPrunedCommitSeq),
31
+ });
32
+ return true;
33
+ }
34
+ /** Report whether a commit pruned its own events without this reader knowing. */
35
+ hasUnhandledSelfPrune(watermark) {
36
+ const handled = this.handledPruneWatermarks.get(watermark.queue);
37
+ return watermark.selfPrunedCommitSeq > (handled?.selfPrunedCommitSeq ?? 0);
38
+ }
39
+ /**
40
+ * Report whether a prune watermark requires an authoritative queue refresh.
41
+ *
42
+ * Two cases need one. A reader at or behind the pruned frontier may have
43
+ * missed discarded history. And a commit that writes more queue events than
44
+ * the retained window prunes its own older events before any reader can
45
+ * observe them, so every reader must reload once per such commit, even one
46
+ * whose applied position has already moved past it. `self_pruned_commit_seq`
47
+ * carries that evidence forward per queue, so a superseding watermark does
48
+ * not drop it; queue destruction erases it with the queue itself, which is
49
+ * safe because the same transaction also invalidates the queue and commit
50
+ * sequence numbers never rewind. A watermark already accounted for never
51
+ * refreshes again. The `<=` comparison is deliberately wider than the
52
+ * observed cases: it also covers a commit split across drain batches, where
53
+ * the applied cursor already equals the pruned frontier while the rest of
54
+ * that commit is still unread.
55
+ */
56
+ requiresRefresh(watermark) {
57
+ const behind = this.appliedThrough(watermark.queue) <= watermark.prunedCommitSeq;
58
+ const selfPruned = this.hasUnhandledSelfPrune(watermark);
59
+ if (!this.remember(watermark))
60
+ return false;
61
+ if (!behind && !selfPruned)
62
+ return false;
63
+ this.applied(watermark.queue, watermark.commitSeq);
64
+ return true;
65
+ }
66
+ }