bunqueue 2.9.0 → 2.9.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/README.md +7 -3
  2. package/dist/application/background/recovery/active.d.ts +1 -0
  3. package/dist/application/background/recovery/active.js +12 -0
  4. package/dist/application/background/recovery/index.js +3 -1
  5. package/dist/application/background/recovery/pending.js +1 -0
  6. package/dist/application/cleanupTasks.js +4 -16
  7. package/dist/application/contextFactory.js +3 -0
  8. package/dist/application/dlqManager.js +1 -0
  9. package/dist/application/dlqRetry.js +1 -0
  10. package/dist/application/eventsManager.d.ts +8 -1
  11. package/dist/application/eventsManager.js +76 -29
  12. package/dist/application/operations/ackHelpers.d.ts +1 -4
  13. package/dist/application/operations/ackHelpers.js +39 -14
  14. package/dist/application/operations/flowPush.js +1 -0
  15. package/dist/application/operations/flowValidation.js +4 -4
  16. package/dist/application/operations/pull.d.ts +7 -2
  17. package/dist/application/operations/pull.js +15 -57
  18. package/dist/application/operations/pullFinalization.d.ts +6 -0
  19. package/dist/application/operations/pullFinalization.js +69 -0
  20. package/dist/application/operations/pullStateTransition.d.ts +8 -7
  21. package/dist/application/operations/pullStateTransition.js +25 -18
  22. package/dist/application/operations/push.js +3 -0
  23. package/dist/application/operations/pushBatch.js +14 -7
  24. package/dist/application/operations/pushContext.d.ts +2 -1
  25. package/dist/application/postgres-queue-manager/control.d.ts +4 -0
  26. package/dist/application/postgres-queue-manager/control.js +24 -0
  27. package/dist/application/postgres-queue-manager/delivery.d.ts +5 -4
  28. package/dist/application/postgres-queue-manager/delivery.js +13 -10
  29. package/dist/application/postgres-queue-manager/groupQueries.d.ts +10 -0
  30. package/dist/application/postgres-queue-manager/groupQueries.js +40 -0
  31. package/dist/application/postgres-queue-manager/queries.d.ts +2 -2
  32. package/dist/application/postgres-queue-manager/queries.js +2 -2
  33. package/dist/application/queue-manager/control.js +1 -0
  34. package/dist/application/queue-manager/delivery.d.ts +5 -4
  35. package/dist/application/queue-manager/delivery.js +15 -8
  36. package/dist/application/queue-manager/limits.d.ts +11 -0
  37. package/dist/application/queue-manager/limits.js +61 -0
  38. package/dist/application/queue-manager/state.js +1 -1
  39. package/dist/application/queueTelemetryJournal.d.ts +2 -0
  40. package/dist/application/queueTelemetryJournal.js +23 -0
  41. package/dist/application/types/ack.d.ts +3 -1
  42. package/dist/cli/commandRouter.d.ts +1 -1
  43. package/dist/client/flowPush.js +6 -0
  44. package/dist/client/groupId.d.ts +1 -0
  45. package/dist/client/groupId.js +1 -0
  46. package/dist/client/index.d.ts +1 -1
  47. package/dist/client/jobHelpers.js +1 -0
  48. package/dist/client/queue/operations/add/bulk.js +4 -4
  49. package/dist/client/queue/operations/add/payload.d.ts +1 -0
  50. package/dist/client/queue/operations/add/payload.js +7 -1
  51. package/dist/client/queue/operations/add/single.js +3 -3
  52. package/dist/client/queue/operations/groups.d.ts +21 -0
  53. package/dist/client/queue/operations/groups.js +88 -0
  54. package/dist/client/queue/runtime/queries.d.ts +13 -0
  55. package/dist/client/queue/runtime/queries.js +31 -0
  56. package/dist/client/queue/runtime/state.d.ts +4 -0
  57. package/dist/client/queue/runtime/state.js +1 -0
  58. package/dist/client/types/index.d.ts +2 -2
  59. package/dist/client/types/options.d.ts +4 -0
  60. package/dist/client/types/worker.d.ts +8 -0
  61. package/dist/client/worker/runtime/execution.js +1 -0
  62. package/dist/client/worker/runtime/options.js +3 -0
  63. package/dist/client/worker/types/options.d.ts +2 -1
  64. package/dist/client/worker/workerPull.d.ts +2 -0
  65. package/dist/client/worker/workerPull.js +6 -4
  66. package/dist/domain/job/groupFifoOrder.d.ts +9 -0
  67. package/dist/domain/job/groupFifoOrder.js +26 -0
  68. package/dist/domain/queue/groupLimiterManager.d.ts +23 -0
  69. package/dist/domain/queue/groupLimiterManager.js +107 -0
  70. package/dist/domain/queue/groupScheduler.d.ts +31 -0
  71. package/dist/domain/queue/groupScheduler.js +256 -0
  72. package/dist/domain/queue/groupSchedulerOrder.d.ts +6 -0
  73. package/dist/domain/queue/groupSchedulerOrder.js +30 -0
  74. package/dist/domain/queue/priorityQueue.d.ts +13 -23
  75. package/dist/domain/queue/priorityQueue.js +53 -42
  76. package/dist/domain/queue/shard/keys.d.ts +1 -0
  77. package/dist/domain/queue/shard/keys.js +25 -2
  78. package/dist/domain/queue/shard/lifecycle.js +4 -0
  79. package/dist/domain/queue/shard/limits.d.ts +20 -1
  80. package/dist/domain/queue/shard/limits.js +56 -0
  81. package/dist/domain/queue/shard/state.d.ts +5 -0
  82. package/dist/domain/queue/shard/state.js +9 -1
  83. package/dist/domain/types/commands/core.d.ts +3 -0
  84. package/dist/domain/types/commands/limits.d.ts +41 -0
  85. package/dist/domain/types/commands/union.d.ts +1 -1
  86. package/dist/domain/types/group.d.ts +21 -0
  87. package/dist/domain/types/group.js +78 -0
  88. package/dist/domain/types/priorityQueue.d.ts +1 -0
  89. package/dist/infrastructure/persistence/jobOptionsBlob.d.ts +1 -0
  90. package/dist/infrastructure/persistence/jobOptionsBlob.js +10 -2
  91. package/dist/infrastructure/persistence/migrations.js +11 -0
  92. package/dist/infrastructure/persistence/postgres/admission.js +5 -2
  93. package/dist/infrastructure/persistence/postgres/batchAdmission.js +22 -4
  94. package/dist/infrastructure/persistence/postgres/claimBatch.d.ts +3 -0
  95. package/dist/infrastructure/persistence/postgres/claimBatch.js +16 -3
  96. package/dist/infrastructure/persistence/postgres/claimSelection.d.ts +8 -1
  97. package/dist/infrastructure/persistence/postgres/claimSelection.js +73 -131
  98. package/dist/infrastructure/persistence/postgres/claims.d.ts +8 -1
  99. package/dist/infrastructure/persistence/postgres/claims.js +19 -3
  100. package/dist/infrastructure/persistence/postgres/eventCatchupCursors.d.ts +32 -0
  101. package/dist/infrastructure/persistence/postgres/eventCatchupCursors.js +66 -0
  102. package/dist/infrastructure/persistence/postgres/eventJournalSchema.d.ts +2 -2
  103. package/dist/infrastructure/persistence/postgres/eventJournalSchema.js +16 -2
  104. package/dist/infrastructure/persistence/postgres/eventPruneWatermarks.d.ts +2 -0
  105. package/dist/infrastructure/persistence/postgres/eventPruneWatermarks.js +21 -3
  106. package/dist/infrastructure/persistence/postgres/events.d.ts +3 -3
  107. package/dist/infrastructure/persistence/postgres/events.js +44 -17
  108. package/dist/infrastructure/persistence/postgres/groupClaims.d.ts +15 -0
  109. package/dist/infrastructure/persistence/postgres/groupClaims.js +95 -0
  110. package/dist/infrastructure/persistence/postgres/groupSchema.d.ts +2 -0
  111. package/dist/infrastructure/persistence/postgres/groupSchema.js +155 -0
  112. package/dist/infrastructure/persistence/postgres/groupSchemaFingerprint.d.ts +3 -0
  113. package/dist/infrastructure/persistence/postgres/groupSchemaFingerprint.js +127 -0
  114. package/dist/infrastructure/persistence/postgres/groupStateRetention.d.ts +3 -0
  115. package/dist/infrastructure/persistence/postgres/groupStateRetention.js +93 -0
  116. package/dist/infrastructure/persistence/postgres/groups.d.ts +11 -0
  117. package/dist/infrastructure/persistence/postgres/groups.js +117 -0
  118. package/dist/infrastructure/persistence/postgres/queueDestruction.js +4 -0
  119. package/dist/infrastructure/persistence/postgres/runtime.d.ts +0 -1
  120. package/dist/infrastructure/persistence/postgres/runtime.js +3 -1
  121. package/dist/infrastructure/persistence/postgres/schema.d.ts +2 -2
  122. package/dist/infrastructure/persistence/postgres/schema.js +8 -4
  123. package/dist/infrastructure/persistence/postgres/schemaInitialization.js +6 -1
  124. package/dist/infrastructure/persistence/postgres/store.d.ts +11 -1
  125. package/dist/infrastructure/persistence/postgres/store.js +47 -3
  126. package/dist/infrastructure/persistence/postgres/telemetry.d.ts +2 -0
  127. package/dist/infrastructure/persistence/postgres/telemetry.js +33 -21
  128. package/dist/infrastructure/persistence/postgres/types.d.ts +1 -0
  129. package/dist/infrastructure/persistence/schema.d.ts +2 -2
  130. package/dist/infrastructure/persistence/schema.js +12 -1
  131. package/dist/infrastructure/persistence/sqlite/control.d.ts +14 -0
  132. package/dist/infrastructure/persistence/sqlite/control.js +69 -0
  133. package/dist/infrastructure/persistence/sqlite/jobs.d.ts +12 -0
  134. package/dist/infrastructure/persistence/sqlite/jobs.js +36 -0
  135. package/dist/infrastructure/persistence/sqlite/telemetry.d.ts +2 -1
  136. package/dist/infrastructure/persistence/sqlite/telemetry.js +5 -64
  137. package/dist/infrastructure/persistence/sqlite/telemetryWrites.d.ts +4 -0
  138. package/dist/infrastructure/persistence/sqlite/telemetryWrites.js +123 -0
  139. package/dist/infrastructure/persistence/sqliteSerializer.js +2 -0
  140. package/dist/infrastructure/server/handler-routes/control.js +13 -0
  141. package/dist/infrastructure/server/handlers/core.d.ts +0 -5
  142. package/dist/infrastructure/server/handlers/core.js +15 -18
  143. package/dist/infrastructure/server/handlers/groups.d.ts +11 -0
  144. package/dist/infrastructure/server/handlers/groups.js +55 -0
  145. package/dist/infrastructure/server/handlers/pushBatchValidation.js +4 -1
  146. package/dist/infrastructure/server/protocol/validation.d.ts +1 -0
  147. package/dist/infrastructure/server/protocol/validation.js +1 -0
  148. package/dist/infrastructure/server/protocol.d.ts +1 -1
  149. package/dist/infrastructure/server/protocol.js +1 -1
  150. package/package.json +2 -1
@@ -1,36 +1,29 @@
1
- /**
2
- * Indexed Priority Queue
3
- * Combines binary heap for O(log n) priority operations
4
- * with Map for O(1) lookups by job ID
5
- */
1
+ import { copyGroupFifoOrder } from '../job/groupFifoOrder';
6
2
  import { comparePriorityEntries } from './priorityQueueOrder';
7
- /**
8
- * Indexed Priority Queue implementation with 4-ary heap
9
- * 4-ary provides better cache locality than binary heap
10
- * O(log₄ n) push, pop, update
11
- * O(1) find, has
12
- */
3
+ /** Indexed map-backed 4-ary heap with O(log4 n) mutations and O(1) lookup. */
13
4
  export class IndexedPriorityQueue {
14
- /** Branching factor - 4 provides optimal cache performance */
5
+ hooks;
6
+ compare;
7
+ order;
15
8
  static D = 4;
16
9
  heap = [];
17
10
  index = new Map();
18
- // Use BigInt to prevent overflow at extreme throughput
19
11
  generation = 0n;
20
- /** Get current size */
12
+ hooksActive = false;
13
+ constructor(hooks, compare = comparePriorityEntries, order) {
14
+ this.hooks = hooks;
15
+ this.compare = compare;
16
+ this.order = order;
17
+ }
21
18
  get size() {
22
19
  return this.index.size;
23
20
  }
24
- /** Check if empty */
25
21
  get isEmpty() {
26
22
  return this.index.size === 0;
27
23
  }
28
- /** Push a job into the queue */
29
24
  push(job) {
30
25
  const gen = this.generation++;
31
- // Store in index
32
26
  this.index.set(job.id, { job, generation: gen });
33
- // Add to heap
34
27
  const entry = {
35
28
  jobId: job.id,
36
29
  priority: job.priority,
@@ -38,32 +31,38 @@ export class IndexedPriorityQueue {
38
31
  lifo: job.lifo,
39
32
  generation: gen,
40
33
  };
34
+ if (this.order) {
35
+ const groupFifoOrder = this.order(job);
36
+ if (groupFifoOrder !== null)
37
+ entry.groupFifoOrder = groupFifoOrder;
38
+ }
41
39
  this.heap.push(entry);
42
40
  this.bubbleUp(this.heap.length - 1);
41
+ if (this.hooks && (this.hooksActive || job.groupId)) {
42
+ this.hooksActive = this.hooks.onInsert(job);
43
+ }
43
44
  }
44
- /** Pop the highest priority job */
45
45
  pop() {
46
46
  while (this.heap.length > 0) {
47
47
  const entry = this.heap[0];
48
48
  const indexed = this.index.get(entry.jobId);
49
- // Skip stale entries (generation mismatch = updated or removed)
50
49
  if (indexed?.generation !== entry.generation) {
51
50
  this.removeTop();
52
51
  continue;
53
52
  }
54
- // Remove from both structures
55
53
  this.removeTop();
56
54
  this.index.delete(entry.jobId);
55
+ if (this.hooks && this.hooksActive) {
56
+ this.hooksActive = this.hooks.onRemove(indexed.job);
57
+ }
57
58
  return indexed.job;
58
59
  }
59
60
  return null;
60
61
  }
61
- /** Peek at the highest priority job without removing */
62
62
  peek() {
63
63
  while (this.heap.length > 0) {
64
64
  const entry = this.heap[0];
65
65
  const indexed = this.index.get(entry.jobId);
66
- // Skip stale entries
67
66
  if (indexed?.generation !== entry.generation) {
68
67
  this.removeTop();
69
68
  continue;
@@ -72,21 +71,20 @@ export class IndexedPriorityQueue {
72
71
  }
73
72
  return null;
74
73
  }
75
- /** Find a job by ID - O(1) */
76
74
  find(jobId) {
77
75
  return this.index.get(jobId)?.job ?? null;
78
76
  }
79
- /** Check if job exists - O(1) */
80
77
  has(jobId) {
81
78
  return this.index.has(jobId);
82
79
  }
83
- /** Remove a job by ID - O(1) for index, heap cleans lazily */
84
80
  remove(jobId) {
85
81
  const indexed = this.index.get(jobId);
86
82
  if (!indexed)
87
83
  return null;
88
84
  this.index.delete(jobId);
89
- // Heap entry becomes stale, will be skipped on pop
85
+ if (this.hooks && this.hooksActive) {
86
+ this.hooksActive = this.hooks.onRemove(indexed.job);
87
+ }
90
88
  return indexed.job;
91
89
  }
92
90
  /** Update job priority - O(log n). Optionally update the LIFO tie-break flag. */
@@ -95,15 +93,13 @@ export class IndexedPriorityQueue {
95
93
  if (!indexed)
96
94
  return false;
97
95
  const lifo = newLifo ?? indexed.job.lifo;
98
- // Create new job with updated priority (immutable pattern)
99
96
  const updatedJob = {
100
97
  ...indexed.job,
101
98
  priority: newPriority,
102
99
  lifo,
103
100
  };
104
- // Create new heap entry with new generation
101
+ copyGroupFifoOrder(indexed.job, updatedJob);
105
102
  const gen = this.generation++;
106
- // Update index with new job and generation
107
103
  this.index.set(jobId, { job: updatedJob, generation: gen });
108
104
  const entry = {
109
105
  jobId: updatedJob.id,
@@ -112,8 +108,16 @@ export class IndexedPriorityQueue {
112
108
  lifo,
113
109
  generation: gen,
114
110
  };
111
+ if (this.order) {
112
+ const groupFifoOrder = this.order(updatedJob);
113
+ if (groupFifoOrder !== null)
114
+ entry.groupFifoOrder = groupFifoOrder;
115
+ }
115
116
  this.heap.push(entry);
116
117
  this.bubbleUp(this.heap.length - 1);
118
+ if (this.hooks && (this.hooksActive || updatedJob.groupId)) {
119
+ this.hooksActive = this.hooks.onInsert(updatedJob);
120
+ }
117
121
  return true;
118
122
  }
119
123
  /** Update job runAt (for delay changes) - O(log n) */
@@ -121,14 +125,12 @@ export class IndexedPriorityQueue {
121
125
  const indexed = this.index.get(jobId);
122
126
  if (!indexed)
123
127
  return false;
124
- // Create new job with updated runAt (immutable pattern)
125
128
  const updatedJob = {
126
129
  ...indexed.job,
127
130
  runAt: newRunAt,
128
131
  };
129
- // Create new heap entry with new generation
132
+ copyGroupFifoOrder(indexed.job, updatedJob);
130
133
  const gen = this.generation++;
131
- // Update index with new job and generation
132
134
  this.index.set(jobId, { job: updatedJob, generation: gen });
133
135
  const entry = {
134
136
  jobId: updatedJob.id,
@@ -137,19 +139,33 @@ export class IndexedPriorityQueue {
137
139
  lifo: updatedJob.lifo,
138
140
  generation: gen,
139
141
  };
142
+ if (this.order) {
143
+ const groupFifoOrder = this.order(updatedJob);
144
+ if (groupFifoOrder !== null)
145
+ entry.groupFifoOrder = groupFifoOrder;
146
+ }
140
147
  this.heap.push(entry);
141
148
  this.bubbleUp(this.heap.length - 1);
149
+ if (this.hooks && (this.hooksActive || updatedJob.groupId)) {
150
+ this.hooksActive = this.hooks.onInsert(updatedJob);
151
+ }
142
152
  return true;
143
153
  }
144
- /** Get all jobs (for iteration) */
145
154
  values() {
146
155
  return Array.from(this.index.values()).map((v) => v.job);
147
156
  }
148
- /** Clear the queue */
149
157
  clear() {
158
+ if (this.hooks) {
159
+ for (const { job } of this.index.values()) {
160
+ if (!this.hooksActive)
161
+ break;
162
+ this.hooksActive = this.hooks.onRemove(job);
163
+ }
164
+ }
150
165
  this.heap = [];
151
166
  this.index.clear();
152
167
  this.generation = 0n;
168
+ this.hooksActive = false;
153
169
  }
154
170
  /**
155
171
  * Get the ratio of stale entries in the heap
@@ -180,11 +196,8 @@ export class IndexedPriorityQueue {
180
196
  this.heap = validEntries;
181
197
  this.heapify();
182
198
  }
183
- /** Rebuild heap property from arbitrary array - O(n) */
184
199
  heapify() {
185
200
  const D = IndexedPriorityQueue.D;
186
- // Start from last non-leaf node and bubble down
187
- // In D-ary heap, last non-leaf is at floor((n-2)/D)
188
201
  for (let i = Math.floor((this.heap.length - 2) / D); i >= 0; i--) {
189
202
  this.bubbleDown(i);
190
203
  }
@@ -193,7 +206,6 @@ export class IndexedPriorityQueue {
193
206
  needsCompaction(threshold = 0.2) {
194
207
  return this.getStaleRatio() > threshold;
195
208
  }
196
- // ============ Heap Operations ============
197
209
  removeTop() {
198
210
  if (this.heap.length <= 1) {
199
211
  this.heap.pop();
@@ -207,7 +219,7 @@ export class IndexedPriorityQueue {
207
219
  const D = IndexedPriorityQueue.D;
208
220
  while (idx > 0) {
209
221
  const parentIdx = Math.floor((idx - 1) / D);
210
- if (comparePriorityEntries(this.heap[idx], this.heap[parentIdx]) >= 0) {
222
+ if (this.compare(this.heap[idx], this.heap[parentIdx]) >= 0) {
211
223
  break;
212
224
  }
213
225
  this.swap(idx, parentIdx);
@@ -223,11 +235,10 @@ export class IndexedPriorityQueue {
223
235
  const firstChild = D * idx + 1;
224
236
  if (firstChild >= length)
225
237
  break;
226
- // Find minimum among up to D children (cache-friendly sequential access)
227
238
  let smallest = idx;
228
239
  const lastChild = Math.min(firstChild + D, length);
229
240
  for (let c = firstChild; c < lastChild; c++) {
230
- if (comparePriorityEntries(heap[c], heap[smallest]) < 0) {
241
+ if (this.compare(heap[c], heap[smallest]) < 0) {
231
242
  smallest = c;
232
243
  }
233
244
  }
@@ -15,4 +15,5 @@ export declare class ShardKeys extends ShardState {
15
15
  isGroupActive(queue: string, groupId: string): boolean;
16
16
  activateGroup(queue: string, groupId: string): void;
17
17
  releaseGroup(queue: string, groupId: string): void;
18
+ getGroupActiveCount(queue: string, groupId: string): number;
18
19
  }
@@ -29,7 +29,7 @@ export class ShardKeys extends ShardState {
29
29
  return this.uniqueKeyManager.getMap();
30
30
  }
31
31
  isGroupActive(queue, groupId) {
32
- return this.activeGroups.get(queue)?.has(groupId) ?? false;
32
+ return (this.activeGroupCounts.get(queue)?.get(groupId) ?? 0) > 0;
33
33
  }
34
34
  activateGroup(queue, groupId) {
35
35
  let groups = this.activeGroups.get(queue);
@@ -38,8 +38,31 @@ export class ShardKeys extends ShardState {
38
38
  this.activeGroups.set(queue, groups);
39
39
  }
40
40
  groups.add(groupId);
41
+ let counts = this.activeGroupCounts.get(queue);
42
+ if (!counts) {
43
+ counts = new Map();
44
+ this.activeGroupCounts.set(queue, counts);
45
+ }
46
+ counts.set(groupId, (counts.get(groupId) ?? 0) + 1);
41
47
  }
42
48
  releaseGroup(queue, groupId) {
43
- this.activeGroups.get(queue)?.delete(groupId);
49
+ const counts = this.activeGroupCounts.get(queue);
50
+ const count = (counts?.get(groupId) ?? 0) - 1;
51
+ if (count > 0) {
52
+ counts?.set(groupId, count);
53
+ }
54
+ else {
55
+ counts?.delete(groupId);
56
+ const groups = this.activeGroups.get(queue);
57
+ groups?.delete(groupId);
58
+ if (groups?.size === 0)
59
+ this.activeGroups.delete(queue);
60
+ }
61
+ if (counts?.size === 0)
62
+ this.activeGroupCounts.delete(queue);
63
+ this.notify(queue);
64
+ }
65
+ getGroupActiveCount(queue, groupId) {
66
+ return this.activeGroupCounts.get(queue)?.get(groupId) ?? 0;
44
67
  }
45
68
  }
@@ -12,6 +12,7 @@ export class ShardLifecycle extends ShardMetrics {
12
12
  this.temporalManager.removeDelayed(job.id);
13
13
  }
14
14
  priorityQueue.clear();
15
+ this.groupScheduler.clearQueue(queue);
15
16
  this.temporalManager.clearIndexForQueue(queue);
16
17
  this.counters.adjustQueued(-count);
17
18
  this.counters.syncDelayedCount();
@@ -37,7 +38,10 @@ export class ShardLifecycle extends ShardMetrics {
37
38
  this.queues.delete(queue);
38
39
  this.uniqueKeyManager.clearQueue(queue);
39
40
  this.limiterManager.deleteQueue(queue);
41
+ this.groupScheduler.clearQueue(queue);
42
+ this.groupLimiterManager.clearQueue(queue);
40
43
  this.activeGroups.delete(queue);
44
+ this.activeGroupCounts.delete(queue);
41
45
  return Array.from(removed);
42
46
  }
43
47
  }
@@ -1,8 +1,27 @@
1
- import type { JobId } from '../../types/job';
1
+ import type { Job, JobId } from '../../types/job';
2
+ import type { GroupPullOptions, GroupRateLimitOverride } from '../../types/group';
3
+ import type { GroupCandidate } from '../groupScheduler';
2
4
  import type { QueueState } from '../../types/queue';
3
5
  import { ShardKeys } from './keys';
4
6
  /** Queue rate, concurrency, and resource ownership operations. */
5
7
  export declare class ShardLimits extends ShardKeys {
8
+ hasGroupScheduler(queue: string): boolean;
9
+ assignGroupFifoOrder(job: Job): void;
10
+ observeGroupFifoOrder(job: Job): void;
11
+ peekGroupCandidate(queue: string, now: number, options?: GroupPullOptions): GroupCandidate;
12
+ acquireGroup(queue: string, groupId: string, options?: GroupPullOptions, now?: number): boolean;
13
+ advanceGroup(queue: string, groupId: string): void;
14
+ getGroupJobsCount(queue: string, groupId: string): number;
15
+ getGroupsJobsCount(queue: string): number;
16
+ cleanExpiredGroupWindows(now: number): void;
17
+ clearEmptyGroupRuntime(queue: string): void;
18
+ setGroupRateLimit(queue: string, groupId: string, max: number, duration: number): void;
19
+ getGroupRateLimit(queue: string, groupId: string): GroupRateLimitOverride | null;
20
+ removeGroupRateLimit(queue: string, groupId: string): number;
21
+ getGroupRateLimitTtl(queue: string, groupId: string, maxJobs?: number): number;
22
+ setGroupConcurrency(queue: string, groupId: string, concurrency: number): void;
23
+ getGroupConcurrency(queue: string, groupId: string): number | null;
24
+ removeGroupConcurrency(queue: string, groupId: string): number;
6
25
  setRateLimit(queue: string, limit: number, durationMs?: number, ttlMs?: number): void;
7
26
  clearRateLimit(queue: string): void;
8
27
  expireRateLimitIfNeeded(queue: string): void;
@@ -1,6 +1,62 @@
1
1
  import { ShardKeys } from './keys';
2
2
  /** Queue rate, concurrency, and resource ownership operations. */
3
3
  export class ShardLimits extends ShardKeys {
4
+ hasGroupScheduler(queue) {
5
+ return this.groupScheduler.isActive(queue);
6
+ }
7
+ assignGroupFifoOrder(job) {
8
+ this.groupScheduler.assignFifoOrder(job);
9
+ }
10
+ observeGroupFifoOrder(job) {
11
+ this.groupScheduler.observeFifoOrder(job);
12
+ }
13
+ peekGroupCandidate(queue, now, options) {
14
+ return this.groupScheduler.peek(queue, now, (groupId) => this.groupLimiterManager.status(queue, groupId, options, this.getGroupActiveCount(queue, groupId), now));
15
+ }
16
+ acquireGroup(queue, groupId, options, now) {
17
+ const acquired = this.groupLimiterManager.acquire(queue, groupId, options, this.getGroupActiveCount(queue, groupId), now);
18
+ if (acquired)
19
+ this.activateGroup(queue, groupId);
20
+ return acquired;
21
+ }
22
+ advanceGroup(queue, groupId) {
23
+ this.groupScheduler.advance(queue, groupId);
24
+ }
25
+ getGroupJobsCount(queue, groupId) {
26
+ return this.groupScheduler.getGroupJobsCount(queue, groupId);
27
+ }
28
+ getGroupsJobsCount(queue) {
29
+ return this.groupScheduler.getGroupsJobsCount(queue);
30
+ }
31
+ cleanExpiredGroupWindows(now) {
32
+ this.groupLimiterManager.pruneExpiredWindows(now);
33
+ }
34
+ clearEmptyGroupRuntime(queue) {
35
+ this.groupScheduler.clearQueue(queue);
36
+ this.activeGroups.delete(queue);
37
+ this.activeGroupCounts.delete(queue);
38
+ }
39
+ setGroupRateLimit(queue, groupId, max, duration) {
40
+ this.groupLimiterManager.setRateLimit(queue, groupId, max, duration);
41
+ }
42
+ getGroupRateLimit(queue, groupId) {
43
+ return this.groupLimiterManager.getRateLimit(queue, groupId);
44
+ }
45
+ removeGroupRateLimit(queue, groupId) {
46
+ return this.groupLimiterManager.removeRateLimit(queue, groupId);
47
+ }
48
+ getGroupRateLimitTtl(queue, groupId, maxJobs) {
49
+ return this.groupLimiterManager.getRateLimitTtl(queue, groupId, maxJobs);
50
+ }
51
+ setGroupConcurrency(queue, groupId, concurrency) {
52
+ this.groupLimiterManager.setConcurrency(queue, groupId, concurrency);
53
+ }
54
+ getGroupConcurrency(queue, groupId) {
55
+ return this.groupLimiterManager.getConcurrency(queue, groupId);
56
+ }
57
+ removeGroupConcurrency(queue, groupId) {
58
+ return this.groupLimiterManager.removeConcurrency(queue, groupId);
59
+ }
4
60
  setRateLimit(queue, limit, durationMs, ttlMs) {
5
61
  this.limiterManager.setRateLimit(queue, limit, durationMs, ttlMs);
6
62
  }
@@ -4,6 +4,8 @@ import { DependencyTracker } from '../dependencyTracker';
4
4
  import { DlqShard } from '../dlqShard';
5
5
  import { LimiterManager } from '../limiterManager';
6
6
  import { IndexedPriorityQueue } from '../priorityQueue';
7
+ import { GroupLimiterManager } from '../groupLimiterManager';
8
+ import { GroupScheduler } from '../groupScheduler';
7
9
  import { ShardCounters } from '../shardCounters';
8
10
  import { TemporalManager } from '../temporalManager';
9
11
  import { UniqueKeyManager } from '../uniqueKeyManager';
@@ -12,9 +14,12 @@ import { WaiterManager } from '../waiterManager';
12
14
  export declare class ShardState {
13
15
  readonly queues: Map<string, IndexedPriorityQueue>;
14
16
  readonly activeGroups: Map<string, Set<string>>;
17
+ readonly activeGroupCounts: Map<string, Map<string, number>>;
15
18
  protected readonly uniqueKeyManager: UniqueKeyManager;
16
19
  protected readonly dlqManager: DlqShard;
17
20
  protected readonly limiterManager: LimiterManager;
21
+ protected readonly groupLimiterManager: GroupLimiterManager;
22
+ protected readonly groupScheduler: GroupScheduler;
18
23
  protected readonly dependencyTracker: DependencyTracker;
19
24
  protected readonly temporalManager: TemporalManager;
20
25
  protected readonly waiterManager: WaiterManager;
@@ -2,6 +2,8 @@ import { DependencyTracker } from '../dependencyTracker';
2
2
  import { DlqShard } from '../dlqShard';
3
3
  import { LimiterManager } from '../limiterManager';
4
4
  import { IndexedPriorityQueue } from '../priorityQueue';
5
+ import { GroupLimiterManager } from '../groupLimiterManager';
6
+ import { GroupScheduler } from '../groupScheduler';
5
7
  import { ShardCounters } from '../shardCounters';
6
8
  import { TemporalManager } from '../temporalManager';
7
9
  import { UniqueKeyManager } from '../uniqueKeyManager';
@@ -10,9 +12,12 @@ import { WaiterManager } from '../waiterManager';
10
12
  export class ShardState {
11
13
  queues = new Map();
12
14
  activeGroups = new Map();
15
+ activeGroupCounts = new Map();
13
16
  uniqueKeyManager = new UniqueKeyManager();
14
17
  dlqManager;
15
18
  limiterManager = new LimiterManager();
19
+ groupLimiterManager = new GroupLimiterManager();
20
+ groupScheduler = new GroupScheduler();
16
21
  dependencyTracker = new DependencyTracker();
17
22
  temporalManager = new TemporalManager();
18
23
  waiterManager = new WaiterManager();
@@ -49,7 +54,10 @@ export class ShardState {
49
54
  getQueue(name) {
50
55
  let queue = this.queues.get(name);
51
56
  if (!queue) {
52
- queue = new IndexedPriorityQueue();
57
+ queue = new IndexedPriorityQueue({
58
+ onInsert: (job) => this.groupScheduler.insert(name, job, () => this.queues.get(name)?.values() ?? []),
59
+ onRemove: (job) => this.groupScheduler.remove(name, job),
60
+ });
53
61
  this.queues.set(name, queue);
54
62
  }
55
63
  return queue;
@@ -1,5 +1,6 @@
1
1
  import type { AtomicFlowJobInput } from '../flow';
2
2
  import type { JobInput } from '../job';
3
+ import type { GroupPullOptions } from '../group';
3
4
  import type { BaseCommand } from './base';
4
5
  export interface PushCommand extends BaseCommand {
5
6
  readonly cmd: 'PUSH';
@@ -72,6 +73,7 @@ export interface PullCommand extends BaseCommand {
72
73
  readonly owner?: string;
73
74
  readonly lockTtl?: number;
74
75
  readonly detach?: boolean;
76
+ readonly group?: GroupPullOptions;
75
77
  }
76
78
  export interface PullBatchCommand extends BaseCommand {
77
79
  readonly cmd: 'PULLB';
@@ -80,6 +82,7 @@ export interface PullBatchCommand extends BaseCommand {
80
82
  readonly timeout?: number;
81
83
  readonly owner?: string;
82
84
  readonly lockTtl?: number;
85
+ readonly group?: GroupPullOptions;
83
86
  }
84
87
  export interface AckCommand extends BaseCommand {
85
88
  readonly cmd: 'ACK';
@@ -52,3 +52,44 @@ export interface GetStallConfigCommand extends BaseCommand {
52
52
  readonly cmd: 'GetStallConfig';
53
53
  readonly queue: string;
54
54
  }
55
+ interface GroupCommand extends BaseCommand {
56
+ readonly queue: string;
57
+ readonly groupId: string;
58
+ }
59
+ export interface GetGroupJobsCountCommand extends GroupCommand {
60
+ readonly cmd: 'GetGroupJobsCount';
61
+ }
62
+ export interface GetGroupsJobsCountCommand extends BaseCommand {
63
+ readonly cmd: 'GetGroupsJobsCount';
64
+ readonly queue: string;
65
+ readonly maxCount?: number;
66
+ }
67
+ export interface GetGroupActiveCountCommand extends GroupCommand {
68
+ readonly cmd: 'GetGroupActiveCount';
69
+ }
70
+ export interface SetGroupRateLimitCommand extends GroupCommand {
71
+ readonly cmd: 'SetGroupRateLimit';
72
+ readonly max: number;
73
+ readonly duration: number;
74
+ }
75
+ export interface GetGroupRateLimitCommand extends GroupCommand {
76
+ readonly cmd: 'GetGroupRateLimit';
77
+ }
78
+ export interface RemoveGroupRateLimitCommand extends GroupCommand {
79
+ readonly cmd: 'RemoveGroupRateLimit';
80
+ }
81
+ export interface GetGroupRateLimitTtlCommand extends GroupCommand {
82
+ readonly cmd: 'GetGroupRateLimitTtl';
83
+ readonly maxJobs?: number;
84
+ }
85
+ export interface SetGroupConcurrencyCommand extends GroupCommand {
86
+ readonly cmd: 'SetGroupConcurrency';
87
+ readonly concurrency: number;
88
+ }
89
+ export interface GetGroupConcurrencyCommand extends GroupCommand {
90
+ readonly cmd: 'GetGroupConcurrency';
91
+ }
92
+ export interface RemoveGroupConcurrencyCommand extends GroupCommand {
93
+ readonly cmd: 'RemoveGroupConcurrency';
94
+ }
95
+ export {};
@@ -9,5 +9,5 @@ import type * as Monitoring from './monitoring';
9
9
  import type * as Protocol from './protocol';
10
10
  import type * as Query from './query';
11
11
  import type * as Workers from './workers';
12
- export type Command = Core.PushCommand | Core.PushBatchCommand | Core.PushFlowCommand | Core.PullCommand | Core.PullBatchCommand | Core.AckCommand | Core.AckBatchCommand | Core.FailCommand | Query.GetJobCommand | Query.GetStateCommand | Query.GetResultCommand | Query.GetJobsCommand | Query.GetJobCountsCommand | Query.GetCountsPerPriorityCommand | Query.GetJobByCustomIdCommand | Query.CountCommand | Query.GetProgressCommand | Query.GetChildrenValuesCommand | Management.CancelCommand | Management.ProgressCommand | Management.UpdateCommand | Management.ChangePriorityCommand | Management.PromoteCommand | Management.WaitJobCommand | Management.MoveToDelayedCommand | Management.DiscardCommand | Management.PauseCommand | Management.ResumeCommand | Management.IsPausedCommand | Management.DrainCommand | Management.ObliterateCommand | Management.ListQueuesCommand | Management.CleanCommand | Dlq.DlqCommand | Dlq.GetDlqStatsCommand | Dlq.RetryDlqCommand | Dlq.PurgeDlqCommand | Dlq.RemoveDlqJobCommand | Dlq.RetryCompletedCommand | Dlq.SetDlqConfigCommand | Dlq.GetDlqConfigCommand | Limits.GetQueueLimitsCommand | Limits.GetDeduplicationJobIdCommand | Limits.RemoveDeduplicationKeyCommand | Limits.RemoveJobDeduplicationKeyCommand | Limits.MoveToWaitingChildrenCommand | Limits.RateLimitCommand | Limits.SetConcurrencyCommand | Limits.RateLimitClearCommand | Limits.ClearConcurrencyCommand | Limits.SetStallConfigCommand | Limits.GetStallConfigCommand | Cron.CronCommand | Cron.CronDeleteCommand | Cron.CronListCommand | Cron.CronGetCommand | Monitoring.AddLogCommand | Monitoring.GetLogsCommand | Monitoring.AddWebhookCommand | Monitoring.RemoveWebhookCommand | Monitoring.ListWebhooksCommand | Monitoring.StatsCommand | Monitoring.MetricsCommand | Monitoring.TrimEventsCommand | Monitoring.PrometheusCommand | Monitoring.StorageStatusCommand | Workers.HeartbeatCommand | Workers.JobHeartbeatCommand | Workers.JobHeartbeatBatchCommand | Workers.PingCommand | Workers.RegisterWorkerCommand | Workers.UnregisterWorkerCommand | Workers.ListWorkersCommand | Extended.ClearLogsCommand | Extended.ExtendLockCommand | Extended.ExtendLocksCommand | Extended.ChangeDelayCommand | Extended.SetWebhookEnabledCommand | Extended.CompactMemoryCommand | Extended.UpdateParentCommand | Extended.GetFailedChildrenValuesCommand | Extended.GetIgnoredChildrenFailuresCommand | Extended.RemoveChildDependencyCommand | Extended.RemoveUnprocessedChildrenCommand | Extended.MoveToWaitCommand | Extended.PromoteJobsCommand | Dashboard.DashboardOverviewCommand | Dashboard.DashboardQueuesCommand | Dashboard.DashboardQueueCommand | Protocol.AuthCommand | Protocol.HelloCommand | Protocol.SubscribeEventsCommand | Protocol.UnsubscribeEventsCommand;
12
+ export type Command = Core.PushCommand | Core.PushBatchCommand | Core.PushFlowCommand | Core.PullCommand | Core.PullBatchCommand | Core.AckCommand | Core.AckBatchCommand | Core.FailCommand | Query.GetJobCommand | Query.GetStateCommand | Query.GetResultCommand | Query.GetJobsCommand | Query.GetJobCountsCommand | Query.GetCountsPerPriorityCommand | Query.GetJobByCustomIdCommand | Query.CountCommand | Query.GetProgressCommand | Query.GetChildrenValuesCommand | Management.CancelCommand | Management.ProgressCommand | Management.UpdateCommand | Management.ChangePriorityCommand | Management.PromoteCommand | Management.WaitJobCommand | Management.MoveToDelayedCommand | Management.DiscardCommand | Management.PauseCommand | Management.ResumeCommand | Management.IsPausedCommand | Management.DrainCommand | Management.ObliterateCommand | Management.ListQueuesCommand | Management.CleanCommand | Dlq.DlqCommand | Dlq.GetDlqStatsCommand | Dlq.RetryDlqCommand | Dlq.PurgeDlqCommand | Dlq.RemoveDlqJobCommand | Dlq.RetryCompletedCommand | Dlq.SetDlqConfigCommand | Dlq.GetDlqConfigCommand | Limits.GetQueueLimitsCommand | Limits.GetDeduplicationJobIdCommand | Limits.RemoveDeduplicationKeyCommand | Limits.RemoveJobDeduplicationKeyCommand | Limits.MoveToWaitingChildrenCommand | Limits.RateLimitCommand | Limits.SetConcurrencyCommand | Limits.RateLimitClearCommand | Limits.ClearConcurrencyCommand | Limits.SetStallConfigCommand | Limits.GetStallConfigCommand | Limits.GetGroupJobsCountCommand | Limits.GetGroupsJobsCountCommand | Limits.GetGroupActiveCountCommand | Limits.SetGroupRateLimitCommand | Limits.GetGroupRateLimitCommand | Limits.RemoveGroupRateLimitCommand | Limits.GetGroupRateLimitTtlCommand | Limits.SetGroupConcurrencyCommand | Limits.GetGroupConcurrencyCommand | Limits.RemoveGroupConcurrencyCommand | Cron.CronCommand | Cron.CronDeleteCommand | Cron.CronListCommand | Cron.CronGetCommand | Monitoring.AddLogCommand | Monitoring.GetLogsCommand | Monitoring.AddWebhookCommand | Monitoring.RemoveWebhookCommand | Monitoring.ListWebhooksCommand | Monitoring.StatsCommand | Monitoring.MetricsCommand | Monitoring.TrimEventsCommand | Monitoring.PrometheusCommand | Monitoring.StorageStatusCommand | Workers.HeartbeatCommand | Workers.JobHeartbeatCommand | Workers.JobHeartbeatBatchCommand | Workers.PingCommand | Workers.RegisterWorkerCommand | Workers.UnregisterWorkerCommand | Workers.ListWorkersCommand | Extended.ClearLogsCommand | Extended.ExtendLockCommand | Extended.ExtendLocksCommand | Extended.ChangeDelayCommand | Extended.SetWebhookEnabledCommand | Extended.CompactMemoryCommand | Extended.UpdateParentCommand | Extended.GetFailedChildrenValuesCommand | Extended.GetIgnoredChildrenFailuresCommand | Extended.RemoveChildDependencyCommand | Extended.RemoveUnprocessedChildrenCommand | Extended.MoveToWaitCommand | Extended.PromoteJobsCommand | Dashboard.DashboardOverviewCommand | Dashboard.DashboardQueuesCommand | Dashboard.DashboardQueueCommand | Protocol.AuthCommand | Protocol.HelloCommand | Protocol.SubscribeEventsCommand | Protocol.UnsubscribeEventsCommand;
13
13
  export type CommandType = Command['cmd'];
@@ -0,0 +1,21 @@
1
+ export interface GroupRateLimit {
2
+ readonly max: number;
3
+ readonly duration: number;
4
+ }
5
+ /** Worker-side defaults enforced authoritatively by the broker. */
6
+ export interface GroupPullOptions {
7
+ readonly concurrency?: number;
8
+ readonly limit?: GroupRateLimit;
9
+ }
10
+ export interface GroupRateLimitOverride extends GroupRateLimit {
11
+ }
12
+ /** Validate the canonical string form shared by every broker and persistence backend. */
13
+ export declare function validateGroupId(value: unknown, required?: boolean): string | null;
14
+ export declare function assertGroupId(value: unknown): asserts value is string;
15
+ export declare function assertOptionalGroupId(value: unknown): asserts value is string | undefined;
16
+ /** Normalize BullMQ-compatible numeric IDs without accepting arbitrary coercion. */
17
+ export declare function normalizeGroupId(value: unknown): string;
18
+ export declare function validatePositiveSafeInteger(value: unknown, name: string): string | null;
19
+ export declare function assertPositiveSafeInteger(value: unknown, name: string): asserts value is number;
20
+ export declare function validateGroupPullOptions(options: unknown): string | null;
21
+ export declare function assertGroupPullOptions(options: unknown): asserts options is GroupPullOptions;
@@ -0,0 +1,78 @@
1
+ /** Validate the canonical string form shared by every broker and persistence backend. */
2
+ export function validateGroupId(value, required = false) {
3
+ if (value === undefined)
4
+ return required ? 'groupId is required' : null;
5
+ if (typeof value !== 'string')
6
+ return 'groupId must be a string';
7
+ if (value.length === 0)
8
+ return 'groupId must be a non-empty string';
9
+ if (value.length > 256)
10
+ return 'groupId must be at most 256 characters';
11
+ if (value.includes('\0'))
12
+ return 'groupId must not contain NUL characters';
13
+ return null;
14
+ }
15
+ export function assertGroupId(value) {
16
+ const error = validateGroupId(value, true);
17
+ if (error)
18
+ throw new Error(error);
19
+ }
20
+ export function assertOptionalGroupId(value) {
21
+ const error = validateGroupId(value);
22
+ if (error)
23
+ throw new Error(error);
24
+ }
25
+ /** Normalize BullMQ-compatible numeric IDs without accepting arbitrary coercion. */
26
+ export function normalizeGroupId(value) {
27
+ if (typeof value !== 'string' && typeof value !== 'number') {
28
+ throw new Error('groupId must be a string or number');
29
+ }
30
+ if (typeof value === 'number' && !Number.isSafeInteger(value)) {
31
+ throw new Error('numeric groupId must be a safe integer');
32
+ }
33
+ const groupId = String(value);
34
+ assertGroupId(groupId);
35
+ return groupId;
36
+ }
37
+ export function validatePositiveSafeInteger(value, name) {
38
+ if (typeof value !== 'number' || !Number.isSafeInteger(value) || value <= 0) {
39
+ return `${name} must be a positive safe integer`;
40
+ }
41
+ return null;
42
+ }
43
+ export function assertPositiveSafeInteger(value, name) {
44
+ const error = validatePositiveSafeInteger(value, name);
45
+ if (error)
46
+ throw new Error(error);
47
+ }
48
+ export function validateGroupPullOptions(options) {
49
+ if (options === undefined)
50
+ return null;
51
+ if (!options || typeof options !== 'object' || Array.isArray(options)) {
52
+ return 'group must be an object';
53
+ }
54
+ const group = options;
55
+ if (group.concurrency !== undefined) {
56
+ const error = validatePositiveSafeInteger(group.concurrency, 'group.concurrency');
57
+ if (error)
58
+ return error;
59
+ }
60
+ if (group.limit !== undefined) {
61
+ if (!group.limit || typeof group.limit !== 'object' || Array.isArray(group.limit)) {
62
+ return 'group.limit must be an object';
63
+ }
64
+ const limit = group.limit;
65
+ const maxError = validatePositiveSafeInteger(limit.max, 'group.limit.max');
66
+ if (maxError)
67
+ return maxError;
68
+ const durationError = validatePositiveSafeInteger(limit.duration, 'group.limit.duration');
69
+ if (durationError)
70
+ return durationError;
71
+ }
72
+ return null;
73
+ }
74
+ export function assertGroupPullOptions(options) {
75
+ const error = validateGroupPullOptions(options);
76
+ if (error)
77
+ throw new Error(error);
78
+ }
@@ -5,6 +5,7 @@ export interface HeapEntry {
5
5
  runAt: number;
6
6
  lifo: boolean;
7
7
  generation: bigint;
8
+ groupFifoOrder?: bigint;
8
9
  }
9
10
  export interface IndexedJob {
10
11
  job: Job;
@@ -11,6 +11,7 @@ export interface StoredJobOptions {
11
11
  debounceId: string | null;
12
12
  debounceTtl: number | null;
13
13
  durable: boolean;
14
+ groupFifoOrder: string | null;
14
15
  }
15
16
  /** Persist generation policies that do not have dedicated legacy columns. */
16
17
  export declare function encodeJobOptions(job: Job): Uint8Array;