bunqueue 2.9.0 → 2.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -3
- package/dist/application/background/recovery/active.d.ts +1 -0
- package/dist/application/background/recovery/active.js +12 -0
- package/dist/application/background/recovery/index.js +3 -1
- package/dist/application/background/recovery/pending.js +1 -0
- package/dist/application/cleanupTasks.js +4 -16
- package/dist/application/contextFactory.js +3 -0
- package/dist/application/dlqManager.js +1 -0
- package/dist/application/dlqRetry.js +1 -0
- package/dist/application/eventsManager.d.ts +8 -1
- package/dist/application/eventsManager.js +76 -29
- package/dist/application/operations/ackHelpers.d.ts +1 -4
- package/dist/application/operations/ackHelpers.js +39 -14
- package/dist/application/operations/flowPush.js +1 -0
- package/dist/application/operations/flowValidation.js +4 -4
- package/dist/application/operations/pull.d.ts +7 -2
- package/dist/application/operations/pull.js +15 -57
- package/dist/application/operations/pullFinalization.d.ts +6 -0
- package/dist/application/operations/pullFinalization.js +69 -0
- package/dist/application/operations/pullStateTransition.d.ts +8 -7
- package/dist/application/operations/pullStateTransition.js +25 -18
- package/dist/application/operations/push.js +3 -0
- package/dist/application/operations/pushBatch.js +14 -7
- package/dist/application/operations/pushContext.d.ts +2 -1
- package/dist/application/postgres-queue-manager/control.d.ts +4 -0
- package/dist/application/postgres-queue-manager/control.js +24 -0
- package/dist/application/postgres-queue-manager/delivery.d.ts +5 -4
- package/dist/application/postgres-queue-manager/delivery.js +13 -10
- package/dist/application/postgres-queue-manager/groupQueries.d.ts +10 -0
- package/dist/application/postgres-queue-manager/groupQueries.js +40 -0
- package/dist/application/postgres-queue-manager/queries.d.ts +2 -2
- package/dist/application/postgres-queue-manager/queries.js +2 -2
- package/dist/application/queue-manager/control.js +1 -0
- package/dist/application/queue-manager/delivery.d.ts +5 -4
- package/dist/application/queue-manager/delivery.js +15 -8
- package/dist/application/queue-manager/limits.d.ts +11 -0
- package/dist/application/queue-manager/limits.js +61 -0
- package/dist/application/queue-manager/state.js +1 -1
- package/dist/application/queueTelemetryJournal.d.ts +2 -0
- package/dist/application/queueTelemetryJournal.js +23 -0
- package/dist/application/types/ack.d.ts +3 -1
- package/dist/cli/commandRouter.d.ts +1 -1
- package/dist/client/flowPush.js +6 -0
- package/dist/client/groupId.d.ts +1 -0
- package/dist/client/groupId.js +1 -0
- package/dist/client/index.d.ts +1 -1
- package/dist/client/jobHelpers.js +1 -0
- package/dist/client/queue/operations/add/bulk.js +4 -4
- package/dist/client/queue/operations/add/payload.d.ts +1 -0
- package/dist/client/queue/operations/add/payload.js +7 -1
- package/dist/client/queue/operations/add/single.js +3 -3
- package/dist/client/queue/operations/groups.d.ts +21 -0
- package/dist/client/queue/operations/groups.js +88 -0
- package/dist/client/queue/runtime/queries.d.ts +13 -0
- package/dist/client/queue/runtime/queries.js +31 -0
- package/dist/client/queue/runtime/state.d.ts +4 -0
- package/dist/client/queue/runtime/state.js +1 -0
- package/dist/client/types/index.d.ts +2 -2
- package/dist/client/types/options.d.ts +4 -0
- package/dist/client/types/worker.d.ts +8 -0
- package/dist/client/worker/runtime/execution.js +1 -0
- package/dist/client/worker/runtime/options.js +3 -0
- package/dist/client/worker/types/options.d.ts +2 -1
- package/dist/client/worker/workerPull.d.ts +2 -0
- package/dist/client/worker/workerPull.js +6 -4
- package/dist/domain/job/groupFifoOrder.d.ts +9 -0
- package/dist/domain/job/groupFifoOrder.js +26 -0
- package/dist/domain/queue/groupLimiterManager.d.ts +23 -0
- package/dist/domain/queue/groupLimiterManager.js +107 -0
- package/dist/domain/queue/groupScheduler.d.ts +31 -0
- package/dist/domain/queue/groupScheduler.js +256 -0
- package/dist/domain/queue/groupSchedulerOrder.d.ts +6 -0
- package/dist/domain/queue/groupSchedulerOrder.js +30 -0
- package/dist/domain/queue/priorityQueue.d.ts +13 -23
- package/dist/domain/queue/priorityQueue.js +53 -42
- package/dist/domain/queue/shard/keys.d.ts +1 -0
- package/dist/domain/queue/shard/keys.js +25 -2
- package/dist/domain/queue/shard/lifecycle.js +4 -0
- package/dist/domain/queue/shard/limits.d.ts +20 -1
- package/dist/domain/queue/shard/limits.js +56 -0
- package/dist/domain/queue/shard/state.d.ts +5 -0
- package/dist/domain/queue/shard/state.js +9 -1
- package/dist/domain/types/commands/core.d.ts +3 -0
- package/dist/domain/types/commands/limits.d.ts +41 -0
- package/dist/domain/types/commands/union.d.ts +1 -1
- package/dist/domain/types/group.d.ts +21 -0
- package/dist/domain/types/group.js +78 -0
- package/dist/domain/types/priorityQueue.d.ts +1 -0
- package/dist/infrastructure/persistence/jobOptionsBlob.d.ts +1 -0
- package/dist/infrastructure/persistence/jobOptionsBlob.js +10 -2
- package/dist/infrastructure/persistence/migrations.js +11 -0
- package/dist/infrastructure/persistence/postgres/admission.js +5 -2
- package/dist/infrastructure/persistence/postgres/batchAdmission.js +22 -4
- package/dist/infrastructure/persistence/postgres/claimBatch.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/claimBatch.js +16 -3
- package/dist/infrastructure/persistence/postgres/claimSelection.d.ts +8 -1
- package/dist/infrastructure/persistence/postgres/claimSelection.js +73 -131
- package/dist/infrastructure/persistence/postgres/claims.d.ts +8 -1
- package/dist/infrastructure/persistence/postgres/claims.js +19 -3
- package/dist/infrastructure/persistence/postgres/eventCatchupCursors.d.ts +32 -0
- package/dist/infrastructure/persistence/postgres/eventCatchupCursors.js +66 -0
- package/dist/infrastructure/persistence/postgres/eventJournalSchema.d.ts +2 -2
- package/dist/infrastructure/persistence/postgres/eventJournalSchema.js +16 -2
- package/dist/infrastructure/persistence/postgres/eventPruneWatermarks.d.ts +2 -0
- package/dist/infrastructure/persistence/postgres/eventPruneWatermarks.js +21 -3
- package/dist/infrastructure/persistence/postgres/events.d.ts +3 -3
- package/dist/infrastructure/persistence/postgres/events.js +44 -17
- package/dist/infrastructure/persistence/postgres/groupClaims.d.ts +15 -0
- package/dist/infrastructure/persistence/postgres/groupClaims.js +95 -0
- package/dist/infrastructure/persistence/postgres/groupSchema.d.ts +2 -0
- package/dist/infrastructure/persistence/postgres/groupSchema.js +155 -0
- package/dist/infrastructure/persistence/postgres/groupSchemaFingerprint.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/groupSchemaFingerprint.js +127 -0
- package/dist/infrastructure/persistence/postgres/groupStateRetention.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/groupStateRetention.js +93 -0
- package/dist/infrastructure/persistence/postgres/groups.d.ts +11 -0
- package/dist/infrastructure/persistence/postgres/groups.js +117 -0
- package/dist/infrastructure/persistence/postgres/queueDestruction.js +4 -0
- package/dist/infrastructure/persistence/postgres/runtime.d.ts +0 -1
- package/dist/infrastructure/persistence/postgres/runtime.js +3 -1
- package/dist/infrastructure/persistence/postgres/schema.d.ts +2 -2
- package/dist/infrastructure/persistence/postgres/schema.js +8 -4
- package/dist/infrastructure/persistence/postgres/schemaInitialization.js +6 -1
- package/dist/infrastructure/persistence/postgres/store.d.ts +11 -1
- package/dist/infrastructure/persistence/postgres/store.js +47 -3
- package/dist/infrastructure/persistence/postgres/telemetry.d.ts +2 -0
- package/dist/infrastructure/persistence/postgres/telemetry.js +33 -21
- package/dist/infrastructure/persistence/postgres/types.d.ts +1 -0
- package/dist/infrastructure/persistence/schema.d.ts +2 -2
- package/dist/infrastructure/persistence/schema.js +12 -1
- package/dist/infrastructure/persistence/sqlite/control.d.ts +14 -0
- package/dist/infrastructure/persistence/sqlite/control.js +69 -0
- package/dist/infrastructure/persistence/sqlite/jobs.d.ts +12 -0
- package/dist/infrastructure/persistence/sqlite/jobs.js +36 -0
- package/dist/infrastructure/persistence/sqlite/telemetry.d.ts +2 -1
- package/dist/infrastructure/persistence/sqlite/telemetry.js +5 -64
- package/dist/infrastructure/persistence/sqlite/telemetryWrites.d.ts +4 -0
- package/dist/infrastructure/persistence/sqlite/telemetryWrites.js +123 -0
- package/dist/infrastructure/persistence/sqliteSerializer.js +2 -0
- package/dist/infrastructure/server/handler-routes/control.js +13 -0
- package/dist/infrastructure/server/handlers/core.d.ts +0 -5
- package/dist/infrastructure/server/handlers/core.js +15 -18
- package/dist/infrastructure/server/handlers/groups.d.ts +11 -0
- package/dist/infrastructure/server/handlers/groups.js +55 -0
- package/dist/infrastructure/server/handlers/pushBatchValidation.js +4 -1
- package/dist/infrastructure/server/protocol/validation.d.ts +1 -0
- package/dist/infrastructure/server/protocol/validation.js +1 -0
- package/dist/infrastructure/server/protocol.d.ts +1 -1
- package/dist/infrastructure/server/protocol.js +1 -1
- package/package.json +2 -1
|
@@ -1,36 +1,29 @@
|
|
|
1
|
-
|
|
2
|
-
* Indexed Priority Queue
|
|
3
|
-
* Combines binary heap for O(log n) priority operations
|
|
4
|
-
* with Map for O(1) lookups by job ID
|
|
5
|
-
*/
|
|
1
|
+
import { copyGroupFifoOrder } from '../job/groupFifoOrder';
|
|
6
2
|
import { comparePriorityEntries } from './priorityQueueOrder';
|
|
7
|
-
/**
|
|
8
|
-
* Indexed Priority Queue implementation with 4-ary heap
|
|
9
|
-
* 4-ary provides better cache locality than binary heap
|
|
10
|
-
* O(log₄ n) push, pop, update
|
|
11
|
-
* O(1) find, has
|
|
12
|
-
*/
|
|
3
|
+
/** Indexed map-backed 4-ary heap with O(log4 n) mutations and O(1) lookup. */
|
|
13
4
|
export class IndexedPriorityQueue {
|
|
14
|
-
|
|
5
|
+
hooks;
|
|
6
|
+
compare;
|
|
7
|
+
order;
|
|
15
8
|
static D = 4;
|
|
16
9
|
heap = [];
|
|
17
10
|
index = new Map();
|
|
18
|
-
// Use BigInt to prevent overflow at extreme throughput
|
|
19
11
|
generation = 0n;
|
|
20
|
-
|
|
12
|
+
hooksActive = false;
|
|
13
|
+
constructor(hooks, compare = comparePriorityEntries, order) {
|
|
14
|
+
this.hooks = hooks;
|
|
15
|
+
this.compare = compare;
|
|
16
|
+
this.order = order;
|
|
17
|
+
}
|
|
21
18
|
get size() {
|
|
22
19
|
return this.index.size;
|
|
23
20
|
}
|
|
24
|
-
/** Check if empty */
|
|
25
21
|
get isEmpty() {
|
|
26
22
|
return this.index.size === 0;
|
|
27
23
|
}
|
|
28
|
-
/** Push a job into the queue */
|
|
29
24
|
push(job) {
|
|
30
25
|
const gen = this.generation++;
|
|
31
|
-
// Store in index
|
|
32
26
|
this.index.set(job.id, { job, generation: gen });
|
|
33
|
-
// Add to heap
|
|
34
27
|
const entry = {
|
|
35
28
|
jobId: job.id,
|
|
36
29
|
priority: job.priority,
|
|
@@ -38,32 +31,38 @@ export class IndexedPriorityQueue {
|
|
|
38
31
|
lifo: job.lifo,
|
|
39
32
|
generation: gen,
|
|
40
33
|
};
|
|
34
|
+
if (this.order) {
|
|
35
|
+
const groupFifoOrder = this.order(job);
|
|
36
|
+
if (groupFifoOrder !== null)
|
|
37
|
+
entry.groupFifoOrder = groupFifoOrder;
|
|
38
|
+
}
|
|
41
39
|
this.heap.push(entry);
|
|
42
40
|
this.bubbleUp(this.heap.length - 1);
|
|
41
|
+
if (this.hooks && (this.hooksActive || job.groupId)) {
|
|
42
|
+
this.hooksActive = this.hooks.onInsert(job);
|
|
43
|
+
}
|
|
43
44
|
}
|
|
44
|
-
/** Pop the highest priority job */
|
|
45
45
|
pop() {
|
|
46
46
|
while (this.heap.length > 0) {
|
|
47
47
|
const entry = this.heap[0];
|
|
48
48
|
const indexed = this.index.get(entry.jobId);
|
|
49
|
-
// Skip stale entries (generation mismatch = updated or removed)
|
|
50
49
|
if (indexed?.generation !== entry.generation) {
|
|
51
50
|
this.removeTop();
|
|
52
51
|
continue;
|
|
53
52
|
}
|
|
54
|
-
// Remove from both structures
|
|
55
53
|
this.removeTop();
|
|
56
54
|
this.index.delete(entry.jobId);
|
|
55
|
+
if (this.hooks && this.hooksActive) {
|
|
56
|
+
this.hooksActive = this.hooks.onRemove(indexed.job);
|
|
57
|
+
}
|
|
57
58
|
return indexed.job;
|
|
58
59
|
}
|
|
59
60
|
return null;
|
|
60
61
|
}
|
|
61
|
-
/** Peek at the highest priority job without removing */
|
|
62
62
|
peek() {
|
|
63
63
|
while (this.heap.length > 0) {
|
|
64
64
|
const entry = this.heap[0];
|
|
65
65
|
const indexed = this.index.get(entry.jobId);
|
|
66
|
-
// Skip stale entries
|
|
67
66
|
if (indexed?.generation !== entry.generation) {
|
|
68
67
|
this.removeTop();
|
|
69
68
|
continue;
|
|
@@ -72,21 +71,20 @@ export class IndexedPriorityQueue {
|
|
|
72
71
|
}
|
|
73
72
|
return null;
|
|
74
73
|
}
|
|
75
|
-
/** Find a job by ID - O(1) */
|
|
76
74
|
find(jobId) {
|
|
77
75
|
return this.index.get(jobId)?.job ?? null;
|
|
78
76
|
}
|
|
79
|
-
/** Check if job exists - O(1) */
|
|
80
77
|
has(jobId) {
|
|
81
78
|
return this.index.has(jobId);
|
|
82
79
|
}
|
|
83
|
-
/** Remove a job by ID - O(1) for index, heap cleans lazily */
|
|
84
80
|
remove(jobId) {
|
|
85
81
|
const indexed = this.index.get(jobId);
|
|
86
82
|
if (!indexed)
|
|
87
83
|
return null;
|
|
88
84
|
this.index.delete(jobId);
|
|
89
|
-
|
|
85
|
+
if (this.hooks && this.hooksActive) {
|
|
86
|
+
this.hooksActive = this.hooks.onRemove(indexed.job);
|
|
87
|
+
}
|
|
90
88
|
return indexed.job;
|
|
91
89
|
}
|
|
92
90
|
/** Update job priority - O(log n). Optionally update the LIFO tie-break flag. */
|
|
@@ -95,15 +93,13 @@ export class IndexedPriorityQueue {
|
|
|
95
93
|
if (!indexed)
|
|
96
94
|
return false;
|
|
97
95
|
const lifo = newLifo ?? indexed.job.lifo;
|
|
98
|
-
// Create new job with updated priority (immutable pattern)
|
|
99
96
|
const updatedJob = {
|
|
100
97
|
...indexed.job,
|
|
101
98
|
priority: newPriority,
|
|
102
99
|
lifo,
|
|
103
100
|
};
|
|
104
|
-
|
|
101
|
+
copyGroupFifoOrder(indexed.job, updatedJob);
|
|
105
102
|
const gen = this.generation++;
|
|
106
|
-
// Update index with new job and generation
|
|
107
103
|
this.index.set(jobId, { job: updatedJob, generation: gen });
|
|
108
104
|
const entry = {
|
|
109
105
|
jobId: updatedJob.id,
|
|
@@ -112,8 +108,16 @@ export class IndexedPriorityQueue {
|
|
|
112
108
|
lifo,
|
|
113
109
|
generation: gen,
|
|
114
110
|
};
|
|
111
|
+
if (this.order) {
|
|
112
|
+
const groupFifoOrder = this.order(updatedJob);
|
|
113
|
+
if (groupFifoOrder !== null)
|
|
114
|
+
entry.groupFifoOrder = groupFifoOrder;
|
|
115
|
+
}
|
|
115
116
|
this.heap.push(entry);
|
|
116
117
|
this.bubbleUp(this.heap.length - 1);
|
|
118
|
+
if (this.hooks && (this.hooksActive || updatedJob.groupId)) {
|
|
119
|
+
this.hooksActive = this.hooks.onInsert(updatedJob);
|
|
120
|
+
}
|
|
117
121
|
return true;
|
|
118
122
|
}
|
|
119
123
|
/** Update job runAt (for delay changes) - O(log n) */
|
|
@@ -121,14 +125,12 @@ export class IndexedPriorityQueue {
|
|
|
121
125
|
const indexed = this.index.get(jobId);
|
|
122
126
|
if (!indexed)
|
|
123
127
|
return false;
|
|
124
|
-
// Create new job with updated runAt (immutable pattern)
|
|
125
128
|
const updatedJob = {
|
|
126
129
|
...indexed.job,
|
|
127
130
|
runAt: newRunAt,
|
|
128
131
|
};
|
|
129
|
-
|
|
132
|
+
copyGroupFifoOrder(indexed.job, updatedJob);
|
|
130
133
|
const gen = this.generation++;
|
|
131
|
-
// Update index with new job and generation
|
|
132
134
|
this.index.set(jobId, { job: updatedJob, generation: gen });
|
|
133
135
|
const entry = {
|
|
134
136
|
jobId: updatedJob.id,
|
|
@@ -137,19 +139,33 @@ export class IndexedPriorityQueue {
|
|
|
137
139
|
lifo: updatedJob.lifo,
|
|
138
140
|
generation: gen,
|
|
139
141
|
};
|
|
142
|
+
if (this.order) {
|
|
143
|
+
const groupFifoOrder = this.order(updatedJob);
|
|
144
|
+
if (groupFifoOrder !== null)
|
|
145
|
+
entry.groupFifoOrder = groupFifoOrder;
|
|
146
|
+
}
|
|
140
147
|
this.heap.push(entry);
|
|
141
148
|
this.bubbleUp(this.heap.length - 1);
|
|
149
|
+
if (this.hooks && (this.hooksActive || updatedJob.groupId)) {
|
|
150
|
+
this.hooksActive = this.hooks.onInsert(updatedJob);
|
|
151
|
+
}
|
|
142
152
|
return true;
|
|
143
153
|
}
|
|
144
|
-
/** Get all jobs (for iteration) */
|
|
145
154
|
values() {
|
|
146
155
|
return Array.from(this.index.values()).map((v) => v.job);
|
|
147
156
|
}
|
|
148
|
-
/** Clear the queue */
|
|
149
157
|
clear() {
|
|
158
|
+
if (this.hooks) {
|
|
159
|
+
for (const { job } of this.index.values()) {
|
|
160
|
+
if (!this.hooksActive)
|
|
161
|
+
break;
|
|
162
|
+
this.hooksActive = this.hooks.onRemove(job);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
150
165
|
this.heap = [];
|
|
151
166
|
this.index.clear();
|
|
152
167
|
this.generation = 0n;
|
|
168
|
+
this.hooksActive = false;
|
|
153
169
|
}
|
|
154
170
|
/**
|
|
155
171
|
* Get the ratio of stale entries in the heap
|
|
@@ -180,11 +196,8 @@ export class IndexedPriorityQueue {
|
|
|
180
196
|
this.heap = validEntries;
|
|
181
197
|
this.heapify();
|
|
182
198
|
}
|
|
183
|
-
/** Rebuild heap property from arbitrary array - O(n) */
|
|
184
199
|
heapify() {
|
|
185
200
|
const D = IndexedPriorityQueue.D;
|
|
186
|
-
// Start from last non-leaf node and bubble down
|
|
187
|
-
// In D-ary heap, last non-leaf is at floor((n-2)/D)
|
|
188
201
|
for (let i = Math.floor((this.heap.length - 2) / D); i >= 0; i--) {
|
|
189
202
|
this.bubbleDown(i);
|
|
190
203
|
}
|
|
@@ -193,7 +206,6 @@ export class IndexedPriorityQueue {
|
|
|
193
206
|
needsCompaction(threshold = 0.2) {
|
|
194
207
|
return this.getStaleRatio() > threshold;
|
|
195
208
|
}
|
|
196
|
-
// ============ Heap Operations ============
|
|
197
209
|
removeTop() {
|
|
198
210
|
if (this.heap.length <= 1) {
|
|
199
211
|
this.heap.pop();
|
|
@@ -207,7 +219,7 @@ export class IndexedPriorityQueue {
|
|
|
207
219
|
const D = IndexedPriorityQueue.D;
|
|
208
220
|
while (idx > 0) {
|
|
209
221
|
const parentIdx = Math.floor((idx - 1) / D);
|
|
210
|
-
if (
|
|
222
|
+
if (this.compare(this.heap[idx], this.heap[parentIdx]) >= 0) {
|
|
211
223
|
break;
|
|
212
224
|
}
|
|
213
225
|
this.swap(idx, parentIdx);
|
|
@@ -223,11 +235,10 @@ export class IndexedPriorityQueue {
|
|
|
223
235
|
const firstChild = D * idx + 1;
|
|
224
236
|
if (firstChild >= length)
|
|
225
237
|
break;
|
|
226
|
-
// Find minimum among up to D children (cache-friendly sequential access)
|
|
227
238
|
let smallest = idx;
|
|
228
239
|
const lastChild = Math.min(firstChild + D, length);
|
|
229
240
|
for (let c = firstChild; c < lastChild; c++) {
|
|
230
|
-
if (
|
|
241
|
+
if (this.compare(heap[c], heap[smallest]) < 0) {
|
|
231
242
|
smallest = c;
|
|
232
243
|
}
|
|
233
244
|
}
|
|
@@ -15,4 +15,5 @@ export declare class ShardKeys extends ShardState {
|
|
|
15
15
|
isGroupActive(queue: string, groupId: string): boolean;
|
|
16
16
|
activateGroup(queue: string, groupId: string): void;
|
|
17
17
|
releaseGroup(queue: string, groupId: string): void;
|
|
18
|
+
getGroupActiveCount(queue: string, groupId: string): number;
|
|
18
19
|
}
|
|
@@ -29,7 +29,7 @@ export class ShardKeys extends ShardState {
|
|
|
29
29
|
return this.uniqueKeyManager.getMap();
|
|
30
30
|
}
|
|
31
31
|
isGroupActive(queue, groupId) {
|
|
32
|
-
return this.
|
|
32
|
+
return (this.activeGroupCounts.get(queue)?.get(groupId) ?? 0) > 0;
|
|
33
33
|
}
|
|
34
34
|
activateGroup(queue, groupId) {
|
|
35
35
|
let groups = this.activeGroups.get(queue);
|
|
@@ -38,8 +38,31 @@ export class ShardKeys extends ShardState {
|
|
|
38
38
|
this.activeGroups.set(queue, groups);
|
|
39
39
|
}
|
|
40
40
|
groups.add(groupId);
|
|
41
|
+
let counts = this.activeGroupCounts.get(queue);
|
|
42
|
+
if (!counts) {
|
|
43
|
+
counts = new Map();
|
|
44
|
+
this.activeGroupCounts.set(queue, counts);
|
|
45
|
+
}
|
|
46
|
+
counts.set(groupId, (counts.get(groupId) ?? 0) + 1);
|
|
41
47
|
}
|
|
42
48
|
releaseGroup(queue, groupId) {
|
|
43
|
-
this.
|
|
49
|
+
const counts = this.activeGroupCounts.get(queue);
|
|
50
|
+
const count = (counts?.get(groupId) ?? 0) - 1;
|
|
51
|
+
if (count > 0) {
|
|
52
|
+
counts?.set(groupId, count);
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
counts?.delete(groupId);
|
|
56
|
+
const groups = this.activeGroups.get(queue);
|
|
57
|
+
groups?.delete(groupId);
|
|
58
|
+
if (groups?.size === 0)
|
|
59
|
+
this.activeGroups.delete(queue);
|
|
60
|
+
}
|
|
61
|
+
if (counts?.size === 0)
|
|
62
|
+
this.activeGroupCounts.delete(queue);
|
|
63
|
+
this.notify(queue);
|
|
64
|
+
}
|
|
65
|
+
getGroupActiveCount(queue, groupId) {
|
|
66
|
+
return this.activeGroupCounts.get(queue)?.get(groupId) ?? 0;
|
|
44
67
|
}
|
|
45
68
|
}
|
|
@@ -12,6 +12,7 @@ export class ShardLifecycle extends ShardMetrics {
|
|
|
12
12
|
this.temporalManager.removeDelayed(job.id);
|
|
13
13
|
}
|
|
14
14
|
priorityQueue.clear();
|
|
15
|
+
this.groupScheduler.clearQueue(queue);
|
|
15
16
|
this.temporalManager.clearIndexForQueue(queue);
|
|
16
17
|
this.counters.adjustQueued(-count);
|
|
17
18
|
this.counters.syncDelayedCount();
|
|
@@ -37,7 +38,10 @@ export class ShardLifecycle extends ShardMetrics {
|
|
|
37
38
|
this.queues.delete(queue);
|
|
38
39
|
this.uniqueKeyManager.clearQueue(queue);
|
|
39
40
|
this.limiterManager.deleteQueue(queue);
|
|
41
|
+
this.groupScheduler.clearQueue(queue);
|
|
42
|
+
this.groupLimiterManager.clearQueue(queue);
|
|
40
43
|
this.activeGroups.delete(queue);
|
|
44
|
+
this.activeGroupCounts.delete(queue);
|
|
41
45
|
return Array.from(removed);
|
|
42
46
|
}
|
|
43
47
|
}
|
|
@@ -1,8 +1,27 @@
|
|
|
1
|
-
import type { JobId } from '../../types/job';
|
|
1
|
+
import type { Job, JobId } from '../../types/job';
|
|
2
|
+
import type { GroupPullOptions, GroupRateLimitOverride } from '../../types/group';
|
|
3
|
+
import type { GroupCandidate } from '../groupScheduler';
|
|
2
4
|
import type { QueueState } from '../../types/queue';
|
|
3
5
|
import { ShardKeys } from './keys';
|
|
4
6
|
/** Queue rate, concurrency, and resource ownership operations. */
|
|
5
7
|
export declare class ShardLimits extends ShardKeys {
|
|
8
|
+
hasGroupScheduler(queue: string): boolean;
|
|
9
|
+
assignGroupFifoOrder(job: Job): void;
|
|
10
|
+
observeGroupFifoOrder(job: Job): void;
|
|
11
|
+
peekGroupCandidate(queue: string, now: number, options?: GroupPullOptions): GroupCandidate;
|
|
12
|
+
acquireGroup(queue: string, groupId: string, options?: GroupPullOptions, now?: number): boolean;
|
|
13
|
+
advanceGroup(queue: string, groupId: string): void;
|
|
14
|
+
getGroupJobsCount(queue: string, groupId: string): number;
|
|
15
|
+
getGroupsJobsCount(queue: string): number;
|
|
16
|
+
cleanExpiredGroupWindows(now: number): void;
|
|
17
|
+
clearEmptyGroupRuntime(queue: string): void;
|
|
18
|
+
setGroupRateLimit(queue: string, groupId: string, max: number, duration: number): void;
|
|
19
|
+
getGroupRateLimit(queue: string, groupId: string): GroupRateLimitOverride | null;
|
|
20
|
+
removeGroupRateLimit(queue: string, groupId: string): number;
|
|
21
|
+
getGroupRateLimitTtl(queue: string, groupId: string, maxJobs?: number): number;
|
|
22
|
+
setGroupConcurrency(queue: string, groupId: string, concurrency: number): void;
|
|
23
|
+
getGroupConcurrency(queue: string, groupId: string): number | null;
|
|
24
|
+
removeGroupConcurrency(queue: string, groupId: string): number;
|
|
6
25
|
setRateLimit(queue: string, limit: number, durationMs?: number, ttlMs?: number): void;
|
|
7
26
|
clearRateLimit(queue: string): void;
|
|
8
27
|
expireRateLimitIfNeeded(queue: string): void;
|
|
@@ -1,6 +1,62 @@
|
|
|
1
1
|
import { ShardKeys } from './keys';
|
|
2
2
|
/** Queue rate, concurrency, and resource ownership operations. */
|
|
3
3
|
export class ShardLimits extends ShardKeys {
|
|
4
|
+
hasGroupScheduler(queue) {
|
|
5
|
+
return this.groupScheduler.isActive(queue);
|
|
6
|
+
}
|
|
7
|
+
assignGroupFifoOrder(job) {
|
|
8
|
+
this.groupScheduler.assignFifoOrder(job);
|
|
9
|
+
}
|
|
10
|
+
observeGroupFifoOrder(job) {
|
|
11
|
+
this.groupScheduler.observeFifoOrder(job);
|
|
12
|
+
}
|
|
13
|
+
peekGroupCandidate(queue, now, options) {
|
|
14
|
+
return this.groupScheduler.peek(queue, now, (groupId) => this.groupLimiterManager.status(queue, groupId, options, this.getGroupActiveCount(queue, groupId), now));
|
|
15
|
+
}
|
|
16
|
+
acquireGroup(queue, groupId, options, now) {
|
|
17
|
+
const acquired = this.groupLimiterManager.acquire(queue, groupId, options, this.getGroupActiveCount(queue, groupId), now);
|
|
18
|
+
if (acquired)
|
|
19
|
+
this.activateGroup(queue, groupId);
|
|
20
|
+
return acquired;
|
|
21
|
+
}
|
|
22
|
+
advanceGroup(queue, groupId) {
|
|
23
|
+
this.groupScheduler.advance(queue, groupId);
|
|
24
|
+
}
|
|
25
|
+
getGroupJobsCount(queue, groupId) {
|
|
26
|
+
return this.groupScheduler.getGroupJobsCount(queue, groupId);
|
|
27
|
+
}
|
|
28
|
+
getGroupsJobsCount(queue) {
|
|
29
|
+
return this.groupScheduler.getGroupsJobsCount(queue);
|
|
30
|
+
}
|
|
31
|
+
cleanExpiredGroupWindows(now) {
|
|
32
|
+
this.groupLimiterManager.pruneExpiredWindows(now);
|
|
33
|
+
}
|
|
34
|
+
clearEmptyGroupRuntime(queue) {
|
|
35
|
+
this.groupScheduler.clearQueue(queue);
|
|
36
|
+
this.activeGroups.delete(queue);
|
|
37
|
+
this.activeGroupCounts.delete(queue);
|
|
38
|
+
}
|
|
39
|
+
setGroupRateLimit(queue, groupId, max, duration) {
|
|
40
|
+
this.groupLimiterManager.setRateLimit(queue, groupId, max, duration);
|
|
41
|
+
}
|
|
42
|
+
getGroupRateLimit(queue, groupId) {
|
|
43
|
+
return this.groupLimiterManager.getRateLimit(queue, groupId);
|
|
44
|
+
}
|
|
45
|
+
removeGroupRateLimit(queue, groupId) {
|
|
46
|
+
return this.groupLimiterManager.removeRateLimit(queue, groupId);
|
|
47
|
+
}
|
|
48
|
+
getGroupRateLimitTtl(queue, groupId, maxJobs) {
|
|
49
|
+
return this.groupLimiterManager.getRateLimitTtl(queue, groupId, maxJobs);
|
|
50
|
+
}
|
|
51
|
+
setGroupConcurrency(queue, groupId, concurrency) {
|
|
52
|
+
this.groupLimiterManager.setConcurrency(queue, groupId, concurrency);
|
|
53
|
+
}
|
|
54
|
+
getGroupConcurrency(queue, groupId) {
|
|
55
|
+
return this.groupLimiterManager.getConcurrency(queue, groupId);
|
|
56
|
+
}
|
|
57
|
+
removeGroupConcurrency(queue, groupId) {
|
|
58
|
+
return this.groupLimiterManager.removeConcurrency(queue, groupId);
|
|
59
|
+
}
|
|
4
60
|
setRateLimit(queue, limit, durationMs, ttlMs) {
|
|
5
61
|
this.limiterManager.setRateLimit(queue, limit, durationMs, ttlMs);
|
|
6
62
|
}
|
|
@@ -4,6 +4,8 @@ import { DependencyTracker } from '../dependencyTracker';
|
|
|
4
4
|
import { DlqShard } from '../dlqShard';
|
|
5
5
|
import { LimiterManager } from '../limiterManager';
|
|
6
6
|
import { IndexedPriorityQueue } from '../priorityQueue';
|
|
7
|
+
import { GroupLimiterManager } from '../groupLimiterManager';
|
|
8
|
+
import { GroupScheduler } from '../groupScheduler';
|
|
7
9
|
import { ShardCounters } from '../shardCounters';
|
|
8
10
|
import { TemporalManager } from '../temporalManager';
|
|
9
11
|
import { UniqueKeyManager } from '../uniqueKeyManager';
|
|
@@ -12,9 +14,12 @@ import { WaiterManager } from '../waiterManager';
|
|
|
12
14
|
export declare class ShardState {
|
|
13
15
|
readonly queues: Map<string, IndexedPriorityQueue>;
|
|
14
16
|
readonly activeGroups: Map<string, Set<string>>;
|
|
17
|
+
readonly activeGroupCounts: Map<string, Map<string, number>>;
|
|
15
18
|
protected readonly uniqueKeyManager: UniqueKeyManager;
|
|
16
19
|
protected readonly dlqManager: DlqShard;
|
|
17
20
|
protected readonly limiterManager: LimiterManager;
|
|
21
|
+
protected readonly groupLimiterManager: GroupLimiterManager;
|
|
22
|
+
protected readonly groupScheduler: GroupScheduler;
|
|
18
23
|
protected readonly dependencyTracker: DependencyTracker;
|
|
19
24
|
protected readonly temporalManager: TemporalManager;
|
|
20
25
|
protected readonly waiterManager: WaiterManager;
|
|
@@ -2,6 +2,8 @@ import { DependencyTracker } from '../dependencyTracker';
|
|
|
2
2
|
import { DlqShard } from '../dlqShard';
|
|
3
3
|
import { LimiterManager } from '../limiterManager';
|
|
4
4
|
import { IndexedPriorityQueue } from '../priorityQueue';
|
|
5
|
+
import { GroupLimiterManager } from '../groupLimiterManager';
|
|
6
|
+
import { GroupScheduler } from '../groupScheduler';
|
|
5
7
|
import { ShardCounters } from '../shardCounters';
|
|
6
8
|
import { TemporalManager } from '../temporalManager';
|
|
7
9
|
import { UniqueKeyManager } from '../uniqueKeyManager';
|
|
@@ -10,9 +12,12 @@ import { WaiterManager } from '../waiterManager';
|
|
|
10
12
|
export class ShardState {
|
|
11
13
|
queues = new Map();
|
|
12
14
|
activeGroups = new Map();
|
|
15
|
+
activeGroupCounts = new Map();
|
|
13
16
|
uniqueKeyManager = new UniqueKeyManager();
|
|
14
17
|
dlqManager;
|
|
15
18
|
limiterManager = new LimiterManager();
|
|
19
|
+
groupLimiterManager = new GroupLimiterManager();
|
|
20
|
+
groupScheduler = new GroupScheduler();
|
|
16
21
|
dependencyTracker = new DependencyTracker();
|
|
17
22
|
temporalManager = new TemporalManager();
|
|
18
23
|
waiterManager = new WaiterManager();
|
|
@@ -49,7 +54,10 @@ export class ShardState {
|
|
|
49
54
|
getQueue(name) {
|
|
50
55
|
let queue = this.queues.get(name);
|
|
51
56
|
if (!queue) {
|
|
52
|
-
queue = new IndexedPriorityQueue(
|
|
57
|
+
queue = new IndexedPriorityQueue({
|
|
58
|
+
onInsert: (job) => this.groupScheduler.insert(name, job, () => this.queues.get(name)?.values() ?? []),
|
|
59
|
+
onRemove: (job) => this.groupScheduler.remove(name, job),
|
|
60
|
+
});
|
|
53
61
|
this.queues.set(name, queue);
|
|
54
62
|
}
|
|
55
63
|
return queue;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { AtomicFlowJobInput } from '../flow';
|
|
2
2
|
import type { JobInput } from '../job';
|
|
3
|
+
import type { GroupPullOptions } from '../group';
|
|
3
4
|
import type { BaseCommand } from './base';
|
|
4
5
|
export interface PushCommand extends BaseCommand {
|
|
5
6
|
readonly cmd: 'PUSH';
|
|
@@ -72,6 +73,7 @@ export interface PullCommand extends BaseCommand {
|
|
|
72
73
|
readonly owner?: string;
|
|
73
74
|
readonly lockTtl?: number;
|
|
74
75
|
readonly detach?: boolean;
|
|
76
|
+
readonly group?: GroupPullOptions;
|
|
75
77
|
}
|
|
76
78
|
export interface PullBatchCommand extends BaseCommand {
|
|
77
79
|
readonly cmd: 'PULLB';
|
|
@@ -80,6 +82,7 @@ export interface PullBatchCommand extends BaseCommand {
|
|
|
80
82
|
readonly timeout?: number;
|
|
81
83
|
readonly owner?: string;
|
|
82
84
|
readonly lockTtl?: number;
|
|
85
|
+
readonly group?: GroupPullOptions;
|
|
83
86
|
}
|
|
84
87
|
export interface AckCommand extends BaseCommand {
|
|
85
88
|
readonly cmd: 'ACK';
|
|
@@ -52,3 +52,44 @@ export interface GetStallConfigCommand extends BaseCommand {
|
|
|
52
52
|
readonly cmd: 'GetStallConfig';
|
|
53
53
|
readonly queue: string;
|
|
54
54
|
}
|
|
55
|
+
interface GroupCommand extends BaseCommand {
|
|
56
|
+
readonly queue: string;
|
|
57
|
+
readonly groupId: string;
|
|
58
|
+
}
|
|
59
|
+
export interface GetGroupJobsCountCommand extends GroupCommand {
|
|
60
|
+
readonly cmd: 'GetGroupJobsCount';
|
|
61
|
+
}
|
|
62
|
+
export interface GetGroupsJobsCountCommand extends BaseCommand {
|
|
63
|
+
readonly cmd: 'GetGroupsJobsCount';
|
|
64
|
+
readonly queue: string;
|
|
65
|
+
readonly maxCount?: number;
|
|
66
|
+
}
|
|
67
|
+
export interface GetGroupActiveCountCommand extends GroupCommand {
|
|
68
|
+
readonly cmd: 'GetGroupActiveCount';
|
|
69
|
+
}
|
|
70
|
+
export interface SetGroupRateLimitCommand extends GroupCommand {
|
|
71
|
+
readonly cmd: 'SetGroupRateLimit';
|
|
72
|
+
readonly max: number;
|
|
73
|
+
readonly duration: number;
|
|
74
|
+
}
|
|
75
|
+
export interface GetGroupRateLimitCommand extends GroupCommand {
|
|
76
|
+
readonly cmd: 'GetGroupRateLimit';
|
|
77
|
+
}
|
|
78
|
+
export interface RemoveGroupRateLimitCommand extends GroupCommand {
|
|
79
|
+
readonly cmd: 'RemoveGroupRateLimit';
|
|
80
|
+
}
|
|
81
|
+
export interface GetGroupRateLimitTtlCommand extends GroupCommand {
|
|
82
|
+
readonly cmd: 'GetGroupRateLimitTtl';
|
|
83
|
+
readonly maxJobs?: number;
|
|
84
|
+
}
|
|
85
|
+
export interface SetGroupConcurrencyCommand extends GroupCommand {
|
|
86
|
+
readonly cmd: 'SetGroupConcurrency';
|
|
87
|
+
readonly concurrency: number;
|
|
88
|
+
}
|
|
89
|
+
export interface GetGroupConcurrencyCommand extends GroupCommand {
|
|
90
|
+
readonly cmd: 'GetGroupConcurrency';
|
|
91
|
+
}
|
|
92
|
+
export interface RemoveGroupConcurrencyCommand extends GroupCommand {
|
|
93
|
+
readonly cmd: 'RemoveGroupConcurrency';
|
|
94
|
+
}
|
|
95
|
+
export {};
|
|
@@ -9,5 +9,5 @@ import type * as Monitoring from './monitoring';
|
|
|
9
9
|
import type * as Protocol from './protocol';
|
|
10
10
|
import type * as Query from './query';
|
|
11
11
|
import type * as Workers from './workers';
|
|
12
|
-
export type Command = Core.PushCommand | Core.PushBatchCommand | Core.PushFlowCommand | Core.PullCommand | Core.PullBatchCommand | Core.AckCommand | Core.AckBatchCommand | Core.FailCommand | Query.GetJobCommand | Query.GetStateCommand | Query.GetResultCommand | Query.GetJobsCommand | Query.GetJobCountsCommand | Query.GetCountsPerPriorityCommand | Query.GetJobByCustomIdCommand | Query.CountCommand | Query.GetProgressCommand | Query.GetChildrenValuesCommand | Management.CancelCommand | Management.ProgressCommand | Management.UpdateCommand | Management.ChangePriorityCommand | Management.PromoteCommand | Management.WaitJobCommand | Management.MoveToDelayedCommand | Management.DiscardCommand | Management.PauseCommand | Management.ResumeCommand | Management.IsPausedCommand | Management.DrainCommand | Management.ObliterateCommand | Management.ListQueuesCommand | Management.CleanCommand | Dlq.DlqCommand | Dlq.GetDlqStatsCommand | Dlq.RetryDlqCommand | Dlq.PurgeDlqCommand | Dlq.RemoveDlqJobCommand | Dlq.RetryCompletedCommand | Dlq.SetDlqConfigCommand | Dlq.GetDlqConfigCommand | Limits.GetQueueLimitsCommand | Limits.GetDeduplicationJobIdCommand | Limits.RemoveDeduplicationKeyCommand | Limits.RemoveJobDeduplicationKeyCommand | Limits.MoveToWaitingChildrenCommand | Limits.RateLimitCommand | Limits.SetConcurrencyCommand | Limits.RateLimitClearCommand | Limits.ClearConcurrencyCommand | Limits.SetStallConfigCommand | Limits.GetStallConfigCommand | Cron.CronCommand | Cron.CronDeleteCommand | Cron.CronListCommand | Cron.CronGetCommand | Monitoring.AddLogCommand | Monitoring.GetLogsCommand | Monitoring.AddWebhookCommand | Monitoring.RemoveWebhookCommand | Monitoring.ListWebhooksCommand | Monitoring.StatsCommand | Monitoring.MetricsCommand | Monitoring.TrimEventsCommand | Monitoring.PrometheusCommand | Monitoring.StorageStatusCommand | Workers.HeartbeatCommand | Workers.JobHeartbeatCommand | Workers.JobHeartbeatBatchCommand | Workers.PingCommand | Workers.RegisterWorkerCommand | Workers.UnregisterWorkerCommand | Workers.ListWorkersCommand | Extended.ClearLogsCommand | Extended.ExtendLockCommand | Extended.ExtendLocksCommand | Extended.ChangeDelayCommand | Extended.SetWebhookEnabledCommand | Extended.CompactMemoryCommand | Extended.UpdateParentCommand | Extended.GetFailedChildrenValuesCommand | Extended.GetIgnoredChildrenFailuresCommand | Extended.RemoveChildDependencyCommand | Extended.RemoveUnprocessedChildrenCommand | Extended.MoveToWaitCommand | Extended.PromoteJobsCommand | Dashboard.DashboardOverviewCommand | Dashboard.DashboardQueuesCommand | Dashboard.DashboardQueueCommand | Protocol.AuthCommand | Protocol.HelloCommand | Protocol.SubscribeEventsCommand | Protocol.UnsubscribeEventsCommand;
|
|
12
|
+
export type Command = Core.PushCommand | Core.PushBatchCommand | Core.PushFlowCommand | Core.PullCommand | Core.PullBatchCommand | Core.AckCommand | Core.AckBatchCommand | Core.FailCommand | Query.GetJobCommand | Query.GetStateCommand | Query.GetResultCommand | Query.GetJobsCommand | Query.GetJobCountsCommand | Query.GetCountsPerPriorityCommand | Query.GetJobByCustomIdCommand | Query.CountCommand | Query.GetProgressCommand | Query.GetChildrenValuesCommand | Management.CancelCommand | Management.ProgressCommand | Management.UpdateCommand | Management.ChangePriorityCommand | Management.PromoteCommand | Management.WaitJobCommand | Management.MoveToDelayedCommand | Management.DiscardCommand | Management.PauseCommand | Management.ResumeCommand | Management.IsPausedCommand | Management.DrainCommand | Management.ObliterateCommand | Management.ListQueuesCommand | Management.CleanCommand | Dlq.DlqCommand | Dlq.GetDlqStatsCommand | Dlq.RetryDlqCommand | Dlq.PurgeDlqCommand | Dlq.RemoveDlqJobCommand | Dlq.RetryCompletedCommand | Dlq.SetDlqConfigCommand | Dlq.GetDlqConfigCommand | Limits.GetQueueLimitsCommand | Limits.GetDeduplicationJobIdCommand | Limits.RemoveDeduplicationKeyCommand | Limits.RemoveJobDeduplicationKeyCommand | Limits.MoveToWaitingChildrenCommand | Limits.RateLimitCommand | Limits.SetConcurrencyCommand | Limits.RateLimitClearCommand | Limits.ClearConcurrencyCommand | Limits.SetStallConfigCommand | Limits.GetStallConfigCommand | Limits.GetGroupJobsCountCommand | Limits.GetGroupsJobsCountCommand | Limits.GetGroupActiveCountCommand | Limits.SetGroupRateLimitCommand | Limits.GetGroupRateLimitCommand | Limits.RemoveGroupRateLimitCommand | Limits.GetGroupRateLimitTtlCommand | Limits.SetGroupConcurrencyCommand | Limits.GetGroupConcurrencyCommand | Limits.RemoveGroupConcurrencyCommand | Cron.CronCommand | Cron.CronDeleteCommand | Cron.CronListCommand | Cron.CronGetCommand | Monitoring.AddLogCommand | Monitoring.GetLogsCommand | Monitoring.AddWebhookCommand | Monitoring.RemoveWebhookCommand | Monitoring.ListWebhooksCommand | Monitoring.StatsCommand | Monitoring.MetricsCommand | Monitoring.TrimEventsCommand | Monitoring.PrometheusCommand | Monitoring.StorageStatusCommand | Workers.HeartbeatCommand | Workers.JobHeartbeatCommand | Workers.JobHeartbeatBatchCommand | Workers.PingCommand | Workers.RegisterWorkerCommand | Workers.UnregisterWorkerCommand | Workers.ListWorkersCommand | Extended.ClearLogsCommand | Extended.ExtendLockCommand | Extended.ExtendLocksCommand | Extended.ChangeDelayCommand | Extended.SetWebhookEnabledCommand | Extended.CompactMemoryCommand | Extended.UpdateParentCommand | Extended.GetFailedChildrenValuesCommand | Extended.GetIgnoredChildrenFailuresCommand | Extended.RemoveChildDependencyCommand | Extended.RemoveUnprocessedChildrenCommand | Extended.MoveToWaitCommand | Extended.PromoteJobsCommand | Dashboard.DashboardOverviewCommand | Dashboard.DashboardQueuesCommand | Dashboard.DashboardQueueCommand | Protocol.AuthCommand | Protocol.HelloCommand | Protocol.SubscribeEventsCommand | Protocol.UnsubscribeEventsCommand;
|
|
13
13
|
export type CommandType = Command['cmd'];
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export interface GroupRateLimit {
|
|
2
|
+
readonly max: number;
|
|
3
|
+
readonly duration: number;
|
|
4
|
+
}
|
|
5
|
+
/** Worker-side defaults enforced authoritatively by the broker. */
|
|
6
|
+
export interface GroupPullOptions {
|
|
7
|
+
readonly concurrency?: number;
|
|
8
|
+
readonly limit?: GroupRateLimit;
|
|
9
|
+
}
|
|
10
|
+
export interface GroupRateLimitOverride extends GroupRateLimit {
|
|
11
|
+
}
|
|
12
|
+
/** Validate the canonical string form shared by every broker and persistence backend. */
|
|
13
|
+
export declare function validateGroupId(value: unknown, required?: boolean): string | null;
|
|
14
|
+
export declare function assertGroupId(value: unknown): asserts value is string;
|
|
15
|
+
export declare function assertOptionalGroupId(value: unknown): asserts value is string | undefined;
|
|
16
|
+
/** Normalize BullMQ-compatible numeric IDs without accepting arbitrary coercion. */
|
|
17
|
+
export declare function normalizeGroupId(value: unknown): string;
|
|
18
|
+
export declare function validatePositiveSafeInteger(value: unknown, name: string): string | null;
|
|
19
|
+
export declare function assertPositiveSafeInteger(value: unknown, name: string): asserts value is number;
|
|
20
|
+
export declare function validateGroupPullOptions(options: unknown): string | null;
|
|
21
|
+
export declare function assertGroupPullOptions(options: unknown): asserts options is GroupPullOptions;
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/** Validate the canonical string form shared by every broker and persistence backend. */
|
|
2
|
+
export function validateGroupId(value, required = false) {
|
|
3
|
+
if (value === undefined)
|
|
4
|
+
return required ? 'groupId is required' : null;
|
|
5
|
+
if (typeof value !== 'string')
|
|
6
|
+
return 'groupId must be a string';
|
|
7
|
+
if (value.length === 0)
|
|
8
|
+
return 'groupId must be a non-empty string';
|
|
9
|
+
if (value.length > 256)
|
|
10
|
+
return 'groupId must be at most 256 characters';
|
|
11
|
+
if (value.includes('\0'))
|
|
12
|
+
return 'groupId must not contain NUL characters';
|
|
13
|
+
return null;
|
|
14
|
+
}
|
|
15
|
+
export function assertGroupId(value) {
|
|
16
|
+
const error = validateGroupId(value, true);
|
|
17
|
+
if (error)
|
|
18
|
+
throw new Error(error);
|
|
19
|
+
}
|
|
20
|
+
export function assertOptionalGroupId(value) {
|
|
21
|
+
const error = validateGroupId(value);
|
|
22
|
+
if (error)
|
|
23
|
+
throw new Error(error);
|
|
24
|
+
}
|
|
25
|
+
/** Normalize BullMQ-compatible numeric IDs without accepting arbitrary coercion. */
|
|
26
|
+
export function normalizeGroupId(value) {
|
|
27
|
+
if (typeof value !== 'string' && typeof value !== 'number') {
|
|
28
|
+
throw new Error('groupId must be a string or number');
|
|
29
|
+
}
|
|
30
|
+
if (typeof value === 'number' && !Number.isSafeInteger(value)) {
|
|
31
|
+
throw new Error('numeric groupId must be a safe integer');
|
|
32
|
+
}
|
|
33
|
+
const groupId = String(value);
|
|
34
|
+
assertGroupId(groupId);
|
|
35
|
+
return groupId;
|
|
36
|
+
}
|
|
37
|
+
export function validatePositiveSafeInteger(value, name) {
|
|
38
|
+
if (typeof value !== 'number' || !Number.isSafeInteger(value) || value <= 0) {
|
|
39
|
+
return `${name} must be a positive safe integer`;
|
|
40
|
+
}
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
export function assertPositiveSafeInteger(value, name) {
|
|
44
|
+
const error = validatePositiveSafeInteger(value, name);
|
|
45
|
+
if (error)
|
|
46
|
+
throw new Error(error);
|
|
47
|
+
}
|
|
48
|
+
export function validateGroupPullOptions(options) {
|
|
49
|
+
if (options === undefined)
|
|
50
|
+
return null;
|
|
51
|
+
if (!options || typeof options !== 'object' || Array.isArray(options)) {
|
|
52
|
+
return 'group must be an object';
|
|
53
|
+
}
|
|
54
|
+
const group = options;
|
|
55
|
+
if (group.concurrency !== undefined) {
|
|
56
|
+
const error = validatePositiveSafeInteger(group.concurrency, 'group.concurrency');
|
|
57
|
+
if (error)
|
|
58
|
+
return error;
|
|
59
|
+
}
|
|
60
|
+
if (group.limit !== undefined) {
|
|
61
|
+
if (!group.limit || typeof group.limit !== 'object' || Array.isArray(group.limit)) {
|
|
62
|
+
return 'group.limit must be an object';
|
|
63
|
+
}
|
|
64
|
+
const limit = group.limit;
|
|
65
|
+
const maxError = validatePositiveSafeInteger(limit.max, 'group.limit.max');
|
|
66
|
+
if (maxError)
|
|
67
|
+
return maxError;
|
|
68
|
+
const durationError = validatePositiveSafeInteger(limit.duration, 'group.limit.duration');
|
|
69
|
+
if (durationError)
|
|
70
|
+
return durationError;
|
|
71
|
+
}
|
|
72
|
+
return null;
|
|
73
|
+
}
|
|
74
|
+
export function assertGroupPullOptions(options) {
|
|
75
|
+
const error = validateGroupPullOptions(options);
|
|
76
|
+
if (error)
|
|
77
|
+
throw new Error(error);
|
|
78
|
+
}
|
|
@@ -11,6 +11,7 @@ export interface StoredJobOptions {
|
|
|
11
11
|
debounceId: string | null;
|
|
12
12
|
debounceTtl: number | null;
|
|
13
13
|
durable: boolean;
|
|
14
|
+
groupFifoOrder: string | null;
|
|
14
15
|
}
|
|
15
16
|
/** Persist generation policies that do not have dedicated legacy columns. */
|
|
16
17
|
export declare function encodeJobOptions(job: Job): Uint8Array;
|