bunqueue 1.9.6 → 1.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/application/backgroundTasks.d.ts +32 -0
- package/dist/application/backgroundTasks.d.ts.map +1 -0
- package/dist/application/backgroundTasks.js +318 -0
- package/dist/application/backgroundTasks.js.map +1 -0
- package/dist/application/cleanupTasks.d.ts +11 -0
- package/dist/application/cleanupTasks.d.ts.map +1 -0
- package/dist/application/cleanupTasks.js +181 -0
- package/dist/application/cleanupTasks.js.map +1 -0
- package/dist/application/lockManager.d.ts +62 -0
- package/dist/application/lockManager.d.ts.map +1 -0
- package/dist/application/lockManager.js +307 -0
- package/dist/application/lockManager.js.map +1 -0
- package/dist/application/queueManager.d.ts +14 -176
- package/dist/application/queueManager.d.ts.map +1 -1
- package/dist/application/queueManager.js +130 -953
- package/dist/application/queueManager.js.map +1 -1
- package/dist/application/statsManager.d.ts +56 -0
- package/dist/application/statsManager.d.ts.map +1 -0
- package/dist/application/statsManager.js +111 -0
- package/dist/application/statsManager.js.map +1 -0
- package/dist/application/types.d.ts +123 -0
- package/dist/application/types.d.ts.map +1 -0
- package/dist/application/types.js +16 -0
- package/dist/application/types.js.map +1 -0
- package/dist/infrastructure/server/tcp.d.ts.map +1 -1
- package/dist/infrastructure/server/tcp.js +14 -8
- package/dist/infrastructure/server/tcp.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lock Manager - Job lock and client tracking
|
|
3
|
+
* Handles BullMQ-style lock-based job ownership
|
|
4
|
+
*/
|
|
5
|
+
import { createJobLock, isLockExpired, renewLock, DEFAULT_LOCK_TTL } from '../domain/types/job';
|
|
6
|
+
import { queueLog } from '../shared/logger';
|
|
7
|
+
import { shardIndex, processingShardIndex } from '../shared/hash';
|
|
8
|
+
import { withWriteLock } from '../shared/lock';
|
|
9
|
+
// ============ Lock Creation & Verification ============
|
|
10
|
+
/**
|
|
11
|
+
* Create a lock for a job when it's pulled for processing.
|
|
12
|
+
* @returns The lock token, or null if job not in processing
|
|
13
|
+
*/
|
|
14
|
+
export function createLock(jobId, owner, ctx, ttl = DEFAULT_LOCK_TTL) {
|
|
15
|
+
const loc = ctx.jobIndex.get(jobId);
|
|
16
|
+
if (loc?.type !== 'processing')
|
|
17
|
+
return null;
|
|
18
|
+
// Check if lock already exists (shouldn't happen, but defensive)
|
|
19
|
+
if (ctx.jobLocks.has(jobId)) {
|
|
20
|
+
queueLog.warn('Lock already exists for job', { jobId: String(jobId), owner });
|
|
21
|
+
return null;
|
|
22
|
+
}
|
|
23
|
+
const lock = createJobLock(jobId, owner, ttl);
|
|
24
|
+
ctx.jobLocks.set(jobId, lock);
|
|
25
|
+
return lock.token;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Verify that a token is valid for a job.
|
|
29
|
+
* @returns true if token matches the active lock
|
|
30
|
+
*/
|
|
31
|
+
export function verifyLock(jobId, token, ctx) {
|
|
32
|
+
const lock = ctx.jobLocks.get(jobId);
|
|
33
|
+
if (!lock)
|
|
34
|
+
return false;
|
|
35
|
+
if (lock.token !== token)
|
|
36
|
+
return false;
|
|
37
|
+
if (isLockExpired(lock))
|
|
38
|
+
return false;
|
|
39
|
+
return true;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Renew a lock with the given token.
|
|
43
|
+
* @returns true if renewal succeeded, false if token invalid or lock expired
|
|
44
|
+
*/
|
|
45
|
+
export function renewJobLock(jobId, token, ctx, newTtl) {
|
|
46
|
+
const lock = ctx.jobLocks.get(jobId);
|
|
47
|
+
if (!lock)
|
|
48
|
+
return false;
|
|
49
|
+
if (lock.token !== token)
|
|
50
|
+
return false;
|
|
51
|
+
if (isLockExpired(lock)) {
|
|
52
|
+
// Lock already expired, remove it
|
|
53
|
+
ctx.jobLocks.delete(jobId);
|
|
54
|
+
return false;
|
|
55
|
+
}
|
|
56
|
+
renewLock(lock, newTtl);
|
|
57
|
+
// Also update lastHeartbeat on the job (for legacy stall detection compatibility)
|
|
58
|
+
const loc = ctx.jobIndex.get(jobId);
|
|
59
|
+
if (loc?.type === 'processing') {
|
|
60
|
+
const job = ctx.processingShards[loc.shardIdx].get(jobId);
|
|
61
|
+
if (job)
|
|
62
|
+
job.lastHeartbeat = Date.now();
|
|
63
|
+
}
|
|
64
|
+
return true;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Renew locks for multiple jobs (batch operation).
|
|
68
|
+
* @returns Array of jobIds that were successfully renewed
|
|
69
|
+
*/
|
|
70
|
+
export function renewJobLockBatch(items, ctx) {
|
|
71
|
+
const renewed = [];
|
|
72
|
+
for (const item of items) {
|
|
73
|
+
if (renewJobLock(item.id, item.token, ctx, item.ttl)) {
|
|
74
|
+
renewed.push(String(item.id));
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
return renewed;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Release a lock when job is completed or failed.
|
|
81
|
+
* Should be called by ACK/FAIL operations.
|
|
82
|
+
*/
|
|
83
|
+
export function releaseLock(jobId, ctx, token) {
|
|
84
|
+
const lock = ctx.jobLocks.get(jobId);
|
|
85
|
+
if (!lock)
|
|
86
|
+
return true; // No lock to release
|
|
87
|
+
// If token provided, verify it matches
|
|
88
|
+
if (token && lock.token !== token) {
|
|
89
|
+
queueLog.warn('Token mismatch on lock release', {
|
|
90
|
+
jobId: String(jobId),
|
|
91
|
+
expected: lock.token.substring(0, 8),
|
|
92
|
+
got: token.substring(0, 8),
|
|
93
|
+
});
|
|
94
|
+
return false;
|
|
95
|
+
}
|
|
96
|
+
ctx.jobLocks.delete(jobId);
|
|
97
|
+
return true;
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Get lock info for a job (for debugging/monitoring).
|
|
101
|
+
*/
|
|
102
|
+
export function getLockInfo(jobId, ctx) {
|
|
103
|
+
return ctx.jobLocks.get(jobId) ?? null;
|
|
104
|
+
}
|
|
105
|
+
// ============ Client-Job Tracking ============
|
|
106
|
+
/**
|
|
107
|
+
* Register a job as owned by a client (called on PULL).
|
|
108
|
+
*/
|
|
109
|
+
export function registerClientJob(clientId, jobId, ctx) {
|
|
110
|
+
let jobs = ctx.clientJobs.get(clientId);
|
|
111
|
+
if (!jobs) {
|
|
112
|
+
jobs = new Set();
|
|
113
|
+
ctx.clientJobs.set(clientId, jobs);
|
|
114
|
+
}
|
|
115
|
+
jobs.add(jobId);
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Unregister a job from a client (called on ACK/FAIL).
|
|
119
|
+
*/
|
|
120
|
+
export function unregisterClientJob(clientId, jobId, ctx) {
|
|
121
|
+
if (!clientId)
|
|
122
|
+
return;
|
|
123
|
+
const jobs = ctx.clientJobs.get(clientId);
|
|
124
|
+
if (jobs) {
|
|
125
|
+
jobs.delete(jobId);
|
|
126
|
+
if (jobs.size === 0) {
|
|
127
|
+
ctx.clientJobs.delete(clientId);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Release all jobs owned by a client back to queue (called on TCP disconnect).
|
|
133
|
+
* Returns the number of jobs released.
|
|
134
|
+
*
|
|
135
|
+
* Uses proper locking to prevent race conditions.
|
|
136
|
+
*/
|
|
137
|
+
export async function releaseClientJobs(clientId, ctx) {
|
|
138
|
+
const jobs = ctx.clientJobs.get(clientId);
|
|
139
|
+
if (!jobs || jobs.size === 0) {
|
|
140
|
+
ctx.clientJobs.delete(clientId);
|
|
141
|
+
return 0;
|
|
142
|
+
}
|
|
143
|
+
// Phase 1: Collect jobs to release (read-only, no locks needed)
|
|
144
|
+
const jobsToRelease = [];
|
|
145
|
+
for (const jobId of jobs) {
|
|
146
|
+
const loc = ctx.jobIndex.get(jobId);
|
|
147
|
+
if (loc?.type !== 'processing')
|
|
148
|
+
continue;
|
|
149
|
+
const procIdx = loc.shardIdx;
|
|
150
|
+
const job = ctx.processingShards[procIdx].get(jobId);
|
|
151
|
+
if (!job)
|
|
152
|
+
continue;
|
|
153
|
+
jobsToRelease.push({
|
|
154
|
+
jobId,
|
|
155
|
+
procIdx,
|
|
156
|
+
queueShardIdx: shardIndex(job.queue),
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
if (jobsToRelease.length === 0) {
|
|
160
|
+
ctx.clientJobs.delete(clientId);
|
|
161
|
+
return 0;
|
|
162
|
+
}
|
|
163
|
+
// Phase 2: Group by processing shard for efficient locking
|
|
164
|
+
const byProcShard = new Map();
|
|
165
|
+
for (const item of jobsToRelease) {
|
|
166
|
+
let list = byProcShard.get(item.procIdx);
|
|
167
|
+
if (!list) {
|
|
168
|
+
list = [];
|
|
169
|
+
byProcShard.set(item.procIdx, list);
|
|
170
|
+
}
|
|
171
|
+
list.push(item);
|
|
172
|
+
}
|
|
173
|
+
let released = 0;
|
|
174
|
+
const now = Date.now();
|
|
175
|
+
// Phase 3: Process each processing shard with proper locking
|
|
176
|
+
for (const [procIdx, items] of byProcShard) {
|
|
177
|
+
await withWriteLock(ctx.processingLocks[procIdx], async () => {
|
|
178
|
+
for (const { jobId, queueShardIdx } of items) {
|
|
179
|
+
const job = ctx.processingShards[procIdx].get(jobId);
|
|
180
|
+
if (!job)
|
|
181
|
+
continue;
|
|
182
|
+
// Acquire shard lock for queue modifications
|
|
183
|
+
await withWriteLock(ctx.shardLocks[queueShardIdx], () => {
|
|
184
|
+
const shard = ctx.shards[queueShardIdx];
|
|
185
|
+
// Remove from processing
|
|
186
|
+
ctx.processingShards[procIdx].delete(jobId);
|
|
187
|
+
// Release lock if exists
|
|
188
|
+
ctx.jobLocks.delete(jobId);
|
|
189
|
+
// Release concurrency
|
|
190
|
+
shard.releaseConcurrency(job.queue);
|
|
191
|
+
// Release group if active
|
|
192
|
+
if (job.groupId) {
|
|
193
|
+
shard.releaseGroup(job.queue, job.groupId);
|
|
194
|
+
}
|
|
195
|
+
// Reset job state for retry
|
|
196
|
+
job.startedAt = null;
|
|
197
|
+
job.lastHeartbeat = now;
|
|
198
|
+
// Re-queue the job
|
|
199
|
+
shard.getQueue(job.queue).push(job);
|
|
200
|
+
const isDelayed = job.runAt > now;
|
|
201
|
+
shard.incrementQueued(jobId, isDelayed, job.createdAt, job.queue, job.runAt);
|
|
202
|
+
ctx.jobIndex.set(jobId, { type: 'queue', shardIdx: queueShardIdx, queueName: job.queue });
|
|
203
|
+
released++;
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
// Clear client tracking
|
|
209
|
+
ctx.clientJobs.delete(clientId);
|
|
210
|
+
if (released > 0) {
|
|
211
|
+
queueLog.info('Released client jobs', { clientId: clientId.substring(0, 8), released });
|
|
212
|
+
}
|
|
213
|
+
return released;
|
|
214
|
+
}
|
|
215
|
+
// ============ Lock Expiration Check ============
|
|
216
|
+
/**
|
|
217
|
+
* Check and handle expired locks.
|
|
218
|
+
* Jobs with expired locks are requeued for retry.
|
|
219
|
+
*
|
|
220
|
+
* Uses proper locking to prevent race conditions.
|
|
221
|
+
*/
|
|
222
|
+
export async function checkExpiredLocks(ctx) {
|
|
223
|
+
const now = Date.now();
|
|
224
|
+
// Phase 1: Collect expired locks (read-only)
|
|
225
|
+
const expired = [];
|
|
226
|
+
for (const [jobId, lock] of ctx.jobLocks) {
|
|
227
|
+
if (isLockExpired(lock, now)) {
|
|
228
|
+
const procIdx = processingShardIndex(String(jobId));
|
|
229
|
+
expired.push({ jobId, lock, procIdx });
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
if (expired.length === 0)
|
|
233
|
+
return;
|
|
234
|
+
// Phase 2: Group by processing shard
|
|
235
|
+
const byProcShard = new Map();
|
|
236
|
+
for (const item of expired) {
|
|
237
|
+
let list = byProcShard.get(item.procIdx);
|
|
238
|
+
if (!list) {
|
|
239
|
+
list = [];
|
|
240
|
+
byProcShard.set(item.procIdx, list);
|
|
241
|
+
}
|
|
242
|
+
list.push(item);
|
|
243
|
+
}
|
|
244
|
+
// Phase 3: Process each shard with proper locking
|
|
245
|
+
for (const [procIdx, items] of byProcShard) {
|
|
246
|
+
await withWriteLock(ctx.processingLocks[procIdx], async () => {
|
|
247
|
+
for (const { jobId, lock } of items) {
|
|
248
|
+
const job = ctx.processingShards[procIdx].get(jobId);
|
|
249
|
+
if (job) {
|
|
250
|
+
const idx = shardIndex(job.queue);
|
|
251
|
+
await withWriteLock(ctx.shardLocks[idx], () => {
|
|
252
|
+
const shard = ctx.shards[idx];
|
|
253
|
+
const queue = shard.getQueue(job.queue);
|
|
254
|
+
// Remove from processing
|
|
255
|
+
ctx.processingShards[procIdx].delete(jobId);
|
|
256
|
+
// Increment attempts and reset state
|
|
257
|
+
job.attempts++;
|
|
258
|
+
job.startedAt = null;
|
|
259
|
+
job.lastHeartbeat = now;
|
|
260
|
+
job.stallCount++;
|
|
261
|
+
// Check if max stalls exceeded
|
|
262
|
+
const stallConfig = shard.getStallConfig(job.queue);
|
|
263
|
+
if (stallConfig.maxStalls > 0 && job.stallCount >= stallConfig.maxStalls) {
|
|
264
|
+
shard.addToDlq(job, "stalled" /* FailureReason.Stalled */, `Lock expired after ${lock.renewalCount} renewals`);
|
|
265
|
+
ctx.jobIndex.set(jobId, { type: 'dlq', queueName: job.queue });
|
|
266
|
+
queueLog.warn('Job moved to DLQ due to lock expiration', {
|
|
267
|
+
jobId: String(jobId),
|
|
268
|
+
queue: job.queue,
|
|
269
|
+
owner: lock.owner,
|
|
270
|
+
renewals: lock.renewalCount,
|
|
271
|
+
stallCount: job.stallCount,
|
|
272
|
+
});
|
|
273
|
+
ctx.eventsManager.broadcast({
|
|
274
|
+
eventType: "failed" /* EventType.Failed */,
|
|
275
|
+
jobId,
|
|
276
|
+
queue: job.queue,
|
|
277
|
+
timestamp: now,
|
|
278
|
+
error: 'Lock expired (max stalls reached)',
|
|
279
|
+
});
|
|
280
|
+
}
|
|
281
|
+
else {
|
|
282
|
+
queue.push(job);
|
|
283
|
+
ctx.jobIndex.set(jobId, { type: 'queue', shardIdx: idx, queueName: job.queue });
|
|
284
|
+
queueLog.info('Job requeued due to lock expiration', {
|
|
285
|
+
jobId: String(jobId),
|
|
286
|
+
queue: job.queue,
|
|
287
|
+
owner: lock.owner,
|
|
288
|
+
renewals: lock.renewalCount,
|
|
289
|
+
attempt: job.attempts,
|
|
290
|
+
});
|
|
291
|
+
ctx.eventsManager.broadcast({
|
|
292
|
+
eventType: "stalled" /* EventType.Stalled */,
|
|
293
|
+
jobId,
|
|
294
|
+
queue: job.queue,
|
|
295
|
+
timestamp: now,
|
|
296
|
+
});
|
|
297
|
+
}
|
|
298
|
+
});
|
|
299
|
+
}
|
|
300
|
+
// Remove the expired lock
|
|
301
|
+
ctx.jobLocks.delete(jobId);
|
|
302
|
+
}
|
|
303
|
+
});
|
|
304
|
+
}
|
|
305
|
+
queueLog.info('Processed expired locks', { count: expired.length });
|
|
306
|
+
}
|
|
307
|
+
//# sourceMappingURL=lockManager.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lockManager.js","sourceRoot":"","sources":["../../src/application/lockManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAGhG,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAClE,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAG/C,yDAAyD;AAEzD;;;GAGG;AACH,MAAM,UAAU,UAAU,CACxB,KAAY,EACZ,KAAa,EACb,GAAgB,EAChB,MAAc,gBAAgB;IAE9B,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACpC,IAAI,GAAG,EAAE,IAAI,KAAK,YAAY;QAAE,OAAO,IAAI,CAAC;IAE5C,iEAAiE;IACjE,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAC5B,QAAQ,CAAC,IAAI,CAAC,6BAA6B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QAC9E,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,IAAI,GAAG,aAAa,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;IAC9C,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC9B,OAAO,IAAI,CAAC,KAAK,CAAC;AACpB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,KAAY,EAAE,KAAa,EAAE,GAAgB;IACtE,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxB,IAAI,IAAI,CAAC,KAAK,KAAK,KAAK;QAAE,OAAO,KAAK,CAAC;IACvC,IAAI,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACtC,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAC1B,KAAY,EACZ,KAAa,EACb,GAAgB,EAChB,MAAe;IAEf,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxB,IAAI,IAAI,CAAC,KAAK,KAAK,KAAK;QAAE,OAAO,KAAK,CAAC;IACvC,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,kCAAkC;QAClC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC3B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAExB,kFAAkF;IAClF,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACpC,IAAI,GAAG,EAAE,IAAI,KAAK,YAAY,EAAE,CAAC;QAC/B,MAAM,GAAG,GAAG,GAAG,CAAC,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC1D,IAAI,GAAG;YAAE,GAAG,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC1C,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAC/B,KAAwD,EACxD,GAAgB;IAEhB,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACrD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,KAAY,EAAE,GAAgB,EAAE,KAAc;IACxE,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC,CAAC,qBAAqB;IAE7C,uCAAuC;IACvC,IAAI,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,KAAK,EAAE,CAAC;QAClC,QAAQ,CAAC,IAAI,CAAC,gCAAgC,EAAE;YAC9C,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC;YACpB,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC;YACpC,GAAG,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC;SAC3B,CAAC,CAAC;QACH,OAAO,KAAK,CAAC;IACf,CAAC;IAED,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC3B,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,KAAY,EAAE,GAAgB;IACxD,OAAO,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC;AACzC,CAAC;AAED,gDAAgD;AAEhD;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAgB,EAAE,KAAY,EAAE,GAAgB;IAChF,IAAI,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACxC,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;QACjB,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IACrC,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CACjC,QAA4B,EAC5B,KAAY,EACZ,GAAgB;IAEhB,IAAI,CAAC,QAAQ;QAAE,OAAO;IACtB,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC1C,IAAI,IAAI,EAAE,CAAC;QACT,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YACpB,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,QAAgB,EAAE,GAAgB;IACxE,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC1C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QAC7B,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAChC,OAAO,CAAC,CAAC;IACX,CAAC;IAED,gEAAgE;IAChE,MAAM,aAAa,GAId,EAAE,CAAC;IAER,KAAK,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACpC,IAAI,GAAG,EAAE,IAAI,KAAK,YAAY;YAAE,SAAS;QAEzC,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC;QAC7B,MAAM,GAAG,GAAG,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACrD,IAAI,CAAC,GAAG;YAAE,SAAS;QAEnB,aAAa,CAAC,IAAI,CAAC;YACjB,KAAK;YACL,OAAO;YACP,aAAa,EAAE,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC;SACrC,CAAC,CAAC;IACL,CAAC;IAED,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAChC,OAAO,CAAC,CAAC;IACX,CAAC;IAED,2DAA2D;IAC3D,MAAM,WAAW,GAAG,IAAI,GAAG,EAAgC,CAAC;IAC5D,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;QACjC,IAAI,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,IAAI,GAAG,EAAE,CAAC;YACV,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEvB,6DAA6D;IAC7D,KAAK,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,WAAW,EAAE,CAAC;QAC3C,MAAM,aAAa,CAAC,GAAG,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,KAAK,IAAI,EAAE;YAC3D,KAAK,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,KAAK,EAAE,CAAC;gBAC7C,MAAM,GAAG,GAAG,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACrD,IAAI,CAAC,GAAG;oBAAE,SAAS;gBAEnB,6CAA6C;gBAC7C,MAAM,aAAa,CAAC,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,GAAG,EAAE;oBACtD,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;oBAExC,yBAAyB;oBACzB,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAE5C,yBAAyB;oBACzB,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAE3B,sBAAsB;oBACtB,KAAK,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;oBAEpC,0BAA0B;oBAC1B,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;wBAChB,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;oBAC7C,CAAC;oBAED,4BAA4B;oBAC5B,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC;oBACrB,GAAG,CAAC,aAAa,GAAG,GAAG,CAAC;oBAExB,mBAAmB;oBACnB,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACpC,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC;oBAClC,KAAK,CAAC,eAAe,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;oBAC7E,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,SAAS,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;oBAE1F,QAAQ,EAAE,CAAC;gBACb,CAAC,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,wBAAwB;IACxB,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAEhC,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;QACjB,QAAQ,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC1F,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,kDAAkD;AAElD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,GAAgB;IACtD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEvB,6CAA6C;IAC7C,MAAM,OAAO,GAA4D,EAAE,CAAC;IAE5E,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QACzC,IAAI,aAAa,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,oBAAoB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YACpD,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAEjC,qCAAqC;IACrC,MAAM,WAAW,GAAG,IAAI,GAAG,EAA0B,CAAC;IACtD,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,IAAI,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,IAAI,GAAG,EAAE,CAAC;YACV,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClB,CAAC;IAED,kDAAkD;IAClD,KAAK,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,WAAW,EAAE,CAAC;QAC3C,MAAM,aAAa,CAAC,GAAG,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,KAAK,IAAI,EAAE;YAC3D,KAAK,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,KAAK,EAAE,CAAC;gBACpC,MAAM,GAAG,GAAG,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAErD,IAAI,GAAG,EAAE,CAAC;oBACR,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;oBAElC,MAAM,aAAa,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE;wBAC5C,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;wBAC9B,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;wBAExC,yBAAyB;wBACzB,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;wBAE5C,qCAAqC;wBACrC,GAAG,CAAC,QAAQ,EAAE,CAAC;wBACf,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC;wBACrB,GAAG,CAAC,aAAa,GAAG,GAAG,CAAC;wBACxB,GAAG,CAAC,UAAU,EAAE,CAAC;wBAEjB,+BAA+B;wBAC/B,MAAM,WAAW,GAAG,KAAK,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;wBACpD,IAAI,WAAW,CAAC,SAAS,GAAG,CAAC,IAAI,GAAG,CAAC,UAAU,IAAI,WAAW,CAAC,SAAS,EAAE,CAAC;4BACzE,KAAK,CAAC,QAAQ,CACZ,GAAG,yCAEH,sBAAsB,IAAI,CAAC,YAAY,WAAW,CACnD,CAAC;4BACF,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;4BAE/D,QAAQ,CAAC,IAAI,CAAC,yCAAyC,EAAE;gCACvD,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC;gCACpB,KAAK,EAAE,GAAG,CAAC,KAAK;gCAChB,KAAK,EAAE,IAAI,CAAC,KAAK;gCACjB,QAAQ,EAAE,IAAI,CAAC,YAAY;gCAC3B,UAAU,EAAE,GAAG,CAAC,UAAU;6BAC3B,CAAC,CAAC;4BAEH,GAAG,CAAC,aAAa,CAAC,SAAS,CAAC;gCAC1B,SAAS,iCAAkB;gCAC3B,KAAK;gCACL,KAAK,EAAE,GAAG,CAAC,KAAK;gCAChB,SAAS,EAAE,GAAG;gCACd,KAAK,EAAE,mCAAmC;6BAC3C,CAAC,CAAC;wBACL,CAAC;6BAAM,CAAC;4BACN,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;4BAChB,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;4BAEhF,QAAQ,CAAC,IAAI,CAAC,qCAAqC,EAAE;gCACnD,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC;gCACpB,KAAK,EAAE,GAAG,CAAC,KAAK;gCAChB,KAAK,EAAE,IAAI,CAAC,KAAK;gCACjB,QAAQ,EAAE,IAAI,CAAC,YAAY;gCAC3B,OAAO,EAAE,GAAG,CAAC,QAAQ;6BACtB,CAAC,CAAC;4BAEH,GAAG,CAAC,aAAa,CAAC,SAAS,CAAC;gCAC1B,SAAS,mCAAmB;gCAC5B,KAAK;gCACL,KAAK,EAAE,GAAG,CAAC,KAAK;gCAChB,SAAS,EAAE,GAAG;6BACf,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC,CAAC,CAAC;gBACL,CAAC;gBAED,0BAA0B;gBAC1B,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,QAAQ,CAAC,IAAI,CAAC,yBAAyB,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;AACtE,CAAC"}
|
|
@@ -6,23 +6,13 @@ import type { Job, JobId, JobInput, JobLock, LockToken } from '../domain/types/j
|
|
|
6
6
|
import type { JobLocation, JobEvent } from '../domain/types/queue';
|
|
7
7
|
import type { CronJob, CronJobInput } from '../domain/types/cron';
|
|
8
8
|
import type { JobLogEntry } from '../domain/types/worker';
|
|
9
|
+
import { Shard } from '../domain/queue/shard';
|
|
9
10
|
import { WebhookManager } from './webhookManager';
|
|
10
11
|
import { WorkerManager } from './workerManager';
|
|
11
12
|
import { type SetLike } from '../shared/lru';
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
maxCompletedJobs?: number;
|
|
16
|
-
maxJobResults?: number;
|
|
17
|
-
maxJobLogs?: number;
|
|
18
|
-
maxCustomIds?: number;
|
|
19
|
-
maxWaitingDeps?: number;
|
|
20
|
-
cleanupIntervalMs?: number;
|
|
21
|
-
jobTimeoutCheckMs?: number;
|
|
22
|
-
dependencyCheckMs?: number;
|
|
23
|
-
stallCheckMs?: number;
|
|
24
|
-
dlqMaintenanceMs?: number;
|
|
25
|
-
}
|
|
13
|
+
import type { QueueManagerConfig } from './types';
|
|
14
|
+
import * as statsMgr from './statsManager';
|
|
15
|
+
export type { QueueManagerConfig };
|
|
26
16
|
/**
|
|
27
17
|
* QueueManager - Central coordinator
|
|
28
18
|
*/
|
|
@@ -39,7 +29,6 @@ export declare class QueueManager {
|
|
|
39
29
|
private readonly customIdMap;
|
|
40
30
|
private readonly jobLogs;
|
|
41
31
|
private readonly pendingDepChecks;
|
|
42
|
-
private depCheckInterval;
|
|
43
32
|
private readonly stalledCandidates;
|
|
44
33
|
private readonly jobLocks;
|
|
45
34
|
private readonly clientJobs;
|
|
@@ -50,17 +39,15 @@ export declare class QueueManager {
|
|
|
50
39
|
private readonly maxLogsPerJob;
|
|
51
40
|
private readonly metrics;
|
|
52
41
|
private readonly startTime;
|
|
53
|
-
private
|
|
54
|
-
private timeoutInterval;
|
|
55
|
-
private stallCheckInterval;
|
|
56
|
-
private dlqMaintenanceInterval;
|
|
57
|
-
private lockCheckInterval;
|
|
42
|
+
private readonly backgroundTaskHandles;
|
|
58
43
|
private readonly queueNamesCache;
|
|
59
44
|
constructor(config?: QueueManagerConfig);
|
|
45
|
+
private getLockContext;
|
|
46
|
+
private getBackgroundContext;
|
|
47
|
+
private getStatsContext;
|
|
60
48
|
private getPushContext;
|
|
61
49
|
private getPullContext;
|
|
62
50
|
private getAckContext;
|
|
63
|
-
/** Handle repeatable job - re-queue with incremented count */
|
|
64
51
|
private handleRepeat;
|
|
65
52
|
private getJobMgmtContext;
|
|
66
53
|
private getQueryContext;
|
|
@@ -68,95 +55,38 @@ export declare class QueueManager {
|
|
|
68
55
|
push(queue: string, input: JobInput): Promise<Job>;
|
|
69
56
|
pushBatch(queue: string, inputs: JobInput[]): Promise<JobId[]>;
|
|
70
57
|
pull(queue: string, timeoutMs?: number): Promise<Job | null>;
|
|
71
|
-
/**
|
|
72
|
-
* Pull a job and create a lock for it (BullMQ-style).
|
|
73
|
-
* Returns both the job and its lock token for ownership verification.
|
|
74
|
-
*/
|
|
75
58
|
pullWithLock(queue: string, owner: string, timeoutMs?: number, lockTtl?: number): Promise<{
|
|
76
59
|
job: Job | null;
|
|
77
60
|
token: string | null;
|
|
78
61
|
}>;
|
|
79
|
-
/** Pull multiple jobs in single lock acquisition - O(1) instead of O(n) locks */
|
|
80
62
|
pullBatch(queue: string, count: number, timeoutMs?: number): Promise<Job[]>;
|
|
81
|
-
/**
|
|
82
|
-
* Pull multiple jobs and create locks for them (BullMQ-style).
|
|
83
|
-
* Returns both jobs and their lock tokens for ownership verification.
|
|
84
|
-
*/
|
|
85
63
|
pullBatchWithLock(queue: string, count: number, owner: string, timeoutMs?: number, lockTtl?: number): Promise<{
|
|
86
64
|
jobs: Job[];
|
|
87
65
|
tokens: string[];
|
|
88
66
|
}>;
|
|
89
67
|
ack(jobId: JobId, result?: unknown, token?: string): Promise<void>;
|
|
90
|
-
/** Acknowledge multiple jobs in parallel with Promise.all */
|
|
91
68
|
ackBatch(jobIds: JobId[], tokens?: string[]): Promise<void>;
|
|
92
|
-
/** Acknowledge multiple jobs with individual results - batch optimized */
|
|
93
69
|
ackBatchWithResults(items: Array<{
|
|
94
70
|
id: JobId;
|
|
95
71
|
result: unknown;
|
|
96
72
|
token?: string;
|
|
97
73
|
}>): Promise<void>;
|
|
98
74
|
fail(jobId: JobId, error?: string, token?: string): Promise<void>;
|
|
99
|
-
/**
|
|
100
|
-
* Update job heartbeat for stall detection (single job).
|
|
101
|
-
* If token is provided, also renews the lock.
|
|
102
|
-
*/
|
|
103
75
|
jobHeartbeat(jobId: JobId, token?: string): boolean;
|
|
104
|
-
/**
|
|
105
|
-
* Update job heartbeat for multiple jobs (batch).
|
|
106
|
-
* If tokens are provided, also renews the locks.
|
|
107
|
-
*/
|
|
108
76
|
jobHeartbeatBatch(jobIds: JobId[], tokens?: string[]): number;
|
|
109
|
-
/**
|
|
110
|
-
* Create a lock for a job when it's pulled for processing.
|
|
111
|
-
* @returns The lock token, or null if job not in processing
|
|
112
|
-
*/
|
|
113
77
|
createLock(jobId: JobId, owner: string, ttl?: number): LockToken | null;
|
|
114
|
-
/**
|
|
115
|
-
* Verify that a token is valid for a job.
|
|
116
|
-
* @returns true if token matches the active lock
|
|
117
|
-
*/
|
|
118
78
|
verifyLock(jobId: JobId, token: string): boolean;
|
|
119
|
-
/**
|
|
120
|
-
* Renew a lock with the given token.
|
|
121
|
-
* @returns true if renewal succeeded, false if token invalid or lock expired
|
|
122
|
-
*/
|
|
123
79
|
renewJobLock(jobId: JobId, token: string, newTtl?: number): boolean;
|
|
124
|
-
/**
|
|
125
|
-
* Renew locks for multiple jobs (batch operation).
|
|
126
|
-
* @returns Array of jobIds that were successfully renewed
|
|
127
|
-
*/
|
|
128
80
|
renewJobLockBatch(items: Array<{
|
|
129
81
|
id: JobId;
|
|
130
82
|
token: string;
|
|
131
83
|
ttl?: number;
|
|
132
84
|
}>): string[];
|
|
133
|
-
/**
|
|
134
|
-
* Release a lock when job is completed or failed.
|
|
135
|
-
* Should be called by ACK/FAIL operations.
|
|
136
|
-
*/
|
|
137
85
|
releaseLock(jobId: JobId, token?: string): boolean;
|
|
138
|
-
/**
|
|
139
|
-
* Get lock info for a job (for debugging/monitoring).
|
|
140
|
-
*/
|
|
141
86
|
getLockInfo(jobId: JobId): JobLock | null;
|
|
142
|
-
/**
|
|
143
|
-
* Register a job as owned by a client (called on PULL).
|
|
144
|
-
*/
|
|
145
87
|
registerClientJob(clientId: string, jobId: JobId): void;
|
|
146
|
-
/**
|
|
147
|
-
* Unregister a job from a client (called on ACK/FAIL).
|
|
148
|
-
*/
|
|
149
88
|
unregisterClientJob(clientId: string | undefined, jobId: JobId): void;
|
|
150
|
-
|
|
151
|
-
* Release all jobs owned by a client back to queue (called on TCP disconnect).
|
|
152
|
-
* Returns the number of jobs released.
|
|
153
|
-
*/
|
|
154
|
-
releaseClientJobs(clientId: string): number;
|
|
155
|
-
/**
|
|
156
|
-
* Check and handle expired locks.
|
|
157
|
-
* Jobs with expired locks are requeued for retry.
|
|
158
|
-
*/
|
|
159
|
-
private checkExpiredLocks;
|
|
89
|
+
releaseClientJobs(clientId: string): Promise<number>;
|
|
160
90
|
getJob(jobId: JobId): Promise<Job | null>;
|
|
161
91
|
getResult(jobId: JobId): unknown;
|
|
162
92
|
getJobByCustomId(customId: string): Job | null;
|
|
@@ -171,19 +101,10 @@ export declare class QueueManager {
|
|
|
171
101
|
drain(queue: string): number;
|
|
172
102
|
obliterate(queue: string): void;
|
|
173
103
|
listQueues(): string[];
|
|
174
|
-
/** Register queue name in cache - called when first job is pushed */
|
|
175
104
|
private registerQueueName;
|
|
176
|
-
/** Unregister queue name from cache - called on obliterate */
|
|
177
105
|
private unregisterQueueName;
|
|
178
106
|
clean(queue: string, graceMs: number, state?: string, limit?: number): number;
|
|
179
|
-
/** Get job counts grouped by priority for a queue */
|
|
180
107
|
getCountsPerPriority(queue: string): Record<number, number>;
|
|
181
|
-
/**
|
|
182
|
-
* Get jobs with filtering and pagination
|
|
183
|
-
* @param queue - Queue name
|
|
184
|
-
* @param options - Filter options
|
|
185
|
-
* @returns Array of jobs matching the criteria
|
|
186
|
-
*/
|
|
187
108
|
getJobs(queue: string, options?: {
|
|
188
109
|
state?: 'waiting' | 'delayed' | 'active' | 'completed' | 'failed';
|
|
189
110
|
start?: number;
|
|
@@ -193,16 +114,7 @@ export declare class QueueManager {
|
|
|
193
114
|
getDlq(queue: string, count?: number): Job[];
|
|
194
115
|
retryDlq(queue: string, jobId?: JobId): number;
|
|
195
116
|
purgeDlq(queue: string): number;
|
|
196
|
-
/**
|
|
197
|
-
* Retry a completed job by re-queueing it
|
|
198
|
-
* @param queue - Queue name
|
|
199
|
-
* @param jobId - Specific job ID to retry (optional - retries all if not specified)
|
|
200
|
-
* @returns Number of jobs retried
|
|
201
|
-
*/
|
|
202
117
|
retryCompleted(queue: string, jobId?: JobId): number;
|
|
203
|
-
/**
|
|
204
|
-
* Internal helper to re-queue a completed job
|
|
205
|
-
*/
|
|
206
118
|
private requeueCompletedJob;
|
|
207
119
|
setRateLimit(queue: string, limit: number): void;
|
|
208
120
|
clearRateLimit(queue: string): void;
|
|
@@ -224,91 +136,17 @@ export declare class QueueManager {
|
|
|
224
136
|
getCron(name: string): CronJob | undefined;
|
|
225
137
|
listCrons(): CronJob[];
|
|
226
138
|
subscribe(callback: (event: JobEvent) => void): () => void;
|
|
227
|
-
/** Wait for job completion - event-driven, no polling */
|
|
228
139
|
waitForJobCompletion(jobId: JobId, timeoutMs: number): Promise<boolean>;
|
|
229
|
-
/** Get job index for dependency validation */
|
|
230
140
|
getJobIndex(): Map<JobId, JobLocation>;
|
|
231
|
-
/** Get completed jobs set for dependency validation */
|
|
232
141
|
getCompletedJobs(): SetLike<JobId>;
|
|
233
|
-
/**
|
|
234
|
-
|
|
235
|
-
* This avoids lock order violations by not iterating shards while holding locks
|
|
236
|
-
*/
|
|
142
|
+
/** Expose shards for testing (internal use only) */
|
|
143
|
+
getShards(): Shard[];
|
|
237
144
|
private onJobCompleted;
|
|
238
|
-
/**
|
|
239
|
-
* Batch version of onJobCompleted - more efficient for large batches
|
|
240
|
-
*/
|
|
241
145
|
private onJobsCompleted;
|
|
242
|
-
/**
|
|
243
|
-
* Check if there are any jobs waiting for dependencies
|
|
244
|
-
* Used to skip dependency tracking when not needed
|
|
245
|
-
*/
|
|
246
146
|
private hasPendingDeps;
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
* Uses reverse index for O(m) where m = jobs waiting on completed deps
|
|
250
|
-
* Instead of O(n) full scan of all waiting deps
|
|
251
|
-
*/
|
|
252
|
-
private processPendingDependencies;
|
|
253
|
-
private startBackgroundTasks;
|
|
254
|
-
private checkJobTimeouts;
|
|
255
|
-
/**
|
|
256
|
-
* Check for stalled jobs and handle them
|
|
257
|
-
* Uses two-phase detection (like BullMQ) to prevent false positives:
|
|
258
|
-
* - Phase 1: Jobs marked as candidates in previous check are confirmed stalled
|
|
259
|
-
* - Phase 2: Current processing jobs are marked as candidates for next check
|
|
260
|
-
*/
|
|
261
|
-
private checkStalledJobs;
|
|
262
|
-
/**
|
|
263
|
-
* Handle a stalled job based on the action
|
|
264
|
-
*/
|
|
265
|
-
private handleStalledJob;
|
|
266
|
-
/**
|
|
267
|
-
* Perform DLQ maintenance: auto-retry and purge expired
|
|
268
|
-
*/
|
|
269
|
-
private performDlqMaintenance;
|
|
270
|
-
private recover;
|
|
271
|
-
private cleanup;
|
|
272
|
-
shutdown(): void;
|
|
273
|
-
getStats(): {
|
|
274
|
-
waiting: number;
|
|
275
|
-
delayed: number;
|
|
276
|
-
active: number;
|
|
277
|
-
dlq: number;
|
|
278
|
-
completed: number;
|
|
279
|
-
totalPushed: bigint;
|
|
280
|
-
totalPulled: bigint;
|
|
281
|
-
totalCompleted: bigint;
|
|
282
|
-
totalFailed: bigint;
|
|
283
|
-
uptime: number;
|
|
284
|
-
cronJobs: number;
|
|
285
|
-
cronPending: number;
|
|
286
|
-
};
|
|
287
|
-
/**
|
|
288
|
-
* Get detailed memory statistics for debugging memory issues.
|
|
289
|
-
* Returns counts of entries in all major collections.
|
|
290
|
-
*/
|
|
291
|
-
getMemoryStats(): {
|
|
292
|
-
jobIndex: number;
|
|
293
|
-
completedJobs: number;
|
|
294
|
-
jobResults: number;
|
|
295
|
-
jobLogs: number;
|
|
296
|
-
customIdMap: number;
|
|
297
|
-
jobLocks: number;
|
|
298
|
-
clientJobs: number;
|
|
299
|
-
clientJobsTotal: number;
|
|
300
|
-
pendingDepChecks: number;
|
|
301
|
-
stalledCandidates: number;
|
|
302
|
-
processingTotal: number;
|
|
303
|
-
queuedTotal: number;
|
|
304
|
-
waitingDepsTotal: number;
|
|
305
|
-
temporalIndexTotal: number;
|
|
306
|
-
delayedHeapTotal: number;
|
|
307
|
-
};
|
|
308
|
-
/**
|
|
309
|
-
* Force compact all collections to reduce memory usage.
|
|
310
|
-
* Use after large batch operations or when memory pressure is high.
|
|
311
|
-
*/
|
|
147
|
+
getStats(): statsMgr.QueueStats;
|
|
148
|
+
getMemoryStats(): statsMgr.MemoryStats;
|
|
312
149
|
compactMemory(): void;
|
|
150
|
+
shutdown(): void;
|
|
313
151
|
}
|
|
314
152
|
//# sourceMappingURL=queueManager.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queueManager.d.ts","sourceRoot":"","sources":["../../src/application/queueManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"queueManager.d.ts","sourceRoot":"","sources":["../../src/application/queueManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAEpF,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACnE,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAG9C,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAmBhD,OAAO,EAAkC,KAAK,OAAO,EAAE,MAAM,eAAe,CAAC;AAG7E,OAAO,KAAK,EAAE,kBAAkB,EAAgD,MAAM,SAAS,CAAC;AAIhG,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAG3C,YAAY,EAAE,kBAAkB,EAAE,CAAC;AAEnC;;GAEG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgD;IACvE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuB;IAG/C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAgB;IAC3C,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAyB;IAC1D,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAgB;IAGhD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAiC;IAC1D,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA8B;IACzD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyB;IACrD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAgC;IAGxD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAoB;IAGrD,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAoB;IAGtD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA6B;IACtD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAiC;IAG5D,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAgB;IAG9C,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;IACxC,QAAQ,CAAC,aAAa,EAAE,aAAa,CAAC;IACtC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAgB;IAG9C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAO;IAGrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAKtB;IACF,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAc;IAGxC,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAA8C;IAGpF,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAqB;gBAEzC,MAAM,GAAE,kBAAuB;IA6C3C,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,oBAAoB;IA4B5B,OAAO,CAAC,eAAe;IAkBvB,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,aAAa;IAqBrB,OAAO,CAAC,YAAY;IAyBpB,OAAO,CAAC,iBAAiB;IAYzB,OAAO,CAAC,eAAe;IAcvB,OAAO,CAAC,aAAa;IAUf,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC;IAKlD,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAK9D,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,GAAE,MAAU,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC;IAI/D,YAAY,CAChB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,SAAS,GAAE,MAAU,EACrB,OAAO,GAAE,MAAyB,GACjC,OAAO,CAAC;QAAE,GAAG,EAAE,GAAG,GAAG,IAAI,CAAC;QAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC;IAO/C,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,GAAE,MAAU,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAI9E,iBAAiB,CACrB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,SAAS,GAAE,MAAU,EACrB,OAAO,GAAE,MAAyB,GACjC,OAAO,CAAC;QAAE,IAAI,EAAE,GAAG,EAAE,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAUvC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQlE,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAiB3D,mBAAmB,CACvB,KAAK,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,KAAK,CAAC;QAAC,MAAM,EAAE,OAAO,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,GAC3D,OAAO,CAAC,IAAI,CAAC;IAYV,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQvE,YAAY,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO;IAiBnD,iBAAiB,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM;IAU7D,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,GAAE,MAAyB,GAAG,SAAS,GAAG,IAAI;IAIzF,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO;IAIhD,YAAY,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO;IAInE,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,KAAK,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,MAAM,EAAE;IAIrF,WAAW,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO;IAIlD,WAAW,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,GAAG,IAAI;IAMzC,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI;IAIvD,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI;IAIrE,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAM9C,MAAM,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC;IAI/C,SAAS,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO;IAIhC,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,GAAG,GAAG,IAAI;IAI9C,WAAW,CAAC,KAAK,EAAE,KAAK;;;;IAIxB,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAM5B,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAI1B,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAI3B,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO;IAIhC,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAI5B,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAK/B,UAAU,IAAI,MAAM,EAAE;IAItB,OAAO,CAAC,iBAAiB;IAIzB,OAAO,CAAC,mBAAmB;IAI3B,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM;IAU7E,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IAM3D,OAAO,CACL,KAAK,EAAE,MAAM,EACb,OAAO,GAAE;QACP,KAAK,CAAC,EAAE,SAAS,GAAG,SAAS,GAAG,QAAQ,GAAG,WAAW,GAAG,QAAQ,CAAC;QAClE,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,GAAG,CAAC,EAAE,OAAO,CAAC;KACV,GACL,GAAG,EAAE;IAwCR,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,GAAG,EAAE;IAI5C,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM;IAI9C,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAI/B,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM;IAepD,OAAO,CAAC,mBAAmB;IAqB3B,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAIhD,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAInC,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAIlD,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAM/B,MAAM,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC;IAItC,cAAc,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAIlF,aAAa,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAI5D,cAAc,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAIhE,OAAO,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC;IAIvC,aAAa,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAI5D,OAAO,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC;IAM7C,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,GAAE,MAAM,GAAG,MAAM,GAAG,OAAgB,GAAG,OAAO;IASzF,OAAO,CAAC,KAAK,EAAE,KAAK,GAAG,WAAW,EAAE;IAQpC,SAAS,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI;IAU7B,oBAAoB,IAAI,MAAM;IAM9B,OAAO,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO;IAMrC,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAMjC,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS;IAI1C,SAAS,IAAI,OAAO,EAAE;IAMtB,SAAS,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,GAAG,MAAM,IAAI;IAI1D,oBAAoB,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAMvE,WAAW,IAAI,GAAG,CAAC,KAAK,EAAE,WAAW,CAAC;IAItC,gBAAgB,IAAI,OAAO,CAAC,KAAK,CAAC;IAIlC,oDAAoD;IACpD,SAAS,IAAI,KAAK,EAAE;IAIpB,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,eAAe;IAIvB,OAAO,CAAC,cAAc;IAStB,QAAQ;IAIR,cAAc;IAId,aAAa,IAAI,IAAI;IAMrB,QAAQ,IAAI,IAAI;CA6BjB"}
|