bunqueue-client 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/dist/canonical-manifest.json +57 -53
- package/dist/embedded.js +1353 -1261
- package/dist/index.js +52 -4
- package/dist/legacy.js +1 -1
- package/dist/types/sdk/typescript/src/index.d.ts +1 -1
- package/dist/types/sdk/typescript/src/legacy.d.ts +1 -1
- package/dist/types/src/application/clientOwnership.d.ts +65 -0
- package/dist/types/src/application/clientTracking.d.ts +7 -11
- package/dist/types/src/application/operations/jobClaim.d.ts +2 -2
- package/dist/types/src/application/operations/jobManagement.d.ts +2 -0
- package/dist/types/src/application/orphanRecovery.d.ts +25 -0
- package/dist/types/src/application/queue-manager/state.d.ts +2 -0
- package/dist/types/src/application/stallDetection.d.ts +16 -0
- package/dist/types/src/application/types/contextFactory.d.ts +2 -0
- package/dist/types/src/application/types/contexts.d.ts +3 -0
- package/dist/types/src/application/types/queueManager.d.ts +2 -0
- package/dist/types/src/client/forwarder.d.ts +4 -3
- package/dist/types/src/client/types/options.d.ts +2 -0
- package/dist/types/src/domain/job/constants.d.ts +2 -0
- package/dist/types/src/domain/job/create.d.ts +7 -0
- package/dist/types/src/domain/job/locks.d.ts +13 -1
- package/dist/types/src/domain/job/state.d.ts +15 -0
- package/dist/types/src/domain/types/commands/core.d.ts +2 -5
- package/dist/types/src/domain/types/cron.d.ts +2 -4
- package/dist/types/src/domain/types/jobs/model.d.ts +1 -4
- package/package.json +1 -1
- package/src/index.ts +1 -1
- package/src/legacy.ts +1 -1
package/dist/embedded.js
CHANGED
|
@@ -199,1069 +199,1409 @@ async function withWriteLock(lock, fn, timeoutMs) {
|
|
|
199
199
|
}
|
|
200
200
|
}
|
|
201
201
|
|
|
202
|
-
// ../../src/application/
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
this.onRecentEvict = onRecentEvict;
|
|
216
|
-
this.maxRecent = Math.max(1, Math.trunc(maxRecent));
|
|
217
|
-
}
|
|
218
|
-
add(jobId, queue) {
|
|
219
|
-
if (queue !== undefined)
|
|
220
|
-
this.owners.set(jobId, queue);
|
|
221
|
-
if (this.pinned.has(jobId) || this.recent.has(jobId))
|
|
222
|
-
return;
|
|
223
|
-
if (this.recent.size >= this.maxRecent)
|
|
224
|
-
this.evictOldestRecent();
|
|
225
|
-
if (this.nextRecentToken >= Number.MAX_SAFE_INTEGER)
|
|
226
|
-
this.compactRecentOrder(true);
|
|
227
|
-
const token = this.nextRecentToken++;
|
|
228
|
-
this.recent.add(jobId);
|
|
229
|
-
this.recentTokens.set(jobId, token);
|
|
230
|
-
this.recentOrder.push(jobId);
|
|
231
|
-
this.recentOrderTokens.push(token);
|
|
232
|
-
}
|
|
233
|
-
pin(jobId, queue) {
|
|
234
|
-
if (queue !== undefined)
|
|
235
|
-
this.owners.set(jobId, queue);
|
|
236
|
-
if (this.recent.delete(jobId)) {
|
|
237
|
-
this.recentTokens.delete(jobId);
|
|
238
|
-
this.compactRecentOrder();
|
|
239
|
-
}
|
|
240
|
-
this.pinned.add(jobId);
|
|
241
|
-
}
|
|
242
|
-
unpin(jobId, retainAsRecent) {
|
|
243
|
-
this.pinned.delete(jobId);
|
|
244
|
-
if (retainAsRecent)
|
|
245
|
-
this.add(jobId);
|
|
246
|
-
else
|
|
247
|
-
this.delete(jobId);
|
|
248
|
-
}
|
|
249
|
-
hydrate(records) {
|
|
250
|
-
this.clear();
|
|
251
|
-
for (const record of records) {
|
|
252
|
-
if (record.pinned)
|
|
253
|
-
this.pin(record.jobId, record.queue);
|
|
254
|
-
else
|
|
255
|
-
this.add(record.jobId, record.queue);
|
|
256
|
-
}
|
|
257
|
-
}
|
|
258
|
-
pinnedValues() {
|
|
259
|
-
return this.pinned.values();
|
|
260
|
-
}
|
|
261
|
-
has(jobId) {
|
|
262
|
-
return this.pinned.has(jobId) || this.recent.has(jobId);
|
|
263
|
-
}
|
|
264
|
-
delete(jobId) {
|
|
265
|
-
const wasPinned = this.pinned.delete(jobId);
|
|
266
|
-
const wasRecent = this.recent.delete(jobId);
|
|
267
|
-
if (wasRecent) {
|
|
268
|
-
this.recentTokens.delete(jobId);
|
|
269
|
-
this.compactRecentOrder();
|
|
270
|
-
}
|
|
271
|
-
this.owners.delete(jobId);
|
|
272
|
-
return wasRecent || wasPinned;
|
|
273
|
-
}
|
|
274
|
-
deleteForQueue(queue) {
|
|
275
|
-
const removed = [];
|
|
276
|
-
for (const [jobId, owner] of this.owners) {
|
|
277
|
-
if (owner !== queue)
|
|
278
|
-
continue;
|
|
279
|
-
if (this.delete(jobId))
|
|
280
|
-
removed.push(jobId);
|
|
281
|
-
}
|
|
282
|
-
return removed;
|
|
283
|
-
}
|
|
284
|
-
clear() {
|
|
285
|
-
this.recent.clear();
|
|
286
|
-
this.recentOrder = [];
|
|
287
|
-
this.recentOrderTokens = [];
|
|
288
|
-
this.recentTokens.clear();
|
|
289
|
-
this.recentOrderHead = 0;
|
|
290
|
-
this.nextRecentToken = 0;
|
|
291
|
-
this.pinned.clear();
|
|
292
|
-
this.owners.clear();
|
|
293
|
-
}
|
|
294
|
-
get size() {
|
|
295
|
-
return this.recent.size + this.pinned.size;
|
|
296
|
-
}
|
|
297
|
-
evictOldestRecent() {
|
|
298
|
-
while (this.recentOrderHead < this.recentOrder.length) {
|
|
299
|
-
const index = this.recentOrderHead++;
|
|
300
|
-
const oldest = this.recentOrder[index];
|
|
301
|
-
const token = this.recentOrderTokens[index];
|
|
302
|
-
if (this.recentTokens.get(oldest) !== token)
|
|
303
|
-
continue;
|
|
304
|
-
this.recent.delete(oldest);
|
|
305
|
-
this.recentTokens.delete(oldest);
|
|
306
|
-
this.owners.delete(oldest);
|
|
307
|
-
this.compactRecentOrder();
|
|
308
|
-
this.onRecentEvict?.(oldest);
|
|
309
|
-
return;
|
|
310
|
-
}
|
|
202
|
+
// ../../src/application/clientOwnership.ts
|
|
203
|
+
function removeFromClient(clientId, jobId, ctx) {
|
|
204
|
+
const jobs = ctx.clientJobs.get(clientId);
|
|
205
|
+
if (!jobs)
|
|
206
|
+
return;
|
|
207
|
+
jobs.delete(jobId);
|
|
208
|
+
if (jobs.size === 0)
|
|
209
|
+
ctx.clientJobs.delete(clientId);
|
|
210
|
+
}
|
|
211
|
+
function registerClientJob(clientId, jobId, ctx) {
|
|
212
|
+
const previous = ctx.clientJobOwners.get(jobId);
|
|
213
|
+
if (previous && previous.clientId !== clientId) {
|
|
214
|
+
removeFromClient(previous.clientId, jobId, ctx);
|
|
311
215
|
}
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
const tokens = [];
|
|
318
|
-
for (let index = this.recentOrderHead;index < this.recentOrder.length; index++) {
|
|
319
|
-
const jobId = this.recentOrder[index];
|
|
320
|
-
const token = this.recentOrderTokens[index];
|
|
321
|
-
if (this.recentTokens.get(jobId) !== token)
|
|
322
|
-
continue;
|
|
323
|
-
const compactedToken = order.length;
|
|
324
|
-
order.push(jobId);
|
|
325
|
-
tokens.push(compactedToken);
|
|
326
|
-
}
|
|
327
|
-
this.recentTokens.clear();
|
|
328
|
-
for (let index = 0;index < order.length; index++) {
|
|
329
|
-
this.recentTokens.set(order[index], tokens[index]);
|
|
330
|
-
}
|
|
331
|
-
this.recentOrder = order;
|
|
332
|
-
this.recentOrderTokens = tokens;
|
|
333
|
-
this.recentOrderHead = 0;
|
|
334
|
-
this.nextRecentToken = order.length;
|
|
216
|
+
ctx.clientJobOwners.set(jobId, { clientId, delivery: ctx.jobIndex.get(jobId) });
|
|
217
|
+
let jobs = ctx.clientJobs.get(clientId);
|
|
218
|
+
if (!jobs) {
|
|
219
|
+
jobs = new Set;
|
|
220
|
+
ctx.clientJobs.set(clientId, jobs);
|
|
335
221
|
}
|
|
222
|
+
jobs.add(jobId);
|
|
336
223
|
}
|
|
337
|
-
function
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
if (pinned)
|
|
344
|
-
ctx.depCompletions?.pin(job.id, job.queue);
|
|
345
|
-
else
|
|
346
|
-
ctx.depCompletions?.add(job.id, job.queue);
|
|
224
|
+
function unregisterClientJob(clientId, jobId, ctx) {
|
|
225
|
+
if (!clientId)
|
|
226
|
+
return;
|
|
227
|
+
removeFromClient(clientId, jobId, ctx);
|
|
228
|
+
if (ctx.clientJobOwners.get(jobId)?.clientId === clientId)
|
|
229
|
+
ctx.clientJobOwners.delete(jobId);
|
|
347
230
|
}
|
|
348
|
-
function
|
|
349
|
-
const
|
|
350
|
-
if (
|
|
231
|
+
function detachClientJob(jobId, ctx) {
|
|
232
|
+
const owner = ctx.clientJobOwners.get(jobId);
|
|
233
|
+
if (!owner)
|
|
351
234
|
return;
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
235
|
+
ctx.clientJobOwners.delete(jobId);
|
|
236
|
+
removeFromClient(owner.clientId, jobId, ctx);
|
|
237
|
+
}
|
|
238
|
+
function ownsCurrentDelivery(clientId, jobId, ctx) {
|
|
239
|
+
const owner = ctx.clientJobOwners.get(jobId);
|
|
240
|
+
if (owner?.clientId !== clientId)
|
|
241
|
+
return false;
|
|
242
|
+
const location = ctx.jobIndex.get(jobId);
|
|
243
|
+
return location?.type === "processing" && location === owner.delivery;
|
|
244
|
+
}
|
|
245
|
+
function dropClient(clientId, ctx) {
|
|
246
|
+
const jobs = ctx.clientJobs.get(clientId);
|
|
247
|
+
ctx.clientJobs.delete(clientId);
|
|
248
|
+
if (!jobs)
|
|
355
249
|
return;
|
|
250
|
+
for (const jobId of jobs) {
|
|
251
|
+
if (ctx.clientJobOwners.get(jobId)?.clientId === clientId)
|
|
252
|
+
ctx.clientJobOwners.delete(jobId);
|
|
356
253
|
}
|
|
357
|
-
for (const id of candidates)
|
|
358
|
-
ctx.depCompletions?.unpin(id, true);
|
|
359
254
|
}
|
|
360
|
-
function
|
|
361
|
-
|
|
362
|
-
for (const
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
return;
|
|
255
|
+
function pruneEndedClientDeliveries(ctx) {
|
|
256
|
+
let pruned = 0;
|
|
257
|
+
for (const [jobId, owner] of ctx.clientJobOwners) {
|
|
258
|
+
const location = ctx.jobIndex.get(jobId);
|
|
259
|
+
if (location?.type === "processing" && location === owner.delivery)
|
|
260
|
+
continue;
|
|
261
|
+
ctx.clientJobOwners.delete(jobId);
|
|
262
|
+
removeFromClient(owner.clientId, jobId, ctx);
|
|
263
|
+
pruned++;
|
|
370
264
|
}
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
265
|
+
return pruned;
|
|
266
|
+
}
|
|
267
|
+
// ../../src/domain/job/constants.ts
|
|
268
|
+
var DEFAULT_MAX_BACKOFF = 3600000;
|
|
269
|
+
var MAX_TIMELINE_ENTRIES = 20;
|
|
270
|
+
var DEFAULT_LOCK_TTL = 30000;
|
|
271
|
+
var JOB_DEFAULTS = {
|
|
272
|
+
priority: 0,
|
|
273
|
+
maxAttempts: 3,
|
|
274
|
+
backoff: 1000,
|
|
275
|
+
lifo: false,
|
|
276
|
+
removeOnComplete: false,
|
|
277
|
+
removeOnFail: false,
|
|
278
|
+
stackTraceLimit: 10
|
|
279
|
+
};
|
|
280
|
+
var MAX_BACKOFF_DELAY = 86400000;
|
|
281
|
+
// ../../src/domain/job/payload.ts
|
|
282
|
+
var DEFAULT_JOB_NAME = "default";
|
|
283
|
+
var MAX_JOB_NAME_LENGTH = 256;
|
|
284
|
+
function legacyEnvelope(data) {
|
|
285
|
+
if (typeof data !== "object" || data === null || Array.isArray(data))
|
|
286
|
+
return null;
|
|
287
|
+
const record = data;
|
|
288
|
+
if (typeof record.name !== "string")
|
|
289
|
+
return null;
|
|
290
|
+
const { name, ...userData } = record;
|
|
291
|
+
return { name, data: userData };
|
|
292
|
+
}
|
|
293
|
+
function validateJobName(name) {
|
|
294
|
+
if (typeof name !== "string" || name.length === 0 || name.length > MAX_JOB_NAME_LENGTH) {
|
|
295
|
+
return "Job name must be a non-empty string of at most 256 characters";
|
|
374
296
|
}
|
|
297
|
+
return null;
|
|
375
298
|
}
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
const rightPoint = right.codePointAt(rightIndex);
|
|
386
|
-
if (leftPoint === undefined || rightPoint === undefined)
|
|
387
|
-
break;
|
|
388
|
-
if (leftPoint !== rightPoint)
|
|
389
|
-
return leftPoint - rightPoint;
|
|
390
|
-
leftIndex += leftPoint > 65535 ? 2 : 1;
|
|
391
|
-
rightIndex += rightPoint > 65535 ? 2 : 1;
|
|
299
|
+
function checkedName(name) {
|
|
300
|
+
const error = validateJobName(name);
|
|
301
|
+
if (error)
|
|
302
|
+
throw new Error(error);
|
|
303
|
+
return name;
|
|
304
|
+
}
|
|
305
|
+
function normalizeJobPayload(source, fallback = DEFAULT_JOB_NAME) {
|
|
306
|
+
if (source.name !== undefined) {
|
|
307
|
+
return { name: checkedName(source.name), data: source.data, legacy: false };
|
|
392
308
|
}
|
|
393
|
-
return
|
|
309
|
+
return { name: checkedName(fallback), data: source.data, legacy: false };
|
|
310
|
+
}
|
|
311
|
+
function normalizeLegacyJobPayload(source, fallback = DEFAULT_JOB_NAME) {
|
|
312
|
+
if (source.name !== undefined)
|
|
313
|
+
return normalizeJobPayload(source, fallback);
|
|
314
|
+
const legacy = legacyEnvelope(source.data);
|
|
315
|
+
if (legacy)
|
|
316
|
+
return { name: checkedName(legacy.name), data: legacy.data, legacy: true };
|
|
317
|
+
return { name: checkedName(fallback), data: source.data, legacy: false };
|
|
394
318
|
}
|
|
395
319
|
|
|
396
|
-
// ../../src/
|
|
397
|
-
function
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
if (queue !== null && location.queueName !== queue)
|
|
402
|
-
return [];
|
|
403
|
-
if (ctx.dependencyResults.hasConsumers(jobId))
|
|
404
|
-
return [];
|
|
405
|
-
const job = ctx.completedJobsData.get(jobId);
|
|
406
|
-
const timestamp = job?.completedAt ?? job?.createdAt ?? 0;
|
|
407
|
-
return timestamp <= completedBefore ? [{ jobId, timestamp }] : [];
|
|
408
|
-
}).sort((left, right) => left.timestamp - right.timestamp || compareSqliteBinaryText(left.jobId, right.jobId)).slice(0, limit).map(({ jobId }) => ({ jobId, queue: ctx.jobIndex.get(jobId)?.queueName ?? "" }));
|
|
320
|
+
// ../../src/domain/job/create.ts
|
|
321
|
+
function parseMaxDelay(value) {
|
|
322
|
+
if (typeof value !== "number" || !Number.isFinite(value))
|
|
323
|
+
return;
|
|
324
|
+
return value >= 0 && value <= MAX_BACKOFF_DELAY ? value : undefined;
|
|
409
325
|
}
|
|
410
|
-
function
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
return
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
const ownsCurrentCompletion = location?.type === "completed" && location.queueName === ownerQueue;
|
|
418
|
-
const hasNoCurrentGeneration = location === undefined;
|
|
419
|
-
const resultOwner = ctx.jobResultQueues?.get(jobId);
|
|
420
|
-
if (resultOwner === ownerQueue || resultOwner === undefined && ownsCurrentCompletion) {
|
|
421
|
-
ctx.jobResults?.delete(jobId);
|
|
422
|
-
ctx.jobResultQueues?.delete(jobId);
|
|
423
|
-
}
|
|
424
|
-
const logOwner = ctx.jobLogQueues?.get(jobId);
|
|
425
|
-
if (logOwner === ownerQueue || logOwner === undefined && ownsCurrentCompletion) {
|
|
426
|
-
ctx.jobLogs?.delete(jobId);
|
|
427
|
-
ctx.jobLogQueues?.delete(jobId);
|
|
428
|
-
}
|
|
429
|
-
if (!hasNoCurrentGeneration && !ownsCurrentCompletion)
|
|
430
|
-
continue;
|
|
431
|
-
ctx.completedJobs.delete(jobId);
|
|
432
|
-
ctx.completedJobsData.delete(jobId);
|
|
433
|
-
ctx.jobIndex.delete(jobId);
|
|
434
|
-
ctx.dependencyResults.releaseConsumer(jobId);
|
|
326
|
+
function parseBackoff(input) {
|
|
327
|
+
if (typeof input === "object" && input !== null) {
|
|
328
|
+
const maxDelay = parseMaxDelay(input.maxDelay);
|
|
329
|
+
return {
|
|
330
|
+
backoff: input.delay,
|
|
331
|
+
backoffConfig: maxDelay === undefined ? { type: input.type, delay: input.delay } : { type: input.type, delay: input.delay, maxDelay }
|
|
332
|
+
};
|
|
435
333
|
}
|
|
436
|
-
return
|
|
334
|
+
return { backoff: input ?? JOB_DEFAULTS.backoff, backoffConfig: null };
|
|
437
335
|
}
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
}
|
|
454
|
-
for (const shard of ctx.shards) {
|
|
455
|
-
for (const job of shard.waitingDeps.values())
|
|
456
|
-
occupied.add(job.queue);
|
|
457
|
-
for (const job of shard.waitingChildren.values())
|
|
458
|
-
occupied.add(job.queue);
|
|
459
|
-
}
|
|
460
|
-
for (const [queue, count] of ctx.pendingQueueAdmissions) {
|
|
461
|
-
if (count > 0)
|
|
462
|
-
occupied.add(queue);
|
|
463
|
-
}
|
|
464
|
-
return occupied;
|
|
336
|
+
function parseRepeatConfig(repeat) {
|
|
337
|
+
if (!repeat)
|
|
338
|
+
return null;
|
|
339
|
+
return {
|
|
340
|
+
every: repeat.every,
|
|
341
|
+
limit: repeat.limit,
|
|
342
|
+
pattern: repeat.pattern,
|
|
343
|
+
count: repeat.count ?? 0,
|
|
344
|
+
startDate: repeat.startDate,
|
|
345
|
+
endDate: repeat.endDate,
|
|
346
|
+
tz: repeat.tz,
|
|
347
|
+
immediately: repeat.immediately,
|
|
348
|
+
prevMillis: repeat.prevMillis,
|
|
349
|
+
offset: repeat.offset,
|
|
350
|
+
jobId: repeat.jobId
|
|
351
|
+
};
|
|
465
352
|
}
|
|
466
|
-
function
|
|
467
|
-
|
|
468
|
-
for (const shard of ctx.shards) {
|
|
469
|
-
for (const queue of shard.getConfiguredQueueNames())
|
|
470
|
-
configured.add(queue);
|
|
471
|
-
}
|
|
472
|
-
return configured;
|
|
353
|
+
function toBoolean(value, fallback) {
|
|
354
|
+
return value === undefined ? fallback : Boolean(value);
|
|
473
355
|
}
|
|
474
|
-
function
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
queues.add(job.queue);
|
|
483
|
-
}
|
|
484
|
-
return queues;
|
|
356
|
+
function parseCoreOptions(input) {
|
|
357
|
+
return {
|
|
358
|
+
priority: input.priority ?? JOB_DEFAULTS.priority,
|
|
359
|
+
lifo: input.lifo ?? JOB_DEFAULTS.lifo,
|
|
360
|
+
maxAttempts: input.maxAttempts ?? JOB_DEFAULTS.maxAttempts,
|
|
361
|
+
removeOnComplete: toBoolean(input.removeOnComplete, JOB_DEFAULTS.removeOnComplete),
|
|
362
|
+
removeOnFail: toBoolean(input.removeOnFail, JOB_DEFAULTS.removeOnFail)
|
|
363
|
+
};
|
|
485
364
|
}
|
|
486
|
-
function
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
}
|
|
497
|
-
return counts;
|
|
365
|
+
function parseOptionalFields(input) {
|
|
366
|
+
return {
|
|
367
|
+
ttl: input.ttl ?? null,
|
|
368
|
+
timeout: input.timeout ?? null,
|
|
369
|
+
uniqueKey: input.uniqueKey ?? null,
|
|
370
|
+
customId: input.customId ?? null,
|
|
371
|
+
parentId: input.parentId ?? null,
|
|
372
|
+
groupId: input.groupId ?? null,
|
|
373
|
+
stallTimeout: input.stallTimeout ?? null
|
|
374
|
+
};
|
|
498
375
|
}
|
|
499
|
-
function
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
continue;
|
|
515
|
-
}
|
|
516
|
-
ctx.storage?.deleteQueueState(queueName);
|
|
517
|
-
ctx.storage?.deleteGroupState(queueName);
|
|
518
|
-
ctx.unregisterQueueName(queueName);
|
|
519
|
-
shard.obliterate(queueName);
|
|
520
|
-
removed.push(queueName);
|
|
521
|
-
}
|
|
522
|
-
for (const shard of ctx.shards)
|
|
523
|
-
shard.cleanOrphanedTemporalEntries();
|
|
524
|
-
for (const queue of removed)
|
|
525
|
-
ctx.dashboardEmit?.("queue:removed", { queue });
|
|
376
|
+
function parseBullMQV5Options(input) {
|
|
377
|
+
return {
|
|
378
|
+
stackTraceLimit: input.stackTraceLimit ?? JOB_DEFAULTS.stackTraceLimit,
|
|
379
|
+
keepLogs: input.keepLogs ?? null,
|
|
380
|
+
sizeLimit: input.sizeLimit ?? null,
|
|
381
|
+
failParentOnFailure: input.failParentOnFailure ?? false,
|
|
382
|
+
removeDependencyOnFailure: input.removeDependencyOnFailure ?? false,
|
|
383
|
+
continueParentOnFailure: input.continueParentOnFailure ?? false,
|
|
384
|
+
ignoreDependencyOnFailure: input.ignoreDependencyOnFailure ?? false,
|
|
385
|
+
deduplicationTtl: input.dedup?.ttl ?? null,
|
|
386
|
+
deduplicationExtend: input.dedup?.extend ?? false,
|
|
387
|
+
deduplicationReplace: input.dedup?.replace ?? false,
|
|
388
|
+
debounceId: input.debounceId ?? null,
|
|
389
|
+
debounceTtl: input.debounceTtl ?? null
|
|
390
|
+
};
|
|
526
391
|
}
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
const
|
|
531
|
-
const
|
|
532
|
-
const
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
392
|
+
function createJob(id, queue, input, now = Date.now()) {
|
|
393
|
+
const { backoff, backoffConfig } = parseBackoff(input.backoff);
|
|
394
|
+
const coreOpts = parseCoreOptions(input);
|
|
395
|
+
const optionalFields = parseOptionalFields(input);
|
|
396
|
+
const v5Opts = parseBullMQV5Options(input);
|
|
397
|
+
const createdAt = input.timestamp ?? now;
|
|
398
|
+
const payload = normalizeJobPayload(input);
|
|
399
|
+
return {
|
|
400
|
+
id,
|
|
401
|
+
queue,
|
|
402
|
+
name: payload.name,
|
|
403
|
+
data: payload.data,
|
|
404
|
+
createdAt,
|
|
405
|
+
runAt: createdAt + (input.delay ?? 0),
|
|
406
|
+
startedAt: null,
|
|
407
|
+
completedAt: null,
|
|
408
|
+
attempts: 0,
|
|
409
|
+
backoff,
|
|
410
|
+
backoffConfig,
|
|
411
|
+
dependsOn: input.dependsOn ?? [],
|
|
412
|
+
childrenIds: input.childrenIds ?? [],
|
|
413
|
+
childrenCompleted: 0,
|
|
414
|
+
tags: input.tags ?? [],
|
|
415
|
+
progress: 0,
|
|
416
|
+
progressMessage: null,
|
|
417
|
+
stacktrace: null,
|
|
418
|
+
repeat: parseRepeatConfig(input.repeat),
|
|
419
|
+
durable: input.durable ?? false,
|
|
420
|
+
lastHeartbeat: createdAt,
|
|
421
|
+
stallCount: 0,
|
|
422
|
+
...coreOpts,
|
|
423
|
+
...optionalFields,
|
|
424
|
+
...v5Opts,
|
|
425
|
+
timeline: []
|
|
426
|
+
};
|
|
556
427
|
}
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
428
|
+
// ../../src/domain/job/ids.ts
|
|
429
|
+
function isWellFormedJobId(id) {
|
|
430
|
+
for (let index = 0;index < id.length; index++) {
|
|
431
|
+
const unit = id.charCodeAt(index);
|
|
432
|
+
if (unit >= 55296 && unit <= 56319) {
|
|
433
|
+
const next = id.charCodeAt(++index);
|
|
434
|
+
if (!(next >= 56320 && next <= 57343))
|
|
435
|
+
return false;
|
|
436
|
+
} else if (unit >= 56320 && unit <= 57343) {
|
|
437
|
+
return false;
|
|
564
438
|
}
|
|
565
|
-
if (orphaned.length === 0)
|
|
566
|
-
continue;
|
|
567
|
-
await withWriteLock(ctx.processingLocks[i], () => {
|
|
568
|
-
let removed = 0;
|
|
569
|
-
for (const jobId of orphaned) {
|
|
570
|
-
const job = ctx.processingShards[i].get(jobId);
|
|
571
|
-
if (job) {
|
|
572
|
-
ctx.processingShards[i].delete(jobId);
|
|
573
|
-
ctx.timeoutScheduler.cancel(jobId);
|
|
574
|
-
ctx.jobIndex.delete(jobId);
|
|
575
|
-
removed++;
|
|
576
|
-
}
|
|
577
|
-
}
|
|
578
|
-
if (removed > 0)
|
|
579
|
-
ctx.dashboardEmit?.("cleanup:orphans-removed", { count: removed });
|
|
580
|
-
});
|
|
581
439
|
}
|
|
440
|
+
return true;
|
|
582
441
|
}
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
const shard = ctx.shards[i];
|
|
587
|
-
const staleIds = [];
|
|
588
|
-
for (const [_id, job] of shard.waitingDeps) {
|
|
589
|
-
if (now - job.createdAt > depTimeout) {
|
|
590
|
-
staleIds.push(job.id);
|
|
591
|
-
}
|
|
592
|
-
}
|
|
593
|
-
if (staleIds.length === 0)
|
|
594
|
-
continue;
|
|
595
|
-
const removed = await withWriteLock(ctx.shardLocks[i], () => {
|
|
596
|
-
let count = 0;
|
|
597
|
-
const released = [];
|
|
598
|
-
for (const id of staleIds) {
|
|
599
|
-
const job = shard.waitingDeps.get(id);
|
|
600
|
-
if (!job || now - job.createdAt <= depTimeout)
|
|
601
|
-
continue;
|
|
602
|
-
ctx.storage?.deleteJob(job.id);
|
|
603
|
-
shard.waitingDeps.delete(job.id);
|
|
604
|
-
shard.unregisterDependencies(job.id, job.dependsOn);
|
|
605
|
-
released.push(...job.dependsOn);
|
|
606
|
-
if (job.uniqueKey && shard.getUniqueKeyEntry(job.queue, job.uniqueKey)?.jobId === job.id) {
|
|
607
|
-
shard.releaseUniqueKeyIfOwned(job.queue, job.uniqueKey, job.id);
|
|
608
|
-
}
|
|
609
|
-
if (job.customId && ctx.customIdMap.get(job.customId) === job.id) {
|
|
610
|
-
ctx.customIdMap.delete(job.customId);
|
|
611
|
-
}
|
|
612
|
-
ctx.dependencyResults?.releaseConsumer(job.id);
|
|
613
|
-
ctx.jobIndex.delete(job.id);
|
|
614
|
-
count++;
|
|
615
|
-
}
|
|
616
|
-
return { count, released };
|
|
617
|
-
});
|
|
618
|
-
releaseDependencyCompletionPins(removed.released, ctx);
|
|
619
|
-
if (removed.count > 0) {
|
|
620
|
-
ctx.dashboardEmit?.("cleanup:stale-deps-removed", { count: removed.count });
|
|
621
|
-
}
|
|
622
|
-
}
|
|
442
|
+
function assertWellFormedJobId(id, name = "job ID") {
|
|
443
|
+
if (!isWellFormedJobId(id))
|
|
444
|
+
throw new Error(`${name} must be well-formed Unicode`);
|
|
623
445
|
}
|
|
624
|
-
function
|
|
625
|
-
|
|
626
|
-
const shard = ctx.shards[i];
|
|
627
|
-
shard.cleanExpiredGroupWindows(now);
|
|
628
|
-
shard.cleanExpiredUniqueKeys();
|
|
629
|
-
for (const [_queueName, keys] of shard.uniqueKeys) {
|
|
630
|
-
if (keys.size > 1000) {
|
|
631
|
-
const toRemove = Math.floor(keys.size / 2);
|
|
632
|
-
const iter = keys.keys();
|
|
633
|
-
for (let j = 0;j < toRemove; j++) {
|
|
634
|
-
const { value, done } = iter.next();
|
|
635
|
-
if (done)
|
|
636
|
-
break;
|
|
637
|
-
keys.delete(value);
|
|
638
|
-
}
|
|
639
|
-
}
|
|
640
|
-
}
|
|
641
|
-
}
|
|
446
|
+
function jobId(id) {
|
|
447
|
+
return id;
|
|
642
448
|
}
|
|
643
|
-
function
|
|
644
|
-
|
|
645
|
-
const loc = ctx.jobIndex.get(jobId);
|
|
646
|
-
if (loc?.type !== "processing") {
|
|
647
|
-
ctx.stalledCandidates.delete(jobId);
|
|
648
|
-
}
|
|
649
|
-
}
|
|
449
|
+
function generateJobId() {
|
|
450
|
+
return Bun.randomUUIDv7();
|
|
650
451
|
}
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
return;
|
|
654
|
-
const processingCandidates = new Map;
|
|
655
|
-
const queueCandidates = new Map;
|
|
656
|
-
for (const [jobId, loc] of ctx.jobIndex) {
|
|
657
|
-
if (loc.type === "processing") {
|
|
658
|
-
const procIdx = processingShardIndex(jobId);
|
|
659
|
-
let list = processingCandidates.get(procIdx);
|
|
660
|
-
if (!list) {
|
|
661
|
-
list = [];
|
|
662
|
-
processingCandidates.set(procIdx, list);
|
|
663
|
-
}
|
|
664
|
-
list.push(jobId);
|
|
665
|
-
} else if (loc.type === "queue") {
|
|
666
|
-
let list = queueCandidates.get(loc.shardIdx);
|
|
667
|
-
if (!list) {
|
|
668
|
-
list = [];
|
|
669
|
-
queueCandidates.set(loc.shardIdx, list);
|
|
670
|
-
}
|
|
671
|
-
list.push({ jobId, queueName: loc.queueName });
|
|
672
|
-
}
|
|
673
|
-
}
|
|
674
|
-
for (const [procIdx, candidates] of processingCandidates) {
|
|
675
|
-
await withWriteLock(ctx.processingLocks[procIdx], () => {
|
|
676
|
-
for (const jobId of candidates) {
|
|
677
|
-
if (!ctx.processingShards[procIdx].has(jobId)) {
|
|
678
|
-
ctx.jobIndex.delete(jobId);
|
|
679
|
-
}
|
|
680
|
-
}
|
|
681
|
-
});
|
|
682
|
-
}
|
|
683
|
-
for (const [shardIdx, candidates] of queueCandidates) {
|
|
684
|
-
await withWriteLock(ctx.shardLocks[shardIdx], () => {
|
|
685
|
-
const shard = ctx.shards[shardIdx];
|
|
686
|
-
for (const { jobId, queueName } of candidates) {
|
|
687
|
-
if (!shard.getQueue(queueName).has(jobId)) {
|
|
688
|
-
ctx.jobIndex.delete(jobId);
|
|
689
|
-
}
|
|
690
|
-
}
|
|
691
|
-
});
|
|
692
|
-
}
|
|
452
|
+
function generateLockToken() {
|
|
453
|
+
return Bun.randomUUIDv7();
|
|
693
454
|
}
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
455
|
+
// ../../src/domain/job/locks.ts
|
|
456
|
+
function createJobLock(jobId, owner, ttl = DEFAULT_LOCK_TTL, now = Date.now()) {
|
|
457
|
+
return {
|
|
458
|
+
jobId,
|
|
459
|
+
token: generateLockToken(),
|
|
460
|
+
owner,
|
|
461
|
+
createdAt: now,
|
|
462
|
+
expiresAt: now + ttl,
|
|
463
|
+
lastRenewalAt: now,
|
|
464
|
+
renewalCount: 0,
|
|
465
|
+
ttl
|
|
466
|
+
};
|
|
701
467
|
}
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
var MAX_TIMELINE_ENTRIES = 20;
|
|
705
|
-
var DEFAULT_LOCK_TTL = 30000;
|
|
706
|
-
var JOB_DEFAULTS = {
|
|
707
|
-
priority: 0,
|
|
708
|
-
maxAttempts: 3,
|
|
709
|
-
backoff: 1000,
|
|
710
|
-
lifo: false,
|
|
711
|
-
removeOnComplete: false,
|
|
712
|
-
removeOnFail: false,
|
|
713
|
-
stackTraceLimit: 10
|
|
714
|
-
};
|
|
715
|
-
// ../../src/domain/job/payload.ts
|
|
716
|
-
var DEFAULT_JOB_NAME = "default";
|
|
717
|
-
var MAX_JOB_NAME_LENGTH = 256;
|
|
718
|
-
function legacyEnvelope(data) {
|
|
719
|
-
if (typeof data !== "object" || data === null || Array.isArray(data))
|
|
720
|
-
return null;
|
|
721
|
-
const record = data;
|
|
722
|
-
if (typeof record.name !== "string")
|
|
723
|
-
return null;
|
|
724
|
-
const { name, ...userData } = record;
|
|
725
|
-
return { name, data: userData };
|
|
468
|
+
function isLockExpired(lock, now = Date.now()) {
|
|
469
|
+
return now >= lock.expiresAt;
|
|
726
470
|
}
|
|
727
|
-
function
|
|
728
|
-
|
|
729
|
-
|
|
471
|
+
function renewLock(lock, newTtl, now = Date.now()) {
|
|
472
|
+
const ttl = newTtl ?? lock.ttl;
|
|
473
|
+
lock.expiresAt = now + ttl;
|
|
474
|
+
lock.lastRenewalAt = now;
|
|
475
|
+
lock.renewalCount++;
|
|
476
|
+
}
|
|
477
|
+
function isLeaseFromEarlierGeneration(job, lock) {
|
|
478
|
+
return job.startedAt !== null && job.startedAt !== undefined && job.startedAt > lock.createdAt;
|
|
479
|
+
}
|
|
480
|
+
// ../../src/domain/job/state.ts
|
|
481
|
+
function normalizeStacktrace(lines, limit) {
|
|
482
|
+
if (!lines || lines.length === 0)
|
|
483
|
+
return null;
|
|
484
|
+
const out = [];
|
|
485
|
+
for (const line of lines) {
|
|
486
|
+
if (out.length >= limit)
|
|
487
|
+
break;
|
|
488
|
+
if (typeof line !== "string")
|
|
489
|
+
continue;
|
|
490
|
+
const trimmed = line.trim();
|
|
491
|
+
if (trimmed)
|
|
492
|
+
out.push(trimmed);
|
|
730
493
|
}
|
|
731
|
-
return null;
|
|
494
|
+
return out.length > 0 ? out : null;
|
|
732
495
|
}
|
|
733
|
-
function
|
|
734
|
-
|
|
735
|
-
if (error)
|
|
736
|
-
throw new Error(error);
|
|
737
|
-
return name;
|
|
496
|
+
function isReady(job, now = Date.now()) {
|
|
497
|
+
return job.runAt <= now;
|
|
738
498
|
}
|
|
739
|
-
function
|
|
740
|
-
if (
|
|
741
|
-
return
|
|
499
|
+
function isExpired(job, now = Date.now()) {
|
|
500
|
+
if (job.ttl === null)
|
|
501
|
+
return false;
|
|
502
|
+
return now > job.createdAt + job.ttl;
|
|
503
|
+
}
|
|
504
|
+
function calculateBackoff(job) {
|
|
505
|
+
const maxDelay = job.backoffConfig?.maxDelay ?? DEFAULT_MAX_BACKOFF;
|
|
506
|
+
if (job.backoffConfig) {
|
|
507
|
+
if (job.backoffConfig.type === "fixed") {
|
|
508
|
+
const base = job.backoffConfig.delay;
|
|
509
|
+
const jittered = base * (0.8 + Math.random() * 0.4);
|
|
510
|
+
return Math.min(jittered, maxDelay);
|
|
511
|
+
}
|
|
512
|
+
const base = job.backoffConfig.delay * Math.pow(2, job.attempts);
|
|
513
|
+
const jittered = base * (0.5 + Math.random());
|
|
514
|
+
return Math.min(jittered, maxDelay);
|
|
742
515
|
}
|
|
743
|
-
|
|
516
|
+
const base = job.backoff * Math.pow(2, job.attempts);
|
|
517
|
+
const jittered = base * (0.5 + Math.random());
|
|
518
|
+
return Math.min(jittered, maxDelay);
|
|
744
519
|
}
|
|
745
|
-
function
|
|
746
|
-
|
|
747
|
-
return normalizeJobPayload(source, fallback);
|
|
748
|
-
const legacy = legacyEnvelope(source.data);
|
|
749
|
-
if (legacy)
|
|
750
|
-
return { name: checkedName(legacy.name), data: legacy.data, legacy: true };
|
|
751
|
-
return { name: checkedName(fallback), data: source.data, legacy: false };
|
|
520
|
+
function canRetry(job) {
|
|
521
|
+
return job.attempts < job.maxAttempts;
|
|
752
522
|
}
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
523
|
+
// ../../src/domain/types/stall.ts
|
|
524
|
+
var DEFAULT_STALL_CONFIG = {
|
|
525
|
+
enabled: true,
|
|
526
|
+
stallInterval: 30000,
|
|
527
|
+
maxStalls: 3,
|
|
528
|
+
gracePeriod: 5000
|
|
529
|
+
};
|
|
530
|
+
function checkStall(job, config = DEFAULT_STALL_CONFIG, now = Date.now()) {
|
|
531
|
+
if (job.startedAt === null) {
|
|
757
532
|
return {
|
|
758
|
-
|
|
759
|
-
|
|
533
|
+
isStalled: false,
|
|
534
|
+
stalledFor: 0,
|
|
535
|
+
shouldMoveToDlq: false,
|
|
536
|
+
newStallCount: job.stallCount
|
|
760
537
|
};
|
|
761
538
|
}
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
539
|
+
if (now - job.startedAt < config.gracePeriod) {
|
|
540
|
+
return {
|
|
541
|
+
isStalled: false,
|
|
542
|
+
stalledFor: 0,
|
|
543
|
+
shouldMoveToDlq: false,
|
|
544
|
+
newStallCount: job.stallCount
|
|
545
|
+
};
|
|
546
|
+
}
|
|
547
|
+
const stallInterval = job.stallTimeout ?? config.stallInterval;
|
|
548
|
+
const stalledFor = now - job.lastHeartbeat;
|
|
549
|
+
const isStalled = stalledFor > stallInterval;
|
|
550
|
+
if (!isStalled) {
|
|
551
|
+
return {
|
|
552
|
+
isStalled: false,
|
|
553
|
+
stalledFor,
|
|
554
|
+
shouldMoveToDlq: false,
|
|
555
|
+
newStallCount: job.stallCount
|
|
556
|
+
};
|
|
557
|
+
}
|
|
558
|
+
const newStallCount = job.stallCount + 1;
|
|
559
|
+
const shouldMoveToDlq = newStallCount >= config.maxStalls;
|
|
560
|
+
return {
|
|
561
|
+
isStalled: true,
|
|
562
|
+
stalledFor,
|
|
563
|
+
shouldMoveToDlq,
|
|
564
|
+
newStallCount
|
|
779
565
|
};
|
|
780
566
|
}
|
|
781
|
-
function
|
|
782
|
-
|
|
567
|
+
function getStallAction(job, config = DEFAULT_STALL_CONFIG, now = Date.now()) {
|
|
568
|
+
const result = checkStall(job, config, now);
|
|
569
|
+
if (!result.isStalled) {
|
|
570
|
+
return "keep" /* Keep */;
|
|
571
|
+
}
|
|
572
|
+
if (result.shouldMoveToDlq) {
|
|
573
|
+
return "move_to_dlq" /* MoveToDlq */;
|
|
574
|
+
}
|
|
575
|
+
return "retry" /* Retry */;
|
|
783
576
|
}
|
|
784
|
-
function
|
|
577
|
+
function incrementStallCount(job) {
|
|
578
|
+
job.stallCount++;
|
|
579
|
+
return job.stallCount;
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
// ../../src/shared/logger.ts
|
|
583
|
+
var LOG_LEVEL_PRIORITY = {
|
|
584
|
+
debug: 0,
|
|
585
|
+
info: 1,
|
|
586
|
+
warn: 2,
|
|
587
|
+
error: 3
|
|
588
|
+
};
|
|
589
|
+
|
|
590
|
+
class Logger {
|
|
591
|
+
component;
|
|
592
|
+
static jsonMode = false;
|
|
593
|
+
static level = "info";
|
|
594
|
+
constructor(component) {
|
|
595
|
+
this.component = component;
|
|
596
|
+
}
|
|
597
|
+
static enableJsonMode() {
|
|
598
|
+
Logger.jsonMode = true;
|
|
599
|
+
}
|
|
600
|
+
static disableJsonMode() {
|
|
601
|
+
Logger.jsonMode = false;
|
|
602
|
+
}
|
|
603
|
+
static setLevel(level) {
|
|
604
|
+
Logger.level = level;
|
|
605
|
+
}
|
|
606
|
+
debug(message, data) {
|
|
607
|
+
this.log("debug", message, data);
|
|
608
|
+
}
|
|
609
|
+
debugToStderr(message, data) {
|
|
610
|
+
this.log("debug", message, data, true);
|
|
611
|
+
}
|
|
612
|
+
info(message, data) {
|
|
613
|
+
this.log("info", message, data);
|
|
614
|
+
}
|
|
615
|
+
infoToStderr(message, data) {
|
|
616
|
+
this.log("info", message, data, true);
|
|
617
|
+
}
|
|
618
|
+
warn(message, data) {
|
|
619
|
+
this.log("warn", message, data);
|
|
620
|
+
}
|
|
621
|
+
error(message, data) {
|
|
622
|
+
this.log("error", message, data);
|
|
623
|
+
}
|
|
624
|
+
log(level, message, data, forceStderr = false) {
|
|
625
|
+
if (LOG_LEVEL_PRIORITY[level] < LOG_LEVEL_PRIORITY[Logger.level])
|
|
626
|
+
return;
|
|
627
|
+
if (Logger.jsonMode) {
|
|
628
|
+
const entry = {
|
|
629
|
+
timestamp: new Date().toISOString(),
|
|
630
|
+
level,
|
|
631
|
+
component: this.component,
|
|
632
|
+
message,
|
|
633
|
+
...data && { data }
|
|
634
|
+
};
|
|
635
|
+
if (forceStderr)
|
|
636
|
+
console.error(JSON.stringify(entry));
|
|
637
|
+
else
|
|
638
|
+
console.log(JSON.stringify(entry));
|
|
639
|
+
} else {
|
|
640
|
+
const prefix = `[${this.component}]`;
|
|
641
|
+
const dataStr = data ? ` ${JSON.stringify(data)}` : "";
|
|
642
|
+
switch (level) {
|
|
643
|
+
case "debug":
|
|
644
|
+
if (forceStderr)
|
|
645
|
+
console.error(`${prefix} ${message}${dataStr}`);
|
|
646
|
+
else
|
|
647
|
+
console.debug(`${prefix} ${message}${dataStr}`);
|
|
648
|
+
break;
|
|
649
|
+
case "info":
|
|
650
|
+
if (forceStderr)
|
|
651
|
+
console.error(`${prefix} ${message}${dataStr}`);
|
|
652
|
+
else
|
|
653
|
+
console.log(`${prefix} ${message}${dataStr}`);
|
|
654
|
+
break;
|
|
655
|
+
case "warn":
|
|
656
|
+
console.warn(`${prefix} ${message}${dataStr}`);
|
|
657
|
+
break;
|
|
658
|
+
case "error":
|
|
659
|
+
console.error(`${prefix} ${message}${dataStr}`);
|
|
660
|
+
break;
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
function createLogger(component) {
|
|
666
|
+
return new Logger(component);
|
|
667
|
+
}
|
|
668
|
+
var serverLog = createLogger("Server");
|
|
669
|
+
var tcpLog = createLogger("TCP");
|
|
670
|
+
var httpLog = createLogger("HTTP");
|
|
671
|
+
var wsLog = createLogger("WS");
|
|
672
|
+
var cronLog = createLogger("Cron");
|
|
673
|
+
var statsLog = createLogger("Stats");
|
|
674
|
+
var storageLog = createLogger("Storage");
|
|
675
|
+
var queueLog = createLogger("Queue");
|
|
676
|
+
var webhookLog = createLogger("Webhook");
|
|
677
|
+
var backupLog = createLogger("Backup");
|
|
678
|
+
|
|
679
|
+
// ../../src/domain/types/dlq.ts
|
|
680
|
+
var DLQ_RETRY_STATE = Symbol("bunqueue.dlqRetryState");
|
|
681
|
+
function getDlqRetryState(job) {
|
|
682
|
+
return job[DLQ_RETRY_STATE] ?? null;
|
|
683
|
+
}
|
|
684
|
+
function setDlqRetryState(job, state) {
|
|
685
|
+
if (state === null) {
|
|
686
|
+
delete job[DLQ_RETRY_STATE];
|
|
687
|
+
return;
|
|
688
|
+
}
|
|
689
|
+
Object.defineProperty(job, DLQ_RETRY_STATE, {
|
|
690
|
+
value: state,
|
|
691
|
+
writable: true,
|
|
692
|
+
configurable: true,
|
|
693
|
+
enumerable: false
|
|
694
|
+
});
|
|
695
|
+
}
|
|
696
|
+
function createAttemptRecord(job, reason, error, failedAt = Date.now()) {
|
|
785
697
|
return {
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
698
|
+
attempt: job.attempts,
|
|
699
|
+
startedAt: job.startedAt ?? job.createdAt,
|
|
700
|
+
failedAt,
|
|
701
|
+
reason,
|
|
702
|
+
error,
|
|
703
|
+
duration: job.startedAt ? failedAt - job.startedAt : 0
|
|
791
704
|
};
|
|
792
705
|
}
|
|
793
|
-
function
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
stallTimeout: input.stallTimeout ?? null
|
|
706
|
+
function recordJobFailureAttempt(job, reason, error, failedAt = Date.now()) {
|
|
707
|
+
const state = getDlqRetryState(job) ?? {
|
|
708
|
+
enteredAt: null,
|
|
709
|
+
expiresAt: null,
|
|
710
|
+
attempts: [],
|
|
711
|
+
retryCount: 0,
|
|
712
|
+
lastRetryAt: null,
|
|
713
|
+
nextRetryAt: null
|
|
802
714
|
};
|
|
715
|
+
state.attempts.push(createAttemptRecord(job, reason, error, failedAt));
|
|
716
|
+
setDlqRetryState(job, state);
|
|
803
717
|
}
|
|
804
|
-
function
|
|
718
|
+
function retainDlqRetryState(job, entry) {
|
|
719
|
+
setDlqRetryState(job, {
|
|
720
|
+
enteredAt: entry.enteredAt,
|
|
721
|
+
expiresAt: entry.expiresAt,
|
|
722
|
+
attempts: [...entry.attempts],
|
|
723
|
+
retryCount: entry.retryCount,
|
|
724
|
+
lastRetryAt: entry.lastRetryAt,
|
|
725
|
+
nextRetryAt: entry.nextRetryAt
|
|
726
|
+
});
|
|
727
|
+
}
|
|
728
|
+
var DEFAULT_DLQ_CONFIG = {
|
|
729
|
+
autoRetry: false,
|
|
730
|
+
autoRetryInterval: 60 * 60 * 1000,
|
|
731
|
+
maxAutoRetries: 3,
|
|
732
|
+
maxAge: 7 * 24 * 60 * 60 * 1000,
|
|
733
|
+
maxEntries: 1e4
|
|
734
|
+
};
|
|
735
|
+
function createDlqEntry(job, reason, error, config = DEFAULT_DLQ_CONFIG, now = Date.now()) {
|
|
736
|
+
const retained = getDlqRetryState(job);
|
|
737
|
+
const alreadyEntered = retained?.enteredAt !== null && retained?.enteredAt !== undefined;
|
|
738
|
+
const attemptRecord = createAttemptRecord(job, reason, error, now);
|
|
739
|
+
setDlqRetryState(job, null);
|
|
805
740
|
return {
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
deduplicationReplace: input.dedup?.replace ?? false,
|
|
816
|
-
debounceId: input.debounceId ?? null,
|
|
817
|
-
debounceTtl: input.debounceTtl ?? null
|
|
741
|
+
job,
|
|
742
|
+
enteredAt: alreadyEntered ? retained.enteredAt : now,
|
|
743
|
+
reason,
|
|
744
|
+
error,
|
|
745
|
+
attempts: [...retained?.attempts ?? [], attemptRecord],
|
|
746
|
+
retryCount: retained?.retryCount ?? 0,
|
|
747
|
+
lastRetryAt: retained?.lastRetryAt ?? null,
|
|
748
|
+
nextRetryAt: alreadyEntered ? retained.nextRetryAt : config.autoRetry ? now + config.autoRetryInterval : null,
|
|
749
|
+
expiresAt: alreadyEntered ? retained.expiresAt : config.maxAge !== null ? now + config.maxAge : null
|
|
818
750
|
};
|
|
819
751
|
}
|
|
820
|
-
function
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
752
|
+
function isDlqEntryExpired(entry, now = Date.now()) {
|
|
753
|
+
return entry.expiresAt !== null && now >= entry.expiresAt;
|
|
754
|
+
}
|
|
755
|
+
function canAutoRetry(entry, config, now = Date.now()) {
|
|
756
|
+
if (!config.autoRetry)
|
|
757
|
+
return false;
|
|
758
|
+
if (entry.retryCount >= config.maxAutoRetries)
|
|
759
|
+
return false;
|
|
760
|
+
if (entry.nextRetryAt === null)
|
|
761
|
+
return false;
|
|
762
|
+
return now >= entry.nextRetryAt;
|
|
763
|
+
}
|
|
764
|
+
function scheduleNextRetry(entry, config) {
|
|
765
|
+
const now = Date.now();
|
|
766
|
+
entry.retryCount++;
|
|
767
|
+
entry.lastRetryAt = now;
|
|
768
|
+
if (entry.retryCount < config.maxAutoRetries) {
|
|
769
|
+
const backoffMultiplier = Math.pow(2, entry.retryCount - 1);
|
|
770
|
+
entry.nextRetryAt = now + config.autoRetryInterval * backoffMultiplier;
|
|
771
|
+
} else {
|
|
772
|
+
entry.nextRetryAt = null;
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
// ../../src/domain/types/queue.ts
|
|
777
|
+
function createQueueState(name) {
|
|
778
|
+
return {
|
|
779
|
+
name,
|
|
780
|
+
paused: false,
|
|
781
|
+
rateLimit: null,
|
|
782
|
+
rateLimitDuration: null,
|
|
783
|
+
rateLimitExpiresAt: null,
|
|
784
|
+
concurrencyLimit: null,
|
|
785
|
+
activeCount: 0
|
|
854
786
|
};
|
|
855
787
|
}
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
788
|
+
|
|
789
|
+
class RateLimiter {
|
|
790
|
+
capacity;
|
|
791
|
+
refillRate;
|
|
792
|
+
tokens;
|
|
793
|
+
lastRefill;
|
|
794
|
+
constructor(capacity, refillRate = capacity) {
|
|
795
|
+
this.capacity = capacity;
|
|
796
|
+
this.refillRate = refillRate;
|
|
797
|
+
this.tokens = capacity;
|
|
798
|
+
this.lastRefill = Date.now();
|
|
799
|
+
}
|
|
800
|
+
tryAcquire() {
|
|
801
|
+
this.refill();
|
|
802
|
+
if (this.tokens >= 1) {
|
|
803
|
+
this.tokens -= 1;
|
|
804
|
+
return true;
|
|
805
|
+
}
|
|
806
|
+
return false;
|
|
807
|
+
}
|
|
808
|
+
refill() {
|
|
809
|
+
const now = Date.now();
|
|
810
|
+
const elapsed = (now - this.lastRefill) / 1000;
|
|
811
|
+
const tokensToAdd = elapsed * this.refillRate;
|
|
812
|
+
this.tokens = Math.min(this.capacity, this.tokens + tokensToAdd);
|
|
813
|
+
this.lastRefill = now;
|
|
814
|
+
}
|
|
815
|
+
getTokens() {
|
|
816
|
+
this.refill();
|
|
817
|
+
return this.tokens;
|
|
818
|
+
}
|
|
819
|
+
getTtl(maxJobs) {
|
|
820
|
+
const tokens = this.getTokens();
|
|
821
|
+
const consumed = this.capacity - tokens;
|
|
822
|
+
if (maxJobs !== undefined && consumed < maxJobs)
|
|
823
|
+
return 0;
|
|
824
|
+
if (tokens >= 1 || this.refillRate <= 0)
|
|
825
|
+
return 0;
|
|
826
|
+
return Math.ceil((1 - tokens) / this.refillRate * 1000);
|
|
827
|
+
}
|
|
828
|
+
}
|
|
829
|
+
|
|
830
|
+
class ConcurrencyLimiter {
|
|
831
|
+
limit;
|
|
832
|
+
active = 0;
|
|
833
|
+
constructor(limit) {
|
|
834
|
+
this.limit = limit;
|
|
835
|
+
}
|
|
836
|
+
tryAcquire() {
|
|
837
|
+
if (this.active < this.limit) {
|
|
838
|
+
this.active += 1;
|
|
839
|
+
return true;
|
|
840
|
+
}
|
|
841
|
+
return false;
|
|
842
|
+
}
|
|
843
|
+
release() {
|
|
844
|
+
if (this.active > 0) {
|
|
845
|
+
this.active -= 1;
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
getActive() {
|
|
849
|
+
return this.active;
|
|
850
|
+
}
|
|
851
|
+
getLimit() {
|
|
852
|
+
return this.limit;
|
|
853
|
+
}
|
|
854
|
+
setLimit(limit) {
|
|
855
|
+
this.limit = limit;
|
|
856
|
+
}
|
|
857
|
+
}
|
|
858
|
+
|
|
859
|
+
// ../../src/application/stallDetection.ts
|
|
860
|
+
function checkStalledJobs(ctx) {
|
|
861
|
+
const now = Date.now();
|
|
862
|
+
const confirmedStalled = [];
|
|
863
|
+
for (const jobId of ctx.stalledCandidates) {
|
|
864
|
+
const procIdx = processingShardIndex(jobId);
|
|
865
|
+
const job = ctx.processingShards[procIdx].get(jobId);
|
|
866
|
+
if (!job) {
|
|
867
|
+
ctx.stalledCandidates.delete(jobId);
|
|
868
|
+
continue;
|
|
869
|
+
}
|
|
870
|
+
const stallConfig = ctx.shards[shardIndex(job.queue)].getStallConfig(job.queue);
|
|
871
|
+
if (!stallConfig.enabled) {
|
|
872
|
+
ctx.stalledCandidates.delete(jobId);
|
|
873
|
+
continue;
|
|
874
|
+
}
|
|
875
|
+
const action = getStallAction(job, stallConfig, now);
|
|
876
|
+
if (action !== "keep" /* Keep */) {
|
|
877
|
+
confirmedStalled.push({ job, action });
|
|
878
|
+
}
|
|
879
|
+
ctx.stalledCandidates.delete(jobId);
|
|
880
|
+
}
|
|
881
|
+
for (let i = 0;i < SHARD_COUNT; i++) {
|
|
882
|
+
const procShard = ctx.processingShards[i];
|
|
883
|
+
for (const [jobId, job] of procShard) {
|
|
884
|
+
const stallConfig = ctx.shards[shardIndex(job.queue)].getStallConfig(job.queue);
|
|
885
|
+
if (!stallConfig.enabled)
|
|
886
|
+
continue;
|
|
887
|
+
const action = getStallAction(job, stallConfig, now);
|
|
888
|
+
if (action !== "keep" /* Keep */) {
|
|
889
|
+
ctx.stalledCandidates.add(jobId);
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
}
|
|
893
|
+
const stillStalled = (job, lockedNow) => {
|
|
894
|
+
const config = ctx.shards[shardIndex(job.queue)].getStallConfig(job.queue);
|
|
895
|
+
return config.enabled && getStallAction(job, config, lockedNow) !== "keep" /* Keep */;
|
|
896
|
+
};
|
|
897
|
+
for (const { job, action } of confirmedStalled) {
|
|
898
|
+
handleStalledJob(job, action, ctx, stillStalled).catch((err) => {
|
|
899
|
+
queueLog.error("Failed to handle stalled job", {
|
|
900
|
+
jobId: String(job.id),
|
|
901
|
+
error: String(err)
|
|
902
|
+
});
|
|
903
|
+
});
|
|
904
|
+
}
|
|
905
|
+
}
|
|
906
|
+
async function handleStalledJob(job, action, ctx, recheck) {
|
|
907
|
+
const idx = shardIndex(job.queue);
|
|
908
|
+
const procIdx = processingShardIndex(job.id);
|
|
909
|
+
let handled = false;
|
|
910
|
+
let handledAction = action;
|
|
911
|
+
await withWriteLock(ctx.shardLocks[idx], async () => {
|
|
912
|
+
await withWriteLock(ctx.processingLocks[procIdx], () => {
|
|
913
|
+
if (ctx.processingShards[procIdx].get(job.id) !== job) {
|
|
914
|
+
return;
|
|
915
|
+
}
|
|
916
|
+
if (recheck && !recheck(job, Date.now()))
|
|
917
|
+
return;
|
|
918
|
+
const shard = ctx.shards[idx];
|
|
919
|
+
const attemptsExhausted = job.attempts + 1 >= job.maxAttempts;
|
|
920
|
+
if (action === "move_to_dlq" /* MoveToDlq */ || attemptsExhausted) {
|
|
921
|
+
handledAction = "move_to_dlq" /* MoveToDlq */;
|
|
922
|
+
moveStalliedJobToDlq(job, ctx, shard, procIdx, attemptsExhausted);
|
|
923
|
+
} else {
|
|
924
|
+
retryStalliedJob(job, ctx, shard, procIdx, idx);
|
|
925
|
+
}
|
|
926
|
+
detachClientJob(job.id, ctx);
|
|
927
|
+
handled = true;
|
|
928
|
+
});
|
|
929
|
+
});
|
|
930
|
+
if (handled) {
|
|
931
|
+
ctx.dashboardEmit?.("job:stalled", {
|
|
932
|
+
jobId: String(job.id),
|
|
933
|
+
queue: job.queue,
|
|
934
|
+
stallCount: job.stallCount,
|
|
935
|
+
action: handledAction
|
|
936
|
+
});
|
|
937
|
+
ctx.eventsManager.broadcast({
|
|
938
|
+
eventType: "stalled" /* Stalled */,
|
|
939
|
+
queue: job.queue,
|
|
940
|
+
jobId: job.id,
|
|
941
|
+
timestamp: Date.now(),
|
|
942
|
+
data: { stallCount: job.stallCount, action: handledAction }
|
|
943
|
+
});
|
|
944
|
+
ctx.webhookManager.trigger("stalled", String(job.id), job.queue, {
|
|
945
|
+
data: { stallCount: job.stallCount, action: handledAction }
|
|
946
|
+
});
|
|
947
|
+
}
|
|
948
|
+
return handled;
|
|
949
|
+
}
|
|
950
|
+
function moveStalliedJobToDlq(job, ctx, shard, procIdx, attemptsExhausted) {
|
|
951
|
+
ctx.processingShards[procIdx].delete(job.id);
|
|
952
|
+
ctx.timeoutScheduler.cancel(job.id);
|
|
953
|
+
shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId, job.id);
|
|
954
|
+
if (job.uniqueKey?.startsWith("cron:")) {
|
|
955
|
+
ctx.jobIndex.delete(job.id);
|
|
956
|
+
ctx.storage?.deleteJob(job.id);
|
|
957
|
+
return;
|
|
958
|
+
}
|
|
959
|
+
incrementStallCount(job);
|
|
960
|
+
job.attempts++;
|
|
961
|
+
job.startedAt = null;
|
|
962
|
+
job.lastHeartbeat = Date.now();
|
|
963
|
+
const entry = shard.addToDlq(job, attemptsExhausted ? "max_attempts_exceeded" /* MaxAttemptsExceeded */ : "stalled" /* Stalled */, attemptsExhausted ? `Job stalled at max attempts (${job.maxAttempts})` : `Job stalled ${job.stallCount} times`);
|
|
964
|
+
ctx.jobIndex.set(job.id, { type: "dlq", queueName: job.queue });
|
|
965
|
+
ctx.storage?.saveDlqEntry(entry);
|
|
966
|
+
ctx.storage?.deleteJob(job.id);
|
|
967
|
+
}
|
|
968
|
+
function retryStalliedJob(job, ctx, shard, procIdx, idx) {
|
|
969
|
+
if (job.uniqueKey?.startsWith("cron:")) {
|
|
970
|
+
ctx.processingShards[procIdx].delete(job.id);
|
|
971
|
+
ctx.timeoutScheduler.cancel(job.id);
|
|
972
|
+
shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId, job.id);
|
|
973
|
+
ctx.jobIndex.delete(job.id);
|
|
974
|
+
ctx.storage?.deleteJob(job.id);
|
|
975
|
+
return;
|
|
976
|
+
}
|
|
977
|
+
incrementStallCount(job);
|
|
978
|
+
job.attempts++;
|
|
979
|
+
const stallNow = Date.now();
|
|
980
|
+
recordJobFailureAttempt(job, "stalled" /* Stalled */, "stalled", stallNow);
|
|
981
|
+
job.startedAt = null;
|
|
982
|
+
job.runAt = stallNow + calculateBackoff(job);
|
|
983
|
+
job.lastHeartbeat = stallNow;
|
|
984
|
+
if (job.timeline.length < MAX_TIMELINE_ENTRIES) {
|
|
985
|
+
job.timeline.push({
|
|
986
|
+
state: "failed",
|
|
987
|
+
timestamp: stallNow,
|
|
988
|
+
error: "stalled",
|
|
989
|
+
attempt: job.attempts
|
|
990
|
+
});
|
|
991
|
+
job.timeline.push({ state: "waiting", timestamp: stallNow, attempt: job.attempts + 1 });
|
|
992
|
+
}
|
|
993
|
+
ctx.processingShards[procIdx].delete(job.id);
|
|
994
|
+
ctx.timeoutScheduler.cancel(job.id);
|
|
995
|
+
shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId, job.id);
|
|
996
|
+
shard.getQueue(job.queue).push(job);
|
|
997
|
+
const isDelayed = job.runAt > Date.now();
|
|
998
|
+
shard.incrementQueued(job.id, isDelayed, job.createdAt, job.queue, job.runAt);
|
|
999
|
+
ctx.jobIndex.set(job.id, { type: "queue", shardIdx: idx, queueName: job.queue });
|
|
1000
|
+
ctx.storage?.updateForRetry(job);
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
// ../../src/application/orphanRecovery.ts
|
|
1004
|
+
var ORPHAN_WINDOW_FLOOR_MS = 30 * 60 * 1000;
|
|
1005
|
+
function isOrphanedProcessingEntry(job, lock, now, ctx) {
|
|
1006
|
+
if (!job.startedAt)
|
|
1007
|
+
return false;
|
|
1008
|
+
const config = ctx.shards[shardIndex(job.queue)].getStallConfig(job.queue);
|
|
1009
|
+
if (!config.enabled)
|
|
1010
|
+
return false;
|
|
1011
|
+
if (now - job.startedAt < config.gracePeriod)
|
|
1012
|
+
return false;
|
|
1013
|
+
if (lock && !isLockExpired(lock, now) && !isLeaseFromEarlierGeneration(job, lock))
|
|
1014
|
+
return false;
|
|
1015
|
+
const window = Math.max(ORPHAN_WINDOW_FLOOR_MS, job.stallTimeout ?? config.stallInterval);
|
|
1016
|
+
const lastActivity = Math.max(job.startedAt, job.lastHeartbeat || 0);
|
|
1017
|
+
return now - lastActivity > window;
|
|
1018
|
+
}
|
|
1019
|
+
function orphanStallAction(job, ctx) {
|
|
1020
|
+
const { maxStalls } = ctx.shards[shardIndex(job.queue)].getStallConfig(job.queue);
|
|
1021
|
+
return maxStalls > 0 && job.stallCount + 1 >= maxStalls ? "move_to_dlq" /* MoveToDlq */ : "retry" /* Retry */;
|
|
1022
|
+
}
|
|
1023
|
+
async function recoverOrphanedProcessingEntries(ctx, now) {
|
|
1024
|
+
const orphans = [];
|
|
1025
|
+
for (let i = 0;i < SHARD_COUNT; i++) {
|
|
1026
|
+
for (const [jobId, job] of ctx.processingShards[i]) {
|
|
1027
|
+
if (isOrphanedProcessingEntry(job, ctx.jobLocks.get(jobId), now, ctx)) {
|
|
1028
|
+
orphans.push(job);
|
|
1029
|
+
}
|
|
1030
|
+
}
|
|
1031
|
+
}
|
|
1032
|
+
if (orphans.length === 0)
|
|
1033
|
+
return;
|
|
1034
|
+
const stillOrphaned = (job, lockedNow) => isOrphanedProcessingEntry(job, ctx.jobLocks.get(job.id), lockedNow, ctx);
|
|
1035
|
+
let recovered = 0;
|
|
1036
|
+
for (const job of orphans) {
|
|
1037
|
+
try {
|
|
1038
|
+
if (await handleStalledJob(job, orphanStallAction(job, ctx), ctx, stillOrphaned)) {
|
|
1039
|
+
recovered++;
|
|
1040
|
+
}
|
|
1041
|
+
} catch (err) {
|
|
1042
|
+
queueLog.error("Failed to recover orphaned processing entry", {
|
|
1043
|
+
jobId: String(job.id),
|
|
1044
|
+
error: String(err)
|
|
1045
|
+
});
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
if (recovered > 0)
|
|
1049
|
+
ctx.dashboardEmit?.("cleanup:orphans-removed", { count: recovered });
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
// ../../src/application/dependencyCompletions.ts
|
|
1053
|
+
class DependencyCompletionTracker {
|
|
1054
|
+
onRecentEvict;
|
|
1055
|
+
recent = new Set;
|
|
1056
|
+
recentOrder = [];
|
|
1057
|
+
recentOrderTokens = [];
|
|
1058
|
+
recentTokens = new Map;
|
|
1059
|
+
recentOrderHead = 0;
|
|
1060
|
+
nextRecentToken = 0;
|
|
1061
|
+
pinned = new Set;
|
|
1062
|
+
owners = new Map;
|
|
1063
|
+
maxRecent;
|
|
1064
|
+
constructor(maxRecent, onRecentEvict) {
|
|
1065
|
+
this.onRecentEvict = onRecentEvict;
|
|
1066
|
+
this.maxRecent = Math.max(1, Math.trunc(maxRecent));
|
|
1067
|
+
}
|
|
1068
|
+
add(jobId, queue) {
|
|
1069
|
+
if (queue !== undefined)
|
|
1070
|
+
this.owners.set(jobId, queue);
|
|
1071
|
+
if (this.pinned.has(jobId) || this.recent.has(jobId))
|
|
1072
|
+
return;
|
|
1073
|
+
if (this.recent.size >= this.maxRecent)
|
|
1074
|
+
this.evictOldestRecent();
|
|
1075
|
+
if (this.nextRecentToken >= Number.MAX_SAFE_INTEGER)
|
|
1076
|
+
this.compactRecentOrder(true);
|
|
1077
|
+
const token = this.nextRecentToken++;
|
|
1078
|
+
this.recent.add(jobId);
|
|
1079
|
+
this.recentTokens.set(jobId, token);
|
|
1080
|
+
this.recentOrder.push(jobId);
|
|
1081
|
+
this.recentOrderTokens.push(token);
|
|
1082
|
+
}
|
|
1083
|
+
pin(jobId, queue) {
|
|
1084
|
+
if (queue !== undefined)
|
|
1085
|
+
this.owners.set(jobId, queue);
|
|
1086
|
+
if (this.recent.delete(jobId)) {
|
|
1087
|
+
this.recentTokens.delete(jobId);
|
|
1088
|
+
this.compactRecentOrder();
|
|
1089
|
+
}
|
|
1090
|
+
this.pinned.add(jobId);
|
|
1091
|
+
}
|
|
1092
|
+
unpin(jobId, retainAsRecent) {
|
|
1093
|
+
this.pinned.delete(jobId);
|
|
1094
|
+
if (retainAsRecent)
|
|
1095
|
+
this.add(jobId);
|
|
1096
|
+
else
|
|
1097
|
+
this.delete(jobId);
|
|
1098
|
+
}
|
|
1099
|
+
hydrate(records) {
|
|
1100
|
+
this.clear();
|
|
1101
|
+
for (const record of records) {
|
|
1102
|
+
if (record.pinned)
|
|
1103
|
+
this.pin(record.jobId, record.queue);
|
|
1104
|
+
else
|
|
1105
|
+
this.add(record.jobId, record.queue);
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
pinnedValues() {
|
|
1109
|
+
return this.pinned.values();
|
|
1110
|
+
}
|
|
1111
|
+
has(jobId) {
|
|
1112
|
+
return this.pinned.has(jobId) || this.recent.has(jobId);
|
|
1113
|
+
}
|
|
1114
|
+
delete(jobId) {
|
|
1115
|
+
const wasPinned = this.pinned.delete(jobId);
|
|
1116
|
+
const wasRecent = this.recent.delete(jobId);
|
|
1117
|
+
if (wasRecent) {
|
|
1118
|
+
this.recentTokens.delete(jobId);
|
|
1119
|
+
this.compactRecentOrder();
|
|
1120
|
+
}
|
|
1121
|
+
this.owners.delete(jobId);
|
|
1122
|
+
return wasRecent || wasPinned;
|
|
1123
|
+
}
|
|
1124
|
+
deleteForQueue(queue) {
|
|
1125
|
+
const removed = [];
|
|
1126
|
+
for (const [jobId, owner] of this.owners) {
|
|
1127
|
+
if (owner !== queue)
|
|
1128
|
+
continue;
|
|
1129
|
+
if (this.delete(jobId))
|
|
1130
|
+
removed.push(jobId);
|
|
1131
|
+
}
|
|
1132
|
+
return removed;
|
|
1133
|
+
}
|
|
1134
|
+
clear() {
|
|
1135
|
+
this.recent.clear();
|
|
1136
|
+
this.recentOrder = [];
|
|
1137
|
+
this.recentOrderTokens = [];
|
|
1138
|
+
this.recentTokens.clear();
|
|
1139
|
+
this.recentOrderHead = 0;
|
|
1140
|
+
this.nextRecentToken = 0;
|
|
1141
|
+
this.pinned.clear();
|
|
1142
|
+
this.owners.clear();
|
|
1143
|
+
}
|
|
1144
|
+
get size() {
|
|
1145
|
+
return this.recent.size + this.pinned.size;
|
|
1146
|
+
}
|
|
1147
|
+
evictOldestRecent() {
|
|
1148
|
+
while (this.recentOrderHead < this.recentOrder.length) {
|
|
1149
|
+
const index = this.recentOrderHead++;
|
|
1150
|
+
const oldest = this.recentOrder[index];
|
|
1151
|
+
const token = this.recentOrderTokens[index];
|
|
1152
|
+
if (this.recentTokens.get(oldest) !== token)
|
|
1153
|
+
continue;
|
|
1154
|
+
this.recent.delete(oldest);
|
|
1155
|
+
this.recentTokens.delete(oldest);
|
|
1156
|
+
this.owners.delete(oldest);
|
|
1157
|
+
this.compactRecentOrder();
|
|
1158
|
+
this.onRecentEvict?.(oldest);
|
|
1159
|
+
return;
|
|
1160
|
+
}
|
|
1161
|
+
}
|
|
1162
|
+
compactRecentOrder(force = false) {
|
|
1163
|
+
const pending = this.recentOrder.length - this.recentOrderHead;
|
|
1164
|
+
if (!force && this.recentOrderHead < this.maxRecent && pending < this.maxRecent * 2)
|
|
1165
|
+
return;
|
|
1166
|
+
const order = [];
|
|
1167
|
+
const tokens = [];
|
|
1168
|
+
for (let index = this.recentOrderHead;index < this.recentOrder.length; index++) {
|
|
1169
|
+
const jobId = this.recentOrder[index];
|
|
1170
|
+
const token = this.recentOrderTokens[index];
|
|
1171
|
+
if (this.recentTokens.get(jobId) !== token)
|
|
1172
|
+
continue;
|
|
1173
|
+
const compactedToken = order.length;
|
|
1174
|
+
order.push(jobId);
|
|
1175
|
+
tokens.push(compactedToken);
|
|
1176
|
+
}
|
|
1177
|
+
this.recentTokens.clear();
|
|
1178
|
+
for (let index = 0;index < order.length; index++) {
|
|
1179
|
+
this.recentTokens.set(order[index], tokens[index]);
|
|
866
1180
|
}
|
|
1181
|
+
this.recentOrder = order;
|
|
1182
|
+
this.recentOrderTokens = tokens;
|
|
1183
|
+
this.recentOrderHead = 0;
|
|
1184
|
+
this.nextRecentToken = order.length;
|
|
867
1185
|
}
|
|
868
|
-
return true;
|
|
869
|
-
}
|
|
870
|
-
function assertWellFormedJobId(id, name = "job ID") {
|
|
871
|
-
if (!isWellFormedJobId(id))
|
|
872
|
-
throw new Error(`${name} must be well-formed Unicode`);
|
|
873
|
-
}
|
|
874
|
-
function jobId(id) {
|
|
875
|
-
return id;
|
|
876
|
-
}
|
|
877
|
-
function generateJobId() {
|
|
878
|
-
return Bun.randomUUIDv7();
|
|
879
1186
|
}
|
|
880
|
-
function
|
|
881
|
-
return
|
|
1187
|
+
function hasDependencyWaiters(shards, jobId) {
|
|
1188
|
+
return shards.some((shard) => (shard.getJobsWaitingFor(jobId)?.size ?? 0) > 0);
|
|
882
1189
|
}
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
expiresAt: now + ttl,
|
|
891
|
-
lastRenewalAt: now,
|
|
892
|
-
renewalCount: 0,
|
|
893
|
-
ttl
|
|
894
|
-
};
|
|
1190
|
+
function commitRemovedCompletion(job, ctx, completedAt = Date.now()) {
|
|
1191
|
+
const pinned = hasDependencyWaiters(ctx.shards, job.id);
|
|
1192
|
+
ctx.storage?.commitRemovedCompletion(job, ctx.maxDependencyCompletions, pinned, completedAt);
|
|
1193
|
+
if (pinned)
|
|
1194
|
+
ctx.depCompletions?.pin(job.id, job.queue);
|
|
1195
|
+
else
|
|
1196
|
+
ctx.depCompletions?.add(job.id, job.queue);
|
|
895
1197
|
}
|
|
896
|
-
function
|
|
897
|
-
|
|
1198
|
+
function releaseDependencyCompletionPins(dependencyIds, ctx) {
|
|
1199
|
+
const candidates = [...new Set(dependencyIds)].filter((id) => !hasDependencyWaiters(ctx.shards, id));
|
|
1200
|
+
if (candidates.length === 0)
|
|
1201
|
+
return;
|
|
1202
|
+
if (ctx.storage) {
|
|
1203
|
+
const records = ctx.storage.unpinDependencyCompletions(candidates, ctx.maxDependencyCompletions);
|
|
1204
|
+
ctx.depCompletions?.hydrate(records);
|
|
1205
|
+
return;
|
|
1206
|
+
}
|
|
1207
|
+
for (const id of candidates)
|
|
1208
|
+
ctx.depCompletions?.unpin(id, true);
|
|
898
1209
|
}
|
|
899
|
-
function
|
|
900
|
-
const
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
1210
|
+
function reconcileDependencyCompletionPins(ctx) {
|
|
1211
|
+
const referenced = new Set;
|
|
1212
|
+
for (const shard of ctx.shards) {
|
|
1213
|
+
for (const dependencyId of shard.dependencyIndex.keys())
|
|
1214
|
+
referenced.add(dependencyId);
|
|
1215
|
+
}
|
|
1216
|
+
if (ctx.storage) {
|
|
1217
|
+
const records = ctx.storage.reconcileDependencyCompletionPins(referenced, ctx.maxDependencyCompletions);
|
|
1218
|
+
ctx.depCompletions?.hydrate(records);
|
|
1219
|
+
return;
|
|
1220
|
+
}
|
|
1221
|
+
for (const id of ctx.depCompletions?.pinnedValues() ?? []) {
|
|
1222
|
+
if (!referenced.has(id))
|
|
1223
|
+
ctx.depCompletions?.unpin(id, true);
|
|
1224
|
+
}
|
|
904
1225
|
}
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
1226
|
+
|
|
1227
|
+
// ../../src/shared/serialization.ts
|
|
1228
|
+
function compareSqliteBinaryText(left, right) {
|
|
1229
|
+
if (left === right)
|
|
1230
|
+
return 0;
|
|
1231
|
+
let leftIndex = 0;
|
|
1232
|
+
let rightIndex = 0;
|
|
1233
|
+
while (leftIndex < left.length && rightIndex < right.length) {
|
|
1234
|
+
const leftPoint = left.codePointAt(leftIndex);
|
|
1235
|
+
const rightPoint = right.codePointAt(rightIndex);
|
|
1236
|
+
if (leftPoint === undefined || rightPoint === undefined)
|
|
912
1237
|
break;
|
|
913
|
-
if (
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
out.push(trimmed);
|
|
1238
|
+
if (leftPoint !== rightPoint)
|
|
1239
|
+
return leftPoint - rightPoint;
|
|
1240
|
+
leftIndex += leftPoint > 65535 ? 2 : 1;
|
|
1241
|
+
rightIndex += rightPoint > 65535 ? 2 : 1;
|
|
918
1242
|
}
|
|
919
|
-
return
|
|
920
|
-
}
|
|
921
|
-
function isReady(job, now = Date.now()) {
|
|
922
|
-
return job.runAt <= now;
|
|
1243
|
+
return left.length - leftIndex - (right.length - rightIndex);
|
|
923
1244
|
}
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
return
|
|
1245
|
+
|
|
1246
|
+
// ../../src/application/completedCleanup.ts
|
|
1247
|
+
function memoryCandidates(queue, completedBefore, limit, ctx) {
|
|
1248
|
+
return [...ctx.jobIndex].flatMap(([jobId, location]) => {
|
|
1249
|
+
if (location.type !== "completed")
|
|
1250
|
+
return [];
|
|
1251
|
+
if (queue !== null && location.queueName !== queue)
|
|
1252
|
+
return [];
|
|
1253
|
+
if (ctx.dependencyResults.hasConsumers(jobId))
|
|
1254
|
+
return [];
|
|
1255
|
+
const job = ctx.completedJobsData.get(jobId);
|
|
1256
|
+
const timestamp = job?.completedAt ?? job?.createdAt ?? 0;
|
|
1257
|
+
return timestamp <= completedBefore ? [{ jobId, timestamp }] : [];
|
|
1258
|
+
}).sort((left, right) => left.timestamp - right.timestamp || compareSqliteBinaryText(left.jobId, right.jobId)).slice(0, limit).map(({ jobId }) => ({ jobId, queue: ctx.jobIndex.get(jobId)?.queueName ?? "" }));
|
|
928
1259
|
}
|
|
929
|
-
function
|
|
930
|
-
const
|
|
931
|
-
if (
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
1260
|
+
function cleanCompletedJobs(queue, completedBefore, limit, ctx) {
|
|
1261
|
+
const normalizedLimit = Number.isFinite(limit) ? Math.max(0, Math.trunc(limit)) : 0;
|
|
1262
|
+
if (normalizedLimit === 0 || !Number.isFinite(completedBefore))
|
|
1263
|
+
return [];
|
|
1264
|
+
const removals = ctx.storage ? ctx.storage.cleanCompletedJobs(queue, completedBefore, normalizedLimit, (jobId) => ctx.dependencyResults.hasConsumers(jobId)) : memoryCandidates(queue, completedBefore, normalizedLimit, ctx);
|
|
1265
|
+
for (const { jobId, queue: ownerQueue } of removals) {
|
|
1266
|
+
const location = ctx.jobIndex.get(jobId);
|
|
1267
|
+
const ownsCurrentCompletion = location?.type === "completed" && location.queueName === ownerQueue;
|
|
1268
|
+
const hasNoCurrentGeneration = location === undefined;
|
|
1269
|
+
const resultOwner = ctx.jobResultQueues?.get(jobId);
|
|
1270
|
+
if (resultOwner === ownerQueue || resultOwner === undefined && ownsCurrentCompletion) {
|
|
1271
|
+
ctx.jobResults?.delete(jobId);
|
|
1272
|
+
ctx.jobResultQueues?.delete(jobId);
|
|
936
1273
|
}
|
|
937
|
-
const
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
const base = job.backoff * Math.pow(2, job.attempts);
|
|
942
|
-
const jittered = base * (0.5 + Math.random());
|
|
943
|
-
return Math.min(jittered, maxDelay);
|
|
944
|
-
}
|
|
945
|
-
function canRetry(job) {
|
|
946
|
-
return job.attempts < job.maxAttempts;
|
|
947
|
-
}
|
|
948
|
-
// ../../src/application/dependencyProcessor.ts
|
|
949
|
-
function dependencyReadyState(job, now) {
|
|
950
|
-
if (job.runAt > now)
|
|
951
|
-
return "delayed";
|
|
952
|
-
return job.priority > 0 ? "prioritized" : "waiting";
|
|
953
|
-
}
|
|
954
|
-
function checkpointDependencyPromotion(job, state, now, storage) {
|
|
955
|
-
if (job.timeline.at(-1)?.state !== state && job.timeline.length < MAX_TIMELINE_ENTRIES) {
|
|
956
|
-
job.timeline.push({ state, timestamp: now });
|
|
957
|
-
}
|
|
958
|
-
storage?.updateFlowParentResolution(job, state);
|
|
959
|
-
}
|
|
960
|
-
async function processPendingDependencies(ctx) {
|
|
961
|
-
if (ctx.pendingDepChecks.size === 0)
|
|
962
|
-
return;
|
|
963
|
-
const completedIds = Array.from(ctx.pendingDepChecks);
|
|
964
|
-
const completedNow = new Set(completedIds);
|
|
965
|
-
ctx.pendingDepChecks.clear();
|
|
966
|
-
const jobsToCheckByShard = new Map;
|
|
967
|
-
const releasedDependencyIds = new Set;
|
|
968
|
-
for (const completedId of completedIds) {
|
|
969
|
-
for (let i = 0;i < SHARD_COUNT; i++) {
|
|
970
|
-
const waitingJobIds = ctx.shards[i].getJobsWaitingFor(completedId);
|
|
971
|
-
if (waitingJobIds && waitingJobIds.size > 0) {
|
|
972
|
-
let shardJobs = jobsToCheckByShard.get(i);
|
|
973
|
-
if (!shardJobs) {
|
|
974
|
-
shardJobs = new Set;
|
|
975
|
-
jobsToCheckByShard.set(i, shardJobs);
|
|
976
|
-
}
|
|
977
|
-
for (const jobId of waitingJobIds) {
|
|
978
|
-
shardJobs.add(jobId);
|
|
979
|
-
}
|
|
980
|
-
}
|
|
1274
|
+
const logOwner = ctx.jobLogQueues?.get(jobId);
|
|
1275
|
+
if (logOwner === ownerQueue || logOwner === undefined && ownsCurrentCompletion) {
|
|
1276
|
+
ctx.jobLogs?.delete(jobId);
|
|
1277
|
+
ctx.jobLogQueues?.delete(jobId);
|
|
981
1278
|
}
|
|
1279
|
+
if (!hasNoCurrentGeneration && !ownsCurrentCompletion)
|
|
1280
|
+
continue;
|
|
1281
|
+
ctx.completedJobs.delete(jobId);
|
|
1282
|
+
ctx.completedJobsData.delete(jobId);
|
|
1283
|
+
ctx.jobIndex.delete(jobId);
|
|
1284
|
+
ctx.dependencyResults.releaseConsumer(jobId);
|
|
982
1285
|
}
|
|
983
|
-
|
|
984
|
-
await withWriteLock(ctx.shardLocks[i], () => {
|
|
985
|
-
const shard = ctx.shards[i];
|
|
986
|
-
const jobsToPromote = [];
|
|
987
|
-
for (const jobId of jobIdsToCheck) {
|
|
988
|
-
const job = shard.waitingDeps.get(jobId);
|
|
989
|
-
if (job?.dependsOn.every((dep) => completedNow.has(dep) || ctx.completedJobs.has(dep) || (ctx.depCompletions?.has(dep) ?? false))) {
|
|
990
|
-
jobsToPromote.push(job);
|
|
991
|
-
}
|
|
992
|
-
}
|
|
993
|
-
if (jobsToPromote.length > 0) {
|
|
994
|
-
promoteJobsToQueue(jobsToPromote, shard, ctx, i, releasedDependencyIds);
|
|
995
|
-
}
|
|
996
|
-
});
|
|
997
|
-
}));
|
|
998
|
-
releaseDependencyCompletionPins(releasedDependencyIds, ctx);
|
|
1286
|
+
return removals.map(({ jobId }) => jobId);
|
|
999
1287
|
}
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
const isDelayed = state === "delayed";
|
|
1012
|
-
shard.incrementQueued(job.id, isDelayed, job.createdAt, job.queue, job.runAt);
|
|
1013
|
-
ctx.jobIndex.set(job.id, { type: "queue", shardIdx, queueName: job.queue });
|
|
1014
|
-
}
|
|
1288
|
+
|
|
1289
|
+
// ../../src/application/emptyQueueCleanup.ts
|
|
1290
|
+
function collectOccupiedQueues(ctx) {
|
|
1291
|
+
const occupied = new Set;
|
|
1292
|
+
for (const [jobId, location] of ctx.jobIndex) {
|
|
1293
|
+
if (location.type !== "processing")
|
|
1294
|
+
continue;
|
|
1295
|
+
occupied.add(location.queueName);
|
|
1296
|
+
const processingJob = ctx.processingShards[location.shardIdx]?.get(jobId);
|
|
1297
|
+
if (processingJob)
|
|
1298
|
+
occupied.add(processingJob.queue);
|
|
1015
1299
|
}
|
|
1016
|
-
|
|
1017
|
-
const
|
|
1018
|
-
|
|
1019
|
-
promotedByQueue.set(job.queue, (promotedByQueue.get(job.queue) ?? 0) + 1);
|
|
1020
|
-
}
|
|
1021
|
-
for (const [queue, count] of promotedByQueue)
|
|
1022
|
-
shard.notifyBatch(queue, count);
|
|
1023
|
-
for (const job of jobsToPromote) {
|
|
1024
|
-
ctx.dashboardEmit?.("job:dependencies-resolved", { jobId: String(job.id), queue: job.queue });
|
|
1025
|
-
}
|
|
1300
|
+
for (const processing of ctx.processingShards) {
|
|
1301
|
+
for (const job of processing.values())
|
|
1302
|
+
occupied.add(job.queue);
|
|
1026
1303
|
}
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
return job[DLQ_RETRY_STATE] ?? null;
|
|
1033
|
-
}
|
|
1034
|
-
function setDlqRetryState(job, state) {
|
|
1035
|
-
if (state === null) {
|
|
1036
|
-
delete job[DLQ_RETRY_STATE];
|
|
1037
|
-
return;
|
|
1304
|
+
for (const shard of ctx.shards) {
|
|
1305
|
+
for (const job of shard.waitingDeps.values())
|
|
1306
|
+
occupied.add(job.queue);
|
|
1307
|
+
for (const job of shard.waitingChildren.values())
|
|
1308
|
+
occupied.add(job.queue);
|
|
1038
1309
|
}
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
});
|
|
1045
|
-
}
|
|
1046
|
-
function createAttemptRecord(job, reason, error, failedAt = Date.now()) {
|
|
1047
|
-
return {
|
|
1048
|
-
attempt: job.attempts,
|
|
1049
|
-
startedAt: job.startedAt ?? job.createdAt,
|
|
1050
|
-
failedAt,
|
|
1051
|
-
reason,
|
|
1052
|
-
error,
|
|
1053
|
-
duration: job.startedAt ? failedAt - job.startedAt : 0
|
|
1054
|
-
};
|
|
1055
|
-
}
|
|
1056
|
-
function recordJobFailureAttempt(job, reason, error, failedAt = Date.now()) {
|
|
1057
|
-
const state = getDlqRetryState(job) ?? {
|
|
1058
|
-
enteredAt: null,
|
|
1059
|
-
expiresAt: null,
|
|
1060
|
-
attempts: [],
|
|
1061
|
-
retryCount: 0,
|
|
1062
|
-
lastRetryAt: null,
|
|
1063
|
-
nextRetryAt: null
|
|
1064
|
-
};
|
|
1065
|
-
state.attempts.push(createAttemptRecord(job, reason, error, failedAt));
|
|
1066
|
-
setDlqRetryState(job, state);
|
|
1067
|
-
}
|
|
1068
|
-
function retainDlqRetryState(job, entry) {
|
|
1069
|
-
setDlqRetryState(job, {
|
|
1070
|
-
enteredAt: entry.enteredAt,
|
|
1071
|
-
expiresAt: entry.expiresAt,
|
|
1072
|
-
attempts: [...entry.attempts],
|
|
1073
|
-
retryCount: entry.retryCount,
|
|
1074
|
-
lastRetryAt: entry.lastRetryAt,
|
|
1075
|
-
nextRetryAt: entry.nextRetryAt
|
|
1076
|
-
});
|
|
1310
|
+
for (const [queue, count] of ctx.pendingQueueAdmissions) {
|
|
1311
|
+
if (count > 0)
|
|
1312
|
+
occupied.add(queue);
|
|
1313
|
+
}
|
|
1314
|
+
return occupied;
|
|
1077
1315
|
}
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
function createDlqEntry(job, reason, error, config = DEFAULT_DLQ_CONFIG, now = Date.now()) {
|
|
1086
|
-
const retained = getDlqRetryState(job);
|
|
1087
|
-
const alreadyEntered = retained?.enteredAt !== null && retained?.enteredAt !== undefined;
|
|
1088
|
-
const attemptRecord = createAttemptRecord(job, reason, error, now);
|
|
1089
|
-
setDlqRetryState(job, null);
|
|
1090
|
-
return {
|
|
1091
|
-
job,
|
|
1092
|
-
enteredAt: alreadyEntered ? retained.enteredAt : now,
|
|
1093
|
-
reason,
|
|
1094
|
-
error,
|
|
1095
|
-
attempts: [...retained?.attempts ?? [], attemptRecord],
|
|
1096
|
-
retryCount: retained?.retryCount ?? 0,
|
|
1097
|
-
lastRetryAt: retained?.lastRetryAt ?? null,
|
|
1098
|
-
nextRetryAt: alreadyEntered ? retained.nextRetryAt : config.autoRetry ? now + config.autoRetryInterval : null,
|
|
1099
|
-
expiresAt: alreadyEntered ? retained.expiresAt : config.maxAge !== null ? now + config.maxAge : null
|
|
1100
|
-
};
|
|
1316
|
+
function collectConfiguredQueues(ctx) {
|
|
1317
|
+
const configured = new Set;
|
|
1318
|
+
for (const shard of ctx.shards) {
|
|
1319
|
+
for (const queue of shard.getConfiguredQueueNames())
|
|
1320
|
+
configured.add(queue);
|
|
1321
|
+
}
|
|
1322
|
+
return configured;
|
|
1101
1323
|
}
|
|
1102
|
-
function
|
|
1103
|
-
|
|
1324
|
+
function collectBufferedCompletedQueues(ctx) {
|
|
1325
|
+
const queues = new Set;
|
|
1326
|
+
for (const job of ctx.storage?.getBufferedJobs() ?? []) {
|
|
1327
|
+
if (job.completedAt === null)
|
|
1328
|
+
continue;
|
|
1329
|
+
const location = ctx.jobIndex.get(job.id);
|
|
1330
|
+
if (location && (location.type !== "completed" || location.queueName !== job.queue))
|
|
1331
|
+
continue;
|
|
1332
|
+
queues.add(job.queue);
|
|
1333
|
+
}
|
|
1334
|
+
return queues;
|
|
1104
1335
|
}
|
|
1105
|
-
function
|
|
1106
|
-
if (
|
|
1107
|
-
return
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1336
|
+
function collectCompletedCounts(ctx, registered) {
|
|
1337
|
+
if (ctx.storage)
|
|
1338
|
+
return ctx.storage.countCompletedJobsByQueue(registered);
|
|
1339
|
+
const registeredSet = new Set(registered);
|
|
1340
|
+
const counts = new Map;
|
|
1341
|
+
for (const [jobId, location] of ctx.jobIndex) {
|
|
1342
|
+
if (location.type !== "completed" || !registeredSet.has(location.queueName) || !ctx.completedJobs.has(jobId)) {
|
|
1343
|
+
continue;
|
|
1344
|
+
}
|
|
1345
|
+
counts.set(location.queueName, (counts.get(location.queueName) ?? 0) + 1);
|
|
1346
|
+
}
|
|
1347
|
+
return counts;
|
|
1113
1348
|
}
|
|
1114
|
-
function
|
|
1115
|
-
const
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1349
|
+
function cleanEmptyQueues(ctx) {
|
|
1350
|
+
const registered = Array.from(ctx.queueNamesCache);
|
|
1351
|
+
const completedCounts = collectCompletedCounts(ctx, registered);
|
|
1352
|
+
const bufferedCompletedQueues = collectBufferedCompletedQueues(ctx);
|
|
1353
|
+
const occupied = collectOccupiedQueues(ctx);
|
|
1354
|
+
const configured = collectConfiguredQueues(ctx);
|
|
1355
|
+
const removed = [];
|
|
1356
|
+
for (const queueName of registered) {
|
|
1357
|
+
const shard = ctx.shards[shardIndex(queueName)];
|
|
1358
|
+
const hasLiveRuntime = (shard.queues.get(queueName)?.size ?? 0) > 0 || shard.getDlqCount(queueName) > 0 || occupied.has(queueName);
|
|
1359
|
+
if (hasLiveRuntime)
|
|
1360
|
+
continue;
|
|
1361
|
+
shard.queues.delete(queueName);
|
|
1362
|
+
shard.clearEmptyGroupRuntime(queueName);
|
|
1363
|
+
if ((completedCounts.get(queueName) ?? 0) > 0 || bufferedCompletedQueues.has(queueName) || configured.has(queueName)) {
|
|
1364
|
+
continue;
|
|
1365
|
+
}
|
|
1366
|
+
ctx.storage?.deleteQueueState(queueName);
|
|
1367
|
+
ctx.storage?.deleteGroupState(queueName);
|
|
1368
|
+
ctx.unregisterQueueName(queueName);
|
|
1369
|
+
shard.obliterate(queueName);
|
|
1370
|
+
removed.push(queueName);
|
|
1123
1371
|
}
|
|
1372
|
+
for (const shard of ctx.shards)
|
|
1373
|
+
shard.cleanOrphanedTemporalEntries();
|
|
1374
|
+
for (const queue of removed)
|
|
1375
|
+
ctx.dashboardEmit?.("queue:removed", { queue });
|
|
1124
1376
|
}
|
|
1125
1377
|
|
|
1126
|
-
// ../../src/
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
return {
|
|
1136
|
-
isStalled: false,
|
|
1137
|
-
stalledFor: 0,
|
|
1138
|
-
shouldMoveToDlq: false,
|
|
1139
|
-
newStallCount: job.stallCount
|
|
1140
|
-
};
|
|
1378
|
+
// ../../src/application/cleanupTasks.ts
|
|
1379
|
+
async function cleanup(ctx) {
|
|
1380
|
+
const now = Date.now();
|
|
1381
|
+
const retention = ctx.config.completedRetentionMs;
|
|
1382
|
+
if (retention !== null && retention !== undefined) {
|
|
1383
|
+
const removed = cleanCompletedJobs(null, now - retention, 1000, ctx);
|
|
1384
|
+
if (removed.length > 0) {
|
|
1385
|
+
ctx.dashboardEmit?.("cleanup:completed-removed", { count: removed.length });
|
|
1386
|
+
}
|
|
1141
1387
|
}
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
isStalled: false,
|
|
1145
|
-
stalledFor: 0,
|
|
1146
|
-
shouldMoveToDlq: false,
|
|
1147
|
-
newStallCount: job.stallCount
|
|
1148
|
-
};
|
|
1388
|
+
for (let i = 0;i < SHARD_COUNT; i++) {
|
|
1389
|
+
ctx.shards[i].refreshDelayedCount(now);
|
|
1149
1390
|
}
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
stalledFor,
|
|
1157
|
-
shouldMoveToDlq: false,
|
|
1158
|
-
newStallCount: job.stallCount
|
|
1159
|
-
};
|
|
1391
|
+
for (let i = 0;i < SHARD_COUNT; i++) {
|
|
1392
|
+
for (const q of ctx.shards[i].queues.values()) {
|
|
1393
|
+
if (q.needsCompaction(0.2)) {
|
|
1394
|
+
q.compact();
|
|
1395
|
+
}
|
|
1396
|
+
}
|
|
1160
1397
|
}
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1398
|
+
await recoverOrphanedProcessingEntries(ctx, now);
|
|
1399
|
+
await cleanStaleWaitingDependencies(ctx, now);
|
|
1400
|
+
cleanUniqueKeysAndGroups(ctx, now);
|
|
1401
|
+
cleanStalledCandidates(ctx);
|
|
1402
|
+
await cleanOrphanedJobIndex(ctx);
|
|
1403
|
+
cleanOrphanedJobLocks(ctx);
|
|
1404
|
+
pruneEndedClientDeliveries(ctx);
|
|
1405
|
+
cleanEmptyQueues(ctx);
|
|
1169
1406
|
}
|
|
1170
|
-
function
|
|
1171
|
-
const
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1407
|
+
async function cleanStaleWaitingDependencies(ctx, now) {
|
|
1408
|
+
const depTimeout = 60 * 60 * 1000;
|
|
1409
|
+
for (let i = 0;i < SHARD_COUNT; i++) {
|
|
1410
|
+
const shard = ctx.shards[i];
|
|
1411
|
+
const staleIds = [];
|
|
1412
|
+
for (const [_id, job] of shard.waitingDeps) {
|
|
1413
|
+
if (now - job.createdAt > depTimeout) {
|
|
1414
|
+
staleIds.push(job.id);
|
|
1415
|
+
}
|
|
1416
|
+
}
|
|
1417
|
+
if (staleIds.length === 0)
|
|
1418
|
+
continue;
|
|
1419
|
+
const removed = await withWriteLock(ctx.shardLocks[i], () => {
|
|
1420
|
+
let count = 0;
|
|
1421
|
+
const released = [];
|
|
1422
|
+
for (const id of staleIds) {
|
|
1423
|
+
const job = shard.waitingDeps.get(id);
|
|
1424
|
+
if (!job || now - job.createdAt <= depTimeout)
|
|
1425
|
+
continue;
|
|
1426
|
+
ctx.storage?.deleteJob(job.id);
|
|
1427
|
+
shard.waitingDeps.delete(job.id);
|
|
1428
|
+
shard.unregisterDependencies(job.id, job.dependsOn);
|
|
1429
|
+
released.push(...job.dependsOn);
|
|
1430
|
+
if (job.uniqueKey && shard.getUniqueKeyEntry(job.queue, job.uniqueKey)?.jobId === job.id) {
|
|
1431
|
+
shard.releaseUniqueKeyIfOwned(job.queue, job.uniqueKey, job.id);
|
|
1432
|
+
}
|
|
1433
|
+
if (job.customId && ctx.customIdMap.get(job.customId) === job.id) {
|
|
1434
|
+
ctx.customIdMap.delete(job.customId);
|
|
1435
|
+
}
|
|
1436
|
+
ctx.dependencyResults?.releaseConsumer(job.id);
|
|
1437
|
+
ctx.jobIndex.delete(job.id);
|
|
1438
|
+
count++;
|
|
1439
|
+
}
|
|
1440
|
+
return { count, released };
|
|
1441
|
+
});
|
|
1442
|
+
releaseDependencyCompletionPins(removed.released, ctx);
|
|
1443
|
+
if (removed.count > 0) {
|
|
1444
|
+
ctx.dashboardEmit?.("cleanup:stale-deps-removed", { count: removed.count });
|
|
1445
|
+
}
|
|
1177
1446
|
}
|
|
1178
|
-
return "retry" /* Retry */;
|
|
1179
|
-
}
|
|
1180
|
-
function incrementStallCount(job) {
|
|
1181
|
-
job.stallCount++;
|
|
1182
|
-
return job.stallCount;
|
|
1183
1447
|
}
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1448
|
+
function cleanUniqueKeysAndGroups(ctx, now) {
|
|
1449
|
+
for (let i = 0;i < SHARD_COUNT; i++) {
|
|
1450
|
+
const shard = ctx.shards[i];
|
|
1451
|
+
shard.cleanExpiredGroupWindows(now);
|
|
1452
|
+
shard.cleanExpiredUniqueKeys();
|
|
1453
|
+
for (const [_queueName, keys] of shard.uniqueKeys) {
|
|
1454
|
+
if (keys.size > 1000) {
|
|
1455
|
+
const toRemove = Math.floor(keys.size / 2);
|
|
1456
|
+
const iter = keys.keys();
|
|
1457
|
+
for (let j = 0;j < toRemove; j++) {
|
|
1458
|
+
const { value, done } = iter.next();
|
|
1459
|
+
if (done)
|
|
1460
|
+
break;
|
|
1461
|
+
keys.delete(value);
|
|
1462
|
+
}
|
|
1463
|
+
}
|
|
1464
|
+
}
|
|
1465
|
+
}
|
|
1196
1466
|
}
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
constructor(capacity, refillRate = capacity) {
|
|
1204
|
-
this.capacity = capacity;
|
|
1205
|
-
this.refillRate = refillRate;
|
|
1206
|
-
this.tokens = capacity;
|
|
1207
|
-
this.lastRefill = Date.now();
|
|
1467
|
+
function cleanStalledCandidates(ctx) {
|
|
1468
|
+
for (const jobId of ctx.stalledCandidates) {
|
|
1469
|
+
const loc = ctx.jobIndex.get(jobId);
|
|
1470
|
+
if (loc?.type !== "processing") {
|
|
1471
|
+
ctx.stalledCandidates.delete(jobId);
|
|
1472
|
+
}
|
|
1208
1473
|
}
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1474
|
+
}
|
|
1475
|
+
async function cleanOrphanedJobIndex(ctx) {
|
|
1476
|
+
if (ctx.jobIndex.size <= 1e5)
|
|
1477
|
+
return;
|
|
1478
|
+
const processingCandidates = new Map;
|
|
1479
|
+
const queueCandidates = new Map;
|
|
1480
|
+
for (const [jobId, loc] of ctx.jobIndex) {
|
|
1481
|
+
if (loc.type === "processing") {
|
|
1482
|
+
const procIdx = processingShardIndex(jobId);
|
|
1483
|
+
let list = processingCandidates.get(procIdx);
|
|
1484
|
+
if (!list) {
|
|
1485
|
+
list = [];
|
|
1486
|
+
processingCandidates.set(procIdx, list);
|
|
1487
|
+
}
|
|
1488
|
+
list.push(jobId);
|
|
1489
|
+
} else if (loc.type === "queue") {
|
|
1490
|
+
let list = queueCandidates.get(loc.shardIdx);
|
|
1491
|
+
if (!list) {
|
|
1492
|
+
list = [];
|
|
1493
|
+
queueCandidates.set(loc.shardIdx, list);
|
|
1494
|
+
}
|
|
1495
|
+
list.push({ jobId, queueName: loc.queueName });
|
|
1214
1496
|
}
|
|
1215
|
-
return false;
|
|
1216
1497
|
}
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1498
|
+
for (const [procIdx, candidates] of processingCandidates) {
|
|
1499
|
+
await withWriteLock(ctx.processingLocks[procIdx], () => {
|
|
1500
|
+
for (const jobId of candidates) {
|
|
1501
|
+
if (!ctx.processingShards[procIdx].has(jobId)) {
|
|
1502
|
+
ctx.jobIndex.delete(jobId);
|
|
1503
|
+
}
|
|
1504
|
+
}
|
|
1505
|
+
});
|
|
1223
1506
|
}
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1507
|
+
for (const [shardIdx, candidates] of queueCandidates) {
|
|
1508
|
+
await withWriteLock(ctx.shardLocks[shardIdx], () => {
|
|
1509
|
+
const shard = ctx.shards[shardIdx];
|
|
1510
|
+
for (const { jobId, queueName } of candidates) {
|
|
1511
|
+
if (!shard.getQueue(queueName).has(jobId)) {
|
|
1512
|
+
ctx.jobIndex.delete(jobId);
|
|
1513
|
+
}
|
|
1514
|
+
}
|
|
1515
|
+
});
|
|
1227
1516
|
}
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
return Math.ceil((1 - tokens) / this.refillRate * 1000);
|
|
1517
|
+
}
|
|
1518
|
+
function cleanOrphanedJobLocks(ctx) {
|
|
1519
|
+
for (const jobId of ctx.jobLocks.keys()) {
|
|
1520
|
+
const loc = ctx.jobIndex.get(jobId);
|
|
1521
|
+
if (loc?.type !== "processing") {
|
|
1522
|
+
ctx.jobLocks.delete(jobId);
|
|
1523
|
+
}
|
|
1236
1524
|
}
|
|
1237
1525
|
}
|
|
1238
1526
|
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1527
|
+
// ../../src/application/dependencyProcessor.ts
|
|
1528
|
+
function dependencyReadyState(job, now) {
|
|
1529
|
+
if (job.runAt > now)
|
|
1530
|
+
return "delayed";
|
|
1531
|
+
return job.priority > 0 ? "prioritized" : "waiting";
|
|
1532
|
+
}
|
|
1533
|
+
function checkpointDependencyPromotion(job, state, now, storage) {
|
|
1534
|
+
if (job.timeline.at(-1)?.state !== state && job.timeline.length < MAX_TIMELINE_ENTRIES) {
|
|
1535
|
+
job.timeline.push({ state, timestamp: now });
|
|
1244
1536
|
}
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1537
|
+
storage?.updateFlowParentResolution(job, state);
|
|
1538
|
+
}
|
|
1539
|
+
async function processPendingDependencies(ctx) {
|
|
1540
|
+
if (ctx.pendingDepChecks.size === 0)
|
|
1541
|
+
return;
|
|
1542
|
+
const completedIds = Array.from(ctx.pendingDepChecks);
|
|
1543
|
+
const completedNow = new Set(completedIds);
|
|
1544
|
+
ctx.pendingDepChecks.clear();
|
|
1545
|
+
const jobsToCheckByShard = new Map;
|
|
1546
|
+
const releasedDependencyIds = new Set;
|
|
1547
|
+
for (const completedId of completedIds) {
|
|
1548
|
+
for (let i = 0;i < SHARD_COUNT; i++) {
|
|
1549
|
+
const waitingJobIds = ctx.shards[i].getJobsWaitingFor(completedId);
|
|
1550
|
+
if (waitingJobIds && waitingJobIds.size > 0) {
|
|
1551
|
+
let shardJobs = jobsToCheckByShard.get(i);
|
|
1552
|
+
if (!shardJobs) {
|
|
1553
|
+
shardJobs = new Set;
|
|
1554
|
+
jobsToCheckByShard.set(i, shardJobs);
|
|
1555
|
+
}
|
|
1556
|
+
for (const jobId of waitingJobIds) {
|
|
1557
|
+
shardJobs.add(jobId);
|
|
1558
|
+
}
|
|
1559
|
+
}
|
|
1249
1560
|
}
|
|
1250
|
-
return false;
|
|
1251
1561
|
}
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1562
|
+
await Promise.all(Array.from(jobsToCheckByShard.entries()).map(async ([i, jobIdsToCheck]) => {
|
|
1563
|
+
await withWriteLock(ctx.shardLocks[i], () => {
|
|
1564
|
+
const shard = ctx.shards[i];
|
|
1565
|
+
const jobsToPromote = [];
|
|
1566
|
+
for (const jobId of jobIdsToCheck) {
|
|
1567
|
+
const job = shard.waitingDeps.get(jobId);
|
|
1568
|
+
if (job?.dependsOn.every((dep) => completedNow.has(dep) || ctx.completedJobs.has(dep) || (ctx.depCompletions?.has(dep) ?? false))) {
|
|
1569
|
+
jobsToPromote.push(job);
|
|
1570
|
+
}
|
|
1571
|
+
}
|
|
1572
|
+
if (jobsToPromote.length > 0) {
|
|
1573
|
+
promoteJobsToQueue(jobsToPromote, shard, ctx, i, releasedDependencyIds);
|
|
1574
|
+
}
|
|
1575
|
+
});
|
|
1576
|
+
}));
|
|
1577
|
+
releaseDependencyCompletionPins(releasedDependencyIds, ctx);
|
|
1578
|
+
}
|
|
1579
|
+
function promoteJobsToQueue(jobsToPromote, shard, ctx, shardIdx, releasedDependencyIds) {
|
|
1580
|
+
const now = Date.now();
|
|
1581
|
+
for (const job of jobsToPromote) {
|
|
1582
|
+
if (shard.waitingDeps.has(job.id)) {
|
|
1583
|
+
const state = dependencyReadyState(job, now);
|
|
1584
|
+
checkpointDependencyPromotion(job, state, now, ctx.storage);
|
|
1585
|
+
shard.waitingDeps.delete(job.id);
|
|
1586
|
+
shard.unregisterDependencies(job.id, job.dependsOn);
|
|
1587
|
+
for (const dependencyId of job.dependsOn)
|
|
1588
|
+
releasedDependencyIds.add(dependencyId);
|
|
1589
|
+
shard.getQueue(job.queue).push(job);
|
|
1590
|
+
const isDelayed = state === "delayed";
|
|
1591
|
+
shard.incrementQueued(job.id, isDelayed, job.createdAt, job.queue, job.runAt);
|
|
1592
|
+
ctx.jobIndex.set(job.id, { type: "queue", shardIdx, queueName: job.queue });
|
|
1255
1593
|
}
|
|
1256
1594
|
}
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1595
|
+
if (jobsToPromote.length > 0) {
|
|
1596
|
+
const promotedByQueue = new Map;
|
|
1597
|
+
for (const job of jobsToPromote) {
|
|
1598
|
+
promotedByQueue.set(job.queue, (promotedByQueue.get(job.queue) ?? 0) + 1);
|
|
1599
|
+
}
|
|
1600
|
+
for (const [queue, count] of promotedByQueue)
|
|
1601
|
+
shard.notifyBatch(queue, count);
|
|
1602
|
+
for (const job of jobsToPromote) {
|
|
1603
|
+
ctx.dashboardEmit?.("job:dependencies-resolved", { jobId: String(job.id), queue: job.queue });
|
|
1604
|
+
}
|
|
1265
1605
|
}
|
|
1266
1606
|
}
|
|
1267
1607
|
|
|
@@ -1273,16 +1613,10 @@ function createLock(jobId, owner, ctx, ttl = DEFAULT_LOCK_TTL) {
|
|
|
1273
1613
|
const existing = ctx.jobLocks.get(jobId);
|
|
1274
1614
|
if (existing) {
|
|
1275
1615
|
const job = ctx.processingShards[loc.shardIdx].get(jobId);
|
|
1276
|
-
|
|
1277
|
-
if (!isNewGeneration)
|
|
1616
|
+
if (!job || !isLeaseFromEarlierGeneration(job, existing))
|
|
1278
1617
|
return null;
|
|
1279
1618
|
ctx.jobLocks.delete(jobId);
|
|
1280
|
-
|
|
1281
|
-
if (!jobs.delete(jobId))
|
|
1282
|
-
continue;
|
|
1283
|
-
if (jobs.size === 0)
|
|
1284
|
-
ctx.clientJobs.delete(clientId);
|
|
1285
|
-
}
|
|
1619
|
+
detachClientJob(jobId, ctx);
|
|
1286
1620
|
}
|
|
1287
1621
|
const lock = createJobLock(jobId, owner, ttl);
|
|
1288
1622
|
ctx.jobLocks.set(jobId, lock);
|
|
@@ -1327,42 +1661,29 @@ function renewJobLockBatch(items, ctx) {
|
|
|
1327
1661
|
return renewed;
|
|
1328
1662
|
}
|
|
1329
1663
|
function releaseLock(jobId, ctx, token) {
|
|
1330
|
-
const lock = ctx.jobLocks.get(jobId);
|
|
1331
|
-
if (!lock)
|
|
1332
|
-
return true;
|
|
1333
|
-
if (token && lock.token !== token) {
|
|
1334
|
-
return false;
|
|
1335
|
-
}
|
|
1336
|
-
ctx.jobLocks.delete(jobId);
|
|
1337
|
-
return true;
|
|
1338
|
-
}
|
|
1339
|
-
function getLockInfo(jobId, ctx) {
|
|
1340
|
-
return ctx.jobLocks.get(jobId) ?? null;
|
|
1341
|
-
}
|
|
1342
|
-
// ../../src/application/clientTracking.ts
|
|
1343
|
-
function registerClientJob(clientId, jobId, ctx) {
|
|
1344
|
-
let jobs = ctx.clientJobs.get(clientId);
|
|
1345
|
-
if (!jobs) {
|
|
1346
|
-
jobs = new Set;
|
|
1347
|
-
ctx.clientJobs.set(clientId, jobs);
|
|
1348
|
-
}
|
|
1349
|
-
jobs.add(jobId);
|
|
1350
|
-
}
|
|
1351
|
-
function unregisterClientJob(clientId, jobId, ctx) {
|
|
1352
|
-
if (!clientId)
|
|
1353
|
-
return;
|
|
1354
|
-
const jobs = ctx.clientJobs.get(clientId);
|
|
1355
|
-
if (jobs) {
|
|
1356
|
-
jobs.delete(jobId);
|
|
1357
|
-
if (jobs.size === 0) {
|
|
1358
|
-
ctx.clientJobs.delete(clientId);
|
|
1359
|
-
}
|
|
1664
|
+
const lock = ctx.jobLocks.get(jobId);
|
|
1665
|
+
if (!lock)
|
|
1666
|
+
return true;
|
|
1667
|
+
if (token && lock.token !== token) {
|
|
1668
|
+
return false;
|
|
1360
1669
|
}
|
|
1670
|
+
ctx.jobLocks.delete(jobId);
|
|
1671
|
+
return true;
|
|
1672
|
+
}
|
|
1673
|
+
function getLockInfo(jobId, ctx) {
|
|
1674
|
+
return ctx.jobLocks.get(jobId) ?? null;
|
|
1675
|
+
}
|
|
1676
|
+
// ../../src/application/clientTracking.ts
|
|
1677
|
+
function isReleasable(clientId, jobId, ctx) {
|
|
1678
|
+
if (!ownsCurrentDelivery(clientId, jobId, ctx))
|
|
1679
|
+
return false;
|
|
1680
|
+
const lock = ctx.jobLocks.get(jobId);
|
|
1681
|
+
return !(lock && lock.renewalCount > 0);
|
|
1361
1682
|
}
|
|
1362
1683
|
async function releaseClientJobs(clientId, ctx) {
|
|
1363
1684
|
const jobs = ctx.clientJobs.get(clientId);
|
|
1364
1685
|
if (!jobs || jobs.size === 0) {
|
|
1365
|
-
|
|
1686
|
+
dropClient(clientId, ctx);
|
|
1366
1687
|
return 0;
|
|
1367
1688
|
}
|
|
1368
1689
|
const jobsToRelease = [];
|
|
@@ -1370,8 +1691,7 @@ async function releaseClientJobs(clientId, ctx) {
|
|
|
1370
1691
|
const loc = ctx.jobIndex.get(jobId);
|
|
1371
1692
|
if (loc?.type !== "processing")
|
|
1372
1693
|
continue;
|
|
1373
|
-
|
|
1374
|
-
if (lock && lock.renewalCount > 0)
|
|
1694
|
+
if (!isReleasable(clientId, jobId, ctx))
|
|
1375
1695
|
continue;
|
|
1376
1696
|
const procIdx = loc.shardIdx;
|
|
1377
1697
|
const job = ctx.processingShards[procIdx].get(jobId);
|
|
@@ -1384,7 +1704,7 @@ async function releaseClientJobs(clientId, ctx) {
|
|
|
1384
1704
|
});
|
|
1385
1705
|
}
|
|
1386
1706
|
if (jobsToRelease.length === 0) {
|
|
1387
|
-
|
|
1707
|
+
dropClient(clientId, ctx);
|
|
1388
1708
|
return 0;
|
|
1389
1709
|
}
|
|
1390
1710
|
const byProcShard = new Map;
|
|
@@ -1414,7 +1734,7 @@ async function releaseClientJobs(clientId, ctx) {
|
|
|
1414
1734
|
await withWriteLock(ctx.processingLocks[procIdx], () => {
|
|
1415
1735
|
for (const { jobId } of shardItems) {
|
|
1416
1736
|
const job = ctx.processingShards[procIdx].get(jobId);
|
|
1417
|
-
if (!job)
|
|
1737
|
+
if (!job || !isReleasable(clientId, jobId, ctx))
|
|
1418
1738
|
continue;
|
|
1419
1739
|
released += releaseJobToQueue({ jobId, job, procIdx, queueShardIdx, ctx, now });
|
|
1420
1740
|
}
|
|
@@ -1424,17 +1744,19 @@ async function releaseClientJobs(clientId, ctx) {
|
|
|
1424
1744
|
}
|
|
1425
1745
|
return released;
|
|
1426
1746
|
} finally {
|
|
1427
|
-
|
|
1747
|
+
dropClient(clientId, ctx);
|
|
1428
1748
|
}
|
|
1429
1749
|
}
|
|
1430
1750
|
function forceReleaseClientJobs(clientId, ctx) {
|
|
1431
1751
|
const jobs = ctx.clientJobs.get(clientId);
|
|
1432
1752
|
if (!jobs || jobs.size === 0) {
|
|
1433
|
-
|
|
1753
|
+
dropClient(clientId, ctx);
|
|
1434
1754
|
return 0;
|
|
1435
1755
|
}
|
|
1436
1756
|
let touched = 0;
|
|
1437
1757
|
for (const jobId of jobs) {
|
|
1758
|
+
if (!ownsCurrentDelivery(clientId, jobId, ctx))
|
|
1759
|
+
continue;
|
|
1438
1760
|
ctx.jobLocks.delete(jobId);
|
|
1439
1761
|
const loc = ctx.jobIndex.get(jobId);
|
|
1440
1762
|
if (loc?.type !== "processing")
|
|
@@ -1447,7 +1769,7 @@ function forceReleaseClientJobs(clientId, ctx) {
|
|
|
1447
1769
|
ctx.timeoutScheduler.cancel(jobId);
|
|
1448
1770
|
touched++;
|
|
1449
1771
|
}
|
|
1450
|
-
|
|
1772
|
+
dropClient(clientId, ctx);
|
|
1451
1773
|
return touched;
|
|
1452
1774
|
}
|
|
1453
1775
|
function releaseJobToQueue(opts) {
|
|
@@ -1456,6 +1778,7 @@ function releaseJobToQueue(opts) {
|
|
|
1456
1778
|
ctx.processingShards[procIdx].delete(jobId);
|
|
1457
1779
|
ctx.timeoutScheduler.cancel(jobId);
|
|
1458
1780
|
ctx.jobLocks.delete(jobId);
|
|
1781
|
+
detachClientJob(jobId, ctx);
|
|
1459
1782
|
shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId, job.id);
|
|
1460
1783
|
if (job.uniqueKey?.startsWith("cron:")) {
|
|
1461
1784
|
ctx.jobIndex.delete(jobId);
|
|
@@ -1479,7 +1802,7 @@ async function checkExpiredLocks(ctx) {
|
|
|
1479
1802
|
if (isLockExpired(lock, now)) {
|
|
1480
1803
|
const procIdx = processingShardIndex(jobId);
|
|
1481
1804
|
const job = ctx.processingShards[procIdx].get(jobId);
|
|
1482
|
-
if (job) {
|
|
1805
|
+
if (job && !isLeaseFromEarlierGeneration(job, lock)) {
|
|
1483
1806
|
const shardIdx = shardIndex(job.queue);
|
|
1484
1807
|
expired.push({ jobId, lock, procIdx, shardIdx, job });
|
|
1485
1808
|
} else {
|
|
@@ -1510,7 +1833,7 @@ async function checkExpiredLocks(ctx) {
|
|
|
1510
1833
|
for (const { jobId, lock, job } of items) {
|
|
1511
1834
|
const currentJob = ctx.processingShards[procIdx].get(jobId);
|
|
1512
1835
|
const currentLock = ctx.jobLocks.get(jobId);
|
|
1513
|
-
if (currentJob !== job || currentLock !== lock || !isLockExpired(currentLock, Date.now())) {
|
|
1836
|
+
if (currentJob !== job || currentLock !== lock || !isLockExpired(currentLock, Date.now()) || isLeaseFromEarlierGeneration(currentJob, currentLock)) {
|
|
1514
1837
|
continue;
|
|
1515
1838
|
}
|
|
1516
1839
|
processExpiredLockInner(jobId, lock, job, shardIdx, procIdx, ctx, now);
|
|
@@ -1525,6 +1848,7 @@ function processExpiredLockInner(jobId, lock, job, shardIdx, procIdx, ctx, now)
|
|
|
1525
1848
|
const queue = shard.getQueue(job.queue);
|
|
1526
1849
|
ctx.processingShards[procIdx].delete(jobId);
|
|
1527
1850
|
ctx.timeoutScheduler.cancel(jobId);
|
|
1851
|
+
detachClientJob(jobId, ctx);
|
|
1528
1852
|
if (job.uniqueKey?.startsWith("cron:")) {
|
|
1529
1853
|
ctx.retiredCronLeaseTokens.set(jobId, lock.token);
|
|
1530
1854
|
shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId, job.id);
|
|
@@ -1752,239 +2076,6 @@ function checkStorageSize(ctx) {
|
|
|
1752
2076
|
}
|
|
1753
2077
|
}
|
|
1754
2078
|
|
|
1755
|
-
// ../../src/shared/logger.ts
|
|
1756
|
-
var LOG_LEVEL_PRIORITY = {
|
|
1757
|
-
debug: 0,
|
|
1758
|
-
info: 1,
|
|
1759
|
-
warn: 2,
|
|
1760
|
-
error: 3
|
|
1761
|
-
};
|
|
1762
|
-
|
|
1763
|
-
class Logger {
|
|
1764
|
-
component;
|
|
1765
|
-
static jsonMode = false;
|
|
1766
|
-
static level = "info";
|
|
1767
|
-
constructor(component) {
|
|
1768
|
-
this.component = component;
|
|
1769
|
-
}
|
|
1770
|
-
static enableJsonMode() {
|
|
1771
|
-
Logger.jsonMode = true;
|
|
1772
|
-
}
|
|
1773
|
-
static disableJsonMode() {
|
|
1774
|
-
Logger.jsonMode = false;
|
|
1775
|
-
}
|
|
1776
|
-
static setLevel(level) {
|
|
1777
|
-
Logger.level = level;
|
|
1778
|
-
}
|
|
1779
|
-
debug(message, data) {
|
|
1780
|
-
this.log("debug", message, data);
|
|
1781
|
-
}
|
|
1782
|
-
debugToStderr(message, data) {
|
|
1783
|
-
this.log("debug", message, data, true);
|
|
1784
|
-
}
|
|
1785
|
-
info(message, data) {
|
|
1786
|
-
this.log("info", message, data);
|
|
1787
|
-
}
|
|
1788
|
-
infoToStderr(message, data) {
|
|
1789
|
-
this.log("info", message, data, true);
|
|
1790
|
-
}
|
|
1791
|
-
warn(message, data) {
|
|
1792
|
-
this.log("warn", message, data);
|
|
1793
|
-
}
|
|
1794
|
-
error(message, data) {
|
|
1795
|
-
this.log("error", message, data);
|
|
1796
|
-
}
|
|
1797
|
-
log(level, message, data, forceStderr = false) {
|
|
1798
|
-
if (LOG_LEVEL_PRIORITY[level] < LOG_LEVEL_PRIORITY[Logger.level])
|
|
1799
|
-
return;
|
|
1800
|
-
if (Logger.jsonMode) {
|
|
1801
|
-
const entry = {
|
|
1802
|
-
timestamp: new Date().toISOString(),
|
|
1803
|
-
level,
|
|
1804
|
-
component: this.component,
|
|
1805
|
-
message,
|
|
1806
|
-
...data && { data }
|
|
1807
|
-
};
|
|
1808
|
-
if (forceStderr)
|
|
1809
|
-
console.error(JSON.stringify(entry));
|
|
1810
|
-
else
|
|
1811
|
-
console.log(JSON.stringify(entry));
|
|
1812
|
-
} else {
|
|
1813
|
-
const prefix = `[${this.component}]`;
|
|
1814
|
-
const dataStr = data ? ` ${JSON.stringify(data)}` : "";
|
|
1815
|
-
switch (level) {
|
|
1816
|
-
case "debug":
|
|
1817
|
-
if (forceStderr)
|
|
1818
|
-
console.error(`${prefix} ${message}${dataStr}`);
|
|
1819
|
-
else
|
|
1820
|
-
console.debug(`${prefix} ${message}${dataStr}`);
|
|
1821
|
-
break;
|
|
1822
|
-
case "info":
|
|
1823
|
-
if (forceStderr)
|
|
1824
|
-
console.error(`${prefix} ${message}${dataStr}`);
|
|
1825
|
-
else
|
|
1826
|
-
console.log(`${prefix} ${message}${dataStr}`);
|
|
1827
|
-
break;
|
|
1828
|
-
case "warn":
|
|
1829
|
-
console.warn(`${prefix} ${message}${dataStr}`);
|
|
1830
|
-
break;
|
|
1831
|
-
case "error":
|
|
1832
|
-
console.error(`${prefix} ${message}${dataStr}`);
|
|
1833
|
-
break;
|
|
1834
|
-
}
|
|
1835
|
-
}
|
|
1836
|
-
}
|
|
1837
|
-
}
|
|
1838
|
-
function createLogger(component) {
|
|
1839
|
-
return new Logger(component);
|
|
1840
|
-
}
|
|
1841
|
-
var serverLog = createLogger("Server");
|
|
1842
|
-
var tcpLog = createLogger("TCP");
|
|
1843
|
-
var httpLog = createLogger("HTTP");
|
|
1844
|
-
var wsLog = createLogger("WS");
|
|
1845
|
-
var cronLog = createLogger("Cron");
|
|
1846
|
-
var statsLog = createLogger("Stats");
|
|
1847
|
-
var storageLog = createLogger("Storage");
|
|
1848
|
-
var queueLog = createLogger("Queue");
|
|
1849
|
-
var webhookLog = createLogger("Webhook");
|
|
1850
|
-
var backupLog = createLogger("Backup");
|
|
1851
|
-
|
|
1852
|
-
// ../../src/application/stallDetection.ts
|
|
1853
|
-
function checkStalledJobs(ctx) {
|
|
1854
|
-
const now = Date.now();
|
|
1855
|
-
const confirmedStalled = [];
|
|
1856
|
-
for (const jobId of ctx.stalledCandidates) {
|
|
1857
|
-
const procIdx = processingShardIndex(jobId);
|
|
1858
|
-
const job = ctx.processingShards[procIdx].get(jobId);
|
|
1859
|
-
if (!job) {
|
|
1860
|
-
ctx.stalledCandidates.delete(jobId);
|
|
1861
|
-
continue;
|
|
1862
|
-
}
|
|
1863
|
-
const stallConfig = ctx.shards[shardIndex(job.queue)].getStallConfig(job.queue);
|
|
1864
|
-
if (!stallConfig.enabled) {
|
|
1865
|
-
ctx.stalledCandidates.delete(jobId);
|
|
1866
|
-
continue;
|
|
1867
|
-
}
|
|
1868
|
-
const action = getStallAction(job, stallConfig, now);
|
|
1869
|
-
if (action !== "keep" /* Keep */) {
|
|
1870
|
-
confirmedStalled.push({ job, action });
|
|
1871
|
-
}
|
|
1872
|
-
ctx.stalledCandidates.delete(jobId);
|
|
1873
|
-
}
|
|
1874
|
-
for (let i = 0;i < SHARD_COUNT; i++) {
|
|
1875
|
-
const procShard = ctx.processingShards[i];
|
|
1876
|
-
for (const [jobId, job] of procShard) {
|
|
1877
|
-
const stallConfig = ctx.shards[shardIndex(job.queue)].getStallConfig(job.queue);
|
|
1878
|
-
if (!stallConfig.enabled)
|
|
1879
|
-
continue;
|
|
1880
|
-
const action = getStallAction(job, stallConfig, now);
|
|
1881
|
-
if (action !== "keep" /* Keep */) {
|
|
1882
|
-
ctx.stalledCandidates.add(jobId);
|
|
1883
|
-
}
|
|
1884
|
-
}
|
|
1885
|
-
}
|
|
1886
|
-
for (const { job, action } of confirmedStalled) {
|
|
1887
|
-
handleStalledJob(job, action, ctx).catch((err) => {
|
|
1888
|
-
queueLog.error("Failed to handle stalled job", {
|
|
1889
|
-
jobId: String(job.id),
|
|
1890
|
-
error: String(err)
|
|
1891
|
-
});
|
|
1892
|
-
});
|
|
1893
|
-
}
|
|
1894
|
-
}
|
|
1895
|
-
async function handleStalledJob(job, action, ctx) {
|
|
1896
|
-
const idx = shardIndex(job.queue);
|
|
1897
|
-
const procIdx = processingShardIndex(job.id);
|
|
1898
|
-
let handled = false;
|
|
1899
|
-
let handledAction = action;
|
|
1900
|
-
await withWriteLock(ctx.shardLocks[idx], async () => {
|
|
1901
|
-
await withWriteLock(ctx.processingLocks[procIdx], () => {
|
|
1902
|
-
if (!ctx.processingShards[procIdx].has(job.id)) {
|
|
1903
|
-
return;
|
|
1904
|
-
}
|
|
1905
|
-
const shard = ctx.shards[idx];
|
|
1906
|
-
const attemptsExhausted = job.attempts + 1 >= job.maxAttempts;
|
|
1907
|
-
if (action === "move_to_dlq" /* MoveToDlq */ || attemptsExhausted) {
|
|
1908
|
-
handledAction = "move_to_dlq" /* MoveToDlq */;
|
|
1909
|
-
moveStalliedJobToDlq(job, ctx, shard, procIdx, attemptsExhausted);
|
|
1910
|
-
} else {
|
|
1911
|
-
retryStalliedJob(job, ctx, shard, procIdx, idx);
|
|
1912
|
-
}
|
|
1913
|
-
handled = true;
|
|
1914
|
-
});
|
|
1915
|
-
});
|
|
1916
|
-
if (handled) {
|
|
1917
|
-
ctx.dashboardEmit?.("job:stalled", {
|
|
1918
|
-
jobId: String(job.id),
|
|
1919
|
-
queue: job.queue,
|
|
1920
|
-
stallCount: job.stallCount,
|
|
1921
|
-
action: handledAction
|
|
1922
|
-
});
|
|
1923
|
-
ctx.eventsManager.broadcast({
|
|
1924
|
-
eventType: "stalled" /* Stalled */,
|
|
1925
|
-
queue: job.queue,
|
|
1926
|
-
jobId: job.id,
|
|
1927
|
-
timestamp: Date.now(),
|
|
1928
|
-
data: { stallCount: job.stallCount, action: handledAction }
|
|
1929
|
-
});
|
|
1930
|
-
ctx.webhookManager.trigger("stalled", String(job.id), job.queue, {
|
|
1931
|
-
data: { stallCount: job.stallCount, action: handledAction }
|
|
1932
|
-
});
|
|
1933
|
-
}
|
|
1934
|
-
}
|
|
1935
|
-
function moveStalliedJobToDlq(job, ctx, shard, procIdx, attemptsExhausted) {
|
|
1936
|
-
ctx.processingShards[procIdx].delete(job.id);
|
|
1937
|
-
ctx.timeoutScheduler.cancel(job.id);
|
|
1938
|
-
shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId, job.id);
|
|
1939
|
-
if (job.uniqueKey?.startsWith("cron:")) {
|
|
1940
|
-
ctx.jobIndex.delete(job.id);
|
|
1941
|
-
ctx.storage?.deleteJob(job.id);
|
|
1942
|
-
return;
|
|
1943
|
-
}
|
|
1944
|
-
incrementStallCount(job);
|
|
1945
|
-
job.attempts++;
|
|
1946
|
-
job.startedAt = null;
|
|
1947
|
-
job.lastHeartbeat = Date.now();
|
|
1948
|
-
const entry = shard.addToDlq(job, attemptsExhausted ? "max_attempts_exceeded" /* MaxAttemptsExceeded */ : "stalled" /* Stalled */, attemptsExhausted ? `Job stalled at max attempts (${job.maxAttempts})` : `Job stalled ${job.stallCount} times`);
|
|
1949
|
-
ctx.jobIndex.set(job.id, { type: "dlq", queueName: job.queue });
|
|
1950
|
-
ctx.storage?.saveDlqEntry(entry);
|
|
1951
|
-
ctx.storage?.deleteJob(job.id);
|
|
1952
|
-
}
|
|
1953
|
-
function retryStalliedJob(job, ctx, shard, procIdx, idx) {
|
|
1954
|
-
if (job.uniqueKey?.startsWith("cron:")) {
|
|
1955
|
-
ctx.processingShards[procIdx].delete(job.id);
|
|
1956
|
-
ctx.timeoutScheduler.cancel(job.id);
|
|
1957
|
-
shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId, job.id);
|
|
1958
|
-
ctx.jobIndex.delete(job.id);
|
|
1959
|
-
ctx.storage?.deleteJob(job.id);
|
|
1960
|
-
return;
|
|
1961
|
-
}
|
|
1962
|
-
incrementStallCount(job);
|
|
1963
|
-
job.attempts++;
|
|
1964
|
-
const stallNow = Date.now();
|
|
1965
|
-
recordJobFailureAttempt(job, "stalled" /* Stalled */, "stalled", stallNow);
|
|
1966
|
-
job.startedAt = null;
|
|
1967
|
-
job.runAt = stallNow + calculateBackoff(job);
|
|
1968
|
-
job.lastHeartbeat = stallNow;
|
|
1969
|
-
if (job.timeline.length < MAX_TIMELINE_ENTRIES) {
|
|
1970
|
-
job.timeline.push({
|
|
1971
|
-
state: "failed",
|
|
1972
|
-
timestamp: stallNow,
|
|
1973
|
-
error: "stalled",
|
|
1974
|
-
attempt: job.attempts
|
|
1975
|
-
});
|
|
1976
|
-
job.timeline.push({ state: "waiting", timestamp: stallNow, attempt: job.attempts + 1 });
|
|
1977
|
-
}
|
|
1978
|
-
ctx.processingShards[procIdx].delete(job.id);
|
|
1979
|
-
ctx.timeoutScheduler.cancel(job.id);
|
|
1980
|
-
shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId, job.id);
|
|
1981
|
-
shard.getQueue(job.queue).push(job);
|
|
1982
|
-
const isDelayed = job.runAt > Date.now();
|
|
1983
|
-
shard.incrementQueued(job.id, isDelayed, job.createdAt, job.queue, job.runAt);
|
|
1984
|
-
ctx.jobIndex.set(job.id, { type: "queue", shardIdx: idx, queueName: job.queue });
|
|
1985
|
-
ctx.storage?.updateForRetry(job);
|
|
1986
|
-
}
|
|
1987
|
-
|
|
1988
2079
|
// ../../src/application/taskErrorTracking.ts
|
|
1989
2080
|
var MAX_CONSECUTIVE_FAILURES = 5;
|
|
1990
2081
|
var taskErrors = {
|
|
@@ -2407,6 +2498,7 @@ function getLockContext(ctx) {
|
|
|
2407
2498
|
jobLocks: ctx.jobLocks,
|
|
2408
2499
|
retiredCronLeaseTokens: ctx.retiredCronLeaseTokens,
|
|
2409
2500
|
clientJobs: ctx.clientJobs,
|
|
2501
|
+
clientJobOwners: ctx.clientJobOwners,
|
|
2410
2502
|
processingShards: ctx.processingShards,
|
|
2411
2503
|
processingLocks: ctx.processingLocks,
|
|
2412
2504
|
shards: ctx.shards,
|
|
@@ -3665,7 +3757,7 @@ var latencyTracker = new LatencyTracker;
|
|
|
3665
3757
|
// ../../package.json
|
|
3666
3758
|
var package_default = {
|
|
3667
3759
|
name: "bunqueue",
|
|
3668
|
-
version: "2.9.
|
|
3760
|
+
version: "2.9.7",
|
|
3669
3761
|
description: "High-performance job queue for Bun and AI agents. Memory or one-file SQLite, optional PostgreSQL 15\u201318 multi-broker persistence, cron, retries, DLQ, and MCP.",
|
|
3670
3762
|
keywords: [
|
|
3671
3763
|
"agentic",
|
|
@@ -4279,12 +4371,7 @@ function isStorageDegraded(status) {
|
|
|
4279
4371
|
// ../../src/application/operations/jobClaim.ts
|
|
4280
4372
|
function releaseClaimedJobOwnership(jobId, ctx) {
|
|
4281
4373
|
ctx.jobLocks.delete(jobId);
|
|
4282
|
-
|
|
4283
|
-
if (!jobs.delete(jobId))
|
|
4284
|
-
continue;
|
|
4285
|
-
if (jobs.size === 0)
|
|
4286
|
-
ctx.clientJobs.delete(clientId);
|
|
4287
|
-
}
|
|
4374
|
+
detachClientJob(jobId, ctx);
|
|
4288
4375
|
}
|
|
4289
4376
|
|
|
4290
4377
|
// ../../src/application/operations/jobMoveOperations.ts
|
|
@@ -7042,9 +7129,10 @@ function validateOptions(input) {
|
|
|
7042
7129
|
if (input.backoff.type !== "fixed" && input.backoff.type !== "exponential") {
|
|
7043
7130
|
throw new Error("backoff.type must be 'fixed' or 'exponential'");
|
|
7044
7131
|
}
|
|
7045
|
-
numeric(input.backoff.delay, "backoff.delay", 0,
|
|
7132
|
+
numeric(input.backoff.delay, "backoff.delay", 0, MAX_BACKOFF_DELAY);
|
|
7133
|
+
numeric(input.backoff.maxDelay, "backoff.maxDelay", 0, MAX_BACKOFF_DELAY);
|
|
7046
7134
|
} else {
|
|
7047
|
-
numeric(input.backoff, "backoff", 0,
|
|
7135
|
+
numeric(input.backoff, "backoff", 0, MAX_BACKOFF_DELAY);
|
|
7048
7136
|
}
|
|
7049
7137
|
for (const name of BOOLEAN_OPTIONS) {
|
|
7050
7138
|
const value = input[name];
|
|
@@ -13832,6 +13920,7 @@ class ContextFactory {
|
|
|
13832
13920
|
jobLocks: this.deps.jobLocks,
|
|
13833
13921
|
retiredCronLeaseTokens: this.deps.retiredCronLeaseTokens,
|
|
13834
13922
|
clientJobs: this.deps.clientJobs,
|
|
13923
|
+
clientJobOwners: this.deps.clientJobOwners,
|
|
13835
13924
|
processingShards: this.deps.processingShards,
|
|
13836
13925
|
processingLocks: this.deps.processingLocks,
|
|
13837
13926
|
shards: this.deps.shards,
|
|
@@ -13865,6 +13954,7 @@ class ContextFactory {
|
|
|
13865
13954
|
jobLocks: this.deps.jobLocks,
|
|
13866
13955
|
retiredCronLeaseTokens: this.deps.retiredCronLeaseTokens,
|
|
13867
13956
|
clientJobs: this.deps.clientJobs,
|
|
13957
|
+
clientJobOwners: this.deps.clientJobOwners,
|
|
13868
13958
|
stalledCandidates: this.deps.stalledCandidates,
|
|
13869
13959
|
pendingDepChecks: this.deps.pendingDepChecks,
|
|
13870
13960
|
pendingQueueAdmissions: this.deps.pendingQueueAdmissions,
|
|
@@ -13988,6 +14078,7 @@ class ContextFactory {
|
|
|
13988
14078
|
jobIndex: this.deps.jobIndex,
|
|
13989
14079
|
jobLocks: this.deps.jobLocks,
|
|
13990
14080
|
clientJobs: this.deps.clientJobs,
|
|
14081
|
+
clientJobOwners: this.deps.clientJobOwners,
|
|
13991
14082
|
dependencyResults: this.deps.dependencyResults,
|
|
13992
14083
|
customIdMap: this.deps.customIdMap,
|
|
13993
14084
|
depCompletions: this.deps.depCompletions,
|
|
@@ -14434,6 +14525,7 @@ function createContextDependencies(state) {
|
|
|
14434
14525
|
jobLogQueues: state.jobLogQueues,
|
|
14435
14526
|
jobLocks: state.jobLocks,
|
|
14436
14527
|
clientJobs: state.clientJobs,
|
|
14528
|
+
clientJobOwners: state.clientJobOwners,
|
|
14437
14529
|
stalledCandidates: state.stalledCandidates,
|
|
14438
14530
|
pendingDepChecks: state.pendingDepChecks,
|
|
14439
14531
|
pendingQueueAdmissions: state.pendingQueueAdmissions,
|
|
@@ -14502,6 +14594,7 @@ class QueueManagerState {
|
|
|
14502
14594
|
depFlushRunning = false;
|
|
14503
14595
|
jobLocks = new Map;
|
|
14504
14596
|
clientJobs = new Map;
|
|
14597
|
+
clientJobOwners = new Map;
|
|
14505
14598
|
repeatChain = new Map;
|
|
14506
14599
|
failedChildrenValues = new Map;
|
|
14507
14600
|
ignoredChildrenFailures = new Map;
|
|
@@ -14688,7 +14781,7 @@ class QueueManagerDelivery extends QueueManagerState {
|
|
|
14688
14781
|
if (location?.type !== "processing")
|
|
14689
14782
|
return;
|
|
14690
14783
|
const job = this.processingShards[location.shardIdx].get(jobId);
|
|
14691
|
-
if (!job || job
|
|
14784
|
+
if (!job || isLeaseFromEarlierGeneration(job, lock)) {
|
|
14692
14785
|
throw new Error(`Invalid or expired lock token for job ${jobId}`);
|
|
14693
14786
|
}
|
|
14694
14787
|
}
|
|
@@ -14700,7 +14793,7 @@ class QueueManagerDelivery extends QueueManagerState {
|
|
|
14700
14793
|
if (lock?.token !== token)
|
|
14701
14794
|
return false;
|
|
14702
14795
|
const job = this.processingShards[location.shardIdx].get(jobId);
|
|
14703
|
-
return !(job && job
|
|
14796
|
+
return !(job && isLeaseFromEarlierGeneration(job, lock));
|
|
14704
14797
|
}
|
|
14705
14798
|
isStallRetried(jobId) {
|
|
14706
14799
|
const location = this.jobIndex.get(jobId);
|
|
@@ -14942,11 +15035,7 @@ class QueueManagerAck extends QueueManagerDelivery {
|
|
|
14942
15035
|
if (lock)
|
|
14943
15036
|
this.retiredTimeoutLeaseTokens.set(lock.token, generation);
|
|
14944
15037
|
releaseLock(job.id, this.contextFactory.getLockContext(), lock?.token);
|
|
14945
|
-
|
|
14946
|
-
jobs.delete(job.id);
|
|
14947
|
-
if (jobs.size === 0)
|
|
14948
|
-
this.clientJobs.delete(clientId);
|
|
14949
|
-
}
|
|
15038
|
+
detachClientJob(job.id, { clientJobs: this.clientJobs, clientJobOwners: this.clientJobOwners });
|
|
14950
15039
|
}
|
|
14951
15040
|
async failWithOptions(jobId, error, token, options, enforceLease = true) {
|
|
14952
15041
|
try {
|
|
@@ -15228,11 +15317,13 @@ class QueueManagerControl extends QueueManagerQueries {
|
|
|
15228
15317
|
toDrop.add(jobId);
|
|
15229
15318
|
}
|
|
15230
15319
|
}
|
|
15320
|
+
const ownership = { clientJobs: this.clientJobs, clientJobOwners: this.clientJobOwners };
|
|
15231
15321
|
for (const jobId of toDrop) {
|
|
15232
15322
|
const location = this.jobIndex.get(jobId);
|
|
15233
15323
|
if (location?.type === "processing") {
|
|
15234
15324
|
this.processingShards[location.shardIdx]?.delete(jobId);
|
|
15235
15325
|
}
|
|
15326
|
+
detachClientJob(jobId, ownership);
|
|
15236
15327
|
this.timeoutScheduler.cancel(jobId);
|
|
15237
15328
|
this.jobIndex.delete(jobId);
|
|
15238
15329
|
this.completedJobs.delete(jobId);
|
|
@@ -16344,6 +16435,7 @@ class QueueManagerLifecycle extends QueueManagerDependencies {
|
|
|
16344
16435
|
this.jobLocks.clear();
|
|
16345
16436
|
this.stalledCandidates.clear();
|
|
16346
16437
|
this.clientJobs.clear();
|
|
16438
|
+
this.clientJobOwners.clear();
|
|
16347
16439
|
this.repeatChain.clear();
|
|
16348
16440
|
this.failedChildrenValues.clear();
|
|
16349
16441
|
this.ignoredChildrenFailures.clear();
|