bunqueue-client 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. package/CHANGELOG.md +15 -0
  2. package/dist/canonical-manifest.json +57 -53
  3. package/dist/embedded.js +1353 -1261
  4. package/dist/index.js +52 -4
  5. package/dist/legacy.js +1 -1
  6. package/dist/types/sdk/typescript/src/index.d.ts +1 -1
  7. package/dist/types/sdk/typescript/src/legacy.d.ts +1 -1
  8. package/dist/types/src/application/clientOwnership.d.ts +65 -0
  9. package/dist/types/src/application/clientTracking.d.ts +7 -11
  10. package/dist/types/src/application/operations/jobClaim.d.ts +2 -2
  11. package/dist/types/src/application/operations/jobManagement.d.ts +2 -0
  12. package/dist/types/src/application/orphanRecovery.d.ts +25 -0
  13. package/dist/types/src/application/queue-manager/state.d.ts +2 -0
  14. package/dist/types/src/application/stallDetection.d.ts +16 -0
  15. package/dist/types/src/application/types/contextFactory.d.ts +2 -0
  16. package/dist/types/src/application/types/contexts.d.ts +3 -0
  17. package/dist/types/src/application/types/queueManager.d.ts +2 -0
  18. package/dist/types/src/client/forwarder.d.ts +4 -3
  19. package/dist/types/src/client/types/options.d.ts +2 -0
  20. package/dist/types/src/domain/job/constants.d.ts +2 -0
  21. package/dist/types/src/domain/job/create.d.ts +7 -0
  22. package/dist/types/src/domain/job/locks.d.ts +13 -1
  23. package/dist/types/src/domain/job/state.d.ts +15 -0
  24. package/dist/types/src/domain/types/commands/core.d.ts +2 -5
  25. package/dist/types/src/domain/types/cron.d.ts +2 -4
  26. package/dist/types/src/domain/types/jobs/model.d.ts +1 -4
  27. package/package.json +1 -1
  28. package/src/index.ts +1 -1
  29. package/src/legacy.ts +1 -1
package/dist/embedded.js CHANGED
@@ -199,1069 +199,1409 @@ async function withWriteLock(lock, fn, timeoutMs) {
199
199
  }
200
200
  }
201
201
 
202
- // ../../src/application/dependencyCompletions.ts
203
- class DependencyCompletionTracker {
204
- onRecentEvict;
205
- recent = new Set;
206
- recentOrder = [];
207
- recentOrderTokens = [];
208
- recentTokens = new Map;
209
- recentOrderHead = 0;
210
- nextRecentToken = 0;
211
- pinned = new Set;
212
- owners = new Map;
213
- maxRecent;
214
- constructor(maxRecent, onRecentEvict) {
215
- this.onRecentEvict = onRecentEvict;
216
- this.maxRecent = Math.max(1, Math.trunc(maxRecent));
217
- }
218
- add(jobId, queue) {
219
- if (queue !== undefined)
220
- this.owners.set(jobId, queue);
221
- if (this.pinned.has(jobId) || this.recent.has(jobId))
222
- return;
223
- if (this.recent.size >= this.maxRecent)
224
- this.evictOldestRecent();
225
- if (this.nextRecentToken >= Number.MAX_SAFE_INTEGER)
226
- this.compactRecentOrder(true);
227
- const token = this.nextRecentToken++;
228
- this.recent.add(jobId);
229
- this.recentTokens.set(jobId, token);
230
- this.recentOrder.push(jobId);
231
- this.recentOrderTokens.push(token);
232
- }
233
- pin(jobId, queue) {
234
- if (queue !== undefined)
235
- this.owners.set(jobId, queue);
236
- if (this.recent.delete(jobId)) {
237
- this.recentTokens.delete(jobId);
238
- this.compactRecentOrder();
239
- }
240
- this.pinned.add(jobId);
241
- }
242
- unpin(jobId, retainAsRecent) {
243
- this.pinned.delete(jobId);
244
- if (retainAsRecent)
245
- this.add(jobId);
246
- else
247
- this.delete(jobId);
248
- }
249
- hydrate(records) {
250
- this.clear();
251
- for (const record of records) {
252
- if (record.pinned)
253
- this.pin(record.jobId, record.queue);
254
- else
255
- this.add(record.jobId, record.queue);
256
- }
257
- }
258
- pinnedValues() {
259
- return this.pinned.values();
260
- }
261
- has(jobId) {
262
- return this.pinned.has(jobId) || this.recent.has(jobId);
263
- }
264
- delete(jobId) {
265
- const wasPinned = this.pinned.delete(jobId);
266
- const wasRecent = this.recent.delete(jobId);
267
- if (wasRecent) {
268
- this.recentTokens.delete(jobId);
269
- this.compactRecentOrder();
270
- }
271
- this.owners.delete(jobId);
272
- return wasRecent || wasPinned;
273
- }
274
- deleteForQueue(queue) {
275
- const removed = [];
276
- for (const [jobId, owner] of this.owners) {
277
- if (owner !== queue)
278
- continue;
279
- if (this.delete(jobId))
280
- removed.push(jobId);
281
- }
282
- return removed;
283
- }
284
- clear() {
285
- this.recent.clear();
286
- this.recentOrder = [];
287
- this.recentOrderTokens = [];
288
- this.recentTokens.clear();
289
- this.recentOrderHead = 0;
290
- this.nextRecentToken = 0;
291
- this.pinned.clear();
292
- this.owners.clear();
293
- }
294
- get size() {
295
- return this.recent.size + this.pinned.size;
296
- }
297
- evictOldestRecent() {
298
- while (this.recentOrderHead < this.recentOrder.length) {
299
- const index = this.recentOrderHead++;
300
- const oldest = this.recentOrder[index];
301
- const token = this.recentOrderTokens[index];
302
- if (this.recentTokens.get(oldest) !== token)
303
- continue;
304
- this.recent.delete(oldest);
305
- this.recentTokens.delete(oldest);
306
- this.owners.delete(oldest);
307
- this.compactRecentOrder();
308
- this.onRecentEvict?.(oldest);
309
- return;
310
- }
202
+ // ../../src/application/clientOwnership.ts
203
+ function removeFromClient(clientId, jobId, ctx) {
204
+ const jobs = ctx.clientJobs.get(clientId);
205
+ if (!jobs)
206
+ return;
207
+ jobs.delete(jobId);
208
+ if (jobs.size === 0)
209
+ ctx.clientJobs.delete(clientId);
210
+ }
211
+ function registerClientJob(clientId, jobId, ctx) {
212
+ const previous = ctx.clientJobOwners.get(jobId);
213
+ if (previous && previous.clientId !== clientId) {
214
+ removeFromClient(previous.clientId, jobId, ctx);
311
215
  }
312
- compactRecentOrder(force = false) {
313
- const pending = this.recentOrder.length - this.recentOrderHead;
314
- if (!force && this.recentOrderHead < this.maxRecent && pending < this.maxRecent * 2)
315
- return;
316
- const order = [];
317
- const tokens = [];
318
- for (let index = this.recentOrderHead;index < this.recentOrder.length; index++) {
319
- const jobId = this.recentOrder[index];
320
- const token = this.recentOrderTokens[index];
321
- if (this.recentTokens.get(jobId) !== token)
322
- continue;
323
- const compactedToken = order.length;
324
- order.push(jobId);
325
- tokens.push(compactedToken);
326
- }
327
- this.recentTokens.clear();
328
- for (let index = 0;index < order.length; index++) {
329
- this.recentTokens.set(order[index], tokens[index]);
330
- }
331
- this.recentOrder = order;
332
- this.recentOrderTokens = tokens;
333
- this.recentOrderHead = 0;
334
- this.nextRecentToken = order.length;
216
+ ctx.clientJobOwners.set(jobId, { clientId, delivery: ctx.jobIndex.get(jobId) });
217
+ let jobs = ctx.clientJobs.get(clientId);
218
+ if (!jobs) {
219
+ jobs = new Set;
220
+ ctx.clientJobs.set(clientId, jobs);
335
221
  }
222
+ jobs.add(jobId);
336
223
  }
337
- function hasDependencyWaiters(shards, jobId) {
338
- return shards.some((shard) => (shard.getJobsWaitingFor(jobId)?.size ?? 0) > 0);
339
- }
340
- function commitRemovedCompletion(job, ctx, completedAt = Date.now()) {
341
- const pinned = hasDependencyWaiters(ctx.shards, job.id);
342
- ctx.storage?.commitRemovedCompletion(job, ctx.maxDependencyCompletions, pinned, completedAt);
343
- if (pinned)
344
- ctx.depCompletions?.pin(job.id, job.queue);
345
- else
346
- ctx.depCompletions?.add(job.id, job.queue);
224
+ function unregisterClientJob(clientId, jobId, ctx) {
225
+ if (!clientId)
226
+ return;
227
+ removeFromClient(clientId, jobId, ctx);
228
+ if (ctx.clientJobOwners.get(jobId)?.clientId === clientId)
229
+ ctx.clientJobOwners.delete(jobId);
347
230
  }
348
- function releaseDependencyCompletionPins(dependencyIds, ctx) {
349
- const candidates = [...new Set(dependencyIds)].filter((id) => !hasDependencyWaiters(ctx.shards, id));
350
- if (candidates.length === 0)
231
+ function detachClientJob(jobId, ctx) {
232
+ const owner = ctx.clientJobOwners.get(jobId);
233
+ if (!owner)
351
234
  return;
352
- if (ctx.storage) {
353
- const records = ctx.storage.unpinDependencyCompletions(candidates, ctx.maxDependencyCompletions);
354
- ctx.depCompletions?.hydrate(records);
235
+ ctx.clientJobOwners.delete(jobId);
236
+ removeFromClient(owner.clientId, jobId, ctx);
237
+ }
238
+ function ownsCurrentDelivery(clientId, jobId, ctx) {
239
+ const owner = ctx.clientJobOwners.get(jobId);
240
+ if (owner?.clientId !== clientId)
241
+ return false;
242
+ const location = ctx.jobIndex.get(jobId);
243
+ return location?.type === "processing" && location === owner.delivery;
244
+ }
245
+ function dropClient(clientId, ctx) {
246
+ const jobs = ctx.clientJobs.get(clientId);
247
+ ctx.clientJobs.delete(clientId);
248
+ if (!jobs)
355
249
  return;
250
+ for (const jobId of jobs) {
251
+ if (ctx.clientJobOwners.get(jobId)?.clientId === clientId)
252
+ ctx.clientJobOwners.delete(jobId);
356
253
  }
357
- for (const id of candidates)
358
- ctx.depCompletions?.unpin(id, true);
359
254
  }
360
- function reconcileDependencyCompletionPins(ctx) {
361
- const referenced = new Set;
362
- for (const shard of ctx.shards) {
363
- for (const dependencyId of shard.dependencyIndex.keys())
364
- referenced.add(dependencyId);
365
- }
366
- if (ctx.storage) {
367
- const records = ctx.storage.reconcileDependencyCompletionPins(referenced, ctx.maxDependencyCompletions);
368
- ctx.depCompletions?.hydrate(records);
369
- return;
255
+ function pruneEndedClientDeliveries(ctx) {
256
+ let pruned = 0;
257
+ for (const [jobId, owner] of ctx.clientJobOwners) {
258
+ const location = ctx.jobIndex.get(jobId);
259
+ if (location?.type === "processing" && location === owner.delivery)
260
+ continue;
261
+ ctx.clientJobOwners.delete(jobId);
262
+ removeFromClient(owner.clientId, jobId, ctx);
263
+ pruned++;
370
264
  }
371
- for (const id of ctx.depCompletions?.pinnedValues() ?? []) {
372
- if (!referenced.has(id))
373
- ctx.depCompletions?.unpin(id, true);
265
+ return pruned;
266
+ }
267
+ // ../../src/domain/job/constants.ts
268
+ var DEFAULT_MAX_BACKOFF = 3600000;
269
+ var MAX_TIMELINE_ENTRIES = 20;
270
+ var DEFAULT_LOCK_TTL = 30000;
271
+ var JOB_DEFAULTS = {
272
+ priority: 0,
273
+ maxAttempts: 3,
274
+ backoff: 1000,
275
+ lifo: false,
276
+ removeOnComplete: false,
277
+ removeOnFail: false,
278
+ stackTraceLimit: 10
279
+ };
280
+ var MAX_BACKOFF_DELAY = 86400000;
281
+ // ../../src/domain/job/payload.ts
282
+ var DEFAULT_JOB_NAME = "default";
283
+ var MAX_JOB_NAME_LENGTH = 256;
284
+ function legacyEnvelope(data) {
285
+ if (typeof data !== "object" || data === null || Array.isArray(data))
286
+ return null;
287
+ const record = data;
288
+ if (typeof record.name !== "string")
289
+ return null;
290
+ const { name, ...userData } = record;
291
+ return { name, data: userData };
292
+ }
293
+ function validateJobName(name) {
294
+ if (typeof name !== "string" || name.length === 0 || name.length > MAX_JOB_NAME_LENGTH) {
295
+ return "Job name must be a non-empty string of at most 256 characters";
374
296
  }
297
+ return null;
375
298
  }
376
-
377
- // ../../src/shared/serialization.ts
378
- function compareSqliteBinaryText(left, right) {
379
- if (left === right)
380
- return 0;
381
- let leftIndex = 0;
382
- let rightIndex = 0;
383
- while (leftIndex < left.length && rightIndex < right.length) {
384
- const leftPoint = left.codePointAt(leftIndex);
385
- const rightPoint = right.codePointAt(rightIndex);
386
- if (leftPoint === undefined || rightPoint === undefined)
387
- break;
388
- if (leftPoint !== rightPoint)
389
- return leftPoint - rightPoint;
390
- leftIndex += leftPoint > 65535 ? 2 : 1;
391
- rightIndex += rightPoint > 65535 ? 2 : 1;
299
+ function checkedName(name) {
300
+ const error = validateJobName(name);
301
+ if (error)
302
+ throw new Error(error);
303
+ return name;
304
+ }
305
+ function normalizeJobPayload(source, fallback = DEFAULT_JOB_NAME) {
306
+ if (source.name !== undefined) {
307
+ return { name: checkedName(source.name), data: source.data, legacy: false };
392
308
  }
393
- return left.length - leftIndex - (right.length - rightIndex);
309
+ return { name: checkedName(fallback), data: source.data, legacy: false };
310
+ }
311
+ function normalizeLegacyJobPayload(source, fallback = DEFAULT_JOB_NAME) {
312
+ if (source.name !== undefined)
313
+ return normalizeJobPayload(source, fallback);
314
+ const legacy = legacyEnvelope(source.data);
315
+ if (legacy)
316
+ return { name: checkedName(legacy.name), data: legacy.data, legacy: true };
317
+ return { name: checkedName(fallback), data: source.data, legacy: false };
394
318
  }
395
319
 
396
- // ../../src/application/completedCleanup.ts
397
- function memoryCandidates(queue, completedBefore, limit, ctx) {
398
- return [...ctx.jobIndex].flatMap(([jobId, location]) => {
399
- if (location.type !== "completed")
400
- return [];
401
- if (queue !== null && location.queueName !== queue)
402
- return [];
403
- if (ctx.dependencyResults.hasConsumers(jobId))
404
- return [];
405
- const job = ctx.completedJobsData.get(jobId);
406
- const timestamp = job?.completedAt ?? job?.createdAt ?? 0;
407
- return timestamp <= completedBefore ? [{ jobId, timestamp }] : [];
408
- }).sort((left, right) => left.timestamp - right.timestamp || compareSqliteBinaryText(left.jobId, right.jobId)).slice(0, limit).map(({ jobId }) => ({ jobId, queue: ctx.jobIndex.get(jobId)?.queueName ?? "" }));
320
+ // ../../src/domain/job/create.ts
321
+ function parseMaxDelay(value) {
322
+ if (typeof value !== "number" || !Number.isFinite(value))
323
+ return;
324
+ return value >= 0 && value <= MAX_BACKOFF_DELAY ? value : undefined;
409
325
  }
410
- function cleanCompletedJobs(queue, completedBefore, limit, ctx) {
411
- const normalizedLimit = Number.isFinite(limit) ? Math.max(0, Math.trunc(limit)) : 0;
412
- if (normalizedLimit === 0 || !Number.isFinite(completedBefore))
413
- return [];
414
- const removals = ctx.storage ? ctx.storage.cleanCompletedJobs(queue, completedBefore, normalizedLimit, (jobId) => ctx.dependencyResults.hasConsumers(jobId)) : memoryCandidates(queue, completedBefore, normalizedLimit, ctx);
415
- for (const { jobId, queue: ownerQueue } of removals) {
416
- const location = ctx.jobIndex.get(jobId);
417
- const ownsCurrentCompletion = location?.type === "completed" && location.queueName === ownerQueue;
418
- const hasNoCurrentGeneration = location === undefined;
419
- const resultOwner = ctx.jobResultQueues?.get(jobId);
420
- if (resultOwner === ownerQueue || resultOwner === undefined && ownsCurrentCompletion) {
421
- ctx.jobResults?.delete(jobId);
422
- ctx.jobResultQueues?.delete(jobId);
423
- }
424
- const logOwner = ctx.jobLogQueues?.get(jobId);
425
- if (logOwner === ownerQueue || logOwner === undefined && ownsCurrentCompletion) {
426
- ctx.jobLogs?.delete(jobId);
427
- ctx.jobLogQueues?.delete(jobId);
428
- }
429
- if (!hasNoCurrentGeneration && !ownsCurrentCompletion)
430
- continue;
431
- ctx.completedJobs.delete(jobId);
432
- ctx.completedJobsData.delete(jobId);
433
- ctx.jobIndex.delete(jobId);
434
- ctx.dependencyResults.releaseConsumer(jobId);
326
+ function parseBackoff(input) {
327
+ if (typeof input === "object" && input !== null) {
328
+ const maxDelay = parseMaxDelay(input.maxDelay);
329
+ return {
330
+ backoff: input.delay,
331
+ backoffConfig: maxDelay === undefined ? { type: input.type, delay: input.delay } : { type: input.type, delay: input.delay, maxDelay }
332
+ };
435
333
  }
436
- return removals.map(({ jobId }) => jobId);
334
+ return { backoff: input ?? JOB_DEFAULTS.backoff, backoffConfig: null };
437
335
  }
438
-
439
- // ../../src/application/emptyQueueCleanup.ts
440
- function collectOccupiedQueues(ctx) {
441
- const occupied = new Set;
442
- for (const [jobId, location] of ctx.jobIndex) {
443
- if (location.type !== "processing")
444
- continue;
445
- occupied.add(location.queueName);
446
- const processingJob = ctx.processingShards[location.shardIdx]?.get(jobId);
447
- if (processingJob)
448
- occupied.add(processingJob.queue);
449
- }
450
- for (const processing of ctx.processingShards) {
451
- for (const job of processing.values())
452
- occupied.add(job.queue);
453
- }
454
- for (const shard of ctx.shards) {
455
- for (const job of shard.waitingDeps.values())
456
- occupied.add(job.queue);
457
- for (const job of shard.waitingChildren.values())
458
- occupied.add(job.queue);
459
- }
460
- for (const [queue, count] of ctx.pendingQueueAdmissions) {
461
- if (count > 0)
462
- occupied.add(queue);
463
- }
464
- return occupied;
336
+ function parseRepeatConfig(repeat) {
337
+ if (!repeat)
338
+ return null;
339
+ return {
340
+ every: repeat.every,
341
+ limit: repeat.limit,
342
+ pattern: repeat.pattern,
343
+ count: repeat.count ?? 0,
344
+ startDate: repeat.startDate,
345
+ endDate: repeat.endDate,
346
+ tz: repeat.tz,
347
+ immediately: repeat.immediately,
348
+ prevMillis: repeat.prevMillis,
349
+ offset: repeat.offset,
350
+ jobId: repeat.jobId
351
+ };
465
352
  }
466
- function collectConfiguredQueues(ctx) {
467
- const configured = new Set;
468
- for (const shard of ctx.shards) {
469
- for (const queue of shard.getConfiguredQueueNames())
470
- configured.add(queue);
471
- }
472
- return configured;
353
+ function toBoolean(value, fallback) {
354
+ return value === undefined ? fallback : Boolean(value);
473
355
  }
474
- function collectBufferedCompletedQueues(ctx) {
475
- const queues = new Set;
476
- for (const job of ctx.storage?.getBufferedJobs() ?? []) {
477
- if (job.completedAt === null)
478
- continue;
479
- const location = ctx.jobIndex.get(job.id);
480
- if (location && (location.type !== "completed" || location.queueName !== job.queue))
481
- continue;
482
- queues.add(job.queue);
483
- }
484
- return queues;
356
+ function parseCoreOptions(input) {
357
+ return {
358
+ priority: input.priority ?? JOB_DEFAULTS.priority,
359
+ lifo: input.lifo ?? JOB_DEFAULTS.lifo,
360
+ maxAttempts: input.maxAttempts ?? JOB_DEFAULTS.maxAttempts,
361
+ removeOnComplete: toBoolean(input.removeOnComplete, JOB_DEFAULTS.removeOnComplete),
362
+ removeOnFail: toBoolean(input.removeOnFail, JOB_DEFAULTS.removeOnFail)
363
+ };
485
364
  }
486
- function collectCompletedCounts(ctx, registered) {
487
- if (ctx.storage)
488
- return ctx.storage.countCompletedJobsByQueue(registered);
489
- const registeredSet = new Set(registered);
490
- const counts = new Map;
491
- for (const [jobId, location] of ctx.jobIndex) {
492
- if (location.type !== "completed" || !registeredSet.has(location.queueName) || !ctx.completedJobs.has(jobId)) {
493
- continue;
494
- }
495
- counts.set(location.queueName, (counts.get(location.queueName) ?? 0) + 1);
496
- }
497
- return counts;
365
+ function parseOptionalFields(input) {
366
+ return {
367
+ ttl: input.ttl ?? null,
368
+ timeout: input.timeout ?? null,
369
+ uniqueKey: input.uniqueKey ?? null,
370
+ customId: input.customId ?? null,
371
+ parentId: input.parentId ?? null,
372
+ groupId: input.groupId ?? null,
373
+ stallTimeout: input.stallTimeout ?? null
374
+ };
498
375
  }
499
- function cleanEmptyQueues(ctx) {
500
- const registered = Array.from(ctx.queueNamesCache);
501
- const completedCounts = collectCompletedCounts(ctx, registered);
502
- const bufferedCompletedQueues = collectBufferedCompletedQueues(ctx);
503
- const occupied = collectOccupiedQueues(ctx);
504
- const configured = collectConfiguredQueues(ctx);
505
- const removed = [];
506
- for (const queueName of registered) {
507
- const shard = ctx.shards[shardIndex(queueName)];
508
- const hasLiveRuntime = (shard.queues.get(queueName)?.size ?? 0) > 0 || shard.getDlqCount(queueName) > 0 || occupied.has(queueName);
509
- if (hasLiveRuntime)
510
- continue;
511
- shard.queues.delete(queueName);
512
- shard.clearEmptyGroupRuntime(queueName);
513
- if ((completedCounts.get(queueName) ?? 0) > 0 || bufferedCompletedQueues.has(queueName) || configured.has(queueName)) {
514
- continue;
515
- }
516
- ctx.storage?.deleteQueueState(queueName);
517
- ctx.storage?.deleteGroupState(queueName);
518
- ctx.unregisterQueueName(queueName);
519
- shard.obliterate(queueName);
520
- removed.push(queueName);
521
- }
522
- for (const shard of ctx.shards)
523
- shard.cleanOrphanedTemporalEntries();
524
- for (const queue of removed)
525
- ctx.dashboardEmit?.("queue:removed", { queue });
376
+ function parseBullMQV5Options(input) {
377
+ return {
378
+ stackTraceLimit: input.stackTraceLimit ?? JOB_DEFAULTS.stackTraceLimit,
379
+ keepLogs: input.keepLogs ?? null,
380
+ sizeLimit: input.sizeLimit ?? null,
381
+ failParentOnFailure: input.failParentOnFailure ?? false,
382
+ removeDependencyOnFailure: input.removeDependencyOnFailure ?? false,
383
+ continueParentOnFailure: input.continueParentOnFailure ?? false,
384
+ ignoreDependencyOnFailure: input.ignoreDependencyOnFailure ?? false,
385
+ deduplicationTtl: input.dedup?.ttl ?? null,
386
+ deduplicationExtend: input.dedup?.extend ?? false,
387
+ deduplicationReplace: input.dedup?.replace ?? false,
388
+ debounceId: input.debounceId ?? null,
389
+ debounceTtl: input.debounceTtl ?? null
390
+ };
526
391
  }
527
-
528
- // ../../src/application/cleanupTasks.ts
529
- async function cleanup(ctx) {
530
- const now = Date.now();
531
- const stallTimeout = 30 * 60 * 1000;
532
- const retention = ctx.config.completedRetentionMs;
533
- if (retention !== null && retention !== undefined) {
534
- const removed = cleanCompletedJobs(null, now - retention, 1000, ctx);
535
- if (removed.length > 0) {
536
- ctx.dashboardEmit?.("cleanup:completed-removed", { count: removed.length });
537
- }
538
- }
539
- for (let i = 0;i < SHARD_COUNT; i++) {
540
- ctx.shards[i].refreshDelayedCount(now);
541
- }
542
- for (let i = 0;i < SHARD_COUNT; i++) {
543
- for (const q of ctx.shards[i].queues.values()) {
544
- if (q.needsCompaction(0.2)) {
545
- q.compact();
546
- }
547
- }
548
- }
549
- await cleanOrphanedProcessingEntries(ctx, now, stallTimeout);
550
- await cleanStaleWaitingDependencies(ctx, now);
551
- cleanUniqueKeysAndGroups(ctx, now);
552
- cleanStalledCandidates(ctx);
553
- await cleanOrphanedJobIndex(ctx);
554
- cleanOrphanedJobLocks(ctx);
555
- cleanEmptyQueues(ctx);
392
+ function createJob(id, queue, input, now = Date.now()) {
393
+ const { backoff, backoffConfig } = parseBackoff(input.backoff);
394
+ const coreOpts = parseCoreOptions(input);
395
+ const optionalFields = parseOptionalFields(input);
396
+ const v5Opts = parseBullMQV5Options(input);
397
+ const createdAt = input.timestamp ?? now;
398
+ const payload = normalizeJobPayload(input);
399
+ return {
400
+ id,
401
+ queue,
402
+ name: payload.name,
403
+ data: payload.data,
404
+ createdAt,
405
+ runAt: createdAt + (input.delay ?? 0),
406
+ startedAt: null,
407
+ completedAt: null,
408
+ attempts: 0,
409
+ backoff,
410
+ backoffConfig,
411
+ dependsOn: input.dependsOn ?? [],
412
+ childrenIds: input.childrenIds ?? [],
413
+ childrenCompleted: 0,
414
+ tags: input.tags ?? [],
415
+ progress: 0,
416
+ progressMessage: null,
417
+ stacktrace: null,
418
+ repeat: parseRepeatConfig(input.repeat),
419
+ durable: input.durable ?? false,
420
+ lastHeartbeat: createdAt,
421
+ stallCount: 0,
422
+ ...coreOpts,
423
+ ...optionalFields,
424
+ ...v5Opts,
425
+ timeline: []
426
+ };
556
427
  }
557
- async function cleanOrphanedProcessingEntries(ctx, now, stallTimeout) {
558
- for (let i = 0;i < SHARD_COUNT; i++) {
559
- const orphaned = [];
560
- for (const [jobId, job] of ctx.processingShards[i]) {
561
- if (job.startedAt && now - job.startedAt > stallTimeout) {
562
- orphaned.push(jobId);
563
- }
428
+ // ../../src/domain/job/ids.ts
429
+ function isWellFormedJobId(id) {
430
+ for (let index = 0;index < id.length; index++) {
431
+ const unit = id.charCodeAt(index);
432
+ if (unit >= 55296 && unit <= 56319) {
433
+ const next = id.charCodeAt(++index);
434
+ if (!(next >= 56320 && next <= 57343))
435
+ return false;
436
+ } else if (unit >= 56320 && unit <= 57343) {
437
+ return false;
564
438
  }
565
- if (orphaned.length === 0)
566
- continue;
567
- await withWriteLock(ctx.processingLocks[i], () => {
568
- let removed = 0;
569
- for (const jobId of orphaned) {
570
- const job = ctx.processingShards[i].get(jobId);
571
- if (job) {
572
- ctx.processingShards[i].delete(jobId);
573
- ctx.timeoutScheduler.cancel(jobId);
574
- ctx.jobIndex.delete(jobId);
575
- removed++;
576
- }
577
- }
578
- if (removed > 0)
579
- ctx.dashboardEmit?.("cleanup:orphans-removed", { count: removed });
580
- });
581
439
  }
440
+ return true;
582
441
  }
583
- async function cleanStaleWaitingDependencies(ctx, now) {
584
- const depTimeout = 60 * 60 * 1000;
585
- for (let i = 0;i < SHARD_COUNT; i++) {
586
- const shard = ctx.shards[i];
587
- const staleIds = [];
588
- for (const [_id, job] of shard.waitingDeps) {
589
- if (now - job.createdAt > depTimeout) {
590
- staleIds.push(job.id);
591
- }
592
- }
593
- if (staleIds.length === 0)
594
- continue;
595
- const removed = await withWriteLock(ctx.shardLocks[i], () => {
596
- let count = 0;
597
- const released = [];
598
- for (const id of staleIds) {
599
- const job = shard.waitingDeps.get(id);
600
- if (!job || now - job.createdAt <= depTimeout)
601
- continue;
602
- ctx.storage?.deleteJob(job.id);
603
- shard.waitingDeps.delete(job.id);
604
- shard.unregisterDependencies(job.id, job.dependsOn);
605
- released.push(...job.dependsOn);
606
- if (job.uniqueKey && shard.getUniqueKeyEntry(job.queue, job.uniqueKey)?.jobId === job.id) {
607
- shard.releaseUniqueKeyIfOwned(job.queue, job.uniqueKey, job.id);
608
- }
609
- if (job.customId && ctx.customIdMap.get(job.customId) === job.id) {
610
- ctx.customIdMap.delete(job.customId);
611
- }
612
- ctx.dependencyResults?.releaseConsumer(job.id);
613
- ctx.jobIndex.delete(job.id);
614
- count++;
615
- }
616
- return { count, released };
617
- });
618
- releaseDependencyCompletionPins(removed.released, ctx);
619
- if (removed.count > 0) {
620
- ctx.dashboardEmit?.("cleanup:stale-deps-removed", { count: removed.count });
621
- }
622
- }
442
+ function assertWellFormedJobId(id, name = "job ID") {
443
+ if (!isWellFormedJobId(id))
444
+ throw new Error(`${name} must be well-formed Unicode`);
623
445
  }
624
- function cleanUniqueKeysAndGroups(ctx, now) {
625
- for (let i = 0;i < SHARD_COUNT; i++) {
626
- const shard = ctx.shards[i];
627
- shard.cleanExpiredGroupWindows(now);
628
- shard.cleanExpiredUniqueKeys();
629
- for (const [_queueName, keys] of shard.uniqueKeys) {
630
- if (keys.size > 1000) {
631
- const toRemove = Math.floor(keys.size / 2);
632
- const iter = keys.keys();
633
- for (let j = 0;j < toRemove; j++) {
634
- const { value, done } = iter.next();
635
- if (done)
636
- break;
637
- keys.delete(value);
638
- }
639
- }
640
- }
641
- }
446
+ function jobId(id) {
447
+ return id;
642
448
  }
643
- function cleanStalledCandidates(ctx) {
644
- for (const jobId of ctx.stalledCandidates) {
645
- const loc = ctx.jobIndex.get(jobId);
646
- if (loc?.type !== "processing") {
647
- ctx.stalledCandidates.delete(jobId);
648
- }
649
- }
449
+ function generateJobId() {
450
+ return Bun.randomUUIDv7();
650
451
  }
651
- async function cleanOrphanedJobIndex(ctx) {
652
- if (ctx.jobIndex.size <= 1e5)
653
- return;
654
- const processingCandidates = new Map;
655
- const queueCandidates = new Map;
656
- for (const [jobId, loc] of ctx.jobIndex) {
657
- if (loc.type === "processing") {
658
- const procIdx = processingShardIndex(jobId);
659
- let list = processingCandidates.get(procIdx);
660
- if (!list) {
661
- list = [];
662
- processingCandidates.set(procIdx, list);
663
- }
664
- list.push(jobId);
665
- } else if (loc.type === "queue") {
666
- let list = queueCandidates.get(loc.shardIdx);
667
- if (!list) {
668
- list = [];
669
- queueCandidates.set(loc.shardIdx, list);
670
- }
671
- list.push({ jobId, queueName: loc.queueName });
672
- }
673
- }
674
- for (const [procIdx, candidates] of processingCandidates) {
675
- await withWriteLock(ctx.processingLocks[procIdx], () => {
676
- for (const jobId of candidates) {
677
- if (!ctx.processingShards[procIdx].has(jobId)) {
678
- ctx.jobIndex.delete(jobId);
679
- }
680
- }
681
- });
682
- }
683
- for (const [shardIdx, candidates] of queueCandidates) {
684
- await withWriteLock(ctx.shardLocks[shardIdx], () => {
685
- const shard = ctx.shards[shardIdx];
686
- for (const { jobId, queueName } of candidates) {
687
- if (!shard.getQueue(queueName).has(jobId)) {
688
- ctx.jobIndex.delete(jobId);
689
- }
690
- }
691
- });
692
- }
452
+ function generateLockToken() {
453
+ return Bun.randomUUIDv7();
693
454
  }
694
- function cleanOrphanedJobLocks(ctx) {
695
- for (const jobId of ctx.jobLocks.keys()) {
696
- const loc = ctx.jobIndex.get(jobId);
697
- if (loc?.type !== "processing") {
698
- ctx.jobLocks.delete(jobId);
699
- }
700
- }
455
+ // ../../src/domain/job/locks.ts
456
+ function createJobLock(jobId, owner, ttl = DEFAULT_LOCK_TTL, now = Date.now()) {
457
+ return {
458
+ jobId,
459
+ token: generateLockToken(),
460
+ owner,
461
+ createdAt: now,
462
+ expiresAt: now + ttl,
463
+ lastRenewalAt: now,
464
+ renewalCount: 0,
465
+ ttl
466
+ };
701
467
  }
702
- // ../../src/domain/job/constants.ts
703
- var DEFAULT_MAX_BACKOFF = 3600000;
704
- var MAX_TIMELINE_ENTRIES = 20;
705
- var DEFAULT_LOCK_TTL = 30000;
706
- var JOB_DEFAULTS = {
707
- priority: 0,
708
- maxAttempts: 3,
709
- backoff: 1000,
710
- lifo: false,
711
- removeOnComplete: false,
712
- removeOnFail: false,
713
- stackTraceLimit: 10
714
- };
715
- // ../../src/domain/job/payload.ts
716
- var DEFAULT_JOB_NAME = "default";
717
- var MAX_JOB_NAME_LENGTH = 256;
718
- function legacyEnvelope(data) {
719
- if (typeof data !== "object" || data === null || Array.isArray(data))
720
- return null;
721
- const record = data;
722
- if (typeof record.name !== "string")
723
- return null;
724
- const { name, ...userData } = record;
725
- return { name, data: userData };
468
+ function isLockExpired(lock, now = Date.now()) {
469
+ return now >= lock.expiresAt;
726
470
  }
727
- function validateJobName(name) {
728
- if (typeof name !== "string" || name.length === 0 || name.length > MAX_JOB_NAME_LENGTH) {
729
- return "Job name must be a non-empty string of at most 256 characters";
471
+ function renewLock(lock, newTtl, now = Date.now()) {
472
+ const ttl = newTtl ?? lock.ttl;
473
+ lock.expiresAt = now + ttl;
474
+ lock.lastRenewalAt = now;
475
+ lock.renewalCount++;
476
+ }
477
+ function isLeaseFromEarlierGeneration(job, lock) {
478
+ return job.startedAt !== null && job.startedAt !== undefined && job.startedAt > lock.createdAt;
479
+ }
480
+ // ../../src/domain/job/state.ts
481
+ function normalizeStacktrace(lines, limit) {
482
+ if (!lines || lines.length === 0)
483
+ return null;
484
+ const out = [];
485
+ for (const line of lines) {
486
+ if (out.length >= limit)
487
+ break;
488
+ if (typeof line !== "string")
489
+ continue;
490
+ const trimmed = line.trim();
491
+ if (trimmed)
492
+ out.push(trimmed);
730
493
  }
731
- return null;
494
+ return out.length > 0 ? out : null;
732
495
  }
733
- function checkedName(name) {
734
- const error = validateJobName(name);
735
- if (error)
736
- throw new Error(error);
737
- return name;
496
+ function isReady(job, now = Date.now()) {
497
+ return job.runAt <= now;
738
498
  }
739
- function normalizeJobPayload(source, fallback = DEFAULT_JOB_NAME) {
740
- if (source.name !== undefined) {
741
- return { name: checkedName(source.name), data: source.data, legacy: false };
499
+ function isExpired(job, now = Date.now()) {
500
+ if (job.ttl === null)
501
+ return false;
502
+ return now > job.createdAt + job.ttl;
503
+ }
504
+ function calculateBackoff(job) {
505
+ const maxDelay = job.backoffConfig?.maxDelay ?? DEFAULT_MAX_BACKOFF;
506
+ if (job.backoffConfig) {
507
+ if (job.backoffConfig.type === "fixed") {
508
+ const base = job.backoffConfig.delay;
509
+ const jittered = base * (0.8 + Math.random() * 0.4);
510
+ return Math.min(jittered, maxDelay);
511
+ }
512
+ const base = job.backoffConfig.delay * Math.pow(2, job.attempts);
513
+ const jittered = base * (0.5 + Math.random());
514
+ return Math.min(jittered, maxDelay);
742
515
  }
743
- return { name: checkedName(fallback), data: source.data, legacy: false };
516
+ const base = job.backoff * Math.pow(2, job.attempts);
517
+ const jittered = base * (0.5 + Math.random());
518
+ return Math.min(jittered, maxDelay);
744
519
  }
745
- function normalizeLegacyJobPayload(source, fallback = DEFAULT_JOB_NAME) {
746
- if (source.name !== undefined)
747
- return normalizeJobPayload(source, fallback);
748
- const legacy = legacyEnvelope(source.data);
749
- if (legacy)
750
- return { name: checkedName(legacy.name), data: legacy.data, legacy: true };
751
- return { name: checkedName(fallback), data: source.data, legacy: false };
520
+ function canRetry(job) {
521
+ return job.attempts < job.maxAttempts;
752
522
  }
753
-
754
- // ../../src/domain/job/create.ts
755
- function parseBackoff(input) {
756
- if (typeof input === "object") {
523
+ // ../../src/domain/types/stall.ts
524
+ var DEFAULT_STALL_CONFIG = {
525
+ enabled: true,
526
+ stallInterval: 30000,
527
+ maxStalls: 3,
528
+ gracePeriod: 5000
529
+ };
530
+ function checkStall(job, config = DEFAULT_STALL_CONFIG, now = Date.now()) {
531
+ if (job.startedAt === null) {
757
532
  return {
758
- backoff: input.delay,
759
- backoffConfig: { type: input.type, delay: input.delay }
533
+ isStalled: false,
534
+ stalledFor: 0,
535
+ shouldMoveToDlq: false,
536
+ newStallCount: job.stallCount
760
537
  };
761
538
  }
762
- return { backoff: input ?? JOB_DEFAULTS.backoff, backoffConfig: null };
763
- }
764
- function parseRepeatConfig(repeat) {
765
- if (!repeat)
766
- return null;
767
- return {
768
- every: repeat.every,
769
- limit: repeat.limit,
770
- pattern: repeat.pattern,
771
- count: repeat.count ?? 0,
772
- startDate: repeat.startDate,
773
- endDate: repeat.endDate,
774
- tz: repeat.tz,
775
- immediately: repeat.immediately,
776
- prevMillis: repeat.prevMillis,
777
- offset: repeat.offset,
778
- jobId: repeat.jobId
539
+ if (now - job.startedAt < config.gracePeriod) {
540
+ return {
541
+ isStalled: false,
542
+ stalledFor: 0,
543
+ shouldMoveToDlq: false,
544
+ newStallCount: job.stallCount
545
+ };
546
+ }
547
+ const stallInterval = job.stallTimeout ?? config.stallInterval;
548
+ const stalledFor = now - job.lastHeartbeat;
549
+ const isStalled = stalledFor > stallInterval;
550
+ if (!isStalled) {
551
+ return {
552
+ isStalled: false,
553
+ stalledFor,
554
+ shouldMoveToDlq: false,
555
+ newStallCount: job.stallCount
556
+ };
557
+ }
558
+ const newStallCount = job.stallCount + 1;
559
+ const shouldMoveToDlq = newStallCount >= config.maxStalls;
560
+ return {
561
+ isStalled: true,
562
+ stalledFor,
563
+ shouldMoveToDlq,
564
+ newStallCount
779
565
  };
780
566
  }
781
- function toBoolean(value, fallback) {
782
- return value === undefined ? fallback : Boolean(value);
567
+ function getStallAction(job, config = DEFAULT_STALL_CONFIG, now = Date.now()) {
568
+ const result = checkStall(job, config, now);
569
+ if (!result.isStalled) {
570
+ return "keep" /* Keep */;
571
+ }
572
+ if (result.shouldMoveToDlq) {
573
+ return "move_to_dlq" /* MoveToDlq */;
574
+ }
575
+ return "retry" /* Retry */;
783
576
  }
784
- function parseCoreOptions(input) {
577
+ function incrementStallCount(job) {
578
+ job.stallCount++;
579
+ return job.stallCount;
580
+ }
581
+
582
+ // ../../src/shared/logger.ts
583
+ var LOG_LEVEL_PRIORITY = {
584
+ debug: 0,
585
+ info: 1,
586
+ warn: 2,
587
+ error: 3
588
+ };
589
+
590
+ class Logger {
591
+ component;
592
+ static jsonMode = false;
593
+ static level = "info";
594
+ constructor(component) {
595
+ this.component = component;
596
+ }
597
+ static enableJsonMode() {
598
+ Logger.jsonMode = true;
599
+ }
600
+ static disableJsonMode() {
601
+ Logger.jsonMode = false;
602
+ }
603
+ static setLevel(level) {
604
+ Logger.level = level;
605
+ }
606
+ debug(message, data) {
607
+ this.log("debug", message, data);
608
+ }
609
+ debugToStderr(message, data) {
610
+ this.log("debug", message, data, true);
611
+ }
612
+ info(message, data) {
613
+ this.log("info", message, data);
614
+ }
615
+ infoToStderr(message, data) {
616
+ this.log("info", message, data, true);
617
+ }
618
+ warn(message, data) {
619
+ this.log("warn", message, data);
620
+ }
621
+ error(message, data) {
622
+ this.log("error", message, data);
623
+ }
624
+ log(level, message, data, forceStderr = false) {
625
+ if (LOG_LEVEL_PRIORITY[level] < LOG_LEVEL_PRIORITY[Logger.level])
626
+ return;
627
+ if (Logger.jsonMode) {
628
+ const entry = {
629
+ timestamp: new Date().toISOString(),
630
+ level,
631
+ component: this.component,
632
+ message,
633
+ ...data && { data }
634
+ };
635
+ if (forceStderr)
636
+ console.error(JSON.stringify(entry));
637
+ else
638
+ console.log(JSON.stringify(entry));
639
+ } else {
640
+ const prefix = `[${this.component}]`;
641
+ const dataStr = data ? ` ${JSON.stringify(data)}` : "";
642
+ switch (level) {
643
+ case "debug":
644
+ if (forceStderr)
645
+ console.error(`${prefix} ${message}${dataStr}`);
646
+ else
647
+ console.debug(`${prefix} ${message}${dataStr}`);
648
+ break;
649
+ case "info":
650
+ if (forceStderr)
651
+ console.error(`${prefix} ${message}${dataStr}`);
652
+ else
653
+ console.log(`${prefix} ${message}${dataStr}`);
654
+ break;
655
+ case "warn":
656
+ console.warn(`${prefix} ${message}${dataStr}`);
657
+ break;
658
+ case "error":
659
+ console.error(`${prefix} ${message}${dataStr}`);
660
+ break;
661
+ }
662
+ }
663
+ }
664
+ }
665
+ function createLogger(component) {
666
+ return new Logger(component);
667
+ }
668
+ var serverLog = createLogger("Server");
669
+ var tcpLog = createLogger("TCP");
670
+ var httpLog = createLogger("HTTP");
671
+ var wsLog = createLogger("WS");
672
+ var cronLog = createLogger("Cron");
673
+ var statsLog = createLogger("Stats");
674
+ var storageLog = createLogger("Storage");
675
+ var queueLog = createLogger("Queue");
676
+ var webhookLog = createLogger("Webhook");
677
+ var backupLog = createLogger("Backup");
678
+
679
+ // ../../src/domain/types/dlq.ts
680
+ var DLQ_RETRY_STATE = Symbol("bunqueue.dlqRetryState");
681
+ function getDlqRetryState(job) {
682
+ return job[DLQ_RETRY_STATE] ?? null;
683
+ }
684
+ function setDlqRetryState(job, state) {
685
+ if (state === null) {
686
+ delete job[DLQ_RETRY_STATE];
687
+ return;
688
+ }
689
+ Object.defineProperty(job, DLQ_RETRY_STATE, {
690
+ value: state,
691
+ writable: true,
692
+ configurable: true,
693
+ enumerable: false
694
+ });
695
+ }
696
+ function createAttemptRecord(job, reason, error, failedAt = Date.now()) {
785
697
  return {
786
- priority: input.priority ?? JOB_DEFAULTS.priority,
787
- lifo: input.lifo ?? JOB_DEFAULTS.lifo,
788
- maxAttempts: input.maxAttempts ?? JOB_DEFAULTS.maxAttempts,
789
- removeOnComplete: toBoolean(input.removeOnComplete, JOB_DEFAULTS.removeOnComplete),
790
- removeOnFail: toBoolean(input.removeOnFail, JOB_DEFAULTS.removeOnFail)
698
+ attempt: job.attempts,
699
+ startedAt: job.startedAt ?? job.createdAt,
700
+ failedAt,
701
+ reason,
702
+ error,
703
+ duration: job.startedAt ? failedAt - job.startedAt : 0
791
704
  };
792
705
  }
793
- function parseOptionalFields(input) {
794
- return {
795
- ttl: input.ttl ?? null,
796
- timeout: input.timeout ?? null,
797
- uniqueKey: input.uniqueKey ?? null,
798
- customId: input.customId ?? null,
799
- parentId: input.parentId ?? null,
800
- groupId: input.groupId ?? null,
801
- stallTimeout: input.stallTimeout ?? null
706
+ function recordJobFailureAttempt(job, reason, error, failedAt = Date.now()) {
707
+ const state = getDlqRetryState(job) ?? {
708
+ enteredAt: null,
709
+ expiresAt: null,
710
+ attempts: [],
711
+ retryCount: 0,
712
+ lastRetryAt: null,
713
+ nextRetryAt: null
802
714
  };
715
+ state.attempts.push(createAttemptRecord(job, reason, error, failedAt));
716
+ setDlqRetryState(job, state);
803
717
  }
804
- function parseBullMQV5Options(input) {
718
+ function retainDlqRetryState(job, entry) {
719
+ setDlqRetryState(job, {
720
+ enteredAt: entry.enteredAt,
721
+ expiresAt: entry.expiresAt,
722
+ attempts: [...entry.attempts],
723
+ retryCount: entry.retryCount,
724
+ lastRetryAt: entry.lastRetryAt,
725
+ nextRetryAt: entry.nextRetryAt
726
+ });
727
+ }
728
+ var DEFAULT_DLQ_CONFIG = {
729
+ autoRetry: false,
730
+ autoRetryInterval: 60 * 60 * 1000,
731
+ maxAutoRetries: 3,
732
+ maxAge: 7 * 24 * 60 * 60 * 1000,
733
+ maxEntries: 1e4
734
+ };
735
+ function createDlqEntry(job, reason, error, config = DEFAULT_DLQ_CONFIG, now = Date.now()) {
736
+ const retained = getDlqRetryState(job);
737
+ const alreadyEntered = retained?.enteredAt !== null && retained?.enteredAt !== undefined;
738
+ const attemptRecord = createAttemptRecord(job, reason, error, now);
739
+ setDlqRetryState(job, null);
805
740
  return {
806
- stackTraceLimit: input.stackTraceLimit ?? JOB_DEFAULTS.stackTraceLimit,
807
- keepLogs: input.keepLogs ?? null,
808
- sizeLimit: input.sizeLimit ?? null,
809
- failParentOnFailure: input.failParentOnFailure ?? false,
810
- removeDependencyOnFailure: input.removeDependencyOnFailure ?? false,
811
- continueParentOnFailure: input.continueParentOnFailure ?? false,
812
- ignoreDependencyOnFailure: input.ignoreDependencyOnFailure ?? false,
813
- deduplicationTtl: input.dedup?.ttl ?? null,
814
- deduplicationExtend: input.dedup?.extend ?? false,
815
- deduplicationReplace: input.dedup?.replace ?? false,
816
- debounceId: input.debounceId ?? null,
817
- debounceTtl: input.debounceTtl ?? null
741
+ job,
742
+ enteredAt: alreadyEntered ? retained.enteredAt : now,
743
+ reason,
744
+ error,
745
+ attempts: [...retained?.attempts ?? [], attemptRecord],
746
+ retryCount: retained?.retryCount ?? 0,
747
+ lastRetryAt: retained?.lastRetryAt ?? null,
748
+ nextRetryAt: alreadyEntered ? retained.nextRetryAt : config.autoRetry ? now + config.autoRetryInterval : null,
749
+ expiresAt: alreadyEntered ? retained.expiresAt : config.maxAge !== null ? now + config.maxAge : null
818
750
  };
819
751
  }
820
- function createJob(id, queue, input, now = Date.now()) {
821
- const { backoff, backoffConfig } = parseBackoff(input.backoff);
822
- const coreOpts = parseCoreOptions(input);
823
- const optionalFields = parseOptionalFields(input);
824
- const v5Opts = parseBullMQV5Options(input);
825
- const createdAt = input.timestamp ?? now;
826
- const payload = normalizeJobPayload(input);
827
- return {
828
- id,
829
- queue,
830
- name: payload.name,
831
- data: payload.data,
832
- createdAt,
833
- runAt: createdAt + (input.delay ?? 0),
834
- startedAt: null,
835
- completedAt: null,
836
- attempts: 0,
837
- backoff,
838
- backoffConfig,
839
- dependsOn: input.dependsOn ?? [],
840
- childrenIds: input.childrenIds ?? [],
841
- childrenCompleted: 0,
842
- tags: input.tags ?? [],
843
- progress: 0,
844
- progressMessage: null,
845
- stacktrace: null,
846
- repeat: parseRepeatConfig(input.repeat),
847
- durable: input.durable ?? false,
848
- lastHeartbeat: createdAt,
849
- stallCount: 0,
850
- ...coreOpts,
851
- ...optionalFields,
852
- ...v5Opts,
853
- timeline: []
752
+ function isDlqEntryExpired(entry, now = Date.now()) {
753
+ return entry.expiresAt !== null && now >= entry.expiresAt;
754
+ }
755
+ function canAutoRetry(entry, config, now = Date.now()) {
756
+ if (!config.autoRetry)
757
+ return false;
758
+ if (entry.retryCount >= config.maxAutoRetries)
759
+ return false;
760
+ if (entry.nextRetryAt === null)
761
+ return false;
762
+ return now >= entry.nextRetryAt;
763
+ }
764
+ function scheduleNextRetry(entry, config) {
765
+ const now = Date.now();
766
+ entry.retryCount++;
767
+ entry.lastRetryAt = now;
768
+ if (entry.retryCount < config.maxAutoRetries) {
769
+ const backoffMultiplier = Math.pow(2, entry.retryCount - 1);
770
+ entry.nextRetryAt = now + config.autoRetryInterval * backoffMultiplier;
771
+ } else {
772
+ entry.nextRetryAt = null;
773
+ }
774
+ }
775
+
776
+ // ../../src/domain/types/queue.ts
777
+ function createQueueState(name) {
778
+ return {
779
+ name,
780
+ paused: false,
781
+ rateLimit: null,
782
+ rateLimitDuration: null,
783
+ rateLimitExpiresAt: null,
784
+ concurrencyLimit: null,
785
+ activeCount: 0
854
786
  };
855
787
  }
856
- // ../../src/domain/job/ids.ts
857
- function isWellFormedJobId(id) {
858
- for (let index = 0;index < id.length; index++) {
859
- const unit = id.charCodeAt(index);
860
- if (unit >= 55296 && unit <= 56319) {
861
- const next = id.charCodeAt(++index);
862
- if (!(next >= 56320 && next <= 57343))
863
- return false;
864
- } else if (unit >= 56320 && unit <= 57343) {
865
- return false;
788
+
789
+ class RateLimiter {
790
+ capacity;
791
+ refillRate;
792
+ tokens;
793
+ lastRefill;
794
+ constructor(capacity, refillRate = capacity) {
795
+ this.capacity = capacity;
796
+ this.refillRate = refillRate;
797
+ this.tokens = capacity;
798
+ this.lastRefill = Date.now();
799
+ }
800
+ tryAcquire() {
801
+ this.refill();
802
+ if (this.tokens >= 1) {
803
+ this.tokens -= 1;
804
+ return true;
805
+ }
806
+ return false;
807
+ }
808
+ refill() {
809
+ const now = Date.now();
810
+ const elapsed = (now - this.lastRefill) / 1000;
811
+ const tokensToAdd = elapsed * this.refillRate;
812
+ this.tokens = Math.min(this.capacity, this.tokens + tokensToAdd);
813
+ this.lastRefill = now;
814
+ }
815
+ getTokens() {
816
+ this.refill();
817
+ return this.tokens;
818
+ }
819
+ getTtl(maxJobs) {
820
+ const tokens = this.getTokens();
821
+ const consumed = this.capacity - tokens;
822
+ if (maxJobs !== undefined && consumed < maxJobs)
823
+ return 0;
824
+ if (tokens >= 1 || this.refillRate <= 0)
825
+ return 0;
826
+ return Math.ceil((1 - tokens) / this.refillRate * 1000);
827
+ }
828
+ }
829
+
830
+ class ConcurrencyLimiter {
831
+ limit;
832
+ active = 0;
833
+ constructor(limit) {
834
+ this.limit = limit;
835
+ }
836
+ tryAcquire() {
837
+ if (this.active < this.limit) {
838
+ this.active += 1;
839
+ return true;
840
+ }
841
+ return false;
842
+ }
843
+ release() {
844
+ if (this.active > 0) {
845
+ this.active -= 1;
846
+ }
847
+ }
848
+ getActive() {
849
+ return this.active;
850
+ }
851
+ getLimit() {
852
+ return this.limit;
853
+ }
854
+ setLimit(limit) {
855
+ this.limit = limit;
856
+ }
857
+ }
858
+
859
+ // ../../src/application/stallDetection.ts
860
+ function checkStalledJobs(ctx) {
861
+ const now = Date.now();
862
+ const confirmedStalled = [];
863
+ for (const jobId of ctx.stalledCandidates) {
864
+ const procIdx = processingShardIndex(jobId);
865
+ const job = ctx.processingShards[procIdx].get(jobId);
866
+ if (!job) {
867
+ ctx.stalledCandidates.delete(jobId);
868
+ continue;
869
+ }
870
+ const stallConfig = ctx.shards[shardIndex(job.queue)].getStallConfig(job.queue);
871
+ if (!stallConfig.enabled) {
872
+ ctx.stalledCandidates.delete(jobId);
873
+ continue;
874
+ }
875
+ const action = getStallAction(job, stallConfig, now);
876
+ if (action !== "keep" /* Keep */) {
877
+ confirmedStalled.push({ job, action });
878
+ }
879
+ ctx.stalledCandidates.delete(jobId);
880
+ }
881
+ for (let i = 0;i < SHARD_COUNT; i++) {
882
+ const procShard = ctx.processingShards[i];
883
+ for (const [jobId, job] of procShard) {
884
+ const stallConfig = ctx.shards[shardIndex(job.queue)].getStallConfig(job.queue);
885
+ if (!stallConfig.enabled)
886
+ continue;
887
+ const action = getStallAction(job, stallConfig, now);
888
+ if (action !== "keep" /* Keep */) {
889
+ ctx.stalledCandidates.add(jobId);
890
+ }
891
+ }
892
+ }
893
+ const stillStalled = (job, lockedNow) => {
894
+ const config = ctx.shards[shardIndex(job.queue)].getStallConfig(job.queue);
895
+ return config.enabled && getStallAction(job, config, lockedNow) !== "keep" /* Keep */;
896
+ };
897
+ for (const { job, action } of confirmedStalled) {
898
+ handleStalledJob(job, action, ctx, stillStalled).catch((err) => {
899
+ queueLog.error("Failed to handle stalled job", {
900
+ jobId: String(job.id),
901
+ error: String(err)
902
+ });
903
+ });
904
+ }
905
+ }
906
+ async function handleStalledJob(job, action, ctx, recheck) {
907
+ const idx = shardIndex(job.queue);
908
+ const procIdx = processingShardIndex(job.id);
909
+ let handled = false;
910
+ let handledAction = action;
911
+ await withWriteLock(ctx.shardLocks[idx], async () => {
912
+ await withWriteLock(ctx.processingLocks[procIdx], () => {
913
+ if (ctx.processingShards[procIdx].get(job.id) !== job) {
914
+ return;
915
+ }
916
+ if (recheck && !recheck(job, Date.now()))
917
+ return;
918
+ const shard = ctx.shards[idx];
919
+ const attemptsExhausted = job.attempts + 1 >= job.maxAttempts;
920
+ if (action === "move_to_dlq" /* MoveToDlq */ || attemptsExhausted) {
921
+ handledAction = "move_to_dlq" /* MoveToDlq */;
922
+ moveStalliedJobToDlq(job, ctx, shard, procIdx, attemptsExhausted);
923
+ } else {
924
+ retryStalliedJob(job, ctx, shard, procIdx, idx);
925
+ }
926
+ detachClientJob(job.id, ctx);
927
+ handled = true;
928
+ });
929
+ });
930
+ if (handled) {
931
+ ctx.dashboardEmit?.("job:stalled", {
932
+ jobId: String(job.id),
933
+ queue: job.queue,
934
+ stallCount: job.stallCount,
935
+ action: handledAction
936
+ });
937
+ ctx.eventsManager.broadcast({
938
+ eventType: "stalled" /* Stalled */,
939
+ queue: job.queue,
940
+ jobId: job.id,
941
+ timestamp: Date.now(),
942
+ data: { stallCount: job.stallCount, action: handledAction }
943
+ });
944
+ ctx.webhookManager.trigger("stalled", String(job.id), job.queue, {
945
+ data: { stallCount: job.stallCount, action: handledAction }
946
+ });
947
+ }
948
+ return handled;
949
+ }
950
+ function moveStalliedJobToDlq(job, ctx, shard, procIdx, attemptsExhausted) {
951
+ ctx.processingShards[procIdx].delete(job.id);
952
+ ctx.timeoutScheduler.cancel(job.id);
953
+ shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId, job.id);
954
+ if (job.uniqueKey?.startsWith("cron:")) {
955
+ ctx.jobIndex.delete(job.id);
956
+ ctx.storage?.deleteJob(job.id);
957
+ return;
958
+ }
959
+ incrementStallCount(job);
960
+ job.attempts++;
961
+ job.startedAt = null;
962
+ job.lastHeartbeat = Date.now();
963
+ const entry = shard.addToDlq(job, attemptsExhausted ? "max_attempts_exceeded" /* MaxAttemptsExceeded */ : "stalled" /* Stalled */, attemptsExhausted ? `Job stalled at max attempts (${job.maxAttempts})` : `Job stalled ${job.stallCount} times`);
964
+ ctx.jobIndex.set(job.id, { type: "dlq", queueName: job.queue });
965
+ ctx.storage?.saveDlqEntry(entry);
966
+ ctx.storage?.deleteJob(job.id);
967
+ }
968
+ function retryStalliedJob(job, ctx, shard, procIdx, idx) {
969
+ if (job.uniqueKey?.startsWith("cron:")) {
970
+ ctx.processingShards[procIdx].delete(job.id);
971
+ ctx.timeoutScheduler.cancel(job.id);
972
+ shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId, job.id);
973
+ ctx.jobIndex.delete(job.id);
974
+ ctx.storage?.deleteJob(job.id);
975
+ return;
976
+ }
977
+ incrementStallCount(job);
978
+ job.attempts++;
979
+ const stallNow = Date.now();
980
+ recordJobFailureAttempt(job, "stalled" /* Stalled */, "stalled", stallNow);
981
+ job.startedAt = null;
982
+ job.runAt = stallNow + calculateBackoff(job);
983
+ job.lastHeartbeat = stallNow;
984
+ if (job.timeline.length < MAX_TIMELINE_ENTRIES) {
985
+ job.timeline.push({
986
+ state: "failed",
987
+ timestamp: stallNow,
988
+ error: "stalled",
989
+ attempt: job.attempts
990
+ });
991
+ job.timeline.push({ state: "waiting", timestamp: stallNow, attempt: job.attempts + 1 });
992
+ }
993
+ ctx.processingShards[procIdx].delete(job.id);
994
+ ctx.timeoutScheduler.cancel(job.id);
995
+ shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId, job.id);
996
+ shard.getQueue(job.queue).push(job);
997
+ const isDelayed = job.runAt > Date.now();
998
+ shard.incrementQueued(job.id, isDelayed, job.createdAt, job.queue, job.runAt);
999
+ ctx.jobIndex.set(job.id, { type: "queue", shardIdx: idx, queueName: job.queue });
1000
+ ctx.storage?.updateForRetry(job);
1001
+ }
1002
+
1003
+ // ../../src/application/orphanRecovery.ts
1004
+ var ORPHAN_WINDOW_FLOOR_MS = 30 * 60 * 1000;
1005
+ function isOrphanedProcessingEntry(job, lock, now, ctx) {
1006
+ if (!job.startedAt)
1007
+ return false;
1008
+ const config = ctx.shards[shardIndex(job.queue)].getStallConfig(job.queue);
1009
+ if (!config.enabled)
1010
+ return false;
1011
+ if (now - job.startedAt < config.gracePeriod)
1012
+ return false;
1013
+ if (lock && !isLockExpired(lock, now) && !isLeaseFromEarlierGeneration(job, lock))
1014
+ return false;
1015
+ const window = Math.max(ORPHAN_WINDOW_FLOOR_MS, job.stallTimeout ?? config.stallInterval);
1016
+ const lastActivity = Math.max(job.startedAt, job.lastHeartbeat || 0);
1017
+ return now - lastActivity > window;
1018
+ }
1019
+ function orphanStallAction(job, ctx) {
1020
+ const { maxStalls } = ctx.shards[shardIndex(job.queue)].getStallConfig(job.queue);
1021
+ return maxStalls > 0 && job.stallCount + 1 >= maxStalls ? "move_to_dlq" /* MoveToDlq */ : "retry" /* Retry */;
1022
+ }
1023
+ async function recoverOrphanedProcessingEntries(ctx, now) {
1024
+ const orphans = [];
1025
+ for (let i = 0;i < SHARD_COUNT; i++) {
1026
+ for (const [jobId, job] of ctx.processingShards[i]) {
1027
+ if (isOrphanedProcessingEntry(job, ctx.jobLocks.get(jobId), now, ctx)) {
1028
+ orphans.push(job);
1029
+ }
1030
+ }
1031
+ }
1032
+ if (orphans.length === 0)
1033
+ return;
1034
+ const stillOrphaned = (job, lockedNow) => isOrphanedProcessingEntry(job, ctx.jobLocks.get(job.id), lockedNow, ctx);
1035
+ let recovered = 0;
1036
+ for (const job of orphans) {
1037
+ try {
1038
+ if (await handleStalledJob(job, orphanStallAction(job, ctx), ctx, stillOrphaned)) {
1039
+ recovered++;
1040
+ }
1041
+ } catch (err) {
1042
+ queueLog.error("Failed to recover orphaned processing entry", {
1043
+ jobId: String(job.id),
1044
+ error: String(err)
1045
+ });
1046
+ }
1047
+ }
1048
+ if (recovered > 0)
1049
+ ctx.dashboardEmit?.("cleanup:orphans-removed", { count: recovered });
1050
+ }
1051
+
1052
+ // ../../src/application/dependencyCompletions.ts
1053
+ class DependencyCompletionTracker {
1054
+ onRecentEvict;
1055
+ recent = new Set;
1056
+ recentOrder = [];
1057
+ recentOrderTokens = [];
1058
+ recentTokens = new Map;
1059
+ recentOrderHead = 0;
1060
+ nextRecentToken = 0;
1061
+ pinned = new Set;
1062
+ owners = new Map;
1063
+ maxRecent;
1064
+ constructor(maxRecent, onRecentEvict) {
1065
+ this.onRecentEvict = onRecentEvict;
1066
+ this.maxRecent = Math.max(1, Math.trunc(maxRecent));
1067
+ }
1068
+ add(jobId, queue) {
1069
+ if (queue !== undefined)
1070
+ this.owners.set(jobId, queue);
1071
+ if (this.pinned.has(jobId) || this.recent.has(jobId))
1072
+ return;
1073
+ if (this.recent.size >= this.maxRecent)
1074
+ this.evictOldestRecent();
1075
+ if (this.nextRecentToken >= Number.MAX_SAFE_INTEGER)
1076
+ this.compactRecentOrder(true);
1077
+ const token = this.nextRecentToken++;
1078
+ this.recent.add(jobId);
1079
+ this.recentTokens.set(jobId, token);
1080
+ this.recentOrder.push(jobId);
1081
+ this.recentOrderTokens.push(token);
1082
+ }
1083
+ pin(jobId, queue) {
1084
+ if (queue !== undefined)
1085
+ this.owners.set(jobId, queue);
1086
+ if (this.recent.delete(jobId)) {
1087
+ this.recentTokens.delete(jobId);
1088
+ this.compactRecentOrder();
1089
+ }
1090
+ this.pinned.add(jobId);
1091
+ }
1092
+ unpin(jobId, retainAsRecent) {
1093
+ this.pinned.delete(jobId);
1094
+ if (retainAsRecent)
1095
+ this.add(jobId);
1096
+ else
1097
+ this.delete(jobId);
1098
+ }
1099
+ hydrate(records) {
1100
+ this.clear();
1101
+ for (const record of records) {
1102
+ if (record.pinned)
1103
+ this.pin(record.jobId, record.queue);
1104
+ else
1105
+ this.add(record.jobId, record.queue);
1106
+ }
1107
+ }
1108
+ pinnedValues() {
1109
+ return this.pinned.values();
1110
+ }
1111
+ has(jobId) {
1112
+ return this.pinned.has(jobId) || this.recent.has(jobId);
1113
+ }
1114
+ delete(jobId) {
1115
+ const wasPinned = this.pinned.delete(jobId);
1116
+ const wasRecent = this.recent.delete(jobId);
1117
+ if (wasRecent) {
1118
+ this.recentTokens.delete(jobId);
1119
+ this.compactRecentOrder();
1120
+ }
1121
+ this.owners.delete(jobId);
1122
+ return wasRecent || wasPinned;
1123
+ }
1124
+ deleteForQueue(queue) {
1125
+ const removed = [];
1126
+ for (const [jobId, owner] of this.owners) {
1127
+ if (owner !== queue)
1128
+ continue;
1129
+ if (this.delete(jobId))
1130
+ removed.push(jobId);
1131
+ }
1132
+ return removed;
1133
+ }
1134
+ clear() {
1135
+ this.recent.clear();
1136
+ this.recentOrder = [];
1137
+ this.recentOrderTokens = [];
1138
+ this.recentTokens.clear();
1139
+ this.recentOrderHead = 0;
1140
+ this.nextRecentToken = 0;
1141
+ this.pinned.clear();
1142
+ this.owners.clear();
1143
+ }
1144
+ get size() {
1145
+ return this.recent.size + this.pinned.size;
1146
+ }
1147
+ evictOldestRecent() {
1148
+ while (this.recentOrderHead < this.recentOrder.length) {
1149
+ const index = this.recentOrderHead++;
1150
+ const oldest = this.recentOrder[index];
1151
+ const token = this.recentOrderTokens[index];
1152
+ if (this.recentTokens.get(oldest) !== token)
1153
+ continue;
1154
+ this.recent.delete(oldest);
1155
+ this.recentTokens.delete(oldest);
1156
+ this.owners.delete(oldest);
1157
+ this.compactRecentOrder();
1158
+ this.onRecentEvict?.(oldest);
1159
+ return;
1160
+ }
1161
+ }
1162
+ compactRecentOrder(force = false) {
1163
+ const pending = this.recentOrder.length - this.recentOrderHead;
1164
+ if (!force && this.recentOrderHead < this.maxRecent && pending < this.maxRecent * 2)
1165
+ return;
1166
+ const order = [];
1167
+ const tokens = [];
1168
+ for (let index = this.recentOrderHead;index < this.recentOrder.length; index++) {
1169
+ const jobId = this.recentOrder[index];
1170
+ const token = this.recentOrderTokens[index];
1171
+ if (this.recentTokens.get(jobId) !== token)
1172
+ continue;
1173
+ const compactedToken = order.length;
1174
+ order.push(jobId);
1175
+ tokens.push(compactedToken);
1176
+ }
1177
+ this.recentTokens.clear();
1178
+ for (let index = 0;index < order.length; index++) {
1179
+ this.recentTokens.set(order[index], tokens[index]);
866
1180
  }
1181
+ this.recentOrder = order;
1182
+ this.recentOrderTokens = tokens;
1183
+ this.recentOrderHead = 0;
1184
+ this.nextRecentToken = order.length;
867
1185
  }
868
- return true;
869
- }
870
- function assertWellFormedJobId(id, name = "job ID") {
871
- if (!isWellFormedJobId(id))
872
- throw new Error(`${name} must be well-formed Unicode`);
873
- }
874
- function jobId(id) {
875
- return id;
876
- }
877
- function generateJobId() {
878
- return Bun.randomUUIDv7();
879
1186
  }
880
- function generateLockToken() {
881
- return Bun.randomUUIDv7();
1187
+ function hasDependencyWaiters(shards, jobId) {
1188
+ return shards.some((shard) => (shard.getJobsWaitingFor(jobId)?.size ?? 0) > 0);
882
1189
  }
883
- // ../../src/domain/job/locks.ts
884
- function createJobLock(jobId, owner, ttl = DEFAULT_LOCK_TTL, now = Date.now()) {
885
- return {
886
- jobId,
887
- token: generateLockToken(),
888
- owner,
889
- createdAt: now,
890
- expiresAt: now + ttl,
891
- lastRenewalAt: now,
892
- renewalCount: 0,
893
- ttl
894
- };
1190
+ function commitRemovedCompletion(job, ctx, completedAt = Date.now()) {
1191
+ const pinned = hasDependencyWaiters(ctx.shards, job.id);
1192
+ ctx.storage?.commitRemovedCompletion(job, ctx.maxDependencyCompletions, pinned, completedAt);
1193
+ if (pinned)
1194
+ ctx.depCompletions?.pin(job.id, job.queue);
1195
+ else
1196
+ ctx.depCompletions?.add(job.id, job.queue);
895
1197
  }
896
- function isLockExpired(lock, now = Date.now()) {
897
- return now >= lock.expiresAt;
1198
+ function releaseDependencyCompletionPins(dependencyIds, ctx) {
1199
+ const candidates = [...new Set(dependencyIds)].filter((id) => !hasDependencyWaiters(ctx.shards, id));
1200
+ if (candidates.length === 0)
1201
+ return;
1202
+ if (ctx.storage) {
1203
+ const records = ctx.storage.unpinDependencyCompletions(candidates, ctx.maxDependencyCompletions);
1204
+ ctx.depCompletions?.hydrate(records);
1205
+ return;
1206
+ }
1207
+ for (const id of candidates)
1208
+ ctx.depCompletions?.unpin(id, true);
898
1209
  }
899
- function renewLock(lock, newTtl, now = Date.now()) {
900
- const ttl = newTtl ?? lock.ttl;
901
- lock.expiresAt = now + ttl;
902
- lock.lastRenewalAt = now;
903
- lock.renewalCount++;
1210
+ function reconcileDependencyCompletionPins(ctx) {
1211
+ const referenced = new Set;
1212
+ for (const shard of ctx.shards) {
1213
+ for (const dependencyId of shard.dependencyIndex.keys())
1214
+ referenced.add(dependencyId);
1215
+ }
1216
+ if (ctx.storage) {
1217
+ const records = ctx.storage.reconcileDependencyCompletionPins(referenced, ctx.maxDependencyCompletions);
1218
+ ctx.depCompletions?.hydrate(records);
1219
+ return;
1220
+ }
1221
+ for (const id of ctx.depCompletions?.pinnedValues() ?? []) {
1222
+ if (!referenced.has(id))
1223
+ ctx.depCompletions?.unpin(id, true);
1224
+ }
904
1225
  }
905
- // ../../src/domain/job/state.ts
906
- function normalizeStacktrace(lines, limit) {
907
- if (!lines || lines.length === 0)
908
- return null;
909
- const out = [];
910
- for (const line of lines) {
911
- if (out.length >= limit)
1226
+
1227
+ // ../../src/shared/serialization.ts
1228
+ function compareSqliteBinaryText(left, right) {
1229
+ if (left === right)
1230
+ return 0;
1231
+ let leftIndex = 0;
1232
+ let rightIndex = 0;
1233
+ while (leftIndex < left.length && rightIndex < right.length) {
1234
+ const leftPoint = left.codePointAt(leftIndex);
1235
+ const rightPoint = right.codePointAt(rightIndex);
1236
+ if (leftPoint === undefined || rightPoint === undefined)
912
1237
  break;
913
- if (typeof line !== "string")
914
- continue;
915
- const trimmed = line.trim();
916
- if (trimmed)
917
- out.push(trimmed);
1238
+ if (leftPoint !== rightPoint)
1239
+ return leftPoint - rightPoint;
1240
+ leftIndex += leftPoint > 65535 ? 2 : 1;
1241
+ rightIndex += rightPoint > 65535 ? 2 : 1;
918
1242
  }
919
- return out.length > 0 ? out : null;
920
- }
921
- function isReady(job, now = Date.now()) {
922
- return job.runAt <= now;
1243
+ return left.length - leftIndex - (right.length - rightIndex);
923
1244
  }
924
- function isExpired(job, now = Date.now()) {
925
- if (job.ttl === null)
926
- return false;
927
- return now > job.createdAt + job.ttl;
1245
+
1246
+ // ../../src/application/completedCleanup.ts
1247
+ function memoryCandidates(queue, completedBefore, limit, ctx) {
1248
+ return [...ctx.jobIndex].flatMap(([jobId, location]) => {
1249
+ if (location.type !== "completed")
1250
+ return [];
1251
+ if (queue !== null && location.queueName !== queue)
1252
+ return [];
1253
+ if (ctx.dependencyResults.hasConsumers(jobId))
1254
+ return [];
1255
+ const job = ctx.completedJobsData.get(jobId);
1256
+ const timestamp = job?.completedAt ?? job?.createdAt ?? 0;
1257
+ return timestamp <= completedBefore ? [{ jobId, timestamp }] : [];
1258
+ }).sort((left, right) => left.timestamp - right.timestamp || compareSqliteBinaryText(left.jobId, right.jobId)).slice(0, limit).map(({ jobId }) => ({ jobId, queue: ctx.jobIndex.get(jobId)?.queueName ?? "" }));
928
1259
  }
929
- function calculateBackoff(job) {
930
- const maxDelay = job.backoffConfig?.maxDelay ?? DEFAULT_MAX_BACKOFF;
931
- if (job.backoffConfig) {
932
- if (job.backoffConfig.type === "fixed") {
933
- const base = job.backoffConfig.delay;
934
- const jittered = base * (0.8 + Math.random() * 0.4);
935
- return Math.min(jittered, maxDelay);
1260
+ function cleanCompletedJobs(queue, completedBefore, limit, ctx) {
1261
+ const normalizedLimit = Number.isFinite(limit) ? Math.max(0, Math.trunc(limit)) : 0;
1262
+ if (normalizedLimit === 0 || !Number.isFinite(completedBefore))
1263
+ return [];
1264
+ const removals = ctx.storage ? ctx.storage.cleanCompletedJobs(queue, completedBefore, normalizedLimit, (jobId) => ctx.dependencyResults.hasConsumers(jobId)) : memoryCandidates(queue, completedBefore, normalizedLimit, ctx);
1265
+ for (const { jobId, queue: ownerQueue } of removals) {
1266
+ const location = ctx.jobIndex.get(jobId);
1267
+ const ownsCurrentCompletion = location?.type === "completed" && location.queueName === ownerQueue;
1268
+ const hasNoCurrentGeneration = location === undefined;
1269
+ const resultOwner = ctx.jobResultQueues?.get(jobId);
1270
+ if (resultOwner === ownerQueue || resultOwner === undefined && ownsCurrentCompletion) {
1271
+ ctx.jobResults?.delete(jobId);
1272
+ ctx.jobResultQueues?.delete(jobId);
936
1273
  }
937
- const base = job.backoffConfig.delay * Math.pow(2, job.attempts);
938
- const jittered = base * (0.5 + Math.random());
939
- return Math.min(jittered, maxDelay);
940
- }
941
- const base = job.backoff * Math.pow(2, job.attempts);
942
- const jittered = base * (0.5 + Math.random());
943
- return Math.min(jittered, maxDelay);
944
- }
945
- function canRetry(job) {
946
- return job.attempts < job.maxAttempts;
947
- }
948
- // ../../src/application/dependencyProcessor.ts
949
- function dependencyReadyState(job, now) {
950
- if (job.runAt > now)
951
- return "delayed";
952
- return job.priority > 0 ? "prioritized" : "waiting";
953
- }
954
- function checkpointDependencyPromotion(job, state, now, storage) {
955
- if (job.timeline.at(-1)?.state !== state && job.timeline.length < MAX_TIMELINE_ENTRIES) {
956
- job.timeline.push({ state, timestamp: now });
957
- }
958
- storage?.updateFlowParentResolution(job, state);
959
- }
960
- async function processPendingDependencies(ctx) {
961
- if (ctx.pendingDepChecks.size === 0)
962
- return;
963
- const completedIds = Array.from(ctx.pendingDepChecks);
964
- const completedNow = new Set(completedIds);
965
- ctx.pendingDepChecks.clear();
966
- const jobsToCheckByShard = new Map;
967
- const releasedDependencyIds = new Set;
968
- for (const completedId of completedIds) {
969
- for (let i = 0;i < SHARD_COUNT; i++) {
970
- const waitingJobIds = ctx.shards[i].getJobsWaitingFor(completedId);
971
- if (waitingJobIds && waitingJobIds.size > 0) {
972
- let shardJobs = jobsToCheckByShard.get(i);
973
- if (!shardJobs) {
974
- shardJobs = new Set;
975
- jobsToCheckByShard.set(i, shardJobs);
976
- }
977
- for (const jobId of waitingJobIds) {
978
- shardJobs.add(jobId);
979
- }
980
- }
1274
+ const logOwner = ctx.jobLogQueues?.get(jobId);
1275
+ if (logOwner === ownerQueue || logOwner === undefined && ownsCurrentCompletion) {
1276
+ ctx.jobLogs?.delete(jobId);
1277
+ ctx.jobLogQueues?.delete(jobId);
981
1278
  }
1279
+ if (!hasNoCurrentGeneration && !ownsCurrentCompletion)
1280
+ continue;
1281
+ ctx.completedJobs.delete(jobId);
1282
+ ctx.completedJobsData.delete(jobId);
1283
+ ctx.jobIndex.delete(jobId);
1284
+ ctx.dependencyResults.releaseConsumer(jobId);
982
1285
  }
983
- await Promise.all(Array.from(jobsToCheckByShard.entries()).map(async ([i, jobIdsToCheck]) => {
984
- await withWriteLock(ctx.shardLocks[i], () => {
985
- const shard = ctx.shards[i];
986
- const jobsToPromote = [];
987
- for (const jobId of jobIdsToCheck) {
988
- const job = shard.waitingDeps.get(jobId);
989
- if (job?.dependsOn.every((dep) => completedNow.has(dep) || ctx.completedJobs.has(dep) || (ctx.depCompletions?.has(dep) ?? false))) {
990
- jobsToPromote.push(job);
991
- }
992
- }
993
- if (jobsToPromote.length > 0) {
994
- promoteJobsToQueue(jobsToPromote, shard, ctx, i, releasedDependencyIds);
995
- }
996
- });
997
- }));
998
- releaseDependencyCompletionPins(releasedDependencyIds, ctx);
1286
+ return removals.map(({ jobId }) => jobId);
999
1287
  }
1000
- function promoteJobsToQueue(jobsToPromote, shard, ctx, shardIdx, releasedDependencyIds) {
1001
- const now = Date.now();
1002
- for (const job of jobsToPromote) {
1003
- if (shard.waitingDeps.has(job.id)) {
1004
- const state = dependencyReadyState(job, now);
1005
- checkpointDependencyPromotion(job, state, now, ctx.storage);
1006
- shard.waitingDeps.delete(job.id);
1007
- shard.unregisterDependencies(job.id, job.dependsOn);
1008
- for (const dependencyId of job.dependsOn)
1009
- releasedDependencyIds.add(dependencyId);
1010
- shard.getQueue(job.queue).push(job);
1011
- const isDelayed = state === "delayed";
1012
- shard.incrementQueued(job.id, isDelayed, job.createdAt, job.queue, job.runAt);
1013
- ctx.jobIndex.set(job.id, { type: "queue", shardIdx, queueName: job.queue });
1014
- }
1288
+
1289
+ // ../../src/application/emptyQueueCleanup.ts
1290
+ function collectOccupiedQueues(ctx) {
1291
+ const occupied = new Set;
1292
+ for (const [jobId, location] of ctx.jobIndex) {
1293
+ if (location.type !== "processing")
1294
+ continue;
1295
+ occupied.add(location.queueName);
1296
+ const processingJob = ctx.processingShards[location.shardIdx]?.get(jobId);
1297
+ if (processingJob)
1298
+ occupied.add(processingJob.queue);
1015
1299
  }
1016
- if (jobsToPromote.length > 0) {
1017
- const promotedByQueue = new Map;
1018
- for (const job of jobsToPromote) {
1019
- promotedByQueue.set(job.queue, (promotedByQueue.get(job.queue) ?? 0) + 1);
1020
- }
1021
- for (const [queue, count] of promotedByQueue)
1022
- shard.notifyBatch(queue, count);
1023
- for (const job of jobsToPromote) {
1024
- ctx.dashboardEmit?.("job:dependencies-resolved", { jobId: String(job.id), queue: job.queue });
1025
- }
1300
+ for (const processing of ctx.processingShards) {
1301
+ for (const job of processing.values())
1302
+ occupied.add(job.queue);
1026
1303
  }
1027
- }
1028
-
1029
- // ../../src/domain/types/dlq.ts
1030
- var DLQ_RETRY_STATE = Symbol("bunqueue.dlqRetryState");
1031
- function getDlqRetryState(job) {
1032
- return job[DLQ_RETRY_STATE] ?? null;
1033
- }
1034
- function setDlqRetryState(job, state) {
1035
- if (state === null) {
1036
- delete job[DLQ_RETRY_STATE];
1037
- return;
1304
+ for (const shard of ctx.shards) {
1305
+ for (const job of shard.waitingDeps.values())
1306
+ occupied.add(job.queue);
1307
+ for (const job of shard.waitingChildren.values())
1308
+ occupied.add(job.queue);
1038
1309
  }
1039
- Object.defineProperty(job, DLQ_RETRY_STATE, {
1040
- value: state,
1041
- writable: true,
1042
- configurable: true,
1043
- enumerable: false
1044
- });
1045
- }
1046
- function createAttemptRecord(job, reason, error, failedAt = Date.now()) {
1047
- return {
1048
- attempt: job.attempts,
1049
- startedAt: job.startedAt ?? job.createdAt,
1050
- failedAt,
1051
- reason,
1052
- error,
1053
- duration: job.startedAt ? failedAt - job.startedAt : 0
1054
- };
1055
- }
1056
- function recordJobFailureAttempt(job, reason, error, failedAt = Date.now()) {
1057
- const state = getDlqRetryState(job) ?? {
1058
- enteredAt: null,
1059
- expiresAt: null,
1060
- attempts: [],
1061
- retryCount: 0,
1062
- lastRetryAt: null,
1063
- nextRetryAt: null
1064
- };
1065
- state.attempts.push(createAttemptRecord(job, reason, error, failedAt));
1066
- setDlqRetryState(job, state);
1067
- }
1068
- function retainDlqRetryState(job, entry) {
1069
- setDlqRetryState(job, {
1070
- enteredAt: entry.enteredAt,
1071
- expiresAt: entry.expiresAt,
1072
- attempts: [...entry.attempts],
1073
- retryCount: entry.retryCount,
1074
- lastRetryAt: entry.lastRetryAt,
1075
- nextRetryAt: entry.nextRetryAt
1076
- });
1310
+ for (const [queue, count] of ctx.pendingQueueAdmissions) {
1311
+ if (count > 0)
1312
+ occupied.add(queue);
1313
+ }
1314
+ return occupied;
1077
1315
  }
1078
- var DEFAULT_DLQ_CONFIG = {
1079
- autoRetry: false,
1080
- autoRetryInterval: 60 * 60 * 1000,
1081
- maxAutoRetries: 3,
1082
- maxAge: 7 * 24 * 60 * 60 * 1000,
1083
- maxEntries: 1e4
1084
- };
1085
- function createDlqEntry(job, reason, error, config = DEFAULT_DLQ_CONFIG, now = Date.now()) {
1086
- const retained = getDlqRetryState(job);
1087
- const alreadyEntered = retained?.enteredAt !== null && retained?.enteredAt !== undefined;
1088
- const attemptRecord = createAttemptRecord(job, reason, error, now);
1089
- setDlqRetryState(job, null);
1090
- return {
1091
- job,
1092
- enteredAt: alreadyEntered ? retained.enteredAt : now,
1093
- reason,
1094
- error,
1095
- attempts: [...retained?.attempts ?? [], attemptRecord],
1096
- retryCount: retained?.retryCount ?? 0,
1097
- lastRetryAt: retained?.lastRetryAt ?? null,
1098
- nextRetryAt: alreadyEntered ? retained.nextRetryAt : config.autoRetry ? now + config.autoRetryInterval : null,
1099
- expiresAt: alreadyEntered ? retained.expiresAt : config.maxAge !== null ? now + config.maxAge : null
1100
- };
1316
+ function collectConfiguredQueues(ctx) {
1317
+ const configured = new Set;
1318
+ for (const shard of ctx.shards) {
1319
+ for (const queue of shard.getConfiguredQueueNames())
1320
+ configured.add(queue);
1321
+ }
1322
+ return configured;
1101
1323
  }
1102
- function isDlqEntryExpired(entry, now = Date.now()) {
1103
- return entry.expiresAt !== null && now >= entry.expiresAt;
1324
+ function collectBufferedCompletedQueues(ctx) {
1325
+ const queues = new Set;
1326
+ for (const job of ctx.storage?.getBufferedJobs() ?? []) {
1327
+ if (job.completedAt === null)
1328
+ continue;
1329
+ const location = ctx.jobIndex.get(job.id);
1330
+ if (location && (location.type !== "completed" || location.queueName !== job.queue))
1331
+ continue;
1332
+ queues.add(job.queue);
1333
+ }
1334
+ return queues;
1104
1335
  }
1105
- function canAutoRetry(entry, config, now = Date.now()) {
1106
- if (!config.autoRetry)
1107
- return false;
1108
- if (entry.retryCount >= config.maxAutoRetries)
1109
- return false;
1110
- if (entry.nextRetryAt === null)
1111
- return false;
1112
- return now >= entry.nextRetryAt;
1336
+ function collectCompletedCounts(ctx, registered) {
1337
+ if (ctx.storage)
1338
+ return ctx.storage.countCompletedJobsByQueue(registered);
1339
+ const registeredSet = new Set(registered);
1340
+ const counts = new Map;
1341
+ for (const [jobId, location] of ctx.jobIndex) {
1342
+ if (location.type !== "completed" || !registeredSet.has(location.queueName) || !ctx.completedJobs.has(jobId)) {
1343
+ continue;
1344
+ }
1345
+ counts.set(location.queueName, (counts.get(location.queueName) ?? 0) + 1);
1346
+ }
1347
+ return counts;
1113
1348
  }
1114
- function scheduleNextRetry(entry, config) {
1115
- const now = Date.now();
1116
- entry.retryCount++;
1117
- entry.lastRetryAt = now;
1118
- if (entry.retryCount < config.maxAutoRetries) {
1119
- const backoffMultiplier = Math.pow(2, entry.retryCount - 1);
1120
- entry.nextRetryAt = now + config.autoRetryInterval * backoffMultiplier;
1121
- } else {
1122
- entry.nextRetryAt = null;
1349
+ function cleanEmptyQueues(ctx) {
1350
+ const registered = Array.from(ctx.queueNamesCache);
1351
+ const completedCounts = collectCompletedCounts(ctx, registered);
1352
+ const bufferedCompletedQueues = collectBufferedCompletedQueues(ctx);
1353
+ const occupied = collectOccupiedQueues(ctx);
1354
+ const configured = collectConfiguredQueues(ctx);
1355
+ const removed = [];
1356
+ for (const queueName of registered) {
1357
+ const shard = ctx.shards[shardIndex(queueName)];
1358
+ const hasLiveRuntime = (shard.queues.get(queueName)?.size ?? 0) > 0 || shard.getDlqCount(queueName) > 0 || occupied.has(queueName);
1359
+ if (hasLiveRuntime)
1360
+ continue;
1361
+ shard.queues.delete(queueName);
1362
+ shard.clearEmptyGroupRuntime(queueName);
1363
+ if ((completedCounts.get(queueName) ?? 0) > 0 || bufferedCompletedQueues.has(queueName) || configured.has(queueName)) {
1364
+ continue;
1365
+ }
1366
+ ctx.storage?.deleteQueueState(queueName);
1367
+ ctx.storage?.deleteGroupState(queueName);
1368
+ ctx.unregisterQueueName(queueName);
1369
+ shard.obliterate(queueName);
1370
+ removed.push(queueName);
1123
1371
  }
1372
+ for (const shard of ctx.shards)
1373
+ shard.cleanOrphanedTemporalEntries();
1374
+ for (const queue of removed)
1375
+ ctx.dashboardEmit?.("queue:removed", { queue });
1124
1376
  }
1125
1377
 
1126
- // ../../src/domain/types/stall.ts
1127
- var DEFAULT_STALL_CONFIG = {
1128
- enabled: true,
1129
- stallInterval: 30000,
1130
- maxStalls: 3,
1131
- gracePeriod: 5000
1132
- };
1133
- function checkStall(job, config = DEFAULT_STALL_CONFIG, now = Date.now()) {
1134
- if (job.startedAt === null) {
1135
- return {
1136
- isStalled: false,
1137
- stalledFor: 0,
1138
- shouldMoveToDlq: false,
1139
- newStallCount: job.stallCount
1140
- };
1378
+ // ../../src/application/cleanupTasks.ts
1379
+ async function cleanup(ctx) {
1380
+ const now = Date.now();
1381
+ const retention = ctx.config.completedRetentionMs;
1382
+ if (retention !== null && retention !== undefined) {
1383
+ const removed = cleanCompletedJobs(null, now - retention, 1000, ctx);
1384
+ if (removed.length > 0) {
1385
+ ctx.dashboardEmit?.("cleanup:completed-removed", { count: removed.length });
1386
+ }
1141
1387
  }
1142
- if (now - job.startedAt < config.gracePeriod) {
1143
- return {
1144
- isStalled: false,
1145
- stalledFor: 0,
1146
- shouldMoveToDlq: false,
1147
- newStallCount: job.stallCount
1148
- };
1388
+ for (let i = 0;i < SHARD_COUNT; i++) {
1389
+ ctx.shards[i].refreshDelayedCount(now);
1149
1390
  }
1150
- const stallInterval = job.stallTimeout ?? config.stallInterval;
1151
- const stalledFor = now - job.lastHeartbeat;
1152
- const isStalled = stalledFor > stallInterval;
1153
- if (!isStalled) {
1154
- return {
1155
- isStalled: false,
1156
- stalledFor,
1157
- shouldMoveToDlq: false,
1158
- newStallCount: job.stallCount
1159
- };
1391
+ for (let i = 0;i < SHARD_COUNT; i++) {
1392
+ for (const q of ctx.shards[i].queues.values()) {
1393
+ if (q.needsCompaction(0.2)) {
1394
+ q.compact();
1395
+ }
1396
+ }
1160
1397
  }
1161
- const newStallCount = job.stallCount + 1;
1162
- const shouldMoveToDlq = newStallCount >= config.maxStalls;
1163
- return {
1164
- isStalled: true,
1165
- stalledFor,
1166
- shouldMoveToDlq,
1167
- newStallCount
1168
- };
1398
+ await recoverOrphanedProcessingEntries(ctx, now);
1399
+ await cleanStaleWaitingDependencies(ctx, now);
1400
+ cleanUniqueKeysAndGroups(ctx, now);
1401
+ cleanStalledCandidates(ctx);
1402
+ await cleanOrphanedJobIndex(ctx);
1403
+ cleanOrphanedJobLocks(ctx);
1404
+ pruneEndedClientDeliveries(ctx);
1405
+ cleanEmptyQueues(ctx);
1169
1406
  }
1170
- function getStallAction(job, config = DEFAULT_STALL_CONFIG, now = Date.now()) {
1171
- const result = checkStall(job, config, now);
1172
- if (!result.isStalled) {
1173
- return "keep" /* Keep */;
1174
- }
1175
- if (result.shouldMoveToDlq) {
1176
- return "move_to_dlq" /* MoveToDlq */;
1407
+ async function cleanStaleWaitingDependencies(ctx, now) {
1408
+ const depTimeout = 60 * 60 * 1000;
1409
+ for (let i = 0;i < SHARD_COUNT; i++) {
1410
+ const shard = ctx.shards[i];
1411
+ const staleIds = [];
1412
+ for (const [_id, job] of shard.waitingDeps) {
1413
+ if (now - job.createdAt > depTimeout) {
1414
+ staleIds.push(job.id);
1415
+ }
1416
+ }
1417
+ if (staleIds.length === 0)
1418
+ continue;
1419
+ const removed = await withWriteLock(ctx.shardLocks[i], () => {
1420
+ let count = 0;
1421
+ const released = [];
1422
+ for (const id of staleIds) {
1423
+ const job = shard.waitingDeps.get(id);
1424
+ if (!job || now - job.createdAt <= depTimeout)
1425
+ continue;
1426
+ ctx.storage?.deleteJob(job.id);
1427
+ shard.waitingDeps.delete(job.id);
1428
+ shard.unregisterDependencies(job.id, job.dependsOn);
1429
+ released.push(...job.dependsOn);
1430
+ if (job.uniqueKey && shard.getUniqueKeyEntry(job.queue, job.uniqueKey)?.jobId === job.id) {
1431
+ shard.releaseUniqueKeyIfOwned(job.queue, job.uniqueKey, job.id);
1432
+ }
1433
+ if (job.customId && ctx.customIdMap.get(job.customId) === job.id) {
1434
+ ctx.customIdMap.delete(job.customId);
1435
+ }
1436
+ ctx.dependencyResults?.releaseConsumer(job.id);
1437
+ ctx.jobIndex.delete(job.id);
1438
+ count++;
1439
+ }
1440
+ return { count, released };
1441
+ });
1442
+ releaseDependencyCompletionPins(removed.released, ctx);
1443
+ if (removed.count > 0) {
1444
+ ctx.dashboardEmit?.("cleanup:stale-deps-removed", { count: removed.count });
1445
+ }
1177
1446
  }
1178
- return "retry" /* Retry */;
1179
- }
1180
- function incrementStallCount(job) {
1181
- job.stallCount++;
1182
- return job.stallCount;
1183
1447
  }
1184
-
1185
- // ../../src/domain/types/queue.ts
1186
- function createQueueState(name) {
1187
- return {
1188
- name,
1189
- paused: false,
1190
- rateLimit: null,
1191
- rateLimitDuration: null,
1192
- rateLimitExpiresAt: null,
1193
- concurrencyLimit: null,
1194
- activeCount: 0
1195
- };
1448
+ function cleanUniqueKeysAndGroups(ctx, now) {
1449
+ for (let i = 0;i < SHARD_COUNT; i++) {
1450
+ const shard = ctx.shards[i];
1451
+ shard.cleanExpiredGroupWindows(now);
1452
+ shard.cleanExpiredUniqueKeys();
1453
+ for (const [_queueName, keys] of shard.uniqueKeys) {
1454
+ if (keys.size > 1000) {
1455
+ const toRemove = Math.floor(keys.size / 2);
1456
+ const iter = keys.keys();
1457
+ for (let j = 0;j < toRemove; j++) {
1458
+ const { value, done } = iter.next();
1459
+ if (done)
1460
+ break;
1461
+ keys.delete(value);
1462
+ }
1463
+ }
1464
+ }
1465
+ }
1196
1466
  }
1197
-
1198
- class RateLimiter {
1199
- capacity;
1200
- refillRate;
1201
- tokens;
1202
- lastRefill;
1203
- constructor(capacity, refillRate = capacity) {
1204
- this.capacity = capacity;
1205
- this.refillRate = refillRate;
1206
- this.tokens = capacity;
1207
- this.lastRefill = Date.now();
1467
+ function cleanStalledCandidates(ctx) {
1468
+ for (const jobId of ctx.stalledCandidates) {
1469
+ const loc = ctx.jobIndex.get(jobId);
1470
+ if (loc?.type !== "processing") {
1471
+ ctx.stalledCandidates.delete(jobId);
1472
+ }
1208
1473
  }
1209
- tryAcquire() {
1210
- this.refill();
1211
- if (this.tokens >= 1) {
1212
- this.tokens -= 1;
1213
- return true;
1474
+ }
1475
+ async function cleanOrphanedJobIndex(ctx) {
1476
+ if (ctx.jobIndex.size <= 1e5)
1477
+ return;
1478
+ const processingCandidates = new Map;
1479
+ const queueCandidates = new Map;
1480
+ for (const [jobId, loc] of ctx.jobIndex) {
1481
+ if (loc.type === "processing") {
1482
+ const procIdx = processingShardIndex(jobId);
1483
+ let list = processingCandidates.get(procIdx);
1484
+ if (!list) {
1485
+ list = [];
1486
+ processingCandidates.set(procIdx, list);
1487
+ }
1488
+ list.push(jobId);
1489
+ } else if (loc.type === "queue") {
1490
+ let list = queueCandidates.get(loc.shardIdx);
1491
+ if (!list) {
1492
+ list = [];
1493
+ queueCandidates.set(loc.shardIdx, list);
1494
+ }
1495
+ list.push({ jobId, queueName: loc.queueName });
1214
1496
  }
1215
- return false;
1216
1497
  }
1217
- refill() {
1218
- const now = Date.now();
1219
- const elapsed = (now - this.lastRefill) / 1000;
1220
- const tokensToAdd = elapsed * this.refillRate;
1221
- this.tokens = Math.min(this.capacity, this.tokens + tokensToAdd);
1222
- this.lastRefill = now;
1498
+ for (const [procIdx, candidates] of processingCandidates) {
1499
+ await withWriteLock(ctx.processingLocks[procIdx], () => {
1500
+ for (const jobId of candidates) {
1501
+ if (!ctx.processingShards[procIdx].has(jobId)) {
1502
+ ctx.jobIndex.delete(jobId);
1503
+ }
1504
+ }
1505
+ });
1223
1506
  }
1224
- getTokens() {
1225
- this.refill();
1226
- return this.tokens;
1507
+ for (const [shardIdx, candidates] of queueCandidates) {
1508
+ await withWriteLock(ctx.shardLocks[shardIdx], () => {
1509
+ const shard = ctx.shards[shardIdx];
1510
+ for (const { jobId, queueName } of candidates) {
1511
+ if (!shard.getQueue(queueName).has(jobId)) {
1512
+ ctx.jobIndex.delete(jobId);
1513
+ }
1514
+ }
1515
+ });
1227
1516
  }
1228
- getTtl(maxJobs) {
1229
- const tokens = this.getTokens();
1230
- const consumed = this.capacity - tokens;
1231
- if (maxJobs !== undefined && consumed < maxJobs)
1232
- return 0;
1233
- if (tokens >= 1 || this.refillRate <= 0)
1234
- return 0;
1235
- return Math.ceil((1 - tokens) / this.refillRate * 1000);
1517
+ }
1518
+ function cleanOrphanedJobLocks(ctx) {
1519
+ for (const jobId of ctx.jobLocks.keys()) {
1520
+ const loc = ctx.jobIndex.get(jobId);
1521
+ if (loc?.type !== "processing") {
1522
+ ctx.jobLocks.delete(jobId);
1523
+ }
1236
1524
  }
1237
1525
  }
1238
1526
 
1239
- class ConcurrencyLimiter {
1240
- limit;
1241
- active = 0;
1242
- constructor(limit) {
1243
- this.limit = limit;
1527
+ // ../../src/application/dependencyProcessor.ts
1528
+ function dependencyReadyState(job, now) {
1529
+ if (job.runAt > now)
1530
+ return "delayed";
1531
+ return job.priority > 0 ? "prioritized" : "waiting";
1532
+ }
1533
+ function checkpointDependencyPromotion(job, state, now, storage) {
1534
+ if (job.timeline.at(-1)?.state !== state && job.timeline.length < MAX_TIMELINE_ENTRIES) {
1535
+ job.timeline.push({ state, timestamp: now });
1244
1536
  }
1245
- tryAcquire() {
1246
- if (this.active < this.limit) {
1247
- this.active += 1;
1248
- return true;
1537
+ storage?.updateFlowParentResolution(job, state);
1538
+ }
1539
+ async function processPendingDependencies(ctx) {
1540
+ if (ctx.pendingDepChecks.size === 0)
1541
+ return;
1542
+ const completedIds = Array.from(ctx.pendingDepChecks);
1543
+ const completedNow = new Set(completedIds);
1544
+ ctx.pendingDepChecks.clear();
1545
+ const jobsToCheckByShard = new Map;
1546
+ const releasedDependencyIds = new Set;
1547
+ for (const completedId of completedIds) {
1548
+ for (let i = 0;i < SHARD_COUNT; i++) {
1549
+ const waitingJobIds = ctx.shards[i].getJobsWaitingFor(completedId);
1550
+ if (waitingJobIds && waitingJobIds.size > 0) {
1551
+ let shardJobs = jobsToCheckByShard.get(i);
1552
+ if (!shardJobs) {
1553
+ shardJobs = new Set;
1554
+ jobsToCheckByShard.set(i, shardJobs);
1555
+ }
1556
+ for (const jobId of waitingJobIds) {
1557
+ shardJobs.add(jobId);
1558
+ }
1559
+ }
1249
1560
  }
1250
- return false;
1251
1561
  }
1252
- release() {
1253
- if (this.active > 0) {
1254
- this.active -= 1;
1562
+ await Promise.all(Array.from(jobsToCheckByShard.entries()).map(async ([i, jobIdsToCheck]) => {
1563
+ await withWriteLock(ctx.shardLocks[i], () => {
1564
+ const shard = ctx.shards[i];
1565
+ const jobsToPromote = [];
1566
+ for (const jobId of jobIdsToCheck) {
1567
+ const job = shard.waitingDeps.get(jobId);
1568
+ if (job?.dependsOn.every((dep) => completedNow.has(dep) || ctx.completedJobs.has(dep) || (ctx.depCompletions?.has(dep) ?? false))) {
1569
+ jobsToPromote.push(job);
1570
+ }
1571
+ }
1572
+ if (jobsToPromote.length > 0) {
1573
+ promoteJobsToQueue(jobsToPromote, shard, ctx, i, releasedDependencyIds);
1574
+ }
1575
+ });
1576
+ }));
1577
+ releaseDependencyCompletionPins(releasedDependencyIds, ctx);
1578
+ }
1579
+ function promoteJobsToQueue(jobsToPromote, shard, ctx, shardIdx, releasedDependencyIds) {
1580
+ const now = Date.now();
1581
+ for (const job of jobsToPromote) {
1582
+ if (shard.waitingDeps.has(job.id)) {
1583
+ const state = dependencyReadyState(job, now);
1584
+ checkpointDependencyPromotion(job, state, now, ctx.storage);
1585
+ shard.waitingDeps.delete(job.id);
1586
+ shard.unregisterDependencies(job.id, job.dependsOn);
1587
+ for (const dependencyId of job.dependsOn)
1588
+ releasedDependencyIds.add(dependencyId);
1589
+ shard.getQueue(job.queue).push(job);
1590
+ const isDelayed = state === "delayed";
1591
+ shard.incrementQueued(job.id, isDelayed, job.createdAt, job.queue, job.runAt);
1592
+ ctx.jobIndex.set(job.id, { type: "queue", shardIdx, queueName: job.queue });
1255
1593
  }
1256
1594
  }
1257
- getActive() {
1258
- return this.active;
1259
- }
1260
- getLimit() {
1261
- return this.limit;
1262
- }
1263
- setLimit(limit) {
1264
- this.limit = limit;
1595
+ if (jobsToPromote.length > 0) {
1596
+ const promotedByQueue = new Map;
1597
+ for (const job of jobsToPromote) {
1598
+ promotedByQueue.set(job.queue, (promotedByQueue.get(job.queue) ?? 0) + 1);
1599
+ }
1600
+ for (const [queue, count] of promotedByQueue)
1601
+ shard.notifyBatch(queue, count);
1602
+ for (const job of jobsToPromote) {
1603
+ ctx.dashboardEmit?.("job:dependencies-resolved", { jobId: String(job.id), queue: job.queue });
1604
+ }
1265
1605
  }
1266
1606
  }
1267
1607
 
@@ -1273,16 +1613,10 @@ function createLock(jobId, owner, ctx, ttl = DEFAULT_LOCK_TTL) {
1273
1613
  const existing = ctx.jobLocks.get(jobId);
1274
1614
  if (existing) {
1275
1615
  const job = ctx.processingShards[loc.shardIdx].get(jobId);
1276
- const isNewGeneration = job?.startedAt !== null && job?.startedAt !== undefined && job.startedAt > existing.createdAt;
1277
- if (!isNewGeneration)
1616
+ if (!job || !isLeaseFromEarlierGeneration(job, existing))
1278
1617
  return null;
1279
1618
  ctx.jobLocks.delete(jobId);
1280
- for (const [clientId, jobs] of ctx.clientJobs) {
1281
- if (!jobs.delete(jobId))
1282
- continue;
1283
- if (jobs.size === 0)
1284
- ctx.clientJobs.delete(clientId);
1285
- }
1619
+ detachClientJob(jobId, ctx);
1286
1620
  }
1287
1621
  const lock = createJobLock(jobId, owner, ttl);
1288
1622
  ctx.jobLocks.set(jobId, lock);
@@ -1327,42 +1661,29 @@ function renewJobLockBatch(items, ctx) {
1327
1661
  return renewed;
1328
1662
  }
1329
1663
  function releaseLock(jobId, ctx, token) {
1330
- const lock = ctx.jobLocks.get(jobId);
1331
- if (!lock)
1332
- return true;
1333
- if (token && lock.token !== token) {
1334
- return false;
1335
- }
1336
- ctx.jobLocks.delete(jobId);
1337
- return true;
1338
- }
1339
- function getLockInfo(jobId, ctx) {
1340
- return ctx.jobLocks.get(jobId) ?? null;
1341
- }
1342
- // ../../src/application/clientTracking.ts
1343
- function registerClientJob(clientId, jobId, ctx) {
1344
- let jobs = ctx.clientJobs.get(clientId);
1345
- if (!jobs) {
1346
- jobs = new Set;
1347
- ctx.clientJobs.set(clientId, jobs);
1348
- }
1349
- jobs.add(jobId);
1350
- }
1351
- function unregisterClientJob(clientId, jobId, ctx) {
1352
- if (!clientId)
1353
- return;
1354
- const jobs = ctx.clientJobs.get(clientId);
1355
- if (jobs) {
1356
- jobs.delete(jobId);
1357
- if (jobs.size === 0) {
1358
- ctx.clientJobs.delete(clientId);
1359
- }
1664
+ const lock = ctx.jobLocks.get(jobId);
1665
+ if (!lock)
1666
+ return true;
1667
+ if (token && lock.token !== token) {
1668
+ return false;
1360
1669
  }
1670
+ ctx.jobLocks.delete(jobId);
1671
+ return true;
1672
+ }
1673
+ function getLockInfo(jobId, ctx) {
1674
+ return ctx.jobLocks.get(jobId) ?? null;
1675
+ }
1676
+ // ../../src/application/clientTracking.ts
1677
+ function isReleasable(clientId, jobId, ctx) {
1678
+ if (!ownsCurrentDelivery(clientId, jobId, ctx))
1679
+ return false;
1680
+ const lock = ctx.jobLocks.get(jobId);
1681
+ return !(lock && lock.renewalCount > 0);
1361
1682
  }
1362
1683
  async function releaseClientJobs(clientId, ctx) {
1363
1684
  const jobs = ctx.clientJobs.get(clientId);
1364
1685
  if (!jobs || jobs.size === 0) {
1365
- ctx.clientJobs.delete(clientId);
1686
+ dropClient(clientId, ctx);
1366
1687
  return 0;
1367
1688
  }
1368
1689
  const jobsToRelease = [];
@@ -1370,8 +1691,7 @@ async function releaseClientJobs(clientId, ctx) {
1370
1691
  const loc = ctx.jobIndex.get(jobId);
1371
1692
  if (loc?.type !== "processing")
1372
1693
  continue;
1373
- const lock = ctx.jobLocks.get(jobId);
1374
- if (lock && lock.renewalCount > 0)
1694
+ if (!isReleasable(clientId, jobId, ctx))
1375
1695
  continue;
1376
1696
  const procIdx = loc.shardIdx;
1377
1697
  const job = ctx.processingShards[procIdx].get(jobId);
@@ -1384,7 +1704,7 @@ async function releaseClientJobs(clientId, ctx) {
1384
1704
  });
1385
1705
  }
1386
1706
  if (jobsToRelease.length === 0) {
1387
- ctx.clientJobs.delete(clientId);
1707
+ dropClient(clientId, ctx);
1388
1708
  return 0;
1389
1709
  }
1390
1710
  const byProcShard = new Map;
@@ -1414,7 +1734,7 @@ async function releaseClientJobs(clientId, ctx) {
1414
1734
  await withWriteLock(ctx.processingLocks[procIdx], () => {
1415
1735
  for (const { jobId } of shardItems) {
1416
1736
  const job = ctx.processingShards[procIdx].get(jobId);
1417
- if (!job)
1737
+ if (!job || !isReleasable(clientId, jobId, ctx))
1418
1738
  continue;
1419
1739
  released += releaseJobToQueue({ jobId, job, procIdx, queueShardIdx, ctx, now });
1420
1740
  }
@@ -1424,17 +1744,19 @@ async function releaseClientJobs(clientId, ctx) {
1424
1744
  }
1425
1745
  return released;
1426
1746
  } finally {
1427
- ctx.clientJobs.delete(clientId);
1747
+ dropClient(clientId, ctx);
1428
1748
  }
1429
1749
  }
1430
1750
  function forceReleaseClientJobs(clientId, ctx) {
1431
1751
  const jobs = ctx.clientJobs.get(clientId);
1432
1752
  if (!jobs || jobs.size === 0) {
1433
- ctx.clientJobs.delete(clientId);
1753
+ dropClient(clientId, ctx);
1434
1754
  return 0;
1435
1755
  }
1436
1756
  let touched = 0;
1437
1757
  for (const jobId of jobs) {
1758
+ if (!ownsCurrentDelivery(clientId, jobId, ctx))
1759
+ continue;
1438
1760
  ctx.jobLocks.delete(jobId);
1439
1761
  const loc = ctx.jobIndex.get(jobId);
1440
1762
  if (loc?.type !== "processing")
@@ -1447,7 +1769,7 @@ function forceReleaseClientJobs(clientId, ctx) {
1447
1769
  ctx.timeoutScheduler.cancel(jobId);
1448
1770
  touched++;
1449
1771
  }
1450
- ctx.clientJobs.delete(clientId);
1772
+ dropClient(clientId, ctx);
1451
1773
  return touched;
1452
1774
  }
1453
1775
  function releaseJobToQueue(opts) {
@@ -1456,6 +1778,7 @@ function releaseJobToQueue(opts) {
1456
1778
  ctx.processingShards[procIdx].delete(jobId);
1457
1779
  ctx.timeoutScheduler.cancel(jobId);
1458
1780
  ctx.jobLocks.delete(jobId);
1781
+ detachClientJob(jobId, ctx);
1459
1782
  shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId, job.id);
1460
1783
  if (job.uniqueKey?.startsWith("cron:")) {
1461
1784
  ctx.jobIndex.delete(jobId);
@@ -1479,7 +1802,7 @@ async function checkExpiredLocks(ctx) {
1479
1802
  if (isLockExpired(lock, now)) {
1480
1803
  const procIdx = processingShardIndex(jobId);
1481
1804
  const job = ctx.processingShards[procIdx].get(jobId);
1482
- if (job) {
1805
+ if (job && !isLeaseFromEarlierGeneration(job, lock)) {
1483
1806
  const shardIdx = shardIndex(job.queue);
1484
1807
  expired.push({ jobId, lock, procIdx, shardIdx, job });
1485
1808
  } else {
@@ -1510,7 +1833,7 @@ async function checkExpiredLocks(ctx) {
1510
1833
  for (const { jobId, lock, job } of items) {
1511
1834
  const currentJob = ctx.processingShards[procIdx].get(jobId);
1512
1835
  const currentLock = ctx.jobLocks.get(jobId);
1513
- if (currentJob !== job || currentLock !== lock || !isLockExpired(currentLock, Date.now())) {
1836
+ if (currentJob !== job || currentLock !== lock || !isLockExpired(currentLock, Date.now()) || isLeaseFromEarlierGeneration(currentJob, currentLock)) {
1514
1837
  continue;
1515
1838
  }
1516
1839
  processExpiredLockInner(jobId, lock, job, shardIdx, procIdx, ctx, now);
@@ -1525,6 +1848,7 @@ function processExpiredLockInner(jobId, lock, job, shardIdx, procIdx, ctx, now)
1525
1848
  const queue = shard.getQueue(job.queue);
1526
1849
  ctx.processingShards[procIdx].delete(jobId);
1527
1850
  ctx.timeoutScheduler.cancel(jobId);
1851
+ detachClientJob(jobId, ctx);
1528
1852
  if (job.uniqueKey?.startsWith("cron:")) {
1529
1853
  ctx.retiredCronLeaseTokens.set(jobId, lock.token);
1530
1854
  shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId, job.id);
@@ -1752,239 +2076,6 @@ function checkStorageSize(ctx) {
1752
2076
  }
1753
2077
  }
1754
2078
 
1755
- // ../../src/shared/logger.ts
1756
- var LOG_LEVEL_PRIORITY = {
1757
- debug: 0,
1758
- info: 1,
1759
- warn: 2,
1760
- error: 3
1761
- };
1762
-
1763
- class Logger {
1764
- component;
1765
- static jsonMode = false;
1766
- static level = "info";
1767
- constructor(component) {
1768
- this.component = component;
1769
- }
1770
- static enableJsonMode() {
1771
- Logger.jsonMode = true;
1772
- }
1773
- static disableJsonMode() {
1774
- Logger.jsonMode = false;
1775
- }
1776
- static setLevel(level) {
1777
- Logger.level = level;
1778
- }
1779
- debug(message, data) {
1780
- this.log("debug", message, data);
1781
- }
1782
- debugToStderr(message, data) {
1783
- this.log("debug", message, data, true);
1784
- }
1785
- info(message, data) {
1786
- this.log("info", message, data);
1787
- }
1788
- infoToStderr(message, data) {
1789
- this.log("info", message, data, true);
1790
- }
1791
- warn(message, data) {
1792
- this.log("warn", message, data);
1793
- }
1794
- error(message, data) {
1795
- this.log("error", message, data);
1796
- }
1797
- log(level, message, data, forceStderr = false) {
1798
- if (LOG_LEVEL_PRIORITY[level] < LOG_LEVEL_PRIORITY[Logger.level])
1799
- return;
1800
- if (Logger.jsonMode) {
1801
- const entry = {
1802
- timestamp: new Date().toISOString(),
1803
- level,
1804
- component: this.component,
1805
- message,
1806
- ...data && { data }
1807
- };
1808
- if (forceStderr)
1809
- console.error(JSON.stringify(entry));
1810
- else
1811
- console.log(JSON.stringify(entry));
1812
- } else {
1813
- const prefix = `[${this.component}]`;
1814
- const dataStr = data ? ` ${JSON.stringify(data)}` : "";
1815
- switch (level) {
1816
- case "debug":
1817
- if (forceStderr)
1818
- console.error(`${prefix} ${message}${dataStr}`);
1819
- else
1820
- console.debug(`${prefix} ${message}${dataStr}`);
1821
- break;
1822
- case "info":
1823
- if (forceStderr)
1824
- console.error(`${prefix} ${message}${dataStr}`);
1825
- else
1826
- console.log(`${prefix} ${message}${dataStr}`);
1827
- break;
1828
- case "warn":
1829
- console.warn(`${prefix} ${message}${dataStr}`);
1830
- break;
1831
- case "error":
1832
- console.error(`${prefix} ${message}${dataStr}`);
1833
- break;
1834
- }
1835
- }
1836
- }
1837
- }
1838
- function createLogger(component) {
1839
- return new Logger(component);
1840
- }
1841
- var serverLog = createLogger("Server");
1842
- var tcpLog = createLogger("TCP");
1843
- var httpLog = createLogger("HTTP");
1844
- var wsLog = createLogger("WS");
1845
- var cronLog = createLogger("Cron");
1846
- var statsLog = createLogger("Stats");
1847
- var storageLog = createLogger("Storage");
1848
- var queueLog = createLogger("Queue");
1849
- var webhookLog = createLogger("Webhook");
1850
- var backupLog = createLogger("Backup");
1851
-
1852
- // ../../src/application/stallDetection.ts
1853
- function checkStalledJobs(ctx) {
1854
- const now = Date.now();
1855
- const confirmedStalled = [];
1856
- for (const jobId of ctx.stalledCandidates) {
1857
- const procIdx = processingShardIndex(jobId);
1858
- const job = ctx.processingShards[procIdx].get(jobId);
1859
- if (!job) {
1860
- ctx.stalledCandidates.delete(jobId);
1861
- continue;
1862
- }
1863
- const stallConfig = ctx.shards[shardIndex(job.queue)].getStallConfig(job.queue);
1864
- if (!stallConfig.enabled) {
1865
- ctx.stalledCandidates.delete(jobId);
1866
- continue;
1867
- }
1868
- const action = getStallAction(job, stallConfig, now);
1869
- if (action !== "keep" /* Keep */) {
1870
- confirmedStalled.push({ job, action });
1871
- }
1872
- ctx.stalledCandidates.delete(jobId);
1873
- }
1874
- for (let i = 0;i < SHARD_COUNT; i++) {
1875
- const procShard = ctx.processingShards[i];
1876
- for (const [jobId, job] of procShard) {
1877
- const stallConfig = ctx.shards[shardIndex(job.queue)].getStallConfig(job.queue);
1878
- if (!stallConfig.enabled)
1879
- continue;
1880
- const action = getStallAction(job, stallConfig, now);
1881
- if (action !== "keep" /* Keep */) {
1882
- ctx.stalledCandidates.add(jobId);
1883
- }
1884
- }
1885
- }
1886
- for (const { job, action } of confirmedStalled) {
1887
- handleStalledJob(job, action, ctx).catch((err) => {
1888
- queueLog.error("Failed to handle stalled job", {
1889
- jobId: String(job.id),
1890
- error: String(err)
1891
- });
1892
- });
1893
- }
1894
- }
1895
- async function handleStalledJob(job, action, ctx) {
1896
- const idx = shardIndex(job.queue);
1897
- const procIdx = processingShardIndex(job.id);
1898
- let handled = false;
1899
- let handledAction = action;
1900
- await withWriteLock(ctx.shardLocks[idx], async () => {
1901
- await withWriteLock(ctx.processingLocks[procIdx], () => {
1902
- if (!ctx.processingShards[procIdx].has(job.id)) {
1903
- return;
1904
- }
1905
- const shard = ctx.shards[idx];
1906
- const attemptsExhausted = job.attempts + 1 >= job.maxAttempts;
1907
- if (action === "move_to_dlq" /* MoveToDlq */ || attemptsExhausted) {
1908
- handledAction = "move_to_dlq" /* MoveToDlq */;
1909
- moveStalliedJobToDlq(job, ctx, shard, procIdx, attemptsExhausted);
1910
- } else {
1911
- retryStalliedJob(job, ctx, shard, procIdx, idx);
1912
- }
1913
- handled = true;
1914
- });
1915
- });
1916
- if (handled) {
1917
- ctx.dashboardEmit?.("job:stalled", {
1918
- jobId: String(job.id),
1919
- queue: job.queue,
1920
- stallCount: job.stallCount,
1921
- action: handledAction
1922
- });
1923
- ctx.eventsManager.broadcast({
1924
- eventType: "stalled" /* Stalled */,
1925
- queue: job.queue,
1926
- jobId: job.id,
1927
- timestamp: Date.now(),
1928
- data: { stallCount: job.stallCount, action: handledAction }
1929
- });
1930
- ctx.webhookManager.trigger("stalled", String(job.id), job.queue, {
1931
- data: { stallCount: job.stallCount, action: handledAction }
1932
- });
1933
- }
1934
- }
1935
- function moveStalliedJobToDlq(job, ctx, shard, procIdx, attemptsExhausted) {
1936
- ctx.processingShards[procIdx].delete(job.id);
1937
- ctx.timeoutScheduler.cancel(job.id);
1938
- shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId, job.id);
1939
- if (job.uniqueKey?.startsWith("cron:")) {
1940
- ctx.jobIndex.delete(job.id);
1941
- ctx.storage?.deleteJob(job.id);
1942
- return;
1943
- }
1944
- incrementStallCount(job);
1945
- job.attempts++;
1946
- job.startedAt = null;
1947
- job.lastHeartbeat = Date.now();
1948
- const entry = shard.addToDlq(job, attemptsExhausted ? "max_attempts_exceeded" /* MaxAttemptsExceeded */ : "stalled" /* Stalled */, attemptsExhausted ? `Job stalled at max attempts (${job.maxAttempts})` : `Job stalled ${job.stallCount} times`);
1949
- ctx.jobIndex.set(job.id, { type: "dlq", queueName: job.queue });
1950
- ctx.storage?.saveDlqEntry(entry);
1951
- ctx.storage?.deleteJob(job.id);
1952
- }
1953
- function retryStalliedJob(job, ctx, shard, procIdx, idx) {
1954
- if (job.uniqueKey?.startsWith("cron:")) {
1955
- ctx.processingShards[procIdx].delete(job.id);
1956
- ctx.timeoutScheduler.cancel(job.id);
1957
- shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId, job.id);
1958
- ctx.jobIndex.delete(job.id);
1959
- ctx.storage?.deleteJob(job.id);
1960
- return;
1961
- }
1962
- incrementStallCount(job);
1963
- job.attempts++;
1964
- const stallNow = Date.now();
1965
- recordJobFailureAttempt(job, "stalled" /* Stalled */, "stalled", stallNow);
1966
- job.startedAt = null;
1967
- job.runAt = stallNow + calculateBackoff(job);
1968
- job.lastHeartbeat = stallNow;
1969
- if (job.timeline.length < MAX_TIMELINE_ENTRIES) {
1970
- job.timeline.push({
1971
- state: "failed",
1972
- timestamp: stallNow,
1973
- error: "stalled",
1974
- attempt: job.attempts
1975
- });
1976
- job.timeline.push({ state: "waiting", timestamp: stallNow, attempt: job.attempts + 1 });
1977
- }
1978
- ctx.processingShards[procIdx].delete(job.id);
1979
- ctx.timeoutScheduler.cancel(job.id);
1980
- shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId, job.id);
1981
- shard.getQueue(job.queue).push(job);
1982
- const isDelayed = job.runAt > Date.now();
1983
- shard.incrementQueued(job.id, isDelayed, job.createdAt, job.queue, job.runAt);
1984
- ctx.jobIndex.set(job.id, { type: "queue", shardIdx: idx, queueName: job.queue });
1985
- ctx.storage?.updateForRetry(job);
1986
- }
1987
-
1988
2079
  // ../../src/application/taskErrorTracking.ts
1989
2080
  var MAX_CONSECUTIVE_FAILURES = 5;
1990
2081
  var taskErrors = {
@@ -2407,6 +2498,7 @@ function getLockContext(ctx) {
2407
2498
  jobLocks: ctx.jobLocks,
2408
2499
  retiredCronLeaseTokens: ctx.retiredCronLeaseTokens,
2409
2500
  clientJobs: ctx.clientJobs,
2501
+ clientJobOwners: ctx.clientJobOwners,
2410
2502
  processingShards: ctx.processingShards,
2411
2503
  processingLocks: ctx.processingLocks,
2412
2504
  shards: ctx.shards,
@@ -3665,7 +3757,7 @@ var latencyTracker = new LatencyTracker;
3665
3757
  // ../../package.json
3666
3758
  var package_default = {
3667
3759
  name: "bunqueue",
3668
- version: "2.9.6",
3760
+ version: "2.9.7",
3669
3761
  description: "High-performance job queue for Bun and AI agents. Memory or one-file SQLite, optional PostgreSQL 15\u201318 multi-broker persistence, cron, retries, DLQ, and MCP.",
3670
3762
  keywords: [
3671
3763
  "agentic",
@@ -4279,12 +4371,7 @@ function isStorageDegraded(status) {
4279
4371
  // ../../src/application/operations/jobClaim.ts
4280
4372
  function releaseClaimedJobOwnership(jobId, ctx) {
4281
4373
  ctx.jobLocks.delete(jobId);
4282
- for (const [clientId, jobs] of ctx.clientJobs) {
4283
- if (!jobs.delete(jobId))
4284
- continue;
4285
- if (jobs.size === 0)
4286
- ctx.clientJobs.delete(clientId);
4287
- }
4374
+ detachClientJob(jobId, ctx);
4288
4375
  }
4289
4376
 
4290
4377
  // ../../src/application/operations/jobMoveOperations.ts
@@ -7042,9 +7129,10 @@ function validateOptions(input) {
7042
7129
  if (input.backoff.type !== "fixed" && input.backoff.type !== "exponential") {
7043
7130
  throw new Error("backoff.type must be 'fixed' or 'exponential'");
7044
7131
  }
7045
- numeric(input.backoff.delay, "backoff.delay", 0, 24 * 60 * 60 * 1000);
7132
+ numeric(input.backoff.delay, "backoff.delay", 0, MAX_BACKOFF_DELAY);
7133
+ numeric(input.backoff.maxDelay, "backoff.maxDelay", 0, MAX_BACKOFF_DELAY);
7046
7134
  } else {
7047
- numeric(input.backoff, "backoff", 0, 24 * 60 * 60 * 1000);
7135
+ numeric(input.backoff, "backoff", 0, MAX_BACKOFF_DELAY);
7048
7136
  }
7049
7137
  for (const name of BOOLEAN_OPTIONS) {
7050
7138
  const value = input[name];
@@ -13832,6 +13920,7 @@ class ContextFactory {
13832
13920
  jobLocks: this.deps.jobLocks,
13833
13921
  retiredCronLeaseTokens: this.deps.retiredCronLeaseTokens,
13834
13922
  clientJobs: this.deps.clientJobs,
13923
+ clientJobOwners: this.deps.clientJobOwners,
13835
13924
  processingShards: this.deps.processingShards,
13836
13925
  processingLocks: this.deps.processingLocks,
13837
13926
  shards: this.deps.shards,
@@ -13865,6 +13954,7 @@ class ContextFactory {
13865
13954
  jobLocks: this.deps.jobLocks,
13866
13955
  retiredCronLeaseTokens: this.deps.retiredCronLeaseTokens,
13867
13956
  clientJobs: this.deps.clientJobs,
13957
+ clientJobOwners: this.deps.clientJobOwners,
13868
13958
  stalledCandidates: this.deps.stalledCandidates,
13869
13959
  pendingDepChecks: this.deps.pendingDepChecks,
13870
13960
  pendingQueueAdmissions: this.deps.pendingQueueAdmissions,
@@ -13988,6 +14078,7 @@ class ContextFactory {
13988
14078
  jobIndex: this.deps.jobIndex,
13989
14079
  jobLocks: this.deps.jobLocks,
13990
14080
  clientJobs: this.deps.clientJobs,
14081
+ clientJobOwners: this.deps.clientJobOwners,
13991
14082
  dependencyResults: this.deps.dependencyResults,
13992
14083
  customIdMap: this.deps.customIdMap,
13993
14084
  depCompletions: this.deps.depCompletions,
@@ -14434,6 +14525,7 @@ function createContextDependencies(state) {
14434
14525
  jobLogQueues: state.jobLogQueues,
14435
14526
  jobLocks: state.jobLocks,
14436
14527
  clientJobs: state.clientJobs,
14528
+ clientJobOwners: state.clientJobOwners,
14437
14529
  stalledCandidates: state.stalledCandidates,
14438
14530
  pendingDepChecks: state.pendingDepChecks,
14439
14531
  pendingQueueAdmissions: state.pendingQueueAdmissions,
@@ -14502,6 +14594,7 @@ class QueueManagerState {
14502
14594
  depFlushRunning = false;
14503
14595
  jobLocks = new Map;
14504
14596
  clientJobs = new Map;
14597
+ clientJobOwners = new Map;
14505
14598
  repeatChain = new Map;
14506
14599
  failedChildrenValues = new Map;
14507
14600
  ignoredChildrenFailures = new Map;
@@ -14688,7 +14781,7 @@ class QueueManagerDelivery extends QueueManagerState {
14688
14781
  if (location?.type !== "processing")
14689
14782
  return;
14690
14783
  const job = this.processingShards[location.shardIdx].get(jobId);
14691
- if (!job || job.startedAt !== null && job.startedAt > lock.createdAt) {
14784
+ if (!job || isLeaseFromEarlierGeneration(job, lock)) {
14692
14785
  throw new Error(`Invalid or expired lock token for job ${jobId}`);
14693
14786
  }
14694
14787
  }
@@ -14700,7 +14793,7 @@ class QueueManagerDelivery extends QueueManagerState {
14700
14793
  if (lock?.token !== token)
14701
14794
  return false;
14702
14795
  const job = this.processingShards[location.shardIdx].get(jobId);
14703
- return !(job && job.startedAt !== null && job.startedAt > lock.createdAt);
14796
+ return !(job && isLeaseFromEarlierGeneration(job, lock));
14704
14797
  }
14705
14798
  isStallRetried(jobId) {
14706
14799
  const location = this.jobIndex.get(jobId);
@@ -14942,11 +15035,7 @@ class QueueManagerAck extends QueueManagerDelivery {
14942
15035
  if (lock)
14943
15036
  this.retiredTimeoutLeaseTokens.set(lock.token, generation);
14944
15037
  releaseLock(job.id, this.contextFactory.getLockContext(), lock?.token);
14945
- for (const [clientId, jobs] of this.clientJobs) {
14946
- jobs.delete(job.id);
14947
- if (jobs.size === 0)
14948
- this.clientJobs.delete(clientId);
14949
- }
15038
+ detachClientJob(job.id, { clientJobs: this.clientJobs, clientJobOwners: this.clientJobOwners });
14950
15039
  }
14951
15040
  async failWithOptions(jobId, error, token, options, enforceLease = true) {
14952
15041
  try {
@@ -15228,11 +15317,13 @@ class QueueManagerControl extends QueueManagerQueries {
15228
15317
  toDrop.add(jobId);
15229
15318
  }
15230
15319
  }
15320
+ const ownership = { clientJobs: this.clientJobs, clientJobOwners: this.clientJobOwners };
15231
15321
  for (const jobId of toDrop) {
15232
15322
  const location = this.jobIndex.get(jobId);
15233
15323
  if (location?.type === "processing") {
15234
15324
  this.processingShards[location.shardIdx]?.delete(jobId);
15235
15325
  }
15326
+ detachClientJob(jobId, ownership);
15236
15327
  this.timeoutScheduler.cancel(jobId);
15237
15328
  this.jobIndex.delete(jobId);
15238
15329
  this.completedJobs.delete(jobId);
@@ -16344,6 +16435,7 @@ class QueueManagerLifecycle extends QueueManagerDependencies {
16344
16435
  this.jobLocks.clear();
16345
16436
  this.stalledCandidates.clear();
16346
16437
  this.clientJobs.clear();
16438
+ this.clientJobOwners.clear();
16347
16439
  this.repeatChain.clear();
16348
16440
  this.failedChildrenValues.clear();
16349
16441
  this.ignoredChildrenFailures.clear();