bunqueue 2.8.53 → 2.8.56
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/application/background/dlq.d.ts +2 -0
- package/dist/application/background/dlq.js +26 -0
- package/dist/application/background/lifecycle.d.ts +4 -0
- package/dist/application/background/lifecycle.js +90 -0
- package/dist/application/background/recovery/active.d.ts +4 -0
- package/dist/application/background/recovery/active.js +69 -0
- package/dist/application/background/recovery/index.d.ts +2 -0
- package/dist/application/background/recovery/index.js +22 -0
- package/dist/application/background/recovery/pending.d.ts +3 -0
- package/dist/application/background/recovery/pending.js +69 -0
- package/dist/application/background/recovery/restore.d.ts +5 -0
- package/dist/application/background/recovery/restore.js +58 -0
- package/dist/application/background/recovery/shared.d.ts +6 -0
- package/dist/application/background/recovery/shared.js +15 -0
- package/dist/application/background/timeouts.d.ts +25 -0
- package/dist/application/background/timeouts.js +191 -0
- package/dist/application/backgroundTasks.d.ts +7 -31
- package/dist/application/backgroundTasks.js +6 -438
- package/dist/application/cleanupTasks.js +2 -1
- package/dist/application/clientTracking.js +3 -1
- package/dist/application/contextFactory.d.ts +13 -90
- package/dist/application/contextFactory.js +9 -0
- package/dist/application/dependencyResultTracker.d.ts +1 -0
- package/dist/application/dependencyResultTracker.js +3 -0
- package/dist/application/dlqManager.d.ts +9 -9
- package/dist/application/dlqManager.js +49 -167
- package/dist/application/dlqRetry.d.ts +12 -0
- package/dist/application/dlqRetry.js +97 -0
- package/dist/application/lockManager.js +24 -4
- package/dist/application/lockOperations.js +16 -3
- package/dist/application/operations/ack/batch.d.ts +8 -0
- package/dist/application/operations/ack/batch.js +64 -0
- package/dist/application/operations/ack/completion.d.ts +3 -0
- package/dist/application/operations/ack/completion.js +74 -0
- package/dist/application/operations/ack/failure.d.ts +3 -0
- package/dist/application/operations/ack/failure.js +160 -0
- package/dist/application/operations/ack.d.ts +5 -79
- package/dist/application/operations/ack.js +4 -319
- package/dist/application/operations/ackHelpers.d.ts +3 -52
- package/dist/application/operations/ackHelpers.js +11 -4
- package/dist/application/operations/customId.d.ts +18 -5
- package/dist/application/operations/customId.js +36 -25
- package/dist/application/operations/flowValidation.js +9 -4
- package/dist/application/operations/jobManagement.js +50 -11
- package/dist/application/operations/jobMoveOperations.js +3 -3
- package/dist/application/operations/jobStateTransitions.d.ts +1 -1
- package/dist/application/operations/jobStateTransitions.js +16 -2
- package/dist/application/operations/parentLink.d.ts +36 -0
- package/dist/application/operations/parentLink.js +153 -0
- package/dist/application/operations/parentLinkInput.d.ts +8 -0
- package/dist/application/operations/parentLinkInput.js +23 -0
- package/dist/application/operations/push.d.ts +5 -44
- package/dist/application/operations/push.js +62 -152
- package/dist/application/operations/pushAdmission.d.ts +32 -0
- package/dist/application/operations/pushAdmission.js +38 -0
- package/dist/application/operations/pushBatch.d.ts +4 -0
- package/dist/application/operations/pushBatch.js +136 -0
- package/dist/application/operations/pushContext.d.ts +39 -0
- package/dist/application/operations/pushContext.js +1 -0
- package/dist/application/operations/pushDeduplication.d.ts +42 -0
- package/dist/application/operations/pushDeduplication.js +131 -0
- package/dist/application/operations/pushInsert.d.ts +13 -1
- package/dist/application/operations/pushInsert.js +33 -17
- package/dist/application/operations/pushLocks.js +13 -9
- package/dist/application/operations/query/collect.d.ts +12 -0
- package/dist/application/operations/query/collect.js +118 -0
- package/dist/application/operations/query/jobLookup.d.ts +9 -0
- package/dist/application/operations/query/jobLookup.js +110 -0
- package/dist/application/operations/query/pagination.d.ts +8 -0
- package/dist/application/operations/query/pagination.js +90 -0
- package/dist/application/operations/query/state.d.ts +4 -0
- package/dist/application/operations/query/state.js +63 -0
- package/dist/application/operations/queryOperations.d.ts +5 -50
- package/dist/application/operations/queryOperations.js +4 -474
- package/dist/application/queue-manager/ack.d.ts +23 -0
- package/dist/application/queue-manager/ack.js +242 -0
- package/dist/application/queue-manager/configuration.d.ts +27 -0
- package/dist/application/queue-manager/configuration.js +84 -0
- package/dist/application/queue-manager/context.d.ts +5 -0
- package/dist/application/queue-manager/context.js +61 -0
- package/dist/application/queue-manager/control.d.ts +17 -0
- package/dist/application/queue-manager/control.js +133 -0
- package/dist/application/queue-manager/delivery.d.ts +32 -0
- package/dist/application/queue-manager/delivery.js +143 -0
- package/dist/application/queue-manager/dependencies.d.ts +8 -0
- package/dist/application/queue-manager/dependencies.js +127 -0
- package/dist/application/queue-manager/dependency-runtime.d.ts +8 -0
- package/dist/application/queue-manager/dependency-runtime.js +40 -0
- package/dist/application/queue-manager/flow-failures.d.ts +18 -0
- package/dist/application/queue-manager/flow-failures.js +107 -0
- package/dist/application/queue-manager/flow-options.d.ts +8 -0
- package/dist/application/queue-manager/flow-options.js +129 -0
- package/dist/application/queue-manager/job-management.d.ts +17 -0
- package/dist/application/queue-manager/job-management.js +100 -0
- package/dist/application/queue-manager/lifecycle.d.ts +4 -0
- package/dist/application/queue-manager/lifecycle.js +41 -0
- package/dist/application/queue-manager/limits.d.ts +41 -0
- package/dist/application/queue-manager/limits.js +108 -0
- package/dist/application/queue-manager/locks.d.ts +21 -0
- package/dist/application/queue-manager/locks.js +59 -0
- package/dist/application/queue-manager/observability.d.ts +16 -0
- package/dist/application/queue-manager/observability.js +44 -0
- package/dist/application/queue-manager/queries.d.ts +17 -0
- package/dist/application/queue-manager/queries.js +118 -0
- package/dist/application/queue-manager/repeat.d.ts +4 -0
- package/dist/application/queue-manager/repeat.js +26 -0
- package/dist/application/queue-manager/services.d.ts +19 -0
- package/dist/application/queue-manager/services.js +80 -0
- package/dist/application/queue-manager/state.d.ts +92 -0
- package/dist/application/queue-manager/state.js +144 -0
- package/dist/application/queue-manager/stats.d.ts +28 -0
- package/dist/application/queue-manager/stats.js +52 -0
- package/dist/application/queueManager.d.ts +4 -380
- package/dist/application/queueManager.js +3 -1880
- package/dist/application/queueTelemetryJournal.d.ts +20 -0
- package/dist/application/queueTelemetryJournal.js +133 -0
- package/dist/application/repeatJobs.d.ts +7 -0
- package/dist/application/repeatJobs.js +132 -0
- package/dist/application/stallDetection.js +9 -4
- package/dist/application/types/ack.d.ts +132 -0
- package/dist/application/types/ack.js +1 -0
- package/dist/application/types/background.d.ts +18 -0
- package/dist/application/types/background.js +1 -0
- package/dist/application/types/config.d.ts +37 -0
- package/dist/application/types/config.js +15 -0
- package/dist/application/types/contextFactory.d.ts +83 -0
- package/dist/application/types/contextFactory.js +1 -0
- package/dist/application/types/contexts.d.ts +110 -0
- package/dist/application/types/contexts.js +1 -0
- package/dist/application/types/index.d.ts +7 -0
- package/dist/application/types/index.js +1 -0
- package/dist/application/types/query.d.ts +23 -0
- package/dist/application/types/query.js +1 -0
- package/dist/application/types/queueManager.d.ts +94 -0
- package/dist/application/types/queueManager.js +1 -0
- package/dist/cli/commandRouter.d.ts +1 -1
- package/dist/cli/commands/core.js +1 -0
- package/dist/cli/output/collections.d.ts +8 -0
- package/dist/cli/output/collections.js +131 -0
- package/dist/cli/output/jobs.d.ts +2 -0
- package/dist/cli/output/jobs.js +61 -0
- package/dist/cli/output/style.d.ts +14 -0
- package/dist/cli/output/style.js +28 -0
- package/dist/cli/output/success.d.ts +1 -0
- package/dist/cli/output/success.js +108 -0
- package/dist/cli/output.d.ts +1 -6
- package/dist/cli/output.js +7 -395
- package/dist/client/bunqueue/dlqRateLimit.d.ts +8 -0
- package/dist/client/bunqueue/dlqRateLimit.js +24 -0
- package/dist/client/bunqueue/runtime.d.ts +35 -0
- package/dist/client/bunqueue/runtime.js +138 -0
- package/dist/client/bunqueue.d.ts +19 -34
- package/dist/client/bunqueue.js +45 -166
- package/dist/client/dlqConversion.d.ts +5 -0
- package/dist/client/dlqConversion.js +28 -0
- package/dist/client/events.d.ts +7 -56
- package/dist/client/events.js +38 -15
- package/dist/client/flow.d.ts +6 -12
- package/dist/client/flow.js +16 -34
- package/dist/client/flowJobDependencies.js +3 -1
- package/dist/client/flowJobFactory.d.ts +9 -1
- package/dist/client/flowJobFactory.js +25 -9
- package/dist/client/flowJobMoveMethods.d.ts +5 -5
- package/dist/client/flowJobMoveMethods.js +37 -17
- package/dist/client/flowLegacyPlan.js +22 -8
- package/dist/client/flowOptions.d.ts +1 -1
- package/dist/client/flowOptions.js +2 -1
- package/dist/client/flowPlan.js +2 -3
- package/dist/client/flowPush.d.ts +1 -0
- package/dist/client/flowPush.js +5 -1
- package/dist/client/flowReader.js +5 -5
- package/dist/client/flowResults.d.ts +11 -0
- package/dist/client/flowResults.js +42 -0
- package/dist/client/index.d.ts +3 -1
- package/dist/client/jobConversion.d.ts +3 -7
- package/dist/client/jobConversion.js +36 -33
- package/dist/client/jobConversionHelpers.d.ts +19 -3
- package/dist/client/jobConversionHelpers.js +20 -16
- package/dist/client/jobConversionTypes.d.ts +8 -0
- package/dist/client/jobDeduplication.d.ts +6 -0
- package/dist/client/jobDeduplication.js +14 -0
- package/dist/client/jobHelpers.d.ts +7 -2
- package/dist/client/jobHelpers.js +16 -8
- package/dist/client/queue/bullmqCompat.d.ts +18 -8
- package/dist/client/queue/bullmqCompat.js +39 -71
- package/dist/client/queue/deduplication.d.ts +2 -0
- package/dist/client/queue/deduplication.js +20 -11
- package/dist/client/queue/dlq.d.ts +11 -6
- package/dist/client/queue/dlq.js +57 -16
- package/dist/client/queue/dlqJobMethods.d.ts +9 -0
- package/dist/client/queue/dlqJobMethods.js +177 -0
- package/dist/client/queue/dlqOps.d.ts +2 -1
- package/dist/client/queue/dlqOps.js +4 -10
- package/dist/client/queue/dlqStats.d.ts +3 -0
- package/dist/client/queue/dlqStats.js +21 -0
- package/dist/client/queue/job-proxy/dependencies.d.ts +9 -0
- package/dist/client/queue/job-proxy/dependencies.js +67 -0
- package/dist/client/queue/job-proxy/reflection.d.ts +2 -0
- package/dist/client/queue/job-proxy/reflection.js +18 -0
- package/dist/client/queue/job-proxy/simple.d.ts +3 -0
- package/dist/client/queue/job-proxy/simple.js +316 -0
- package/dist/client/queue/job-proxy/tcp.d.ts +4 -0
- package/dist/client/queue/job-proxy/tcp.js +191 -0
- package/dist/client/queue/jobMetadata.d.ts +10 -0
- package/dist/client/queue/jobMetadata.js +24 -0
- package/dist/client/queue/jobMove.d.ts +4 -4
- package/dist/client/queue/jobMove.js +87 -24
- package/dist/client/queue/jobProxy.d.ts +4 -50
- package/dist/client/queue/jobProxy.js +3 -511
- package/dist/client/queue/operations/add/bulk.d.ts +7 -0
- package/dist/client/queue/operations/add/bulk.js +129 -0
- package/dist/client/queue/operations/add/payload.d.ts +24 -0
- package/dist/client/queue/operations/add/payload.js +85 -0
- package/dist/client/queue/operations/add/single.d.ts +3 -0
- package/dist/client/queue/operations/add/single.js +107 -0
- package/dist/client/queue/operations/add.d.ts +2 -47
- package/dist/client/queue/operations/add.js +2 -347
- package/dist/client/queue/operations/index.d.ts +1 -0
- package/dist/client/queue/operations/index.js +1 -0
- package/dist/client/queue/operations/management.d.ts +0 -4
- package/dist/client/queue/operations/management.js +35 -6
- package/dist/client/queue/operations/query.d.ts +1 -20
- package/dist/client/queue/operations/query.js +38 -116
- package/dist/client/queue/operations/queryStates.d.ts +13 -0
- package/dist/client/queue/operations/queryStates.js +32 -0
- package/dist/client/queue/operations/queryTcpPages.d.ts +14 -0
- package/dist/client/queue/operations/queryTcpPages.js +40 -0
- package/dist/client/queue/queue.d.ts +6 -189
- package/dist/client/queue/queue.js +6 -547
- package/dist/client/queue/rateLimit.d.ts +4 -4
- package/dist/client/queue/rateLimit.js +17 -8
- package/dist/client/queue/runtime/compatibility.d.ts +13 -0
- package/dist/client/queue/runtime/compatibility.js +38 -0
- package/dist/client/queue/runtime/configuration.d.ts +42 -0
- package/dist/client/queue/runtime/configuration.js +109 -0
- package/dist/client/queue/runtime/connection.d.ts +6 -0
- package/dist/client/queue/runtime/connection.js +16 -0
- package/dist/client/queue/runtime/control.d.ts +41 -0
- package/dist/client/queue/runtime/control.js +87 -0
- package/dist/client/queue/runtime/queries.d.ts +48 -0
- package/dist/client/queue/runtime/queries.js +95 -0
- package/dist/client/queue/runtime/scheduling.d.ts +33 -0
- package/dist/client/queue/runtime/scheduling.js +56 -0
- package/dist/client/queue/runtime/state.d.ts +96 -0
- package/dist/client/queue/runtime/state.js +149 -0
- package/dist/client/queue/scheduler.d.ts +2 -5
- package/dist/client/queue/scheduler.js +25 -23
- package/dist/client/queue/schedulerPagination.d.ts +7 -0
- package/dist/client/queue/schedulerPagination.js +10 -0
- package/dist/client/queue/types/add.d.ts +40 -0
- package/dist/client/queue/types/add.js +1 -0
- package/dist/client/queue/types/contexts.d.ts +33 -0
- package/dist/client/queue/types/contexts.js +5 -0
- package/dist/client/queue/types/index.d.ts +3 -0
- package/dist/client/queue/types/index.js +1 -0
- package/dist/client/queue/types/job.d.ts +50 -0
- package/dist/client/queue/types/job.js +1 -0
- package/dist/client/queue/types/runtime.d.ts +35 -0
- package/dist/client/queue/types/runtime.js +1 -0
- package/dist/client/queue/workers.d.ts +16 -8
- package/dist/client/queue/workers.js +42 -16
- package/dist/client/queue-events/tcpSubscription.d.ts +23 -0
- package/dist/client/queue-events/tcpSubscription.js +99 -0
- package/dist/client/queueGroup.d.ts +11 -0
- package/dist/client/queueGroup.js +28 -1
- package/dist/client/sandboxed/queueOps.d.ts +2 -2
- package/dist/client/sandboxed/queueOps.js +32 -5
- package/dist/client/sandboxed/runtime/dispatch.d.ts +13 -0
- package/dist/client/sandboxed/runtime/dispatch.js +186 -0
- package/dist/client/sandboxed/runtime/lifecycle.d.ts +20 -0
- package/dist/client/sandboxed/runtime/lifecycle.js +95 -0
- package/dist/client/sandboxed/runtime/log.d.ts +1 -0
- package/dist/client/sandboxed/runtime/log.js +15 -0
- package/dist/client/sandboxed/runtime/pool.d.ts +11 -0
- package/dist/client/sandboxed/runtime/pool.js +121 -0
- package/dist/client/sandboxed/runtime/recovery.d.ts +11 -0
- package/dist/client/sandboxed/runtime/recovery.js +122 -0
- package/dist/client/sandboxed/runtime/state.d.ts +39 -0
- package/dist/client/sandboxed/runtime/state.js +59 -0
- package/dist/client/sandboxed/types/index.d.ts +2 -0
- package/dist/client/sandboxed/types/index.js +1 -0
- package/dist/client/sandboxed/types/options.d.ts +27 -0
- package/dist/client/sandboxed/types/options.js +1 -0
- package/dist/client/sandboxed/types/process.d.ts +30 -0
- package/dist/client/sandboxed/types/process.js +1 -0
- package/dist/client/sandboxed/worker.d.ts +3 -85
- package/dist/client/sandboxed/worker.js +3 -557
- package/dist/client/sandboxed/wrapper.js +1 -0
- package/dist/client/tcp/client.d.ts +5 -112
- package/dist/client/tcp/client.js +5 -481
- package/dist/client/tcp/commandQueue.d.ts +23 -0
- package/dist/client/tcp/commandQueue.js +89 -0
- package/dist/client/tcp/connection.d.ts +4 -85
- package/dist/client/tcp/connection.js +3 -315
- package/dist/client/tcp/errors.d.ts +6 -0
- package/dist/client/tcp/errors.js +18 -0
- package/dist/client/tcp/runtime/commands.d.ts +9 -0
- package/dist/client/tcp/runtime/commands.js +124 -0
- package/dist/client/tcp/runtime/connectivity.d.ts +13 -0
- package/dist/client/tcp/runtime/connectivity.js +77 -0
- package/dist/client/tcp/runtime/health.d.ts +14 -0
- package/dist/client/tcp/runtime/health.js +127 -0
- package/dist/client/tcp/runtime/lifecycle.d.ts +8 -0
- package/dist/client/tcp/runtime/lifecycle.js +32 -0
- package/dist/client/tcp/runtime/state.d.ts +50 -0
- package/dist/client/tcp/runtime/state.js +45 -0
- package/dist/client/tcp/transport.d.ts +18 -0
- package/dist/client/tcp/transport.js +158 -0
- package/dist/client/tcp/types/command.d.ts +9 -0
- package/dist/client/tcp/types/command.js +1 -0
- package/dist/client/tcp/types/connection.d.ts +31 -0
- package/dist/client/tcp/types/connection.js +17 -0
- package/dist/client/tcp/types/index.d.ts +5 -0
- package/dist/client/tcp/types/index.js +1 -0
- package/dist/client/tcp/types/socket.d.ts +8 -0
- package/dist/client/tcp/types/socket.js +1 -0
- package/dist/client/tcp/types/tls.d.ts +4 -0
- package/dist/client/tcp/types/tls.js +1 -0
- package/dist/client/tcp/types.d.ts +2 -106
- package/dist/client/tcp/types.js +2 -22
- package/dist/client/types/connection.d.ts +29 -0
- package/dist/client/types/connection.js +1 -0
- package/dist/client/types/dlq.d.ts +45 -0
- package/dist/client/types/dlq.js +1 -0
- package/dist/client/types/events.d.ts +54 -0
- package/dist/client/types/events.js +1 -0
- package/dist/client/types/flow.d.ts +10 -0
- package/dist/client/types/flow.js +1 -0
- package/dist/client/types/index.d.ts +10 -0
- package/dist/client/types/index.js +1 -0
- package/dist/client/types/job.d.ts +127 -0
- package/dist/client/types/job.js +1 -0
- package/dist/client/types/metrics.d.ts +1 -0
- package/dist/client/types/metrics.js +1 -0
- package/dist/client/types/options.d.ts +62 -0
- package/dist/client/types/options.js +1 -0
- package/dist/client/types/worker.d.ts +38 -0
- package/dist/client/types/worker.js +1 -0
- package/dist/client/types.d.ts +2 -583
- package/dist/client/types.js +2 -6
- package/dist/client/worker/ackBatcher.d.ts +1 -1
- package/dist/client/worker/ackBatcher.js +17 -5
- package/dist/client/worker/ackOutcome.d.ts +8 -0
- package/dist/client/worker/ackOutcome.js +58 -0
- package/dist/client/worker/constants.d.ts +7 -0
- package/dist/client/worker/constants.js +7 -0
- package/dist/client/worker/groupConcurrency.js +3 -0
- package/dist/client/worker/handlers/completion.d.ts +28 -0
- package/dist/client/worker/handlers/completion.js +68 -0
- package/dist/client/worker/handlers/dependencies.d.ts +11 -0
- package/dist/client/worker/handlers/dependencies.js +107 -0
- package/dist/client/worker/handlers/mutations.d.ts +23 -0
- package/dist/client/worker/handlers/mutations.js +211 -0
- package/dist/client/worker/handlers/queries.d.ts +15 -0
- package/dist/client/worker/handlers/queries.js +100 -0
- package/dist/client/worker/index.d.ts +2 -2
- package/dist/client/worker/index.js +1 -1
- package/dist/client/worker/jobParser.js +4 -2
- package/dist/client/worker/processor.d.ts +2 -15
- package/dist/client/worker/processor.js +68 -125
- package/dist/client/worker/processorHandlers.d.ts +4 -65
- package/dist/client/worker/processorHandlers.js +4 -400
- package/dist/client/worker/processorOutcome.d.ts +14 -0
- package/dist/client/worker/processorOutcome.js +125 -0
- package/dist/client/worker/runtime/buffer.d.ts +8 -0
- package/dist/client/worker/runtime/buffer.js +50 -0
- package/dist/client/worker/runtime/control.d.ts +26 -0
- package/dist/client/worker/runtime/control.js +175 -0
- package/dist/client/worker/runtime/execution.d.ts +11 -0
- package/dist/client/worker/runtime/execution.js +123 -0
- package/dist/client/worker/runtime/lifecycle.d.ts +7 -0
- package/dist/client/worker/runtime/lifecycle.js +114 -0
- package/dist/client/worker/runtime/manual.d.ts +10 -0
- package/dist/client/worker/runtime/manual.js +137 -0
- package/dist/client/worker/runtime/options.d.ts +5 -0
- package/dist/client/worker/runtime/options.js +39 -0
- package/dist/client/worker/runtime/polling.d.ts +12 -0
- package/dist/client/worker/runtime/polling.js +123 -0
- package/dist/client/worker/runtime/state.d.ts +110 -0
- package/dist/client/worker/runtime/state.js +156 -0
- package/dist/client/worker/types/index.d.ts +5 -0
- package/dist/client/worker/types/index.js +1 -0
- package/dist/client/worker/types/manual.d.ts +7 -0
- package/dist/client/worker/types/manual.js +1 -0
- package/dist/client/worker/types/options.d.ts +24 -0
- package/dist/client/worker/types/options.js +1 -0
- package/dist/client/worker/types/outcome.d.ts +23 -0
- package/dist/client/worker/types/outcome.js +1 -0
- package/dist/client/worker/types/processor.d.ts +22 -0
- package/dist/client/worker/types/processor.js +1 -0
- package/dist/client/worker/types/transport.d.ts +11 -0
- package/dist/client/worker/types/transport.js +1 -0
- package/dist/client/worker/worker.d.ts +3 -160
- package/dist/client/worker/worker.js +3 -904
- package/dist/client/worker/workerRateLimiter.d.ts +5 -1
- package/dist/client/worker/workerRateLimiter.js +40 -20
- package/dist/client/worker.d.ts +1 -0
- package/dist/client/workflow/compensationChild.d.ts +2 -1
- package/dist/client/workflow/compensationChild.js +1 -1
- package/dist/client/workflow/compensationClaim.d.ts +11 -0
- package/dist/client/workflow/compensationClaim.js +30 -0
- package/dist/client/workflow/compensator.d.ts +7 -2
- package/dist/client/workflow/compensator.js +5 -11
- package/dist/client/workflow/recovery.js +25 -2
- package/dist/client/workflow/rollbackControl.js +1 -1
- package/dist/domain/job/constants.d.ts +12 -0
- package/dist/domain/job/constants.js +12 -0
- package/dist/domain/job/create.d.ts +2 -0
- package/dist/domain/job/create.js +103 -0
- package/dist/domain/job/ids.d.ts +5 -0
- package/dist/domain/job/ids.js +12 -0
- package/dist/domain/job/locks.d.ts +4 -0
- package/dist/domain/job/locks.js +23 -0
- package/dist/domain/job/payload.d.ts +18 -0
- package/dist/domain/job/payload.js +43 -0
- package/dist/domain/job/state.d.ts +8 -0
- package/dist/domain/job/state.js +51 -0
- package/dist/domain/job/terminal.d.ts +3 -0
- package/dist/domain/job/terminal.js +9 -0
- package/dist/domain/queue/dlqShard.d.ts +2 -0
- package/dist/domain/queue/dlqShard.js +34 -9
- package/dist/domain/queue/limiterManager.d.ts +11 -0
- package/dist/domain/queue/limiterManager.js +28 -0
- package/dist/domain/queue/priorityQueue.js +3 -39
- package/dist/domain/queue/priorityQueueOrder.d.ts +3 -0
- package/dist/domain/queue/priorityQueueOrder.js +21 -0
- package/dist/domain/queue/shard/dependencies.d.ts +11 -0
- package/dist/domain/queue/shard/dependencies.js +22 -0
- package/dist/domain/queue/shard/dlq.d.ts +25 -0
- package/dist/domain/queue/shard/dlq.js +71 -0
- package/dist/domain/queue/shard/keys.d.ts +18 -0
- package/dist/domain/queue/shard/keys.js +45 -0
- package/dist/domain/queue/shard/lifecycle.d.ts +10 -0
- package/dist/domain/queue/shard/lifecycle.js +43 -0
- package/dist/domain/queue/shard/limits.d.ts +24 -0
- package/dist/domain/queue/shard/limits.js +57 -0
- package/dist/domain/queue/shard/metrics.d.ts +32 -0
- package/dist/domain/queue/shard/metrics.js +72 -0
- package/dist/domain/queue/shard/state.d.ts +34 -0
- package/dist/domain/queue/shard/state.js +80 -0
- package/dist/domain/queue/shard.d.ts +5 -137
- package/dist/domain/queue/shard.js +4 -390
- package/dist/domain/queue/uniqueKeyManager.d.ts +3 -1
- package/dist/domain/queue/uniqueKeyManager.js +9 -1
- package/dist/domain/response/builders.d.ts +15 -0
- package/dist/domain/response/builders.js +39 -0
- package/dist/domain/types/command.d.ts +2 -522
- package/dist/domain/types/command.js +2 -5
- package/dist/domain/types/commands/base.d.ts +5 -0
- package/dist/domain/types/commands/base.js +1 -0
- package/dist/domain/types/commands/core.d.ts +106 -0
- package/dist/domain/types/commands/core.js +1 -0
- package/dist/domain/types/commands/cron.d.ts +36 -0
- package/dist/domain/types/commands/cron.js +1 -0
- package/dist/domain/types/commands/dashboard.d.ts +13 -0
- package/dist/domain/types/commands/dashboard.js +1 -0
- package/dist/domain/types/commands/dlq.d.ts +39 -0
- package/dist/domain/types/commands/dlq.js +1 -0
- package/dist/domain/types/commands/extended.d.ts +63 -0
- package/dist/domain/types/commands/extended.js +1 -0
- package/dist/domain/types/commands/index.d.ts +12 -0
- package/dist/domain/types/commands/index.js +12 -0
- package/dist/domain/types/commands/limits.d.ts +54 -0
- package/dist/domain/types/commands/limits.js +1 -0
- package/dist/domain/types/commands/management.d.ts +73 -0
- package/dist/domain/types/commands/management.js +1 -0
- package/dist/domain/types/commands/monitoring.d.ts +49 -0
- package/dist/domain/types/commands/monitoring.js +1 -0
- package/dist/domain/types/commands/protocol.d.ts +20 -0
- package/dist/domain/types/commands/protocol.js +1 -0
- package/dist/domain/types/commands/query.d.ts +46 -0
- package/dist/domain/types/commands/query.js +1 -0
- package/dist/domain/types/commands/union.d.ts +13 -0
- package/dist/domain/types/commands/union.js +1 -0
- package/dist/domain/types/commands/workers.d.ts +39 -0
- package/dist/domain/types/commands/workers.js +1 -0
- package/dist/domain/types/cron.d.ts +4 -2
- package/dist/domain/types/cron.js +3 -0
- package/dist/domain/types/dlq.d.ts +17 -0
- package/dist/domain/types/dlq.js +70 -14
- package/dist/domain/types/index.d.ts +2 -0
- package/dist/domain/types/index.js +1 -0
- package/dist/domain/types/job.d.ts +8 -269
- package/dist/domain/types/job.js +8 -234
- package/dist/domain/types/jobs/model.d.ts +152 -0
- package/dist/domain/types/jobs/model.js +1 -0
- package/dist/domain/types/metrics.d.ts +16 -0
- package/dist/domain/types/metrics.js +1 -0
- package/dist/domain/types/priorityQueue.d.ts +12 -0
- package/dist/domain/types/priorityQueue.js +1 -0
- package/dist/domain/types/protocol.d.ts +5 -0
- package/dist/domain/types/protocol.js +4 -0
- package/dist/domain/types/queue.d.ts +7 -0
- package/dist/domain/types/queue.js +13 -0
- package/dist/domain/types/response.d.ts +3 -192
- package/dist/domain/types/response.js +3 -101
- package/dist/domain/types/responses/model.d.ts +152 -0
- package/dist/domain/types/responses/model.js +1 -0
- package/dist/domain/types/shard.d.ts +4 -0
- package/dist/domain/types/shard.js +1 -0
- package/dist/infrastructure/cloud/cloudAgent.js +6 -0
- package/dist/infrastructure/cloud/commandHandler.d.ts +2 -41
- package/dist/infrastructure/cloud/commandHandler.js +1 -1
- package/dist/infrastructure/cloud/commands/integrations.d.ts +2 -0
- package/dist/infrastructure/cloud/commands/integrations.js +31 -0
- package/dist/infrastructure/cloud/commands/jobMapper.d.ts +56 -0
- package/dist/infrastructure/cloud/commands/jobMapper.js +72 -0
- package/dist/infrastructure/cloud/commands/jobs.d.ts +2 -0
- package/dist/infrastructure/cloud/commands/jobs.js +111 -0
- package/dist/infrastructure/cloud/commands/queues.d.ts +2 -0
- package/dist/infrastructure/cloud/commands/queues.js +126 -0
- package/dist/infrastructure/cloud/commands.d.ts +3 -10
- package/dist/infrastructure/cloud/commands.js +7 -344
- package/dist/infrastructure/cloud/snapshot/analytics.d.ts +16 -0
- package/dist/infrastructure/cloud/snapshot/analytics.js +132 -0
- package/dist/infrastructure/cloud/snapshot/collector.d.ts +24 -0
- package/dist/infrastructure/cloud/snapshot/collector.js +105 -0
- package/dist/infrastructure/cloud/snapshot/config.d.ts +5 -0
- package/dist/infrastructure/cloud/snapshot/config.js +86 -0
- package/dist/infrastructure/cloud/snapshot/jobs.d.ts +5 -0
- package/dist/infrastructure/cloud/snapshot/jobs.js +148 -0
- package/dist/infrastructure/cloud/snapshot/stalls.d.ts +3 -0
- package/dist/infrastructure/cloud/snapshot/stalls.js +34 -0
- package/dist/infrastructure/cloud/snapshotCollector.d.ts +2 -53
- package/dist/infrastructure/cloud/snapshotCollector.js +5 -107
- package/dist/infrastructure/cloud/snapshotHelpers.d.ts +5 -45
- package/dist/infrastructure/cloud/snapshotHelpers.js +4 -426
- package/dist/infrastructure/cloud/types/collector.d.ts +54 -0
- package/dist/infrastructure/cloud/types/collector.js +1 -0
- package/dist/infrastructure/cloud/types/command.d.ts +44 -0
- package/dist/infrastructure/cloud/types/command.js +1 -0
- package/dist/infrastructure/cloud/types/config.d.ts +18 -0
- package/dist/infrastructure/cloud/types/config.js +1 -0
- package/dist/infrastructure/cloud/types/event.d.ts +14 -0
- package/dist/infrastructure/cloud/types/event.js +1 -0
- package/dist/infrastructure/cloud/types/redact.d.ts +4 -0
- package/dist/infrastructure/cloud/types/redact.js +1 -0
- package/dist/infrastructure/cloud/types/snapshot.d.ts +52 -0
- package/dist/infrastructure/cloud/types/snapshot.js +1 -0
- package/dist/infrastructure/cloud/types/snapshotCore.d.ts +125 -0
- package/dist/infrastructure/cloud/types/snapshotCore.js +1 -0
- package/dist/infrastructure/cloud/types/snapshotJobs.d.ts +124 -0
- package/dist/infrastructure/cloud/types/snapshotJobs.js +1 -0
- package/dist/infrastructure/cloud/types/snapshotTelemetry.d.ts +100 -0
- package/dist/infrastructure/cloud/types/snapshotTelemetry.js +1 -0
- package/dist/infrastructure/cloud/types.d.ts +3 -445
- package/dist/infrastructure/cloud/types.js +0 -4
- package/dist/infrastructure/persistence/batchInsert.d.ts +13 -0
- package/dist/infrastructure/persistence/batchInsert.js +96 -0
- package/dist/infrastructure/persistence/jobOptionsBlob.d.ts +18 -0
- package/dist/infrastructure/persistence/jobOptionsBlob.js +56 -0
- package/dist/infrastructure/persistence/legacyNameMigration.d.ts +5 -0
- package/dist/infrastructure/persistence/legacyNameMigration.js +42 -0
- package/dist/infrastructure/persistence/migrations.d.ts +3 -0
- package/dist/infrastructure/persistence/migrations.js +120 -0
- package/dist/infrastructure/persistence/schema.d.ts +3 -5
- package/dist/infrastructure/persistence/schema.js +40 -136
- package/dist/infrastructure/persistence/sqlite/admission.d.ts +9 -0
- package/dist/infrastructure/persistence/sqlite/admission.js +34 -0
- package/dist/infrastructure/persistence/sqlite/control.d.ts +32 -0
- package/dist/infrastructure/persistence/sqlite/control.js +90 -0
- package/dist/infrastructure/persistence/sqlite/flows.d.ts +26 -0
- package/dist/infrastructure/persistence/sqlite/flows.js +167 -0
- package/dist/infrastructure/persistence/sqlite/jobs.d.ts +23 -0
- package/dist/infrastructure/persistence/sqlite/jobs.js +166 -0
- package/dist/infrastructure/persistence/sqlite/lifecycle.d.ts +6 -0
- package/dist/infrastructure/persistence/sqlite/lifecycle.js +30 -0
- package/dist/infrastructure/persistence/sqlite/mutations.d.ts +20 -0
- package/dist/infrastructure/persistence/sqlite/mutations.js +149 -0
- package/dist/infrastructure/persistence/sqlite/queries.d.ts +23 -0
- package/dist/infrastructure/persistence/sqlite/queries.js +89 -0
- package/dist/infrastructure/persistence/sqlite/records.d.ts +24 -0
- package/dist/infrastructure/persistence/sqlite/records.js +115 -0
- package/dist/infrastructure/persistence/sqlite/state.d.ts +36 -0
- package/dist/infrastructure/persistence/sqlite/state.js +207 -0
- package/dist/infrastructure/persistence/sqlite/telemetry.d.ts +16 -0
- package/dist/infrastructure/persistence/sqlite/telemetry.js +126 -0
- package/dist/infrastructure/persistence/sqlite.d.ts +5 -276
- package/dist/infrastructure/persistence/sqlite.js +3 -997
- package/dist/infrastructure/persistence/sqliteBatch.d.ts +4 -121
- package/dist/infrastructure/persistence/sqliteBatch.js +3 -422
- package/dist/infrastructure/persistence/sqliteSerializer.d.ts +6 -1
- package/dist/infrastructure/persistence/sqliteSerializer.js +33 -11
- package/dist/infrastructure/persistence/statements.d.ts +5 -1
- package/dist/infrastructure/persistence/statements.js +14 -9
- package/dist/infrastructure/persistence/types/admission.d.ts +8 -0
- package/dist/infrastructure/persistence/types/admission.js +1 -0
- package/dist/infrastructure/persistence/types/batch.d.ts +12 -0
- package/dist/infrastructure/persistence/types/batch.js +1 -0
- package/dist/infrastructure/persistence/types/sqlite.d.ts +17 -0
- package/dist/infrastructure/persistence/types/sqlite.js +1 -0
- package/dist/infrastructure/persistence/writeBuffer.d.ts +38 -0
- package/dist/infrastructure/persistence/writeBuffer.js +221 -0
- package/dist/infrastructure/scheduler/cron/execution.d.ts +12 -0
- package/dist/infrastructure/scheduler/cron/execution.js +150 -0
- package/dist/infrastructure/scheduler/cron/runtime.d.ts +31 -0
- package/dist/infrastructure/scheduler/cron/runtime.js +143 -0
- package/dist/infrastructure/scheduler/cronScheduler.d.ts +4 -117
- package/dist/infrastructure/scheduler/cronScheduler.js +3 -436
- package/dist/infrastructure/scheduler/types/cronScheduler.d.ts +16 -0
- package/dist/infrastructure/scheduler/types/cronScheduler.js +1 -0
- package/dist/infrastructure/server/bootstrap.js +3 -0
- package/dist/infrastructure/server/handler-routes/control.d.ts +8 -0
- package/dist/infrastructure/server/handler-routes/control.js +82 -0
- package/dist/infrastructure/server/handler-routes/jobs.d.ts +6 -0
- package/dist/infrastructure/server/handler-routes/jobs.js +110 -0
- package/dist/infrastructure/server/handler-routes/monitoring.d.ts +5 -0
- package/dist/infrastructure/server/handler-routes/monitoring.js +65 -0
- package/dist/infrastructure/server/handlerRoutes.d.ts +3 -26
- package/dist/infrastructure/server/handlerRoutes.js +3 -257
- package/dist/infrastructure/server/handlers/advanced/configNumbers.d.ts +2 -0
- package/dist/infrastructure/server/handlers/advanced/configNumbers.js +26 -0
- package/dist/infrastructure/server/handlers/advanced/dependencies.d.ts +15 -0
- package/dist/infrastructure/server/handlers/advanced/dependencies.js +38 -0
- package/dist/infrastructure/server/handlers/advanced/jobs.d.ts +33 -0
- package/dist/infrastructure/server/handlers/advanced/jobs.js +136 -0
- package/dist/infrastructure/server/handlers/advanced/queue.d.ts +42 -0
- package/dist/infrastructure/server/handlers/advanced/queue.js +111 -0
- package/dist/infrastructure/server/handlers/advanced.d.ts +3 -121
- package/dist/infrastructure/server/handlers/advanced.js +3 -336
- package/dist/infrastructure/server/handlers/core.d.ts +1 -4
- package/dist/infrastructure/server/handlers/core.js +37 -20
- package/dist/infrastructure/server/handlers/cron.js +9 -1
- package/dist/infrastructure/server/handlers/dlq.d.ts +3 -0
- package/dist/infrastructure/server/handlers/dlq.js +18 -4
- package/dist/infrastructure/server/handlers/flow.js +6 -1
- package/dist/infrastructure/server/handlers/introspection.d.ts +19 -0
- package/dist/infrastructure/server/handlers/introspection.js +21 -0
- package/dist/infrastructure/server/handlers/management.d.ts +8 -0
- package/dist/infrastructure/server/handlers/management.js +30 -0
- package/dist/infrastructure/server/handlers/monitoring/health.d.ts +13 -0
- package/dist/infrastructure/server/handlers/monitoring/health.js +61 -0
- package/dist/infrastructure/server/handlers/monitoring/operations.d.ts +8 -0
- package/dist/infrastructure/server/handlers/monitoring/operations.js +28 -0
- package/dist/infrastructure/server/handlers/monitoring/webhooks.d.ts +7 -0
- package/dist/infrastructure/server/handlers/monitoring/webhooks.js +56 -0
- package/dist/infrastructure/server/handlers/monitoring/workers.d.ts +6 -0
- package/dist/infrastructure/server/handlers/monitoring/workers.js +57 -0
- package/dist/infrastructure/server/handlers/monitoring.d.ts +4 -30
- package/dist/infrastructure/server/handlers/monitoring.js +4 -222
- package/dist/infrastructure/server/handlers/pushBatchValidation.d.ts +1 -1
- package/dist/infrastructure/server/handlers/pushBatchValidation.js +7 -0
- package/dist/infrastructure/server/handlers/query.js +13 -4
- package/dist/infrastructure/server/http-routes/jobAdvanced.d.ts +2 -0
- package/dist/infrastructure/server/http-routes/jobAdvanced.js +106 -0
- package/dist/infrastructure/server/http-routes/jobManagement.d.ts +2 -0
- package/dist/infrastructure/server/http-routes/jobManagement.js +74 -0
- package/dist/infrastructure/server/http-routes/queueJobs.d.ts +2 -0
- package/dist/infrastructure/server/http-routes/queueJobs.js +108 -0
- package/dist/infrastructure/server/httpRouteJobs.d.ts +1 -8
- package/dist/infrastructure/server/httpRouteJobs.js +33 -222
- package/dist/infrastructure/server/httpRouteQueues.d.ts +1 -6
- package/dist/infrastructure/server/httpRouteQueues.js +51 -217
- package/dist/infrastructure/server/httpRouteResources.js +1 -0
- package/dist/infrastructure/server/protocol/commands.d.ts +8 -0
- package/dist/infrastructure/server/protocol/commands.js +31 -0
- package/dist/infrastructure/server/protocol/frameParser.d.ts +16 -0
- package/dist/infrastructure/server/protocol/frameParser.js +65 -0
- package/dist/infrastructure/server/protocol/lineBuffer.d.ts +6 -0
- package/dist/infrastructure/server/protocol/lineBuffer.js +21 -0
- package/dist/infrastructure/server/protocol/validation.d.ts +9 -0
- package/dist/infrastructure/server/protocol/validation.js +75 -0
- package/dist/infrastructure/server/protocol.d.ts +5 -77
- package/dist/infrastructure/server/protocol.js +4 -265
- package/dist/infrastructure/server/rateLimiter.js +1 -0
- package/dist/infrastructure/server/socketWriteQueue.d.ts +1 -1
- package/dist/infrastructure/server/socketWriteQueue.js +8 -5
- package/dist/infrastructure/server/sse/constants.d.ts +5 -0
- package/dist/infrastructure/server/sse/constants.js +5 -0
- package/dist/infrastructure/server/sse/eventBuffer.d.ts +3 -0
- package/dist/infrastructure/server/sse/eventBuffer.js +21 -0
- package/dist/infrastructure/server/sseHandler.d.ts +2 -6
- package/dist/infrastructure/server/sseHandler.js +20 -96
- package/dist/infrastructure/server/tcp/clientRelease.d.ts +2 -0
- package/dist/infrastructure/server/tcp/clientRelease.js +18 -0
- package/dist/infrastructure/server/tcp/connections.d.ts +24 -0
- package/dist/infrastructure/server/tcp/connections.js +162 -0
- package/dist/infrastructure/server/tcp/constants.d.ts +3 -0
- package/dist/infrastructure/server/tcp/constants.js +3 -0
- package/dist/infrastructure/server/tcp/eventSubscriptions.d.ts +10 -0
- package/dist/infrastructure/server/tcp/eventSubscriptions.js +17 -0
- package/dist/infrastructure/server/tcp/responses.d.ts +3 -0
- package/dist/infrastructure/server/tcp/responses.js +8 -0
- package/dist/infrastructure/server/tcp.d.ts +10 -68
- package/dist/infrastructure/server/tcp.js +47 -256
- package/dist/infrastructure/server/types/protocol.d.ts +4 -0
- package/dist/infrastructure/server/types/protocol.js +1 -0
- package/dist/infrastructure/server/types/sse.d.ts +11 -0
- package/dist/infrastructure/server/types/sse.js +1 -0
- package/dist/infrastructure/server/types/tcpServer.d.ts +25 -0
- package/dist/infrastructure/server/types/tcpServer.js +1 -0
- package/dist/infrastructure/server/types/ws.d.ts +6 -0
- package/dist/infrastructure/server/types/ws.js +1 -0
- package/dist/infrastructure/server/ws/constants.d.ts +6 -0
- package/dist/infrastructure/server/ws/constants.js +130 -0
- package/dist/infrastructure/server/ws/snapshots.d.ts +4 -0
- package/dist/infrastructure/server/ws/snapshots.js +52 -0
- package/dist/infrastructure/server/ws/subscriptions.d.ts +10 -0
- package/dist/infrastructure/server/ws/subscriptions.js +39 -0
- package/dist/infrastructure/server/wsHandler.d.ts +2 -10
- package/dist/infrastructure/server/wsHandler.js +12 -237
- package/dist/mcp/adapter.d.ts +5 -403
- package/dist/mcp/adapter.js +6 -778
- package/dist/mcp/backend/embedded/base.d.ts +7 -0
- package/dist/mcp/backend/embedded/base.js +15 -0
- package/dist/mcp/backend/embedded/index.d.ts +14 -0
- package/dist/mcp/backend/embedded/index.js +22 -0
- package/dist/mcp/backend/embedded/jobs.d.ts +44 -0
- package/dist/mcp/backend/embedded/jobs.js +98 -0
- package/dist/mcp/backend/embedded/queues.d.ts +27 -0
- package/dist/mcp/backend/embedded/queues.js +84 -0
- package/dist/mcp/backend/embedded/services.d.ts +29 -0
- package/dist/mcp/backend/embedded/services.js +107 -0
- package/dist/mcp/backend/serializers.d.ts +7 -0
- package/dist/mcp/backend/serializers.js +32 -0
- package/dist/mcp/backend/tcp/base.d.ts +19 -0
- package/dist/mcp/backend/tcp/base.js +54 -0
- package/dist/mcp/backend/tcp/index.d.ts +14 -0
- package/dist/mcp/backend/tcp/index.js +20 -0
- package/dist/mcp/backend/tcp/jobs.d.ts +44 -0
- package/dist/mcp/backend/tcp/jobs.js +122 -0
- package/dist/mcp/backend/tcp/queues.d.ts +27 -0
- package/dist/mcp/backend/tcp/queues.js +89 -0
- package/dist/mcp/backend/tcp/services.d.ts +30 -0
- package/dist/mcp/backend/tcp/services.js +161 -0
- package/dist/mcp/types/adapter.d.ts +176 -0
- package/dist/mcp/types/adapter.js +1 -0
- package/dist/shared/asyncLock.d.ts +13 -0
- package/dist/shared/asyncLock.js +65 -0
- package/dist/shared/lock.d.ts +10 -72
- package/dist/shared/lock.js +7 -231
- package/dist/shared/lockError.d.ts +3 -0
- package/dist/shared/lockError.js +6 -0
- package/dist/shared/lockTimeout.d.ts +2 -0
- package/dist/shared/lockTimeout.js +2 -0
- package/dist/shared/rwLock.d.ts +22 -0
- package/dist/shared/rwLock.js +124 -0
- package/dist/shared/serialization.js +1 -0
- package/dist/shared/skipList.d.ts +0 -5
- package/dist/shared/skipList.js +10 -57
- package/dist/shared/skipListNodes.d.ts +6 -0
- package/dist/shared/skipListNodes.js +43 -0
- package/dist/shared/types/lock.d.ts +7 -0
- package/dist/shared/types/lock.js +1 -0
- package/dist/shared/types/skipList.d.ts +4 -0
- package/dist/shared/types/skipList.js +1 -0
- package/package.json +2 -1
|
@@ -1,1881 +1,4 @@
|
|
|
1
|
-
/**
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
*/
|
|
5
|
-
import { DEFAULT_LOCK_TTL } from '../domain/types/job';
|
|
6
|
-
import { DEFAULT_STALL_CONFIG } from '../domain/types/stall';
|
|
7
|
-
import { DEFAULT_DLQ_CONFIG } from '../domain/types/dlq';
|
|
8
|
-
import { Shard } from '../domain/queue/shard';
|
|
9
|
-
import { SqliteStorage } from '../infrastructure/persistence/sqlite';
|
|
10
|
-
import { CronScheduler } from '../infrastructure/scheduler/cronScheduler';
|
|
11
|
-
import { WebhookManager } from './webhookManager';
|
|
12
|
-
import { WorkerManager } from './workerManager';
|
|
13
|
-
import { EventsManager } from './eventsManager';
|
|
14
|
-
import { createMonitoringState } from './monitoringChecks';
|
|
15
|
-
import { RWLock, withWriteLock } from '../shared/lock';
|
|
16
|
-
import { processingShardIndex, shardIndex, SHARD_COUNT } from '../shared/hash';
|
|
17
|
-
import { pushJob, pushJobBatch } from './operations/push';
|
|
18
|
-
import { pushFlowBatch } from './operations/flowPush';
|
|
19
|
-
import { pullJob, pullJobBatch } from './operations/pull';
|
|
20
|
-
import { ackJob, ackJobBatch, ackJobBatchWithResults, failJob } from './operations/ack';
|
|
21
|
-
import * as queueControl from './operations/queueControl';
|
|
22
|
-
import * as jobMgmt from './operations/jobManagement';
|
|
23
|
-
import * as jobPromotion from './operations/jobPromotion';
|
|
24
|
-
import * as jobTransitions from './operations/jobStateTransitions';
|
|
25
|
-
import * as queryOps from './operations/queryOperations';
|
|
26
|
-
import * as dlqOps from './dlqManager';
|
|
27
|
-
import * as logsOps from './jobLogsManager';
|
|
28
|
-
import { generatePrometheusMetrics } from './metricsExporter';
|
|
29
|
-
import { selectPrometheusQueues } from './prometheusOperationalMetrics';
|
|
30
|
-
import { LRUMap, BoundedSet, BoundedMap } from '../shared/lru';
|
|
31
|
-
import { DEFAULT_CONFIG } from './types';
|
|
32
|
-
import * as lockMgr from './lockManager';
|
|
33
|
-
import * as bgTasks from './backgroundTasks';
|
|
34
|
-
import * as statsMgr from './statsManager';
|
|
35
|
-
import { ContextFactory } from './contextFactory';
|
|
36
|
-
import { processPendingDependencies } from './dependencyProcessor';
|
|
37
|
-
import { handleTaskError, handleTaskSuccess } from './taskErrorTracking';
|
|
38
|
-
import { DependencyResultTracker } from './dependencyResultTracker';
|
|
39
|
-
import { recoverFlowFailures } from './flowFailureRecovery';
|
|
40
|
-
import { assertFlowParentOwnership, backpatchDeclaredFlowChild, canAcceptRemovedFlowChild, flowChildFailureError, isDeclaredFlowChild, } from './flowParentBackpatch';
|
|
41
|
-
import { commitRemovedCompletion, DependencyCompletionTracker, reconcileDependencyCompletionPins, releaseDependencyCompletionPins, } from './dependencyCompletions';
|
|
42
|
-
/**
|
|
43
|
-
* QueueManager - Central coordinator
|
|
44
|
-
*/
|
|
45
|
-
export class QueueManager {
|
|
46
|
-
config;
|
|
47
|
-
storage;
|
|
48
|
-
// Sharded data structures
|
|
49
|
-
shards = [];
|
|
50
|
-
shardLocks = [];
|
|
51
|
-
customIdLock = new RWLock();
|
|
52
|
-
processingShards = [];
|
|
53
|
-
processingLocks = [];
|
|
54
|
-
// Global indexes (bounded with LRU eviction)
|
|
55
|
-
jobIndex = new Map();
|
|
56
|
-
completedJobs;
|
|
57
|
-
completedJobsData;
|
|
58
|
-
// Bare removeOnComplete evidence: bounded recent IDs plus IDs pinned by live
|
|
59
|
-
// dependency edges. It is never surfaced as completed job data or statistics.
|
|
60
|
-
depCompletions;
|
|
61
|
-
// Ids of jobs failed by the timeout sweep. A late ACK whose lock token no
|
|
62
|
-
// longer matches (the job was requeued for retry) is discarded for these,
|
|
63
|
-
// instead of phantom-completing the job and skipping the retry. Bounded;
|
|
64
|
-
// never needs explicit clearing because a legit retry ACK carries a valid
|
|
65
|
-
// current token and bypasses the stale-token recovery path entirely.
|
|
66
|
-
timedOutJobs;
|
|
67
|
-
jobResults;
|
|
68
|
-
dependencyResults = new DependencyResultTracker();
|
|
69
|
-
customIdMap;
|
|
70
|
-
jobLogs;
|
|
71
|
-
// Deferred dependency resolution queue
|
|
72
|
-
pendingDepChecks = new Set();
|
|
73
|
-
// Two-phase stall detection
|
|
74
|
-
stalledCandidates = new Set();
|
|
75
|
-
// Event-driven dependency flush state
|
|
76
|
-
depFlushScheduled = false;
|
|
77
|
-
depFlushRunning = false;
|
|
78
|
-
// Lock-based job ownership tracking
|
|
79
|
-
jobLocks = new Map();
|
|
80
|
-
clientJobs = new Map();
|
|
81
|
-
// Repeat chain: maps completed job ID -> successor repeat job ID
|
|
82
|
-
repeatChain = new Map();
|
|
83
|
-
// BullMQ v5 flow failure tracking
|
|
84
|
-
/** Stores failed children values for continueParentOnFailure — parentId → { childKey: error } */
|
|
85
|
-
failedChildrenValues = new Map();
|
|
86
|
-
/** Stores ignored children failures for ignoreDependencyOnFailure — parentId → { childKey: error } */
|
|
87
|
-
ignoredChildrenFailures = new Map();
|
|
88
|
-
// Cron scheduler
|
|
89
|
-
cronScheduler;
|
|
90
|
-
// Managers
|
|
91
|
-
webhookManager;
|
|
92
|
-
workerManager;
|
|
93
|
-
eventsManager;
|
|
94
|
-
// Dashboard event callback (set by HTTP server for WS broadcast)
|
|
95
|
-
dashboardEmit = null;
|
|
96
|
-
recoveryStats = null;
|
|
97
|
-
// Job logs config
|
|
98
|
-
maxLogsPerJob = 100;
|
|
99
|
-
// Metrics
|
|
100
|
-
metrics = {
|
|
101
|
-
totalPushed: { value: 0n },
|
|
102
|
-
totalPulled: { value: 0n },
|
|
103
|
-
totalCompleted: { value: 0n },
|
|
104
|
-
totalFailed: { value: 0n },
|
|
105
|
-
};
|
|
106
|
-
operationalMetricsProvider = null;
|
|
107
|
-
// LRU-bounded so high-cardinality / dynamically-named queues cannot grow it
|
|
108
|
-
// without bound. Live queues stay resident (recently accessed on every
|
|
109
|
-
// ack/fail); only long-idle ephemeral names are evicted. obliterate() also
|
|
110
|
-
// deletes the entry explicitly. Reclaiming on a transient drain is avoided
|
|
111
|
-
// on purpose so cumulative per-queue counters survive idle periods.
|
|
112
|
-
// Assigned in the constructor (needs this.config).
|
|
113
|
-
perQueueMetrics;
|
|
114
|
-
startTime = Date.now();
|
|
115
|
-
// Background task handles
|
|
116
|
-
backgroundTaskHandles;
|
|
117
|
-
// Queue names cache
|
|
118
|
-
queueNamesCache = new Set();
|
|
119
|
-
// Monitoring state
|
|
120
|
-
monitoringState = createMonitoringState();
|
|
121
|
-
// Context factory
|
|
122
|
-
contextFactory;
|
|
123
|
-
constructor(config = {}) {
|
|
124
|
-
this.config = { ...DEFAULT_CONFIG, ...config };
|
|
125
|
-
this.storage = config.dataPath ? new SqliteStorage({ path: config.dataPath }) : null;
|
|
126
|
-
// Initialize bounded collections
|
|
127
|
-
this.completedJobsData = new BoundedMap(this.config.maxCompletedJobs);
|
|
128
|
-
this.completedJobs = new BoundedSet(this.config.maxCompletedJobs, (jobId) => {
|
|
129
|
-
this.jobIndex.delete(jobId);
|
|
130
|
-
this.completedJobsData.delete(jobId);
|
|
131
|
-
});
|
|
132
|
-
this.depCompletions = new DependencyCompletionTracker(this.config.maxCompletedJobs, (jobId) => {
|
|
133
|
-
this.storage?.deleteDependencyCompletion(jobId);
|
|
134
|
-
});
|
|
135
|
-
this.timedOutJobs = new BoundedSet(this.config.maxCompletedJobs);
|
|
136
|
-
this.perQueueMetrics = new LRUMap(this.config.maxCustomIds);
|
|
137
|
-
this.jobResults = new LRUMap(this.config.maxJobResults);
|
|
138
|
-
this.customIdMap = new LRUMap(this.config.maxCustomIds);
|
|
139
|
-
this.jobLogs = new LRUMap(this.config.maxJobLogs);
|
|
140
|
-
// Initialize shards
|
|
141
|
-
for (let i = 0; i < SHARD_COUNT; i++) {
|
|
142
|
-
this.shards.push(new Shard());
|
|
143
|
-
this.shardLocks.push(new RWLock());
|
|
144
|
-
this.processingShards.push(new Map());
|
|
145
|
-
this.processingLocks.push(new RWLock());
|
|
146
|
-
}
|
|
147
|
-
// Initialize cron scheduler
|
|
148
|
-
this.cronScheduler = new CronScheduler();
|
|
149
|
-
this.cronScheduler.setPushCallback(async (queue, input) => {
|
|
150
|
-
await this.push(queue, input);
|
|
151
|
-
});
|
|
152
|
-
// Set up persistence callback for cron state
|
|
153
|
-
if (this.storage) {
|
|
154
|
-
const storage = this.storage;
|
|
155
|
-
this.cronScheduler.setPersistCallback((name, executions, nextRun) => {
|
|
156
|
-
storage.updateCron(name, executions, nextRun);
|
|
157
|
-
});
|
|
158
|
-
}
|
|
159
|
-
// Initialize managers
|
|
160
|
-
this.webhookManager = new WebhookManager({ validateUrls: config.validateWebhookUrls });
|
|
161
|
-
this.workerManager = new WorkerManager();
|
|
162
|
-
// Wire worker check for skipIfNoWorker crons
|
|
163
|
-
this.cronScheduler.setWorkerCheckCallback((queue) => {
|
|
164
|
-
return this.workerManager.getForQueue(queue).length > 0;
|
|
165
|
-
});
|
|
166
|
-
this.eventsManager = new EventsManager(this.webhookManager);
|
|
167
|
-
// Initialize context factory
|
|
168
|
-
this.contextFactory = new ContextFactory(this.getContextDependencies(), this.getContextCallbacks());
|
|
169
|
-
// Load and start
|
|
170
|
-
bgTasks.recover(this.contextFactory.getBackgroundContext());
|
|
171
|
-
if (this.storage) {
|
|
172
|
-
recoverFlowFailures({
|
|
173
|
-
storage: this.storage,
|
|
174
|
-
shards: this.shards,
|
|
175
|
-
jobIndex: this.jobIndex,
|
|
176
|
-
completedJobs: this.completedJobs,
|
|
177
|
-
depCompletions: this.depCompletions,
|
|
178
|
-
maxDependencyCompletions: this.config.maxCompletedJobs,
|
|
179
|
-
dependencyResults: this.dependencyResults,
|
|
180
|
-
failedChildrenValues: this.failedChildrenValues,
|
|
181
|
-
ignoredChildrenFailures: this.ignoredChildrenFailures,
|
|
182
|
-
});
|
|
183
|
-
}
|
|
184
|
-
this.recoveryStats = { queues: this.queueNamesCache.size, jobs: this.jobIndex.size };
|
|
185
|
-
if (this.storage) {
|
|
186
|
-
this.cronScheduler.load(this.storage.loadCronJobs());
|
|
187
|
-
}
|
|
188
|
-
this.backgroundTaskHandles = bgTasks.startBackgroundTasks(this.contextFactory.getBackgroundContext(), this.cronScheduler);
|
|
189
|
-
}
|
|
190
|
-
// ============ Context Dependencies ============
|
|
191
|
-
getContextDependencies() {
|
|
192
|
-
return {
|
|
193
|
-
config: this.config,
|
|
194
|
-
storage: this.storage,
|
|
195
|
-
shards: this.shards,
|
|
196
|
-
shardLocks: this.shardLocks,
|
|
197
|
-
customIdLock: this.customIdLock,
|
|
198
|
-
processingShards: this.processingShards,
|
|
199
|
-
processingLocks: this.processingLocks,
|
|
200
|
-
jobIndex: this.jobIndex,
|
|
201
|
-
completedJobs: this.completedJobs,
|
|
202
|
-
completedJobsData: this.completedJobsData,
|
|
203
|
-
depCompletions: this.depCompletions,
|
|
204
|
-
timedOutJobs: this.timedOutJobs,
|
|
205
|
-
jobResults: this.jobResults,
|
|
206
|
-
dependencyResults: this.dependencyResults,
|
|
207
|
-
customIdMap: this.customIdMap,
|
|
208
|
-
jobLogs: this.jobLogs,
|
|
209
|
-
jobLocks: this.jobLocks,
|
|
210
|
-
clientJobs: this.clientJobs,
|
|
211
|
-
stalledCandidates: this.stalledCandidates,
|
|
212
|
-
pendingDepChecks: this.pendingDepChecks,
|
|
213
|
-
queueNamesCache: this.queueNamesCache,
|
|
214
|
-
repeatChain: this.repeatChain,
|
|
215
|
-
eventsManager: this.eventsManager,
|
|
216
|
-
webhookManager: this.webhookManager,
|
|
217
|
-
workerManager: this.workerManager,
|
|
218
|
-
monitoringState: this.monitoringState,
|
|
219
|
-
metrics: this.metrics,
|
|
220
|
-
startTime: this.startTime,
|
|
221
|
-
maxLogsPerJob: this.maxLogsPerJob,
|
|
222
|
-
perQueueMetrics: this.perQueueMetrics,
|
|
223
|
-
};
|
|
224
|
-
}
|
|
225
|
-
getContextCallbacks() {
|
|
226
|
-
return {
|
|
227
|
-
fail: this.fail.bind(this),
|
|
228
|
-
registerQueueName: this.registerQueueName.bind(this),
|
|
229
|
-
unregisterQueueName: this.unregisterQueueName.bind(this),
|
|
230
|
-
onJobCompleted: this.onJobCompleted.bind(this),
|
|
231
|
-
onJobFailed: this.onJobFailed.bind(this),
|
|
232
|
-
onJobsCompleted: this.onJobsCompleted.bind(this),
|
|
233
|
-
hasPendingDeps: this.hasPendingDeps.bind(this),
|
|
234
|
-
onRepeat: this.handleRepeat.bind(this),
|
|
235
|
-
emitDashboardEvent: this.emitDashboardEvent.bind(this),
|
|
236
|
-
onChildTerminalFailure: this.failParentOnChildFailure.bind(this),
|
|
237
|
-
onChildDependencyOption: this.onChildDependencyOption.bind(this),
|
|
238
|
-
};
|
|
239
|
-
}
|
|
240
|
-
handleRepeat(job) {
|
|
241
|
-
if (!job.repeat)
|
|
242
|
-
return;
|
|
243
|
-
const delay = job.repeat.every ?? 0;
|
|
244
|
-
const oldId = job.id;
|
|
245
|
-
void this.push(job.queue, {
|
|
246
|
-
data: job.data,
|
|
247
|
-
priority: job.priority,
|
|
248
|
-
delay,
|
|
249
|
-
maxAttempts: job.maxAttempts,
|
|
250
|
-
backoff: job.backoff,
|
|
251
|
-
ttl: job.ttl ?? undefined,
|
|
252
|
-
timeout: job.timeout ?? undefined,
|
|
253
|
-
tags: job.tags,
|
|
254
|
-
groupId: job.groupId ?? undefined,
|
|
255
|
-
lifo: job.lifo,
|
|
256
|
-
removeOnComplete: job.removeOnComplete,
|
|
257
|
-
removeOnFail: job.removeOnFail,
|
|
258
|
-
repeat: {
|
|
259
|
-
every: job.repeat.every,
|
|
260
|
-
limit: job.repeat.limit,
|
|
261
|
-
pattern: job.repeat.pattern,
|
|
262
|
-
count: job.repeat.count + 1,
|
|
263
|
-
},
|
|
264
|
-
}).then((newJob) => {
|
|
265
|
-
// Store mapping so updateJobData can find the successor
|
|
266
|
-
this.repeatChain.set(oldId, newJob.id);
|
|
267
|
-
// Limit chain size to prevent memory leak (only keep recent mappings)
|
|
268
|
-
if (this.repeatChain.size > 10000) {
|
|
269
|
-
const first = this.repeatChain.keys().next().value;
|
|
270
|
-
if (first !== undefined)
|
|
271
|
-
this.repeatChain.delete(first);
|
|
272
|
-
}
|
|
273
|
-
});
|
|
274
|
-
}
|
|
275
|
-
// ============ Core Operations ============
|
|
276
|
-
async push(queue, input) {
|
|
277
|
-
this.registerQueueName(queue);
|
|
278
|
-
return pushJob(queue, input, this.contextFactory.getPushContext());
|
|
279
|
-
}
|
|
280
|
-
async pushBatch(queue, inputs) {
|
|
281
|
-
this.registerQueueName(queue);
|
|
282
|
-
return pushJobBatch(queue, inputs, this.contextFactory.getPushContext());
|
|
283
|
-
}
|
|
284
|
-
async pushFlow(batch) {
|
|
285
|
-
return pushFlowBatch(batch, this.contextFactory.getPushContext());
|
|
286
|
-
}
|
|
287
|
-
async pull(queue, timeoutMs = 0, signal) {
|
|
288
|
-
return pullJob(queue, timeoutMs, this.contextFactory.getPullContext(), signal);
|
|
289
|
-
}
|
|
290
|
-
async pullWithLock(queue, owner, timeoutMs = 0, lockTtl = DEFAULT_LOCK_TTL, signal) {
|
|
291
|
-
const job = await pullJob(queue, timeoutMs, this.contextFactory.getPullContext(), signal);
|
|
292
|
-
if (!job)
|
|
293
|
-
return { job: null, token: null };
|
|
294
|
-
const token = lockMgr.createLock(job.id, owner, this.contextFactory.getLockContext(), lockTtl);
|
|
295
|
-
return { job, token };
|
|
296
|
-
}
|
|
297
|
-
async pullBatch(queue, count, timeoutMs = 0, signal) {
|
|
298
|
-
return pullJobBatch(queue, count, timeoutMs, this.contextFactory.getPullContext(), signal);
|
|
299
|
-
}
|
|
300
|
-
// biome-ignore lint/complexity/useMaxParams: preserves the public API while adding cancellation
|
|
301
|
-
async pullBatchWithLock(queue, count, owner, timeoutMs = 0, lockTtl = DEFAULT_LOCK_TTL, signal) {
|
|
302
|
-
const jobs = await pullJobBatch(queue, count, timeoutMs, this.contextFactory.getPullContext(), signal);
|
|
303
|
-
const tokens = [];
|
|
304
|
-
for (const job of jobs) {
|
|
305
|
-
const token = lockMgr.createLock(job.id, owner, this.contextFactory.getLockContext(), lockTtl);
|
|
306
|
-
tokens.push(token ?? '');
|
|
307
|
-
}
|
|
308
|
-
return { jobs, tokens };
|
|
309
|
-
}
|
|
310
|
-
async ack(jobId, result, token) {
|
|
311
|
-
const lockCtx = this.contextFactory.getLockContext();
|
|
312
|
-
if (token &&
|
|
313
|
-
!lockMgr.verifyLock(jobId, token, lockCtx) &&
|
|
314
|
-
!this.isExpiredButOwned(jobId, token, lockCtx)) {
|
|
315
|
-
// #101: if the lock is expired but still OURS and the job is still in
|
|
316
|
-
// `processing`, isExpiredButOwned() short-circuits this block so the
|
|
317
|
-
// completion falls through to ackJob() rather than being lost.
|
|
318
|
-
this.throwIfOwnershipConflict(jobId, lockCtx);
|
|
319
|
-
// No ownership conflict. If job is still in processing (dedup case
|
|
320
|
-
// from Issue #33: lock removed but job still there), proceed with ACK.
|
|
321
|
-
// Otherwise (lock expiration: job requeued), return gracefully.
|
|
322
|
-
const loc = this.jobIndex.get(jobId);
|
|
323
|
-
if (loc?.type !== 'processing') {
|
|
324
|
-
// Job may have been stall-retried to queue while we processed it.
|
|
325
|
-
// Complete it from queue to prevent duplicate execution (Issue #33).
|
|
326
|
-
if (loc?.type === 'queue') {
|
|
327
|
-
// BUT a job failed by the timeout sweep is requeued for RETRY — a late
|
|
328
|
-
// ACK from the timed-out worker must not complete it (that would skip
|
|
329
|
-
// the retry and silently override the timeout). Discard it gracefully.
|
|
330
|
-
if (this.timedOutJobs.has(jobId)) {
|
|
331
|
-
lockMgr.releaseLock(jobId, lockCtx, token);
|
|
332
|
-
return;
|
|
333
|
-
}
|
|
334
|
-
await this.completeStallRetriedJob(jobId, result);
|
|
335
|
-
lockMgr.releaseLock(jobId, lockCtx, token);
|
|
336
|
-
}
|
|
337
|
-
return;
|
|
338
|
-
}
|
|
339
|
-
}
|
|
340
|
-
try {
|
|
341
|
-
await ackJob(jobId, result, this.contextFactory.getAckContext());
|
|
342
|
-
}
|
|
343
|
-
catch (err) {
|
|
344
|
-
// Job removed from processing by stall detection and re-queued.
|
|
345
|
-
// Try to complete from queue to prevent duplicate execution (Issue #33).
|
|
346
|
-
// With token: always attempt (worker had ownership via lock).
|
|
347
|
-
// Without token: only if job was stall-retried (attempts > 0), to avoid
|
|
348
|
-
// completing freshly-pushed jobs that were never pulled.
|
|
349
|
-
if (err instanceof Error && err.message.includes('not found')) {
|
|
350
|
-
// A timeout-failed job requeued for retry must not be completed by a
|
|
351
|
-
// stale ACK from the timed-out worker — discard it so the retry wins.
|
|
352
|
-
if (this.timedOutJobs.has(jobId)) {
|
|
353
|
-
if (token)
|
|
354
|
-
lockMgr.releaseLock(jobId, lockCtx, token);
|
|
355
|
-
return;
|
|
356
|
-
}
|
|
357
|
-
const shouldRecover = token ?? this.isStallRetried(jobId);
|
|
358
|
-
if (shouldRecover && (await this.completeStallRetriedJob(jobId, result))) {
|
|
359
|
-
if (token)
|
|
360
|
-
lockMgr.releaseLock(jobId, lockCtx, token);
|
|
361
|
-
return;
|
|
362
|
-
}
|
|
363
|
-
}
|
|
364
|
-
throw err;
|
|
365
|
-
}
|
|
366
|
-
lockMgr.releaseLock(jobId, lockCtx, token);
|
|
367
|
-
}
|
|
368
|
-
async ackBatch(jobIds, tokens) {
|
|
369
|
-
const lockCtx = this.contextFactory.getLockContext();
|
|
370
|
-
const validJobIds = [];
|
|
371
|
-
const validTokens = tokens ? [] : undefined;
|
|
372
|
-
if (tokens?.length === jobIds.length) {
|
|
373
|
-
for (let i = 0; i < jobIds.length; i++) {
|
|
374
|
-
const t = tokens[i];
|
|
375
|
-
if (t &&
|
|
376
|
-
!lockMgr.verifyLock(jobIds[i], t, lockCtx) &&
|
|
377
|
-
!this.isExpiredButOwned(jobIds[i], t, lockCtx)) {
|
|
378
|
-
this.throwIfOwnershipConflict(jobIds[i], lockCtx);
|
|
379
|
-
// Recover stall-retried job (#75): lock expired and job was
|
|
380
|
-
// re-queued by lock expiration or stall detection. Complete it
|
|
381
|
-
// from the queue to prevent duplicate execution.
|
|
382
|
-
const loc = this.jobIndex.get(jobIds[i]);
|
|
383
|
-
if (loc?.type === 'queue') {
|
|
384
|
-
// Skip completion for a timeout-requeued job (retry must win); else
|
|
385
|
-
// recover the stall-retried job to prevent duplicate execution (#75).
|
|
386
|
-
if (!this.timedOutJobs.has(jobIds[i])) {
|
|
387
|
-
await this.completeStallRetriedJob(jobIds[i], undefined);
|
|
388
|
-
}
|
|
389
|
-
lockMgr.releaseLock(jobIds[i], lockCtx, t);
|
|
390
|
-
}
|
|
391
|
-
continue;
|
|
392
|
-
}
|
|
393
|
-
// #101 grace window: an expired-but-still-ours lock on a still-processing
|
|
394
|
-
// job is accepted (isExpiredButOwned), not lost.
|
|
395
|
-
validJobIds.push(jobIds[i]);
|
|
396
|
-
if (validTokens)
|
|
397
|
-
validTokens.push(t);
|
|
398
|
-
}
|
|
399
|
-
}
|
|
400
|
-
else {
|
|
401
|
-
validJobIds.push(...jobIds);
|
|
402
|
-
}
|
|
403
|
-
if (validJobIds.length > 0) {
|
|
404
|
-
await ackJobBatch(validJobIds, this.contextFactory.getAckContext());
|
|
405
|
-
}
|
|
406
|
-
if (validTokens) {
|
|
407
|
-
for (let i = 0; i < validJobIds.length; i++) {
|
|
408
|
-
lockMgr.releaseLock(validJobIds[i], lockCtx, validTokens[i]);
|
|
409
|
-
}
|
|
410
|
-
}
|
|
411
|
-
else if (tokens) {
|
|
412
|
-
for (let i = 0; i < jobIds.length; i++) {
|
|
413
|
-
lockMgr.releaseLock(jobIds[i], lockCtx, tokens[i]);
|
|
414
|
-
}
|
|
415
|
-
}
|
|
416
|
-
}
|
|
417
|
-
async ackBatchWithResults(items) {
|
|
418
|
-
const lockCtx = this.contextFactory.getLockContext();
|
|
419
|
-
const validItems = [];
|
|
420
|
-
for (const item of items) {
|
|
421
|
-
if (item.token &&
|
|
422
|
-
!lockMgr.verifyLock(item.id, item.token, lockCtx) &&
|
|
423
|
-
!this.isExpiredButOwned(item.id, item.token, lockCtx)) {
|
|
424
|
-
this.throwIfOwnershipConflict(item.id, lockCtx);
|
|
425
|
-
// Recover stall-retried job (#75): lock expired and job was
|
|
426
|
-
// re-queued by lock expiration or stall detection. Complete it
|
|
427
|
-
// from the queue to prevent duplicate execution.
|
|
428
|
-
const loc = this.jobIndex.get(item.id);
|
|
429
|
-
if (loc?.type === 'queue') {
|
|
430
|
-
// Skip completion for a timeout-requeued job (retry must win); else
|
|
431
|
-
// recover the stall-retried job to prevent duplicate execution (#75).
|
|
432
|
-
if (!this.timedOutJobs.has(item.id)) {
|
|
433
|
-
await this.completeStallRetriedJob(item.id, item.result);
|
|
434
|
-
}
|
|
435
|
-
lockMgr.releaseLock(item.id, lockCtx, item.token);
|
|
436
|
-
}
|
|
437
|
-
continue;
|
|
438
|
-
}
|
|
439
|
-
// #101 grace window (isExpiredButOwned true): the lock TTL elapsed while
|
|
440
|
-
// the handler ran, but the lock is still OURS and the job is still in
|
|
441
|
-
// `processing` — accept the completion instead of losing the work.
|
|
442
|
-
validItems.push(item);
|
|
443
|
-
}
|
|
444
|
-
if (validItems.length > 0) {
|
|
445
|
-
await ackJobBatchWithResults(validItems, this.contextFactory.getAckContext());
|
|
446
|
-
}
|
|
447
|
-
for (const item of validItems) {
|
|
448
|
-
lockMgr.releaseLock(item.id, lockCtx, item.token);
|
|
449
|
-
}
|
|
450
|
-
}
|
|
451
|
-
async fail(jobId, error, token, unrecoverable = false, stack) {
|
|
452
|
-
const lockCtx = this.contextFactory.getLockContext();
|
|
453
|
-
if (token && !lockMgr.verifyLock(jobId, token, lockCtx)) {
|
|
454
|
-
this.throwIfOwnershipConflict(jobId, lockCtx);
|
|
455
|
-
const loc = this.jobIndex.get(jobId);
|
|
456
|
-
if (loc?.type !== 'processing')
|
|
457
|
-
return;
|
|
458
|
-
}
|
|
459
|
-
try {
|
|
460
|
-
await failJob(jobId, error, this.contextFactory.getAckContext(), unrecoverable, stack);
|
|
461
|
-
}
|
|
462
|
-
catch (err) {
|
|
463
|
-
// Job removed from processing by stall detection. The stall retry
|
|
464
|
-
// already handles requeuing, so the fail is redundant — return silently.
|
|
465
|
-
// Only applies when caller has a lock token (worker with ownership).
|
|
466
|
-
if (token && err instanceof Error && err.message.includes('not found')) {
|
|
467
|
-
const loc = this.jobIndex.get(jobId);
|
|
468
|
-
if (loc?.type === 'queue')
|
|
469
|
-
return;
|
|
470
|
-
}
|
|
471
|
-
throw err;
|
|
472
|
-
}
|
|
473
|
-
lockMgr.releaseLock(jobId, lockCtx, token);
|
|
474
|
-
}
|
|
475
|
-
/**
|
|
476
|
-
* Check if a failed lock verification is a genuine ownership conflict.
|
|
477
|
-
* If the job is still in processing with a different lock, throw.
|
|
478
|
-
* If the job was already requeued by the background lock expiration task, return silently.
|
|
479
|
-
*/
|
|
480
|
-
throwIfOwnershipConflict(jobId, lockCtx) {
|
|
481
|
-
const loc = this.jobIndex.get(jobId);
|
|
482
|
-
if (loc?.type === 'processing' && lockCtx.jobLocks.has(jobId)) {
|
|
483
|
-
throw new Error(`Invalid or expired lock token for job ${jobId}`);
|
|
484
|
-
}
|
|
485
|
-
}
|
|
486
|
-
/**
|
|
487
|
-
* Issue #101 grace window: decide whether an ACK whose lock failed
|
|
488
|
-
* verification (because the TTL expired) should still be honored.
|
|
489
|
-
*
|
|
490
|
-
* Returns true ONLY when ALL hold:
|
|
491
|
-
* 1. the job is still in `processing`,
|
|
492
|
-
* 2. the lock entry's token still matches the presenting worker, and
|
|
493
|
-
* 3. the lock belongs to the CURRENT processing instance — its `createdAt`
|
|
494
|
-
* is not older than the job's `startedAt`.
|
|
495
|
-
*
|
|
496
|
-
* Condition 3 is the re-lease guard. A lock-expiry re-lease (checkExpiredLocks)
|
|
497
|
-
* deletes the stale lock, so a new lease installs a NEW token and condition 2
|
|
498
|
-
* already fails. But the STALL path (stallDetection retry/moveToDlq) requeues
|
|
499
|
-
* the job WITHOUT deleting the lock — the original (now-expired) lock lingers
|
|
500
|
-
* with the original token. If another worker then re-pulls the job, its
|
|
501
|
-
* `startedAt` is reset to a newer time than the lingering lock's `createdAt`,
|
|
502
|
-
* so condition 3 fails and the timed-out worker's late ack is rejected
|
|
503
|
-
* (preventing a double-completion the skeptic confirmed). In the genuine #101
|
|
504
|
-
* case — the same worker finishing just after its lock expired, no re-pull —
|
|
505
|
-
* `startedAt` is unchanged and `createdAt >= startedAt`, so the grace is granted
|
|
506
|
-
* and the successful completion is recorded instead of being lost to a stall.
|
|
507
|
-
*
|
|
508
|
-
* Without this, a successful completion arriving just after lock expiry is
|
|
509
|
-
* rejected as "Invalid or expired lock token", the client drops it, and the
|
|
510
|
-
* job stalls to `failed` despite having been processed correctly.
|
|
511
|
-
*/
|
|
512
|
-
isExpiredButOwned(jobId, token, lockCtx) {
|
|
513
|
-
const loc = this.jobIndex.get(jobId);
|
|
514
|
-
if (loc?.type !== 'processing')
|
|
515
|
-
return false;
|
|
516
|
-
const lock = lockCtx.jobLocks.get(jobId);
|
|
517
|
-
if (lock?.token !== token)
|
|
518
|
-
return false;
|
|
519
|
-
// Re-lease guard: a re-pulled job has a startedAt newer than the lingering
|
|
520
|
-
// lock's createdAt → the lock no longer owns the current processing instance.
|
|
521
|
-
const job = this.processingShards[loc.shardIdx].get(jobId);
|
|
522
|
-
if (job && job.startedAt !== null && job.startedAt > lock.createdAt)
|
|
523
|
-
return false;
|
|
524
|
-
return true;
|
|
525
|
-
}
|
|
526
|
-
/** Check if a queued job was stall-retried (has been processed before). */
|
|
527
|
-
isStallRetried(jobId) {
|
|
528
|
-
const loc = this.jobIndex.get(jobId);
|
|
529
|
-
if (loc?.type !== 'queue')
|
|
530
|
-
return false;
|
|
531
|
-
const shard = this.shards[loc.shardIdx];
|
|
532
|
-
const pq = shard.getQueue(loc.queueName);
|
|
533
|
-
const job = pq.find(jobId);
|
|
534
|
-
return job !== null && job.attempts > 0;
|
|
535
|
-
}
|
|
536
|
-
/**
|
|
537
|
-
* Complete a job that stall detection moved back to the queue while still processing.
|
|
538
|
-
* Removes from queue and marks completed to prevent duplicate execution (Issue #33).
|
|
539
|
-
*/
|
|
540
|
-
async completeStallRetriedJob(jId, result) {
|
|
541
|
-
const loc = this.jobIndex.get(jId);
|
|
542
|
-
if (loc?.type !== 'queue')
|
|
543
|
-
return false;
|
|
544
|
-
const idx = loc.shardIdx;
|
|
545
|
-
const queueName = loc.queueName;
|
|
546
|
-
const shard = this.shards[idx];
|
|
547
|
-
let job = null;
|
|
548
|
-
await withWriteLock(this.shardLocks[idx], () => {
|
|
549
|
-
const pq = shard.getQueue(queueName);
|
|
550
|
-
job = pq.remove(jId);
|
|
551
|
-
if (job) {
|
|
552
|
-
shard.decrementQueued(jId);
|
|
553
|
-
shard.releaseJobResources(queueName, job.uniqueKey, job.groupId);
|
|
554
|
-
}
|
|
555
|
-
});
|
|
556
|
-
if (!job) {
|
|
557
|
-
// Job was already pulled from queue by another worker — can't prevent.
|
|
558
|
-
return false;
|
|
559
|
-
}
|
|
560
|
-
const ctx = this.contextFactory.getAckContext();
|
|
561
|
-
if (!job.removeOnComplete) {
|
|
562
|
-
ctx.completedJobs.add(jId);
|
|
563
|
-
ctx.completedJobsData.set(jId, job);
|
|
564
|
-
if (result !== undefined) {
|
|
565
|
-
ctx.jobResults.set(jId, result);
|
|
566
|
-
ctx.storage?.storeResult(jId, result);
|
|
567
|
-
}
|
|
568
|
-
ctx.jobIndex.set(jId, { type: 'completed', queueName: job.queue });
|
|
569
|
-
ctx.storage?.markCompleted(jId, Date.now(), job.timeline);
|
|
570
|
-
}
|
|
571
|
-
else {
|
|
572
|
-
commitRemovedCompletion(job, ctx);
|
|
573
|
-
ctx.jobIndex.delete(jId);
|
|
574
|
-
}
|
|
575
|
-
if (result !== undefined)
|
|
576
|
-
ctx.dependencyResults.retain(jId, result);
|
|
577
|
-
ctx.dependencyResults.releaseConsumer(jId);
|
|
578
|
-
ctx.totalCompleted.value++;
|
|
579
|
-
ctx.broadcast({
|
|
580
|
-
eventType: 'completed',
|
|
581
|
-
queue: queueName,
|
|
582
|
-
jobId: jId,
|
|
583
|
-
timestamp: Date.now(),
|
|
584
|
-
data: result,
|
|
585
|
-
});
|
|
586
|
-
ctx.onJobCompleted(jId);
|
|
587
|
-
return true;
|
|
588
|
-
}
|
|
589
|
-
jobHeartbeat(jobId, token) {
|
|
590
|
-
const loc = this.jobIndex.get(jobId);
|
|
591
|
-
if (loc?.type !== 'processing')
|
|
592
|
-
return false;
|
|
593
|
-
if (token) {
|
|
594
|
-
return lockMgr.renewJobLock(jobId, token, this.contextFactory.getLockContext());
|
|
595
|
-
}
|
|
596
|
-
const processing = this.processingShards[loc.shardIdx];
|
|
597
|
-
const job = processing.get(jobId);
|
|
598
|
-
if (job) {
|
|
599
|
-
job.lastHeartbeat = Date.now();
|
|
600
|
-
return true;
|
|
601
|
-
}
|
|
602
|
-
return false;
|
|
603
|
-
}
|
|
604
|
-
jobHeartbeatBatch(jobIds, tokens) {
|
|
605
|
-
let count = 0;
|
|
606
|
-
for (let i = 0; i < jobIds.length; i++) {
|
|
607
|
-
if (this.jobHeartbeat(jobIds[i], tokens?.[i]))
|
|
608
|
-
count++;
|
|
609
|
-
}
|
|
610
|
-
return count;
|
|
611
|
-
}
|
|
612
|
-
// ============ Lock Management ============
|
|
613
|
-
/** Remove a lock unconditionally (used by worker dedup to clear stale locks) */
|
|
614
|
-
removeLock(jobId) {
|
|
615
|
-
this.contextFactory.getLockContext().jobLocks.delete(jobId);
|
|
616
|
-
}
|
|
617
|
-
createLock(jobId, owner, ttl = DEFAULT_LOCK_TTL) {
|
|
618
|
-
return lockMgr.createLock(jobId, owner, this.contextFactory.getLockContext(), ttl);
|
|
619
|
-
}
|
|
620
|
-
verifyLock(jobId, token) {
|
|
621
|
-
return lockMgr.verifyLock(jobId, token, this.contextFactory.getLockContext());
|
|
622
|
-
}
|
|
623
|
-
renewJobLock(jobId, token, newTtl) {
|
|
624
|
-
return lockMgr.renewJobLock(jobId, token, this.contextFactory.getLockContext(), newTtl);
|
|
625
|
-
}
|
|
626
|
-
renewJobLockBatch(items) {
|
|
627
|
-
return lockMgr.renewJobLockBatch(items, this.contextFactory.getLockContext());
|
|
628
|
-
}
|
|
629
|
-
releaseLock(jobId, token) {
|
|
630
|
-
return lockMgr.releaseLock(jobId, this.contextFactory.getLockContext(), token);
|
|
631
|
-
}
|
|
632
|
-
getLockInfo(jobId) {
|
|
633
|
-
return lockMgr.getLockInfo(jobId, this.contextFactory.getLockContext());
|
|
634
|
-
}
|
|
635
|
-
// ============ Client-Job Tracking ============
|
|
636
|
-
registerClientJob(clientId, jobId) {
|
|
637
|
-
lockMgr.registerClientJob(clientId, jobId, this.contextFactory.getLockContext());
|
|
638
|
-
}
|
|
639
|
-
unregisterClientJob(clientId, jobId) {
|
|
640
|
-
lockMgr.unregisterClientJob(clientId, jobId, this.contextFactory.getLockContext());
|
|
641
|
-
}
|
|
642
|
-
releaseClientJobs(clientId) {
|
|
643
|
-
return lockMgr.releaseClientJobs(clientId, this.contextFactory.getLockContext());
|
|
644
|
-
}
|
|
645
|
-
/**
|
|
646
|
-
* Force-release client tracking without acquiring queue locks. Last-resort
|
|
647
|
-
* fallback when releaseClientJobs has exhausted its retry budget — clears
|
|
648
|
-
* the clientJobs map entry to prevent leaks and resets job heartbeats so
|
|
649
|
-
* the stall detector recovers orphaned active jobs on its next tick.
|
|
650
|
-
*/
|
|
651
|
-
forceReleaseClientJobs(clientId) {
|
|
652
|
-
return lockMgr.forceReleaseClientJobs(clientId, this.contextFactory.getLockContext());
|
|
653
|
-
}
|
|
654
|
-
// ============ Query Operations ============
|
|
655
|
-
async getJob(jobId) {
|
|
656
|
-
return queryOps.getJob(jobId, this.contextFactory.getQueryContext());
|
|
657
|
-
}
|
|
658
|
-
async getJobState(jobId) {
|
|
659
|
-
return queryOps.getJobState(jobId, this.contextFactory.getQueryContext());
|
|
660
|
-
}
|
|
661
|
-
getResult(jobId) {
|
|
662
|
-
return queryOps.getJobResult(jobId, this.contextFactory.getQueryContext());
|
|
663
|
-
}
|
|
664
|
-
/**
|
|
665
|
-
* Get return values from all children of a parent job.
|
|
666
|
-
* Returns a Record where keys are job keys (queueName:jobId) and values are return values.
|
|
667
|
-
* BullMQ v5 compatible.
|
|
668
|
-
*/
|
|
669
|
-
async getChildrenValues(parentJobId) {
|
|
670
|
-
const job = await this.getJob(parentJobId);
|
|
671
|
-
if (!job?.childrenIds || job.childrenIds.length === 0) {
|
|
672
|
-
return {};
|
|
673
|
-
}
|
|
674
|
-
const ctx = this.contextFactory.getQueryContext();
|
|
675
|
-
const results = {};
|
|
676
|
-
for (const childId of job.childrenIds) {
|
|
677
|
-
const result = queryOps.getJobResult(childId, ctx);
|
|
678
|
-
if (result !== undefined) {
|
|
679
|
-
// Get child job to find its queue name
|
|
680
|
-
const childJob = await this.getJob(childId);
|
|
681
|
-
const key = childJob ? `${childJob.queue}:${childId}` : childId;
|
|
682
|
-
results[key] = result;
|
|
683
|
-
}
|
|
684
|
-
}
|
|
685
|
-
return results;
|
|
686
|
-
}
|
|
687
|
-
/**
|
|
688
|
-
* Update a job's parent reference.
|
|
689
|
-
* Used by FlowProducer when creating flows where children need to reference parent.
|
|
690
|
-
*/
|
|
691
|
-
async updateJobParent(childJobId, parentJobId) {
|
|
692
|
-
if (childJobId === parentJobId)
|
|
693
|
-
throw new Error('A flow job cannot be its own parent');
|
|
694
|
-
let childJob = await this.getJob(childJobId);
|
|
695
|
-
const parentJob = await this.getJob(parentJobId);
|
|
696
|
-
if (!childJob) {
|
|
697
|
-
if (canAcceptRemovedFlowChild(parentJob, childJobId, this.depCompletions))
|
|
698
|
-
return;
|
|
699
|
-
throw new Error(`Child job not found: ${String(childJobId)}`);
|
|
700
|
-
}
|
|
701
|
-
if (!parentJob)
|
|
702
|
-
throw new Error(`Parent job not found: ${String(parentJobId)}`);
|
|
703
|
-
assertFlowParentOwnership(childJob, parentJobId);
|
|
704
|
-
if (isDeclaredFlowChild(parentJob, childJobId)) {
|
|
705
|
-
childJob = await backpatchDeclaredFlowChild(childJob, parentJob, {
|
|
706
|
-
storage: this.storage,
|
|
707
|
-
customIdLock: this.customIdLock,
|
|
708
|
-
shards: this.shards,
|
|
709
|
-
shardLocks: this.shardLocks,
|
|
710
|
-
processingShards: this.processingShards,
|
|
711
|
-
processingLocks: this.processingLocks,
|
|
712
|
-
jobIndex: this.jobIndex,
|
|
713
|
-
completedJobsData: this.completedJobsData,
|
|
714
|
-
depCompletions: this.depCompletions,
|
|
715
|
-
});
|
|
716
|
-
}
|
|
717
|
-
else {
|
|
718
|
-
const parentLocation = this.jobIndex.get(parentJobId);
|
|
719
|
-
if (parentLocation?.type !== 'queue') {
|
|
720
|
-
throw new Error(`Parent job ${String(parentJobId)} is not linkable`);
|
|
721
|
-
}
|
|
722
|
-
const childLocation = this.jobIndex.get(childJobId);
|
|
723
|
-
const shardIndexes = [
|
|
724
|
-
...new Set([shardIndex(childJob.queue), shardIndex(parentJob.queue)]),
|
|
725
|
-
].sort((a, b) => a - b);
|
|
726
|
-
const processingIndexes = childLocation?.type === 'processing' ? [processingShardIndex(childJobId)] : [];
|
|
727
|
-
const guards = [await this.customIdLock.acquireWrite()];
|
|
728
|
-
try {
|
|
729
|
-
for (const index of shardIndexes)
|
|
730
|
-
guards.push(await this.shardLocks[index].acquireWrite());
|
|
731
|
-
for (const index of processingIndexes) {
|
|
732
|
-
guards.push(await this.processingLocks[index].acquireWrite());
|
|
733
|
-
}
|
|
734
|
-
if (this.jobIndex.get(parentJobId)?.type !== 'queue') {
|
|
735
|
-
throw new Error(`Parent job ${String(parentJobId)} changed state while linking`);
|
|
736
|
-
}
|
|
737
|
-
const childData = {
|
|
738
|
-
...childJob.data,
|
|
739
|
-
__parentId: String(parentJobId),
|
|
740
|
-
__parentQueue: parentJob.queue,
|
|
741
|
-
};
|
|
742
|
-
const childrenIds = [...parentJob.childrenIds, childJobId];
|
|
743
|
-
const dependsOn = parentJob.dependsOn.includes(childJobId)
|
|
744
|
-
? [...parentJob.dependsOn]
|
|
745
|
-
: [...parentJob.dependsOn, childJobId];
|
|
746
|
-
const parentData = {
|
|
747
|
-
...parentJob.data,
|
|
748
|
-
__childrenIds: childrenIds.map(String),
|
|
749
|
-
};
|
|
750
|
-
const linkedChild = { ...childJob, data: childData, parentId: parentJobId };
|
|
751
|
-
const linkedParent = { ...parentJob, data: parentData, childrenIds, dependsOn };
|
|
752
|
-
const childFinished = this.completedJobs.has(childJobId) || this.depCompletions.has(childJobId);
|
|
753
|
-
const parentState = childFinished
|
|
754
|
-
? parentJob.runAt > Date.now()
|
|
755
|
-
? 'delayed'
|
|
756
|
-
: parentJob.priority > 0
|
|
757
|
-
? 'prioritized'
|
|
758
|
-
: 'waiting'
|
|
759
|
-
: 'waiting-children';
|
|
760
|
-
this.storage?.updateFlowLink(linkedChild, linkedParent, parentState);
|
|
761
|
-
childJob.parentId = parentJobId;
|
|
762
|
-
childJob.data = childData;
|
|
763
|
-
parentJob.childrenIds = childrenIds;
|
|
764
|
-
parentJob.dependsOn = dependsOn;
|
|
765
|
-
parentJob.data = parentData;
|
|
766
|
-
if (!childFinished) {
|
|
767
|
-
const shard = this.shards[shardIndex(parentJob.queue)];
|
|
768
|
-
if (!shard.waitingDeps.has(parentJobId)) {
|
|
769
|
-
const removed = shard.getQueue(parentJob.queue).remove(parentJobId);
|
|
770
|
-
if (removed)
|
|
771
|
-
shard.decrementQueued(parentJobId);
|
|
772
|
-
shard.waitingDeps.set(parentJobId, parentJob);
|
|
773
|
-
}
|
|
774
|
-
shard.registerDependencies(parentJobId, [childJobId]);
|
|
775
|
-
this.dependencyResults.registerConsumer(parentJobId, dependsOn);
|
|
776
|
-
}
|
|
777
|
-
}
|
|
778
|
-
finally {
|
|
779
|
-
for (let index = guards.length - 1; index >= 0; index--)
|
|
780
|
-
guards[index].release();
|
|
781
|
-
}
|
|
782
|
-
}
|
|
783
|
-
// Handle race condition: child may have already terminally failed
|
|
784
|
-
// before parent linkage was established (parentId was 'pending').
|
|
785
|
-
// If so, propagate failParentOnFailure now with the real parent ID.
|
|
786
|
-
const childLoc = this.jobIndex.get(childJobId);
|
|
787
|
-
const childFailureError = childLoc?.type === 'dlq'
|
|
788
|
-
? (flowChildFailureError(childJob, this.shards) ?? 'Child job failed')
|
|
789
|
-
: 'Child job failed';
|
|
790
|
-
if (childLoc?.type === 'dlq' && childJob.failParentOnFailure) {
|
|
791
|
-
await this.moveParentToFailed(parentJobId, childJob, childFailureError);
|
|
792
|
-
}
|
|
793
|
-
if (childLoc?.type === 'dlq' &&
|
|
794
|
-
(childJob.removeDependencyOnFailure ||
|
|
795
|
-
childJob.ignoreDependencyOnFailure ||
|
|
796
|
-
childJob.continueParentOnFailure)) {
|
|
797
|
-
await this.onChildDependencyOption(childJob, childFailureError);
|
|
798
|
-
}
|
|
799
|
-
}
|
|
800
|
-
getJobByCustomId(customId) {
|
|
801
|
-
return queryOps.getJobByCustomId(customId, this.contextFactory.getQueryContext());
|
|
802
|
-
}
|
|
803
|
-
getProgress(jobId) {
|
|
804
|
-
return queryOps.getJobProgress(jobId, this.contextFactory.getQueryContext());
|
|
805
|
-
}
|
|
806
|
-
count(queue) {
|
|
807
|
-
return queueControl.getQueueCount(queue, this.contextFactory.getQueueControlContext());
|
|
808
|
-
}
|
|
809
|
-
// ============ Queue Control ============
|
|
810
|
-
pause(queue) {
|
|
811
|
-
queueControl.pauseQueue(queue, this.contextFactory.getQueueControlContext());
|
|
812
|
-
this.persistQueueState(queue);
|
|
813
|
-
this.dashboardEmit?.('queue:paused', { queue });
|
|
814
|
-
this.eventsManager.broadcast({
|
|
815
|
-
eventType: "paused" /* EventType.Paused */,
|
|
816
|
-
queue,
|
|
817
|
-
jobId: '',
|
|
818
|
-
timestamp: Date.now(),
|
|
819
|
-
});
|
|
820
|
-
}
|
|
821
|
-
resume(queue) {
|
|
822
|
-
queueControl.resumeQueue(queue, this.contextFactory.getQueueControlContext());
|
|
823
|
-
this.persistQueueState(queue);
|
|
824
|
-
this.dashboardEmit?.('queue:resumed', { queue });
|
|
825
|
-
this.eventsManager.broadcast({
|
|
826
|
-
eventType: "resumed" /* EventType.Resumed */,
|
|
827
|
-
queue,
|
|
828
|
-
jobId: '',
|
|
829
|
-
timestamp: Date.now(),
|
|
830
|
-
});
|
|
831
|
-
}
|
|
832
|
-
isPaused(queue) {
|
|
833
|
-
return queueControl.isQueuePaused(queue, this.contextFactory.getQueueControlContext());
|
|
834
|
-
}
|
|
835
|
-
drain(queue) {
|
|
836
|
-
const count = queueControl.drainQueue(queue, this.contextFactory.getQueueControlContext());
|
|
837
|
-
if (count > 0)
|
|
838
|
-
this.dashboardEmit?.('queue:drained', { queue, count });
|
|
839
|
-
return count;
|
|
840
|
-
}
|
|
841
|
-
obliterate(queue) {
|
|
842
|
-
const shardJobs = queueControl.obliterateQueue(queue, this.contextFactory.getQueueControlContext());
|
|
843
|
-
dlqOps.purgeDlqJobs(queue, this.contextFactory.getDlqContext());
|
|
844
|
-
// obliterateQueue() returns every queued, DLQ, and dependency-gated job it
|
|
845
|
-
// removed. Active jobs in processingShards, plus completed/result/log/lock
|
|
846
|
-
// state in global indexes, still need to be discovered and purged here.
|
|
847
|
-
const toDrop = new Set(shardJobs);
|
|
848
|
-
for (const [jid, loc] of this.jobIndex) {
|
|
849
|
-
if (loc.type === 'processing') {
|
|
850
|
-
const job = this.processingShards[loc.shardIdx]?.get(jid);
|
|
851
|
-
if (job?.queue === queue)
|
|
852
|
-
toDrop.add(jid);
|
|
853
|
-
}
|
|
854
|
-
else if (loc.queueName === queue) {
|
|
855
|
-
toDrop.add(jid);
|
|
856
|
-
}
|
|
857
|
-
}
|
|
858
|
-
for (const jid of toDrop) {
|
|
859
|
-
const loc = this.jobIndex.get(jid);
|
|
860
|
-
if (loc?.type === 'processing') {
|
|
861
|
-
this.processingShards[loc.shardIdx]?.delete(jid);
|
|
862
|
-
}
|
|
863
|
-
this.jobIndex.delete(jid);
|
|
864
|
-
this.completedJobs.delete(jid);
|
|
865
|
-
this.completedJobsData.delete(jid);
|
|
866
|
-
this.jobResults.delete(jid);
|
|
867
|
-
this.jobLogs.delete(jid);
|
|
868
|
-
this.jobLocks.delete(jid);
|
|
869
|
-
this.dependencyResults.releaseConsumer(jid);
|
|
870
|
-
this.failedChildrenValues.delete(jid);
|
|
871
|
-
this.ignoredChildrenFailures.delete(jid);
|
|
872
|
-
this.pendingDepChecks.delete(jid);
|
|
873
|
-
this.stalledCandidates.delete(jid);
|
|
874
|
-
this.repeatChain.delete(jid);
|
|
875
|
-
this.storage?.deleteJob(jid);
|
|
876
|
-
}
|
|
877
|
-
// repeatChain maps oldId → newId; drop rows whose value is now a ghost.
|
|
878
|
-
const chainKeysToDelete = [];
|
|
879
|
-
for (const [oldId, newId] of this.repeatChain) {
|
|
880
|
-
if (toDrop.has(newId))
|
|
881
|
-
chainKeysToDelete.push(oldId);
|
|
882
|
-
}
|
|
883
|
-
for (const oldId of chainKeysToDelete)
|
|
884
|
-
this.repeatChain.delete(oldId);
|
|
885
|
-
// Drop customId → JobId mappings that point at a dropped job
|
|
886
|
-
const customIdsToDelete = [];
|
|
887
|
-
for (const [cid, jid] of this.customIdMap.entries()) {
|
|
888
|
-
if (toDrop.has(jid))
|
|
889
|
-
customIdsToDelete.push(cid);
|
|
890
|
-
}
|
|
891
|
-
for (const cid of customIdsToDelete)
|
|
892
|
-
this.customIdMap.delete(cid);
|
|
893
|
-
// removeOnComplete jobs have no jobIndex entry, but their bounded durable
|
|
894
|
-
// dependency proofs still belong to the obliterated queue.
|
|
895
|
-
const removedCompletions = this.storage?.deleteDependencyCompletionsForQueue(queue) ?? [];
|
|
896
|
-
for (const jobId of removedCompletions)
|
|
897
|
-
this.depCompletions.delete(jobId);
|
|
898
|
-
this.reconcileCompletionPins();
|
|
899
|
-
// Per-queue cumulative counters are keyed by queue name and never expire on
|
|
900
|
-
// their own; obliterate is the documented way to reclaim ALL state for a
|
|
901
|
-
// queue, so drop its metrics entry too (prevents unbounded growth for
|
|
902
|
-
// ephemeral/dynamically-named queues).
|
|
903
|
-
this.purgeQueueMetadata(queue);
|
|
904
|
-
this.unregisterQueueName(queue);
|
|
905
|
-
this.dashboardEmit?.('queue:obliterated', { queue });
|
|
906
|
-
this.dashboardEmit?.('queue:removed', { queue });
|
|
907
|
-
}
|
|
908
|
-
/**
|
|
909
|
-
* Drop per-queue metadata that obliterate is responsible for reclaiming:
|
|
910
|
-
* cumulative metrics (keyed by name, never self-expiring) and the persisted
|
|
911
|
-
* control-state row (#100 — so a stale pause/limit can't resurrect on the
|
|
912
|
-
* next restart).
|
|
913
|
-
*/
|
|
914
|
-
purgeQueueMetadata(queue) {
|
|
915
|
-
this.perQueueMetrics.delete(queue);
|
|
916
|
-
this.storage?.deleteQueueState(queue);
|
|
917
|
-
}
|
|
918
|
-
listQueues() {
|
|
919
|
-
return Array.from(this.queueNamesCache);
|
|
920
|
-
}
|
|
921
|
-
registerQueueName(queue) {
|
|
922
|
-
const isNew = !this.queueNamesCache.has(queue);
|
|
923
|
-
this.queueNamesCache.add(queue);
|
|
924
|
-
if (isNew)
|
|
925
|
-
this.dashboardEmit?.('queue:created', { queue });
|
|
926
|
-
}
|
|
927
|
-
unregisterQueueName(queue) {
|
|
928
|
-
this.queueNamesCache.delete(queue);
|
|
929
|
-
}
|
|
930
|
-
releaseCompletionPins(dependencyIds) {
|
|
931
|
-
releaseDependencyCompletionPins(dependencyIds, {
|
|
932
|
-
storage: this.storage,
|
|
933
|
-
shards: this.shards,
|
|
934
|
-
depCompletions: this.depCompletions,
|
|
935
|
-
maxDependencyCompletions: this.config.maxCompletedJobs,
|
|
936
|
-
});
|
|
937
|
-
}
|
|
938
|
-
reconcileCompletionPins() {
|
|
939
|
-
reconcileDependencyCompletionPins({
|
|
940
|
-
storage: this.storage,
|
|
941
|
-
shards: this.shards,
|
|
942
|
-
depCompletions: this.depCompletions,
|
|
943
|
-
maxDependencyCompletions: this.config.maxCompletedJobs,
|
|
944
|
-
});
|
|
945
|
-
}
|
|
946
|
-
clean(queue, graceMs, state, limit) {
|
|
947
|
-
return queueControl.cleanQueue(queue, graceMs, this.contextFactory.getQueueControlContext(), state, limit);
|
|
948
|
-
}
|
|
949
|
-
getCountsPerPriority(queue) {
|
|
950
|
-
const idx = shardIndex(queue);
|
|
951
|
-
const counts = this.shards[idx].getCountsPerPriority(queue);
|
|
952
|
-
return Object.fromEntries(counts);
|
|
953
|
-
}
|
|
954
|
-
getJobs(queue, options = {}) {
|
|
955
|
-
const idx = shardIndex(queue);
|
|
956
|
-
return queryOps.getJobs(queue, idx, options, {
|
|
957
|
-
...this.contextFactory.getQueryContext(),
|
|
958
|
-
shardCount: SHARD_COUNT,
|
|
959
|
-
});
|
|
960
|
-
}
|
|
961
|
-
// ============ DLQ Operations ============
|
|
962
|
-
getDlq(queue, count) {
|
|
963
|
-
return dlqOps.getDlqJobs(queue, this.contextFactory.getDlqContext(), count);
|
|
964
|
-
}
|
|
965
|
-
getDlqEntries(queue, filter) {
|
|
966
|
-
return dlqOps.getDlqEntries(queue, this.contextFactory.getDlqContext(), filter);
|
|
967
|
-
}
|
|
968
|
-
getDlqCount(queue) {
|
|
969
|
-
return this.shards[shardIndex(queue)].getDlqCount(queue);
|
|
970
|
-
}
|
|
971
|
-
getDlqStats(queue) {
|
|
972
|
-
return dlqOps.getDlqStats(queue, this.contextFactory.getDlqContext());
|
|
973
|
-
}
|
|
974
|
-
retryDlq(queue, jobId, limit) {
|
|
975
|
-
return dlqOps.retryDlqJobs(queue, this.contextFactory.getDlqContext(), jobId, limit);
|
|
976
|
-
}
|
|
977
|
-
purgeDlq(queue) {
|
|
978
|
-
return dlqOps.purgeDlqJobs(queue, this.contextFactory.getDlqContext());
|
|
979
|
-
}
|
|
980
|
-
retryCompleted(queue, jobId) {
|
|
981
|
-
return dlqOps.retryCompletedJobs(queue, this.contextFactory.getRetryCompletedContext(), jobId);
|
|
982
|
-
}
|
|
983
|
-
// ============ Rate Limiting ============
|
|
984
|
-
setRateLimit(queue, limit, durationMs, ttlMs) {
|
|
985
|
-
this.shards[shardIndex(queue)].setRateLimit(queue, limit, durationMs, ttlMs);
|
|
986
|
-
this.persistQueueState(queue);
|
|
987
|
-
}
|
|
988
|
-
clearRateLimit(queue) {
|
|
989
|
-
this.shards[shardIndex(queue)].clearRateLimit(queue);
|
|
990
|
-
this.persistQueueState(queue);
|
|
991
|
-
}
|
|
992
|
-
setConcurrency(queue, limit) {
|
|
993
|
-
this.shards[shardIndex(queue)].setConcurrency(queue, limit);
|
|
994
|
-
this.persistQueueState(queue);
|
|
995
|
-
}
|
|
996
|
-
clearConcurrency(queue) {
|
|
997
|
-
this.shards[shardIndex(queue)].clearConcurrency(queue);
|
|
998
|
-
this.persistQueueState(queue);
|
|
999
|
-
}
|
|
1000
|
-
/**
|
|
1001
|
-
* Issue #100: write-through the current control-state (paused / rate-limit /
|
|
1002
|
-
* concurrency) to the `queue_state` table so it survives a server restart.
|
|
1003
|
-
* Reads the post-mutation state from the owning shard and UPSERTs the row.
|
|
1004
|
-
*/
|
|
1005
|
-
persistQueueState(queue) {
|
|
1006
|
-
if (!this.storage)
|
|
1007
|
-
return;
|
|
1008
|
-
const state = this.shards[shardIndex(queue)].getState(queue);
|
|
1009
|
-
const stallConfig = this.shards[shardIndex(queue)].getStallConfig(queue);
|
|
1010
|
-
const dlqConfig = this.shards[shardIndex(queue)].getDlqConfig(queue);
|
|
1011
|
-
const hasCustomStallConfig = stallConfig.enabled !== DEFAULT_STALL_CONFIG.enabled ||
|
|
1012
|
-
stallConfig.stallInterval !== DEFAULT_STALL_CONFIG.stallInterval ||
|
|
1013
|
-
stallConfig.maxStalls !== DEFAULT_STALL_CONFIG.maxStalls ||
|
|
1014
|
-
stallConfig.gracePeriod !== DEFAULT_STALL_CONFIG.gracePeriod;
|
|
1015
|
-
const hasCustomDlqConfig = dlqConfig.autoRetry !== DEFAULT_DLQ_CONFIG.autoRetry ||
|
|
1016
|
-
dlqConfig.autoRetryInterval !== DEFAULT_DLQ_CONFIG.autoRetryInterval ||
|
|
1017
|
-
dlqConfig.maxAutoRetries !== DEFAULT_DLQ_CONFIG.maxAutoRetries ||
|
|
1018
|
-
dlqConfig.maxAge !== DEFAULT_DLQ_CONFIG.maxAge ||
|
|
1019
|
-
dlqConfig.maxEntries !== DEFAULT_DLQ_CONFIG.maxEntries;
|
|
1020
|
-
// When control-state returns fully to default (not paused, no limits), drop
|
|
1021
|
-
// the row instead of persisting an all-default placeholder. Keeps the table
|
|
1022
|
-
// free of noise rows for ephemeral queues that only ever call resume/clear*,
|
|
1023
|
-
// and recovers identically (absent row → default state).
|
|
1024
|
-
if (!state.paused &&
|
|
1025
|
-
state.rateLimit === null &&
|
|
1026
|
-
state.concurrencyLimit === null &&
|
|
1027
|
-
!hasCustomStallConfig &&
|
|
1028
|
-
!hasCustomDlqConfig) {
|
|
1029
|
-
this.storage.deleteQueueState(queue);
|
|
1030
|
-
return;
|
|
1031
|
-
}
|
|
1032
|
-
this.storage.saveQueueState(queue, {
|
|
1033
|
-
paused: state.paused,
|
|
1034
|
-
rateLimit: state.rateLimit,
|
|
1035
|
-
concurrencyLimit: state.concurrencyLimit,
|
|
1036
|
-
rateLimitDuration: state.rateLimitDuration,
|
|
1037
|
-
rateLimitExpiresAt: state.rateLimitExpiresAt,
|
|
1038
|
-
stallConfig,
|
|
1039
|
-
dlqConfig,
|
|
1040
|
-
});
|
|
1041
|
-
}
|
|
1042
|
-
/** Get rate limit and concurrency limit for a queue */
|
|
1043
|
-
getQueueLimits(queue) {
|
|
1044
|
-
const shard = this.shards[shardIndex(queue)];
|
|
1045
|
-
// Lazy TTL expiry so reads never report a limit that no longer throttles.
|
|
1046
|
-
shard.expireRateLimitIfNeeded(queue);
|
|
1047
|
-
const state = shard.getState(queue);
|
|
1048
|
-
return { rateLimit: state.rateLimit, concurrencyLimit: state.concurrencyLimit };
|
|
1049
|
-
}
|
|
1050
|
-
/** Get all job results (for cloud telemetry) */
|
|
1051
|
-
getAllJobResults() {
|
|
1052
|
-
const map = new Map();
|
|
1053
|
-
for (const [k, v] of this.jobResults.entries())
|
|
1054
|
-
map.set(k, v);
|
|
1055
|
-
return map;
|
|
1056
|
-
}
|
|
1057
|
-
/** Get all job logs (for cloud telemetry) */
|
|
1058
|
-
getAllJobLogs() {
|
|
1059
|
-
const map = new Map();
|
|
1060
|
-
for (const [k, v] of this.jobLogs.entries())
|
|
1061
|
-
map.set(k, v);
|
|
1062
|
-
return map;
|
|
1063
|
-
}
|
|
1064
|
-
/** Get all active job locks (for cloud telemetry) */
|
|
1065
|
-
getAllJobLocks() {
|
|
1066
|
-
return this.jobLocks;
|
|
1067
|
-
}
|
|
1068
|
-
// ============ Stall & DLQ Config ============
|
|
1069
|
-
setStallConfig(queue, config) {
|
|
1070
|
-
this.shards[shardIndex(queue)].setStallConfig(queue, config);
|
|
1071
|
-
this.persistQueueState(queue);
|
|
1072
|
-
}
|
|
1073
|
-
getStallConfig(queue) {
|
|
1074
|
-
return this.shards[shardIndex(queue)].getStallConfig(queue);
|
|
1075
|
-
}
|
|
1076
|
-
setDlqConfig(queue, config) {
|
|
1077
|
-
this.shards[shardIndex(queue)].setDlqConfig(queue, config);
|
|
1078
|
-
this.persistQueueState(queue);
|
|
1079
|
-
}
|
|
1080
|
-
getDlqConfig(queue) {
|
|
1081
|
-
return this.shards[shardIndex(queue)].getDlqConfig(queue);
|
|
1082
|
-
}
|
|
1083
|
-
/** Get extended telemetry data for cloud snapshot */
|
|
1084
|
-
getCloudTelemetry(queueNames) {
|
|
1085
|
-
const perQueue = {};
|
|
1086
|
-
for (const name of queueNames) {
|
|
1087
|
-
const idx = shardIndex(name);
|
|
1088
|
-
const shard = this.shards[idx];
|
|
1089
|
-
const uniqueMap = shard.uniqueKeys.get(name);
|
|
1090
|
-
const groupSet = shard.activeGroups.get(name);
|
|
1091
|
-
let waitingDeps = 0;
|
|
1092
|
-
for (const j of shard.waitingDeps.values()) {
|
|
1093
|
-
if (j.queue === name)
|
|
1094
|
-
waitingDeps++;
|
|
1095
|
-
}
|
|
1096
|
-
let waitingChildren = 0;
|
|
1097
|
-
for (const j of shard.waitingChildren.values()) {
|
|
1098
|
-
if (j.queue === name)
|
|
1099
|
-
waitingChildren++;
|
|
1100
|
-
}
|
|
1101
|
-
perQueue[name] = {
|
|
1102
|
-
uniqueKeys: uniqueMap?.size ?? 0,
|
|
1103
|
-
activeGroups: groupSet?.size ?? 0,
|
|
1104
|
-
waitingDeps,
|
|
1105
|
-
waitingChildren,
|
|
1106
|
-
};
|
|
1107
|
-
}
|
|
1108
|
-
return {
|
|
1109
|
-
perQueue,
|
|
1110
|
-
eventSubscribers: this.eventsManager.subscriberCount,
|
|
1111
|
-
pendingDepChecks: this.pendingDepChecks.size,
|
|
1112
|
-
};
|
|
1113
|
-
}
|
|
1114
|
-
// ============ Job Management ============
|
|
1115
|
-
async cancel(jobId) {
|
|
1116
|
-
return jobMgmt.cancelJob(jobId, this.contextFactory.getJobMgmtContext());
|
|
1117
|
-
}
|
|
1118
|
-
async updateProgress(jobId, progress, message) {
|
|
1119
|
-
return jobMgmt.updateJobProgress(jobId, progress, this.contextFactory.getJobMgmtContext(), message);
|
|
1120
|
-
}
|
|
1121
|
-
async updateJobData(jobId, data) {
|
|
1122
|
-
return jobMgmt.updateJobData(jobId, data, this.contextFactory.getJobMgmtContext());
|
|
1123
|
-
}
|
|
1124
|
-
async changePriority(jobId, priority, lifo) {
|
|
1125
|
-
return jobMgmt.changeJobPriority(jobId, priority, this.contextFactory.getJobMgmtContext(), lifo);
|
|
1126
|
-
}
|
|
1127
|
-
async promote(jobId) {
|
|
1128
|
-
return jobPromotion.promoteJob(jobId, this.contextFactory.getJobMgmtContext());
|
|
1129
|
-
}
|
|
1130
|
-
async promoteJobs(queue, count) {
|
|
1131
|
-
return jobPromotion.promoteJobs(queue, count, this.contextFactory.getJobMgmtContext());
|
|
1132
|
-
}
|
|
1133
|
-
async moveToDelayed(jobId, delay) {
|
|
1134
|
-
// moveToDelayed and changeDelay share identical routing in this engine: a job
|
|
1135
|
-
// already in the queue (waiting/prioritized/delayed) gets its runAt updated in
|
|
1136
|
-
// place so it becomes (or stays) delayed; an active/processing job is moved back
|
|
1137
|
-
// to the queue with the new delay. Delegate to changeDelay so the in-queue case
|
|
1138
|
-
// is handled — previously moveJobToDelayed only handled `processing` jobs, making
|
|
1139
|
-
// a WAITING job a silent no-op over TCP/HTTP/MCP. Mirrors the embedded client's
|
|
1140
|
-
// moveJobToDelayed branching (changeWaitingDelay for in-queue, moveJobToDelayed
|
|
1141
|
-
// for active). See src/client/queue/jobMove.ts.
|
|
1142
|
-
return this.changeDelay(jobId, delay);
|
|
1143
|
-
}
|
|
1144
|
-
async changeDelay(jobId, delay) {
|
|
1145
|
-
const ctx = this.contextFactory.getJobMgmtContext();
|
|
1146
|
-
const loc = ctx.jobIndex.get(jobId);
|
|
1147
|
-
// Jobs already in queue (waiting/delayed): mutate runAt in place
|
|
1148
|
-
if (loc?.type === 'queue') {
|
|
1149
|
-
return jobTransitions.changeWaitingDelay(jobId, delay, ctx);
|
|
1150
|
-
}
|
|
1151
|
-
// Active/processing jobs: move back to queue with new delay
|
|
1152
|
-
return jobMgmt.moveJobToDelayed(jobId, delay, ctx);
|
|
1153
|
-
}
|
|
1154
|
-
async moveActiveToWait(jobId) {
|
|
1155
|
-
return jobTransitions.moveActiveToWait(jobId, this.contextFactory.getJobMgmtContext());
|
|
1156
|
-
}
|
|
1157
|
-
async changeWaitingDelay(jobId, delay) {
|
|
1158
|
-
return jobTransitions.changeWaitingDelay(jobId, delay, this.contextFactory.getJobMgmtContext());
|
|
1159
|
-
}
|
|
1160
|
-
async moveToWaitingChildren(jobId) {
|
|
1161
|
-
return jobTransitions.moveToWaitingChildren(jobId, this.contextFactory.getJobMgmtContext());
|
|
1162
|
-
}
|
|
1163
|
-
async extendLock(jobId, token, duration) {
|
|
1164
|
-
const jid = typeof jobId === 'string' ? jobId : jobId;
|
|
1165
|
-
const ctx = this.contextFactory.getLockContext();
|
|
1166
|
-
if (token) {
|
|
1167
|
-
// Use provided token for verification
|
|
1168
|
-
return lockMgr.renewJobLock(jid, token, ctx, duration);
|
|
1169
|
-
}
|
|
1170
|
-
// Fall back to looking up the token
|
|
1171
|
-
const lockInfo = lockMgr.getLockInfo(jid, ctx);
|
|
1172
|
-
if (lockInfo) {
|
|
1173
|
-
return lockMgr.renewJobLock(jid, lockInfo.token, ctx, duration);
|
|
1174
|
-
}
|
|
1175
|
-
return false;
|
|
1176
|
-
}
|
|
1177
|
-
async discard(jobId) {
|
|
1178
|
-
return jobMgmt.discardJob(jobId, this.contextFactory.getJobMgmtContext());
|
|
1179
|
-
}
|
|
1180
|
-
// ============ Job Logs ============
|
|
1181
|
-
addLog(jobId, message, level = 'info') {
|
|
1182
|
-
return logsOps.addJobLog(jobId, message, this.contextFactory.getLogsContext(), level);
|
|
1183
|
-
}
|
|
1184
|
-
getLogs(jobId) {
|
|
1185
|
-
return logsOps.getJobLogs(jobId, this.contextFactory.getLogsContext());
|
|
1186
|
-
}
|
|
1187
|
-
clearLogs(jobId, keepLogs) {
|
|
1188
|
-
logsOps.clearJobLogs(jobId, this.contextFactory.getLogsContext(), keepLogs);
|
|
1189
|
-
}
|
|
1190
|
-
// ============ Metrics ============
|
|
1191
|
-
getPerQueueStats() {
|
|
1192
|
-
return statsMgr.getPerQueueStats(this.contextFactory.getStatsContext(), this.queueNamesCache);
|
|
1193
|
-
}
|
|
1194
|
-
getPrometheusMetrics() {
|
|
1195
|
-
const storageStatus = this.getStorageStatus();
|
|
1196
|
-
const memory = process.memoryUsage();
|
|
1197
|
-
const queueSelection = selectPrometheusQueues(this.queueNamesCache, this.config.maxPrometheusQueues);
|
|
1198
|
-
return generatePrometheusMetrics(this.getStats(), this.workerManager, this.webhookManager, statsMgr.getPerQueueStats(this.contextFactory.getStatsContext(), queueSelection.selected), {
|
|
1199
|
-
storageDegraded: storageStatus.diskFull,
|
|
1200
|
-
storageDiskFull: storageStatus.diskFull,
|
|
1201
|
-
...(this.storage && { sqliteDatabaseSizeBytes: this.storage.getSize() }),
|
|
1202
|
-
processHeapUsedBytes: memory.heapUsed,
|
|
1203
|
-
processHeapTotalBytes: memory.heapTotal,
|
|
1204
|
-
processResidentMemoryBytes: memory.rss,
|
|
1205
|
-
perQueueMetricsExported: queueSelection.selected.size,
|
|
1206
|
-
perQueueMetricsOmitted: queueSelection.omitted,
|
|
1207
|
-
operational: this.operationalMetricsProvider?.(),
|
|
1208
|
-
});
|
|
1209
|
-
}
|
|
1210
|
-
/** Attach metrics owned by server components outside the queue state machine. */
|
|
1211
|
-
setOperationalMetricsProvider(provider) {
|
|
1212
|
-
this.operationalMetricsProvider = provider;
|
|
1213
|
-
}
|
|
1214
|
-
// ============ Cron Operations ============
|
|
1215
|
-
addCron(input) {
|
|
1216
|
-
// On upsert with preventOverlap, remove any orphaned queued job (#73).
|
|
1217
|
-
// Between disconnect and reconnect, a cron tick may push a job that no
|
|
1218
|
-
// worker consumes. When the client re-upserts, that stale job would be
|
|
1219
|
-
// picked up immediately by the new worker. Remove it so the cron
|
|
1220
|
-
// scheduler creates a fresh job at the correct next-run time.
|
|
1221
|
-
if (input.preventOverlap) {
|
|
1222
|
-
const uniqueKey = input.uniqueKey ?? `cron:${input.name}`;
|
|
1223
|
-
this.removeOrphanedCronJob(input.queue, uniqueKey);
|
|
1224
|
-
}
|
|
1225
|
-
const cron = this.cronScheduler.add(input);
|
|
1226
|
-
this.storage?.saveCron(cron);
|
|
1227
|
-
return cron;
|
|
1228
|
-
}
|
|
1229
|
-
/** Remove an orphaned cron job from the queue by its uniqueKey */
|
|
1230
|
-
removeOrphanedCronJob(queue, uniqueKey) {
|
|
1231
|
-
const idx = shardIndex(queue);
|
|
1232
|
-
const shard = this.shards[idx];
|
|
1233
|
-
const entry = shard.getUniqueKeyEntry(queue, uniqueKey);
|
|
1234
|
-
if (!entry)
|
|
1235
|
-
return;
|
|
1236
|
-
const jobId = entry.jobId;
|
|
1237
|
-
const location = this.jobIndex.get(jobId);
|
|
1238
|
-
// Only remove waiting/queued jobs, never processing jobs
|
|
1239
|
-
if (location?.type !== 'queue')
|
|
1240
|
-
return;
|
|
1241
|
-
const job = shard.getQueue(queue).remove(jobId);
|
|
1242
|
-
if (job) {
|
|
1243
|
-
shard.decrementQueued(jobId);
|
|
1244
|
-
shard.releaseUniqueKey(queue, uniqueKey);
|
|
1245
|
-
this.jobIndex.delete(jobId);
|
|
1246
|
-
this.storage?.deleteJob(jobId);
|
|
1247
|
-
}
|
|
1248
|
-
}
|
|
1249
|
-
removeCron(name) {
|
|
1250
|
-
const removed = this.cronScheduler.remove(name);
|
|
1251
|
-
if (removed)
|
|
1252
|
-
this.storage?.deleteCron(name);
|
|
1253
|
-
return removed;
|
|
1254
|
-
}
|
|
1255
|
-
getCron(name) {
|
|
1256
|
-
return this.cronScheduler.get(name);
|
|
1257
|
-
}
|
|
1258
|
-
listCrons() {
|
|
1259
|
-
return this.cronScheduler.list();
|
|
1260
|
-
}
|
|
1261
|
-
// ============ Events ============
|
|
1262
|
-
/** Register dashboard event emitter (for WS pub/sub) */
|
|
1263
|
-
setDashboardEmit(fn) {
|
|
1264
|
-
this.dashboardEmit = fn;
|
|
1265
|
-
this.cronScheduler.setDashboardEmit(fn);
|
|
1266
|
-
this.webhookManager.setDashboardEmit(fn);
|
|
1267
|
-
this.workerManager.setDashboardEmit(fn);
|
|
1268
|
-
if (this.recoveryStats) {
|
|
1269
|
-
fn('server:recovered', this.recoveryStats);
|
|
1270
|
-
this.recoveryStats = null;
|
|
1271
|
-
}
|
|
1272
|
-
}
|
|
1273
|
-
/** Emit a dashboard event (callable from handlers) */
|
|
1274
|
-
emitDashboardEvent(event, data) {
|
|
1275
|
-
this.dashboardEmit?.(event, data);
|
|
1276
|
-
}
|
|
1277
|
-
subscribe(callback) {
|
|
1278
|
-
return this.eventsManager.subscribe(callback);
|
|
1279
|
-
}
|
|
1280
|
-
waitForJobCompletion(jobId, timeoutMs) {
|
|
1281
|
-
return this.eventsManager.waitForJobCompletion(jobId, timeoutMs);
|
|
1282
|
-
}
|
|
1283
|
-
/** Register worker with dashboard event */
|
|
1284
|
-
registerWorker(name, queues, concurrency, opts) {
|
|
1285
|
-
const worker = this.workerManager.register(name, queues, concurrency, opts);
|
|
1286
|
-
this.dashboardEmit?.('worker:connected', {
|
|
1287
|
-
workerId: worker.id,
|
|
1288
|
-
name: worker.name,
|
|
1289
|
-
queues: worker.queues,
|
|
1290
|
-
hostname: worker.hostname,
|
|
1291
|
-
pid: worker.pid,
|
|
1292
|
-
});
|
|
1293
|
-
return worker;
|
|
1294
|
-
}
|
|
1295
|
-
/** Unregister worker with dashboard event */
|
|
1296
|
-
unregisterWorker(workerId) {
|
|
1297
|
-
const result = this.workerManager.unregister(workerId);
|
|
1298
|
-
if (result) {
|
|
1299
|
-
this.dashboardEmit?.('worker:disconnected', { workerId });
|
|
1300
|
-
}
|
|
1301
|
-
return result;
|
|
1302
|
-
}
|
|
1303
|
-
/** Unregister all workers associated with a TCP client ID */
|
|
1304
|
-
unregisterWorkersByClientId(clientId) {
|
|
1305
|
-
return this.workerManager.unregisterByClientId(clientId);
|
|
1306
|
-
}
|
|
1307
|
-
// ============ Internal State Access ============
|
|
1308
|
-
getJobIndex() {
|
|
1309
|
-
return this.jobIndex;
|
|
1310
|
-
}
|
|
1311
|
-
getCompletedJobs() {
|
|
1312
|
-
return this.completedJobs;
|
|
1313
|
-
}
|
|
1314
|
-
// Bare completion ids of removeOnComplete jobs. The PUSH gate consults this so a
|
|
1315
|
-
// late dependent on an evicted removeOnComplete parent is admitted (same window
|
|
1316
|
-
// the readiness path / dependency processor already honor).
|
|
1317
|
-
getDepCompletions() {
|
|
1318
|
-
return this.depCompletions;
|
|
1319
|
-
}
|
|
1320
|
-
getShards() {
|
|
1321
|
-
return this.shards;
|
|
1322
|
-
}
|
|
1323
|
-
onJobCompleted(completedId) {
|
|
1324
|
-
// Release flow-failure tracking once a parent job reaches terminal completion
|
|
1325
|
-
// (AUDIT H8). The parent consumes these values via getFailedChildrenValues()/
|
|
1326
|
-
// getIgnoredChildrenFailures() while it is processing (before it acks), so by the
|
|
1327
|
-
// time it completes they are no longer needed. Keyed by parentId; obliterate()
|
|
1328
|
-
// and shutdown() clear them too. Without this, every parent completion that
|
|
1329
|
-
// involved a failed child with continueParentOnFailure / ignoreDependencyOnFailure
|
|
1330
|
-
// leaked an entry permanently.
|
|
1331
|
-
this.failedChildrenValues.delete(completedId);
|
|
1332
|
-
this.ignoredChildrenFailures.delete(completedId);
|
|
1333
|
-
this.storage?.deleteFlowFailure(completedId);
|
|
1334
|
-
this.pendingDepChecks.add(completedId);
|
|
1335
|
-
this.scheduleDependencyFlush();
|
|
1336
|
-
void this.checkFlowCompleted(completedId);
|
|
1337
|
-
}
|
|
1338
|
-
/** Release failure metadata owned by a parent that reached terminal failure. */
|
|
1339
|
-
onJobFailed(failedId) {
|
|
1340
|
-
this.failedChildrenValues.delete(failedId);
|
|
1341
|
-
this.ignoredChildrenFailures.delete(failedId);
|
|
1342
|
-
this.storage?.deleteFlowFailure(failedId);
|
|
1343
|
-
}
|
|
1344
|
-
/** Check if completing this job completes an entire flow */
|
|
1345
|
-
async checkFlowCompleted(completedId) {
|
|
1346
|
-
const job = await this.getJob(completedId);
|
|
1347
|
-
if (!job?.parentId)
|
|
1348
|
-
return;
|
|
1349
|
-
const parent = await this.getJob(job.parentId);
|
|
1350
|
-
if (!parent?.childrenIds || parent.childrenIds.length === 0)
|
|
1351
|
-
return;
|
|
1352
|
-
const allDone = parent.childrenIds.every((childId) => this.completedJobs.has(childId));
|
|
1353
|
-
if (allDone) {
|
|
1354
|
-
this.dashboardEmit?.('flow:completed', {
|
|
1355
|
-
parentJobId: String(parent.id),
|
|
1356
|
-
queue: parent.queue,
|
|
1357
|
-
childrenCount: parent.childrenIds.length,
|
|
1358
|
-
});
|
|
1359
|
-
}
|
|
1360
|
-
}
|
|
1361
|
-
/**
|
|
1362
|
-
* BullMQ v5: failParentOnFailure — when a child terminally fails
|
|
1363
|
-
* and has failParentOnFailure: true, also fail the parent job.
|
|
1364
|
-
*/
|
|
1365
|
-
async failParentOnChildFailure(childJob, error) {
|
|
1366
|
-
const parentId = childJob.parentId;
|
|
1367
|
-
if (!parentId)
|
|
1368
|
-
return;
|
|
1369
|
-
await this.moveParentToFailed(parentId, childJob, error);
|
|
1370
|
-
}
|
|
1371
|
-
/** Move a parent job to DLQ/failed state because a child failed */
|
|
1372
|
-
async moveParentToFailed(parentId, childJob, error) {
|
|
1373
|
-
const parentJob = await this.getJob(parentId);
|
|
1374
|
-
if (!parentJob)
|
|
1375
|
-
return;
|
|
1376
|
-
const parentLoc = this.jobIndex.get(parentId);
|
|
1377
|
-
if (!parentLoc)
|
|
1378
|
-
return;
|
|
1379
|
-
// Only fail parent if it's in a queue (waitingDeps/waitingChildren) state
|
|
1380
|
-
if (parentLoc.type !== 'queue')
|
|
1381
|
-
return;
|
|
1382
|
-
const idx = shardIndex(parentJob.queue);
|
|
1383
|
-
let releasedDependencies = [];
|
|
1384
|
-
await withWriteLock(this.shardLocks[idx], () => {
|
|
1385
|
-
// Re-check inside lock to prevent duplicate DLQ entries (TOCTOU guard)
|
|
1386
|
-
if (this.jobIndex.get(parentId)?.type !== 'queue')
|
|
1387
|
-
return;
|
|
1388
|
-
const shard = this.shards[idx];
|
|
1389
|
-
// Remove from waitingDeps if present
|
|
1390
|
-
if (shard.waitingDeps.has(parentId)) {
|
|
1391
|
-
releasedDependencies = [...parentJob.dependsOn];
|
|
1392
|
-
shard.waitingDeps.delete(parentId);
|
|
1393
|
-
shard.unregisterDependencies(parentId, parentJob.dependsOn);
|
|
1394
|
-
}
|
|
1395
|
-
// Remove from waitingChildren if present
|
|
1396
|
-
if (shard.waitingChildren.has(parentId)) {
|
|
1397
|
-
shard.waitingChildren.delete(parentId);
|
|
1398
|
-
}
|
|
1399
|
-
// Remove from queue if present
|
|
1400
|
-
const queue = shard.getQueue(parentJob.queue);
|
|
1401
|
-
if (queue.find(parentId)) {
|
|
1402
|
-
queue.remove(parentId);
|
|
1403
|
-
shard.decrementQueued(parentId);
|
|
1404
|
-
}
|
|
1405
|
-
// Add parent to DLQ with child_failed reason
|
|
1406
|
-
const failError = `Child job ${childJob.id} failed: ${error ?? 'unknown error'}`;
|
|
1407
|
-
const entry = shard.addToDlq(parentJob, "unknown" /* FailureReason.Unknown */, failError);
|
|
1408
|
-
this.jobIndex.set(parentId, { type: 'dlq', queueName: parentJob.queue });
|
|
1409
|
-
this.storage?.saveDlqEntry(entry);
|
|
1410
|
-
this.storage?.deleteJob(parentId);
|
|
1411
|
-
this.storage?.deleteFlowFailure(parentId, childJob.id);
|
|
1412
|
-
});
|
|
1413
|
-
this.releaseCompletionPins(releasedDependencies);
|
|
1414
|
-
// Parent reached a terminal (DLQ) state — release its flow-failure tracking.
|
|
1415
|
-
this.failedChildrenValues.delete(parentId);
|
|
1416
|
-
this.ignoredChildrenFailures.delete(parentId);
|
|
1417
|
-
this.dependencyResults.releaseConsumer(parentId);
|
|
1418
|
-
// Broadcast failed event for parent
|
|
1419
|
-
this.eventsManager.broadcast({
|
|
1420
|
-
eventType: 'failed',
|
|
1421
|
-
queue: parentJob.queue,
|
|
1422
|
-
jobId: parentId,
|
|
1423
|
-
timestamp: Date.now(),
|
|
1424
|
-
error: `Child job ${childJob.id} failed: ${error ?? 'unknown error'}`,
|
|
1425
|
-
data: parentJob.data,
|
|
1426
|
-
});
|
|
1427
|
-
this.dashboardEmit?.('flow:failed', {
|
|
1428
|
-
parentJobId: String(parentId),
|
|
1429
|
-
failedChildId: String(childJob.id),
|
|
1430
|
-
queue: parentJob.queue,
|
|
1431
|
-
error: error ?? 'Child job failed',
|
|
1432
|
-
});
|
|
1433
|
-
}
|
|
1434
|
-
/**
|
|
1435
|
-
* Handle child dependency options: removeDependencyOnFailure, ignoreDependencyOnFailure, continueParentOnFailure
|
|
1436
|
-
*/
|
|
1437
|
-
async onChildDependencyOption(childJob, error) {
|
|
1438
|
-
if (!childJob.parentId)
|
|
1439
|
-
return;
|
|
1440
|
-
if (childJob.continueParentOnFailure) {
|
|
1441
|
-
await this.continueParentOnChildFailure(childJob, error);
|
|
1442
|
-
}
|
|
1443
|
-
else {
|
|
1444
|
-
// removeDependencyOnFailure or ignoreDependencyOnFailure
|
|
1445
|
-
await this.removeChildFromParentDeps(childJob, error, childJob.ignoreDependencyOnFailure);
|
|
1446
|
-
}
|
|
1447
|
-
}
|
|
1448
|
-
/**
|
|
1449
|
-
* continueParentOnFailure: move parent to queue when a child fails.
|
|
1450
|
-
* Stores the failure info for getFailedChildrenValues().
|
|
1451
|
-
*/
|
|
1452
|
-
async continueParentOnChildFailure(childJob, error) {
|
|
1453
|
-
const parentId = childJob.parentId;
|
|
1454
|
-
if (!parentId)
|
|
1455
|
-
return;
|
|
1456
|
-
const parentJob = await this.getJob(parentId);
|
|
1457
|
-
if (!parentJob)
|
|
1458
|
-
return;
|
|
1459
|
-
const parentLoc = this.jobIndex.get(parentId);
|
|
1460
|
-
if (parentLoc?.type !== 'queue')
|
|
1461
|
-
return;
|
|
1462
|
-
// Store failed child value
|
|
1463
|
-
const childKey = `${childJob.queue}:${childJob.id}`;
|
|
1464
|
-
const existing = this.failedChildrenValues.get(parentId) ?? {};
|
|
1465
|
-
existing[childKey] = error ?? 'unknown error';
|
|
1466
|
-
this.failedChildrenValues.set(parentId, existing);
|
|
1467
|
-
const idx = shardIndex(parentJob.queue);
|
|
1468
|
-
let releasedDependencies = [];
|
|
1469
|
-
await withWriteLock(this.shardLocks[idx], () => {
|
|
1470
|
-
if (this.jobIndex.get(parentId)?.type !== 'queue')
|
|
1471
|
-
return;
|
|
1472
|
-
const shard = this.shards[idx];
|
|
1473
|
-
const dependencies = [...parentJob.dependsOn];
|
|
1474
|
-
releasedDependencies = dependencies;
|
|
1475
|
-
shard.unregisterDependencies(parentId, dependencies);
|
|
1476
|
-
for (const dependency of dependencies) {
|
|
1477
|
-
this.dependencyResults.releaseDependency(parentId, dependency);
|
|
1478
|
-
}
|
|
1479
|
-
parentJob.dependsOn = [];
|
|
1480
|
-
this.storage?.updateFlowParentResolution(parentJob);
|
|
1481
|
-
});
|
|
1482
|
-
this.releaseCompletionPins(releasedDependencies);
|
|
1483
|
-
await this.promoteParentAfterChildFailure(parentId, parentJob, idx);
|
|
1484
|
-
}
|
|
1485
|
-
/**
|
|
1486
|
-
* Move a flow parent from its waiting state into the run queue after a child failure
|
|
1487
|
-
* (continueParentOnFailure / last-dependency resolution). Deferred to the next tick
|
|
1488
|
-
* so the failed child's worker batch drains first — the parent is then picked up on
|
|
1489
|
-
* a subsequent poll rather than racing to completion in the same synchronous cascade.
|
|
1490
|
-
*/
|
|
1491
|
-
async promoteParentAfterChildFailure(parentId, parentJob, idx) {
|
|
1492
|
-
await new Promise((resolve) => {
|
|
1493
|
-
const timer = setTimeout(resolve, 0);
|
|
1494
|
-
timer.unref?.();
|
|
1495
|
-
});
|
|
1496
|
-
let promoted = false;
|
|
1497
|
-
let releasedDependencies = [];
|
|
1498
|
-
await withWriteLock(this.shardLocks[idx], () => {
|
|
1499
|
-
// TOCTOU guard
|
|
1500
|
-
if (this.jobIndex.get(parentId)?.type !== 'queue')
|
|
1501
|
-
return;
|
|
1502
|
-
const shard = this.shards[idx];
|
|
1503
|
-
// Remove from waitingDeps
|
|
1504
|
-
if (shard.waitingDeps.has(parentId)) {
|
|
1505
|
-
releasedDependencies = [...parentJob.dependsOn];
|
|
1506
|
-
shard.waitingDeps.delete(parentId);
|
|
1507
|
-
shard.unregisterDependencies(parentId, parentJob.dependsOn);
|
|
1508
|
-
}
|
|
1509
|
-
// Remove from waitingChildren
|
|
1510
|
-
if (shard.waitingChildren.has(parentId)) {
|
|
1511
|
-
shard.waitingChildren.delete(parentId);
|
|
1512
|
-
}
|
|
1513
|
-
// Add to queue
|
|
1514
|
-
const queue = shard.getQueue(parentJob.queue);
|
|
1515
|
-
if (!queue.find(parentId)) {
|
|
1516
|
-
const now = Date.now();
|
|
1517
|
-
parentJob.runAt = now;
|
|
1518
|
-
queue.push(parentJob);
|
|
1519
|
-
shard.incrementQueued(parentId, false, parentJob.createdAt, parentJob.queue, now);
|
|
1520
|
-
this.jobIndex.set(parentId, { type: 'queue', shardIdx: idx, queueName: parentJob.queue });
|
|
1521
|
-
this.storage?.updateFlowParentResolution(parentJob);
|
|
1522
|
-
shard.notify(parentJob.queue);
|
|
1523
|
-
promoted = true;
|
|
1524
|
-
}
|
|
1525
|
-
});
|
|
1526
|
-
this.releaseCompletionPins(releasedDependencies);
|
|
1527
|
-
if (promoted) {
|
|
1528
|
-
this.eventsManager.broadcast({
|
|
1529
|
-
eventType: 'waiting',
|
|
1530
|
-
queue: parentJob.queue,
|
|
1531
|
-
jobId: parentId,
|
|
1532
|
-
timestamp: Date.now(),
|
|
1533
|
-
prev: 'waiting-children',
|
|
1534
|
-
});
|
|
1535
|
-
}
|
|
1536
|
-
}
|
|
1537
|
-
/**
|
|
1538
|
-
* removeDependencyOnFailure / ignoreDependencyOnFailure:
|
|
1539
|
-
* Remove child from parent's pending deps. If last dep, promote parent.
|
|
1540
|
-
* If ignoreDependencyOnFailure, also store failure reason.
|
|
1541
|
-
*/
|
|
1542
|
-
async removeChildFromParentDeps(childJob, error, storeIgnored) {
|
|
1543
|
-
const parentId = childJob.parentId;
|
|
1544
|
-
if (!parentId)
|
|
1545
|
-
return;
|
|
1546
|
-
const parentJob = await this.getJob(parentId);
|
|
1547
|
-
if (!parentJob)
|
|
1548
|
-
return;
|
|
1549
|
-
const parentLoc = this.jobIndex.get(parentId);
|
|
1550
|
-
if (parentLoc?.type !== 'queue')
|
|
1551
|
-
return;
|
|
1552
|
-
if (storeIgnored) {
|
|
1553
|
-
const childKey = `${childJob.queue}:${childJob.id}`;
|
|
1554
|
-
const existing = this.ignoredChildrenFailures.get(parentId) ?? {};
|
|
1555
|
-
existing[childKey] = error ?? 'unknown error';
|
|
1556
|
-
this.ignoredChildrenFailures.set(parentId, existing);
|
|
1557
|
-
}
|
|
1558
|
-
const idx = shardIndex(parentJob.queue);
|
|
1559
|
-
// Remove the failed child from the parent's pending deps synchronously so
|
|
1560
|
-
// dependency tracking stays consistent; defer only the promotion decision.
|
|
1561
|
-
let readyToPromote = false;
|
|
1562
|
-
let releasedDependency = null;
|
|
1563
|
-
await withWriteLock(this.shardLocks[idx], () => {
|
|
1564
|
-
if (this.jobIndex.get(parentId)?.type !== 'queue')
|
|
1565
|
-
return;
|
|
1566
|
-
const shard = this.shards[idx];
|
|
1567
|
-
const parentInDeps = shard.waitingDeps.get(parentId);
|
|
1568
|
-
if (!parentInDeps)
|
|
1569
|
-
return;
|
|
1570
|
-
// Remove child from parent's dependsOn
|
|
1571
|
-
const depIndex = parentJob.dependsOn.indexOf(childJob.id);
|
|
1572
|
-
if (depIndex !== -1) {
|
|
1573
|
-
parentJob.dependsOn.splice(depIndex, 1);
|
|
1574
|
-
shard.unregisterDependencies(parentId, [childJob.id]);
|
|
1575
|
-
releasedDependency = childJob.id;
|
|
1576
|
-
this.dependencyResults.releaseDependency(parentId, childJob.id);
|
|
1577
|
-
this.storage?.updateFlowParentResolution(parentJob);
|
|
1578
|
-
}
|
|
1579
|
-
// If no more pending deps, the parent is ready to be promoted
|
|
1580
|
-
readyToPromote =
|
|
1581
|
-
parentJob.dependsOn.length === 0 ||
|
|
1582
|
-
parentJob.dependsOn.every((dep) => this.completedJobs.has(dep));
|
|
1583
|
-
});
|
|
1584
|
-
if (releasedDependency)
|
|
1585
|
-
this.releaseCompletionPins([releasedDependency]);
|
|
1586
|
-
if (readyToPromote) {
|
|
1587
|
-
await this.promoteParentAfterChildFailure(parentId, parentJob, idx);
|
|
1588
|
-
}
|
|
1589
|
-
if (!storeIgnored)
|
|
1590
|
-
this.storage?.deleteFlowFailure(parentId, childJob.id);
|
|
1591
|
-
}
|
|
1592
|
-
/**
|
|
1593
|
-
* Get failed children values for a parent job (populated by continueParentOnFailure).
|
|
1594
|
-
*/
|
|
1595
|
-
async getFailedChildrenValues(parentJobId) {
|
|
1596
|
-
return this.failedChildrenValues.get(parentJobId) ?? {};
|
|
1597
|
-
}
|
|
1598
|
-
/**
|
|
1599
|
-
* Get ignored children failures for a parent job (populated by ignoreDependencyOnFailure).
|
|
1600
|
-
*/
|
|
1601
|
-
async getIgnoredChildrenFailures(parentJobId) {
|
|
1602
|
-
return this.ignoredChildrenFailures.get(parentJobId) ?? {};
|
|
1603
|
-
}
|
|
1604
|
-
/**
|
|
1605
|
-
* Remove a child job's dependency from its parent.
|
|
1606
|
-
* If this was the last pending child, promotes parent to queue.
|
|
1607
|
-
* Throws if the job has no parent.
|
|
1608
|
-
*/
|
|
1609
|
-
async removeChildDependency(childJobId) {
|
|
1610
|
-
const childJob = await this.getJob(childJobId);
|
|
1611
|
-
if (!childJob)
|
|
1612
|
-
throw new Error(`Job not found: ${childJobId}`);
|
|
1613
|
-
if (!childJob.parentId)
|
|
1614
|
-
throw new Error(`Job ${childJobId} has no parent`);
|
|
1615
|
-
const parentId = childJob.parentId;
|
|
1616
|
-
const parentJob = await this.getJob(parentId);
|
|
1617
|
-
if (!parentJob)
|
|
1618
|
-
return false;
|
|
1619
|
-
const parentLoc = this.jobIndex.get(parentId);
|
|
1620
|
-
if (parentLoc?.type !== 'queue')
|
|
1621
|
-
return false;
|
|
1622
|
-
const childLoc = this.jobIndex.get(childJobId);
|
|
1623
|
-
if (childLoc?.type !== 'queue' && childLoc?.type !== 'processing')
|
|
1624
|
-
return false;
|
|
1625
|
-
const parentIdx = shardIndex(parentJob.queue);
|
|
1626
|
-
const shardIndexes = [...new Set([parentIdx, shardIndex(childJob.queue)])].sort((a, b) => a - b);
|
|
1627
|
-
const processingIndexes = childLoc.type === 'processing' ? [processingShardIndex(childJobId)] : [];
|
|
1628
|
-
const guards = [];
|
|
1629
|
-
let removed = false;
|
|
1630
|
-
let promoted = false;
|
|
1631
|
-
let releasedDependencies = [];
|
|
1632
|
-
try {
|
|
1633
|
-
for (const index of shardIndexes)
|
|
1634
|
-
guards.push(await this.shardLocks[index].acquireWrite());
|
|
1635
|
-
for (const index of processingIndexes) {
|
|
1636
|
-
guards.push(await this.processingLocks[index].acquireWrite());
|
|
1637
|
-
}
|
|
1638
|
-
if (this.jobIndex.get(parentId)?.type !== 'queue' ||
|
|
1639
|
-
this.jobIndex.get(childJobId)?.type !== childLoc.type) {
|
|
1640
|
-
return false;
|
|
1641
|
-
}
|
|
1642
|
-
const shard = this.shards[parentIdx];
|
|
1643
|
-
if (!shard.waitingDeps.has(parentId) || !parentJob.dependsOn.includes(childJobId)) {
|
|
1644
|
-
return false;
|
|
1645
|
-
}
|
|
1646
|
-
const unresolved = parentJob.dependsOn.filter((dependency) => dependency !== childJobId &&
|
|
1647
|
-
!this.completedJobs.has(dependency) &&
|
|
1648
|
-
!this.depCompletions.has(dependency));
|
|
1649
|
-
const released = parentJob.dependsOn.filter((dependency) => !unresolved.includes(dependency));
|
|
1650
|
-
releasedDependencies = released;
|
|
1651
|
-
const childrenIds = parentJob.childrenIds.filter((childId) => childId !== childJobId);
|
|
1652
|
-
const childData = { ...childJob.data };
|
|
1653
|
-
delete childData.__parentId;
|
|
1654
|
-
delete childData.__parentQueue;
|
|
1655
|
-
const parentData = { ...parentJob.data };
|
|
1656
|
-
if (childrenIds.length > 0)
|
|
1657
|
-
parentData.__childrenIds = childrenIds.map(String);
|
|
1658
|
-
else
|
|
1659
|
-
delete parentData.__childrenIds;
|
|
1660
|
-
const runAt = unresolved.length === 0 ? Date.now() : parentJob.runAt;
|
|
1661
|
-
const parentState = unresolved.length > 0
|
|
1662
|
-
? 'waiting-children'
|
|
1663
|
-
: parentJob.priority > 0
|
|
1664
|
-
? 'prioritized'
|
|
1665
|
-
: 'waiting';
|
|
1666
|
-
const detachedChild = { ...childJob, parentId: null, data: childData };
|
|
1667
|
-
const detachedParent = {
|
|
1668
|
-
...parentJob,
|
|
1669
|
-
childrenIds,
|
|
1670
|
-
dependsOn: unresolved,
|
|
1671
|
-
data: parentData,
|
|
1672
|
-
runAt,
|
|
1673
|
-
};
|
|
1674
|
-
this.storage?.removeFlowLink(detachedChild, detachedParent, parentState);
|
|
1675
|
-
childJob.parentId = null;
|
|
1676
|
-
childJob.data = childData;
|
|
1677
|
-
parentJob.childrenIds = childrenIds;
|
|
1678
|
-
parentJob.dependsOn = unresolved;
|
|
1679
|
-
parentJob.data = parentData;
|
|
1680
|
-
parentJob.runAt = runAt;
|
|
1681
|
-
shard.unregisterDependencies(parentId, released);
|
|
1682
|
-
for (const dependency of released) {
|
|
1683
|
-
this.dependencyResults.releaseDependency(parentId, dependency);
|
|
1684
|
-
}
|
|
1685
|
-
if (unresolved.length === 0) {
|
|
1686
|
-
shard.waitingDeps.delete(parentId);
|
|
1687
|
-
shard.getQueue(parentJob.queue).push(parentJob);
|
|
1688
|
-
shard.incrementQueued(parentId, false, parentJob.createdAt, parentJob.queue, runAt);
|
|
1689
|
-
this.jobIndex.set(parentId, {
|
|
1690
|
-
type: 'queue',
|
|
1691
|
-
shardIdx: parentIdx,
|
|
1692
|
-
queueName: parentJob.queue,
|
|
1693
|
-
});
|
|
1694
|
-
shard.notify(parentJob.queue);
|
|
1695
|
-
promoted = true;
|
|
1696
|
-
}
|
|
1697
|
-
removed = true;
|
|
1698
|
-
}
|
|
1699
|
-
finally {
|
|
1700
|
-
for (let index = guards.length - 1; index >= 0; index--)
|
|
1701
|
-
guards[index].release();
|
|
1702
|
-
}
|
|
1703
|
-
this.releaseCompletionPins(releasedDependencies);
|
|
1704
|
-
if (promoted) {
|
|
1705
|
-
this.eventsManager.broadcast({
|
|
1706
|
-
eventType: 'waiting',
|
|
1707
|
-
queue: parentJob.queue,
|
|
1708
|
-
jobId: parentId,
|
|
1709
|
-
timestamp: Date.now(),
|
|
1710
|
-
prev: 'waiting-children',
|
|
1711
|
-
});
|
|
1712
|
-
}
|
|
1713
|
-
return removed;
|
|
1714
|
-
}
|
|
1715
|
-
/**
|
|
1716
|
-
* Remove all unprocessed (waiting/delayed) children of a parent job.
|
|
1717
|
-
* Active, completed, and failed children are not affected.
|
|
1718
|
-
*/
|
|
1719
|
-
async removeUnprocessedChildren(parentJobId) {
|
|
1720
|
-
const parent = await this.getJob(parentJobId);
|
|
1721
|
-
if (!parent?.childrenIds || parent.childrenIds.length === 0)
|
|
1722
|
-
return;
|
|
1723
|
-
for (const childId of parent.childrenIds) {
|
|
1724
|
-
const loc = this.jobIndex.get(childId);
|
|
1725
|
-
// Only cancel children in 'queue' state (waiting/delayed), not active/completed/failed
|
|
1726
|
-
if (loc?.type === 'queue') {
|
|
1727
|
-
try {
|
|
1728
|
-
await this.cancel(childId);
|
|
1729
|
-
}
|
|
1730
|
-
catch {
|
|
1731
|
-
// Best-effort
|
|
1732
|
-
}
|
|
1733
|
-
}
|
|
1734
|
-
}
|
|
1735
|
-
}
|
|
1736
|
-
onJobsCompleted(completedIds) {
|
|
1737
|
-
for (const id of completedIds)
|
|
1738
|
-
this.pendingDepChecks.add(id);
|
|
1739
|
-
this.scheduleDependencyFlush();
|
|
1740
|
-
}
|
|
1741
|
-
/**
|
|
1742
|
-
* Schedule dependency flush on next microtask.
|
|
1743
|
-
* Coalesces multiple onJobCompleted calls in the same tick.
|
|
1744
|
-
*/
|
|
1745
|
-
scheduleDependencyFlush() {
|
|
1746
|
-
if (this.depFlushScheduled)
|
|
1747
|
-
return;
|
|
1748
|
-
this.depFlushScheduled = true;
|
|
1749
|
-
queueMicrotask(() => {
|
|
1750
|
-
this.depFlushScheduled = false;
|
|
1751
|
-
if (this.depFlushRunning)
|
|
1752
|
-
return;
|
|
1753
|
-
void this.runDependencyFlush();
|
|
1754
|
-
});
|
|
1755
|
-
}
|
|
1756
|
-
/**
|
|
1757
|
-
* Run dependency flush with reentrancy loop.
|
|
1758
|
-
* The while-loop handles new completions arriving during async lock waits.
|
|
1759
|
-
*/
|
|
1760
|
-
async runDependencyFlush() {
|
|
1761
|
-
this.depFlushRunning = true;
|
|
1762
|
-
try {
|
|
1763
|
-
while (this.pendingDepChecks.size > 0) {
|
|
1764
|
-
await processPendingDependencies(this.contextFactory.getBackgroundContext());
|
|
1765
|
-
handleTaskSuccess('dependency');
|
|
1766
|
-
}
|
|
1767
|
-
}
|
|
1768
|
-
catch (err) {
|
|
1769
|
-
handleTaskError('dependency', err);
|
|
1770
|
-
}
|
|
1771
|
-
finally {
|
|
1772
|
-
this.depFlushRunning = false;
|
|
1773
|
-
if (this.pendingDepChecks.size > 0) {
|
|
1774
|
-
this.scheduleDependencyFlush();
|
|
1775
|
-
}
|
|
1776
|
-
}
|
|
1777
|
-
}
|
|
1778
|
-
hasPendingDeps() {
|
|
1779
|
-
for (const shard of this.shards) {
|
|
1780
|
-
if (shard.waitingDeps.size > 0)
|
|
1781
|
-
return true;
|
|
1782
|
-
}
|
|
1783
|
-
return false;
|
|
1784
|
-
}
|
|
1785
|
-
// ============ Stats ============
|
|
1786
|
-
getStats() {
|
|
1787
|
-
return statsMgr.getStats(this.contextFactory.getStatsContext(), this.cronScheduler);
|
|
1788
|
-
}
|
|
1789
|
-
/** Get summary of all queues: name, paused, counts */
|
|
1790
|
-
getQueuesSummary() {
|
|
1791
|
-
const ctx = this.contextFactory.getStatsContext();
|
|
1792
|
-
const queues = this.listQueues();
|
|
1793
|
-
const result = [];
|
|
1794
|
-
const countsByQueue = statsMgr.getAllQueueJobCounts(queues, ctx);
|
|
1795
|
-
for (const name of queues) {
|
|
1796
|
-
const c = countsByQueue.get(name);
|
|
1797
|
-
if (!c)
|
|
1798
|
-
continue;
|
|
1799
|
-
result.push({
|
|
1800
|
-
name,
|
|
1801
|
-
paused: this.isPaused(name),
|
|
1802
|
-
counts: {
|
|
1803
|
-
waiting: c.waiting,
|
|
1804
|
-
prioritized: c.prioritized,
|
|
1805
|
-
active: c.active,
|
|
1806
|
-
completed: c.completed,
|
|
1807
|
-
failed: c.failed,
|
|
1808
|
-
delayed: c.delayed,
|
|
1809
|
-
},
|
|
1810
|
-
});
|
|
1811
|
-
}
|
|
1812
|
-
return result;
|
|
1813
|
-
}
|
|
1814
|
-
/** Get counts for every registered queue with one global aggregation pass. */
|
|
1815
|
-
getAllQueueJobCounts() {
|
|
1816
|
-
return this.getQueueJobCountsBatch(this.queueNamesCache);
|
|
1817
|
-
}
|
|
1818
|
-
/** Aggregate a selected group of queues without repeating global scans. */
|
|
1819
|
-
getQueueJobCountsBatch(queueNames) {
|
|
1820
|
-
const ctx = this.contextFactory.getStatsContext();
|
|
1821
|
-
return statsMgr.getAllQueueJobCounts(queueNames, ctx);
|
|
1822
|
-
}
|
|
1823
|
-
/** Get job counts for a specific queue */
|
|
1824
|
-
getQueueJobCounts(queueName) {
|
|
1825
|
-
return statsMgr.getQueueJobCounts(queueName, this.contextFactory.getStatsContext());
|
|
1826
|
-
}
|
|
1827
|
-
getMemoryStats() {
|
|
1828
|
-
return statsMgr.getMemoryStats(this.contextFactory.getStatsContext());
|
|
1829
|
-
}
|
|
1830
|
-
/** Get storage health status (disk full detection) */
|
|
1831
|
-
getStorageStatus() {
|
|
1832
|
-
if (!this.storage)
|
|
1833
|
-
return { diskFull: false, error: null, since: null };
|
|
1834
|
-
return this.storage.getDiskFullStatus();
|
|
1835
|
-
}
|
|
1836
|
-
/** Persist pending buffered writes before an external SQLite snapshot. */
|
|
1837
|
-
flushPersistence() {
|
|
1838
|
-
return this.storage?.flushWriteBuffer() ?? 0;
|
|
1839
|
-
}
|
|
1840
|
-
compactMemory() {
|
|
1841
|
-
statsMgr.compactMemory(this.contextFactory.getStatsContext());
|
|
1842
|
-
this.dashboardEmit?.('memory:compacted', {});
|
|
1843
|
-
}
|
|
1844
|
-
// ============ Lifecycle ============
|
|
1845
|
-
shutdown() {
|
|
1846
|
-
this.cronScheduler.stop();
|
|
1847
|
-
this.workerManager.stop();
|
|
1848
|
-
this.eventsManager.clear();
|
|
1849
|
-
if (this.backgroundTaskHandles) {
|
|
1850
|
-
bgTasks.stopBackgroundTasks(this.backgroundTaskHandles);
|
|
1851
|
-
}
|
|
1852
|
-
this.storage?.close();
|
|
1853
|
-
// Clear in-memory collections
|
|
1854
|
-
this.jobIndex.clear();
|
|
1855
|
-
this.completedJobs.clear();
|
|
1856
|
-
this.completedJobsData.clear();
|
|
1857
|
-
this.depCompletions.clear();
|
|
1858
|
-
this.timedOutJobs.clear();
|
|
1859
|
-
this.jobResults.clear();
|
|
1860
|
-
this.dependencyResults.clear();
|
|
1861
|
-
this.jobLogs.clear();
|
|
1862
|
-
this.customIdMap.clear();
|
|
1863
|
-
this.pendingDepChecks.clear();
|
|
1864
|
-
this.queueNamesCache.clear();
|
|
1865
|
-
this.jobLocks.clear();
|
|
1866
|
-
this.stalledCandidates.clear();
|
|
1867
|
-
this.clientJobs.clear();
|
|
1868
|
-
this.repeatChain.clear();
|
|
1869
|
-
this.failedChildrenValues.clear();
|
|
1870
|
-
this.ignoredChildrenFailures.clear();
|
|
1871
|
-
for (const shard of this.processingShards)
|
|
1872
|
-
shard.clear();
|
|
1873
|
-
for (const shard of this.shards) {
|
|
1874
|
-
shard.waitingDeps.clear();
|
|
1875
|
-
shard.dependencyIndex.clear();
|
|
1876
|
-
shard.waitingChildren.clear();
|
|
1877
|
-
shard.uniqueKeys.clear();
|
|
1878
|
-
shard.activeGroups.clear();
|
|
1879
|
-
}
|
|
1880
|
-
}
|
|
1
|
+
/** Public QueueManager façade. Implementation lives in queue-manager/. */
|
|
2
|
+
import { QueueManagerLifecycle } from './queue-manager/lifecycle';
|
|
3
|
+
export class QueueManager extends QueueManagerLifecycle {
|
|
1881
4
|
}
|