bunqueue 2.8.60 → 2.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +114 -78
- package/dist/application/background/dlq.js +2 -0
- package/dist/application/background/recovery/active.d.ts +1 -0
- package/dist/application/background/recovery/active.js +12 -0
- package/dist/application/background/recovery/index.js +3 -1
- package/dist/application/background/recovery/pending.js +1 -0
- package/dist/application/cleanupTasks.js +4 -16
- package/dist/application/contextFactory.js +8 -0
- package/dist/application/dlqManager.d.ts +10 -0
- package/dist/application/dlqManager.js +48 -9
- package/dist/application/dlqRetry.js +60 -38
- package/dist/application/eventsManager.d.ts +9 -2
- package/dist/application/eventsManager.js +106 -55
- package/dist/application/operations/ackHelpers.d.ts +1 -4
- package/dist/application/operations/ackHelpers.js +39 -14
- package/dist/application/operations/flowPush.js +1 -0
- package/dist/application/operations/flowValidation.js +4 -4
- package/dist/application/operations/jobManagement.d.ts +4 -0
- package/dist/application/operations/jobMoveOperations.js +9 -4
- package/dist/application/operations/pull.d.ts +7 -2
- package/dist/application/operations/pull.js +15 -57
- package/dist/application/operations/pullFinalization.d.ts +6 -0
- package/dist/application/operations/pullFinalization.js +69 -0
- package/dist/application/operations/pullStateTransition.d.ts +8 -7
- package/dist/application/operations/pullStateTransition.js +25 -18
- package/dist/application/operations/push.js +3 -0
- package/dist/application/operations/pushBatch.js +14 -7
- package/dist/application/operations/pushContext.d.ts +2 -1
- package/dist/application/postgres-queue-manager/batchSnapshot.d.ts +16 -0
- package/dist/application/postgres-queue-manager/batchSnapshot.js +17 -0
- package/dist/application/postgres-queue-manager/cloud.d.ts +30 -0
- package/dist/application/postgres-queue-manager/cloud.js +55 -0
- package/dist/application/postgres-queue-manager/config.d.ts +5 -0
- package/dist/application/postgres-queue-manager/config.js +1 -0
- package/dist/application/postgres-queue-manager/control.d.ts +30 -0
- package/dist/application/postgres-queue-manager/control.js +210 -0
- package/dist/application/postgres-queue-manager/deferredWrites.d.ts +13 -0
- package/dist/application/postgres-queue-manager/deferredWrites.js +60 -0
- package/dist/application/postgres-queue-manager/delivery.d.ts +20 -0
- package/dist/application/postgres-queue-manager/delivery.js +83 -0
- package/dist/application/postgres-queue-manager/groupQueries.d.ts +10 -0
- package/dist/application/postgres-queue-manager/groupQueries.js +40 -0
- package/dist/application/postgres-queue-manager/lease.d.ts +13 -0
- package/dist/application/postgres-queue-manager/lease.js +88 -0
- package/dist/application/postgres-queue-manager/lifetimeMetrics.d.ts +32 -0
- package/dist/application/postgres-queue-manager/lifetimeMetrics.js +53 -0
- package/dist/application/postgres-queue-manager/maintenance.d.ts +14 -0
- package/dist/application/postgres-queue-manager/maintenance.js +100 -0
- package/dist/application/postgres-queue-manager/operationGate.d.ts +26 -0
- package/dist/application/postgres-queue-manager/operationGate.js +96 -0
- package/dist/application/postgres-queue-manager/operations.d.ts +25 -0
- package/dist/application/postgres-queue-manager/operations.js +194 -0
- package/dist/application/postgres-queue-manager/projectionRefreshes.d.ts +43 -0
- package/dist/application/postgres-queue-manager/projectionRefreshes.js +170 -0
- package/dist/application/postgres-queue-manager/queries.d.ts +142 -0
- package/dist/application/postgres-queue-manager/queries.js +231 -0
- package/dist/application/postgres-queue-manager/relationships.d.ts +10 -0
- package/dist/application/postgres-queue-manager/relationships.js +45 -0
- package/dist/application/postgres-queue-manager/services.d.ts +31 -0
- package/dist/application/postgres-queue-manager/services.js +149 -0
- package/dist/application/postgres-queue-manager/snapshot.d.ts +51 -0
- package/dist/application/postgres-queue-manager/snapshot.js +236 -0
- package/dist/application/postgres-queue-manager/snapshotViews.d.ts +15 -0
- package/dist/application/postgres-queue-manager/snapshotViews.js +100 -0
- package/dist/application/postgres-queue-manager/startupEventBuffer.d.ts +12 -0
- package/dist/application/postgres-queue-manager/startupEventBuffer.js +36 -0
- package/dist/application/postgres-queue-manager/state.d.ts +47 -0
- package/dist/application/postgres-queue-manager/state.js +273 -0
- package/dist/application/postgres-queue-manager/terminalDelivery.d.ts +20 -0
- package/dist/application/postgres-queue-manager/terminalDelivery.js +100 -0
- package/dist/application/postgresQueueManager.d.ts +2 -0
- package/dist/application/postgresQueueManager.js +1 -0
- package/dist/application/queue-manager/control.js +1 -0
- package/dist/application/queue-manager/delivery.d.ts +5 -4
- package/dist/application/queue-manager/delivery.js +15 -8
- package/dist/application/queue-manager/flow-failures.js +4 -0
- package/dist/application/queue-manager/limits.d.ts +13 -0
- package/dist/application/queue-manager/limits.js +65 -0
- package/dist/application/queue-manager/observability.js +15 -3
- package/dist/application/queue-manager/queries.d.ts +7 -0
- package/dist/application/queue-manager/queries.js +9 -0
- package/dist/application/queue-manager/services.d.ts +1 -1
- package/dist/application/queue-manager/services.js +4 -2
- package/dist/application/queue-manager/state.js +17 -2
- package/dist/application/queueTelemetryJournal.d.ts +2 -0
- package/dist/application/queueTelemetryJournal.js +23 -0
- package/dist/application/types/ack.d.ts +3 -1
- package/dist/cli/commandRouter.d.ts +1 -1
- package/dist/cli/commands/server.js +1 -1
- package/dist/client/flowPush.js +6 -0
- package/dist/client/forwarder.d.ts +6 -4
- package/dist/client/forwarder.js +5 -3
- package/dist/client/groupId.d.ts +1 -0
- package/dist/client/groupId.js +1 -0
- package/dist/client/index.d.ts +1 -1
- package/dist/client/jobHelpers.js +1 -0
- package/dist/client/queue/dlq.d.ts +2 -0
- package/dist/client/queue/dlq.js +14 -0
- package/dist/client/queue/dlqOps.d.ts +2 -0
- package/dist/client/queue/dlqOps.js +4 -0
- package/dist/client/queue/helpers.js +2 -0
- package/dist/client/queue/operations/add/bulk.js +4 -4
- package/dist/client/queue/operations/add/payload.d.ts +1 -0
- package/dist/client/queue/operations/add/payload.js +7 -1
- package/dist/client/queue/operations/add/single.js +3 -3
- package/dist/client/queue/operations/groups.d.ts +21 -0
- package/dist/client/queue/operations/groups.js +88 -0
- package/dist/client/queue/runtime/configuration.d.ts +4 -0
- package/dist/client/queue/runtime/configuration.js +8 -0
- package/dist/client/queue/runtime/queries.d.ts +13 -0
- package/dist/client/queue/runtime/queries.js +31 -0
- package/dist/client/queue/runtime/state.d.ts +4 -0
- package/dist/client/queue/runtime/state.js +1 -0
- package/dist/client/types/index.d.ts +2 -2
- package/dist/client/types/options.d.ts +4 -0
- package/dist/client/types/worker.d.ts +8 -0
- package/dist/client/worker/runtime/execution.js +4 -8
- package/dist/client/worker/runtime/options.js +3 -0
- package/dist/client/worker/runtime/polling.d.ts +1 -0
- package/dist/client/worker/runtime/polling.js +9 -0
- package/dist/client/worker/types/options.d.ts +2 -1
- package/dist/client/worker/workerPull.d.ts +2 -0
- package/dist/client/worker/workerPull.js +6 -4
- package/dist/config/resolve.d.ts +14 -0
- package/dist/config/resolve.js +50 -5
- package/dist/config/types.d.ts +18 -0
- package/dist/domain/job/groupFifoOrder.d.ts +9 -0
- package/dist/domain/job/groupFifoOrder.js +26 -0
- package/dist/domain/queue/dlqShard.d.ts +2 -0
- package/dist/domain/queue/dlqShard.js +16 -0
- package/dist/domain/queue/groupLimiterManager.d.ts +23 -0
- package/dist/domain/queue/groupLimiterManager.js +107 -0
- package/dist/domain/queue/groupScheduler.d.ts +31 -0
- package/dist/domain/queue/groupScheduler.js +256 -0
- package/dist/domain/queue/groupSchedulerOrder.d.ts +6 -0
- package/dist/domain/queue/groupSchedulerOrder.js +30 -0
- package/dist/domain/queue/priorityQueue.d.ts +13 -23
- package/dist/domain/queue/priorityQueue.js +53 -42
- package/dist/domain/queue/shard/dlq.d.ts +1 -0
- package/dist/domain/queue/shard/dlq.js +3 -0
- package/dist/domain/queue/shard/keys.d.ts +1 -0
- package/dist/domain/queue/shard/keys.js +25 -2
- package/dist/domain/queue/shard/lifecycle.js +4 -0
- package/dist/domain/queue/shard/limits.d.ts +20 -1
- package/dist/domain/queue/shard/limits.js +56 -0
- package/dist/domain/queue/shard/state.d.ts +5 -0
- package/dist/domain/queue/shard/state.js +9 -1
- package/dist/domain/types/commands/core.d.ts +3 -0
- package/dist/domain/types/commands/dlq.d.ts +5 -0
- package/dist/domain/types/commands/limits.d.ts +41 -0
- package/dist/domain/types/commands/union.d.ts +1 -1
- package/dist/domain/types/cron.d.ts +5 -0
- package/dist/domain/types/cron.js +14 -3
- package/dist/domain/types/dlq.d.ts +1 -1
- package/dist/domain/types/dlq.js +1 -2
- package/dist/domain/types/group.d.ts +21 -0
- package/dist/domain/types/group.js +78 -0
- package/dist/domain/types/jobs/model.d.ts +1 -1
- package/dist/domain/types/priorityQueue.d.ts +1 -0
- package/dist/infrastructure/cloud/commandHandler.js +5 -3
- package/dist/infrastructure/cloud/commands/integrations.js +11 -11
- package/dist/infrastructure/cloud/commands/jobs.js +48 -40
- package/dist/infrastructure/cloud/commands/queues.js +61 -75
- package/dist/infrastructure/cloud/queueAdapter/local.d.ts +41 -0
- package/dist/infrastructure/cloud/queueAdapter/local.js +187 -0
- package/dist/infrastructure/cloud/queueAdapter/postgres.d.ts +42 -0
- package/dist/infrastructure/cloud/queueAdapter/postgres.js +188 -0
- package/dist/infrastructure/cloud/queueAdapter/registry.d.ts +4 -0
- package/dist/infrastructure/cloud/queueAdapter/registry.js +15 -0
- package/dist/infrastructure/cloud/queueAdapter/types.d.ts +87 -0
- package/dist/infrastructure/cloud/queueAdapter/types.js +1 -0
- package/dist/infrastructure/cloud/snapshot/analytics.d.ts +2 -0
- package/dist/infrastructure/cloud/snapshot/analytics.js +7 -0
- package/dist/infrastructure/cloud/snapshot/collector.d.ts +4 -0
- package/dist/infrastructure/cloud/snapshot/collector.js +39 -1
- package/dist/infrastructure/cloud/snapshot/config.d.ts +3 -0
- package/dist/infrastructure/cloud/snapshot/config.js +43 -0
- package/dist/infrastructure/cloud/snapshot/jobs.d.ts +3 -0
- package/dist/infrastructure/cloud/snapshot/jobs.js +32 -1
- package/dist/infrastructure/cloud/snapshotCollector.js +20 -35
- package/dist/infrastructure/cloud/snapshotHelpers.d.ts +3 -3
- package/dist/infrastructure/cloud/snapshotHelpers.js +3 -3
- package/dist/infrastructure/cloud/statsRefresh.js +2 -1
- package/dist/infrastructure/cloud/types/command.d.ts +2 -2
- package/dist/infrastructure/persistence/jobOptionsBlob.d.ts +1 -0
- package/dist/infrastructure/persistence/jobOptionsBlob.js +10 -2
- package/dist/infrastructure/persistence/migrations.js +11 -0
- package/dist/infrastructure/persistence/postgres/admission.d.ts +18 -0
- package/dist/infrastructure/persistence/postgres/admission.js +196 -0
- package/dist/infrastructure/persistence/postgres/admissionParent.d.ts +5 -0
- package/dist/infrastructure/persistence/postgres/admissionParent.js +35 -0
- package/dist/infrastructure/persistence/postgres/admissionResult.d.ts +9 -0
- package/dist/infrastructure/persistence/postgres/admissionResult.js +1 -0
- package/dist/infrastructure/persistence/postgres/admissionStore.d.ts +12 -0
- package/dist/infrastructure/persistence/postgres/admissionStore.js +101 -0
- package/dist/infrastructure/persistence/postgres/advisoryLocks.d.ts +2 -0
- package/dist/infrastructure/persistence/postgres/advisoryLocks.js +13 -0
- package/dist/infrastructure/persistence/postgres/batchAdmission.d.ts +11 -0
- package/dist/infrastructure/persistence/postgres/batchAdmission.js +128 -0
- package/dist/infrastructure/persistence/postgres/batchCompletion.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/batchCompletion.js +174 -0
- package/dist/infrastructure/persistence/postgres/batchEvents.d.ts +14 -0
- package/dist/infrastructure/persistence/postgres/batchEvents.js +72 -0
- package/dist/infrastructure/persistence/postgres/brokerSessions.d.ts +14 -0
- package/dist/infrastructure/persistence/postgres/brokerSessions.js +94 -0
- package/dist/infrastructure/persistence/postgres/claimBatch.d.ts +14 -0
- package/dist/infrastructure/persistence/postgres/claimBatch.js +69 -0
- package/dist/infrastructure/persistence/postgres/claimSelection.d.ts +11 -0
- package/dist/infrastructure/persistence/postgres/claimSelection.js +140 -0
- package/dist/infrastructure/persistence/postgres/claims.d.ts +10 -0
- package/dist/infrastructure/persistence/postgres/claims.js +153 -0
- package/dist/infrastructure/persistence/postgres/cloudReadModel.d.ts +26 -0
- package/dist/infrastructure/persistence/postgres/cloudReadModel.js +123 -0
- package/dist/infrastructure/persistence/postgres/codec.d.ts +30 -0
- package/dist/infrastructure/persistence/postgres/codec.js +82 -0
- package/dist/infrastructure/persistence/postgres/completionEvents.d.ts +11 -0
- package/dist/infrastructure/persistence/postgres/completionEvents.js +21 -0
- package/dist/infrastructure/persistence/postgres/completionLifecycle.d.ts +11 -0
- package/dist/infrastructure/persistence/postgres/completionLifecycle.js +145 -0
- package/dist/infrastructure/persistence/postgres/completionOutcome.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/completionOutcome.js +112 -0
- package/dist/infrastructure/persistence/postgres/completionQueries.d.ts +10 -0
- package/dist/infrastructure/persistence/postgres/completionQueries.js +87 -0
- package/dist/infrastructure/persistence/postgres/connection.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/connection.js +15 -0
- package/dist/infrastructure/persistence/postgres/context.d.ts +28 -0
- package/dist/infrastructure/persistence/postgres/context.js +74 -0
- package/dist/infrastructure/persistence/postgres/control.d.ts +23 -0
- package/dist/infrastructure/persistence/postgres/control.js +105 -0
- package/dist/infrastructure/persistence/postgres/crons.d.ts +18 -0
- package/dist/infrastructure/persistence/postgres/crons.js +205 -0
- package/dist/infrastructure/persistence/postgres/deduplication.d.ts +9 -0
- package/dist/infrastructure/persistence/postgres/deduplication.js +47 -0
- package/dist/infrastructure/persistence/postgres/dependencyDestruction.d.ts +14 -0
- package/dist/infrastructure/persistence/postgres/dependencyDestruction.js +74 -0
- package/dist/infrastructure/persistence/postgres/dependencyPromotion.d.ts +29 -0
- package/dist/infrastructure/persistence/postgres/dependencyPromotion.js +212 -0
- package/dist/infrastructure/persistence/postgres/destructiveMutations.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/destructiveMutations.js +111 -0
- package/dist/infrastructure/persistence/postgres/dlqLifecycle.d.ts +13 -0
- package/dist/infrastructure/persistence/postgres/dlqLifecycle.js +184 -0
- package/dist/infrastructure/persistence/postgres/dlqMaintenance.d.ts +6 -0
- package/dist/infrastructure/persistence/postgres/dlqMaintenance.js +36 -0
- package/dist/infrastructure/persistence/postgres/dlqRetryPlan.d.ts +21 -0
- package/dist/infrastructure/persistence/postgres/dlqRetryPlan.js +92 -0
- package/dist/infrastructure/persistence/postgres/eventCatchupCursors.d.ts +32 -0
- package/dist/infrastructure/persistence/postgres/eventCatchupCursors.js +66 -0
- package/dist/infrastructure/persistence/postgres/eventCommitGc.d.ts +27 -0
- package/dist/infrastructure/persistence/postgres/eventCommitGc.js +62 -0
- package/dist/infrastructure/persistence/postgres/eventJournal.d.ts +18 -0
- package/dist/infrastructure/persistence/postgres/eventJournal.js +76 -0
- package/dist/infrastructure/persistence/postgres/eventJournalSchema.d.ts +5 -0
- package/dist/infrastructure/persistence/postgres/eventJournalSchema.js +267 -0
- package/dist/infrastructure/persistence/postgres/eventPruneWatermarks.d.ts +26 -0
- package/dist/infrastructure/persistence/postgres/eventPruneWatermarks.js +129 -0
- package/dist/infrastructure/persistence/postgres/eventRetention.d.ts +18 -0
- package/dist/infrastructure/persistence/postgres/eventRetention.js +76 -0
- package/dist/infrastructure/persistence/postgres/events.d.ts +45 -0
- package/dist/infrastructure/persistence/postgres/events.js +276 -0
- package/dist/infrastructure/persistence/postgres/expiry.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/expiry.js +74 -0
- package/dist/infrastructure/persistence/postgres/flowFailures.d.ts +17 -0
- package/dist/infrastructure/persistence/postgres/flowFailures.js +199 -0
- package/dist/infrastructure/persistence/postgres/groupClaims.d.ts +15 -0
- package/dist/infrastructure/persistence/postgres/groupClaims.js +95 -0
- package/dist/infrastructure/persistence/postgres/groupSchema.d.ts +2 -0
- package/dist/infrastructure/persistence/postgres/groupSchema.js +155 -0
- package/dist/infrastructure/persistence/postgres/groupSchemaFingerprint.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/groupSchemaFingerprint.js +127 -0
- package/dist/infrastructure/persistence/postgres/groupStateRetention.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/groupStateRetention.js +93 -0
- package/dist/infrastructure/persistence/postgres/groups.d.ts +11 -0
- package/dist/infrastructure/persistence/postgres/groups.js +117 -0
- package/dist/infrastructure/persistence/postgres/jobData.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/jobData.js +32 -0
- package/dist/infrastructure/persistence/postgres/leaseRelease.d.ts +9 -0
- package/dist/infrastructure/persistence/postgres/leaseRelease.js +120 -0
- package/dist/infrastructure/persistence/postgres/leaseRenewal.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/leaseRenewal.js +35 -0
- package/dist/infrastructure/persistence/postgres/lifetimeMetrics.d.ts +15 -0
- package/dist/infrastructure/persistence/postgres/lifetimeMetrics.js +46 -0
- package/dist/infrastructure/persistence/postgres/lifetimeMetricsFinalizer.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/lifetimeMetricsFinalizer.js +13 -0
- package/dist/infrastructure/persistence/postgres/logs.d.ts +6 -0
- package/dist/infrastructure/persistence/postgres/logs.js +70 -0
- package/dist/infrastructure/persistence/postgres/maintenance.d.ts +6 -0
- package/dist/infrastructure/persistence/postgres/maintenance.js +138 -0
- package/dist/infrastructure/persistence/postgres/maintenanceFlights.d.ts +14 -0
- package/dist/infrastructure/persistence/postgres/maintenanceFlights.js +66 -0
- package/dist/infrastructure/persistence/postgres/metricWrites.d.ts +9 -0
- package/dist/infrastructure/persistence/postgres/metricWrites.js +94 -0
- package/dist/infrastructure/persistence/postgres/mutations.d.ts +9 -0
- package/dist/infrastructure/persistence/postgres/mutations.js +171 -0
- package/dist/infrastructure/persistence/postgres/outcomes.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/outcomes.js +194 -0
- package/dist/infrastructure/persistence/postgres/policies.d.ts +11 -0
- package/dist/infrastructure/persistence/postgres/policies.js +18 -0
- package/dist/infrastructure/persistence/postgres/postCommitMaintenance.d.ts +17 -0
- package/dist/infrastructure/persistence/postgres/postCommitMaintenance.js +117 -0
- package/dist/infrastructure/persistence/postgres/promotion.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/promotion.js +43 -0
- package/dist/infrastructure/persistence/postgres/queries.d.ts +22 -0
- package/dist/infrastructure/persistence/postgres/queries.js +152 -0
- package/dist/infrastructure/persistence/postgres/queueDestruction.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/queueDestruction.js +158 -0
- package/dist/infrastructure/persistence/postgres/queueLifecycle.d.ts +10 -0
- package/dist/infrastructure/persistence/postgres/queueLifecycle.js +60 -0
- package/dist/infrastructure/persistence/postgres/rateLimit.d.ts +6 -0
- package/dist/infrastructure/persistence/postgres/rateLimit.js +10 -0
- package/dist/infrastructure/persistence/postgres/readModels.d.ts +32 -0
- package/dist/infrastructure/persistence/postgres/readModels.js +72 -0
- package/dist/infrastructure/persistence/postgres/recovery.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/recovery.js +178 -0
- package/dist/infrastructure/persistence/postgres/relationships.d.ts +9 -0
- package/dist/infrastructure/persistence/postgres/relationships.js +201 -0
- package/dist/infrastructure/persistence/postgres/removeUnprocessedChildren.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/removeUnprocessedChildren.js +241 -0
- package/dist/infrastructure/persistence/postgres/repeatMutations.d.ts +5 -0
- package/dist/infrastructure/persistence/postgres/repeatMutations.js +37 -0
- package/dist/infrastructure/persistence/postgres/runtime.d.ts +45 -0
- package/dist/infrastructure/persistence/postgres/runtime.js +238 -0
- package/dist/infrastructure/persistence/postgres/runtimeConfig.d.ts +21 -0
- package/dist/infrastructure/persistence/postgres/runtimeConfig.js +39 -0
- package/dist/infrastructure/persistence/postgres/schema.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/schema.js +261 -0
- package/dist/infrastructure/persistence/postgres/schemaInitialization.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/schemaInitialization.js +226 -0
- package/dist/infrastructure/persistence/postgres/serialAdmission.d.ts +6 -0
- package/dist/infrastructure/persistence/postgres/serialAdmission.js +96 -0
- package/dist/infrastructure/persistence/postgres/snapshotBudget.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/snapshotBudget.js +83 -0
- package/dist/infrastructure/persistence/postgres/storageHealth.d.ts +10 -0
- package/dist/infrastructure/persistence/postgres/storageHealth.js +29 -0
- package/dist/infrastructure/persistence/postgres/store.d.ts +100 -0
- package/dist/infrastructure/persistence/postgres/store.js +180 -0
- package/dist/infrastructure/persistence/postgres/telemetry.d.ts +8 -0
- package/dist/infrastructure/persistence/postgres/telemetry.js +100 -0
- package/dist/infrastructure/persistence/postgres/terminal.d.ts +8 -0
- package/dist/infrastructure/persistence/postgres/terminal.js +137 -0
- package/dist/infrastructure/persistence/postgres/transactionRetry.d.ts +6 -0
- package/dist/infrastructure/persistence/postgres/transactionRetry.js +42 -0
- package/dist/infrastructure/persistence/postgres/types.d.ts +176 -0
- package/dist/infrastructure/persistence/postgres/types.js +1 -0
- package/dist/infrastructure/persistence/postgres/workers.d.ts +15 -0
- package/dist/infrastructure/persistence/postgres/workers.js +126 -0
- package/dist/infrastructure/persistence/postgres.d.ts +2 -0
- package/dist/infrastructure/persistence/postgres.js +1 -0
- package/dist/infrastructure/persistence/schema.d.ts +2 -2
- package/dist/infrastructure/persistence/schema.js +12 -1
- package/dist/infrastructure/persistence/sqlite/control.d.ts +14 -0
- package/dist/infrastructure/persistence/sqlite/control.js +69 -0
- package/dist/infrastructure/persistence/sqlite/jobs.d.ts +12 -0
- package/dist/infrastructure/persistence/sqlite/jobs.js +38 -0
- package/dist/infrastructure/persistence/sqlite/telemetry.d.ts +2 -1
- package/dist/infrastructure/persistence/sqlite/telemetry.js +5 -64
- package/dist/infrastructure/persistence/sqlite/telemetryWrites.d.ts +4 -0
- package/dist/infrastructure/persistence/sqlite/telemetryWrites.js +123 -0
- package/dist/infrastructure/persistence/sqliteSerializer.js +2 -0
- package/dist/infrastructure/scheduler/cron/persisted.d.ts +5 -0
- package/dist/infrastructure/scheduler/cron/persisted.js +27 -0
- package/dist/infrastructure/scheduler/cron/runtime.js +5 -10
- package/dist/infrastructure/scheduler/cron/validation.d.ts +3 -0
- package/dist/infrastructure/scheduler/cron/validation.js +11 -0
- package/dist/infrastructure/scheduler/cronParser.d.ts +1 -1
- package/dist/infrastructure/scheduler/cronParser.js +144 -11
- package/dist/infrastructure/server/bootstrap.d.ts +2 -3
- package/dist/infrastructure/server/bootstrap.js +33 -59
- package/dist/infrastructure/server/errors.d.ts +4 -0
- package/dist/infrastructure/server/errors.js +54 -0
- package/dist/infrastructure/server/handler-routes/control.d.ts +5 -5
- package/dist/infrastructure/server/handler-routes/control.js +16 -1
- package/dist/infrastructure/server/handler-routes/monitoring.d.ts +2 -2
- package/dist/infrastructure/server/handler.js +9 -11
- package/dist/infrastructure/server/handlers/advanced/dependencies.js +7 -6
- package/dist/infrastructure/server/handlers/advanced/jobs.js +19 -6
- package/dist/infrastructure/server/handlers/advanced/queue.d.ts +9 -9
- package/dist/infrastructure/server/handlers/advanced/queue.js +91 -27
- package/dist/infrastructure/server/handlers/core.d.ts +0 -5
- package/dist/infrastructure/server/handlers/core.js +47 -49
- package/dist/infrastructure/server/handlers/cron.d.ts +4 -4
- package/dist/infrastructure/server/handlers/cron.js +60 -59
- package/dist/infrastructure/server/handlers/dashboard.d.ts +1 -1
- package/dist/infrastructure/server/handlers/dashboard.js +21 -8
- package/dist/infrastructure/server/handlers/dlq.d.ts +6 -3
- package/dist/infrastructure/server/handlers/dlq.js +35 -9
- package/dist/infrastructure/server/handlers/flow.js +2 -1
- package/dist/infrastructure/server/handlers/groups.d.ts +11 -0
- package/dist/infrastructure/server/handlers/groups.js +55 -0
- package/dist/infrastructure/server/handlers/management.d.ts +5 -5
- package/dist/infrastructure/server/handlers/management.js +36 -9
- package/dist/infrastructure/server/handlers/monitoring/health.d.ts +5 -5
- package/dist/infrastructure/server/handlers/monitoring/health.js +45 -16
- package/dist/infrastructure/server/handlers/monitoring/operations.d.ts +1 -1
- package/dist/infrastructure/server/handlers/monitoring/operations.js +8 -1
- package/dist/infrastructure/server/handlers/monitoring/workers.d.ts +3 -3
- package/dist/infrastructure/server/handlers/monitoring/workers.js +31 -7
- package/dist/infrastructure/server/handlers/pushBatchValidation.d.ts +3 -1
- package/dist/infrastructure/server/handlers/pushBatchValidation.js +56 -8
- package/dist/infrastructure/server/handlers/query.d.ts +1 -1
- package/dist/infrastructure/server/handlers/query.js +5 -4
- package/dist/infrastructure/server/http.js +2 -2
- package/dist/infrastructure/server/httpDashboardEndpoints.d.ts +1 -1
- package/dist/infrastructure/server/httpDashboardEndpoints.js +19 -6
- package/dist/infrastructure/server/httpEndpoints.js +18 -7
- package/dist/infrastructure/server/httpRouteQueues.js +11 -1
- package/dist/infrastructure/server/protocol/validation.d.ts +1 -0
- package/dist/infrastructure/server/protocol/validation.js +1 -0
- package/dist/infrastructure/server/protocol.d.ts +1 -1
- package/dist/infrastructure/server/protocol.js +1 -1
- package/dist/infrastructure/server/shutdownCoordinator.d.ts +24 -0
- package/dist/infrastructure/server/shutdownCoordinator.js +106 -0
- package/dist/infrastructure/server/sseHandler.js +8 -3
- package/dist/infrastructure/server/storageAdapter.d.ts +29 -0
- package/dist/infrastructure/server/storageAdapter.js +61 -0
- package/dist/infrastructure/server/storageManager.d.ts +12 -0
- package/dist/infrastructure/server/storageManager.js +102 -0
- package/dist/infrastructure/server/tcp.js +2 -3
- package/dist/infrastructure/server/ws/snapshots.d.ts +1 -1
- package/dist/infrastructure/server/ws/snapshots.js +19 -4
- package/dist/infrastructure/server/wsHandler.js +10 -5
- package/dist/main.js +1 -1
- package/dist/mcp/backend/embedded/services.js +2 -1
- package/dist/mcp/index.d.ts +2 -2
- package/dist/mcp/index.js +2 -2
- package/dist/shared/storageHealth.d.ts +9 -0
- package/dist/shared/storageHealth.js +12 -0
- package/package.json +75 -65
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
import { getGroupFifoOrder, setGroupFifoOrder } from '../job/groupFifoOrder';
|
|
2
|
+
import { isExpired, isReady } from '../types/job';
|
|
3
|
+
import { IndexedPriorityQueue } from './priorityQueue';
|
|
4
|
+
import { compareGroupFifoEntries, compareGroupWakeEntries, nextGroupCheckAt, orderedGroupClone, } from './groupSchedulerOrder';
|
|
5
|
+
/** Lazy secondary ready/group indexes; the primary queue remains authoritative. */
|
|
6
|
+
export class GroupScheduler {
|
|
7
|
+
queues = new Map();
|
|
8
|
+
nextFifoOrder = 1n;
|
|
9
|
+
assignFifoOrder(job) {
|
|
10
|
+
if (!job.groupId)
|
|
11
|
+
return;
|
|
12
|
+
const existing = getGroupFifoOrder(job);
|
|
13
|
+
if (existing !== null) {
|
|
14
|
+
this.observeFifoOrder(job);
|
|
15
|
+
return;
|
|
16
|
+
}
|
|
17
|
+
setGroupFifoOrder(job, this.nextFifoOrder++);
|
|
18
|
+
}
|
|
19
|
+
observeFifoOrder(job) {
|
|
20
|
+
const order = getGroupFifoOrder(job);
|
|
21
|
+
if (order !== null && order >= this.nextFifoOrder)
|
|
22
|
+
this.nextFifoOrder = order + 1n;
|
|
23
|
+
}
|
|
24
|
+
insert(queue, job, primaryJobs, now = Date.now()) {
|
|
25
|
+
this.assignFifoOrder(job);
|
|
26
|
+
let state = this.queues.get(queue);
|
|
27
|
+
const previous = state?.jobs.get(job.id);
|
|
28
|
+
if (previous) {
|
|
29
|
+
this.remove(queue, previous);
|
|
30
|
+
state = this.queues.get(queue);
|
|
31
|
+
}
|
|
32
|
+
if (!state) {
|
|
33
|
+
if (!job.groupId)
|
|
34
|
+
return false;
|
|
35
|
+
state = this.createState();
|
|
36
|
+
this.queues.set(queue, state);
|
|
37
|
+
for (const current of primaryJobs()) {
|
|
38
|
+
this.assignFifoOrder(current);
|
|
39
|
+
this.insertTracked(state, current, now);
|
|
40
|
+
}
|
|
41
|
+
return true;
|
|
42
|
+
}
|
|
43
|
+
this.insertTracked(state, job, now);
|
|
44
|
+
return true;
|
|
45
|
+
}
|
|
46
|
+
remove(queue, job) {
|
|
47
|
+
const state = this.queues.get(queue);
|
|
48
|
+
if (!state)
|
|
49
|
+
return false;
|
|
50
|
+
const tracked = state.jobs.get(job.id) ?? job;
|
|
51
|
+
const placement = state.placements.get(job.id);
|
|
52
|
+
if (!placement)
|
|
53
|
+
return true;
|
|
54
|
+
if (placement.kind === 'plain') {
|
|
55
|
+
state.plain.remove(job.id);
|
|
56
|
+
}
|
|
57
|
+
else if (placement.kind === 'delayed') {
|
|
58
|
+
state.delayed.remove(job.id);
|
|
59
|
+
}
|
|
60
|
+
else {
|
|
61
|
+
const lane = state.lanes.get(placement.groupId);
|
|
62
|
+
lane?.jobs.remove(job.id);
|
|
63
|
+
if (lane?.jobs.isEmpty)
|
|
64
|
+
this.detachLane(state, placement.groupId, lane);
|
|
65
|
+
}
|
|
66
|
+
state.placements.delete(job.id);
|
|
67
|
+
state.jobs.delete(job.id);
|
|
68
|
+
if (tracked.groupId) {
|
|
69
|
+
const count = (state.groupCounts.get(tracked.groupId) ?? 1) - 1;
|
|
70
|
+
if (count > 0)
|
|
71
|
+
state.groupCounts.set(tracked.groupId, count);
|
|
72
|
+
else {
|
|
73
|
+
state.groupCounts.delete(tracked.groupId);
|
|
74
|
+
const lane = state.lanes.get(tracked.groupId);
|
|
75
|
+
if (lane?.jobs.isEmpty && !lane.inRotation)
|
|
76
|
+
state.lanes.delete(tracked.groupId);
|
|
77
|
+
}
|
|
78
|
+
state.groupedCount = Math.max(0, state.groupedCount - 1);
|
|
79
|
+
if (state.groupedCount === 0)
|
|
80
|
+
this.queues.delete(queue);
|
|
81
|
+
}
|
|
82
|
+
return this.queues.has(queue);
|
|
83
|
+
}
|
|
84
|
+
isActive(queue) {
|
|
85
|
+
return this.queues.has(queue);
|
|
86
|
+
}
|
|
87
|
+
peek(queue, now, eligibility) {
|
|
88
|
+
const state = this.queues.get(queue);
|
|
89
|
+
if (!state)
|
|
90
|
+
return { job: null, nextRunAt: null };
|
|
91
|
+
while (true) {
|
|
92
|
+
this.promoteDue(state, now);
|
|
93
|
+
let plain = state.plain.peek();
|
|
94
|
+
while (plain) {
|
|
95
|
+
const job = state.jobs.get(plain.id) ?? plain;
|
|
96
|
+
if (isReady(job, now) || isExpired(job, now))
|
|
97
|
+
return { job, nextRunAt: null };
|
|
98
|
+
this.demoteReady(state, job);
|
|
99
|
+
plain = state.plain.peek();
|
|
100
|
+
}
|
|
101
|
+
let nextRunAt = state.delayed.peek()?.runAt ?? null;
|
|
102
|
+
let groupId = state.cursor;
|
|
103
|
+
let restart = false;
|
|
104
|
+
for (let visited = 0; groupId && visited < state.rotationSize; visited++) {
|
|
105
|
+
const lane = state.lanes.get(groupId);
|
|
106
|
+
if (!lane?.inRotation)
|
|
107
|
+
break;
|
|
108
|
+
const candidate = lane.jobs.peek();
|
|
109
|
+
if (candidate) {
|
|
110
|
+
const job = state.jobs.get(candidate.id) ?? candidate;
|
|
111
|
+
if (!isReady(job, now) && !isExpired(job, now)) {
|
|
112
|
+
this.demoteReady(state, job);
|
|
113
|
+
restart = true;
|
|
114
|
+
break;
|
|
115
|
+
}
|
|
116
|
+
const status = eligibility(groupId);
|
|
117
|
+
if (status.eligible)
|
|
118
|
+
return { job, nextRunAt };
|
|
119
|
+
if (status.retryAt !== null) {
|
|
120
|
+
nextRunAt = nextRunAt === null ? status.retryAt : Math.min(nextRunAt, status.retryAt);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
groupId = lane.next;
|
|
124
|
+
}
|
|
125
|
+
if (!restart)
|
|
126
|
+
return { job: null, nextRunAt };
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
advance(queue, groupId) {
|
|
130
|
+
const state = this.queues.get(queue);
|
|
131
|
+
const lane = state?.lanes.get(groupId);
|
|
132
|
+
if (state && lane?.inRotation)
|
|
133
|
+
state.cursor = lane.next;
|
|
134
|
+
}
|
|
135
|
+
getGroupJobsCount(queue, groupId) {
|
|
136
|
+
return this.queues.get(queue)?.groupCounts.get(groupId) ?? 0;
|
|
137
|
+
}
|
|
138
|
+
getGroupsJobsCount(queue) {
|
|
139
|
+
return this.queues.get(queue)?.groupedCount ?? 0;
|
|
140
|
+
}
|
|
141
|
+
clearQueue(queue) {
|
|
142
|
+
this.queues.delete(queue);
|
|
143
|
+
}
|
|
144
|
+
createState() {
|
|
145
|
+
return {
|
|
146
|
+
jobs: new Map(),
|
|
147
|
+
placements: new Map(),
|
|
148
|
+
plain: new IndexedPriorityQueue(),
|
|
149
|
+
delayed: new IndexedPriorityQueue(undefined, compareGroupWakeEntries, getGroupFifoOrder),
|
|
150
|
+
lanes: new Map(),
|
|
151
|
+
groupCounts: new Map(),
|
|
152
|
+
groupedCount: 0,
|
|
153
|
+
cursor: null,
|
|
154
|
+
rotationSize: 0,
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
insertTracked(state, job, now) {
|
|
158
|
+
state.jobs.set(job.id, job);
|
|
159
|
+
if (job.groupId) {
|
|
160
|
+
state.groupCounts.set(job.groupId, (state.groupCounts.get(job.groupId) ?? 0) + 1);
|
|
161
|
+
state.groupedCount++;
|
|
162
|
+
}
|
|
163
|
+
if (nextGroupCheckAt(job) > now) {
|
|
164
|
+
state.delayed.push(orderedGroupClone(job, nextGroupCheckAt(job)));
|
|
165
|
+
state.placements.set(job.id, { kind: 'delayed' });
|
|
166
|
+
return;
|
|
167
|
+
}
|
|
168
|
+
this.insertReady(state, job);
|
|
169
|
+
}
|
|
170
|
+
promoteDue(state, now) {
|
|
171
|
+
while (true) {
|
|
172
|
+
const scheduled = state.delayed.peek();
|
|
173
|
+
if (!scheduled || scheduled.runAt > now)
|
|
174
|
+
return;
|
|
175
|
+
state.delayed.pop();
|
|
176
|
+
const job = state.jobs.get(scheduled.id);
|
|
177
|
+
if (!job || state.placements.get(job.id)?.kind !== 'delayed')
|
|
178
|
+
continue;
|
|
179
|
+
this.insertReady(state, job);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
insertReady(state, job) {
|
|
183
|
+
if (!job.groupId) {
|
|
184
|
+
state.plain.push(job);
|
|
185
|
+
state.placements.set(job.id, { kind: 'plain' });
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
let lane = state.lanes.get(job.groupId);
|
|
189
|
+
if (!lane) {
|
|
190
|
+
lane = {
|
|
191
|
+
jobs: new IndexedPriorityQueue(undefined, compareGroupFifoEntries, getGroupFifoOrder),
|
|
192
|
+
previous: job.groupId,
|
|
193
|
+
next: job.groupId,
|
|
194
|
+
inRotation: false,
|
|
195
|
+
};
|
|
196
|
+
state.lanes.set(job.groupId, lane);
|
|
197
|
+
}
|
|
198
|
+
lane.jobs.push(orderedGroupClone(job));
|
|
199
|
+
state.placements.set(job.id, { kind: 'group', groupId: job.groupId });
|
|
200
|
+
if (!lane.inRotation)
|
|
201
|
+
this.attachLane(state, job.groupId, lane);
|
|
202
|
+
}
|
|
203
|
+
demoteReady(state, job) {
|
|
204
|
+
const placement = state.placements.get(job.id);
|
|
205
|
+
if (placement?.kind === 'plain') {
|
|
206
|
+
state.plain.remove(job.id);
|
|
207
|
+
}
|
|
208
|
+
else if (placement?.kind === 'group') {
|
|
209
|
+
const lane = state.lanes.get(placement.groupId);
|
|
210
|
+
lane?.jobs.remove(job.id);
|
|
211
|
+
if (lane?.jobs.isEmpty)
|
|
212
|
+
this.detachLane(state, placement.groupId, lane);
|
|
213
|
+
}
|
|
214
|
+
else {
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
state.delayed.push(orderedGroupClone(job, nextGroupCheckAt(job)));
|
|
218
|
+
state.placements.set(job.id, { kind: 'delayed' });
|
|
219
|
+
}
|
|
220
|
+
attachLane(state, groupId, lane) {
|
|
221
|
+
if (state.cursor === null) {
|
|
222
|
+
lane.previous = groupId;
|
|
223
|
+
lane.next = groupId;
|
|
224
|
+
state.cursor = groupId;
|
|
225
|
+
}
|
|
226
|
+
else {
|
|
227
|
+
const cursor = state.lanes.get(state.cursor);
|
|
228
|
+
const tail = state.lanes.get(cursor.previous);
|
|
229
|
+
lane.previous = cursor.previous;
|
|
230
|
+
lane.next = state.cursor;
|
|
231
|
+
tail.next = groupId;
|
|
232
|
+
cursor.previous = groupId;
|
|
233
|
+
}
|
|
234
|
+
lane.inRotation = true;
|
|
235
|
+
state.rotationSize++;
|
|
236
|
+
}
|
|
237
|
+
detachLane(state, groupId, lane) {
|
|
238
|
+
if (!lane.inRotation)
|
|
239
|
+
return;
|
|
240
|
+
if (state.rotationSize === 1) {
|
|
241
|
+
state.cursor = null;
|
|
242
|
+
}
|
|
243
|
+
else {
|
|
244
|
+
const previous = state.lanes.get(lane.previous);
|
|
245
|
+
const next = state.lanes.get(lane.next);
|
|
246
|
+
previous.next = lane.next;
|
|
247
|
+
next.previous = lane.previous;
|
|
248
|
+
if (state.cursor === groupId)
|
|
249
|
+
state.cursor = lane.next;
|
|
250
|
+
}
|
|
251
|
+
lane.inRotation = false;
|
|
252
|
+
state.rotationSize--;
|
|
253
|
+
if (lane.jobs.isEmpty && !state.groupCounts.has(groupId))
|
|
254
|
+
state.lanes.delete(groupId);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { Job } from '../types/job';
|
|
2
|
+
import type { HeapEntry } from '../types/priorityQueue';
|
|
3
|
+
export declare function orderedGroupClone(job: Job, runAt?: number): Job;
|
|
4
|
+
export declare function nextGroupCheckAt(job: Job): number;
|
|
5
|
+
export declare function compareGroupFifoEntries(left: HeapEntry, right: HeapEntry): number;
|
|
6
|
+
export declare function compareGroupWakeEntries(left: HeapEntry, right: HeapEntry): number;
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { copyGroupFifoOrder } from '../job/groupFifoOrder';
|
|
2
|
+
export function orderedGroupClone(job, runAt = job.runAt) {
|
|
3
|
+
const clone = { ...job, priority: 0, lifo: false, runAt };
|
|
4
|
+
copyGroupFifoOrder(job, clone);
|
|
5
|
+
return clone;
|
|
6
|
+
}
|
|
7
|
+
export function nextGroupCheckAt(job) {
|
|
8
|
+
if (job.ttl === null)
|
|
9
|
+
return job.runAt;
|
|
10
|
+
return Math.min(job.runAt, job.createdAt + job.ttl + 1);
|
|
11
|
+
}
|
|
12
|
+
function compareOrder(left, right) {
|
|
13
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
14
|
+
}
|
|
15
|
+
export function compareGroupFifoEntries(left, right) {
|
|
16
|
+
const order = compareOrder(left.groupFifoOrder ?? left.generation, right.groupFifoOrder ?? right.generation);
|
|
17
|
+
if (order !== 0)
|
|
18
|
+
return order;
|
|
19
|
+
return left.jobId < right.jobId ? -1 : left.jobId > right.jobId ? 1 : 0;
|
|
20
|
+
}
|
|
21
|
+
export function compareGroupWakeEntries(left, right) {
|
|
22
|
+
if (left.runAt !== right.runAt)
|
|
23
|
+
return left.runAt - right.runAt;
|
|
24
|
+
if (left.groupFifoOrder !== undefined && right.groupFifoOrder !== undefined) {
|
|
25
|
+
const order = compareOrder(left.groupFifoOrder, right.groupFifoOrder);
|
|
26
|
+
if (order !== 0)
|
|
27
|
+
return order;
|
|
28
|
+
}
|
|
29
|
+
return left.jobId < right.jobId ? -1 : left.jobId > right.jobId ? 1 : 0;
|
|
30
|
+
}
|
|
@@ -1,44 +1,35 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Indexed Priority Queue
|
|
3
|
-
* Combines binary heap for O(log n) priority operations
|
|
4
|
-
* with Map for O(1) lookups by job ID
|
|
5
|
-
*/
|
|
6
1
|
import type { Job, JobId } from '../types/job';
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
2
|
+
import type { HeapEntry } from '../types/priorityQueue';
|
|
3
|
+
export interface PriorityQueueHooks {
|
|
4
|
+
onInsert(job: Job): boolean;
|
|
5
|
+
onRemove(job: Job): boolean;
|
|
6
|
+
}
|
|
7
|
+
export type PriorityQueueComparator = (left: HeapEntry, right: HeapEntry) => number;
|
|
8
|
+
export type PriorityQueueOrder = (job: Job) => bigint | null;
|
|
9
|
+
/** Indexed map-backed 4-ary heap with O(log4 n) mutations and O(1) lookup. */
|
|
13
10
|
export declare class IndexedPriorityQueue {
|
|
14
|
-
|
|
11
|
+
private readonly hooks?;
|
|
12
|
+
private readonly compare;
|
|
13
|
+
private readonly order?;
|
|
15
14
|
private static readonly D;
|
|
16
15
|
private heap;
|
|
17
16
|
private readonly index;
|
|
18
17
|
private generation;
|
|
19
|
-
|
|
18
|
+
private hooksActive;
|
|
19
|
+
constructor(hooks?: PriorityQueueHooks | undefined, compare?: PriorityQueueComparator, order?: PriorityQueueOrder | undefined);
|
|
20
20
|
get size(): number;
|
|
21
|
-
/** Check if empty */
|
|
22
21
|
get isEmpty(): boolean;
|
|
23
|
-
/** Push a job into the queue */
|
|
24
22
|
push(job: Job): void;
|
|
25
|
-
/** Pop the highest priority job */
|
|
26
23
|
pop(): Job | null;
|
|
27
|
-
/** Peek at the highest priority job without removing */
|
|
28
24
|
peek(): Job | null;
|
|
29
|
-
/** Find a job by ID - O(1) */
|
|
30
25
|
find(jobId: JobId): Job | null;
|
|
31
|
-
/** Check if job exists - O(1) */
|
|
32
26
|
has(jobId: JobId): boolean;
|
|
33
|
-
/** Remove a job by ID - O(1) for index, heap cleans lazily */
|
|
34
27
|
remove(jobId: JobId): Job | null;
|
|
35
28
|
/** Update job priority - O(log n). Optionally update the LIFO tie-break flag. */
|
|
36
29
|
updatePriority(jobId: JobId, newPriority: number, newLifo?: boolean): boolean;
|
|
37
30
|
/** Update job runAt (for delay changes) - O(log n) */
|
|
38
31
|
updateRunAt(jobId: JobId, newRunAt: number): boolean;
|
|
39
|
-
/** Get all jobs (for iteration) */
|
|
40
32
|
values(): Job[];
|
|
41
|
-
/** Clear the queue */
|
|
42
33
|
clear(): void;
|
|
43
34
|
/**
|
|
44
35
|
* Get the ratio of stale entries in the heap
|
|
@@ -51,7 +42,6 @@ export declare class IndexedPriorityQueue {
|
|
|
51
42
|
* O(n) operation but reclaims memory and improves performance
|
|
52
43
|
*/
|
|
53
44
|
compact(): void;
|
|
54
|
-
/** Rebuild heap property from arbitrary array - O(n) */
|
|
55
45
|
private heapify;
|
|
56
46
|
/** Check if compaction is needed (stale ratio > threshold) */
|
|
57
47
|
needsCompaction(threshold?: number): boolean;
|
|
@@ -1,36 +1,29 @@
|
|
|
1
|
-
|
|
2
|
-
* Indexed Priority Queue
|
|
3
|
-
* Combines binary heap for O(log n) priority operations
|
|
4
|
-
* with Map for O(1) lookups by job ID
|
|
5
|
-
*/
|
|
1
|
+
import { copyGroupFifoOrder } from '../job/groupFifoOrder';
|
|
6
2
|
import { comparePriorityEntries } from './priorityQueueOrder';
|
|
7
|
-
/**
|
|
8
|
-
* Indexed Priority Queue implementation with 4-ary heap
|
|
9
|
-
* 4-ary provides better cache locality than binary heap
|
|
10
|
-
* O(log₄ n) push, pop, update
|
|
11
|
-
* O(1) find, has
|
|
12
|
-
*/
|
|
3
|
+
/** Indexed map-backed 4-ary heap with O(log4 n) mutations and O(1) lookup. */
|
|
13
4
|
export class IndexedPriorityQueue {
|
|
14
|
-
|
|
5
|
+
hooks;
|
|
6
|
+
compare;
|
|
7
|
+
order;
|
|
15
8
|
static D = 4;
|
|
16
9
|
heap = [];
|
|
17
10
|
index = new Map();
|
|
18
|
-
// Use BigInt to prevent overflow at extreme throughput
|
|
19
11
|
generation = 0n;
|
|
20
|
-
|
|
12
|
+
hooksActive = false;
|
|
13
|
+
constructor(hooks, compare = comparePriorityEntries, order) {
|
|
14
|
+
this.hooks = hooks;
|
|
15
|
+
this.compare = compare;
|
|
16
|
+
this.order = order;
|
|
17
|
+
}
|
|
21
18
|
get size() {
|
|
22
19
|
return this.index.size;
|
|
23
20
|
}
|
|
24
|
-
/** Check if empty */
|
|
25
21
|
get isEmpty() {
|
|
26
22
|
return this.index.size === 0;
|
|
27
23
|
}
|
|
28
|
-
/** Push a job into the queue */
|
|
29
24
|
push(job) {
|
|
30
25
|
const gen = this.generation++;
|
|
31
|
-
// Store in index
|
|
32
26
|
this.index.set(job.id, { job, generation: gen });
|
|
33
|
-
// Add to heap
|
|
34
27
|
const entry = {
|
|
35
28
|
jobId: job.id,
|
|
36
29
|
priority: job.priority,
|
|
@@ -38,32 +31,38 @@ export class IndexedPriorityQueue {
|
|
|
38
31
|
lifo: job.lifo,
|
|
39
32
|
generation: gen,
|
|
40
33
|
};
|
|
34
|
+
if (this.order) {
|
|
35
|
+
const groupFifoOrder = this.order(job);
|
|
36
|
+
if (groupFifoOrder !== null)
|
|
37
|
+
entry.groupFifoOrder = groupFifoOrder;
|
|
38
|
+
}
|
|
41
39
|
this.heap.push(entry);
|
|
42
40
|
this.bubbleUp(this.heap.length - 1);
|
|
41
|
+
if (this.hooks && (this.hooksActive || job.groupId)) {
|
|
42
|
+
this.hooksActive = this.hooks.onInsert(job);
|
|
43
|
+
}
|
|
43
44
|
}
|
|
44
|
-
/** Pop the highest priority job */
|
|
45
45
|
pop() {
|
|
46
46
|
while (this.heap.length > 0) {
|
|
47
47
|
const entry = this.heap[0];
|
|
48
48
|
const indexed = this.index.get(entry.jobId);
|
|
49
|
-
// Skip stale entries (generation mismatch = updated or removed)
|
|
50
49
|
if (indexed?.generation !== entry.generation) {
|
|
51
50
|
this.removeTop();
|
|
52
51
|
continue;
|
|
53
52
|
}
|
|
54
|
-
// Remove from both structures
|
|
55
53
|
this.removeTop();
|
|
56
54
|
this.index.delete(entry.jobId);
|
|
55
|
+
if (this.hooks && this.hooksActive) {
|
|
56
|
+
this.hooksActive = this.hooks.onRemove(indexed.job);
|
|
57
|
+
}
|
|
57
58
|
return indexed.job;
|
|
58
59
|
}
|
|
59
60
|
return null;
|
|
60
61
|
}
|
|
61
|
-
/** Peek at the highest priority job without removing */
|
|
62
62
|
peek() {
|
|
63
63
|
while (this.heap.length > 0) {
|
|
64
64
|
const entry = this.heap[0];
|
|
65
65
|
const indexed = this.index.get(entry.jobId);
|
|
66
|
-
// Skip stale entries
|
|
67
66
|
if (indexed?.generation !== entry.generation) {
|
|
68
67
|
this.removeTop();
|
|
69
68
|
continue;
|
|
@@ -72,21 +71,20 @@ export class IndexedPriorityQueue {
|
|
|
72
71
|
}
|
|
73
72
|
return null;
|
|
74
73
|
}
|
|
75
|
-
/** Find a job by ID - O(1) */
|
|
76
74
|
find(jobId) {
|
|
77
75
|
return this.index.get(jobId)?.job ?? null;
|
|
78
76
|
}
|
|
79
|
-
/** Check if job exists - O(1) */
|
|
80
77
|
has(jobId) {
|
|
81
78
|
return this.index.has(jobId);
|
|
82
79
|
}
|
|
83
|
-
/** Remove a job by ID - O(1) for index, heap cleans lazily */
|
|
84
80
|
remove(jobId) {
|
|
85
81
|
const indexed = this.index.get(jobId);
|
|
86
82
|
if (!indexed)
|
|
87
83
|
return null;
|
|
88
84
|
this.index.delete(jobId);
|
|
89
|
-
|
|
85
|
+
if (this.hooks && this.hooksActive) {
|
|
86
|
+
this.hooksActive = this.hooks.onRemove(indexed.job);
|
|
87
|
+
}
|
|
90
88
|
return indexed.job;
|
|
91
89
|
}
|
|
92
90
|
/** Update job priority - O(log n). Optionally update the LIFO tie-break flag. */
|
|
@@ -95,15 +93,13 @@ export class IndexedPriorityQueue {
|
|
|
95
93
|
if (!indexed)
|
|
96
94
|
return false;
|
|
97
95
|
const lifo = newLifo ?? indexed.job.lifo;
|
|
98
|
-
// Create new job with updated priority (immutable pattern)
|
|
99
96
|
const updatedJob = {
|
|
100
97
|
...indexed.job,
|
|
101
98
|
priority: newPriority,
|
|
102
99
|
lifo,
|
|
103
100
|
};
|
|
104
|
-
|
|
101
|
+
copyGroupFifoOrder(indexed.job, updatedJob);
|
|
105
102
|
const gen = this.generation++;
|
|
106
|
-
// Update index with new job and generation
|
|
107
103
|
this.index.set(jobId, { job: updatedJob, generation: gen });
|
|
108
104
|
const entry = {
|
|
109
105
|
jobId: updatedJob.id,
|
|
@@ -112,8 +108,16 @@ export class IndexedPriorityQueue {
|
|
|
112
108
|
lifo,
|
|
113
109
|
generation: gen,
|
|
114
110
|
};
|
|
111
|
+
if (this.order) {
|
|
112
|
+
const groupFifoOrder = this.order(updatedJob);
|
|
113
|
+
if (groupFifoOrder !== null)
|
|
114
|
+
entry.groupFifoOrder = groupFifoOrder;
|
|
115
|
+
}
|
|
115
116
|
this.heap.push(entry);
|
|
116
117
|
this.bubbleUp(this.heap.length - 1);
|
|
118
|
+
if (this.hooks && (this.hooksActive || updatedJob.groupId)) {
|
|
119
|
+
this.hooksActive = this.hooks.onInsert(updatedJob);
|
|
120
|
+
}
|
|
117
121
|
return true;
|
|
118
122
|
}
|
|
119
123
|
/** Update job runAt (for delay changes) - O(log n) */
|
|
@@ -121,14 +125,12 @@ export class IndexedPriorityQueue {
|
|
|
121
125
|
const indexed = this.index.get(jobId);
|
|
122
126
|
if (!indexed)
|
|
123
127
|
return false;
|
|
124
|
-
// Create new job with updated runAt (immutable pattern)
|
|
125
128
|
const updatedJob = {
|
|
126
129
|
...indexed.job,
|
|
127
130
|
runAt: newRunAt,
|
|
128
131
|
};
|
|
129
|
-
|
|
132
|
+
copyGroupFifoOrder(indexed.job, updatedJob);
|
|
130
133
|
const gen = this.generation++;
|
|
131
|
-
// Update index with new job and generation
|
|
132
134
|
this.index.set(jobId, { job: updatedJob, generation: gen });
|
|
133
135
|
const entry = {
|
|
134
136
|
jobId: updatedJob.id,
|
|
@@ -137,19 +139,33 @@ export class IndexedPriorityQueue {
|
|
|
137
139
|
lifo: updatedJob.lifo,
|
|
138
140
|
generation: gen,
|
|
139
141
|
};
|
|
142
|
+
if (this.order) {
|
|
143
|
+
const groupFifoOrder = this.order(updatedJob);
|
|
144
|
+
if (groupFifoOrder !== null)
|
|
145
|
+
entry.groupFifoOrder = groupFifoOrder;
|
|
146
|
+
}
|
|
140
147
|
this.heap.push(entry);
|
|
141
148
|
this.bubbleUp(this.heap.length - 1);
|
|
149
|
+
if (this.hooks && (this.hooksActive || updatedJob.groupId)) {
|
|
150
|
+
this.hooksActive = this.hooks.onInsert(updatedJob);
|
|
151
|
+
}
|
|
142
152
|
return true;
|
|
143
153
|
}
|
|
144
|
-
/** Get all jobs (for iteration) */
|
|
145
154
|
values() {
|
|
146
155
|
return Array.from(this.index.values()).map((v) => v.job);
|
|
147
156
|
}
|
|
148
|
-
/** Clear the queue */
|
|
149
157
|
clear() {
|
|
158
|
+
if (this.hooks) {
|
|
159
|
+
for (const { job } of this.index.values()) {
|
|
160
|
+
if (!this.hooksActive)
|
|
161
|
+
break;
|
|
162
|
+
this.hooksActive = this.hooks.onRemove(job);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
150
165
|
this.heap = [];
|
|
151
166
|
this.index.clear();
|
|
152
167
|
this.generation = 0n;
|
|
168
|
+
this.hooksActive = false;
|
|
153
169
|
}
|
|
154
170
|
/**
|
|
155
171
|
* Get the ratio of stale entries in the heap
|
|
@@ -180,11 +196,8 @@ export class IndexedPriorityQueue {
|
|
|
180
196
|
this.heap = validEntries;
|
|
181
197
|
this.heapify();
|
|
182
198
|
}
|
|
183
|
-
/** Rebuild heap property from arbitrary array - O(n) */
|
|
184
199
|
heapify() {
|
|
185
200
|
const D = IndexedPriorityQueue.D;
|
|
186
|
-
// Start from last non-leaf node and bubble down
|
|
187
|
-
// In D-ary heap, last non-leaf is at floor((n-2)/D)
|
|
188
201
|
for (let i = Math.floor((this.heap.length - 2) / D); i >= 0; i--) {
|
|
189
202
|
this.bubbleDown(i);
|
|
190
203
|
}
|
|
@@ -193,7 +206,6 @@ export class IndexedPriorityQueue {
|
|
|
193
206
|
needsCompaction(threshold = 0.2) {
|
|
194
207
|
return this.getStaleRatio() > threshold;
|
|
195
208
|
}
|
|
196
|
-
// ============ Heap Operations ============
|
|
197
209
|
removeTop() {
|
|
198
210
|
if (this.heap.length <= 1) {
|
|
199
211
|
this.heap.pop();
|
|
@@ -207,7 +219,7 @@ export class IndexedPriorityQueue {
|
|
|
207
219
|
const D = IndexedPriorityQueue.D;
|
|
208
220
|
while (idx > 0) {
|
|
209
221
|
const parentIdx = Math.floor((idx - 1) / D);
|
|
210
|
-
if (
|
|
222
|
+
if (this.compare(this.heap[idx], this.heap[parentIdx]) >= 0) {
|
|
211
223
|
break;
|
|
212
224
|
}
|
|
213
225
|
this.swap(idx, parentIdx);
|
|
@@ -223,11 +235,10 @@ export class IndexedPriorityQueue {
|
|
|
223
235
|
const firstChild = D * idx + 1;
|
|
224
236
|
if (firstChild >= length)
|
|
225
237
|
break;
|
|
226
|
-
// Find minimum among up to D children (cache-friendly sequential access)
|
|
227
238
|
let smallest = idx;
|
|
228
239
|
const lastChild = Math.min(firstChild + D, length);
|
|
229
240
|
for (let c = firstChild; c < lastChild; c++) {
|
|
230
|
-
if (
|
|
241
|
+
if (this.compare(heap[c], heap[smallest]) < 0) {
|
|
231
242
|
smallest = c;
|
|
232
243
|
}
|
|
233
244
|
}
|
|
@@ -18,6 +18,7 @@ export declare class ShardDlq extends ShardDependencies {
|
|
|
18
18
|
getDlq(queue: string, count?: number): Job[];
|
|
19
19
|
getDlqFiltered(queue: string, filter: DlqFilter): DlqEntry[];
|
|
20
20
|
removeFromDlq(queue: string, jobId: JobId): DlqEntry | null;
|
|
21
|
+
removeAllFromDlq(queue: string, jobId: JobId): DlqEntry[];
|
|
21
22
|
getAutoRetryEntries(queue: string, now?: number): DlqEntry[];
|
|
22
23
|
getExpiredEntries(queue: string, now?: number): DlqEntry[];
|
|
23
24
|
purgeExpired(queue: string, now?: number): number;
|
|
@@ -56,6 +56,9 @@ export class ShardDlq extends ShardDependencies {
|
|
|
56
56
|
removeFromDlq(queue, jobId) {
|
|
57
57
|
return this.dlqManager.remove(queue, jobId);
|
|
58
58
|
}
|
|
59
|
+
removeAllFromDlq(queue, jobId) {
|
|
60
|
+
return this.dlqManager.removeAll(queue, jobId);
|
|
61
|
+
}
|
|
59
62
|
getAutoRetryEntries(queue, now = Date.now()) {
|
|
60
63
|
return this.dlqManager.getAutoRetryEntries(queue, now);
|
|
61
64
|
}
|
|
@@ -15,4 +15,5 @@ export declare class ShardKeys extends ShardState {
|
|
|
15
15
|
isGroupActive(queue: string, groupId: string): boolean;
|
|
16
16
|
activateGroup(queue: string, groupId: string): void;
|
|
17
17
|
releaseGroup(queue: string, groupId: string): void;
|
|
18
|
+
getGroupActiveCount(queue: string, groupId: string): number;
|
|
18
19
|
}
|
|
@@ -29,7 +29,7 @@ export class ShardKeys extends ShardState {
|
|
|
29
29
|
return this.uniqueKeyManager.getMap();
|
|
30
30
|
}
|
|
31
31
|
isGroupActive(queue, groupId) {
|
|
32
|
-
return this.
|
|
32
|
+
return (this.activeGroupCounts.get(queue)?.get(groupId) ?? 0) > 0;
|
|
33
33
|
}
|
|
34
34
|
activateGroup(queue, groupId) {
|
|
35
35
|
let groups = this.activeGroups.get(queue);
|
|
@@ -38,8 +38,31 @@ export class ShardKeys extends ShardState {
|
|
|
38
38
|
this.activeGroups.set(queue, groups);
|
|
39
39
|
}
|
|
40
40
|
groups.add(groupId);
|
|
41
|
+
let counts = this.activeGroupCounts.get(queue);
|
|
42
|
+
if (!counts) {
|
|
43
|
+
counts = new Map();
|
|
44
|
+
this.activeGroupCounts.set(queue, counts);
|
|
45
|
+
}
|
|
46
|
+
counts.set(groupId, (counts.get(groupId) ?? 0) + 1);
|
|
41
47
|
}
|
|
42
48
|
releaseGroup(queue, groupId) {
|
|
43
|
-
this.
|
|
49
|
+
const counts = this.activeGroupCounts.get(queue);
|
|
50
|
+
const count = (counts?.get(groupId) ?? 0) - 1;
|
|
51
|
+
if (count > 0) {
|
|
52
|
+
counts?.set(groupId, count);
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
counts?.delete(groupId);
|
|
56
|
+
const groups = this.activeGroups.get(queue);
|
|
57
|
+
groups?.delete(groupId);
|
|
58
|
+
if (groups?.size === 0)
|
|
59
|
+
this.activeGroups.delete(queue);
|
|
60
|
+
}
|
|
61
|
+
if (counts?.size === 0)
|
|
62
|
+
this.activeGroupCounts.delete(queue);
|
|
63
|
+
this.notify(queue);
|
|
64
|
+
}
|
|
65
|
+
getGroupActiveCount(queue, groupId) {
|
|
66
|
+
return this.activeGroupCounts.get(queue)?.get(groupId) ?? 0;
|
|
44
67
|
}
|
|
45
68
|
}
|
|
@@ -12,6 +12,7 @@ export class ShardLifecycle extends ShardMetrics {
|
|
|
12
12
|
this.temporalManager.removeDelayed(job.id);
|
|
13
13
|
}
|
|
14
14
|
priorityQueue.clear();
|
|
15
|
+
this.groupScheduler.clearQueue(queue);
|
|
15
16
|
this.temporalManager.clearIndexForQueue(queue);
|
|
16
17
|
this.counters.adjustQueued(-count);
|
|
17
18
|
this.counters.syncDelayedCount();
|
|
@@ -37,7 +38,10 @@ export class ShardLifecycle extends ShardMetrics {
|
|
|
37
38
|
this.queues.delete(queue);
|
|
38
39
|
this.uniqueKeyManager.clearQueue(queue);
|
|
39
40
|
this.limiterManager.deleteQueue(queue);
|
|
41
|
+
this.groupScheduler.clearQueue(queue);
|
|
42
|
+
this.groupLimiterManager.clearQueue(queue);
|
|
40
43
|
this.activeGroups.delete(queue);
|
|
44
|
+
this.activeGroupCounts.delete(queue);
|
|
41
45
|
return Array.from(removed);
|
|
42
46
|
}
|
|
43
47
|
}
|