bunqueue 2.8.60 → 2.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +114 -78
- package/dist/application/background/dlq.js +2 -0
- package/dist/application/background/recovery/active.d.ts +1 -0
- package/dist/application/background/recovery/active.js +12 -0
- package/dist/application/background/recovery/index.js +3 -1
- package/dist/application/background/recovery/pending.js +1 -0
- package/dist/application/cleanupTasks.js +4 -16
- package/dist/application/contextFactory.js +8 -0
- package/dist/application/dlqManager.d.ts +10 -0
- package/dist/application/dlqManager.js +48 -9
- package/dist/application/dlqRetry.js +60 -38
- package/dist/application/eventsManager.d.ts +9 -2
- package/dist/application/eventsManager.js +106 -55
- package/dist/application/operations/ackHelpers.d.ts +1 -4
- package/dist/application/operations/ackHelpers.js +39 -14
- package/dist/application/operations/flowPush.js +1 -0
- package/dist/application/operations/flowValidation.js +4 -4
- package/dist/application/operations/jobManagement.d.ts +4 -0
- package/dist/application/operations/jobMoveOperations.js +9 -4
- package/dist/application/operations/pull.d.ts +7 -2
- package/dist/application/operations/pull.js +15 -57
- package/dist/application/operations/pullFinalization.d.ts +6 -0
- package/dist/application/operations/pullFinalization.js +69 -0
- package/dist/application/operations/pullStateTransition.d.ts +8 -7
- package/dist/application/operations/pullStateTransition.js +25 -18
- package/dist/application/operations/push.js +3 -0
- package/dist/application/operations/pushBatch.js +14 -7
- package/dist/application/operations/pushContext.d.ts +2 -1
- package/dist/application/postgres-queue-manager/batchSnapshot.d.ts +16 -0
- package/dist/application/postgres-queue-manager/batchSnapshot.js +17 -0
- package/dist/application/postgres-queue-manager/cloud.d.ts +30 -0
- package/dist/application/postgres-queue-manager/cloud.js +55 -0
- package/dist/application/postgres-queue-manager/config.d.ts +5 -0
- package/dist/application/postgres-queue-manager/config.js +1 -0
- package/dist/application/postgres-queue-manager/control.d.ts +30 -0
- package/dist/application/postgres-queue-manager/control.js +210 -0
- package/dist/application/postgres-queue-manager/deferredWrites.d.ts +13 -0
- package/dist/application/postgres-queue-manager/deferredWrites.js +60 -0
- package/dist/application/postgres-queue-manager/delivery.d.ts +20 -0
- package/dist/application/postgres-queue-manager/delivery.js +83 -0
- package/dist/application/postgres-queue-manager/groupQueries.d.ts +10 -0
- package/dist/application/postgres-queue-manager/groupQueries.js +40 -0
- package/dist/application/postgres-queue-manager/lease.d.ts +13 -0
- package/dist/application/postgres-queue-manager/lease.js +88 -0
- package/dist/application/postgres-queue-manager/lifetimeMetrics.d.ts +32 -0
- package/dist/application/postgres-queue-manager/lifetimeMetrics.js +53 -0
- package/dist/application/postgres-queue-manager/maintenance.d.ts +14 -0
- package/dist/application/postgres-queue-manager/maintenance.js +100 -0
- package/dist/application/postgres-queue-manager/operationGate.d.ts +26 -0
- package/dist/application/postgres-queue-manager/operationGate.js +96 -0
- package/dist/application/postgres-queue-manager/operations.d.ts +25 -0
- package/dist/application/postgres-queue-manager/operations.js +194 -0
- package/dist/application/postgres-queue-manager/projectionRefreshes.d.ts +43 -0
- package/dist/application/postgres-queue-manager/projectionRefreshes.js +170 -0
- package/dist/application/postgres-queue-manager/queries.d.ts +142 -0
- package/dist/application/postgres-queue-manager/queries.js +231 -0
- package/dist/application/postgres-queue-manager/relationships.d.ts +10 -0
- package/dist/application/postgres-queue-manager/relationships.js +45 -0
- package/dist/application/postgres-queue-manager/services.d.ts +31 -0
- package/dist/application/postgres-queue-manager/services.js +149 -0
- package/dist/application/postgres-queue-manager/snapshot.d.ts +51 -0
- package/dist/application/postgres-queue-manager/snapshot.js +236 -0
- package/dist/application/postgres-queue-manager/snapshotViews.d.ts +15 -0
- package/dist/application/postgres-queue-manager/snapshotViews.js +100 -0
- package/dist/application/postgres-queue-manager/startupEventBuffer.d.ts +12 -0
- package/dist/application/postgres-queue-manager/startupEventBuffer.js +36 -0
- package/dist/application/postgres-queue-manager/state.d.ts +47 -0
- package/dist/application/postgres-queue-manager/state.js +273 -0
- package/dist/application/postgres-queue-manager/terminalDelivery.d.ts +20 -0
- package/dist/application/postgres-queue-manager/terminalDelivery.js +100 -0
- package/dist/application/postgresQueueManager.d.ts +2 -0
- package/dist/application/postgresQueueManager.js +1 -0
- package/dist/application/queue-manager/control.js +1 -0
- package/dist/application/queue-manager/delivery.d.ts +5 -4
- package/dist/application/queue-manager/delivery.js +15 -8
- package/dist/application/queue-manager/flow-failures.js +4 -0
- package/dist/application/queue-manager/limits.d.ts +13 -0
- package/dist/application/queue-manager/limits.js +65 -0
- package/dist/application/queue-manager/observability.js +15 -3
- package/dist/application/queue-manager/queries.d.ts +7 -0
- package/dist/application/queue-manager/queries.js +9 -0
- package/dist/application/queue-manager/services.d.ts +1 -1
- package/dist/application/queue-manager/services.js +4 -2
- package/dist/application/queue-manager/state.js +17 -2
- package/dist/application/queueTelemetryJournal.d.ts +2 -0
- package/dist/application/queueTelemetryJournal.js +23 -0
- package/dist/application/types/ack.d.ts +3 -1
- package/dist/cli/commandRouter.d.ts +1 -1
- package/dist/cli/commands/server.js +1 -1
- package/dist/client/flowPush.js +6 -0
- package/dist/client/forwarder.d.ts +6 -4
- package/dist/client/forwarder.js +5 -3
- package/dist/client/groupId.d.ts +1 -0
- package/dist/client/groupId.js +1 -0
- package/dist/client/index.d.ts +1 -1
- package/dist/client/jobHelpers.js +1 -0
- package/dist/client/queue/dlq.d.ts +2 -0
- package/dist/client/queue/dlq.js +14 -0
- package/dist/client/queue/dlqOps.d.ts +2 -0
- package/dist/client/queue/dlqOps.js +4 -0
- package/dist/client/queue/helpers.js +2 -0
- package/dist/client/queue/operations/add/bulk.js +4 -4
- package/dist/client/queue/operations/add/payload.d.ts +1 -0
- package/dist/client/queue/operations/add/payload.js +7 -1
- package/dist/client/queue/operations/add/single.js +3 -3
- package/dist/client/queue/operations/groups.d.ts +21 -0
- package/dist/client/queue/operations/groups.js +88 -0
- package/dist/client/queue/runtime/configuration.d.ts +4 -0
- package/dist/client/queue/runtime/configuration.js +8 -0
- package/dist/client/queue/runtime/queries.d.ts +13 -0
- package/dist/client/queue/runtime/queries.js +31 -0
- package/dist/client/queue/runtime/state.d.ts +4 -0
- package/dist/client/queue/runtime/state.js +1 -0
- package/dist/client/types/index.d.ts +2 -2
- package/dist/client/types/options.d.ts +4 -0
- package/dist/client/types/worker.d.ts +8 -0
- package/dist/client/worker/runtime/execution.js +4 -8
- package/dist/client/worker/runtime/options.js +3 -0
- package/dist/client/worker/runtime/polling.d.ts +1 -0
- package/dist/client/worker/runtime/polling.js +9 -0
- package/dist/client/worker/types/options.d.ts +2 -1
- package/dist/client/worker/workerPull.d.ts +2 -0
- package/dist/client/worker/workerPull.js +6 -4
- package/dist/config/resolve.d.ts +14 -0
- package/dist/config/resolve.js +50 -5
- package/dist/config/types.d.ts +18 -0
- package/dist/domain/job/groupFifoOrder.d.ts +9 -0
- package/dist/domain/job/groupFifoOrder.js +26 -0
- package/dist/domain/queue/dlqShard.d.ts +2 -0
- package/dist/domain/queue/dlqShard.js +16 -0
- package/dist/domain/queue/groupLimiterManager.d.ts +23 -0
- package/dist/domain/queue/groupLimiterManager.js +107 -0
- package/dist/domain/queue/groupScheduler.d.ts +31 -0
- package/dist/domain/queue/groupScheduler.js +256 -0
- package/dist/domain/queue/groupSchedulerOrder.d.ts +6 -0
- package/dist/domain/queue/groupSchedulerOrder.js +30 -0
- package/dist/domain/queue/priorityQueue.d.ts +13 -23
- package/dist/domain/queue/priorityQueue.js +53 -42
- package/dist/domain/queue/shard/dlq.d.ts +1 -0
- package/dist/domain/queue/shard/dlq.js +3 -0
- package/dist/domain/queue/shard/keys.d.ts +1 -0
- package/dist/domain/queue/shard/keys.js +25 -2
- package/dist/domain/queue/shard/lifecycle.js +4 -0
- package/dist/domain/queue/shard/limits.d.ts +20 -1
- package/dist/domain/queue/shard/limits.js +56 -0
- package/dist/domain/queue/shard/state.d.ts +5 -0
- package/dist/domain/queue/shard/state.js +9 -1
- package/dist/domain/types/commands/core.d.ts +3 -0
- package/dist/domain/types/commands/dlq.d.ts +5 -0
- package/dist/domain/types/commands/limits.d.ts +41 -0
- package/dist/domain/types/commands/union.d.ts +1 -1
- package/dist/domain/types/cron.d.ts +5 -0
- package/dist/domain/types/cron.js +14 -3
- package/dist/domain/types/dlq.d.ts +1 -1
- package/dist/domain/types/dlq.js +1 -2
- package/dist/domain/types/group.d.ts +21 -0
- package/dist/domain/types/group.js +78 -0
- package/dist/domain/types/jobs/model.d.ts +1 -1
- package/dist/domain/types/priorityQueue.d.ts +1 -0
- package/dist/infrastructure/cloud/commandHandler.js +5 -3
- package/dist/infrastructure/cloud/commands/integrations.js +11 -11
- package/dist/infrastructure/cloud/commands/jobs.js +48 -40
- package/dist/infrastructure/cloud/commands/queues.js +61 -75
- package/dist/infrastructure/cloud/queueAdapter/local.d.ts +41 -0
- package/dist/infrastructure/cloud/queueAdapter/local.js +187 -0
- package/dist/infrastructure/cloud/queueAdapter/postgres.d.ts +42 -0
- package/dist/infrastructure/cloud/queueAdapter/postgres.js +188 -0
- package/dist/infrastructure/cloud/queueAdapter/registry.d.ts +4 -0
- package/dist/infrastructure/cloud/queueAdapter/registry.js +15 -0
- package/dist/infrastructure/cloud/queueAdapter/types.d.ts +87 -0
- package/dist/infrastructure/cloud/queueAdapter/types.js +1 -0
- package/dist/infrastructure/cloud/snapshot/analytics.d.ts +2 -0
- package/dist/infrastructure/cloud/snapshot/analytics.js +7 -0
- package/dist/infrastructure/cloud/snapshot/collector.d.ts +4 -0
- package/dist/infrastructure/cloud/snapshot/collector.js +39 -1
- package/dist/infrastructure/cloud/snapshot/config.d.ts +3 -0
- package/dist/infrastructure/cloud/snapshot/config.js +43 -0
- package/dist/infrastructure/cloud/snapshot/jobs.d.ts +3 -0
- package/dist/infrastructure/cloud/snapshot/jobs.js +32 -1
- package/dist/infrastructure/cloud/snapshotCollector.js +20 -35
- package/dist/infrastructure/cloud/snapshotHelpers.d.ts +3 -3
- package/dist/infrastructure/cloud/snapshotHelpers.js +3 -3
- package/dist/infrastructure/cloud/statsRefresh.js +2 -1
- package/dist/infrastructure/cloud/types/command.d.ts +2 -2
- package/dist/infrastructure/persistence/jobOptionsBlob.d.ts +1 -0
- package/dist/infrastructure/persistence/jobOptionsBlob.js +10 -2
- package/dist/infrastructure/persistence/migrations.js +11 -0
- package/dist/infrastructure/persistence/postgres/admission.d.ts +18 -0
- package/dist/infrastructure/persistence/postgres/admission.js +196 -0
- package/dist/infrastructure/persistence/postgres/admissionParent.d.ts +5 -0
- package/dist/infrastructure/persistence/postgres/admissionParent.js +35 -0
- package/dist/infrastructure/persistence/postgres/admissionResult.d.ts +9 -0
- package/dist/infrastructure/persistence/postgres/admissionResult.js +1 -0
- package/dist/infrastructure/persistence/postgres/admissionStore.d.ts +12 -0
- package/dist/infrastructure/persistence/postgres/admissionStore.js +101 -0
- package/dist/infrastructure/persistence/postgres/advisoryLocks.d.ts +2 -0
- package/dist/infrastructure/persistence/postgres/advisoryLocks.js +13 -0
- package/dist/infrastructure/persistence/postgres/batchAdmission.d.ts +11 -0
- package/dist/infrastructure/persistence/postgres/batchAdmission.js +128 -0
- package/dist/infrastructure/persistence/postgres/batchCompletion.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/batchCompletion.js +174 -0
- package/dist/infrastructure/persistence/postgres/batchEvents.d.ts +14 -0
- package/dist/infrastructure/persistence/postgres/batchEvents.js +72 -0
- package/dist/infrastructure/persistence/postgres/brokerSessions.d.ts +14 -0
- package/dist/infrastructure/persistence/postgres/brokerSessions.js +94 -0
- package/dist/infrastructure/persistence/postgres/claimBatch.d.ts +14 -0
- package/dist/infrastructure/persistence/postgres/claimBatch.js +69 -0
- package/dist/infrastructure/persistence/postgres/claimSelection.d.ts +11 -0
- package/dist/infrastructure/persistence/postgres/claimSelection.js +140 -0
- package/dist/infrastructure/persistence/postgres/claims.d.ts +10 -0
- package/dist/infrastructure/persistence/postgres/claims.js +153 -0
- package/dist/infrastructure/persistence/postgres/cloudReadModel.d.ts +26 -0
- package/dist/infrastructure/persistence/postgres/cloudReadModel.js +123 -0
- package/dist/infrastructure/persistence/postgres/codec.d.ts +30 -0
- package/dist/infrastructure/persistence/postgres/codec.js +82 -0
- package/dist/infrastructure/persistence/postgres/completionEvents.d.ts +11 -0
- package/dist/infrastructure/persistence/postgres/completionEvents.js +21 -0
- package/dist/infrastructure/persistence/postgres/completionLifecycle.d.ts +11 -0
- package/dist/infrastructure/persistence/postgres/completionLifecycle.js +145 -0
- package/dist/infrastructure/persistence/postgres/completionOutcome.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/completionOutcome.js +112 -0
- package/dist/infrastructure/persistence/postgres/completionQueries.d.ts +10 -0
- package/dist/infrastructure/persistence/postgres/completionQueries.js +87 -0
- package/dist/infrastructure/persistence/postgres/connection.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/connection.js +15 -0
- package/dist/infrastructure/persistence/postgres/context.d.ts +28 -0
- package/dist/infrastructure/persistence/postgres/context.js +74 -0
- package/dist/infrastructure/persistence/postgres/control.d.ts +23 -0
- package/dist/infrastructure/persistence/postgres/control.js +105 -0
- package/dist/infrastructure/persistence/postgres/crons.d.ts +18 -0
- package/dist/infrastructure/persistence/postgres/crons.js +205 -0
- package/dist/infrastructure/persistence/postgres/deduplication.d.ts +9 -0
- package/dist/infrastructure/persistence/postgres/deduplication.js +47 -0
- package/dist/infrastructure/persistence/postgres/dependencyDestruction.d.ts +14 -0
- package/dist/infrastructure/persistence/postgres/dependencyDestruction.js +74 -0
- package/dist/infrastructure/persistence/postgres/dependencyPromotion.d.ts +29 -0
- package/dist/infrastructure/persistence/postgres/dependencyPromotion.js +212 -0
- package/dist/infrastructure/persistence/postgres/destructiveMutations.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/destructiveMutations.js +111 -0
- package/dist/infrastructure/persistence/postgres/dlqLifecycle.d.ts +13 -0
- package/dist/infrastructure/persistence/postgres/dlqLifecycle.js +184 -0
- package/dist/infrastructure/persistence/postgres/dlqMaintenance.d.ts +6 -0
- package/dist/infrastructure/persistence/postgres/dlqMaintenance.js +36 -0
- package/dist/infrastructure/persistence/postgres/dlqRetryPlan.d.ts +21 -0
- package/dist/infrastructure/persistence/postgres/dlqRetryPlan.js +92 -0
- package/dist/infrastructure/persistence/postgres/eventCatchupCursors.d.ts +32 -0
- package/dist/infrastructure/persistence/postgres/eventCatchupCursors.js +66 -0
- package/dist/infrastructure/persistence/postgres/eventCommitGc.d.ts +27 -0
- package/dist/infrastructure/persistence/postgres/eventCommitGc.js +62 -0
- package/dist/infrastructure/persistence/postgres/eventJournal.d.ts +18 -0
- package/dist/infrastructure/persistence/postgres/eventJournal.js +76 -0
- package/dist/infrastructure/persistence/postgres/eventJournalSchema.d.ts +5 -0
- package/dist/infrastructure/persistence/postgres/eventJournalSchema.js +267 -0
- package/dist/infrastructure/persistence/postgres/eventPruneWatermarks.d.ts +26 -0
- package/dist/infrastructure/persistence/postgres/eventPruneWatermarks.js +129 -0
- package/dist/infrastructure/persistence/postgres/eventRetention.d.ts +18 -0
- package/dist/infrastructure/persistence/postgres/eventRetention.js +76 -0
- package/dist/infrastructure/persistence/postgres/events.d.ts +45 -0
- package/dist/infrastructure/persistence/postgres/events.js +276 -0
- package/dist/infrastructure/persistence/postgres/expiry.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/expiry.js +74 -0
- package/dist/infrastructure/persistence/postgres/flowFailures.d.ts +17 -0
- package/dist/infrastructure/persistence/postgres/flowFailures.js +199 -0
- package/dist/infrastructure/persistence/postgres/groupClaims.d.ts +15 -0
- package/dist/infrastructure/persistence/postgres/groupClaims.js +95 -0
- package/dist/infrastructure/persistence/postgres/groupSchema.d.ts +2 -0
- package/dist/infrastructure/persistence/postgres/groupSchema.js +155 -0
- package/dist/infrastructure/persistence/postgres/groupSchemaFingerprint.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/groupSchemaFingerprint.js +127 -0
- package/dist/infrastructure/persistence/postgres/groupStateRetention.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/groupStateRetention.js +93 -0
- package/dist/infrastructure/persistence/postgres/groups.d.ts +11 -0
- package/dist/infrastructure/persistence/postgres/groups.js +117 -0
- package/dist/infrastructure/persistence/postgres/jobData.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/jobData.js +32 -0
- package/dist/infrastructure/persistence/postgres/leaseRelease.d.ts +9 -0
- package/dist/infrastructure/persistence/postgres/leaseRelease.js +120 -0
- package/dist/infrastructure/persistence/postgres/leaseRenewal.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/leaseRenewal.js +35 -0
- package/dist/infrastructure/persistence/postgres/lifetimeMetrics.d.ts +15 -0
- package/dist/infrastructure/persistence/postgres/lifetimeMetrics.js +46 -0
- package/dist/infrastructure/persistence/postgres/lifetimeMetricsFinalizer.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/lifetimeMetricsFinalizer.js +13 -0
- package/dist/infrastructure/persistence/postgres/logs.d.ts +6 -0
- package/dist/infrastructure/persistence/postgres/logs.js +70 -0
- package/dist/infrastructure/persistence/postgres/maintenance.d.ts +6 -0
- package/dist/infrastructure/persistence/postgres/maintenance.js +138 -0
- package/dist/infrastructure/persistence/postgres/maintenanceFlights.d.ts +14 -0
- package/dist/infrastructure/persistence/postgres/maintenanceFlights.js +66 -0
- package/dist/infrastructure/persistence/postgres/metricWrites.d.ts +9 -0
- package/dist/infrastructure/persistence/postgres/metricWrites.js +94 -0
- package/dist/infrastructure/persistence/postgres/mutations.d.ts +9 -0
- package/dist/infrastructure/persistence/postgres/mutations.js +171 -0
- package/dist/infrastructure/persistence/postgres/outcomes.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/outcomes.js +194 -0
- package/dist/infrastructure/persistence/postgres/policies.d.ts +11 -0
- package/dist/infrastructure/persistence/postgres/policies.js +18 -0
- package/dist/infrastructure/persistence/postgres/postCommitMaintenance.d.ts +17 -0
- package/dist/infrastructure/persistence/postgres/postCommitMaintenance.js +117 -0
- package/dist/infrastructure/persistence/postgres/promotion.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/promotion.js +43 -0
- package/dist/infrastructure/persistence/postgres/queries.d.ts +22 -0
- package/dist/infrastructure/persistence/postgres/queries.js +152 -0
- package/dist/infrastructure/persistence/postgres/queueDestruction.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/queueDestruction.js +158 -0
- package/dist/infrastructure/persistence/postgres/queueLifecycle.d.ts +10 -0
- package/dist/infrastructure/persistence/postgres/queueLifecycle.js +60 -0
- package/dist/infrastructure/persistence/postgres/rateLimit.d.ts +6 -0
- package/dist/infrastructure/persistence/postgres/rateLimit.js +10 -0
- package/dist/infrastructure/persistence/postgres/readModels.d.ts +32 -0
- package/dist/infrastructure/persistence/postgres/readModels.js +72 -0
- package/dist/infrastructure/persistence/postgres/recovery.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/recovery.js +178 -0
- package/dist/infrastructure/persistence/postgres/relationships.d.ts +9 -0
- package/dist/infrastructure/persistence/postgres/relationships.js +201 -0
- package/dist/infrastructure/persistence/postgres/removeUnprocessedChildren.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/removeUnprocessedChildren.js +241 -0
- package/dist/infrastructure/persistence/postgres/repeatMutations.d.ts +5 -0
- package/dist/infrastructure/persistence/postgres/repeatMutations.js +37 -0
- package/dist/infrastructure/persistence/postgres/runtime.d.ts +45 -0
- package/dist/infrastructure/persistence/postgres/runtime.js +238 -0
- package/dist/infrastructure/persistence/postgres/runtimeConfig.d.ts +21 -0
- package/dist/infrastructure/persistence/postgres/runtimeConfig.js +39 -0
- package/dist/infrastructure/persistence/postgres/schema.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/schema.js +261 -0
- package/dist/infrastructure/persistence/postgres/schemaInitialization.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/schemaInitialization.js +226 -0
- package/dist/infrastructure/persistence/postgres/serialAdmission.d.ts +6 -0
- package/dist/infrastructure/persistence/postgres/serialAdmission.js +96 -0
- package/dist/infrastructure/persistence/postgres/snapshotBudget.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/snapshotBudget.js +83 -0
- package/dist/infrastructure/persistence/postgres/storageHealth.d.ts +10 -0
- package/dist/infrastructure/persistence/postgres/storageHealth.js +29 -0
- package/dist/infrastructure/persistence/postgres/store.d.ts +100 -0
- package/dist/infrastructure/persistence/postgres/store.js +180 -0
- package/dist/infrastructure/persistence/postgres/telemetry.d.ts +8 -0
- package/dist/infrastructure/persistence/postgres/telemetry.js +100 -0
- package/dist/infrastructure/persistence/postgres/terminal.d.ts +8 -0
- package/dist/infrastructure/persistence/postgres/terminal.js +137 -0
- package/dist/infrastructure/persistence/postgres/transactionRetry.d.ts +6 -0
- package/dist/infrastructure/persistence/postgres/transactionRetry.js +42 -0
- package/dist/infrastructure/persistence/postgres/types.d.ts +176 -0
- package/dist/infrastructure/persistence/postgres/types.js +1 -0
- package/dist/infrastructure/persistence/postgres/workers.d.ts +15 -0
- package/dist/infrastructure/persistence/postgres/workers.js +126 -0
- package/dist/infrastructure/persistence/postgres.d.ts +2 -0
- package/dist/infrastructure/persistence/postgres.js +1 -0
- package/dist/infrastructure/persistence/schema.d.ts +2 -2
- package/dist/infrastructure/persistence/schema.js +12 -1
- package/dist/infrastructure/persistence/sqlite/control.d.ts +14 -0
- package/dist/infrastructure/persistence/sqlite/control.js +69 -0
- package/dist/infrastructure/persistence/sqlite/jobs.d.ts +12 -0
- package/dist/infrastructure/persistence/sqlite/jobs.js +38 -0
- package/dist/infrastructure/persistence/sqlite/telemetry.d.ts +2 -1
- package/dist/infrastructure/persistence/sqlite/telemetry.js +5 -64
- package/dist/infrastructure/persistence/sqlite/telemetryWrites.d.ts +4 -0
- package/dist/infrastructure/persistence/sqlite/telemetryWrites.js +123 -0
- package/dist/infrastructure/persistence/sqliteSerializer.js +2 -0
- package/dist/infrastructure/scheduler/cron/persisted.d.ts +5 -0
- package/dist/infrastructure/scheduler/cron/persisted.js +27 -0
- package/dist/infrastructure/scheduler/cron/runtime.js +5 -10
- package/dist/infrastructure/scheduler/cron/validation.d.ts +3 -0
- package/dist/infrastructure/scheduler/cron/validation.js +11 -0
- package/dist/infrastructure/scheduler/cronParser.d.ts +1 -1
- package/dist/infrastructure/scheduler/cronParser.js +144 -11
- package/dist/infrastructure/server/bootstrap.d.ts +2 -3
- package/dist/infrastructure/server/bootstrap.js +33 -59
- package/dist/infrastructure/server/errors.d.ts +4 -0
- package/dist/infrastructure/server/errors.js +54 -0
- package/dist/infrastructure/server/handler-routes/control.d.ts +5 -5
- package/dist/infrastructure/server/handler-routes/control.js +16 -1
- package/dist/infrastructure/server/handler-routes/monitoring.d.ts +2 -2
- package/dist/infrastructure/server/handler.js +9 -11
- package/dist/infrastructure/server/handlers/advanced/dependencies.js +7 -6
- package/dist/infrastructure/server/handlers/advanced/jobs.js +19 -6
- package/dist/infrastructure/server/handlers/advanced/queue.d.ts +9 -9
- package/dist/infrastructure/server/handlers/advanced/queue.js +91 -27
- package/dist/infrastructure/server/handlers/core.d.ts +0 -5
- package/dist/infrastructure/server/handlers/core.js +47 -49
- package/dist/infrastructure/server/handlers/cron.d.ts +4 -4
- package/dist/infrastructure/server/handlers/cron.js +60 -59
- package/dist/infrastructure/server/handlers/dashboard.d.ts +1 -1
- package/dist/infrastructure/server/handlers/dashboard.js +21 -8
- package/dist/infrastructure/server/handlers/dlq.d.ts +6 -3
- package/dist/infrastructure/server/handlers/dlq.js +35 -9
- package/dist/infrastructure/server/handlers/flow.js +2 -1
- package/dist/infrastructure/server/handlers/groups.d.ts +11 -0
- package/dist/infrastructure/server/handlers/groups.js +55 -0
- package/dist/infrastructure/server/handlers/management.d.ts +5 -5
- package/dist/infrastructure/server/handlers/management.js +36 -9
- package/dist/infrastructure/server/handlers/monitoring/health.d.ts +5 -5
- package/dist/infrastructure/server/handlers/monitoring/health.js +45 -16
- package/dist/infrastructure/server/handlers/monitoring/operations.d.ts +1 -1
- package/dist/infrastructure/server/handlers/monitoring/operations.js +8 -1
- package/dist/infrastructure/server/handlers/monitoring/workers.d.ts +3 -3
- package/dist/infrastructure/server/handlers/monitoring/workers.js +31 -7
- package/dist/infrastructure/server/handlers/pushBatchValidation.d.ts +3 -1
- package/dist/infrastructure/server/handlers/pushBatchValidation.js +56 -8
- package/dist/infrastructure/server/handlers/query.d.ts +1 -1
- package/dist/infrastructure/server/handlers/query.js +5 -4
- package/dist/infrastructure/server/http.js +2 -2
- package/dist/infrastructure/server/httpDashboardEndpoints.d.ts +1 -1
- package/dist/infrastructure/server/httpDashboardEndpoints.js +19 -6
- package/dist/infrastructure/server/httpEndpoints.js +18 -7
- package/dist/infrastructure/server/httpRouteQueues.js +11 -1
- package/dist/infrastructure/server/protocol/validation.d.ts +1 -0
- package/dist/infrastructure/server/protocol/validation.js +1 -0
- package/dist/infrastructure/server/protocol.d.ts +1 -1
- package/dist/infrastructure/server/protocol.js +1 -1
- package/dist/infrastructure/server/shutdownCoordinator.d.ts +24 -0
- package/dist/infrastructure/server/shutdownCoordinator.js +106 -0
- package/dist/infrastructure/server/sseHandler.js +8 -3
- package/dist/infrastructure/server/storageAdapter.d.ts +29 -0
- package/dist/infrastructure/server/storageAdapter.js +61 -0
- package/dist/infrastructure/server/storageManager.d.ts +12 -0
- package/dist/infrastructure/server/storageManager.js +102 -0
- package/dist/infrastructure/server/tcp.js +2 -3
- package/dist/infrastructure/server/ws/snapshots.d.ts +1 -1
- package/dist/infrastructure/server/ws/snapshots.js +19 -4
- package/dist/infrastructure/server/wsHandler.js +10 -5
- package/dist/main.js +1 -1
- package/dist/mcp/backend/embedded/services.js +2 -1
- package/dist/mcp/index.d.ts +2 -2
- package/dist/mcp/index.js +2 -2
- package/dist/shared/storageHealth.d.ts +9 -0
- package/dist/shared/storageHealth.js +12 -0
- package/package.json +75 -65
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
import { DEFAULT_DLQ_CONFIG } from '../../domain/types/dlq';
|
|
2
|
+
import { DEFAULT_STALL_CONFIG } from '../../domain/types/stall';
|
|
3
|
+
import { normalizePostgresRateLimit } from '../../infrastructure/persistence/postgres/rateLimit';
|
|
4
|
+
import { PostgresQueueManagerMaintenance } from './maintenance';
|
|
5
|
+
const PENDING_STATES = ['waiting', 'prioritized', 'delayed', 'waiting-children'];
|
|
6
|
+
function normalizeDlq(current, patch) {
|
|
7
|
+
const next = { ...current };
|
|
8
|
+
if (typeof patch.autoRetry === 'boolean')
|
|
9
|
+
next.autoRetry = patch.autoRetry;
|
|
10
|
+
if (Number.isFinite(patch.autoRetryInterval)) {
|
|
11
|
+
next.autoRetryInterval = Math.max(0, Math.floor(patch.autoRetryInterval));
|
|
12
|
+
}
|
|
13
|
+
if (Number.isFinite(patch.maxAutoRetries)) {
|
|
14
|
+
next.maxAutoRetries = Math.max(0, Math.floor(patch.maxAutoRetries));
|
|
15
|
+
}
|
|
16
|
+
if (patch.maxAge === null)
|
|
17
|
+
next.maxAge = null;
|
|
18
|
+
else if (Number.isFinite(patch.maxAge)) {
|
|
19
|
+
next.maxAge = Math.max(0, Math.floor(patch.maxAge));
|
|
20
|
+
}
|
|
21
|
+
if (Number.isFinite(patch.maxEntries)) {
|
|
22
|
+
next.maxEntries = Math.max(1, Math.floor(patch.maxEntries));
|
|
23
|
+
}
|
|
24
|
+
return next;
|
|
25
|
+
}
|
|
26
|
+
/** Durable queue controls and policies shared by every PostgreSQL broker. */
|
|
27
|
+
export class PostgresQueueManagerControl extends PostgresQueueManagerMaintenance {
|
|
28
|
+
async setGroupRateLimit(queue, groupId, max, duration) {
|
|
29
|
+
return await this.runPostgresOperation(async () => {
|
|
30
|
+
await this.postgresReady;
|
|
31
|
+
await this.postgresStore.setGroupRateLimit(queue, groupId, max, duration);
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
async removeGroupRateLimit(queue, groupId) {
|
|
35
|
+
return await this.runPostgresOperation(async () => {
|
|
36
|
+
await this.postgresReady;
|
|
37
|
+
return await this.postgresStore.removeGroupRateLimit(queue, groupId);
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
async setGroupConcurrency(queue, groupId, concurrency) {
|
|
41
|
+
return await this.runPostgresOperation(async () => {
|
|
42
|
+
await this.postgresReady;
|
|
43
|
+
await this.postgresStore.setGroupConcurrency(queue, groupId, concurrency);
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
async removeGroupConcurrency(queue, groupId) {
|
|
47
|
+
return await this.runPostgresOperation(async () => {
|
|
48
|
+
await this.postgresReady;
|
|
49
|
+
return await this.postgresStore.removeGroupConcurrency(queue, groupId);
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
pause(queue) {
|
|
53
|
+
this.operations.runSync(() => {
|
|
54
|
+
this.setLocalQueueState(queue, { paused: true });
|
|
55
|
+
this.enqueueWrite(() => this.postgresStore.pause(queue, true));
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
resume(queue) {
|
|
59
|
+
this.operations.runSync(() => {
|
|
60
|
+
this.setLocalQueueState(queue, { paused: false });
|
|
61
|
+
this.enqueueWrite(() => this.postgresStore.pause(queue, false));
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
async pauseDurable(queue, paused) {
|
|
65
|
+
return await this.runPostgresOperation(async () => {
|
|
66
|
+
await this.postgresReady;
|
|
67
|
+
await this.postgresStore.pause(queue, paused);
|
|
68
|
+
await this.refreshQueueAfterCommit(queue);
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
drain(queue) {
|
|
72
|
+
return this.operations.runSync(() => {
|
|
73
|
+
const count = this.postgresSnapshot.removeStates(queue, PENDING_STATES);
|
|
74
|
+
this.enqueueWrite(async () => {
|
|
75
|
+
await this.postgresStore.drain(queue);
|
|
76
|
+
await this.refreshQueueAfterCommit(queue);
|
|
77
|
+
});
|
|
78
|
+
return count;
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
async drainDurable(queue) {
|
|
82
|
+
return await this.runPostgresOperation(async () => {
|
|
83
|
+
await this.postgresReady;
|
|
84
|
+
const count = await this.postgresStore.drain(queue);
|
|
85
|
+
await this.refreshQueueAfterCommit(queue);
|
|
86
|
+
return count;
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
obliterate(queue) {
|
|
90
|
+
this.operations.runSync(() => {
|
|
91
|
+
this.postgresSnapshot.removeStates(queue, [
|
|
92
|
+
...PENDING_STATES,
|
|
93
|
+
'active',
|
|
94
|
+
'completed',
|
|
95
|
+
'failed',
|
|
96
|
+
]);
|
|
97
|
+
this.enqueueWrite(async () => {
|
|
98
|
+
await this.postgresStore.obliterate(queue);
|
|
99
|
+
await this.refreshQueueAfterCommit(queue);
|
|
100
|
+
});
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
async obliterateDurable(queue) {
|
|
104
|
+
return await this.runPostgresOperation(async () => {
|
|
105
|
+
await this.postgresReady;
|
|
106
|
+
await this.postgresStore.obliterate(queue);
|
|
107
|
+
await this.refreshQueueAfterCommit(queue);
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
setRateLimit(queue, limit, durationMs, ttlMs) {
|
|
111
|
+
this.operations.runSync(() => {
|
|
112
|
+
const normalized = normalizePostgresRateLimit(durationMs, ttlMs);
|
|
113
|
+
this.setLocalQueueState(queue, {
|
|
114
|
+
rateLimit: limit,
|
|
115
|
+
rateDurationMs: normalized.durationMs,
|
|
116
|
+
rateExpiresAt: normalized.ttlMs === null ? null : Date.now() + normalized.ttlMs,
|
|
117
|
+
});
|
|
118
|
+
this.enqueueWrite(() => this.postgresStore.setRateLimit(queue, limit, normalized.durationMs, normalized.ttlMs));
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
clearRateLimit(queue) {
|
|
122
|
+
this.operations.runSync(() => {
|
|
123
|
+
this.setLocalQueueState(queue, {
|
|
124
|
+
rateLimit: null,
|
|
125
|
+
rateDurationMs: null,
|
|
126
|
+
rateExpiresAt: null,
|
|
127
|
+
});
|
|
128
|
+
this.enqueueWrite(() => this.postgresStore.setRateLimit(queue, null, null));
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
setConcurrency(queue, limit) {
|
|
132
|
+
this.operations.runSync(() => {
|
|
133
|
+
this.setLocalQueueState(queue, { concurrencyLimit: limit });
|
|
134
|
+
this.enqueueWrite(() => this.postgresStore.setConcurrency(queue, limit));
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
clearConcurrency(queue) {
|
|
138
|
+
this.operations.runSync(() => {
|
|
139
|
+
this.setLocalQueueState(queue, { concurrencyLimit: null });
|
|
140
|
+
this.enqueueWrite(() => this.postgresStore.setConcurrency(queue, null));
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
async setRateLimitDurable(queue, limit, durationMs, ttlMs) {
|
|
144
|
+
return await this.runPostgresOperation(async () => {
|
|
145
|
+
await this.postgresReady;
|
|
146
|
+
const normalized = normalizePostgresRateLimit(durationMs, ttlMs);
|
|
147
|
+
await this.postgresStore.setRateLimit(queue, limit, normalized.durationMs, normalized.ttlMs);
|
|
148
|
+
await this.refreshQueueAfterCommit(queue);
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
async setConcurrencyDurable(queue, limit) {
|
|
152
|
+
return await this.runPostgresOperation(async () => {
|
|
153
|
+
await this.postgresReady;
|
|
154
|
+
await this.postgresStore.setConcurrency(queue, limit);
|
|
155
|
+
await this.refreshQueueAfterCommit(queue);
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
setStallConfig(queue, patch) {
|
|
159
|
+
this.operations.runSync(() => {
|
|
160
|
+
const config = { ...this.getStallConfig(queue), ...patch };
|
|
161
|
+
this.setLocalQueueState(queue, { stallConfig: config });
|
|
162
|
+
this.enqueueWrite(() => this.postgresStore.setStallConfig(queue, config));
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
getStallConfig(queue) {
|
|
166
|
+
return this.postgresSnapshot.queueState(queue)?.stallConfig ?? DEFAULT_STALL_CONFIG;
|
|
167
|
+
}
|
|
168
|
+
async setStallConfigDurable(queue, patch) {
|
|
169
|
+
return await this.runPostgresOperation(async () => {
|
|
170
|
+
await this.postgresReady;
|
|
171
|
+
const state = await this.postgresStore.getQueueState(queue);
|
|
172
|
+
const config = { ...state.stallConfig, ...patch };
|
|
173
|
+
await this.postgresStore.setStallConfig(queue, config);
|
|
174
|
+
await this.refreshQueueAfterCommit(queue);
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
setDlqConfig(queue, patch) {
|
|
178
|
+
this.operations.runSync(() => {
|
|
179
|
+
const config = normalizeDlq(this.getDlqConfig(queue), patch);
|
|
180
|
+
this.setLocalQueueState(queue, { dlqConfig: config });
|
|
181
|
+
this.enqueueWrite(() => this.postgresStore.setDlqConfig(queue, config));
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
getDlqConfig(queue) {
|
|
185
|
+
return this.postgresSnapshot.queueState(queue)?.dlqConfig ?? DEFAULT_DLQ_CONFIG;
|
|
186
|
+
}
|
|
187
|
+
async setDlqConfigDurable(queue, patch) {
|
|
188
|
+
return await this.runPostgresOperation(async () => {
|
|
189
|
+
await this.postgresReady;
|
|
190
|
+
const state = await this.postgresStore.getQueueState(queue);
|
|
191
|
+
await this.postgresStore.setDlqConfig(queue, normalizeDlq(state.dlqConfig, patch));
|
|
192
|
+
await this.refreshQueueAfterCommit(queue);
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
setLocalQueueState(queue, patch) {
|
|
196
|
+
const current = this.postgresSnapshot.queueState(queue) ?? {
|
|
197
|
+
queue,
|
|
198
|
+
paused: false,
|
|
199
|
+
rateLimit: null,
|
|
200
|
+
rateDurationMs: null,
|
|
201
|
+
rateWindowStartedAt: null,
|
|
202
|
+
rateExpiresAt: null,
|
|
203
|
+
rateCount: 0,
|
|
204
|
+
concurrencyLimit: null,
|
|
205
|
+
stallConfig: DEFAULT_STALL_CONFIG,
|
|
206
|
+
dlqConfig: DEFAULT_DLQ_CONFIG,
|
|
207
|
+
};
|
|
208
|
+
this.postgresSnapshot.setQueueState({ ...current, ...patch });
|
|
209
|
+
}
|
|
210
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export declare function throwDeferredWriteErrors(errors: readonly unknown[], message?: string): void;
|
|
2
|
+
/** Serializes fire-and-forget writes while retaining failures until an observed drain. */
|
|
3
|
+
export declare class PostgresDeferredWriteQueue {
|
|
4
|
+
private accepting;
|
|
5
|
+
private tail;
|
|
6
|
+
private nextSequence;
|
|
7
|
+
private readonly failures;
|
|
8
|
+
private checkpoint;
|
|
9
|
+
enqueue(operation: () => Promise<void>): void;
|
|
10
|
+
closeAndDrain(): Promise<unknown[]>;
|
|
11
|
+
drain(): Promise<unknown[]>;
|
|
12
|
+
flush(): Promise<void>;
|
|
13
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
export function throwDeferredWriteErrors(errors, message = 'Multiple deferred PostgreSQL writes failed') {
|
|
2
|
+
if (errors.length === 0)
|
|
3
|
+
return;
|
|
4
|
+
if (errors.length === 1)
|
|
5
|
+
throw errors[0];
|
|
6
|
+
throw new AggregateError(errors, message);
|
|
7
|
+
}
|
|
8
|
+
/** Serializes fire-and-forget writes while retaining failures until an observed drain. */
|
|
9
|
+
export class PostgresDeferredWriteQueue {
|
|
10
|
+
accepting = true;
|
|
11
|
+
tail = Promise.resolve();
|
|
12
|
+
nextSequence = 0;
|
|
13
|
+
failures = [];
|
|
14
|
+
checkpoint = null;
|
|
15
|
+
enqueue(operation) {
|
|
16
|
+
if (!this.accepting)
|
|
17
|
+
throw new Error('PostgreSQL queue manager is shutting down');
|
|
18
|
+
const sequence = ++this.nextSequence;
|
|
19
|
+
this.tail = this.tail.then(async () => {
|
|
20
|
+
try {
|
|
21
|
+
await operation();
|
|
22
|
+
}
|
|
23
|
+
catch (error) {
|
|
24
|
+
this.failures.push({ sequence, error });
|
|
25
|
+
}
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
closeAndDrain() {
|
|
29
|
+
this.accepting = false;
|
|
30
|
+
return this.drain();
|
|
31
|
+
}
|
|
32
|
+
async drain() {
|
|
33
|
+
const throughSequence = this.nextSequence;
|
|
34
|
+
const pending = this.checkpoint;
|
|
35
|
+
if (pending?.sequence === throughSequence)
|
|
36
|
+
return [...(await pending.result)];
|
|
37
|
+
const throughTail = this.tail;
|
|
38
|
+
const checkpoint = {
|
|
39
|
+
sequence: throughSequence,
|
|
40
|
+
result: throughTail.then(() => {
|
|
41
|
+
const firstLaterFailure = this.failures.findIndex(({ sequence }) => sequence > throughSequence);
|
|
42
|
+
const observed = firstLaterFailure === -1
|
|
43
|
+
? this.failures.splice(0)
|
|
44
|
+
: this.failures.splice(0, firstLaterFailure);
|
|
45
|
+
return observed.map(({ error }) => error);
|
|
46
|
+
}),
|
|
47
|
+
};
|
|
48
|
+
this.checkpoint = checkpoint;
|
|
49
|
+
try {
|
|
50
|
+
return [...(await checkpoint.result)];
|
|
51
|
+
}
|
|
52
|
+
finally {
|
|
53
|
+
if (this.checkpoint === checkpoint)
|
|
54
|
+
this.checkpoint = null;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
async flush() {
|
|
58
|
+
throwDeferredWriteErrors(await this.drain());
|
|
59
|
+
}
|
|
60
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { AtomicFlowBatchInput, AtomicFlowBatchResult } from '../../domain/types/flow';
|
|
2
|
+
import { type GroupPullOptions } from '../../domain/types/group';
|
|
3
|
+
import { type Job, type JobId, type JobInput } from '../../domain/types/job';
|
|
4
|
+
import { PostgresQueueManagerTerminalDelivery } from './terminalDelivery';
|
|
5
|
+
export declare class PostgresQueueManagerDelivery extends PostgresQueueManagerTerminalDelivery {
|
|
6
|
+
push(queue: string, input: JobInput): Promise<Job>;
|
|
7
|
+
pushBatch(queue: string, inputs: JobInput[]): Promise<JobId[]>;
|
|
8
|
+
pushFlow(batch: AtomicFlowBatchInput): Promise<AtomicFlowBatchResult>;
|
|
9
|
+
pull(queue: string, timeoutMs?: number, signal?: AbortSignal, groupOptions?: GroupPullOptions): Promise<Job | null>;
|
|
10
|
+
pullWithLock(queue: string, owner: string, timeoutMs?: number, lockTtl?: number, signal?: AbortSignal, groupOptions?: GroupPullOptions): Promise<{
|
|
11
|
+
job: Job | null;
|
|
12
|
+
token: string | null;
|
|
13
|
+
}>;
|
|
14
|
+
pullBatch(queue: string, count: number, timeoutMs?: number, signal?: AbortSignal, groupOptions?: GroupPullOptions): Promise<Job[]>;
|
|
15
|
+
pullBatchWithLock(queue: string, count: number, owner: string, timeoutMs?: number, lockTtl?: number, signal?: AbortSignal, groupOptions?: GroupPullOptions): Promise<{
|
|
16
|
+
jobs: Job[];
|
|
17
|
+
tokens: string[];
|
|
18
|
+
}>;
|
|
19
|
+
private claimUntil;
|
|
20
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { assertGroupPullOptions } from '../../domain/types/group';
|
|
2
|
+
import { createJob, DEFAULT_LOCK_TTL, generateJobId, jobId, } from '../../domain/types/job';
|
|
3
|
+
import { validateRepeatJobInput } from '../repeatJobs';
|
|
4
|
+
import { PostgresQueueManagerTerminalDelivery } from './terminalDelivery';
|
|
5
|
+
export class PostgresQueueManagerDelivery extends PostgresQueueManagerTerminalDelivery {
|
|
6
|
+
async push(queue, input) {
|
|
7
|
+
return await this.runPostgresOperation(async () => {
|
|
8
|
+
await this.postgresReady;
|
|
9
|
+
validateRepeatJobInput(input);
|
|
10
|
+
const id = input.customId ? jobId(input.customId) : generateJobId();
|
|
11
|
+
const admitted = await this.postgresStore.insert(createJob(id, queue, input, await this.postgresStore.now()));
|
|
12
|
+
await this.refreshJob(admitted.job.id, queue);
|
|
13
|
+
return admitted.job;
|
|
14
|
+
});
|
|
15
|
+
}
|
|
16
|
+
async pushBatch(queue, inputs) {
|
|
17
|
+
return await this.runPostgresOperation(async () => {
|
|
18
|
+
await this.postgresReady;
|
|
19
|
+
for (const input of inputs)
|
|
20
|
+
validateRepeatJobInput(input);
|
|
21
|
+
const now = await this.postgresStore.now();
|
|
22
|
+
const jobs = inputs.map((input) => createJob(input.customId ? jobId(input.customId) : generateJobId(), queue, input, now));
|
|
23
|
+
const stored = await this.postgresStore.insertMany(jobs);
|
|
24
|
+
await this.refreshJobs(stored.map((job) => job.id), queue);
|
|
25
|
+
return stored.map((job) => job.id);
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
async pushFlow(batch) {
|
|
29
|
+
return await this.runPostgresOperation(async () => {
|
|
30
|
+
await this.postgresReady;
|
|
31
|
+
const jobs = await this.postgresStore.insertFlow(batch);
|
|
32
|
+
await Promise.all(new Set(jobs.map((job) => job.queue))
|
|
33
|
+
.values()
|
|
34
|
+
.map((queue) => this.refreshQueueAfterCommit(queue)));
|
|
35
|
+
return { jobs };
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
async pull(queue, timeoutMs = 0, signal, groupOptions) {
|
|
39
|
+
const claims = await this.claimUntil(queue, 1, this.postgresStore.config.brokerId, timeoutMs, undefined, signal, groupOptions);
|
|
40
|
+
return claims[0]?.job ?? null;
|
|
41
|
+
}
|
|
42
|
+
// oxlint-disable-next-line max-params -- public API includes cancellation and lease policy
|
|
43
|
+
async pullWithLock(queue, owner, timeoutMs = 0, lockTtl = DEFAULT_LOCK_TTL, signal, groupOptions) {
|
|
44
|
+
const claims = await this.claimUntil(queue, 1, owner, timeoutMs, lockTtl, signal, groupOptions);
|
|
45
|
+
const claim = claims[0];
|
|
46
|
+
return claim ? { job: claim.job, token: claim.token } : { job: null, token: null };
|
|
47
|
+
}
|
|
48
|
+
async pullBatch(queue, count, timeoutMs = 0, signal, groupOptions) {
|
|
49
|
+
const claims = await this.claimUntil(queue, count, this.postgresStore.config.brokerId, timeoutMs, undefined, signal, groupOptions);
|
|
50
|
+
return claims.map((claim) => claim.job);
|
|
51
|
+
}
|
|
52
|
+
// oxlint-disable-next-line max-params -- public API includes cancellation and lease policy
|
|
53
|
+
async pullBatchWithLock(queue, count, owner, timeoutMs = 0, lockTtl = DEFAULT_LOCK_TTL, signal, groupOptions) {
|
|
54
|
+
const claims = await this.claimUntil(queue, count, owner, timeoutMs, lockTtl, signal, groupOptions);
|
|
55
|
+
return {
|
|
56
|
+
jobs: claims.map((claim) => claim.job),
|
|
57
|
+
tokens: claims.map((claim) => claim.token),
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
// oxlint-disable-next-line max-params -- internal bridge mirrors the public pull contract
|
|
61
|
+
async claimUntil(queue, count, owner, timeoutMs, leaseDurationMs, signal, groupOptions) {
|
|
62
|
+
assertGroupPullOptions(groupOptions);
|
|
63
|
+
await this.postgresReady;
|
|
64
|
+
await this.flushPostgresWrites();
|
|
65
|
+
const deadline = Date.now() + Math.max(0, timeoutMs);
|
|
66
|
+
do {
|
|
67
|
+
const claims = await this.runPostgresOperation(async () => {
|
|
68
|
+
const admitted = await this.postgresStore.claim(queue, count, owner, leaseDurationMs, groupOptions);
|
|
69
|
+
for (const claim of admitted) {
|
|
70
|
+
this.applyPostgresClaim(claim);
|
|
71
|
+
}
|
|
72
|
+
return admitted;
|
|
73
|
+
});
|
|
74
|
+
if (claims.length > 0 || timeoutMs <= 0 || signal?.aborted)
|
|
75
|
+
return claims;
|
|
76
|
+
const remaining = deadline - Date.now();
|
|
77
|
+
if (remaining <= 0)
|
|
78
|
+
return [];
|
|
79
|
+
await this.postgresStore.waitForWork(queue, Math.min(remaining, this.postgresStore.config.pollIntervalMs), signal);
|
|
80
|
+
} while (!signal?.aborted);
|
|
81
|
+
return [];
|
|
82
|
+
}
|
|
83
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { PostgresQueueManagerState } from './state';
|
|
2
|
+
/** Database-authoritative group getters shared by every PostgreSQL broker. */
|
|
3
|
+
export declare class PostgresQueueManagerGroupQueries extends PostgresQueueManagerState {
|
|
4
|
+
getGroupJobsCount(queue: string, groupId: string): Promise<number>;
|
|
5
|
+
getGroupsJobsCount(queue: string): Promise<number>;
|
|
6
|
+
getGroupActiveCount(queue: string, groupId: string): Promise<number>;
|
|
7
|
+
getGroupRateLimit(queue: string, groupId: string): Promise<import("../../domain/types/group").GroupRateLimitOverride | null>;
|
|
8
|
+
getGroupRateLimitTtl(queue: string, groupId: string, maxJobs?: number): Promise<number>;
|
|
9
|
+
getGroupConcurrency(queue: string, groupId: string): Promise<number | null>;
|
|
10
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { PostgresQueueManagerState } from './state';
|
|
2
|
+
/** Database-authoritative group getters shared by every PostgreSQL broker. */
|
|
3
|
+
export class PostgresQueueManagerGroupQueries extends PostgresQueueManagerState {
|
|
4
|
+
async getGroupJobsCount(queue, groupId) {
|
|
5
|
+
return await this.runPostgresOperation(async () => {
|
|
6
|
+
await this.postgresReady;
|
|
7
|
+
return await this.postgresStore.getGroupJobsCount(queue, groupId);
|
|
8
|
+
});
|
|
9
|
+
}
|
|
10
|
+
async getGroupsJobsCount(queue) {
|
|
11
|
+
return await this.runPostgresOperation(async () => {
|
|
12
|
+
await this.postgresReady;
|
|
13
|
+
return await this.postgresStore.getGroupJobsCount(queue);
|
|
14
|
+
});
|
|
15
|
+
}
|
|
16
|
+
async getGroupActiveCount(queue, groupId) {
|
|
17
|
+
return await this.runPostgresOperation(async () => {
|
|
18
|
+
await this.postgresReady;
|
|
19
|
+
return await this.postgresStore.getGroupActiveCount(queue, groupId);
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
async getGroupRateLimit(queue, groupId) {
|
|
23
|
+
return await this.runPostgresOperation(async () => {
|
|
24
|
+
await this.postgresReady;
|
|
25
|
+
return await this.postgresStore.getGroupRateLimit(queue, groupId);
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
async getGroupRateLimitTtl(queue, groupId, maxJobs) {
|
|
29
|
+
return await this.runPostgresOperation(async () => {
|
|
30
|
+
await this.postgresReady;
|
|
31
|
+
return await this.postgresStore.getGroupRateLimitTtl(queue, groupId, maxJobs);
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
async getGroupConcurrency(queue, groupId) {
|
|
35
|
+
return await this.runPostgresOperation(async () => {
|
|
36
|
+
await this.postgresReady;
|
|
37
|
+
return await this.postgresStore.getGroupConcurrency(queue, groupId);
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { type JobId, type JobLock, type LockToken } from '../../domain/types/job';
|
|
2
|
+
import { PostgresQueueManagerDelivery } from './delivery';
|
|
3
|
+
export declare class PostgresQueueManagerLease extends PostgresQueueManagerDelivery {
|
|
4
|
+
extendLock(id: JobId | string, token: string | null, duration: number): Promise<boolean>;
|
|
5
|
+
heartbeatDurable(id: JobId, token?: string, duration?: number): Promise<boolean>;
|
|
6
|
+
heartbeatBatchDurable(ids: JobId[], tokens?: string[]): Promise<number>;
|
|
7
|
+
jobHeartbeat(id: JobId, token?: string): boolean;
|
|
8
|
+
jobHeartbeatBatch(ids: JobId[], tokens?: string[]): number;
|
|
9
|
+
verifyLock(id: JobId, token: string): boolean;
|
|
10
|
+
renewJobLock(id: JobId, token: string, ttl?: number): boolean;
|
|
11
|
+
getLockInfo(id: JobId): JobLock | null;
|
|
12
|
+
createLock(id: JobId, _owner: string, _ttl?: number): LockToken | null;
|
|
13
|
+
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { DEFAULT_LOCK_TTL, jobId, lockToken, } from '../../domain/types/job';
|
|
2
|
+
import { PostgresQueueManagerDelivery } from './delivery';
|
|
3
|
+
export class PostgresQueueManagerLease extends PostgresQueueManagerDelivery {
|
|
4
|
+
async extendLock(id, token, duration) {
|
|
5
|
+
return await this.runPostgresOperation(async () => {
|
|
6
|
+
await this.postgresReady;
|
|
7
|
+
const target = jobId(String(id));
|
|
8
|
+
const leaseToken = token ?? this.tokenFor(target);
|
|
9
|
+
if (!leaseToken)
|
|
10
|
+
return false;
|
|
11
|
+
const expiresAt = await this.postgresStore.renew(target, leaseToken, duration);
|
|
12
|
+
if (expiresAt !== null) {
|
|
13
|
+
await this.refreshJob(target);
|
|
14
|
+
return true;
|
|
15
|
+
}
|
|
16
|
+
this.forgetToken(target);
|
|
17
|
+
await this.refreshJob(target);
|
|
18
|
+
return false;
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
async heartbeatDurable(id, token, duration) {
|
|
22
|
+
return await this.runPostgresOperation(async () => {
|
|
23
|
+
const leaseToken = this.tokenFor(id, token);
|
|
24
|
+
if (!leaseToken)
|
|
25
|
+
return false;
|
|
26
|
+
return await this.extendLock(id, leaseToken, duration ?? this.postgresStore.config.leaseDurationMs);
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
async heartbeatBatchDurable(ids, tokens) {
|
|
30
|
+
return await this.runPostgresOperation(async () => {
|
|
31
|
+
const renewed = await Promise.all(ids.map((id, index) => this.heartbeatDurable(id, tokens?.[index])));
|
|
32
|
+
return renewed.filter(Boolean).length;
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
jobHeartbeat(id, token) {
|
|
36
|
+
return this.operations.runSync(() => {
|
|
37
|
+
const leaseToken = this.tokenFor(id, token);
|
|
38
|
+
const row = this.postgresSnapshot.get(id);
|
|
39
|
+
if (!leaseToken || row?.state !== 'active')
|
|
40
|
+
return false;
|
|
41
|
+
this.enqueueWrite(async () => {
|
|
42
|
+
const renewed = await this.postgresStore.renew(id, leaseToken, this.postgresStore.config.leaseDurationMs);
|
|
43
|
+
if (renewed !== null)
|
|
44
|
+
await this.refreshJob(id);
|
|
45
|
+
});
|
|
46
|
+
return true;
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
jobHeartbeatBatch(ids, tokens) {
|
|
50
|
+
return this.operations.runSync(() => ids.reduce((count, id, index) => count + (this.jobHeartbeat(id, tokens?.[index]) ? 1 : 0), 0));
|
|
51
|
+
}
|
|
52
|
+
verifyLock(id, token) {
|
|
53
|
+
const row = this.postgresSnapshot.get(id);
|
|
54
|
+
return row?.state === 'active' && this.tokenFor(id) === token;
|
|
55
|
+
}
|
|
56
|
+
renewJobLock(id, token, ttl = DEFAULT_LOCK_TTL) {
|
|
57
|
+
return this.operations.runSync(() => {
|
|
58
|
+
if (!this.verifyLock(id, token))
|
|
59
|
+
return false;
|
|
60
|
+
this.enqueueWrite(async () => {
|
|
61
|
+
const renewed = await this.postgresStore.renew(id, token, ttl);
|
|
62
|
+
if (renewed !== null)
|
|
63
|
+
await this.refreshJob(id);
|
|
64
|
+
});
|
|
65
|
+
return true;
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
getLockInfo(id) {
|
|
69
|
+
const row = this.postgresSnapshot.get(id);
|
|
70
|
+
const token = this.tokenFor(id);
|
|
71
|
+
if (row?.state !== 'active' || !token || row.leaseUntil === null)
|
|
72
|
+
return null;
|
|
73
|
+
return {
|
|
74
|
+
jobId: id,
|
|
75
|
+
token: lockToken(token),
|
|
76
|
+
owner: row.leaseOwner ?? this.postgresStore.config.brokerId,
|
|
77
|
+
createdAt: row.job.startedAt ?? Date.now(),
|
|
78
|
+
expiresAt: row.leaseUntil,
|
|
79
|
+
lastRenewalAt: row.job.lastHeartbeat,
|
|
80
|
+
renewalCount: row.leaseRenewals,
|
|
81
|
+
ttl: Math.max(1, row.leaseUntil - row.job.lastHeartbeat),
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
createLock(id, _owner, _ttl = DEFAULT_LOCK_TTL) {
|
|
85
|
+
const token = this.tokenFor(id);
|
|
86
|
+
return token ? lockToken(token) : null;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { EventType } from '../../domain/types/queue';
|
|
2
|
+
import type { PostgresStoreEvent } from '../../infrastructure/persistence/postgres';
|
|
3
|
+
import type { PostgresLifetimeMetricsSnapshot } from '../../infrastructure/persistence/postgres/lifetimeMetrics';
|
|
4
|
+
interface PostgresMetricCounters {
|
|
5
|
+
readonly totalPushed: {
|
|
6
|
+
value: bigint;
|
|
7
|
+
};
|
|
8
|
+
readonly totalPulled: {
|
|
9
|
+
value: bigint;
|
|
10
|
+
};
|
|
11
|
+
readonly totalCompleted: {
|
|
12
|
+
value: bigint;
|
|
13
|
+
};
|
|
14
|
+
readonly totalFailed: {
|
|
15
|
+
value: bigint;
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
interface PostgresPerQueueMetrics {
|
|
19
|
+
clear(): void;
|
|
20
|
+
get(queue: string): {
|
|
21
|
+
totalCompleted: bigint;
|
|
22
|
+
totalFailed: bigint;
|
|
23
|
+
} | undefined;
|
|
24
|
+
set(queue: string, totals: {
|
|
25
|
+
totalCompleted: bigint;
|
|
26
|
+
totalFailed: bigint;
|
|
27
|
+
}): unknown;
|
|
28
|
+
}
|
|
29
|
+
export declare function postgresEventType(value: string): EventType | null;
|
|
30
|
+
export declare function hydratePostgresLifetimeMetrics(metrics: PostgresMetricCounters, perQueue: PostgresPerQueueMetrics, snapshot: PostgresLifetimeMetricsSnapshot): void;
|
|
31
|
+
export declare function applyPostgresEventMetrics(metrics: PostgresMetricCounters, perQueue: PostgresPerQueueMetrics, event: PostgresStoreEvent, includeTerminal: boolean): void;
|
|
32
|
+
export {};
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
export function postgresEventType(value) {
|
|
2
|
+
switch (value) {
|
|
3
|
+
case "pushed" /* EventType.Pushed */:
|
|
4
|
+
case "pulled" /* EventType.Pulled */:
|
|
5
|
+
case "completed" /* EventType.Completed */:
|
|
6
|
+
case "failed" /* EventType.Failed */:
|
|
7
|
+
case "progress" /* EventType.Progress */:
|
|
8
|
+
case "stalled" /* EventType.Stalled */:
|
|
9
|
+
case "removed" /* EventType.Removed */:
|
|
10
|
+
case "delayed" /* EventType.Delayed */:
|
|
11
|
+
case "duplicated" /* EventType.Duplicated */:
|
|
12
|
+
case "retried" /* EventType.Retried */:
|
|
13
|
+
case "waiting-children" /* EventType.WaitingChildren */:
|
|
14
|
+
case "drained" /* EventType.Drained */:
|
|
15
|
+
case "paused" /* EventType.Paused */:
|
|
16
|
+
case "resumed" /* EventType.Resumed */:
|
|
17
|
+
return value;
|
|
18
|
+
default:
|
|
19
|
+
return null;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
export function hydratePostgresLifetimeMetrics(metrics, perQueue, snapshot) {
|
|
23
|
+
metrics.totalCompleted.value = snapshot.totalCompleted;
|
|
24
|
+
metrics.totalFailed.value = snapshot.totalFailed;
|
|
25
|
+
perQueue.clear();
|
|
26
|
+
for (const totals of snapshot.queues) {
|
|
27
|
+
perQueue.set(totals.queue, {
|
|
28
|
+
totalCompleted: totals.totalCompleted,
|
|
29
|
+
totalFailed: totals.totalFailed,
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
export function applyPostgresEventMetrics(metrics, perQueue, event, includeTerminal) {
|
|
34
|
+
const type = postgresEventType(event.type);
|
|
35
|
+
if (type === "pushed" /* EventType.Pushed */)
|
|
36
|
+
metrics.totalPushed.value++;
|
|
37
|
+
if (type === "pulled" /* EventType.Pulled */)
|
|
38
|
+
metrics.totalPulled.value++;
|
|
39
|
+
const completed = includeTerminal && type === "completed" /* EventType.Completed */;
|
|
40
|
+
const failed = includeTerminal && type === "failed" /* EventType.Failed */ && event.state === 'failed';
|
|
41
|
+
if (!completed && !failed)
|
|
42
|
+
return;
|
|
43
|
+
const totals = perQueue.get(event.queue) ?? { totalCompleted: 0n, totalFailed: 0n };
|
|
44
|
+
if (completed) {
|
|
45
|
+
metrics.totalCompleted.value++;
|
|
46
|
+
totals.totalCompleted++;
|
|
47
|
+
}
|
|
48
|
+
if (failed) {
|
|
49
|
+
metrics.totalFailed.value++;
|
|
50
|
+
totals.totalFailed++;
|
|
51
|
+
}
|
|
52
|
+
perQueue.set(event.queue, totals);
|
|
53
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { DlqFilter } from '../../domain/types/dlq';
|
|
2
|
+
import type { JobId } from '../../domain/types/job';
|
|
3
|
+
import { PostgresQueueManagerRelationships } from './relationships';
|
|
4
|
+
/** Durable maintenance commands that cannot use the synchronous SQLite façade. */
|
|
5
|
+
export declare class PostgresQueueManagerMaintenance extends PostgresQueueManagerRelationships {
|
|
6
|
+
retryDlqDurable(queue: string, id?: JobId, count?: number, filter?: DlqFilter): Promise<number>;
|
|
7
|
+
purgeDlqDurable(queue: string): Promise<number>;
|
|
8
|
+
removeDlqJobDurable(queue: string, id: JobId): Promise<boolean>;
|
|
9
|
+
retryCompletedDurable(queue: string, id?: JobId, options?: {
|
|
10
|
+
limit?: number;
|
|
11
|
+
timestamp?: number;
|
|
12
|
+
}): Promise<number>;
|
|
13
|
+
cleanDurable(queue: string, graceMs: number, state?: string, limit?: number): Promise<JobId[]>;
|
|
14
|
+
}
|