bunqueue 2.8.59 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +109 -77
- package/dist/application/background/dlq.js +2 -0
- package/dist/application/contextFactory.js +5 -0
- package/dist/application/dlqManager.d.ts +11 -1
- package/dist/application/dlqManager.js +48 -10
- package/dist/application/dlqRetry.js +59 -38
- package/dist/application/eventsManager.d.ts +1 -1
- package/dist/application/eventsManager.js +30 -26
- package/dist/application/lockManager.js +1 -1
- package/dist/application/operations/jobManagement.d.ts +4 -0
- package/dist/application/operations/jobMoveOperations.js +9 -4
- package/dist/application/postgres-queue-manager/batchSnapshot.d.ts +16 -0
- package/dist/application/postgres-queue-manager/batchSnapshot.js +17 -0
- package/dist/application/postgres-queue-manager/cloud.d.ts +30 -0
- package/dist/application/postgres-queue-manager/cloud.js +55 -0
- package/dist/application/postgres-queue-manager/config.d.ts +5 -0
- package/dist/application/postgres-queue-manager/config.js +1 -0
- package/dist/application/postgres-queue-manager/control.d.ts +26 -0
- package/dist/application/postgres-queue-manager/control.js +186 -0
- package/dist/application/postgres-queue-manager/deferredWrites.d.ts +13 -0
- package/dist/application/postgres-queue-manager/deferredWrites.js +60 -0
- package/dist/application/postgres-queue-manager/delivery.d.ts +19 -0
- package/dist/application/postgres-queue-manager/delivery.js +80 -0
- package/dist/application/postgres-queue-manager/lease.d.ts +13 -0
- package/dist/application/postgres-queue-manager/lease.js +88 -0
- package/dist/application/postgres-queue-manager/lifetimeMetrics.d.ts +32 -0
- package/dist/application/postgres-queue-manager/lifetimeMetrics.js +53 -0
- package/dist/application/postgres-queue-manager/maintenance.d.ts +14 -0
- package/dist/application/postgres-queue-manager/maintenance.js +100 -0
- package/dist/application/postgres-queue-manager/operationGate.d.ts +26 -0
- package/dist/application/postgres-queue-manager/operationGate.js +96 -0
- package/dist/application/postgres-queue-manager/operations.d.ts +25 -0
- package/dist/application/postgres-queue-manager/operations.js +194 -0
- package/dist/application/postgres-queue-manager/projectionRefreshes.d.ts +43 -0
- package/dist/application/postgres-queue-manager/projectionRefreshes.js +170 -0
- package/dist/application/postgres-queue-manager/queries.d.ts +142 -0
- package/dist/application/postgres-queue-manager/queries.js +231 -0
- package/dist/application/postgres-queue-manager/relationships.d.ts +10 -0
- package/dist/application/postgres-queue-manager/relationships.js +45 -0
- package/dist/application/postgres-queue-manager/services.d.ts +31 -0
- package/dist/application/postgres-queue-manager/services.js +149 -0
- package/dist/application/postgres-queue-manager/snapshot.d.ts +51 -0
- package/dist/application/postgres-queue-manager/snapshot.js +236 -0
- package/dist/application/postgres-queue-manager/snapshotViews.d.ts +15 -0
- package/dist/application/postgres-queue-manager/snapshotViews.js +100 -0
- package/dist/application/postgres-queue-manager/startupEventBuffer.d.ts +12 -0
- package/dist/application/postgres-queue-manager/startupEventBuffer.js +36 -0
- package/dist/application/postgres-queue-manager/state.d.ts +47 -0
- package/dist/application/postgres-queue-manager/state.js +273 -0
- package/dist/application/postgres-queue-manager/terminalDelivery.d.ts +20 -0
- package/dist/application/postgres-queue-manager/terminalDelivery.js +100 -0
- package/dist/application/postgresQueueManager.d.ts +2 -0
- package/dist/application/postgresQueueManager.js +1 -0
- package/dist/application/queue-manager/ack.js +1 -1
- package/dist/application/queue-manager/delivery.js +1 -1
- package/dist/application/queue-manager/flow-failures.js +4 -0
- package/dist/application/queue-manager/limits.d.ts +2 -0
- package/dist/application/queue-manager/limits.js +4 -0
- package/dist/application/queue-manager/observability.js +15 -3
- package/dist/application/queue-manager/queries.d.ts +7 -0
- package/dist/application/queue-manager/queries.js +9 -0
- package/dist/application/queue-manager/services.d.ts +1 -1
- package/dist/application/queue-manager/services.js +4 -2
- package/dist/application/queue-manager/state.js +16 -1
- package/dist/application/statsManager.d.ts +1 -1
- package/dist/application/statsManager.js +1 -1
- package/dist/cli/commands/server.js +1 -1
- package/dist/client/events.js +2 -2
- package/dist/client/flow.js +0 -1
- package/dist/client/forwarder.d.ts +6 -4
- package/dist/client/forwarder.js +5 -3
- package/dist/client/queue/dlq.d.ts +2 -0
- package/dist/client/queue/dlq.js +14 -0
- package/dist/client/queue/dlqOps.d.ts +2 -0
- package/dist/client/queue/dlqOps.js +4 -0
- package/dist/client/queue/helpers.js +2 -0
- package/dist/client/queue/operations/add/single.js +1 -1
- package/dist/client/queue/runtime/configuration.d.ts +4 -0
- package/dist/client/queue/runtime/configuration.js +8 -0
- package/dist/client/sandboxed/runtime/state.js +2 -2
- package/dist/client/tcp/connection.d.ts +1 -1
- package/dist/client/tcp/connection.js +1 -1
- package/dist/client/tcp/runtime/state.js +2 -2
- package/dist/client/types/metrics.d.ts +1 -1
- package/dist/client/worker/runtime/control.js +0 -1
- package/dist/client/worker/runtime/execution.js +3 -8
- package/dist/client/worker/runtime/lifecycle.js +0 -1
- package/dist/client/worker/runtime/polling.d.ts +1 -0
- package/dist/client/worker/runtime/polling.js +9 -0
- package/dist/client/worker/runtime/state.js +3 -3
- package/dist/client/workflow/compensationChild.d.ts +2 -0
- package/dist/client/workflow/compensationChild.js +3 -1
- package/dist/client/workflow/compensationPass.d.ts +1 -0
- package/dist/client/workflow/compensationPass.js +18 -2
- package/dist/client/workflow/compensator.d.ts +2 -1
- package/dist/client/workflow/compensator.js +11 -3
- package/dist/client/workflow/engine.js +1 -1
- package/dist/client/workflow/executionFence.d.ts +15 -0
- package/dist/client/workflow/executionFence.js +29 -0
- package/dist/client/workflow/executor.d.ts +5 -11
- package/dist/client/workflow/executor.js +48 -47
- package/dist/client/workflow/executorLifecycle.d.ts +1 -0
- package/dist/client/workflow/executorLifecycle.js +41 -10
- package/dist/client/workflow/executorNodes.d.ts +1 -0
- package/dist/client/workflow/executorNodes.js +45 -16
- package/dist/client/workflow/executorQueue.d.ts +9 -0
- package/dist/client/workflow/executorQueue.js +15 -0
- package/dist/client/workflow/forEachRunner.d.ts +4 -0
- package/dist/client/workflow/forEachRunner.js +70 -0
- package/dist/client/workflow/loops.d.ts +4 -5
- package/dist/client/workflow/loops.js +27 -93
- package/dist/client/workflow/mapRunner.d.ts +1 -1
- package/dist/client/workflow/mapRunner.js +11 -2
- package/dist/client/workflow/recovery.d.ts +1 -0
- package/dist/client/workflow/recovery.js +24 -18
- package/dist/client/workflow/rollbackControl.d.ts +1 -0
- package/dist/client/workflow/rollbackControl.js +5 -2
- package/dist/client/workflow/runner.d.ts +3 -1
- package/dist/client/workflow/runner.js +26 -3
- package/dist/client/workflow/stepTypes.js +1 -1
- package/dist/client/workflow/subWorkflowRunner.d.ts +1 -0
- package/dist/client/workflow/subWorkflowRunner.js +7 -0
- package/dist/client/workflow/waitFor.d.ts +4 -0
- package/dist/client/workflow/waitFor.js +23 -11
- package/dist/client/workflow/workflowDecisions.d.ts +1 -1
- package/dist/client/workflow/workflowDecisions.js +3 -1
- package/dist/config/resolve.d.ts +14 -0
- package/dist/config/resolve.js +50 -5
- package/dist/config/types.d.ts +18 -0
- package/dist/domain/queue/dlqShard.d.ts +2 -0
- package/dist/domain/queue/dlqShard.js +16 -0
- package/dist/domain/queue/shard/dlq.d.ts +1 -0
- package/dist/domain/queue/shard/dlq.js +3 -0
- package/dist/domain/types/commands/dlq.d.ts +5 -0
- package/dist/domain/types/commands/union.d.ts +1 -1
- package/dist/domain/types/cron.d.ts +5 -0
- package/dist/domain/types/cron.js +14 -3
- package/dist/domain/types/dlq.d.ts +1 -1
- package/dist/domain/types/dlq.js +1 -2
- package/dist/domain/types/jobs/model.d.ts +1 -1
- package/dist/infrastructure/cloud/commandHandler.js +5 -3
- package/dist/infrastructure/cloud/commands/integrations.js +11 -11
- package/dist/infrastructure/cloud/commands/jobs.js +48 -40
- package/dist/infrastructure/cloud/commands/queues.js +61 -75
- package/dist/infrastructure/cloud/queueAdapter/local.d.ts +41 -0
- package/dist/infrastructure/cloud/queueAdapter/local.js +187 -0
- package/dist/infrastructure/cloud/queueAdapter/postgres.d.ts +42 -0
- package/dist/infrastructure/cloud/queueAdapter/postgres.js +188 -0
- package/dist/infrastructure/cloud/queueAdapter/registry.d.ts +4 -0
- package/dist/infrastructure/cloud/queueAdapter/registry.js +15 -0
- package/dist/infrastructure/cloud/queueAdapter/types.d.ts +87 -0
- package/dist/infrastructure/cloud/queueAdapter/types.js +1 -0
- package/dist/infrastructure/cloud/snapshot/analytics.d.ts +2 -0
- package/dist/infrastructure/cloud/snapshot/analytics.js +7 -0
- package/dist/infrastructure/cloud/snapshot/collector.d.ts +4 -0
- package/dist/infrastructure/cloud/snapshot/collector.js +39 -1
- package/dist/infrastructure/cloud/snapshot/config.d.ts +3 -0
- package/dist/infrastructure/cloud/snapshot/config.js +43 -0
- package/dist/infrastructure/cloud/snapshot/jobs.d.ts +3 -0
- package/dist/infrastructure/cloud/snapshot/jobs.js +32 -1
- package/dist/infrastructure/cloud/snapshotCollector.js +20 -35
- package/dist/infrastructure/cloud/snapshotHelpers.d.ts +3 -3
- package/dist/infrastructure/cloud/snapshotHelpers.js +3 -3
- package/dist/infrastructure/cloud/statsRefresh.js +2 -1
- package/dist/infrastructure/cloud/types/command.d.ts +2 -2
- package/dist/infrastructure/persistence/postgres/admission.d.ts +18 -0
- package/dist/infrastructure/persistence/postgres/admission.js +193 -0
- package/dist/infrastructure/persistence/postgres/admissionParent.d.ts +5 -0
- package/dist/infrastructure/persistence/postgres/admissionParent.js +35 -0
- package/dist/infrastructure/persistence/postgres/admissionResult.d.ts +9 -0
- package/dist/infrastructure/persistence/postgres/admissionResult.js +1 -0
- package/dist/infrastructure/persistence/postgres/admissionStore.d.ts +12 -0
- package/dist/infrastructure/persistence/postgres/admissionStore.js +101 -0
- package/dist/infrastructure/persistence/postgres/advisoryLocks.d.ts +2 -0
- package/dist/infrastructure/persistence/postgres/advisoryLocks.js +13 -0
- package/dist/infrastructure/persistence/postgres/batchAdmission.d.ts +11 -0
- package/dist/infrastructure/persistence/postgres/batchAdmission.js +110 -0
- package/dist/infrastructure/persistence/postgres/batchCompletion.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/batchCompletion.js +174 -0
- package/dist/infrastructure/persistence/postgres/batchEvents.d.ts +14 -0
- package/dist/infrastructure/persistence/postgres/batchEvents.js +72 -0
- package/dist/infrastructure/persistence/postgres/brokerSessions.d.ts +14 -0
- package/dist/infrastructure/persistence/postgres/brokerSessions.js +94 -0
- package/dist/infrastructure/persistence/postgres/claimBatch.d.ts +11 -0
- package/dist/infrastructure/persistence/postgres/claimBatch.js +56 -0
- package/dist/infrastructure/persistence/postgres/claimSelection.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/claimSelection.js +198 -0
- package/dist/infrastructure/persistence/postgres/claims.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/claims.js +137 -0
- package/dist/infrastructure/persistence/postgres/cloudReadModel.d.ts +26 -0
- package/dist/infrastructure/persistence/postgres/cloudReadModel.js +123 -0
- package/dist/infrastructure/persistence/postgres/codec.d.ts +30 -0
- package/dist/infrastructure/persistence/postgres/codec.js +82 -0
- package/dist/infrastructure/persistence/postgres/completionEvents.d.ts +11 -0
- package/dist/infrastructure/persistence/postgres/completionEvents.js +21 -0
- package/dist/infrastructure/persistence/postgres/completionLifecycle.d.ts +11 -0
- package/dist/infrastructure/persistence/postgres/completionLifecycle.js +145 -0
- package/dist/infrastructure/persistence/postgres/completionOutcome.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/completionOutcome.js +112 -0
- package/dist/infrastructure/persistence/postgres/completionQueries.d.ts +10 -0
- package/dist/infrastructure/persistence/postgres/completionQueries.js +87 -0
- package/dist/infrastructure/persistence/postgres/connection.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/connection.js +15 -0
- package/dist/infrastructure/persistence/postgres/context.d.ts +28 -0
- package/dist/infrastructure/persistence/postgres/context.js +74 -0
- package/dist/infrastructure/persistence/postgres/control.d.ts +23 -0
- package/dist/infrastructure/persistence/postgres/control.js +105 -0
- package/dist/infrastructure/persistence/postgres/crons.d.ts +18 -0
- package/dist/infrastructure/persistence/postgres/crons.js +205 -0
- package/dist/infrastructure/persistence/postgres/deduplication.d.ts +9 -0
- package/dist/infrastructure/persistence/postgres/deduplication.js +47 -0
- package/dist/infrastructure/persistence/postgres/dependencyDestruction.d.ts +14 -0
- package/dist/infrastructure/persistence/postgres/dependencyDestruction.js +74 -0
- package/dist/infrastructure/persistence/postgres/dependencyPromotion.d.ts +29 -0
- package/dist/infrastructure/persistence/postgres/dependencyPromotion.js +212 -0
- package/dist/infrastructure/persistence/postgres/destructiveMutations.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/destructiveMutations.js +111 -0
- package/dist/infrastructure/persistence/postgres/dlqLifecycle.d.ts +13 -0
- package/dist/infrastructure/persistence/postgres/dlqLifecycle.js +184 -0
- package/dist/infrastructure/persistence/postgres/dlqMaintenance.d.ts +6 -0
- package/dist/infrastructure/persistence/postgres/dlqMaintenance.js +36 -0
- package/dist/infrastructure/persistence/postgres/dlqRetryPlan.d.ts +21 -0
- package/dist/infrastructure/persistence/postgres/dlqRetryPlan.js +92 -0
- package/dist/infrastructure/persistence/postgres/eventCommitGc.d.ts +27 -0
- package/dist/infrastructure/persistence/postgres/eventCommitGc.js +62 -0
- package/dist/infrastructure/persistence/postgres/eventJournal.d.ts +18 -0
- package/dist/infrastructure/persistence/postgres/eventJournal.js +76 -0
- package/dist/infrastructure/persistence/postgres/eventJournalSchema.d.ts +5 -0
- package/dist/infrastructure/persistence/postgres/eventJournalSchema.js +253 -0
- package/dist/infrastructure/persistence/postgres/eventPruneWatermarks.d.ts +24 -0
- package/dist/infrastructure/persistence/postgres/eventPruneWatermarks.js +111 -0
- package/dist/infrastructure/persistence/postgres/eventRetention.d.ts +18 -0
- package/dist/infrastructure/persistence/postgres/eventRetention.js +76 -0
- package/dist/infrastructure/persistence/postgres/events.d.ts +45 -0
- package/dist/infrastructure/persistence/postgres/events.js +249 -0
- package/dist/infrastructure/persistence/postgres/expiry.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/expiry.js +74 -0
- package/dist/infrastructure/persistence/postgres/flowFailures.d.ts +17 -0
- package/dist/infrastructure/persistence/postgres/flowFailures.js +199 -0
- package/dist/infrastructure/persistence/postgres/jobData.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/jobData.js +32 -0
- package/dist/infrastructure/persistence/postgres/leaseRelease.d.ts +9 -0
- package/dist/infrastructure/persistence/postgres/leaseRelease.js +120 -0
- package/dist/infrastructure/persistence/postgres/leaseRenewal.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/leaseRenewal.js +35 -0
- package/dist/infrastructure/persistence/postgres/lifetimeMetrics.d.ts +15 -0
- package/dist/infrastructure/persistence/postgres/lifetimeMetrics.js +46 -0
- package/dist/infrastructure/persistence/postgres/lifetimeMetricsFinalizer.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/lifetimeMetricsFinalizer.js +13 -0
- package/dist/infrastructure/persistence/postgres/logs.d.ts +6 -0
- package/dist/infrastructure/persistence/postgres/logs.js +70 -0
- package/dist/infrastructure/persistence/postgres/maintenance.d.ts +6 -0
- package/dist/infrastructure/persistence/postgres/maintenance.js +138 -0
- package/dist/infrastructure/persistence/postgres/maintenanceFlights.d.ts +14 -0
- package/dist/infrastructure/persistence/postgres/maintenanceFlights.js +66 -0
- package/dist/infrastructure/persistence/postgres/metricWrites.d.ts +9 -0
- package/dist/infrastructure/persistence/postgres/metricWrites.js +94 -0
- package/dist/infrastructure/persistence/postgres/mutations.d.ts +9 -0
- package/dist/infrastructure/persistence/postgres/mutations.js +171 -0
- package/dist/infrastructure/persistence/postgres/outcomes.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/outcomes.js +194 -0
- package/dist/infrastructure/persistence/postgres/policies.d.ts +11 -0
- package/dist/infrastructure/persistence/postgres/policies.js +18 -0
- package/dist/infrastructure/persistence/postgres/postCommitMaintenance.d.ts +17 -0
- package/dist/infrastructure/persistence/postgres/postCommitMaintenance.js +117 -0
- package/dist/infrastructure/persistence/postgres/promotion.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/promotion.js +43 -0
- package/dist/infrastructure/persistence/postgres/queries.d.ts +22 -0
- package/dist/infrastructure/persistence/postgres/queries.js +152 -0
- package/dist/infrastructure/persistence/postgres/queueDestruction.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/queueDestruction.js +154 -0
- package/dist/infrastructure/persistence/postgres/queueLifecycle.d.ts +10 -0
- package/dist/infrastructure/persistence/postgres/queueLifecycle.js +60 -0
- package/dist/infrastructure/persistence/postgres/rateLimit.d.ts +6 -0
- package/dist/infrastructure/persistence/postgres/rateLimit.js +10 -0
- package/dist/infrastructure/persistence/postgres/readModels.d.ts +32 -0
- package/dist/infrastructure/persistence/postgres/readModels.js +72 -0
- package/dist/infrastructure/persistence/postgres/recovery.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/recovery.js +178 -0
- package/dist/infrastructure/persistence/postgres/relationships.d.ts +9 -0
- package/dist/infrastructure/persistence/postgres/relationships.js +201 -0
- package/dist/infrastructure/persistence/postgres/removeUnprocessedChildren.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/removeUnprocessedChildren.js +241 -0
- package/dist/infrastructure/persistence/postgres/repeatMutations.d.ts +5 -0
- package/dist/infrastructure/persistence/postgres/repeatMutations.js +37 -0
- package/dist/infrastructure/persistence/postgres/runtime.d.ts +46 -0
- package/dist/infrastructure/persistence/postgres/runtime.js +236 -0
- package/dist/infrastructure/persistence/postgres/runtimeConfig.d.ts +21 -0
- package/dist/infrastructure/persistence/postgres/runtimeConfig.js +39 -0
- package/dist/infrastructure/persistence/postgres/schema.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/schema.js +257 -0
- package/dist/infrastructure/persistence/postgres/schemaInitialization.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/schemaInitialization.js +221 -0
- package/dist/infrastructure/persistence/postgres/serialAdmission.d.ts +6 -0
- package/dist/infrastructure/persistence/postgres/serialAdmission.js +96 -0
- package/dist/infrastructure/persistence/postgres/snapshotBudget.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/snapshotBudget.js +83 -0
- package/dist/infrastructure/persistence/postgres/storageHealth.d.ts +10 -0
- package/dist/infrastructure/persistence/postgres/storageHealth.js +29 -0
- package/dist/infrastructure/persistence/postgres/store.d.ts +90 -0
- package/dist/infrastructure/persistence/postgres/store.js +136 -0
- package/dist/infrastructure/persistence/postgres/telemetry.d.ts +6 -0
- package/dist/infrastructure/persistence/postgres/telemetry.js +88 -0
- package/dist/infrastructure/persistence/postgres/terminal.d.ts +8 -0
- package/dist/infrastructure/persistence/postgres/terminal.js +137 -0
- package/dist/infrastructure/persistence/postgres/transactionRetry.d.ts +6 -0
- package/dist/infrastructure/persistence/postgres/transactionRetry.js +42 -0
- package/dist/infrastructure/persistence/postgres/types.d.ts +175 -0
- package/dist/infrastructure/persistence/postgres/types.js +1 -0
- package/dist/infrastructure/persistence/postgres/workers.d.ts +15 -0
- package/dist/infrastructure/persistence/postgres/workers.js +126 -0
- package/dist/infrastructure/persistence/postgres.d.ts +2 -0
- package/dist/infrastructure/persistence/postgres.js +1 -0
- package/dist/infrastructure/persistence/sqlite/jobs.js +2 -0
- package/dist/infrastructure/persistence/sqlite.d.ts +1 -1
- package/dist/infrastructure/scheduler/cron/persisted.d.ts +5 -0
- package/dist/infrastructure/scheduler/cron/persisted.js +27 -0
- package/dist/infrastructure/scheduler/cron/runtime.js +5 -10
- package/dist/infrastructure/scheduler/cron/validation.d.ts +3 -0
- package/dist/infrastructure/scheduler/cron/validation.js +11 -0
- package/dist/infrastructure/scheduler/cronParser.d.ts +1 -1
- package/dist/infrastructure/scheduler/cronParser.js +144 -11
- package/dist/infrastructure/server/bootstrap.d.ts +2 -3
- package/dist/infrastructure/server/bootstrap.js +33 -59
- package/dist/infrastructure/server/errors.d.ts +4 -0
- package/dist/infrastructure/server/errors.js +54 -0
- package/dist/infrastructure/server/handler-routes/control.d.ts +5 -5
- package/dist/infrastructure/server/handler-routes/control.js +3 -1
- package/dist/infrastructure/server/handler-routes/jobs.js +0 -3
- package/dist/infrastructure/server/handler-routes/monitoring.d.ts +2 -2
- package/dist/infrastructure/server/handler.js +9 -11
- package/dist/infrastructure/server/handlerRoutes.d.ts +2 -2
- package/dist/infrastructure/server/handlerRoutes.js +2 -2
- package/dist/infrastructure/server/handlers/advanced/dependencies.js +7 -6
- package/dist/infrastructure/server/handlers/advanced/jobs.js +19 -6
- package/dist/infrastructure/server/handlers/advanced/queue.d.ts +9 -9
- package/dist/infrastructure/server/handlers/advanced/queue.js +91 -27
- package/dist/infrastructure/server/handlers/core.js +32 -31
- package/dist/infrastructure/server/handlers/cron.d.ts +4 -4
- package/dist/infrastructure/server/handlers/cron.js +60 -59
- package/dist/infrastructure/server/handlers/dashboard.d.ts +1 -1
- package/dist/infrastructure/server/handlers/dashboard.js +21 -8
- package/dist/infrastructure/server/handlers/dlq.d.ts +6 -3
- package/dist/infrastructure/server/handlers/dlq.js +35 -9
- package/dist/infrastructure/server/handlers/flow.js +2 -1
- package/dist/infrastructure/server/handlers/management.d.ts +5 -5
- package/dist/infrastructure/server/handlers/management.js +36 -9
- package/dist/infrastructure/server/handlers/monitoring/health.d.ts +5 -5
- package/dist/infrastructure/server/handlers/monitoring/health.js +45 -16
- package/dist/infrastructure/server/handlers/monitoring/operations.d.ts +1 -1
- package/dist/infrastructure/server/handlers/monitoring/operations.js +8 -1
- package/dist/infrastructure/server/handlers/monitoring/workers.d.ts +3 -3
- package/dist/infrastructure/server/handlers/monitoring/workers.js +31 -7
- package/dist/infrastructure/server/handlers/pushBatchValidation.d.ts +3 -1
- package/dist/infrastructure/server/handlers/pushBatchValidation.js +52 -7
- package/dist/infrastructure/server/handlers/query.d.ts +1 -1
- package/dist/infrastructure/server/handlers/query.js +5 -4
- package/dist/infrastructure/server/http.js +2 -2
- package/dist/infrastructure/server/httpDashboardEndpoints.d.ts +1 -1
- package/dist/infrastructure/server/httpDashboardEndpoints.js +19 -6
- package/dist/infrastructure/server/httpEndpoints.js +20 -7
- package/dist/infrastructure/server/httpRouteQueues.js +11 -1
- package/dist/infrastructure/server/shutdownCoordinator.d.ts +24 -0
- package/dist/infrastructure/server/shutdownCoordinator.js +106 -0
- package/dist/infrastructure/server/sseHandler.js +8 -3
- package/dist/infrastructure/server/storageAdapter.d.ts +29 -0
- package/dist/infrastructure/server/storageAdapter.js +61 -0
- package/dist/infrastructure/server/storageManager.d.ts +12 -0
- package/dist/infrastructure/server/storageManager.js +102 -0
- package/dist/infrastructure/server/tcp.js +2 -3
- package/dist/infrastructure/server/ws/snapshots.d.ts +1 -1
- package/dist/infrastructure/server/ws/snapshots.js +19 -4
- package/dist/infrastructure/server/wsHandler.js +10 -5
- package/dist/main.js +1 -1
- package/dist/mcp/backend/embedded/services.js +2 -1
- package/dist/mcp/index.d.ts +2 -2
- package/dist/mcp/index.js +2 -2
- package/dist/shared/storageHealth.d.ts +9 -0
- package/dist/shared/storageHealth.js +12 -0
- package/package.json +85 -74
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { PostgresEventStream } from './events';
|
|
2
|
+
import type { PostgresContext } from './context';
|
|
3
|
+
import type { PostgresDeliveredStoreEvent, PostgresStorageConfig, PostgresStorageHealth } from './types';
|
|
4
|
+
/** Connection, schema, event stream, health, and maintenance lifecycle. */
|
|
5
|
+
export declare class PostgresQueueStoreRuntime {
|
|
6
|
+
readonly context: PostgresContext;
|
|
7
|
+
readonly events: PostgresEventStream;
|
|
8
|
+
private heartbeatTimer;
|
|
9
|
+
private recoveryTimer;
|
|
10
|
+
private dlqTimer;
|
|
11
|
+
private cronTimer;
|
|
12
|
+
private readyPromise;
|
|
13
|
+
private closing;
|
|
14
|
+
private closed;
|
|
15
|
+
private closeAttempt;
|
|
16
|
+
private leasesReleased;
|
|
17
|
+
private workersRemoved;
|
|
18
|
+
private brokerUnregistered;
|
|
19
|
+
private eventsClosed;
|
|
20
|
+
private eventStatusUnsubscribed;
|
|
21
|
+
private sqlClosed;
|
|
22
|
+
private brokerRegistrationStarted;
|
|
23
|
+
private readonly healthTracker;
|
|
24
|
+
private readonly unsubscribeEventStatus;
|
|
25
|
+
private readonly postCommitMaintenance;
|
|
26
|
+
private readonly maintenanceFlights;
|
|
27
|
+
private readonly eventCommitGc;
|
|
28
|
+
constructor(config: PostgresStorageConfig);
|
|
29
|
+
get config(): import("./types").ResolvedPostgresStorageConfig;
|
|
30
|
+
initialize(): Promise<void>;
|
|
31
|
+
close(): Promise<void>;
|
|
32
|
+
private stopMaintenance;
|
|
33
|
+
private closeOnce;
|
|
34
|
+
private tryCloseStep;
|
|
35
|
+
health(): PostgresStorageHealth;
|
|
36
|
+
reportQueueRefresh(queue: string, error: unknown): void;
|
|
37
|
+
reportProjectionRefresh(queue: string, id: string, error: unknown): void;
|
|
38
|
+
private reportMaintenance;
|
|
39
|
+
onEvent(listener: (event: PostgresDeliveredStoreEvent) => void): () => void;
|
|
40
|
+
onInvalidation(listener: (queue: string) => void): () => void;
|
|
41
|
+
waitForWork(queue: string, timeoutMs: number, signal?: AbortSignal): Promise<void>;
|
|
42
|
+
private initializeOnce;
|
|
43
|
+
private startMaintenance;
|
|
44
|
+
private runMaintenance;
|
|
45
|
+
private heartbeatBroker;
|
|
46
|
+
}
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
import { recoverExpiredPostgresLeases } from './recovery';
|
|
2
|
+
import { releasePostgresBrokerLeases } from './leaseRelease';
|
|
3
|
+
import { PostgresEventStream } from './events';
|
|
4
|
+
import { prunePostgresEventCommits } from './eventJournal';
|
|
5
|
+
import { maintainPostgresDlq, repairPostgresDlq } from './dlqMaintenance';
|
|
6
|
+
import { listPostgresCrons, processPostgresCrons, reconcilePostgresCronsOnStartup } from './crons';
|
|
7
|
+
import { purgeStalePostgresWorkers, removePostgresBrokerWorkers } from './workers';
|
|
8
|
+
import { initializePostgresSchema } from './schemaInitialization';
|
|
9
|
+
import { PostgresPostCommitMaintenance } from './postCommitMaintenance';
|
|
10
|
+
import { sweepPostgresEventRetention } from './telemetry';
|
|
11
|
+
import { prunePostgresCompletionTombstones } from './completionLifecycle';
|
|
12
|
+
import { PostgresMaintenanceFlights } from './maintenanceFlights';
|
|
13
|
+
import { resolvePostgresRuntimeConfig } from './runtimeConfig';
|
|
14
|
+
import { PostgresEventCommitGc } from './eventCommitGc';
|
|
15
|
+
import { createPostgresConnection } from './connection';
|
|
16
|
+
import { PostgresStorageHealthTracker } from './storageHealth';
|
|
17
|
+
import { heartbeatPostgresBroker, PostgresBrokerSessionFencedError, purgeStalePostgresBrokers, registerPostgresBroker, unregisterPostgresBroker, } from './brokerSessions';
|
|
18
|
+
/** Connection, schema, event stream, health, and maintenance lifecycle. */
|
|
19
|
+
export class PostgresQueueStoreRuntime {
|
|
20
|
+
context;
|
|
21
|
+
events;
|
|
22
|
+
heartbeatTimer = null;
|
|
23
|
+
recoveryTimer = null;
|
|
24
|
+
dlqTimer = null;
|
|
25
|
+
cronTimer = null;
|
|
26
|
+
readyPromise = null;
|
|
27
|
+
closing = false;
|
|
28
|
+
closed = false;
|
|
29
|
+
closeAttempt = null;
|
|
30
|
+
leasesReleased = false;
|
|
31
|
+
workersRemoved = false;
|
|
32
|
+
brokerUnregistered = false;
|
|
33
|
+
eventsClosed = false;
|
|
34
|
+
eventStatusUnsubscribed = false;
|
|
35
|
+
sqlClosed = false;
|
|
36
|
+
brokerRegistrationStarted = false;
|
|
37
|
+
healthTracker;
|
|
38
|
+
unsubscribeEventStatus;
|
|
39
|
+
postCommitMaintenance;
|
|
40
|
+
maintenanceFlights;
|
|
41
|
+
eventCommitGc;
|
|
42
|
+
constructor(config) {
|
|
43
|
+
const resolved = resolvePostgresRuntimeConfig(config);
|
|
44
|
+
const sql = createPostgresConnection(resolved);
|
|
45
|
+
this.healthTracker = new PostgresStorageHealthTracker(resolved);
|
|
46
|
+
this.postCommitMaintenance = new PostgresPostCommitMaintenance((subsystem, error) => this.reportMaintenance(subsystem, error), Math.max(25, resolved.pollIntervalMs));
|
|
47
|
+
this.maintenanceFlights = new PostgresMaintenanceFlights((subsystem, error) => this.reportMaintenance(subsystem, error));
|
|
48
|
+
this.context = {
|
|
49
|
+
sql,
|
|
50
|
+
config: resolved,
|
|
51
|
+
postCommitMaintenance: (subsystem, operation) => this.postCommitMaintenance.run(subsystem, operation),
|
|
52
|
+
};
|
|
53
|
+
this.eventCommitGc = new PostgresEventCommitGc((batchSize, maxBatches) => prunePostgresEventCommits(this.context, batchSize, maxBatches), (subsystem, error) => this.reportMaintenance(subsystem, error));
|
|
54
|
+
this.events = new PostgresEventStream(this.context);
|
|
55
|
+
this.unsubscribeEventStatus = this.events.subscribeDrainStatus((error) => {
|
|
56
|
+
if (error === null)
|
|
57
|
+
this.healthTracker.clear('event-stream');
|
|
58
|
+
else
|
|
59
|
+
this.healthTracker.record('event-stream', error);
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
get config() {
|
|
63
|
+
return this.context.config;
|
|
64
|
+
}
|
|
65
|
+
initialize() {
|
|
66
|
+
this.readyPromise ??= this.initializeOnce();
|
|
67
|
+
return this.readyPromise;
|
|
68
|
+
}
|
|
69
|
+
close() {
|
|
70
|
+
if (this.closed)
|
|
71
|
+
return Promise.resolve();
|
|
72
|
+
if (this.closeAttempt)
|
|
73
|
+
return this.closeAttempt;
|
|
74
|
+
this.closing = true;
|
|
75
|
+
this.stopMaintenance();
|
|
76
|
+
const attempt = this.closeOnce().finally(() => {
|
|
77
|
+
this.closeAttempt = null;
|
|
78
|
+
});
|
|
79
|
+
this.closeAttempt = attempt;
|
|
80
|
+
return attempt;
|
|
81
|
+
}
|
|
82
|
+
stopMaintenance() {
|
|
83
|
+
this.postCommitMaintenance.close();
|
|
84
|
+
this.maintenanceFlights.close();
|
|
85
|
+
this.eventCommitGc.close();
|
|
86
|
+
if (this.heartbeatTimer)
|
|
87
|
+
clearInterval(this.heartbeatTimer);
|
|
88
|
+
if (this.recoveryTimer)
|
|
89
|
+
clearInterval(this.recoveryTimer);
|
|
90
|
+
if (this.dlqTimer)
|
|
91
|
+
clearInterval(this.dlqTimer);
|
|
92
|
+
if (this.cronTimer)
|
|
93
|
+
clearInterval(this.cronTimer);
|
|
94
|
+
this.heartbeatTimer = null;
|
|
95
|
+
this.recoveryTimer = null;
|
|
96
|
+
this.dlqTimer = null;
|
|
97
|
+
this.cronTimer = null;
|
|
98
|
+
}
|
|
99
|
+
async closeOnce() {
|
|
100
|
+
const errors = [];
|
|
101
|
+
let initialized = true;
|
|
102
|
+
try {
|
|
103
|
+
await this.initialize();
|
|
104
|
+
}
|
|
105
|
+
catch (error) {
|
|
106
|
+
initialized = false;
|
|
107
|
+
errors.push(error);
|
|
108
|
+
}
|
|
109
|
+
if (initialized || this.brokerRegistrationStarted) {
|
|
110
|
+
await this.maintenanceFlights.drain();
|
|
111
|
+
await this.eventCommitGc.drain();
|
|
112
|
+
await this.postCommitMaintenance.drain();
|
|
113
|
+
this.leasesReleased = await this.tryCloseStep(this.leasesReleased, () => releasePostgresBrokerLeases(this.context), errors);
|
|
114
|
+
this.workersRemoved = await this.tryCloseStep(this.workersRemoved, () => removePostgresBrokerWorkers(this.context), errors);
|
|
115
|
+
this.brokerUnregistered = await this.tryCloseStep(this.brokerUnregistered, async () => {
|
|
116
|
+
await unregisterPostgresBroker(this.context);
|
|
117
|
+
}, errors);
|
|
118
|
+
}
|
|
119
|
+
this.eventsClosed = await this.tryCloseStep(this.eventsClosed, () => this.events.close(), errors);
|
|
120
|
+
if (this.eventsClosed && !this.eventStatusUnsubscribed) {
|
|
121
|
+
this.unsubscribeEventStatus();
|
|
122
|
+
this.eventStatusUnsubscribed = true;
|
|
123
|
+
}
|
|
124
|
+
const databaseClosed = !this.brokerRegistrationStarted ||
|
|
125
|
+
(this.leasesReleased && this.workersRemoved && this.brokerUnregistered);
|
|
126
|
+
if (databaseClosed) {
|
|
127
|
+
this.sqlClosed = await this.tryCloseStep(this.sqlClosed, () => this.context.sql.close({ timeout: 5 }), errors);
|
|
128
|
+
}
|
|
129
|
+
if (this.eventsClosed && this.sqlClosed && (!initialized || databaseClosed)) {
|
|
130
|
+
this.closed = true;
|
|
131
|
+
}
|
|
132
|
+
if (errors.length === 1)
|
|
133
|
+
throw errors[0];
|
|
134
|
+
if (errors.length > 1)
|
|
135
|
+
throw new AggregateError(errors, 'PostgreSQL shutdown failed');
|
|
136
|
+
}
|
|
137
|
+
async tryCloseStep(completed, operation, errors) {
|
|
138
|
+
if (completed)
|
|
139
|
+
return true;
|
|
140
|
+
try {
|
|
141
|
+
await operation();
|
|
142
|
+
return true;
|
|
143
|
+
}
|
|
144
|
+
catch (error) {
|
|
145
|
+
errors.push(error);
|
|
146
|
+
return false;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
health() {
|
|
150
|
+
return this.healthTracker.snapshot();
|
|
151
|
+
}
|
|
152
|
+
reportQueueRefresh(queue, error) {
|
|
153
|
+
const key = `queue-refresh:${queue}`;
|
|
154
|
+
if (error === null) {
|
|
155
|
+
this.healthTracker.clear(key);
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
this.healthTracker.record(key, error, `Queue refresh ${queue}: `);
|
|
159
|
+
}
|
|
160
|
+
reportProjectionRefresh(queue, id, error) {
|
|
161
|
+
const key = `projection-refresh:${id}`;
|
|
162
|
+
if (error === null) {
|
|
163
|
+
this.healthTracker.clear(key);
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
this.healthTracker.record(key, error, `Projection refresh ${queue}/${id}: `);
|
|
167
|
+
}
|
|
168
|
+
reportMaintenance(subsystem, error) {
|
|
169
|
+
const key = `maintenance:${subsystem}`;
|
|
170
|
+
if (error === null)
|
|
171
|
+
this.healthTracker.clear(key);
|
|
172
|
+
else
|
|
173
|
+
this.healthTracker.record(key, error);
|
|
174
|
+
}
|
|
175
|
+
onEvent(listener) {
|
|
176
|
+
return this.events.subscribe(listener);
|
|
177
|
+
}
|
|
178
|
+
onInvalidation(listener) {
|
|
179
|
+
return this.events.subscribeInvalidation(listener);
|
|
180
|
+
}
|
|
181
|
+
waitForWork(queue, timeoutMs, signal) {
|
|
182
|
+
return this.events.wait(queue, timeoutMs, signal);
|
|
183
|
+
}
|
|
184
|
+
async initializeOnce() {
|
|
185
|
+
try {
|
|
186
|
+
await this.context.sql.connect();
|
|
187
|
+
await initializePostgresSchema(this.context);
|
|
188
|
+
await listPostgresCrons(this.context);
|
|
189
|
+
await registerPostgresBroker(this.context);
|
|
190
|
+
this.brokerRegistrationStarted = true;
|
|
191
|
+
await reconcilePostgresCronsOnStartup(this.context);
|
|
192
|
+
await recoverExpiredPostgresLeases(this.context);
|
|
193
|
+
await repairPostgresDlq(this.context);
|
|
194
|
+
await prunePostgresCompletionTombstones(this.context);
|
|
195
|
+
await sweepPostgresEventRetention(this.context);
|
|
196
|
+
await prunePostgresEventCommits(this.context);
|
|
197
|
+
await this.events.start();
|
|
198
|
+
if (!this.closing)
|
|
199
|
+
this.startMaintenance();
|
|
200
|
+
this.healthTracker.clear('lifecycle');
|
|
201
|
+
}
|
|
202
|
+
catch (error) {
|
|
203
|
+
this.healthTracker.record('lifecycle', error);
|
|
204
|
+
throw error;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
startMaintenance() {
|
|
208
|
+
this.eventCommitGc.start();
|
|
209
|
+
const heartbeatMs = Math.max(1000, Math.floor(this.config.leaseDurationMs / 3));
|
|
210
|
+
this.heartbeatTimer = setInterval(() => void this.runMaintenance(() => this.heartbeatBroker(), 'heartbeat'), heartbeatMs);
|
|
211
|
+
this.recoveryTimer = setInterval(() => void this.runMaintenance(() => recoverExpiredPostgresLeases(this.context), 'recovery'), Math.max(500, Math.floor(this.config.leaseDurationMs / 2)));
|
|
212
|
+
this.dlqTimer = setInterval(() => {
|
|
213
|
+
void this.runMaintenance(async () => {
|
|
214
|
+
await maintainPostgresDlq(this.context);
|
|
215
|
+
await purgeStalePostgresWorkers(this.context);
|
|
216
|
+
await purgeStalePostgresBrokers(this.context);
|
|
217
|
+
}, 'dlq');
|
|
218
|
+
void this.runMaintenance(() => sweepPostgresEventRetention(this.context), 'event-retention');
|
|
219
|
+
void this.postCommitMaintenance.run('completion-retention', () => prunePostgresCompletionTombstones(this.context));
|
|
220
|
+
}, 60_000);
|
|
221
|
+
this.cronTimer = setInterval(() => void this.runMaintenance(() => processPostgresCrons(this.context), 'cron'), this.config.pollIntervalMs);
|
|
222
|
+
}
|
|
223
|
+
async runMaintenance(operation, subsystem = 'maintenance') {
|
|
224
|
+
await this.maintenanceFlights.run(subsystem, operation);
|
|
225
|
+
}
|
|
226
|
+
async heartbeatBroker() {
|
|
227
|
+
try {
|
|
228
|
+
await heartbeatPostgresBroker(this.context);
|
|
229
|
+
}
|
|
230
|
+
catch (error) {
|
|
231
|
+
if (error instanceof PostgresBrokerSessionFencedError)
|
|
232
|
+
this.stopMaintenance();
|
|
233
|
+
throw error;
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { PostgresStorageConfig } from './types';
|
|
2
|
+
export declare function resolvePostgresRuntimeConfig(config: PostgresStorageConfig): {
|
|
3
|
+
url: string;
|
|
4
|
+
namespace: string;
|
|
5
|
+
brokerId: string;
|
|
6
|
+
brokerSessionId: string;
|
|
7
|
+
poolSize: number;
|
|
8
|
+
leaseDurationMs: number;
|
|
9
|
+
pollIntervalMs: number;
|
|
10
|
+
statementTimeoutMs: number;
|
|
11
|
+
lockTimeoutMs: number;
|
|
12
|
+
idleTransactionTimeoutMs: number;
|
|
13
|
+
maxConcurrentOperations: number;
|
|
14
|
+
maxQueuedOperations: number;
|
|
15
|
+
maxSnapshotJobs: number;
|
|
16
|
+
maxSnapshotPayloadBytes: number;
|
|
17
|
+
maxQueueEvents: number;
|
|
18
|
+
maxMetricDataPoints: number;
|
|
19
|
+
maxCompletedJobs: number;
|
|
20
|
+
maxJobResults: number;
|
|
21
|
+
};
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { hostname } from 'node:os';
|
|
2
|
+
function integerAtLeast(value, fallback, minimum) {
|
|
3
|
+
if (!Number.isFinite(value))
|
|
4
|
+
return fallback;
|
|
5
|
+
return Math.max(minimum, Math.floor(value));
|
|
6
|
+
}
|
|
7
|
+
export function resolvePostgresRuntimeConfig(config) {
|
|
8
|
+
const url = new URL(config.url);
|
|
9
|
+
if (url.protocol !== 'postgres:' && url.protocol !== 'postgresql:') {
|
|
10
|
+
throw new Error('PostgreSQL storage requires a postgres:// or postgresql:// URL');
|
|
11
|
+
}
|
|
12
|
+
if (config.maxQueueEvents !== undefined &&
|
|
13
|
+
Number.isFinite(config.maxQueueEvents) &&
|
|
14
|
+
config.maxQueueEvents < 1) {
|
|
15
|
+
throw new Error('PostgreSQL storage requires maxQueueEvents to be at least 1');
|
|
16
|
+
}
|
|
17
|
+
const poolSize = integerAtLeast(config.poolSize, 4, 2);
|
|
18
|
+
const maxConcurrentOperations = integerAtLeast(config.maxConcurrentOperations, 16, 1);
|
|
19
|
+
return {
|
|
20
|
+
url: config.url,
|
|
21
|
+
namespace: config.namespace?.trim() || 'default',
|
|
22
|
+
brokerId: config.brokerId?.trim() || `${hostname()}:${process.pid}:${Bun.randomUUIDv7().slice(-12)}`,
|
|
23
|
+
brokerSessionId: Bun.randomUUIDv7(),
|
|
24
|
+
poolSize,
|
|
25
|
+
leaseDurationMs: integerAtLeast(config.leaseDurationMs, 30_000, 1000),
|
|
26
|
+
pollIntervalMs: integerAtLeast(config.pollIntervalMs, 250, 25),
|
|
27
|
+
statementTimeoutMs: integerAtLeast(config.statementTimeoutMs, 30_000, 1),
|
|
28
|
+
lockTimeoutMs: integerAtLeast(config.lockTimeoutMs, 5_000, 1),
|
|
29
|
+
idleTransactionTimeoutMs: integerAtLeast(config.idleTransactionTimeoutMs, 30_000, 1),
|
|
30
|
+
maxConcurrentOperations,
|
|
31
|
+
maxQueuedOperations: integerAtLeast(config.maxQueuedOperations, 128, 0),
|
|
32
|
+
maxSnapshotJobs: integerAtLeast(config.maxSnapshotJobs, 100_000, 1),
|
|
33
|
+
maxSnapshotPayloadBytes: integerAtLeast(config.maxSnapshotPayloadBytes, 256 * 1024 * 1024, 1),
|
|
34
|
+
maxQueueEvents: integerAtLeast(config.maxQueueEvents, 10_000, 1),
|
|
35
|
+
maxMetricDataPoints: integerAtLeast(config.maxMetricDataPoints, 20_160, 0),
|
|
36
|
+
maxCompletedJobs: integerAtLeast(config.maxCompletedJobs, 50_000, 1),
|
|
37
|
+
maxJobResults: integerAtLeast(config.maxJobResults, 10_000, 0),
|
|
38
|
+
};
|
|
39
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export declare const POSTGRES_SCHEMA_VERSION = 17;
|
|
2
|
+
/** PostgreSQL schema for the database-authoritative distributed queue engine. */
|
|
3
|
+
export declare const POSTGRES_SCHEMA = "\nCREATE TABLE IF NOT EXISTS bunqueue_schema_migrations (\n version INTEGER PRIMARY KEY,\n applied_at BIGINT NOT NULL\n);\n\nCREATE TABLE IF NOT EXISTS bunqueue_jobs (\n namespace TEXT NOT NULL,\n id TEXT NOT NULL,\n queue TEXT NOT NULL,\n payload BYTEA NOT NULL,\n state TEXT NOT NULL CHECK (\n state IN ('waiting', 'prioritized', 'delayed', 'waiting-children',\n 'active', 'completed', 'failed')\n ),\n priority INTEGER NOT NULL DEFAULT 0,\n lifo BOOLEAN NOT NULL DEFAULT FALSE,\n run_at BIGINT NOT NULL,\n created_at BIGINT NOT NULL,\n started_at BIGINT,\n completed_at BIGINT,\n attempts INTEGER NOT NULL DEFAULT 0,\n max_attempts INTEGER NOT NULL DEFAULT 3,\n ttl BIGINT,\n timeout BIGINT,\n unique_key TEXT,\n unique_expires_at BIGINT,\n custom_id TEXT,\n group_id TEXT,\n parent_id TEXT,\n lease_owner TEXT,\n lease_broker_id TEXT,\n lease_broker_session_id TEXT,\n lease_token TEXT,\n lease_until BIGINT,\n lease_renewals INTEGER NOT NULL DEFAULT 0,\n result BYTEA,\n dlq_entry BYTEA,\n dlq_retry_state BYTEA,\n error TEXT,\n failure_reason TEXT,\n version BIGINT NOT NULL DEFAULT 0,\n PRIMARY KEY (namespace, id)\n);\n\nCREATE INDEX IF NOT EXISTS bunqueue_jobs_ready_idx\n ON bunqueue_jobs(namespace, queue, priority DESC, run_at ASC, id ASC)\n WHERE state IN ('waiting', 'prioritized', 'delayed');\nCREATE INDEX IF NOT EXISTS bunqueue_jobs_state_idx\n ON bunqueue_jobs(namespace, queue, state, created_at, id);\nCREATE INDEX IF NOT EXISTS bunqueue_jobs_lease_idx\n ON bunqueue_jobs(namespace, lease_until)\n WHERE state = 'active';\nCREATE INDEX IF NOT EXISTS bunqueue_jobs_parent_idx\n ON bunqueue_jobs(namespace, parent_id)\n WHERE parent_id IS NOT NULL;\nCREATE INDEX IF NOT EXISTS bunqueue_jobs_group_ready_idx\n ON bunqueue_jobs(namespace, queue, group_id, priority DESC, run_at ASC, id ASC)\n WHERE group_id IS NOT NULL AND state IN ('waiting', 'prioritized', 'delayed');\nCREATE INDEX IF NOT EXISTS bunqueue_jobs_group_active_idx\n ON bunqueue_jobs(namespace, queue, group_id, lease_until)\n WHERE group_id IS NOT NULL AND state = 'active';\nCREATE INDEX IF NOT EXISTS bunqueue_jobs_lifo_ready_idx\n ON bunqueue_jobs(namespace, queue, priority DESC, id DESC)\n WHERE group_id IS NULL AND lifo AND state IN ('waiting', 'prioritized', 'delayed');\nCREATE INDEX IF NOT EXISTS bunqueue_jobs_ttl_pending_idx\n ON bunqueue_jobs(namespace, queue, created_at, id)\n WHERE ttl IS NOT NULL AND state IN ('waiting', 'prioritized', 'delayed');\nALTER TABLE bunqueue_jobs\n ADD COLUMN IF NOT EXISTS unique_expires_at BIGINT;\nALTER TABLE bunqueue_jobs\n ADD COLUMN IF NOT EXISTS lease_broker_id TEXT;\nALTER TABLE bunqueue_jobs\n ADD COLUMN IF NOT EXISTS lease_broker_session_id TEXT;\nALTER TABLE bunqueue_jobs\n ADD COLUMN IF NOT EXISTS lease_renewals INTEGER NOT NULL DEFAULT 0;\nALTER TABLE bunqueue_jobs\n ADD COLUMN IF NOT EXISTS dlq_retry_state BYTEA;\nCREATE INDEX IF NOT EXISTS bunqueue_jobs_broker_session_lease_idx\n ON bunqueue_jobs(namespace, lease_broker_id, lease_broker_session_id, id)\n WHERE state = 'active';\n\nCREATE TABLE IF NOT EXISTS bunqueue_dependencies (\n namespace TEXT NOT NULL,\n job_id TEXT NOT NULL,\n dependency_id TEXT NOT NULL,\n PRIMARY KEY (namespace, job_id, dependency_id)\n);\nCREATE INDEX IF NOT EXISTS bunqueue_dependencies_dependency_idx\n ON bunqueue_dependencies(namespace, dependency_id, job_id);\n\nCREATE TABLE IF NOT EXISTS bunqueue_flow_failures (\n namespace TEXT NOT NULL,\n parent_id TEXT NOT NULL,\n child_id TEXT NOT NULL,\n child_queue TEXT NOT NULL,\n mode TEXT NOT NULL CHECK (mode IN ('fail', 'remove', 'ignore', 'continue')),\n error TEXT NOT NULL,\n created_at BIGINT NOT NULL,\n PRIMARY KEY (namespace, parent_id, child_id)\n);\nCREATE INDEX IF NOT EXISTS bunqueue_flow_failures_parent_idx\n ON bunqueue_flow_failures(namespace, parent_id, mode);\n\nCREATE TABLE IF NOT EXISTS bunqueue_repeat_links (\n namespace TEXT NOT NULL,\n original_id TEXT NOT NULL,\n successor_id TEXT NOT NULL,\n PRIMARY KEY (namespace, original_id)\n);\nCREATE INDEX IF NOT EXISTS bunqueue_repeat_links_successor_idx\n ON bunqueue_repeat_links(namespace, successor_id);\n\nCREATE TABLE IF NOT EXISTS bunqueue_job_logs (\n id BIGSERIAL PRIMARY KEY,\n namespace TEXT NOT NULL,\n job_id TEXT NOT NULL,\n timestamp BIGINT NOT NULL,\n level TEXT NOT NULL CHECK (level IN ('info', 'warn', 'error')),\n message TEXT NOT NULL\n);\nCREATE INDEX IF NOT EXISTS bunqueue_job_logs_job_idx\n ON bunqueue_job_logs(namespace, job_id, id);\n\nCREATE TABLE IF NOT EXISTS bunqueue_crons (\n namespace TEXT NOT NULL,\n name TEXT NOT NULL,\n payload BYTEA NOT NULL,\n next_run BIGINT NOT NULL,\n executions INTEGER NOT NULL DEFAULT 0,\n max_limit INTEGER,\n updated_at BIGINT NOT NULL,\n PRIMARY KEY (namespace, name)\n);\nCREATE INDEX IF NOT EXISTS bunqueue_crons_due_idx\n ON bunqueue_crons(namespace, next_run, name);\n\nCREATE TABLE IF NOT EXISTS bunqueue_workers (\n namespace TEXT NOT NULL,\n id TEXT NOT NULL,\n broker_id TEXT NOT NULL,\n broker_session_id TEXT,\n client_id TEXT,\n queues TEXT[] NOT NULL,\n payload BYTEA NOT NULL,\n last_seen BIGINT NOT NULL,\n PRIMARY KEY (namespace, id)\n);\nCREATE INDEX IF NOT EXISTS bunqueue_workers_queue_idx\n ON bunqueue_workers USING GIN(queues);\nCREATE INDEX IF NOT EXISTS bunqueue_workers_broker_idx\n ON bunqueue_workers(namespace, broker_id, client_id);\nALTER TABLE bunqueue_workers\n ADD COLUMN IF NOT EXISTS broker_session_id TEXT;\nCREATE INDEX IF NOT EXISTS bunqueue_workers_broker_session_idx\n ON bunqueue_workers(namespace, broker_id, broker_session_id, client_id);\n\nCREATE TABLE IF NOT EXISTS bunqueue_metric_buckets (\n namespace TEXT NOT NULL,\n queue TEXT NOT NULL,\n metric_type TEXT NOT NULL CHECK (metric_type IN ('completed', 'failed')),\n minute BIGINT NOT NULL,\n count INTEGER NOT NULL,\n PRIMARY KEY (namespace, queue, metric_type, minute)\n);\nCREATE TABLE IF NOT EXISTS bunqueue_metric_totals (\n namespace TEXT NOT NULL,\n queue TEXT NOT NULL,\n metric_type TEXT NOT NULL CHECK (metric_type IN ('completed', 'failed')),\n total_count BIGINT NOT NULL,\n prev_ts BIGINT NOT NULL,\n prev_count INTEGER NOT NULL,\n PRIMARY KEY (namespace, queue, metric_type)\n);\n\nCREATE TABLE IF NOT EXISTS bunqueue_completions (\n namespace TEXT NOT NULL,\n job_id TEXT NOT NULL,\n queue TEXT NOT NULL,\n completed_at BIGINT NOT NULL,\n result BYTEA,\n PRIMARY KEY (namespace, job_id)\n);\nCREATE INDEX IF NOT EXISTS bunqueue_completions_queue_idx\n ON bunqueue_completions(namespace, queue, job_id);\nCREATE INDEX IF NOT EXISTS bunqueue_completions_recent_idx\n ON bunqueue_completions(namespace, completed_at DESC, job_id DESC);\n\nCREATE TABLE IF NOT EXISTS bunqueue_queue_state (\n namespace TEXT NOT NULL,\n queue TEXT NOT NULL,\n paused BOOLEAN NOT NULL DEFAULT FALSE,\n rate_limit INTEGER,\n rate_duration_ms BIGINT,\n rate_window_started_at BIGINT,\n rate_expires_at BIGINT,\n rate_count INTEGER NOT NULL DEFAULT 0,\n concurrency_limit INTEGER,\n stall_config BYTEA,\n dlq_config BYTEA,\n PRIMARY KEY (namespace, queue)\n);\n\nALTER TABLE bunqueue_queue_state\n ADD COLUMN IF NOT EXISTS rate_expires_at BIGINT;\n\nINSERT INTO bunqueue_queue_state (namespace, queue)\nSELECT DISTINCT namespace, queue FROM bunqueue_jobs\nON CONFLICT (namespace, queue) DO NOTHING;\n\nCREATE TABLE IF NOT EXISTS bunqueue_events (\n id BIGSERIAL PRIMARY KEY,\n namespace TEXT NOT NULL,\n queue TEXT NOT NULL,\n event_type TEXT NOT NULL,\n job_id TEXT NOT NULL,\n occurred_at BIGINT NOT NULL,\n payload BYTEA\n);\nCREATE INDEX IF NOT EXISTS bunqueue_events_namespace_id_idx\n ON bunqueue_events(namespace, id);\nCREATE INDEX IF NOT EXISTS bunqueue_events_queue_id_idx\n ON bunqueue_events(namespace, queue, id DESC);\n\nCREATE TABLE IF NOT EXISTS bunqueue_event_prune_watermarks (\n namespace TEXT NOT NULL,\n queue TEXT NOT NULL,\n source_event_id BIGINT NOT NULL,\n pruned_through BIGINT NOT NULL,\n PRIMARY KEY (namespace, queue, source_event_id)\n);\nCREATE INDEX IF NOT EXISTS bunqueue_event_prune_watermarks_namespace_queue_idx\n ON bunqueue_event_prune_watermarks(namespace, queue, pruned_through DESC);\n\n\nCREATE SEQUENCE IF NOT EXISTS bunqueue_event_commit_seq AS BIGINT CACHE 1;\nALTER SEQUENCE bunqueue_event_commit_seq\n AS BIGINT INCREMENT BY 1 NO MINVALUE NO MAXVALUE NO CYCLE CACHE 1;\n\nCREATE TABLE IF NOT EXISTS bunqueue_event_commits (\n namespace TEXT NOT NULL,\n transaction_id BIGINT NOT NULL,\n commit_seq BIGINT,\n PRIMARY KEY (namespace, transaction_id)\n);\n\nALTER TABLE bunqueue_event_commits\n ADD COLUMN IF NOT EXISTS namespace TEXT;\nALTER TABLE bunqueue_event_commits\n ADD COLUMN IF NOT EXISTS transaction_id BIGINT;\nALTER TABLE bunqueue_event_commits\n ADD COLUMN IF NOT EXISTS commit_seq BIGINT;\nALTER TABLE bunqueue_event_commits\n ALTER COLUMN namespace TYPE TEXT USING namespace::text,\n ALTER COLUMN transaction_id TYPE BIGINT USING transaction_id::bigint,\n ALTER COLUMN commit_seq TYPE BIGINT USING commit_seq::bigint;\n\nALTER TABLE bunqueue_events\n ADD COLUMN IF NOT EXISTS transaction_id BIGINT;\nALTER TABLE bunqueue_events\n ALTER COLUMN transaction_id DROP DEFAULT;\nALTER TABLE bunqueue_events\n ALTER COLUMN transaction_id TYPE BIGINT USING transaction_id::bigint;\nALTER TABLE bunqueue_events\n ALTER COLUMN transaction_id SET DEFAULT (pg_current_xact_id()::text::bigint);\n\nALTER TABLE bunqueue_event_prune_watermarks\n ADD COLUMN IF NOT EXISTS transaction_id BIGINT;\nALTER TABLE bunqueue_event_prune_watermarks\n ADD COLUMN IF NOT EXISTS commit_seq BIGINT;\nALTER TABLE bunqueue_event_prune_watermarks\n ADD COLUMN IF NOT EXISTS pruned_commit_seq BIGINT;\nALTER TABLE bunqueue_event_prune_watermarks\n ADD COLUMN IF NOT EXISTS prunes_current_transaction BOOLEAN NOT NULL DEFAULT FALSE;\nALTER TABLE bunqueue_event_prune_watermarks\n ALTER COLUMN transaction_id DROP DEFAULT,\n ALTER COLUMN pruned_commit_seq DROP DEFAULT,\n ALTER COLUMN prunes_current_transaction DROP DEFAULT;\nALTER TABLE bunqueue_event_prune_watermarks\n ALTER COLUMN transaction_id TYPE BIGINT USING transaction_id::bigint,\n ALTER COLUMN commit_seq TYPE BIGINT USING commit_seq::bigint,\n ALTER COLUMN pruned_commit_seq TYPE BIGINT USING pruned_commit_seq::bigint,\n ALTER COLUMN prunes_current_transaction TYPE BOOLEAN\n USING prunes_current_transaction::boolean;\nALTER TABLE bunqueue_event_prune_watermarks\n ALTER COLUMN transaction_id SET DEFAULT (pg_current_xact_id()::text::bigint);\nALTER TABLE bunqueue_event_prune_watermarks\n ALTER COLUMN pruned_commit_seq SET DEFAULT 0,\n ALTER COLUMN prunes_current_transaction SET DEFAULT FALSE;\n\nUPDATE bunqueue_events\nSET transaction_id = -id\nWHERE transaction_id IS NULL;\nUPDATE bunqueue_event_prune_watermarks\nSET transaction_id = COALESCE(transaction_id, -source_event_id),\n commit_seq = COALESCE(commit_seq, source_event_id),\n pruned_commit_seq = COALESCE(pruned_commit_seq, commit_seq, source_event_id)\nWHERE transaction_id IS NULL OR commit_seq IS NULL OR pruned_commit_seq IS NULL;\n\nALTER TABLE bunqueue_events\n ALTER COLUMN transaction_id SET NOT NULL;\nALTER TABLE bunqueue_event_commits\n ALTER COLUMN namespace SET NOT NULL,\n ALTER COLUMN transaction_id SET NOT NULL;\nALTER TABLE bunqueue_event_prune_watermarks\n ALTER COLUMN transaction_id SET NOT NULL,\n ALTER COLUMN pruned_commit_seq SET NOT NULL,\n ALTER COLUMN prunes_current_transaction SET NOT NULL;\n\nDO $migration$\nBEGIN\n IF EXISTS (\n SELECT 1 FROM pg_attribute\n WHERE attrelid = to_regclass('bunqueue_events')\n AND attname = 'commit_seq' AND NOT attisdropped\n ) THEN\n EXECUTE $sql$\n INSERT INTO bunqueue_event_commits (namespace, transaction_id, commit_seq)\n SELECT namespace, transaction_id, MAX(commit_seq)\n FROM bunqueue_events\n WHERE commit_seq IS NOT NULL\n GROUP BY namespace, transaction_id\n ON CONFLICT (namespace, transaction_id) DO UPDATE SET\n commit_seq = GREATEST(\n COALESCE(bunqueue_event_commits.commit_seq, 0),\n excluded.commit_seq\n )\n $sql$;\n ELSE\n INSERT INTO bunqueue_event_commits (namespace, transaction_id, commit_seq)\n SELECT namespace, transaction_id, MAX(id)\n FROM bunqueue_events\n GROUP BY namespace, transaction_id\n ON CONFLICT (namespace, transaction_id) DO NOTHING;\n END IF;\nEND;\n$migration$;\n\nINSERT INTO bunqueue_event_commits (namespace, transaction_id, commit_seq)\nSELECT namespace, transaction_id, MAX(commit_seq)\nFROM bunqueue_event_prune_watermarks\nWHERE commit_seq IS NOT NULL\nGROUP BY namespace, transaction_id\nON CONFLICT (namespace, transaction_id) DO UPDATE SET\n commit_seq = GREATEST(\n COALESCE(bunqueue_event_commits.commit_seq, 0),\n excluded.commit_seq\n );\n\nDO $block$\nDECLARE\n journal_floor BIGINT := 0;\n legacy_head_floor BIGINT := 0;\n sequence_floor BIGINT := 0;\nBEGIN\n SELECT COALESCE(MAX(commit_seq), 0) INTO journal_floor\n FROM bunqueue_event_commits;\n SELECT last_value INTO sequence_floor FROM bunqueue_event_commit_seq;\n IF to_regclass('bunqueue_event_commit_heads') IS NOT NULL THEN\n EXECUTE 'SELECT COALESCE(MAX(commit_seq), 0) FROM bunqueue_event_commit_heads'\n INTO legacy_head_floor;\n END IF;\n PERFORM setval(\n 'bunqueue_event_commit_seq',\n GREATEST(journal_floor, legacy_head_floor, sequence_floor, 1),\n TRUE\n );\nEND;\n$block$;\n\nDROP TABLE IF EXISTS bunqueue_event_commit_heads;\n\nDROP TRIGGER IF EXISTS bunqueue_events_register_commit ON bunqueue_events;\nDROP TRIGGER IF EXISTS bunqueue_watermarks_insert_register_commit\n ON bunqueue_event_prune_watermarks;\nDROP TRIGGER IF EXISTS bunqueue_watermarks_update_register_commit\n ON bunqueue_event_prune_watermarks;\nDROP TABLE IF EXISTS bunqueue_event_commit_assignments CASCADE;\nDROP INDEX IF EXISTS bunqueue_events_commit_idx;\nDROP INDEX IF EXISTS bunqueue_events_pending_commit_idx;\nALTER TABLE bunqueue_events DROP COLUMN IF EXISTS commit_seq;\n\nDROP INDEX IF EXISTS bunqueue_events_transaction_idx;\nDROP INDEX IF EXISTS bunqueue_event_commits_replay_idx;\nDROP INDEX IF EXISTS bunqueue_event_prune_watermarks_commit_idx;\nDROP INDEX IF EXISTS bunqueue_event_prune_watermarks_pending_commit_idx;\nDROP INDEX IF EXISTS bunqueue_event_prune_watermarks_transaction_idx;\n\nCREATE INDEX bunqueue_events_transaction_idx\n ON bunqueue_events(namespace, transaction_id, id);\nCREATE INDEX bunqueue_event_commits_replay_idx\n ON bunqueue_event_commits(namespace, commit_seq, transaction_id)\n WHERE commit_seq IS NOT NULL;\nCREATE INDEX bunqueue_event_prune_watermarks_commit_idx\n ON bunqueue_event_prune_watermarks(namespace, queue, commit_seq DESC)\n WHERE commit_seq IS NOT NULL;\nCREATE INDEX bunqueue_event_prune_watermarks_pending_commit_idx\n ON bunqueue_event_prune_watermarks(namespace, transaction_id)\n WHERE commit_seq IS NULL;\nCREATE INDEX bunqueue_event_prune_watermarks_transaction_idx\n ON bunqueue_event_prune_watermarks(namespace, transaction_id);\n\nCREATE OR REPLACE FUNCTION bunqueue_register_event_rows()\nRETURNS TRIGGER LANGUAGE plpgsql AS $function$\nBEGIN\n INSERT INTO bunqueue_event_commits (namespace, transaction_id)\n SELECT DISTINCT namespace, transaction_id FROM new_event_rows\n ORDER BY namespace, transaction_id\n ON CONFLICT DO NOTHING;\n RETURN NULL;\nEND;\n$function$;\n\nCREATE OR REPLACE FUNCTION bunqueue_register_watermark_rows()\nRETURNS TRIGGER LANGUAGE plpgsql AS $function$\nBEGIN\n INSERT INTO bunqueue_event_commits (namespace, transaction_id)\n SELECT DISTINCT namespace, transaction_id FROM new_watermark_rows\n ORDER BY namespace, transaction_id\n ON CONFLICT DO NOTHING;\n RETURN NULL;\nEND;\n$function$;\n\nCREATE OR REPLACE FUNCTION bunqueue_assign_event_commit()\nRETURNS TRIGGER LANGUAGE plpgsql AS $function$\nDECLARE\n assigned_commit_seq BIGINT;\nBEGIN\n PERFORM pg_advisory_xact_lock(hashtextextended(\n 'bunqueue:event-commit:' || length(NEW.namespace)::text || ':' || NEW.namespace,\n 0\n ));\n assigned_commit_seq := nextval('bunqueue_event_commit_seq');\n\n UPDATE bunqueue_event_commits\n SET commit_seq = assigned_commit_seq\n WHERE namespace = NEW.namespace\n AND transaction_id = NEW.transaction_id\n AND commit_seq IS NULL;\n UPDATE bunqueue_event_prune_watermarks\n SET commit_seq = assigned_commit_seq,\n pruned_commit_seq = CASE\n WHEN prunes_current_transaction THEN GREATEST(\n COALESCE(pruned_commit_seq, 0), assigned_commit_seq\n )\n ELSE pruned_commit_seq\n END,\n prunes_current_transaction = FALSE\n WHERE namespace = NEW.namespace\n AND transaction_id = NEW.transaction_id\n AND commit_seq IS NULL;\n RETURN NULL;\nEND;\n$function$;\n\nDROP TRIGGER IF EXISTS bunqueue_events_register_commit ON bunqueue_events;\nCREATE TRIGGER bunqueue_events_register_commit\nAFTER INSERT ON bunqueue_events\nREFERENCING NEW TABLE AS new_event_rows\nFOR EACH STATEMENT EXECUTE FUNCTION bunqueue_register_event_rows();\n\nDROP TRIGGER IF EXISTS bunqueue_watermarks_insert_register_commit\n ON bunqueue_event_prune_watermarks;\nCREATE TRIGGER bunqueue_watermarks_insert_register_commit\nAFTER INSERT ON bunqueue_event_prune_watermarks\nREFERENCING NEW TABLE AS new_watermark_rows\nFOR EACH STATEMENT EXECUTE FUNCTION bunqueue_register_watermark_rows();\n\nDROP TRIGGER IF EXISTS bunqueue_watermarks_update_register_commit\n ON bunqueue_event_prune_watermarks;\nCREATE TRIGGER bunqueue_watermarks_update_register_commit\nAFTER UPDATE ON bunqueue_event_prune_watermarks\nREFERENCING NEW TABLE AS new_watermark_rows\nFOR EACH STATEMENT EXECUTE FUNCTION bunqueue_register_watermark_rows();\n\nDROP TRIGGER IF EXISTS bunqueue_assign_event_commit\n ON bunqueue_event_commits;\nCREATE CONSTRAINT TRIGGER bunqueue_assign_event_commit\nAFTER INSERT ON bunqueue_event_commits\nDEFERRABLE INITIALLY DEFERRED\nFOR EACH ROW EXECUTE FUNCTION bunqueue_assign_event_commit();\n\n\nCREATE TABLE IF NOT EXISTS bunqueue_brokers (\n namespace TEXT NOT NULL,\n broker_id TEXT NOT NULL,\n session_id TEXT,\n started_at BIGINT NOT NULL,\n heartbeat_at BIGINT NOT NULL,\n PRIMARY KEY (namespace, broker_id)\n);\nALTER TABLE bunqueue_brokers\n ADD COLUMN IF NOT EXISTS session_id TEXT;\n\nDROP INDEX IF EXISTS bunqueue_jobs_live_unique_key_idx;\nCREATE UNIQUE INDEX bunqueue_jobs_live_unique_key_idx\n ON bunqueue_jobs(namespace, queue, unique_key)\n WHERE unique_key IS NOT NULL\n AND state IN ('waiting', 'prioritized', 'delayed', 'waiting-children', 'active');\n";
|
|
4
|
+
export declare const POSTGRES_EVENT_CHANNEL = "bunqueue_jobs_changed";
|
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
import { POSTGRES_EVENT_JOURNAL_SCHEMA } from './eventJournalSchema';
|
|
2
|
+
export const POSTGRES_SCHEMA_VERSION = 17;
|
|
3
|
+
/** PostgreSQL schema for the database-authoritative distributed queue engine. */
|
|
4
|
+
export const POSTGRES_SCHEMA = `
|
|
5
|
+
CREATE TABLE IF NOT EXISTS bunqueue_schema_migrations (
|
|
6
|
+
version INTEGER PRIMARY KEY,
|
|
7
|
+
applied_at BIGINT NOT NULL
|
|
8
|
+
);
|
|
9
|
+
|
|
10
|
+
CREATE TABLE IF NOT EXISTS bunqueue_jobs (
|
|
11
|
+
namespace TEXT NOT NULL,
|
|
12
|
+
id TEXT NOT NULL,
|
|
13
|
+
queue TEXT NOT NULL,
|
|
14
|
+
payload BYTEA NOT NULL,
|
|
15
|
+
state TEXT NOT NULL CHECK (
|
|
16
|
+
state IN ('waiting', 'prioritized', 'delayed', 'waiting-children',
|
|
17
|
+
'active', 'completed', 'failed')
|
|
18
|
+
),
|
|
19
|
+
priority INTEGER NOT NULL DEFAULT 0,
|
|
20
|
+
lifo BOOLEAN NOT NULL DEFAULT FALSE,
|
|
21
|
+
run_at BIGINT NOT NULL,
|
|
22
|
+
created_at BIGINT NOT NULL,
|
|
23
|
+
started_at BIGINT,
|
|
24
|
+
completed_at BIGINT,
|
|
25
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
26
|
+
max_attempts INTEGER NOT NULL DEFAULT 3,
|
|
27
|
+
ttl BIGINT,
|
|
28
|
+
timeout BIGINT,
|
|
29
|
+
unique_key TEXT,
|
|
30
|
+
unique_expires_at BIGINT,
|
|
31
|
+
custom_id TEXT,
|
|
32
|
+
group_id TEXT,
|
|
33
|
+
parent_id TEXT,
|
|
34
|
+
lease_owner TEXT,
|
|
35
|
+
lease_broker_id TEXT,
|
|
36
|
+
lease_broker_session_id TEXT,
|
|
37
|
+
lease_token TEXT,
|
|
38
|
+
lease_until BIGINT,
|
|
39
|
+
lease_renewals INTEGER NOT NULL DEFAULT 0,
|
|
40
|
+
result BYTEA,
|
|
41
|
+
dlq_entry BYTEA,
|
|
42
|
+
dlq_retry_state BYTEA,
|
|
43
|
+
error TEXT,
|
|
44
|
+
failure_reason TEXT,
|
|
45
|
+
version BIGINT NOT NULL DEFAULT 0,
|
|
46
|
+
PRIMARY KEY (namespace, id)
|
|
47
|
+
);
|
|
48
|
+
|
|
49
|
+
CREATE INDEX IF NOT EXISTS bunqueue_jobs_ready_idx
|
|
50
|
+
ON bunqueue_jobs(namespace, queue, priority DESC, run_at ASC, id ASC)
|
|
51
|
+
WHERE state IN ('waiting', 'prioritized', 'delayed');
|
|
52
|
+
CREATE INDEX IF NOT EXISTS bunqueue_jobs_state_idx
|
|
53
|
+
ON bunqueue_jobs(namespace, queue, state, created_at, id);
|
|
54
|
+
CREATE INDEX IF NOT EXISTS bunqueue_jobs_lease_idx
|
|
55
|
+
ON bunqueue_jobs(namespace, lease_until)
|
|
56
|
+
WHERE state = 'active';
|
|
57
|
+
CREATE INDEX IF NOT EXISTS bunqueue_jobs_parent_idx
|
|
58
|
+
ON bunqueue_jobs(namespace, parent_id)
|
|
59
|
+
WHERE parent_id IS NOT NULL;
|
|
60
|
+
CREATE INDEX IF NOT EXISTS bunqueue_jobs_group_ready_idx
|
|
61
|
+
ON bunqueue_jobs(namespace, queue, group_id, priority DESC, run_at ASC, id ASC)
|
|
62
|
+
WHERE group_id IS NOT NULL AND state IN ('waiting', 'prioritized', 'delayed');
|
|
63
|
+
CREATE INDEX IF NOT EXISTS bunqueue_jobs_group_active_idx
|
|
64
|
+
ON bunqueue_jobs(namespace, queue, group_id, lease_until)
|
|
65
|
+
WHERE group_id IS NOT NULL AND state = 'active';
|
|
66
|
+
CREATE INDEX IF NOT EXISTS bunqueue_jobs_lifo_ready_idx
|
|
67
|
+
ON bunqueue_jobs(namespace, queue, priority DESC, id DESC)
|
|
68
|
+
WHERE group_id IS NULL AND lifo AND state IN ('waiting', 'prioritized', 'delayed');
|
|
69
|
+
CREATE INDEX IF NOT EXISTS bunqueue_jobs_ttl_pending_idx
|
|
70
|
+
ON bunqueue_jobs(namespace, queue, created_at, id)
|
|
71
|
+
WHERE ttl IS NOT NULL AND state IN ('waiting', 'prioritized', 'delayed');
|
|
72
|
+
ALTER TABLE bunqueue_jobs
|
|
73
|
+
ADD COLUMN IF NOT EXISTS unique_expires_at BIGINT;
|
|
74
|
+
ALTER TABLE bunqueue_jobs
|
|
75
|
+
ADD COLUMN IF NOT EXISTS lease_broker_id TEXT;
|
|
76
|
+
ALTER TABLE bunqueue_jobs
|
|
77
|
+
ADD COLUMN IF NOT EXISTS lease_broker_session_id TEXT;
|
|
78
|
+
ALTER TABLE bunqueue_jobs
|
|
79
|
+
ADD COLUMN IF NOT EXISTS lease_renewals INTEGER NOT NULL DEFAULT 0;
|
|
80
|
+
ALTER TABLE bunqueue_jobs
|
|
81
|
+
ADD COLUMN IF NOT EXISTS dlq_retry_state BYTEA;
|
|
82
|
+
CREATE INDEX IF NOT EXISTS bunqueue_jobs_broker_session_lease_idx
|
|
83
|
+
ON bunqueue_jobs(namespace, lease_broker_id, lease_broker_session_id, id)
|
|
84
|
+
WHERE state = 'active';
|
|
85
|
+
|
|
86
|
+
CREATE TABLE IF NOT EXISTS bunqueue_dependencies (
|
|
87
|
+
namespace TEXT NOT NULL,
|
|
88
|
+
job_id TEXT NOT NULL,
|
|
89
|
+
dependency_id TEXT NOT NULL,
|
|
90
|
+
PRIMARY KEY (namespace, job_id, dependency_id)
|
|
91
|
+
);
|
|
92
|
+
CREATE INDEX IF NOT EXISTS bunqueue_dependencies_dependency_idx
|
|
93
|
+
ON bunqueue_dependencies(namespace, dependency_id, job_id);
|
|
94
|
+
|
|
95
|
+
CREATE TABLE IF NOT EXISTS bunqueue_flow_failures (
|
|
96
|
+
namespace TEXT NOT NULL,
|
|
97
|
+
parent_id TEXT NOT NULL,
|
|
98
|
+
child_id TEXT NOT NULL,
|
|
99
|
+
child_queue TEXT NOT NULL,
|
|
100
|
+
mode TEXT NOT NULL CHECK (mode IN ('fail', 'remove', 'ignore', 'continue')),
|
|
101
|
+
error TEXT NOT NULL,
|
|
102
|
+
created_at BIGINT NOT NULL,
|
|
103
|
+
PRIMARY KEY (namespace, parent_id, child_id)
|
|
104
|
+
);
|
|
105
|
+
CREATE INDEX IF NOT EXISTS bunqueue_flow_failures_parent_idx
|
|
106
|
+
ON bunqueue_flow_failures(namespace, parent_id, mode);
|
|
107
|
+
|
|
108
|
+
CREATE TABLE IF NOT EXISTS bunqueue_repeat_links (
|
|
109
|
+
namespace TEXT NOT NULL,
|
|
110
|
+
original_id TEXT NOT NULL,
|
|
111
|
+
successor_id TEXT NOT NULL,
|
|
112
|
+
PRIMARY KEY (namespace, original_id)
|
|
113
|
+
);
|
|
114
|
+
CREATE INDEX IF NOT EXISTS bunqueue_repeat_links_successor_idx
|
|
115
|
+
ON bunqueue_repeat_links(namespace, successor_id);
|
|
116
|
+
|
|
117
|
+
CREATE TABLE IF NOT EXISTS bunqueue_job_logs (
|
|
118
|
+
id BIGSERIAL PRIMARY KEY,
|
|
119
|
+
namespace TEXT NOT NULL,
|
|
120
|
+
job_id TEXT NOT NULL,
|
|
121
|
+
timestamp BIGINT NOT NULL,
|
|
122
|
+
level TEXT NOT NULL CHECK (level IN ('info', 'warn', 'error')),
|
|
123
|
+
message TEXT NOT NULL
|
|
124
|
+
);
|
|
125
|
+
CREATE INDEX IF NOT EXISTS bunqueue_job_logs_job_idx
|
|
126
|
+
ON bunqueue_job_logs(namespace, job_id, id);
|
|
127
|
+
|
|
128
|
+
CREATE TABLE IF NOT EXISTS bunqueue_crons (
|
|
129
|
+
namespace TEXT NOT NULL,
|
|
130
|
+
name TEXT NOT NULL,
|
|
131
|
+
payload BYTEA NOT NULL,
|
|
132
|
+
next_run BIGINT NOT NULL,
|
|
133
|
+
executions INTEGER NOT NULL DEFAULT 0,
|
|
134
|
+
max_limit INTEGER,
|
|
135
|
+
updated_at BIGINT NOT NULL,
|
|
136
|
+
PRIMARY KEY (namespace, name)
|
|
137
|
+
);
|
|
138
|
+
CREATE INDEX IF NOT EXISTS bunqueue_crons_due_idx
|
|
139
|
+
ON bunqueue_crons(namespace, next_run, name);
|
|
140
|
+
|
|
141
|
+
CREATE TABLE IF NOT EXISTS bunqueue_workers (
|
|
142
|
+
namespace TEXT NOT NULL,
|
|
143
|
+
id TEXT NOT NULL,
|
|
144
|
+
broker_id TEXT NOT NULL,
|
|
145
|
+
broker_session_id TEXT,
|
|
146
|
+
client_id TEXT,
|
|
147
|
+
queues TEXT[] NOT NULL,
|
|
148
|
+
payload BYTEA NOT NULL,
|
|
149
|
+
last_seen BIGINT NOT NULL,
|
|
150
|
+
PRIMARY KEY (namespace, id)
|
|
151
|
+
);
|
|
152
|
+
CREATE INDEX IF NOT EXISTS bunqueue_workers_queue_idx
|
|
153
|
+
ON bunqueue_workers USING GIN(queues);
|
|
154
|
+
CREATE INDEX IF NOT EXISTS bunqueue_workers_broker_idx
|
|
155
|
+
ON bunqueue_workers(namespace, broker_id, client_id);
|
|
156
|
+
ALTER TABLE bunqueue_workers
|
|
157
|
+
ADD COLUMN IF NOT EXISTS broker_session_id TEXT;
|
|
158
|
+
CREATE INDEX IF NOT EXISTS bunqueue_workers_broker_session_idx
|
|
159
|
+
ON bunqueue_workers(namespace, broker_id, broker_session_id, client_id);
|
|
160
|
+
|
|
161
|
+
CREATE TABLE IF NOT EXISTS bunqueue_metric_buckets (
|
|
162
|
+
namespace TEXT NOT NULL,
|
|
163
|
+
queue TEXT NOT NULL,
|
|
164
|
+
metric_type TEXT NOT NULL CHECK (metric_type IN ('completed', 'failed')),
|
|
165
|
+
minute BIGINT NOT NULL,
|
|
166
|
+
count INTEGER NOT NULL,
|
|
167
|
+
PRIMARY KEY (namespace, queue, metric_type, minute)
|
|
168
|
+
);
|
|
169
|
+
CREATE TABLE IF NOT EXISTS bunqueue_metric_totals (
|
|
170
|
+
namespace TEXT NOT NULL,
|
|
171
|
+
queue TEXT NOT NULL,
|
|
172
|
+
metric_type TEXT NOT NULL CHECK (metric_type IN ('completed', 'failed')),
|
|
173
|
+
total_count BIGINT NOT NULL,
|
|
174
|
+
prev_ts BIGINT NOT NULL,
|
|
175
|
+
prev_count INTEGER NOT NULL,
|
|
176
|
+
PRIMARY KEY (namespace, queue, metric_type)
|
|
177
|
+
);
|
|
178
|
+
|
|
179
|
+
CREATE TABLE IF NOT EXISTS bunqueue_completions (
|
|
180
|
+
namespace TEXT NOT NULL,
|
|
181
|
+
job_id TEXT NOT NULL,
|
|
182
|
+
queue TEXT NOT NULL,
|
|
183
|
+
completed_at BIGINT NOT NULL,
|
|
184
|
+
result BYTEA,
|
|
185
|
+
PRIMARY KEY (namespace, job_id)
|
|
186
|
+
);
|
|
187
|
+
CREATE INDEX IF NOT EXISTS bunqueue_completions_queue_idx
|
|
188
|
+
ON bunqueue_completions(namespace, queue, job_id);
|
|
189
|
+
CREATE INDEX IF NOT EXISTS bunqueue_completions_recent_idx
|
|
190
|
+
ON bunqueue_completions(namespace, completed_at DESC, job_id DESC);
|
|
191
|
+
|
|
192
|
+
CREATE TABLE IF NOT EXISTS bunqueue_queue_state (
|
|
193
|
+
namespace TEXT NOT NULL,
|
|
194
|
+
queue TEXT NOT NULL,
|
|
195
|
+
paused BOOLEAN NOT NULL DEFAULT FALSE,
|
|
196
|
+
rate_limit INTEGER,
|
|
197
|
+
rate_duration_ms BIGINT,
|
|
198
|
+
rate_window_started_at BIGINT,
|
|
199
|
+
rate_expires_at BIGINT,
|
|
200
|
+
rate_count INTEGER NOT NULL DEFAULT 0,
|
|
201
|
+
concurrency_limit INTEGER,
|
|
202
|
+
stall_config BYTEA,
|
|
203
|
+
dlq_config BYTEA,
|
|
204
|
+
PRIMARY KEY (namespace, queue)
|
|
205
|
+
);
|
|
206
|
+
|
|
207
|
+
ALTER TABLE bunqueue_queue_state
|
|
208
|
+
ADD COLUMN IF NOT EXISTS rate_expires_at BIGINT;
|
|
209
|
+
|
|
210
|
+
INSERT INTO bunqueue_queue_state (namespace, queue)
|
|
211
|
+
SELECT DISTINCT namespace, queue FROM bunqueue_jobs
|
|
212
|
+
ON CONFLICT (namespace, queue) DO NOTHING;
|
|
213
|
+
|
|
214
|
+
CREATE TABLE IF NOT EXISTS bunqueue_events (
|
|
215
|
+
id BIGSERIAL PRIMARY KEY,
|
|
216
|
+
namespace TEXT NOT NULL,
|
|
217
|
+
queue TEXT NOT NULL,
|
|
218
|
+
event_type TEXT NOT NULL,
|
|
219
|
+
job_id TEXT NOT NULL,
|
|
220
|
+
occurred_at BIGINT NOT NULL,
|
|
221
|
+
payload BYTEA
|
|
222
|
+
);
|
|
223
|
+
CREATE INDEX IF NOT EXISTS bunqueue_events_namespace_id_idx
|
|
224
|
+
ON bunqueue_events(namespace, id);
|
|
225
|
+
CREATE INDEX IF NOT EXISTS bunqueue_events_queue_id_idx
|
|
226
|
+
ON bunqueue_events(namespace, queue, id DESC);
|
|
227
|
+
|
|
228
|
+
CREATE TABLE IF NOT EXISTS bunqueue_event_prune_watermarks (
|
|
229
|
+
namespace TEXT NOT NULL,
|
|
230
|
+
queue TEXT NOT NULL,
|
|
231
|
+
source_event_id BIGINT NOT NULL,
|
|
232
|
+
pruned_through BIGINT NOT NULL,
|
|
233
|
+
PRIMARY KEY (namespace, queue, source_event_id)
|
|
234
|
+
);
|
|
235
|
+
CREATE INDEX IF NOT EXISTS bunqueue_event_prune_watermarks_namespace_queue_idx
|
|
236
|
+
ON bunqueue_event_prune_watermarks(namespace, queue, pruned_through DESC);
|
|
237
|
+
|
|
238
|
+
${POSTGRES_EVENT_JOURNAL_SCHEMA}
|
|
239
|
+
|
|
240
|
+
CREATE TABLE IF NOT EXISTS bunqueue_brokers (
|
|
241
|
+
namespace TEXT NOT NULL,
|
|
242
|
+
broker_id TEXT NOT NULL,
|
|
243
|
+
session_id TEXT,
|
|
244
|
+
started_at BIGINT NOT NULL,
|
|
245
|
+
heartbeat_at BIGINT NOT NULL,
|
|
246
|
+
PRIMARY KEY (namespace, broker_id)
|
|
247
|
+
);
|
|
248
|
+
ALTER TABLE bunqueue_brokers
|
|
249
|
+
ADD COLUMN IF NOT EXISTS session_id TEXT;
|
|
250
|
+
|
|
251
|
+
DROP INDEX IF EXISTS bunqueue_jobs_live_unique_key_idx;
|
|
252
|
+
CREATE UNIQUE INDEX bunqueue_jobs_live_unique_key_idx
|
|
253
|
+
ON bunqueue_jobs(namespace, queue, unique_key)
|
|
254
|
+
WHERE unique_key IS NOT NULL
|
|
255
|
+
AND state IN ('waiting', 'prioritized', 'delayed', 'waiting-children', 'active');
|
|
256
|
+
`;
|
|
257
|
+
export const POSTGRES_EVENT_CHANNEL = 'bunqueue_jobs_changed';
|