bunqueue 2.8.60 → 2.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +114 -78
- package/dist/application/background/dlq.js +2 -0
- package/dist/application/background/recovery/active.d.ts +1 -0
- package/dist/application/background/recovery/active.js +12 -0
- package/dist/application/background/recovery/index.js +3 -1
- package/dist/application/background/recovery/pending.js +1 -0
- package/dist/application/cleanupTasks.js +4 -16
- package/dist/application/contextFactory.js +8 -0
- package/dist/application/dlqManager.d.ts +10 -0
- package/dist/application/dlqManager.js +48 -9
- package/dist/application/dlqRetry.js +60 -38
- package/dist/application/eventsManager.d.ts +9 -2
- package/dist/application/eventsManager.js +106 -55
- package/dist/application/operations/ackHelpers.d.ts +1 -4
- package/dist/application/operations/ackHelpers.js +39 -14
- package/dist/application/operations/flowPush.js +1 -0
- package/dist/application/operations/flowValidation.js +4 -4
- package/dist/application/operations/jobManagement.d.ts +4 -0
- package/dist/application/operations/jobMoveOperations.js +9 -4
- package/dist/application/operations/pull.d.ts +7 -2
- package/dist/application/operations/pull.js +15 -57
- package/dist/application/operations/pullFinalization.d.ts +6 -0
- package/dist/application/operations/pullFinalization.js +69 -0
- package/dist/application/operations/pullStateTransition.d.ts +8 -7
- package/dist/application/operations/pullStateTransition.js +25 -18
- package/dist/application/operations/push.js +3 -0
- package/dist/application/operations/pushBatch.js +14 -7
- package/dist/application/operations/pushContext.d.ts +2 -1
- package/dist/application/postgres-queue-manager/batchSnapshot.d.ts +16 -0
- package/dist/application/postgres-queue-manager/batchSnapshot.js +17 -0
- package/dist/application/postgres-queue-manager/cloud.d.ts +30 -0
- package/dist/application/postgres-queue-manager/cloud.js +55 -0
- package/dist/application/postgres-queue-manager/config.d.ts +5 -0
- package/dist/application/postgres-queue-manager/config.js +1 -0
- package/dist/application/postgres-queue-manager/control.d.ts +30 -0
- package/dist/application/postgres-queue-manager/control.js +210 -0
- package/dist/application/postgres-queue-manager/deferredWrites.d.ts +13 -0
- package/dist/application/postgres-queue-manager/deferredWrites.js +60 -0
- package/dist/application/postgres-queue-manager/delivery.d.ts +20 -0
- package/dist/application/postgres-queue-manager/delivery.js +83 -0
- package/dist/application/postgres-queue-manager/groupQueries.d.ts +10 -0
- package/dist/application/postgres-queue-manager/groupQueries.js +40 -0
- package/dist/application/postgres-queue-manager/lease.d.ts +13 -0
- package/dist/application/postgres-queue-manager/lease.js +88 -0
- package/dist/application/postgres-queue-manager/lifetimeMetrics.d.ts +32 -0
- package/dist/application/postgres-queue-manager/lifetimeMetrics.js +53 -0
- package/dist/application/postgres-queue-manager/maintenance.d.ts +14 -0
- package/dist/application/postgres-queue-manager/maintenance.js +100 -0
- package/dist/application/postgres-queue-manager/operationGate.d.ts +26 -0
- package/dist/application/postgres-queue-manager/operationGate.js +96 -0
- package/dist/application/postgres-queue-manager/operations.d.ts +25 -0
- package/dist/application/postgres-queue-manager/operations.js +194 -0
- package/dist/application/postgres-queue-manager/projectionRefreshes.d.ts +43 -0
- package/dist/application/postgres-queue-manager/projectionRefreshes.js +170 -0
- package/dist/application/postgres-queue-manager/queries.d.ts +142 -0
- package/dist/application/postgres-queue-manager/queries.js +231 -0
- package/dist/application/postgres-queue-manager/relationships.d.ts +10 -0
- package/dist/application/postgres-queue-manager/relationships.js +45 -0
- package/dist/application/postgres-queue-manager/services.d.ts +31 -0
- package/dist/application/postgres-queue-manager/services.js +149 -0
- package/dist/application/postgres-queue-manager/snapshot.d.ts +51 -0
- package/dist/application/postgres-queue-manager/snapshot.js +236 -0
- package/dist/application/postgres-queue-manager/snapshotViews.d.ts +15 -0
- package/dist/application/postgres-queue-manager/snapshotViews.js +100 -0
- package/dist/application/postgres-queue-manager/startupEventBuffer.d.ts +12 -0
- package/dist/application/postgres-queue-manager/startupEventBuffer.js +36 -0
- package/dist/application/postgres-queue-manager/state.d.ts +47 -0
- package/dist/application/postgres-queue-manager/state.js +273 -0
- package/dist/application/postgres-queue-manager/terminalDelivery.d.ts +20 -0
- package/dist/application/postgres-queue-manager/terminalDelivery.js +100 -0
- package/dist/application/postgresQueueManager.d.ts +2 -0
- package/dist/application/postgresQueueManager.js +1 -0
- package/dist/application/queue-manager/control.js +1 -0
- package/dist/application/queue-manager/delivery.d.ts +5 -4
- package/dist/application/queue-manager/delivery.js +15 -8
- package/dist/application/queue-manager/flow-failures.js +4 -0
- package/dist/application/queue-manager/limits.d.ts +13 -0
- package/dist/application/queue-manager/limits.js +65 -0
- package/dist/application/queue-manager/observability.js +15 -3
- package/dist/application/queue-manager/queries.d.ts +7 -0
- package/dist/application/queue-manager/queries.js +9 -0
- package/dist/application/queue-manager/services.d.ts +1 -1
- package/dist/application/queue-manager/services.js +4 -2
- package/dist/application/queue-manager/state.js +17 -2
- package/dist/application/queueTelemetryJournal.d.ts +2 -0
- package/dist/application/queueTelemetryJournal.js +23 -0
- package/dist/application/types/ack.d.ts +3 -1
- package/dist/cli/commandRouter.d.ts +1 -1
- package/dist/cli/commands/server.js +1 -1
- package/dist/client/flowPush.js +6 -0
- package/dist/client/forwarder.d.ts +6 -4
- package/dist/client/forwarder.js +5 -3
- package/dist/client/groupId.d.ts +1 -0
- package/dist/client/groupId.js +1 -0
- package/dist/client/index.d.ts +1 -1
- package/dist/client/jobHelpers.js +1 -0
- package/dist/client/queue/dlq.d.ts +2 -0
- package/dist/client/queue/dlq.js +14 -0
- package/dist/client/queue/dlqOps.d.ts +2 -0
- package/dist/client/queue/dlqOps.js +4 -0
- package/dist/client/queue/helpers.js +2 -0
- package/dist/client/queue/operations/add/bulk.js +4 -4
- package/dist/client/queue/operations/add/payload.d.ts +1 -0
- package/dist/client/queue/operations/add/payload.js +7 -1
- package/dist/client/queue/operations/add/single.js +3 -3
- package/dist/client/queue/operations/groups.d.ts +21 -0
- package/dist/client/queue/operations/groups.js +88 -0
- package/dist/client/queue/runtime/configuration.d.ts +4 -0
- package/dist/client/queue/runtime/configuration.js +8 -0
- package/dist/client/queue/runtime/queries.d.ts +13 -0
- package/dist/client/queue/runtime/queries.js +31 -0
- package/dist/client/queue/runtime/state.d.ts +4 -0
- package/dist/client/queue/runtime/state.js +1 -0
- package/dist/client/types/index.d.ts +2 -2
- package/dist/client/types/options.d.ts +4 -0
- package/dist/client/types/worker.d.ts +8 -0
- package/dist/client/worker/runtime/execution.js +4 -8
- package/dist/client/worker/runtime/options.js +3 -0
- package/dist/client/worker/runtime/polling.d.ts +1 -0
- package/dist/client/worker/runtime/polling.js +9 -0
- package/dist/client/worker/types/options.d.ts +2 -1
- package/dist/client/worker/workerPull.d.ts +2 -0
- package/dist/client/worker/workerPull.js +6 -4
- package/dist/config/resolve.d.ts +14 -0
- package/dist/config/resolve.js +50 -5
- package/dist/config/types.d.ts +18 -0
- package/dist/domain/job/groupFifoOrder.d.ts +9 -0
- package/dist/domain/job/groupFifoOrder.js +26 -0
- package/dist/domain/queue/dlqShard.d.ts +2 -0
- package/dist/domain/queue/dlqShard.js +16 -0
- package/dist/domain/queue/groupLimiterManager.d.ts +23 -0
- package/dist/domain/queue/groupLimiterManager.js +107 -0
- package/dist/domain/queue/groupScheduler.d.ts +31 -0
- package/dist/domain/queue/groupScheduler.js +256 -0
- package/dist/domain/queue/groupSchedulerOrder.d.ts +6 -0
- package/dist/domain/queue/groupSchedulerOrder.js +30 -0
- package/dist/domain/queue/priorityQueue.d.ts +13 -23
- package/dist/domain/queue/priorityQueue.js +53 -42
- package/dist/domain/queue/shard/dlq.d.ts +1 -0
- package/dist/domain/queue/shard/dlq.js +3 -0
- package/dist/domain/queue/shard/keys.d.ts +1 -0
- package/dist/domain/queue/shard/keys.js +25 -2
- package/dist/domain/queue/shard/lifecycle.js +4 -0
- package/dist/domain/queue/shard/limits.d.ts +20 -1
- package/dist/domain/queue/shard/limits.js +56 -0
- package/dist/domain/queue/shard/state.d.ts +5 -0
- package/dist/domain/queue/shard/state.js +9 -1
- package/dist/domain/types/commands/core.d.ts +3 -0
- package/dist/domain/types/commands/dlq.d.ts +5 -0
- package/dist/domain/types/commands/limits.d.ts +41 -0
- package/dist/domain/types/commands/union.d.ts +1 -1
- package/dist/domain/types/cron.d.ts +5 -0
- package/dist/domain/types/cron.js +14 -3
- package/dist/domain/types/dlq.d.ts +1 -1
- package/dist/domain/types/dlq.js +1 -2
- package/dist/domain/types/group.d.ts +21 -0
- package/dist/domain/types/group.js +78 -0
- package/dist/domain/types/jobs/model.d.ts +1 -1
- package/dist/domain/types/priorityQueue.d.ts +1 -0
- package/dist/infrastructure/cloud/commandHandler.js +5 -3
- package/dist/infrastructure/cloud/commands/integrations.js +11 -11
- package/dist/infrastructure/cloud/commands/jobs.js +48 -40
- package/dist/infrastructure/cloud/commands/queues.js +61 -75
- package/dist/infrastructure/cloud/queueAdapter/local.d.ts +41 -0
- package/dist/infrastructure/cloud/queueAdapter/local.js +187 -0
- package/dist/infrastructure/cloud/queueAdapter/postgres.d.ts +42 -0
- package/dist/infrastructure/cloud/queueAdapter/postgres.js +188 -0
- package/dist/infrastructure/cloud/queueAdapter/registry.d.ts +4 -0
- package/dist/infrastructure/cloud/queueAdapter/registry.js +15 -0
- package/dist/infrastructure/cloud/queueAdapter/types.d.ts +87 -0
- package/dist/infrastructure/cloud/queueAdapter/types.js +1 -0
- package/dist/infrastructure/cloud/snapshot/analytics.d.ts +2 -0
- package/dist/infrastructure/cloud/snapshot/analytics.js +7 -0
- package/dist/infrastructure/cloud/snapshot/collector.d.ts +4 -0
- package/dist/infrastructure/cloud/snapshot/collector.js +39 -1
- package/dist/infrastructure/cloud/snapshot/config.d.ts +3 -0
- package/dist/infrastructure/cloud/snapshot/config.js +43 -0
- package/dist/infrastructure/cloud/snapshot/jobs.d.ts +3 -0
- package/dist/infrastructure/cloud/snapshot/jobs.js +32 -1
- package/dist/infrastructure/cloud/snapshotCollector.js +20 -35
- package/dist/infrastructure/cloud/snapshotHelpers.d.ts +3 -3
- package/dist/infrastructure/cloud/snapshotHelpers.js +3 -3
- package/dist/infrastructure/cloud/statsRefresh.js +2 -1
- package/dist/infrastructure/cloud/types/command.d.ts +2 -2
- package/dist/infrastructure/persistence/jobOptionsBlob.d.ts +1 -0
- package/dist/infrastructure/persistence/jobOptionsBlob.js +10 -2
- package/dist/infrastructure/persistence/migrations.js +11 -0
- package/dist/infrastructure/persistence/postgres/admission.d.ts +18 -0
- package/dist/infrastructure/persistence/postgres/admission.js +196 -0
- package/dist/infrastructure/persistence/postgres/admissionParent.d.ts +5 -0
- package/dist/infrastructure/persistence/postgres/admissionParent.js +35 -0
- package/dist/infrastructure/persistence/postgres/admissionResult.d.ts +9 -0
- package/dist/infrastructure/persistence/postgres/admissionResult.js +1 -0
- package/dist/infrastructure/persistence/postgres/admissionStore.d.ts +12 -0
- package/dist/infrastructure/persistence/postgres/admissionStore.js +101 -0
- package/dist/infrastructure/persistence/postgres/advisoryLocks.d.ts +2 -0
- package/dist/infrastructure/persistence/postgres/advisoryLocks.js +13 -0
- package/dist/infrastructure/persistence/postgres/batchAdmission.d.ts +11 -0
- package/dist/infrastructure/persistence/postgres/batchAdmission.js +128 -0
- package/dist/infrastructure/persistence/postgres/batchCompletion.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/batchCompletion.js +174 -0
- package/dist/infrastructure/persistence/postgres/batchEvents.d.ts +14 -0
- package/dist/infrastructure/persistence/postgres/batchEvents.js +72 -0
- package/dist/infrastructure/persistence/postgres/brokerSessions.d.ts +14 -0
- package/dist/infrastructure/persistence/postgres/brokerSessions.js +94 -0
- package/dist/infrastructure/persistence/postgres/claimBatch.d.ts +14 -0
- package/dist/infrastructure/persistence/postgres/claimBatch.js +69 -0
- package/dist/infrastructure/persistence/postgres/claimSelection.d.ts +11 -0
- package/dist/infrastructure/persistence/postgres/claimSelection.js +140 -0
- package/dist/infrastructure/persistence/postgres/claims.d.ts +10 -0
- package/dist/infrastructure/persistence/postgres/claims.js +153 -0
- package/dist/infrastructure/persistence/postgres/cloudReadModel.d.ts +26 -0
- package/dist/infrastructure/persistence/postgres/cloudReadModel.js +123 -0
- package/dist/infrastructure/persistence/postgres/codec.d.ts +30 -0
- package/dist/infrastructure/persistence/postgres/codec.js +82 -0
- package/dist/infrastructure/persistence/postgres/completionEvents.d.ts +11 -0
- package/dist/infrastructure/persistence/postgres/completionEvents.js +21 -0
- package/dist/infrastructure/persistence/postgres/completionLifecycle.d.ts +11 -0
- package/dist/infrastructure/persistence/postgres/completionLifecycle.js +145 -0
- package/dist/infrastructure/persistence/postgres/completionOutcome.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/completionOutcome.js +112 -0
- package/dist/infrastructure/persistence/postgres/completionQueries.d.ts +10 -0
- package/dist/infrastructure/persistence/postgres/completionQueries.js +87 -0
- package/dist/infrastructure/persistence/postgres/connection.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/connection.js +15 -0
- package/dist/infrastructure/persistence/postgres/context.d.ts +28 -0
- package/dist/infrastructure/persistence/postgres/context.js +74 -0
- package/dist/infrastructure/persistence/postgres/control.d.ts +23 -0
- package/dist/infrastructure/persistence/postgres/control.js +105 -0
- package/dist/infrastructure/persistence/postgres/crons.d.ts +18 -0
- package/dist/infrastructure/persistence/postgres/crons.js +205 -0
- package/dist/infrastructure/persistence/postgres/deduplication.d.ts +9 -0
- package/dist/infrastructure/persistence/postgres/deduplication.js +47 -0
- package/dist/infrastructure/persistence/postgres/dependencyDestruction.d.ts +14 -0
- package/dist/infrastructure/persistence/postgres/dependencyDestruction.js +74 -0
- package/dist/infrastructure/persistence/postgres/dependencyPromotion.d.ts +29 -0
- package/dist/infrastructure/persistence/postgres/dependencyPromotion.js +212 -0
- package/dist/infrastructure/persistence/postgres/destructiveMutations.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/destructiveMutations.js +111 -0
- package/dist/infrastructure/persistence/postgres/dlqLifecycle.d.ts +13 -0
- package/dist/infrastructure/persistence/postgres/dlqLifecycle.js +184 -0
- package/dist/infrastructure/persistence/postgres/dlqMaintenance.d.ts +6 -0
- package/dist/infrastructure/persistence/postgres/dlqMaintenance.js +36 -0
- package/dist/infrastructure/persistence/postgres/dlqRetryPlan.d.ts +21 -0
- package/dist/infrastructure/persistence/postgres/dlqRetryPlan.js +92 -0
- package/dist/infrastructure/persistence/postgres/eventCatchupCursors.d.ts +32 -0
- package/dist/infrastructure/persistence/postgres/eventCatchupCursors.js +66 -0
- package/dist/infrastructure/persistence/postgres/eventCommitGc.d.ts +27 -0
- package/dist/infrastructure/persistence/postgres/eventCommitGc.js +62 -0
- package/dist/infrastructure/persistence/postgres/eventJournal.d.ts +18 -0
- package/dist/infrastructure/persistence/postgres/eventJournal.js +76 -0
- package/dist/infrastructure/persistence/postgres/eventJournalSchema.d.ts +5 -0
- package/dist/infrastructure/persistence/postgres/eventJournalSchema.js +267 -0
- package/dist/infrastructure/persistence/postgres/eventPruneWatermarks.d.ts +26 -0
- package/dist/infrastructure/persistence/postgres/eventPruneWatermarks.js +129 -0
- package/dist/infrastructure/persistence/postgres/eventRetention.d.ts +18 -0
- package/dist/infrastructure/persistence/postgres/eventRetention.js +76 -0
- package/dist/infrastructure/persistence/postgres/events.d.ts +45 -0
- package/dist/infrastructure/persistence/postgres/events.js +276 -0
- package/dist/infrastructure/persistence/postgres/expiry.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/expiry.js +74 -0
- package/dist/infrastructure/persistence/postgres/flowFailures.d.ts +17 -0
- package/dist/infrastructure/persistence/postgres/flowFailures.js +199 -0
- package/dist/infrastructure/persistence/postgres/groupClaims.d.ts +15 -0
- package/dist/infrastructure/persistence/postgres/groupClaims.js +95 -0
- package/dist/infrastructure/persistence/postgres/groupSchema.d.ts +2 -0
- package/dist/infrastructure/persistence/postgres/groupSchema.js +155 -0
- package/dist/infrastructure/persistence/postgres/groupSchemaFingerprint.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/groupSchemaFingerprint.js +127 -0
- package/dist/infrastructure/persistence/postgres/groupStateRetention.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/groupStateRetention.js +93 -0
- package/dist/infrastructure/persistence/postgres/groups.d.ts +11 -0
- package/dist/infrastructure/persistence/postgres/groups.js +117 -0
- package/dist/infrastructure/persistence/postgres/jobData.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/jobData.js +32 -0
- package/dist/infrastructure/persistence/postgres/leaseRelease.d.ts +9 -0
- package/dist/infrastructure/persistence/postgres/leaseRelease.js +120 -0
- package/dist/infrastructure/persistence/postgres/leaseRenewal.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/leaseRenewal.js +35 -0
- package/dist/infrastructure/persistence/postgres/lifetimeMetrics.d.ts +15 -0
- package/dist/infrastructure/persistence/postgres/lifetimeMetrics.js +46 -0
- package/dist/infrastructure/persistence/postgres/lifetimeMetricsFinalizer.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/lifetimeMetricsFinalizer.js +13 -0
- package/dist/infrastructure/persistence/postgres/logs.d.ts +6 -0
- package/dist/infrastructure/persistence/postgres/logs.js +70 -0
- package/dist/infrastructure/persistence/postgres/maintenance.d.ts +6 -0
- package/dist/infrastructure/persistence/postgres/maintenance.js +138 -0
- package/dist/infrastructure/persistence/postgres/maintenanceFlights.d.ts +14 -0
- package/dist/infrastructure/persistence/postgres/maintenanceFlights.js +66 -0
- package/dist/infrastructure/persistence/postgres/metricWrites.d.ts +9 -0
- package/dist/infrastructure/persistence/postgres/metricWrites.js +94 -0
- package/dist/infrastructure/persistence/postgres/mutations.d.ts +9 -0
- package/dist/infrastructure/persistence/postgres/mutations.js +171 -0
- package/dist/infrastructure/persistence/postgres/outcomes.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/outcomes.js +194 -0
- package/dist/infrastructure/persistence/postgres/policies.d.ts +11 -0
- package/dist/infrastructure/persistence/postgres/policies.js +18 -0
- package/dist/infrastructure/persistence/postgres/postCommitMaintenance.d.ts +17 -0
- package/dist/infrastructure/persistence/postgres/postCommitMaintenance.js +117 -0
- package/dist/infrastructure/persistence/postgres/promotion.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/promotion.js +43 -0
- package/dist/infrastructure/persistence/postgres/queries.d.ts +22 -0
- package/dist/infrastructure/persistence/postgres/queries.js +152 -0
- package/dist/infrastructure/persistence/postgres/queueDestruction.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/queueDestruction.js +158 -0
- package/dist/infrastructure/persistence/postgres/queueLifecycle.d.ts +10 -0
- package/dist/infrastructure/persistence/postgres/queueLifecycle.js +60 -0
- package/dist/infrastructure/persistence/postgres/rateLimit.d.ts +6 -0
- package/dist/infrastructure/persistence/postgres/rateLimit.js +10 -0
- package/dist/infrastructure/persistence/postgres/readModels.d.ts +32 -0
- package/dist/infrastructure/persistence/postgres/readModels.js +72 -0
- package/dist/infrastructure/persistence/postgres/recovery.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/recovery.js +178 -0
- package/dist/infrastructure/persistence/postgres/relationships.d.ts +9 -0
- package/dist/infrastructure/persistence/postgres/relationships.js +201 -0
- package/dist/infrastructure/persistence/postgres/removeUnprocessedChildren.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/removeUnprocessedChildren.js +241 -0
- package/dist/infrastructure/persistence/postgres/repeatMutations.d.ts +5 -0
- package/dist/infrastructure/persistence/postgres/repeatMutations.js +37 -0
- package/dist/infrastructure/persistence/postgres/runtime.d.ts +45 -0
- package/dist/infrastructure/persistence/postgres/runtime.js +238 -0
- package/dist/infrastructure/persistence/postgres/runtimeConfig.d.ts +21 -0
- package/dist/infrastructure/persistence/postgres/runtimeConfig.js +39 -0
- package/dist/infrastructure/persistence/postgres/schema.d.ts +4 -0
- package/dist/infrastructure/persistence/postgres/schema.js +261 -0
- package/dist/infrastructure/persistence/postgres/schemaInitialization.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/schemaInitialization.js +226 -0
- package/dist/infrastructure/persistence/postgres/serialAdmission.d.ts +6 -0
- package/dist/infrastructure/persistence/postgres/serialAdmission.js +96 -0
- package/dist/infrastructure/persistence/postgres/snapshotBudget.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/snapshotBudget.js +83 -0
- package/dist/infrastructure/persistence/postgres/storageHealth.d.ts +10 -0
- package/dist/infrastructure/persistence/postgres/storageHealth.js +29 -0
- package/dist/infrastructure/persistence/postgres/store.d.ts +100 -0
- package/dist/infrastructure/persistence/postgres/store.js +180 -0
- package/dist/infrastructure/persistence/postgres/telemetry.d.ts +8 -0
- package/dist/infrastructure/persistence/postgres/telemetry.js +100 -0
- package/dist/infrastructure/persistence/postgres/terminal.d.ts +8 -0
- package/dist/infrastructure/persistence/postgres/terminal.js +137 -0
- package/dist/infrastructure/persistence/postgres/transactionRetry.d.ts +6 -0
- package/dist/infrastructure/persistence/postgres/transactionRetry.js +42 -0
- package/dist/infrastructure/persistence/postgres/types.d.ts +176 -0
- package/dist/infrastructure/persistence/postgres/types.js +1 -0
- package/dist/infrastructure/persistence/postgres/workers.d.ts +15 -0
- package/dist/infrastructure/persistence/postgres/workers.js +126 -0
- package/dist/infrastructure/persistence/postgres.d.ts +2 -0
- package/dist/infrastructure/persistence/postgres.js +1 -0
- package/dist/infrastructure/persistence/schema.d.ts +2 -2
- package/dist/infrastructure/persistence/schema.js +12 -1
- package/dist/infrastructure/persistence/sqlite/control.d.ts +14 -0
- package/dist/infrastructure/persistence/sqlite/control.js +69 -0
- package/dist/infrastructure/persistence/sqlite/jobs.d.ts +12 -0
- package/dist/infrastructure/persistence/sqlite/jobs.js +38 -0
- package/dist/infrastructure/persistence/sqlite/telemetry.d.ts +2 -1
- package/dist/infrastructure/persistence/sqlite/telemetry.js +5 -64
- package/dist/infrastructure/persistence/sqlite/telemetryWrites.d.ts +4 -0
- package/dist/infrastructure/persistence/sqlite/telemetryWrites.js +123 -0
- package/dist/infrastructure/persistence/sqliteSerializer.js +2 -0
- package/dist/infrastructure/scheduler/cron/persisted.d.ts +5 -0
- package/dist/infrastructure/scheduler/cron/persisted.js +27 -0
- package/dist/infrastructure/scheduler/cron/runtime.js +5 -10
- package/dist/infrastructure/scheduler/cron/validation.d.ts +3 -0
- package/dist/infrastructure/scheduler/cron/validation.js +11 -0
- package/dist/infrastructure/scheduler/cronParser.d.ts +1 -1
- package/dist/infrastructure/scheduler/cronParser.js +144 -11
- package/dist/infrastructure/server/bootstrap.d.ts +2 -3
- package/dist/infrastructure/server/bootstrap.js +33 -59
- package/dist/infrastructure/server/errors.d.ts +4 -0
- package/dist/infrastructure/server/errors.js +54 -0
- package/dist/infrastructure/server/handler-routes/control.d.ts +5 -5
- package/dist/infrastructure/server/handler-routes/control.js +16 -1
- package/dist/infrastructure/server/handler-routes/monitoring.d.ts +2 -2
- package/dist/infrastructure/server/handler.js +9 -11
- package/dist/infrastructure/server/handlers/advanced/dependencies.js +7 -6
- package/dist/infrastructure/server/handlers/advanced/jobs.js +19 -6
- package/dist/infrastructure/server/handlers/advanced/queue.d.ts +9 -9
- package/dist/infrastructure/server/handlers/advanced/queue.js +91 -27
- package/dist/infrastructure/server/handlers/core.d.ts +0 -5
- package/dist/infrastructure/server/handlers/core.js +47 -49
- package/dist/infrastructure/server/handlers/cron.d.ts +4 -4
- package/dist/infrastructure/server/handlers/cron.js +60 -59
- package/dist/infrastructure/server/handlers/dashboard.d.ts +1 -1
- package/dist/infrastructure/server/handlers/dashboard.js +21 -8
- package/dist/infrastructure/server/handlers/dlq.d.ts +6 -3
- package/dist/infrastructure/server/handlers/dlq.js +35 -9
- package/dist/infrastructure/server/handlers/flow.js +2 -1
- package/dist/infrastructure/server/handlers/groups.d.ts +11 -0
- package/dist/infrastructure/server/handlers/groups.js +55 -0
- package/dist/infrastructure/server/handlers/management.d.ts +5 -5
- package/dist/infrastructure/server/handlers/management.js +36 -9
- package/dist/infrastructure/server/handlers/monitoring/health.d.ts +5 -5
- package/dist/infrastructure/server/handlers/monitoring/health.js +45 -16
- package/dist/infrastructure/server/handlers/monitoring/operations.d.ts +1 -1
- package/dist/infrastructure/server/handlers/monitoring/operations.js +8 -1
- package/dist/infrastructure/server/handlers/monitoring/workers.d.ts +3 -3
- package/dist/infrastructure/server/handlers/monitoring/workers.js +31 -7
- package/dist/infrastructure/server/handlers/pushBatchValidation.d.ts +3 -1
- package/dist/infrastructure/server/handlers/pushBatchValidation.js +56 -8
- package/dist/infrastructure/server/handlers/query.d.ts +1 -1
- package/dist/infrastructure/server/handlers/query.js +5 -4
- package/dist/infrastructure/server/http.js +2 -2
- package/dist/infrastructure/server/httpDashboardEndpoints.d.ts +1 -1
- package/dist/infrastructure/server/httpDashboardEndpoints.js +19 -6
- package/dist/infrastructure/server/httpEndpoints.js +18 -7
- package/dist/infrastructure/server/httpRouteQueues.js +11 -1
- package/dist/infrastructure/server/protocol/validation.d.ts +1 -0
- package/dist/infrastructure/server/protocol/validation.js +1 -0
- package/dist/infrastructure/server/protocol.d.ts +1 -1
- package/dist/infrastructure/server/protocol.js +1 -1
- package/dist/infrastructure/server/shutdownCoordinator.d.ts +24 -0
- package/dist/infrastructure/server/shutdownCoordinator.js +106 -0
- package/dist/infrastructure/server/sseHandler.js +8 -3
- package/dist/infrastructure/server/storageAdapter.d.ts +29 -0
- package/dist/infrastructure/server/storageAdapter.js +61 -0
- package/dist/infrastructure/server/storageManager.d.ts +12 -0
- package/dist/infrastructure/server/storageManager.js +102 -0
- package/dist/infrastructure/server/tcp.js +2 -3
- package/dist/infrastructure/server/ws/snapshots.d.ts +1 -1
- package/dist/infrastructure/server/ws/snapshots.js +19 -4
- package/dist/infrastructure/server/wsHandler.js +10 -5
- package/dist/main.js +1 -1
- package/dist/mcp/backend/embedded/services.js +2 -1
- package/dist/mcp/index.d.ts +2 -2
- package/dist/mcp/index.js +2 -2
- package/dist/shared/storageHealth.d.ts +9 -0
- package/dist/shared/storageHealth.js +12 -0
- package/package.json +75 -65
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import { DEFAULT_DLQ_CONFIG } from '../../../domain/types/dlq';
|
|
2
|
+
import { DEFAULT_STALL_CONFIG } from '../../../domain/types/stall';
|
|
3
|
+
import { claimPostgresCandidates } from './claimBatch';
|
|
4
|
+
import { decodePostgresValue } from './codec';
|
|
5
|
+
import { databaseNow } from './context';
|
|
6
|
+
import { expirePendingPostgresJobs } from './expiry';
|
|
7
|
+
import { repairResolvedPostgresDependencyConsumers } from './dependencyPromotion';
|
|
8
|
+
import { lockPostgresBrokerSession } from './brokerSessions';
|
|
9
|
+
import { runPostgresTransactionWithRetry } from './transactionRetry';
|
|
10
|
+
import { hasClaimablePostgresGroup } from './claimSelection';
|
|
11
|
+
import { ensurePostgresGroupStates, refreshPostgresGroupRateWindows } from './groupClaims';
|
|
12
|
+
function toQueueState(row) {
|
|
13
|
+
return {
|
|
14
|
+
queue: row.queue,
|
|
15
|
+
paused: row.paused,
|
|
16
|
+
rateLimit: row.rate_limit,
|
|
17
|
+
rateDurationMs: row.rate_duration_ms === null ? null : Number(row.rate_duration_ms),
|
|
18
|
+
rateWindowStartedAt: row.rate_window_started_at === null ? null : Number(row.rate_window_started_at),
|
|
19
|
+
rateExpiresAt: row.rate_expires_at === null ? null : Number(row.rate_expires_at),
|
|
20
|
+
rateCount: row.rate_count,
|
|
21
|
+
concurrencyLimit: row.concurrency_limit,
|
|
22
|
+
stallConfig: decodePostgresValue(row.stall_config, DEFAULT_STALL_CONFIG, 'stallConfig'),
|
|
23
|
+
dlqConfig: decodePostgresValue(row.dlq_config, DEFAULT_DLQ_CONFIG, 'dlqConfig'),
|
|
24
|
+
groupSequence: Number(row.group_sequence),
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
async function lockQueueState(tx, ctx, queue, exclusive) {
|
|
28
|
+
const select = async () => exclusive
|
|
29
|
+
? await tx `
|
|
30
|
+
SELECT queue, paused, rate_limit, rate_duration_ms, rate_window_started_at,
|
|
31
|
+
rate_expires_at, rate_count, concurrency_limit, stall_config, dlq_config,
|
|
32
|
+
group_sequence
|
|
33
|
+
FROM bunqueue_queue_state
|
|
34
|
+
WHERE namespace = ${ctx.config.namespace} AND queue = ${queue}
|
|
35
|
+
FOR UPDATE
|
|
36
|
+
`
|
|
37
|
+
: await tx `
|
|
38
|
+
SELECT queue, paused, rate_limit, rate_duration_ms, rate_window_started_at,
|
|
39
|
+
rate_expires_at, rate_count, concurrency_limit, stall_config, dlq_config,
|
|
40
|
+
group_sequence
|
|
41
|
+
FROM bunqueue_queue_state
|
|
42
|
+
WHERE namespace = ${ctx.config.namespace} AND queue = ${queue}
|
|
43
|
+
FOR SHARE
|
|
44
|
+
`;
|
|
45
|
+
let rows = await select();
|
|
46
|
+
if (rows.length === 0) {
|
|
47
|
+
await tx `
|
|
48
|
+
INSERT INTO bunqueue_queue_state (namespace, queue)
|
|
49
|
+
VALUES (${ctx.config.namespace}, ${queue})
|
|
50
|
+
ON CONFLICT DO NOTHING
|
|
51
|
+
`;
|
|
52
|
+
rows = await select();
|
|
53
|
+
}
|
|
54
|
+
const [row] = rows;
|
|
55
|
+
if (!row)
|
|
56
|
+
throw new Error(`Missing PostgreSQL queue state for ${queue}`);
|
|
57
|
+
return toQueueState(row);
|
|
58
|
+
}
|
|
59
|
+
async function refreshRateWindow(tx, ctx, state, now) {
|
|
60
|
+
if (state.rateLimit === null)
|
|
61
|
+
return state;
|
|
62
|
+
if (state.rateExpiresAt !== null && state.rateExpiresAt <= now) {
|
|
63
|
+
await tx `
|
|
64
|
+
UPDATE bunqueue_queue_state
|
|
65
|
+
SET rate_limit = NULL, rate_duration_ms = NULL, rate_window_started_at = NULL,
|
|
66
|
+
rate_expires_at = NULL, rate_count = 0
|
|
67
|
+
WHERE namespace = ${ctx.config.namespace} AND queue = ${state.queue}
|
|
68
|
+
`;
|
|
69
|
+
return {
|
|
70
|
+
...state,
|
|
71
|
+
rateLimit: null,
|
|
72
|
+
rateDurationMs: null,
|
|
73
|
+
rateWindowStartedAt: null,
|
|
74
|
+
rateExpiresAt: null,
|
|
75
|
+
rateCount: 0,
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
const duration = state.rateDurationMs ?? 1000;
|
|
79
|
+
if (state.rateWindowStartedAt !== null && now < state.rateWindowStartedAt + duration) {
|
|
80
|
+
return state;
|
|
81
|
+
}
|
|
82
|
+
await tx `
|
|
83
|
+
UPDATE bunqueue_queue_state
|
|
84
|
+
SET rate_window_started_at = ${now}, rate_count = 0
|
|
85
|
+
WHERE namespace = ${ctx.config.namespace} AND queue = ${state.queue}
|
|
86
|
+
`;
|
|
87
|
+
return { ...state, rateWindowStartedAt: now, rateCount: 0 };
|
|
88
|
+
}
|
|
89
|
+
async function activeCount(tx, ctx, queue, now) {
|
|
90
|
+
const [row] = await tx `
|
|
91
|
+
SELECT COUNT(*)::bigint AS count
|
|
92
|
+
FROM bunqueue_jobs
|
|
93
|
+
WHERE namespace = ${ctx.config.namespace}
|
|
94
|
+
AND queue = ${queue}
|
|
95
|
+
AND state = 'active'
|
|
96
|
+
AND lease_until > ${now}
|
|
97
|
+
`;
|
|
98
|
+
return Number(row.count);
|
|
99
|
+
}
|
|
100
|
+
export async function claimPostgresJobs(ctx, input) {
|
|
101
|
+
const { queue, count, owner, leaseDurationMs, groupOptions } = input;
|
|
102
|
+
if (count <= 0)
|
|
103
|
+
return [];
|
|
104
|
+
await expirePendingPostgresJobs(ctx, queue);
|
|
105
|
+
const claimWithLock = async (exclusive) => await runPostgresTransactionWithRetry(ctx, async (tx) => {
|
|
106
|
+
await lockPostgresBrokerSession(tx, ctx);
|
|
107
|
+
const now = await databaseNow(tx);
|
|
108
|
+
let state = await lockQueueState(tx, ctx, queue, exclusive);
|
|
109
|
+
if (!exclusive && (state.rateLimit !== null || state.concurrencyLimit !== null)) {
|
|
110
|
+
return { claimed: [], retryExclusive: true };
|
|
111
|
+
}
|
|
112
|
+
if (exclusive)
|
|
113
|
+
state = await refreshRateWindow(tx, ctx, state, now);
|
|
114
|
+
if (state.paused)
|
|
115
|
+
return { claimed: [], retryExclusive: false };
|
|
116
|
+
const active = state.concurrencyLimit === null ? 0 : await activeCount(tx, ctx, queue, now);
|
|
117
|
+
const concurrencyCapacity = state.concurrencyLimit === null ? count : Math.max(0, state.concurrencyLimit - active);
|
|
118
|
+
const rateCapacity = state.rateLimit === null ? count : Math.max(0, state.rateLimit - state.rateCount);
|
|
119
|
+
const capacity = Math.min(count, concurrencyCapacity, rateCapacity);
|
|
120
|
+
if (capacity <= 0)
|
|
121
|
+
return { claimed: [], retryExclusive: false };
|
|
122
|
+
await repairResolvedPostgresDependencyConsumers(tx, ctx, queue, now);
|
|
123
|
+
const hasGroups = await hasClaimablePostgresGroup(tx, ctx, queue, now);
|
|
124
|
+
if (!exclusive && hasGroups)
|
|
125
|
+
return { claimed: [], retryExclusive: true };
|
|
126
|
+
let groupSequence = state.groupSequence;
|
|
127
|
+
if (hasGroups) {
|
|
128
|
+
groupSequence = await ensurePostgresGroupStates(tx, ctx, queue, groupSequence);
|
|
129
|
+
await refreshPostgresGroupRateWindows(tx, ctx, queue, groupOptions, now);
|
|
130
|
+
}
|
|
131
|
+
const claimed = await claimPostgresCandidates(tx, ctx, {
|
|
132
|
+
queue,
|
|
133
|
+
capacity,
|
|
134
|
+
owner,
|
|
135
|
+
requestedLeaseMs: leaseDurationMs,
|
|
136
|
+
now,
|
|
137
|
+
groupOptions,
|
|
138
|
+
groupSequence,
|
|
139
|
+
});
|
|
140
|
+
if (state.rateLimit !== null && claimed.length > 0) {
|
|
141
|
+
await tx `
|
|
142
|
+
UPDATE bunqueue_queue_state
|
|
143
|
+
SET rate_count = rate_count + ${claimed.length}
|
|
144
|
+
WHERE namespace = ${ctx.config.namespace} AND queue = ${queue}
|
|
145
|
+
`;
|
|
146
|
+
}
|
|
147
|
+
return { claimed, retryExclusive: false };
|
|
148
|
+
});
|
|
149
|
+
const sharedAttempt = await claimWithLock(false);
|
|
150
|
+
if (!sharedAttempt.retryExclusive)
|
|
151
|
+
return sharedAttempt.claimed;
|
|
152
|
+
return (await claimWithLock(true)).claimed;
|
|
153
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { CronJob } from '../../../domain/types/cron';
|
|
2
|
+
import type { JobLogEntry, Worker } from '../../../domain/types/worker';
|
|
3
|
+
import type { PostgresContext } from './context';
|
|
4
|
+
import type { PostgresCompletionResult, PostgresJobState, PostgresQueueState, PostgresStoredJob } from './types';
|
|
5
|
+
export interface PostgresCloudQueueCount {
|
|
6
|
+
readonly queue: string;
|
|
7
|
+
readonly state: PostgresJobState;
|
|
8
|
+
readonly count: number;
|
|
9
|
+
}
|
|
10
|
+
export interface PostgresCloudQueueTotal {
|
|
11
|
+
readonly queue: string;
|
|
12
|
+
readonly metricType: 'completed' | 'failed';
|
|
13
|
+
readonly count: bigint;
|
|
14
|
+
}
|
|
15
|
+
export interface PostgresCloudReadModel {
|
|
16
|
+
readonly jobs: readonly PostgresStoredJob[];
|
|
17
|
+
readonly queueStates: readonly PostgresQueueState[];
|
|
18
|
+
readonly counts: readonly PostgresCloudQueueCount[];
|
|
19
|
+
readonly totals: readonly PostgresCloudQueueTotal[];
|
|
20
|
+
readonly results: readonly PostgresCompletionResult[];
|
|
21
|
+
readonly logs: ReadonlyMap<string, readonly JobLogEntry[]>;
|
|
22
|
+
readonly workers: readonly Worker[];
|
|
23
|
+
readonly crons: readonly CronJob[];
|
|
24
|
+
}
|
|
25
|
+
/** Read one coherent, bounded Cloud view from the durable PostgreSQL source. */
|
|
26
|
+
export declare function loadPostgresCloudReadModel(ctx: PostgresContext): Promise<PostgresCloudReadModel>;
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import { decodePostgresJob, decodePostgresValue } from './codec';
|
|
2
|
+
import { decodePostgresQueueState } from './control';
|
|
3
|
+
import { decodePostgresCron } from './crons';
|
|
4
|
+
import { jobId } from '../../../domain/types/job';
|
|
5
|
+
/** Read one coherent, bounded Cloud view from the durable PostgreSQL source. */
|
|
6
|
+
export async function loadPostgresCloudReadModel(ctx) {
|
|
7
|
+
return await ctx.sql.begin('isolation level repeatable read read only', async (tx) => {
|
|
8
|
+
const [jobRows, stateRows, countRows, totalRows, resultRows, workerRows, cronRows] = await Promise.all([
|
|
9
|
+
tx `
|
|
10
|
+
WITH ranked_live AS MATERIALIZED (
|
|
11
|
+
SELECT job.*, row_number() OVER (
|
|
12
|
+
PARTITION BY queue, state ORDER BY created_at, id
|
|
13
|
+
) AS cloud_rank
|
|
14
|
+
FROM bunqueue_jobs AS job
|
|
15
|
+
WHERE namespace = ${ctx.config.namespace} AND state <> 'completed'
|
|
16
|
+
), retained_completed AS MATERIALIZED (
|
|
17
|
+
SELECT job.*, 1::bigint AS cloud_rank
|
|
18
|
+
FROM bunqueue_jobs AS job
|
|
19
|
+
WHERE namespace = ${ctx.config.namespace} AND state = 'completed'
|
|
20
|
+
ORDER BY completed_at DESC NULLS LAST, id DESC
|
|
21
|
+
LIMIT ${ctx.config.maxCompletedJobs}
|
|
22
|
+
)
|
|
23
|
+
SELECT cloud_jobs.*
|
|
24
|
+
FROM (
|
|
25
|
+
SELECT * FROM ranked_live WHERE cloud_rank <= 1000
|
|
26
|
+
UNION ALL SELECT * FROM retained_completed
|
|
27
|
+
) AS cloud_jobs
|
|
28
|
+
ORDER BY created_at, id
|
|
29
|
+
`,
|
|
30
|
+
tx `
|
|
31
|
+
SELECT queue, paused, rate_limit, rate_duration_ms, rate_window_started_at,
|
|
32
|
+
rate_expires_at, rate_count, concurrency_limit, stall_config, dlq_config
|
|
33
|
+
FROM bunqueue_queue_state
|
|
34
|
+
WHERE namespace = ${ctx.config.namespace}
|
|
35
|
+
ORDER BY queue
|
|
36
|
+
`,
|
|
37
|
+
tx `
|
|
38
|
+
SELECT queue, state, COUNT(*)::bigint AS count
|
|
39
|
+
FROM bunqueue_jobs
|
|
40
|
+
WHERE namespace = ${ctx.config.namespace}
|
|
41
|
+
GROUP BY queue, state ORDER BY queue, state
|
|
42
|
+
`,
|
|
43
|
+
tx `
|
|
44
|
+
SELECT queue, metric_type, total_count
|
|
45
|
+
FROM bunqueue_metric_totals
|
|
46
|
+
WHERE namespace = ${ctx.config.namespace}
|
|
47
|
+
ORDER BY queue, metric_type
|
|
48
|
+
`,
|
|
49
|
+
tx `
|
|
50
|
+
SELECT job_id, queue, result
|
|
51
|
+
FROM bunqueue_completions
|
|
52
|
+
WHERE namespace = ${ctx.config.namespace}
|
|
53
|
+
ORDER BY completed_at DESC, job_id DESC
|
|
54
|
+
LIMIT ${ctx.config.maxJobResults}
|
|
55
|
+
`,
|
|
56
|
+
tx `
|
|
57
|
+
SELECT id, payload, last_seen FROM bunqueue_workers
|
|
58
|
+
WHERE namespace = ${ctx.config.namespace}
|
|
59
|
+
ORDER BY id
|
|
60
|
+
`,
|
|
61
|
+
tx `
|
|
62
|
+
SELECT name, payload, next_run, executions, max_limit
|
|
63
|
+
FROM bunqueue_crons
|
|
64
|
+
WHERE namespace = ${ctx.config.namespace}
|
|
65
|
+
ORDER BY name
|
|
66
|
+
`,
|
|
67
|
+
]);
|
|
68
|
+
const ids = jobRows.map((row) => row.id);
|
|
69
|
+
const logRows = ids.length === 0
|
|
70
|
+
? []
|
|
71
|
+
: await tx `
|
|
72
|
+
SELECT job_id, timestamp, level, message
|
|
73
|
+
FROM bunqueue_job_logs
|
|
74
|
+
WHERE namespace = ${ctx.config.namespace}
|
|
75
|
+
AND job_id = ANY(${tx.array(ids, 'TEXT')})
|
|
76
|
+
ORDER BY job_id, id
|
|
77
|
+
`;
|
|
78
|
+
return decodeCloudReadModel({
|
|
79
|
+
jobRows,
|
|
80
|
+
stateRows,
|
|
81
|
+
countRows,
|
|
82
|
+
totalRows,
|
|
83
|
+
resultRows,
|
|
84
|
+
logRows,
|
|
85
|
+
workerRows,
|
|
86
|
+
cronRows,
|
|
87
|
+
});
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
function decodeCloudReadModel(rows) {
|
|
91
|
+
const { jobRows, stateRows, countRows, totalRows, resultRows, logRows, workerRows, cronRows } = rows;
|
|
92
|
+
const logs = new Map();
|
|
93
|
+
for (const { job_id, timestamp, level, message } of logRows) {
|
|
94
|
+
const entries = logs.get(job_id) ?? [];
|
|
95
|
+
entries.push({ timestamp: Number(timestamp), level, message });
|
|
96
|
+
logs.set(job_id, entries);
|
|
97
|
+
}
|
|
98
|
+
return {
|
|
99
|
+
jobs: jobRows.map(decodePostgresJob),
|
|
100
|
+
queueStates: stateRows.map((row) => decodePostgresQueueState(row.queue, row)),
|
|
101
|
+
counts: countRows.map((row) => ({ ...row, count: Number(row.count) })),
|
|
102
|
+
totals: totalRows.map((row) => ({
|
|
103
|
+
queue: row.queue,
|
|
104
|
+
metricType: row.metric_type,
|
|
105
|
+
count: BigInt(row.total_count),
|
|
106
|
+
})),
|
|
107
|
+
results: resultRows.map((row) => ({
|
|
108
|
+
jobId: jobId(row.job_id),
|
|
109
|
+
queue: row.queue,
|
|
110
|
+
result: decodePostgresValue(row.result, null, `postgresCloudResult:${row.job_id}`),
|
|
111
|
+
pinned: false,
|
|
112
|
+
})),
|
|
113
|
+
logs,
|
|
114
|
+
workers: workerRows.map((row) => {
|
|
115
|
+
const worker = decodePostgresValue(row.payload, null, `postgresCloudWorker:${row.id}`);
|
|
116
|
+
if (!worker)
|
|
117
|
+
throw new Error(`Corrupt PostgreSQL worker payload for ${row.id}`);
|
|
118
|
+
worker.lastSeen = Number(row.last_seen);
|
|
119
|
+
return worker;
|
|
120
|
+
}),
|
|
121
|
+
crons: cronRows.map(decodePostgresCron),
|
|
122
|
+
};
|
|
123
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { type DlqEntry, type DlqRetryState } from '../../../domain/types/dlq';
|
|
2
|
+
import type { Job, JobId } from '../../../domain/types/job';
|
|
3
|
+
import type { PostgresJobRow, PostgresStoredJob, PostgresStoreEvent, PostgresJobState } from './types';
|
|
4
|
+
export declare function numeric(value: number | string | bigint | null): number | null;
|
|
5
|
+
export declare function encodePostgresValue(value: unknown): Uint8Array;
|
|
6
|
+
export declare function postgresByteaBase64(value: Uint8Array): string;
|
|
7
|
+
export declare function decodePostgresValue<T>(value: Uint8Array | null, fallback: T, context: string): T;
|
|
8
|
+
export declare function decodePostgresJob(row: PostgresJobRow): PostgresStoredJob;
|
|
9
|
+
interface EventPayload {
|
|
10
|
+
job?: Job;
|
|
11
|
+
invalidationQueue?: string;
|
|
12
|
+
state?: PostgresJobState;
|
|
13
|
+
result?: unknown;
|
|
14
|
+
error?: string;
|
|
15
|
+
removed?: boolean;
|
|
16
|
+
dlqEntry?: DlqEntry | null;
|
|
17
|
+
dlqRetryState?: DlqRetryState | null;
|
|
18
|
+
}
|
|
19
|
+
export declare function decodePostgresEvent(row: {
|
|
20
|
+
id: number | string | bigint;
|
|
21
|
+
queue: string;
|
|
22
|
+
event_type: string;
|
|
23
|
+
job_id: string;
|
|
24
|
+
occurred_at: number | string | bigint;
|
|
25
|
+
payload: Uint8Array | null;
|
|
26
|
+
}): PostgresStoreEvent;
|
|
27
|
+
export declare function postgresStateForJob(job: Job, dependenciesSatisfied: boolean, now?: number): Extract<PostgresJobState, 'waiting' | 'prioritized' | 'delayed' | 'waiting-children'>;
|
|
28
|
+
export declare function eventPayload(input: EventPayload): Uint8Array;
|
|
29
|
+
export declare function asJobId(value: string): JobId;
|
|
30
|
+
export {};
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { setDlqRetryState } from '../../../domain/types/dlq';
|
|
2
|
+
import { Buffer } from 'node:buffer';
|
|
3
|
+
import { jobId } from '../../../domain/types/job';
|
|
4
|
+
import { pack, unpack } from '../sqliteSerializer';
|
|
5
|
+
export function numeric(value) {
|
|
6
|
+
if (value === null)
|
|
7
|
+
return null;
|
|
8
|
+
const converted = Number(value);
|
|
9
|
+
if (!Number.isSafeInteger(converted)) {
|
|
10
|
+
throw new Error(`PostgreSQL integer is outside the JavaScript safe range: ${String(value)}`);
|
|
11
|
+
}
|
|
12
|
+
return converted;
|
|
13
|
+
}
|
|
14
|
+
export function encodePostgresValue(value) {
|
|
15
|
+
return pack(value);
|
|
16
|
+
}
|
|
17
|
+
export function postgresByteaBase64(value) {
|
|
18
|
+
return Buffer.from(value).toString('base64');
|
|
19
|
+
}
|
|
20
|
+
export function decodePostgresValue(value, fallback, context) {
|
|
21
|
+
return value ? unpack(value, fallback, context) : fallback;
|
|
22
|
+
}
|
|
23
|
+
export function decodePostgresJob(row) {
|
|
24
|
+
const job = decodePostgresValue(row.payload, null, `postgresJob:${row.id}`);
|
|
25
|
+
if (!job)
|
|
26
|
+
throw new Error(`Corrupt PostgreSQL payload for job ${row.id}`);
|
|
27
|
+
const dlqRetryState = decodePostgresValue(row.dlq_retry_state, null, `postgresDlqRetry:${row.id}`);
|
|
28
|
+
const normalizedJob = { ...job, id: jobId(String(job.id)) };
|
|
29
|
+
setDlqRetryState(normalizedJob, dlqRetryState);
|
|
30
|
+
return {
|
|
31
|
+
job: normalizedJob,
|
|
32
|
+
state: row.state,
|
|
33
|
+
result: decodePostgresValue(row.result, null, `postgresResult:${row.id}`),
|
|
34
|
+
error: row.error,
|
|
35
|
+
dlqEntry: decodePostgresValue(row.dlq_entry, null, `postgresDlq:${row.id}`),
|
|
36
|
+
dlqRetryState,
|
|
37
|
+
token: row.lease_token,
|
|
38
|
+
leaseOwner: row.lease_owner,
|
|
39
|
+
leaseBrokerId: row.lease_broker_id,
|
|
40
|
+
leaseUntil: numeric(row.lease_until),
|
|
41
|
+
leaseRenewals: row.lease_renewals,
|
|
42
|
+
version: numeric(row.version) ?? 0,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
export function decodePostgresEvent(row) {
|
|
46
|
+
const payload = decodePostgresValue(row.payload, {}, `postgresEvent:${row.id}`);
|
|
47
|
+
const dlqEntry = payload.dlqEntry
|
|
48
|
+
? {
|
|
49
|
+
...payload.dlqEntry,
|
|
50
|
+
job: { ...payload.dlqEntry.job, id: jobId(String(payload.dlqEntry.job.id)) },
|
|
51
|
+
}
|
|
52
|
+
: payload.dlqEntry;
|
|
53
|
+
return {
|
|
54
|
+
id: numeric(row.id) ?? 0,
|
|
55
|
+
queue: payload.invalidationQueue ?? row.queue,
|
|
56
|
+
type: row.event_type,
|
|
57
|
+
jobId: jobId(row.job_id),
|
|
58
|
+
occurredAt: numeric(row.occurred_at) ?? 0,
|
|
59
|
+
job: payload.job ? { ...payload.job, id: jobId(String(payload.job.id)) } : null,
|
|
60
|
+
state: payload.state ?? null,
|
|
61
|
+
...(Object.hasOwn(payload, 'result') && { result: payload.result }),
|
|
62
|
+
...(payload.error !== undefined && { error: payload.error }),
|
|
63
|
+
...(payload.removed !== undefined && { removed: payload.removed }),
|
|
64
|
+
...(Object.hasOwn(payload, 'dlqEntry') && { dlqEntry }),
|
|
65
|
+
...(Object.hasOwn(payload, 'dlqRetryState') && {
|
|
66
|
+
dlqRetryState: payload.dlqRetryState,
|
|
67
|
+
}),
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
export function postgresStateForJob(job, dependenciesSatisfied, now = Date.now()) {
|
|
71
|
+
if (!dependenciesSatisfied)
|
|
72
|
+
return 'waiting-children';
|
|
73
|
+
if (job.runAt > now)
|
|
74
|
+
return 'delayed';
|
|
75
|
+
return job.priority > 0 ? 'prioritized' : 'waiting';
|
|
76
|
+
}
|
|
77
|
+
export function eventPayload(input) {
|
|
78
|
+
return encodePostgresValue(input);
|
|
79
|
+
}
|
|
80
|
+
export function asJobId(value) {
|
|
81
|
+
return jobId(value);
|
|
82
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { TransactionSQL } from 'bun';
|
|
2
|
+
import type { Job } from '../../../domain/types/job';
|
|
3
|
+
import type { PostgresContext } from './context';
|
|
4
|
+
interface CompletionEventInput {
|
|
5
|
+
readonly job: Job;
|
|
6
|
+
readonly result: unknown;
|
|
7
|
+
readonly removed: boolean;
|
|
8
|
+
}
|
|
9
|
+
/** Persist one completion event per job while amortizing queue-level bookkeeping. */
|
|
10
|
+
export declare function recordPostgresCompletionEvents(tx: TransactionSQL, ctx: PostgresContext, inputs: readonly CompletionEventInput[], now: number): Promise<void>;
|
|
11
|
+
export {};
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { recordPostgresJobEvents } from './batchEvents';
|
|
2
|
+
import { recordPostgresMetricAdditions } from './metricWrites';
|
|
3
|
+
async function updateMetrics(tx, ctx, inputs, now) {
|
|
4
|
+
const counts = new Map();
|
|
5
|
+
for (const input of inputs)
|
|
6
|
+
counts.set(input.job.queue, (counts.get(input.job.queue) ?? 0) + 1);
|
|
7
|
+
await recordPostgresMetricAdditions(tx, ctx, 'completed', [...counts].map(([queue, count]) => ({ queue, count })), now);
|
|
8
|
+
}
|
|
9
|
+
/** Persist one completion event per job while amortizing queue-level bookkeeping. */
|
|
10
|
+
export async function recordPostgresCompletionEvents(tx, ctx, inputs, now) {
|
|
11
|
+
if (inputs.length === 0)
|
|
12
|
+
return;
|
|
13
|
+
await recordPostgresJobEvents(tx, ctx, inputs.map((input) => ({
|
|
14
|
+
job: input.job,
|
|
15
|
+
type: 'completed',
|
|
16
|
+
state: 'completed',
|
|
17
|
+
result: input.result,
|
|
18
|
+
removed: input.removed,
|
|
19
|
+
})), now);
|
|
20
|
+
await updateMetrics(tx, ctx, inputs, now);
|
|
21
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { TransactionSQL } from 'bun';
|
|
2
|
+
import { type JobId } from '../../../domain/types/job';
|
|
3
|
+
import { type PostgresContext } from './context';
|
|
4
|
+
/** Reject retirement while any non-terminal consumer still owns the old generation. */
|
|
5
|
+
export declare function assertNoLivePostgresCompletionConsumers(tx: TransactionSQL, ctx: PostgresContext, ids: readonly JobId[], deletingConsumerIds?: readonly JobId[]): Promise<void>;
|
|
6
|
+
/** Remove completion evidence for IDs that are about to start a new generation. */
|
|
7
|
+
export declare function retirePostgresCompletionGenerations(tx: TransactionSQL, ctx: PostgresContext, ids: readonly JobId[], deletingConsumerIds?: readonly JobId[]): Promise<string[]>;
|
|
8
|
+
/** Delete queue-owned completion-only rows inside an already serialized obliterate. */
|
|
9
|
+
export declare function deletePostgresQueueCompletions(tx: TransactionSQL, ctx: PostgresContext, queue: string): Promise<JobId[]>;
|
|
10
|
+
/** Bound unreferenced removeOnComplete tombstones without pruning live dependency evidence. */
|
|
11
|
+
export declare function prunePostgresCompletionTombstones(ctx: PostgresContext): Promise<number>;
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { jobId } from '../../../domain/types/job';
|
|
2
|
+
import { postgresAdvisoryLockName } from './advisoryLocks';
|
|
3
|
+
import { databaseNow, recordPostgresEvent } from './context';
|
|
4
|
+
import { lockPostgresDependencyCompletions } from './dependencyPromotion';
|
|
5
|
+
const LIVE_CONSUMER_STATES = [
|
|
6
|
+
'waiting',
|
|
7
|
+
'prioritized',
|
|
8
|
+
'delayed',
|
|
9
|
+
'waiting-children',
|
|
10
|
+
'active',
|
|
11
|
+
];
|
|
12
|
+
const COMPLETION_PRUNE_BATCH_SIZE = 1000;
|
|
13
|
+
function uniqueIds(ids) {
|
|
14
|
+
return [...new Set(ids.map(String))].sort();
|
|
15
|
+
}
|
|
16
|
+
/** Reject retirement while any non-terminal consumer still owns the old generation. */
|
|
17
|
+
export async function assertNoLivePostgresCompletionConsumers(tx, ctx, ids, deletingConsumerIds = []) {
|
|
18
|
+
const candidates = uniqueIds(ids);
|
|
19
|
+
if (candidates.length === 0)
|
|
20
|
+
return;
|
|
21
|
+
const deleting = uniqueIds(deletingConsumerIds);
|
|
22
|
+
const rows = await tx `
|
|
23
|
+
SELECT dependency.dependency_id
|
|
24
|
+
FROM bunqueue_dependencies AS dependency
|
|
25
|
+
JOIN bunqueue_jobs AS consumer
|
|
26
|
+
ON consumer.namespace = dependency.namespace AND consumer.id = dependency.job_id
|
|
27
|
+
WHERE dependency.namespace = ${ctx.config.namespace}
|
|
28
|
+
AND dependency.dependency_id = ANY(${tx.array(candidates, 'TEXT')})
|
|
29
|
+
AND consumer.state = ANY(${tx.array([...LIVE_CONSUMER_STATES], 'TEXT')})
|
|
30
|
+
AND (
|
|
31
|
+
${deleting.length} = 0
|
|
32
|
+
OR consumer.id <> ALL(${tx.array(deleting, 'TEXT')})
|
|
33
|
+
)
|
|
34
|
+
ORDER BY dependency.dependency_id, consumer.id
|
|
35
|
+
LIMIT 1
|
|
36
|
+
`;
|
|
37
|
+
if (rows[0]) {
|
|
38
|
+
throw new Error(`Custom ID ${rows[0].dependency_id} still has unresolved dependency consumers`);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
/** Remove completion evidence for IDs that are about to start a new generation. */
|
|
42
|
+
export async function retirePostgresCompletionGenerations(tx, ctx, ids, deletingConsumerIds = []) {
|
|
43
|
+
const candidates = uniqueIds(ids);
|
|
44
|
+
if (candidates.length === 0)
|
|
45
|
+
return [];
|
|
46
|
+
const existing = await tx `
|
|
47
|
+
SELECT job_id FROM bunqueue_completions
|
|
48
|
+
WHERE namespace = ${ctx.config.namespace}
|
|
49
|
+
AND job_id = ANY(${tx.array(candidates, 'TEXT')})
|
|
50
|
+
ORDER BY job_id
|
|
51
|
+
`;
|
|
52
|
+
if (existing.length === 0)
|
|
53
|
+
return [];
|
|
54
|
+
await assertNoLivePostgresCompletionConsumers(tx, ctx, existing.map((row) => jobId(row.job_id)), deletingConsumerIds);
|
|
55
|
+
const rows = await tx `
|
|
56
|
+
DELETE FROM bunqueue_completions
|
|
57
|
+
WHERE namespace = ${ctx.config.namespace}
|
|
58
|
+
AND job_id = ANY(${tx.array(existing.map((row) => row.job_id), 'TEXT')})
|
|
59
|
+
RETURNING job_id
|
|
60
|
+
`;
|
|
61
|
+
return rows.map((row) => row.job_id);
|
|
62
|
+
}
|
|
63
|
+
/** Delete queue-owned completion-only rows inside an already serialized obliterate. */
|
|
64
|
+
export async function deletePostgresQueueCompletions(tx, ctx, queue) {
|
|
65
|
+
const rows = await tx `
|
|
66
|
+
DELETE FROM bunqueue_completions
|
|
67
|
+
WHERE namespace = ${ctx.config.namespace} AND queue = ${queue}
|
|
68
|
+
RETURNING job_id
|
|
69
|
+
`;
|
|
70
|
+
return rows.map((row) => jobId(row.job_id));
|
|
71
|
+
}
|
|
72
|
+
async function prunePostgresCompletionTombstoneBatch(ctx) {
|
|
73
|
+
const lockName = postgresAdvisoryLockName('completion-retention', ctx.config.namespace);
|
|
74
|
+
return await ctx.sql.begin(async (tx) => {
|
|
75
|
+
await tx `
|
|
76
|
+
SELECT pg_advisory_xact_lock(hashtextextended(${lockName}, 0))
|
|
77
|
+
`;
|
|
78
|
+
const candidates = await tx `
|
|
79
|
+
SELECT completion.job_id
|
|
80
|
+
FROM bunqueue_completions AS completion
|
|
81
|
+
WHERE completion.namespace = ${ctx.config.namespace}
|
|
82
|
+
AND NOT EXISTS (
|
|
83
|
+
SELECT 1 FROM bunqueue_jobs AS retained
|
|
84
|
+
WHERE retained.namespace = completion.namespace
|
|
85
|
+
AND retained.id = completion.job_id
|
|
86
|
+
)
|
|
87
|
+
AND NOT EXISTS (
|
|
88
|
+
SELECT 1
|
|
89
|
+
FROM bunqueue_dependencies AS dependency
|
|
90
|
+
JOIN bunqueue_jobs AS consumer
|
|
91
|
+
ON consumer.namespace = dependency.namespace AND consumer.id = dependency.job_id
|
|
92
|
+
WHERE dependency.namespace = completion.namespace
|
|
93
|
+
AND dependency.dependency_id = completion.job_id
|
|
94
|
+
AND consumer.state = ANY(${tx.array([...LIVE_CONSUMER_STATES], 'TEXT')})
|
|
95
|
+
)
|
|
96
|
+
ORDER BY completion.completed_at DESC, completion.job_id DESC
|
|
97
|
+
OFFSET ${ctx.config.maxCompletedJobs}
|
|
98
|
+
LIMIT ${COMPLETION_PRUNE_BATCH_SIZE}
|
|
99
|
+
`;
|
|
100
|
+
const ids = candidates.map((row) => jobId(row.job_id));
|
|
101
|
+
if (ids.length === 0)
|
|
102
|
+
return { selected: 0, removed: 0 };
|
|
103
|
+
await lockPostgresDependencyCompletions(tx, ctx, ids);
|
|
104
|
+
const removed = await tx `
|
|
105
|
+
DELETE FROM bunqueue_completions AS completion
|
|
106
|
+
WHERE completion.namespace = ${ctx.config.namespace}
|
|
107
|
+
AND completion.job_id = ANY(${tx.array(ids.map(String), 'TEXT')})
|
|
108
|
+
AND NOT EXISTS (
|
|
109
|
+
SELECT 1 FROM bunqueue_jobs AS retained
|
|
110
|
+
WHERE retained.namespace = completion.namespace
|
|
111
|
+
AND retained.id = completion.job_id
|
|
112
|
+
)
|
|
113
|
+
AND NOT EXISTS (
|
|
114
|
+
SELECT 1
|
|
115
|
+
FROM bunqueue_dependencies AS dependency
|
|
116
|
+
JOIN bunqueue_jobs AS consumer
|
|
117
|
+
ON consumer.namespace = dependency.namespace AND consumer.id = dependency.job_id
|
|
118
|
+
WHERE dependency.namespace = completion.namespace
|
|
119
|
+
AND dependency.dependency_id = completion.job_id
|
|
120
|
+
AND consumer.state = ANY(${tx.array([...LIVE_CONSUMER_STATES], 'TEXT')})
|
|
121
|
+
)
|
|
122
|
+
RETURNING completion.job_id, completion.queue
|
|
123
|
+
`;
|
|
124
|
+
const now = await databaseNow(tx);
|
|
125
|
+
for (const queue of [...new Set(removed.map((row) => row.queue))].sort()) {
|
|
126
|
+
await recordPostgresEvent(tx, ctx, {
|
|
127
|
+
queue,
|
|
128
|
+
type: 'queue-completions-pruned',
|
|
129
|
+
jobId: jobId(`__queue__:${queue}`),
|
|
130
|
+
occurredAt: now,
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
return { selected: ids.length, removed: removed.length };
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
/** Bound unreferenced removeOnComplete tombstones without pruning live dependency evidence. */
|
|
137
|
+
export async function prunePostgresCompletionTombstones(ctx) {
|
|
138
|
+
let removed = 0;
|
|
139
|
+
while (true) {
|
|
140
|
+
const batch = await prunePostgresCompletionTombstoneBatch(ctx);
|
|
141
|
+
removed += batch.removed;
|
|
142
|
+
if (batch.selected === 0)
|
|
143
|
+
return removed;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { type JobId } from '../../../domain/types/job';
|
|
2
|
+
import { type PostgresContext } from './context';
|
|
3
|
+
import type { PostgresTransitionResult } from './types';
|
|
4
|
+
export declare function completePostgresJob(ctx: PostgresContext, id: JobId, token: string, result: unknown, removeOnComplete?: boolean): Promise<PostgresTransitionResult>;
|