pgbus 0.13.1 → 0.13.3

data/lib/pgbus/process/supervisor.rb

@@ -44,6 +44,16 @@ module Pgbus
         @pending_restarts = pending_restarts
         @crash_counts = Hash.new(0)
         @notify_hub = notify_hub
+        @intended_children = 0
+        @readiness = Concurrent::AtomicReference.new(
+          ReadinessSnapshot.new(booted: false, shutting_down: shutting_down, expected: 0, live: forks.size)
+        )
+      end
+
+      # The current container-local readiness state. Safe to call from any
+      # thread (the health server's accept thread reads it per probe).
+      def readiness_snapshot
+        @readiness.get
       end
 
       def shutting_down?
@@ -97,6 +107,7 @@ module Pgbus
         start_notify_hub
 
         boot_processes
+        mark_booted
         monitor_loop
       ensure
         shutdown
@@ -105,17 +116,51 @@ module Pgbus
       def graceful_shutdown
         Pgbus.logger.info { "[Pgbus] Supervisor: graceful shutdown requested" }
         @shutting_down = true
+        refresh_readiness
         signal_children("TERM")
       end
 
       def immediate_shutdown
         Pgbus.logger.warn { "[Pgbus] Supervisor: immediate shutdown requested" }
         @shutting_down = true
+        refresh_readiness
         signal_children("QUIT")
       end
 
       private
 
+      # Boot is complete: connection verified, queues bootstrapped, every
+      # configured child fork ATTEMPTED. The baseline is the larger of the
+      # intended-attempt count and the fork-table size: a boot-time fork
+      # failure (EAGAIN/ENOMEM, logged-and-swallowed in fork_*) leaves
+      # intended > live, so the readiness gate reports DEGRADED instead of
+      # blessing a container that is missing workers. Roles that legitimately
+      # declined to boot (scheduler with no recurring tasks) never reach a
+      # fork_* method and are counted by neither side.
+      def mark_booted
+        @booted = true
+        @expected_children = [@intended_children, @forks.size].max
+        refresh_readiness
+      end
+
+      # Count a child the configuration intends this boot to run. Called at
+      # the top of every fork_* method — before the fork can fail — and only
+      # pre-boot, so restart_child's re-forks never inflate the baseline.
+      def note_intended_child
+        @intended_children += 1 unless @booted
+      end
+
+      # Publish a fresh snapshot; the swapped-in Data is immutable, so the
+      # health server's accept thread always reads a consistent state.
+      def refresh_readiness
+        @readiness.set(
+          ReadinessSnapshot.new(
+            booted: !!@booted, shutting_down: @shutting_down,
+            expected: @expected_children || 0, live: @forks.size
+          )
+        )
+      end
+
       # Log a single boot diagnostics banner: the settings that actually
       # determine whether this deployment works. One consecutive block of
       # "[Pgbus] boot:"-prefixed info lines so it reads cleanly under both the
@@ -253,6 +298,7 @@ module Pgbus
       end
 
       def fork_worker(worker_config, slot: nil)
+        note_intended_child
         queues = worker_config[:queues] || [config.default_queue]
         threads = worker_config[:threads] || 5
         single_active = worker_config[:single_active_consumer] || false
@@ -337,6 +383,7 @@ module Pgbus
       end
 
       def fork_dispatcher
+        note_intended_child
         pid = fork do
           restore_signals
           setup_child_process
@@ -364,6 +411,7 @@ module Pgbus
       end
 
       def fork_scheduler
+        note_intended_child
         pid = fork do
           restore_signals
           setup_child_process
@@ -427,6 +475,7 @@ module Pgbus
       end
 
       def fork_consumer(consumer_config, slot: nil)
+        note_intended_child
         # Array() so a consumer entry without :topics can't NoMethodError the
         # supervisor on the topics.join log lines below.
         topics = Array(consumer_config[:topics])
@@ -504,6 +553,7 @@ module Pgbus
       end
 
       def fork_outbox_poller
+        note_intended_child
         pid = fork do
           restore_signals
           setup_child_process
@@ -537,6 +587,9 @@ module Pgbus
             # refresh, and fork status broadcast (issue #381).
             @notify_hub&.tick
           end
+          # After reap + restarts so a clean recycle (reaped and re-forked in
+          # the same pass) never dips the published live count (issue #386).
+          refresh_readiness
           interruptible_sleep(FORK_WAIT)
         end
       end
@@ -821,7 +874,12 @@ module Pgbus
       def start_health_server
         return unless config.health_port
 
-        @health_server = Pgbus::Web::HealthServer.new(port: config.health_port, bind: config.health_bind)
+        # The standalone server answers /readyz from THIS supervisor's
+        # container-local snapshot — a rolling deploy's health gate must
+        # measure the new container, not the fleet-wide verdict a sibling
+        # container's workers can satisfy (issue #386).
+        app = Pgbus::Web::HealthApp.new(local_readiness: -> { readiness_snapshot })
+        @health_server = Pgbus::Web::HealthServer.new(port: config.health_port, bind: config.health_bind, app: app)
         @health_server.start
       end
 
@@ -876,8 +934,11 @@ module Pgbus
       end
 
       def shutdown
-        # Wait for all children with timeout
-        deadline = Time.now + 30
+        # Wait for children to drain and exit, bounded by config.shutdown_timeout
+        # (default drain_timeout + 5) so raising the drain window can never
+        # mean SIGKILLing workers mid-drain. An orchestrator's stop grace
+        # period should exceed this value (issue #386).
+        deadline = Time.now + config.shutdown_timeout
 
         until @forks.empty? || Time.now > deadline
           reap_children

data/lib/pgbus/process/worker.rb

@@ -155,6 +155,12 @@ module Pgbus
       NOTIFY_RETRY_BASE_SECONDS = 5
       NOTIFY_RETRY_MAX_SECONDS = 300
 
+      # Residual pool wait in #shutdown, AFTER the drain loop already spent up
+      # to config.drain_timeout on in-flight jobs. Short by design: a job still
+      # running has proven it won't finish, and this wait competes with the
+      # supervisor's shutdown_timeout deadline (issue #386).
+      POOL_TERMINATION_WAIT = 5
+
       def run
         setup_signals
         start_heartbeat
@@ -175,9 +181,9 @@ module Pgbus
 
           break if @lifecycle.stopped?
           # quiesced? (all slots free), not idle? (any slot free) — exiting
-          # with work still in flight abandons those jobs to the 30s
-          # wait_for_termination timeout in shutdown. Bounded by
-          # config.drain_timeout so a stuck job can't wedge the loop forever.
+          # with work still in flight abandons those jobs to shutdown's short
+          # POOL_TERMINATION_WAIT residual. Bounded by config.drain_timeout so
+          # a stuck job can't wedge the loop forever.
           break if @lifecycle.draining? && (@pool.quiesced? || drain_deadline_exceeded?)
 
           claim_and_execute if @lifecycle.can_process?
@@ -792,7 +798,11 @@ module Pgbus
         Pgbus.logger.info { "[Pgbus] Worker draining thread pool..." }
         stop_wake_source
         @pool.shutdown
-        @pool.wait_for_termination(30)
+        # Residual wait only: the drain loop already waited up to
+        # config.drain_timeout for in-flight jobs. A job still running here has
+        # proven it won't finish; waiting another full window would push the
+        # worker past the supervisor's shutdown_timeout deadline (issue #386).
+        @pool.wait_for_termination(POOL_TERMINATION_WAIT)
         @stat_buffer&.stop
         @queue_lock&.unlock_all
         @heartbeat&.stop

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.13.1"
+  VERSION = "0.13.3"
 end

data/lib/pgbus/web/data_source.rb

@@ -1177,7 +1177,8 @@ module Pgbus
               (SELECT EXTRACT(epoch FROM (NOW() - min(enqueued_at)))::int FROM pgmq.#{qtable}) AS oldest_msg_age_sec,
               (SELECT CASE WHEN is_called THEN last_value ELSE 0 END FROM pgmq.#{seq_name}) AS total_messages,
               (SELECT max(read_ct) FROM pgmq.#{qtable}) AS max_read_ct,
-              (SELECT count(*) FROM pgmq.#{qtable} WHERE vt <= NOW() AND read_ct = 0) AS visible_unread_length
+              (SELECT count(*) FROM pgmq.#{qtable} WHERE vt <= NOW() AND read_ct = 0) AS visible_unread_length,
+              (SELECT EXTRACT(epoch FROM (NOW() - min(vt)))::int FROM pgmq.#{qtable} WHERE vt <= NOW()) AS oldest_claimable_age_sec
           SQL
         rescue StandardError => e
           Pgbus.logger.debug { "[Pgbus::Web] Skipping queue metrics for #{name}: #{e.message}" }
@@ -1193,7 +1194,9 @@ module Pgbus
             name: row["queue_name"],
             queue_length: row["queue_length"].to_i,
             queue_visible_length: row["queue_visible_length"].to_i,
+            parked_length: row["queue_length"].to_i - row["queue_visible_length"].to_i,
             oldest_msg_age_sec: row["oldest_msg_age_sec"]&.to_i,
+            oldest_claimable_age_sec: row["oldest_claimable_age_sec"]&.to_i,
             newest_msg_age_sec: row["newest_msg_age_sec"]&.to_i,
             total_messages: row["total_messages"].to_i,
             max_read_ct: row["max_read_ct"]&.to_i,
@@ -1216,6 +1219,7 @@ module Pgbus
               count(CASE WHEN vt <= NOW() THEN 1 END) AS queue_visible_length,
               EXTRACT(epoch FROM (NOW() - max(enqueued_at)))::int AS newest_msg_age_sec,
               EXTRACT(epoch FROM (NOW() - min(enqueued_at)))::int AS oldest_msg_age_sec,
+              EXTRACT(epoch FROM (NOW() - min(vt) FILTER (WHERE vt <= NOW())))::int AS oldest_claimable_age_sec,
               max(read_ct) AS max_read_ct,
               count(CASE WHEN vt <= NOW() AND read_ct = 0 THEN 1 END) AS visible_unread_length
             FROM pgmq.#{qtable}
@@ -1229,6 +1233,7 @@ module Pgbus
             q_summary.queue_visible_length,
             q_summary.newest_msg_age_sec,
             q_summary.oldest_msg_age_sec,
+            q_summary.oldest_claimable_age_sec,
             q_summary.max_read_ct,
             q_summary.visible_unread_length,
             all_metrics.total_messages
@@ -1241,7 +1246,9 @@ module Pgbus
           name: queue_name,
           queue_length: row["queue_length"].to_i,
           queue_visible_length: row["queue_visible_length"].to_i,
+          parked_length: row["queue_length"].to_i - row["queue_visible_length"].to_i,
           oldest_msg_age_sec: row["oldest_msg_age_sec"]&.to_i,
+          oldest_claimable_age_sec: row["oldest_claimable_age_sec"]&.to_i,
           newest_msg_age_sec: row["newest_msg_age_sec"]&.to_i,
           total_messages: row["total_messages"].to_i,
           max_read_ct: row["max_read_ct"]&.to_i,

data/lib/pgbus/web/health_app.rb

@@ -49,8 +49,15 @@ module Pgbus
       # @param data_source [Pgbus::Web::DataSource, nil] read layer for /readyz.
       #   nil (the default) builds a fresh DataSource per readiness check, which
       #   avoids serving stale metrics from a long-lived app's memoized instance.
-      def initialize(data_source: nil)
+      # @param local_readiness [#call, nil] when set, /readyz answers from this
+      #   callable's {Process::ReadinessSnapshot} instead of the cluster-wide
+      #   analyzer — the supervisor's standalone HealthServer passes its own
+      #   snapshot so a rolling deploy's health gate measures THIS container,
+      #   not the fleet (issue #386). The Rails-mounted app leaves it nil and
+      #   keeps the cluster verdict.
+      def initialize(data_source: nil, local_readiness: nil)
         @data_source = data_source
+        @local_readiness = local_readiness
       end
 
       def call(env)
@@ -69,6 +76,8 @@ module Pgbus
       end
 
       def readyz
+        return local_readyz if @local_readiness
+
         # HealthAnalyzer lives in the MCP namespace, which is excluded from
         # Zeitwerk (its *tools* subclass the optional `mcp` gem). The analyzer
         # itself has no gem dependency, so require just that one file — the
@@ -83,6 +92,19 @@ module Pgbus
         [503, JSON_HEADERS.dup, [{ status: "ERROR", error: e.message }.to_json]]
       end
 
+      # Container-local readiness: no database, no analyzer — just the
+      # supervisor's published snapshot. The error path mirrors the cluster
+      # readyz: 503 ERROR, logged, never swallowed.
+      def local_readyz
+        snapshot = @local_readiness.call
+        status = snapshot.ready? ? 200 : 503
+        body = { status: snapshot.status, expected: snapshot.expected, live: snapshot.live }
+        [status, JSON_HEADERS.dup, [body.to_json]]
+      rescue StandardError => e
+        Pgbus.logger.error { "[Pgbus::Web::HealthApp] local readiness check failed: #{e.class}: #{e.message}" }
+        [503, JSON_HEADERS.dup, [{ status: "ERROR", error: e.message }.to_json]]
+      end
+
       # Reuse an injected DataSource (tests, an app that wants one shared
       # instance); otherwise build a fresh one each check so per-instance
       # memoization can never serve stale queue/process metrics.

data/lib/pgbus/web/metrics_serializer.rb

@@ -54,6 +54,15 @@ module Pgbus
           end
         end
 
+        gauge(lines, "pgbus_queue_oldest_claimable_age_seconds",
+              "Age of the oldest message eligible for pickup (visibility timeout elapsed)") do
+          queues.filter_map do |q|
+            next unless q[:oldest_claimable_age_sec]
+
+            [q[:oldest_claimable_age_sec], { queue: q[:name] }]
+          end
+        end
+
         gauge(lines, "pgbus_queue_paused", "Whether the queue is paused (1) or active (0)") do
           queues.map { |q| [q[:paused] ? 1 : 0, { queue: q[:name] }] }
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgbus
 version: !ruby/object:Gem::Version
-  version: 0.13.1
+  version: 0.13.3
 platform: ruby
 authors:
 - Mikael Henriksson
@@ -114,6 +114,7 @@ email:
 - mikael@mhenrixon.com
 executables:
 - pgbus
+- pgbus-health
 extensions: []
 extra_rdoc_files: []
 files:
@@ -207,6 +208,7 @@ files:
 - config/locales/sv.yml
 - config/routes.rb
 - exe/pgbus
+- exe/pgbus-health
 - lib/active_job/queue_adapters/pgbus_adapter.rb
 - lib/generators/pgbus/add_failed_events_index_generator.rb
 - lib/generators/pgbus/add_job_stats_generator.rb
@@ -214,6 +216,7 @@ files:
 - lib/generators/pgbus/add_job_stats_queue_index_generator.rb
 - lib/generators/pgbus/add_outbox_generator.rb
 - lib/generators/pgbus/add_presence_generator.rb
+- lib/generators/pgbus/add_processed_event_completion_generator.rb
 - lib/generators/pgbus/add_queue_states_generator.rb
 - lib/generators/pgbus/add_recurring_generator.rb
 - lib/generators/pgbus/add_stream_queues_generator.rb
@@ -228,6 +231,7 @@ files:
 - lib/generators/pgbus/templates/add_job_stats_queue_index.rb.erb
 - lib/generators/pgbus/templates/add_outbox.rb.erb
 - lib/generators/pgbus/templates/add_presence.rb.erb
+- lib/generators/pgbus/templates/add_processed_event_completion.rb.erb
 - lib/generators/pgbus/templates/add_queue_states.rb.erb
 - lib/generators/pgbus/templates/add_recurring_tables.rb.erb
 - lib/generators/pgbus/templates/add_stream_queues.rb.erb
@@ -282,6 +286,7 @@ files:
 - lib/pgbus/failed_event_recorder.rb
 - lib/pgbus/generators/database_target_detector.rb
 - lib/pgbus/generators/migration_detector.rb
+- lib/pgbus/health_probe.rb
 - lib/pgbus/instrumentation.rb
 - lib/pgbus/integrations/appsignal.rb
 - lib/pgbus/integrations/appsignal/dashboard.json
@@ -333,6 +338,7 @@ files:
 - lib/pgbus/process/notify_probe.rb
 - lib/pgbus/process/primary_validator.rb
 - lib/pgbus/process/queue_lock.rb
+- lib/pgbus/process/readiness_snapshot.rb
 - lib/pgbus/process/signal_handler.rb
 - lib/pgbus/process/supervisor.rb
 - lib/pgbus/process/wake_pipe.rb