pgbus 0.18.1 → 0.19.0

data/lib/pgbus/configuration.rb

@@ -51,6 +51,12 @@ module Pgbus
     # wait, so recycling/deploy never wedges on a permanently-stuck job.
     attr_accessor :stall_threshold, :read_timeout, :drain_timeout
 
+    # immediate_shutdown_grace (default 2s): on QUIT, a worker or consumer
+    # stops taking work and gives the jobs already running this long to finish
+    # before it kills their threads. Killed jobs' ActiveRecord connections are
+    # reclaimed, so none stays checked out by a dead thread.
+    attr_accessor :immediate_shutdown_grace
+
     # shutdown_timeout bounds how long the supervisor waits for its children
     # after forwarding TERM before escalating to SIGKILL. nil (default) derives
     # drain_timeout + SHUTDOWN_TIMEOUT_MARGIN, so raising drain_timeout keeps
@@ -132,8 +138,11 @@ module Pgbus
     # Event bus
     attr_reader :idempotency_ttl
 
-    # Logging
-    attr_accessor :logger
+    # Logging. Unset means Rails.logger when Rails has one at the time of the
+    # read, else a $stdout Logger. Resolved on every read rather than when the
+    # configuration is built, because Pgbus builds it when the gem loads,
+    # which in a Rails app is before Rails.logger exists.
+    attr_writer :logger
     attr_reader :log_format # rubocop:disable Style/AccessorGrouping
 
     # Error reporting — array of callable objects invoked on caught exceptions.
@@ -296,6 +305,7 @@ module Pgbus
       @stall_threshold = 90
       @read_timeout = 30
       @drain_timeout = 30
+      @immediate_shutdown_grace = 2
       @shutdown_timeout = nil
 
       @dispatch_interval = 1.0
@@ -327,8 +337,10 @@ module Pgbus
       @idempotency_ttl = 7 * 24 * 3600 # 7 days
       @allowed_global_id_models = nil # nil = allow all (for backwards compat)
 
-      @logger = (defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger) || Logger.new($stdout)
+      @logger = nil
+      @stdout_logger = Logger.new($stdout)
       @log_format = :text
+      @log_format_set = false # see #apply_log_format!
       @error_reporters = []
 
       @listen_notify = true
@@ -566,6 +578,7 @@ module Pgbus
       @statsd_port = 8125
 
       @eager_validation = true
+      @finalized = false
     end
 
     def queue_name(name)
@@ -619,6 +632,10 @@ module Pgbus
       ExecutionPools.normalize_mode(mode)
     end
 
+    def logger
+      @logger || rails_logger || @stdout_logger
+    end
+
     VALID_LOG_FORMATS = %i[text json].freeze
 
     def log_format=(format)
@@ -628,15 +645,26 @@ module Pgbus
       end
 
       @log_format = format
+      @log_format_set = true
+      apply_log_format!
+    end
+
+    # Installs log_format's formatter on the logger pgbus resolves now; a
+    # no-op until log_format is set. The configuration exists before
+    # Rails.logger does, so a log_format set that early (config/application.rb)
+    # formats only the $stdout fallback: the engine calls this again once
+    # Rails.logger exists.
+    def apply_log_format!
+      return unless @log_format_set
 
       # Only install pgbus's formatter when the logger still carries a nil or
       # pgbus-installed formatter. A user who set a custom logger with a custom
       # formatter keeps it — log_format only records the intended format for
       # pgbus's own defaults, it must not silently clobber their formatting.
-      old_formatter = @logger&.formatter
+      old_formatter = logger&.formatter
       return unless pgbus_installable_formatter?(old_formatter)
 
-      new_formatter = case format
+      new_formatter = case log_format
                       when :json then LogFormatter::JSON.new
                       when :text then LogFormatter::Text.new
                       end
@@ -649,7 +677,7 @@ module Pgbus
       # formatter. Re-extend the new formatter with the same module when the old
       # one carried it, so tagging keeps working (issue #334).
       preserve_tagged_logging(old_formatter, new_formatter)
-      @logger.formatter = new_formatter
+      logger.formatter = new_formatter
     end
 
     VALID_BROADCAST_MODES = %i[ephemeral durable].freeze
@@ -799,6 +827,36 @@ module Pgbus
       @worker_notify_scope = coerced
     end
 
+    # Settings finalize! never freezes: @error_reporters is a list of
+    # callables apps append to (`c.error_reporters << ...`), and
+    # @deprecation_warned is this object's own bookkeeping.
+    FINALIZE_SKIP = %i[@error_reporters @deprecation_warned @finalized].freeze
+    private_constant :FINALIZE_SKIP
+
+    # Replaces every plain-data setting (Strings, Symbols, numbers, booleans,
+    # nil, Regexps, classes, and Arrays/Hashes/Sets/Ranges of those) with a
+    # deep-frozen, Ractor-shareable copy, so each reader hands back a value
+    # any Ractor can read (issue #527). Settings that hold a callable or an
+    # object (the logger, error reporters, hooks, the web data source, a
+    # metrics backend) are left as they are: they stay main-Ractor-only.
+    # Writers keep working (a setter assigns a new value; Pgbus.configure
+    # finalizes again); only in-place mutation of a frozen collection raises.
+    # Called once at boot by Pgbus.finalize!.
+    def finalize!
+      instance_variables.each do |ivar|
+        next if FINALIZE_SKIP.include?(ivar)
+
+        value = instance_variable_get(ivar)
+        instance_variable_set(ivar, Shareable.freeze_copy(value)) if Shareable.data?(value)
+      end
+      @finalized = true
+      self
+    end
+
+    def finalized?
+      @finalized
+    end
+
     def validate!
       if pool_size && !(pool_size.is_a?(Numeric) && pool_size.positive?)
         raise Pgbus::ConfigurationError, "pool_size must be a positive number or nil (auto-tune)"
@@ -829,6 +887,8 @@ module Pgbus
       end
       raise Pgbus::ConfigurationError, "drain_timeout must be > 0" unless drain_timeout.is_a?(Numeric) && drain_timeout.positive?
 
+      validate_immediate_shutdown_grace!
+
       validate_shutdown_timeout!
 
       unless stats_flush_size.is_a?(Integer) && stats_flush_size.positive?
@@ -1279,15 +1339,17 @@ module Pgbus
       validate_processes!(options[:processes]) unless options[:processes].nil?
 
       normalized_name = name.to_s
-      @workers ||= []
+      existing = @workers || []
 
-      if @workers.any? { |c| capsule_name(c) == normalized_name }
+      if existing.any? { |c| capsule_name(c) == normalized_name }
         raise Pgbus::ConfigurationError, "capsule #{name.inspect} is already defined"
       end
 
       validate_no_queue_overlap!(queues)
 
-      @workers << { name: normalized_name, queues: queues, threads: threads, **options }
+      # A new list rather than <<: after finalize! @workers is frozen, and a
+      # writer keeps working (a later Pgbus.configure finalizes again).
+      @workers = [*existing, { name: normalized_name, queues: queues, threads: threads, **options }]
     end
 
     # Look up a capsule by its name. Accepts symbol or string. Returns the
@@ -1631,6 +1693,10 @@ module Pgbus
       end
     end
 
+    def rails_logger
+      ::Rails.logger if defined?(::Rails) && ::Rails.respond_to?(:logger)
+    end
+
     # When the logger's previous formatter was extended with
     # ActiveSupport::TaggedLogging::Formatter (the case for a Rails
     # Rails.logger), extend the replacement pgbus formatter with the same module
@@ -1718,6 +1784,15 @@ module Pgbus
             "#{name} must be a Numeric (seconds), ActiveSupport::Duration, or nil to disable, got #{value.inspect}"
     end
 
+    # A finite real number >= 0: NaN is not negative?, and an infinite grace
+    # would never kill a job that does not finish.
+    def validate_immediate_shutdown_grace!
+      grace = immediate_shutdown_grace
+      return if grace.is_a?(Numeric) && grace.real? && grace.finite? && grace >= 0
+
+      raise Pgbus::ConfigurationError, "immediate_shutdown_grace must be a finite number >= 0"
+    end
+
     def validate_positive_duration!(numeric, name)
       raise Pgbus::ConfigurationError, "#{name} must be a positive number, got #{numeric}" unless numeric.positive?
 

data/lib/pgbus/current_attributes.rb

@@ -20,6 +20,8 @@ module Pgbus
   module CurrentAttributes
     METADATA_KEY = "pgbus_current"
 
+    # The names already warned about, so each missing class is logged once.
+    # Built at load; Concurrent::Map#compute_if_absent is atomic.
     @missing_warned = Concurrent::Map.new
 
     class << self
@@ -115,7 +117,7 @@ module Pgbus
       def resolve_class(name)
         name.constantize
       rescue NameError
-        @missing_warned.compute_if_absent(name) do
+        @missing_warned.compute_if_absent(name) do # rubocop:disable ThreadSafety/ClassInstanceVariable -- see @missing_warned
           Pgbus.logger.warn { "[Pgbus] current_attributes: #{name} is not defined — skipping" }
           true
         end

data/lib/pgbus/dedup_cache.rb

@@ -17,6 +17,10 @@ module Pgbus
       @ttl = ttl
       @cache = Concurrent::Map.new
       @insertion_order = Concurrent::Array.new
+      # Serializes the check-then-act writers (mark!, expiry eviction, clear!)
+      # so the size bound and @insertion_order stay consistent (issue #527).
+      # seen? reads @cache without it.
+      @lock = Mutex.new
     end
 
     # Returns true if this key was already seen (duplicate).
@@ -26,7 +30,7 @@ module Pgbus
       return false unless entry
 
       if expired?(entry)
-        evict(key)
+        @lock.synchronize { evict(key) if @cache[key].equal?(entry) }
         return false
       end
 
@@ -36,11 +40,13 @@ module Pgbus
     # Mark a key as seen. Call this after successfully claiming idempotency
     # in the database so future lookups skip the DB.
     def mark!(key)
-      already_present = @cache.key?(key)
-      evict_oldest if !already_present && @cache.size >= @max_size
+      @lock.synchronize do
+        already_present = @cache.key?(key)
+        evict_oldest if !already_present && @cache.size >= @max_size
 
-      @cache[key] = monotonic_now
-      @insertion_order << key unless already_present
+        @cache[key] = monotonic_now
+        @insertion_order << key unless already_present
+      end
     end
 
     def size
@@ -48,8 +54,10 @@ module Pgbus
     end
 
     def clear!
-      @cache.clear
-      @insertion_order.clear
+      @lock.synchronize do
+        @cache.clear
+        @insertion_order.clear
+      end
     end
 
     private

data/lib/pgbus/engine.rb

@@ -81,6 +81,9 @@ module Pgbus
     end
 
     initializer "pgbus.logger" do
+      # Rails.logger exists now: a log_format set before it did went to the
+      # $stdout fallback only.
+      Pgbus.configuration.apply_log_format!
       ActiveSupport.on_load(:after_initialize) do
         Pgbus.configuration.logger ||= Rails.logger
       end
@@ -209,5 +212,13 @@ module Pgbus
         end
       end
     end
+
+    # Last: once the app's initializers have configured pgbus, freeze the
+    # configuration's plain-data settings into Ractor-shareable values
+    # (Pgbus.finalize!, issue #527). Registered after the app's initializer
+    # files, so it runs after the after_initialize blocks they add.
+    initializer "pgbus.finalize", after: :load_config_initializers do
+      ActiveSupport.on_load(:after_initialize) { Pgbus.finalize! }
+    end
   end
 end

data/lib/pgbus/event_bus/handler.rb

@@ -4,6 +4,19 @@ module Pgbus
   module EventBus
     class Handler
       class << self
+        # One DedupCache per handler class, built when the class is defined
+        # (this class body, and #inherited for every subclass), never by a
+        # lazy `||=` that two first deliveries could each win (issue #527).
+        attr_reader :dedup_cache
+
+        def inherited(subclass)
+          super
+          subclass.instance_variable_set(:@dedup_cache, DedupCache.new)
+        end
+
+        # Class-body macro: `idempotent!` runs while the handler class loads,
+        # and #idempotent? only reads the flag afterwards.
+        # rubocop:disable ThreadSafety/ClassInstanceVariable
         def idempotent!
           @idempotent = true
         end
@@ -11,10 +24,7 @@ module Pgbus
         def idempotent?
           @idempotent == true
         end
-
-        def dedup_cache
-          @dedup_cache ||= DedupCache.new
-        end
+        # rubocop:enable ThreadSafety/ClassInstanceVariable
 
         # How long a pending claim may stay silent before its holder counts as
         # dead. ClaimBeat refreshes a live claim from the visibility heartbeat,
@@ -34,6 +44,8 @@ module Pgbus
         end
       end
 
+      @dedup_cache = DedupCache.new
+
       # Outcome of the two-phase claim. `age` is how long the losing delivery
       # found the existing claim to have been silent, in seconds (nil unless
       # the claim was pending).

data/lib/pgbus/execution_pools/async_pool.rb

@@ -3,6 +3,10 @@
 module Pgbus
   module ExecutionPools
     class AsyncPool
+      # How long #kill waits for the killed reactor thread (and the job
+      # fibers it cancels) to unwind before giving up on it.
+      KILL_JOIN_TIMEOUT = 1
+
       attr_reader :capacity
 
       def initialize(capacity:, on_state_change: nil)
@@ -70,9 +74,15 @@ module Pgbus
         @reactor_thread&.join(timeout)
       end
 
+      # Killing the reactor thread cancels its job fibers; wait for them to
+      # unwind so the caller can reclaim what they held (ExecutionPools.terminate).
       def kill
         shutdown
-        @reactor_thread&.kill
+        thread = @reactor_thread
+        return unless thread
+
+        thread.kill
+        join_killed(thread)
       end
 
       def metadata
@@ -86,6 +96,14 @@ module Pgbus
 
       private
 
+      # The reactor's own failure is already recorded (register_fatal_error);
+      # Thread#join would raise it again here.
+      def join_killed(thread)
+        thread.join(KILL_JOIN_TIMEOUT)
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus] Async pool reactor ended with #{e.class}: #{e.message}" }
+      end
+
       def validate_dependencies!
         require "async"
         require "async/semaphore"
@@ -95,9 +113,12 @@ module Pgbus
               "Add `gem \"async\"` to your Gemfile. Original error: #{e.message}"
       end
 
+      # The pool's one reactor thread, owned by this pool: started once by
+      # #initialize, it exits when #shutdown lets the reactor drain, and
+      # #wait_for_termination / #kill join it.
       # rubocop:disable-next Lint/RescueException
       def start_reactor
-        Thread.new do
+        Thread.new do # rubocop:disable ThreadSafety/NewThread
           Thread.current.name = "pgbus-async-reactor-#{object_id}"
           Async do |task|
             semaphore = Async::Semaphore.new(@capacity, parent: task)

data/lib/pgbus/execution_pools/thread_pool.rb

@@ -5,6 +5,10 @@ require "concurrent"
 module Pgbus
   module ExecutionPools
     class ThreadPool
+      # How long #kill waits, in all, for the killed jobs to unwind (run their
+      # ensure blocks) before giving up on those still running.
+      KILL_JOIN_TIMEOUT = 1
+
       attr_reader :capacity
 
       def initialize(capacity:, on_state_change: nil)
@@ -12,14 +16,17 @@ module Pgbus
         @on_state_change = on_state_change
         @available_capacity = Concurrent::AtomicFixnum.new(capacity)
         @pool = Concurrent::FixedThreadPool.new(capacity)
+        @running = Concurrent::Map.new
       end
 
       def post(&block)
         @available_capacity.decrement
         begin
           @pool.post do
+            @running[Thread.current] = true
             block.call
           ensure
+            @running.delete(Thread.current)
             @available_capacity.increment
             @on_state_change&.call
           end
@@ -52,8 +59,17 @@ module Pgbus
         @pool.wait_for_termination(timeout)
       end
 
+      # Thread#kill only asks a thread to stop: wait for the killed jobs to
+      # unwind, so the caller can reclaim what they held (ExecutionPools.terminate).
+      # The running jobs are listed after the kill: a job that started in
+      # between is killed too, and a job leaves the list only once it has
+      # unwound, so every one still unwinding is joined. One budget for all
+      # of them: stuck jobs must not hold a shutdown for capacity x the timeout.
       def kill
         @pool.kill
+        unwinding = @running.keys
+        deadline = monotonic_now + KILL_JOIN_TIMEOUT
+        unwinding.each { |thread| thread.join([deadline - monotonic_now, 0].max) }
       end
 
       def metadata
@@ -63,6 +79,12 @@ module Pgbus
           busy: @capacity - available_capacity
         }
       end
+
+      private
+
+      def monotonic_now
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
     end
   end
 end

data/lib/pgbus/execution_pools.rb

@@ -22,6 +22,32 @@ module Pgbus
           raise ArgumentError, "Unknown execution_mode: #{mode.inspect}. Expected one of: :threads, :async, :fiber"
         end
       end
+
+      # Stop +pool+ taking work, give the jobs already running +grace+
+      # seconds to finish, and kill whatever is still running after that.
+      # Either way, reclaim the ActiveRecord connections the jobs left checked
+      # out: a killed thread or fiber never checks its own back in, and the
+      # pool would stay that many connections short until its reaper ran.
+      # Also when the wait raises (an async pool re-raises a fatal reactor
+      # error). Returns true when every job finished within the grace (issue #527).
+      def terminate(pool, grace:)
+        pool.shutdown
+        finished = pool.wait_for_termination(grace) ? true : false
+        pool.kill unless finished
+        finished
+      ensure
+        reclaim_connections
+      end
+
+      # The caller's own connection, then every connection whose owning
+      # thread or fiber is dead. A no-op without ActiveRecord.
+      def reclaim_connections
+        return unless defined?(::ActiveRecord::Base)
+
+        handler = ::ActiveRecord::Base.connection_handler
+        handler.clear_active_connections!(:all)
+        handler.connection_pool_list(:all).each(&:reap)
+      end
     end
   end
 end

data/lib/pgbus/integrations/appsignal/probe.rb

@@ -24,23 +24,31 @@ module Pgbus
         METRIC_PREFIX = "pgbus_"
         private_constant :METRIC_PREFIX
 
+        # Registered once at boot; @install_lock makes the check-then-register
+        # atomic so the probe is never registered twice (issue #527).
+        @install_lock = Mutex.new
+        @installed = false
+
         class << self
-          def install! # rubocop:disable Naming/PredicateMethod
-            return false if @installed
+          def install!
+            @install_lock.synchronize do
+              next false if @installed
 
-            ::Appsignal::Probes.register :pgbus, new_probe_instance
-            @installed = true
-            true
+              ::Appsignal::Probes.register :pgbus, new_probe_instance
+              @installed = true
+            end
           end
 
           def installed?
-            @installed == true
+            @install_lock.synchronize { @installed }
           end
 
           def reset!
-            ::Appsignal::Probes.unregister(:pgbus) if defined?(::Appsignal::Probes) &&
-                                                      ::Appsignal::Probes.respond_to?(:unregister)
-            @installed = false
+            @install_lock.synchronize do
+              ::Appsignal::Probes.unregister(:pgbus) if defined?(::Appsignal::Probes) &&
+                                                        ::Appsignal::Probes.respond_to?(:unregister)
+              @installed = false
+            end
           end
 
           # Visible for testing — returns a fresh runnable probe.

data/lib/pgbus/integrations/appsignal/subscriber.rb

@@ -21,14 +21,39 @@ module Pgbus
         METRIC_PREFIX = "pgbus_"
         private_constant :BACKGROUND_JOB, :METRIC_PREFIX
 
-        # Tracked so we can detach in reset! (used by specs).
-        @subscriptions = []
+        # Tracked so we can detach in reset! (used by specs). Installed once
+        # at boot (the engine's after_initialize); @install_lock makes the
+        # check-then-install atomic so nothing subscribes twice (issue #527).
+        @install_lock = Mutex.new
+        @subscriptions = [].freeze
+        @installed = false
 
         class << self
           def install!
-            return false if @installed
+            @install_lock.synchronize do
+              next false if @installed
 
-            @subscriptions = [
+              @subscriptions = subscribe_all
+              @installed = true
+            end
+          end
+
+          def installed?
+            @install_lock.synchronize { @installed }
+          end
+
+          def reset!
+            @install_lock.synchronize do
+              @subscriptions.each { |s| ActiveSupport::Notifications.unsubscribe(s) }
+              @subscriptions = [].freeze
+              @installed = false
+            end
+          end
+
+          private
+
+          def subscribe_all
+            [
               subscribe("pgbus.executor.execute") { |event| on_executor_execute(event) },
               subscribe("pgbus.job_completed") { |event| on_job_completed(event) },
               subscribe("pgbus.job_failed") { |event| on_job_failed(event) },
@@ -48,22 +73,9 @@ module Pgbus
               # pgbus_pool_available gauges for AppSignal (see probe.rb).
               # Subscribing here too would double-report on every heartbeat
               # and skew the gauges.
-            ]
-            @installed = true
-          end
-
-          def installed?
-            @installed == true
+            ].freeze
           end
 
-          def reset!
-            @subscriptions&.each { |s| ActiveSupport::Notifications.unsubscribe(s) }
-            @subscriptions = []
-            @installed = false
-          end
-
-          private
-
           def subscribe(name, &block)
             # silence rubocop unused
             ActiveSupport::Notifications.subscribe(name) do |*args|

data/lib/pgbus/integrations/appsignal.rb

@@ -28,41 +28,51 @@ module Pgbus
       DASHBOARD_PATH = File.expand_path("appsignal/dashboard.json", __dir__).freeze
       DASHBOARDS_DIR = File.expand_path("appsignal/dashboards", __dir__).freeze
 
+      # Installed once at boot (the engine's after_initialize); @lock makes
+      # the check-then-install atomic and guards the parsed-dashboard memos
+      # (issue #527).
+      @lock = Mutex.new
+      @installed = false
+
       module_function
 
-      def install! # rubocop:disable Naming/PredicateMethod
+      def install!
         return false unless defined?(::Appsignal)
-        return false if @installed
 
-        Subscriber.install!
-        Probe.install! if Pgbus.configuration.appsignal_probe_enabled
-        @installed = true
-        Pgbus.logger.info { "[Pgbus] AppSignal integration installed" }
-        true
+        installed_now = @lock.synchronize do
+          next false if @installed
+
+          Subscriber.install!
+          Probe.install! if Pgbus.configuration.appsignal_probe_enabled
+          @installed = true
+        end
+        Pgbus.logger.info { "[Pgbus] AppSignal integration installed" } if installed_now
+        installed_now
       end
 
       def installed?
-        @installed == true
+        @lock.synchronize { @installed }
       end
 
       def dashboard_definition
-        @dashboard_definition ||= JSON.parse(File.read(DASHBOARD_PATH))
+        @lock.synchronize { @dashboard_definition ||= JSON.parse(File.read(DASHBOARD_PATH)) }
       end
 
       def dashboard_definitions
-        @dashboard_definitions ||=
-          Dir[File.join(DASHBOARDS_DIR, "*.json")].map do |path|
-            JSON.parse(File.read(path))
-          end
+        @lock.synchronize do
+          @dashboard_definitions ||= Dir[File.join(DASHBOARDS_DIR, "*.json")].map { |path| JSON.parse(File.read(path)) }
+        end
       end
 
       # Test hook: tear everything down so a fresh install! can run.
       def reset!
         Subscriber.reset! if defined?(Subscriber)
         Probe.reset! if defined?(Probe)
-        @installed = false
-        @dashboard_definition = nil
-        @dashboard_definitions = nil
+        @lock.synchronize do
+          @installed = false
+          @dashboard_definition = nil
+          @dashboard_definitions = nil
+        end
       end
     end
   end

data/lib/pgbus/mcp/rack_app.rb

@@ -37,7 +37,7 @@ module Pgbus
       # First mcp release with the transport's allowed_hosts / allowed_origins /
       # dns_rebinding_protection options. Older gems would raise ArgumentError
       # on the pass-through; fail with the fix spelled out instead.
-      MIN_MCP_VERSION = Gem::Version.new("0.23.0")
+      MIN_MCP_VERSION = Gem::Version.new("0.23.0").freeze
       # Only the JSON body string is frozen and reused. The outer response triple
       # and its headers hash MUST be built fresh per call (#unauthorized) so
       # downstream Rack middleware can mutate them — Rack::TempfileReaper assigns