pgbus 0.18.2 → 0.19.1

data/lib/pgbus/client.rb

@@ -20,9 +20,6 @@ module Pgbus
 
     attr_reader :pgmq, :config, :connection_health
 
-    PGMQ_REQUIRE_MUTEX = Mutex.new
-    private_constant :PGMQ_REQUIRE_MUTEX
-
     # Fixed advisory-lock key serializing pgmq schema installation across
     # processes (issue #397). "pgmqinst" in ASCII hex — arbitrary but stable;
     # it only has to be identical in every process that can install.
@@ -55,6 +52,11 @@ module Pgbus
     # (issue #397, forensics in getzazu/app#3413).
     @pgmq_install_mutex = Mutex.new
 
+    # Serializes the one-time `require "pgmq"` (see load_pgmq_gem!). Class
+    # state built at load, not a constant: a constant holding a Mutex is
+    # unreadable from a non-main Ractor (issue #527).
+    @pgmq_require_mutex = Mutex.new
+
     class << self
       attr_reader :pgmq_install_mutex
     end
@@ -82,7 +84,10 @@ module Pgbus
     # instead of the global Kernel#require, which — if stubbed before pgmq is
     # genuinely loaded — permanently prevents the real gem from ever loading.
     def self.load_pgmq_gem!
-      PGMQ_REQUIRE_MUTEX.synchronize do
+      # The lock is Client's class state, which a subclass does not inherit.
+      return Client.load_pgmq_gem! unless equal?(Client)
+
+      @pgmq_require_mutex.synchronize do
         Object.const_set(:PGMQ, Module.new) unless defined?(::PGMQ)
         require "pgmq"
       end
@@ -920,10 +925,33 @@ module Pgbus
 
     # --- LISTEN/NOTIFY management (PGMQ v1.11.0+) ---
 
-    def wait_for_notify(queue_name, timeout: nil, &block)
-      full_name = config.queue_name(queue_name)
-      with_stale_connection_retry do
-        synchronized { @pgmq.wait_for_notify(full_name, timeout: timeout, &block) }
+    # Blocks until PGMQ's insert trigger notifies +queue_name+ or +timeout+
+    # seconds pass (nil waits indefinitely); returns the channel, or nil on a
+    # timeout, and yields (channel, pid, payload) like
+    # PG::Connection#wait_for_notify. The wait gets a dedicated connection of
+    # its own (worker_notify_connection_options), opened for it and closed
+    # after: never a pool connection, and never under @pgmq_mutex, which on
+    # the shared-AR path serializes every PGMQ call, so holding it across a
+    # blocking wait stalled the whole client (issue #527). The shared-AR path
+    # therefore needs worker_notify_database_url (or a base database_url).
+    # A connection found dead before it was used (STALE_CONNECTION_PATTERNS)
+    # is replaced and the wait starts again, as with_stale_connection_retry
+    # did for the pooled wait; a connection lost mid-wait is not retried.
+    def wait_for_notify(queue_name, timeout: nil, &)
+      channel = "pgmq.q_#{config.queue_name(queue_name)}.INSERT"
+      attempts = 0
+      begin
+        listen_on_dedicated_connection(channel, timeout, &)
+      rescue StandardError => e
+        attempts += 1
+        raise unless attempts <= STALE_RETRY_ATTEMPTS && stale_pg_error?(e)
+
+        sleep STALE_RETRY_DELAYS[[attempts - 1, STALE_RETRY_DELAYS.size - 1].min]
+        Pgbus.logger.warn do
+          "[Pgbus::Client] Retrying wait_for_notify after a stale connection " \
+            "(attempt #{attempts}/#{STALE_RETRY_ATTEMPTS}): #{e.message}"
+        end
+        retry
       end
     end
 
@@ -1199,7 +1227,8 @@ module Pgbus
     def ensure_pgmq_schema
       return if @schema_ensured
 
-      self.class.pgmq_install_mutex.synchronize do
+      # Client's lock, also for a subclass: it serializes every client's install.
+      Client.pgmq_install_mutex.synchronize do
         return if @schema_ensured
 
         # Cache only a durable result: true only when this call owned the
@@ -2069,6 +2098,22 @@ module Pgbus
       end
     end
 
+    def listen_on_dedicated_connection(channel, timeout, &)
+      conn = DedicatedConnection.connect(config.worker_notify_connection_options)
+      begin
+        conn.exec("LISTEN #{conn.quote_ident(channel)}")
+        conn.wait_for_notify(timeout, &)
+      ensure
+        DedicatedConnection.close_quietly(conn)
+      end
+    end
+
+    # A raw PG::Error (the dedicated connection is not wrapped by pgmq-ruby)
+    # whose message says the connection was dead before it was used.
+    def stale_pg_error?(error)
+      defined?(::PG::Error) && error.is_a?(::PG::Error) && stale_connection_error?(error)
+    end
+
     def stale_connection_error?(error)
       # Substring match against any stale-connection pattern. Deliberately a
       # single Regexp rather than `STALE_CONNECTION_PATTERNS.any? { |p| msg.include?(p) }`:

data/lib/pgbus/concurrency.rb

@@ -12,6 +12,9 @@ module Pgbus
     # parked job whose class no longer resolves.
     DEFAULT_DURATION = 15 * 60
 
+    # Class-body macro: it runs while the job class loads (at boot, or under the
+    # autoloader's lock), stores a frozen Hash, and the reader only reads it.
+    # rubocop:disable-next ThreadSafety/ClassInstanceVariable
     class_methods do
       # Limit concurrent execution of jobs with the same key.
       #

data/lib/pgbus/configuration/deploy_settings.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Configuration
+    # Deploy tasks (issue #536): one-time jobs the scheduler enqueues once
+    # every live pgbus process runs the new app revision.
+    #
+    #   config.app_revision = -> { File.read("REVISION").strip } # or a String
+    #   config.deploy_tasks_files = ["config/deploy_tasks.yml"]
+    #   config.deploy_task_run_retention = 30.days
+    module DeploySettings
+      DEFAULT_DEPLOY_TASK_RUN_RETENTION = 90 * 24 * 3600
+
+      attr_accessor :app_revision, :deploy_tasks, :deploy_tasks_files, :deploy_tasks_enabled
+      attr_reader :deploy_task_run_retention
+
+      def deploy_task_run_retention=(value)
+        @deploy_task_run_retention = coerce_duration!(value, :deploy_task_run_retention)
+      end
+
+      private
+
+      def initialize_deploy_settings
+        @app_revision = nil
+        @deploy_tasks = nil
+        @deploy_tasks_files = nil
+        @deploy_tasks_enabled = true
+        @deploy_task_run_retention = DEFAULT_DEPLOY_TASK_RUN_RETENTION
+      end
+
+      def validate_deploy_settings!
+        unless app_revision.nil? || app_revision.is_a?(String) || app_revision.respond_to?(:call)
+          raise Pgbus::ConfigurationError, "app_revision must be a String, respond to #call, or be nil"
+        end
+
+        raise Pgbus::ConfigurationError, "deploy_tasks must be a Hash or nil" unless deploy_tasks.nil? || deploy_tasks.is_a?(Hash)
+        unless deploy_tasks_files.nil? || deploy_tasks_files.is_a?(Array)
+          raise Pgbus::ConfigurationError, "deploy_tasks_files must be an Array of paths or nil"
+        end
+        return if [true, false].include?(deploy_tasks_enabled)
+
+        raise Pgbus::ConfigurationError, "deploy_tasks_enabled must be true or false"
+      end
+    end
+  end
+end

data/lib/pgbus/configuration.rb

@@ -6,6 +6,7 @@ module Pgbus
   class Configuration
     include PresenceSettings
     include StreamAuthorization
+    include DeploySettings
 
     # Connection settings
     attr_accessor :database_url, :connection_params, :pool_size, :pool_timeout
@@ -51,6 +52,12 @@ module Pgbus
     # wait, so recycling/deploy never wedges on a permanently-stuck job.
     attr_accessor :stall_threshold, :read_timeout, :drain_timeout
 
+    # immediate_shutdown_grace (default 2s): on QUIT, a worker or consumer
+    # stops taking work and gives the jobs already running this long to finish
+    # before it kills their threads. Killed jobs' ActiveRecord connections are
+    # reclaimed, so none stays checked out by a dead thread.
+    attr_accessor :immediate_shutdown_grace
+
     # shutdown_timeout bounds how long the supervisor waits for its children
     # after forwarding TERM before escalating to SIGKILL. nil (default) derives
     # drain_timeout + SHUTDOWN_TIMEOUT_MARGIN, so raising drain_timeout keeps
@@ -132,8 +139,11 @@ module Pgbus
     # Event bus
     attr_reader :idempotency_ttl
 
-    # Logging
-    attr_accessor :logger
+    # Logging. Unset means Rails.logger when Rails has one at the time of the
+    # read, else a $stdout Logger. Resolved on every read rather than when the
+    # configuration is built, because Pgbus builds it when the gem loads,
+    # which in a Rails app is before Rails.logger exists.
+    attr_writer :logger
     attr_reader :log_format # rubocop:disable Style/AccessorGrouping
 
     # Error reporting — array of callable objects invoked on caught exceptions.
@@ -296,6 +306,7 @@ module Pgbus
       @stall_threshold = 90
       @read_timeout = 30
       @drain_timeout = 30
+      @immediate_shutdown_grace = 2
       @shutdown_timeout = nil
 
       @dispatch_interval = 1.0
@@ -327,8 +338,10 @@ module Pgbus
       @idempotency_ttl = 7 * 24 * 3600 # 7 days
       @allowed_global_id_models = nil # nil = allow all (for backwards compat)
 
-      @logger = (defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger) || Logger.new($stdout)
+      @logger = nil
+      @stdout_logger = Logger.new($stdout)
       @log_format = :text
+      @log_format_set = false # see #apply_log_format!
       @error_reporters = []
 
       @listen_notify = true
@@ -351,6 +364,7 @@ module Pgbus
       @recurring_tasks_files = nil
       @recurring_enabled = true
       @recurring_execution_retention = 7 * 24 * 3600 # 7 days
+      initialize_deploy_settings
 
       @zombie_detection = true
 
@@ -566,6 +580,7 @@ module Pgbus
       @statsd_port = 8125
 
       @eager_validation = true
+      @finalized = false
     end
 
     def queue_name(name)
@@ -619,6 +634,10 @@ module Pgbus
       ExecutionPools.normalize_mode(mode)
     end
 
+    def logger
+      @logger || rails_logger || @stdout_logger
+    end
+
     VALID_LOG_FORMATS = %i[text json].freeze
 
     def log_format=(format)
@@ -628,15 +647,26 @@ module Pgbus
       end
 
       @log_format = format
+      @log_format_set = true
+      apply_log_format!
+    end
+
+    # Installs log_format's formatter on the logger pgbus resolves now; a
+    # no-op until log_format is set. The configuration exists before
+    # Rails.logger does, so a log_format set that early (config/application.rb)
+    # formats only the $stdout fallback: the engine calls this again once
+    # Rails.logger exists.
+    def apply_log_format!
+      return unless @log_format_set
 
       # Only install pgbus's formatter when the logger still carries a nil or
       # pgbus-installed formatter. A user who set a custom logger with a custom
       # formatter keeps it — log_format only records the intended format for
       # pgbus's own defaults, it must not silently clobber their formatting.
-      old_formatter = @logger&.formatter
+      old_formatter = logger&.formatter
       return unless pgbus_installable_formatter?(old_formatter)
 
-      new_formatter = case format
+      new_formatter = case log_format
                       when :json then LogFormatter::JSON.new
                       when :text then LogFormatter::Text.new
                       end
@@ -649,7 +679,7 @@ module Pgbus
       # formatter. Re-extend the new formatter with the same module when the old
       # one carried it, so tagging keeps working (issue #334).
       preserve_tagged_logging(old_formatter, new_formatter)
-      @logger.formatter = new_formatter
+      logger.formatter = new_formatter
     end
 
     VALID_BROADCAST_MODES = %i[ephemeral durable].freeze
@@ -799,6 +829,36 @@ module Pgbus
       @worker_notify_scope = coerced
     end
 
+    # Settings finalize! never freezes: @error_reporters is a list of
+    # callables apps append to (`c.error_reporters << ...`), and
+    # @deprecation_warned is this object's own bookkeeping.
+    FINALIZE_SKIP = %i[@error_reporters @deprecation_warned @finalized].freeze
+    private_constant :FINALIZE_SKIP
+
+    # Replaces every plain-data setting (Strings, Symbols, numbers, booleans,
+    # nil, Regexps, classes, and Arrays/Hashes/Sets/Ranges of those) with a
+    # deep-frozen, Ractor-shareable copy, so each reader hands back a value
+    # any Ractor can read (issue #527). Settings that hold a callable or an
+    # object (the logger, error reporters, hooks, the web data source, a
+    # metrics backend) are left as they are: they stay main-Ractor-only.
+    # Writers keep working (a setter assigns a new value; Pgbus.configure
+    # finalizes again); only in-place mutation of a frozen collection raises.
+    # Called once at boot by Pgbus.finalize!.
+    def finalize!
+      instance_variables.each do |ivar|
+        next if FINALIZE_SKIP.include?(ivar)
+
+        value = instance_variable_get(ivar)
+        instance_variable_set(ivar, Shareable.freeze_copy(value)) if Shareable.data?(value)
+      end
+      @finalized = true
+      self
+    end
+
+    def finalized?
+      @finalized
+    end
+
     def validate!
       if pool_size && !(pool_size.is_a?(Numeric) && pool_size.positive?)
         raise Pgbus::ConfigurationError, "pool_size must be a positive number or nil (auto-tune)"
@@ -829,6 +889,8 @@ module Pgbus
       end
       raise Pgbus::ConfigurationError, "drain_timeout must be > 0" unless drain_timeout.is_a?(Numeric) && drain_timeout.positive?
 
+      validate_immediate_shutdown_grace!
+
       validate_shutdown_timeout!
 
       unless stats_flush_size.is_a?(Integer) && stats_flush_size.positive?
@@ -884,6 +946,7 @@ module Pgbus
       validate_streams!
       validate_metrics_backend!
       validate_fair_share!
+      validate_deploy_settings!
 
       self
     end
@@ -1279,15 +1342,17 @@ module Pgbus
       validate_processes!(options[:processes]) unless options[:processes].nil?
 
       normalized_name = name.to_s
-      @workers ||= []
+      existing = @workers || []
 
-      if @workers.any? { |c| capsule_name(c) == normalized_name }
+      if existing.any? { |c| capsule_name(c) == normalized_name }
         raise Pgbus::ConfigurationError, "capsule #{name.inspect} is already defined"
       end
 
       validate_no_queue_overlap!(queues)
 
-      @workers << { name: normalized_name, queues: queues, threads: threads, **options }
+      # A new list rather than <<: after finalize! @workers is frozen, and a
+      # writer keeps working (a later Pgbus.configure finalizes again).
+      @workers = [*existing, { name: normalized_name, queues: queues, threads: threads, **options }]
     end
 
     # Look up a capsule by its name. Accepts symbol or string. Returns the
@@ -1631,6 +1696,10 @@ module Pgbus
       end
     end
 
+    def rails_logger
+      ::Rails.logger if defined?(::Rails) && ::Rails.respond_to?(:logger)
+    end
+
     # When the logger's previous formatter was extended with
     # ActiveSupport::TaggedLogging::Formatter (the case for a Rails
     # Rails.logger), extend the replacement pgbus formatter with the same module
@@ -1718,6 +1787,15 @@ module Pgbus
             "#{name} must be a Numeric (seconds), ActiveSupport::Duration, or nil to disable, got #{value.inspect}"
     end
 
+    # A finite real number >= 0: NaN is not negative?, and an infinite grace
+    # would never kill a job that does not finish.
+    def validate_immediate_shutdown_grace!
+      grace = immediate_shutdown_grace
+      return if grace.is_a?(Numeric) && grace.real? && grace.finite? && grace >= 0
+
+      raise Pgbus::ConfigurationError, "immediate_shutdown_grace must be a finite number >= 0"
+    end
+
     def validate_positive_duration!(numeric, name)
       raise Pgbus::ConfigurationError, "#{name} must be a positive number, got #{numeric}" unless numeric.positive?
 

data/lib/pgbus/current_attributes.rb

@@ -20,6 +20,8 @@ module Pgbus
   module CurrentAttributes
     METADATA_KEY = "pgbus_current"
 
+    # The names already warned about, so each missing class is logged once.
+    # Built at load; Concurrent::Map#compute_if_absent is atomic.
     @missing_warned = Concurrent::Map.new
 
     class << self
@@ -115,7 +117,7 @@ module Pgbus
       def resolve_class(name)
         name.constantize
       rescue NameError
-        @missing_warned.compute_if_absent(name) do
+        @missing_warned.compute_if_absent(name) do # rubocop:disable ThreadSafety/ClassInstanceVariable -- see @missing_warned
           Pgbus.logger.warn { "[Pgbus] current_attributes: #{name} is not defined — skipping" }
           true
         end

data/lib/pgbus/dedup_cache.rb

@@ -17,6 +17,10 @@ module Pgbus
       @ttl = ttl
       @cache = Concurrent::Map.new
       @insertion_order = Concurrent::Array.new
+      # Serializes the check-then-act writers (mark!, expiry eviction, clear!)
+      # so the size bound and @insertion_order stay consistent (issue #527).
+      # seen? reads @cache without it.
+      @lock = Mutex.new
     end
 
     # Returns true if this key was already seen (duplicate).
@@ -26,7 +30,7 @@ module Pgbus
       return false unless entry
 
       if expired?(entry)
-        evict(key)
+        @lock.synchronize { evict(key) if @cache[key].equal?(entry) }
         return false
       end
 
@@ -36,11 +40,13 @@ module Pgbus
     # Mark a key as seen. Call this after successfully claiming idempotency
     # in the database so future lookups skip the DB.
     def mark!(key)
-      already_present = @cache.key?(key)
-      evict_oldest if !already_present && @cache.size >= @max_size
+      @lock.synchronize do
+        already_present = @cache.key?(key)
+        evict_oldest if !already_present && @cache.size >= @max_size
 
-      @cache[key] = monotonic_now
-      @insertion_order << key unless already_present
+        @cache[key] = monotonic_now
+        @insertion_order << key unless already_present
+      end
     end
 
     def size
@@ -48,8 +54,10 @@ module Pgbus
     end
 
     def clear!
-      @cache.clear
-      @insertion_order.clear
+      @lock.synchronize do
+        @cache.clear
+        @insertion_order.clear
+      end
     end
 
     private

data/lib/pgbus/deploy/revision.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Deploy
+    # The app revision this process runs (issue #536). Every heartbeat row
+    # carries it in metadata["revision"]; the deploy-task rollover gate
+    # compares those against the scheduler's own.
+    #
+    # First match wins, blank counts as absent:
+    #   config.app_revision (String or callable) → PGBUS_REVISION → DASH_VERSION
+    #   → KAMAL_VERSION → the REVISION file at Rails.root (Capistrano) → nil
+    #
+    # Computed per call, never memoized: it is read once per process
+    # registration, and a module-level memo would be unguarded shared state.
+    module Revision
+      ENV_KEYS = %w[PGBUS_REVISION DASH_VERSION KAMAL_VERSION].freeze
+
+      module_function
+
+      def current(config = Pgbus.configuration)
+        from_config(config) || from_env || from_file
+      end
+
+      def from_config(config)
+        value = config.app_revision
+        value = value.call if value.respond_to?(:call)
+        presence(value)
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] app_revision raised #{e.class}: #{e.message}; trying the environment" }
+        nil
+      end
+
+      def from_env
+        ENV_KEYS.each do |key|
+          value = presence(ENV.fetch(key, nil))
+          return value if value
+        end
+        nil
+      end
+
+      def from_file
+        return nil unless defined?(Rails) && Rails.respond_to?(:root) && Rails.root
+
+        path = Rails.root.join("REVISION")
+        File.exist?(path) ? presence(File.read(path)) : nil
+      rescue SystemCallError, IOError => e
+        # Read while every heartbeat initializes: an unreadable optional file
+        # must not stop a process from starting.
+        Pgbus.logger.warn { "[Pgbus] Could not read #{path}: #{e.class}: #{e.message}" }
+        nil
+      end
+
+      def presence(value)
+        string = value.to_s.strip
+        string.empty? ? nil : string
+      end
+    end
+  end
+end

data/lib/pgbus/deploy/rollover.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Pgbus
+  module Deploy
+    # Has the fleet rolled over to +revision+? (issue #536). A pure read of
+    # pgbus_processes: live rows (heartbeat within Heartbeat::ALIVE_THRESHOLD)
+    # whose metadata["revision"] differs from +revision+ are lagging (a missing
+    # revision is a pre-upgrade process, so it lags too), and at least one live
+    # worker on +revision+ must exist. The scheduler and the dashboard both
+    # consume the Status, so the page prints the reason the scheduler acts on.
+    module Rollover
+      # reason – :ready, :no_revision, :lagging or :no_workers.
+      # lagging – [{ kind:, hostname:, pid:, revision: }].
+      # live_by_revision – { revision => live process count }, nil for rows without one.
+      Status = Data.define(:revision, :lagging, :workers_live, :reason, :live_by_revision) do
+        def rolled_over?
+          reason == :ready
+        end
+      end
+
+      module_function
+
+      def check(revision:, now: Time.current)
+        cutoff = now - Process::Heartbeat::ALIVE_THRESHOLD
+        rows = ProcessEntry.where(last_heartbeat_at: cutoff..).pluck(:kind, :hostname, :pid, :metadata)
+        evaluate(rows.map { |kind, hostname, pid, metadata| { kind:, hostname:, pid:, metadata: } }, revision: revision)
+      end
+
+      # [live, revision] of the newest live scheduler row: the revision the
+      # dashboard judges the rollover against (the one the scheduler acts on).
+      def scheduler_revision(now: Time.current)
+        row = ProcessEntry.where(kind: "scheduler", last_heartbeat_at: (now - Process::Heartbeat::ALIVE_THRESHOLD)..)
+                          .order(created_at: :desc, id: :desc).first
+        row ? [true, revision_of(row.metadata)] : [false, nil]
+      end
+
+      # rows – live process rows as { kind:, hostname:, pid:, metadata: }.
+      def evaluate(rows, revision:)
+        live = rows.map { |r| { kind: r[:kind], hostname: r[:hostname], pid: r[:pid], revision: revision_of(r[:metadata]) } }
+        lagging = live.reject { |r| revision && r[:revision] == revision }
+        workers_live = live.count { |r| r[:kind] == "worker" && revision && r[:revision] == revision }
+
+        Status.new(revision: revision, lagging: revision ? lagging : [], workers_live: workers_live,
+                   reason: reason_for(revision, lagging, workers_live), live_by_revision: live.map { |r| r[:revision] }.tally)
+      end
+
+      def reason_for(revision, lagging, workers_live)
+        return :no_revision if revision.nil?
+        return :lagging if lagging.any?
+        return :no_workers if workers_live.zero?
+
+        :ready
+      end
+
+      def revision_of(metadata)
+        metadata = JSON.parse(metadata) if metadata.is_a?(String)
+        metadata.is_a?(Hash) ? metadata["revision"] : nil
+      rescue JSON::ParserError
+        nil
+      end
+    end
+  end
+end