pgbus 0.16.7 → 0.17.1

data/lib/pgbus/pgmq_schema.rb

@@ -14,6 +14,7 @@ module Pgbus
     class VersionNotFoundError < Pgbus::Error; end
 
     SCHEMA_DIR = File.expand_path("pgmq_schema", __dir__).freeze
+    FIXUPS_DIR = File.join(SCHEMA_DIR, "fixups").freeze
 
     class << self
       # Returns the latest vendored PGMQ version string.
@@ -23,9 +24,37 @@ module Pgbus
 
       # Returns sorted list of all vendored PGMQ versions.
       def available_versions
-        Dir.glob(File.join(SCHEMA_DIR, "pgmq_v*.sql"))
-           .map { |f| File.basename(f).match(/pgmq_v(.+)\.sql/)[1] }
-           .sort_by { |v| Gem::Version.new(v) }
+        versions_in(SCHEMA_DIR)
+      end
+
+      # Versions that ship a table fixup, in version order.
+      #
+      # An upgrade drops every pgmq function and composite type and re-runs the
+      # target version's schema, which recreates both — but never touches an
+      # existing table. An upstream hop that ALTERs one (1.13.0 moves
+      # partitioned queues' msg_id from GENERATED ALWAYS to BY DEFAULT) needs
+      # its own step, and that is what these files are.
+      def fixup_versions
+        versions_in(FIXUPS_DIR)
+      end
+
+      # Concatenated fixups for every version in (after, upto], in version
+      # order. Returns "" when the hop crosses none.
+      #
+      # @param after [String, nil] the installed version; nil (no recorded
+      #   version) applies every fixup up to the target, which is safe because
+      #   each one is idempotent.
+      # @param upto [String] the version being upgraded to
+      def fixups_sql(after:, upto: latest_version)
+        ceiling = Gem::Version.new(upto)
+        floor = after && Gem::Version.new(after)
+
+        applicable = fixup_versions.select do |version|
+          candidate = Gem::Version.new(version)
+          candidate <= ceiling && (floor.nil? || candidate > floor)
+        end
+
+        applicable.map { |version| File.read(File.join(FIXUPS_DIR, "pgmq_v#{version}.sql")) }.join("\n")
       end
 
       # Returns the filesystem path to the vendored SQL file for a given version.
@@ -177,6 +206,15 @@ module Pgbus
 
       private
 
+      # Vendored versions discovered in a directory, in version order. Schema
+      # files and fixups share this so the two can never drift into different
+      # discovery rules.
+      def versions_in(dir)
+        Dir.glob(File.join(dir, "pgmq_v*.sql"))
+           .map { |f| File.basename(f).match(/pgmq_v(.+)\.sql/)[1] }
+           .sort_by { |v| Gem::Version.new(v) }
+      end
+
       # Strips extension-specific blocks (pg_extension_config_dump, pg_depend checks)
       # that only work when pgmq is installed as an extension.
       def strip_extension_only_blocks(sql)

data/lib/pgbus/process/claim_buffer.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Process
+    # The read-ahead buffer a Worker or Consumer owns (issue #486).
+    #
+    # Under database latency the claim loop used to issue one read round trip
+    # per job: once the pool is full, slots free one at a time and each loop
+    # step read `qty = free slots` = 1. With `read_ahead = N` the loop claims
+    # up to N messages beyond its free slots and parks them here, so one read
+    # returns a batch and the pool never waits on the reader.
+    #
+    # Surplus claims cannot live inside the execution pool: AsyncPool#post
+    # raises at zero capacity and a queued block could not be handed back on
+    # drain. So the buffer is drained into the pool only while it has a free
+    # slot, and returned to the queue (vt: 0) on drain, recycle or shutdown.
+    #
+    # Every buffered message is kept invisible by a VisibilityHeartbeat hold
+    # from claim until the run starts, under the executor's key
+    # (`source_queue || queue_name`, `prefixed: source_queue.nil?`), so the
+    # run's own tracking takes over the same entry with no gap. The hold does
+    # not know the job class, so a job that opted out with
+    # `pgbus_visibility_heartbeat false` is still held while it waits here:
+    # the opt-out covers its run, not the wait. With the heartbeat disabled
+    # globally there is no hold at all (Configuration#validate! warns).
+    #
+    # Only ever touched from the owning process's loop thread, so no mutex.
+    class ClaimBuffer
+      Claim = Struct.new(:queue_name, :message, :source_queue, :hold)
+
+      HOLD_LABEL = "(read-ahead)"
+
+      def initialize(config: Pgbus.configuration)
+        @config = config
+        @claims = []
+      end
+
+      def size
+        @claims.size
+      end
+
+      def empty?
+        @claims.empty?
+      end
+
+      def push(queue_name, message, source_queue = nil, client: Pgbus.client)
+        hold = VisibilityHeartbeat.hold(
+          client: client, queue_name: source_queue || queue_name, prefixed: source_queue.nil?,
+          msg_id: message.msg_id, job_class: HOLD_LABEL, config: @config
+        )
+        @claims << Claim.new(queue_name, message, source_queue, hold)
+        self
+      end
+
+      def shift
+        @claims.shift
+      end
+
+      # Yields the oldest claims, one per free pool slot, and returns how many.
+      # Capacity is re-read before every claim: the block's post takes a slot.
+      def drain_into(pool)
+        drained = 0
+        while !@claims.empty? && pool.available_capacity.positive?
+          yield @claims.shift
+          drained += 1
+        end
+        drained
+      end
+
+      # How many messages the next read should claim: enough to fill the free
+      # slots and top the buffer back up to `read_ahead`, never more than
+      # `prefetch_room` (prefetch_limit minus everything already claimed).
+      # With read_ahead 0 and an empty buffer this is `free_slots`: the
+      # pre-#486 qty.
+      def deficit(free_slots:, read_ahead:, prefetch_room: nil)
+        want = free_slots + read_ahead - @claims.size
+        want = [want, prefetch_room].min if prefetch_room
+        want.clamp(0..)
+      end
+
+      # Hand every buffered message back to the queue right away instead of
+      # leaving it invisible until its timeout. Their read_ct has already been
+      # bumped, the same cost a crash or a stale claim pays. A failed return
+      # is logged and skipped: the hold is already dropped, so the message
+      # reappears when its current timeout runs out. Returns the count. One
+      # claim at a time, so #size stays true while the returns are written.
+      def return_all!(client: Pgbus.client)
+        returned = 0
+        while (claim = @claims.shift)
+          return_claim(claim, client)
+          returned += 1
+        end
+        returned
+      end
+
+      private
+
+      # The hold goes first: a heartbeat tick that still held the entry could
+      # otherwise write its extension after the vt: 0 and hide the message
+      # again for a full visibility_timeout. (VisibilityHeartbeat#extend!
+      # re-checks registration, which shrinks the remaining window to the
+      # tick's own check-then-write.)
+      def return_claim(claim, client)
+        VisibilityHeartbeat.release(claim.hold)
+        queue = claim.source_queue || claim.queue_name
+        client.set_visibility_timeout(queue, claim.message.msg_id.to_i, vt: 0, prefixed: claim.source_queue.nil?)
+      rescue StandardError => e
+        Pgbus.logger.warn do
+          "[Pgbus] Could not return read-ahead message msg_id=#{claim.message.msg_id} queue=#{queue}: " \
+            "#{e.class}: #{e.message}"
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/process/consumer.rb

@@ -7,7 +7,7 @@ module Pgbus
     class Consumer
       include SignalHandler
 
-      attr_reader :topics, :threads, :config, :execution_mode,
+      attr_reader :topics, :threads, :config, :execution_mode, :read_ahead,
                   :queue_names, :wake_signal, :notify_retry_backoff, :circuit_breaker
       # notify_listener is writable so tests can simulate a start_notify_listener
       # success from inside a stub (production sets it in start_notify_listener).
@@ -30,6 +30,11 @@ module Pgbus
         @jobs_processed.value
       end
 
+      # Messages claimed by read-ahead and not yet handed to the pool (#486).
+      def buffered
+        @claim_buffer.size
+      end
+
       # Seed the processed-job counter. Used by tests to drive the recycle
       # thresholds without running thousands of real jobs; production only ever
       # increments it via the AtomicFixnum during message handling.
@@ -60,7 +65,7 @@ module Pgbus
                      queue_names: nil, liveness_pipe: nil, stat_buffer: :default,
                      notify_listener: nil, notify_retry_at: 0.0,
                      notify_retry_backoff: NOTIFY_RETRY_BASE_SECONDS,
-                     started_at_monotonic: nil, wake_pipe: nil)
+                     started_at_monotonic: nil, wake_pipe: nil, read_ahead: nil)
         @topics = Array(topics)
         @threads = threads
         @config = config
@@ -68,6 +73,8 @@ module Pgbus
         @shutting_down = false
         @recycling = false
         @jobs_processed = Concurrent::AtomicFixnum.new(0)
+        @read_ahead = read_ahead || config.read_ahead
+        @claim_buffer = ClaimBuffer.new(config: config)
         @loop_tick_at = Concurrent::AtomicReference.new(nil)
         @started_at_monotonic = started_at_monotonic || monotonic_now
         @wake_signal = WakeSignal.new
@@ -134,7 +141,12 @@ module Pgbus
           check_recycle
           ensure_notify_listener
 
-          break if @shutting_down
+          if @shutting_down
+            # No drain loop here: buffered claims go straight back to the
+            # queue; shutdown then waits for the ones already running.
+            @claim_buffer.return_all!(client: Pgbus.client) unless @claim_buffer.empty?
+            break
+          end
 
           consume
           @stat_buffer&.flush_if_due
@@ -171,20 +183,35 @@ module Pgbus
         @queue_names = @registry.queue_names_for_topics(topics)
       end
 
+      # Same loop step as Worker#claim_and_execute (issue #486): feed
+      # buffered claims to free slots, read the deficit, feed again, and wait
+      # only when the step moved nothing.
       def consume
-        idle = @pool.available_capacity
-        return @wake_signal.wait(timeout: wake_timeout) if idle <= 0
-
-        tagged_messages = fetch_messages(idle)
-
-        if tagged_messages.empty?
-          @wake_signal.wait(timeout: wake_timeout)
-          return
+        drained = drain_claim_buffer
+        claimed = claim_deficit
+        drained += drain_claim_buffer
+        @wake_signal.wait(timeout: wake_timeout) if drained.zero? && claimed.zero?
+      end
+
+      # The hold is released before handle_message's own heartbeat tracking
+      # registers the same key, so the message is never untracked in between.
+      def drain_claim_buffer
+        @claim_buffer.drain_into(@pool) do |claim|
+          @pool.post do
+            VisibilityHeartbeat.release(claim.hold)
+            handle_message(claim.message, claim.queue_name)
+          end
         end
+      end
 
-        tagged_messages.each do |queue_name, message|
-          @pool.post { handle_message(message, queue_name) }
-        end
+      def claim_deficit
+        want = @claim_buffer.deficit(free_slots: @pool.available_capacity, read_ahead: @read_ahead)
+        return 0 if want.zero?
+
+        tagged_messages = fetch_messages(want)
+        client = Pgbus.client
+        tagged_messages.each { |queue_name, message| @claim_buffer.push(queue_name, message, client: client) }
+        tagged_messages.size
       end
 
       # Returns an array of [queue_name, message] pairs. Queues whose circuit

data/lib/pgbus/process/supervisor.rb

@@ -171,7 +171,7 @@ module Pgbus
       private_constant :ROLE_FLAGS
 
       def log_boot_banner
-        Pgbus.logger.info { "[Pgbus] boot: pgbus #{Pgbus::VERSION} pid=#{::Process.pid}" }
+        Pgbus.logger.info { "[Pgbus] boot: pgbus #{Pgbus::VERSION} pid=#{::Process.pid} jit=#{RubyJit.label}" }
         Pgbus.logger.info do
           "[Pgbus] boot: connection=#{redacted_connection_target} pool=#{banner_field { config.resolved_pool_size }}"
         end
@@ -330,6 +330,7 @@ module Pgbus
             queues: queues, threads: threads, config: config,
             single_active_consumer: single_active, consumer_priority: priority,
             execution_mode: exec_mode, group_mode: grp_mode,
+            read_ahead: config.read_ahead_for(worker_config),
             liveness_pipe: liveness_writer, wake_pipe: wake_reader
           )
           worker.run
@@ -498,6 +499,7 @@ module Pgbus
           setup_child_process
           load_rails_app
           consumer = Consumer.new(topics: topics, threads: threads, config: config,
+                                  read_ahead: config.read_ahead_for(consumer_config),
                                   liveness_pipe: liveness_writer, wake_pipe: wake_reader)
           consumer.run
         end

data/lib/pgbus/process/worker.rb

@@ -7,7 +7,7 @@ module Pgbus
     class Worker
       include SignalHandler
 
-      attr_reader :queues, :threads, :config, :execution_mode,
+      attr_reader :queues, :threads, :config, :execution_mode, :read_ahead,
                   :rate_counter, :wake_signal, :restore_streak, :lifecycle
       # stat_buffer is writable so a test can swap in a buffer double after
       # construction to assert graceful_shutdown / check_recycle flush it. The
@@ -35,7 +35,7 @@ module Pgbus
                      rate_counter: nil, wake_signal: nil, stat_buffer: :default,
                      notify_listener: nil, notify_retry_at: 0.0,
                      notify_retry_backoff: NOTIFY_RETRY_BASE_SECONDS,
-                     started_at_monotonic: nil, wake_pipe: nil)
+                     started_at_monotonic: nil, wake_pipe: nil, read_ahead: nil)
         @queues = Array(queues)
         @initial_queues = @queues.dup.freeze
         @wildcard = @queues.include?("*")
@@ -65,7 +65,10 @@ module Pgbus
         @last_wildcard_resolve = nil
         @jobs_processed = Concurrent::AtomicFixnum.new(0)
         @jobs_failed = Concurrent::AtomicFixnum.new(0)
+        # in_flight counts every claimed message: buffered and running.
         @in_flight = Concurrent::AtomicFixnum.new(0)
+        @read_ahead = read_ahead || config.read_ahead
+        @claim_buffer = ClaimBuffer.new(config: config)
         @loop_tick_at = Concurrent::AtomicReference.new(nil)
         @rate_counter = rate_counter || RateCounter.new(:processed, :failed, :dequeued)
         @started_at = Time.current
@@ -119,6 +122,7 @@ module Pgbus
           jobs_processed: @jobs_processed.value,
           jobs_failed: @jobs_failed.value,
           in_flight: @in_flight.value,
+          buffered: @claim_buffer.size,
           state: @lifecycle.state,
           execution_mode: @execution_mode,
           consumer_priority: @consumer_priority,
@@ -184,6 +188,9 @@ module Pgbus
           check_recycle
           refresh_wildcard_queues
           ensure_notify_listener
+          # Draining, paused or stopped: claims nobody will run here go back
+          # to the queue now rather than when their timeout runs out.
+          return_claims unless @lifecycle.can_process?
 
           break if @lifecycle.stopped?
           # quiesced? (all slots free), not idle? (any slot free) — exiting
@@ -240,32 +247,53 @@ module Pgbus
 
       private
 
+      # One loop step (issue #486): feed buffered claims to free slots, read
+      # the deficit (free slots + read_ahead - buffered, capped by
+      # prefetch_limit), feed again. With read_ahead 0 the buffer never
+      # outlives the step and qty is the free slot count, as before. Waits
+      # only when the step moved nothing, so a full pool with a topped-up
+      # buffer sleeps until a slot frees (the pool's on_state_change wakes it).
       def claim_and_execute
-        poll_interval = wake_timeout
-
-        idle = @pool.available_capacity
-        return @wake_signal.wait(timeout: poll_interval) if idle <= 0
-
-        if config.prefetch_limit
-          available = config.prefetch_limit - @in_flight.value
-          return @wake_signal.wait(timeout: poll_interval) if available <= 0
-
-          idle = [idle, available].min
+        drained = drain_claim_buffer
+        claimed = claim_deficit
+        drained += drain_claim_buffer
+        @wake_signal.wait(timeout: wake_timeout) if drained.zero? && claimed.zero?
+      end
+
+      # The hold is released before the executor's own heartbeat tracking
+      # registers the same key, so the message is never untracked in between.
+      def drain_claim_buffer
+        @claim_buffer.drain_into(@pool) do |claim|
+          @pool.post do
+            VisibilityHeartbeat.release(claim.hold)
+            process_message(claim.message, claim.queue_name, source_queue: claim.source_queue)
+          end
         end
+      end
 
-        tagged_messages = fetch_messages(idle)
+      def claim_deficit
+        prefetch_room = config.prefetch_limit && (config.prefetch_limit - @in_flight.value)
+        want = @claim_buffer.deficit(free_slots: @pool.available_capacity, read_ahead: @read_ahead,
+                                     prefetch_room: prefetch_room)
+        return 0 if want.zero?
 
-        if tagged_messages.empty?
-          @wake_signal.wait(timeout: poll_interval)
-          return
-        end
+        tagged_messages = fetch_messages(want)
+        return 0 if tagged_messages.empty?
 
         @rate_counter.increment(:dequeued, tagged_messages.size)
+        client = Pgbus.client
         tagged_messages.each do |queue_name, message, source_queue|
           detect_zombie(queue_name, message)
           @in_flight.increment
-          @pool.post { process_message(message, queue_name, source_queue: source_queue) }
+          @claim_buffer.push(queue_name, message, source_queue, client: client)
         end
+        tagged_messages.size
+      end
+
+      def return_claims
+        return if @claim_buffer.empty?
+
+        @in_flight.decrement(@claim_buffer.return_all!(client: Pgbus.client))
       end
 
       # Returns an array of [queue_name, message] pairs so we always know
@@ -831,7 +859,8 @@ module Pgbus
           "rates" => @rate_counter.rates.transform_keys(&:to_s),
           "jobs_processed" => @jobs_processed.value,
           "jobs_failed" => @jobs_failed.value,
-          "in_flight" => @in_flight.value
+          "in_flight" => @in_flight.value,
+          "buffered" => @claim_buffer.size
         }
       end
 

data/lib/pgbus/ruby_jit.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Pgbus
+  # Reports which Ruby JIT this process runs under (issue #484). Read-only:
+  # Rails owns turning YJIT on (config.yjit), pgbus only makes the state
+  # visible in the boot log and `pgbus doctor`.
+  module RubyJit
+    module_function
+
+    def label
+      return "yjit" if yjit_available? && RubyVM::YJIT.enabled?
+      return "zjit" if defined?(RubyVM::ZJIT) && RubyVM::ZJIT.enabled?
+
+      "none"
+    end
+
+    def yjit_available?
+      defined?(RubyVM::YJIT) ? true : false
+    end
+  end
+end

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.16.7"
+  VERSION = "0.17.1"
 end

data/lib/pgbus/visibility_heartbeat.rb

@@ -30,8 +30,9 @@ module Pgbus
     #
     # The ninth member takes the struct out of the 80-byte slot it used to fit
     # (measured: 80 → 160). That is paid at most once per in-flight message, so
-    # the whole table is bounded by the execution pool's capacity — a handful of
-    # entries per process, not one per enqueued job.
+    # the whole table is bounded by the execution pool's capacity plus the
+    # read-ahead buffer (threads + read_ahead per process, issue #486) — a
+    # handful of entries per process, not one per enqueued job.
     Entry = Struct.new(:client, :queue_name, :prefixed, :msg_id, :job_class, :extended_at, :extensions,
                        :concurrency, :on_beat, keyword_init: true)
 
@@ -81,6 +82,25 @@ module Pgbus
         end
       end
 
+      # Keep a claimed-but-not-started message invisible until #release.
+      # Used by Process::ClaimBuffer for read-ahead (issue #486): a buffered
+      # message waits for a free slot and must not be redelivered meanwhile.
+      # Same key as #track, so the run's own tracking takes over after the
+      # hold is released. Returns the entry, or nil when the heartbeat is off.
+      def hold(client:, queue_name:, msg_id:, prefixed: true, job_class: nil, config: Pgbus.configuration)
+        return unless config.visibility_heartbeat
+
+        entry = Entry.new(client: client, queue_name: queue_name, prefixed: prefixed, msg_id: msg_id.to_i,
+                          job_class: job_class, extended_at: monotonic_now, extensions: 0)
+        register(entry, config)
+        entry
+      end
+
+      # Drop a #hold. A nil entry (heartbeat off) is a no-op.
+      def release(entry)
+        unregister(entry) if entry
+      end
+
       # Extend every tracked message whose last extension is older than the
       # heartbeat interval. Public so tests and callers without the thread
       # can drive it.
@@ -127,11 +147,22 @@ module Pgbus
         end
       end
 
+      # Only the entry that is registered under the key: a stale hold released
+      # after #track re-registered the same message must not drop the running
+      # job's entry (issue #486).
       def unregister(entry)
-        synchronize { entries.delete(key_for(entry)) }
+        synchronize do
+          key = key_for(entry)
+          entries.delete(key) if entries[key].equal?(entry)
+        end
       end
 
       def extend!(entry, now:, config:)
+        # tick! picked this entry under the mutex but runs here without it: if
+        # the entry was released meanwhile (a read-ahead claim handed back with
+        # vt: 0, a job that finished), extending it would undo that.
+        return unless registered?(entry)
+
         vt = config.visibility_timeout
         entry.client.set_visibility_timeout(entry.queue_name, entry.msg_id, vt: vt, prefixed: entry.prefixed)
         entry.extended_at = now
@@ -218,6 +249,10 @@ module Pgbus
         @entries ||= {}
       end
 
+      def registered?(entry)
+        synchronize { entries[key_for(entry)].equal?(entry) }
+      end
+
       def key_for(entry)
         [entry.queue_name, entry.msg_id]
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgbus
 version: !ruby/object:Gem::Version
-  version: 0.16.7
+  version: 0.17.1
 platform: ruby
 authors:
 - Mikael Henriksson
@@ -343,9 +343,12 @@ files:
 - lib/pgbus/outbox.rb
 - lib/pgbus/outbox/poller.rb
 - lib/pgbus/pgmq_schema.rb
+- lib/pgbus/pgmq_schema/fixups/pgmq_v1.13.0.sql
 - lib/pgbus/pgmq_schema/pgmq_v1.11.0.sql
 - lib/pgbus/pgmq_schema/pgmq_v1.11.1.sql
 - lib/pgbus/pgmq_schema/pgmq_v1.12.0.sql
+- lib/pgbus/pgmq_schema/pgmq_v1.13.0.sql
+- lib/pgbus/process/claim_buffer.rb
 - lib/pgbus/process/consumer.rb
 - lib/pgbus/process/consumer_priority.rb
 - lib/pgbus/process/dispatcher.rb
@@ -374,6 +377,7 @@ files:
 - lib/pgbus/recurring/scheduler.rb
 - lib/pgbus/recurring/task.rb
 - lib/pgbus/retry_backoff.rb
+- lib/pgbus/ruby_jit.rb
 - lib/pgbus/serializer.rb
 - lib/pgbus/stat_buffer.rb
 - lib/pgbus/streams.rb