pgbus 0.9.10 → 0.10.0

data/lib/generators/pgbus/update_generator.rb

@@ -1,50 +1,26 @@
 # frozen_string_literal: true
 
 require "rails/generators"
-require "pgbus/generators/config_converter"
 require "pgbus/generators/migration_detector"
 require "pgbus/generators/database_target_detector"
 
 module Pgbus
   module Generators
-    # Upgrade command with two independent jobs:
-    #
-    #   1. Config conversion: if config/pgbus.yml exists, convert it to
-    #      config/initializers/pgbus.rb using the modern Ruby DSL. Skip
-    #      silently if the initializer already exists or the YAML is
-    #      absent — safe to re-run.
-    #
-    #   2. Migration detection: inspect the live database and add any
-    #      missing pgbus migrations to db/migrate (or db/pgbus_migrate
-    #      if a separate database is configured). Invokes each matching
-    #      sub-generator in-process via Thor's invoke, so this mirrors
-    #      what the user would get running each generator by hand.
+    # Upgrade command: inspect the live database and add any missing pgbus
+    # migrations to db/migrate (or db/pgbus_migrate if a separate database is
+    # configured). Invokes each matching sub-generator in-process via Thor's
+    # invoke, so this mirrors what the user would get running each generator
+    # by hand.
     #
     # Usage:
     #
     #   bin/rails generate pgbus:update
     #   bin/rails generate pgbus:update --dry-run
-    #   bin/rails generate pgbus:update --skip-config
     #   bin/rails generate pgbus:update --skip-migrations
     #   bin/rails generate pgbus:update --database=pgbus
     #   bin/rails generate pgbus:update --quiet
     class UpdateGenerator < Rails::Generators::Base
-      desc "Upgrade pgbus: convert YAML config + add any missing migrations"
-
-      class_option :source,
-                   type: :string,
-                   default: "config/pgbus.yml",
-                   desc: "Path to an existing YAML config to convert (default: config/pgbus.yml)"
-
-      class_option :destination,
-                   type: :string,
-                   default: "config/initializers/pgbus.rb",
-                   desc: "Path to the generated initializer (default: config/initializers/pgbus.rb)"
-
-      class_option :skip_config,
-                   type: :boolean,
-                   default: false,
-                   desc: "Skip the YAML → Ruby initializer conversion step"
+      desc "Upgrade pgbus: add any missing migrations"
 
       class_option :skip_migrations,
                    type: :boolean,
@@ -67,30 +43,6 @@ module Pgbus
                    default: false,
                    desc: "Suppress verbose per-step output"
 
-      def convert_yaml_if_present
-        return if options[:skip_config]
-
-        source_path = File.expand_path(options[:source], destination_root)
-        destination_path = File.expand_path(options[:destination], destination_root)
-
-        unless File.exist?(source_path)
-          log "YAML config not found at #{options[:source]}; skipping config conversion."
-          return
-        end
-
-        if File.exist?(destination_path)
-          log "Initializer already exists at #{options[:destination]}; skipping config conversion."
-          return
-        end
-
-        ruby_source = load_and_convert(source_path)
-        if options[:dry_run]
-          log_change "[dry-run] would create #{options[:destination]}"
-        else
-          create_file destination_path, ruby_source
-        end
-      end
-
       def detect_and_install_missing_migrations
         return if options[:skip_migrations]
 
@@ -172,18 +124,6 @@ module Pgbus
 
       private
 
-      # Wrap converter and YAML errors as Thor::Error so the generator
-      # surfaces them with the standard "in red, no backtrace, exit 1"
-      # behavior. Catches:
-      #   - ConfigConverter::Error (validation, missing file race)
-      #   - Psych::Exception (malformed YAML, disallowed types)
-      #   - Errno::ENOENT / Errno::EACCES (file disappeared / not readable)
-      def load_and_convert(source_path)
-        ConfigConverter.from_yaml(source_path)
-      rescue ConfigConverter::Error, Psych::Exception, Errno::ENOENT, Errno::EACCES => e
-        raise Thor::Error, "Failed to convert #{options[:source]}: #{e.message}"
-      end
-
       def active_record_available?
         defined?(::ActiveRecord::Base) && ::ActiveRecord::Base.respond_to?(:connection)
       end

data/lib/pgbus/client/ensure_stream_queue.rb

@@ -22,7 +22,14 @@ module Pgbus
         full_name = config.queue_name(stream_name)
 
         with_stale_connection_retry do
-          ensure_queue(stream_name)
+          # Create the BARE queue directly. ensure_queue would fan out through
+          # the priority strategy to _p0.._pN under priority_levels>1, leaving
+          # the bare queue — the one the streamer NOTIFYs on and read_after
+          # peeks — uncreated, and the enable_notify_if_needed below would then
+          # raise on the missing bare table. Streams never use priority
+          # sub-queues (issue #310).
+          ensure_pgmq_schema
+          ensure_single_queue(full_name)
 
           # PGMQ's default NOTIFY throttle is 250ms — meant to coalesce
           # high-frequency worker queue inserts. Streams are latency-
@@ -38,6 +45,8 @@ module Pgbus
         # requires a roundtrip and a brief ACCESS SHARE lock on the archive
         # table. Broadcast-per-after_commit loops can hit this 1000x/sec on
         # the same stream, so memoize per-process after the first success.
+        # The StreamQueue registration shares this memo — both are one-time
+        # per stream per process and must both survive a first-broadcast.
         return if @stream_indexes_created[stream_name]
 
         sanitized = QueueNameValidator.sanitize!(full_name)
@@ -52,6 +61,11 @@ module Pgbus
           end
         end
 
+        # Record the physical queue name so maintenance (stream-archive prune,
+        # orphan sweep, compact_archives) and wildcard workers can tell this
+        # queue apart from a job queue. No-ops on unmigrated installs.
+        Pgbus::StreamQueue.record!(full_name)
+
         @stream_indexes_created[stream_name] = true
       end
     end

data/lib/pgbus/client/read_after.rb

@@ -18,7 +18,7 @@ module Pgbus
         sql = build_read_after_sql(sanitized)
 
         rows = synchronized do
-          with_raw_connection do |conn|
+          with_streams_connection do |conn|
             conn.exec_params(sql, [after_id.to_i, limit.to_i]).to_a
           end
         end
@@ -34,7 +34,7 @@ module Pgbus
         sanitized = sanitized_queue(stream_name)
         sql = "SELECT COALESCE(MAX(msg_id), 0) AS max FROM pgmq.q_#{sanitized}"
         synchronized do
-          with_raw_connection do |conn|
+          with_streams_connection do |conn|
             conn.exec(sql).first.fetch("max").to_i
           end
         end
@@ -53,7 +53,7 @@ module Pgbus
           ) AS least
         SQL
         synchronized do
-          with_raw_connection do |conn|
+          with_streams_connection do |conn|
             value = conn.exec(sql).first.fetch("least")
             value&.to_i
           end
@@ -84,9 +84,10 @@ module Pgbus
       # See issues #101 and #104. The comparison is case-insensitive because
       # Postgres downcases unquoted identifiers in its error output, while
       # `sanitized` can contain uppercase characters for GlobalID-keyed streams
-      # (e.g. `pgbus_stream_Z2lkOi8vY29zbW9zL1VzZXIvMQ` from
-      # `pgbus_stream_from Current.user`). A case-sensitive substring match
-      # would miss the downcased relation name and let the exception escape.
+      # (e.g. `pgbus_Z2lkOi8vY29zbW9zL1VzZXIvMQ` from
+      # `pgbus_stream_from Current.user` — stream queues share the job prefix,
+      # see Configuration#queue_name). A case-sensitive substring match would
+      # miss the downcased relation name and let the exception escape.
       #
       # The regex uses `\b` word boundaries so `pgmq.q_<needle>` doesn't
       # accidentally match longer related identifiers like

data/lib/pgbus/client/resizable_pool.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+require "concurrent"
+
+module Pgbus
+  class Client
+    # Owns the current streams PGMQ::Client behind a Concurrent::AtomicReference
+    # and resizes it by BUILDING A NEW pool, atomically swapping the reference,
+    # then bounded-draining + shutting down the old one (issue #323 spike).
+    #
+    # connection_pool 2.x cannot resize in place (@size is frozen at
+    # construction; only shutdown/reload/available exist), so growing REQUIRES a
+    # new PGMQ::Client + a reference swap.
+    #
+    # DRAIN CORRECTNESS — the load-bearing detail. We do NOT trust
+    # connection_pool's `available == size` as "drained": pgmq-ruby's
+    # #with_connection retries a lost connection (checkout → checkin → checkout
+    # again), so between the two checkouts the connection sits idle in the queue
+    # and `available` momentarily equals `size` while a produce/read is still
+    # mid-flight. Closing the pool in that window makes the retry's checkout raise
+    # PoolShuttingDownError → a genuinely LOST durable broadcast (the
+    # PgBouncer/failover mid-INSERT case). Instead each reader op is bracketed by
+    # an in-flight AtomicFixnum on the CAPTURED pool, and drain waits until THAT
+    # hits zero. This is immune to the internal retry and independent of the
+    # connection_pool version.
+    #
+    # DURABLE BROADCASTS are loss-safe by construction: #produce captures the
+    # Pool once, runs its whole retrying checkout→INSERT→COMMIT→checkin on that
+    # pool, then decrements. Both old and new pools connect to the same Postgres,
+    # so any INSERT commits; and close waits for the captured pool's inflight to
+    # reach zero, so a pool is never shut while a produce is in flight on it.
+    class ResizablePool
+      # The current pool: a PGMQ::Client plus its own in-flight checkout counter.
+      Pool = Data.define(:pgmq, :inflight)
+
+      SwapStats = Data.define(:swap_count, :last_drain_seconds, :last_conns_closed,
+                              :last_from_size, :last_to_size, :last_drained)
+
+      DRAIN_POLL = 0.01 # 10ms
+
+      # `shared:` is accepted for symmetry with the Client's two connection paths
+      # but is not stored — the shared-AR alias guard lives in #close_current via
+      # the `job_pool:` identity check, and #swap is simply never called on the
+      # shared path (Client#resize_streams_pool short-circuits there).
+      def initialize(pgmq, shared:, drain_timeout:, logger:, clock: ::Process) # rubocop:disable Lint/UnusedMethodArgument
+        @ref = Concurrent::AtomicReference.new(new_pool(pgmq))
+        @drain_timeout = drain_timeout
+        @logger = logger
+        @clock = clock
+        @swap_mutex = Mutex.new # serializes swap AND close_current; reads never take it
+        @swap_count = 0
+        @last = nil
+      end
+
+      # Hot path: one volatile load, no lock.
+      def current = @ref.get
+
+      # Bracketed reader ops — inflight decrements only after the WHOLE retrying
+      # call returns (so drain can't close under an in-flight produce/read).
+      def produce(*, **)
+        pool = @ref.get
+        pool.inflight.increment
+        begin
+          pool.pgmq.produce(*, **)
+        ensure
+          pool.inflight.decrement
+        end
+      end
+
+      def with_connection(&)
+        pool = @ref.get
+        pool.inflight.increment
+        begin
+          pool.pgmq.with_connection(&)
+        ensure
+          pool.inflight.decrement
+        end
+      end
+
+      def stats = @ref.get.pgmq.stats
+
+      # Swap in a freshly-built PGMQ::Client, then drain + close the old Pool.
+      def swap(new_pgmq, from_size:, to_size:)
+        @swap_mutex.synchronize do
+          old = @ref.get
+          @ref.set(new_pool(new_pgmq)) # new checkouts now resolve to the new pool
+          drained, secs, closed = drain_and_close(old)
+          @swap_count += 1
+          @last = SwapStats.new(swap_count: @swap_count, last_drain_seconds: secs,
+                                last_conns_closed: closed, last_from_size: from_size,
+                                last_to_size: to_size, last_drained: drained)
+        end
+      end
+
+      # Teardown of the current pool (from Client#close), race-free vs swap.
+      # Skips the close when the current pool aliases the job pool (shared-AR path).
+      def close_current(job_pool:)
+        @swap_mutex.synchronize do
+          pool = @ref.get
+          pool.pgmq.close unless pool.pgmq.equal?(job_pool)
+        end
+      end
+
+      def stats_snapshot
+        @last || SwapStats.new(swap_count: 0, last_drain_seconds: 0.0, last_conns_closed: 0,
+                               last_from_size: nil, last_to_size: nil, last_drained: nil)
+      end
+
+      private
+
+      def new_pool(pgmq)
+        Pool.new(pgmq: pgmq, inflight: Concurrent::AtomicFixnum.new(0))
+      end
+
+      # Bounded drain on the OLD pool's inflight counter, then shutdown. Mirrors
+      # OutboundPump#stop: fixed budget, log-and-abandon, NEVER Thread#kill /
+      # Timeout.timeout. Returns [drained?, wall_secs, conns_closed].
+      def drain_and_close(old)
+        t0 = mono
+        drained = wait_until_drained(old.inflight)
+        # Count connections about to be torn down BEFORE close (after close it's ~0).
+        conns = conns_open(old.pgmq)
+        unless drained
+          @logger.warn do
+            "[Pgbus::ResizablePool] streams pool did not fully drain in " \
+              "#{@drain_timeout}s (inflight=#{old.inflight.value}); shutting down anyway — " \
+              "in-flight connections self-close on checkin"
+          end
+        end
+        old.pgmq.close
+        [drained, mono - t0, conns]
+      rescue StandardError => e
+        @logger.error { "[Pgbus::ResizablePool] old pool teardown failed: #{e.class}: #{e.message}" }
+        [false, mono - t0, 0]
+      end
+
+      def wait_until_drained(inflight)
+        deadline = mono + @drain_timeout
+        loop do
+          return true if inflight.value <= 0
+          return false if mono >= deadline
+
+          sleep(DRAIN_POLL) # NOT Timeout.timeout — Pgbus/NoRubyTimeout cop
+        end
+      end
+
+      # size - available = connections currently open (created & not idle-in-queue).
+      def conns_open(pgmq)
+        stats = pgmq.stats
+        [stats[:size] - stats[:available], 0].max
+      rescue StandardError
+        0
+      end
+
+      # Qualify ::Process — Pgbus::Process (the worker/supervisor namespace)
+      # shadows the top-level constant here.
+      def mono = @clock.clock_gettime(::Process::CLOCK_MONOTONIC)
+    end
+  end
+end