pgbus 0.4.1 → 0.5.0

data/app/controllers/pgbus/dead_letter_controller.rb

@@ -13,9 +13,7 @@ module Pgbus
 
     def retry
       queue_name = params[:queue_name].to_s
-      unless queue_name.end_with?(Pgbus.configuration.dead_letter_queue_suffix)
-        return redirect_to dead_letter_index_path, alert: "Invalid DLQ queue."
-      end
+      return redirect_to dead_letter_index_path, alert: "Invalid DLQ queue." unless queue_name.end_with?(Pgbus::DEAD_LETTER_SUFFIX)
 
       if data_source.retry_dlq_message(queue_name, params[:id])
         redirect_to dead_letter_index_path, notice: "Message re-enqueued to original queue."
@@ -26,9 +24,7 @@ module Pgbus
 
     def discard
       queue_name = params[:queue_name].to_s
-      unless queue_name.end_with?(Pgbus.configuration.dead_letter_queue_suffix)
-        return redirect_to dead_letter_index_path, alert: "Invalid DLQ queue."
-      end
+      return redirect_to dead_letter_index_path, alert: "Invalid DLQ queue." unless queue_name.end_with?(Pgbus::DEAD_LETTER_SUFFIX)
 
       if data_source.discard_dlq_message(queue_name, params[:id])
         redirect_to dead_letter_index_path, notice: "Message discarded."
@@ -57,7 +53,7 @@ module Pgbus
       count = 0
       selections.each do |sel|
         queue_name = sel[:queue_name].to_s
-        next unless queue_name.end_with?(Pgbus.configuration.dead_letter_queue_suffix)
+        next unless queue_name.end_with?(Pgbus::DEAD_LETTER_SUFFIX)
 
         count += 1 if data_source.discard_dlq_message(queue_name, sel[:msg_id])
       end

data/app/views/pgbus/dead_letter/_messages_table.html.erb

@@ -24,8 +24,7 @@
       <tbody class="divide-y divide-gray-100 dark:divide-gray-700">
         <% @messages.each do |m| %>
           <% payload = pgbus_parse_message(m[:message]) %>
-          <% dlq_suffix = Pgbus.configuration.dead_letter_queue_suffix %>
-          <% source_queue = m[:queue_name].to_s.delete_suffix(dlq_suffix) %>
+          <% source_queue = m[:queue_name].to_s.delete_suffix(Pgbus::DEAD_LETTER_SUFFIX) %>
           <tr>
             <td class="w-10 px-4 py-3 align-top">
               <input type="checkbox" data-bulk-item data-pgbus-action="bulk-row-toggle"

data/lib/generators/pgbus/templates/pgbus.yml.erb

@@ -8,13 +8,15 @@ default: &default
   # Default queue name (without prefix)
   default_queue: default
 
-  # Connection pool for PGMQ client
-  pool_size: 5
+  # Connection pool for PGMQ client.
+  # pool_size auto-tunes from your worker thread counts:
+  #   sum(workers.threads) + sum(event_consumers.threads) + 2
+  # Set explicitly only if you need a tighter or looser pool than that.
+  # pool_size: 5
   pool_timeout: 5
 
   # Use PostgreSQL LISTEN/NOTIFY for instant job wake-up
   listen_notify: true
-  notify_throttle_ms: 250
 
   # Visibility timeout in seconds (how long a message is invisible after being read)
   visibility_timeout: 30

data/lib/generators/pgbus/update_generator.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "pgbus/generators/config_converter"
+
+module Pgbus
+  module Generators
+    # Converts an existing config/pgbus.yml to a Ruby initializer at
+    # config/initializers/pgbus.rb using the modern DSL.
+    #
+    # The original YAML file is left in place — the user reviews the
+    # generated initializer and deletes the YAML when ready.
+    #
+    # Usage:
+    #
+    #   bin/rails generate pgbus:update
+    #   bin/rails generate pgbus:update --force      # overwrite existing initializer
+    #   bin/rails generate pgbus:update --source=path/to/pgbus.yml
+    class UpdateGenerator < Rails::Generators::Base
+      desc "Convert config/pgbus.yml to config/initializers/pgbus.rb using the Ruby DSL"
+
+      class_option :source,
+                   type: :string,
+                   default: "config/pgbus.yml",
+                   desc: "Path to the existing YAML config (default: config/pgbus.yml)"
+
+      class_option :destination,
+                   type: :string,
+                   default: "config/initializers/pgbus.rb",
+                   desc: "Path to the generated initializer (default: config/initializers/pgbus.rb)"
+
+      def convert
+        source_path = File.expand_path(options[:source], destination_root)
+        destination_path = File.expand_path(options[:destination], destination_root)
+
+        # Thor::Error is the idiomatic way to abort a Rails generator. Thor
+        # catches it, prints the message in red, and exits with status 1
+        # without a Ruby backtrace. exit 1 would skip the framework's
+        # cleanup hooks and is hard to test.
+        raise Thor::Error, "Source file not found: #{options[:source]}" unless File.exist?(source_path)
+
+        ruby_source = load_and_convert(source_path)
+        create_file destination_path, ruby_source
+      end
+
+      def display_post_install
+        say ""
+        say "Pgbus initializer generated at #{options[:destination]}!", :green
+        say ""
+        say "Next steps:"
+        say "  1. Review the generated initializer for correctness"
+        say "  2. Boot your app and verify everything still works"
+        say "  3. Delete #{options[:source]} when satisfied (Pgbus will stop reading it)"
+        say ""
+        say "If you spot a setting that didn't translate cleanly, please open an issue:"
+        say "  https://github.com/mhenrixon/pgbus/issues", :cyan
+        say ""
+      end
+
+      private
+
+      # Wrap converter and YAML errors as Thor::Error so the generator
+      # surfaces them with the standard "in red, no backtrace, exit 1"
+      # behavior. Catches:
+      #   - ConfigConverter::Error (validation, missing file race)
+      #   - Psych::Exception (malformed YAML, disallowed types)
+      #   - Errno::ENOENT / Errno::EACCES (file disappeared / not readable)
+      def load_and_convert(source_path)
+        ConfigConverter.from_yaml(source_path)
+      rescue ConfigConverter::Error, Psych::Exception, Errno::ENOENT, Errno::EACCES => e
+        raise Thor::Error, "Failed to convert #{options[:source]}: #{e.message}"
+      end
+    end
+  end
+end

data/lib/pgbus/circuit_breaker.rb

@@ -4,6 +4,20 @@ require "concurrent"
 
 module Pgbus
   class CircuitBreaker
+    # Number of consecutive failures on a queue before the breaker trips
+    # and pauses the queue. Tuned via constants rather than configuration
+    # because the value rarely needs adjusting and exposing it as a setting
+    # never proved useful in practice.
+    THRESHOLD = 5
+
+    # Initial backoff (seconds) on the first trip. Doubles on each
+    # subsequent trip up to MAX_BACKOFF.
+    BASE_BACKOFF = 30
+
+    # Cap on the exponential backoff (seconds). After ~5 trips the curve
+    # plateaus here so a perpetually-failing queue stops spamming retries.
+    MAX_BACKOFF = 600
+
     attr_reader :config
 
     def initialize(config: Pgbus.configuration)
@@ -22,7 +36,7 @@ module Pgbus
 
       count = @failure_counts.compute(queue_name) { |val| (val || 0) + 1 }
 
-      return unless count >= config.circuit_breaker_threshold
+      return unless count >= THRESHOLD
 
       trip!(queue_name, count)
     end
@@ -105,8 +119,8 @@ module Pgbus
     end
 
     def calculate_backoff(trip_count)
-      backoff = config.circuit_breaker_base_backoff * (2**(trip_count - 1))
-      [backoff, config.circuit_breaker_max_backoff].min
+      backoff = BASE_BACKOFF * (2**(trip_count - 1))
+      [backoff, MAX_BACKOFF].min
     end
   end
 end

data/lib/pgbus/cli.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "optparse"
+
 module Pgbus
   module CLI
     module_function
@@ -9,7 +11,7 @@ module Pgbus
 
       case command
       when "start"
-        start_supervisor
+        start_supervisor(args[1..] || [])
       when "status"
         show_status
       when "queues"
@@ -25,13 +27,89 @@ module Pgbus
       end
     end
 
-    def start_supervisor
+    def start_supervisor(args = [])
+      apply_start_options(args)
       Pgbus.logger.info { "[Pgbus] Starting Pgbus #{Pgbus::VERSION}..." }
 
       supervisor = Process::Supervisor.new
       supervisor.run
     end
 
+    # Parses CLI flags for `pgbus start` and applies them to the global
+    # configuration before the supervisor boots. Designed to override the
+    # initializer config without requiring a redeploy.
+    def apply_start_options(args)
+      options = parse_start_options(args)
+
+      Pgbus.configuration.workers = options[:queues] if options[:queues]
+      apply_capsule_filter(options[:capsule]) if options[:capsule]
+      apply_role_filter(options)
+    end
+
+    def parse_start_options(args)
+      options = {}
+      OptionParser.new do |opts|
+        opts.banner = "Usage: pgbus start [options]"
+
+        opts.on("--queues STRING", "Override worker capsules (e.g. \"critical: 5; default: 10\")") do |v|
+          options[:queues] = v
+        end
+
+        opts.on("--capsule NAME", "Run only the named capsule from the configured workers") do |v|
+          options[:capsule] = v
+        end
+
+        opts.on("--workers-only", "Run only the worker processes (no scheduler/dispatcher/consumers/outbox)") do
+          options[:workers_only] = true
+        end
+
+        opts.on("--scheduler-only", "Run only the recurring scheduler (the cron pod pattern)") do
+          options[:scheduler_only] = true
+        end
+
+        opts.on("--dispatcher-only", "Run only the dispatcher (the maintenance pod pattern)") do
+          options[:dispatcher_only] = true
+        end
+      end.parse!(args.dup)
+      options
+    end
+
+    ROLE_FLAG_TO_ROLE = {
+      workers_only: :workers,
+      scheduler_only: :scheduler,
+      dispatcher_only: :dispatcher
+    }.freeze
+    private_constant :ROLE_FLAG_TO_ROLE
+
+    # Translates --workers-only / --scheduler-only / --dispatcher-only into
+    # the corresponding +Pgbus.configuration.roles+ array. Mutually exclusive —
+    # passing more than one of the three flags raises ArgumentError.
+    def apply_role_filter(options)
+      role_flags = options.slice(*ROLE_FLAG_TO_ROLE.keys).compact
+      return if role_flags.empty?
+
+      if role_flags.size > 1
+        raise ArgumentError,
+              "--workers-only, --scheduler-only, and --dispatcher-only are mutually exclusive " \
+              "(got: #{role_flags.keys.map { |k| "--#{k.to_s.tr("_", "-")}" }.join(", ")})"
+      end
+
+      Pgbus.configuration.roles = [ROLE_FLAG_TO_ROLE.fetch(role_flags.keys.first)]
+    end
+
+    def apply_capsule_filter(name)
+      capsule = Pgbus.configuration.capsule_named(name)
+      unless capsule
+        available = (Pgbus.configuration.workers || []).filter_map { |c| c[:name] || c["name"] }
+        raise ArgumentError,
+              "no capsule named #{name.inspect} (available: #{available.join(", ")})"
+      end
+
+      # Go through the public setter so any future normalization/validation
+      # in workers= is applied consistently to the CLI override path too.
+      Pgbus.configuration.workers = [capsule]
+    end
+
     def show_status
       processes = ProcessEntry.order(:kind, :created_at)
                               .select(:kind, :hostname, :pid, :metadata, :last_heartbeat_at)
@@ -65,7 +143,7 @@ module Pgbus
 
     def print_help
       puts <<~HELP
-        Usage: pgbus <command>
+        Usage: pgbus <command> [options]
 
         Commands:
           start    Start the Pgbus supervisor (workers + dispatcher)
@@ -73,6 +151,20 @@ module Pgbus
           queues   List queues with metrics
           version  Show version
           help     Show this help
+
+        Options for `start`:
+          --queues STRING    Override worker capsules from the CLI
+                             (e.g. "critical: 5; default: 10")
+          --capsule NAME     Run only the named capsule from the configured
+                             workers (useful for one-capsule-per-process
+                             deployments)
+          --workers-only     Run only worker processes (no scheduler/
+                             dispatcher/consumers/outbox — for worker-only
+                             containers)
+          --scheduler-only   Run only the recurring scheduler (the cron pod
+                             pattern — exactly one of these per deployment)
+          --dispatcher-only  Run only the dispatcher (the maintenance pod
+                             pattern)
       HELP
     end
   end

data/lib/pgbus/client.rb

@@ -9,6 +9,14 @@ module Pgbus
     PGMQ_REQUIRE_MUTEX = Mutex.new
     private_constant :PGMQ_REQUIRE_MUTEX
 
+    # Throttle window for PGMQ's enable_notify_insert trigger. Postgres
+    # NOTIFYs are coalesced into one wake-up per window, so a value of 250ms
+    # means: at most 4 broadcasts/sec per queue, regardless of insert rate.
+    # The trigger is a Postgres-level concern; exposing it as a setting
+    # never came up in practice and changing it on the fly would require
+    # re-running the trigger DDL on every queue.
+    NOTIFY_THROTTLE_MS = 250
+
     def initialize(config = Pgbus.configuration)
       # Define the PGMQ module before requiring the gem so that Zeitwerk's
       # eager_load (called inside pgmq.rb) can resolve the constant.
@@ -33,9 +41,10 @@ module Pgbus
       else
         # With a String URL or Hash params, pgmq-ruby creates its own dedicated
         # PG::Connection per pool slot — no shared state with ActiveRecord.
-        # Use the configured pool_size and let pgmq-ruby's connection_pool handle
+        # Use the resolved pool size (auto-tuned from worker thread counts
+        # unless explicitly set) and let pgmq-ruby's connection_pool handle
         # concurrency internally (no mutex needed).
-        @pgmq = PGMQ::Client.new(conn_opts, pool_size: config.pool_size, pool_timeout: config.pool_timeout)
+        @pgmq = PGMQ::Client.new(conn_opts, pool_size: config.resolved_pool_size, pool_timeout: config.pool_timeout)
         @pgmq_mutex = nil
       end
 
@@ -431,7 +440,7 @@ module Pgbus
       @queues_created.compute_if_absent(full_name) do
         synchronized do
           @pgmq.create(full_name)
-          @pgmq.enable_notify_insert(full_name, throttle_interval_ms: config.notify_throttle_ms) if config.listen_notify
+          @pgmq.enable_notify_insert(full_name, throttle_interval_ms: NOTIFY_THROTTLE_MS) if config.listen_notify
         end
         true
       end

data/lib/pgbus/configuration/capsule_dsl.rb

@@ -0,0 +1,190 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Configuration
+    # Parses the capsule string DSL into the internal worker config array.
+    #
+    # The DSL lets users describe worker thread pools (capsules) compactly:
+    #
+    #   "*: 5"                                # one capsule, all queues, 5 threads
+    #   "critical: 5; default: 10"            # two capsules
+    #   "critical, default: 5"                # one capsule, list = strict priority
+    #   "default, mailers: 10; *: 2"          # specialized + fallback wildcard
+    #   "staging_*: 3"                        # prefix wildcard
+    #
+    # Operators:
+    #   ,    queue separator within a capsule (list order = strict priority)
+    #   ;    capsule separator (each becomes its own thread pool)
+    #   :N   thread count for the capsule (default 5)
+    #   *    wildcard, matches all queues
+    #   *_   trailing wildcard, prefix match (e.g. "staging_*")
+    #
+    # Returns +Array<Hash>+ in the same shape as the legacy +workers:+ array,
+    # so the rest of the codebase can consume it without changes:
+    #
+    #   [{ queues: ["critical"], threads: 5 },
+    #    { queues: ["default", "mailers"], threads: 10 }]
+    #
+    # Validation runs at parse time. Errors include the offending input and
+    # a clear message naming the rule that was violated.
+    class CapsuleDSL
+      DEFAULT_THREADS = 5
+      WILDCARD = "*"
+
+      # Valid queue tokens accepted by the DSL:
+      #   "*"         bare wildcard (matches all queues at runtime)
+      #   "default"   bare queue name
+      #   "staging_*" prefix wildcard (matches queues by prefix at runtime)
+      #
+      # The character class for queue names — alphanumerics and underscores
+      # only — intentionally matches Pgbus::QueueNameValidator::VALID_QUEUE_NAME_PATTERN.
+      # Hyphens, dots, and other punctuation are NOT permitted because PGMQ
+      # interpolates queue names directly into SQL identifiers (q_<name>,
+      # a_<name>) and only those characters are safe there. If you change
+      # this pattern, also update QueueNameValidator to keep them in sync.
+      #
+      # The "*" tokens (bare and trailing) are DSL-only — they get expanded
+      # to concrete queue names at runtime before reaching QueueNameValidator.
+      QUEUE_NAME_PATTERN = /\A(?:\*|[a-zA-Z0-9_]+\*?)\z/
+      THREAD_COUNT_PATTERN = /\A\d+\z/
+
+      class ParseError < ArgumentError; end
+
+      def self.parse(input)
+        new(input).parse
+      end
+
+      def initialize(input)
+        @input = input
+      end
+
+      def parse
+        validate_input_type!
+        validate_input_not_empty!
+
+        capsules = split_capsules(@input).map { |segment| parse_capsule(segment) }
+        validate_no_duplicate_queues_across_capsules!(capsules)
+        capsules
+      end
+
+      private
+
+      def validate_input_type!
+        return if @input.is_a?(String)
+
+        raise ParseError,
+              "expected String, got #{@input.class} (#{@input.inspect})"
+      end
+
+      def validate_input_not_empty!
+        return if @input.strip != ""
+
+        raise ParseError, "empty capsule string — must declare at least one queue"
+      end
+
+      # Splits on `;` and trims whitespace, dropping a trailing empty segment
+      # so `"a; b;"` is treated the same as `"a; b"`. Leading-empty segments
+      # ("; a") are kept and rejected by parse_capsule with a clearer error.
+      def split_capsules(string)
+        segments = string.strip.split(";").map(&:strip)
+        segments.pop if segments.last == "" # trailing semicolon
+        segments
+      end
+
+      def parse_capsule(segment)
+        if segment == ""
+          raise ParseError,
+                "empty capsule in #{@input.inspect} — leading or doubled semicolons are not allowed"
+        end
+
+        queues_part, threads_part = split_on_threads(segment)
+        queues = parse_queue_list(queues_part, segment)
+        threads = parse_thread_count(threads_part, segment)
+        validate_no_duplicates_within_capsule!(queues, segment)
+
+        { queues: queues, threads: threads }
+      end
+
+      # Splits a capsule segment on the LAST `:` so queue names containing
+      # underscores or digits are unaffected. Returns [queues_part, threads_part]
+      # where threads_part is nil when no `:` is present.
+      def split_on_threads(segment)
+        idx = segment.rindex(":")
+        return [segment, nil] unless idx
+
+        [segment[0...idx].strip, segment[(idx + 1)..].strip]
+      end
+
+      def parse_queue_list(queues_part, original_segment)
+        if queues_part.empty?
+          raise ParseError,
+                "expected queue name before ':' in #{original_segment.inspect}"
+        end
+
+        queues = queues_part.split(",").map(&:strip)
+        queues.each { |q| validate_queue_name!(q, original_segment) }
+        queues
+      end
+
+      def validate_queue_name!(name, original_segment)
+        return if name.match?(QUEUE_NAME_PATTERN)
+
+        raise ParseError,
+              "invalid character in queue name #{name.inspect} (in #{original_segment.inspect}) — " \
+              "queue names must match /[a-zA-Z0-9_]+\\*?/"
+      end
+
+      def parse_thread_count(threads_part, original_segment)
+        return DEFAULT_THREADS if threads_part.nil?
+
+        if threads_part.empty?
+          raise ParseError,
+                "expected thread count after ':' in #{original_segment.inspect}"
+        end
+
+        unless threads_part.match?(THREAD_COUNT_PATTERN)
+          raise ParseError,
+                "invalid thread count #{threads_part.inspect} in #{original_segment.inspect} — " \
+                "must be a positive integer"
+        end
+
+        n = threads_part.to_i
+        if n.zero?
+          raise ParseError,
+                "thread count must be a positive integer, got 0 in #{original_segment.inspect}"
+        end
+
+        n
+      end
+
+      def validate_no_duplicates_within_capsule!(queues, original_segment)
+        seen = {}
+        queues.each do |q|
+          if seen[q]
+            raise ParseError,
+                  "queue #{q.inspect} listed twice in capsule #{original_segment.inspect} — " \
+                  "duplicate queues within a capsule are not allowed"
+          end
+          seen[q] = true
+        end
+      end
+
+      def validate_no_duplicate_queues_across_capsules!(capsules)
+        return if capsules.size < 2
+
+        seen = {}
+        capsules.each_with_index do |capsule, idx|
+          capsule[:queues].each do |q|
+            if seen[q]
+              label = (q == WILDCARD ? "wildcard '*'" : "queue #{q.inspect}")
+              raise ParseError,
+                    "#{label} appears in two capsules (positions #{seen[q]} and #{idx + 1}) " \
+                    "in #{@input.inspect} — each queue can only be assigned to one capsule"
+            end
+            seen[q] = idx + 1
+          end
+        end
+      end
+    end
+  end
+end