pgbus 0.4.1 → 0.5.0

data/lib/pgbus/generators/config_converter.rb

@@ -0,0 +1,323 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module Pgbus
+  module Generators
+    # Converts a +config/pgbus.yml+ file into a Ruby initializer
+    # (+config/initializers/pgbus.rb+) using the Ruby DSL added in
+    # Pgbus 0.5+ — capsule string DSL, ActiveSupport::Duration coercion,
+    # auto-tuned pool size, named capsules, etc.
+    #
+    # Drops settings that:
+    #   - match the gem default (no point restating it)
+    #   - are deprecated (e.g. pool_size, which is now auto-tuned)
+    #
+    # Converts seconds to durations when they evenly divide into a clean
+    # unit (7 days, 30 days, 10 minutes). Falls back to the raw integer
+    # otherwise.
+    #
+    # When the YAML has multiple environments with different values for
+    # the same setting, emits Rails.env-aware code:
+    #
+    #   - 2 envs with same value → unconditional line
+    #   - 2 envs, one differs → `unless Rails.env.X?` modifier
+    #   - 3+ envs with differences → `case Rails.env when ... end`
+    #
+    # Backwards compatible: the original YAML file is NOT touched. The
+    # generator's CLI wrapper writes the new initializer and tells the
+    # user to delete the YAML when ready.
+    class ConfigConverter
+      class Error < StandardError; end
+
+      # Setters that accept ActiveSupport::Duration (PR 5).
+      DURATION_SETTINGS = %w[
+        visibility_timeout archive_retention idempotency_ttl
+        outbox_retention stats_retention recurring_execution_retention
+      ].freeze
+
+      # Settings that no longer exist in the public API. The converter
+      # silently drops these from the generated initializer so users on
+      # legacy YAML get a clean migration.
+      #
+      #   - pool_size              -> auto-tuned from worker thread counts
+      #   - notify_throttle_ms     -> Pgbus::Client::NOTIFY_THROTTLE_MS
+      #   - circuit_breaker_*      -> Pgbus::CircuitBreaker constants
+      #   - archive_compaction_*   -> Pgbus::Process::Dispatcher constants
+      #   - dead_letter_queue_suffix -> Pgbus::DEAD_LETTER_SUFFIX (frozen)
+      DEPRECATED_SETTINGS = %w[
+        pool_size
+        notify_throttle_ms
+        circuit_breaker_threshold circuit_breaker_base_backoff circuit_breaker_max_backoff
+        archive_compaction_interval archive_compaction_batch_size
+        dead_letter_queue_suffix
+      ].freeze
+
+      # Settings whose default we know how to compute by inspecting
+      # Pgbus::Configuration.new. Any setting not listed here is emitted
+      # as-is (we can't tell if it matches the default).
+      KNOWN_SETTINGS = %w[
+        queue_prefix default_queue pool_timeout listen_notify
+        visibility_timeout max_retries idempotency_ttl
+        max_jobs_per_worker max_memory_mb max_worker_lifetime
+        dispatch_interval prefetch_limit
+        circuit_breaker_enabled
+        archive_retention
+        outbox_enabled outbox_poll_interval outbox_batch_size outbox_retention
+        stats_enabled stats_retention
+        recurring_schedule_interval recurring_execution_retention skip_recurring
+        polling_interval default_priority priority_levels
+        return_to_app_url workers
+      ].freeze
+
+      def self.from_yaml(path)
+        raise Error, "config file not found: #{path}" unless File.exist?(path)
+
+        parsed = YAML.safe_load_file(path, aliases: true, permitted_classes: [Symbol])
+        from_hash(parsed)
+      end
+
+      def self.from_hash(envs_hash)
+        new(envs_hash).render
+      end
+
+      def initialize(envs_hash)
+        @envs = (envs_hash || {}).reject { |env, _| env.start_with?("default") }
+        @envs = { "production" => envs_hash } if @envs.empty? && envs_hash
+        @defaults = build_defaults
+      end
+
+      def render
+        lines = []
+        lines << "# frozen_string_literal: true"
+        lines << "#"
+        lines << "# Generated by `rails generate pgbus:update` from config/pgbus.yml."
+        lines << "# Review and adjust as needed, then delete config/pgbus.yml."
+        lines << ""
+        lines << "Pgbus.configure do |c|"
+
+        body = render_body
+        body.each { |line| lines << "  #{line}" }
+
+        lines << "end"
+        "#{lines.join("\n")}\n"
+      end
+
+      private
+
+      def build_defaults
+        config = Pgbus::Configuration.new
+        KNOWN_SETTINGS.each_with_object({}) do |key, h|
+          h[key] = config.public_send(key) if config.respond_to?(key)
+        end
+      end
+
+      def render_body
+        all_settings = collect_all_settings
+        return [] if all_settings.empty?
+
+        constant_settings, varying_settings = partition_by_variance(all_settings)
+        special_keys = %w[workers]
+
+        lines = []
+        # Special: workers comes first (most user-visible)
+        special_keys.each do |key|
+          rendered = render_workers(constant_settings[key], varying_settings[key])
+          lines.concat(rendered) if rendered
+        end
+
+        constant_settings.each do |key, value|
+          next if special_keys.include?(key)
+          next if drop?(key, value)
+
+          lines << render_setting(key, value)
+        end
+
+        varying_settings.each do |key, env_values|
+          next if special_keys.include?(key)
+          next if env_values.values.all? { |v| drop?(key, v) }
+
+          lines.concat(render_varying_setting(key, env_values))
+        end
+
+        lines
+      end
+
+      def collect_all_settings
+        all_keys = @envs.values.flat_map { |env_settings| env_settings&.keys || [] }.uniq
+        all_keys.to_h do |key|
+          [key, @envs.transform_values { |env_settings| env_settings&.fetch(key, :__missing__) }]
+        end
+      end
+
+      # Returns [constant_settings, varying_settings].
+      # constant_settings: { "key" => value }   (same value across all envs)
+      # varying_settings:  { "key" => { env => value, ... } }
+      def partition_by_variance(all_settings)
+        constant = {}
+        varying = {}
+        all_settings.each do |key, env_values|
+          present_values = env_values.reject { |_, v| v == :__missing__ }
+          unique_values = present_values.values.uniq
+          if unique_values.size <= 1
+            constant[key] = unique_values.first
+          else
+            varying[key] = present_values
+          end
+        end
+        [constant, varying]
+      end
+
+      def drop?(key, value)
+        return true if DEPRECATED_SETTINGS.include?(key)
+        return true if value == :__missing__
+        return true if @defaults.key?(key) && @defaults[key] == value
+
+        false
+      end
+
+      def render_setting(key, value)
+        "c.#{key} = #{render_value(key, value)}"
+      end
+
+      def render_varying_setting(key, env_values)
+        envs = env_values.keys
+        if envs.size == 2 && envs.include?("development")
+          # Special case: "everything except dev" — common pattern
+          non_dev_value = env_values.except("development").values.first
+          dev_value = env_values["development"]
+          return ["c.#{key} = #{render_value(key, non_dev_value)} unless Rails.env.development?"] if dev_value.nil? && non_dev_value
+        end
+
+        # Fallback: case Rails.env block — `when` clauses indented one
+        # level inside the case, `end` flush with `c.X` (standard Ruby
+        # formatting for assigned case expressions).
+        lines = ["c.#{key} = case Rails.env"]
+        env_values.each do |env, value|
+          lines << "  when \"#{env}\" then #{render_value(key, value)}"
+        end
+        lines << "end"
+        lines
+      end
+
+      def render_workers(constant_workers, varying_workers)
+        if constant_workers && !varying_workers
+          rendered = render_workers_value(constant_workers)
+          return rendered unless rendered.nil? || rendered.empty?
+        end
+
+        if varying_workers
+          # Different worker config per env. when clauses indented one
+          # level inside the case, end flush with c.workers (standard
+          # Ruby formatting for assigned case expressions).
+          lines = ["c.workers = case Rails.env"]
+          varying_workers.each do |env, workers|
+            string_form = workers_as_string(workers)
+            value = string_form ? string_form.inspect : workers.inspect
+            lines << "  when \"#{env}\" then #{value}"
+          end
+          lines << "end"
+          return lines
+        end
+
+        nil
+      end
+
+      def render_workers_value(workers)
+        return nil if workers.nil? || workers == :__missing__
+
+        # Drop if matches the gem default ([{queues: %w[default], threads: 5}]).
+        # Compare via normalized form (string keys both sides) so YAML's
+        # string-keyed hashes match the symbol-keyed default.
+        return nil if normalize_workers(workers) == normalize_workers(@defaults["workers"])
+
+        if workers_simple?(workers)
+          string_form = workers_as_string(workers)
+          ["c.workers = #{string_form.inspect}"]
+        else
+          workers.map.with_index do |capsule, idx|
+            render_capsule_call(capsule, idx)
+          end
+        end
+      end
+
+      # Normalize a workers array to symbol-keyed hashes with array-of-string
+      # queues for stable comparison. Doesn't mutate input.
+      def normalize_workers(workers)
+        return nil if workers.nil?
+
+        workers.map do |capsule|
+          {
+            queues: Array(capsule[:queues] || capsule["queues"]).map(&:to_s),
+            threads: (capsule[:threads] || capsule["threads"] || 5).to_i
+          }
+        end
+      end
+
+      def workers_simple?(workers)
+        workers.all? do |capsule|
+          extras = capsule.keys.map(&:to_s) - %w[queues threads]
+          extras.empty?
+        end
+      end
+
+      def workers_as_string(workers)
+        workers.map do |capsule|
+          queues = (capsule["queues"] || capsule[:queues]).join(", ")
+          threads = capsule["threads"] || capsule[:threads] || 5
+          "#{queues}: #{threads}"
+        end.join("; ")
+      end
+
+      def render_capsule_call(capsule, _idx)
+        queues = capsule["queues"] || capsule[:queues]
+        threads = capsule["threads"] || capsule[:threads] || 5
+        name = capsule["name"] || capsule[:name] || queues.first
+        opts = capsule.reject { |k, _| %w[queues threads name].include?(k.to_s) }
+
+        parts = ["queues: #{queues.inspect}", "threads: #{threads}"]
+        opts.each { |k, v| parts << "#{k}: #{v.inspect}" }
+
+        "c.capsule :#{name}, #{parts.join(", ")}"
+      end
+
+      def render_value(key, value)
+        return "nil" if value.nil?
+
+        if DURATION_SETTINGS.include?(key) && value.is_a?(Integer) && value.positive?
+          duration_form = format_duration(value)
+          return duration_form if duration_form
+        end
+
+        if value.is_a?(Integer) && value >= 1000
+          format_integer_with_underscores(value)
+        else
+          value.inspect
+        end
+      end
+
+      def format_duration(seconds)
+        units = [
+          [86_400, "day", "days"],
+          [3600, "hour", "hours"],
+          [60, "minute", "minutes"],
+          [1, "second", "seconds"]
+        ]
+
+        units.each do |unit_seconds, singular, plural|
+          next unless (seconds % unit_seconds).zero?
+
+          count = seconds / unit_seconds
+          unit_name = count == 1 ? singular : plural
+          return "#{count}.#{unit_name}"
+        end
+
+        nil
+      end
+
+      def format_integer_with_underscores(int)
+        int.to_s.reverse.scan(/\d{1,3}/).join("_").reverse
+      end
+    end
+  end
+end

data/lib/pgbus/process/dispatcher.rb

@@ -16,6 +16,12 @@ module Pgbus
       JOB_LOCK_CLEANUP_INTERVAL = 300 # Run job lock cleanup every 5 minutes
       STATS_CLEANUP_INTERVAL = 3600 # Run stats cleanup every hour
 
+      # Page size for archive compaction. Each cycle deletes up to this
+      # many archived rows per queue. Tuned via constant rather than
+      # configuration because the value rarely needs adjusting and a
+      # too-small value just delays cleanup, never breaks anything.
+      ARCHIVE_COMPACTION_BATCH_SIZE = 1000
+
       attr_reader :config
 
       def initialize(config: Pgbus.configuration)
@@ -72,7 +78,7 @@ module Pgbus
         run_if_due(now, :@last_concurrency_at, CONCURRENCY_INTERVAL) { cleanup_concurrency }
         run_if_due(now, :@last_batch_cleanup_at, BATCH_CLEANUP_INTERVAL) { cleanup_batches }
         run_if_due(now, :@last_recurring_cleanup_at, RECURRING_CLEANUP_INTERVAL) { cleanup_recurring_executions }
-        run_if_due(now, :@last_archive_compaction_at, archive_compaction_interval) { compact_archives }
+        run_if_due(now, :@last_archive_compaction_at, ARCHIVE_COMPACTION_INTERVAL) { compact_archives }
         run_if_due(now, :@last_outbox_cleanup_at, OUTBOX_CLEANUP_INTERVAL) { cleanup_outbox }
         run_if_due(now, :@last_job_lock_cleanup_at, JOB_LOCK_CLEANUP_INTERVAL) { cleanup_job_locks }
         run_if_due(now, :@last_stats_cleanup_at, STATS_CLEANUP_INTERVAL) { cleanup_stats }
@@ -211,16 +217,12 @@ module Pgbus
         Pgbus.logger.warn { "[Pgbus] Outbox cleanup failed: #{e.message}" }
       end
 
-      def archive_compaction_interval
-        config.archive_compaction_interval || ARCHIVE_COMPACTION_INTERVAL
-      end
-
       def compact_archives
         retention = config.archive_retention
         return unless retention&.positive?
 
         cutoff = Time.current - retention
-        batch_size = config.archive_compaction_batch_size || 1000
+        batch_size = ARCHIVE_COMPACTION_BATCH_SIZE
         prefix = config.queue_prefix
 
         conn = config.connects_to ? Pgbus::BusRecord.connection : ActiveRecord::Base.connection

data/lib/pgbus/process/supervisor.rb

@@ -42,22 +42,17 @@ module Pgbus
       private
 
       def boot_processes
-        # Boot workers
-        config.workers.each do |worker_config|
-          fork_worker(worker_config)
-        end
-
-        # Boot dispatcher
-        fork_dispatcher
-
-        # Boot recurring scheduler if configured
-        boot_scheduler
-
-        # Boot event consumers if configured
-        boot_consumers
-
-        # Boot outbox poller if configured
-        boot_outbox_poller
+        # Boot workers (workers may be nil for scheduler-only or
+        # dispatcher-only deployments via --workers-only / --scheduler-only /
+        # --dispatcher-only CLI flags). Each role is gated by
+        # config.role_enabled?, which returns true unless +config.roles+ has
+        # been narrowed.
+        Array(config.workers).each { |worker_config| fork_worker(worker_config) } if config.role_enabled?(:workers)
+
+        fork_dispatcher if config.role_enabled?(:dispatcher)
+        boot_scheduler if config.role_enabled?(:scheduler)
+        boot_consumers if config.role_enabled?(:consumers)
+        boot_outbox_poller if config.role_enabled?(:outbox)
       end
 
       def fork_worker(worker_config)

data/lib/pgbus/process/worker.rb

@@ -217,7 +217,7 @@ module Pgbus
       def resolve_wildcard_queues
         return unless @wildcard
 
-        dlq_suffix = config.dead_letter_queue_suffix
+        dlq_suffix = Pgbus::DEAD_LETTER_SUFFIX
         prefix = "#{config.queue_prefix}_"
 
         conn = Pgbus.configuration.connects_to ? Pgbus::BusRecord.connection : ActiveRecord::Base.connection

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.4.1"
+  VERSION = "0.5.0"
 end

data/lib/pgbus/web/data_source.rb

@@ -16,7 +16,7 @@ module Pgbus
         queues = queues_with_metrics
         total_depth = queues.sum { |q| q[:queue_length] }
         total_visible = queues.sum { |q| q[:queue_visible_length] }
-        dlq_suffix = Pgbus.configuration.dead_letter_queue_suffix
+        dlq_suffix = Pgbus::DEAD_LETTER_SUFFIX
         dlq_depth = queues.select { |q| q[:name].end_with?(dlq_suffix) }.sum { |q| q[:queue_length] }
 
         throughput = compute_throughput(queues)
@@ -120,7 +120,7 @@ module Pgbus
       end
 
       def discard_all_enqueued
-        dlq_suffix = Pgbus.configuration.dead_letter_queue_suffix
+        dlq_suffix = Pgbus::DEAD_LETTER_SUFFIX
         queues = queues_with_metrics.reject { |q| q[:name].end_with?(dlq_suffix) }
         total = 0
 
@@ -259,7 +259,7 @@ module Pgbus
       # Note: DLQ queue names from queues_with_metrics are already fully qualified
       # (e.g., "pgbus_default_dlq"), so we use them directly without re-prefixing.
       def dlq_messages(page: 1, per_page: 25)
-        dlq_suffix = Pgbus.configuration.dead_letter_queue_suffix
+        dlq_suffix = Pgbus::DEAD_LETTER_SUFFIX
         queues = queues_with_metrics.select { |q| q[:name].end_with?(dlq_suffix) }
         offset = (page - 1) * per_page
 
@@ -274,7 +274,7 @@ module Pgbus
       end
 
       def dlq_message_detail(msg_id)
-        dlq_suffix = Pgbus.configuration.dead_letter_queue_suffix
+        dlq_suffix = Pgbus::DEAD_LETTER_SUFFIX
         queues = queues_with_metrics.select { |q| q[:name].end_with?(dlq_suffix) }
         queues.each do |q|
           row = connection.select_one(
@@ -292,7 +292,7 @@ module Pgbus
 
       def retry_dlq_message(queue_name, msg_id)
         # queue_name here is the full DLQ name (already prefixed)
-        dlq_suffix = Pgbus.configuration.dead_letter_queue_suffix
+        dlq_suffix = Pgbus::DEAD_LETTER_SUFFIX
         original_queue = queue_name.delete_suffix(dlq_suffix)
 
         row = connection.select_one(
@@ -664,7 +664,7 @@ module Pgbus
       end
 
       def all_queue_messages(limit, offset)
-        dlq_suffix = Pgbus.configuration.dead_letter_queue_suffix
+        dlq_suffix = Pgbus::DEAD_LETTER_SUFFIX
         queues = queues_with_metrics.reject { |q| q[:name].end_with?(dlq_suffix) }
         messages = queues.flat_map do |q|
           query_queue_messages_raw(q[:name], limit + offset, 0)

data/lib/pgbus.rb

@@ -3,6 +3,13 @@
 require "zeitwerk"
 
 module Pgbus
+  # Suffix appended to a queue name to derive its dead-letter companion
+  # (e.g. "pgbus_default" -> "pgbus_default_dlq"). Hard-coded here because
+  # changing it on a running deployment would orphan every existing DLQ
+  # message; nothing in the codebase or in user reports has ever needed
+  # this to be configurable.
+  DEAD_LETTER_SUFFIX = "_dlq"
+
   class Error < StandardError; end
   class ConfigurationError < Error; end
   class SerializationError < Error; end
@@ -24,7 +31,12 @@ module Pgbus
     def loader
       @loader ||= begin
         loader = Zeitwerk::Loader.for_gem
-        loader.inflector.inflect("pgbus" => "Pgbus", "cli" => "CLI", "dsl" => "DSL")
+        loader.inflector.inflect(
+          "pgbus" => "Pgbus",
+          "cli" => "CLI",
+          "dsl" => "DSL",
+          "capsule_dsl" => "CapsuleDSL"
+        )
         loader.ignore("#{__dir__}/generators")
         loader.ignore("#{__dir__}/active_job")
         loader

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgbus
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Mikael Henriksson
@@ -221,6 +221,7 @@ files:
 - lib/generators/pgbus/templates/pgbus_binstub.erb
 - lib/generators/pgbus/templates/recurring.yml.erb
 - lib/generators/pgbus/templates/upgrade_pgmq.rb.erb
+- lib/generators/pgbus/update_generator.rb
 - lib/generators/pgbus/upgrade_pgmq_generator.rb
 - lib/pgbus.rb
 - lib/pgbus/active_job/adapter.rb
@@ -235,6 +236,7 @@ files:
 - lib/pgbus/concurrency/semaphore.rb
 - lib/pgbus/config_loader.rb
 - lib/pgbus/configuration.rb
+- lib/pgbus/configuration/capsule_dsl.rb
 - lib/pgbus/dedup_cache.rb
 - lib/pgbus/engine.rb
 - lib/pgbus/event.rb
@@ -243,6 +245,7 @@ files:
 - lib/pgbus/event_bus/registry.rb
 - lib/pgbus/event_bus/subscriber.rb
 - lib/pgbus/failed_event_recorder.rb
+- lib/pgbus/generators/config_converter.rb
 - lib/pgbus/instrumentation.rb
 - lib/pgbus/outbox.rb
 - lib/pgbus/outbox/poller.rb