pgbus 0.1.5 → 0.1.7

data/lib/pgbus/process/wake_signal.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "concurrent"
+
+module Pgbus
+  module Process
+    # Wake signal inspired by LavinMQ's BoolChannel pattern.
+    # Replaces polling-based coordination with instant state-change signaling.
+    #
+    # IMPORTANT: Single-waiter only. The +wait+ method calls @event.reset
+    # immediately after waking, which means concurrent waiters may miss
+    # notifications. Callers must ensure only one thread calls +wait+ at
+    # a time. In pgbus this is guaranteed because each Worker has exactly
+    # one main loop thread that calls +wait+, while +notify!+ can be
+    # called from any thread (signal handlers, lifecycle transitions).
+    #
+    # Usage:
+    #   signal = WakeSignal.new
+    #   # In worker thread (single waiter):
+    #   signal.wait(timeout: 5)  # blocks until signaled or timeout
+    #   # In another thread (e.g. signal handler, lifecycle transition):
+    #   signal.notify!           # wakes the waiting thread
+    class WakeSignal
+      def initialize
+        @event = Concurrent::Event.new
+      end
+
+      # Block until +notify!+ is called or timeout expires.
+      # Returns true if signaled, false if timed out.
+      # Resets the event after waking — only safe with a single waiter.
+      def wait(timeout: nil)
+        result = @event.wait(timeout)
+        @event.reset
+        result
+      end
+
+      # Wake all waiting threads immediately.
+      def notify!
+        @event.set
+      end
+
+      # Check if a notification is pending without blocking.
+      def pending?
+        @event.set?
+      end
+
+      # Clear the pending notification.
+      def reset!
+        @event.reset
+      end
+    end
+  end
+end

data/lib/pgbus/process/worker.rb

@@ -9,10 +9,13 @@ module Pgbus
 
       attr_reader :queues, :threads, :config
 
-      def initialize(queues:, threads: 5, config: Pgbus.configuration)
+      def initialize(queues:, threads: 5, config: Pgbus.configuration,
+                     single_active_consumer: false, consumer_priority: 0)
         @queues = Array(queues)
         @threads = threads
         @config = config
+        @single_active_consumer = single_active_consumer
+        @consumer_priority = consumer_priority
         @lifecycle = Lifecycle.new
         @jobs_processed = Concurrent::AtomicFixnum.new(0)
         @jobs_failed = Concurrent::AtomicFixnum.new(0)
@@ -22,6 +25,8 @@ module Pgbus
         @executor = Pgbus::ActiveJob::Executor.new
         @pool = Concurrent::FixedThreadPool.new(threads)
         @circuit_breaker = Pgbus::CircuitBreaker.new(config: config)
+        @queue_lock = QueueLock.new if @single_active_consumer
+        @wake_signal = WakeSignal.new
       end
 
       def stats
@@ -30,6 +35,9 @@ module Pgbus
           jobs_failed: @jobs_failed.value,
           in_flight: @in_flight.value,
           state: @lifecycle.state,
+          consumer_priority: @consumer_priority,
+          single_active_consumer: @single_active_consumer,
+          locked_queues: @queue_lock&.held_queues || [],
           rates: @rate_counter.rates,
           started_at: @started_at
         }
@@ -50,7 +58,7 @@ module Pgbus
           break if @lifecycle.draining? && @pool.queue_length.zero?
 
           claim_and_execute if @lifecycle.can_process?
-          interruptible_sleep(config.polling_interval) if @lifecycle.draining? || @lifecycle.paused?
+          @wake_signal.wait(timeout: config.polling_interval) if @lifecycle.draining? || @lifecycle.paused?
         end
 
         shutdown
@@ -59,23 +67,27 @@ module Pgbus
       def graceful_shutdown
         Pgbus.logger.info { "[Pgbus] Worker shutting down gracefully..." }
         @lifecycle.transition_to(:draining)
+        @wake_signal.notify!
       end
 
       def immediate_shutdown
         Pgbus.logger.warn { "[Pgbus] Worker shutting down immediately!" }
         @lifecycle.transition_to!(:stopped)
+        @wake_signal.notify!
         @pool.kill
       end
 
       private
 
       def claim_and_execute
+        poll_interval = effective_polling_interval
+
         idle = @pool.max_length - @pool.queue_length
-        return interruptible_sleep(config.polling_interval) if idle <= 0
+        return @wake_signal.wait(timeout: poll_interval) if idle <= 0
 
         if config.prefetch_limit
           available = config.prefetch_limit - @in_flight.value
-          return interruptible_sleep(config.polling_interval) if available <= 0
+          return @wake_signal.wait(timeout: poll_interval) if available <= 0
 
           idle = [idle, available].min
         end
@@ -83,7 +95,7 @@ module Pgbus
         tagged_messages = fetch_messages(idle)
 
         if tagged_messages.empty?
-          interruptible_sleep(config.polling_interval)
+          @wake_signal.wait(timeout: poll_interval)
           return
         end
 
@@ -98,6 +110,7 @@ module Pgbus
       # which queue each message came from (PGMQ messages don't carry this).
       def fetch_messages(qty)
         active_queues = queues.reject { |q| @circuit_breaker.paused?(q) }
+        active_queues = active_queues.select { |q| @queue_lock.try_lock(q) } if @single_active_consumer
         return [] if active_queues.empty?
 
         if priority_enabled?
@@ -188,6 +201,7 @@ module Pgbus
         return unless @lifecycle.running? && recycle_needed?
 
         @lifecycle.transition_to(:draining)
+        @wake_signal.notify!
       end
 
       def recycle_needed?
@@ -227,10 +241,25 @@ module Pgbus
         end
       end
 
+      def effective_polling_interval
+        return config.polling_interval if @consumer_priority.zero?
+
+        ConsumerPriority.effective_polling_interval(
+          base_interval: config.polling_interval,
+          my_priority: @consumer_priority,
+          max_priority: ConsumerPriority.max_active_priority(queues, ::Process.pid)
+        )
+      rescue StandardError
+        config.polling_interval
+      end
+
       def start_heartbeat
         @heartbeat = Heartbeat.new(
           kind: "worker",
-          metadata: { queues: queues, threads: threads, pid: ::Process.pid },
+          metadata: {
+            queues: queues, threads: threads, pid: ::Process.pid,
+            consumer_priority: @consumer_priority
+          },
           on_beat: -> { @rate_counter.snapshot! }
         )
         @heartbeat.start
@@ -240,6 +269,7 @@ module Pgbus
         Pgbus.logger.info { "[Pgbus] Worker draining thread pool..." }
         @pool.shutdown
         @pool.wait_for_termination(30)
+        @queue_lock&.unlock_all
         @heartbeat&.stop
         restore_signals
         Pgbus.logger.info { "[Pgbus] Worker stopped. Processed: #{@jobs_processed.value}, Failed: #{@jobs_failed.value}" }

data/lib/pgbus/queue_factory.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Pgbus
+  # Dispatches queue operations based on queue type (standard vs priority).
+  # Replaces conditional `priority_enabled?` checks scattered through Client
+  # with a single strategy object selected at initialization.
+  #
+  # Inspired by LavinMQ's QueueFactory which dispatches queue creation by
+  # type: standard, durable, priority, stream, delayed.
+  module QueueFactory
+    def self.for(config)
+      if config.priority_levels && config.priority_levels > 1
+        PriorityStrategy.new(config)
+      else
+        StandardStrategy.new(config)
+      end
+    end
+
+    # Standard single-queue strategy: one PGMQ queue per logical name.
+    class StandardStrategy
+      def initialize(config)
+        @config = config
+      end
+
+      def physical_queue_names(name)
+        [@config.queue_name(name)]
+      end
+
+      def target_queue(name, _priority)
+        @config.queue_name(name)
+      end
+
+      def priority?
+        false
+      end
+    end
+
+    # Priority sub-queue strategy: N PGMQ queues per logical name (_p0.._pN).
+    class PriorityStrategy
+      def initialize(config)
+        @config = config
+      end
+
+      def physical_queue_names(name)
+        @config.priority_queue_names(name)
+      end
+
+      def target_queue(name, priority)
+        level = if priority
+                  priority.clamp(0, @config.priority_levels - 1)
+                else
+                  @config.default_priority.clamp(0, @config.priority_levels - 1)
+                end
+        @config.priority_queue_name(name, level)
+      end
+
+      def priority?
+        true
+      end
+    end
+  end
+end

data/lib/pgbus/uniqueness.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require "active_support/concern"
+require "socket"
+
+module Pgbus
+  # Job uniqueness guarantees: prevent duplicate jobs from running concurrently.
+  #
+  # Unlike concurrency limits (which allow N concurrent jobs for the same key),
+  # uniqueness ensures AT MOST ONE job with a given key exists in the system
+  # at any time — from enqueue through completion.
+  #
+  # Lock lifecycle:
+  #   1. Enqueue: lock acquired (state: queued), no owner yet
+  #   2. Execution start: lock transitions to (state: executing, owner_pid: PID)
+  #   3. Completion/DLQ: lock released (row deleted)
+  #   4. Crash recovery: reaper detects orphaned locks by cross-referencing
+  #      owner_pid against pgbus_processes heartbeats
+  #   5. Last resort: expires_at TTL (default 24h) handles the case where
+  #      even the reaper can't run (entire supervisor dead)
+  #
+  # Strategies:
+  #   :until_executed  — Lock acquired at enqueue, held through execution, released on
+  #                      completion or DLQ. Prevents duplicate enqueue AND duplicate execution.
+  #
+  #   :while_executing — Lock acquired at execution start, released on completion.
+  #                      Allows duplicate enqueue (multiple copies in queue) but only one
+  #                      executes at a time.
+  #
+  # Usage:
+  #   class ImportOrderJob < ApplicationJob
+  #     ensures_uniqueness strategy: :until_executed,
+  #                        key: ->(order_id) { "import-order-#{order_id}" },
+  #                        on_conflict: :reject
+  #
+  #     def perform(order_id)
+  #       # Only one instance of this job per order_id can exist at a time
+  #     end
+  #   end
+  module Uniqueness
+    extend ActiveSupport::Concern
+
+    METADATA_KEY = "pgbus_uniqueness_key"
+    STRATEGY_KEY = "pgbus_uniqueness_strategy"
+    TTL_KEY = "pgbus_uniqueness_lock_ttl"
+
+    # 24 hours — last-resort fallback only. The reaper handles normal crash recovery.
+    DEFAULT_LOCK_TTL = 24 * 60 * 60
+
+    VALID_STRATEGIES = %i[until_executed while_executing].freeze
+    VALID_CONFLICTS = %i[reject discard log].freeze
+
+    class_methods do
+      def ensures_uniqueness(strategy: :until_executed, key: nil, lock_ttl: DEFAULT_LOCK_TTL, on_conflict: :reject)
+        raise ArgumentError, "strategy must be one of: #{VALID_STRATEGIES.join(", ")}" unless VALID_STRATEGIES.include?(strategy)
+        raise ArgumentError, "on_conflict must be one of: #{VALID_CONFLICTS.join(", ")}" unless VALID_CONFLICTS.include?(on_conflict)
+
+        valid_ttl = lock_ttl.is_a?(Numeric) || (defined?(ActiveSupport::Duration) && lock_ttl.is_a?(ActiveSupport::Duration))
+        raise ArgumentError, "lock_ttl must be a positive number or Duration" unless valid_ttl && lock_ttl.positive?
+        raise ArgumentError, "key must be callable (Proc or lambda)" if key && !key.respond_to?(:call)
+
+        @pgbus_uniqueness = {
+          strategy: strategy,
+          key: key || ->(*) { name },
+          lock_ttl: lock_ttl,
+          on_conflict: on_conflict
+        }.freeze
+      end
+
+      def pgbus_uniqueness
+        @pgbus_uniqueness
+      end
+    end
+
+    class << self
+      def resolve_key(active_job)
+        config = uniqueness_config(active_job)
+        return nil unless config
+
+        args = active_job.arguments
+        last = args.last
+        if last.is_a?(Hash) && last.each_key.all?(Symbol)
+          config[:key].call(*args[...-1], **last)
+        else
+          config[:key].call(*args)
+        end
+      end
+
+      def inject_metadata(active_job, payload_hash)
+        config = uniqueness_config(active_job)
+        return payload_hash unless config
+
+        key = resolve_key(active_job)
+        return payload_hash unless key
+
+        payload_hash.merge(
+          METADATA_KEY => key,
+          STRATEGY_KEY => config[:strategy].to_s,
+          TTL_KEY => config[:lock_ttl]
+        )
+      end
+
+      def extract_key(payload)
+        payload&.dig(METADATA_KEY)
+      end
+
+      def extract_strategy(payload)
+        payload&.dig(STRATEGY_KEY)&.to_sym
+      end
+
+      def uniqueness_config(active_job)
+        return nil unless active_job.class.respond_to?(:pgbus_uniqueness)
+
+        active_job.class.pgbus_uniqueness
+      end
+
+      # Acquire the uniqueness lock at enqueue time (:until_executed only).
+      # Lock state: queued, no owner_pid yet.
+      # Returns :acquired or :locked.
+      def acquire_enqueue_lock(key, active_job)
+        config = uniqueness_config(active_job)
+        return :no_lock unless config
+        return :no_lock unless config[:strategy] == :until_executed
+
+        acquired = JobLock.acquire!(
+          key,
+          job_class: active_job.class.name,
+          job_id: active_job.job_id,
+          state: "queued",
+          ttl: config[:lock_ttl]
+        )
+        acquired ? :acquired : :locked
+      end
+
+      # Transition a queued lock to executing state when the worker picks it up.
+      # Called for :until_executed jobs at execution start.
+      def claim_for_execution!(key, ttl:)
+        JobLock.claim_for_execution!(key, owner_pid: ::Process.pid, owner_hostname: Socket.gethostname, ttl: ttl)
+      end
+
+      # Acquire the uniqueness lock at execution time (:while_executing only).
+      # Lock state: executing with owner_pid.
+      # Returns true if acquired, false if another instance is running.
+      def acquire_execution_lock(key, payload)
+        strategy = extract_strategy(payload)
+        return true unless strategy == :while_executing
+
+        ttl = payload[TTL_KEY] || DEFAULT_LOCK_TTL
+
+        JobLock.acquire!(
+          key,
+          job_class: payload["job_class"],
+          job_id: payload["job_id"],
+          state: "executing",
+          owner_pid: ::Process.pid,
+          owner_hostname: Socket.gethostname,
+          ttl: ttl
+        )
+      end
+
+      # Release the uniqueness lock after execution completes.
+      def release_lock(key)
+        return unless key
+
+        JobLock.release!(key)
+      end
+    end
+  end
+end

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.1.5"
+  VERSION = "0.1.7"
 end

data/lib/pgbus/web/data_source.rb

@@ -497,6 +497,55 @@ module Pgbus
         []
       end
 
+      # Job locks
+      def job_locks
+        JobLock.order(locked_at: :desc).limit(100).map do |lock|
+          {
+            lock_key: lock.lock_key,
+            job_class: lock.job_class,
+            job_id: lock.job_id,
+            state: lock.state,
+            owner_pid: lock.owner_pid,
+            owner_hostname: lock.owner_hostname,
+            locked_at: lock.locked_at,
+            expires_at: lock.expires_at,
+            age_seconds: lock.locked_at ? (Time.current - lock.locked_at).to_i : nil
+          }
+        end
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching job locks: #{e.message}" }
+        []
+      end
+
+      # Job stats
+      def job_stats_summary(minutes: 60)
+        JobStat.summary(minutes: minutes)
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching job stats summary: #{e.message}" }
+        { total: 0, success: 0, failed: 0, dead_lettered: 0, avg_duration_ms: 0, max_duration_ms: 0 }
+      end
+
+      def job_throughput(minutes: 60)
+        JobStat.throughput(minutes: minutes).map { |time, count| { time: time, count: count } }
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching throughput: #{e.message}" }
+        []
+      end
+
+      def job_status_counts(minutes: 60)
+        JobStat.status_counts(minutes: minutes)
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching status counts: #{e.message}" }
+        {}
+      end
+
+      def slowest_job_classes(limit: 10, minutes: 60)
+        JobStat.slowest_classes(limit: limit, minutes: minutes)
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching slowest classes: #{e.message}" }
+        []
+      end
+
       # Subscriber registry
       def registered_subscribers
         EventBus::Registry.instance.subscribers.map do |s|

data/lib/pgbus.rb

@@ -9,6 +9,7 @@ module Pgbus
   class QueueNotFoundError < Error; end
   class DeadLetterError < Error; end
   class ConcurrencyLimitExceeded < Error; end
+  class JobNotUnique < Error; end
   class SchemaNotReady < Error; end
 
   class << self

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgbus
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.7
 platform: ruby
 authors:
 - Mikael Henriksson
@@ -107,12 +107,15 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- app/controllers/pgbus/api/insights_controller.rb
 - app/controllers/pgbus/api/stats_controller.rb
 - app/controllers/pgbus/application_controller.rb
 - app/controllers/pgbus/dashboard_controller.rb
 - app/controllers/pgbus/dead_letter_controller.rb
 - app/controllers/pgbus/events_controller.rb
+- app/controllers/pgbus/insights_controller.rb
 - app/controllers/pgbus/jobs_controller.rb
+- app/controllers/pgbus/locks_controller.rb
 - app/controllers/pgbus/outbox_controller.rb
 - app/controllers/pgbus/processes_controller.rb
 - app/controllers/pgbus/queues_controller.rb
@@ -121,6 +124,8 @@ files:
 - app/models/pgbus/application_record.rb
 - app/models/pgbus/batch_entry.rb
 - app/models/pgbus/blocked_execution.rb
+- app/models/pgbus/job_lock.rb
+- app/models/pgbus/job_stat.rb
 - app/models/pgbus/outbox_entry.rb
 - app/models/pgbus/process_entry.rb
 - app/models/pgbus/processed_event.rb
@@ -139,10 +144,12 @@ files:
 - app/views/pgbus/dead_letter/show.html.erb
 - app/views/pgbus/events/index.html.erb
 - app/views/pgbus/events/show.html.erb
+- app/views/pgbus/insights/show.html.erb
 - app/views/pgbus/jobs/_enqueued_table.html.erb
 - app/views/pgbus/jobs/_failed_table.html.erb
 - app/views/pgbus/jobs/index.html.erb
 - app/views/pgbus/jobs/show.html.erb
+- app/views/pgbus/locks/index.html.erb
 - app/views/pgbus/outbox/index.html.erb
 - app/views/pgbus/processes/_processes_table.html.erb
 - app/views/pgbus/processes/index.html.erb
@@ -155,10 +162,14 @@ files:
 - config/routes.rb
 - exe/pgbus
 - lib/active_job/queue_adapters/pgbus_adapter.rb
+- lib/generators/pgbus/add_job_locks_generator.rb
+- lib/generators/pgbus/add_job_stats_generator.rb
 - lib/generators/pgbus/add_outbox_generator.rb
 - lib/generators/pgbus/add_queue_states_generator.rb
 - lib/generators/pgbus/add_recurring_generator.rb
 - lib/generators/pgbus/install_generator.rb
+- lib/generators/pgbus/templates/add_job_locks.rb.erb
+- lib/generators/pgbus/templates/add_job_stats.rb.erb
 - lib/generators/pgbus/templates/add_outbox.rb.erb
 - lib/generators/pgbus/templates/add_queue_states.rb.erb
 - lib/generators/pgbus/templates/add_recurring_tables.rb.erb
@@ -193,12 +204,16 @@ files:
 - lib/pgbus/pgmq_schema.rb
 - lib/pgbus/pgmq_schema/pgmq_v1.11.0.sql
 - lib/pgbus/process/consumer.rb
+- lib/pgbus/process/consumer_priority.rb
 - lib/pgbus/process/dispatcher.rb
 - lib/pgbus/process/heartbeat.rb
 - lib/pgbus/process/lifecycle.rb
+- lib/pgbus/process/queue_lock.rb
 - lib/pgbus/process/signal_handler.rb
 - lib/pgbus/process/supervisor.rb
+- lib/pgbus/process/wake_signal.rb
 - lib/pgbus/process/worker.rb
+- lib/pgbus/queue_factory.rb
 - lib/pgbus/rate_counter.rb
 - lib/pgbus/recurring/already_recorded.rb
 - lib/pgbus/recurring/command_job.rb
@@ -207,6 +222,7 @@ files:
 - lib/pgbus/recurring/scheduler.rb
 - lib/pgbus/recurring/task.rb
 - lib/pgbus/serializer.rb
+- lib/pgbus/uniqueness.rb
 - lib/pgbus/version.rb
 - lib/pgbus/web/authentication.rb
 - lib/pgbus/web/data_source.rb