pgbus 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2741aec28cfa0ab8a9f2b88131a74a304198dbaa3426b0aca4b54b9f4c615a35
-  data.tar.gz: 0f164183827f32996556d304ad13dc5a00a9491f2a3ca19cbaa9a3b8aa21c975
+  metadata.gz: 2aad79af8c595a48d879b9bad0ebaaf43d40843cdbcb9dcc7a2dad0023257e53
+  data.tar.gz: 53bda3ba2e7d1d0f1935d183193bff5b48adeca8a4da44b6d55f9ba743e62d8e
 SHA512:
-  metadata.gz: d2ce8858dcd751ff70d653b847ffc8db202e655ff77d70276f8b538141bce8c964589e6ee2ed9ff68f9b78f080c952ac6661fd16ce2e555fd0d1578659e75adc
-  data.tar.gz: 2a55f808a68f8dbe879a7a6fdc674372fe42f3cb8ecbc91388b0f54b0c3c74f7e7b8424767f566caee1395e4393110f698b2b1c73c13d78fd2fe0105b59ed20a
+  metadata.gz: c9cfcd463d6b91591f9f037969ca1b30c2dc29ba0ce7a8eeefa829ef2b12eee71c4aba0f5b8d02b3336d6cc38b04db7cc51d749ca97774d4afa4e2246f8d6ccc
+  data.tar.gz: a50591b61b5bf49a2b5e14750622ba7ca0681d3db5f47d801542a23eb3147a8b9d5a605fcd0c55264ad4068e75f3d827ad550d2d58cd2026fa8a4351f72eea8a

data/lib/pgbus/active_job/adapter.rb

@@ -7,7 +7,7 @@ module Pgbus
     class Adapter
       def enqueue(active_job)
         queue = active_job.queue_name || Pgbus.configuration.default_queue
-        payload_hash = JSON.parse(Serializer.serialize_job(active_job))
+        payload_hash = Serializer.serialize_job_hash(active_job)
         payload_hash = Concurrency.inject_metadata(active_job, payload_hash)
         payload_hash = inject_batch_metadata(payload_hash)
 
@@ -16,7 +16,7 @@ module Pgbus
 
       def enqueue_at(active_job, timestamp)
         queue = active_job.queue_name || Pgbus.configuration.default_queue
-        payload_hash = JSON.parse(Serializer.serialize_job(active_job))
+        payload_hash = Serializer.serialize_job_hash(active_job)
         payload_hash = Concurrency.inject_metadata(active_job, payload_hash)
         payload_hash = inject_batch_metadata(payload_hash)
         delay = [(timestamp - Time.now.to_f).ceil, 0].max
@@ -88,7 +88,7 @@ module Pgbus
       def enqueue_immediate(queue, jobs)
         return if jobs.empty?
 
-        payloads = jobs.map { |j| JSON.parse(Serializer.serialize_job(j)) }
+        payloads = jobs.map { |j| Serializer.serialize_job_hash(j) }
         msg_ids = Pgbus.client.send_batch(queue, payloads)
 
         unless msg_ids.is_a?(Array) && msg_ids.size == jobs.size

data/lib/pgbus/active_job/executor.rb

@@ -28,8 +28,8 @@ module Pgbus
         Instrumentation.instrument("pgbus.executor.execute", queue: queue_name, job_class: job_class) do
           job = ::ActiveJob::Base.deserialize(payload)
           execute_job(job)
-          job_succeeded = true
           client.archive_message(queue_name, message.msg_id.to_i)
+          job_succeeded = true
         end
 
         instrument("pgbus.job_completed", queue: queue_name, job_class: job_class)
@@ -41,10 +41,9 @@ module Pgbus
         # Semaphore is released only on success or dead-lettering.
         :failed
       ensure
-        # Signal concurrency and batch AFTER archive, in an ensure block so they
-        # fire even if archive_message raises. This prevents the semaphore slot
-        # from being stuck until expiry when the archive DB call fails after the
-        # job has already completed successfully.
+        # Signal concurrency and batch only when the job was archived successfully.
+        # job_succeeded is set AFTER archive_message, so if archive fails the
+        # semaphore slot stays held until VT expires and the job is retried.
         if job_succeeded
           signal_concurrency(payload)
           signal_batch_completed(payload)

data/lib/pgbus/batch.rb

@@ -134,10 +134,12 @@ module Pgbus
       end
 
       def enqueue_callback(class_name, properties)
-        job_class = class_name.constantize
+        job_class = class_name.safe_constantize
+        unless job_class && job_class < ::ActiveJob::Base
+          Pgbus.logger.error { "[Pgbus] Batch callback class invalid or not an ActiveJob: #{class_name}" }
+          return
+        end
         job_class.perform_later(properties)
-      rescue NameError => e
-        Pgbus.logger.error { "[Pgbus] Batch callback class not found: #{class_name}: #{e.message}" }
       end
     end
   end

data/lib/pgbus/client.rb

@@ -24,28 +24,27 @@ module Pgbus
         pool_size: config.pool_size,
         pool_timeout: config.pool_timeout
       )
-      @queues_created = {}
-      @mutex = Mutex.new
+      @queues_created = Concurrent::Map.new
     end
 
     def ensure_queue(name)
       full_name = config.queue_name(name)
-      @mutex.synchronize do
-        return if @queues_created[full_name]
+      return if @queues_created[full_name]
 
+      @queues_created.compute_if_absent(full_name) do
         @pgmq.create(full_name)
         @pgmq.enable_notify_insert(full_name, throttle_interval_ms: config.notify_throttle_ms) if config.listen_notify
-        @queues_created[full_name] = true
+        true
       end
     end
 
     def ensure_dead_letter_queue(name)
       dlq_name = config.dead_letter_queue_name(name)
-      @mutex.synchronize do
-        return if @queues_created[dlq_name]
+      return if @queues_created[dlq_name]
 
+      @queues_created.compute_if_absent(dlq_name) do
         @pgmq.create(dlq_name)
-        @queues_created[dlq_name] = true
+        true
       end
     end
 

data/lib/pgbus/configuration.rb

@@ -116,6 +116,21 @@ module Pgbus
       @pgmq_schema_mode = mode
     end
 
+    def validate!
+      raise ArgumentError, "pool_size must be > 0" unless pool_size.is_a?(Numeric) && pool_size.positive?
+      raise ArgumentError, "pool_timeout must be > 0" unless pool_timeout.is_a?(Numeric) && pool_timeout.positive?
+      raise ArgumentError, "polling_interval must be > 0" unless polling_interval.is_a?(Numeric) && polling_interval.positive?
+      raise ArgumentError, "visibility_timeout must be > 0" unless visibility_timeout.is_a?(Numeric) && visibility_timeout.positive?
+      raise ArgumentError, "max_retries must be >= 0" unless max_retries.is_a?(Integer) && max_retries >= 0
+
+      workers.each do |w|
+        threads = w[:threads] || w["threads"] || 5
+        raise ArgumentError, "worker threads must be > 0" unless threads.is_a?(Integer) && threads.positive?
+      end
+
+      self
+    end
+
     def connection_options
       if database_url
         database_url

data/lib/pgbus/event_bus/handler.rb

@@ -17,11 +17,7 @@ module Pgbus
         raw = JSON.parse(message.message)
         event = build_event(raw)
 
-        if self.class.idempotent?
-          return :skipped if already_processed?(event.event_id)
-
-          mark_processed!(event.event_id)
-        end
+        return :skipped if self.class.idempotent? && !claim_idempotency?(event.event_id)
 
         handle(event)
         instrument("pgbus.event_processed", event_id: event.event_id, handler: self.class.name)
@@ -51,15 +47,16 @@ module Pgbus
         ActiveSupport::Notifications.instrument(event_name, payload)
       end
 
-      def already_processed?(event_id)
-        ProcessedEvent.exists?(event_id: event_id, handler_class: self.class.name)
-      end
-
-      def mark_processed!(event_id)
-        ProcessedEvent.upsert(
+      # Atomically claim idempotency: INSERT ... ON CONFLICT DO NOTHING.
+      # Returns true if this handler claimed the event (row was inserted),
+      # false if another handler already processed it (conflict, no insert).
+      def claim_idempotency?(event_id)
+        result = ProcessedEvent.insert(
           { event_id: event_id, handler_class: self.class.name, processed_at: Time.now.utc },
           unique_by: %i[event_id handler_class]
         )
+        # insert returns an InsertAll::Result; inserted row count > 0 means we claimed it
+        result.rows.any?
       end
     end
   end

data/lib/pgbus/process/consumer.rb

@@ -71,6 +71,12 @@ module Pgbus
       end
 
       def handle_message(message, queue_name)
+        if message.read_ct.to_i > config.max_retries
+          Pgbus.logger.warn { "[Pgbus] Consumer moving message #{message.msg_id} to DLQ after #{message.read_ct} reads" }
+          Pgbus.client.move_to_dead_letter(queue_name, message)
+          return
+        end
+
         raw = JSON.parse(message.message)
         routing_key = raw.dig("headers", "routing_key") || raw["routing_key"]
 
@@ -83,6 +89,9 @@ module Pgbus
         Pgbus.client.archive_message(queue_name, message.msg_id.to_i)
       rescue StandardError => e
         Pgbus.logger.error { "[Pgbus] Consumer error: #{e.class}: #{e.message}" }
+        # Message stays in queue; VT will expire and it becomes available again.
+        # read_ct tracks delivery attempts — when it exceeds max_retries,
+        # the next read will route to DLQ above.
       end
 
       def pattern_overlaps?(topic_filter, subscription_pattern)

data/lib/pgbus/process/dispatcher.rb

@@ -59,30 +59,20 @@ module Pgbus
       def run_maintenance
         now = Time.now
 
-        if now - @last_cleanup_at >= CLEANUP_INTERVAL
-          cleanup_processed_events
-          @last_cleanup_at = now
-        end
-
-        if now - @last_reap_at >= REAP_INTERVAL
-          reap_stale_processes
-          @last_reap_at = now
-        end
-
-        if now - @last_concurrency_at >= CONCURRENCY_INTERVAL
-          cleanup_concurrency
-          @last_concurrency_at = now
-        end
+        run_if_due(now, :@last_cleanup_at, CLEANUP_INTERVAL) { cleanup_processed_events }
+        run_if_due(now, :@last_reap_at, REAP_INTERVAL) { reap_stale_processes }
+        run_if_due(now, :@last_concurrency_at, CONCURRENCY_INTERVAL) { cleanup_concurrency }
+        run_if_due(now, :@last_batch_cleanup_at, BATCH_CLEANUP_INTERVAL) { cleanup_batches }
+        run_if_due(now, :@last_recurring_cleanup_at, RECURRING_CLEANUP_INTERVAL) { cleanup_recurring_executions }
+      end
 
-        if now - @last_batch_cleanup_at >= BATCH_CLEANUP_INTERVAL
-          cleanup_batches
-          @last_batch_cleanup_at = now
-        end
+      # Only update the timestamp when the block succeeds.
+      # On failure, the next tick retries instead of waiting the full interval.
+      def run_if_due(now, ivar, interval)
+        return unless now - instance_variable_get(ivar) >= interval
 
-        if now - @last_recurring_cleanup_at >= RECURRING_CLEANUP_INTERVAL
-          cleanup_recurring_executions
-          @last_recurring_cleanup_at = now
-        end
+        yield
+        instance_variable_set(ivar, now)
       rescue StandardError => e
         Pgbus.logger.error { "[Pgbus] Dispatcher maintenance error: #{e.message}" }
       end

data/lib/pgbus/process/supervisor.rb

@@ -69,8 +69,15 @@ module Pgbus
           worker.run
         end
 
+        unless pid
+          Pgbus.logger.error { "[Pgbus] Failed to fork worker for queues=#{queues.join(",")}" }
+          return
+        end
+
         @forks[pid] = { type: :worker, config: worker_config }
         Pgbus.logger.info { "[Pgbus] Forked worker pid=#{pid} queues=#{queues.join(",")}" }
+      rescue Errno::EAGAIN, Errno::ENOMEM => e
+        Pgbus.logger.error { "[Pgbus] Fork failed for worker: #{e.message}" }
       end
 
       def fork_dispatcher
@@ -82,8 +89,15 @@ module Pgbus
           dispatcher.run
         end
 
+        unless pid
+          Pgbus.logger.error { "[Pgbus] Failed to fork dispatcher" }
+          return
+        end
+
         @forks[pid] = { type: :dispatcher }
         Pgbus.logger.info { "[Pgbus] Forked dispatcher pid=#{pid}" }
+      rescue Errno::EAGAIN, Errno::ENOMEM => e
+        Pgbus.logger.error { "[Pgbus] Fork failed for dispatcher: #{e.message}" }
       end
 
       def boot_scheduler
@@ -103,8 +117,15 @@ module Pgbus
           scheduler.run
         end
 
+        unless pid
+          Pgbus.logger.error { "[Pgbus] Failed to fork scheduler" }
+          return
+        end
+
         @forks[pid] = { type: :scheduler }
         Pgbus.logger.info { "[Pgbus] Forked scheduler pid=#{pid}" }
+      rescue Errno::EAGAIN, Errno::ENOMEM => e
+        Pgbus.logger.error { "[Pgbus] Fork failed for scheduler: #{e.message}" }
       end
 
       def recurring_tasks_configured?
@@ -150,8 +171,15 @@ module Pgbus
           consumer.run
         end
 
+        unless pid
+          Pgbus.logger.error { "[Pgbus] Failed to fork consumer for topics=#{topics.join(",")}" }
+          return
+        end
+
         @forks[pid] = { type: :consumer, config: consumer_config }
         Pgbus.logger.info { "[Pgbus] Forked consumer pid=#{pid} topics=#{topics.join(",")}" }
+      rescue Errno::EAGAIN, Errno::ENOMEM => e
+        Pgbus.logger.error { "[Pgbus] Fork failed for consumer: #{e.message}" }
       end
 
       def monitor_loop

data/lib/pgbus/process/worker.rb

@@ -32,11 +32,12 @@ module Pgbus
         Pgbus.logger.info { "[Pgbus] Worker started: queues=#{queues.join(",")} threads=#{threads} pid=#{::Process.pid}" }
 
         loop do
-          break if @shutting_down
-          break if recycle_needed?
-
           process_signals
-          claim_and_execute
+          break if @shutting_down && @pool.queue_length.zero?
+          break if recycle_needed? && @pool.queue_length.zero?
+
+          claim_and_execute unless @shutting_down || recycle_needed?
+          interruptible_sleep(config.polling_interval) if (@shutting_down || recycle_needed?) && !@pool.queue_length.zero?
         end
 
         shutdown

data/lib/pgbus/recurring/command_job.rb

@@ -3,13 +3,25 @@
 module Pgbus
   module Recurring
     # Job class for command-based recurring tasks.
-    # Executes a Ruby command string, similar to solid_queue's RecurringJob.
+    # Evaluates a method call chain on a constant, e.g. "OldRecord.cleanup!".
     #
-    # NOTE: Only use this with trusted commands from config/recurring.yml.
-    # Never expose this to user input.
+    # Only supports `ConstantName.method_name` and `ConstantName.method_name(args)`.
+    # Rejects arbitrary Ruby expressions for safety.
     class CommandJob < ::ActiveJob::Base
+      SAFE_COMMAND = /\A([A-Z][A-Za-z0-9_]*(?:::[A-Z][A-Za-z0-9_]*)*)\.([a-z_][a-z0-9_!?]*)\z/
+
       def perform(command)
-        eval(command) # rubocop:disable Security/Eval
+        match = SAFE_COMMAND.match(command)
+        unless match
+          raise ArgumentError,
+                "Unsafe recurring command: #{command.inspect}. " \
+                "Must be in the form 'ClassName.method_name'."
+        end
+
+        klass = match[1].safe_constantize
+        raise ArgumentError, "Unknown class in recurring command: #{match[1]}" unless klass
+
+        klass.public_send(match[2])
       end
     end
   end

data/lib/pgbus/serializer.rb

@@ -15,6 +15,12 @@ module Pgbus
       end
     end
 
+    def serialize_job_hash(active_job)
+      Instrumentation.instrument("pgbus.serializer.serialize", kind: :job) do
+        active_job.serialize
+      end
+    end
+
     def deserialize_job(json_string)
       Instrumentation.instrument("pgbus.serializer.deserialize", kind: :job) do
         data = JSON.parse(json_string)

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/lib/pgbus/web/data_source.rb

@@ -129,7 +129,9 @@ module Pgbus
 
         connection.transaction do
           @client.send_message(event["queue_name"], payload, headers: headers)
-          connection.execute("DELETE FROM pgbus_failed_events WHERE id = #{id.to_i}")
+          connection.exec_delete(
+            "DELETE FROM pgbus_failed_events WHERE id = $1", "Pgbus Delete Failed Event", [id.to_i]
+          )
         end
         true
       rescue StandardError => e
@@ -138,7 +140,9 @@ module Pgbus
       end
 
       def discard_failed_event(id)
-        connection.execute("DELETE FROM pgbus_failed_events WHERE id = #{id.to_i}")
+        connection.exec_delete(
+          "DELETE FROM pgbus_failed_events WHERE id = $1", "Pgbus Delete Failed Event", [id.to_i]
+        )
         true
       rescue StandardError => e
         Pgbus.logger.debug { "[Pgbus::Web] Error discarding failed event #{id}: #{e.message}" }
@@ -147,18 +151,27 @@ module Pgbus
 
       def retry_all_failed
         count = 0
-        connection.select_all("SELECT * FROM pgbus_failed_events").each do |event|
-          payload = JSON.parse(event["payload"])
-          headers = event["headers"]
-          headers = JSON.parse(headers) if headers.is_a?(String)
-
-          connection.transaction do
-            @client.send_message(event["queue_name"], payload, headers: headers)
-            connection.execute("DELETE FROM pgbus_failed_events WHERE id = #{event["id"].to_i}")
+        loop do
+          batch = connection.select_all(
+            "SELECT * FROM pgbus_failed_events ORDER BY id LIMIT 100", "Pgbus Retry Batch"
+          ).to_a
+          break if batch.empty?
+
+          batch.each do |event|
+            payload = JSON.parse(event["payload"])
+            headers = event["headers"]
+            headers = JSON.parse(headers) if headers.is_a?(String)
+
+            connection.transaction do
+              @client.send_message(event["queue_name"], payload, headers: headers)
+              connection.exec_delete(
+                "DELETE FROM pgbus_failed_events WHERE id = $1", "Pgbus Delete Failed Event", [event["id"].to_i]
+              )
+            end
+            count += 1
+          rescue StandardError => e
+            Pgbus.logger.error { "[Pgbus::Web] Failed to retry event #{event["id"]}: #{e.message}" }
           end
-          count += 1
-        rescue StandardError => e
-          Pgbus.logger.error { "[Pgbus::Web] Failed to retry event #{event["id"]}: #{e.message}" }
         end
         count
       end
@@ -552,7 +565,10 @@ module Pgbus
       end
 
       def sanitize_name(name)
-        name.gsub(/[^a-zA-Z0-9_]/, "")
+        sanitized = name.gsub(/[^a-zA-Z0-9_]/, "")
+        raise ArgumentError, "Invalid queue name: #{name.inspect}" if sanitized.empty?
+
+        sanitized
       end
 
       def parse_arguments(args)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgbus
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Mikael Henriksson