pgbus 0.0.1 → 0.1.2

data/lib/pgbus/process/worker.rb

@@ -28,14 +28,16 @@ module Pgbus
       def run
         setup_signals
         start_heartbeat
+        resolve_wildcard_queues
         Pgbus.logger.info { "[Pgbus] Worker started: queues=#{queues.join(",")} threads=#{threads} pid=#{::Process.pid}" }
 
         loop do
-          break if @shutting_down
-          break if recycle_needed?
-
           process_signals
-          claim_and_execute
+          break if @shutting_down && @pool.queue_length.zero?
+          break if recycle_needed? && @pool.queue_length.zero?
+
+          claim_and_execute unless @shutting_down || recycle_needed?
+          interruptible_sleep(config.polling_interval) if (@shutting_down || recycle_needed?) && !@pool.queue_length.zero?
         end
 
         shutdown
@@ -56,29 +58,31 @@ module Pgbus
 
       def claim_and_execute
         idle = @pool.max_length - @pool.queue_length
-        return sleep(config.polling_interval) if idle <= 0
+        return interruptible_sleep(config.polling_interval) if idle <= 0
 
-        messages = fetch_messages(idle)
+        tagged_messages = fetch_messages(idle)
 
-        if messages.empty?
-          sleep(config.polling_interval)
+        if tagged_messages.empty?
+          interruptible_sleep(config.polling_interval)
           return
         end
 
-        messages.each do |message|
-          queue_name = message.respond_to?(:queue_name) ? message.queue_name : queues.first
+        tagged_messages.each do |queue_name, message|
           @pool.post { process_message(message, queue_name) }
         end
       end
 
+      # Returns an array of [queue_name, message] pairs so we always know
+      # which queue each message came from (PGMQ messages don't carry this).
       def fetch_messages(qty)
         if queues.size == 1
-          Pgbus.client.read_batch(queues.first, qty: qty) || []
+          queue = queues.first
+          messages = Pgbus.client.read_batch(queue, qty: qty) || []
+          messages.map { |m| [queue, m] }
         else
-          # Multi-queue read: read from each queue proportionally
           per_queue = [(qty / queues.size.to_f).ceil, 1].max
           queues.flat_map do |q|
-            Pgbus.client.read_batch(q, qty: per_queue) || []
+            (Pgbus.client.read_batch(q, qty: per_queue) || []).map { |m| [q, m] }
           end.first(qty)
         end
       rescue StandardError => e
@@ -95,6 +99,31 @@ module Pgbus
         Pgbus.logger.error { "[Pgbus] Unhandled error processing message: #{e.message}" }
       end
 
+      # Resolve "*" to all non-DLQ queues from pgmq.meta, stripping the prefix.
+      # Called once at startup. If no wildcard, this is a no-op.
+      def resolve_wildcard_queues
+        return unless @queues.include?("*")
+
+        dlq_suffix = config.dead_letter_queue_suffix
+        prefix = "#{config.queue_prefix}_"
+
+        all_queues = ActiveRecord::Base.connection.select_values("SELECT queue_name FROM pgmq.meta ORDER BY queue_name")
+        resolved = all_queues
+                   .reject { |q| q.end_with?(dlq_suffix) }
+                   .map { |q| q.delete_prefix(prefix) }
+
+        if resolved.empty?
+          Pgbus.logger.warn { "[Pgbus] Wildcard queue '*' resolved to no queues — falling back to default" }
+          @queues = [config.default_queue]
+        else
+          @queues = resolved
+          Pgbus.logger.info { "[Pgbus] Wildcard queue '*' resolved to: #{@queues.join(", ")}" }
+        end
+      rescue StandardError => e
+        Pgbus.logger.error { "[Pgbus] Failed to resolve wildcard queues: #{e.message} — falling back to default" }
+        @queues = [config.default_queue]
+      end
+
       def recycle_needed?
         exceeded_max_jobs? || exceeded_max_memory? || exceeded_max_lifetime?
       end

data/lib/pgbus/recurring/already_recorded.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Recurring
+    class AlreadyRecorded < Pgbus::Error; end
+  end
+end

data/lib/pgbus/recurring/command_job.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Recurring
+    # Job class for command-based recurring tasks.
+    # Evaluates a method call chain on a constant, e.g. "OldRecord.cleanup!".
+    #
+    # Only supports `ConstantName.method_name` and `ConstantName.method_name(args)`.
+    # Rejects arbitrary Ruby expressions for safety.
+    class CommandJob < ::ActiveJob::Base
+      SAFE_COMMAND = /\A([A-Z][A-Za-z0-9_]*(?:::[A-Z][A-Za-z0-9_]*)*)\.([a-z_][a-z0-9_!?]*)\z/
+
+      def perform(command)
+        match = SAFE_COMMAND.match(command)
+        unless match
+          raise ArgumentError,
+                "Unsafe recurring command: #{command.inspect}. " \
+                "Must be in the form 'ClassName.method_name'."
+        end
+
+        klass = match[1].safe_constantize
+        raise ArgumentError, "Unknown class in recurring command: #{match[1]}" unless klass
+
+        klass.public_send(match[2])
+      end
+    end
+  end
+end

data/lib/pgbus/recurring/config_loader.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "erb"
+
+module Pgbus
+  module Recurring
+    module ConfigLoader
+      module_function
+
+      def load(path, env: nil)
+        return {} unless path && File.exist?(path.to_s)
+
+        env ||= detect_env
+        raw = File.read(path)
+        parsed = YAML.safe_load(ERB.new(raw).result, permitted_classes: [Symbol], aliases: true)
+        return {} unless parsed.is_a?(Hash)
+
+        # If the parsed hash has an environment key, use that subtree
+        parsed.key?(env) ? parsed.fetch(env, {}) : parsed
+      rescue StandardError => e
+        Pgbus.logger.error { "[Pgbus] Failed to load recurring config from #{path}: #{e.message}" }
+        {}
+      end
+
+      def detect_env
+        if defined?(Rails)
+          Rails.env.to_s
+        else
+          ENV.fetch("PGBUS_ENV", "development")
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/recurring/schedule.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Recurring
+    class Schedule
+      attr_reader :tasks
+
+      def initialize(config: Pgbus.configuration)
+        @config = config
+        @tasks = load_tasks
+      end
+
+      def due_tasks(time = Time.now)
+        tasks.select { |task| task_due?(task, time) }
+      end
+
+      def enqueue_task(task, run_at:)
+        queue = resolve_queue(task)
+
+        RecurringExecution.record(task.key, run_at) do
+          payload = build_payload(task)
+          headers = build_headers(task, run_at)
+
+          Pgbus.client.ensure_queue(queue)
+          Pgbus.client.send_message(queue, payload, headers: headers)
+
+          Pgbus.logger.info do
+            "[Pgbus] Enqueued recurring task #{task.key} (#{task.class_name || task.command}) " \
+              "for run_at=#{run_at.iso8601}"
+          end
+        end
+      rescue AlreadyRecorded
+        Pgbus.logger.debug { "[Pgbus] Recurring task #{task.key} already enqueued for #{run_at.iso8601}" }
+      end
+
+      def build_payload(task)
+        if task.command
+          {
+            "job_class" => "Pgbus::Recurring::CommandJob",
+            "arguments" => [task.command],
+            "queue_name" => task.queue_name || @config.default_queue,
+            "priority" => nil
+          }
+        else
+          {
+            "job_class" => task.class_name,
+            "arguments" => task.arguments,
+            "queue_name" => task.queue_name || @config.default_queue,
+            "priority" => task.priority.zero? ? nil : task.priority
+          }
+        end
+      end
+
+      private
+
+      def load_tasks
+        raw = @config.recurring_tasks || {}
+        raw.filter_map do |key, options|
+          options = options.transform_keys(&:to_s).transform_keys(&:to_sym) if options.is_a?(Hash)
+          task = Task.from_configuration(key, **(options || {}))
+          if task.valid?
+            task
+          else
+            Pgbus.logger.warn { "[Pgbus] Skipping invalid recurring task '#{key}': #{task.errors.join(", ")}" }
+            nil
+          end
+        end
+      end
+
+      def task_due?(task, time)
+        # A task is due when its most recent cron occurrence (previous_time)
+        # falls within the current tick window. We also check match? to
+        # handle the exact-boundary case where time == cron time.
+        cron = task.parsed_schedule
+        return false unless cron
+
+        # Check if `time` itself matches the cron (exact boundary hit)
+        return true if cron.match?(time)
+
+        # Check if the previous occurrence was recent enough that we should
+        # still fire it (handles the case where we tick slightly after the
+        # cron time). The window is the scheduler interval.
+        prev = task.previous_time(time)
+        return false unless prev
+
+        (time - prev) <= @config.recurring_schedule_interval
+      end
+
+      def resolve_queue(task)
+        @config.queue_name(task.queue_name || @config.default_queue)
+      end
+
+      def build_headers(task, run_at)
+        {
+          "pgbus.recurring_key" => task.key,
+          "pgbus.recurring_run_at" => run_at.iso8601,
+          "pgbus.recurring_schedule" => task.schedule
+        }
+      end
+    end
+  end
+end

data/lib/pgbus/recurring/scheduler.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Recurring
+    class Scheduler
+      include Process::SignalHandler
+
+      attr_reader :schedule, :config
+
+      def initialize(config: Pgbus.configuration)
+        @config = config
+        @schedule = Schedule.new(config: config)
+        @shutting_down = false
+        @last_runs = {}
+      end
+
+      def run
+        setup_signals
+        start_heartbeat
+
+        Pgbus.logger.info do
+          "[Pgbus] Scheduler started: #{schedule.tasks.size} recurring tasks, " \
+            "interval=#{config.recurring_schedule_interval}s"
+        end
+
+        loop do
+          break if @shutting_down
+
+          process_signals
+          break if @shutting_down
+
+          tick(Time.now)
+          break if @shutting_down
+
+          interruptible_sleep(config.recurring_schedule_interval)
+        end
+
+        shutdown
+      end
+
+      def tick(now)
+        schedule.due_tasks(now).each do |task|
+          run_at = task.previous_time(now)
+          next unless run_at
+
+          schedule.enqueue_task(task, run_at: run_at)
+          @last_runs[task.key] = now
+        rescue StandardError => e
+          Pgbus.logger.error do
+            "[Pgbus] Error scheduling recurring task #{task.key}: #{e.class}: #{e.message}"
+          end
+        end
+      end
+
+      def last_run_at(key)
+        @last_runs[key]
+      end
+
+      def task_statuses
+        schedule.tasks.map do |task|
+          {
+            key: task.key,
+            class_name: task.class_name,
+            command: task.command,
+            schedule: task.schedule,
+            human_schedule: task.human_schedule,
+            queue_name: task.queue_name,
+            arguments: task.arguments,
+            priority: task.priority,
+            description: task.description,
+            next_run_at: task.next_time,
+            last_run_at: @last_runs[task.key]
+          }
+        end
+      end
+
+      def graceful_shutdown
+        @shutting_down = true
+      end
+
+      def immediate_shutdown
+        @shutting_down = true
+      end
+
+      private
+
+      def start_heartbeat
+        @heartbeat = Process::Heartbeat.new(
+          kind: "scheduler",
+          metadata: { pid: ::Process.pid, tasks: schedule.tasks.size }
+        )
+        @heartbeat.start
+      end
+
+      def shutdown
+        @heartbeat&.stop
+        restore_signals
+        Pgbus.logger.info { "[Pgbus] Scheduler stopped" }
+      end
+    end
+  end
+end

data/lib/pgbus/recurring/task.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require "fugit"
+
+module Pgbus
+  module Recurring
+    class Task
+      attr_reader :key, :class_name, :command, :schedule, :queue_name,
+                  :arguments, :priority, :description
+
+      def self.from_configuration(key, **options)
+        options = options.transform_keys(&:to_sym)
+        new(
+          key: key,
+          class_name: options[:class],
+          command: options[:command],
+          schedule: options[:schedule],
+          queue_name: options[:queue],
+          arguments: Array(options[:args]),
+          priority: options.fetch(:priority, 0).to_i,
+          description: options[:description]
+        )
+      end
+
+      def initialize(key:, class_name: nil, command: nil, schedule: nil,
+                     queue_name: nil, arguments: [], priority: 0, description: nil)
+        @key = key
+        @class_name = class_name
+        @command = command
+        @schedule = schedule
+        @queue_name = queue_name
+        @arguments = arguments || []
+        @priority = priority || 0
+        @description = description
+        @errors = []
+      end
+
+      def valid?
+        @errors = []
+        validate!
+        @errors.empty?
+      end
+
+      def errors
+        valid? unless defined?(@validated)
+        @errors
+      end
+
+      def parsed_schedule
+        @parsed_schedule ||= parse_schedule
+      end
+
+      def next_time(from = Time.now)
+        parsed_schedule&.next_time(from)&.to_t
+      end
+
+      def previous_time(from = Time.now)
+        parsed_schedule&.previous_time(from)&.to_t
+      end
+
+      def human_schedule
+        return nil unless parsed_schedule
+
+        parsed_schedule.to_cron_s
+      end
+
+      def job_class
+        class_name&.safe_constantize
+      end
+
+      def to_h
+        {
+          key: key,
+          class_name: class_name,
+          command: command,
+          schedule: schedule,
+          queue_name: queue_name,
+          arguments: arguments,
+          priority: priority,
+          description: description
+        }
+      end
+
+      private
+
+      def validate!
+        @validated = true
+
+        if schedule.nil? || schedule.to_s.strip.empty?
+          @errors << "Schedule is required"
+          return
+        end
+
+        @errors << "Either class or command is required" if class_name.to_s.strip.empty? && command.to_s.strip.empty?
+
+        return if parsed_schedule.is_a?(Fugit::Cron)
+
+        @errors << "Schedule '#{schedule}' is not a valid cron expression"
+      end
+
+      def parse_schedule
+        return nil if schedule.nil? || schedule.to_s.strip.empty?
+
+        parsed = Fugit.parse(schedule.to_s, multi: :fail)
+        parsed.is_a?(Fugit::Cron) ? parsed : nil
+      rescue ArgumentError
+        nil
+      end
+    end
+  end
+end

data/lib/pgbus/serializer.rb

@@ -7,15 +7,25 @@ module Pgbus
     module_function
 
     def serialize_job(active_job)
-      data = active_job.serialize
-      # GlobalID is handled by ActiveJob's serialize — it converts AR objects
-      # to GlobalID URIs automatically. We just JSON-encode the result.
-      JSON.generate(data)
+      Instrumentation.instrument("pgbus.serializer.serialize", kind: :job) do
+        data = active_job.serialize
+        # GlobalID is handled by ActiveJob's serialize — it converts AR objects
+        # to GlobalID URIs automatically. We just JSON-encode the result.
+        JSON.generate(data)
+      end
+    end
+
+    def serialize_job_hash(active_job)
+      Instrumentation.instrument("pgbus.serializer.serialize", kind: :job) do
+        active_job.serialize
+      end
     end
 
     def deserialize_job(json_string)
-      data = JSON.parse(json_string)
-      ActiveJob::Base.deserialize(data)
+      Instrumentation.instrument("pgbus.serializer.deserialize", kind: :job) do
+        data = JSON.parse(json_string)
+        ActiveJob::Base.deserialize(data)
+      end
     end
 
     def serialize_event(event)

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.0.1"
+  VERSION = "0.1.2"
 end