postburner 0.8.0 → 1.0.0.pre.1

data/lib/postburner/active_job/payload.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+module Postburner
+  module ActiveJob
+    # Handles serialization/deserialization of ActiveJob payloads for Beanstalkd.
+    #
+    # Provides consistent payload format for both default and tracked jobs, with
+    # versioning support for future format changes.
+    #
+    # ## Payload Format (V1)
+    #
+    # Both default and tracked jobs store the same ActiveJob data in Beanstalkd.
+    # The only difference is tracked jobs also persist to PostgreSQL and include
+    # a `postburner_job_id` reference.
+    #
+    # @example Default job payload
+    #   {
+    #     v: 1,
+    #     tracked: false,
+    #     postburner_job_id: nil,
+    #     job_class: "SendEmail",
+    #     job_id: "abc-123",
+    #     queue_name: "mailers",
+    #     arguments: [[1, 2, 3]],
+    #     retry_count: 0,
+    #     ...
+    #   }
+    #
+    # @example Tracked job payload
+    #   {
+    #     v: 1,
+    #     tracked: true,
+    #     postburner_job_id: 456,  # References postburner_jobs.id
+    #     job_class: "ProcessPayment",
+    #     job_id: "def-456",
+    #     queue_name: "critical",
+    #     arguments: [[789]],
+    #     retry_count: 0,
+    #     ...
+    #   }
+    #
+    class Payload
+      VERSION = 1
+
+      class << self
+        # Generates payload structure for an ActiveJob.
+        #
+        # @param job [ActiveJob::Base] The ActiveJob instance
+        # @param tracked [Boolean] Whether this is a tracked job
+        # @param postburner_job_id [Integer, nil] Postburner::Job ID for tracked jobs
+        #
+        # @return [Hash] Payload hash
+        #
+        # @api private
+        #
+        def for_job(job, tracked: false, postburner_job_id: nil)
+          {
+            v: VERSION,
+            tracked: tracked,
+            postburner_job_id: postburner_job_id,
+            job_class: job.class.name,
+            job_id: job.job_id,
+            queue_name: job.queue_name,
+            priority: job.priority,
+            arguments: ::ActiveJob::Arguments.serialize(job.arguments),
+            executions: job.executions,
+            exception_executions: job.exception_executions || {},
+            locale: job.locale,
+            timezone: job.timezone,
+            enqueued_at: job.enqueued_at&.iso8601,
+            retry_count: 0  # For default job retry tracking
+          }
+        end
+
+        # Generates JSON payload for a default job.
+        #
+        # @param job [ActiveJob::Base] The ActiveJob instance
+        #
+        # @return [String] JSON-encoded payload
+        #
+        def default_payload(job)
+          JSON.generate(for_job(job, tracked: false))
+        end
+
+        # Generates JSON payload for a tracked job.
+        #
+        # @param job [ActiveJob::Base] The ActiveJob instance
+        # @param postburner_job_id [Integer] Postburner::Job ID
+        #
+        # @return [String] JSON-encoded payload
+        #
+        def tracked_payload(job, postburner_job_id)
+          JSON.generate(for_job(job, tracked: true, postburner_job_id: postburner_job_id))
+        end
+
+        # Serializes ActiveJob data for PostgreSQL storage (tracked jobs).
+        #
+        # Returns the same structure as for_job but as a plain Hash
+        # (not JSON string) for storage in Postburner::Job args column.
+        #
+        # @param job [ActiveJob::Base] The ActiveJob instance
+        #
+        # @return [Hash] Payload hash for PostgreSQL storage
+        #
+        def serialize_for_tracked(job)
+          for_job(job, tracked: true).stringify_keys
+        end
+
+        # Parses and validates a JSON payload from Beanstalkd.
+        #
+        # Handles version checking and returns the parsed data.
+        #
+        # @param json_string [String] JSON payload from Beanstalkd
+        #
+        # @return [Hash] Parsed payload with string keys
+        #
+        # @raise [ArgumentError] if payload version is unknown
+        #
+        def parse(json_string)
+          data = JSON.parse(json_string)
+
+          # Handle version
+          version = data['v'] || data[:v] || 1
+
+          case version
+          when 1
+            data.is_a?(Hash) ? data.stringify_keys : data
+          else
+            raise ArgumentError, "Unknown payload version: #{version}"
+          end
+        end
+
+        # Detects if a parsed payload is for a tracked job.
+        #
+        # @param payload [Hash] Parsed payload
+        #
+        # @return [Boolean] true if tracked, false otherwise
+        #
+        def tracked?(payload)
+          payload['tracked'] == true || payload[:tracked] == true
+        end
+
+        # Detects if a parsed payload is for a legacy Postburner::Job.
+        #
+        # Legacy format: { "class" => "JobClassName", "args" => [job_id] }
+        #
+        # @param payload [Hash] Parsed payload
+        #
+        # @return [Boolean] true if legacy format
+        #
+        def legacy_format?(payload)
+          payload.key?('class') && payload.key?('args') && !payload.key?('v')
+        end
+      end
+    end
+  end
+end

data/lib/postburner/beanstalkd.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module Postburner
+  # Beanstalkd-specific configuration DSL for ActiveJob classes using Postburner.
+  #
+  # Provides consistent API with Postburner::Job for setting Beanstalkd queue
+  # priority and TTR (time-to-run). Include this module in your ActiveJob classes
+  # to use Beanstalkd-specific configuration.
+  #
+  # @note Automatically included when you include Postburner::Tracked
+  #
+  # @example Default job with Beanstalkd config
+  #   class ProcessPayment < ApplicationJob
+  #     include Postburner::Beanstalkd
+  #
+  #     queue_as :critical
+  #     queue_priority 0      # Highest priority
+  #     queue_ttr 300         # 5 minutes to complete
+  #
+  #     def perform(payment_id)
+  #       # ...
+  #     end
+  #   end
+  #
+  # @example With tracking (Beanstalkd automatically included)
+  #   class ProcessPayment < ApplicationJob
+  #     include Postburner::Tracked  # Includes Beanstalkd automatically
+  #
+  #     queue_priority 0
+  #     queue_ttr 600
+  #
+  #     def perform(payment_id)
+  #       log "Processing payment"
+  #       # ...
+  #     end
+  #   end
+  #
+  module Beanstalkd
+      extend ActiveSupport::Concern
+
+      included do
+        class_attribute :postburner_priority, default: nil
+        class_attribute :postburner_ttr, default: nil
+      end
+
+      class_methods do
+        # Sets or returns the queue priority.
+        #
+        # Lower numbers = higher priority in Beanstalkd (0 is highest).
+        # Falls back to Postburner.configuration.default_priority if not set.
+        #
+        # @param pri [Integer, nil] Priority to set (0-4294967295), or nil to get current value
+        #
+        # @return [Integer, nil] Current priority when getting, nil when setting
+        #
+        # @example Set priority
+        #   queue_priority 0  # Highest priority
+        #
+        # @example Get priority
+        #   ProcessPayment.queue_priority  # => 0
+        #
+        def queue_priority(pri = nil)
+          if pri
+            self.postburner_priority = pri
+            nil
+          else
+            postburner_priority
+          end
+        end
+
+        # Sets or returns the queue TTR (time to run).
+        #
+        # Number of seconds Beanstalkd will wait for job completion before
+        # making it available again. Falls back to Postburner.configuration.default_ttr
+        # if not set.
+        #
+        # @param ttr [Integer, nil] Timeout in seconds, or nil to get current value
+        #
+        # @return [Integer, nil] Current TTR when getting, nil when setting
+        #
+        # @example Set TTR
+        #   queue_ttr 300  # 5 minutes
+        #
+        # @example Get TTR
+        #   ProcessPayment.queue_ttr  # => 300
+        #
+        def queue_ttr(ttr = nil)
+          if ttr
+            self.postburner_ttr = ttr
+            nil
+          else
+            postburner_ttr
+          end
+        end
+      end
+  end
+end

data/lib/postburner/configuration.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+module Postburner
+  # Configuration system for Postburner workers and connections.
+  #
+  # Supports both programmatic configuration and YAML file loading.
+  # Configuration can be set per-environment and controls worker behavior,
+  # Beanstalkd connection, and execution strategies.
+  #
+  # @example Programmatic configuration
+  #   Postburner.configure do |config|
+  #     config.beanstalk_url = 'beanstalk://localhost:11300'
+  #     config.worker_type = :threads_on_fork
+  #     config.logger = Rails.logger
+  #   end
+  #
+  # @example Loading from YAML
+  #   config = Postburner::Configuration.load_yaml('config/postburner.yml', 'production')
+  #
+  class Configuration
+    attr_accessor :beanstalk_url, :logger, :queues, :default_queue, :default_priority, :default_ttr, :default_threads, :default_forks, :default_gc_limit
+
+    # @param options [Hash] Configuration options
+    # @option options [String] :beanstalk_url Beanstalkd URL (default: ENV['BEANSTALK_URL'] or localhost)
+    # @option options [Logger] :logger Logger instance (default: Rails.logger)
+    # @option options [Hash] :queues Queue configurations
+    # @option options [String] :default_queue Default queue name (default: 'default')
+    # @option options [Integer] :default_priority Default job priority (default: 65536, lower = higher priority)
+    # @option options [Integer] :default_ttr Default time-to-run in seconds (default: 300)
+    # @option options [Integer] :default_threads Default thread count per queue (default: 1)
+    # @option options [Integer] :default_forks Default fork count per queue (default: 0, single process)
+    # @option options [Integer] :default_gc_limit Default GC limit for worker restarts (default: nil, no limit)
+    #
+    def initialize(options = {})
+      @beanstalk_url = options[:beanstalk_url] || ENV['BEANSTALK_URL'] || 'beanstalk://localhost:11300'
+      @logger = options[:logger] || (defined?(Rails) ? Rails.logger : Logger.new(STDOUT))
+      @queues = options[:queues] || { 'default' => {} }
+      @default_queue = options[:default_queue] || 'default'
+      @default_priority = options[:default_priority] || 65536
+      @default_ttr = options[:default_ttr] || 300
+      @default_threads = options[:default_threads] || 1
+      @default_forks = options[:default_forks] || 0
+      @default_gc_limit = options[:default_gc_limit]
+    end
+
+    # Loads configuration from a YAML file.
+    #
+    # @param path [String] Path to YAML configuration file
+    # @param env [String] Environment name (e.g., 'production', 'development')
+    # @param worker_name [String, nil] Worker name to load (nil = auto-select if single worker)
+    #
+    # @return [Configuration] Configured instance
+    #
+    # @raise [Errno::ENOENT] if file doesn't exist
+    # @raise [ArgumentError] if environment not found in YAML
+    # @raise [ArgumentError] if multiple workers defined but worker_name not specified
+    # @raise [ArgumentError] if worker_name specified but not found
+    #
+    # @example Single worker (auto-selected)
+    #   config = Postburner::Configuration.load_yaml('config/postburner.yml', 'production')
+    #
+    # @example Multiple workers (must specify)
+    #   config = Postburner::Configuration.load_yaml('config/postburner.yml', 'production', 'imports')
+    #
+    def self.load_yaml(path, env = 'development', worker_name = nil)
+      yaml = YAML.load_file(path)
+      env_config = yaml[env.to_s] || yaml[env.to_sym]
+
+      raise ArgumentError, "Environment '#{env}' not found in #{path}" unless env_config
+
+      workers = env_config['workers']
+      raise ArgumentError, "No 'workers:' section found in #{path} for environment '#{env}'" unless workers
+
+      # Auto-select single worker or validate worker_name
+      if worker_name.nil?
+        if workers.size == 1
+          worker_name = workers.keys.first
+        else
+          raise ArgumentError, "Configuration has multiple workers, but --worker not specified\nAvailable workers: #{workers.keys.join(', ')}\nUsage: bin/postburner --worker <name>"
+        end
+      else
+        unless workers.key?(worker_name)
+          raise ArgumentError, "Worker '#{worker_name}' not found in #{path}\nAvailable workers: #{workers.keys.join(', ')}"
+        end
+      end
+
+      worker_config = workers[worker_name]
+
+      # Convert queue array to hash format expected by rest of system
+      queue_list = worker_config['queues'] || []
+      queues_hash = {}
+      queue_list.each do |queue_name|
+        queues_hash[queue_name] = {}
+      end
+
+      options = {
+        beanstalk_url: env_config['beanstalk_url'],
+        queues: queues_hash,
+        default_queue: worker_config['default_queue'] || env_config['default_queue'],
+        default_priority: worker_config['default_priority'] || env_config['default_priority'],
+        default_ttr: worker_config['default_ttr'] || env_config['default_ttr'],
+        default_threads: worker_config['default_threads'] || env_config['default_threads'],
+        default_forks: worker_config['default_forks'] || env_config['default_forks'],
+        default_gc_limit: worker_config['default_gc_limit'] || env_config['default_gc_limit']
+      }
+
+      new(options)
+    end
+
+    # Returns queue configuration for a specific queue name.
+    #
+    # @param queue_name [String, Symbol] Name of the queue
+    #
+    # @return [Hash] Queue configuration with threads, gc_limit, etc.
+    #
+    # @example
+    #   config.queue_config('critical')  # => { threads: 1, gc_limit: 100 }
+    #
+    def queue_config(queue_name)
+      @queues[queue_name.to_s] || @queues[queue_name.to_sym] || {}
+    end
+
+    # Returns array of all configured queue names.
+    #
+    # @return [Array<String>] Queue names
+    #
+    # @example
+    #   config.queue_names  # => ['default', 'critical', 'mailers']
+    #
+    def queue_names
+      @queues.keys.map(&:to_s)
+    end
+
+    # Expands queue name to full tube name with environment prefix.
+    #
+    # Converts a simple queue name (e.g., 'default', 'critical') to the full
+    # Beanstalkd tube name with environment namespace (e.g.,
+    # 'postburner.production.critical').
+    #
+    # @param queue_name [String, Symbol, nil] Base queue name (defaults to configured default_queue)
+    # @param env [String, Symbol, nil] Environment name (defaults to Rails.env or 'development')
+    #
+    # @return [String] Full tube name with environment prefix
+    #
+    # @example
+    #   config.expand_tube_name('critical', 'production')
+    #   # => "postburner.production.critical"
+    #
+    # @example With configured default
+    #   config.default_queue = 'background'
+    #   config.expand_tube_name
+    #   # => "postburner.development.background"
+    #
+    def expand_tube_name(queue_name = nil, env = nil)
+      env ||= defined?(Rails) ? Rails.env : nil
+      queue_name ||= @default_queue
+      [
+        'postburner',
+        env,
+        queue_name,
+      ].compact.join('.')
+    end
+
+
+    # Returns array of expanded tube names with environment prefix.
+    #
+    # @param env [String, Symbol, nil] Environment name (defaults to Rails.env or 'development')
+    #
+    # @return [Array<String>] Array of expanded tube names
+    #
+    # @example
+    #   config.expanded_tube_names('production')  # => ['postburner.production.default', 'postburner.production.critical']
+    def expanded_tube_names(env = nil)
+      queue_names.map { |q| expand_tube_name(q, env) }
+    end
+  end
+
+  # Returns the global configuration instance.
+  #
+  # @return [Configuration] Global configuration
+  #
+  def self.configuration
+    @configuration ||= Configuration.new
+  end
+
+  # Configures Postburner via block.
+  #
+  # @yield [config] Configuration instance
+  # @yieldparam config [Configuration] The configuration to modify
+  #
+  # @return [void]
+  #
+  # @example
+  #   Postburner.configure do |config|
+  #     config.beanstalk_url = 'beanstalk://localhost:11300'
+  #     config.worker_type = :threads_on_fork
+  #   end
+  #
+  def self.configure
+    yield(configuration)
+  end
+end

data/lib/postburner/connection.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module Postburner
+  # Wrapper around Beaneater connection with automatic reconnection.
+  #
+  # Provides a simplified interface to Beanstalkd via Beaneater, with
+  # automatic connection management and reconnection on failures.
+  #
+  # @example Direct usage
+  #   conn = Postburner::Connection.new
+  #   conn.tubes['default'].put('job data')
+  #
+  # @example With reconnection
+  #   conn = Postburner::Connection.new
+  #   conn.reconnect!  # Force fresh connection
+  #
+  class Connection
+    attr_reader :url
+
+    # @param url [String] Beanstalkd URL (e.g., 'beanstalk://localhost:11300')
+    #
+    def initialize(url = nil)
+      @url = url || Postburner.configuration.beanstalk_url
+      @pool = nil
+      connect!
+    end
+
+    # Returns the tubes interface for Beanstalkd operations.
+    #
+    # Automatically ensures connection is active before returning.
+    #
+    # @return [Beaneater::Tubes] Tubes interface
+    #
+    # @raise [Beaneater::NotConnected] if connection fails
+    #
+    # @example
+    #   conn.tubes['critical'].put('{"job": "data"}')
+    #   conn.tubes.watch!('default', 'critical')
+    #
+    def tubes
+      ensure_connected!
+      @pool.tubes
+    end
+
+    # Returns the underlying Beaneater pool.
+    #
+    # @return [Beaneater::Pool] Beaneater connection pool
+    #
+    # @example
+    #   conn.beanstalk.stats
+    #
+    def beanstalk
+      ensure_connected!
+      @pool
+    end
+
+    # Checks if currently connected to Beanstalkd.
+    #
+    # @return [Boolean] true if connected, false otherwise
+    #
+    def connected?
+      @pool && @pool.respond_to?(:connection) && @pool.connection
+    rescue
+      false
+    end
+
+    # Forces reconnection to Beanstalkd.
+    #
+    # Closes existing connection (if any) and establishes a fresh one.
+    # Use this when connection has gone stale or after network issues.
+    #
+    # @return [void]
+    #
+    # @example
+    #   conn.reconnect!
+    #
+    def reconnect!
+      close
+      connect!
+    end
+
+    # Closes the Beanstalkd connection.
+    #
+    # @return [void]
+    #
+    def close
+      @pool&.close rescue nil
+      @pool = nil
+    end
+
+    private
+
+    # Establishes connection to Beanstalkd.
+    #
+    # @return [void]
+    #
+    # @raise [Beaneater::NotConnected] if connection fails
+    #
+    def connect!
+      # Beaneater expects 'host:port' format, not 'beanstalk://host:port'
+      clean_url = @url.sub(%r{^beanstalk://}, '')
+      @pool = Beaneater.new(clean_url)
+    end
+
+    # Ensures connection is active, reconnecting if necessary.
+    #
+    # @return [void]
+    #
+    def ensure_connected!
+      reconnect! unless connected?
+    end
+  end
+end

data/lib/postburner/engine.rb

@@ -1,4 +1,4 @@
-require 'backburner'
+require 'beaneater'
 require 'haml-rails'
 
 module Postburner