postburner 0.9.0.rc.1 → 1.0.0.pre.2

data/lib/postburner/configuration.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+module Postburner
+  # Configuration system for Postburner workers and connections.
+  #
+  # Supports both programmatic configuration and YAML file loading.
+  # Configuration can be set per-environment and controls worker behavior,
+  # Beanstalkd connection, and execution strategies.
+  #
+  # @example Programmatic configuration
+  #   Postburner.configure do |config|
+  #     config.beanstalk_url = 'beanstalk://localhost:11300'
+  #     config.worker_type = :threads_on_fork
+  #     config.logger = Rails.logger
+  #   end
+  #
+  # @example Loading from YAML
+  #   config = Postburner::Configuration.load_yaml('config/postburner.yml', 'production')
+  #
+  class Configuration
+    attr_accessor :beanstalk_url, :logger, :queues, :default_queue, :default_priority, :default_ttr, :default_threads, :default_forks, :default_gc_limit
+
+    # @param options [Hash] Configuration options
+    # @option options [String] :beanstalk_url Beanstalkd URL (default: ENV['BEANSTALK_URL'] or localhost)
+    # @option options [Logger] :logger Logger instance (default: Rails.logger)
+    # @option options [Hash] :queues Queue configurations
+    # @option options [String] :default_queue Default queue name (default: 'default')
+    # @option options [Integer] :default_priority Default job priority (default: 65536, lower = higher priority)
+    # @option options [Integer] :default_ttr Default time-to-run in seconds (default: 300)
+    # @option options [Integer] :default_threads Default thread count per queue (default: 1)
+    # @option options [Integer] :default_forks Default fork count per queue (default: 0, single process)
+    # @option options [Integer] :default_gc_limit Default GC limit for worker restarts (default: nil, no limit)
+    #
+    def initialize(options = {})
+      @beanstalk_url = options[:beanstalk_url] || ENV['BEANSTALK_URL'] || 'beanstalk://localhost:11300'
+      @logger = options[:logger] || (defined?(Rails) ? Rails.logger : Logger.new(STDOUT))
+      @queues = options[:queues] || { 'default' => {} }
+      @default_queue = options[:default_queue] || 'default'
+      @default_priority = options[:default_priority] || 65536
+      @default_ttr = options[:default_ttr] || 300
+      @default_threads = options[:default_threads] || 1
+      @default_forks = options[:default_forks] || 0
+      @default_gc_limit = options[:default_gc_limit]
+    end
+
+    # Loads configuration from a YAML file.
+    #
+    # @param path [String] Path to YAML configuration file
+    # @param env [String] Environment name (e.g., 'production', 'development')
+    # @param worker_name [String, nil] Worker name to load (nil = auto-select if single worker)
+    #
+    # @return [Configuration] Configured instance
+    #
+    # @raise [Errno::ENOENT] if file doesn't exist
+    # @raise [ArgumentError] if environment not found in YAML
+    # @raise [ArgumentError] if multiple workers defined but worker_name not specified
+    # @raise [ArgumentError] if worker_name specified but not found
+    #
+    # @example Single worker (auto-selected)
+    #   config = Postburner::Configuration.load_yaml('config/postburner.yml', 'production')
+    #
+    # @example Multiple workers (must specify)
+    #   config = Postburner::Configuration.load_yaml('config/postburner.yml', 'production', 'imports')
+    #
+    def self.load_yaml(path, env = 'development', worker_name = nil)
+      yaml = YAML.load_file(path, aliases: true)
+      env_config = yaml[env.to_s] || yaml[env.to_sym]
+
+      raise ArgumentError, "Environment '#{env}' not found in #{path}" unless env_config
+
+      workers = env_config['workers']
+      raise ArgumentError, "No 'workers:' section found in #{path} for environment '#{env}'" unless workers
+
+      # Auto-select single worker or validate worker_name
+      if worker_name.nil?
+        if workers.size == 1
+          worker_name = workers.keys.first
+        else
+          raise ArgumentError, "Configuration has multiple workers, but --worker not specified\nAvailable workers: #{workers.keys.join(', ')}\nUsage: bin/postburner --worker <name>"
+        end
+      else
+        unless workers.key?(worker_name)
+          raise ArgumentError, "Worker '#{worker_name}' not found in #{path}\nAvailable workers: #{workers.keys.join(', ')}"
+        end
+      end
+
+      worker_config = workers[worker_name]
+
+      # Convert queue array to hash format expected by rest of system
+      queue_list = worker_config['queues'] || []
+      queues_hash = {}
+      queue_list.each do |queue_name|
+        queues_hash[queue_name] = {}
+      end
+
+      options = {
+        beanstalk_url: env_config['beanstalk_url'],
+        queues: queues_hash,
+        default_queue: worker_config['default_queue'] || env_config['default_queue'],
+        default_priority: worker_config['default_priority'] || env_config['default_priority'],
+        default_ttr: worker_config['default_ttr'] || env_config['default_ttr'],
+        default_threads: worker_config['default_threads'] || env_config['default_threads'],
+        default_forks: worker_config['default_forks'] || env_config['default_forks'],
+        default_gc_limit: worker_config['default_gc_limit'] || env_config['default_gc_limit']
+      }
+
+      new(options)
+    end
+
+    # Returns queue configuration for a specific queue name.
+    #
+    # @param queue_name [String, Symbol] Name of the queue
+    #
+    # @return [Hash] Queue configuration with threads, gc_limit, etc.
+    #
+    # @example
+    #   config.queue_config('critical')  # => { threads: 1, gc_limit: 100 }
+    #
+    def queue_config(queue_name)
+      @queues[queue_name.to_s] || @queues[queue_name.to_sym] || {}
+    end
+
+    # Returns array of all configured queue names.
+    #
+    # @return [Array<String>] Queue names
+    #
+    # @example
+    #   config.queue_names  # => ['default', 'critical', 'mailers']
+    #
+    def queue_names
+      @queues.keys.map(&:to_s)
+    end
+
+    # Expands queue name to full tube name with environment prefix.
+    #
+    # Converts a simple queue name (e.g., 'default', 'critical') to the full
+    # Beanstalkd tube name with environment namespace (e.g.,
+    # 'postburner.production.critical').
+    #
+    # @param queue_name [String, Symbol, nil] Base queue name (defaults to configured default_queue)
+    # @param env [String, Symbol, nil] Environment name (defaults to Rails.env or 'development')
+    #
+    # @return [String] Full tube name with environment prefix
+    #
+    # @example
+    #   config.expand_tube_name('critical', 'production')
+    #   # => "postburner.production.critical"
+    #
+    # @example With configured default
+    #   config.default_queue = 'background'
+    #   config.expand_tube_name
+    #   # => "postburner.development.background"
+    #
+    def expand_tube_name(queue_name = nil, env = nil)
+      env ||= defined?(Rails) ? Rails.env : nil
+      queue_name ||= @default_queue
+      [
+        'postburner',
+        env,
+        queue_name,
+      ].compact.join('.')
+    end
+
+
+    # Returns array of expanded tube names with environment prefix.
+    #
+    # @param env [String, Symbol, nil] Environment name (defaults to Rails.env or 'development')
+    #
+    # @return [Array<String>] Array of expanded tube names
+    #
+    # @example
+    #   config.expanded_tube_names('production')  # => ['postburner.production.default', 'postburner.production.critical']
+    def expanded_tube_names(env = nil)
+      queue_names.map { |q| expand_tube_name(q, env) }
+    end
+  end
+
+  # Returns the global configuration instance.
+  #
+  # @return [Configuration] Global configuration
+  #
+  def self.configuration
+    @configuration ||= Configuration.new
+  end
+
+  # Configures Postburner via block.
+  #
+  # @yield [config] Configuration instance
+  # @yieldparam config [Configuration] The configuration to modify
+  #
+  # @return [void]
+  #
+  # @example
+  #   Postburner.configure do |config|
+  #     config.beanstalk_url = 'beanstalk://localhost:11300'
+  #     config.worker_type = :threads_on_fork
+  #   end
+  #
+  def self.configure
+    yield(configuration)
+  end
+end

data/lib/postburner/connection.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module Postburner
+  # Wrapper around Beaneater connection with automatic reconnection.
+  #
+  # Provides a simplified interface to Beanstalkd via Beaneater, with
+  # automatic connection management and reconnection on failures.
+  #
+  # @example Direct usage
+  #   conn = Postburner::Connection.new
+  #   conn.tubes['default'].put('job data')
+  #
+  # @example With reconnection
+  #   conn = Postburner::Connection.new
+  #   conn.reconnect!  # Force fresh connection
+  #
+  class Connection
+    attr_reader :url
+
+    # @param url [String] Beanstalkd URL (e.g., 'beanstalk://localhost:11300')
+    #
+    def initialize(url = nil)
+      @url = url || Postburner.configuration.beanstalk_url
+      @pool = nil
+      connect!
+    end
+
+    # Returns the tubes interface for Beanstalkd operations.
+    #
+    # Automatically ensures connection is active before returning.
+    #
+    # @return [Beaneater::Tubes] Tubes interface
+    #
+    # @raise [Beaneater::NotConnected] if connection fails
+    #
+    # @example
+    #   conn.tubes['critical'].put('{"job": "data"}')
+    #   conn.tubes.watch!('default', 'critical')
+    #
+    def tubes
+      ensure_connected!
+      @pool.tubes
+    end
+
+    # Returns the underlying Beaneater pool.
+    #
+    # @return [Beaneater::Pool] Beaneater connection pool
+    #
+    # @example
+    #   conn.beanstalk.stats
+    #
+    def beanstalk
+      ensure_connected!
+      @pool
+    end
+
+    # Checks if currently connected to Beanstalkd.
+    #
+    # @return [Boolean] true if connected, false otherwise
+    #
+    def connected?
+      @pool && @pool.respond_to?(:connection) && @pool.connection
+    rescue
+      false
+    end
+
+    # Forces reconnection to Beanstalkd.
+    #
+    # Closes existing connection (if any) and establishes a fresh one.
+    # Use this when connection has gone stale or after network issues.
+    #
+    # @return [void]
+    #
+    # @example
+    #   conn.reconnect!
+    #
+    def reconnect!
+      close
+      connect!
+    end
+
+    # Closes the Beanstalkd connection.
+    #
+    # @return [void]
+    #
+    def close
+      @pool&.close rescue nil
+      @pool = nil
+    end
+
+    private
+
+    # Establishes connection to Beanstalkd.
+    #
+    # @return [void]
+    #
+    # @raise [Beaneater::NotConnected] if connection fails
+    #
+    def connect!
+      # Beaneater expects 'host:port' format, not 'beanstalk://host:port'
+      clean_url = @url.sub(%r{^beanstalk://}, '')
+      @pool = Beaneater.new(clean_url)
+    end
+
+    # Ensures connection is active, reconnecting if necessary.
+    #
+    # @return [void]
+    #
+    def ensure_connected!
+      reconnect! unless connected?
+    end
+  end
+end

data/lib/postburner/engine.rb

@@ -1,4 +1,4 @@
-require 'backburner'
+require 'beaneater'
 require 'haml-rails'
 
 module Postburner

data/lib/postburner/queue_config.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+module Postburner
+  # Queue configuration module for Postburner::Job classes.
+  #
+  # Provides DSL methods for configuring queue behavior (name, priority, TTR, retries).
+  # Replaces Backburner::Queue with cleaner implementation that doesn't interfere
+  # with ActiveSupport::Callbacks.
+  #
+  # @example Basic usage
+  #   class ProcessPayment < Postburner::Job
+  #     queue 'critical'
+  #     queue_priority 0
+  #     queue_ttr 300
+  #     queue_max_job_retries 3
+  #
+  #     def perform(args)
+  #       # ...
+  #     end
+  #   end
+  #
+  module QueueConfig
+    extend ActiveSupport::Concern
+
+    included do
+      class_attribute :postburner_queue_name, default: 'default'
+      class_attribute :postburner_priority, default: nil
+      class_attribute :postburner_ttr, default: nil
+      class_attribute :postburner_max_retries, default: nil
+      class_attribute :postburner_retry_delay, default: nil
+    end
+
+    class_methods do
+      # Sets or returns the queue name.
+      #
+      # @param name [String, Symbol, nil] Queue name to set, or nil to get current value
+      #
+      # @return [String, nil] Current queue name when getting, nil when setting
+      #
+      # @example Set queue
+      #   queue 'critical'
+      #
+      # @example Get queue
+      #   ProcessPayment.queue  # => 'critical'
+      #
+      def queue(name = nil)
+        if name
+          self.postburner_queue_name = name.to_s
+          nil  # Return nil to avoid callback interference
+        else
+          postburner_queue_name
+        end
+      end
+
+      # Sets or returns the queue priority.
+      #
+      # Lower numbers = higher priority in Beanstalkd.
+      #
+      # @param pri [Integer, nil] Priority to set (0-4294967295), or nil to get current value
+      #
+      # @return [Integer, nil] Current priority when getting, nil when setting
+      #
+      # @example Set priority
+      #   queue_priority 0  # Highest priority
+      #
+      # @example Get priority
+      #   ProcessPayment.queue_priority  # => 0
+      #
+      def queue_priority(pri = nil)
+        if pri
+          self.postburner_priority = pri
+          nil
+        else
+          postburner_priority
+        end
+      end
+
+      # Sets or returns the queue TTR (time-to-run).
+      #
+      # Number of seconds Beanstalkd will wait for job completion before
+      # making it available again.
+      #
+      # @param ttr [Integer, nil] Timeout in seconds, or nil to get current value
+      #
+      # @return [Integer, nil] Current TTR when getting, nil when setting
+      #
+      # @example Set TTR
+      #   queue_ttr 300  # 5 minutes
+      #
+      # @example Get TTR
+      #   ProcessPayment.queue_ttr  # => 300
+      #
+      def queue_ttr(ttr = nil)
+        if ttr
+          self.postburner_ttr = ttr
+          nil
+        else
+          postburner_ttr
+        end
+      end
+
+      # Sets or returns maximum number of job retries.
+      #
+      # @param retries [Integer, nil] Max retries, or nil to get current value
+      #
+      # @return [Integer, nil] Current max retries when getting, nil when setting
+      #
+      # @example Set max retries
+      #   queue_max_job_retries 3
+      #
+      # @example Get max retries
+      #   ProcessPayment.queue_max_job_retries  # => 3
+      #
+      def queue_max_job_retries(retries = nil)
+        if retries
+          self.postburner_max_retries = retries
+          nil
+        else
+          postburner_max_retries
+        end
+      end
+
+      # Sets or returns the retry delay.
+      #
+      # Can accept either a fixed delay (Integer) or a proc for dynamic calculation.
+      # The proc receives the retry count and returns delay in seconds.
+      #
+      # @param delay [Integer, Proc, nil] Delay in seconds, proc, or nil to get current value
+      #
+      # @return [Integer, Proc, nil] Current delay when getting, nil when setting
+      #
+      # @example Set fixed retry delay
+      #   queue_retry_delay 10
+      #
+      # @example Set exponential backoff with proc
+      #   queue_retry_delay ->(retries) { 2 ** retries }
+      #
+      # @example Get retry delay
+      #   ProcessPayment.queue_retry_delay  # => 10 or #<Proc>
+      #
+      def queue_retry_delay(delay = nil)
+        if delay
+          self.postburner_retry_delay = delay
+          nil
+        else
+          postburner_retry_delay
+        end
+      end
+    end
+  end
+end

data/lib/postburner/strategies/queue.rb

@@ -66,19 +66,33 @@ module Postburner
       # @option options [Integer] :pri Priority (lower = higher priority)
       # @option options [Integer] :ttr Time-to-run before job times out
       #
-      # @return [Hash] Backburner response with :id and :status
+      # @return [Hash] Response with :id (beanstalkd job id) and :status
       #
       # @raise [Beaneater::NotConnected] if Beanstalkd connection fails
       #
       # @api private
       #
       def insert(job, options = {})
+        #debugger
         Postburner::Job.transaction do
-          Backburner::Worker.enqueue(
-            Postburner::Job,
-            job.id,
-            options
-          )
+          Postburner.connected do |conn|
+            tube_name = job.tube_name
+            data = { class: job.class.name, args: [job.id] }
+
+            # Get priority, TTR from job instance (respects instance overrides) or options
+            pri = options[:pri] || job.queue_priority || Postburner.configuration.default_priority
+            delay = options[:delay] || 0
+            ttr = options[:ttr] || job.queue_ttr || Postburner.configuration.default_ttr
+
+            response = conn.tubes[tube_name].put(
+              JSON.generate(data),
+              pri: pri,
+              delay: delay,
+              ttr: ttr
+            )
+
+            response
+          end
         end
       end
 

data/lib/postburner/tracked.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+module Postburner
+  # Concern for ActiveJob classes to opt-in to PostgreSQL tracking.
+  #
+  # Simply include this module in your ActiveJob class to enable full audit
+  # trail persistence in PostgreSQL. Without this, jobs execute as "default" jobs
+  # (Beanstalkd only, no PostgreSQL overhead).
+  #
+  # @example Opt-in to tracking
+  #   class ProcessPayment < ApplicationJob
+  #     include Postburner::Tracked  # ← Enables PostgreSQL audit trail
+  #
+  #     def perform(payment_id)
+  #       log "Processing payment #{payment_id}"
+  #       # ... work ...
+  #       log! "Payment processed successfully"
+  #     end
+  #   end
+  #
+  # @example Without tracking (default mode)
+  #   class SendEmail < ApplicationJob
+  #     # No Postburner::Tracked - executes as default job
+  #
+  #     def perform(email)
+  #       # Fast execution, no PostgreSQL overhead
+  #     end
+  #   end
+  #
+  module Tracked
+    extend ActiveSupport::Concern
+
+    included do
+      # Include Beanstalkd configuration DSL automatically
+      include Postburner::Beanstalkd
+
+      # Reference to Postburner::Job during execution (set by worker)
+      attr_accessor :postburner_job
+    end
+
+    # Appends a log message to the job's audit trail.
+    #
+    # Only available for tracked jobs. Logs are stored in the
+    # Postburner::Job record's `logs` JSONB array.
+    #
+    # @param message [String] Log message
+    # @param level [Symbol] Log level (:debug, :info, :warning, :error)
+    #
+    # @return [void]
+    #
+    # @example
+    #   def perform(user_id)
+    #     log "Starting user processing"
+    #     # ... work ...
+    #     log "Completed successfully", level: :info
+    #   end
+    #
+    def log(message, level: :info)
+      postburner_job&.log(message, level: level)
+    end
+
+    # Appends a log message and immediately persists to database.
+    #
+    # Use this for important log messages that should be saved immediately
+    # rather than batched with other updates.
+    #
+    # @param message [String] Log message
+    # @param level [Symbol] Log level (:debug, :info, :warning, :error)
+    #
+    # @return [void]
+    #
+    # @example
+    #   def perform(payment_id)
+    #     log! "Critical: Processing high-value payment #{payment_id}"
+    #   end
+    #
+    def log!(message, level: :info)
+      postburner_job&.log!(message, level: level)
+    end
+
+    # Tracks an exception in the job's errata array.
+    #
+    # Appends exception details (class, message, backtrace) to the
+    # in-memory errata array. Does NOT persist immediately.
+    #
+    # @param exception [Exception] The exception to track
+    #
+    # @return [void]
+    #
+    # @example
+    #   def perform(user_id)
+    #     process_user(user_id)
+    #   rescue => e
+    #     log_exception(e)
+    #     raise
+    #   end
+    #
+    def log_exception(exception)
+      postburner_job&.log_exception(exception)
+    end
+
+    # Tracks an exception and immediately persists to database.
+    #
+    # @param exception [Exception] The exception to track
+    #
+    # @return [void]
+    #
+    # @example
+    #   def perform(user_id)
+    #     process_user(user_id)
+    #   rescue CriticalError => e
+    #     log_exception!(e)
+    #     raise
+    #   end
+    #
+    def log_exception!(exception)
+      postburner_job&.log_exception!(exception)
+    end
+
+    # Returns the Beanstalkd job object for direct queue operations.
+    #
+    # Provides access to the underlying Beaneater job object through the
+    # TrackedJob wrapper. Use this to perform Beanstalkd operations like
+    # touch, bury, release, etc.
+    #
+    # @return [Beaneater::Job, nil] Beanstalkd job object or nil if not available
+    #
+    # @example Extend TTR during long operation
+    #   def perform(file_id)
+    #     file = File.find(file_id)
+    #     file.each_line do |line|
+    #       # ... process line ...
+    #       bk&.touch  # Extend TTR
+    #     end
+    #   end
+    #
+    # @example Other Beanstalkd operations
+    #   bk&.bury              # Bury the job
+    #   bk&.release(pri: 0)   # Release with priority
+    #   bk&.stats             # Get job statistics
+    #
+    # @see #extend!
+    #
+    def bk
+      postburner_job&.bk
+    end
+
+    # Extends the job's time-to-run (TTR) in Beanstalkd.
+    #
+    # Convenience method that calls touch on the Beanstalkd job, extending
+    # the TTR by the original TTR value. Use this during long-running operations
+    # to prevent the job from timing out.
+    #
+    # @return [void]
+    #
+    # @example Process large file line by line
+    #   def perform(file_id)
+    #     file = File.find(file_id)
+    #     file.each_line do |line|
+    #       # ... process line ...
+    #       extend!  # Extend TTR to prevent timeout
+    #     end
+    #   end
+    #
+    # @see #bk
+    #
+    def extend!
+      postburner_job&.extend!
+    end
+  end
+end

data/lib/postburner/version.rb

@@ -1,3 +1,3 @@
 module Postburner
-  VERSION = '0.9.0.rc.1'
+  VERSION = '1.0.0.pre.2'
 end