postburner 0.9.0.rc.1 → 1.0.0.pre.1

data/app/models/postburner/job.rb

@@ -70,45 +70,13 @@ module Postburner
   # @attr_reader [Array<Hash>] errata Array of error details with timestamps
   # @attr_reader [Array<Hash>] logs Array of log entries with timestamps
   #
-  # Module to override Backburner::Queue methods to return nil when setting values
-  # This prevents return values from interfering with callback definitions
-  module BackburnerQueueOverrides
-    def queue(name=nil)
-      result = super
-      name.nil? ? result : nil
-    end
-
-    def queue_priority(pri=nil)
-      result = super
-      pri.nil? ? result : nil
-    end
-
-    def queue_respond_timeout(ttr=nil)
-      result = super
-      ttr.nil? ? result : nil
-    end
-
-    def queue_max_job_retries(retries=nil)
-      result = super
-      retries.nil? ? result : nil
-    end
-
-    def queue_retry_delay(delay=nil)
-      result = super
-      delay.nil? ? result : nil
-    end
-
-    def queue_retry_delay_proc(proc=nil)
-      result = super
-      proc.nil? ? result : nil
-    end
-  end
-
   class Job < ApplicationRecord
-    include Backburner::Queue
-    singleton_class.prepend BackburnerQueueOverrides
+    include QueueConfig
     include Callbacks
 
+    # Instance-level queue configuration (overrides class-level defaults)
+    attr_writer :queue_priority, :queue_ttr
+
     LOG_LEVELS = [
       :debug,
       :info,
@@ -524,7 +492,7 @@ module Postburner
     # @return [Beaneater::Job, nil] Beanstalkd job object or nil if no bkid
     #
     # @example Direct Beanstalkd operations
-    #   bk_job = job.beanstalk_job
+    #   bk_job = job.bk
     #   bk_job.stats           # Get job statistics
     #   bk_job.bury            # Bury the job
     #   bk_job.release(pri: 0) # Release with priority
@@ -533,7 +501,7 @@ module Postburner
     # @see #delete!
     # @see #kick!
     #
-    def beanstalk_job
+    def bk
       return unless self.bkid.present?
       return @_beanstalk_job if @_beanstalk_job
 
@@ -542,6 +510,8 @@ module Postburner
       @_beanstalk_job
     end
 
+    alias_method :beanstalk_job, :bk
+
     # Returns the Beanstalkd job object with cache invalidation.
     #
     # Same as {#beanstalk_job} but clears the cached instance variable first,
@@ -554,11 +524,163 @@ module Postburner
     #
     # @see #beanstalk_job
     #
-    def beanstalk_job!
+    def bk!
       @_beanstalk_job = nil
       self.beanstalk_job
     end
 
+    alias_method :beanstalk_job!, :bk!
+
+    # Returns job statistics including Beanstalkd job state.
+    #
+    # Fetches current job state from Beanstalkd and returns combined statistics
+    # about the job's PostgreSQL record and its current Beanstalkd status.
+    #
+    # @return [Hash] Statistics hash with the following keys:
+    #   - id: PostgreSQL job ID
+    #   - bkid: Beanstalkd job ID
+    #   - queue: Queue name configured for this job class (e.g., 'sleep-jobs')
+    #   - tube: Derived tube name with environment prefix (e.g., 'postburner.development.sleep-jobs')
+    #   - watched: Boolean indicating if tube is in configured watch list
+    #   - beanstalk: Hash of Beanstalkd job statistics:
+    #     - id: Beanstalkd job ID
+    #     - tube: Tube name where job resides
+    #     - state: Job state (ready, reserved, delayed, buried)
+    #     - pri: Priority (0 = highest, 4294967295 = lowest)
+    #     - age: Seconds since job was created
+    #     - delay: Seconds remaining before job becomes ready (0 if ready now)
+    #     - ttr: Time-to-run in seconds (time allowed for job processing)
+    #     - time_left: Seconds remaining before job times out (0 if not reserved)
+    #     - file: Binlog file number containing job
+    #     - reserves: Number of times job has been reserved
+    #     - timeouts: Number of times job has timed out during processing
+    #     - releases: Number of times job has been released back to ready
+    #     - buries: Number of times job has been buried
+    #     - kicks: Number of times job has been kicked from buried/delayed
+    #
+    # @raise [Beaneater::NotFoundError] if job no longer exists in Beanstalkd
+    #
+    # @example
+    #   job.stats
+    #   # => {
+    #   #   id: 5,
+    #   #   bkid: 1,
+    #   #   queue: "sleep-jobs",
+    #   #   tube: "postburner.development.sleep-jobs",
+    #   #   watched: false,
+    #   #   beanstalk: {
+    #   #     id: 1,
+    #   #     tube: "postburner.development.sleep-jobs",
+    #   #     state: "ready",
+    #   #     pri: 50,
+    #   #     age: 1391,
+    #   #     delay: 0,
+    #   #     ttr: 120,
+    #   #     time_left: 0,
+    #   #     file: 0,
+    #   #     reserves: 0,
+    #   #     timeouts: 0,
+    #   #     releases: 0,
+    #   #     buries: 0,
+    #   #     kicks: 0
+    #   #   }
+    #   # }
+    #
+    def stats
+      # Get configured watched tubes (expanded with environment prefix)
+      watched = Postburner.watched_tube_names.include?(self.tube_name)
+
+      {
+        id: self.id,
+        bkid: self.bkid,
+        queue: queue_name,
+        tube: tube_name,
+        watched: watched,
+        beanstalk: self.bk.stats.to_h.symbolize_keys,
+      }
+    end
+
+    alias_method :beanstalk_job_stats, :stats
+
+    # Returns the queue name for this job instance.
+    #
+    # Checks instance-level override first, then falls back to class-level configuration.
+    #
+    # @return [String] Queue name
+    #
+    # @example Class-level configuration
+    #   class MyJob < Postburner::Job
+    #     queue 'critical'
+    #   end
+    #   job = MyJob.create!(args: {})
+    #   job.queue_name  # => 'critical'
+    #
+    def queue_name
+      self.class.queue
+    end
+
+    # Returns the queue priority for this job instance.
+    #
+    # Checks instance-level override first, then falls back to class-level configuration.
+    #
+    # @return [Integer, nil] Priority (lower = higher priority)
+    #
+    # @example Instance-level override
+    #   job = MyJob.create!(args: {}, queue_priority: 1500)
+    #   job.queue_priority  # => 1500
+    #
+    def queue_priority
+      @queue_priority || self.class.queue_priority
+    end
+
+    # Returns the queue TTR (time-to-run) for this job instance.
+    #
+    # Checks instance-level override first, then falls back to class-level configuration.
+    #
+    # @return [Integer, nil] TTR in seconds
+    #
+    # @example Instance-level override
+    #   job = MyJob.create!(args: {}, queue_ttr: 600)
+    #   job.queue_ttr  # => 600
+    #
+    def queue_ttr
+      @queue_ttr || self.class.queue_ttr
+    end
+
+    def tube_name
+      Postburner.configuration.expand_tube_name(queue_name)
+    end
+
+    # Extends the job's time-to-run (TTR) in Beanstalkd.
+    #
+    # Calls touch on the Beanstalkd job, extending the TTR by the original
+    # TTR value. Use this during long-running operations to prevent the job
+    # from timing out.
+    #
+    # @return [void]
+    #
+    # @note Does nothing if job has no bkid (e.g., in test mode)
+    #
+    # @example Process large file line by line
+    #   def perform(args)
+    #     file = File.find(args['file_id'])
+    #     file.each_line do |line|
+    #       # ... process line ...
+    #       extend!  # Extend TTR to prevent timeout
+    #     end
+    #   end
+    #
+    # @see #bk
+    #
+    def extend!
+      return unless self.bk
+      begin
+        self.bk.touch
+      rescue Beaneater::NotConnected => e
+        self.bk!.touch
+      end
+    end
+
     # Tracks an exception in the job's errata array.
     #
     # Appends exception details to the in-memory errata array with timestamp,
@@ -797,6 +919,7 @@ module Postburner
     # @api private
     #
     def insert_if_queued!
+      #debugger
       return unless self.will_insert?
       insert!(@_insert_options)
     end
@@ -815,6 +938,7 @@ module Postburner
     #
     def insert!(options={})
       response = Postburner.queue_strategy.insert(self, options)
+      #debugger
 
       # Response must be a hash with an :id key (value can be nil)
       # Backburner returns symbol keys

data/app/models/postburner/tracked_job.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Postburner
+  # Job wrapper for executing tracked ActiveJob instances.
+  #
+  # TrackedJob is a Postburner::Job subclass that deserializes and executes
+  # ActiveJob instances that opted-in to PostgreSQL tracking via the
+  # `Postburner::Tracked` concern.
+  #
+  # When an ActiveJob with `tracked` is enqueued, the adapter:
+  # 1. Creates a TrackedJob record in PostgreSQL
+  # 2. Stores the ActiveJob data in the `args` JSONB column
+  # 3. Queues a minimal payload to Beanstalkd with the TrackedJob ID
+  #
+  # When the worker executes the job:
+  # 1. Loads the TrackedJob record
+  # 2. Deserializes the ActiveJob from `args`
+  # 3. Executes it with full audit trail (logs, timing, errors)
+  #
+  # @example
+  #   # User's ActiveJob (opts in to tracking)
+  #   class ProcessPayment < ApplicationJob
+  #     include Postburner::Tracked
+  #     tracked
+  #
+  #     def perform(payment_id)
+  #       log "Processing payment #{payment_id}"
+  #       # ...
+  #     end
+  #   end
+  #
+  #   # ActiveJob adapter creates TrackedJob
+  #   ProcessPayment.perform_later(123)
+  #   # => Creates TrackedJob record, queues to Beanstalkd
+  #
+  #   # Worker executes
+  #   Postburner::Job.perform(tracked_job.id)
+  #   # => Deserializes ProcessPayment job and executes with audit trail
+  #
+  class TrackedJob < Job
+    # Executes the wrapped ActiveJob instance.
+    #
+    # Deserializes the ActiveJob from args, restores metadata, sets up
+    # the bidirectional link for logging, and executes the job.
+    #
+    # @param args [Hash] JSONB args containing serialized ActiveJob data
+    #
+    # @return [void]
+    #
+    # @raise [Exception] Any exception raised by the ActiveJob is logged and re-raised
+    #
+    def perform(args)
+      # Extract ActiveJob metadata from args
+      job_class = args['job_class'].constantize
+      arguments = ::ActiveJob::Arguments.deserialize(args['arguments'])
+
+      # Instantiate the ActiveJob
+      job = job_class.new(*arguments)
+
+      # Restore ActiveJob metadata
+      job.job_id = args['job_id']
+      job.queue_name = args['queue_name']
+      job.priority = args['priority']
+      job.executions = args['executions'] || 0
+      job.exception_executions = args['exception_executions'] || {}
+      job.locale = args['locale']
+      job.timezone = args['timezone']
+
+      if args['enqueued_at']
+        job.enqueued_at = Time.iso8601(args['enqueued_at'])
+      end
+
+      # Give the ActiveJob access to this Postburner::Job for logging
+      if job.respond_to?(:postburner_job=)
+        job.postburner_job = self
+      end
+
+      # Execute the job (ActiveJob handles retry_on/discard_on)
+      # Exceptions are caught by Postburner::Job#perform! and logged to errata
+      job.perform_now
+    end
+  end
+end

data/bin/postburner

@@ -0,0 +1,91 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Postburner worker executable
+#
+# Loads configuration from YAML and starts the appropriate worker type.
+#
+# Usage:
+#   bin/postburner [--config PATH] [--env ENVIRONMENT]
+#
+# Examples:
+#   bin/postburner
+#   bin/postburner --config config/postburner.yml --env production
+#
+
+require 'optparse'
+
+# Parse command-line options
+options = {
+  config: 'config/postburner.yml',
+  env: ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development',
+  worker: nil,
+  queues: nil
+}
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: bin/postburner [options]"
+
+  opts.on('-c', '--config PATH', 'Path to YAML configuration file (default: config/postburner.yml)') do |path|
+    options[:config] = path
+  end
+
+  opts.on('-e', '--env ENVIRONMENT', 'Environment (default: RAILS_ENV or development)') do |env|
+    options[:env] = env
+  end
+
+  opts.on('-w', '--worker WORKER', 'Worker name from config (required if multiple workers defined)') do |worker|
+    options[:worker] = worker
+  end
+
+  opts.on('-q', '--queues QUEUES', 'Comma-separated list of queues to process (default: all configured queues)') do |queues|
+    options[:queues] = queues.split(',').map(&:strip)
+  end
+
+  opts.on('-h', '--help', 'Show this help message') do
+    puts opts
+    exit
+  end
+end.parse!
+
+# Load Rails environment from current directory
+# This executable should be run from your Rails application root
+ENV['RAILS_ENV'] ||= options[:env]
+require File.expand_path('config/environment', Dir.pwd)
+
+# Postburner is loaded automatically via the Rails engine when the gem is in your Gemfile
+
+# Load configuration
+config_path = File.expand_path(options[:config], Dir.pwd)
+
+begin
+  config = Postburner::Configuration.load_yaml(config_path, options[:env], options[:worker])
+rescue ArgumentError => e
+  Rails.logger.error "[Postburner] ERROR: #{e.message}"
+  exit 1
+end
+
+# Filter queues if --queues option provided
+if options[:queues]
+  # Validate that all specified queues exist in config
+  invalid_queues = options[:queues] - config.queue_names
+  unless invalid_queues.empty?
+    config.logger.error "[Postburner] ERROR: Unknown queue(s): #{invalid_queues.join(', ')}"
+    config.logger.error "[Postburner] Available queues: #{config.queue_names.join(', ')}"
+    exit 1
+  end
+
+  # Filter config to only include specified queues
+  filtered_queues = config.queues.select { |name, _| options[:queues].include?(name.to_s) }
+  config.queues = filtered_queues
+end
+
+config.logger.info "[Postburner] Configuration: #{config_path}"
+config.logger.info "[Postburner] Environment: #{options[:env]}"
+config.logger.info "[Postburner] Worker: #{options[:worker] || '(auto-selected)'}" if options[:worker] || options[:queues].nil?
+config.logger.info "[Postburner] Queues: #{config.queue_names.join(', ')}"
+config.logger.info "[Postburner] Defaults: forks=#{config.default_forks}, threads=#{config.default_threads}, gc_limit=#{config.default_gc_limit || 'none'}"
+
+# Create and start worker
+worker = Postburner::Workers::Worker.new(config)
+worker.start

data/bin/rails

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+# This command will automatically be run when you run "rails" with Rails gems
+# installed from the root of your application.
+
+ENGINE_ROOT = File.expand_path('..', __dir__)
+ENGINE_PATH = File.expand_path('../lib/postburner/engine', __dir__)
+APP_PATH = File.expand_path('../test/dummy/config/application', __dir__)
+
+# Set up gems listed in the Gemfile.
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
+require "bundler/setup" if File.exist?(ENV["BUNDLE_GEMFILE"])
+
+require "rails/all"
+require "rails/engine/commands"

data/config/environment.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+# Leave me here for vim-rails to work in an engine.

data/config/postburner.yml

@@ -0,0 +1,22 @@
+development:
+  beanstalk_url: beanstalk://localhost:11300
+  worker_type: simple
+  queues:
+    default: {}
+
+test:
+  beanstalk_url: beanstalk://localhost:11300
+  worker_type: simple
+  queues:
+    default: {}
+
+production:
+  beanstalk_url: beanstalk://localhost:11300
+  worker_type: threads_on_fork
+  queues:
+    critical:
+      threads: 1
+      gc_limit: 100
+    default:
+      threads: 5
+      gc_limit: 500

data/config/postburner.yml.example

@@ -0,0 +1,142 @@
+# Postburner Configuration Example
+#
+# Copy this file to config/postburner.yml and customize for your environment.
+#
+# ## Named Workers Configuration
+#
+# Postburner uses named worker configurations to support different deployment patterns:
+# - Single worker: bin/postburner (auto-selects the single worker)
+# - Multiple workers: bin/postburner --worker <name> (must specify which worker)
+#
+# Each worker can have different fork/thread settings and process different queues.
+# This enables running different queue groups in separate OS processes with distinct
+# concurrency profiles.
+#
+# ## Puma-Style Architecture
+#
+# - **forks: 0** = Single process with thread pool (development/staging)
+# - **forks: 1+** = Multiple processes with thread pools (production)
+#
+# Scale by adjusting forks and threads per worker:
+# - Development: forks=0, threads=1 (single-threaded, easiest debugging)
+# - Staging: forks=0, threads=10 (multi-threaded, moderate load)
+# - Production: forks=4, threads=10 (40 concurrent jobs per queue)
+#
+
+default: &default
+  # Beanstalkd connection URL
+  # Override with ENV['BEANSTALK_URL'] if set
+  beanstalk_url: <%= ENV['BEANSTALK_URL'] || 'beanstalk://localhost:11300' %>
+
+development:
+  <<: *default
+
+  workers:
+    default:
+      # Single-threaded, single process (simplest for debugging)
+      # Defaults: forks=0, threads=1, gc_limit=nil
+      queues:
+        - default
+        - mailers
+
+test:
+  <<: *default
+
+  workers:
+    default:
+      # Test mode uses inline strategies automatically
+      # Defaults: forks=0, threads=1, gc_limit=nil
+      queues:
+        - default
+
+staging:
+  <<: *default
+
+  workers:
+    default:
+      # Multi-threaded, single process (moderate concurrency)
+      default_threads: 10
+      default_gc_limit: 5000
+      queues:
+        - critical
+        - default
+        - mailers
+
+production:
+  <<: *default
+
+  # Example 1: Single worker processing all queues with same settings
+  # Run: bin/postburner
+  #
+  # workers:
+  #   default:
+  #     default_forks: 4
+  #     default_threads: 10
+  #     default_gc_limit: 5000
+  #     queues:
+  #       - critical
+  #       - default
+  #       - mailers
+  #       - imports
+
+  # Example 2: Multiple workers with different concurrency profiles
+  # Run separate processes:
+  #   bin/postburner --worker imports    (4 forks, 1 thread each)
+  #   bin/postburner --worker general    (2 forks, 100 threads each)
+  #
+  workers:
+    # Heavy, memory-intensive jobs - more processes, fewer threads
+    imports:
+      default_forks: 4
+      default_threads: 1
+      default_gc_limit: 500
+      queues:
+        - imports
+        - data_processing
+
+    # General jobs - fewer processes, many threads
+    general:
+      default_forks: 2
+      default_threads: 100
+      default_gc_limit: 5000
+      queues:
+        - default
+        - mailers
+        - notifications
+
+  # Example 3: Fine-grained control with multiple specialized workers
+  # Run separate processes:
+  #   bin/postburner --worker critical
+  #   bin/postburner --worker default
+  #   bin/postburner --worker mailers
+  #
+  # workers:
+  #   critical:
+  #     default_forks: 1
+  #     default_threads: 1
+  #     default_gc_limit: 100
+  #     queues:
+  #       - critical
+  #
+  #   default:
+  #     default_forks: 4
+  #     default_threads: 10
+  #     default_gc_limit: 5000
+  #     queues:
+  #       - default
+  #
+  #   mailers:
+  #     default_forks: 2
+  #     default_threads: 5
+  #     default_gc_limit: 2000
+  #     queues:
+  #       - mailers
+
+# Global Defaults (can be overridden per worker):
+#
+# default_queue: default           # Default queue name (optional)
+# default_priority: 65536          # Lower = higher priority (optional, 0 is highest)
+# default_ttr: 300                 # Time-to-run in seconds (optional)
+# default_threads: 1               # Thread count per fork (optional, defaults to 1)
+# default_forks: 0                 # Fork count (optional, defaults to 0 = single process)
+# default_gc_limit: nil            # Exit after N jobs for restart (optional, nil = no limit)