postburner 0.8.0 → 1.0.0.pre.1

data/app/models/postburner/tracked_job.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Postburner
+  # Job wrapper for executing tracked ActiveJob instances.
+  #
+  # TrackedJob is a Postburner::Job subclass that deserializes and executes
+  # ActiveJob instances that opted-in to PostgreSQL tracking via the
+  # `Postburner::Tracked` concern.
+  #
+  # When an ActiveJob with `tracked` is enqueued, the adapter:
+  # 1. Creates a TrackedJob record in PostgreSQL
+  # 2. Stores the ActiveJob data in the `args` JSONB column
+  # 3. Queues a minimal payload to Beanstalkd with the TrackedJob ID
+  #
+  # When the worker executes the job:
+  # 1. Loads the TrackedJob record
+  # 2. Deserializes the ActiveJob from `args`
+  # 3. Executes it with full audit trail (logs, timing, errors)
+  #
+  # @example
+  #   # User's ActiveJob (opts in to tracking)
+  #   class ProcessPayment < ApplicationJob
+  #     include Postburner::Tracked
+  #     tracked
+  #
+  #     def perform(payment_id)
+  #       log "Processing payment #{payment_id}"
+  #       # ...
+  #     end
+  #   end
+  #
+  #   # ActiveJob adapter creates TrackedJob
+  #   ProcessPayment.perform_later(123)
+  #   # => Creates TrackedJob record, queues to Beanstalkd
+  #
+  #   # Worker executes
+  #   Postburner::Job.perform(tracked_job.id)
+  #   # => Deserializes ProcessPayment job and executes with audit trail
+  #
+  class TrackedJob < Job
+    # Executes the wrapped ActiveJob instance.
+    #
+    # Deserializes the ActiveJob from args, restores metadata, sets up
+    # the bidirectional link for logging, and executes the job.
+    #
+    # @param args [Hash] JSONB args containing serialized ActiveJob data
+    #
+    # @return [void]
+    #
+    # @raise [Exception] Any exception raised by the ActiveJob is logged and re-raised
+    #
+    def perform(args)
+      # Extract ActiveJob metadata from args
+      job_class = args['job_class'].constantize
+      arguments = ::ActiveJob::Arguments.deserialize(args['arguments'])
+
+      # Instantiate the ActiveJob
+      job = job_class.new(*arguments)
+
+      # Restore ActiveJob metadata
+      job.job_id = args['job_id']
+      job.queue_name = args['queue_name']
+      job.priority = args['priority']
+      job.executions = args['executions'] || 0
+      job.exception_executions = args['exception_executions'] || {}
+      job.locale = args['locale']
+      job.timezone = args['timezone']
+
+      if args['enqueued_at']
+        job.enqueued_at = Time.iso8601(args['enqueued_at'])
+      end
+
+      # Give the ActiveJob access to this Postburner::Job for logging
+      if job.respond_to?(:postburner_job=)
+        job.postburner_job = self
+      end
+
+      # Execute the job (ActiveJob handles retry_on/discard_on)
+      # Exceptions are caught by Postburner::Job#perform! and logged to errata
+      job.perform_now
+    end
+  end
+end

data/bin/postburner

@@ -0,0 +1,91 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Postburner worker executable
+#
+# Loads configuration from YAML and starts the appropriate worker type.
+#
+# Usage:
+#   bin/postburner [--config PATH] [--env ENVIRONMENT]
+#
+# Examples:
+#   bin/postburner
+#   bin/postburner --config config/postburner.yml --env production
+#
+
+require 'optparse'
+
+# Parse command-line options
+options = {
+  config: 'config/postburner.yml',
+  env: ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development',
+  worker: nil,
+  queues: nil
+}
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: bin/postburner [options]"
+
+  opts.on('-c', '--config PATH', 'Path to YAML configuration file (default: config/postburner.yml)') do |path|
+    options[:config] = path
+  end
+
+  opts.on('-e', '--env ENVIRONMENT', 'Environment (default: RAILS_ENV or development)') do |env|
+    options[:env] = env
+  end
+
+  opts.on('-w', '--worker WORKER', 'Worker name from config (required if multiple workers defined)') do |worker|
+    options[:worker] = worker
+  end
+
+  opts.on('-q', '--queues QUEUES', 'Comma-separated list of queues to process (default: all configured queues)') do |queues|
+    options[:queues] = queues.split(',').map(&:strip)
+  end
+
+  opts.on('-h', '--help', 'Show this help message') do
+    puts opts
+    exit
+  end
+end.parse!
+
+# Load Rails environment from current directory
+# This executable should be run from your Rails application root
+ENV['RAILS_ENV'] ||= options[:env]
+require File.expand_path('config/environment', Dir.pwd)
+
+# Postburner is loaded automatically via the Rails engine when the gem is in your Gemfile
+
+# Load configuration
+config_path = File.expand_path(options[:config], Dir.pwd)
+
+begin
+  config = Postburner::Configuration.load_yaml(config_path, options[:env], options[:worker])
+rescue ArgumentError => e
+  Rails.logger.error "[Postburner] ERROR: #{e.message}"
+  exit 1
+end
+
+# Filter queues if --queues option provided
+if options[:queues]
+  # Validate that all specified queues exist in config
+  invalid_queues = options[:queues] - config.queue_names
+  unless invalid_queues.empty?
+    config.logger.error "[Postburner] ERROR: Unknown queue(s): #{invalid_queues.join(', ')}"
+    config.logger.error "[Postburner] Available queues: #{config.queue_names.join(', ')}"
+    exit 1
+  end
+
+  # Filter config to only include specified queues
+  filtered_queues = config.queues.select { |name, _| options[:queues].include?(name.to_s) }
+  config.queues = filtered_queues
+end
+
+config.logger.info "[Postburner] Configuration: #{config_path}"
+config.logger.info "[Postburner] Environment: #{options[:env]}"
+config.logger.info "[Postburner] Worker: #{options[:worker] || '(auto-selected)'}" if options[:worker] || options[:queues].nil?
+config.logger.info "[Postburner] Queues: #{config.queue_names.join(', ')}"
+config.logger.info "[Postburner] Defaults: forks=#{config.default_forks}, threads=#{config.default_threads}, gc_limit=#{config.default_gc_limit || 'none'}"
+
+# Create and start worker
+worker = Postburner::Workers::Worker.new(config)
+worker.start

data/bin/rails

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+# This command will automatically be run when you run "rails" with Rails gems
+# installed from the root of your application.
+
+ENGINE_ROOT = File.expand_path('..', __dir__)
+ENGINE_PATH = File.expand_path('../lib/postburner/engine', __dir__)
+APP_PATH = File.expand_path('../test/dummy/config/application', __dir__)
+
+# Set up gems listed in the Gemfile.
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
+require "bundler/setup" if File.exist?(ENV["BUNDLE_GEMFILE"])
+
+require "rails/all"
+require "rails/engine/commands"

data/config/environment.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+# Leave me here for vim-rails to work in an engine.

data/config/postburner.yml

@@ -0,0 +1,22 @@
+development:
+  beanstalk_url: beanstalk://localhost:11300
+  worker_type: simple
+  queues:
+    default: {}
+
+test:
+  beanstalk_url: beanstalk://localhost:11300
+  worker_type: simple
+  queues:
+    default: {}
+
+production:
+  beanstalk_url: beanstalk://localhost:11300
+  worker_type: threads_on_fork
+  queues:
+    critical:
+      threads: 1
+      gc_limit: 100
+    default:
+      threads: 5
+      gc_limit: 500

data/config/postburner.yml.example

@@ -0,0 +1,142 @@
+# Postburner Configuration Example
+#
+# Copy this file to config/postburner.yml and customize for your environment.
+#
+# ## Named Workers Configuration
+#
+# Postburner uses named worker configurations to support different deployment patterns:
+# - Single worker: bin/postburner (auto-selects the single worker)
+# - Multiple workers: bin/postburner --worker <name> (must specify which worker)
+#
+# Each worker can have different fork/thread settings and process different queues.
+# This enables running different queue groups in separate OS processes with distinct
+# concurrency profiles.
+#
+# ## Puma-Style Architecture
+#
+# - **forks: 0** = Single process with thread pool (development/staging)
+# - **forks: 1+** = Multiple processes with thread pools (production)
+#
+# Scale by adjusting forks and threads per worker:
+# - Development: forks=0, threads=1 (single-threaded, easiest debugging)
+# - Staging: forks=0, threads=10 (multi-threaded, moderate load)
+# - Production: forks=4, threads=10 (40 concurrent jobs per queue)
+#
+
+default: &default
+  # Beanstalkd connection URL
+  # Override with ENV['BEANSTALK_URL'] if set
+  beanstalk_url: <%= ENV['BEANSTALK_URL'] || 'beanstalk://localhost:11300' %>
+
+development:
+  <<: *default
+
+  workers:
+    default:
+      # Single-threaded, single process (simplest for debugging)
+      # Defaults: forks=0, threads=1, gc_limit=nil
+      queues:
+        - default
+        - mailers
+
+test:
+  <<: *default
+
+  workers:
+    default:
+      # Test mode uses inline strategies automatically
+      # Defaults: forks=0, threads=1, gc_limit=nil
+      queues:
+        - default
+
+staging:
+  <<: *default
+
+  workers:
+    default:
+      # Multi-threaded, single process (moderate concurrency)
+      default_threads: 10
+      default_gc_limit: 5000
+      queues:
+        - critical
+        - default
+        - mailers
+
+production:
+  <<: *default
+
+  # Example 1: Single worker processing all queues with same settings
+  # Run: bin/postburner
+  #
+  # workers:
+  #   default:
+  #     default_forks: 4
+  #     default_threads: 10
+  #     default_gc_limit: 5000
+  #     queues:
+  #       - critical
+  #       - default
+  #       - mailers
+  #       - imports
+
+  # Example 2: Multiple workers with different concurrency profiles
+  # Run separate processes:
+  #   bin/postburner --worker imports    (4 forks, 1 thread each)
+  #   bin/postburner --worker general    (2 forks, 100 threads each)
+  #
+  workers:
+    # Heavy, memory-intensive jobs - more processes, fewer threads
+    imports:
+      default_forks: 4
+      default_threads: 1
+      default_gc_limit: 500
+      queues:
+        - imports
+        - data_processing
+
+    # General jobs - fewer processes, many threads
+    general:
+      default_forks: 2
+      default_threads: 100
+      default_gc_limit: 5000
+      queues:
+        - default
+        - mailers
+        - notifications
+
+  # Example 3: Fine-grained control with multiple specialized workers
+  # Run separate processes:
+  #   bin/postburner --worker critical
+  #   bin/postburner --worker default
+  #   bin/postburner --worker mailers
+  #
+  # workers:
+  #   critical:
+  #     default_forks: 1
+  #     default_threads: 1
+  #     default_gc_limit: 100
+  #     queues:
+  #       - critical
+  #
+  #   default:
+  #     default_forks: 4
+  #     default_threads: 10
+  #     default_gc_limit: 5000
+  #     queues:
+  #       - default
+  #
+  #   mailers:
+  #     default_forks: 2
+  #     default_threads: 5
+  #     default_gc_limit: 2000
+  #     queues:
+  #       - mailers
+
+# Global Defaults (can be overridden per worker):
+#
+# default_queue: default           # Default queue name (optional)
+# default_priority: 65536          # Lower = higher priority (optional, 0 is highest)
+# default_ttr: 300                 # Time-to-run in seconds (optional)
+# default_threads: 1               # Thread count per fork (optional, defaults to 1)
+# default_forks: 0                 # Fork count (optional, defaults to 0 = single process)
+# default_gc_limit: nil            # Exit after N jobs for restart (optional, nil = no limit)

data/lib/postburner/active_job/adapter.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  module QueueAdapters
+    # Postburner adapter for ActiveJob.
+    #
+    # Provides dual-mode job execution:
+    # - **Default mode**: Fast execution via Beanstalkd only
+    # - **Tracked mode** (opt-in): Full PostgreSQL audit trail
+    #
+    # ## Usage
+    #
+    # @example Configure in Rails
+    #   # config/application.rb
+    #   config.active_job.queue_adapter = :postburner
+    #
+    # @example Default job (fast)
+    #   class SendEmail < ApplicationJob
+    #     def perform(user_id)
+    #       # No PostgreSQL overhead, executes quickly
+    #     end
+    #   end
+    #
+    # @example Tracked job (opt-in for audit trail)
+    #   class ProcessPayment < ApplicationJob
+    #     include Postburner::Tracked
+    #     tracked  # ← Enables PostgreSQL tracking
+    #
+    #     def perform(payment_id)
+    #       log "Processing payment..."
+    #       # Full audit trail: logs, timing, errors
+    #     end
+    #   end
+    #
+    class PostburnerAdapter
+      # Enqueues a job for immediate execution.
+      #
+      # @param job [ActiveJob::Base] The job to enqueue
+      #
+      # @return [void]
+      #
+      def enqueue(job)
+        enqueue_at(job, nil)
+      end
+
+      # Enqueues a job for execution at a specific time.
+      #
+      # @param job [ActiveJob::Base] The job to enqueue
+      # @param timestamp [Time, nil] When to execute the job (nil = immediate)
+      #
+      # @return [void]
+      #
+      def enqueue_at(job, timestamp)
+        if tracked?(job)
+          enqueue_tracked(job, timestamp)
+        else
+          enqueue_default(job, timestamp)
+        end
+      end
+
+      private
+
+      # Checks if a job should be tracked in PostgreSQL.
+      #
+      # Detects tracking by checking if the job class includes the
+      # Postburner::Tracked module.
+      #
+      # @param job [ActiveJob::Base] The job instance
+      #
+      # @return [Boolean] true if tracked, false otherwise
+      #
+      def tracked?(job)
+        job.class.included_modules.include?(Postburner::Tracked)
+      end
+
+      # Enqueues a tracked job (with PostgreSQL audit trail).
+      #
+      # Creates a Postburner::TrackedJob record, then queues minimal payload
+      # to Beanstalkd with just the job ID reference.
+      #
+      # @param job [ActiveJob::Base] The job instance
+      # @param timestamp [Time, nil] When to execute the job
+      #
+      # @return [void]
+      #
+      def enqueue_tracked(job, timestamp)
+        # Create Postburner::TrackedJob record
+        tracked_job = Postburner::TrackedJob.create!(
+          args: Postburner::ActiveJob::Payload.serialize_for_tracked(job),
+          run_at: timestamp,
+          queued_at: Time.zone.now
+        )
+
+        # Calculate delay for Beanstalkd
+        delay = timestamp ? [(timestamp.to_f - Time.now.to_f).to_i, 0].max : 0
+
+        # Queue to Beanstalkd with minimal payload
+        Postburner.connected do |conn|
+          tube_name = expand_tube_name(job.queue_name)
+
+          # Get priority and TTR from class configuration or fall back to defaults
+          pri = job.class.respond_to?(:queue_priority) && job.class.queue_priority ||
+                Postburner.configuration.default_priority
+          ttr = job.class.respond_to?(:queue_ttr) && job.class.queue_ttr ||
+                Postburner.configuration.default_ttr
+
+          bkid = conn.tubes[tube_name].put(
+            Postburner::ActiveJob::Payload.tracked_payload(job, tracked_job.id),
+            pri: pri,
+            delay: delay,
+            ttr: ttr
+          )
+
+          # Update tracked_job with Beanstalkd ID
+          tracked_job.update_column(:bkid, bkid)
+        end
+      end
+
+      # Enqueues a default job (Beanstalkd only, no PostgreSQL).
+      #
+      # Queues full job data to Beanstalkd for fast execution without
+      # PostgreSQL overhead.
+      #
+      # @param job [ActiveJob::Base] The job instance
+      # @param timestamp [Time, nil] When to execute the job
+      #
+      # @return [void]
+      #
+      def enqueue_default(job, timestamp)
+        delay = timestamp ? [(timestamp.to_f - Time.now.to_f).to_i, 0].max : 0
+
+        Postburner.connected do |conn|
+          tube_name = expand_tube_name(job.queue_name)
+
+          # Get priority and TTR from class configuration or fall back to defaults
+          pri = job.class.respond_to?(:queue_priority) && job.class.queue_priority ||
+                Postburner.configuration.default_priority
+          ttr = job.class.respond_to?(:queue_ttr) && job.class.queue_ttr ||
+                Postburner.configuration.default_ttr
+
+          conn.tubes[tube_name].put(
+            Postburner::ActiveJob::Payload.default_payload(job),
+            pri: pri,
+            delay: delay,
+            ttr: ttr
+          )
+        end
+      end
+
+      # Expands queue name to full tube name with environment prefix.
+      #
+      # Delegates to Postburner::Configuration for consistent tube naming.
+      #
+      # @param queue_name [String] Queue name from ActiveJob
+      #
+      # @return [String] Full tube name (e.g., 'postburner.production.critical')
+      #
+      def expand_tube_name(queue_name)
+        Postburner.configuration.expand_tube_name(queue_name)
+      end
+    end
+  end
+end

data/lib/postburner/active_job/execution.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Postburner
+  module ActiveJob
+    # Handles execution of ActiveJob jobs from Beanstalkd payloads.
+    #
+    # Supports both default jobs (fast, no PostgreSQL) and tracked jobs
+    # (full audit trail). Also handles legacy Postburner::Job format
+    # for backward compatibility.
+    #
+    class Execution
+      class << self
+        # Executes a job from a Beanstalkd payload.
+        #
+        # Automatically detects payload type (default, tracked, or legacy)
+        # and routes to the appropriate execution path.
+        #
+        # @param payload_json [String] JSON payload from Beanstalkd
+        #
+        # @return [void]
+        #
+        # @raise [StandardError] if job execution fails
+        #
+        def execute(payload_json)
+          payload = Payload.parse(payload_json)
+
+          if Payload.legacy_format?(payload)
+            execute_legacy(payload)
+          elsif Payload.tracked?(payload)
+            execute_tracked(payload)
+          else
+            execute_default(payload)
+          end
+        end
+
+        private
+
+        # Executes a tracked job (with PostgreSQL audit trail).
+        #
+        # Loads the Postburner::Job record and delegates to its perform! method,
+        # which handles all the audit trail logging, timing, and error tracking.
+        #
+        # @param payload [Hash] Parsed payload
+        #
+        # @return [void]
+        #
+        def execute_tracked(payload)
+          job_id = payload['postburner_job_id']
+
+          unless job_id
+            raise ArgumentError, "Tracked job missing postburner_job_id"
+          end
+
+          # Delegate to Postburner::Job.perform which handles the full lifecycle
+          Postburner::Job.perform(job_id)
+        end
+
+        # Executes a default job (Beanstalkd only, no PostgreSQL).
+        #
+        # Deserializes the ActiveJob, restores its metadata, and executes it.
+        # Handles retries via ActiveJob's retry_on/discard_on mechanisms.
+        #
+        # @param payload [Hash] Parsed payload
+        #
+        # @return [void]
+        #
+        def execute_default(payload)
+          job_class = payload['job_class'].constantize
+          arguments = ::ActiveJob::Arguments.deserialize(payload['arguments'])
+
+          # Instantiate the job
+          job = job_class.new(*arguments)
+
+          # Restore job metadata
+          job.job_id = payload['job_id']
+          job.queue_name = payload['queue_name']
+          job.priority = payload['priority']
+          job.executions = payload['executions'] || 0
+          job.exception_executions = payload['exception_executions'] || {}
+          job.locale = payload['locale']
+          job.timezone = payload['timezone']
+
+          if payload['enqueued_at']
+            job.enqueued_at = Time.iso8601(payload['enqueued_at'])
+          end
+
+          # Execute the job (ActiveJob handles retry_on/discard_on)
+          job.perform_now
+        end
+
+        # Executes a legacy Postburner::Job (backward compatibility).
+        #
+        # Legacy format: { "class" => "JobClassName", "args" => [job_id] }
+        #
+        # @param payload [Hash] Parsed legacy payload
+        #
+        # @return [void]
+        #
+        def execute_legacy(payload)
+          job_class = payload['class'].constantize
+          job_id = payload['args'].first
+
+          # Delegate to the class's perform method (Postburner::Job.perform)
+          job_class.perform(job_id)
+        end
+      end
+    end
+  end
+end