postburner 0.9.0.rc.1 → 1.0.0.pre.2

data/lib/generators/postburner/install/templates/config/postburner.yml

@@ -0,0 +1,142 @@
+# Postburner Configuration Example
+#
+# Copy this file to config/postburner.yml and customize for your environment.
+#
+# ## Named Workers Configuration
+#
+# Postburner uses named worker configurations to support different deployment patterns:
+# - Single worker: bin/postburner (auto-selects the single worker)
+# - Multiple workers: bin/postburner --worker <name> (must specify which worker)
+#
+# Each worker can have different fork/thread settings and process different queues.
+# This enables running different queue groups in separate OS processes with distinct
+# concurrency profiles.
+#
+# ## Puma-Style Architecture
+#
+# - **forks: 0** = Single process with thread pool (development/staging)
+# - **forks: 1+** = Multiple processes with thread pools (production)
+#
+# Scale by adjusting forks and threads per worker:
+# - Development: forks=0, threads=1 (single-threaded, easiest debugging)
+# - Staging: forks=0, threads=10 (multi-threaded, moderate load)
+# - Production: forks=4, threads=10 (40 concurrent jobs per queue)
+#
+
+default: &default
+  # Beanstalkd connection URL
+  # Override with ENV['BEANSTALK_URL'] if set
+  beanstalk_url: <%= ENV['BEANSTALK_URL'] || 'beanstalk://localhost:11300' %>
+
+development:
+  <<: *default
+
+  workers:
+    default:
+      # Single-threaded, single process (simplest for debugging)
+      # Defaults: forks=0, threads=1, gc_limit=nil
+      queues:
+        - default
+        - mailers
+
+test:
+  <<: *default
+
+  workers:
+    default:
+      # Test mode uses inline strategies automatically
+      # Defaults: forks=0, threads=1, gc_limit=nil
+      queues:
+        - default
+
+staging:
+  <<: *default
+
+  workers:
+    default:
+      # Multi-threaded, single process (moderate concurrency)
+      default_threads: 10
+      default_gc_limit: 5000
+      queues:
+        - critical
+        - default
+        - mailers
+
+production:
+  <<: *default
+
+  # Example 1: Single worker processing all queues with same settings
+  # Run: bin/postburner
+  #
+  # workers:
+  #   default:
+  #     default_forks: 4
+  #     default_threads: 10
+  #     default_gc_limit: 5000
+  #     queues:
+  #       - critical
+  #       - default
+  #       - mailers
+  #       - imports
+
+  # Example 2: Multiple workers with different concurrency profiles
+  # Run separate processes:
+  #   bin/postburner --worker imports    (4 forks, 1 thread each)
+  #   bin/postburner --worker general    (2 forks, 100 threads each)
+  #
+  workers:
+    # Heavy, memory-intensive jobs - more processes, fewer threads
+    imports:
+      default_forks: 4
+      default_threads: 1
+      default_gc_limit: 500
+      queues:
+        - imports
+        - data_processing
+
+    # General jobs - fewer processes, many threads
+    general:
+      default_forks: 2
+      default_threads: 100
+      default_gc_limit: 5000
+      queues:
+        - default
+        - mailers
+        - notifications
+
+  # Example 3: Fine-grained control with multiple specialized workers
+  # Run separate processes:
+  #   bin/postburner --worker critical
+  #   bin/postburner --worker default
+  #   bin/postburner --worker mailers
+  #
+  # workers:
+  #   critical:
+  #     default_forks: 1
+  #     default_threads: 1
+  #     default_gc_limit: 100
+  #     queues:
+  #       - critical
+  #
+  #   default:
+  #     default_forks: 4
+  #     default_threads: 10
+  #     default_gc_limit: 5000
+  #     queues:
+  #       - default
+  #
+  #   mailers:
+  #     default_forks: 2
+  #     default_threads: 5
+  #     default_gc_limit: 2000
+  #     queues:
+  #       - mailers
+
+# Global Defaults (can be overridden per worker):
+#
+# default_queue: default           # Default queue name (optional)
+# default_priority: 65536          # Lower = higher priority (optional, 0 is highest)
+# default_ttr: 300                 # Time-to-run in seconds (optional)
+# default_threads: 1               # Thread count per fork (optional, defaults to 1)
+# default_forks: 0                 # Fork count (optional, defaults to 0 = single process)
+# default_gc_limit: nil            # Exit after N jobs for restart (optional, nil = no limit)

data/lib/postburner/active_job/adapter.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  module QueueAdapters
+    # Postburner adapter for ActiveJob.
+    #
+    # Provides dual-mode job execution:
+    # - **Default mode**: Fast execution via Beanstalkd only
+    # - **Tracked mode** (opt-in): Full PostgreSQL audit trail
+    #
+    # ## Usage
+    #
+    # @example Configure in Rails
+    #   # config/application.rb
+    #   config.active_job.queue_adapter = :postburner
+    #
+    # @example Default job (fast)
+    #   class SendEmail < ApplicationJob
+    #     def perform(user_id)
+    #       # No PostgreSQL overhead, executes quickly
+    #     end
+    #   end
+    #
+    # @example Tracked job (opt-in for audit trail)
+    #   class ProcessPayment < ApplicationJob
+    #     include Postburner::Tracked
+    #     tracked  # ← Enables PostgreSQL tracking
+    #
+    #     def perform(payment_id)
+    #       log "Processing payment..."
+    #       # Full audit trail: logs, timing, errors
+    #     end
+    #   end
+    #
+    class PostburnerAdapter
+      # Enqueues a job for immediate execution.
+      #
+      # @param job [ActiveJob::Base] The job to enqueue
+      #
+      # @return [void]
+      #
+      def enqueue(job)
+        enqueue_at(job, nil)
+      end
+
+      # Enqueues a job for execution at a specific time.
+      #
+      # @param job [ActiveJob::Base] The job to enqueue
+      # @param timestamp [Time, nil] When to execute the job (nil = immediate)
+      #
+      # @return [void]
+      #
+      def enqueue_at(job, timestamp)
+        if tracked?(job)
+          enqueue_tracked(job, timestamp)
+        else
+          enqueue_default(job, timestamp)
+        end
+      end
+
+      # Indicates whether the adapter supports enqueuing after transaction commit.
+      #
+      # Returns true because Postburner uses Beanstalkd (external queue),
+      # which means jobs are safely enqueued outside the database transaction.
+      # This allows Rails to automatically defer job enqueuing until after
+      # the current database transaction commits.
+      #
+      # @return [Boolean] Always returns true
+      #
+      def enqueue_after_transaction_commit?
+        true
+      end
+
+      private
+
+      # Checks if a job should be tracked in PostgreSQL.
+      #
+      # Detects tracking by checking if the job class includes the
+      # Postburner::Tracked module.
+      #
+      # @param job [ActiveJob::Base] The job instance
+      #
+      # @return [Boolean] true if tracked, false otherwise
+      #
+      def tracked?(job)
+        job.class.included_modules.include?(Postburner::Tracked)
+      end
+
+      # Enqueues a tracked job (with PostgreSQL audit trail).
+      #
+      # Creates a Postburner::TrackedJob record, then queues minimal payload
+      # to Beanstalkd with just the job ID reference.
+      #
+      # @param job [ActiveJob::Base] The job instance
+      # @param timestamp [Time, nil] When to execute the job
+      #
+      # @return [void]
+      #
+      def enqueue_tracked(job, timestamp)
+        # Create Postburner::TrackedJob record
+        tracked_job = Postburner::TrackedJob.create!(
+          args: Postburner::ActiveJob::Payload.serialize_for_tracked(job),
+          run_at: timestamp,
+          queued_at: Time.zone.now
+        )
+
+        # Calculate delay for Beanstalkd
+        delay = timestamp ? [(timestamp.to_f - Time.now.to_f).to_i, 0].max : 0
+
+        # Queue to Beanstalkd with minimal payload
+        Postburner.connected do |conn|
+          tube_name = expand_tube_name(job.queue_name)
+
+          # Get priority and TTR from class configuration or fall back to defaults
+          pri = job.class.respond_to?(:queue_priority) && job.class.queue_priority ||
+                Postburner.configuration.default_priority
+          ttr = job.class.respond_to?(:queue_ttr) && job.class.queue_ttr ||
+                Postburner.configuration.default_ttr
+
+          bkid = conn.tubes[tube_name].put(
+            Postburner::ActiveJob::Payload.tracked_payload(job, tracked_job.id),
+            pri: pri,
+            delay: delay,
+            ttr: ttr
+          )
+
+          # Update tracked_job with Beanstalkd ID
+          tracked_job.update_column(:bkid, bkid)
+        end
+      end
+
+      # Enqueues a default job (Beanstalkd only, no PostgreSQL).
+      #
+      # Queues full job data to Beanstalkd for fast execution without
+      # PostgreSQL overhead.
+      #
+      # @param job [ActiveJob::Base] The job instance
+      # @param timestamp [Time, nil] When to execute the job
+      #
+      # @return [void]
+      #
+      def enqueue_default(job, timestamp)
+        delay = timestamp ? [(timestamp.to_f - Time.now.to_f).to_i, 0].max : 0
+
+        Postburner.connected do |conn|
+          tube_name = expand_tube_name(job.queue_name)
+
+          # Get priority and TTR from class configuration or fall back to defaults
+          pri = job.class.respond_to?(:queue_priority) && job.class.queue_priority ||
+                Postburner.configuration.default_priority
+          ttr = job.class.respond_to?(:queue_ttr) && job.class.queue_ttr ||
+                Postburner.configuration.default_ttr
+
+          conn.tubes[tube_name].put(
+            Postburner::ActiveJob::Payload.default_payload(job),
+            pri: pri,
+            delay: delay,
+            ttr: ttr
+          )
+        end
+      end
+
+      # Expands queue name to full tube name with environment prefix.
+      #
+      # Delegates to Postburner::Configuration for consistent tube naming.
+      #
+      # @param queue_name [String] Queue name from ActiveJob
+      #
+      # @return [String] Full tube name (e.g., 'postburner.production.critical')
+      #
+      def expand_tube_name(queue_name)
+        Postburner.configuration.expand_tube_name(queue_name)
+      end
+    end
+  end
+end

data/lib/postburner/active_job/execution.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Postburner
+  module ActiveJob
+    # Handles execution of ActiveJob jobs from Beanstalkd payloads.
+    #
+    # Supports both default jobs (fast, no PostgreSQL) and tracked jobs
+    # (full audit trail). Also handles legacy Postburner::Job format
+    # for backward compatibility.
+    #
+    class Execution
+      class << self
+        # Executes a job from a Beanstalkd payload.
+        #
+        # Automatically detects payload type (default, tracked, or legacy)
+        # and routes to the appropriate execution path.
+        #
+        # @param payload_json [String] JSON payload from Beanstalkd
+        #
+        # @return [void]
+        #
+        # @raise [StandardError] if job execution fails
+        #
+        def execute(payload_json)
+          payload = Payload.parse(payload_json)
+
+          if Payload.legacy_format?(payload)
+            execute_legacy(payload)
+          elsif Payload.tracked?(payload)
+            execute_tracked(payload)
+          else
+            execute_default(payload)
+          end
+        end
+
+        private
+
+        # Executes a tracked job (with PostgreSQL audit trail).
+        #
+        # Loads the Postburner::Job record and delegates to its perform! method,
+        # which handles all the audit trail logging, timing, and error tracking.
+        #
+        # @param payload [Hash] Parsed payload
+        #
+        # @return [void]
+        #
+        def execute_tracked(payload)
+          job_id = payload['postburner_job_id']
+
+          unless job_id
+            raise ArgumentError, "Tracked job missing postburner_job_id"
+          end
+
+          # Delegate to Postburner::Job.perform which handles the full lifecycle
+          Postburner::Job.perform(job_id)
+        end
+
+        # Executes a default job (Beanstalkd only, no PostgreSQL).
+        #
+        # Deserializes the ActiveJob, restores its metadata, and executes it.
+        # Handles retries via ActiveJob's retry_on/discard_on mechanisms.
+        #
+        # @param payload [Hash] Parsed payload
+        #
+        # @return [void]
+        #
+        def execute_default(payload)
+          job_class = payload['job_class'].constantize
+          arguments = ::ActiveJob::Arguments.deserialize(payload['arguments'])
+
+          # Instantiate the job
+          job = job_class.new(*arguments)
+
+          # Restore job metadata
+          job.job_id = payload['job_id']
+          job.queue_name = payload['queue_name']
+          job.priority = payload['priority']
+          job.executions = payload['executions'] || 0
+          job.exception_executions = payload['exception_executions'] || {}
+          job.locale = payload['locale']
+          job.timezone = payload['timezone']
+
+          if payload['enqueued_at']
+            job.enqueued_at = Time.iso8601(payload['enqueued_at'])
+          end
+
+          # Execute the job (ActiveJob handles retry_on/discard_on)
+          job.perform_now
+        end
+
+        # Executes a legacy Postburner::Job (backward compatibility).
+        #
+        # Legacy format: { "class" => "JobClassName", "args" => [job_id] }
+        #
+        # @param payload [Hash] Parsed legacy payload
+        #
+        # @return [void]
+        #
+        def execute_legacy(payload)
+          job_class = payload['class'].constantize
+          job_id = payload['args'].first
+
+          # Delegate to the class's perform method (Postburner::Job.perform)
+          job_class.perform(job_id)
+        end
+      end
+    end
+  end
+end

data/lib/postburner/active_job/payload.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+module Postburner
+  module ActiveJob
+    # Handles serialization/deserialization of ActiveJob payloads for Beanstalkd.
+    #
+    # Provides consistent payload format for both default and tracked jobs, with
+    # versioning support for future format changes.
+    #
+    # ## Payload Format (V1)
+    #
+    # Both default and tracked jobs store the same ActiveJob data in Beanstalkd.
+    # The only difference is tracked jobs also persist to PostgreSQL and include
+    # a `postburner_job_id` reference.
+    #
+    # @example Default job payload
+    #   {
+    #     v: 1,
+    #     tracked: false,
+    #     postburner_job_id: nil,
+    #     job_class: "SendEmail",
+    #     job_id: "abc-123",
+    #     queue_name: "mailers",
+    #     arguments: [[1, 2, 3]],
+    #     retry_count: 0,
+    #     ...
+    #   }
+    #
+    # @example Tracked job payload
+    #   {
+    #     v: 1,
+    #     tracked: true,
+    #     postburner_job_id: 456,  # References postburner_jobs.id
+    #     job_class: "ProcessPayment",
+    #     job_id: "def-456",
+    #     queue_name: "critical",
+    #     arguments: [[789]],
+    #     retry_count: 0,
+    #     ...
+    #   }
+    #
+    class Payload
+      VERSION = 1
+
+      class << self
+        # Generates payload structure for an ActiveJob.
+        #
+        # @param job [ActiveJob::Base] The ActiveJob instance
+        # @param tracked [Boolean] Whether this is a tracked job
+        # @param postburner_job_id [Integer, nil] Postburner::Job ID for tracked jobs
+        #
+        # @return [Hash] Payload hash
+        #
+        # @api private
+        #
+        def for_job(job, tracked: false, postburner_job_id: nil)
+          {
+            v: VERSION,
+            tracked: tracked,
+            postburner_job_id: postburner_job_id,
+            job_class: job.class.name,
+            job_id: job.job_id,
+            queue_name: job.queue_name,
+            priority: job.priority,
+            arguments: ::ActiveJob::Arguments.serialize(job.arguments),
+            executions: job.executions,
+            exception_executions: job.exception_executions || {},
+            locale: job.locale,
+            timezone: job.timezone,
+            enqueued_at: job.enqueued_at&.iso8601,
+            retry_count: 0  # For default job retry tracking
+          }
+        end
+
+        # Generates JSON payload for a default job.
+        #
+        # @param job [ActiveJob::Base] The ActiveJob instance
+        #
+        # @return [String] JSON-encoded payload
+        #
+        def default_payload(job)
+          JSON.generate(for_job(job, tracked: false))
+        end
+
+        # Generates JSON payload for a tracked job.
+        #
+        # @param job [ActiveJob::Base] The ActiveJob instance
+        # @param postburner_job_id [Integer] Postburner::Job ID
+        #
+        # @return [String] JSON-encoded payload
+        #
+        def tracked_payload(job, postburner_job_id)
+          JSON.generate(for_job(job, tracked: true, postburner_job_id: postburner_job_id))
+        end
+
+        # Serializes ActiveJob data for PostgreSQL storage (tracked jobs).
+        #
+        # Returns the same structure as for_job but as a plain Hash
+        # (not JSON string) for storage in Postburner::Job args column.
+        #
+        # @param job [ActiveJob::Base] The ActiveJob instance
+        #
+        # @return [Hash] Payload hash for PostgreSQL storage
+        #
+        def serialize_for_tracked(job)
+          for_job(job, tracked: true).stringify_keys
+        end
+
+        # Parses and validates a JSON payload from Beanstalkd.
+        #
+        # Handles version checking and returns the parsed data.
+        #
+        # @param json_string [String] JSON payload from Beanstalkd
+        #
+        # @return [Hash] Parsed payload with string keys
+        #
+        # @raise [ArgumentError] if payload version is unknown
+        #
+        def parse(json_string)
+          data = JSON.parse(json_string)
+
+          # Handle version
+          version = data['v'] || data[:v] || 1
+
+          case version
+          when 1
+            data.is_a?(Hash) ? data.stringify_keys : data
+          else
+            raise ArgumentError, "Unknown payload version: #{version}"
+          end
+        end
+
+        # Detects if a parsed payload is for a tracked job.
+        #
+        # @param payload [Hash] Parsed payload
+        #
+        # @return [Boolean] true if tracked, false otherwise
+        #
+        def tracked?(payload)
+          payload['tracked'] == true || payload[:tracked] == true
+        end
+
+        # Detects if a parsed payload is for a legacy Postburner::Job.
+        #
+        # Legacy format: { "class" => "JobClassName", "args" => [job_id] }
+        #
+        # @param payload [Hash] Parsed payload
+        #
+        # @return [Boolean] true if legacy format
+        #
+        def legacy_format?(payload)
+          payload.key?('class') && payload.key?('args') && !payload.key?('v')
+        end
+      end
+    end
+  end
+end

data/lib/postburner/beanstalkd.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module Postburner
+  # Beanstalkd-specific configuration DSL for ActiveJob classes using Postburner.
+  #
+  # Provides consistent API with Postburner::Job for setting Beanstalkd queue
+  # priority and TTR (time-to-run). Include this module in your ActiveJob classes
+  # to use Beanstalkd-specific configuration.
+  #
+  # @note Automatically included when you include Postburner::Tracked
+  #
+  # @example Default job with Beanstalkd config
+  #   class ProcessPayment < ApplicationJob
+  #     include Postburner::Beanstalkd
+  #
+  #     queue_as :critical
+  #     queue_priority 0      # Highest priority
+  #     queue_ttr 300         # 5 minutes to complete
+  #
+  #     def perform(payment_id)
+  #       # ...
+  #     end
+  #   end
+  #
+  # @example With tracking (Beanstalkd automatically included)
+  #   class ProcessPayment < ApplicationJob
+  #     include Postburner::Tracked  # Includes Beanstalkd automatically
+  #
+  #     queue_priority 0
+  #     queue_ttr 600
+  #
+  #     def perform(payment_id)
+  #       log "Processing payment"
+  #       # ...
+  #     end
+  #   end
+  #
+  module Beanstalkd
+      extend ActiveSupport::Concern
+
+      included do
+        class_attribute :postburner_priority, default: nil
+        class_attribute :postburner_ttr, default: nil
+      end
+
+      class_methods do
+        # Sets or returns the queue priority.
+        #
+        # Lower numbers = higher priority in Beanstalkd (0 is highest).
+        # Falls back to Postburner.configuration.default_priority if not set.
+        #
+        # @param pri [Integer, nil] Priority to set (0-4294967295), or nil to get current value
+        #
+        # @return [Integer, nil] Current priority when getting, nil when setting
+        #
+        # @example Set priority
+        #   queue_priority 0  # Highest priority
+        #
+        # @example Get priority
+        #   ProcessPayment.queue_priority  # => 0
+        #
+        def queue_priority(pri = nil)
+          if pri
+            self.postburner_priority = pri
+            nil
+          else
+            postburner_priority
+          end
+        end
+
+        # Sets or returns the queue TTR (time to run).
+        #
+        # Number of seconds Beanstalkd will wait for job completion before
+        # making it available again. Falls back to Postburner.configuration.default_ttr
+        # if not set.
+        #
+        # @param ttr [Integer, nil] Timeout in seconds, or nil to get current value
+        #
+        # @return [Integer, nil] Current TTR when getting, nil when setting
+        #
+        # @example Set TTR
+        #   queue_ttr 300  # 5 minutes
+        #
+        # @example Get TTR
+        #   ProcessPayment.queue_ttr  # => 300
+        #
+        def queue_ttr(ttr = nil)
+          if ttr
+            self.postburner_ttr = ttr
+            nil
+          else
+            postburner_ttr
+          end
+        end
+      end
+  end
+end