postburner 0.9.0.rc.1 → 1.0.0.pre.2

data/lib/postburner/workers/base.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+module Postburner
+  module Workers
+    # Base worker class with shared functionality for all worker types.
+    #
+    # Provides common methods for signal handling, job execution, error handling,
+    # and retry logic. Subclasses implement the specific execution strategy
+    # (simple, forking, threads_on_fork).
+    #
+    class Base
+      attr_reader :config, :logger
+
+      # @param config [Postburner::Configuration] Worker configuration
+      #
+      def initialize(config)
+        @config = config
+        @logger = config.logger
+        @shutdown = false
+        setup_signal_handlers
+      end
+
+      # Starts the worker loop.
+      #
+      # Subclasses must implement this method to define their execution strategy.
+      #
+      # @return [void]
+      #
+      # @raise [NotImplementedError] if not implemented by subclass
+      #
+      def start
+        raise NotImplementedError, "Subclasses must implement #start"
+      end
+
+      # Initiates graceful shutdown.
+      #
+      # Sets shutdown flag to stop processing new jobs. Current jobs
+      # are allowed to finish.
+      #
+      # @return [void]
+      #
+      def shutdown
+        @shutdown = true
+      end
+
+      # Checks if shutdown has been requested.
+      #
+      # @return [Boolean] true if shutdown requested, false otherwise
+      #
+      def shutdown?
+        @shutdown
+      end
+
+      protected
+
+      # Sets up signal handlers for graceful shutdown.
+      #
+      # TERM and INT signals trigger graceful shutdown.
+      #
+      # @return [void]
+      #
+      def setup_signal_handlers
+        Signal.trap('TERM') { shutdown }
+        Signal.trap('INT') { shutdown }
+      end
+
+      # Expands queue name to full tube name with environment prefix.
+      #
+      # Delegates to Postburner::Configuration#expand_tube_name.
+      #
+      # @param queue_name [String] Base queue name
+      #
+      # @return [String] Full tube name (e.g., 'postburner.production.critical')
+      #
+      def expand_tube_name(queue_name)
+        config.expand_tube_name(queue_name)
+      end
+
+      # Executes a job from Beanstalkd.
+      #
+      # Delegates to Postburner::ActiveJob::Execution to handle default,
+      # tracked, and legacy job formats.
+      #
+      # @param beanstalk_job [Beaneater::Job] Job from Beanstalkd
+      #
+      # @return [void]
+      #
+      def execute_job(beanstalk_job)
+        logger.info "[Postburner] Executing #{beanstalk_job.class.name} #{beanstalk_job.id}"
+        Postburner::ActiveJob::Execution.execute(beanstalk_job.body)
+        logger.info "[Postburner] Deleting #{beanstalk_job.class.name} #{beanstalk_job.id} (success)"
+        beanstalk_job.delete
+      rescue => e
+        handle_error(beanstalk_job, e)
+      end
+
+      # Handles job execution errors with retry logic.
+      #
+      # Implements retry strategy:
+      # - Parses payload to determine job type
+      # - For default jobs: manages retry count, re-queues with backoff
+      # - For tracked jobs: buries job (Postburner::Job handles retries)
+      # - For legacy jobs: buries job
+      #
+      # @param beanstalk_job [Beaneater::Job] Job from Beanstalkd
+      # @param error [Exception] The exception that was raised
+      #
+      # @return [void]
+      #
+      def handle_error(beanstalk_job, error)
+        logger.error "[Postburner] Job failed: #{error.class} - #{error.message}"
+        logger.error error.backtrace.join("\n")
+
+        begin
+          payload = JSON.parse(beanstalk_job.body)
+
+          if payload['tracked'] || Postburner::ActiveJob::Payload.legacy_format?(payload)
+            # Tracked and legacy jobs: bury for inspection
+            # (Postburner::Job has its own retry logic)
+            logger.info "[Postburner] Burying tracked/legacy job for inspection"
+            beanstalk_job.bury
+          else
+            # Default job: handle retry logic
+            handle_default_retry(beanstalk_job, payload, error)
+          end
+        rescue => retry_error
+          logger.error "[Postburner] Error handling failure: #{retry_error.message}"
+          beanstalk_job.bury rescue nil
+        end
+      end
+
+      # Handles retry logic for default jobs.
+      #
+      # Checks ActiveJob's retry_on configuration, increments retry count,
+      # and re-queues with exponential backoff if retries remaining.
+      #
+      # @param beanstalk_job [Beaneater::Job] Job from Beanstalkd
+      # @param payload [Hash] Parsed job payload
+      # @param error [Exception] The exception that was raised
+      #
+      # @return [void]
+      #
+      def handle_default_retry(beanstalk_job, payload, error)
+        retry_count = payload['retry_count'] || 0
+        job_class = payload['job_class'].constantize
+
+        # Check if job class wants to retry this error
+        # (This is simplified - full implementation would check retry_on config)
+        max_retries = 5  # Default max retries
+
+        if retry_count < max_retries
+          # Increment retry count
+          payload['retry_count'] = retry_count + 1
+          payload['executions'] = (payload['executions'] || 0) + 1
+
+          # Calculate backoff delay (exponential: 1s, 2s, 4s, 8s, 16s...)
+          delay = calculate_backoff(retry_count)
+
+          # Delete old job and insert new one with updated payload
+          beanstalk_job.delete
+
+          Postburner.connected do |conn|
+            tube_name = expand_tube_name(payload['queue_name'])
+            conn.tubes[tube_name].put(
+              JSON.generate(payload),
+              pri: payload['priority'] || 0,
+              delay: delay,
+              ttr: 120
+            )
+          end
+
+          logger.info "[Postburner] Retrying default job #{payload['job_id']}, attempt #{retry_count + 1} in #{delay}s"
+        else
+          # Max retries exceeded
+          logger.error "[Postburner] Discarding default job #{payload['job_id']} after #{retry_count} retries"
+          beanstalk_job.delete
+          # TODO: Call after_discard callback if configured
+        end
+      end
+
+      # Calculates exponential backoff delay for retries.
+      #
+      # @param retry_count [Integer] Number of retries so far
+      #
+      # @return [Integer] Delay in seconds (capped at 1 hour)
+      #
+      def calculate_backoff(retry_count)
+        # Exponential backoff: 2^retry_count, capped at 3600 seconds (1 hour)
+        [2 ** retry_count, 3600].min
+      end
+
+      # Watches the configured queues in Beanstalkd.
+      #
+      # @param connection [Postburner::Connection] Beanstalkd connection
+      # @param queue_name [String, nil] Optional specific queue name to watch (watches all if nil)
+      #
+      # @return [void]
+      #
+      def watch_queues(connection, queue_name = nil)
+        if queue_name
+          tube_name = config.expand_tube_name(queue_name)
+          connection.beanstalk.tubes.watch!(tube_name)
+        else
+          tube_names = config.expanded_tube_names
+          connection.beanstalk.tubes.watch!(*tube_names)
+        end
+      end
+    end
+  end
+end

data/lib/postburner/workers/worker.rb

@@ -0,0 +1,480 @@
+# frozen_string_literal: true
+
+require 'concurrent'
+
+module Postburner
+  module Workers
+    # Puma-style worker with configurable forks and threads per queue.
+    #
+    # This is the universal Postburner worker that scales from development
+    # to production using forks and threads configuration. Just like Puma:
+    # - **0 forks** = Single process with thread pool
+    # - **1+ forks** = Multiple processes (forks) with thread pools
+    #
+    # ## Architecture
+    #
+    # ### Single Process Mode (forks: 0)
+    # ```
+    # Main Process
+    # ├─ Queue 'default' Thread Pool (10 threads)
+    # ├─ Queue 'critical' Thread Pool (1 thread)
+    # └─ Queue 'mailers' Thread Pool (5 threads)
+    # ```
+    #
+    # ### Multi-Process Mode (forks: 1+)
+    # ```
+    # Parent Process
+    # ├─ Fork 0 (queue: critical)
+    # │   └─ Thread 1
+    # ├─ Fork 0 (queue: default)
+    # │   ├─ Thread 1-10
+    # ├─ Fork 1 (queue: default)  # Puma-style: multiple forks of same queue
+    # │   ├─ Thread 1-10
+    # ├─ Fork 2 (queue: default)
+    # │   └─ Thread 1-10
+    # ├─ Fork 3 (queue: default)
+    # │   └─ Thread 1-10
+    # │   └─ Total: 40 concurrent jobs for 'default' (4 forks × 10 threads)
+    # └─ Fork 0 (queue: mailers)
+    #     └─ Thread 1-5
+    # ```
+    #
+    # ## Scaling Strategy
+    #
+    # **Development:**
+    # ```yaml
+    # default_forks: 0
+    # default_threads: 1
+    # ```
+    # Single-threaded, single-process (simplest debugging)
+    #
+    # **Staging:**
+    # ```yaml
+    # default_forks: 0
+    # default_threads: 10
+    # ```
+    # Multi-threaded, single-process (moderate concurrency)
+    #
+    # **Production:**
+    # ```yaml
+    # default_forks: 4
+    # default_threads: 10
+    # ```
+    # 4 processes × 10 threads = 40 concurrent jobs per queue
+    #
+    # ## Configuration
+    #
+    # @example Development (single-threaded)
+    #   development:
+    #     default_forks: 0
+    #     default_threads: 1
+    #     queues:
+    #       default: {}
+    #       mailers: {}
+    #
+    # @example Staging (multi-threaded, single process)
+    #   staging:
+    #     default_forks: 0
+    #     default_threads: 10
+    #     default_gc_limit: 5000
+    #     queues:
+    #       critical:
+    #         threads: 1      # Single-threaded for critical jobs
+    #       default:
+    #         threads: 10     # 10 concurrent threads
+    #       mailers:
+    #         threads: 5      # 5 concurrent emails
+    #
+    # @example Production (Puma-style: forks × threads)
+    #   production:
+    #     default_forks: 2
+    #     default_threads: 10
+    #     default_gc_limit: 5000
+    #     queues:
+    #       critical:
+    #         forks: 1        # Single fork
+    #         threads: 1      # Single thread = 1 concurrent job
+    #       default:
+    #         forks: 4        # 4 forks (Puma-style)
+    #         threads: 10     # 10 threads per fork = 40 total concurrent jobs
+    #       mailers:
+    #         forks: 2
+    #         threads: 5      # 2×5 = 10 total email senders
+    #
+    class Worker < Base
+      # Starts the worker.
+      #
+      # Detects whether to run in single-process mode (forks: 0) or
+      # multi-process mode (forks: 1+) and starts accordingly.
+      #
+      # @return [void]
+      #
+      def start
+        logger.info "[Postburner::Worker] Starting..."
+        logger.info "[Postburner::Worker] Queues: #{config.queue_names.join(', ')}"
+        logger.info "[Postburner] #{config.beanstalk_url} watching tubes: #{config.expanded_tube_names.join(', ')}"
+
+        # Detect mode based on fork configuration
+        if using_forks?
+          start_forked_mode
+        else
+          start_single_process_mode
+        end
+      end
+
+      private
+
+      # Checks if any queue is configured to use forks.
+      #
+      # @return [Boolean] true if any queue has forks > 0
+      #
+      def using_forks?
+        config.queue_names.any? do |queue_name|
+          queue_config = config.queue_config(queue_name)
+          fork_count = queue_config['forks'] || queue_config[:forks] || config.default_forks
+          fork_count > 0
+        end
+      end
+
+      # Starts worker in single-process mode (forks: 0).
+      #
+      # Creates thread pools for each queue in the main process.
+      # Suitable for development and moderate concurrency needs.
+      #
+      # @return [void]
+      #
+      def start_single_process_mode
+        logger.info "[Postburner::Worker] Mode: Single process (forks: 0)"
+
+        # Track total jobs processed across all threads
+        @jobs_processed = Concurrent::AtomicFixnum.new(0)
+        @gc_limit = config.default_gc_limit
+
+        # Create thread pools for each queue
+        @pools = {}
+        config.queue_names.each do |queue_name|
+          spawn_queue_threads(queue_name)
+        end
+
+        # Monitor for shutdown or GC limit
+        until shutdown? || (@gc_limit && @jobs_processed.value >= @gc_limit)
+          sleep 0.5
+        end
+
+        # Shutdown pools gracefully
+        logger.info "[Postburner::Worker] Shutting down..."
+        shutdown_pools
+
+        if @gc_limit && @jobs_processed.value >= @gc_limit
+          logger.info "[Postburner::Worker] Reached GC limit (#{@jobs_processed.value} jobs), exiting for restart..."
+          exit 99  # Special exit code for GC restart
+        else
+          logger.info "[Postburner::Worker] Shutdown complete"
+        end
+      end
+
+      # Spawns thread pool for a specific queue in single-process mode.
+      #
+      # @param queue_name [String] Name of the queue to process
+      #
+      # @return [void]
+      #
+      def spawn_queue_threads(queue_name)
+        queue_config = config.queue_config(queue_name)
+        thread_count = queue_config['threads'] || queue_config[:threads] || config.default_threads
+
+        logger.info "[Postburner::Worker] Queue '#{queue_name}': #{thread_count} threads"
+
+        # Create thread pool
+        pool = Concurrent::FixedThreadPool.new(thread_count)
+        @pools[queue_name] = pool
+
+        # Spawn worker threads
+        thread_count.times do
+          pool.post do
+            process_jobs_in_single_process(queue_name)
+          end
+        end
+      end
+
+      # Processes jobs in a single thread (single-process mode).
+      #
+      # Each thread has its own Beanstalkd connection and reserves jobs
+      # from the specified queue.
+      #
+      # @param queue_name [String] Name of the queue to process
+      #
+      # @return [void]
+      #
+      def process_jobs_in_single_process(queue_name)
+        connection = Postburner::Connection.new
+
+        # Watch only this queue
+        watch_queues(connection, queue_name)
+
+        until shutdown? || (@gc_limit && @jobs_processed.value >= @gc_limit)
+          begin
+            # Reserve with short timeout
+            job = connection.beanstalk.tubes.reserve(timeout: 1)
+
+            if job
+              logger.debug "[Postburner::Worker] Thread #{Thread.current.object_id} reserved job #{job.id}"
+              execute_job(job)
+              @jobs_processed.increment
+            end
+          rescue Beaneater::TimedOutError
+            # Normal timeout, continue
+            next
+          rescue Beaneater::NotConnected => e
+            logger.error "[Postburner::Worker] Thread disconnected: #{e.message}"
+            sleep 1
+            connection.reconnect!
+          rescue => e
+            logger.error "[Postburner::Worker] Thread error: #{e.class} - #{e.message}"
+            logger.error e.backtrace.join("\n")
+            sleep 1
+          end
+        end
+      ensure
+        connection&.close rescue nil
+      end
+
+      # Gracefully shuts down all thread pools (single-process mode).
+      #
+      # @return [void]
+      #
+      def shutdown_pools
+        @pools.each do |queue_name, pool|
+          pool.shutdown
+          pool.wait_for_termination(30)
+          logger.info "[Postburner::Worker] Queue '#{queue_name}' shutdown complete"
+        end
+      end
+
+      # Starts worker in forked mode (forks: 1+).
+      #
+      # Forks multiple child processes for each queue, each running
+      # a thread pool. Parent process monitors children and restarts them when they exit.
+      #
+      # @return [void]
+      #
+      def start_forked_mode
+        logger.info "[Postburner::Worker] Mode: Multi-process (forks: 1+)"
+
+        # Track children: { pid => { queue: 'name', fork_num: 0 } }
+        @children = {}
+
+        # Spawn configured number of forks for each queue
+        config.queue_names.each do |queue_name|
+          spawn_queue_workers(queue_name)
+        end
+
+        # Parent process monitors children
+        until shutdown?
+          begin
+            pid, status = Process.wait2(-1, Process::WNOHANG)
+
+            if pid
+              child_info = @children.delete(pid)
+              exit_code = status.exitstatus
+              queue_name = child_info[:queue]
+              fork_num = child_info[:fork_num]
+
+              if exit_code == 99
+                # GC restart - this is normal
+                logger.info "[Postburner::Worker] Queue '#{queue_name}' fork #{fork_num} reached GC limit, restarting..."
+                spawn_queue_worker(queue_name, fork_num) unless shutdown?
+              else
+                logger.error "[Postburner::Worker] Queue '#{queue_name}' fork #{fork_num} exited unexpectedly (code: #{exit_code})"
+                spawn_queue_worker(queue_name, fork_num) unless shutdown?
+              end
+            end
+
+            sleep 0.5
+          rescue Errno::ECHILD
+            # No children left
+            break
+          rescue => e
+            logger.error "[Postburner::Worker] Monitor error: #{e.message}"
+            sleep 1
+          end
+        end
+
+        # Shutdown - wait for all children
+        logger.info "[Postburner::Worker] Shutting down, waiting for children..."
+        shutdown_children
+        logger.info "[Postburner::Worker] Shutdown complete"
+      end
+
+      # Spawns all forked worker processes for a specific queue.
+      #
+      # @param queue_name [String] Name of the queue to process
+      #
+      # @return [void]
+      #
+      def spawn_queue_workers(queue_name)
+        queue_config = config.queue_config(queue_name)
+        fork_count = queue_config['forks'] || queue_config[:forks] || config.default_forks
+        thread_count = queue_config['threads'] || queue_config[:threads] || config.default_threads
+
+        # Skip if this queue has 0 forks (shouldn't happen in forked mode, but be defensive)
+        return if fork_count == 0
+
+        total_concurrency = fork_count * thread_count
+        logger.info "[Postburner::Worker] Queue '#{queue_name}': #{fork_count} forks × #{thread_count} threads = #{total_concurrency} total concurrency"
+
+        fork_count.times do |fork_num|
+          spawn_queue_worker(queue_name, fork_num)
+        end
+      end
+
+      # Spawns a single forked worker process for a specific queue.
+      #
+      # @param queue_name [String] Name of the queue to process
+      # @param fork_num [Integer] Fork number (0-indexed)
+      #
+      # @return [void]
+      #
+      def spawn_queue_worker(queue_name, fork_num)
+        pid = fork do
+          # Child process
+          run_queue_worker(queue_name, fork_num)
+        end
+
+        @children[pid] = { queue: queue_name, fork_num: fork_num }
+        logger.info "[Postburner::Worker] Spawned worker for queue '#{queue_name}' fork #{fork_num} (pid: #{pid})"
+      end
+
+      # Runs the thread pool worker for a specific queue fork.
+      #
+      # This runs in the child process. Creates a thread pool and processes
+      # jobs until GC limit is reached or shutdown is requested.
+      #
+      # @param queue_name [String] Name of the queue to process
+      # @param fork_num [Integer] Fork number (for logging)
+      #
+      # @return [void]
+      #
+      def run_queue_worker(queue_name, fork_num)
+        queue_config = config.queue_config(queue_name)
+        thread_count = queue_config['threads'] || queue_config[:threads] || config.default_threads
+        gc_limit = queue_config['gc_limit'] || queue_config[:gc_limit] || config.default_gc_limit
+
+        logger.info "[Postburner::Worker] Queue '#{queue_name}' fork #{fork_num}: #{thread_count} threads, GC limit #{gc_limit || 'unlimited'}"
+
+        # Track jobs processed in this fork
+        jobs_processed = Concurrent::AtomicFixnum.new(0)
+
+        # Create thread pool
+        pool = Concurrent::FixedThreadPool.new(thread_count)
+
+        # Each thread needs its own Beanstalkd connection
+        thread_count.times do
+          pool.post do
+            process_jobs_in_fork(queue_name, fork_num, jobs_processed, gc_limit)
+          end
+        end
+
+        # Wait for shutdown or GC limit
+        until shutdown? || (gc_limit && jobs_processed.value >= gc_limit)
+          sleep 0.5
+        end
+
+        # Shutdown pool gracefully
+        pool.shutdown
+        pool.wait_for_termination(30)
+
+        if gc_limit && jobs_processed.value >= gc_limit
+          logger.info "[Postburner::Worker] Queue '#{queue_name}' fork #{fork_num} reached GC limit (#{jobs_processed.value} jobs), exiting for restart..."
+          exit 99  # Special exit code for GC restart
+        else
+          logger.info "[Postburner::Worker] Queue '#{queue_name}' fork #{fork_num} shutting down gracefully..."
+          exit 0
+        end
+      rescue => e
+        logger.error "[Postburner::Worker] Queue '#{queue_name}' fork #{fork_num} error: #{e.message}"
+        logger.error e.backtrace.join("\n")
+        exit 1
+      end
+
+      # Processes jobs in a single thread within a fork (forked mode).
+      #
+      # Each thread has its own Beanstalkd connection and reserves jobs
+      # from the specified queue.
+      #
+      # @param queue_name [String] Name of the queue to process
+      # @param fork_num [Integer] Fork number (for logging)
+      # @param jobs_processed [Concurrent::AtomicFixnum] Shared counter of jobs processed
+      # @param gc_limit [Integer, nil] Maximum jobs before triggering GC restart (nil = unlimited)
+      #
+      # @return [void]
+      #
+      def process_jobs_in_fork(queue_name, fork_num, jobs_processed, gc_limit)
+        connection = Postburner::Connection.new
+
+        # Watch only this queue
+        watch_queues(connection, queue_name)
+
+        until shutdown? || (gc_limit && jobs_processed.value >= gc_limit)
+          begin
+            # Reserve with short timeout
+            job = connection.beanstalk.tubes.reserve(timeout: 1)
+
+            if job
+              logger.debug "[Postburner::Worker] Queue '#{queue_name}' fork #{fork_num} thread #{Thread.current.object_id} reserved job #{job.id}"
+              execute_job(job)
+              jobs_processed.increment
+            end
+          rescue Beaneater::TimedOutError
+            # Normal timeout, continue
+            next
+          rescue Beaneater::NotConnected => e
+            logger.error "[Postburner::Worker] Thread disconnected: #{e.message}"
+            sleep 1
+            connection.reconnect!
+          rescue => e
+            logger.error "[Postburner::Worker] Thread error: #{e.class} - #{e.message}"
+            logger.error e.backtrace.join("\n")
+            sleep 1
+          end
+        end
+      ensure
+        connection&.close rescue nil
+      end
+
+      # Gracefully shuts down all child processes (forked mode).
+      #
+      # Sends TERM signal to all children and waits for them to exit.
+      #
+      # @return [void]
+      #
+      def shutdown_children
+        @children.keys.each do |pid|
+          begin
+            Process.kill('TERM', pid)
+          rescue Errno::ESRCH
+            # Process already exited
+          end
+        end
+
+        # Wait up to 30 seconds for children to exit
+        timeout = Time.now + 30
+        until @children.empty? || Time.now > timeout
+          pid, status = Process.wait2(-1, Process::WNOHANG)
+          @children.delete(pid) if pid
+          sleep 0.5
+        end
+
+        # Force kill any remaining children
+        @children.keys.each do |pid|
+          begin
+            Process.kill('KILL', pid)
+            Process.wait(pid)
+          rescue Errno::ESRCH
+            # Already exited
+          end
+        end
+      end
+    end
+  end
+end