postburner 0.8.0 → 1.0.0.pre.1

data/lib/postburner/workers/worker.rb

@@ -0,0 +1,480 @@
+# frozen_string_literal: true
+
+require 'concurrent'
+
+module Postburner
+  module Workers
+    # Puma-style worker with configurable forks and threads per queue.
+    #
+    # This is the universal Postburner worker that scales from development
+    # to production using forks and threads configuration. Just like Puma:
+    # - **0 forks** = Single process with thread pool
+    # - **1+ forks** = Multiple processes (forks) with thread pools
+    #
+    # ## Architecture
+    #
+    # ### Single Process Mode (forks: 0)
+    # ```
+    # Main Process
+    # ├─ Queue 'default' Thread Pool (10 threads)
+    # ├─ Queue 'critical' Thread Pool (1 thread)
+    # └─ Queue 'mailers' Thread Pool (5 threads)
+    # ```
+    #
+    # ### Multi-Process Mode (forks: 1+)
+    # ```
+    # Parent Process
+    # ├─ Fork 0 (queue: critical)
+    # │   └─ Thread 1
+    # ├─ Fork 0 (queue: default)
+    # │   ├─ Thread 1-10
+    # ├─ Fork 1 (queue: default)  # Puma-style: multiple forks of same queue
+    # │   ├─ Thread 1-10
+    # ├─ Fork 2 (queue: default)
+    # │   └─ Thread 1-10
+    # ├─ Fork 3 (queue: default)
+    # │   └─ Thread 1-10
+    # │   └─ Total: 40 concurrent jobs for 'default' (4 forks × 10 threads)
+    # └─ Fork 0 (queue: mailers)
+    #     └─ Thread 1-5
+    # ```
+    #
+    # ## Scaling Strategy
+    #
+    # **Development:**
+    # ```yaml
+    # default_forks: 0
+    # default_threads: 1
+    # ```
+    # Single-threaded, single-process (simplest debugging)
+    #
+    # **Staging:**
+    # ```yaml
+    # default_forks: 0
+    # default_threads: 10
+    # ```
+    # Multi-threaded, single-process (moderate concurrency)
+    #
+    # **Production:**
+    # ```yaml
+    # default_forks: 4
+    # default_threads: 10
+    # ```
+    # 4 processes × 10 threads = 40 concurrent jobs per queue
+    #
+    # ## Configuration
+    #
+    # @example Development (single-threaded)
+    #   development:
+    #     default_forks: 0
+    #     default_threads: 1
+    #     queues:
+    #       default: {}
+    #       mailers: {}
+    #
+    # @example Staging (multi-threaded, single process)
+    #   staging:
+    #     default_forks: 0
+    #     default_threads: 10
+    #     default_gc_limit: 5000
+    #     queues:
+    #       critical:
+    #         threads: 1      # Single-threaded for critical jobs
+    #       default:
+    #         threads: 10     # 10 concurrent threads
+    #       mailers:
+    #         threads: 5      # 5 concurrent emails
+    #
+    # @example Production (Puma-style: forks × threads)
+    #   production:
+    #     default_forks: 2
+    #     default_threads: 10
+    #     default_gc_limit: 5000
+    #     queues:
+    #       critical:
+    #         forks: 1        # Single fork
+    #         threads: 1      # Single thread = 1 concurrent job
+    #       default:
+    #         forks: 4        # 4 forks (Puma-style)
+    #         threads: 10     # 10 threads per fork = 40 total concurrent jobs
+    #       mailers:
+    #         forks: 2
+    #         threads: 5      # 2×5 = 10 total email senders
+    #
+    class Worker < Base
+      # Starts the worker.
+      #
+      # Detects whether to run in single-process mode (forks: 0) or
+      # multi-process mode (forks: 1+) and starts accordingly.
+      #
+      # @return [void]
+      #
+      def start
+        logger.info "[Postburner::Worker] Starting..."
+        logger.info "[Postburner::Worker] Queues: #{config.queue_names.join(', ')}"
+        logger.info "[Postburner] #{config.beanstalk_url} watching tubes: #{config.expanded_tube_names.join(', ')}"
+
+        # Detect mode based on fork configuration
+        if using_forks?
+          start_forked_mode
+        else
+          start_single_process_mode
+        end
+      end
+
+      private
+
+      # Checks if any queue is configured to use forks.
+      #
+      # @return [Boolean] true if any queue has forks > 0
+      #
+      def using_forks?
+        config.queue_names.any? do |queue_name|
+          queue_config = config.queue_config(queue_name)
+          fork_count = queue_config['forks'] || queue_config[:forks] || config.default_forks
+          fork_count > 0
+        end
+      end
+
+      # Starts worker in single-process mode (forks: 0).
+      #
+      # Creates thread pools for each queue in the main process.
+      # Suitable for development and moderate concurrency needs.
+      #
+      # @return [void]
+      #
+      def start_single_process_mode
+        logger.info "[Postburner::Worker] Mode: Single process (forks: 0)"
+
+        # Track total jobs processed across all threads
+        @jobs_processed = Concurrent::AtomicFixnum.new(0)
+        @gc_limit = config.default_gc_limit
+
+        # Create thread pools for each queue
+        @pools = {}
+        config.queue_names.each do |queue_name|
+          spawn_queue_threads(queue_name)
+        end
+
+        # Monitor for shutdown or GC limit
+        until shutdown? || (@gc_limit && @jobs_processed.value >= @gc_limit)
+          sleep 0.5
+        end
+
+        # Shutdown pools gracefully
+        logger.info "[Postburner::Worker] Shutting down..."
+        shutdown_pools
+
+        if @gc_limit && @jobs_processed.value >= @gc_limit
+          logger.info "[Postburner::Worker] Reached GC limit (#{@jobs_processed.value} jobs), exiting for restart..."
+          exit 99  # Special exit code for GC restart
+        else
+          logger.info "[Postburner::Worker] Shutdown complete"
+        end
+      end
+
+      # Spawns thread pool for a specific queue in single-process mode.
+      #
+      # @param queue_name [String] Name of the queue to process
+      #
+      # @return [void]
+      #
+      def spawn_queue_threads(queue_name)
+        queue_config = config.queue_config(queue_name)
+        thread_count = queue_config['threads'] || queue_config[:threads] || config.default_threads
+
+        logger.info "[Postburner::Worker] Queue '#{queue_name}': #{thread_count} threads"
+
+        # Create thread pool
+        pool = Concurrent::FixedThreadPool.new(thread_count)
+        @pools[queue_name] = pool
+
+        # Spawn worker threads
+        thread_count.times do
+          pool.post do
+            process_jobs_in_single_process(queue_name)
+          end
+        end
+      end
+
+      # Processes jobs in a single thread (single-process mode).
+      #
+      # Each thread has its own Beanstalkd connection and reserves jobs
+      # from the specified queue.
+      #
+      # @param queue_name [String] Name of the queue to process
+      #
+      # @return [void]
+      #
+      def process_jobs_in_single_process(queue_name)
+        connection = Postburner::Connection.new
+
+        # Watch only this queue
+        watch_queues(connection, queue_name)
+
+        until shutdown? || (@gc_limit && @jobs_processed.value >= @gc_limit)
+          begin
+            # Reserve with short timeout
+            job = connection.beanstalk.tubes.reserve(timeout: 1)
+
+            if job
+              logger.debug "[Postburner::Worker] Thread #{Thread.current.object_id} reserved job #{job.id}"
+              execute_job(job)
+              @jobs_processed.increment
+            end
+          rescue Beaneater::TimedOutError
+            # Normal timeout, continue
+            next
+          rescue Beaneater::NotConnected => e
+            logger.error "[Postburner::Worker] Thread disconnected: #{e.message}"
+            sleep 1
+            connection.reconnect!
+          rescue => e
+            logger.error "[Postburner::Worker] Thread error: #{e.class} - #{e.message}"
+            logger.error e.backtrace.join("\n")
+            sleep 1
+          end
+        end
+      ensure
+        connection&.close rescue nil
+      end
+
+      # Gracefully shuts down all thread pools (single-process mode).
+      #
+      # @return [void]
+      #
+      def shutdown_pools
+        @pools.each do |queue_name, pool|
+          pool.shutdown
+          pool.wait_for_termination(30)
+          logger.info "[Postburner::Worker] Queue '#{queue_name}' shutdown complete"
+        end
+      end
+
+      # Starts worker in forked mode (forks: 1+).
+      #
+      # Forks multiple child processes for each queue, each running
+      # a thread pool. Parent process monitors children and restarts them when they exit.
+      #
+      # @return [void]
+      #
+      def start_forked_mode
+        logger.info "[Postburner::Worker] Mode: Multi-process (forks: 1+)"
+
+        # Track children: { pid => { queue: 'name', fork_num: 0 } }
+        @children = {}
+
+        # Spawn configured number of forks for each queue
+        config.queue_names.each do |queue_name|
+          spawn_queue_workers(queue_name)
+        end
+
+        # Parent process monitors children
+        until shutdown?
+          begin
+            pid, status = Process.wait2(-1, Process::WNOHANG)
+
+            if pid
+              child_info = @children.delete(pid)
+              exit_code = status.exitstatus
+              queue_name = child_info[:queue]
+              fork_num = child_info[:fork_num]
+
+              if exit_code == 99
+                # GC restart - this is normal
+                logger.info "[Postburner::Worker] Queue '#{queue_name}' fork #{fork_num} reached GC limit, restarting..."
+                spawn_queue_worker(queue_name, fork_num) unless shutdown?
+              else
+                logger.error "[Postburner::Worker] Queue '#{queue_name}' fork #{fork_num} exited unexpectedly (code: #{exit_code})"
+                spawn_queue_worker(queue_name, fork_num) unless shutdown?
+              end
+            end
+
+            sleep 0.5
+          rescue Errno::ECHILD
+            # No children left
+            break
+          rescue => e
+            logger.error "[Postburner::Worker] Monitor error: #{e.message}"
+            sleep 1
+          end
+        end
+
+        # Shutdown - wait for all children
+        logger.info "[Postburner::Worker] Shutting down, waiting for children..."
+        shutdown_children
+        logger.info "[Postburner::Worker] Shutdown complete"
+      end
+
+      # Spawns all forked worker processes for a specific queue.
+      #
+      # @param queue_name [String] Name of the queue to process
+      #
+      # @return [void]
+      #
+      def spawn_queue_workers(queue_name)
+        queue_config = config.queue_config(queue_name)
+        fork_count = queue_config['forks'] || queue_config[:forks] || config.default_forks
+        thread_count = queue_config['threads'] || queue_config[:threads] || config.default_threads
+
+        # Skip if this queue has 0 forks (shouldn't happen in forked mode, but be defensive)
+        return if fork_count == 0
+
+        total_concurrency = fork_count * thread_count
+        logger.info "[Postburner::Worker] Queue '#{queue_name}': #{fork_count} forks × #{thread_count} threads = #{total_concurrency} total concurrency"
+
+        fork_count.times do |fork_num|
+          spawn_queue_worker(queue_name, fork_num)
+        end
+      end
+
+      # Spawns a single forked worker process for a specific queue.
+      #
+      # @param queue_name [String] Name of the queue to process
+      # @param fork_num [Integer] Fork number (0-indexed)
+      #
+      # @return [void]
+      #
+      def spawn_queue_worker(queue_name, fork_num)
+        pid = fork do
+          # Child process
+          run_queue_worker(queue_name, fork_num)
+        end
+
+        @children[pid] = { queue: queue_name, fork_num: fork_num }
+        logger.info "[Postburner::Worker] Spawned worker for queue '#{queue_name}' fork #{fork_num} (pid: #{pid})"
+      end
+
+      # Runs the thread pool worker for a specific queue fork.
+      #
+      # This runs in the child process. Creates a thread pool and processes
+      # jobs until GC limit is reached or shutdown is requested.
+      #
+      # @param queue_name [String] Name of the queue to process
+      # @param fork_num [Integer] Fork number (for logging)
+      #
+      # @return [void]
+      #
+      def run_queue_worker(queue_name, fork_num)
+        queue_config = config.queue_config(queue_name)
+        thread_count = queue_config['threads'] || queue_config[:threads] || config.default_threads
+        gc_limit = queue_config['gc_limit'] || queue_config[:gc_limit] || config.default_gc_limit
+
+        logger.info "[Postburner::Worker] Queue '#{queue_name}' fork #{fork_num}: #{thread_count} threads, GC limit #{gc_limit || 'unlimited'}"
+
+        # Track jobs processed in this fork
+        jobs_processed = Concurrent::AtomicFixnum.new(0)
+
+        # Create thread pool
+        pool = Concurrent::FixedThreadPool.new(thread_count)
+
+        # Each thread needs its own Beanstalkd connection
+        thread_count.times do
+          pool.post do
+            process_jobs_in_fork(queue_name, fork_num, jobs_processed, gc_limit)
+          end
+        end
+
+        # Wait for shutdown or GC limit
+        until shutdown? || (gc_limit && jobs_processed.value >= gc_limit)
+          sleep 0.5
+        end
+
+        # Shutdown pool gracefully
+        pool.shutdown
+        pool.wait_for_termination(30)
+
+        if gc_limit && jobs_processed.value >= gc_limit
+          logger.info "[Postburner::Worker] Queue '#{queue_name}' fork #{fork_num} reached GC limit (#{jobs_processed.value} jobs), exiting for restart..."
+          exit 99  # Special exit code for GC restart
+        else
+          logger.info "[Postburner::Worker] Queue '#{queue_name}' fork #{fork_num} shutting down gracefully..."
+          exit 0
+        end
+      rescue => e
+        logger.error "[Postburner::Worker] Queue '#{queue_name}' fork #{fork_num} error: #{e.message}"
+        logger.error e.backtrace.join("\n")
+        exit 1
+      end
+
+      # Processes jobs in a single thread within a fork (forked mode).
+      #
+      # Each thread has its own Beanstalkd connection and reserves jobs
+      # from the specified queue.
+      #
+      # @param queue_name [String] Name of the queue to process
+      # @param fork_num [Integer] Fork number (for logging)
+      # @param jobs_processed [Concurrent::AtomicFixnum] Shared counter of jobs processed
+      # @param gc_limit [Integer, nil] Maximum jobs before triggering GC restart (nil = unlimited)
+      #
+      # @return [void]
+      #
+      def process_jobs_in_fork(queue_name, fork_num, jobs_processed, gc_limit)
+        connection = Postburner::Connection.new
+
+        # Watch only this queue
+        watch_queues(connection, queue_name)
+
+        until shutdown? || (gc_limit && jobs_processed.value >= gc_limit)
+          begin
+            # Reserve with short timeout
+            job = connection.beanstalk.tubes.reserve(timeout: 1)
+
+            if job
+              logger.debug "[Postburner::Worker] Queue '#{queue_name}' fork #{fork_num} thread #{Thread.current.object_id} reserved job #{job.id}"
+              execute_job(job)
+              jobs_processed.increment
+            end
+          rescue Beaneater::TimedOutError
+            # Normal timeout, continue
+            next
+          rescue Beaneater::NotConnected => e
+            logger.error "[Postburner::Worker] Thread disconnected: #{e.message}"
+            sleep 1
+            connection.reconnect!
+          rescue => e
+            logger.error "[Postburner::Worker] Thread error: #{e.class} - #{e.message}"
+            logger.error e.backtrace.join("\n")
+            sleep 1
+          end
+        end
+      ensure
+        connection&.close rescue nil
+      end
+
+      # Gracefully shuts down all child processes (forked mode).
+      #
+      # Sends TERM signal to all children and waits for them to exit.
+      #
+      # @return [void]
+      #
+      def shutdown_children
+        @children.keys.each do |pid|
+          begin
+            Process.kill('TERM', pid)
+          rescue Errno::ESRCH
+            # Process already exited
+          end
+        end
+
+        # Wait up to 30 seconds for children to exit
+        timeout = Time.now + 30
+        until @children.empty? || Time.now > timeout
+          pid, status = Process.wait2(-1, Process::WNOHANG)
+          @children.delete(pid) if pid
+          sleep 0.5
+        end
+
+        # Force kill any remaining children
+        @children.keys.each do |pid|
+          begin
+            Process.kill('KILL', pid)
+            Process.wait(pid)
+          rescue Errno::ESRCH
+            # Already exited
+          end
+        end
+      end
+    end
+  end
+end