postburner 1.0.0.pre.11 → 1.0.0.pre.12

data/lib/postburner/workers/worker.rb

@@ -1,396 +0,0 @@
-# frozen_string_literal: true
-
-require 'concurrent'
-
-module Postburner
-  module Workers
-    # Puma-style worker with configurable forks and threads.
-    #
-    # This is the universal Postburner worker that scales from development
-    # to production using forks and threads configuration. Just like Puma:
-    # - **0 forks** = Single process with thread pool
-    # - **1+ forks** = Multiple processes (forks) with thread pools
-    #
-    # ## Architecture
-    #
-    # ### Single Process Mode (forks: 0)
-    # ```
-    # Main Process
-    # └─ Thread Pool (N threads watching all queues)
-    # ```
-    #
-    # ### Multi-Process Mode (forks: 1+)
-    # ```
-    # Parent Process
-    # ├─ Fork 0
-    # │   └─ Thread Pool (N threads watching all queues)
-    # ├─ Fork 1
-    # │   └─ Thread Pool (N threads watching all queues)
-    # └─ Fork 2
-    #     └─ Thread Pool (N threads watching all queues)
-    # ```
-    #
-    # ## Scaling Strategy
-    #
-    # **Development:**
-    # ```yaml
-    # forks: 0
-    # threads: 1
-    # ```
-    # Single-threaded, single-process (simplest debugging)
-    #
-    # **Staging:**
-    # ```yaml
-    # forks: 0
-    # threads: 10
-    # ```
-    # Multi-threaded, single-process (moderate concurrency)
-    #
-    # **Production:**
-    # ```yaml
-    # forks: 4
-    # threads: 10
-    # ```
-    # 4 processes × 10 threads = 40 concurrent jobs
-    #
-    # ## Configuration
-    #
-    # @example Development (single-threaded)
-    #   development:
-    #     workers:
-    #       default:
-    #         forks: 0
-    #         threads: 1
-    #         queues:
-    #           - default
-    #           - mailers
-    #
-    # @example Production (Puma-style: forks × threads)
-    #   production:
-    #     default_forks: 2
-    #     default_threads: 10
-    #     workers:
-    #       default:
-    #         forks: 4        # Overrides default_forks
-    #         threads: 10     # Overrides default_threads
-    #         queues:
-    #           - critical
-    #           - default
-    #           - mailers
-    #
-    class Worker < Base
-      # Starts the worker.
-      #
-      # Detects whether to run in single-process mode (forks: 0) or
-      # multi-process mode (forks: 1+) and starts accordingly.
-      #
-      # @return [void]
-      #
-      def start
-        logger.info "[Postburner::Worker] Starting worker '#{worker_config[:name]}'..."
-        logger.info "[Postburner::Worker] Queues: #{config.queue_names.join(', ')}"
-        logger.info "[Postburner::Worker] Config: #{worker_config[:forks]} forks, #{worker_config[:threads]} threads, gc_limit: #{worker_config[:gc_limit] || 'unlimited'}, timeout: #{worker_config[:timeout]}s"
-        logger.info "[Postburner] #{config.beanstalk_url} watching tubes: #{config.expanded_tube_names.join(', ')}"
-
-        # Detect mode based on fork configuration
-        if worker_config[:forks] > 0
-          start_forked_mode
-        else
-          start_single_process_mode
-        end
-      end
-
-      private
-
-      # Returns the worker configuration hash.
-      #
-      # @return [Hash] Worker config with :name, :queues, :forks, :threads, :gc_limit, :timeout
-      #
-      def worker_config
-        config.worker_config
-      end
-
-      # Starts worker in single-process mode (forks: 0).
-      #
-      # Creates a thread pool that watches all configured queues.
-      # Suitable for development and moderate concurrency needs.
-      #
-      # @return [void]
-      #
-      def start_single_process_mode
-        logger.info "[Postburner::Worker] Mode: Single process (forks: 0)"
-
-        # Track total jobs processed across all threads
-        @jobs_processed = Concurrent::AtomicFixnum.new(0)
-        @gc_limit = worker_config[:gc_limit]
-
-        # Create thread pool
-        thread_count = worker_config[:threads]
-        @pool = Concurrent::FixedThreadPool.new(thread_count)
-
-        # Spawn worker threads
-        thread_count.times do
-          @pool.post do
-            process_jobs
-          end
-        end
-
-        # Monitor for shutdown or GC limit
-        until shutdown? || (@gc_limit && @jobs_processed.value >= @gc_limit)
-          sleep 0.5
-        end
-
-        # Shutdown pool gracefully
-        logger.info "[Postburner::Worker] Shutting down..."
-        @pool.shutdown
-        @pool.wait_for_termination(30)
-
-        if @gc_limit && @jobs_processed.value >= @gc_limit
-          logger.info "[Postburner::Worker] Reached GC limit (#{@jobs_processed.value} jobs), exiting for restart..."
-          exit 99  # Special exit code for GC restart
-        else
-          logger.info "[Postburner::Worker] Shutdown complete"
-        end
-      end
-
-      # Processes jobs in a single thread.
-      #
-      # Each thread has its own Beanstalkd connection and reserves jobs
-      # from all configured queues.
-      #
-      # @return [void]
-      #
-      def process_jobs
-        connection = Postburner::Connection.new
-        timeout = worker_config[:timeout]
-
-        # Watch all configured queues
-        watch_queues(connection, config.queue_names)
-
-        until shutdown? || (@gc_limit && @jobs_processed.value >= @gc_limit)
-          begin
-            # Reserve with configured timeout
-            job = connection.beanstalk.tubes.reserve(timeout: timeout)
-
-            if job
-              logger.debug "[Postburner::Worker] Thread #{Thread.current.object_id} reserved job #{job.id}"
-              execute_job(job)
-              @jobs_processed.increment
-            end
-          rescue Beaneater::TimedOutError
-            # Normal timeout, continue
-            next
-          rescue Beaneater::NotConnected => e
-            logger.error "[Postburner::Worker] Thread disconnected: #{e.message}"
-            sleep 1
-            connection.reconnect!
-          rescue => e
-            logger.error "[Postburner::Worker] Thread error: #{e.class} - #{e.message}"
-            logger.error e.backtrace.join("\n")
-            sleep 1
-          end
-        end
-      ensure
-        connection&.close rescue nil
-      end
-
-      # Starts worker in forked mode (forks: 1+).
-      #
-      # Forks multiple child processes, each running a thread pool.
-      # Parent process monitors children and restarts them when they exit.
-      #
-      # @return [void]
-      #
-      def start_forked_mode
-        logger.info "[Postburner::Worker] Mode: Multi-process (#{worker_config[:forks]} forks)"
-
-        # Track children: { pid => fork_num }
-        @children = {}
-
-        # Spawn configured number of forks
-        worker_config[:forks].times do |fork_num|
-          spawn_fork(fork_num)
-        end
-
-        # Parent process monitors children
-        until shutdown?
-          begin
-            pid, status = Process.wait2(-1, Process::WNOHANG)
-
-            if pid
-              fork_num = @children.delete(pid)
-              exit_code = status.exitstatus
-
-              if exit_code == 99
-                # GC restart - this is normal
-                logger.info "[Postburner::Worker] Fork #{fork_num} reached GC limit, restarting..."
-                spawn_fork(fork_num) unless shutdown?
-              else
-                logger.error "[Postburner::Worker] Fork #{fork_num} exited unexpectedly (code: #{exit_code})"
-                spawn_fork(fork_num) unless shutdown?
-              end
-            end
-
-            sleep 0.5
-          rescue Errno::ECHILD
-            # No children left
-            break
-          rescue => e
-            logger.error "[Postburner::Worker] Monitor error: #{e.message}"
-            sleep 1
-          end
-        end
-
-        # Shutdown - wait for all children
-        logger.info "[Postburner::Worker] Shutting down, waiting for children..."
-        shutdown_children
-        logger.info "[Postburner::Worker] Shutdown complete"
-      end
-
-      # Spawns a single forked worker process.
-      #
-      # @param fork_num [Integer] Fork number (0-indexed)
-      #
-      # @return [void]
-      #
-      def spawn_fork(fork_num)
-        pid = fork do
-          # Child process
-          run_fork(fork_num)
-        end
-
-        @children[pid] = fork_num
-        logger.info "[Postburner::Worker] Spawned fork #{fork_num} (pid: #{pid})"
-      end
-
-      # Runs the thread pool worker in a forked process.
-      #
-      # This runs in the child process. Creates a thread pool and processes
-      # jobs until GC limit is reached or shutdown is requested.
-      #
-      # @param fork_num [Integer] Fork number (for logging)
-      #
-      # @return [void]
-      #
-      def run_fork(fork_num)
-        thread_count = worker_config[:threads]
-        gc_limit = worker_config[:gc_limit]
-
-        logger.info "[Postburner::Worker] Fork #{fork_num}: #{thread_count} threads, GC limit #{gc_limit || 'unlimited'}"
-
-        # Track jobs processed in this fork
-        jobs_processed = Concurrent::AtomicFixnum.new(0)
-
-        # Create thread pool
-        pool = Concurrent::FixedThreadPool.new(thread_count)
-
-        # Each thread needs its own Beanstalkd connection
-        thread_count.times do
-          pool.post do
-            process_jobs_in_fork(fork_num, jobs_processed, gc_limit)
-          end
-        end
-
-        # Wait for shutdown or GC limit
-        until shutdown? || (gc_limit && jobs_processed.value >= gc_limit)
-          sleep 0.5
-        end
-
-        # Shutdown pool gracefully
-        pool.shutdown
-        pool.wait_for_termination(30)
-
-        if gc_limit && jobs_processed.value >= gc_limit
-          logger.info "[Postburner::Worker] Fork #{fork_num} reached GC limit (#{jobs_processed.value} jobs), exiting for restart..."
-          exit 99  # Special exit code for GC restart
-        else
-          logger.info "[Postburner::Worker] Fork #{fork_num} shutting down gracefully..."
-          exit 0
-        end
-      rescue => e
-        logger.error "[Postburner::Worker] Fork #{fork_num} error: #{e.message}"
-        logger.error e.backtrace.join("\n")
-        exit 1
-      end
-
-      # Processes jobs in a single thread within a fork.
-      #
-      # Each thread has its own Beanstalkd connection and reserves jobs
-      # from all configured queues.
-      #
-      # @param fork_num [Integer] Fork number (for logging)
-      # @param jobs_processed [Concurrent::AtomicFixnum] Shared counter of jobs processed
-      # @param gc_limit [Integer, nil] Maximum jobs before triggering GC restart (nil = unlimited)
-      #
-      # @return [void]
-      #
-      def process_jobs_in_fork(fork_num, jobs_processed, gc_limit)
-        connection = Postburner::Connection.new
-        timeout = worker_config[:timeout]
-
-        # Watch all configured queues
-        watch_queues(connection, config.queue_names)
-
-        until shutdown? || (gc_limit && jobs_processed.value >= gc_limit)
-          begin
-            # Reserve with configured timeout
-            job = connection.beanstalk.tubes.reserve(timeout: timeout)
-
-            if job
-              logger.debug "[Postburner::Worker] Fork #{fork_num} thread #{Thread.current.object_id} reserved job #{job.id}"
-              execute_job(job)
-              jobs_processed.increment
-            end
-          rescue Beaneater::TimedOutError
-            # Normal timeout, continue
-            next
-          rescue Beaneater::NotConnected => e
-            logger.error "[Postburner::Worker] Thread disconnected: #{e.message}"
-            sleep 1
-            connection.reconnect!
-          rescue => e
-            logger.error "[Postburner::Worker] Thread error: #{e.class} - #{e.message}"
-            logger.error e.backtrace.join("\n")
-            sleep 1
-          end
-        end
-      ensure
-        connection&.close rescue nil
-      end
-
-      # Gracefully shuts down all child processes (forked mode).
-      #
-      # Sends TERM signal to all children and waits for them to exit.
-      #
-      # @return [void]
-      #
-      def shutdown_children
-        @children.keys.each do |pid|
-          begin
-            Process.kill('TERM', pid)
-          rescue Errno::ESRCH
-            # Process already exited
-          end
-        end
-
-        # Wait up to 30 seconds for children to exit
-        timeout = Time.zone.now + 30
-        until @children.empty? || Time.zone.now > timeout
-          pid, status = Process.wait2(-1, Process::WNOHANG)
-          @children.delete(pid) if pid
-          sleep 0.5
-        end
-
-        # Force kill any remaining children
-        @children.keys.each do |pid|
-          begin
-            Process.kill('KILL', pid)
-            Process.wait(pid)
-          rescue Errno::ESRCH
-            # Already exited
-          end
-        end
-      end
-    end
-  end
-end