postburner 1.0.0.pre.11 → 1.0.0.pre.12

data/lib/postburner/worker.rb

@@ -0,0 +1,684 @@
+# frozen_string_literal: true
+
+require 'concurrent'
+require 'postburner/scheduler'
+
+module Postburner
+  # Puma-style worker with configurable forks and threads.
+  #
+  # Processes jobs from Beanstalkd queues using a scalable architecture that
+  # adapts from development to production. Workers can run in single-process
+  # mode (forks: 0) or multi-process mode (forks: 1+), each with configurable
+  # thread pools.
+  #
+  # ## Architecture
+  #
+  # ### Single Process Mode (forks: 0)
+  #
+  #     Main Process
+  #     └─ Thread Pool (N threads watching all queues)
+  #
+  # ### Multi-Process Mode (forks: 1+)
+  #
+  #     Parent Process
+  #     ├─ Fork 0
+  #     │   └─ Thread Pool (N threads watching all queues)
+  #     ├─ Fork 1
+  #     │   └─ Thread Pool (N threads watching all queues)
+  #     └─ Fork 2
+  #         └─ Thread Pool (N threads watching all queues)
+  #
+  # ## Scaling Strategy
+  #
+  # **Development** (forks: 0, threads: 1):
+  # Single-threaded, single-process for simplest debugging.
+  #
+  # **Staging** (forks: 0, threads: 10):
+  # Multi-threaded, single-process for moderate concurrency.
+  #
+  # **Production** (forks: 4, threads: 10):
+  # 4 processes × 10 threads = 40 concurrent jobs.
+  #
+  # ## Connection Model
+  #
+  # Each worker thread maintains its own Beanstalkd connection for thread safety.
+  #
+  # ## GC Limit
+  #
+  # Workers support a GC limit (gc_limit) that causes the process to exit
+  # after processing N jobs. In multi-process mode, the parent automatically
+  # respawns exited children. This prevents memory bloat in long-running workers.
+  #
+  # ## Scheduler Integration
+  #
+  # Workers automatically watch the scheduler tube and process watchdog jobs.
+  # On reserve timeout, workers check if a watchdog exists and create one if
+  # missing, ensuring the scheduler continues running even if jobs are sparse.
+  #
+  # @example Starting a worker programmatically
+  #   config = Postburner::Configuration.load_yaml('config/postburner.yml', 'production', 'default')
+  #   worker = Postburner::Worker.new(config)
+  #   worker.start
+  #
+  # @example Configuration in postburner.yml
+  #   production:
+  #     workers:
+  #       default:
+  #         forks: 4
+  #         threads: 10
+  #         gc_limit: 5000
+  #         queues:
+  #           - default
+  #           - critical
+  #
+  # @see Postburner::Runner
+  # @see Postburner::Configuration
+  # @see Postburner::Scheduler
+  #
+  class Worker
+    # @!attribute [r] config
+    #   @return [Postburner::Configuration] The worker configuration
+    # @!attribute [r] logger
+    #   @return [Logger] The logger instance
+    attr_reader :config, :logger
+
+    # Initialize a new worker.
+    #
+    # @param config [Postburner::Configuration] Worker configuration loaded from YAML
+    #
+    # @example
+    #   config = Postburner::Configuration.load_yaml('config/postburner.yml', 'production', 'default')
+    #   worker = Postburner::Worker.new(config)
+    #
+    def initialize(config)
+      @config = config
+      @logger = config.logger
+      @shutdown = false
+      setup_signal_handlers
+    end
+
+    # Starts the worker.
+    #
+    # Detects whether to run in single-process mode (forks: 0) or
+    # multi-process mode (forks: 1+) and starts accordingly.
+    #
+    # @return [void]
+    def start
+      logger.info "[Postburner::Worker] Starting worker '#{worker_config[:name]}'..."
+      logger.info "[Postburner::Worker] Queues: #{config.queue_names.join(', ')}"
+      logger.info "[Postburner::Worker] Config: #{worker_config[:forks]} forks, #{worker_config[:threads]} threads, gc_limit: #{worker_config[:gc_limit] || 'unlimited'}, timeout: #{worker_config[:timeout]}s"
+      all_tubes = config.expanded_tube_names + [config.scheduler_tube_name]
+      logger.info "[Postburner] #{config.beanstalk_url} known tubes: #{all_tubes.join(', ')}"
+      log_next_scheduler_watchdog
+
+      if worker_config[:forks] > 0
+        start_forked_mode
+      else
+        start_single_process_mode
+      end
+    end
+
+    # Initiates graceful shutdown.
+    #
+    # Sets shutdown flag to stop processing new jobs. Current jobs
+    # are allowed to finish.
+    #
+    # @return [void]
+    def shutdown
+      @shutdown = true
+    end
+
+    # Checks if shutdown has been requested.
+    #
+    # @return [Boolean] true if shutdown requested, false otherwise
+    def shutdown?
+      @shutdown
+    end
+
+    private
+
+    # Returns the worker configuration hash.
+    #
+    # @return [Hash] Worker config with :name, :queues, :forks, :threads, :gc_limit, :timeout
+    # @api private
+    def worker_config
+      config.worker_config
+    end
+
+    # Sets up signal handlers for graceful shutdown.
+    #
+    # Traps TERM and INT signals to initiate graceful shutdown.
+    #
+    # @return [void]
+    # @api private
+    def setup_signal_handlers
+      Signal.trap('TERM') { shutdown }
+      Signal.trap('INT') { shutdown }
+    end
+
+    # Expands queue name to full tube name with environment prefix.
+    #
+    # @param queue_name [String] Base queue name
+    # @return [String] Full tube name (e.g., 'postburner.production.critical')
+    # @api private
+    def expand_tube_name(queue_name)
+      config.expand_tube_name(queue_name)
+    end
+
+    # Logs info about the next scheduler watchdog job.
+    #
+    # Peeks at the scheduler tube to find the next watchdog job and logs
+    # when it will run. Useful for debugging scheduler timing.
+    #
+    # @return [void]
+    # @api private
+    def log_next_scheduler_watchdog
+      tube_name = config.scheduler_tube_name
+
+      Postburner.connected do |conn|
+        tube = conn.beanstalk.tubes[tube_name]
+
+        # Check delayed first, then ready
+        job = tube.peek(:delayed) rescue nil
+        job ||= tube.peek(:ready) rescue nil
+
+        if job
+          stats = job.stats
+          time_left = stats.time_left.to_i
+          at = Time.current + time_left
+
+          logger.info "[Postburner::Worker] Next scheduler watchdog at #{at.iso8601} (job #{job.id}, time-left: #{time_left}s); config: #{config.default_scheduler_interval}s"
+        else
+          logger.info "[Postburner::Worker] Schedule watchdog not found (tube: #{tube_name})"
+        end
+      end
+    rescue => e
+      logger.warn "[Postburner::Scheduler] Failed to lookup next scheduler watchdog: #{e.message}"
+    end
+
+    # Starts worker in single-process mode (forks: 0).
+    #
+    # Creates a thread pool and processes jobs until shutdown or GC limit
+    # is reached. Exits with code 99 when GC limit is reached to signal
+    # the need for a restart.
+    #
+    # @return [void]
+    # @api private
+    def start_single_process_mode
+      logger.info "[Postburner::Worker] Mode: Single process (forks: 0)"
+
+      @jobs_processed = Concurrent::AtomicFixnum.new(0)
+      @gc_limit = worker_config[:gc_limit]
+
+      thread_count = worker_config[:threads]
+      @pool = Concurrent::FixedThreadPool.new(thread_count)
+
+      thread_count.times do
+        @pool.post { process_jobs }
+      end
+
+      until shutdown? || (@gc_limit && @jobs_processed.value >= @gc_limit)
+        sleep 0.5
+      end
+
+      logger.info "[Postburner::Worker] Shutting down..."
+      @pool.shutdown
+      @pool.wait_for_termination(worker_config[:shutdown_timeout])
+
+      if @gc_limit && @jobs_processed.value >= @gc_limit
+        logger.debug "[Postburner::Worker] Reached GC limit (#{@jobs_processed.value} jobs), exiting for restart..."
+        exit 99
+      else
+        logger.info "[Postburner::Worker] Shutdown complete"
+      end
+    end
+
+    # Processes jobs in a single thread.
+    #
+    # Reserves jobs from watched tubes and executes them. Handles connection
+    # errors with automatic reconnection. Checks for scheduler watchdog on
+    # timeout.
+    #
+    # @return [void]
+    # @api private
+    def process_jobs
+      connection = Postburner::Connection.new
+      timeout = worker_config[:timeout]
+
+      watch_queues(connection, config.queue_names)
+
+      until shutdown? || (@gc_limit && @jobs_processed.value >= @gc_limit)
+        begin
+          job = connection.beanstalk.tubes.reserve(timeout: timeout)
+
+          if job
+            logger.debug "[Postburner::Worker] Thread #{Thread.current.object_id} reserved job #{job.id}"
+            execute_job(job)
+            @jobs_processed.increment
+          else
+            ensure_scheduler_watchdog!(connection)
+          end
+        rescue Beaneater::TimedOutError
+          ensure_scheduler_watchdog!(connection)
+          next
+        rescue Beaneater::NotConnected => e
+          logger.error "[Postburner::Worker] Thread disconnected: #{e.message}"
+          sleep 1
+          connection.reconnect!
+          watch_queues(connection, config.queue_names)
+        rescue => e
+          logger.error "[Postburner::Worker] Thread error: #{e.class} - #{e.message}"
+          logger.error e.backtrace.join("\n")
+          sleep 1
+        end
+      end
+    ensure
+      connection&.close rescue nil
+    end
+
+    # Starts worker in forked mode (forks: 1+).
+    #
+    # Spawns child processes and monitors them for exit. Automatically
+    # respawns children that exit with code 99 (GC limit) or unexpectedly.
+    # Initiates graceful shutdown when shutdown flag is set.
+    #
+    # @return [void]
+    # @api private
+    def start_forked_mode
+      logger.info "[Postburner::Worker] Mode: Multi-process (#{worker_config[:forks]} forks)"
+
+      @children = {}
+
+      worker_config[:forks].times do |fork_num|
+        spawn_fork(fork_num)
+      end
+
+      until shutdown?
+        begin
+          pid, status = Process.wait2(-1, Process::WNOHANG)
+
+          if pid
+            fork_num = @children.delete(pid)
+            exit_code = status.exitstatus
+
+            if exit_code == 99
+              logger.debug "[Postburner::Worker] Fork #{fork_num} reached GC limit, restarting..."
+              spawn_fork(fork_num) unless shutdown?
+            else
+              logger.error "[Postburner::Worker] Fork #{fork_num} exited unexpectedly (code: #{exit_code})"
+              spawn_fork(fork_num) unless shutdown?
+            end
+          end
+
+          sleep 0.5
+        rescue Errno::ECHILD
+          break
+        rescue => e
+          logger.error "[Postburner::Worker] Monitor error: #{e.message}"
+          sleep 1
+        end
+      end
+
+      logger.info "[Postburner::Worker] Shutting down, waiting for children..."
+      shutdown_children
+      logger.info "[Postburner::Worker] Shutdown complete"
+    end
+
+    # Spawns a single forked worker process.
+    #
+    # @param fork_num [Integer] Fork identifier (0-based index)
+    # @return [void]
+    # @api private
+    def spawn_fork(fork_num)
+      pid = fork { run_fork(fork_num) }
+      @children[pid] = fork_num
+      logger.info "[Postburner::Worker] Spawned fork #{fork_num} (pid: #{pid})"
+    end
+
+    # Runs the thread pool worker in a forked process.
+    #
+    # Creates a thread pool within the forked process and processes jobs
+    # until shutdown or GC limit is reached. Exits with code 99 on GC limit,
+    # 0 on graceful shutdown, or 1 on error.
+    #
+    # @param fork_num [Integer] Fork identifier for logging
+    # @return [void]
+    # @api private
+    def run_fork(fork_num)
+      thread_count = worker_config[:threads]
+      gc_limit = worker_config[:gc_limit]
+
+      logger.info "[Postburner::Worker] Fork #{fork_num}: #{thread_count} threads, GC limit #{gc_limit || 'unlimited'}"
+
+      jobs_processed = Concurrent::AtomicFixnum.new(0)
+      pool = Concurrent::FixedThreadPool.new(thread_count)
+
+      thread_count.times do
+        pool.post { process_jobs_in_fork(fork_num, jobs_processed, gc_limit) }
+      end
+
+      until shutdown? || (gc_limit && jobs_processed.value >= gc_limit)
+        sleep 0.5
+      end
+
+      pool.shutdown
+      pool.wait_for_termination(worker_config[:shutdown_timeout])
+
+      if gc_limit && jobs_processed.value >= gc_limit
+        logger.debug "[Postburner::Worker] Fork #{fork_num} reached GC limit (#{jobs_processed.value} jobs), exiting for restart..."
+        exit 99
+      else
+        logger.info "[Postburner::Worker] Fork #{fork_num} shutting down gracefully..."
+        exit 0
+      end
+    rescue => e
+      logger.error "[Postburner::Worker] Fork #{fork_num} error: #{e.message}"
+      logger.error e.backtrace.join("\n")
+      exit 1
+    end
+
+    # Processes jobs in a single thread within a fork.
+    #
+    # Similar to {#process_jobs} but uses shared atomic counter for
+    # tracking jobs processed across all threads in the fork.
+    #
+    # @param fork_num [Integer] Fork identifier for logging
+    # @param jobs_processed [Concurrent::AtomicFixnum] Shared job counter
+    # @param gc_limit [Integer, nil] Maximum jobs before exit (nil = unlimited)
+    # @return [void]
+    # @api private
+    def process_jobs_in_fork(fork_num, jobs_processed, gc_limit)
+      connection = Postburner::Connection.new
+      timeout = worker_config[:timeout]
+
+      watch_queues(connection, config.queue_names)
+
+      until shutdown? || (gc_limit && jobs_processed.value >= gc_limit)
+        begin
+          job = connection.beanstalk.tubes.reserve(timeout: timeout)
+
+          if job
+            logger.debug "[Postburner::Worker] Fork #{fork_num} thread #{Thread.current.object_id} reserved job #{job.id}"
+            execute_job(job)
+            jobs_processed.increment
+          else
+            ensure_scheduler_watchdog!(connection)
+          end
+        rescue Beaneater::TimedOutError
+          ensure_scheduler_watchdog!(connection)
+          next
+        rescue Beaneater::NotConnected => e
+          logger.error "[Postburner::Worker] Thread disconnected: #{e.message}"
+          sleep 1
+          connection.reconnect!
+          watch_queues(connection, config.queue_names)
+        rescue => e
+          logger.error "[Postburner::Worker] Thread error: #{e.class} - #{e.message}"
+          logger.error e.backtrace.join("\n")
+          sleep 1
+        end
+      end
+    ensure
+      connection&.close rescue nil
+    end
+
+    # Gracefully shuts down all child processes.
+    #
+    # Sends TERM signal to all children and waits up to shutdown_timeout
+    # (defaults to default_ttr) for graceful shutdown. Force kills any
+    # remaining children with KILL. Jobs killed mid-execution will be
+    # re-released by Beanstalkd after their TTR expires.
+    #
+    # @return [void]
+    # @api private
+    def shutdown_children
+      @children.keys.each do |pid|
+        Process.kill('TERM', pid) rescue nil
+      end
+
+      timeout = Time.current + worker_config[:shutdown_timeout]
+      until @children.empty? || Time.current > timeout
+        pid, _status = Process.wait2(-1, Process::WNOHANG)
+        @children.delete(pid) if pid
+        sleep 0.5
+      end
+
+      @children.keys.each do |pid|
+        begin
+          Process.kill('KILL', pid)
+          Process.wait(pid)
+        rescue Errno::ESRCH
+          # Already exited
+        end
+      end
+    end
+
+    # Executes a job from Beanstalkd.
+    #
+    # Parses the job payload and dispatches to either scheduler watchdog
+    # execution or regular job execution based on payload type.
+    #
+    # @param beanstalk_job [Beaneater::Job] Reserved job from Beanstalkd
+    # @return [void]
+    # @api private
+    def execute_job(beanstalk_job)
+      payload = JSON.parse(beanstalk_job.body)
+
+      if payload['scheduler'] == true
+        execute_scheduler_job(beanstalk_job, payload)
+      else
+        execute_regular_job(beanstalk_job, payload)
+      end
+    rescue => e
+      handle_error(beanstalk_job, e)
+    end
+
+    # Execute a scheduler watchdog job.
+    #
+    # Instantiates the Scheduler and runs it to process due schedules.
+    # Deletes the job from Beanstalkd after completion.
+    #
+    # @param beanstalk_job [Beaneater::Job] Reserved scheduler job
+    # @param payload [Hash] Parsed job body with 'scheduler' and 'interval' keys
+    # @return [void]
+    # @see Postburner::Scheduler#perform
+    # @api private
+    def execute_scheduler_job(beanstalk_job, payload)
+      logger.info "[Postburner] Executing scheduler watchdog #{beanstalk_job.id}"
+
+      interval = payload['interval'] || 300
+      scheduler = Postburner::Scheduler.new(interval: interval, logger: logger)
+      scheduler.perform
+
+      logger.info "[Postburner] Completed scheduler watchdog #{beanstalk_job.id}"
+      delete_job!(beanstalk_job)
+    end
+
+    # Execute a regular job (ActiveJob or Postburner::Job).
+    #
+    # Delegates execution to {Postburner::ActiveJob::Execution} which handles
+    # both ActiveJob and legacy Postburner::Job formats. Deletes the job
+    # from Beanstalkd after successful completion.
+    #
+    # Instruments with ActiveSupport::Notifications:
+    # - perform_start.postburner: Before job execution
+    # - perform.postburner: Around job execution (includes duration)
+    #
+    # @param beanstalk_job [Beaneater::Job] Reserved job from Beanstalkd
+    # @param payload [Hash] Parsed job body
+    # @return [void]
+    # @see Postburner::ActiveJob::Execution.execute
+    # @api private
+    def execute_regular_job(beanstalk_job, payload)
+      job_description = format_job_description(payload)
+      instrument_payload = { payload: payload, beanstalk_job_id: beanstalk_job.id }
+
+      logger.info "[Postburner] Executing #{job_description} (bkid: #{beanstalk_job.id})"
+
+      ActiveSupport::Notifications.instrument('perform_start.postburner', instrument_payload)
+
+      ActiveSupport::Notifications.instrument('perform.postburner', instrument_payload) do
+        Postburner::ActiveJob::Execution.execute(beanstalk_job.body)
+      end
+
+      logger.info "[Postburner] Completed #{job_description} (bkid: #{beanstalk_job.id})"
+      delete_job!(beanstalk_job)
+    end
+
+    # Format job description for logging.
+    #
+    # For Postburner::Job subclasses: Shows class name and database ID (e.g., HeartbeatJob#123)
+    # For ActiveJob classes: Shows class name only (e.g., SomeActiveJob)
+    #
+    # @param payload [Hash] Parsed job body
+    # @return [String] Human-readable job description
+    # @api private
+    def format_job_description(payload)
+      if Postburner::ActiveJob::Payload.legacy_format?(payload)
+        # Legacy Postburner::Job format: { "class" => "HeartbeatJob", "args" => [123] }
+        job_class = payload['class'] || 'UnknownJob'
+        job_id = payload['args']&.first
+        job_id ? "#{job_class}##{job_id}" : job_class
+      else
+        # ActiveJob format: { "job_class" => "SomeActiveJob", ... }
+        payload['job_class'] || 'UnknownJob'
+      end
+    end
+
+    # Handles job execution errors with retry logic.
+    #
+    # For tracked jobs and legacy Postburner::Job: Buries the job for inspection.
+    # For default ActiveJob: Applies exponential backoff retry with max 5 attempts.
+    #
+    # @param beanstalk_job [Beaneater::Job] Failed job
+    # @param error [Exception] The error that caused the failure
+    # @return [void]
+    # @api private
+    def handle_error(beanstalk_job, error)
+      logger.error "[Postburner] Job failed: #{error.class} - #{error.message}"
+      logger.error error.backtrace.join("\n")
+
+      begin
+        payload = JSON.parse(beanstalk_job.body)
+
+        if payload['tracked'] || Postburner::ActiveJob::Payload.legacy_format?(payload)
+          logger.info "[Postburner] Burying tracked/legacy job for inspection"
+          beanstalk_job.bury
+        else
+          handle_default_retry(beanstalk_job, payload, error)
+        end
+      rescue => retry_error
+        logger.error "[Postburner] Error handling failure: #{retry_error.message}"
+        beanstalk_job.bury rescue nil
+      end
+    end
+
+    # Handles retry logic for default jobs.
+    #
+    # Applies exponential backoff (2^retry_count seconds, max 1 hour).
+    # After 5 failed attempts, discards the job permanently.
+    #
+    # Instruments with ActiveSupport::Notifications:
+    # - retry.postburner: When job is retried
+    # - discard.postburner: When job is discarded after max retries
+    #
+    # @param beanstalk_job [Beaneater::Job] Failed job to retry
+    # @param payload [Hash] Parsed job body (modified with retry_count)
+    # @param error [Exception] The error that caused the failure
+    # @return [void]
+    # @api private
+    def handle_default_retry(beanstalk_job, payload, error)
+      retry_count = payload['retry_count'] || 0
+      max_retries = 5
+
+      if retry_count < max_retries
+        payload['retry_count'] = retry_count + 1
+        payload['executions'] = (payload['executions'] || 0) + 1
+
+        delay = [2 ** retry_count, 3600].min
+
+        beanstalk_job.delete
+
+        Postburner.connected do |conn|
+          tube_name = expand_tube_name(payload['queue_name'])
+          conn.tubes[tube_name].put(
+            JSON.generate(payload),
+            pri: payload['priority'] || config.default_priority,
+            delay: delay,
+            ttr: payload['ttr'] || config.default_ttr
+          )
+        end
+
+        ActiveSupport::Notifications.instrument('retry.postburner', {
+          payload: payload,
+          beanstalk_job_id: beanstalk_job.id,
+          error: error,
+          wait: delay,
+          attempt: retry_count + 1
+        })
+
+        logger.info "[Postburner] Retrying default job #{payload['job_id']}, attempt #{retry_count + 1} in #{delay}s"
+      else
+        ActiveSupport::Notifications.instrument('discard.postburner', {
+          payload: payload,
+          beanstalk_job_id: beanstalk_job.id,
+          error: error
+        })
+
+        logger.error "[Postburner] Discarding default job #{payload['job_id']} after #{retry_count} retries"
+        beanstalk_job.delete
+      end
+    end
+
+    # Watches all configured queues in Beanstalkd.
+    #
+    # Expands queue names to full tube names and sets up the connection
+    # to watch them. Always includes the scheduler tube.
+    #
+    # @param connection [Postburner::Connection] Beanstalkd connection
+    # @param queue_names [Array<String>, nil] Queue names to watch (uses config if nil)
+    # @return [void]
+    # @api private
+    def watch_queues(connection, queue_names = nil)
+      tube_names = queue_names.map { |q| config.expand_tube_name(q) }
+      tube_names = config.expanded_tube_names if tube_names.empty?
+
+      # Always watch scheduler tube
+      tube_names << config.scheduler_tube_name unless tube_names.include?(config.scheduler_tube_name)
+
+      connection.beanstalk.tubes.watch!(*tube_names)
+      logger.debug "[Postburner::Worker] Thread #{Thread.current.object_id} watching: #{tube_names.join(', ')}"
+    end
+
+    # Ensure scheduler watchdog exists in Beanstalkd.
+    #
+    # Called on reserve timeout to verify the watchdog job exists.
+    # Creates a new watchdog if none found, ensuring the scheduler
+    # continues running even when job volume is low.
+    #
+    # @param connection [Postburner::Connection] Beanstalkd connection
+    # @return [void]
+    # @see Postburner::Scheduler.ensure_watchdog!
+    # @api private
+    def ensure_scheduler_watchdog!(connection)
+      Postburner::Scheduler.ensure_watchdog!(connection: connection)
+    end
+
+    # Deletes a job from Beanstalkd with one retry on failure.
+    #
+    # If delete fails (e.g., network blip), waits 1 second and retries once.
+    # This reduces the window for duplicate job execution while keeping
+    # the implementation simple.
+    #
+    # @param beanstalk_job [Beaneater::Job] Job to delete
+    # @return [void]
+    # @api private
+    def delete_job!(beanstalk_job)
+      beanstalk_job.delete
+    rescue => e
+      logger.warn "[Postburner] Delete failed (#{e.message}), retrying..."
+      sleep 1
+      beanstalk_job.delete rescue nil
+    end
+  end
+end