async-background 1.0.2 → 1.1.1

data/lib/async/background/runner/lifecycle.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module Async
+  module Background
+    class Runner
+      module Lifecycle
+        private
+
+        def setup_signal_handlers
+          @signal_r, @signal_w = IO.pipe
+
+          %w[INT TERM].each do |signal|
+            Signal.trap(signal) { stop_from_signal }
+          end
+        end
+
+        def stop_from_signal
+          @running = false
+          @signal_w.write_nonblock('.')
+        rescue StandardError
+          nil
+        end
+
+        def start_signal_watcher
+          @services.spawn(name: 'signal-watcher') { watch_signals }
+        end
+
+        def watch_signals
+          while running?
+            result = read_signal
+            next if result == :wait
+            break unless result
+
+            logger.info { 'Async::Background: stopping gracefully' } unless running?
+            shutdown.signal_all
+            @queue_waker&.signal
+          end
+        end
+
+        def read_signal
+          @signal_r.wait_readable
+          @signal_r.read_nonblock(256)
+        rescue IO::WaitReadable
+          :wait
+        rescue IOError, Errno::EBADF
+          false
+        end
+
+        def error_handler
+          @error_handler ||= build_error_handler
+        end
+
+        def build_error_handler
+          log = logger
+          lambda do |task, error|
+            log.error('Async::Background') do
+              "task #{task&.name || 'unnamed'} died: #{error.class} #{error.message}\n" \
+                "#{Array(error.backtrace).join("\n")}"
+            end
+          end
+        end
+
+        def warn_unsafe_timeouts
+          return if Runtime.native_timeouts?
+
+          logger.warn('Async::Background') do
+            "#{Fiber.scheduler.class} does not implement #timeout_after; job timeouts " \
+              'fall back to stdlib Timeout and may interrupt an unrelated fiber'
+          end
+        end
+
+        def wake_signal_watcher
+          @signal_w&.write_nonblock('.')
+        rescue StandardError
+          nil
+        end
+
+        def shutdown_gracefully
+          wake_services
+          drain_jobs
+          stop_services
+          close_queue
+          close_signal_pipe
+        end
+
+        def wake_services
+          shutdown.signal_all
+          wake_signal_watcher
+          @queue_waker&.signal
+        end
+
+        def drain_jobs
+          return if @jobs.empty?
+
+          logger.info { "Async::Background: draining #{@jobs.size} in-flight job(s)" }
+          @jobs.wait(@drain_timeout)
+        rescue Runtime::TimeoutError
+          cancel_draining_jobs
+        end
+
+        def cancel_draining_jobs
+          logger.warn('Async::Background') do
+            "#{@jobs.size} job(s) still running after #{@drain_timeout}s, cancelling"
+          end
+          @jobs.stop_all(JOB_CANCEL_GRACE)
+        end
+
+        def stop_services
+          return if @services.empty?
+
+          @services.wait(SERVICE_SHUTDOWN_GRACE)
+        rescue Runtime::TimeoutError
+          logger.warn('Async::Background') { 'service tasks did not exit in time, cancelling' }
+          @services.stop_all(SERVICE_CANCEL_GRACE)
+        end
+
+        def close_queue
+          @queue_waker&.close
+          @queue_store&.close
+        end
+
+        def close_signal_pipe
+          @signal_r&.close
+          @signal_w&.close
+        rescue IOError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/async/background/runner/queue_execution.rb

@@ -6,10 +6,15 @@ module Async
       # Queue-only lifecycle. Keeping it separate from recurring scheduling makes
       # the different delivery guarantees visible without adding runtime objects.
       module QueueExecution
+        QueueRun = Data.define(:job, :class_name, :claim_token, :options, :started_at, :job_class)
+        QueueFailure = Data.define(:job, :options, :message, :error, :duration, :timeout, :backtrace)
+        private_constant :QueueRun, :QueueFailure
+
         private
 
         def setup_queue(queue_socket_dir, queue_db_path, queue_mmap)
-          @listen_queue = !!queue_socket_dir && !isolated_worker?
+          @queue_saturated = false
+          @listen_queue = !queue_socket_dir.nil? && !isolated_worker?
           return unless @listen_queue
 
           require_relative '../queue/client'
@@ -18,7 +23,7 @@ module Async
 
           @queue_store = Queue::Store.new(
             path: queue_db_path || Queue::Store.default_path,
-            options: {mmap: queue_mmap}
+            options: { mmap: queue_mmap }
           )
 
           @queue_waker = Queue::SocketWaker.new(queue_socket_path(queue_socket_dir))
@@ -26,20 +31,36 @@ module Async
           recover_queue_jobs
         end
 
-        def start_queue_listener(task)
-          @queue_waker.start_accept_loop(task)
+        def start_queue_listener
+          @queue_waker.start_accept_loop
 
-          task.async do
+          @services.spawn(name: 'queue-listener') do
             logger.info { "Async::Background queue: listening on worker #{worker_index}" }
 
+            failures = 0
+
             while running?
-              @queue_waker.wait(timeout: next_wait_timeout)
-              dispatch_available_queue_jobs
+              begin
+                @queue_waker.wait(timeout: next_wait_timeout)
+                break unless running?
+
+                dispatch_available_queue_jobs
+                failures = 0
+              rescue StandardError => e
+                failures += 1
+                backoff = [QUEUE_ERROR_BACKOFF * failures, QUEUE_POLL_INTERVAL].min
+                logger.error('Async::Background') do
+                  "queue listener: #{e.class} #{e.message}; retrying in #{backoff}s"
+                end
+                shutdown.wait(backoff)
+              end
             end
           end
         end
 
         def next_wait_timeout
+          return QUEUE_POLL_INTERVAL if @queue_saturated
+
           next_due = @queue_store.next_pending_run_at
           return QUEUE_POLL_INTERVAL unless next_due
 
@@ -50,49 +71,56 @@ module Async
         end
 
         def run_queue_job(job_task, job)
-          class_name = job[:class_name]
-          claim_token = job[:claim_token]
-          options = nil
-          started_at = nil
+          context = QueueRun.new(job, job[:class_name], job[:claim_token], nil, nil, nil)
+          context = resolve_queue_run(context)
           metrics_started = false
+          metrics_started = start_queue_metrics(context)
+          return unless metrics_started
+
+          context = context.with(started_at: monotonic_now)
+          perform_queue_run(job_task, context)
+          complete_queue_job!(context)
+        rescue ConfigError => e
+          record_invalid_queue_job!(context, e)
+        rescue Runtime::TimeoutError => e
+          handle_queue_failure(queue_failure(context, e, timeout: true))
+        rescue StandardError => e
+          handle_queue_failure(queue_failure(context, e, timeout: false))
+        ensure
+          metrics.job_stopped(nil) if metrics_started
+        end
 
-          klass = resolve_job_class(class_name)
+        def resolve_queue_run(context)
+          job, class_name, = context.deconstruct
+          job_class = resolve_job_class(class_name)
           options = parse_job_options(job[:options])
-          return unless start_queue_job!(job, class_name, claim_token)
+          context.with(options: options, job_class: job_class)
+        end
+
+        def start_queue_metrics(context)
+          job, class_name, claim_token, = context.deconstruct
+          return false unless start_queue_job!(job, class_name, claim_token)
 
           metrics.job_started(nil)
-          metrics_started = true
-          started_at = monotonic_now
-          job_task.with_timeout(options.timeout) { klass.perform_now(*job[:args]) }
-
-          complete_queue_job!(job, class_name, claim_token, started_at)
-        rescue ConfigError => error
-          record_invalid_queue_job!(job, class_name, claim_token, error)
-        rescue ::Async::TimeoutError => error
-          handle_queue_failure(
-            job,
-            options,
-            "timed out after #{options&.timeout}s",
-            error: error,
-            duration: started_at && (monotonic_now - started_at),
-            timeout: true,
-            backtrace: nil
-          )
-        rescue StandardError => error
-          handle_queue_failure(
-            job,
-            options,
-            "#{error.class} #{error.message}",
-            error: error,
-            duration: started_at && (monotonic_now - started_at),
-            timeout: false,
-            backtrace: error.backtrace
-          )
-        ensure
-          metrics.job_stopped(nil) if metrics_started
+          true
+        end
+
+        def perform_queue_run(job_task, context)
+          job, _class_name, _claim_token, options, _started_at, job_class = context.deconstruct
+          job_task.with_timeout(options.timeout) do
+            job_class.perform_now(*job[:args])
+          end
+        end
+
+        def queue_failure(context, error, timeout:)
+          job, _class_name, _claim_token, options, started_at = context.deconstruct
+          message = timeout ? "timed out after #{options&.timeout}s" : "#{error.class} #{error.message}"
+          duration = started_at && (monotonic_now - started_at)
+          QueueFailure.new(job, options, message, error, duration, timeout, timeout ? nil : error.backtrace)
         end
 
-        def handle_queue_failure(job, options, message, error:, duration:, timeout:, backtrace:)
+        def handle_queue_failure(failure)
+          job, options, message, error, duration, timeout, backtrace = failure.deconstruct
           class_name = job[:class_name]
           result = @queue_store.retry_or_fail(
             job[:id],
@@ -105,14 +133,17 @@ module Async
           return log_stale_queue_failure(job, class_name, timeout) unless result
 
           timeout ? metrics.job_timed_out(nil) : metrics.job_failed(nil, error)
-          result == :retried ? log_queue_retry(job, class_name, message, options) :
+          if result == :retried
+            log_queue_retry(job, class_name, message, options)
+          else
             log_queue_failure(class_name, message, backtrace)
+          end
         end
 
         def parse_job_options(raw)
           Job::Options.new(**(raw || {}))
-        rescue ArgumentError, TypeError => error
-          raise ConfigError, "invalid queue options: #{error.message}"
+        rescue ArgumentError, TypeError => e
+          raise ConfigError, "invalid queue options: #{e.message}"
         end
 
         def isolated_worker?
@@ -129,36 +160,47 @@ module Async
         end
 
         def dispatch_available_queue_jobs
+          @queue_saturated = false
+
           while running?
+            if @jobs.size >= @semaphore.limit
+              @queue_saturated = true
+              return
+            end
+
             job = @queue_store.fetch(worker_index)
-            break unless job
+            return if job.nil?
 
-            semaphore.async { |job_task| run_queue_job(job_task, job) }
+            spawn_job { |job_task| run_queue_job(job_task, job) }
           end
         end
 
         def start_queue_job!(job, class_name, claim_token)
           return true if @queue_store.mark_started!(job[:id], claim_token: claim_token)
 
-          logger.warn('Async::Background') {
+          logger.warn('Async::Background') do
             "queue(#{class_name}): lost lease before start for job #{job[:id]}, ignored"
-          }
+          end
           false
         end
 
-        def complete_queue_job!(job, class_name, claim_token, started_at)
+        def complete_queue_job!(context)
+          job, class_name, claim_token, _options, started_at = context.deconstruct
           duration = monotonic_now - started_at
           if @queue_store.complete(job[:id], claim_token: claim_token, duration_ms: duration_ms(duration))
             metrics.job_succeeded(nil, duration)
-            logger.info('Async::Background') { "queue(#{class_name}): completed in #{duration.round(2)}s" }
+            logger.info('Async::Background') do
+              "queue(#{class_name}): completed in #{duration.round(2)}s"
+            end
           else
-            logger.warn('Async::Background') {
+            logger.warn('Async::Background') do
               "queue(#{class_name}): complete: stale lease for job #{job[:id]}, ignored"
-            }
+            end
           end
         end
 
-        def record_invalid_queue_job!(job, class_name, claim_token, error)
+        def record_invalid_queue_job!(context, error)
+          job, class_name, claim_token, = context.deconstruct
           recorded = @queue_store.fail(
             job[:id],
             claim_token: claim_token,
@@ -166,7 +208,9 @@ module Async
             error_message: error.message
           )
           metrics.job_failed(nil, error) if recorded
-          logger.error('Async::Background') { "queue(#{class_name}): #{error.class} #{error.message}" }
+          logger.error('Async::Background') do
+            "queue(#{class_name}): #{error.class} #{error.message}"
+          end
         end
 
         def log_stale_queue_failure(job, class_name, timeout)
@@ -176,7 +220,7 @@ module Async
           end
         end
 
-        def log_queue_retry(job, class_name, message, options)
+        def log_queue_retry(_job, class_name, message, options)
           @queue_waker&.signal
           logger.warn('Async::Background') do
             "queue(#{class_name}): #{message}; retry #{options&.next_attempt}/#{options&.retry}"

data/lib/async/background/runner/schedule.rb

@@ -59,12 +59,15 @@ module Async
         end
 
         def build_entries(schedule, now)
-          schedule.each_with_object(MinHeap.new) do |(name, config), heap|
-            next unless assigned_worker(config, name) == worker_index
+          schedule
+            .filter_map { |name, config| entry_for(name, config, now) }
+            .each_with_object(MinHeap.new) { |entry, heap| heap.push(entry) }
+        end
 
-            task = build_task_config(name, config)
-            heap.push(build_entry(name, task, now))
-          end
+        def entry_for(name, config, now)
+          return unless assigned_worker(config, name) == worker_index
+
+          build_entry(name, build_task_config(name, config), now)
         end
 
         def assigned_worker(config, name)
@@ -93,29 +96,29 @@ module Async
 
         def resolve_scheduled_job(name, class_name)
           resolve_job_class(class_name)
-        rescue ConfigError => error
-          raise ConfigError, "[#{name}] #{error.message}"
+        rescue ConfigError => e
+          raise ConfigError, "[#{name}] #{e.message}"
         end
 
         def parse_interval(name, value)
-          value&.then do |interval|
-            interval = interval.to_i
-            raise ConfigError, "[#{name}] 'every' must be > 0" unless interval.positive?
+          return if value.nil?
 
-            interval
-          end
+          interval = value.to_i
+          raise ConfigError, "[#{name}] 'every' must be > 0" unless interval.positive?
+
+          interval
         end
 
         def parse_cron(name, value)
-          value&.then do |expression|
-            Fugit::Cron.new(expression) || raise(ConfigError, "[#{name}] invalid cron: #{expression}")
-          end
+          return if value.nil?
+
+          Fugit::Cron.new(value) || raise(ConfigError, "[#{name}] invalid cron: #{value}")
         end
 
         def parse_timeout(name, config)
           Job::Options.new(timeout: config.fetch('timeout', DEFAULT_TIMEOUT)).timeout
-        rescue ArgumentError, TypeError => error
-          raise ConfigError, "[#{name}] #{error.message}"
+        rescue ArgumentError, TypeError => e
+          raise ConfigError, "[#{name}] #{e.message}"
         end
 
         def validate_schedule_frequency!(name, interval, cron)