async-background 1.0.2 → 1.1.1

data/lib/async/background/runner.rb

@@ -1,25 +1,37 @@
 # frozen_string_literal: true
 
-require 'async/barrier'
-
+require_relative 'runtime'
+require_relative 'runner/lifecycle'
 require_relative 'runner/queue_execution'
 require_relative 'runner/schedule'
 
 module Async
   module Background
-    class ConfigError < StandardError; end
+    class ConfigError < Error; end
 
     DEFAULT_TIMEOUT = 30
     MIN_SLEEP_TIME = 0.1
     MAX_JITTER = 5
     QUEUE_POLL_INTERVAL = 5
     MIN_QUEUE_WAIT = 0.001
+    QUEUE_ERROR_BACKOFF = 0.5
+    SERVICE_SHUTDOWN_GRACE = 5
+    SERVICE_CANCEL_GRACE = 2
+    DRAIN_GRACE = 30
+    JOB_CANCEL_GRACE = 5
 
     class Runner
       include Clock
+      include Lifecycle
       include QueueExecution
       include Schedule
 
+      Settings = Data.define(
+        :config_path, :job_count, :queue_socket_dir, :queue_db_path, :queue_mmap,
+        :drain_timeout, :metrics_shm_path
+      )
+      private_constant :Settings
+
       attr_reader :logger,
                   :semaphore,
                   :heap,
@@ -27,44 +39,59 @@ module Async
                   :total_workers,
                   :shutdown,
                   :metrics,
-                  :queue_store
-
-      # `config_path: nil` explicitly disables recurring jobs. This keeps the
-      # dynamic SQLite queue usable on its own; a supplied path remains strict
-      # so a typo cannot silently disable scheduled work.
-      def initialize(
-        config_path: nil,
-        job_count: 2,
-        worker_index:,
-        total_workers:,
-        queue_socket_dir: nil,
-        queue_db_path: nil,
-        queue_mmap: true,
-        metrics_shm_path: Metrics.default_shm_path
-      )
+                  :queue_store,
+                  :jobs,
+                  :services
+
+      def initialize(worker_index:, total_workers:, **options)
+        settings = Settings.new(**default_settings.merge(options))
+        initialize_worker(worker_index, total_workers, settings)
+      end
+
+      def default_settings
+        {
+          config_path: nil,
+          job_count: 2,
+          queue_socket_dir: nil,
+          queue_db_path: nil,
+          queue_mmap: true,
+          drain_timeout: DRAIN_GRACE,
+          metrics_shm_path: Metrics.default_shm_path
+        }
+      end
+
+      def initialize_worker(worker_index, total_workers, settings)
+        config_path = settings.config_path
+        job_count = settings.job_count
         @logger = Console.logger
         @worker_index = worker_index
         @total_workers = total_workers
         @running = true
-        @shutdown = ::Async::Condition.new
-        @metrics = Metrics.new(worker_index: worker_index, total_workers: total_workers, shm_path: metrics_shm_path)
+        @drain_timeout = settings.drain_timeout
+        @shutdown = Runtime::Notification.new
+        @metrics = Metrics.new(worker_index: worker_index, total_workers: total_workers, shm_path: settings.metrics_shm_path)
         logger.info { "Async::Background worker_index=#{worker_index}/#{total_workers}, job_count=#{job_count}" }
 
-        @drain_barrier = ::Async::Barrier.new
-        @semaphore = ::Async::Semaphore.new(job_count, parent: @drain_barrier)
+        @jobs = Runtime::TaskGroup.new(on_error: error_handler, on_release: method(:job_released))
+        @services = Runtime::TaskGroup.new(on_error: error_handler)
+        @semaphore = Runtime::Semaphore.new(job_count)
         @heap = config_path.nil? ? MinHeap.new : build_heap(config_path)
-        setup_queue(queue_socket_dir, queue_db_path, queue_mmap)
+        setup_queue(settings.queue_socket_dir, settings.queue_db_path, settings.queue_mmap)
         validate_work_source!(config_path)
       end
+      private :default_settings, :initialize_worker
 
       def run
-        Async do |task|
+        Runtime.scheduler!
+        warn_unsafe_timeouts
+
+        Runtime.with_error_handler(error_handler) do
           setup_signal_handlers
-          start_signal_watcher(task)
-          start_queue_listener(task) if @listen_queue
+          start_signal_watcher
+          start_queue_listener if @listen_queue
 
-          scheduler_loop(task)
-          drain_and_close_queue
+          scheduler_loop
+          shutdown_gracefully
         end
       end
 
@@ -72,26 +99,21 @@ module Async
         return unless @running
 
         @running = false
-        logger.info { 'Async::Background: stopping gracefully' }
-        shutdown.signal
-        @queue_waker&.signal
+        wake_signal_watcher
       end
 
       def running? = @running
 
       private
 
-      def scheduler_loop(task)
-        # Queue-only workers have no heap entry to sleep on. Keep the runner
-        # alive until #stop / SIGTERM wakes this condition; the queue listener
-        # continues independently in its own Async task.
+      def scheduler_loop
         return shutdown.wait if heap.empty? && @listen_queue
 
         while running?
           entry = heap.peek
           break unless entry
 
-          wait_for_next_entry(task, entry)
+          wait_for_next_entry(entry)
           break unless running?
 
           dispatch_due_entries
@@ -104,9 +126,9 @@ module Async
         raise ConfigError, 'Runner requires config_path or queue_socket_dir'
       end
 
-      def wait_for_next_entry(task, entry)
+      def wait_for_next_entry(entry)
         wait = [entry.next_run_at - monotonic_now, MIN_SLEEP_TIME].max
-        wait_with_shutdown(task, wait)
+        wait_with_shutdown(wait)
       end
 
       def dispatch_due_entries
@@ -131,66 +153,65 @@ module Async
 
       def execute_entry(entry)
         entry.running = true
-        semaphore.async do |job_task|
+        spawn_job do |job_task|
           run_job(job_task, entry)
         ensure
           entry.running = false
         end
       end
 
+      def spawn_job
+        @jobs.spawn(name: 'job') { |task| semaphore.acquire { yield(task) } }
+      end
+
+      def job_released(_task)
+        return unless @queue_saturated
+
+        @queue_waker&.signal
+      end
+
       def run_job(job_task, entry)
         metrics_started = false
+        metrics_started = start_job_metrics(entry)
+        execute_entry_job(job_task, entry)
+      rescue Runtime::TimeoutError
+        record_job_timeout(entry)
+      rescue StandardError => e
+        record_job_failure(entry, e)
+      ensure
+        metrics.job_stopped(entry) if metrics_started
+      end
+
+      def start_job_metrics(entry)
         metrics.job_started(entry)
-        metrics_started = true
+        true
+      end
+
+      def execute_entry_job(job_task, entry)
         started_at = monotonic_now
         job_task.with_timeout(entry.timeout) { entry.job_class.perform_now }
+        record_job_success(entry, monotonic_now - started_at)
+      end
 
-        duration = monotonic_now - started_at
+      def record_job_success(entry, duration)
         metrics.job_succeeded(entry, duration)
         logger.info('Async::Background') { "#{entry.name}: completed in #{duration.round(2)}s" }
-      rescue ::Async::TimeoutError
-        metrics.job_timed_out(entry)
-        logger.error('Async::Background') { "#{entry.name}: timed out after #{entry.timeout}s" }
-      rescue StandardError => error
-        metrics.job_failed(entry, error)
-        logger.error('Async::Background') {
-          "#{entry.name}: #{error.class} #{error.message}\n#{error.backtrace.join("\n")}"
-        }
-      ensure
-        metrics.job_stopped(entry) if metrics_started
       end
 
-      def setup_signal_handlers
-        @signal_r, @signal_w = IO.pipe
-
-        %w[INT TERM].each do |signal|
-          Signal.trap(signal) do
-            @running = false
-            @signal_w.write_nonblock('.') rescue nil
-          end
-        end
+      def record_job_timeout(entry)
+        metrics.job_timed_out(entry)
+        logger.error('Async::Background') { "#{entry.name}: timed out after #{entry.timeout}s" }
       end
 
-      def start_signal_watcher(task)
-        task.async(transient: true) do
-          while running?
-            @signal_r.wait_readable
-            @signal_r.read_nonblock(256) rescue nil
-            shutdown.signal
-            @queue_waker&.signal
-          end
+      def record_job_failure(entry, error)
+        metrics.job_failed(entry, error)
+        logger.error('Async::Background') do
+          "#{entry.name}: #{error.class} #{error.message}\n#{error.backtrace.join("\n")}"
         end
       end
 
-      def wait_with_shutdown(task, duration)
-        task.with_timeout(duration) { shutdown.wait }
-      rescue ::Async::TimeoutError
-      end
-
-      def drain_and_close_queue
-        @drain_barrier.wait
-        @queue_store&.close
-        @queue_waker&.close
+      def wait_with_shutdown(duration)
+        shutdown.wait(duration)
       end
     end
   end

data/lib/async/background/runtime/notification.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Async
+  module Background
+    module Runtime
+      class Notification
+        def initialize
+          @waiters = []
+        end
+
+        def waiting = @waiters.size
+
+        def wait(timeout = nil)
+          wait_until(Runtime.deadline_for(timeout))
+        end
+
+        def wait_until(deadline = nil)
+          Runtime.with_waiter(self, @waiters) do |waiter|
+            Runtime.park(self, waiter, deadline) { false }
+          end
+        end
+
+        def signal
+          while (waiter = @waiters.shift)
+            return true if Runtime.wake_dequeued(waiter, self)
+          end
+
+          false
+        end
+
+        def signal_all
+          return nil if @waiters.empty?
+
+          pending = @waiters
+          @waiters = []
+          pending.each { |waiter| Runtime.wake_dequeued(waiter, self) }
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/async/background/runtime/semaphore.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+module Async
+  module Background
+    module Runtime
+      class Semaphore
+        attr_reader :limit, :available
+
+        def initialize(limit)
+          @limit = Integer(limit)
+          raise ArgumentError, 'limit must be >= 1' unless @limit.positive?
+
+          @available = @limit
+          @waiting = WaitList.new
+        end
+
+        def waiting = @waiting.size
+
+        def acquire
+          wait
+          return @available unless block_given?
+
+          begin
+            yield
+          ensure
+            release
+          end
+        end
+
+        def release
+          while (node = @waiting.shift)
+            node.grant!
+            return @available if node.resume
+          end
+
+          @available += 1
+          @available
+        end
+
+        private
+
+        def wait
+          return (@available -= 1) if @available.positive?
+
+          task = Runtime.current_task
+          node = FiberNode.new(Fiber.current, self)
+          @waiting.push(node)
+          task&.enter_block(node.waiter)
+          wait_for_grant(node, task)
+        ensure
+          release_wait_state(node, task) if node
+        end
+
+        def wait_for_grant(node, task)
+          until node.granted?
+            node.suspend
+            abandon_wait!(node, task) if task&.cancelled?
+          end
+          abandon_wait!(node, task) if task&.cancelled?
+        end
+
+        def release_wait_state(node, task)
+          task&.exit_block
+          @waiting.remove(node) unless node.granted? || node.list.nil?
+        end
+
+        def abandon_wait!(node, task)
+          node.granted? ? release : @waiting.remove(node)
+          task.raise_if_cancelled!
+        end
+
+        class WaitList
+          attr_reader :size
+
+          def initialize
+            @head = @tail = nil
+            @size = 0
+          end
+
+          def first = @head
+          def empty? = @head.nil?
+
+          def push(node)
+            raise ArgumentError, 'node already queued' if node.list
+
+            node.list = self
+            node.prev = @tail
+            node.next = nil
+            @tail ? (@tail.next = node) : (@head = node)
+            @tail = node
+            @size += 1
+            node
+          end
+
+          def shift
+            node = @head or return nil
+
+            remove(node)
+          end
+
+          def remove(node)
+            return nil unless node.list.equal?(self)
+
+            node.prev ? (node.prev.next = node.next) : (@head = node.next)
+            node.next ? (node.next.prev = node.prev) : (@tail = node.prev)
+            node.list = node.prev = node.next = nil
+            @size -= 1
+            node
+          end
+        end
+
+        class FiberNode
+          attr_accessor :list, :prev, :next
+          attr_reader :fiber, :waiter, :blocker
+
+          def initialize(fiber, blocker)
+            @fiber = fiber
+            @blocker = blocker
+            @list = @prev = @next = nil
+            @granted = false
+            @scheduler = Runtime.scheduler!
+            @waiter = { fiber: fiber, scheduler: @scheduler, ready: false, blocker: blocker, queued: true }
+          end
+
+          def granted? = @granted
+
+          def grant!
+            @granted = true
+            @waiter[:ready] = true
+            @waiter[:queued] = false
+            self
+          end
+
+          def suspend
+            @scheduler.block(@blocker, nil)
+          end
+
+          def resume
+            fiber = @fiber
+            return false unless fiber.alive?
+
+            @scheduler.unblock(@blocker, fiber)
+            true
+          end
+        end
+
+        private_constant :WaitList, :FiberNode
+      end
+    end
+  end
+end

data/lib/async/background/runtime/task.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+module Async
+  module Background
+    module Runtime
+      class Task
+        attr_reader :fiber, :name, :error
+
+        def self.spawn(name: nil, on_error: UNSET, &block)
+          task = new(name: name, on_error: on_error)
+          task.start(&block)
+          task
+        end
+
+        def initialize(name: nil, group: nil, on_error: UNSET)
+          @fiber = @scheduler = @blocker = @result = @error = nil
+          @done = @cancelled = false
+
+          @name = name
+          @group = group
+          @on_error = on_error
+          @waiters = nil
+        end
+
+        def start(&block)
+          raise ArgumentError, 'block required' unless block
+          raise Error, 'task already started' if @fiber || @done
+
+          @scheduler = Runtime.scheduler!
+          takes_task = !block.arity.zero?
+
+          scheduled = Fiber.schedule { run_scheduled_task { takes_task ? yield(self) : yield } }
+
+          @fiber ||= scheduled if scheduled.is_a?(Fiber)
+          self
+        end
+
+        def wait(timeout = nil)
+          unless @done
+            deadline = Runtime.deadline_for(timeout)
+            raise TimeoutError, 'task did not finish in time' unless join(deadline)
+          end
+
+          raise @error if @error
+
+          @result
+        end
+
+        def with_timeout(duration, &block)
+          Runtime.with_timeout(duration, &block)
+        end
+
+        def stop
+          return false if @done
+
+          @cancelled = true
+
+          return true if interrupt_fiber
+          return true if release_blocker
+
+          false
+        end
+
+        def cancelled? = @cancelled
+        def finished? = @done
+        def waiting = @waiters ? @waiters.size : 0
+
+        def raise_if_cancelled!
+          raise Cancel, 'task stopped' if @cancelled
+        end
+
+        def enter_block(waiter)
+          @blocker = waiter
+        end
+
+        def exit_block
+          @blocker = nil
+        end
+
+        private
+
+        def run_scheduled_task
+          adopt_fiber(Fiber.current)
+          previous = Runtime.current_task
+          Runtime.current_task = self
+          complete(yield, nil)
+        rescue Cancel
+          complete(nil, nil)
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          complete(nil, e)
+        ensure
+          Runtime.current_task = previous
+        end
+
+        def adopt_fiber(fiber)
+          @adopt_fiber ||= fiber
+        end
+
+        def interrupt_fiber
+          fiber = @fiber
+          scheduler = @scheduler
+          return false unless fiber&.alive?
+          return false unless scheduler.respond_to?(:fiber_interrupt)
+
+          result = scheduler.fiber_interrupt(fiber, Cancel.new('task stopped'))
+          result != false
+        rescue FiberError
+          false
+        end
+
+        def release_blocker
+          waiter = @blocker or return false
+
+          Runtime.wake(waiter, waiter[:blocker]) && waiter[:fiber].alive?
+        end
+
+        def complete(result, error)
+          return if @done
+
+          @result = result
+          @error = error
+          @done = true
+
+          Runtime.report_error(self, error, @on_error) if error && waiting.zero?
+          @group&.release(self)
+          wake_waiters
+        end
+
+        def join(deadline)
+          waiters = (@waiters ||= [])
+          Runtime.with_waiter(self, waiters) do |waiter|
+            Runtime.park(self, waiter, deadline) { @done }
+          end
+        end
+
+        def wake_waiters
+          return if @waiters.nil? || @waiters.empty?
+
+          pending = @waiters
+          @waiters = []
+          pending.each { |waiter| Runtime.wake_dequeued(waiter, self) }
+        end
+      end
+    end
+  end
+end

data/lib/async/background/runtime/task_group.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Async
+  module Background
+    module Runtime
+      class TaskGroup
+        attr_accessor :on_release
+
+        def initialize(on_error: UNSET, on_release: nil)
+          @members = {}
+          @drained = Notification.new
+          @on_error = on_error
+          @on_release = on_release
+        end
+
+        def size = @members.size
+        def empty? = @members.empty?
+        def tasks = @members.keys
+
+        def spawn(name: nil, &block)
+          raise ArgumentError, 'block required' unless block
+
+          task = Task.new(name: name, group: self, on_error: @on_error)
+          @members[task] = true
+
+          begin
+            task.start(&block)
+          rescue Exception
+            release(task)
+            raise
+          end
+
+          task
+        end
+
+        def wait(timeout = nil)
+          deadline = Runtime.deadline_for(timeout)
+
+          until @members.empty?
+            raise TimeoutError, 'tasks did not finish in time' unless @drained.wait_until(deadline)
+          end
+
+          true
+        end
+
+        def stop_all(grace = nil)
+          tasks.each do |task|
+            task.stop
+          rescue StandardError
+            nil
+          end
+
+          return @members.empty? if grace.nil?
+
+          begin
+            wait(grace)
+          rescue TimeoutError
+            false
+          end
+        end
+
+        def release(task)
+          return unless @members.delete(task)
+
+          @drained.signal_all if @members.empty?
+          notify_release(task)
+        end
+
+        private
+
+        def notify_release(task)
+          handler = @on_release or return
+
+          handler.call(task)
+        rescue StandardError => e
+          Runtime.report_error(task, e, @on_error)
+        end
+      end
+    end
+  end
+end