async-background 1.0.1 → 1.1.0

data/lib/async/background/runner.rb

@@ -1,19 +1,23 @@
 # frozen_string_literal: true
 
-require 'async/barrier'
-
+require_relative 'runtime'
 require_relative 'runner/queue_execution'
 require_relative 'runner/schedule'
 
 module Async
   module Background
-    class ConfigError < StandardError; end
+    class ConfigError < Error; end
 
     DEFAULT_TIMEOUT = 30
     MIN_SLEEP_TIME = 0.1
     MAX_JITTER = 5
     QUEUE_POLL_INTERVAL = 5
     MIN_QUEUE_WAIT = 0.001
+    QUEUE_ERROR_BACKOFF = 0.5
+    SERVICE_SHUTDOWN_GRACE = 5
+    SERVICE_CANCEL_GRACE = 2
+    DRAIN_GRACE = 30
+    JOB_CANCEL_GRACE = 5
 
     class Runner
       include Clock
@@ -27,11 +31,10 @@ module Async
                   :total_workers,
                   :shutdown,
                   :metrics,
-                  :queue_store
+                  :queue_store,
+                  :jobs,
+                  :services
 
-      # `config_path: nil` explicitly disables recurring jobs. This keeps the
-      # dynamic SQLite queue usable on its own; a supplied path remains strict
-      # so a typo cannot silently disable scheduled work.
       def initialize(
         config_path: nil,
         job_count: 2,
@@ -40,35 +43,37 @@ module Async
         queue_socket_dir: nil,
         queue_db_path: nil,
         queue_mmap: true,
+        drain_timeout: DRAIN_GRACE,
         metrics_shm_path: Metrics.default_shm_path
       )
         @logger = Console.logger
         @worker_index = worker_index
         @total_workers = total_workers
         @running = true
-        @shutdown = ::Async::Condition.new
-        @metrics = Metrics.new(
-          worker_index: worker_index,
-          total_workers: total_workers,
-          shm_path: metrics_shm_path
-        )
+        @drain_timeout = drain_timeout
+        @shutdown = Runtime::Notification.new
+        @metrics = Metrics.new(worker_index: worker_index, total_workers: total_workers, shm_path: metrics_shm_path)
         logger.info { "Async::Background worker_index=#{worker_index}/#{total_workers}, job_count=#{job_count}" }
 
-        @drain_barrier = ::Async::Barrier.new
-        @semaphore = ::Async::Semaphore.new(job_count, parent: @drain_barrier)
+        @jobs = Runtime::TaskGroup.new(on_error: error_handler, on_release: method(:job_released))
+        @services = Runtime::TaskGroup.new(on_error: error_handler)
+        @semaphore = Runtime::Semaphore.new(job_count)
         @heap = config_path.nil? ? MinHeap.new : build_heap(config_path)
         setup_queue(queue_socket_dir, queue_db_path, queue_mmap)
         validate_work_source!(config_path)
       end
 
       def run
-        Async do |task|
+        Runtime.scheduler!
+        warn_unsafe_timeouts
+
+        Runtime.with_error_handler(error_handler) do
           setup_signal_handlers
-          start_signal_watcher(task)
-          start_queue_listener(task) if @listen_queue
+          start_signal_watcher
+          start_queue_listener if @listen_queue
 
-          scheduler_loop(task)
-          drain_and_close_queue
+          scheduler_loop
+          shutdown_gracefully
         end
       end
 
@@ -76,26 +81,21 @@ module Async
         return unless @running
 
         @running = false
-        logger.info { 'Async::Background: stopping gracefully' }
-        shutdown.signal
-        @queue_waker&.signal
+        wake_signal_watcher
       end
 
       def running? = @running
 
       private
 
-      def scheduler_loop(task)
-        # Queue-only workers have no heap entry to sleep on. Keep the runner
-        # alive until #stop / SIGTERM wakes this condition; the queue listener
-        # continues independently in its own Async task.
+      def scheduler_loop
         return shutdown.wait if heap.empty? && @listen_queue
 
-        loop do
+        while running?
           entry = heap.peek
           break unless entry
 
-          wait_for_next_entry(task, entry)
+          wait_for_next_entry(entry)
           break unless running?
 
           dispatch_due_entries
@@ -108,9 +108,9 @@ module Async
         raise ConfigError, 'Runner requires config_path or queue_socket_dir'
       end
 
-      def wait_for_next_entry(task, entry)
+      def wait_for_next_entry(entry)
         wait = [entry.next_run_at - monotonic_now, MIN_SLEEP_TIME].max
-        wait_with_shutdown(task, wait)
+        wait_with_shutdown(wait)
       end
 
       def dispatch_due_entries
@@ -123,12 +123,7 @@ module Async
       end
 
       def dispatch_entry(entry)
-        if entry.running
-          skip_entry(entry)
-        else
-          execute_entry(entry)
-        end
-
+        entry.running ? skip_entry(entry) : execute_entry(entry)
         entry.reschedule(monotonic_now)
         heap.replace_top(entry)
       end
@@ -140,13 +135,23 @@ module Async
 
       def execute_entry(entry)
         entry.running = true
-        semaphore.async do |job_task|
+        spawn_job do |job_task|
           run_job(job_task, entry)
         ensure
           entry.running = false
         end
       end
 
+      def spawn_job(&block)
+        @jobs.spawn(name: 'job') { |task| semaphore.acquire { block.call(task) } }
+      end
+
+      def job_released(_task)
+        return unless @queue_saturated
+
+        @queue_waker&.signal
+      end
+
       def run_job(job_task, entry)
         metrics_started = false
         metrics.job_started(entry)
@@ -157,7 +162,7 @@ module Async
         duration = monotonic_now - started_at
         metrics.job_succeeded(entry, duration)
         logger.info('Async::Background') { "#{entry.name}: completed in #{duration.round(2)}s" }
-      rescue ::Async::TimeoutError
+      rescue Runtime::TimeoutError
         metrics.job_timed_out(entry)
         logger.error('Async::Background') { "#{entry.name}: timed out after #{entry.timeout}s" }
       rescue StandardError => error
@@ -180,27 +185,102 @@ module Async
         end
       end
 
-      def start_signal_watcher(task)
-        task.async(transient: true) do
-          loop do
-            @signal_r.wait_readable
-            @signal_r.read_nonblock(256) rescue nil
-            shutdown.signal
+      def start_signal_watcher
+        @services.spawn(name: 'signal-watcher') do
+          while running?
+            begin
+              @signal_r.wait_readable
+              @signal_r.read_nonblock(256)
+            rescue IO::WaitReadable
+              next
+            rescue IOError, Errno::EBADF
+              break
+            end
+
+            logger.info { 'Async::Background: stopping gracefully' } unless running?
+
+            shutdown.signal_all
             @queue_waker&.signal
-            break unless running?
           end
         end
       end
 
-      def wait_with_shutdown(task, duration)
-        task.with_timeout(duration) { shutdown.wait }
-      rescue ::Async::TimeoutError
+      def error_handler
+        @error_handler ||= begin
+          log = logger
+          lambda do |task, error|
+            log.error('Async::Background') do
+              "task #{task&.name || 'unnamed'} died: #{error.class} #{error.message}\n" \
+                "#{Array(error.backtrace).join("\n")}"
+            end
+          end
+        end
       end
 
-      def drain_and_close_queue
-        @drain_barrier.wait
-        @queue_store&.close
+      def warn_unsafe_timeouts
+        return if Runtime.native_timeouts?
+
+        logger.warn('Async::Background') do
+          "#{Fiber.scheduler.class} does not implement #timeout_after; job timeouts " \
+            'fall back to stdlib Timeout and may interrupt an unrelated fiber'
+        end
+      end
+
+      def wake_signal_watcher
+        @signal_w&.write_nonblock('.')
+      rescue StandardError
+        nil
+      end
+
+      def wait_with_shutdown(duration)
+        shutdown.wait(duration)
+      end
+
+      def shutdown_gracefully
+        wake_services
+        drain_jobs
+        stop_services
+        close_queue
+        close_signal_pipe
+      end
+
+      def wake_services
+        shutdown.signal_all
+        wake_signal_watcher
+        @queue_waker&.signal
+      end
+
+      def drain_jobs
+        return if @jobs.empty?
+
+        logger.info { "Async::Background: draining #{@jobs.size} in-flight job(s)" }
+        @jobs.wait(@drain_timeout)
+      rescue Runtime::TimeoutError
+        logger.warn('Async::Background') do
+          "#{@jobs.size} job(s) still running after #{@drain_timeout}s, cancelling"
+        end
+        @jobs.stop_all(JOB_CANCEL_GRACE)
+      end
+
+      def stop_services
+        return if @services.empty?
+
+        @services.wait(SERVICE_SHUTDOWN_GRACE)
+      rescue Runtime::TimeoutError
+        logger.warn('Async::Background') { 'service tasks did not exit in time, cancelling' }
+        @services.stop_all(SERVICE_CANCEL_GRACE)
+      end
+
+      def close_queue
         @queue_waker&.close
+        @queue_store&.close
+      end
+
+      def close_signal_pipe
+        @signal_r&.close
+        @signal_w&.close
+      rescue IOError
+        nil
       end
     end
   end

data/lib/async/background/runtime/notification.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Async
+  module Background
+    module Runtime
+      class Notification
+        def initialize
+          @waiters = []
+        end
+
+        def waiting = @waiters.size
+
+        def wait(timeout = nil)
+          wait_until(Runtime.deadline_for(timeout))
+        end
+
+        def wait_until(deadline = nil)
+          Runtime.with_waiter(self, @waiters) do |waiter|
+            Runtime.park(self, waiter, deadline) { false }
+          end
+        end
+
+        def signal
+          while (waiter = @waiters.shift)
+            return true if Runtime.wake_dequeued(waiter, self)
+          end
+
+          false
+        end
+
+        def signal_all
+          return nil if @waiters.empty?
+
+          pending, @waiters = @waiters, []
+          pending.each { |waiter| Runtime.wake_dequeued(waiter, self) }
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/async/background/runtime/semaphore.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module Async
+  module Background
+    module Runtime
+      class Semaphore
+        attr_reader :limit
+
+        def initialize(limit)
+          @limit = Integer(limit)
+          raise ArgumentError, 'limit must be >= 1' unless @limit.positive?
+
+          @available = @limit
+          @waiting = WaitList.new
+        end
+
+        def available = @available
+        def waiting = @waiting.size
+
+        def acquire
+          wait
+          return @available unless block_given?
+
+          begin
+            yield
+          ensure
+            release
+          end
+        end
+
+        def release
+          while (node = @waiting.shift)
+            node.grant!
+            return @available if node.resume
+          end
+
+          @available += 1
+          @available
+        end
+
+        private
+
+        def wait
+          return (@available -= 1) if @available.positive?
+
+          task = Runtime.current_task
+          node = FiberNode.new(Fiber.current, self)
+          @waiting.push(node)
+          task&.enter_block(node.waiter)
+
+          begin
+            until node.granted?
+              node.suspend
+              abandon_wait!(node, task) if task&.cancelled?
+            end
+            abandon_wait!(node, task) if task&.cancelled?
+          ensure
+            task&.exit_block
+            @waiting.remove(node) unless node.granted? || node.list.nil?
+          end
+        end
+
+        def abandon_wait!(node, task)
+          node.granted? ? release : @waiting.remove(node)
+          task.raise_if_cancelled!
+        end
+
+        class WaitList
+          attr_reader :size
+
+          def initialize
+            @head = @tail = nil
+            @size = 0
+          end
+
+          def first = @head
+          def empty? = @head.nil?
+
+          def push(node)
+            raise ArgumentError, 'node already queued' if node.list
+
+            node.list = self
+            node.prev = @tail
+            node.next = nil
+            @tail ? (@tail.next = node) : (@head = node)
+            @tail = node
+            @size += 1
+            node
+          end
+
+          def shift
+            node = @head or return nil
+
+            remove(node)
+          end
+
+          def remove(node)
+            return nil unless node.list.equal?(self)
+
+            node.prev ? (node.prev.next = node.next) : (@head = node.next)
+            node.next ? (node.next.prev = node.prev) : (@tail = node.prev)
+            node.list = node.prev = node.next = nil
+            @size -= 1
+            node
+          end
+        end
+
+        class FiberNode
+          attr_accessor :list, :prev, :next
+          attr_reader :fiber, :waiter, :blocker
+
+          def initialize(fiber, blocker)
+            @fiber = fiber
+            @blocker = blocker
+            @list = @prev = @next = nil
+            @granted = false
+            @scheduler = Runtime.scheduler!
+            @waiter = {fiber: fiber, scheduler: @scheduler, ready: false, blocker: blocker, queued: true}
+          end
+
+          def granted? = @granted
+
+          def grant!
+            @granted = true
+            @waiter[:ready] = true
+            @waiter[:queued] = false
+            self
+          end
+
+          def suspend
+            @scheduler.block(@blocker, nil)
+          end
+
+          def resume
+            fiber = @fiber
+            return false unless fiber.alive?
+
+            @scheduler.unblock(@blocker, fiber)
+            true
+          end
+        end
+
+        private_constant :WaitList, :FiberNode
+      end
+    end
+  end
+end

data/lib/async/background/runtime/task.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+module Async
+  module Background
+    module Runtime
+      class Task
+        attr_reader :fiber, :name, :error
+
+        def self.spawn(name: nil, on_error: UNSET, &block)
+          task = new(name: name, on_error: on_error)
+          task.start(&block)
+          task
+        end
+
+        def initialize(name: nil, group: nil, on_error: UNSET)
+          @fiber = @scheduler = @blocker = @result = @error = nil
+          @done = @cancelled = false
+
+          @name = name
+          @group = group
+          @on_error = on_error
+          @waiters = nil
+        end
+
+        def start(&block)
+          raise ArgumentError, 'block required' unless block
+          raise Error, 'task already started' if @fiber || @done
+
+          @scheduler = Runtime.scheduler!
+          takes_task = !block.arity.zero?
+
+          scheduled = Fiber.schedule do
+            adopt_fiber(Fiber.current)
+            previous = Runtime.current_task
+            Runtime.current_task = self
+            begin
+              complete(takes_task ? block.call(self) : block.call, nil)
+            rescue Cancel
+              complete(nil, nil)
+            rescue Exception => e # rubocop:disable Lint/RescueException
+              complete(nil, e)
+            ensure
+              Runtime.current_task = previous
+            end
+          end
+
+          @fiber ||= scheduled if scheduled.is_a?(Fiber)
+          self
+        end
+
+        def wait(timeout = nil)
+          unless @done
+            deadline = Runtime.deadline_for(timeout)
+            raise TimeoutError, 'task did not finish in time' unless join(deadline)
+          end
+
+          raise @error if @error
+
+          @result
+        end
+
+        def with_timeout(duration, &block)
+          Runtime.with_timeout(duration, &block)
+        end
+
+        def stop
+          return false if @done
+
+          @cancelled = true
+
+          return true if interrupt_fiber
+          return true if release_blocker
+
+          false
+        end
+
+        def cancelled? = @cancelled
+        def finished? = @done
+        def waiting = @waiters ? @waiters.size : 0
+
+        def raise_if_cancelled!
+          raise Cancel, 'task stopped' if @cancelled
+        end
+
+        def enter_block(waiter)
+          @blocker = waiter
+        end
+
+        def exit_block
+          @blocker = nil
+        end
+
+        private
+
+        def adopt_fiber(fiber)
+          @fiber ||= fiber
+        end
+
+        def interrupt_fiber
+          fiber = @fiber
+          scheduler = @scheduler
+          return false unless fiber&.alive?
+          return false unless scheduler&.respond_to?(:fiber_interrupt)
+
+          result = scheduler.fiber_interrupt(fiber, Cancel.new('task stopped'))
+          result != false
+        rescue FiberError
+          false
+        end
+
+        def release_blocker
+          waiter = @blocker or return false
+
+          Runtime.wake(waiter, waiter[:blocker]) && waiter[:fiber].alive?
+        end
+
+        def complete(result, error)
+          return if @done
+
+          @result = result
+          @error = error
+          @done = true
+
+          Runtime.report_error(self, error, @on_error) if error && waiting.zero?
+          @group&.release(self)
+          wake_waiters
+        end
+
+        def join(deadline)
+          waiters = (@waiters ||= [])
+          Runtime.with_waiter(self, waiters) do |waiter|
+            Runtime.park(self, waiter, deadline) { @done }
+          end
+        end
+
+        def wake_waiters
+          return if @waiters.nil? || @waiters.empty?
+
+          pending, @waiters = @waiters, []
+          pending.each { |waiter| Runtime.wake_dequeued(waiter, self) }
+        end
+      end
+    end
+  end
+end

data/lib/async/background/runtime/task_group.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Async
+  module Background
+    module Runtime
+      class TaskGroup
+        attr_accessor :on_release
+
+        def initialize(on_error: UNSET, on_release: nil)
+          @members = {}
+          @drained = Notification.new
+          @on_error = on_error
+          @on_release = on_release
+        end
+
+        def size = @members.size
+        def empty? = @members.empty?
+        def tasks = @members.keys
+
+        def spawn(name: nil, &block)
+          raise ArgumentError, 'block required' unless block
+
+          task = Task.new(name: name, group: self, on_error: @on_error)
+          @members[task] = true
+
+          begin
+            task.start(&block)
+          rescue Exception
+            release(task)
+            raise
+          end
+
+          task
+        end
+
+        def wait(timeout = nil)
+          deadline = Runtime.deadline_for(timeout)
+
+          until @members.empty?
+            raise TimeoutError, 'tasks did not finish in time' unless @drained.wait_until(deadline)
+          end
+
+          true
+        end
+
+        def stop_all(grace = nil)
+          tasks.each do |task|
+            task.stop
+          rescue StandardError
+            nil
+          end
+
+          return @members.empty? if grace.nil?
+
+          begin
+            wait(grace)
+          rescue TimeoutError
+            false
+          end
+        end
+
+        def release(task)
+          return unless @members.delete(task)
+
+          @drained.signal_all if @members.empty?
+          notify_release(task)
+        end
+
+        private
+
+        def notify_release(task)
+          handler = @on_release or return
+
+          handler.call(task)
+        rescue StandardError => error
+          Runtime.report_error(task, error, @on_error)
+        end
+      end
+    end
+  end
+end