async-background 1.0.1 → 1.1.0

data/lib/async/background/queue/socket_waker.rb

@@ -1,101 +1,167 @@
 # frozen_string_literal: true
 
 require 'socket'
-require 'async/notification'
 require 'fileutils'
+require_relative '../runtime'
 
 module Async
   module Background
     module Queue
       class SocketWaker
+        CLOSE_GRACE = 2
+
         attr_reader :path
 
         def initialize(path)
           @path = path
           @server = nil
-          @notification = ::Async::Notification.new
+          @notification = Runtime::Notification.new
           @running = false
           @accept_task = nil
+          @clients = Runtime::TaskGroup.new
+          @sockets = {}
         end
 
         def open!
-          cleanup_stale_socket
           ensure_directory
+          cleanup_stale_socket
           @server = UNIXServer.new(@path)
-          File.chmod(0600, @path)
           @running = true
+          self
         rescue Errno::EADDRINUSE
           raise "Socket #{@path} is already in use by another process"
         end
 
-        def start_accept_loop(parent_task)
-          @accept_task = parent_task.async do |task|
-            while @running
-              begin
-                client = @server.accept_nonblock
-                handle_client(task, client)
-              rescue IO::WaitReadable
-                @server.wait_readable
-              rescue Errno::EBADF, IOError
-                break
-              rescue => e
-                Console.logger.error(self) { "SocketWaker accept error: #{e.class} #{e.message}" }
-              end
-            end
-          rescue => e
-            Console.logger.error(self) { "SocketWaker loop crashed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}" }
-          ensure
-            @accept_task = nil
-          end
+        def start_accept_loop(_parent_task = nil)
+          @accept_task = Runtime.spawn(name: 'socket-waker-accept') { accept_loop }
         end
 
         def wait(timeout: nil)
-          if timeout
-            ::Async::Task.current.with_timeout(timeout) { @notification.wait }
-          else
-            @notification.wait
-          end
-        rescue ::Async::TimeoutError
-          # Timeout is normal - listener will fall back to polling
+          @notification.wait(timeout)
         end
 
         def signal
-          @notification.signal
+          @notification.signal_all
+          true
         end
 
         def close
+          return unless @running || @server
+
           @running = false
-          if @accept_task && !@accept_task.finished?
-            @accept_task.stop rescue nil
-          end
+          stop_accept_loop
+          @notification.signal_all
+          hang_up_clients
+          stop_clients
 
           @server&.close rescue nil
           @server = nil
+          @accept_task = nil
           File.unlink(@path) rescue nil
         end
 
         private
 
-        def handle_client(parent_task, client)
-          parent_task.async do
+        def accept_loop
+          while @running
             begin
-              loop do
-                client.read_nonblock(256)
-              rescue IO::WaitReadable
-                client.wait_readable
-                retry
-              rescue EOFError, Errno::ECONNRESET
-                break
-              end
-            rescue => e
-              Console.logger.warn(self) { "SocketWaker client handler error: #{e.class} #{e.message}" }
-            ensure
-              client.close rescue nil
-              @notification.signal
+              client = @server.accept_nonblock
+            rescue IO::WaitReadable
+              @server.wait_readable
+              next
+            rescue Errno::EBADF, IOError
+              break
+            rescue StandardError => e
+              Console.logger.error(self) { "SocketWaker accept error: #{e.class} #{e.message}" }
+              next
             end
+
+            break unless @running
+
+            handle_client(client)
+          end
+        rescue StandardError => e
+          Console.logger.error(self) { "SocketWaker loop crashed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}" }
+        end
+
+        def handle_client(client)
+          @sockets[client] = true
+
+          @clients.spawn(name: 'socket-waker-client') do
+            loop do
+              client.read_nonblock(256)
+              @notification.signal_all
+            rescue IO::WaitReadable
+              client.wait_readable
+              retry
+            rescue EOFError, Errno::ECONNRESET, Errno::EBADF, IOError
+              break
+            end
+          rescue StandardError => e
+            Console.logger.warn(self) { "SocketWaker client handler error: #{e.class} #{e.message}" }
+          ensure
+            @sockets.delete(client)
+            client.close rescue nil
+            @notification.signal_all
+          end
+        end
+
+        def stop_accept_loop
+          task = @accept_task or return
+
+          wake_accept_loop
+          return if await(task, CLOSE_GRACE)
+
+          task.stop
+          await(task, CLOSE_GRACE)
+        end
+
+        def wake_accept_loop
+          return unless @server
+
+          UNIXSocket.open(@path) { |s| s.write_nonblock("\x00") rescue nil }
+        rescue StandardError
+          nil
+        end
+
+        def hang_up_clients
+          @sockets.keys.each do |socket|
+            socket.close
+          rescue StandardError
+            nil
           end
         end
 
+        def stop_clients
+          return if @clients.empty?
+
+          return if await_group(@clients, CLOSE_GRACE)
+
+          @clients.stop_all(CLOSE_GRACE)
+        end
+
+        def await(task, grace)
+          return true unless Runtime.scheduler
+
+          task.wait(grace)
+          true
+        rescue Runtime::TimeoutError
+          false
+        rescue Exception # rubocop:disable Lint/RescueException
+          true
+        end
+
+        def await_group(group, grace)
+          return true unless Runtime.scheduler
+
+          group.wait(grace)
+          true
+        rescue Runtime::TimeoutError
+          false
+        rescue StandardError
+          true
+        end
+
         def cleanup_stale_socket
           return unless File.exist?(@path)
 

data/lib/async/background/queue/sql.rb

@@ -9,7 +9,8 @@ module Async
         BUSY_TIMEOUT = 'PRAGMA busy_timeout'.freeze
         TABLE_INFO = 'PRAGMA table_info(jobs)'.freeze
         OPTIMIZE = 'PRAGMA optimize'.freeze
-        INCREMENTAL_VACUUM = 'PRAGMA incremental_vacuum'.freeze
+        INCREMENTAL_VACUUM_PAGES = 64
+        INCREMENTAL_VACUUM = "PRAGMA incremental_vacuum(#{INCREMENTAL_VACUUM_PAGES})".freeze
         AUTO_VACUUM_INCREMENTAL = 'PRAGMA auto_vacuum = INCREMENTAL'.freeze
         BEGIN_IMMEDIATE = 'BEGIN IMMEDIATE'.freeze
         COMMIT = 'COMMIT'.freeze

data/lib/async/background/queue/store.rb

@@ -22,6 +22,7 @@ module Async
         CLEANUP_INTERVAL = 300
         CLEANUP_AGE = 3600
         FAILED_RETENTION_AGE = 7 * 24 * 3600
+        CLEANUP_VACUUM_THRESHOLD = 100
         ERROR_MESSAGE_MAX_LEN = 2_000
         EMPTY_ARGS_JSON = '[]'.freeze
 
@@ -80,7 +81,13 @@ module Async
         def enqueue(class_name, args = EMPTY_ARGS, run_at = nil, options: EMPTY_OPTIONS)
           ensure_connection
           now = realtime_now
-          @enqueue_stmt.execute(class_name, dump_args(args), dump_options(options), now, run_at || now)
+          stepped(@enqueue_stmt) do |statement|
+            statement.bind_param(1, class_name)
+            statement.bind_param(2, dump_args(args))
+            statement.bind_param(3, dump_options(options))
+            statement.bind_param(4, now)
+            statement.bind_param(5, run_at || now)
+          end
           @db.last_insert_row_id
         end
 
@@ -90,9 +97,14 @@ module Async
           now = realtime_now
 
           row = transaction do
-            with_statement(@fetch_stmt) { |statement| statement.execute(worker_id, now, token, now).first }
+            stepped(@fetch_stmt) do |statement|
+              statement.bind_param(1, worker_id)
+              statement.bind_param(2, now)
+              statement.bind_param(3, token)
+              statement.bind_param(4, now)
+            end
           end
-          return unless row
+          return if row.nil? || row.empty?
 
           maybe_cleanup
           job_from_row(row, token)
@@ -100,26 +112,28 @@ module Async
 
         def mark_started!(job_id, claim_token:, started_at: realtime_now)
           ensure_connection
-          @mark_started_stmt.execute(started_at, job_id, claim_token)
+          stepped(@mark_started_stmt) do |statement|
+            statement.bind_param(1, started_at)
+            statement.bind_param(2, job_id)
+            statement.bind_param(3, claim_token)
+          end
           @db.changes.positive?
         end
 
         def complete(job_id, claim_token:, finished_at: realtime_now, duration_ms: nil)
           ensure_connection
-          @complete_stmt.execute(finished_at, duration_ms, job_id, claim_token)
+          stepped(@complete_stmt) do |statement|
+            statement.bind_param(1, finished_at)
+            statement.bind_param(2, duration_ms)
+            statement.bind_param(3, job_id)
+            statement.bind_param(4, claim_token)
+          end
           @db.changes.positive?
         end
 
         def fail(job_id, claim_token:, error_class: nil, error_message: nil, finished_at: realtime_now, duration_ms: nil)
           ensure_connection
-          @fail_stmt.execute(
-            finished_at,
-            duration_ms,
-            error_class&.to_s,
-            truncate_message(error_message),
-            job_id,
-            claim_token
-          )
+          bind_failure(@fail_stmt, finished_at, duration_ms, error_class, error_message, job_id, claim_token)
           @db.changes.positive?
         end
 
@@ -146,13 +160,13 @@ module Async
 
         def recover(worker_id)
           ensure_connection
-          @requeue_stmt.execute(worker_id)
+          stepped(@requeue_stmt) { |statement| statement.bind_param(1, worker_id) }
           @db.changes
         end
 
         def next_pending_run_at
           ensure_connection
-          with_statement(@next_pending_stmt) { |statement| statement.execute.first&.first }
+          stepped(@next_pending_stmt)&.first
         end
 
         def data_version
@@ -243,9 +257,11 @@ module Async
         end
 
         def stored_options_for(job_id, claim_token)
-          with_statement(@retry_state_stmt) do |statement|
-            load_options(statement.execute(job_id, claim_token).first&.first)
+          row = stepped(@retry_state_stmt) do |statement|
+            statement.bind_param(1, job_id)
+            statement.bind_param(2, claim_token)
           end
+          load_options(row&.first)
         end
 
         def retry_policy(stored_options, fallback_options)
@@ -260,29 +276,33 @@ module Async
 
         def retry_job!(job_id, claim_token, policy, error_class, error_message)
           advanced = policy.with_attempt(policy.next_attempt)
-          @retry_stmt.execute(
-            realtime_now + advanced.next_retry_delay(advanced.attempt),
-            dump_options(advanced.to_h.compact),
-            error_class&.to_s,
-            truncate_message(error_message),
-            job_id,
-            claim_token
-          )
+          stepped(@retry_stmt) do |statement|
+            statement.bind_param(1, realtime_now + advanced.next_retry_delay(advanced.attempt))
+            statement.bind_param(2, dump_options(advanced.to_h.compact))
+            statement.bind_param(3, error_class&.to_s)
+            statement.bind_param(4, truncate_message(error_message))
+            statement.bind_param(5, job_id)
+            statement.bind_param(6, claim_token)
+          end
           @db.changes.positive? ? :retried : nil
         end
 
         def fail_job!(job_id, claim_token, error_class, error_message, finished_at, duration_ms)
-          @fail_stmt.execute(
-            finished_at,
-            duration_ms,
-            error_class&.to_s,
-            truncate_message(error_message),
-            job_id,
-            claim_token
-          )
+          bind_failure(@fail_stmt, finished_at, duration_ms, error_class, error_message, job_id, claim_token)
           @db.changes.positive? ? :failed : nil
         end
 
+        def bind_failure(statement, finished_at, duration_ms, error_class, error_message, job_id, claim_token)
+          stepped(statement) do |s|
+            s.bind_param(1, finished_at)
+            s.bind_param(2, duration_ms)
+            s.bind_param(3, error_class&.to_s)
+            s.bind_param(4, truncate_message(error_message))
+            s.bind_param(5, job_id)
+            s.bind_param(6, claim_token)
+          end
+        end
+
         def generate_claim_token = SecureRandom.hex(16)
 
         def truncate_message(message)
@@ -293,23 +313,30 @@ module Async
         end
 
         def lease_alive?(job_id, claim_token)
-          with_statement(@lease_check_stmt) do |statement|
-            !statement.execute(job_id, claim_token).first.nil?
-          end
+          !stepped(@lease_check_stmt) do |statement|
+            statement.bind_param(1, job_id)
+            statement.bind_param(2, claim_token)
+          end.nil?
         end
 
         def transaction
-          @db.execute(SQL::BEGIN_IMMEDIATE)
+          stepped(@begin_stmt)
           result = yield
-          @db.execute(SQL::COMMIT)
+          stepped(@commit_stmt)
           result
         rescue StandardError
-          @db.execute(SQL::ROLLBACK) rescue nil
+          begin
+            stepped(@rollback_stmt)
+          rescue StandardError
+            nil
+          end
           raise
         end
 
-        def with_statement(statement)
-          yield statement
+        def stepped(statement)
+          statement.reset!
+          yield statement if block_given?
+          statement.step
         ensure
           statement.reset! rescue nil
         end
@@ -333,19 +360,26 @@ module Async
           Job::Options.new(**options)
         end
 
+        STATEMENTS = {
+          :@enqueue_stmt => SQL::INSERT_JOB,
+          :@fetch_stmt => SQL::FETCH_NEXT_JOB,
+          :@mark_started_stmt => SQL::MARK_STARTED,
+          :@complete_stmt => SQL::COMPLETE_JOB,
+          :@fail_stmt => SQL::FAIL_JOB,
+          :@retry_state_stmt => SQL::RETRY_STATE,
+          :@lease_check_stmt => SQL::LEASE_ALIVE,
+          :@retry_stmt => SQL::RETRY_JOB,
+          :@requeue_stmt => SQL::RECOVER_WORKER,
+          :@cleanup_done_stmt => SQL::CLEANUP_DONE,
+          :@cleanup_failed_stmt => SQL::CLEANUP_FAILED,
+          :@next_pending_stmt => SQL::NEXT_PENDING_RUN_AT,
+          :@begin_stmt => SQL::BEGIN_IMMEDIATE,
+          :@commit_stmt => SQL::COMMIT,
+          :@rollback_stmt => SQL::ROLLBACK
+        }.freeze
+
         def prepare_statements
-          @enqueue_stmt = @db.prepare(SQL::INSERT_JOB)
-          @fetch_stmt = @db.prepare(SQL::FETCH_NEXT_JOB)
-          @mark_started_stmt = @db.prepare(SQL::MARK_STARTED)
-          @complete_stmt = @db.prepare(SQL::COMPLETE_JOB)
-          @fail_stmt = @db.prepare(SQL::FAIL_JOB)
-          @retry_state_stmt = @db.prepare(SQL::RETRY_STATE)
-          @lease_check_stmt = @db.prepare(SQL::LEASE_ALIVE)
-          @retry_stmt = @db.prepare(SQL::RETRY_JOB)
-          @requeue_stmt = @db.prepare(SQL::RECOVER_WORKER)
-          @cleanup_done_stmt = @db.prepare(SQL::CLEANUP_DONE)
-          @cleanup_failed_stmt = @db.prepare(SQL::CLEANUP_FAILED)
-          @next_pending_stmt = @db.prepare(SQL::NEXT_PENDING_RUN_AT)
+          STATEMENTS.each { |name, sql| instance_variable_set(name, @db.prepare(sql)) }
         end
 
         def finalize_statements
@@ -354,28 +388,11 @@ module Async
         end
 
         def statements
-          [
-            @enqueue_stmt,
-            @fetch_stmt,
-            @mark_started_stmt,
-            @complete_stmt,
-            @fail_stmt,
-            @retry_state_stmt,
-            @lease_check_stmt,
-            @retry_stmt,
-            @requeue_stmt,
-            @cleanup_done_stmt,
-            @cleanup_failed_stmt,
-            @next_pending_stmt
-          ]
+          STATEMENTS.keys.map { |name| instance_variable_get(name) }
         end
 
         def clear_statements
-          @enqueue_stmt = @fetch_stmt = @mark_started_stmt = nil
-          @complete_stmt = @fail_stmt = @retry_state_stmt = @lease_check_stmt = nil
-          @retry_stmt = @requeue_stmt = nil
-          @cleanup_done_stmt = @cleanup_failed_stmt = nil
-          @next_pending_stmt = nil
+          STATEMENTS.each_key { |name| instance_variable_set(name, nil) }
         end
 
         def maybe_cleanup
@@ -387,9 +404,15 @@ module Async
         end
 
         def cleanup_finished_jobs(now)
-          @cleanup_done_stmt.execute(now - CLEANUP_AGE)
-          @cleanup_failed_stmt.execute(now - FAILED_RETENTION_AGE)
-          @db.execute(SQL::INCREMENTAL_VACUUM) if @db.changes > 100
+          deleted = 0
+
+          stepped(@cleanup_done_stmt) { |statement| statement.bind_param(1, now - CLEANUP_AGE) }
+          deleted += @db.changes
+          stepped(@cleanup_failed_stmt) { |statement| statement.bind_param(1, now - FAILED_RETENTION_AGE) }
+          deleted += @db.changes
+
+          @db.execute(SQL::INCREMENTAL_VACUUM) if deleted > CLEANUP_VACUUM_THRESHOLD
+          deleted
         end
       end
     end

data/lib/async/background/runner/queue_execution.rb

@@ -9,6 +9,7 @@ module Async
         private
 
         def setup_queue(queue_socket_dir, queue_db_path, queue_mmap)
+          @queue_saturated = false
           @listen_queue = !!queue_socket_dir && !isolated_worker?
           return unless @listen_queue
 
@@ -26,20 +27,36 @@ module Async
           recover_queue_jobs
         end
 
-        def start_queue_listener(task)
-          @queue_waker.start_accept_loop(task)
+        def start_queue_listener
+          @queue_waker.start_accept_loop
 
-          task.async do
+          @services.spawn(name: 'queue-listener') do
             logger.info { "Async::Background queue: listening on worker #{worker_index}" }
 
+            failures = 0
+
             while running?
-              @queue_waker.wait(timeout: next_wait_timeout)
-              dispatch_available_queue_jobs
+              begin
+                @queue_waker.wait(timeout: next_wait_timeout)
+                break unless running?
+
+                dispatch_available_queue_jobs
+                failures = 0
+              rescue StandardError => error
+                failures += 1
+                backoff = [QUEUE_ERROR_BACKOFF * failures, QUEUE_POLL_INTERVAL].min
+                logger.error('Async::Background') do
+                  "queue listener: #{error.class} #{error.message}; retrying in #{backoff}s"
+                end
+                shutdown.wait(backoff)
+              end
             end
           end
         end
 
         def next_wait_timeout
+          return QUEUE_POLL_INTERVAL if @queue_saturated
+
           next_due = @queue_store.next_pending_run_at
           return QUEUE_POLL_INTERVAL unless next_due
 
@@ -68,7 +85,7 @@ module Async
           complete_queue_job!(job, class_name, claim_token, started_at)
         rescue ConfigError => error
           record_invalid_queue_job!(job, class_name, claim_token, error)
-        rescue ::Async::TimeoutError => error
+        rescue Runtime::TimeoutError => error
           handle_queue_failure(
             job,
             options,
@@ -129,11 +146,18 @@ module Async
         end
 
         def dispatch_available_queue_jobs
+          @queue_saturated = false
+
           while running?
+            if @jobs.size >= @semaphore.limit
+              @queue_saturated = true
+              return
+            end
+
             job = @queue_store.fetch(worker_index)
-            break unless job
+            return if job.nil?
 
-            semaphore.async { |job_task| run_queue_job(job_task, job) }
+            spawn_job { |job_task| run_queue_job(job_task, job) }
           end
         end
 

data/lib/async/background/runner/schedule.rb

@@ -59,12 +59,15 @@ module Async
         end
 
         def build_entries(schedule, now)
-          schedule.each_with_object(MinHeap.new) do |(name, config), heap|
-            next unless assigned_worker(config, name) == worker_index
+          schedule
+            .filter_map { |name, config| entry_for(name, config, now) }
+            .each_with_object(MinHeap.new) { |entry, heap| heap.push(entry) }
+        end
 
-            task = build_task_config(name, config)
-            heap.push(build_entry(name, task, now))
-          end
+        def entry_for(name, config, now)
+          return unless assigned_worker(config, name) == worker_index
+
+          build_entry(name, build_task_config(name, config), now)
         end
 
         def assigned_worker(config, name)
@@ -98,18 +101,18 @@ module Async
         end
 
         def parse_interval(name, value)
-          value&.then do |interval|
-            interval = interval.to_i
-            raise ConfigError, "[#{name}] 'every' must be > 0" unless interval.positive?
+          return if value.nil?
 
-            interval
-          end
+          interval = value.to_i
+          raise ConfigError, "[#{name}] 'every' must be > 0" unless interval.positive?
+
+          interval
         end
 
         def parse_cron(name, value)
-          value&.then do |expression|
-            Fugit::Cron.new(expression) || raise(ConfigError, "[#{name}] invalid cron: #{expression}")
-          end
+          return if value.nil?
+
+          Fugit::Cron.new(value) || raise(ConfigError, "[#{name}] invalid cron: #{value}")
         end
 
         def parse_timeout(name, config)