async-background 1.1.0 → 1.1.1

data/lib/async/background/queue/sql.rb

@@ -4,17 +4,17 @@ module Async
   module Background
     module Queue
       module SQL
-        USER_VERSION = 'PRAGMA user_version'.freeze
-        DATA_VERSION = 'PRAGMA data_version'.freeze
-        BUSY_TIMEOUT = 'PRAGMA busy_timeout'.freeze
-        TABLE_INFO = 'PRAGMA table_info(jobs)'.freeze
-        OPTIMIZE = 'PRAGMA optimize'.freeze
+        USER_VERSION = 'PRAGMA user_version'
+        DATA_VERSION = 'PRAGMA data_version'
+        BUSY_TIMEOUT = 'PRAGMA busy_timeout'
+        TABLE_INFO = 'PRAGMA table_info(jobs)'
+        OPTIMIZE = 'PRAGMA optimize'
         INCREMENTAL_VACUUM_PAGES = 64
         INCREMENTAL_VACUUM = "PRAGMA incremental_vacuum(#{INCREMENTAL_VACUUM_PAGES})".freeze
-        AUTO_VACUUM_INCREMENTAL = 'PRAGMA auto_vacuum = INCREMENTAL'.freeze
-        BEGIN_IMMEDIATE = 'BEGIN IMMEDIATE'.freeze
-        COMMIT = 'COMMIT'.freeze
-        ROLLBACK = 'ROLLBACK'.freeze
+        AUTO_VACUUM_INCREMENTAL = 'PRAGMA auto_vacuum = INCREMENTAL'
+        BEGIN_IMMEDIATE = 'BEGIN IMMEDIATE'
+        COMMIT = 'COMMIT'
+        ROLLBACK = 'ROLLBACK'
 
         def self.busy_timeout(milliseconds)
           "PRAGMA busy_timeout = #{Integer(milliseconds)}"
@@ -28,7 +28,7 @@ module Async
           "ALTER TABLE jobs ADD COLUMN #{name} #{sql_type}"
         end
 
-        CREATE_SCHEMA = <<~SQL.freeze
+        CREATE_SCHEMA = <<~SQL
           CREATE TABLE IF NOT EXISTS jobs (
             id                 INTEGER PRIMARY KEY,
             class_name         TEXT    NOT NULL,
@@ -50,12 +50,12 @@ module Async
             ON jobs(run_at) WHERE status = 'pending';
         SQL
 
-        INSERT_JOB = <<~SQL.freeze
+        INSERT_JOB = <<~SQL
           INSERT INTO jobs (class_name, args, options, created_at, run_at)
           VALUES (?, ?, ?, ?, ?)
         SQL
 
-        FETCH_NEXT_JOB = <<~SQL.freeze
+        FETCH_NEXT_JOB = <<~SQL
           UPDATE jobs
           SET    status      = 'running',
                  locked_by   = ?,
@@ -73,13 +73,13 @@ module Async
           RETURNING id, class_name, args, options
         SQL
 
-        MARK_STARTED = <<~SQL.freeze
+        MARK_STARTED = <<~SQL
           UPDATE jobs
           SET    started_at = ?
           WHERE  id = ? AND claim_token = ? AND status = 'running' AND started_at IS NULL
         SQL
 
-        COMPLETE_JOB = <<~SQL.freeze
+        COMPLETE_JOB = <<~SQL
           UPDATE jobs
           SET    status      = 'done',
                  locked_by   = NULL,
@@ -89,7 +89,7 @@ module Async
           WHERE  id = ? AND claim_token = ? AND status = 'running'
         SQL
 
-        FAIL_JOB = <<~SQL.freeze
+        FAIL_JOB = <<~SQL
           UPDATE jobs
           SET    status             = 'failed',
                  locked_by          = NULL,
@@ -101,19 +101,19 @@ module Async
           WHERE  id = ? AND claim_token = ? AND status = 'running'
         SQL
 
-        RETRY_STATE = <<~SQL.freeze
+        RETRY_STATE = <<~SQL
           SELECT options
           FROM jobs
           WHERE id = ? AND claim_token = ? AND status = 'running'
         SQL
 
-        LEASE_ALIVE = <<~SQL.freeze
+        LEASE_ALIVE = <<~SQL
           SELECT 1
           FROM jobs
           WHERE id = ? AND claim_token = ? AND status = 'running'
         SQL
 
-        RETRY_JOB = <<~SQL.freeze
+        RETRY_JOB = <<~SQL
           UPDATE jobs
           SET    status             = 'pending',
                  locked_by          = NULL,
@@ -129,7 +129,7 @@ module Async
           WHERE  id = ? AND claim_token = ? AND status = 'running'
         SQL
 
-        RECOVER_WORKER = <<~SQL.freeze
+        RECOVER_WORKER = <<~SQL
           UPDATE jobs
           SET    status      = 'pending',
                  locked_by   = NULL,
@@ -139,67 +139,67 @@ module Async
           WHERE  status = 'running' AND locked_by = ?
         SQL
 
-        BACKFILL_FINISHED_AT = <<~SQL.freeze
+        BACKFILL_FINISHED_AT = <<~SQL
           UPDATE jobs
           SET finished_at = created_at
           WHERE finished_at IS NULL AND status IN ('done', 'failed')
         SQL
 
-        CLEANUP_DONE = <<~SQL.freeze
+        CLEANUP_DONE = <<~SQL
           DELETE FROM jobs
           WHERE status = 'done' AND finished_at IS NOT NULL AND finished_at < ?
         SQL
 
-        CLEANUP_FAILED = <<~SQL.freeze
+        CLEANUP_FAILED = <<~SQL
           DELETE FROM jobs
           WHERE status = 'failed' AND finished_at IS NOT NULL AND finished_at < ?
         SQL
 
-        NEXT_PENDING_RUN_AT = <<~SQL.freeze
+        NEXT_PENDING_RUN_AT = <<~SQL
           SELECT MIN(run_at)
           FROM jobs
           WHERE status = 'pending'
         SQL
 
-        JOBS_TABLE_EXISTS = <<~SQL.freeze
+        JOBS_TABLE_EXISTS = <<~SQL
           SELECT 1
           FROM sqlite_master
           WHERE type = 'table' AND name = 'jobs'
           LIMIT 1
         SQL
 
-        JOB_INDEX_NAMES = <<~SQL.freeze
+        JOB_INDEX_NAMES = <<~SQL
           SELECT name
           FROM sqlite_master
           WHERE type = 'index' AND tbl_name = 'jobs'
         SQL
 
-        DROP_LEGACY_PENDING_INDEX = 'DROP INDEX IF EXISTS idx_jobs_status_run_at_id'.freeze
+        DROP_LEGACY_PENDING_INDEX = 'DROP INDEX IF EXISTS idx_jobs_status_run_at_id'
 
-        CREATE_PENDING_INDEX = <<~SQL.freeze
+        CREATE_PENDING_INDEX = <<~SQL
           CREATE INDEX IF NOT EXISTS idx_jobs_pending
           ON jobs(run_at) WHERE status = 'pending'
         SQL
 
-        CREATE_DONE_INDEX = <<~SQL.freeze
+        CREATE_DONE_INDEX = <<~SQL
           CREATE INDEX IF NOT EXISTS idx_jobs_done_finished_at
           ON jobs(finished_at DESC, id DESC)
           WHERE status = 'done'
         SQL
 
-        CREATE_FAILED_INDEX = <<~SQL.freeze
+        CREATE_FAILED_INDEX = <<~SQL
           CREATE INDEX IF NOT EXISTS idx_jobs_failed_finished_at
           ON jobs(finished_at DESC, id DESC)
           WHERE status = 'failed'
         SQL
 
-        CREATE_EXECUTING_INDEX = <<~SQL.freeze
+        CREATE_EXECUTING_INDEX = <<~SQL
           CREATE INDEX IF NOT EXISTS idx_jobs_executing_started_at
           ON jobs(started_at)
           WHERE status = 'running' AND started_at IS NOT NULL
         SQL
 
-        CREATE_CLAIMED_INDEX = <<~SQL.freeze
+        CREATE_CLAIMED_INDEX = <<~SQL
           CREATE INDEX IF NOT EXISTS idx_jobs_claimed_locked_at
           ON jobs(locked_at)
           WHERE status = 'running' AND started_at IS NULL

data/lib/async/background/queue/store.rb

@@ -24,7 +24,7 @@ module Async
         FAILED_RETENTION_AGE = 7 * 24 * 3600
         CLEANUP_VACUUM_THRESHOLD = 100
         ERROR_MESSAGE_MAX_LEN = 2_000
-        EMPTY_ARGS_JSON = '[]'.freeze
+        EMPTY_ARGS_JSON = '[]'
 
         attr_reader :path, :options
 
@@ -131,7 +131,8 @@ module Async
           @db.changes.positive?
         end
 
-        def fail(job_id, claim_token:, error_class: nil, error_message: nil, finished_at: realtime_now, duration_ms: nil)
+        def fail(job_id, claim_token:, error_class: nil, error_message: nil, finished_at: realtime_now,
+                 duration_ms: nil)
           ensure_connection
           bind_failure(@fail_stmt, finished_at, duration_ms, error_class, error_message, job_id, claim_token)
           @db.changes.positive?
@@ -153,8 +154,11 @@ module Async
             next unless lease_alive?(job_id, claim_token)
 
             policy = retry_policy(stored_options, fallback_options)
-            policy_retries?(policy) ? retry_job!(job_id, claim_token, policy, error_class, error_message) :
+            if policy_retries?(policy)
+              retry_job!(job_id, claim_token, policy, error_class, error_message)
+            else
               fail_job!(job_id, claim_token, error_class, error_message, finished_at, duration_ms)
+            end
           end
         end
 
@@ -178,7 +182,11 @@ module Async
           return unless connected?
 
           finalize_statements
-          @db.execute(SQL::OPTIMIZE) rescue nil
+          begin
+            @db.execute(SQL::OPTIMIZE)
+          rescue StandardError
+            nil
+          end
           @db.close
           @db = nil
           @schema_checked = false
@@ -211,7 +219,7 @@ module Async
           require 'sqlite3'
         rescue LoadError
           raise LoadError,
-                "sqlite3 gem is required for Async::Background::Queue. " \
+                'sqlite3 gem is required for Async::Background::Queue. ' \
                 "Add `gem 'sqlite3', '~> 2.0'` to your Gemfile."
         end
 
@@ -338,7 +346,11 @@ module Async
           yield statement if block_given?
           statement.step
         ensure
-          statement.reset! rescue nil
+          begin
+            statement.reset!
+          rescue StandardError
+            nil
+          end
         end
 
         def dump_args(args)
@@ -383,7 +395,11 @@ module Async
         end
 
         def finalize_statements
-          statements.each { |statement| statement&.close rescue nil }
+          statements.each do |statement|
+            statement&.close
+          rescue StandardError
+            nil
+          end
           clear_statements
         end
 

data/lib/async/background/runner/lifecycle.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module Async
+  module Background
+    class Runner
+      module Lifecycle
+        private
+
+        def setup_signal_handlers
+          @signal_r, @signal_w = IO.pipe
+
+          %w[INT TERM].each do |signal|
+            Signal.trap(signal) { stop_from_signal }
+          end
+        end
+
+        def stop_from_signal
+          @running = false
+          @signal_w.write_nonblock('.')
+        rescue StandardError
+          nil
+        end
+
+        def start_signal_watcher
+          @services.spawn(name: 'signal-watcher') { watch_signals }
+        end
+
+        def watch_signals
+          while running?
+            result = read_signal
+            next if result == :wait
+            break unless result
+
+            logger.info { 'Async::Background: stopping gracefully' } unless running?
+            shutdown.signal_all
+            @queue_waker&.signal
+          end
+        end
+
+        def read_signal
+          @signal_r.wait_readable
+          @signal_r.read_nonblock(256)
+        rescue IO::WaitReadable
+          :wait
+        rescue IOError, Errno::EBADF
+          false
+        end
+
+        def error_handler
+          @error_handler ||= build_error_handler
+        end
+
+        def build_error_handler
+          log = logger
+          lambda do |task, error|
+            log.error('Async::Background') do
+              "task #{task&.name || 'unnamed'} died: #{error.class} #{error.message}\n" \
+                "#{Array(error.backtrace).join("\n")}"
+            end
+          end
+        end
+
+        def warn_unsafe_timeouts
+          return if Runtime.native_timeouts?
+
+          logger.warn('Async::Background') do
+            "#{Fiber.scheduler.class} does not implement #timeout_after; job timeouts " \
+              'fall back to stdlib Timeout and may interrupt an unrelated fiber'
+          end
+        end
+
+        def wake_signal_watcher
+          @signal_w&.write_nonblock('.')
+        rescue StandardError
+          nil
+        end
+
+        def shutdown_gracefully
+          wake_services
+          drain_jobs
+          stop_services
+          close_queue
+          close_signal_pipe
+        end
+
+        def wake_services
+          shutdown.signal_all
+          wake_signal_watcher
+          @queue_waker&.signal
+        end
+
+        def drain_jobs
+          return if @jobs.empty?
+
+          logger.info { "Async::Background: draining #{@jobs.size} in-flight job(s)" }
+          @jobs.wait(@drain_timeout)
+        rescue Runtime::TimeoutError
+          cancel_draining_jobs
+        end
+
+        def cancel_draining_jobs
+          logger.warn('Async::Background') do
+            "#{@jobs.size} job(s) still running after #{@drain_timeout}s, cancelling"
+          end
+          @jobs.stop_all(JOB_CANCEL_GRACE)
+        end
+
+        def stop_services
+          return if @services.empty?
+
+          @services.wait(SERVICE_SHUTDOWN_GRACE)
+        rescue Runtime::TimeoutError
+          logger.warn('Async::Background') { 'service tasks did not exit in time, cancelling' }
+          @services.stop_all(SERVICE_CANCEL_GRACE)
+        end
+
+        def close_queue
+          @queue_waker&.close
+          @queue_store&.close
+        end
+
+        def close_signal_pipe
+          @signal_r&.close
+          @signal_w&.close
+        rescue IOError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/async/background/runner/queue_execution.rb

@@ -6,11 +6,15 @@ module Async
       # Queue-only lifecycle. Keeping it separate from recurring scheduling makes
       # the different delivery guarantees visible without adding runtime objects.
       module QueueExecution
+        QueueRun = Data.define(:job, :class_name, :claim_token, :options, :started_at, :job_class)
+        QueueFailure = Data.define(:job, :options, :message, :error, :duration, :timeout, :backtrace)
+        private_constant :QueueRun, :QueueFailure
+
         private
 
         def setup_queue(queue_socket_dir, queue_db_path, queue_mmap)
           @queue_saturated = false
-          @listen_queue = !!queue_socket_dir && !isolated_worker?
+          @listen_queue = !queue_socket_dir.nil? && !isolated_worker?
           return unless @listen_queue
 
           require_relative '../queue/client'
@@ -19,7 +23,7 @@ module Async
 
           @queue_store = Queue::Store.new(
             path: queue_db_path || Queue::Store.default_path,
-            options: {mmap: queue_mmap}
+            options: { mmap: queue_mmap }
           )
 
           @queue_waker = Queue::SocketWaker.new(queue_socket_path(queue_socket_dir))
@@ -42,11 +46,11 @@ module Async
 
                 dispatch_available_queue_jobs
                 failures = 0
-              rescue StandardError => error
+              rescue StandardError => e
                 failures += 1
                 backoff = [QUEUE_ERROR_BACKOFF * failures, QUEUE_POLL_INTERVAL].min
                 logger.error('Async::Background') do
-                  "queue listener: #{error.class} #{error.message}; retrying in #{backoff}s"
+                  "queue listener: #{e.class} #{e.message}; retrying in #{backoff}s"
                 end
                 shutdown.wait(backoff)
               end
@@ -67,49 +71,56 @@ module Async
         end
 
         def run_queue_job(job_task, job)
-          class_name = job[:class_name]
-          claim_token = job[:claim_token]
-          options = nil
-          started_at = nil
+          context = QueueRun.new(job, job[:class_name], job[:claim_token], nil, nil, nil)
+          context = resolve_queue_run(context)
           metrics_started = false
+          metrics_started = start_queue_metrics(context)
+          return unless metrics_started
+
+          context = context.with(started_at: monotonic_now)
+          perform_queue_run(job_task, context)
+          complete_queue_job!(context)
+        rescue ConfigError => e
+          record_invalid_queue_job!(context, e)
+        rescue Runtime::TimeoutError => e
+          handle_queue_failure(queue_failure(context, e, timeout: true))
+        rescue StandardError => e
+          handle_queue_failure(queue_failure(context, e, timeout: false))
+        ensure
+          metrics.job_stopped(nil) if metrics_started
+        end
 
-          klass = resolve_job_class(class_name)
+        def resolve_queue_run(context)
+          job, class_name, = context.deconstruct
+          job_class = resolve_job_class(class_name)
           options = parse_job_options(job[:options])
-          return unless start_queue_job!(job, class_name, claim_token)
+          context.with(options: options, job_class: job_class)
+        end
+
+        def start_queue_metrics(context)
+          job, class_name, claim_token, = context.deconstruct
+          return false unless start_queue_job!(job, class_name, claim_token)
 
           metrics.job_started(nil)
-          metrics_started = true
-          started_at = monotonic_now
-          job_task.with_timeout(options.timeout) { klass.perform_now(*job[:args]) }
-
-          complete_queue_job!(job, class_name, claim_token, started_at)
-        rescue ConfigError => error
-          record_invalid_queue_job!(job, class_name, claim_token, error)
-        rescue Runtime::TimeoutError => error
-          handle_queue_failure(
-            job,
-            options,
-            "timed out after #{options&.timeout}s",
-            error: error,
-            duration: started_at && (monotonic_now - started_at),
-            timeout: true,
-            backtrace: nil
-          )
-        rescue StandardError => error
-          handle_queue_failure(
-            job,
-            options,
-            "#{error.class} #{error.message}",
-            error: error,
-            duration: started_at && (monotonic_now - started_at),
-            timeout: false,
-            backtrace: error.backtrace
-          )
-        ensure
-          metrics.job_stopped(nil) if metrics_started
+          true
         end
 
-        def handle_queue_failure(job, options, message, error:, duration:, timeout:, backtrace:)
+        def perform_queue_run(job_task, context)
+          job, _class_name, _claim_token, options, _started_at, job_class = context.deconstruct
+          job_task.with_timeout(options.timeout) do
+            job_class.perform_now(*job[:args])
+          end
+        end
+
+        def queue_failure(context, error, timeout:)
+          job, _class_name, _claim_token, options, started_at = context.deconstruct
+          message = timeout ? "timed out after #{options&.timeout}s" : "#{error.class} #{error.message}"
+          duration = started_at && (monotonic_now - started_at)
+          QueueFailure.new(job, options, message, error, duration, timeout, timeout ? nil : error.backtrace)
+        end
+
+        def handle_queue_failure(failure)
+          job, options, message, error, duration, timeout, backtrace = failure.deconstruct
           class_name = job[:class_name]
           result = @queue_store.retry_or_fail(
             job[:id],
@@ -122,14 +133,17 @@ module Async
           return log_stale_queue_failure(job, class_name, timeout) unless result
 
           timeout ? metrics.job_timed_out(nil) : metrics.job_failed(nil, error)
-          result == :retried ? log_queue_retry(job, class_name, message, options) :
+          if result == :retried
+            log_queue_retry(job, class_name, message, options)
+          else
             log_queue_failure(class_name, message, backtrace)
+          end
         end
 
         def parse_job_options(raw)
           Job::Options.new(**(raw || {}))
-        rescue ArgumentError, TypeError => error
-          raise ConfigError, "invalid queue options: #{error.message}"
+        rescue ArgumentError, TypeError => e
+          raise ConfigError, "invalid queue options: #{e.message}"
         end
 
         def isolated_worker?
@@ -164,25 +178,29 @@ module Async
         def start_queue_job!(job, class_name, claim_token)
           return true if @queue_store.mark_started!(job[:id], claim_token: claim_token)
 
-          logger.warn('Async::Background') {
+          logger.warn('Async::Background') do
             "queue(#{class_name}): lost lease before start for job #{job[:id]}, ignored"
-          }
+          end
           false
         end
 
-        def complete_queue_job!(job, class_name, claim_token, started_at)
+        def complete_queue_job!(context)
+          job, class_name, claim_token, _options, started_at = context.deconstruct
           duration = monotonic_now - started_at
           if @queue_store.complete(job[:id], claim_token: claim_token, duration_ms: duration_ms(duration))
             metrics.job_succeeded(nil, duration)
-            logger.info('Async::Background') { "queue(#{class_name}): completed in #{duration.round(2)}s" }
+            logger.info('Async::Background') do
+              "queue(#{class_name}): completed in #{duration.round(2)}s"
+            end
           else
-            logger.warn('Async::Background') {
+            logger.warn('Async::Background') do
               "queue(#{class_name}): complete: stale lease for job #{job[:id]}, ignored"
-            }
+            end
           end
         end
 
-        def record_invalid_queue_job!(job, class_name, claim_token, error)
+        def record_invalid_queue_job!(context, error)
+          job, class_name, claim_token, = context.deconstruct
           recorded = @queue_store.fail(
             job[:id],
             claim_token: claim_token,
@@ -190,7 +208,9 @@ module Async
             error_message: error.message
           )
           metrics.job_failed(nil, error) if recorded
-          logger.error('Async::Background') { "queue(#{class_name}): #{error.class} #{error.message}" }
+          logger.error('Async::Background') do
+            "queue(#{class_name}): #{error.class} #{error.message}"
+          end
         end
 
         def log_stale_queue_failure(job, class_name, timeout)
@@ -200,7 +220,7 @@ module Async
           end
         end
 
-        def log_queue_retry(job, class_name, message, options)
+        def log_queue_retry(_job, class_name, message, options)
           @queue_waker&.signal
           logger.warn('Async::Background') do
             "queue(#{class_name}): #{message}; retry #{options&.next_attempt}/#{options&.retry}"

data/lib/async/background/runner/schedule.rb

@@ -96,8 +96,8 @@ module Async
 
         def resolve_scheduled_job(name, class_name)
           resolve_job_class(class_name)
-        rescue ConfigError => error
-          raise ConfigError, "[#{name}] #{error.message}"
+        rescue ConfigError => e
+          raise ConfigError, "[#{name}] #{e.message}"
         end
 
         def parse_interval(name, value)
@@ -117,8 +117,8 @@ module Async
 
         def parse_timeout(name, config)
           Job::Options.new(timeout: config.fetch('timeout', DEFAULT_TIMEOUT)).timeout
-        rescue ArgumentError, TypeError => error
-          raise ConfigError, "[#{name}] #{error.message}"
+        rescue ArgumentError, TypeError => e
+          raise ConfigError, "[#{name}] #{e.message}"
         end
 
         def validate_schedule_frequency!(name, interval, cron)