async-background 0.3.0 → 0.4.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 00b7ba6035c743fb4c4568d2309da4bc7f454a3cbe5e17ac5d2251a417a4a31c
-  data.tar.gz: a3dff9b2b51f590a095120a18f1e3d5424b339ab5a6cff90b148f8e9316ee242
+  metadata.gz: c9a1ac97495dc7d6a39adf802a847b62e3601df0fe0ef4aa70dbcc248a2f77b8
+  data.tar.gz: 74bab717411caf3e8a7b469666e9df5f8f59d540ebd98446c5e44e167cacd3c9
 SHA512:
-  metadata.gz: b9c36893b1cb14ba251d9c5a1c614ac50b988ffcbd69c43a876c1e0a2995ee749592618c2e328d2e05573ccf46186616a59615d7c01a785b28fe5897a1e0650f
-  data.tar.gz: 3f47cf30226b45b6c543f48812e95778206e9cccc07445c5066e32cdf88a71f68bf88be9f0134750344cd7e0ed8df1a907107d0e5a3c910ce00a3a493ea700e7
+  metadata.gz: fe41a129f66a72c51016397b866ee9ff31b973d5e92ee6af1f61c05ce87f8f85f9824a68499caefcf950cdb33186c74749817e0ae8b89a483ede580d412e13fc
+  data.tar.gz: 92208ce41c49095cdf652b798ed892e4db5ae12c504cfe94c45bf124054256a94906fba14f68365f938a6d7289c76ca79d7ae7f6fac53bb76d8097d463db684f

data/lib/async/background/metrics.rb

@@ -23,10 +23,11 @@ module Async
         @enabled  = false
         @registry = ::Async::Utilization::Registry.new
         @enabled  = true
-
         ensure_shm!(total_workers, shm_path)
         attach_observer!(worker_index, total_workers, shm_path)
-      rescue LoadError
+      rescue LoadError, ArgumentError
+        @registry = nil
+        @enabled  = false
       end
 
       def enabled?

data/lib/async/background/queue/client.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Async
+  module Background
+    module Queue
+      # Usage:
+      #   Async::Background::Queue.enqueue(SendEmailJob, user_id, "welcome")
+      #
+      class Client
+        def initialize(store:, notifier: nil)
+          @store    = store
+          @notifier = notifier
+        end
+
+        def push(class_name, args = [])
+          id = @store.enqueue(class_name, args)
+          @notifier&.notify
+          id
+        end
+      end
+
+      class << self
+        attr_accessor :default_client
+
+        def enqueue(job_class, *args)
+          raise "Async::Background::Queue not configured" unless default_client
+
+          if job_class.is_a?(String)
+            class_name = job_class
+          else
+            raise ArgumentError, "#{job_class} must implement .perform_now" unless job_class.respond_to?(:perform_now)
+            class_name = job_class.name
+          end
+
+          default_client.push(class_name, args)
+        end
+      end
+    end
+  end
+end

data/lib/async/background/queue/notifier.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Async
+  module Background
+    module Queue
+      class Notifier
+        attr_reader :reader, :writer
+
+        def initialize
+          @reader, @writer = IO.pipe
+          @reader.binmode
+          @writer.binmode
+        end
+
+        def notify
+          @writer.write_nonblock("\x01")
+        rescue IO::WaitWritable, Errno::EAGAIN
+          # pipe buffer full — consumer is already behind, skip
+        rescue IOError
+          # pipe closed
+        end
+
+        def wait(timeout: nil)
+          @reader.wait_readable(timeout)
+          drain
+        end
+
+        def close_writer
+          @writer.close unless @writer.closed?
+        end
+
+        def close_reader
+          @reader.close unless @reader.closed?
+        end
+
+        def close
+          close_reader
+          close_writer
+        end
+
+        def for_producer!
+          close_reader
+        end
+
+        def for_consumer!
+          close_writer
+        end
+
+        private
+
+        def drain
+          loop do
+            @reader.read_nonblock(256)
+          rescue IO::WaitReadable, EOFError
+            break
+          end
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/async/background/queue/store.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Async
+  module Background
+    module Queue
+      class Store
+        SCHEMA = <<~SQL
+          PRAGMA auto_vacuum = INCREMENTAL;
+          CREATE TABLE IF NOT EXISTS jobs (
+            id         INTEGER PRIMARY KEY,
+            class_name TEXT    NOT NULL,
+            args       TEXT    NOT NULL DEFAULT '[]',
+            status     TEXT    NOT NULL DEFAULT 'pending',
+            created_at REAL    NOT NULL,
+            locked_by  INTEGER,
+            locked_at  REAL
+          );
+          CREATE INDEX IF NOT EXISTS idx_jobs_status_id ON jobs(status, id);
+        SQL
+
+        MMAP_SIZE = 268_435_456
+        PRAGMAS = ->(mmap_size) {
+          <<~SQL
+            PRAGMA journal_mode       = WAL;
+            PRAGMA synchronous        = NORMAL;
+            PRAGMA mmap_size          = #{mmap_size};
+            PRAGMA cache_size         = -16000;
+            PRAGMA temp_store         = MEMORY;
+            PRAGMA busy_timeout       = 5000;
+            PRAGMA journal_size_limit = 67108864;
+          SQL
+        }.freeze
+
+        CLEANUP_INTERVAL = 300
+        CLEANUP_AGE      = 3600
+
+        attr_reader :path
+
+        def initialize(path: self.class.default_path, mmap: true)
+          @path = path
+          @mmap = mmap
+          @db   = nil
+          @schema_checked = false
+          @last_cleanup_at = nil
+        end
+
+        def ensure_database!
+          require_sqlite3
+          db = SQLite3::Database.new(@path)
+          db.execute_batch(PRAGMAS.call(@mmap ? MMAP_SIZE : 0))
+          db.execute_batch(SCHEMA)
+          db.execute("PRAGMA wal_checkpoint(TRUNCATE)")
+          db.close
+          @schema_checked = true
+        end
+
+        def enqueue(class_name, args = [])
+          ensure_connection
+          @enqueue_stmt.execute(class_name, JSON.generate(args), realtime_now)
+          @db.last_insert_row_id
+        end
+
+        def fetch(worker_id)
+          ensure_connection
+          @db.execute("BEGIN IMMEDIATE")
+          row = @fetch_stmt.execute(worker_id, realtime_now).first
+          @db.execute("COMMIT")
+          return unless row
+
+          maybe_cleanup
+          { id: row[0], class_name: row[1], args: JSON.parse(row[2]) }
+        rescue
+          @db.execute("ROLLBACK") rescue nil
+          raise
+        end
+
+        def complete(job_id)
+          ensure_connection
+          @complete_stmt.execute(job_id)
+        end
+
+        def fail(job_id)
+          ensure_connection
+          @fail_stmt.execute(job_id)
+        end
+
+        def recover(worker_id)
+          ensure_connection
+          @requeue_stmt.execute(worker_id)
+          @db.changes
+        end
+
+        def close
+          return unless @db && !@db.closed?
+
+          finalize_statements
+          @db.execute("PRAGMA optimize") rescue nil
+          @db.close
+          @db = nil
+        end
+
+        def self.default_path
+          "async_background_queue.db"
+        end
+
+        private
+
+        def require_sqlite3
+          require 'sqlite3'
+        rescue LoadError
+          raise LoadError,
+            "sqlite3 gem is required for Async::Background::Queue. " \
+            "Add `gem 'sqlite3', '~> 2.0'` to your Gemfile."
+        end
+
+        def ensure_connection
+          return if @db && !@db.closed?
+
+          require_sqlite3
+          finalize_statements
+          @db = SQLite3::Database.new(@path)
+          @db.execute_batch(PRAGMAS.call(@mmap ? MMAP_SIZE : 0))
+
+          unless @schema_checked
+            @db.execute_batch(SCHEMA)
+            @schema_checked = true
+          end
+
+          prepare_statements
+          @last_cleanup_at = monotonic_now
+        end
+
+        def prepare_statements
+          @enqueue_stmt = @db.prepare(
+            "INSERT INTO jobs (class_name, args, created_at) VALUES (?, ?, ?)"
+          )
+
+          @fetch_stmt = @db.prepare(<<~SQL)
+            UPDATE jobs
+            SET    status = 'running', locked_by = ?, locked_at = ?
+            WHERE  id = (
+              SELECT id FROM jobs
+              WHERE  status = 'pending'
+              ORDER BY id
+              LIMIT 1
+            )
+            RETURNING id, class_name, args
+          SQL
+
+          @complete_stmt = @db.prepare("UPDATE jobs SET status = 'done' WHERE id = ?")
+          @fail_stmt     = @db.prepare("UPDATE jobs SET status = 'failed' WHERE id = ?")
+
+          @requeue_stmt = @db.prepare(
+            "UPDATE jobs SET status = 'pending', locked_by = NULL, locked_at = NULL " \
+            "WHERE status = 'running' AND locked_by = ?"
+          )
+
+          @cleanup_stmt = @db.prepare("DELETE FROM jobs WHERE status = 'done' AND created_at < ?")
+        end
+
+        def finalize_statements
+          %i[enqueue_stmt fetch_stmt complete_stmt fail_stmt requeue_stmt cleanup_stmt].each do |name|
+            stmt = instance_variable_get(:"@#{name}")
+            stmt&.close rescue nil
+            instance_variable_set(:"@#{name}", nil)
+          end
+        end
+
+        def maybe_cleanup
+          now = monotonic_now
+          return if (now - @last_cleanup_at) < CLEANUP_INTERVAL
+
+          @last_cleanup_at = now
+          @cleanup_stmt.execute(realtime_now - CLEANUP_AGE)
+
+          if @db.changes > 100
+            @db.execute("PRAGMA incremental_vacuum")
+          end
+        end
+
+        def realtime_now
+          Process.clock_gettime(Process::CLOCK_REALTIME)
+        end
+
+        def monotonic_now
+          Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        end
+      end
+    end
+  end
+end

data/lib/async/background/runner.rb

@@ -10,11 +10,15 @@ module Async
     DEFAULT_TIMEOUT = 30
     MIN_SLEEP_TIME  = 0.1
     MAX_JITTER      = 5
+    QUEUE_POLL_INTERVAL = 5
 
     class Runner
-      attr_reader :logger, :semaphore, :heap, :worker_index, :total_workers, :shutdown, :metrics
+      attr_reader :logger, :semaphore, :heap, :worker_index, :total_workers, :shutdown, :metrics, :queue_store
 
-      def initialize(config_path:, job_count: 2, worker_index:, total_workers:)
+      def initialize(
+        config_path:, job_count: 2, worker_index:, total_workers:,
+        queue_notifier: nil, queue_db_path: nil, queue_mmap: true
+      )
         @logger        = Console.logger
         @worker_index  = worker_index
         @total_workers = total_workers
@@ -26,44 +30,20 @@ module Async
 
         @semaphore = ::Async::Semaphore.new(job_count)
         @heap      = build_heap(config_path)
+
+        setup_queue(queue_notifier, queue_db_path, queue_mmap)
       end
 
       def run
         Async do |task|
           setup_signal_handlers
           start_signal_watcher(task)
+          start_queue_listener(task) if @listen_queue
 
-          loop do
-            entry = heap.peek
-            break unless entry
-
-            now = monotonic_now
-            wait = [entry.next_run_at - now, MIN_SLEEP_TIME].max
-            wait_with_shutdown(task, wait)
-            break unless running?
-
-            now = monotonic_now
-            while (entry = heap.peek) && entry.next_run_at <= now
-              break unless running?
-
-              if entry.running
-                logger.warn('Async::Background') { "#{entry.name}: skipped, previous run still active" }
-                metrics.job_skipped(entry)
-              else
-                entry.running = true
-                semaphore.async do
-                  run_job(task, entry)
-                ensure
-                  entry.running = false
-                end
-              end
-
-              entry.reschedule(monotonic_now)
-              heap.replace_top(entry)
-            end
-          end
+          scheduler_loop(task)
 
           semaphore.acquire {}
+          @queue_store&.close
         end
       end
 
@@ -73,6 +53,7 @@ module Async
         @running = false
         logger.info { "Async::Background: stopping gracefully" }
         shutdown.signal
+        @queue_notifier&.notify  # unblock queue listener from @reader.wait_readable
       end
 
       def running?
@@ -81,6 +62,121 @@ module Async
 
       private
 
+      def setup_queue(queue_notifier, queue_db_path, queue_mmap)
+        @listen_queue = false
+        return unless queue_notifier
+
+        # Lazy require — only loaded when queue is actually used
+        require_relative 'queue/store'
+        require_relative 'queue/notifier'
+        require_relative 'queue/client'
+
+        isolated = ENV.fetch("ISOLATION_FORKS", "").split(",").map(&:to_i)
+        return if isolated.include?(worker_index)
+
+        @listen_queue   = true
+        @queue_notifier = queue_notifier
+        @queue_store    = Queue::Store.new(
+          path: queue_db_path || Queue::Store.default_path,
+          mmap: queue_mmap
+        )
+
+        recovered = @queue_store.recover(worker_index)
+        logger.info { "Async::Background queue: recovered #{recovered} stale jobs" } if recovered > 0
+      end
+
+      def start_queue_listener(task)
+        task.async do
+          logger.info { "Async::Background queue: listening on worker #{worker_index}" }
+
+          while running?
+            @queue_notifier.wait(timeout: QUEUE_POLL_INTERVAL)
+
+            while running?
+              job = @queue_store.fetch(worker_index)
+              break unless job
+
+              semaphore.async { run_queue_job(task, job) }
+            end
+          end
+        end
+      end
+
+      def run_queue_job(task, job)
+        class_name = job[:class_name]
+        args       = job[:args]
+        klass      = resolve_job_class(class_name)
+
+        metrics.job_started(nil)
+        t = monotonic_now
+
+        task.with_timeout(DEFAULT_TIMEOUT) { klass.perform_now(*args) }
+
+        duration = monotonic_now - t
+        metrics.job_finished(nil, duration)
+        @queue_store.complete(job[:id])
+
+        logger.info('Async::Background') {
+          "queue(#{class_name}): completed in #{duration.round(2)}s"
+        }
+      rescue ::Async::TimeoutError
+        metrics.job_timed_out(nil)
+        @queue_store.fail(job[:id])
+        logger.error('Async::Background') { "queue(#{class_name}): timed out" }
+      rescue => e
+        metrics.job_failed(nil, e)
+        @queue_store.fail(job[:id])
+        logger.error('Async::Background') {
+          "queue(#{class_name}): #{e.class} #{e.message}\n#{e.backtrace.join("\n")}"
+        }
+      end
+
+      def resolve_job_class(class_name)
+        raise ConfigError, "empty class name in queue job" if class_name.nil? || class_name.strip.empty?
+
+        names = class_name.split("::")
+        klass = names.reduce(Object) do |mod, name|
+          raise ConfigError, "unknown class: #{class_name}" unless mod.const_defined?(name, false)
+          mod.const_get(name, false)
+        end
+
+        raise ConfigError, "#{class_name} must implement .perform_now" unless klass.respond_to?(:perform_now)
+
+        klass
+      end
+
+      def scheduler_loop(task)
+        loop do
+          entry = heap.peek
+          break unless entry
+
+          now = monotonic_now
+          wait = [entry.next_run_at - now, MIN_SLEEP_TIME].max
+          wait_with_shutdown(task, wait)
+          break unless running?
+
+          now = monotonic_now
+          while (entry = heap.peek) && entry.next_run_at <= now
+            break unless running?
+
+            if entry.running
+              logger.warn('Async::Background') { "#{entry.name}: skipped, previous run still active" }
+              metrics.job_skipped(entry)
+            else
+              entry.running = true
+              semaphore.async do
+                run_job(task, entry)
+              ensure
+                entry.running = false
+              end
+            end
+
+            entry.reschedule(monotonic_now)
+            heap.replace_top(entry)
+          end
+        end
+      end
+
       def setup_signal_handlers
         @signal_r, @signal_w = IO.pipe
 
@@ -98,6 +194,7 @@ module Async
             @signal_r.wait_readable
             @signal_r.read_nonblock(256) rescue nil
             shutdown.signal
+            @queue_notifier&.notify
             break unless running?
           end
         end
@@ -183,10 +280,10 @@ module Async
 
       def run_job(task, entry)
         metrics.job_started(entry)
-        t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        t = monotonic_now
         task.with_timeout(entry.timeout) { entry.job_class.perform_now }
 
-        duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - t
+        duration = monotonic_now - t
         metrics.job_finished(entry, duration)
         logger.info('Async::Background') {
           "#{entry.name}: completed in #{duration.round(2)}s"

data/lib/async/background/version.rb

@@ -2,6 +2,6 @@
 
 module Async
   module Background
-    VERSION = '0.3.0'
+    VERSION = '0.4.5'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: async-background
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.5
 platform: ruby
 authors:
 - Roman Hajdarov
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-03-25 00:00:00.000000000 Z
+date: 2026-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: async
@@ -94,6 +94,9 @@ files:
 - lib/async/background/entry.rb
 - lib/async/background/metrics.rb
 - lib/async/background/min_heap.rb
+- lib/async/background/queue/client.rb
+- lib/async/background/queue/notifier.rb
+- lib/async/background/queue/store.rb
 - lib/async/background/runner.rb
 - lib/async/background/version.rb
 homepage: https://github.com/roman-haidarov/async-background