gemstack-jobs 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: f76474daac16429de13f512235ca7eb314b8981ea0f700fedba334b5f8242efa
+  data.tar.gz: 2791f654094f13f0e0c7745106f5bff32b0027be4a69e7c166c9a44c0ef31fcb
+SHA512:
+  metadata.gz: d7ccc72bc94799b9883013b5a1a2761dc9757f77290604a0aa210cdac0a3d62dd829564a38a1fa4a556451aa33202644dcb6255477612debfe4e36607a9c86e4
+  data.tar.gz: a3a95da5e5ef21f12d292b38228e25ff9a9a561ef42179c155dfbe61b17c000421852b3d7970f705bfbcd7055681b96d619bf384020fab2f0b52f73bae05f99e

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+# Changelog
+
+## 0.1.0
+
+First release. See the [GemStack changelog](https://github.com/gemstack-rb/gemstack/blob/main/CHANGELOG.md).

data/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Shoaib Malik
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,28 @@
+# gemstack-jobs
+
+GemStack background jobs: a PostgreSQL queue by default, swappable adapters.
+
+Part of [GemStack](https://github.com/gemstack-rb/gemstack), a modular Ruby API framework for Next.js
+applications. All GemStack gems are developed together in that repository and released with the same
+version.
+
+## Installation
+
+Optional module — added by `gemstack new`:
+
+```ruby
+gem "gemstack-jobs", "~> 0.1"
+```
+
+## Documentation
+
+- [Guide](https://github.com/gemstack-rb/gemstack/blob/main/docs/background-jobs.md)
+- [All guides](https://github.com/gemstack-rb/gemstack/tree/main/docs) ·
+  [Architecture](https://github.com/gemstack-rb/gemstack/blob/main/ARCHITECTURE.md)
+
+Source, issues and pull requests: [gemstack-rb/gemstack](https://github.com/gemstack-rb/gemstack)
+(this gem lives in `gems/gemstack-jobs`).
+
+## License
+
+MIT — see [LICENSE.txt](LICENSE.txt).

data/lib/gemstack/job.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+module GemStack
+  # Base class for background jobs. See GemStack::Jobs.
+  #
+  #   class ImportProducts < GemStack::Job
+  #     queue :imports
+  #     priority 10                                          # lower runs first (default 100)
+  #     retry_on Faraday::TimeoutError, attempts: 5, wait: :exponential
+  #     discard_on GemStack::NotFound                        # the record is gone; nothing to do
+  #
+  #     def perform(import_id, options = {})
+  #       ...
+  #     end
+  #   end
+  #
+  #   ImportProducts.perform_later(import.id)                       # => job id
+  #   ImportProducts.set(wait: 300, queue: "slow").perform_later(1)  # in 5 minutes
+  #   ImportProducts.set(at: Time.now + 3600).perform_later(1)
+  #   ImportProducts.perform_now(1)                                 # synchronously
+  #
+  # Arguments must be JSON values (strings, numbers, booleans, nil, arrays,
+  # hashes with string or symbol keys). Pass ids, not records. Hash keys come
+  # back as strings.
+  class Job
+    RetryRule = Struct.new(:classes, :attempts, :wait)
+
+    class << self
+      def queue(name = nil)
+        @queue = name.to_s if name
+        @queue || (superclass.respond_to?(:queue) ? superclass.queue : Jobs.config.default_queue)
+      end
+
+      def priority(value = nil)
+        @priority = Integer(value) if value
+        @priority || (superclass.respond_to?(:priority) ? superclass.priority : Jobs.config.default_priority)
+      end
+
+      # retry_on Net::ReadTimeout, attempts: 5, wait: 30          # fixed seconds
+      # retry_on StandardError, wait: :exponential                 # default
+      # retry_on Api::RateLimited, wait: ->(attempt) { attempt * 60 }
+      # The first matching rule (most recently declared first) decides.
+      def retry_on(*classes, attempts: nil, wait: :exponential)
+        retry_rules.unshift(RetryRule.new(classes, attempts, wait))
+      end
+
+      # Errors that mean the job should simply be dropped (logged, not retried).
+      def discard_on(*classes)
+        discard_classes.concat(classes)
+      end
+
+      def retry_rules = @retry_rules ||= superclass.respond_to?(:retry_rules) ? superclass.retry_rules.dup : []
+
+      def discard_classes
+        @discard_classes ||= superclass.respond_to?(:discard_classes) ? superclass.discard_classes.dup : []
+      end
+
+      def perform_later(*) = Enqueuer.new(self).perform_later(*)
+      def set(**) = Enqueuer.new(self, **)
+
+      def perform_now(*args)
+        new.perform(*Jobs::Arguments.load(Jobs::Arguments.dump(args)))
+      end
+
+      # [max attempts, seconds to wait before the next attempt] for an error
+      # raised on attempt number `attempt` (1-based); nil wait = give up.
+      def retry_decision(error, attempt)
+        rule = retry_rules.find { |r| r.classes.any? { |klass| error.is_a?(klass) } }
+        max = rule&.attempts || Jobs.config.default_max_attempts
+        return [max, nil] if attempt >= max
+
+        [max, backoff(rule&.wait || :exponential, attempt)]
+      end
+
+      def discard?(error) = discard_classes.any? { |klass| error.is_a?(klass) }
+
+      private
+
+      # Sidekiq's curve: 16s, 31s, 96s, 271s, … ≈ 4 hours over 10 attempts.
+      def backoff(wait, attempt)
+        case wait
+        when :exponential then (attempt**4) + 15 + (rand(10) * attempt)
+        when Proc then Float(wait.call(attempt))
+        else Float(wait)
+        end
+      end
+    end
+
+    # Builds the payload for a (possibly customised) enqueue.
+    class Enqueuer
+      def initialize(job_class, wait: nil, at: nil, queue: nil, priority: nil)
+        @job_class = job_class
+        @run_at = at || (wait && (Time.now + Float(wait)))
+        @queue = queue&.to_s
+        @priority = priority
+      end
+
+      def perform_later(*args)
+        payload = {
+          "job_class" => @job_class.name,
+          "queue" => @queue || @job_class.queue,
+          "priority" => @priority || @job_class.priority,
+          "args" => Jobs::Arguments.dump(args),
+          "run_at" => @run_at
+        }
+        raise ArgumentError, "anonymous job classes can't be enqueued" unless payload["job_class"]
+
+        id = Jobs.adapter.enqueue(payload)
+        Jobs.instrument(:enqueued, job_class: payload["job_class"], job_id: id, queue: payload["queue"],
+                                   run_at: payload["run_at"])
+        id
+      end
+    end
+
+    # Subclasses implement this.
+    def perform(*)
+      raise NotImplementedError, "#{self.class.name}#perform is not implemented"
+    end
+
+    # Set by the executor for the running job.
+    attr_accessor :job_id, :attempt
+  end
+
+  module Jobs
+    # Validates and converts arguments to/from their JSON form.
+    module Arguments
+      module_function
+
+      def dump(args) = args.map { |arg| dump_value(arg, "argument") }
+
+      def load(json) = json
+
+      def dump_value(value, path)
+        case value
+        when String, Integer, true, false, nil then value
+        when Float then value.finite? ? value : invalid(value, path)
+        when Symbol then value.name
+        when Array then value.each_with_index.map { |v, i| dump_value(v, "#{path}[#{i}]") }
+        when Hash
+          value.to_h do |key, v|
+            invalid(key, "#{path} key") unless key.is_a?(String) || key.is_a?(Symbol)
+            [key.to_s, dump_value(v, "#{path}[#{key.inspect}]")]
+          end
+        else invalid(value, path)
+        end
+      end
+
+      def invalid(value, path)
+        hint = value.respond_to?(:pk) || value.respond_to?(:id) ? " — pass its id instead" : ""
+        raise SerializationError, "#{path} #{value.class} can't be serialized to JSON#{hint}"
+      end
+    end
+  end
+end

data/lib/gemstack/jobs/adapters/async.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module GemStack
+  module Jobs
+    module Adapters
+      # An in-process thread pool with an in-memory schedule (retries and
+      # `set(wait:)` work). Jobs are lost when the process exits, so it is
+      # meant for development and apps without a database — use :postgres
+      # or :sidekiq in production.
+      class Async
+        include AfterCommit
+
+        Entry = Struct.new(:run_at, :sequence, :payload)
+
+        def initialize(concurrency: Jobs.config.concurrency)
+          @concurrency = concurrency
+          @entries = []
+          @mutex = Mutex.new
+          @available = ConditionVariable.new
+          @sequence = 0
+          @running = 0
+          @threads = nil
+        end
+
+        def enqueue(payload)
+          id = SecureRandom.uuid
+          after_commit { schedule(payload.merge("id" => id, "attempts" => 0), payload["run_at"] || Time.now) }
+          id
+        end
+
+        # Blocks until no jobs are queued or running (mainly for tests).
+        def drain(timeout: 10)
+          deadline = monotonic + timeout
+          sleep 0.01 until @mutex.synchronize { @entries.empty? && @running.zero? } || monotonic > deadline
+        end
+
+        def shutdown
+          @mutex.synchronize do
+            @stopping = true
+            @available.broadcast
+          end
+          @threads&.each { |thread| thread.join(Jobs.config.shutdown_timeout) }
+          @threads = nil
+        end
+
+        private
+
+        def schedule(payload, run_at)
+          worker_threads
+          @mutex.synchronize do
+            @sequence += 1
+            @entries << Entry.new(run_at, @sequence, payload)
+            @entries.sort_by! { |e| [e.run_at, e.payload["priority"] || 100, e.sequence] }
+            @available.broadcast
+          end
+        end
+
+        # Started lazily on the first job; shutdown joins them.
+        def worker_threads
+          @mutex.synchronize do
+            @threads ||= Array.new(@concurrency) { Thread.new { work } } # rubocop:disable Naming/MemoizedInstanceVariableName
+          end
+        end
+
+        def work
+          loop do
+            entry = next_entry or break
+            outcome = Executor.execute(entry.payload)
+            schedule(entry.payload.merge("attempts" => outcome.attempt), outcome.run_at) if outcome.status == :retry
+          ensure
+            @mutex.synchronize { @running -= 1 } if entry
+          end
+        end
+
+        def next_entry
+          @mutex.synchronize do
+            loop do
+              return nil if @stopping
+
+              first = @entries.first
+              if first && first.run_at <= Time.now
+                @running += 1
+                return @entries.shift
+              end
+              @available.wait(@mutex, first ? [first.run_at - Time.now, 0.01].max : nil)
+            end
+          end
+        end
+
+        def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/gemstack/jobs/adapters/inline.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module GemStack
+  module Jobs
+    module Adapters
+      # Runs jobs immediately, in the caller's thread, ignoring schedules and
+      # retries. Errors propagate, so nothing is silently swallowed. Handy for
+      # scripts and debugging.
+      class Inline
+        def enqueue(payload)
+          id = SecureRandom.uuid
+          job = Executor.resolve(payload["job_class"]).new
+          job.job_id = id
+          job.attempt = 1
+          job.perform(*Arguments.load(payload["args"]))
+          id
+        end
+      end
+
+      # Defers a block until the surrounding database transaction commits
+      # (when gemstack-db is in use), so a job never runs before — or without —
+      # the data it depends on. The :postgres adapter gets this for free by
+      # inserting into the same transaction.
+      module AfterCommit
+        def after_commit(&)
+          db = defined?(GemStack::DB) && GemStack::DB.connected? ? GemStack::DB.connection : nil
+          return yield unless db&.in_transaction?
+
+          db.after_commit(&)
+        end
+      end
+    end
+  end
+end

data/lib/gemstack/jobs/adapters/postgres.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+module GemStack
+  module Jobs
+    # The migration new apps get (and `gemstack jobs:install` writes). Kept
+    # as source text so the app owns a plain, readable migration while tests
+    # and the generator share one definition.
+    module Migration
+      SOURCE = <<~RUBY
+        # frozen_string_literal: true
+
+        # The GemStack job queue (docs/background-jobs.md). Workers claim rows with
+        # FOR UPDATE SKIP LOCKED; finished jobs are deleted, exhausted ones keep failed_at.
+        Sequel.migration do
+          change do
+            create_table(:gemstack_jobs) do
+              primary_key :id, type: :Bignum
+              String :queue, null: false, default: "default"
+              Integer :priority, null: false, default: 100
+              String :job_class, null: false
+              column :args, :jsonb, null: false, default: Sequel.lit("'[]'::jsonb")
+              column :run_at, :timestamptz, null: false, default: Sequel::CURRENT_TIMESTAMP
+              Integer :attempts, null: false, default: 0
+              String :last_error, text: true
+              column :locked_at, :timestamptz
+              String :locked_by
+              column :failed_at, :timestamptz
+              column :created_at, :timestamptz, null: false, default: Sequel::CURRENT_TIMESTAMP
+
+              # The fetch query: ready jobs by queue, in priority/run_at order.
+              index %i[queue priority run_at id], name: :gemstack_jobs_ready,
+                                                  where: Sequel.lit("failed_at IS NULL AND locked_at IS NULL")
+              index :locked_at, name: :gemstack_jobs_locked, where: Sequel.lit("locked_at IS NOT NULL")
+              index :failed_at, name: :gemstack_jobs_failed, where: Sequel.lit("failed_at IS NOT NULL")
+            end
+          end
+        end
+      RUBY
+
+      def self.apply(db, direction = :up)
+        Sequel.extension :migration
+        eval(SOURCE, TOPLEVEL_BINDING, "gemstack_jobs_migration.rb").apply(db, direction) # rubocop:disable Security/Eval
+      end
+    end
+
+    module Adapters
+      # The default adapter: jobs are rows in PostgreSQL (DECISIONS D-040).
+      #
+      # - Enqueueing is an INSERT on the current connection, so inside
+      #   GemStack.transaction a job exists only if the transaction commits.
+      # - NOTIFY (also transactional) wakes idle workers immediately; polling
+      #   every config.jobs.poll_interval is the safety net.
+      # - Workers claim one job at a time with FOR UPDATE SKIP LOCKED, so
+      #   workers never block each other or run the same job twice at once.
+      class Postgres
+        CHANNEL = "gemstack_jobs"
+
+        def initialize(db: nil, table: Jobs.config.table)
+          @db = db
+          @table = table
+        end
+
+        def db
+          @db || begin
+            require "gemstack/db"
+            GemStack::DB.connection
+          end
+        end
+
+        def dataset = db[@table]
+
+        def enqueue(payload)
+          id = dataset.insert(
+            job_class: payload["job_class"], queue: payload["queue"], priority: payload["priority"],
+            args: Sequel.pg_jsonb_wrap(payload["args"]), run_at: payload["run_at"] || Sequel::CURRENT_TIMESTAMP
+          )
+          db.notify(CHANNEL, payload: payload["queue"])
+          id
+        end
+
+        # Claims the next ready job for these queues ("*" = all). Returns a
+        # payload Hash or nil.
+        def claim(queues, worker)
+          ready = dataset.where(failed_at: nil, locked_at: nil).where { run_at <= Sequel::CURRENT_TIMESTAMP }
+          ready = ready.where(queue: queues) unless queues.include?("*")
+          next_id = ready.order(:priority, :run_at, :id).limit(1).for_update.skip_locked.select(:id)
+          row = dataset.where(id: next_id).returning(*returned_columns)
+                       .update(locked_at: Sequel::CURRENT_TIMESTAMP, locked_by: worker).first
+          row && payload_for(row)
+        end
+
+        def complete(id) = dataset.where(id: id).delete
+
+        def reschedule(id, run_at:, attempts:, error:)
+          dataset.where(id: id).update(locked_at: nil, locked_by: nil, run_at: run_at, attempts: attempts,
+                                       last_error: describe(error))
+        end
+
+        def fail(id, attempts:, error:)
+          return complete(id) unless Jobs.config.keep_failed
+
+          dataset.where(id: id).update(locked_at: nil, locked_by: nil, failed_at: Sequel::CURRENT_TIMESTAMP,
+                                       attempts: attempts, last_error: describe(error))
+        end
+
+        # Releases this worker's claimed jobs (graceful shutdown).
+        def release(worker) = dataset.where(locked_by: worker).update(locked_at: nil, locked_by: nil)
+
+        # Releases jobs locked longer than `timeout` seconds (their worker died).
+        def release_stale(timeout)
+          cutoff = Sequel.lit("CURRENT_TIMESTAMP - make_interval(secs => ?)", Float(timeout))
+          dataset.where { locked_at < cutoff }.update(locked_at: nil, locked_by: nil)
+        end
+
+        # Failed jobs back to the queue: all, or the given ids.
+        def retry_failed(ids = nil)
+          failed = dataset.exclude(failed_at: nil)
+          failed = failed.where(id: ids) if ids
+          failed.update(failed_at: nil, attempts: 0, run_at: Sequel::CURRENT_TIMESTAMP, last_error: nil)
+        end
+
+        def discard_failed(ids = nil)
+          failed = dataset.exclude(failed_at: nil)
+          failed = failed.where(id: ids) if ids
+          failed.delete
+        end
+
+        def failed(limit: 20)
+          dataset.exclude(failed_at: nil).order(Sequel.desc(:failed_at)).limit(limit)
+                 .select(:id, :queue, :job_class, :attempts, :failed_at, :last_error).all
+        end
+
+        # { "default" => { ready:, scheduled:, running:, failed: }, ... }
+        def stats
+          now = Sequel::CURRENT_TIMESTAMP
+          states = Sequel.case(
+            [[Sequel.~(failed_at: nil), "failed"], [Sequel.~(locked_at: nil), "running"],
+             [Sequel[:run_at] > now, "scheduled"]], "ready"
+          )
+          dataset.group_and_count(:queue, states.as(:state)).all.each_with_object({}) do |row, result|
+            (result[row[:queue]] ||= { ready: 0, scheduled: 0, running: 0, failed: 0 })[row[:state].to_sym] =
+              row[:count]
+          end
+        end
+
+        private
+
+        # args as text, parsed with the json gem: jobs receive plain Hash/Array
+        # values rather than Sequel's JSONB wrappers. (Lazy: Sequel may not be loaded.)
+        def returned_columns
+          @returned_columns ||= [:id, :job_class, :queue, :priority, :attempts,
+                                 Sequel.cast(:args, :text).as(:args_json)].freeze
+        end
+
+        def payload_for(row)
+          { "id" => row[:id], "job_class" => row[:job_class], "queue" => row[:queue],
+            "priority" => row[:priority], "args" => JSON.parse(row[:args_json]), "attempts" => row[:attempts] }
+        end
+
+        def describe(error)
+          "#{error.class}: #{error.message}\n#{Array(error.backtrace).first(20).join("\n")}"[0, 10_000]
+        end
+      end
+    end
+  end
+end

data/lib/gemstack/jobs/adapters/sidekiq.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module GemStack
+  module Jobs
+    module Adapters
+      # Runs GemStack jobs on Sidekiq (for teams already running Redis +
+      # Sidekiq). Add `gem "sidekiq"`, set `config.jobs.adapter = :sidekiq`,
+      # and run `bundle exec sidekiq -r ./config/sidekiq.rb` (see docs/background-jobs.md).
+      #
+      # GemStack's executor still decides retries and discards, so job
+      # classes behave identically on every adapter; Sidekiq's own retries
+      # are off and exhausted jobs go to Sidekiq's Dead set.
+      class Sidekiq
+        include AfterCommit
+
+        def initialize
+          require "sidekiq"
+          Runner.define!
+        rescue LoadError
+          raise ConfigurationError, 'the :sidekiq job adapter needs `gem "sidekiq"` in the Gemfile'
+        end
+
+        def enqueue(payload)
+          jid = SecureRandom.hex(12)
+          after_commit { Runner.push(payload.merge("attempts" => 0), jid: jid) }
+          jid
+        end
+
+        # The Sidekiq job class that executes GemStack payloads.
+        module Runner
+          module_function
+
+          def define!
+            return if defined?(GemStack::Jobs::SidekiqRunner)
+
+            klass = Class.new do
+              include ::Sidekiq::Job
+
+              sidekiq_options retry: 0 # exhausted jobs → Dead set; GemStack schedules retries itself
+
+              def perform(payload)
+                payload = payload.merge("id" => jid)
+                outcome = Executor.execute(payload)
+                case outcome.status
+                when :retry then Runner.push(payload.merge("attempts" => outcome.attempt), at: outcome.run_at)
+                when :failed then raise outcome.error
+                end
+              end
+            end
+            GemStack::Jobs.const_set(:SidekiqRunner, klass)
+          end
+
+          def push(payload, at: nil, jid: nil)
+            item = { "class" => GemStack::Jobs::SidekiqRunner, "queue" => payload["queue"],
+                     "args" => [payload.except("run_at", "id")] }
+            item["jid"] = jid if jid
+            run_at = at || payload["run_at"]
+            item["at"] = run_at.to_f if run_at
+            ::Sidekiq::Client.push(item)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gemstack/jobs/adapters/test.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module GemStack
+  module Jobs
+    module Adapters
+      # Records jobs instead of running them (the default in tests). See
+      # GemStack::Jobs::Testing for assertions and perform_enqueued_jobs.
+      class Test
+        attr_reader :enqueued, :performed
+
+        def initialize
+          @enqueued = []
+          @performed = []
+          @mutex = Mutex.new
+          @sequence = 0
+        end
+
+        def enqueue(payload)
+          @mutex.synchronize do
+            @sequence += 1
+            @enqueued << payload.merge("id" => @sequence, "attempts" => 0)
+            @sequence
+          end
+        end
+
+        # Runs enqueued jobs (and jobs they enqueue) until none are left, or
+        # only those matching `only`. An error a job raises is re-raised (unless
+        # the job discards it), so failing jobs fail the test. Returns the outcomes.
+        def perform_enqueued(only: nil, except_ids: [])
+          names = only && Array(only).map(&:to_s)
+          outcomes = []
+          loop do
+            payload = @mutex.synchronize do
+              index = @enqueued.index do |p|
+                (names.nil? || names.include?(p["job_class"])) && !except_ids.include?(p["id"])
+              end
+              index && @enqueued.delete_at(index)
+            end
+            break unless payload
+
+            outcome = Executor.execute(payload)
+            raise outcome.error if outcome.error && outcome.status != :discarded
+
+            @performed << payload
+            outcomes << outcome
+          end
+          outcomes
+        end
+
+        def clear
+          @mutex.synchronize do
+            @enqueued.clear
+            @performed.clear
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gemstack/jobs/executor.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module GemStack
+  module Jobs
+    # Runs one job payload and decides what happens next. Shared by every
+    # adapter, so retries, discards, failures and instrumentation behave the
+    # same everywhere.
+    #
+    # Returns an Outcome: :performed, :retry (with run_at), :discarded or :failed.
+    module Executor
+      Outcome = Struct.new(:status, :error, :run_at, :attempt, keyword_init: true)
+
+      module_function
+
+      # payload: "job_class", "args", "queue", "attempts" (runs so far), "id".
+      def execute(payload)
+        attempt = Integer(payload["attempts"] || 0) + 1
+        fields = { job_class: payload["job_class"], job_id: payload["id"], queue: payload["queue"], attempt: attempt }
+        job_class = resolve(payload["job_class"])
+        started = monotonic
+        run(job_class, payload, attempt)
+        Jobs.instrument(:performed, **fields, duration_ms: elapsed(started))
+        Outcome.new(status: :performed, attempt: attempt)
+      rescue UnknownJob => e
+        Jobs.instrument(:failed, **fields, error: e)
+        Outcome.new(status: :failed, error: e, attempt: attempt)
+      rescue StandardError => e
+        failure(job_class, e, attempt, fields.merge(duration_ms: elapsed(started)))
+      end
+
+      def run(job_class, payload, attempt)
+        job = job_class.new
+        job.job_id = payload["id"]
+        job.attempt = attempt
+        job.perform(*Arguments.load(payload["args"] || []))
+      end
+
+      def failure(job_class, error, attempt, fields)
+        if job_class.discard?(error)
+          Jobs.instrument(:discarded, **fields, error: error)
+          return Outcome.new(status: :discarded, error: error, attempt: attempt)
+        end
+
+        _max, wait = job_class.retry_decision(error, attempt)
+        if wait
+          run_at = Time.now + wait
+          Jobs.instrument(:retried, **fields, error: error, run_at: run_at)
+          Outcome.new(status: :retry, error: error, run_at: run_at, attempt: attempt)
+        else
+          Jobs.instrument(:failed, **fields, error: error)
+          Outcome.new(status: :failed, error: error, attempt: attempt)
+        end
+      end
+
+      # Only GemStack::Job subclasses can run — a queue row naming another
+      # constant is never instantiated.
+      def resolve(name)
+        klass = Object.const_get(name.to_s)
+        raise UnknownJob, "#{name} is not a GemStack::Job" unless klass.is_a?(Class) && klass < Job
+
+        klass
+      rescue NameError
+        raise UnknownJob, "unknown job class #{name.inspect}"
+      end
+
+      def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      def elapsed(started) = started && ((monotonic - started) * 1000).round(2)
+    end
+  end
+end

data/lib/gemstack/jobs/testing.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "gemstack/jobs"
+
+module GemStack
+  module Jobs
+    # Test helpers (included into GemStack::TestCase by the generated test helper):
+    #
+    #   def test_signup_sends_welcome_email
+    #     post_json "/api/signups", { email: "a@b.c" }
+    #
+    #     assert_enqueued SendWelcomeEmail, args: [User.last.id]
+    #     perform_enqueued_jobs
+    #     assert_equal 1, Mailer.deliveries.size
+    #   end
+    module Testing
+      def self.included(base)
+        base.class_eval do
+          def before_setup
+            super
+            GemStack::Jobs.adapter = GemStack::Jobs::Adapters::Test.new
+          end
+        end
+      end
+
+      def enqueued_jobs = Jobs.adapter.enqueued
+
+      # Jobs of job_class (optionally with these args / on this queue) were enqueued.
+      def assert_enqueued(job_class, args: nil, queue: nil, count: nil)
+        matching = enqueued_jobs.select do |job|
+          job["job_class"] == job_class.name && (args.nil? || job["args"] == Arguments.dump(args)) &&
+            (queue.nil? || job["queue"] == queue.to_s)
+        end
+        message = "Expected #{job_class.name}#{" with #{args.inspect}" if args} to be enqueued; " \
+                  "enqueued: #{enqueued_jobs.map { |j| [j["job_class"], j["args"]] }.inspect}"
+        count ? assert_equal(count, matching.size, message) : assert(!matching.empty?, message)
+      end
+
+      def refute_enqueued(job_class)
+        assert(enqueued_jobs.none? { |job| job["job_class"] == job_class.name },
+               "Expected no #{job_class.name} to be enqueued")
+      end
+
+      # Runs enqueued jobs (and any they enqueue). With a block: only jobs
+      # enqueued inside it. Errors raised by jobs propagate.
+      def perform_enqueued_jobs(only: nil)
+        return Jobs.adapter.perform_enqueued(only: only) unless block_given?
+
+        before = enqueued_jobs.map { |job| job["id"] }
+        yield
+        Jobs.adapter.perform_enqueued(only: only, except_ids: before)
+      end
+    end
+  end
+end

data/lib/gemstack/jobs/worker.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require "socket"
+
+module GemStack
+  module Jobs
+    # Processes jobs from the PostgreSQL queue (`gemstack jobs`).
+    #
+    # - `concurrency` threads each claim and run one job at a time.
+    # - A listener thread LISTENs for NOTIFY and wakes idle threads at once;
+    #   idle threads also re-check every poll_interval.
+    # - A reaper releases jobs whose lock is older than lock_timeout (a worker
+    #   crashed mid-job), so they run again (at-least-once delivery).
+    # - stop (SIGINT/SIGTERM) lets running jobs finish for shutdown_timeout,
+    #   then releases the rest back to the queue.
+    #
+    # Needs concurrency + 2 database connections (the CLI sizes the pool).
+    class Worker
+      attr_reader :id
+
+      def initialize(store: Adapters::Postgres.new, queues: Jobs.config.queues, concurrency: Jobs.config.concurrency,
+                     poll_interval: Jobs.config.poll_interval, lock_timeout: Jobs.config.lock_timeout,
+                     shutdown_timeout: Jobs.config.shutdown_timeout)
+        @store = store
+        @queues = Array(queues).map(&:to_s)
+        @concurrency = concurrency
+        @poll_interval = poll_interval
+        @lock_timeout = lock_timeout
+        @shutdown_timeout = shutdown_timeout
+        @id = "#{Socket.gethostname}:#{Process.pid}:#{SecureRandom.hex(3)}"
+        @mutex = Mutex.new
+        @wakeup = ConditionVariable.new
+        @running = false
+        @threads = []
+      end
+
+      def running? = @running
+
+      # Starts the threads and returns immediately.
+      def start
+        @running = true
+        GemStack.logger.info("jobs worker started", worker: @id, queues: @queues.join(","), concurrency: @concurrency)
+        @threads = Array.new(@concurrency) { |i| Thread.new { work_loop(i) } }
+        @listener = Thread.new { listen_loop }
+        @reaper = Thread.new { reap_loop }
+        self
+      end
+
+      # Blocks until SIGINT/SIGTERM, then shuts down gracefully.
+      def run
+        %w[INT TERM].each { |signal| trap(signal) { @running = false } }
+        start
+        sleep 0.2 while @running
+        shutdown
+      end
+
+      def stop
+        @running = false
+        wake
+      end
+
+      def shutdown
+        stop
+        deadline = monotonic + @shutdown_timeout
+        @threads.each { |thread| thread.join([deadline - monotonic, 0].max) }
+        unfinished = @threads.count(&:alive?)
+        @threads.each(&:kill)
+        released = @store.release(@id)
+        [@listener, @reaper].compact.each { |thread| thread.kill.join(1) }
+        GemStack.logger.info("jobs worker stopped", worker: @id, released: released, unfinished: unfinished)
+      end
+
+      # Wakes idle threads (NOTIFY arrived, or shutting down).
+      def wake = @mutex.synchronize { @wakeup.broadcast }
+
+      # Runs one job if one is ready. Returns the outcome, or nil when idle.
+      def work_once
+        payload = @store.claim(@queues, @id) or return nil
+
+        outcome = Executor.execute(payload)
+        settle(payload, outcome)
+        outcome
+      end
+
+      private
+
+      def work_loop(_index)
+        while @running
+          begin
+            idle(@poll_interval) unless work_once
+          rescue Sequel::DatabaseConnectionError, Sequel::PoolTimeout => e
+            GemStack.logger.warn("jobs worker: database unavailable, retrying", error: e.message)
+            idle(@poll_interval)
+          rescue StandardError => e # a bug in the worker itself; keep the thread alive
+            GemStack.logger.error("jobs worker error", error: e, backtrace: Array(e.backtrace).first(10))
+            idle(@poll_interval)
+          end
+        end
+      end
+
+      def settle(payload, outcome)
+        case outcome.status
+        when :performed, :discarded then @store.complete(payload["id"])
+        when :retry
+          @store.reschedule(payload["id"], run_at: outcome.run_at, attempts: outcome.attempt, error: outcome.error)
+        when :failed then @store.fail(payload["id"], attempts: outcome.attempt, error: outcome.error)
+        end
+      end
+
+      def idle(seconds)
+        @mutex.synchronize { @wakeup.wait(@mutex, seconds) if @running }
+      end
+
+      def listen_loop
+        @store.db.listen(Adapters::Postgres::CHANNEL, loop: ->(_conn) { throw :stop unless @running },
+                                                      timeout: @poll_interval) { wake }
+      rescue Sequel::DatabaseConnectionError => e
+        GemStack.logger.warn("jobs worker: LISTEN failed, falling back to polling", error: e.message)
+      end
+
+      def reap_loop
+        interval = [@lock_timeout / 4.0, 1].max
+        while @running
+          begin
+            released = @store.release_stale(@lock_timeout)
+            GemStack.logger.warn("jobs: released stale locks", count: released) if released.positive?
+          rescue Sequel::Error => e
+            GemStack.logger.warn("jobs reaper error", error: e.message)
+          end
+          sleep interval
+        end
+      end
+
+      def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/gemstack/jobs.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require "json"
+require "securerandom"
+require "gemstack/core"
+
+module GemStack
+  # Background jobs (ARCHITECTURE §9, DECISIONS D-040).
+  #
+  #   class SendWelcomeEmail < GemStack::Job
+  #     queue :mailers
+  #     retry_on Net::ReadTimeout, attempts: 5
+  #     def perform(user_id) = Mailer.welcome(User.find(user_id))
+  #   end
+  #
+  #   SendWelcomeEmail.perform_later(user.id)
+  #   SendWelcomeEmail.set(wait: 600).perform_later(user.id)
+  #
+  # Work is only ever asynchronous when you ask for it with perform_later.
+  # Delivery is at-least-once: make perform idempotent.
+  module Jobs
+    class Config < Settings
+      # :postgres (default with gemstack-db), :async (in-process threads),
+      # :inline (run immediately), :test (record only; default in tests),
+      # :sidekiq, or an adapter object responding to #enqueue(payload).
+      setting :adapter, default: lambda {
+        if GemStack.env.test? then :test
+        elsif defined?(GemStack::DB) then :postgres
+        else :async
+        end
+      }
+      setting :default_queue, default: "default"
+      setting :default_priority, default: 100 # lower runs first
+      setting :default_max_attempts, default: 10
+      # Worker settings (`gemstack jobs`).
+      setting :queues, default: -> { ENV.fetch("GEMSTACK_JOB_QUEUES", "*").split(",").map(&:strip) }
+      setting :concurrency, default: -> { Integer(ENV.fetch("GEMSTACK_JOB_CONCURRENCY", 5)) }
+      # Seconds between polls when no NOTIFY arrives (a safety net; NOTIFY wakes workers instantly).
+      setting :poll_interval, default: 5
+      # A job locked longer than this is assumed abandoned (worker crashed) and is released.
+      # Jobs that legitimately run longer must raise it.
+      setting :lock_timeout, default: 30 * 60
+      # How long a stopping worker waits for running jobs before releasing them.
+      setting :shutdown_timeout, default: 25
+      setting :table, default: :gemstack_jobs
+      # Keep exhausted jobs (failed_at set) for inspection and `gemstack jobs:retry`.
+      setting :keep_failed, default: true
+    end
+
+    autoload :Worker, "gemstack/jobs/worker"
+    autoload :Testing, "gemstack/jobs/testing"
+
+    # Raised for arguments that can't round-trip through JSON.
+    class SerializationError < Error; end
+
+    # A job's name didn't resolve to a GemStack::Job subclass when it ran.
+    class UnknownJob < Error; end
+
+    Event = Struct.new(:name, :job_class, :job_id, :queue, :attempt, :duration_ms, :error, :run_at, keyword_init: true)
+
+    @subscribers = Hash.new { |hash, key| hash[key] = [] }
+    @mutex = Mutex.new
+
+    class << self
+      def config = GemStack.config.jobs
+
+      def adapter
+        @adapter || @mutex.synchronize { @adapter ||= build_adapter(config.adapter) }
+      end
+
+      attr_writer :adapter
+
+      def build_adapter(setting)
+        case setting
+        when :postgres, "postgres" then Adapters::Postgres.new
+        when :async, "async" then Adapters::Async.new
+        when :inline, "inline" then Adapters::Inline.new
+        when :test, "test" then Adapters::Test.new
+        when :sidekiq, "sidekiq" then Adapters::Sidekiq.new
+        else
+          raise ConfigurationError, "a job adapter must respond to #enqueue" unless setting.respond_to?(:enqueue)
+
+          setting
+        end
+      end
+
+      # Instrumentation for metrics/monitoring:
+      #   GemStack::Jobs.subscribe(:failed) { |event| Sentry.capture_message(...) }
+      # Events: :enqueued, :performed, :retried, :failed, :discarded.
+      def subscribe(name = :all, &block)
+        @mutex.synchronize { @subscribers[name.to_sym] << block }
+        block
+      end
+
+      def unsubscribe(block)
+        @mutex.synchronize { @subscribers.each_value { |list| list.delete(block) } }
+      end
+
+      def instrument(name, **fields)
+        event = Event.new(name: name, **fields)
+        log(event)
+        (@subscribers[name] + @subscribers[:all]).each do |subscriber|
+          subscriber.call(event)
+        rescue StandardError => e
+          GemStack.logger.error("job subscriber failed", error: e)
+        end
+        event
+      end
+
+      def reset!
+        @adapter = nil
+      end
+
+      private
+
+      def log(event)
+        fields = { job: event.job_class, id: event.job_id, queue: event.queue, attempt: event.attempt,
+                   ms: event.duration_ms, run_at: event.run_at&.utc&.iso8601 }.compact
+        fields[:error] = "#{event.error.class}: #{event.error.message}" if event.error
+        level = { failed: :error, retried: :warn }.fetch(event.name, :info)
+        GemStack.logger.public_send(level, "job.#{event.name}", **fields)
+      end
+    end
+  end
+
+  class << self
+    def jobs = Jobs.adapter
+  end
+end
+
+require_relative "job"
+require_relative "jobs/executor"
+require_relative "jobs/adapters/inline"
+require_relative "jobs/adapters/test"
+require_relative "jobs/adapters/async"
+require_relative "jobs/adapters/postgres"
+require_relative "jobs/adapters/sidekiq"
+
+GemStack::Config.namespace(:jobs, GemStack::Jobs::Config)

metadata

@@ -0,0 +1,71 @@
+--- !ruby/object:Gem::Specification
+name: gemstack-jobs
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Shoaib Malik
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: gemstack-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+email:
+- gemstack26@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- lib/gemstack/job.rb
+- lib/gemstack/jobs.rb
+- lib/gemstack/jobs/adapters/async.rb
+- lib/gemstack/jobs/adapters/inline.rb
+- lib/gemstack/jobs/adapters/postgres.rb
+- lib/gemstack/jobs/adapters/sidekiq.rb
+- lib/gemstack/jobs/adapters/test.rb
+- lib/gemstack/jobs/executor.rb
+- lib/gemstack/jobs/testing.rb
+- lib/gemstack/jobs/worker.rb
+homepage: https://github.com/gemstack-rb/gemstack
+licenses:
+- MIT
+metadata:
+  rubygems_mfa_required: 'true'
+  source_code_uri: https://github.com/gemstack-rb/gemstack/tree/main/gems/gemstack-jobs
+  changelog_uri: https://github.com/gemstack-rb/gemstack/blob/main/gems/gemstack-jobs/CHANGELOG.md
+  bug_tracker_uri: https://github.com/gemstack-rb/gemstack/issues
+  documentation_uri: https://github.com/gemstack-rb/gemstack/tree/main/docs
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '4.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.20
+specification_version: 4
+summary: 'GemStack background jobs: a PostgreSQL queue by default, swappable adapters'
+test_files: []