gemstack-jobs 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f76474daac16429de13f512235ca7eb314b8981ea0f700fedba334b5f8242efa
-  data.tar.gz: 2791f654094f13f0e0c7745106f5bff32b0027be4a69e7c166c9a44c0ef31fcb
+  metadata.gz: '0802dfa203bae021ece72db2d1a796c66e7a3b34fd157aa731d1812ccdda10c4'
+  data.tar.gz: 9e8c843b6018c6a33b594f739e1cb985a54f0ca54e0aafafd855e16a79e4629f
 SHA512:
-  metadata.gz: d7ccc72bc94799b9883013b5a1a2761dc9757f77290604a0aa210cdac0a3d62dd829564a38a1fa4a556451aa33202644dcb6255477612debfe4e36607a9c86e4
-  data.tar.gz: a3a95da5e5ef21f12d292b38228e25ff9a9a561ef42179c155dfbe61b17c000421852b3d7970f705bfbcd7055681b96d619bf384020fab2f0b52f73bae05f99e
+  metadata.gz: 65afed0b493a60301ecc39b8750d6b9ba812b511873ce76dc6a634768e60fec703ff91da8b5d476da8d960f40ed39424820db0e0a357c0f6d68e011fad2c5261
+  data.tar.gz: e7de052e354fd15863d5a3df6cf7aa64f45abba4fb1b08a13141862d141376026547a7874303556ab3f7a28b1e60db70512d4c2fcff86463d3d335f37f15eed8

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.2.0
+
+See the [GemStack changelog](https://github.com/gemstack-rb/gemstack/blob/main/CHANGELOG.md).
+
 ## 0.1.0
 
 First release. See the [GemStack changelog](https://github.com/gemstack-rb/gemstack/blob/main/CHANGELOG.md).

data/README.md

@@ -1,6 +1,6 @@
 # gemstack-jobs
 
-GemStack background jobs: a PostgreSQL queue by default, swappable adapters.
+GemStack background jobs: a queue in the app's database by default, swappable adapters.
 
 Part of [GemStack](https://github.com/gemstack-rb/gemstack), a modular Ruby API framework for Next.js
 applications. All GemStack gems are developed together in that repository and released with the same

data/lib/gemstack/jobs/adapters/{postgres.rb → database.rb}

@@ -9,31 +9,37 @@ module GemStack
       SOURCE = <<~RUBY
         # frozen_string_literal: true
 
-        # The GemStack job queue (docs/background-jobs.md). Workers claim rows with
-        # FOR UPDATE SKIP LOCKED; finished jobs are deleted, exhausted ones keep failed_at.
+        # The GemStack job queue (docs/background-jobs.md), for PostgreSQL, MySQL and
+        # SQLite. Workers claim rows one at a time (FOR UPDATE SKIP LOCKED where the
+        # database has it); finished jobs are deleted, exhausted ones keep failed_at.
         Sequel.migration do
-          change do
+          up do
+            partial = database_type != :mysql # MySQL has no partial indexes
+            ready = partial ? { where: Sequel.lit("failed_at IS NULL AND locked_at IS NULL") } : {}
             create_table(:gemstack_jobs) do
               primary_key :id, type: :Bignum
               String :queue, null: false, default: "default"
               Integer :priority, null: false, default: 100
               String :job_class, null: false
-              column :args, :jsonb, null: false, default: Sequel.lit("'[]'::jsonb")
-              column :run_at, :timestamptz, null: false, default: Sequel::CURRENT_TIMESTAMP
+              column :args, :jsonb, null: false # json on MySQL and SQLite
+              column :run_at, :timestamptz, null: false
               Integer :attempts, null: false, default: 0
               String :last_error, text: true
               column :locked_at, :timestamptz
               String :locked_by
               column :failed_at, :timestamptz
-              column :created_at, :timestamptz, null: false, default: Sequel::CURRENT_TIMESTAMP
+              column :created_at, :timestamptz, null: false
 
               # The fetch query: ready jobs by queue, in priority/run_at order.
-              index %i[queue priority run_at id], name: :gemstack_jobs_ready,
-                                                  where: Sequel.lit("failed_at IS NULL AND locked_at IS NULL")
-              index :locked_at, name: :gemstack_jobs_locked, where: Sequel.lit("locked_at IS NOT NULL")
-              index :failed_at, name: :gemstack_jobs_failed, where: Sequel.lit("failed_at IS NOT NULL")
+              index %i[queue priority run_at id], name: :gemstack_jobs_ready, **ready
+              index :locked_at, name: :gemstack_jobs_locked
+              index :failed_at, name: :gemstack_jobs_failed
             end
           end
+
+          down do
+            drop_table(:gemstack_jobs)
+          end
         end
       RUBY
 
@@ -44,15 +50,19 @@ module GemStack
     end
 
     module Adapters
-      # The default adapter: jobs are rows in PostgreSQL (DECISIONS D-040).
+      # The default adapter: jobs are rows in the application's database
+      # (DECISIONS D-040, D-063) — PostgreSQL, MySQL or SQLite.
       #
       # - Enqueueing is an INSERT on the current connection, so inside
       #   GemStack.transaction a job exists only if the transaction commits.
-      # - NOTIFY (also transactional) wakes idle workers immediately; polling
-      #   every config.jobs.poll_interval is the safety net.
-      # - Workers claim one job at a time with FOR UPDATE SKIP LOCKED, so
-      #   workers never block each other or run the same job twice at once.
-      class Postgres
+      # - Workers claim one job at a time in a short transaction: with
+      #   FOR UPDATE SKIP LOCKED on PostgreSQL and MySQL 8, so workers never
+      #   block each other; on SQLite by taking the write lock up front.
+      # - PostgreSQL also NOTIFYs idle workers at once; elsewhere (and as a
+      #   safety net) workers poll every config.jobs.poll_interval.
+      # - Times are set from Ruby, in UTC, so the database clock and time zone
+      #   never matter.
+      class Database
         CHANNEL = "gemstack_jobs"
 
         def initialize(db: nil, table: Jobs.config.table)
@@ -68,25 +78,30 @@ module GemStack
         end
 
         def dataset = db[@table]
+        def postgres? = db.database_type == :postgres
 
         def enqueue(payload)
-          id = dataset.insert(
-            job_class: payload["job_class"], queue: payload["queue"], priority: payload["priority"],
-            args: Sequel.pg_jsonb_wrap(payload["args"]), run_at: payload["run_at"] || Sequel::CURRENT_TIMESTAMP
-          )
-          db.notify(CHANNEL, payload: payload["queue"])
+          now = Time.now
+          args = postgres? ? Sequel.pg_jsonb_wrap(payload["args"]) : JSON.generate(payload["args"])
+          id = dataset.insert(job_class: payload["job_class"], queue: payload["queue"], priority: payload["priority"],
+                              args: args, run_at: payload["run_at"] || now, created_at: now)
+          db.notify(CHANNEL, payload: payload["queue"]) if postgres?
           id
         end
 
         # Claims the next ready job for these queues ("*" = all). Returns a
         # payload Hash or nil.
         def claim(queues, worker)
-          ready = dataset.where(failed_at: nil, locked_at: nil).where { run_at <= Sequel::CURRENT_TIMESTAMP }
-          ready = ready.where(queue: queues) unless queues.include?("*")
-          next_id = ready.order(:priority, :run_at, :id).limit(1).for_update.skip_locked.select(:id)
-          row = dataset.where(id: next_id).returning(*returned_columns)
-                       .update(locked_at: Sequel::CURRENT_TIMESTAMP, locked_by: worker).first
-          row && payload_for(row)
+          now = Time.now
+          db.transaction(**claim_options) do
+            ready = dataset.where(failed_at: nil, locked_at: nil).where { run_at <= now }
+            ready = ready.where(queue: queues) unless queues.include?("*")
+            ready = ready.order(:priority, :run_at, :id).limit(1)
+            ready = ready.for_update.skip_locked if ready.supports_skip_locked?
+            row = ready.select(*returned_columns).first
+            dataset.where(id: row[:id]).update(locked_at: now, locked_by: worker) if row
+            row && payload_for(row)
+          end
         end
 
         def complete(id) = dataset.where(id: id).delete
@@ -99,7 +114,7 @@ module GemStack
         def fail(id, attempts:, error:)
           return complete(id) unless Jobs.config.keep_failed
 
-          dataset.where(id: id).update(locked_at: nil, locked_by: nil, failed_at: Sequel::CURRENT_TIMESTAMP,
+          dataset.where(id: id).update(locked_at: nil, locked_by: nil, failed_at: Time.now,
                                        attempts: attempts, last_error: describe(error))
         end
 
@@ -108,7 +123,7 @@ module GemStack
 
         # Releases jobs locked longer than `timeout` seconds (their worker died).
         def release_stale(timeout)
-          cutoff = Sequel.lit("CURRENT_TIMESTAMP - make_interval(secs => ?)", Float(timeout))
+          cutoff = Time.now - Float(timeout)
           dataset.where { locked_at < cutoff }.update(locked_at: nil, locked_by: nil)
         end
 
@@ -116,7 +131,7 @@ module GemStack
         def retry_failed(ids = nil)
           failed = dataset.exclude(failed_at: nil)
           failed = failed.where(id: ids) if ids
-          failed.update(failed_at: nil, attempts: 0, run_at: Sequel::CURRENT_TIMESTAMP, last_error: nil)
+          failed.update(failed_at: nil, attempts: 0, run_at: Time.now, last_error: nil)
         end
 
         def discard_failed(ids = nil)
@@ -132,7 +147,7 @@ module GemStack
 
         # { "default" => { ready:, scheduled:, running:, failed: }, ... }
         def stats
-          now = Sequel::CURRENT_TIMESTAMP
+          now = Time.now
           states = Sequel.case(
             [[Sequel.~(failed_at: nil), "failed"], [Sequel.~(locked_at: nil), "running"],
              [Sequel[:run_at] > now, "scheduled"]], "ready"
@@ -145,22 +160,29 @@ module GemStack
 
         private
 
+        # SQLite: take the write lock when the transaction starts, so two
+        # workers can't both read the same ready row.
+        def claim_options = db.database_type == :sqlite ? { mode: :immediate } : {}
+
         # args as text, parsed with the json gem: jobs receive plain Hash/Array
         # values rather than Sequel's JSONB wrappers. (Lazy: Sequel may not be loaded.)
         def returned_columns
           @returned_columns ||= [:id, :job_class, :queue, :priority, :attempts,
-                                 Sequel.cast(:args, :text).as(:args_json)].freeze
+                                 postgres? ? Sequel.cast(:args, :text).as(:args_json) : Sequel.as(:args, :args_json)]
         end
 
         def payload_for(row)
           { "id" => row[:id], "job_class" => row[:job_class], "queue" => row[:queue],
-            "priority" => row[:priority], "args" => JSON.parse(row[:args_json]), "attempts" => row[:attempts] }
+            "priority" => row[:priority], "args" => JSON.parse(row[:args_json].to_s), "attempts" => row[:attempts] }
         end
 
         def describe(error)
           "#{error.class}: #{error.message}\n#{Array(error.backtrace).first(20).join("\n")}"[0, 10_000]
         end
       end
+
+      # The adapter's name before it supported MySQL and SQLite; `adapter: :postgres` still works.
+      Postgres = Database
     end
   end
 end

data/lib/gemstack/jobs/worker.rb

@@ -18,13 +18,13 @@ module GemStack
     class Worker
       attr_reader :id
 
-      def initialize(store: Adapters::Postgres.new, queues: Jobs.config.queues, concurrency: Jobs.config.concurrency,
+      def initialize(store: Adapters::Database.new, queues: Jobs.config.queues, concurrency: Jobs.config.concurrency,
                      poll_interval: Jobs.config.poll_interval, lock_timeout: Jobs.config.lock_timeout,
                      shutdown_timeout: Jobs.config.shutdown_timeout)
         @store = store
         @queues = Array(queues).map(&:to_s)
         @concurrency = concurrency
-        @poll_interval = poll_interval
+        @poll_interval = poll_interval || (store.respond_to?(:postgres?) && store.postgres? ? 5 : 1)
         @lock_timeout = lock_timeout
         @shutdown_timeout = shutdown_timeout
         @id = "#{Socket.gethostname}:#{Process.pid}:#{SecureRandom.hex(3)}"
@@ -41,7 +41,7 @@ module GemStack
         @running = true
         GemStack.logger.info("jobs worker started", worker: @id, queues: @queues.join(","), concurrency: @concurrency)
         @threads = Array.new(@concurrency) { |i| Thread.new { work_loop(i) } }
-        @listener = Thread.new { listen_loop }
+        @listener = Thread.new { listen_loop } if @store.respond_to?(:postgres?) && @store.postgres?
         @reaper = Thread.new { reap_loop }
         self
       end
@@ -112,7 +112,7 @@ module GemStack
       end
 
       def listen_loop
-        @store.db.listen(Adapters::Postgres::CHANNEL, loop: ->(_conn) { throw :stop unless @running },
+        @store.db.listen(Adapters::Database::CHANNEL, loop: ->(_conn) { throw :stop unless @running },
                                                       timeout: @poll_interval) { wake }
       rescue Sequel::DatabaseConnectionError => e
         GemStack.logger.warn("jobs worker: LISTEN failed, falling back to polling", error: e.message)

data/lib/gemstack/jobs.rb

@@ -20,12 +20,13 @@ module GemStack
   # Delivery is at-least-once: make perform idempotent.
   module Jobs
     class Config < Settings
-      # :postgres (default with gemstack-db), :async (in-process threads),
+      # :database (default with gemstack-db: the app's PostgreSQL, MySQL or
+      # SQLite; :postgres is an alias), :async (in-process threads),
       # :inline (run immediately), :test (record only; default in tests),
       # :sidekiq, or an adapter object responding to #enqueue(payload).
       setting :adapter, default: lambda {
         if GemStack.env.test? then :test
-        elsif defined?(GemStack::DB) then :postgres
+        elsif defined?(GemStack::DB) then :database
         else :async
         end
       }
@@ -35,8 +36,9 @@ module GemStack
       # Worker settings (`gemstack jobs`).
       setting :queues, default: -> { ENV.fetch("GEMSTACK_JOB_QUEUES", "*").split(",").map(&:strip) }
       setting :concurrency, default: -> { Integer(ENV.fetch("GEMSTACK_JOB_CONCURRENCY", 5)) }
-      # Seconds between polls when no NOTIFY arrives (a safety net; NOTIFY wakes workers instantly).
-      setting :poll_interval, default: 5
+      # Seconds between polls. PostgreSQL NOTIFY wakes workers instantly, so
+      # there polling is only a safety net (5 s); MySQL and SQLite rely on it (1 s).
+      setting :poll_interval, default: nil
       # A job locked longer than this is assumed abandoned (worker crashed) and is released.
       # Jobs that legitimately run longer must raise it.
       setting :lock_timeout, default: 30 * 60
@@ -72,7 +74,7 @@ module GemStack
 
       def build_adapter(setting)
         case setting
-        when :postgres, "postgres" then Adapters::Postgres.new
+        when :database, "database", :postgres, "postgres" then Adapters::Database.new
         when :async, "async" then Adapters::Async.new
         when :inline, "inline" then Adapters::Inline.new
         when :test, "test" then Adapters::Test.new
@@ -84,6 +86,9 @@ module GemStack
         end
       end
 
+      # True when jobs are rows in the application's database (and need a worker).
+      def database_queue?(setting = config.adapter) = %w[database postgres].include?(setting.to_s)
+
       # Instrumentation for metrics/monitoring:
       #   GemStack::Jobs.subscribe(:failed) { |event| Sentry.capture_message(...) }
       # Events: :enqueued, :performed, :retried, :failed, :discarded.
@@ -133,7 +138,7 @@ require_relative "jobs/executor"
 require_relative "jobs/adapters/inline"
 require_relative "jobs/adapters/test"
 require_relative "jobs/adapters/async"
-require_relative "jobs/adapters/postgres"
+require_relative "jobs/adapters/database"
 require_relative "jobs/adapters/sidekiq"
 
 GemStack::Config.namespace(:jobs, GemStack::Jobs::Config)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gemstack-jobs
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Shoaib Malik
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: 0.2.0
 email:
 - gemstack26@gmail.com
 executables: []
@@ -35,8 +35,8 @@ files:
 - lib/gemstack/job.rb
 - lib/gemstack/jobs.rb
 - lib/gemstack/jobs/adapters/async.rb
+- lib/gemstack/jobs/adapters/database.rb
 - lib/gemstack/jobs/adapters/inline.rb
-- lib/gemstack/jobs/adapters/postgres.rb
 - lib/gemstack/jobs/adapters/sidekiq.rb
 - lib/gemstack/jobs/adapters/test.rb
 - lib/gemstack/jobs/executor.rb
@@ -67,5 +67,6 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 rubygems_version: 4.0.20
 specification_version: 4
-summary: 'GemStack background jobs: a PostgreSQL queue by default, swappable adapters'
+summary: 'GemStack background jobs: a queue in the app''s database by default, swappable
+  adapters'
 test_files: []