disqualified 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fc97aaa11362ee9b3b2de020309c6ad083dce7c9c0dfa8337efdac9dc6194f81
-  data.tar.gz: d78e23de661b6e78cffa4a46253ad3abd7a00715593307fe043d5d24776df820
+  metadata.gz: 34ac6d57608468421c25cb83e842ce8cadc028b922e4437e39580ba8f8009106
+  data.tar.gz: 8bf4b146c66caaeae5a322c013b9be859436e17940f6747643289f76edb84abc
 SHA512:
-  metadata.gz: ddca0e4fa03d30f0c974003baa82be69d56e17128eb8a4a3d9643fda4a021562eb805b128b02f1d797213d23ef4be5c9d93e053f7a15abb52c9725a8fd51d2a6
-  data.tar.gz: 79aa2ca84d392afb379c9a3ad9987b0df49ce0189deb1bde5d0ba3fc325e43b14da03e394296feacc785d32ff01955dabbcd7422930f442e53197d851011af50
+  metadata.gz: 2c5c1472c10779a1f55aa88b946801e3ef7ba725538bda4c415572ca8f103c6df81e709399455cc901b419dec15534714fde445d09503f495d9660c7c4bf7f16
+  data.tar.gz: 6eda9531c46d5002efd09c6784551cd2e1cefc7b912143010a92f97ce87db1d478bbcc05c507d22dcb1272f63d97afca57f9d4a758cd67185e86f4588fb7e876

data/README.md

@@ -6,13 +6,26 @@ Since SQLite doesn't have any features like Postgres' `LISTEN`/`NOTIFY`,
 Disqualified resorts to polling the database. This might _disqualify_ it as an
 option for you, but it works well enough for my workload.
 
-Disqualified only works with Rails. You can use it with ActiveJob or by itself.
+Note that:
+
+* Disqualified only works with Rails.
+* Disqualified does not support multiple queues.
+* Each Disqualified process assumes it's the only process running. Running
+  multiple instances of Disqualified should not hurt, but it is not supported.
 
 
 ## Usage
 
 Run `bundle exec disqualified --help` for more information on how to run the
-Disqualified server.
+Disqualified server. This is what I use in production:
+
+```
+env RAILS_ENV=production bundle exec disqualified
+```
+
+You can use Disqualified with ActiveJob, or you can use it by itself.
+The examples below detail how to use it by by itself. See Installation
+instructions for information on how to set up integration with ActiveJob.
 
 
 ### Defining a job

data/lib/disqualified/cli.rb

@@ -31,12 +31,16 @@ class Disqualified::CLI
     logger.info { '                /_/' + "v#{Disqualified::VERSION}".rjust(32, " ") }
     logger.info { Disqualified.server_options.to_s }
 
-    pool = Disqualified::Pool.new(delay_range:, pool_size:, logger:) do |args|
-      args => { promise_index:, running: }
-      logger.debug { format_log("[Disqualified::CLI#run <block>] ##{promise_index}") }
+    pool = Disqualified::Pool.new(delay_range:, pool_size:, error_hooks:, logger:) do |args|
+      args => {promise_index:}
+      logger.debug { format_log("Disqualified::CLI#run <block>", "##{promise_index}") }
       Disqualified::Main.new(error_hooks:, logger:).call
     end
     pool.run!
+  rescue Interrupt
+    pool.shutdown
+    puts
+    puts "Gracefully quitting..."
   end
 
   private

data/lib/disqualified/logging.rb

@@ -1,7 +1,21 @@
 module Disqualified::Logging
   module_function
 
-  def format_log(message)
-    "[#{Time.now.iso8601(3)}] #{message}"
+  def format_log(*parts)
+    *extras, message = parts
+
+    if extras.empty?
+      message
+    else
+      "#{extras.map { |x| "[#{x}]" }.join(" ")} #{message}"
+    end
+  end
+
+  def handle_error(error_hooks, error, context)
+    error_hooks.each do |hook|
+      hook.call(error, context)
+    rescue
+      nil
+    end
   end
 end

data/lib/disqualified/main.rb

@@ -18,7 +18,7 @@ class Disqualified::Main
           .limit(1)
           .update_all(locked_by: run_id, locked_at: Time.now, updated_at: Time.now, attempts: Arel.sql("attempts + 1"))
 
-      @logger.debug { format_log("[Disqualified::Main#run] [Runner #{run_id}] Claimed #{claimed_count}") }
+      @logger.debug { format_log("Disqualified::Main#call", "Runner #{run_id}", "Claimed #{claimed_count}") }
 
       next if claimed_count == 0
 
@@ -30,12 +30,8 @@ class Disqualified::Main
         handler_class = job.handler.constantize
         arguments = JSON.parse(job.arguments)
 
-        begin
-          @logger.info do
-            format_log("[#{run_id}] Running `#{job.handler}' with #{arguments.size} argument(s)")
-          end
-        rescue
-          nil
+        @logger.info do
+          format_log("Disqualified::Main#call", "Runner #{run_id}" "Running `#{job.handler}'")
         end
 
         # Run the job
@@ -44,20 +40,12 @@ class Disqualified::Main
 
         finish(job)
 
-        begin
-          @logger.info do
-            format_log("[#{run_id}] Done")
-          end
-        rescue
-          nil
+        @logger.info do
+          format_log("Disqualified::Main#call", "Runner #{run_id}", "Done")
         end
       rescue => e
-        @error_hooks.each do |hook|
-          hook.call(e)
-        rescue
-          nil
-        end
-        @logger.error { format_log("[Disqualified::Main#run] [Runner #{run_id}] Rescued Record ##{job&.id}") }
+        handle_error(@error_hooks, e, {record: job.attributes})
+        @logger.error { format_log("Disqualified::Main#run", "Runner #{run_id}", "Rescued Record ##{job&.id}") }
         requeue(job)
       end
     end
@@ -73,7 +61,7 @@ class Disqualified::Main
     # Formula from the Sidekiq wiki
     retry_count = job.attempts - 1
     sleep = (retry_count**4) + 15 + (rand(10) * (retry_count + 1))
-    @logger.error { format_log("[Disqualified::Main#requeue] Sleeping job for ##{sleep} seconds") }
+    @logger.debug { format_log("Disqualified::Main#requeue", "Sleeping job for #{sleep} seconds") }
     job.update!(locked_by: nil, locked_at: nil, run_at: Time.now + sleep)
   end
 end

data/lib/disqualified/pool.rb

@@ -1,58 +1,91 @@
 class Disqualified::Pool
   include Disqualified::Logging
 
-  def initialize(delay_range:, logger:, pool_size:, &task)
+  CHECK = :check
+  QUIT = :quit
+  RUN = :run
+
+  def initialize(delay_range:, logger:, pool_size:, error_hooks:, &task)
     @delay_range = delay_range
     @logger = logger
     @pool_size = pool_size
+    @error_hooks = error_hooks
     @task = task
     @running = Concurrent::AtomicBoolean.new(true)
+    @command_queue = Thread::Queue.new
   end
 
   def run!
+    clock.execute
     Concurrent::Promises
       .zip(*pool)
-      .rescue { |error| handle_error(error) }
+      .rescue { |error| handle_error(@error_hooks, error, {}) }
       .run
       .value!
   end
 
+  def shutdown
+    @running.make_false
+    clock.shutdown
+    @pool_size.times do
+      @command_queue.push(Disqualified::Pool::QUIT)
+    end
+  end
+
+  def clock
+    @clock ||= Concurrent::TimerTask.new(run_now: true) do |clock_task|
+      @logger.debug { format_log("Disqualified::Pool#clock", "Starting") }
+      clock_task.execution_interval = random_interval
+      @command_queue.push(Disqualified::Pool::CHECK)
+      @logger.debug { format_log("Disqualified::Pool#clock", "Next run in #{clock_task.execution_interval}") }
+    rescue => e
+      handle_error(@error_hooks, e, {})
+    end
+  end
+
   def pool
     @pool ||=
       @pool_size.times.map do |promise_index|
-        initial_delay = random_interval * promise_index / @pool_size
-        Concurrent::Promises
-          .schedule(initial_delay) do
-            repeat(promise_index:, schedule: false, previous_delay: initial_delay)
-          end
-          .rescue { |error| handle_error(error) }
+        repeat(promise_index:)
           .run
       end
   end
 
-  def repeat(promise_index:, schedule:, previous_delay:)
+  def repeat(promise_index:)
     if @running.false?
       return
     end
 
-    interval =
-      if schedule
-        random_interval
-      else
-        0
-      end
-
-    @logger.debug { format_log("[Disqualified::Pool#repeat(#{promise_index})] Interval #{interval}") }
+    @logger.debug { format_log("Disqualified::Pool#repeat(#{promise_index})", "Started") }
 
-    args = {
-      promise_index:,
-      running: @running
-    }
+    args = {promise_index:}
 
     Concurrent::Promises
-      .schedule(interval, args, &@task)
-      .then { repeat(promise_index:, schedule: true, previous_delay: interval) }
-      .rescue { |error| handle_error(error) }
+      .future(args) do |args|
+        @logger.debug { format_log("Disqualified::Pool#repeat(#{promise_index}) <pre-exec>", "Waiting for command") }
+        command = @command_queue.pop
+        @logger.debug { format_log("Disqualified::Pool#repeat(#{promise_index}) <pre-exec>", "Command: #{command}") }
+
+        case command
+        when Disqualified::Pool::QUIT
+          nil
+        when Disqualified::Pool::CHECK
+          Rails.application.reloader.wrap do
+            pending_job_count = Disqualified::Record
+              .where(finished_at: nil, run_at: (..Time.now), locked_by: nil)
+              .count
+
+            pending_job_count.times do
+              @command_queue.push(Disqualified::Pool::RUN)
+            end
+          end
+        when Disqualified::Pool::RUN
+          @task.call(args)
+        end
+      rescue => e
+        handle_error(@error_hooks, e, {})
+      end
+      .then { repeat(promise_index:) }
   end
 
   private
@@ -60,10 +93,4 @@ class Disqualified::Pool
   def random_interval
     rand(@delay_range)
   end
-
-  def handle_error(error)
-    pp error
-    puts "Gracefully quitting..."
-    @running.make_false
-  end
 end

data/lib/disqualified/version.rb

@@ -1,3 +1,3 @@
 module Disqualified
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/lib/generators/disqualified/install/templates/20220703062536_create_disqualified_jobs.rb

@@ -1,4 +1,4 @@
-class CreateDisqualifiedJobs < ActiveRecord::Migration[5.0]
+class CreateDisqualifiedJobs < ActiveRecord::Migration[7.0]
   def change
     create_table :disqualified_jobs do |t|
       t.string :handler, null: false

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: disqualified
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Zach Ahn
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-08-17 00:00:00.000000000 Z
+date: 2023-04-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -115,7 +115,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.4.7
 signing_key:
 specification_version: 4
 summary: A background job processor tuned for SQLite