disqualified 0.1.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ec05ac0a0d2eadb072a6cfddd164ae66d3dc14642234663917ebb02c824cecec
-  data.tar.gz: 05f192048112ba00432dc7a1e9844e106f0515e7a7f835e65b2a646f9909b8b6
+  metadata.gz: 34ac6d57608468421c25cb83e842ce8cadc028b922e4437e39580ba8f8009106
+  data.tar.gz: 8bf4b146c66caaeae5a322c013b9be859436e17940f6747643289f76edb84abc
 SHA512:
-  metadata.gz: 23ac21c0d08bd70540746321ba5f6cf427d75027acf49f2f4a4295ad6a52fbcaa821f35b40369d99108e4d6dd7d0ad5c04100cb4d7b6a9a10419cd8129fb8c2f
-  data.tar.gz: eb6495ae699c5a1a9414f54924af2a90b583b5ab249e6437f067696416f2807a3871d164b8ac8ee2c4139ae7342e644658b64fa207f4a8d6c7c664c37a42842e
+  metadata.gz: 2c5c1472c10779a1f55aa88b946801e3ef7ba725538bda4c415572ca8f103c6df81e709399455cc901b419dec15534714fde445d09503f495d9660c7c4bf7f16
+  data.tar.gz: 6eda9531c46d5002efd09c6784551cd2e1cefc7b912143010a92f97ce87db1d478bbcc05c507d22dcb1272f63d97afca57f9d4a758cd67185e86f4588fb7e876

data/README.md

@@ -6,12 +6,48 @@ Since SQLite doesn't have any features like Postgres' `LISTEN`/`NOTIFY`,
 Disqualified resorts to polling the database. This might _disqualify_ it as an
 option for you, but it works well enough for my workload.
 
-Disqualified only works with Rails. It does not work with ActiveJob.
+Note that:
+
+* Disqualified only works with Rails.
+* Disqualified does not support multiple queues.
+* Each Disqualified process assumes it's the only process running. Running
+  multiple instances of Disqualified should not hurt, but it is not supported.
 
 
 ## Usage
 
-Run `bundle exec disqualified --help`
+Run `bundle exec disqualified --help` for more information on how to run the
+Disqualified server. This is what I use in production:
+
+```
+env RAILS_ENV=production bundle exec disqualified
+```
+
+You can use Disqualified with ActiveJob, or you can use it by itself.
+The examples below detail how to use it by by itself. See Installation
+instructions for information on how to set up integration with ActiveJob.
+
+
+### Defining a job
+
+```ruby
+class ComplicatedJob
+  include Disqualified::Job
+
+  def perform(arg1, arg2)
+    # ...
+  end
+end
+```
+
+
+### Queuing
+
+```ruby
+ComplicatedJob.perform_async(1, 2)
+ComplicatedJob.perform_in(1.minute, 1, 2)
+ComplicatedJob.perform_at(3.days.from_now, 1, 2)
+```
 
 
 ## Installation
@@ -25,6 +61,24 @@ bundle binstub disqualified
 ```
 
 
+### ActiveJob
+
+You can optionally set up Disqualified as ActiveJob's default backend.
+
+Usually, you'll just need to update your `config/environments/production.rb`
+file to include something like this.
+
+```ruby
+require "disqualified/active_job"
+
+Rails.application.configure do
+  # ...
+  config.active_job.queue_adapter = :disqualified
+  # ...
+end
+```
+
+
 ## Contributing
 
 PRs are welcome! Please confirm the change with me before you start working;

data/lib/disqualified/active_job.rb

@@ -0,0 +1,25 @@
+require "active_job"
+require "disqualified"
+
+class Disqualified::ActiveJobAdapter
+  include Disqualified::Job
+
+  def perform(serialized_job_data)
+    ::ActiveJob::Base.execute(serialized_job_data)
+  end
+end
+
+module ActiveJob
+  module QueueAdapters
+    class DisqualifiedAdapter
+      def enqueue(job_data)
+        Disqualified::ActiveJobAdapter.perform_async(job_data.serialize)
+      end
+
+      def enqueue_at(job_data, timestamp)
+        timestamp = Time.at(timestamp)
+        Disqualified::ActiveJobAdapter.perform_at(timestamp, job_data.serialize)
+      end
+    end
+  end
+end

data/lib/disqualified/cli.rb

@@ -29,14 +29,18 @@ class Disqualified::CLI
     logger.info { ' / /_/ / (__  ) /_/ / /_/ / /_/ / / / __/ /  __/ /_/ /' }
     logger.info { '/_____/_/____/\__, /\__,_/\__,_/_/_/_/ /_/\___/\__,_/' }
     logger.info { '                /_/' + "v#{Disqualified::VERSION}".rjust(32, " ") }
-    logger.info { "#{Disqualified.server_options.to_s}" }
+    logger.info { Disqualified.server_options.to_s }
 
-    pool = Disqualified::Pool.new(delay_range:, pool_size:, logger:) do |args|
-      args => { promise_index:, running: }
-      logger.debug { format_log("[Disqualified::CLI#run <block>] ##{promise_index}") }
+    pool = Disqualified::Pool.new(delay_range:, pool_size:, error_hooks:, logger:) do |args|
+      args => {promise_index:}
+      logger.debug { format_log("Disqualified::CLI#run <block>", "##{promise_index}") }
       Disqualified::Main.new(error_hooks:, logger:).call
     end
     pool.run!
+  rescue Interrupt
+    pool.shutdown
+    puts
+    puts "Gracefully quitting..."
   end
 
   private

data/lib/disqualified/job.rb

@@ -6,7 +6,7 @@ module Disqualified::Job
   module ClassMethods
     def perform_at(the_time, *args)
       Disqualified::Record.create(
-        handler: self.name,
+        handler: name,
         arguments: JSON.dump(args),
         queue: "default",
         run_at: the_time

data/lib/disqualified/logging.rb

@@ -1,7 +1,21 @@
 module Disqualified::Logging
   module_function
 
-  def format_log(message)
-    "[#{Time.now.iso8601(3)}] #{message}"
+  def format_log(*parts)
+    *extras, message = parts
+
+    if extras.empty?
+      message
+    else
+      "#{extras.map { |x| "[#{x}]" }.join(" ")} #{message}"
+    end
+  end
+
+  def handle_error(error_hooks, error, context)
+    error_hooks.each do |hook|
+      hook.call(error, context)
+    rescue
+      nil
+    end
   end
 end

data/lib/disqualified/main.rb

@@ -9,7 +9,7 @@ class Disqualified::Main
   def call
     run_id = SecureRandom.uuid
 
-    Rails.application.executor.wrap do
+    Rails.application.reloader.wrap do
       # Claim a job
       claimed_count =
         Disqualified::Record
@@ -18,7 +18,7 @@ class Disqualified::Main
           .limit(1)
           .update_all(locked_by: run_id, locked_at: Time.now, updated_at: Time.now, attempts: Arel.sql("attempts + 1"))
 
-      @logger.debug { format_log("[Disqualified::Main#run] [Runner #{run_id}] Claimed #{claimed_count}") }
+      @logger.debug { format_log("Disqualified::Main#call", "Runner #{run_id}", "Claimed #{claimed_count}") }
 
       next if claimed_count == 0
 
@@ -30,12 +30,8 @@ class Disqualified::Main
         handler_class = job.handler.constantize
         arguments = JSON.parse(job.arguments)
 
-        begin
-          @logger.info do
-            format_log("[#{run_id}] Running `#{job.handler}' with #{arguments.size} argument(s)")
-          end
-        rescue
-          nil
+        @logger.info do
+          format_log("Disqualified::Main#call", "Runner #{run_id}" "Running `#{job.handler}'")
         end
 
         # Run the job
@@ -44,20 +40,12 @@ class Disqualified::Main
 
         finish(job)
 
-        begin
-          @logger.info do
-            format_log("[#{run_id}] Done")
-          end
-        rescue
-          nil
+        @logger.info do
+          format_log("Disqualified::Main#call", "Runner #{run_id}", "Done")
         end
       rescue => e
-        @error_hooks.each do |hook|
-          hook.call(e)
-        rescue
-          nil
-        end
-        @logger.error { format_log("[Disqualified::Main#run] [Runner #{run_id}] Rescued Record ##{job&.id}") }
+        handle_error(@error_hooks, e, {record: job.attributes})
+        @logger.error { format_log("Disqualified::Main#run", "Runner #{run_id}", "Rescued Record ##{job&.id}") }
         requeue(job)
       end
     end
@@ -73,7 +61,7 @@ class Disqualified::Main
     # Formula from the Sidekiq wiki
     retry_count = job.attempts - 1
     sleep = (retry_count**4) + 15 + (rand(10) * (retry_count + 1))
-    @logger.error { format_log("[Disqualified::Main#requeue] Sleeping job for ##{sleep} seconds") }
+    @logger.debug { format_log("Disqualified::Main#requeue", "Sleeping job for #{sleep} seconds") }
     job.update!(locked_by: nil, locked_at: nil, run_at: Time.now + sleep)
   end
 end

data/lib/disqualified/pool.rb

@@ -1,58 +1,91 @@
 class Disqualified::Pool
   include Disqualified::Logging
 
-  def initialize(delay_range:, logger:, pool_size:, &task)
+  CHECK = :check
+  QUIT = :quit
+  RUN = :run
+
+  def initialize(delay_range:, logger:, pool_size:, error_hooks:, &task)
     @delay_range = delay_range
     @logger = logger
     @pool_size = pool_size
+    @error_hooks = error_hooks
     @task = task
     @running = Concurrent::AtomicBoolean.new(true)
+    @command_queue = Thread::Queue.new
   end
 
   def run!
+    clock.execute
     Concurrent::Promises
       .zip(*pool)
-      .rescue { |error| handle_error(error) }
+      .rescue { |error| handle_error(@error_hooks, error, {}) }
       .run
       .value!
   end
 
+  def shutdown
+    @running.make_false
+    clock.shutdown
+    @pool_size.times do
+      @command_queue.push(Disqualified::Pool::QUIT)
+    end
+  end
+
+  def clock
+    @clock ||= Concurrent::TimerTask.new(run_now: true) do |clock_task|
+      @logger.debug { format_log("Disqualified::Pool#clock", "Starting") }
+      clock_task.execution_interval = random_interval
+      @command_queue.push(Disqualified::Pool::CHECK)
+      @logger.debug { format_log("Disqualified::Pool#clock", "Next run in #{clock_task.execution_interval}") }
+    rescue => e
+      handle_error(@error_hooks, e, {})
+    end
+  end
+
   def pool
     @pool ||=
       @pool_size.times.map do |promise_index|
-        initial_delay = random_interval * promise_index / @pool_size
-        Concurrent::Promises
-          .schedule(initial_delay) do
-            repeat(promise_index:, schedule: false, previous_delay: initial_delay)
-          end
-          .rescue { |error| handle_error(error) }
+        repeat(promise_index:)
           .run
       end
   end
 
-  def repeat(promise_index:, schedule:, previous_delay:)
+  def repeat(promise_index:)
     if @running.false?
       return
     end
 
-    interval =
-      if schedule
-        random_interval
-      else
-        0
-      end
-
-    @logger.debug { format_log("[Disqualified::Pool#repeat(#{promise_index})] Interval #{interval}") }
+    @logger.debug { format_log("Disqualified::Pool#repeat(#{promise_index})", "Started") }
 
-    args = {
-      promise_index:,
-      running: @running
-    }
+    args = {promise_index:}
 
     Concurrent::Promises
-      .schedule(interval, args, &@task)
-      .then { repeat(promise_index:, schedule: true, previous_delay: interval) }
-      .rescue { |error| handle_error(error) }
+      .future(args) do |args|
+        @logger.debug { format_log("Disqualified::Pool#repeat(#{promise_index}) <pre-exec>", "Waiting for command") }
+        command = @command_queue.pop
+        @logger.debug { format_log("Disqualified::Pool#repeat(#{promise_index}) <pre-exec>", "Command: #{command}") }
+
+        case command
+        when Disqualified::Pool::QUIT
+          nil
+        when Disqualified::Pool::CHECK
+          Rails.application.reloader.wrap do
+            pending_job_count = Disqualified::Record
+              .where(finished_at: nil, run_at: (..Time.now), locked_by: nil)
+              .count
+
+            pending_job_count.times do
+              @command_queue.push(Disqualified::Pool::RUN)
+            end
+          end
+        when Disqualified::Pool::RUN
+          @task.call(args)
+        end
+      rescue => e
+        handle_error(@error_hooks, e, {})
+      end
+      .then { repeat(promise_index:) }
   end
 
   private
@@ -60,10 +93,4 @@ class Disqualified::Pool
   def random_interval
     rand(@delay_range)
   end
-
-  def handle_error(error)
-    pp error
-    puts "Gracefully quitting..."
-    @running.make_false
-  end
 end

data/lib/disqualified/version.rb

@@ -1,3 +1,3 @@
 module Disqualified
-  VERSION = "0.1.1"
+  VERSION = "0.3.0"
 end

data/lib/generators/disqualified/install/templates/20220703062536_create_disqualified_jobs.rb

@@ -1,4 +1,4 @@
-class CreateDisqualifiedJobs < ActiveRecord::Migration[5.0]
+class CreateDisqualifiedJobs < ActiveRecord::Migration[7.0]
   def change
     create_table :disqualified_jobs do |t|
       t.string :handler, null: false

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: disqualified
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Zach Ahn
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-07-18 00:00:00.000000000 Z
+date: 2023-04-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 7.0.0
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: mocktail
   requirement: !ruby/object:Gem::Requirement
@@ -62,12 +76,12 @@ extra_rdoc_files: []
 files:
 - LICENSE
 - README.md
-- app/assets/config/disqualified_manifest.js
 - app/models/disqualified/base_record.rb
 - app/models/disqualified/record.rb
 - config/routes.rb
 - exe/disqualified
 - lib/disqualified.rb
+- lib/disqualified/active_job.rb
 - lib/disqualified/cli.rb
 - lib/disqualified/engine.rb
 - lib/disqualified/job.rb
@@ -79,7 +93,6 @@ files:
 - lib/generators/disqualified/install/USAGE
 - lib/generators/disqualified/install/install_generator.rb
 - lib/generators/disqualified/install/templates/20220703062536_create_disqualified_jobs.rb
-- lib/tasks/disqualified_tasks.rake
 homepage: https://github.com/zachahn/disqualified
 licenses:
 - LGPL-3.0-only
@@ -102,7 +115,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.4.7
 signing_key:
 specification_version: 4
 summary: A background job processor tuned for SQLite

data/lib/tasks/disqualified_tasks.rake

@@ -1,4 +0,0 @@
-# desc "Explaining what the task does"
-# task :disqualified do
-#   # Task goes here
-# end