disqualified 0.3.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 34ac6d57608468421c25cb83e842ce8cadc028b922e4437e39580ba8f8009106
-  data.tar.gz: 8bf4b146c66caaeae5a322c013b9be859436e17940f6747643289f76edb84abc
+  metadata.gz: fbe745886eb9d17ecb6537e23724c48423523695e7c3c1243e166116710d99c0
+  data.tar.gz: f4f9f74ec95f4d3f1f3a8cd2a95907b7d247708676b9c7134c88026813bb7363
 SHA512:
-  metadata.gz: 2c5c1472c10779a1f55aa88b946801e3ef7ba725538bda4c415572ca8f103c6df81e709399455cc901b419dec15534714fde445d09503f495d9660c7c4bf7f16
-  data.tar.gz: 6eda9531c46d5002efd09c6784551cd2e1cefc7b912143010a92f97ce87db1d478bbcc05c507d22dcb1272f63d97afca57f9d4a758cd67185e86f4588fb7e876
+  metadata.gz: 1b2d18b8f154938fcb9a6231aa9ba4edd544b06d9a78dba0f7e1def10b12c597d2258f60a5fd7e7ad9704700dab1f3584c96d94a732a26cceba8f4989166d5a7
+  data.tar.gz: 5c648be8f80189afadc35f90a9e6f3c713550626dfc8744629a3867e8b989552c8455f449fef530c9aa13bbeea4be4acbc9654f36a765b48767051af0e731cbb

data/README.md

@@ -10,6 +10,8 @@ Note that:
 
 * Disqualified only works with Rails.
 * Disqualified does not support multiple queues.
+* Disqualified supports Postgres and MySQL, but it isn't particularly optimized
+  for them.
 * Each Disqualified process assumes it's the only process running. Running
   multiple instances of Disqualified should not hurt, but it is not supported.
 

data/app/models/disqualified/base_record.rb

@@ -1,2 +1,3 @@
 class Disqualified::BaseRecord < ActiveRecord::Base
+  self.abstract_class = true
 end

data/app/models/disqualified/record.rb

@@ -1,3 +1,93 @@
 class Disqualified::Record < Disqualified::BaseRecord
   self.table_name = "disqualified_jobs"
+
+  belongs_to :disqualified_sequence,
+    class_name: "Disqualified::SequenceRecord",
+    foreign_key: "sequence_uuid",
+    primary_key: "uuid",
+    optional: true
+
+  scope :with_sequence, -> {
+    joins("LEFT OUTER JOIN disqualified_sequences ds ON ds.uuid = sequence_uuid AND ds.current_step = sequence_step")
+      .where("ds.uuid = sequence_uuid OR (ds.uuid IS NULL AND sequence_uuid IS NULL)")
+  }
+  scope :pending, -> { where(finished_at: nil, run_at: (..Time.now), locked_by: nil) }
+  scope :runnable, -> { with_sequence.pending }
+
+  def self.claim_one!(id: nil)
+    run_id = SecureRandom.uuid
+    association =
+      Disqualified::Record
+        .runnable
+        .order(run_at: :asc)
+        .limit(1)
+
+    if id
+      association = association.where(id:)
+    end
+
+    claimed_count = association.update_all(
+      locked_by: run_id,
+      locked_at: Time.now,
+      updated_at: Time.now,
+      attempts: Arel.sql("attempts + 1")
+    )
+
+    raise Disqualified::Error::NoClaimableJob if claimed_count == 0
+
+    Disqualified::Record.find_by!(locked_by: run_id)
+  rescue ActiveRecord::RecordNotFound
+    raise Disqualified::Error::NoClaimableJob
+  end
+
+  def run!
+    record = self.class.claim_one!(id:)
+    begin
+      record.send(:instantiate_handler_and_perform_with_args)
+    rescue => e
+      record.unclaim
+      raise e
+    else
+      record.finish
+    end
+    record
+  end
+
+  def finish
+    transaction do
+      update!(locked_by: nil, locked_at: nil, finished_at: Time.now)
+      if sequence_uuid && sequence_step
+        Disqualified::SequenceRecord
+          .where(uuid: sequence_uuid, current_step: sequence_step)
+          .update_all(
+            current_step: sequence_step + 1,
+            updated_at: Time.now
+          )
+      end
+    end
+  end
+
+  def requeue
+    retry_count = attempts - 1
+    sleep = (retry_count**4) + 15 + (rand(10) * (retry_count + 1))
+    unclaim(next_run_at: Time.now + sleep)
+  end
+
+  def unclaim(next_run_at: nil)
+    if next_run_at
+      update!(locked_by: nil, locked_at: nil, run_at: next_run_at)
+    else
+      update!(locked_by: nil, locked_at: nil)
+    end
+  end
+
+  private def instantiate_handler_and_perform_with_args
+    raise Disqualified::Error::JobAlreadyFinished if !finished_at.nil?
+    raise Disqualified::Error::JobNotClaimed if locked_by.nil?
+
+    job_class = handler.constantize
+    parsed_arguments = JSON.parse(arguments)
+    job = job_class.new
+    job.perform(*parsed_arguments)
+  end
 end

data/app/models/disqualified/sequence_record.rb

@@ -0,0 +1,3 @@
+class Disqualified::SequenceRecord < Disqualified::BaseRecord
+  self.table_name = "disqualified_sequences"
+end

data/disqualified.gemspec

@@ -0,0 +1,27 @@
+require_relative "lib/disqualified/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "disqualified"
+  spec.version = Disqualified::VERSION
+  spec.authors = ["Zach Ahn"]
+  spec.email = ["engineering@zachahn.com"]
+  spec.homepage = "https://github.com/zachahn/disqualified"
+  spec.summary = "A background job processor tuned for SQLite"
+  spec.license = "LGPL-3.0-only"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/zachahn/disqualified"
+  spec.metadata["changelog_uri"] = "https://github.com/zachahn/disqualified/blob/main/CHANGELOG.md"
+
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    Dir["{app,exe,lib}/**/*", "*.gemspec", "LICENSE", "README.md"]
+  end
+
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+
+  spec.required_ruby_version = ">= 3.1.0"
+
+  spec.add_dependency "rails", ">= 7.0.0"
+  spec.add_dependency "concurrent-ruby", "~> 1.0"
+end

data/lib/disqualified/active_job.rb

@@ -12,6 +12,10 @@ end
 module ActiveJob
   module QueueAdapters
     class DisqualifiedAdapter
+      def enqueue_after_transaction_commit?
+        Disqualified.client_options.enqueue_after_transaction_commit
+      end
+
       def enqueue(job_data)
         Disqualified::ActiveJobAdapter.perform_async(job_data.serialize)
       end

data/lib/disqualified/cli.rb

@@ -21,14 +21,20 @@ class Disqualified::CLI
 
     option_parser.parse(@original_argv)
 
-    Disqualified.server_options.to_h => {delay_range:, error_hooks:, logger:, pool_size:}
+    server_options = Disqualified.server_options
+    delay_range = server_options.delay_range
+    error_hooks = server_options.error_hooks
+    logger = server_options.logger
+    pool_size = server_options.pool_size
 
+    # standard:disable Style/StringLiterals
     logger.info { '    ____  _                        ___ _____          __' }
     logger.info { '   / __ \(_)________ ___  ______ _/ (_) __(_)__  ____/ /' }
     logger.info { '  / / / / / ___/ __ `/ / / / __ `/ / / /_/ / _ \/ __  /' }
     logger.info { ' / /_/ / (__  ) /_/ / /_/ / /_/ / / / __/ /  __/ /_/ /' }
     logger.info { '/_____/_/____/\__, /\__,_/\__,_/_/_/_/ /_/\___/\__,_/' }
     logger.info { '                /_/' + "v#{Disqualified::VERSION}".rjust(32, " ") }
+    # standard:enable Style/StringLiterals
     logger.info { Disqualified.server_options.to_s }
 
     pool = Disqualified::Pool.new(delay_range:, pool_size:, error_hooks:, logger:) do |args|
@@ -38,7 +44,7 @@ class Disqualified::CLI
     end
     pool.run!
   rescue Interrupt
-    pool.shutdown
+    pool&.shutdown
     puts
     puts "Gracefully quitting..."
   end
@@ -46,19 +52,23 @@ class Disqualified::CLI
   private
 
   def option_parser
-    @option_parser ||= OptionParser.new do |opts|
+    return @option_parser if instance_variable_defined?(:@option_parser)
+
+    option_parser = OptionParser.new do |opts|
       opts.banner = "Usage: #{File.basename($0)} [OPTIONS]"
 
-      opts.on("--delay-low SECONDS", Numeric, "Default: #{Disqualified.server_options.delay_low}") do |value|
-        Disqualified.server_options.delay_low = value
+      server_options = Disqualified.server_options
+
+      opts.on("--delay-low SECONDS", Numeric, "Default: #{server_options.delay_low}") do |value|
+        server_options.delay_low = value
       end
 
-      opts.on("--delay-high SECONDS", Numeric, "Default: #{Disqualified.server_options.delay_high}") do |value|
-        Disqualified.server_options.delay_high = value
+      opts.on("--delay-high SECONDS", Numeric, "Default: #{server_options.delay_high}") do |value|
+        server_options.delay_high = value
       end
 
-      opts.on("--pool COUNT", Integer, "Default: #{Disqualified.server_options.pool_size}") do |value|
-        Disqualified.server_options.pool_size = value
+      opts.on("--pool COUNT", Integer, "Default: #{server_options.pool_size}") do |value|
+        server_options.pool_size = value
       end
 
       opts.on("-h", "--help", "Prints this help") do
@@ -66,5 +76,7 @@ class Disqualified::CLI
         exit
       end
     end
+
+    @option_parser ||= option_parser
   end
 end

data/lib/disqualified/configuration.rb

@@ -0,0 +1,21 @@
+module Disqualified
+  @client_options = ClientConfiguration.new
+
+  class << self
+    attr_accessor :client_options
+
+    attr_accessor :server_options
+
+    def configure_client(&block)
+      block.call(client_options)
+    end
+
+    # While client options are always run, server options only run in the "disqualified" process
+
+    def configure_server(&block)
+      if server_options
+        block.call(server_options)
+      end
+    end
+  end
+end

data/lib/disqualified/{server_configuration.rb → configuration_structs.rb}

@@ -1,30 +1,31 @@
-module Disqualified
-  class << self
-    attr_accessor :server_options
+class Disqualified::ClientConfiguration
+  def initialize
+    @enqueue_after_transaction_commit = false
   end
 
-  def self.configure_server
-    if server_options
-      yield server_options
-    end
-  end
+  attr_accessor :enqueue_after_transaction_commit
 end
 
 class Disqualified::ServerConfiguration
   def initialize
-    self.delay_high = 5.0
-    self.delay_low = 1.0
-    self.logger = Rails.logger
-    self.pool_size = 5
-    self.pwd = Dir.pwd
-    self.error_hooks = []
+    @delay_high = 5.0
+    @delay_low = 1.0
+    @logger = Rails.logger
+    @pool_size = 5
+    @pwd = Dir.pwd
+    @error_hooks = []
   end
 
   attr_accessor :delay_high
+
   attr_accessor :delay_low
+
   attr_accessor :error_hooks
+
   attr_accessor :logger
+
   attr_accessor :pool_size
+
   attr_accessor :pwd
 
   private :error_hooks=
@@ -37,18 +38,6 @@ class Disqualified::ServerConfiguration
     error_hooks.push(block)
   end
 
-  def to_h
-    {
-      delay_high:,
-      delay_low:,
-      delay_range:,
-      error_hooks:,
-      logger:,
-      pool_size:,
-      pwd:
-    }
-  end
-
   def to_s
     "{ delay: #{delay_range}, pool_size: #{pool_size}, error_hooks_size: #{error_hooks.size} }"
   end

data/lib/disqualified/error.rb

@@ -0,0 +1,15 @@
+module Disqualified
+  module Error
+    class DisqualifiedError < StandardError
+    end
+
+    class JobAlreadyFinished < DisqualifiedError
+    end
+
+    class JobNotClaimed < DisqualifiedError
+    end
+
+    class NoClaimableJob < DisqualifiedError
+    end
+  end
+end

data/lib/disqualified/job.rb

@@ -1,15 +1,19 @@
 module Disqualified::Job
-  def self.included(base)
-    base.extend ClassMethods
-  end
-
   module ClassMethods
     def perform_at(the_time, *args)
-      Disqualified::Record.create(
+      if Thread.current[Disqualified::Sequence::UUID]
+        Thread.current[Disqualified::Sequence::COUNT] += 1
+        sequence_uuid = Thread.current[Disqualified::Sequence::UUID]
+        sequence_step = Thread.current[Disqualified::Sequence::COUNT]
+      end
+
+      Disqualified::Record.create!(
         handler: name,
         arguments: JSON.dump(args),
         queue: "default",
-        run_at: the_time
+        run_at: the_time,
+        sequence_uuid:,
+        sequence_step:
       )
     end
 
@@ -21,4 +25,8 @@ module Disqualified::Job
       perform_at(Time.now + delay, *args)
     end
   end
+
+  def self.included(other)
+    other.extend(ClassMethods)
+  end
 end

data/lib/disqualified/logging.rb

@@ -1,4 +1,12 @@
 module Disqualified::Logging
+  ERROR_CONTEXT_TYPE = T.type_alias do
+    T::Hash[T.untyped, T.untyped]
+  end
+
+  ERROR_HOOK_TYPE = T.type_alias do
+    T.proc.params(arg0: Exception, arg1: ERROR_CONTEXT_TYPE).void
+  end
+
   module_function
 
   def format_log(*parts)

data/lib/disqualified/main.rb

@@ -7,61 +7,22 @@ class Disqualified::Main
   end
 
   def call
-    run_id = SecureRandom.uuid
-
     Rails.application.reloader.wrap do
-      # Claim a job
-      claimed_count =
-        Disqualified::Record
-          .where(finished_at: nil, run_at: (..Time.now), locked_by: nil)
-          .order(run_at: :asc)
-          .limit(1)
-          .update_all(locked_by: run_id, locked_at: Time.now, updated_at: Time.now, attempts: Arel.sql("attempts + 1"))
-
-      @logger.debug { format_log("Disqualified::Main#call", "Runner #{run_id}", "Claimed #{claimed_count}") }
-
-      next if claimed_count == 0
-
-      # Find the job that we claimed; quit early if none was claimed
-      job = Disqualified::Record.find_by!(locked_by: run_id)
-
-      begin
-        # Deserialize job
-        handler_class = job.handler.constantize
-        arguments = JSON.parse(job.arguments)
-
-        @logger.info do
-          format_log("Disqualified::Main#call", "Runner #{run_id}" "Running `#{job.handler}'")
-        end
-
-        # Run the job
-        handler = handler_class.new
-        handler.perform(*arguments)
-
-        finish(job)
-
-        @logger.info do
-          format_log("Disqualified::Main#call", "Runner #{run_id}", "Done")
-        end
-      rescue => e
-        handle_error(@error_hooks, e, {record: job.attributes})
-        @logger.error { format_log("Disqualified::Main#run", "Runner #{run_id}", "Rescued Record ##{job&.id}") }
-        requeue(job)
+      record = Disqualified::Record.claim_one!
+      run_id = record.locked_by
+      record.send(:instantiate_handler_and_perform_with_args)
+      record.finish
+      @logger.info do
+        format_log("Disqualified::Main#call", "Runner #{run_id}", "Done")
+      end
+    rescue Disqualified::Error::NoClaimableJob
+      @logger.warn do
+        format_log("Disqualified::Main#call", "No claimable jobs")
       end
+    rescue => e
+      handle_error(@error_hooks, e, {record: record.attributes})
+      @logger.error { format_log("Disqualified::Main#run", "Runner #{run_id}", "Rescued Record ##{record.id}") }
+      record.requeue
     end
   end
-
-  private
-
-  def finish(job)
-    job.update!(locked_by: nil, locked_at: nil, finished_at: Time.now)
-  end
-
-  def requeue(job)
-    # Formula from the Sidekiq wiki
-    retry_count = job.attempts - 1
-    sleep = (retry_count**4) + 15 + (rand(10) * (retry_count + 1))
-    @logger.debug { format_log("Disqualified::Main#requeue", "Sleeping job for #{sleep} seconds") }
-    job.update!(locked_by: nil, locked_at: nil, run_at: Time.now + sleep)
-  end
 end

data/lib/disqualified/pool.rb

@@ -44,11 +44,10 @@ class Disqualified::Pool
   end
 
   def pool
-    @pool ||=
-      @pool_size.times.map do |promise_index|
-        repeat(promise_index:)
-          .run
-      end
+    @pool ||= @pool_size.times.map do |promise_index|
+      repeat(promise_index:)
+        &.run
+    end
   end
 
   def repeat(promise_index:)

data/lib/disqualified/sequence.rb

@@ -0,0 +1,21 @@
+class Disqualified::Sequence
+  UUID = :disqualified_sequence_uuid
+  COUNT = :disqualified_sequence_count
+
+  def self.queue(description: nil, &block)
+    Disqualified::SequenceRecord.transaction do
+      Thread.current[UUID] = SecureRandom.uuid
+      Thread.current[COUNT] = 0
+      yield
+      Disqualified::SequenceRecord.create!(
+        uuid: Thread.current[UUID],
+        current_step: 1,
+        final_step: Thread.current[COUNT],
+        description:
+      )
+    end
+  ensure
+    Thread.current[UUID] = nil
+    Thread.current[COUNT] = nil
+  end
+end

data/lib/disqualified/version.rb

@@ -1,3 +1,3 @@
 module Disqualified
-  VERSION = "0.3.0"
+  VERSION = "0.5.0"
 end

data/lib/disqualified.rb

@@ -5,12 +5,17 @@ require "optparse"
 
 require "concurrent"
 require "rails"
+require "sorbet-runtime"
 
+require_relative "disqualified/error"
 require_relative "disqualified/logging"
 
+require_relative "disqualified/configuration_structs"
+
+require_relative "disqualified/configuration"
 require_relative "disqualified/engine"
 require_relative "disqualified/job"
 require_relative "disqualified/main"
 require_relative "disqualified/pool"
-require_relative "disqualified/server_configuration"
+require_relative "disqualified/sequence"
 require_relative "disqualified/version"

data/lib/generators/disqualified/install/USAGE

@@ -3,6 +3,8 @@ Description:
 
 Example:
     bin/rails generate disqualified:install
+    bin/rails generate disqualified:install --database primary
+    bin/rails g disqualified:install --db=animals
 
     This will create:
         db/migrate/20220703062536_create_disqualified_jobs.rb

data/lib/generators/disqualified/install/install_generator.rb

@@ -1,8 +1,20 @@
+require "rails/generators/active_record"
+
 class Disqualified::InstallGenerator < Rails::Generators::Base
+  include ActiveRecord::Generators::Migration
+
   source_root File.expand_path("templates", __dir__)
 
+  class_option :database,
+    type: :string,
+    aliases: %i[--db],
+    desc: "The database for your migration. By default, the current environment's primary database is used."
+
   def copy_migration_file
     basename = "20220703062536_create_disqualified_jobs.rb"
-    copy_file basename, "db/migrate/#{basename}"
+    copy_file(basename, File.join(db_migrate_path, basename))
+
+    basename = "20241119023328_create_disqualified_sequences.rb"
+    copy_file(basename, File.join(db_migrate_path, basename))
   end
 end

data/lib/generators/disqualified/install/templates/20241119023328_create_disqualified_sequences.rb

@@ -0,0 +1,17 @@
+class CreateDisqualifiedSequences < ActiveRecord::Migration[7.2]
+  def change
+    create_table :disqualified_sequences do |t|
+      t.text :uuid, null: false
+      t.integer :current_step, null: false
+      t.integer :final_step, null: false
+      t.text :description
+      t.timestamps
+
+      t.index :uuid, unique: true
+    end
+
+    add_column :disqualified_jobs, :sequence_uuid, :text
+    add_column :disqualified_jobs, :sequence_step, :integer
+    add_index :disqualified_jobs, [:sequence_uuid, :sequence_step], unique: true
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: disqualified
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Zach Ahn
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-04-16 00:00:00.000000000 Z
+date: 2024-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -28,44 +28,16 @@ dependencies:
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: mocktail
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: standard
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
 description:
 email:
 - engineering@zachahn.com
@@ -78,21 +50,26 @@ files:
 - README.md
 - app/models/disqualified/base_record.rb
 - app/models/disqualified/record.rb
-- config/routes.rb
+- app/models/disqualified/sequence_record.rb
+- disqualified.gemspec
 - exe/disqualified
 - lib/disqualified.rb
 - lib/disqualified/active_job.rb
 - lib/disqualified/cli.rb
+- lib/disqualified/configuration.rb
+- lib/disqualified/configuration_structs.rb
 - lib/disqualified/engine.rb
+- lib/disqualified/error.rb
 - lib/disqualified/job.rb
 - lib/disqualified/logging.rb
 - lib/disqualified/main.rb
 - lib/disqualified/pool.rb
-- lib/disqualified/server_configuration.rb
+- lib/disqualified/sequence.rb
 - lib/disqualified/version.rb
 - lib/generators/disqualified/install/USAGE
 - lib/generators/disqualified/install/install_generator.rb
 - lib/generators/disqualified/install/templates/20220703062536_create_disqualified_jobs.rb
+- lib/generators/disqualified/install/templates/20241119023328_create_disqualified_sequences.rb
 homepage: https://github.com/zachahn/disqualified
 licenses:
 - LGPL-3.0-only
@@ -115,7 +92,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.7
+rubygems_version: 3.5.11
 signing_key:
 specification_version: 4
 summary: A background job processor tuned for SQLite

data/config/routes.rb

@@ -1,2 +0,0 @@
-Rails.application.routes.draw do
-end