disqualified 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a30c5111164b062b8615ff07beb0904fca3c38b1c279e92a09c31e30fccecb9f
+  data.tar.gz: ee299e3d06536fcb745f14281b68599c89c1f72bedfa70892d76efd9f2953324
+SHA512:
+  metadata.gz: 282d988287c0d221ad5e0a05d0c386d0f1869cca2e617b8d4ad4e87743e46b0f0467232059e1e7ac32eedb3a162794b710564f3337a00d54adbbf98edb781506
+  data.tar.gz: 02e1a353c89adcb35ff1bce7960265a0576280b280de404c2532ecddea8affd496e4cb3f58ca93c15a27651150a70f8496bb9d359cf5366c5c5c68c1c8616112

data/LICENSE

@@ -0,0 +1,165 @@
+                   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.

data/README.md

@@ -0,0 +1,40 @@
+# Disqualified
+
+I wanted a small background job framework that used SQLite as the backend.
+
+Since SQLite doesn't have any features like Postgres' `LISTEN`/`NOTIFY`,
+Disqualified resorts to polling the database. This might _disqualify_ it as an
+option for you, but it works well enough for my workload.
+
+Disqualified only works with Rails. It does not work with ActiveJob.
+
+
+## Usage
+
+Run `bundle exec disqualified --help`
+
+
+## Installation
+
+Run this in your shell, in your Rails app.
+
+```bash
+bundle add disqualified
+bundle exec rails generate disqualified:install
+bundle binstub disqualified
+```
+
+
+## Contributing
+
+PRs are welcome! Please confirm the change with me before you start working;
+some features might be better off as a plugin or as a fork.
+
+When submitting a PR, and if you prefer, please `Allow edits from maintainers`.
+This will help get your contributions merged in a bit faster.
+
+
+## License
+
+The gem is available as open source under the terms of the
+[LGPL v3 License](https://opensource.org/licenses/LGPL-3.0).

data/app/models/disqualified/base_record.rb

@@ -0,0 +1,2 @@
+class Disqualified::BaseRecord < ActiveRecord::Base
+end

data/app/models/disqualified/record.rb

@@ -0,0 +1,3 @@
+class Disqualified::Record < Disqualified::BaseRecord
+  self.table_name = "disqualified_jobs"
+end

data/config/routes.rb

@@ -0,0 +1,2 @@
+Rails.application.routes.draw do
+end

data/exe/disqualified

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require_relative "../lib/disqualified"
+require_relative "../lib/disqualified/cli"
+
+Disqualified::CLI.run

data/lib/disqualified/cli.rb

@@ -0,0 +1,66 @@
+class Disqualified::CLI
+  include Disqualified::Logging
+
+  def self.run
+    cli = new(ARGV)
+    cli.run
+  end
+
+  class ServerEngine < Rails::Engine
+    config.before_initialize do
+      Disqualified.server_options = Disqualified::ServerConfiguration.new
+    end
+  end
+
+  def initialize(argv)
+    @original_argv = argv
+  end
+
+  def run
+    require File.join(Dir.pwd, "config/environment")
+
+    option_parser.parse(@original_argv)
+
+    Disqualified.server_options.to_h => {delay_range:, error_hooks:, logger:, pool_size:}
+
+    logger.info { '    ____  _                        ___ _____          __' }
+    logger.info { '   / __ \(_)________ ___  ______ _/ (_) __(_)__  ____/ /' }
+    logger.info { '  / / / / / ___/ __ `/ / / / __ `/ / / /_/ / _ \/ __  /' }
+    logger.info { ' / /_/ / (__  ) /_/ / /_/ / /_/ / / / __/ /  __/ /_/ /' }
+    logger.info { '/_____/_/____/\__, /\__,_/\__,_/_/_/_/ /_/\___/\__,_/' }
+    logger.info { '                /_/' + "v#{Disqualified::VERSION}".rjust(32, " ") }
+    logger.info { "#{Disqualified.server_options.to_s}" }
+
+    pool = Disqualified::Pool.new(delay_range:, pool_size:, logger:) do |args|
+      args => { promise_index:, running: }
+      logger.debug { format_log("[Disqualified::CLI#run <block>] ##{promise_index}") }
+      Disqualified::Main.new(error_hooks:, logger:).call
+    end
+    pool.run!
+  end
+
+  private
+
+  def option_parser
+    @option_parser ||= OptionParser.new do |opts|
+      opts.banner = "Usage: #{File.basename($0)} [OPTIONS]"
+
+      opts.on("--delay-low SECONDS", Numeric, "Default: #{Disqualified.server_options.delay_low}") do |value|
+        Disqualified.server_options.delay_low = value
+      end
+
+      opts.on("--delay-high SECONDS", Numeric, "Default: #{Disqualified.server_options.delay_high}") do |value|
+        Disqualified.server_options.delay_high = value
+      end
+
+      opts.on("--pool COUNT", Integer, "Default: #{Disqualified.server_options.pool_size}") do |value|
+        Disqualified.server_options.pool_size = value
+      end
+
+      opts.on("-h", "--help", "Prints this help") do
+        puts opts
+        exit
+      end
+    end
+  end
+end

data/lib/disqualified/engine.rb

@@ -0,0 +1,4 @@
+module Disqualified
+  class Engine < ::Rails::Engine
+  end
+end

data/lib/disqualified/job.rb

@@ -0,0 +1,24 @@
+module Disqualified::Job
+  def self.included(base)
+    base.extend ClassMethods
+  end
+
+  module ClassMethods
+    def perform_at(the_time, *args)
+      Disqualified::Record.create(
+        handler: self.name,
+        arguments: JSON.dump(args),
+        queue: "default",
+        run_at: the_time
+      )
+    end
+
+    def perform_async(*args)
+      perform_at(Time.now, *args)
+    end
+
+    def perform_in(delay, *args)
+      perform_at(Time.now + delay, *args)
+    end
+  end
+end

data/lib/disqualified/logging.rb

@@ -0,0 +1,7 @@
+module Disqualified::Logging
+  module_function
+
+  def format_log(message)
+    "[#{Time.now.iso8601(3)}] #{message}"
+  end
+end

data/lib/disqualified/main.rb

@@ -0,0 +1,79 @@
+class Disqualified::Main
+  include Disqualified::Logging
+
+  def initialize(error_hooks:, logger:)
+    @error_hooks = error_hooks
+    @logger = logger
+  end
+
+  def call
+    run_id = SecureRandom.uuid
+
+    Rails.application.executor.wrap do
+      # Claim a job
+      claimed_count =
+        Disqualified::Record
+          .where(finished_at: nil, run_at: (..Time.now), locked_by: nil)
+          .order(run_at: :asc)
+          .limit(1)
+          .update_all(locked_by: run_id, locked_at: Time.now, updated_at: Time.now, attempts: Arel.sql("attempts + 1"))
+
+      @logger.debug { format_log("[Disqualified::Main#run] [Runner #{run_id}] Claimed #{claimed_count}") }
+
+      next if claimed_count == 0
+
+      # Find the job that we claimed; quit early if none was claimed
+      job = Disqualified::Record.find_by!(locked_by: run_id)
+
+      begin
+        # Deserialize job
+        handler_class = job.handler.constantize
+        arguments = JSON.parse(job.arguments)
+
+        begin
+          @logger.info do
+            format_log("[#{run_id}] Running `#{job.handler}' with #{arguments.size} argument(s)")
+          end
+        rescue
+          nil
+        end
+
+        # Run the job
+        handler = handler_class.new
+        handler.perform(*arguments)
+
+        finish(job)
+
+        begin
+          @logger.info do
+            format_log("[#{run_id}] Done")
+          end
+        rescue
+          nil
+        end
+      rescue => e
+        @error_hooks.each do |hook|
+          hook.call(e)
+        rescue
+          nil
+        end
+        @logger.error { format_log("[Disqualified::Main#run] [Runner #{run_id}] Rescued Record ##{job&.id}") }
+        requeue(job)
+      end
+    end
+  end
+
+  private
+
+  def finish(job)
+    job.update!(locked_by: nil, locked_at: nil, finished_at: Time.now)
+  end
+
+  def requeue(job)
+    # Formula from the Sidekiq wiki
+    retry_count = job.attempts - 1
+    sleep = (retry_count**4) + 15 + (rand(10) * (retry_count + 1))
+    @logger.error { format_log("[Disqualified::Main#requeue] Sleeping job for ##{sleep} seconds") }
+    job.update!(locked_by: nil, locked_at: nil, run_at: Time.now + sleep)
+  end
+end

data/lib/disqualified/pool.rb

@@ -0,0 +1,69 @@
+class Disqualified::Pool
+  include Disqualified::Logging
+
+  def initialize(delay_range:, logger:, pool_size:, &task)
+    @delay_range = delay_range
+    @logger = logger
+    @pool_size = pool_size
+    @task = task
+    @running = Concurrent::AtomicBoolean.new(true)
+  end
+
+  def run!
+    Concurrent::Promises
+      .zip(*pool)
+      .rescue { |error| handle_error(error) }
+      .run
+      .value!
+  end
+
+  def pool
+    @pool ||=
+      @pool_size.times.map do |promise_index|
+        initial_delay = random_interval * promise_index / @pool_size
+        Concurrent::Promises
+          .schedule(initial_delay) do
+            repeat(promise_index:, schedule: false, previous_delay: initial_delay)
+          end
+          .rescue { |error| handle_error(error) }
+          .run
+      end
+  end
+
+  def repeat(promise_index:, schedule:, previous_delay:)
+    if @running.false?
+      return
+    end
+
+    interval =
+      if schedule
+        random_interval
+      else
+        0
+      end
+
+    @logger.debug { format_log("[Disqualified::Pool#repeat(#{promise_index})] Interval #{interval}") }
+
+    args = {
+      promise_index:,
+      running: @running
+    }
+
+    Concurrent::Promises
+      .schedule(interval, args, &@task)
+      .then { repeat(promise_index:, schedule: true, previous_delay: interval) }
+      .rescue { |error| handle_error(error) }
+  end
+
+  private
+
+  def random_interval
+    rand(@delay_range)
+  end
+
+  def handle_error(error)
+    pp error
+    puts "Gracefully quitting..."
+    @running.make_false
+  end
+end

data/lib/disqualified/server_configuration.rb

@@ -0,0 +1,55 @@
+module Disqualified
+  class << self
+    attr_accessor :server_options
+  end
+
+  def self.configure_server
+    if server_options
+      yield server_options
+    end
+  end
+end
+
+class Disqualified::ServerConfiguration
+  def initialize
+    self.delay_high = 5.0
+    self.delay_low = 1.0
+    self.logger = Rails.logger
+    self.pool_size = 5
+    self.pwd = Dir.pwd
+    self.error_hooks = []
+  end
+
+  attr_accessor :delay_high
+  attr_accessor :delay_low
+  attr_accessor :error_hooks
+  attr_accessor :logger
+  attr_accessor :pool_size
+  attr_accessor :pwd
+
+  private :error_hooks=
+
+  def delay_range
+    delay_low.to_f..delay_high.to_f
+  end
+
+  def on_error(&block)
+    error_hooks.push(block)
+  end
+
+  def to_h
+    {
+      delay_high:,
+      delay_low:,
+      delay_range:,
+      error_hooks:,
+      logger:,
+      pool_size:,
+      pwd:
+    }
+  end
+
+  def to_s
+    "{ delay: #{delay_range}, pool_size: #{pool_size}, error_hooks_size: #{error_hooks.size} }"
+  end
+end

data/lib/disqualified/version.rb

@@ -0,0 +1,3 @@
+module Disqualified
+  VERSION = "0.1.0"
+end

data/lib/disqualified.rb

@@ -0,0 +1,17 @@
+require "optparse"
+
+require "concurrent"
+require "rails"
+
+require_relative "disqualified/logging"
+
+require_relative "disqualified/engine"
+require_relative "disqualified/job"
+require_relative "disqualified/main"
+require_relative "disqualified/pool"
+require_relative "disqualified/server_configuration"
+require_relative "disqualified/version"
+
+module Disqualified
+  # Your code goes here...
+end

data/lib/generators/disqualified/install/USAGE

@@ -0,0 +1,8 @@
+Description:
+    Installs Disqualified
+
+Example:
+    bin/rails generate disqualified:install
+
+    This will create:
+        db/migrate/20220703062536_create_disqualified_jobs.rb

data/lib/generators/disqualified/install/install_generator.rb

@@ -0,0 +1,8 @@
+class Disqualified::InstallGenerator < Rails::Generators::Base
+  source_root File.expand_path("templates", __dir__)
+
+  def copy_migration_file
+    basename = "20220703062536_create_disqualified_jobs.rb"
+    copy_file basename, "db/migrate/#{basename}"
+  end
+end

data/lib/generators/disqualified/install/templates/20220703062536_create_disqualified_jobs.rb

@@ -0,0 +1,16 @@
+class CreateDisqualifiedJobs < ActiveRecord::Migration[5.0]
+  def change
+    create_table :disqualified_jobs do |t|
+      t.string :handler, null: false
+      t.text :arguments, null: false
+      t.string :queue, null: false
+      t.datetime :run_at, null: false
+      t.integer :attempts, null: false, default: 0
+      t.datetime :locked_at
+      t.string :locked_by
+      t.datetime :finished_at
+
+      t.timestamps
+    end
+  end
+end

data/lib/tasks/disqualified_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :disqualified do
+#   # Task goes here
+# end

metadata

@@ -0,0 +1,109 @@
+--- !ruby/object:Gem::Specification
+name: disqualified
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Zach Ahn
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2022-07-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 7.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 7.0.0
+- !ruby/object:Gem::Dependency
+  name: mocktail
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: standard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description:
+email:
+- engineering@zachahn.com
+executables:
+- disqualified
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- app/assets/config/disqualified_manifest.js
+- app/models/disqualified/base_record.rb
+- app/models/disqualified/record.rb
+- config/routes.rb
+- exe/disqualified
+- lib/disqualified.rb
+- lib/disqualified/cli.rb
+- lib/disqualified/engine.rb
+- lib/disqualified/job.rb
+- lib/disqualified/logging.rb
+- lib/disqualified/main.rb
+- lib/disqualified/pool.rb
+- lib/disqualified/server_configuration.rb
+- lib/disqualified/version.rb
+- lib/generators/disqualified/install/USAGE
+- lib/generators/disqualified/install/install_generator.rb
+- lib/generators/disqualified/install/templates/20220703062536_create_disqualified_jobs.rb
+- lib/tasks/disqualified_tasks.rake
+homepage: https://github.com/zachahn/disqualified
+licenses:
+- LGPL-3.0-only
+metadata:
+  homepage_uri: https://github.com/zachahn/disqualified
+  source_code_uri: https://github.com/zachahn/disqualified
+  changelog_uri: https://github.com/zachahn/disqualified/blob/main/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.7
+signing_key:
+specification_version: 4
+summary: A background job processor tuned for SQLite
+test_files: []