good_job 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 9604c74cf4073b178401ec0cb606e55732f087464058bebe968b666c31c242d4
+  data.tar.gz: f4af6118d55f231d15c3ffa2a3aac9f625911889a4f1adce5c14dcb2626af4d5
+SHA512:
+  metadata.gz: 46e92a0efba3937d274942ea55371f65cc3aa0451bc625bff6856be09525446960d192473548820bf1c289713151ee82e5cf1bdef21252da6a0927baafe5cc9c
+  data.tar.gz: 9e1d108190398d5d698713e12ba981d63ee121f095185990f1f0963bee36e4f4760a22c9f64f8148624ba6be931a89943e48a601c4395ad825254d5d23c86c58

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright 2020 Ben Sheldon
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,87 @@
+# GoodJob
+
+GoodJob is a multithreaded, Postgres-based ActiveJob backend for Ruby on Rails.
+
+## Usage
+
+1. Create a database migration:
+    ```bash
+    bin/rails g migration CreateGoodJobs
+    ```
+
+    And then add to the newly created file:
+
+    ```ruby
+    class CreateGoodJobs < ActiveRecord::Migration[6.0]
+      def change
+        enable_extension 'pgcrypto'
+
+        create_table :good_jobs, id: :uuid do |t|
+          t.timestamps
+
+          t.text :queue_name
+          t.integer :priority
+          t.jsonb :serialized_params
+          t.timestamp :scheduled_at
+        end
+      end
+    end
+    ```
+1. Configure the ActiveJob adapter:
+    ```ruby
+    # config/environments/production.rb
+    config.active_job.queue_adapter = GoodJob::Adapter.new
+
+    # config/environments/development.rb
+    config.active_job.queue_adapter = GoodJob::Adapter.new(inline: true)
+    ```
+
+1. In production, the scheduler is designed to run in its own process:
+    ```bash
+    $ bundle exec good_job
+    ```
+
+## Installation
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'good_job', github: 'bensheldon/good_job'
+```
+
+And then execute:
+```bash
+$ bundle
+```
+
+## Development
+
+To run tests:
+
+```bash
+# Clone the repository locally
+$ git clone git@github.com:bensheldon/good_job.git
+
+# Set up the local environment
+$ bin/setup_test
+
+# Run the tests
+$ bin/rspec
+```
+
+For developing locally within another Ruby on Rails project:
+
+```bash
+# Within Ruby on Rails directory...
+$ bundle config local.good_job /path/to/local/git/repository
+
+# Confirm that the local copy is used
+$ bundle install
+
+# => Using good_job 0.1.0 from https://github.com/bensheldon/good_job.git (at /Users/You/Projects/good_job@dc57fb0)
+```
+
+## Contributing
+Contribution directions go here.
+
+## License
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/bin/good_job

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require 'good_job/cli'
+GoodJob::CLI.start(ARGV)

data/lib/good_job/adapter.rb

@@ -0,0 +1,32 @@
+module GoodJob
+  class Adapter
+    def initialize(options = {})
+      @options = options
+      @scheduler = InlineScheduler.new if inline?
+    end
+
+    def inline?
+      @options.fetch(:inline, false)
+    end
+
+    def enqueue(job)
+      enqueue_at(job, nil)
+    end
+
+    def enqueue_at(job, timestamp)
+      params = {
+        queue_name: job.queue_name,
+        priority: job.priority,
+        serialized_params: job.serialize,
+      }
+      params[:scheduled_at] = Time.at(timestamp) if timestamp
+
+      good_job = GoodJob::Job.create(params)
+      @scheduler.enqueue(good_job) if inline?
+    end
+
+    def shutdown(wait: true)
+      @scheduler&.shutdown(wait: wait)
+    end
+  end
+end

data/lib/good_job/cli.rb

@@ -0,0 +1,19 @@
+require 'thor'
+
+module GoodJob
+  class CLI < Thor
+    RAILS_ENVIRONMENT_RB = File.expand_path("config/environment.rb")
+
+    desc :start, "Start jobs"
+    def start
+      require RAILS_ENVIRONMENT_RB
+
+      scheduler = GoodJob::Scheduler.new
+      Kernel.loop do
+        sleep 1
+      end
+    end
+
+    default_task :start
+  end
+end

data/lib/good_job/inline_scheduler.rb

@@ -0,0 +1,10 @@
+module GoodJob
+  class InlineScheduler
+    def enqueue(good_job)
+      JobWrapper.new(good_job).perform
+    end
+
+    def shutdown(wait: true)
+    end
+  end
+end

data/lib/good_job/job.rb

@@ -0,0 +1,6 @@
+module GoodJob
+  class Job < ActiveRecord::Base
+    include Lockable
+    self.table_name = 'good_jobs'
+  end
+end

data/lib/good_job/job_wrapper.rb

@@ -0,0 +1,16 @@
+module GoodJob
+  class JobWrapper
+    def initialize(good_job)
+      @good_job = good_job
+    end
+
+    def perform
+      serialized_params = @good_job.serialized_params.merge(
+        "provider_job_id" => @good_job.id
+      )
+      ActiveJob::Base.execute(serialized_params)
+
+      @good_job.destroy!
+    end
+  end
+end

data/lib/good_job/lockable.rb

@@ -0,0 +1,97 @@
+module GoodJob
+  module Lockable
+    extend ActiveSupport::Concern
+
+    RecordAlreadyAdvisoryLockedError = Class.new(StandardError)
+
+    included do
+      scope :joins_advisory_locks, (lambda do
+        joins(<<~SQL)
+          LEFT JOIN pg_locks ON pg_locks.locktype = 'advisory'
+            AND pg_locks.objsubid = 1
+            AND pg_locks.classid = ('x'||substr(md5(good_jobs.id::text), 1, 16))::bit(32)::int
+            AND pg_locks.objid = (('x'||substr(md5(good_jobs.id::text), 1, 16))::bit(64) << 32)::bit(32)::int
+        SQL
+      end)
+
+      scope :advisory_unlocked, -> { joins_advisory_locks.where(pg_locks: { locktype: nil }) }
+      scope :with_advisory_lock, (lambda do
+        where(<<~SQL)
+           pg_try_advisory_lock(('x'||substr(md5(id::text), 1, 16))::bit(64)::bigint)
+        SQL
+      end)
+
+      def self.first_advisory_locked_row(query)
+        find_by_sql(<<~SQL)
+          WITH rows AS (#{query.to_sql})
+          SELECT rows.id
+          FROM rows
+          WHERE pg_try_advisory_lock(('x'||substr(md5(id::text), 1, 16))::bit(64)::bigint)
+        SQL
+      end
+      # private_class_method :first_advisory_locked_row
+
+      # https://www.postgresql.org/docs/9.6/view-pg-locks.html
+      # Advisory locks can be acquired on keys consisting of either a single bigint value or two integer values.
+      # A bigint key is displayed with its high-order half in the classid column, its low-order half in the objid column, and objsubid equal to 1.
+      # The original bigint value can be reassembled with the expression (classid::bigint << 32) | objid::bigint.
+      # Integer keys are displayed with the first key in the classid column, the second key in the objid column, and objsubid equal to 2.
+      # The actual meaning of the keys is up to the user. Advisory locks are local to each database, so the database column is meaningful for an advisory lock.
+      def self.advisory_lock_details
+        connection.select("SELECT * FROM pg_locks WHERE locktype = 'advisory' AND objsubid = 1")
+      end
+
+      def advisory_lock
+        self.class.connection.execute(self.class.sanitize_sql_for_conditions(["SELECT 1 as one WHERE pg_try_advisory_lock(('x'||substr(md5(?), 1, 16))::bit(64)::bigint)", id])).ntuples > 0
+      end
+
+      def advisory_lock!
+        result = advisory_lock
+        result || raise(RecordAlreadyAdvisoryLockedError)
+      end
+
+      def with_advisory_lock
+        begin
+          advisory_lock!
+          yield
+        rescue StandardError => e
+          advisory_unlock unless e.is_a? RecordAlreadyAdvisoryLockedError
+          raise
+        end
+      end
+
+      def advisory_locked?
+        self.class.connection.execute(<<~SQL).ntuples > 0
+          SELECT 1 as one
+          FROM pg_locks
+          WHERE
+            locktype = 'advisory'
+            AND objsubid = 1
+            AND classid = ('x'||substr(md5('#{id}'), 1, 16))::bit(32)::int
+            AND objid = (('x'||substr(md5('#{id}'), 1, 16))::bit(64) << 32)::bit(32)::int
+        SQL
+      end
+
+      def owns_advisory_lock?
+        self.class.connection.execute(<<~SQL).ntuples > 0
+          SELECT 1 as one
+          FROM pg_locks
+          WHERE
+            locktype = 'advisory'
+            AND objsubid = 1
+            AND classid = ('x'||substr(md5('#{id}'), 1, 16))::bit(32)::int
+            AND objid = (('x'||substr(md5('#{id}'), 1, 16))::bit(64) << 32)::bit(32)::int
+            AND pid = pg_backend_pid()
+        SQL
+      end
+
+      def advisory_unlock
+        self.class.connection.execute("SELECT pg_advisory_unlock(('x'||substr(md5('#{id}'), 1, 16))::bit(64)::bigint)").first["pg_advisory_unlock"]
+      end
+
+      def advisory_unlock!
+        advisory_unlock while advisory_locked?
+      end
+    end
+  end
+end

data/lib/good_job/railtie.rb

@@ -0,0 +1,4 @@
+module GoodJob
+  class Railtie < ::Rails::Railtie
+  end
+end

data/lib/good_job/scheduler.rb

@@ -0,0 +1,89 @@
+require "concurrent/scheduled_task"
+require "concurrent/executor/thread_pool_executor"
+require "concurrent/utility/processor_counter"
+
+module GoodJob
+  class Scheduler
+    MINIMUM_EXECUTION_INTERVAL = 0.1
+
+    DEFAULT_TIMER_OPTIONS = {
+      execution_interval: 1,
+      timeout_interval: 1,
+      run_now: true
+    }.freeze
+
+    MAX_THREADS = Concurrent.processor_count
+
+    DEFAULT_POOL_OPTIONS = {
+      name:            'good_job',
+      min_threads:     0,
+      max_threads:     MAX_THREADS,
+      auto_terminate:  true,
+      idletime:        0,
+      max_queue:       0,
+      fallback_policy:  :abort # shouldn't matter -- 0 max queue
+    }.freeze
+
+    def initialize(query = GoodJob::Job.all, **options)
+      @query = query
+
+      @pool = Concurrent::ThreadPoolExecutor.new(DEFAULT_POOL_OPTIONS)
+      @timer = Concurrent::TimerTask.new(DEFAULT_TIMER_OPTIONS) do
+        idle_threads = @pool.max_length - @pool.length
+        puts "There are idle_threads: #{idle_threads}"
+        create_thread if idle_threads.positive?
+        true
+      end
+      @timer.execute
+    end
+
+    def execute
+    end
+
+    def shutdown(wait: true)
+      if @timer.running?
+        @timer.shutdown
+        @timer.wait_for_termination if wait
+      end
+
+      if @pool.running?
+        @pool.shutdown
+        @pool.wait_for_termination if wait
+      end
+    end
+
+    def create_thread
+      future = Concurrent::Future.new(args: [@query], executor: @pool) do |query|
+        Rails.application.executor.wrap do
+          thread_name = Thread.current.name || Thread.current.object_id
+          while job = query.with_advisory_lock.first
+            puts "Executing job #{job.id} in thread #{thread_name}"
+
+            JobWrapper.new(job).perform
+
+            job.advisory_unlock
+          end
+          true
+        end
+      end
+      future.add_observer(TaskObserver.new(self))
+      future.execute
+    end
+
+    class TaskObserver
+      def initialize(scheduler)
+        @scheduler = scheduler
+      end
+
+      def update(time, result, ex)
+        if result
+          puts "(#{time}) Execution successfully returned #{result}\n"
+        elsif ex.is_a?(Concurrent::TimeoutError)
+          puts "(#{time}) Execution timed out\n"
+        else
+          puts "(#{time}) Execution failed with error #{result} #{ex}\n"
+        end
+      end
+    end
+  end
+end

data/lib/good_job/version.rb

@@ -0,0 +1,3 @@
+module GoodJob
+  VERSION = '0.1.0'
+end

data/lib/good_job.rb

@@ -0,0 +1,12 @@
+require "rails"
+
+require 'good_job/lockable'
+require 'good_job/job'
+require 'good_job/inline_scheduler'
+require "good_job/scheduler"
+require "good_job/job_wrapper"
+require 'good_job/adapter'
+
+module GoodJob
+  # Your code goes here...
+end

metadata

@@ -0,0 +1,155 @@
+--- !ruby/object:Gem::Specification
+name: good_job
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Ben Sheldon
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-03-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: database_cleaner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: gem-release
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: GoodJob is a minimal postgres based job queue system for Rails
+email:
+- bensheldon@gmail.com
+executables:
+- good_job
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- bin/good_job
+- lib/good_job.rb
+- lib/good_job/adapter.rb
+- lib/good_job/cli.rb
+- lib/good_job/inline_scheduler.rb
+- lib/good_job/job.rb
+- lib/good_job/job_wrapper.rb
+- lib/good_job/lockable.rb
+- lib/good_job/railtie.rb
+- lib/good_job/scheduler.rb
+- lib/good_job/version.rb
+homepage: https://github.com/benheldon/good_job
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key: 
+specification_version: 4
+summary: GoodJob is a minimal postgres based job queue system for Rails
+test_files: []