good_job 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a808beca38372ee8dcd23871600c9ed9b0181aa7756656c46e0382e9eef927b0
-  data.tar.gz: 7fa59a1e31b4e720149ed19009f91691e840096570d38e519d37eb9b930cb655
+  metadata.gz: 3fb26ee806945cc3f179365b97150d570d9054c7907a94044f28a2afdd46d389
+  data.tar.gz: ab7d3247d78ad394e729c4b4787157ea3b60b0eeb01d0a31a2c93694a3320a62
 SHA512:
-  metadata.gz: 9ce51a00e0aeabbd2be39e796168343d0daf96b368cb75ae8011f05bec891fcab820323189c43c79b97ae5a4c2586a7ec4996cd68717a83bb66613b7db523a95
-  data.tar.gz: d0cb6bc52ba43a1e86a4ba12a8a09204d90f3b393de0d0dc54de64de37c85290f8228362e76f8c023a0d78f28d63706b4c8f2b04626134bc4240d9ad055544db
+  metadata.gz: 8065515ccdf0533d6069688a2afba7ffdb71a2ba9ad45660fda62b3c6c8347eff8e7d6a0a735bfcd974176a1e1e2e1168efd5dc74aa6fbe155326cb33f7700c3
+  data.tar.gz: 6e8877daa20867a5748b62a3e40130ee8a270c9c2db3ba1a8a460c76d33b9a48e6d8f127e779fe0808d245cc75276d36492adf7e70850640e729adbe2863e715

data/CHANGELOG.md

@@ -1,11 +1,15 @@
 # Changelog
 
-## [0.2.2](https://github.com/bensheldon/good_job/tree/0.2.2) (2020-03-07)
+## [0.3.0](https://github.com/bensheldon/good_job/tree/0.3.0) (2020-03-22)
 
-[Full Changelog](https://github.com/bensheldon/good_job/compare/v0.2.1...0.2.2)
+[Full Changelog](https://github.com/bensheldon/good_job/compare/v0.2.1...0.3.0)
 
 **Merged pull requests:**
 
+- Update development Ruby to 2.6.5 [\#22](https://github.com/bensheldon/good_job/pull/22) ([bensheldon](https://github.com/bensheldon))
+- Simplify the internal API, removing JobWrapper and InlineScheduler [\#21](https://github.com/bensheldon/good_job/pull/21) ([bensheldon](https://github.com/bensheldon))
+- Generate a new future for every executed job [\#20](https://github.com/bensheldon/good_job/pull/20) ([bensheldon](https://github.com/bensheldon))
+- Configuration for maximum number of job execution threads [\#18](https://github.com/bensheldon/good_job/pull/18) ([bensheldon](https://github.com/bensheldon))
 - Gracefully shutdown Scheduler when executable receives TERM or INT [\#17](https://github.com/bensheldon/good_job/pull/17) ([bensheldon](https://github.com/bensheldon))
 - Update Appraisals [\#16](https://github.com/bensheldon/good_job/pull/16) ([bensheldon](https://github.com/bensheldon))
 

data/README.md

@@ -67,6 +67,17 @@ $ bundle
     ```bash
     $ bundle exec good_job
     ```
+   
+### Configuring Job Execution Threads
+    
+GoodJob executes enqueued jobs using threads. There is a lot than can be said about [multithreaded behavior in Ruby on Rails](https://guides.rubyonrails.org/threading_and_code_execution.html), but briefly:
+
+- Each GoodJob execution thread requires its own database connection, which are automatically checked out from Rails’s connection pool. _Allowing GoodJob to schedule more threads than are available in the database connection pool can lead to timeouts and is not recommended._ 
+- The maximum number of GoodJob threads can be configured, in decreasing precedence:
+    1. `$ bundle exec good_job --max_threads 4`
+    2. `$ GOOD_JOB_MAX_THREADS=4 bundle exec good_job`
+    3. `$ RAILS_MAX_THREADS=4 bundle exec good_job`
+    4. Implicitly via Rails's database connection pool size (`ActiveRecord::Base.connection_pool.size`)
 
 ## Development
 

data/lib/good_job.rb

@@ -4,10 +4,9 @@ require 'good_job/railtie'
 require 'good_job/logging'
 require 'good_job/lockable'
 require 'good_job/job'
-require 'good_job/inline_scheduler'
 require "good_job/scheduler"
-require "good_job/job_wrapper"
 require 'good_job/adapter'
+require 'good_job/pg_locks'
 
 module GoodJob
   include Logging

data/lib/good_job/adapter.rb

@@ -1,41 +1,36 @@
 module GoodJob
   class Adapter
-    def initialize(options = {})
-      @options = options
-      @scheduler = InlineScheduler.new if inline?
+    def initialize(inline: false)
+      @inline = inline
     end
 
-    def enqueue(job)
-      enqueue_at(job, nil)
+    def enqueue(active_job)
+      enqueue_at(active_job, nil)
     end
 
-    def enqueue_at(job, timestamp)
-      params = {
-        queue_name: job.queue_name,
-        priority: job.priority,
-        serialized_params: job.serialize,
-      }
-      params[:scheduled_at] = Time.at(timestamp) if timestamp
+    def enqueue_at(active_job, timestamp)
+      good_job = GoodJob::Job.enqueue(
+        active_job,
+        scheduled_at: timestamp ? Time.at(timestamp) : nil,
+        create_with_advisory_lock: inline?
+      )
 
-      good_job = GoodJob::Job.create(params)
-      job.provider_job_id = good_job.id
-
-      GoodJob.tag_logger do
-        ActiveSupport::Notifications.instrument("create.good_job", { good_job: good_job, job: job })
-        @scheduler.enqueue(good_job) if inline?
+      if inline?
+        good_job.perform
+        good_job.advisory_unlock
       end
 
       good_job
     end
 
-    def shutdown(wait: true)
-      @scheduler&.shutdown(wait: wait)
+    def shutdown(wait: true) # rubocop:disable Lint/UnusedMethodArgument
+      nil
     end
 
     private
 
     def inline?
-      @options.fetch(:inline, false)
+      @inline
     end
   end
 end

data/lib/good_job/cli.rb

@@ -5,18 +5,23 @@ module GoodJob
     RAILS_ENVIRONMENT_RB = File.expand_path("config/environment.rb")
 
     desc :start, "Start jobs"
+    method_option :max_threads, type: :numeric
     def start
       require RAILS_ENVIRONMENT_RB
 
-      scheduler = GoodJob::Scheduler.new
+      max_threads = options[:max_threads] ||
+                    ENV['GOOD_JOB_MAX_THREADS'] ||
+                    ENV['RAILS_MAX_THREADS'] ||
+                    ActiveRecord::Base.connection_pool.size
+
+      $stdout.puts "GoodJob starting with max_threads=#{max_threads}"
+      scheduler = GoodJob::Scheduler.new(pool_options: { max_threads: max_threads })
 
       %w[INT TERM].each do |signal|
         trap(signal) { @stop_good_job_executable = true }
       end
       @stop_good_job_executable = false
 
-      $stdout.puts "GoodJob waiting for jobs..."
-
       Kernel.loop do
         sleep 0.1
         break if @stop_good_job_executable || scheduler.shutdown?

data/lib/good_job/job.rb

@@ -1,6 +1,39 @@
 module GoodJob
   class Job < ActiveRecord::Base
     include Lockable
+
     self.table_name = 'good_jobs'
+
+    def self.enqueue(active_job, scheduled_at: nil, create_with_advisory_lock: false)
+      good_job = nil
+      ActiveSupport::Notifications.instrument("enqueue_job.good_job", { active_job: active_job, scheduled_at: scheduled_at, create_with_advisory_lock: create_with_advisory_lock }) do |instrument_payload|
+        good_job = GoodJob::Job.new(
+          queue_name: active_job.queue_name,
+          priority: active_job.priority,
+          serialized_params: active_job.serialize,
+          scheduled_at: scheduled_at,
+          create_with_advisory_lock: create_with_advisory_lock
+        )
+
+        instrument_payload[:good_job] = good_job
+
+        good_job.save!
+        active_job.provider_job_id = good_job.id
+      end
+
+      good_job
+    end
+
+    def perform
+      ActiveSupport::Notifications.instrument("before_perform_job.good_job", { good_job: self })
+      ActiveSupport::Notifications.instrument("perform_job.good_job", { good_job: self }) do
+        params = serialized_params.merge(
+          "provider_job_id" => id
+        )
+        ActiveJob::Base.execute(params)
+
+        destroy!
+      end
+    end
   end
 end

data/lib/good_job/lockable.rb

@@ -15,88 +15,79 @@ module GoodJob
       end)
 
       scope :advisory_unlocked, -> { joins_advisory_locks.where(pg_locks: { locktype: nil }) }
-      scope :with_advisory_lock, (lambda do
-        where(<<~SQL)
-          pg_try_advisory_lock(('x'||substr(md5(id::text), 1, 16))::bit(64)::bigint)
-        SQL
-      end)
 
-      def self.first_advisory_locked_row(query)
-        find_by_sql(<<~SQL)
+      attr_accessor :create_with_advisory_lock
+
+      after_create -> { advisory_lock }, if: :create_with_advisory_lock
+    end
+
+    class_methods do
+      def first_advisory_locked_row(query)
+        find_by_sql(<<~SQL).first
           WITH rows AS (#{query.to_sql})
-          SELECT rows.id
+          SELECT rows.*
           FROM rows
           WHERE pg_try_advisory_lock(('x'||substr(md5(id::text), 1, 16))::bit(64)::bigint)
+          LIMIT 1
         SQL
       end
-      # private_class_method :first_advisory_locked_row
-
-      # https://www.postgresql.org/docs/9.6/view-pg-locks.html
-      # Advisory locks can be acquired on keys consisting of either a single bigint value or two integer values.
-      # A bigint key is displayed with its high-order half in the classid column, its low-order half in the objid column, and objsubid equal to 1.
-      # The original bigint value can be reassembled with the expression (classid::bigint << 32) | objid::bigint.
-      # Integer keys are displayed with the first key in the classid column, the second key in the objid column, and objsubid equal to 2.
-      # The actual meaning of the keys is up to the user. Advisory locks are local to each database, so the database column is meaningful for an advisory lock.
-      def self.advisory_lock_details
-        connection.select("SELECT * FROM pg_locks WHERE locktype = 'advisory' AND objsubid = 1")
-      end
+    end
 
-      def advisory_lock
-        self.class.connection.execute(sanitize_sql_for_conditions(["SELECT 1 as one WHERE pg_try_advisory_lock(('x'||substr(md5(?), 1, 16))::bit(64)::bigint)", id])).ntuples.positive?
-      end
+    def advisory_lock
+      self.class.connection.execute(sanitize_sql_for_conditions(["SELECT 1 as one WHERE pg_try_advisory_lock(('x'||substr(md5(?), 1, 16))::bit(64)::bigint)", id])).ntuples.positive?
+    end
 
-      def advisory_lock!
-        result = advisory_lock
-        result || raise(RecordAlreadyAdvisoryLockedError)
-      end
+    def advisory_lock!
+      result = advisory_lock
+      result || raise(RecordAlreadyAdvisoryLockedError)
+    end
 
-      def with_advisory_lock
-        advisory_lock!
-        yield
-      rescue StandardError => e
-        advisory_unlock unless e.is_a? RecordAlreadyAdvisoryLockedError
-        raise
-      end
+    def with_advisory_lock
+      advisory_lock!
+      yield
+    rescue StandardError => e
+      advisory_unlock unless e.is_a? RecordAlreadyAdvisoryLockedError
+      raise
+    end
 
-      def advisory_locked?
-        self.class.connection.execute(<<~SQL).ntuples.positive?
-          SELECT 1 as one
-          FROM pg_locks
-          WHERE
-            locktype = 'advisory'
-            AND objsubid = 1
-            AND classid = ('x'||substr(md5('#{id}'), 1, 16))::bit(32)::int
-            AND objid = (('x'||substr(md5('#{id}'), 1, 16))::bit(64) << 32)::bit(32)::int
-        SQL
-      end
+    def advisory_locked?
+      self.class.connection.execute(<<~SQL).ntuples.positive?
+        SELECT 1 as one
+        FROM pg_locks
+        WHERE
+          locktype = 'advisory'
+          AND objsubid = 1
+          AND classid = ('x'||substr(md5('#{id}'), 1, 16))::bit(32)::int
+          AND objid = (('x'||substr(md5('#{id}'), 1, 16))::bit(64) << 32)::bit(32)::int
+      SQL
+    end
 
-      def owns_advisory_lock?
-        self.class.connection.execute(<<~SQL).ntuples.positive?
-          SELECT 1 as one
-          FROM pg_locks
-          WHERE
-            locktype = 'advisory'
-            AND objsubid = 1
-            AND classid = ('x'||substr(md5('#{id}'), 1, 16))::bit(32)::int
-            AND objid = (('x'||substr(md5('#{id}'), 1, 16))::bit(64) << 32)::bit(32)::int
-            AND pid = pg_backend_pid()
-        SQL
-      end
+    def owns_advisory_lock?
+      self.class.connection.execute(<<~SQL).ntuples.positive?
+        SELECT 1 as one
+        FROM pg_locks
+        WHERE
+          locktype = 'advisory'
+          AND objsubid = 1
+          AND classid = ('x'||substr(md5('#{id}'), 1, 16))::bit(32)::int
+          AND objid = (('x'||substr(md5('#{id}'), 1, 16))::bit(64) << 32)::bit(32)::int
+          AND pid = pg_backend_pid()
+      SQL
+    end
 
-      def advisory_unlock
-        self.class.connection.execute("SELECT pg_advisory_unlock(('x'||substr(md5('#{id}'), 1, 16))::bit(64)::bigint)").first["pg_advisory_unlock"]
-      end
+    def advisory_unlock
+      self.class.connection.execute("SELECT pg_advisory_unlock(('x'||substr(md5('#{id}'), 1, 16))::bit(64)::bigint)").first["pg_advisory_unlock"]
+    end
 
-      def advisory_unlock!
-        advisory_unlock while advisory_locked?
-      end
+    def advisory_unlock!
+      advisory_unlock while advisory_locked?
+    end
 
-      private
+    private
 
-      def sanitize_sql_for_conditions(*args)
-        # Made public in Rails 5.2
-        self.class.send(:sanitize_sql_for_conditions, *args)
-      end
+    def sanitize_sql_for_conditions(*args)
+      # Made public in Rails 5.2
+      self.class.send(:sanitize_sql_for_conditions, *args)
     end
   end
 end

data/lib/good_job/pg_locks.rb

@@ -0,0 +1,21 @@
+module GoodJob
+  class PgLocks < ActiveRecord::Base
+    self.table_name = 'pg_locks'.freeze
+
+    # https://www.postgresql.org/docs/9.6/view-pg-locks.html
+    # Advisory locks can be acquired on keys consisting of either a single bigint value or two integer values.
+    # A bigint key is displayed with its high-order half in the classid column, its low-order half in the objid column, and objsubid equal to 1.
+    # The original bigint value can be reassembled with the expression (classid::bigint << 32) | objid::bigint.
+    # Integer keys are displayed with the first key in the classid column, the second key in the objid column, and objsubid equal to 2.
+    # The actual meaning of the keys is up to the user. Advisory locks are local to each database, so the database column is meaningful for an advisory lock.
+    def self.advisory_lock_details
+      connection.select <<~SQL
+        SELECT *
+        FROM pg_locks
+        WHERE
+          locktype = 'advisory' AND
+          objsubid = 1
+      SQL
+    end
+  end
+end

data/lib/good_job/scheduler.rb

@@ -1,11 +1,9 @@
-require "concurrent/scheduled_task"
 require "concurrent/executor/thread_pool_executor"
+require "concurrent/timer_task"
 require "concurrent/utility/processor_counter"
 
 module GoodJob
   class Scheduler
-    MAX_THREADS = Concurrent.processor_count
-
     DEFAULT_TIMER_OPTIONS = {
       execution_interval: 1,
       timeout_interval: 1,
@@ -15,22 +13,22 @@ module GoodJob
     DEFAULT_POOL_OPTIONS = {
       name: 'good_job',
       min_threads: 0,
-      max_threads: MAX_THREADS,
+      max_threads: Concurrent.processor_count,
       auto_terminate: true,
       idletime: 0,
       max_queue: 0,
       fallback_policy: :abort, # shouldn't matter -- 0 max queue
     }.freeze
 
-    def initialize(query = GoodJob::Job.all, **_options)
+    def initialize(query = GoodJob::Job.all, timer_options: {}, pool_options: {})
       @query = query
 
-      @pool = Concurrent::ThreadPoolExecutor.new(DEFAULT_POOL_OPTIONS)
-      @timer = Concurrent::TimerTask.new(DEFAULT_TIMER_OPTIONS) do
+      @pool = Concurrent::ThreadPoolExecutor.new(DEFAULT_POOL_OPTIONS.merge(pool_options))
+      @timer = Concurrent::TimerTask.new(DEFAULT_TIMER_OPTIONS.merge(timer_options)) do
         idle_threads = @pool.max_length - @pool.length
         create_thread if idle_threads.positive?
       end
-      @timer.add_observer(TimerObserver.new)
+      @timer.add_observer(self, :timer_observer)
       @timer.execute
     end
 
@@ -64,35 +62,30 @@ module GoodJob
 
     def create_thread
       future = Concurrent::Future.new(args: [ordered_query], executor: @pool) do |query|
-        Rails.application.executor.wrap do
-          loop do
-            good_job = query.with_advisory_lock.first
-            break unless good_job
-
-            ActiveSupport::Notifications.instrument("job_started.good_job", { good_job: good_job })
+        executed_job = false
 
-            JobWrapper.new(good_job).perform
+        Rails.application.executor.wrap do
+          good_job = GoodJob::Job.first_advisory_locked_row(query)
+          break unless good_job
 
-            good_job.advisory_unlock
-          end
+          executed_job = true
+          good_job.perform
+          good_job.advisory_unlock
         end
 
-        true
+        executed_job
       end
-      future.add_observer(TaskObserver.new)
+      future.add_observer(self, :task_observer)
       future.execute
     end
 
-    class TimerObserver
-      def update(time, result, error)
-        ActiveSupport::Notifications.instrument("timer_task_finished.good_job", { result: result, error: error, time: time })
-      end
+    def timer_observer(time, executed_task, error)
+      ActiveSupport::Notifications.instrument("finished_timer_task.good_job", { result: executed_task, error: error, time: time })
     end
 
-    class TaskObserver
-      def update(time, result, error)
-        ActiveSupport::Notifications.instrument("job_finished.good_job", { result: result, error: error, time: time })
-      end
+    def task_observer(time, executed_task, error)
+      ActiveSupport::Notifications.instrument("finished_job_task.good_job", { result: executed_task, error: error, time: time })
+      create_thread if executed_task
     end
   end
 end

data/lib/good_job/version.rb

@@ -1,3 +1,3 @@
 module GoodJob
-  VERSION = '0.2.2'.freeze
+  VERSION = '0.3.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: good_job
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - Ben Sheldon
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-08 00:00:00.000000000 Z
+date: 2020-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -122,6 +122,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec-rails
   requirement: !ruby/object:Gem::Requirement
@@ -182,11 +196,10 @@ files:
 - lib/good_job.rb
 - lib/good_job/adapter.rb
 - lib/good_job/cli.rb
-- lib/good_job/inline_scheduler.rb
 - lib/good_job/job.rb
-- lib/good_job/job_wrapper.rb
 - lib/good_job/lockable.rb
 - lib/good_job/logging.rb
+- lib/good_job/pg_locks.rb
 - lib/good_job/railtie.rb
 - lib/good_job/scheduler.rb
 - lib/good_job/version.rb
@@ -221,7 +234,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: A multithreaded, Postgres-based ActiveJob backend for Ruby on Rails

data/lib/good_job/inline_scheduler.rb

@@ -1,10 +0,0 @@
-module GoodJob
-  class InlineScheduler
-    def enqueue(good_job)
-      JobWrapper.new(good_job).perform
-    end
-
-    def shutdown(wait: true)
-    end
-  end
-end

data/lib/good_job/job_wrapper.rb

@@ -1,16 +0,0 @@
-module GoodJob
-  class JobWrapper
-    def initialize(good_job)
-      @good_job = good_job
-    end
-
-    def perform
-      serialized_params = @good_job.serialized_params.merge(
-        "provider_job_id" => @good_job.id
-      )
-      ActiveJob::Base.execute(serialized_params)
-
-      @good_job.destroy!
-    end
-  end
-end