good_job 1.10.1 → 1.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f2621d47f3d0de0bb0411e1b6984e5e684ec2b54a218a29e3c01219995c46f53
-  data.tar.gz: 7772377aeb51daed1530d7f01fcfe92d06ba7ca992ff010846e595450a8bd6b2
+  metadata.gz: 6c0b45fcd80fbd536af0f6fed44ef53f446ee6574f7e90d41a38585ec8b3ba63
+  data.tar.gz: 8b45f77d94bd809b25fb725f57622f1966f188157f235e03c56442817e31afd4
 SHA512:
-  metadata.gz: 1f6640c27c6f3d7e2b2c20952b655e70d1079fa59cf5b130c403cf613b200902c8e5b57d36647747165cec2ec944e183f0720974c8a73bfaab3c64968003ce91
-  data.tar.gz: b514789fd4df9589f012f2c2156bfaef0fd05d2106a10345e23105e6260a160816d7a8b4ba5d97eb9095ba917ecaf3d0630e9c121cb0a23a0563dbc641cd8f45
+  metadata.gz: 820a1847c2dfce6456d925fe73e8373d5b2f19ff9bc2dd143dbf942acfb0c3872b4fdd1dc888741ae7f3974c7f6869ef82ef8c54bd238ceea03bb63ff8829593
+  data.tar.gz: 4faa2442654f37475c22ce7c3e3b440351a5f91a53ac2ada273cba764c409e7f7f108c26b463a736b9880d4097ef9096775905c092bf5860805e3cf7bffede86

data/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## [v1.11.0](https://github.com/bensheldon/good_job/tree/v1.11.0) (2021-07-07)
+
+[Full Changelog](https://github.com/bensheldon/good_job/compare/v1.10.1...v1.11.0)
+
+**Implemented enhancements:**
+
+- Add concurrency extension for ActiveJob [\#281](https://github.com/bensheldon/good_job/pull/281) ([bensheldon](https://github.com/bensheldon))
+
+**Closed issues:**
+
+- Investigate GoodJob concurrency [\#289](https://github.com/bensheldon/good_job/issues/289)
+- Problem with migrating database on 1.10.0 [\#287](https://github.com/bensheldon/good_job/issues/287)
+- Support migration --database option for install task? [\#267](https://github.com/bensheldon/good_job/issues/267)
+- Add GoodJob to Ruby Toolbox [\#243](https://github.com/bensheldon/good_job/issues/243)
+- Custom advisory locks to prevent certain jobs from being worked on concurrently? [\#206](https://github.com/bensheldon/good_job/issues/206)
+
 ## [v1.10.1](https://github.com/bensheldon/good_job/tree/v1.10.1) (2021-06-30)
 
 [Full Changelog](https://github.com/bensheldon/good_job/compare/v1.10.0...v1.10.1)

data/README.md

@@ -38,6 +38,7 @@ For more of the story of GoodJob, read the [introductory blog post](https://isla
     - [Configuration options](#configuration-options)
     - [Global options](#global-options)
     - [Dashboard](#dashboard)
+    - [ActiveJob Concurrency](#activejob-concurrency)
     - [Updating](#updating)
 - [Go deeper](#go-deeper)
     - [Exceptions, retries, and reliability](#exceptions-retries-and-reliability)
@@ -319,6 +320,35 @@ GoodJob includes a Dashboard as a mountable `Rails::Engine`.
     end
     ```
 
+### ActiveJob Concurrency
+
+GoodJob can extend ActiveJob to provide limits on concurrently running jobs, either at time of _enqueue_ or at _perform_.
+
+**Note:** Limiting concurrency at _enqueue_ requires Rails 6.0+ because Rails 5.2 does not support `throw :abort` in ActiveJob callbacks.
+
+```ruby
+class MyJob < ApplicationJob
+  include GoodJob::ActiveJobExtensions::Concurrency
+
+  good_job_control_concurrency_with(
+    # Maximum number of jobs with the concurrency key to be concurrently enqueued
+    enqueue_limit: 2,
+
+    # Maximum number of jobs with the concurrency key to be concurrently performed
+    perform_limit: 1,
+
+    # A unique key to be globally locked against.
+    # Can be String or Lambda/Proc that is invoked in the context of the job.
+    # Note: Arguments passed to #perform_later must be accessed through `arguments` method.
+    key: -> { "Unique-#{arguments.first}" } #  MyJob.perform_later("Alice") => "Unique-Alice"
+  )
+
+  def perform(first_name)
+    # do work
+  end
+end
+```
+
 ### Updating
 
 GoodJob follows semantic versioning, though updates may be encouraged through deprecation warnings in minor versions.

data/lib/good_job/active_job_extensions.rb

@@ -0,0 +1,4 @@
+module GoodJob
+  module ActiveJobExtensions
+  end
+end

data/lib/good_job/active_job_extensions/concurrency.rb

@@ -0,0 +1,68 @@
+module GoodJob
+  module ActiveJobExtensions
+    module Concurrency
+      extend ActiveSupport::Concern
+
+      ConcurrencyExceededError = Class.new(StandardError)
+
+      included do
+        class_attribute :good_job_concurrency_config, instance_accessor: false, default: {}
+
+        before_enqueue do |job|
+          # Always allow jobs to be retried because the current job's execution will complete momentarily
+          next if CurrentExecution.active_job_id == job.job_id
+
+          limit = job.class.good_job_concurrency_config.fetch(:enqueue_limit, Float::INFINITY)
+          next if limit.blank? || (0...Float::INFINITY).exclude?(limit)
+
+          key = job.good_job_concurrency_key
+          next if key.blank?
+
+          GoodJob::Job.new.with_advisory_lock(key: key, function: "pg_advisory_lock") do
+            # TODO: Why is `unscoped` necessary? Nested scope is bleeding into subsequent query?
+            enqueue_concurrency = GoodJob::Job.unscoped.where(concurrency_key: key).unfinished.count
+            # The job has not yet been enqueued, so check if adding it will go over the limit
+            throw :abort if enqueue_concurrency + 1 > limit
+          end
+        end
+
+        retry_on(
+          GoodJob::ActiveJobExtensions::Concurrency::ConcurrencyExceededError,
+          attempts: Float::INFINITY,
+          wait: :exponentially_longer
+        )
+
+        before_perform do |job|
+          limit = job.class.good_job_concurrency_config.fetch(:perform_limit, Float::INFINITY)
+          next if limit.blank? || (0...Float::INFINITY).exclude?(limit)
+
+          key = job.good_job_concurrency_key
+          next if key.blank?
+
+          GoodJob::Job.new.with_advisory_lock(key: key, function: "pg_advisory_lock") do
+            perform_concurrency = GoodJob::Job.unscoped.where(concurrency_key: key).advisory_locked.count
+            # The current job has already been locked and will appear in the previous query
+            raise GoodJob::ActiveJobExtensions::Concurrency::ConcurrencyExceededError if perform_concurrency > limit
+          end
+        end
+      end
+
+      class_methods do
+        def good_job_control_concurrency_with(config)
+          self.good_job_concurrency_config = config
+        end
+      end
+
+      def good_job_concurrency_key
+        key = self.class.good_job_concurrency_config[:key]
+        return if key.blank?
+
+        if key.respond_to? :call
+          instance_exec(&key)
+        else
+          key
+        end
+      end
+    end
+  end
+end

data/lib/good_job/current_execution.rb

@@ -4,11 +4,11 @@ module GoodJob
   # Thread-local attributes for passing values from Instrumentation.
   # (Cannot use ActiveSupport::CurrentAttributes because ActiveJob resets it)
   module CurrentExecution
-    # @!attribute [rw] error_on_retry
+    # @!attribute [rw] active_job_id
     #   @!scope class
-    #   Error captured by retry_on
-    #   @return [Exception, nil]
-    thread_mattr_accessor :error_on_retry
+    #   ActiveJob ID
+    #   @return [String, nil]
+    thread_mattr_accessor :active_job_id
 
     # @!attribute [rw] error_on_discard
     #   @!scope class
@@ -16,11 +16,18 @@ module GoodJob
     #   @return [Exception, nil]
     thread_mattr_accessor :error_on_discard
 
+    # @!attribute [rw] error_on_retry
+    #   @!scope class
+    #   Error captured by retry_on
+    #   @return [Exception, nil]
+    thread_mattr_accessor :error_on_retry
+
     # Resets attributes
     # @return [void]
     def self.reset
-      self.error_on_retry = nil
+      self.active_job_id = nil
       self.error_on_discard = nil
+      self.error_on_retry = nil
     end
 
     # @return [Integer] Current process ID

data/lib/good_job/job.rb

@@ -219,6 +219,21 @@ module GoodJob
           DEPRECATION
         end
 
+        if column_names.include?('concurrency_key')
+          good_job_args[:concurrency_key] = active_job.good_job_concurrency_key if active_job.respond_to?(:good_job_concurrency_key)
+        else
+          ActiveSupport::Deprecation.warn(<<~DEPRECATION)
+            GoodJob has pending database migrations. To create the migration files, run:
+
+                rails generate good_job:update
+
+            To apply the migration files, run:
+
+                rails db:migrate
+
+          DEPRECATION
+        end
+
         good_job = GoodJob::Job.new(**good_job_args)
 
         instrument_payload[:good_job] = good_job
@@ -264,6 +279,10 @@ module GoodJob
       self.class.unscoped.unfinished.owns_advisory_locked.exists?(id: id)
     end
 
+    def active_job_id
+      super || serialized_params['job_id']
+    end
+
     private
 
     # @return [ExecutionResult]
@@ -273,6 +292,7 @@ module GoodJob
       )
 
       GoodJob::CurrentExecution.reset
+      GoodJob::CurrentExecution.active_job_id = active_job_id
       ActiveSupport::Notifications.instrument("perform_job.good_job", { good_job: self, process_id: GoodJob::CurrentExecution.process_id, thread_name: GoodJob::CurrentExecution.thread_name }) do
         value = ActiveJob::Base.execute(params)
 

data/lib/good_job/lockable.rb

@@ -201,9 +201,16 @@ module GoodJob
     # @param function [String, Symbol] Postgres Advisory Lock function name to use
     # @return [Boolean] whether the lock was acquired.
     def advisory_lock(key: lockable_key, function: advisory_lockable_function)
-      query = <<~SQL.squish
-        SELECT #{function}(('x'||substr(md5($1::text), 1, 16))::bit(64)::bigint) AS locked
-      SQL
+      query = if function.include? "_try_"
+                <<~SQL.squish
+                  SELECT #{function}(('x'||substr(md5($1::text), 1, 16))::bit(64)::bigint) AS locked
+                SQL
+              else
+                <<~SQL.squish
+                  SELECT #{function}(('x'||substr(md5($1::text), 1, 16))::bit(64)::bigint)::text AS locked
+                SQL
+              end
+
       binds = [[nil, key]]
       self.class.connection.exec_query(pg_or_jdbc_query(query), 'GoodJob::Lockable Advisory Lock', binds).first['locked']
     end

data/lib/good_job/version.rb

@@ -1,4 +1,4 @@
 module GoodJob
   # GoodJob gem version.
-  VERSION = '1.10.1'.freeze
+  VERSION = '1.11.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: good_job
 version: !ruby/object:Gem::Version
-  version: 1.10.1
+  version: 1.11.0
 platform: ruby
 authors:
 - Ben Sheldon
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-06-30 00:00:00.000000000 Z
+date: 2021-07-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activejob
@@ -362,6 +362,8 @@ files:
 - lib/generators/good_job/templates/update/migrations/03_add_active_job_id_index_and_concurrency_key_index_to_good_jobs.rb
 - lib/generators/good_job/update_generator.rb
 - lib/good_job.rb
+- lib/good_job/active_job_extensions.rb
+- lib/good_job/active_job_extensions/concurrency.rb
 - lib/good_job/adapter.rb
 - lib/good_job/cli.rb
 - lib/good_job/configuration.rb