job-iteration 1.5.0 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cdace86b05a5a1d98777e2310d85d75679073ca63b5e52b0e83bd000891198cf
-  data.tar.gz: 4ed5b475931e24bf56d2b8b126bd9354064c438d0ea16f67c10276fb92ccd97b
+  metadata.gz: 55bbe81f1fe209219d6dbb5aa469f027834d6b09dbdd6485b305fb4e3aa9bcb1
+  data.tar.gz: cf611985f1b8b6dc2cb1d00026fb53d6716ff353d0e59a96235f0e8538e1878b
 SHA512:
-  metadata.gz: cf9d7a54a8881146a4a0a1750e99dddf8d32c9db3339411fc61c8477a89f5766e9bb99f493257e4484d56e18ef443cb6a2a0300958342774d2e7ebd2b3e23c32
-  data.tar.gz: 966c39a5e89ae26343e07000111ba582724b85b4bf9b7a17f790a10b679e297dca94105d3fb42830f987498fb8982037c80cf3e8130b8bede845f9c7532ade41
+  metadata.gz: e8857b87a8a999ec88b584896ff7c384310badbf7c2e7c553b9b64fed108cb7756fdc365a0008766d9179c40279ed221c96eca0b363fd5283f6c7d3d676ab82f
+  data.tar.gz: 89e9e6d584fef095c6c04b96a369abbf710283dffe9856091eef3fc7492222bd7591f22a8ebaaa7ea76f5d3281cd2c8163ba57d37ead43b095fb08fb377aea3a

data/.ruby-version

@@ -1 +1 @@
-3.3.0
+3.3.3

data/CHANGELOG.md

@@ -1,6 +1,17 @@
 ### Main (unreleased)
+
 Nil
 
+## v1.6.0 (Sep 24, 2024)
+
+### Features
+
+- [464](https://github.com/Shopify/job-iteration/pull/464) - Add interruption adapter for [GoodJob](https://github.com/bensheldon/good_job).
+- [505](https://github.com/Shopify/job-iteration/pull/505) - Add interruption adapter for [Solid Queue](https://github.com/rails/solid_queue).
+
+## v1.5.1 (May 29,2024)
+- [483](https://github.com/Shopify/job-iteration/pull/483) - Reverts [#456 Use Arel instead of String for AR Enumerator conditionals](https://github.com/Shopify/job-iteration/pull/456)
+
 ## v1.5.0 (May 29, 2024)
 ### Changes
 

data/Gemfile.lock

@@ -1,73 +1,92 @@
 GIT
   remote: https://github.com/brianmario/mysql2
-  revision: 43ea8af635f5e23f054294ef7759320d47f30e5f
+  revision: f6a9b68b42a51d1a370403f11eb88527dcb42dc6
   specs:
     mysql2 (0.5.6)
+      bigdecimal
 
 PATH
   remote: .
   specs:
-    job-iteration (1.5.0)
+    job-iteration (1.6.0)
       activejob (>= 5.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (7.0.7)
-      activesupport (= 7.0.7)
+    activejob (7.1.3.4)
+      activesupport (= 7.1.3.4)
       globalid (>= 0.3.6)
-    activemodel (7.0.7)
-      activesupport (= 7.0.7)
-    activerecord (7.0.7)
-      activemodel (= 7.0.7)
-      activesupport (= 7.0.7)
-    activesupport (7.0.7)
+    activemodel (7.1.3.4)
+      activesupport (= 7.1.3.4)
+    activerecord (7.1.3.4)
+      activemodel (= 7.1.3.4)
+      activesupport (= 7.1.3.4)
+      timeout (>= 0.4.0)
+    activesupport (7.1.3.4)
+      base64
+      bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
+      connection_pool (>= 2.2.5)
+      drb
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
+      mutex_m
       tzinfo (~> 2.0)
     ast (2.4.2)
+    base64 (0.2.0)
+    bigdecimal (3.1.8)
     coderay (1.1.3)
-    concurrent-ruby (1.2.3)
+    concurrent-ruby (1.3.4)
     connection_pool (2.4.1)
     csv (3.3.0)
-    globalid (1.1.0)
-      activesupport (>= 5.0)
-    i18n (1.14.4)
+    drb (2.2.1)
+    globalid (1.2.1)
+      activesupport (>= 6.1)
+    i18n (1.14.6)
       concurrent-ruby (~> 1.0)
-    json (2.7.1)
+    json (2.7.2)
     language_server-protocol (3.17.0.3)
-    method_source (1.0.0)
-    minitest (5.19.0)
-    mocha (2.2.0)
+    method_source (1.1.0)
+    minitest (5.24.0)
+    mocha (2.4.5)
       ruby2_keywords (>= 0.0.5)
     mono_logger (1.1.2)
     multi_json (1.15.0)
-    parallel (1.24.0)
-    parser (3.3.0.5)
+    mustermann (3.0.0)
+      ruby2_keywords (~> 0.0.1)
+    mutex_m (0.2.0)
+    parallel (1.25.1)
+    parser (3.3.3.0)
       ast (~> 2.4.1)
       racc
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    racc (1.7.3)
-    rack (3.0.9.1)
+    racc (1.8.0)
+    rack (3.1.5)
+    rack-protection (4.0.0)
+      base64 (>= 0.1.0)
+      rack (>= 3.0.0, < 4)
+    rack-session (2.0.0)
+      rack (>= 3.0.0)
     rainbow (3.1.1)
     rake (13.2.1)
-    redis (5.2.0)
+    redis (5.3.0)
       redis-client (>= 0.22.0)
-    redis-client (0.22.1)
+    redis-client (0.22.2)
       connection_pool
     redis-namespace (1.11.0)
       redis (>= 4)
-    regexp_parser (2.9.0)
+    regexp_parser (2.9.2)
     resque (2.6.0)
       mono_logger (~> 1.0)
       multi_json (~> 1.0)
       redis-namespace (~> 1.6)
       sinatra (>= 0.9.2)
-    rexml (3.2.6)
-    rubocop (1.62.1)
+    rexml (3.3.6)
+      strscan
+    rubocop (1.64.1)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
@@ -78,24 +97,31 @@ GEM
       rubocop-ast (>= 1.31.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.31.2)
-      parser (>= 3.3.0.4)
-    rubocop-shopify (2.14.0)
+    rubocop-ast (1.31.3)
+      parser (>= 3.3.1.0)
+    rubocop-shopify (2.15.1)
       rubocop (~> 1.51)
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
-    sidekiq (7.1.2)
+    sidekiq (7.2.4)
       concurrent-ruby (< 2)
       connection_pool (>= 2.3.0)
       rack (>= 2.2.4)
-      redis-client (>= 0.14.0)
-    sinatra (1.0)
-      rack (>= 1.0)
-    sorbet-runtime (0.5.10978)
+      redis-client (>= 0.19.0)
+    sinatra (4.0.0)
+      mustermann (~> 3.0)
+      rack (>= 3.0.0, < 4)
+      rack-protection (= 4.0.0)
+      rack-session (>= 2.0.0, < 3)
+      tilt (~> 2.0)
+    sorbet-runtime (0.5.11460)
+    strscan (3.1.0)
+    tilt (2.4.0)
+    timeout (0.4.1)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.5.0)
-    yard (0.9.36)
+    yard (0.9.37)
 
 PLATFORMS
   ruby
@@ -118,4 +144,4 @@ DEPENDENCIES
   yard
 
 BUNDLED WITH
-   2.5.7
+   2.5.14

data/guides/iteration-how-it-works.md

@@ -29,6 +29,10 @@ In other words, if you are trying to process 100 records but the job consistentl
 
 If no retries are configured or retries are exhausted, Active Job 'bubbles up' the exception to the job backend. Retries by the backend (e.g. Sidekiq) are not supported, meaning that jobs retried by the job backend instead of Active Job will restart from the beginning.
 
+## Stopping a job
+
+Because jobs typically retry when exceptions are thrown, there is a special mechanism to fully stop a job that still has iterations remaining. To do this, you can `throw(:abort)`. This is then caught by job-iteration and signals that the job should complete now, regardless of its iteration state.
+
 ## Signals
 
 It's critical to know [UNIX signals](https://www.tutorialspoint.com/unix/unix-signals-traps.htm) in order to understand how interruption works. There are two main signals that Sidekiq and Resque use: `SIGTERM` and `SIGKILL`. `SIGTERM` is the graceful termination signal which means that the process should exit _soon_, not immediately. For Iteration, it means that we have time to wait for the last iteration to finish and to push job back to the queue with the last cursor position.

data/lib/job-iteration/active_record_cursor.rb

@@ -18,8 +18,12 @@ module JobIteration
       end
     end
 
-    def initialize(relation, columns, position = nil)
-      @columns = columns
+    def initialize(relation, columns = nil, position = nil)
+      @columns = if columns
+        Array(columns)
+      else
+        Array(relation.primary_key).map { |pk| "#{relation.table_name}.#{pk}" }
+      end
       self.position = Array.wrap(position)
       raise ArgumentError, "Must specify at least one column" if columns.empty?
       if relation.joins_values.present? && !@columns.all? { |column| column.to_s.include?(".") }
@@ -30,7 +34,7 @@ module JobIteration
         raise ConditionNotSupportedError
       end
 
-      @base_relation = relation.reorder(*@columns)
+      @base_relation = relation.reorder(@columns.join(","))
       @reached_end = false
     end
 
@@ -50,10 +54,12 @@ module JobIteration
 
     def update_from_record(record)
       self.position = @columns.map do |column|
-        if ActiveRecord.version >= Gem::Version.new("7.1.0.alpha") && column.name == "id"
+        method = column.to_s.split(".").last
+
+        if ActiveRecord.version >= Gem::Version.new("7.1.0.alpha") && method == "id"
           record.id_value
         else
-          record.send(column.name)
+          record.send(method.to_sym)
         end
       end
     end
@@ -83,14 +89,14 @@ module JobIteration
       i = @position.size - 1
       column = @columns[i]
       conditions = if @columns.size == @position.size
-        column.gt(@position[i])
+        "#{column} > ?"
       else
-        column.gteq(@position[i])
+        "#{column} >= ?"
       end
       while i > 0
         i -= 1
         column = @columns[i]
-        conditions = column.gt(@position[i]).or(column.eq(@position[i]).and(conditions))
+        conditions = "#{column} > ? OR (#{column} = ? AND (#{conditions}))"
       end
       ret = @position.reduce([conditions]) { |params, value| params << value << value }
       ret.pop

data/lib/job-iteration/active_record_enumerator.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative "./active_record_cursor"
+require_relative "active_record_cursor"
 module JobIteration
   # Builds Enumerator based on ActiveRecord Relation. Supports enumerating on rows and batches.
   # @see EnumeratorBuilder
@@ -11,9 +11,9 @@ module JobIteration
       @relation = relation
       @batch_size = batch_size
       @columns = if columns
-        Array(columns).map { |col| relation.arel_table[col.to_sym] }
+        Array(columns)
       else
-        Array(relation.primary_key).map { |pk| relation.arel_table[pk.to_sym] }
+        Array(relation.primary_key).map { |pk| "#{relation.table_name}.#{pk}" }
       end
       @cursor = cursor
     end
@@ -45,7 +45,7 @@ module JobIteration
 
     def cursor_value(record)
       positions = @columns.map do |column|
-        attribute_name = column.name.to_sym
+        attribute_name = column.to_s.split(".").last
         column_value(record, attribute_name)
       end
       return positions.first if positions.size == 1
@@ -58,8 +58,8 @@ module JobIteration
     end
 
     def column_value(record, attribute)
-      value = record.read_attribute(attribute)
-      case record.class.columns_hash.fetch(attribute.to_s).type
+      value = record.read_attribute(attribute.to_sym)
+      case record.class.columns_hash.fetch(attribute).type
       when :datetime
         value.strftime(SQL_DATETIME_WITH_NSEC)
       else

data/lib/job-iteration/enumerator_builder.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
-require_relative "./active_record_batch_enumerator"
-require_relative "./active_record_enumerator"
-require_relative "./csv_enumerator"
-require_relative "./throttle_enumerator"
-require_relative "./nested_enumerator"
+require_relative "active_record_batch_enumerator"
+require_relative "active_record_enumerator"
+require_relative "csv_enumerator"
+require_relative "throttle_enumerator"
+require_relative "nested_enumerator"
 require "forwardable"
 
 module JobIteration

data/lib/job-iteration/interruption_adapters/good_job_adapter.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+begin
+  require "good_job"
+rescue LoadError
+  # GoodJob is not available, no need to load the adapter
+  return
+end
+
+begin
+  # GoodJob.current_thread_shutting_down? was introduced in GoodJob 3.26
+  gem("good_job", ">= 3.26")
+rescue Gem::LoadError
+  warn("job-iteration's interruption adapter for GoodJob requires GoodJob 3.26 or newer")
+  return
+end
+
+module JobIteration
+  module InterruptionAdapters
+    module GoodJobAdapter
+      class << self
+        def call
+          !!::GoodJob.current_thread_shutting_down?
+        end
+      end
+    end
+
+    register(:good_job, GoodJobAdapter)
+  end
+end

data/lib/job-iteration/interruption_adapters/solid_queue_adapter.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+begin
+  require "solid_queue"
+rescue LoadError
+  # SolidQueue is not available, no need to load the adapter
+  return
+end
+
+begin
+  # SolidQueue.on_worker_stop was introduced in SolidQueue 0.7.1
+  gem("solid_queue", ">= 0.7.1")
+rescue Gem::LoadError
+  warn("job-iteration's interruption adapter for SolidQueue requires SolidQueue 0.7.1 or newer")
+  return
+end
+
+module JobIteration
+  module InterruptionAdapters
+    module SolidQueueAdapter
+      class << self
+        attr_accessor :stopping
+
+        def call
+          stopping
+        end
+      end
+
+      SolidQueue.on_worker_stop do
+        SolidQueueAdapter.stopping = true
+      end
+    end
+
+    register(:solid_queue, SolidQueueAdapter)
+  end
+end

data/lib/job-iteration/interruption_adapters.rb

@@ -4,7 +4,7 @@ require_relative "interruption_adapters/null_adapter"
 
 module JobIteration
   module InterruptionAdapters
-    BUNDLED_ADAPTERS = [:resque, :sidekiq].freeze # @api private
+    BUNDLED_ADAPTERS = [:good_job, :resque, :sidekiq, :solid_queue].freeze # @api private
 
     class << self
       # Returns adapter for specified name.

data/lib/job-iteration/iteration.rb

@@ -14,7 +14,7 @@ module JobIteration
     # The time when the job starts running. If the job is interrupted and runs again, the value is updated.
     attr_accessor :start_time
 
-    # The total time the job has been running, including multiple iterations.
+    # The total time (in seconds) the job has been running, including multiple iterations.
     # The time isn't reset if the job is interrupted.
     attr_accessor :total_time
 

data/lib/job-iteration/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JobIteration
-  VERSION = "1.5.0"
+  VERSION = "1.6.0"
 end

data/lib/job-iteration.rb

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
 require "active_job"
-require_relative "./job-iteration/version"
-require_relative "./job-iteration/enumerator_builder"
-require_relative "./job-iteration/interruption_adapters"
-require_relative "./job-iteration/iteration"
-require_relative "./job-iteration/log_subscriber"
-require_relative "./job-iteration/railtie"
+require_relative "job-iteration/version"
+require_relative "job-iteration/enumerator_builder"
+require_relative "job-iteration/interruption_adapters"
+require_relative "job-iteration/iteration"
+require_relative "job-iteration/log_subscriber"
+require_relative "job-iteration/railtie"
 
 module JobIteration
   Deprecation = ActiveSupport::Deprecation.new("2.0", "JobIteration")
@@ -62,9 +62,9 @@ module JobIteration
   # Overrides interruption checks based on queue adapter.
   # @deprecated - Use JobIteration::InterruptionAdapters.register(:foo, callable) instead.
   def interruption_adapter=(adapter)
-    Deprecation.warn("Setting JobIteration.interruption_adapter is deprecated. "\
-      "Use JobIteration::InterruptionAdapters.register(:foo, callable) instead "\
-      "to register the callable (a proc, method, or other object responding to #call) "\
+    Deprecation.warn("Setting JobIteration.interruption_adapter is deprecated. " \
+      "Use JobIteration::InterruptionAdapters.register(:foo, callable) instead " \
+      "to register the callable (a proc, method, or other object responding to #call) " \
       "as the interruption adapter for queue adapter :foo.")
     @interruption_adapter = adapter
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: job-iteration
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.6.0
 platform: ruby
 authors:
 - Shopify
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-05-29 00:00:00.000000000 Z
+date: 2024-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activejob
@@ -54,7 +54,6 @@ files:
 - guides/custom-enumerator.md
 - guides/iteration-how-it-works.md
 - guides/throttling.md
-- isogun.yml
 - job-iteration.gemspec
 - lib/job-iteration.rb
 - lib/job-iteration/active_record_batch_enumerator.rb
@@ -63,9 +62,11 @@ files:
 - lib/job-iteration/csv_enumerator.rb
 - lib/job-iteration/enumerator_builder.rb
 - lib/job-iteration/interruption_adapters.rb
+- lib/job-iteration/interruption_adapters/good_job_adapter.rb
 - lib/job-iteration/interruption_adapters/null_adapter.rb
 - lib/job-iteration/interruption_adapters/resque_adapter.rb
 - lib/job-iteration/interruption_adapters/sidekiq_adapter.rb
+- lib/job-iteration/interruption_adapters/solid_queue_adapter.rb
 - lib/job-iteration/iteration.rb
 - lib/job-iteration/log_subscriber.rb
 - lib/job-iteration/nested_enumerator.rb
@@ -79,7 +80,7 @@ licenses:
 metadata:
   changelog_uri: https://github.com/Shopify/job-iteration/blob/main/CHANGELOG.md
   allowed_push_host: https://rubygems.org
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -94,8 +95,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.10
-signing_key: 
+rubygems_version: 3.5.18
+signing_key:
 specification_version: 4
 summary: Makes your background jobs interruptible and resumable.
 test_files: []

data/isogun.yml

@@ -1,11 +0,0 @@
-name: job-iteration
-
-vm:
-  ip_address: 192.168.64.142
-  memory:     1G
-  cores:      2
-
-services:
-  - redis
-  - mysql
-