job-iteration 1.5.1 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b07cecd2f672db0710d7e756d1de20db69581ba98bd46109c0023dd8d991ad76
-  data.tar.gz: 598a4a368abc83bd4765aebf61b205592675d76b544806f4458ecec9779005b1
+  metadata.gz: 0c3d42266a1de05ed85c35d99e76b55a131144054db3e3b83026aa3e180488c6
+  data.tar.gz: c272d3a7e0316cbb0a9b34326f8b1cdff5cb43c2ff58828dea1767a7f47eecb9
 SHA512:
-  metadata.gz: 61cda3903925247899c9051d5de5fac59c9cd36f94172e124e2085174aff5be0677011693018096710a8149582c1589662a1359bb11b0bbdea3c9b7c53916946
-  data.tar.gz: 3773b812a06b4f5700abf4d3c44b65d5aae412545825091c895e294862ca4290aa87b83b7e59a182085fea60d939035f68f2f11ef7ab173c7968cb6cfdff7fd3
+  metadata.gz: 500cefbc1ab8e7f02a4679b13f8c1401662e63021d20b2b4b5209d979f70c208d5b2a91ed5765d5dc5739666772fbe86e6f1a3e658bd9b1dad03f4b42d97b270
+  data.tar.gz: 37e100ba000a3ab67a07e78937ac12c58822a13e8f5641e6a3312a49a51733da3a1d4d3c8bf1a1f1928fc45c3aa026caf003fb97565a412e4c371b389513c7bb

data/.github/workflows/ci.yml

@@ -15,29 +15,37 @@ jobs:
     strategy:
       matrix:
         ruby: ["2.6", "2.7", "3.0", "3.1", "3.2", "3.3"]
-        rails: ["5.2", "6.0", "6.1", "7.0", "7.1", "edge"]
+        rails: ["5.2", "6.0", "6.1", "7.0", "7.1", "7.2", "edge"]
         gemfile: [rails_gems]
         exclude:
           - ruby: "2.6"
             rails: "7.0"
           - ruby: "2.6"
             rails: "7.1"
+          - ruby: "2.6"
+            rails: "7.2"
           - ruby: "2.6"
             rails: "edge"
           - ruby: "2.7"
             rails: "7.1"
+          - ruby: "2.7"
+            rails: "7.2"
           - ruby: "2.7"
             rails: "edge"
           - ruby: "3.0"
             rails: "5.2"
           - ruby: "3.0"
             rails: "7.1"
+          - ruby: "3.0"
+            rails: "7.2"
           - ruby: "3.0"
             rails: "edge"
           - ruby: "3.1"
             rails: "5.2"
           - ruby: "3.1"
             rails: "6.0"
+          - ruby: "3.1"
+            rails: "edge"
           - ruby: "3.2"
             rails: "5.2"
           - ruby: "3.2"

data/.ruby-version

@@ -1 +1 @@
-3.3.0
+3.3.3

data/CHANGELOG.md

@@ -1,10 +1,29 @@
 ### Main (unreleased)
+
 Nil
 
+## v1.7.0 (Oct 11, 2024)
+
+### Features
+
+- [509](https://github.com/Shopify/job-iteration/pull/509) - Added CSV batching functionality to `EnumeratorBuilder` with `build_csv_enumerator_on_batches` method and `csv_on_batches` alias.
+- [512](https://github.com/Shopify/job-iteration/pull/512) - Added support for custom timezones on `ActiveRecordEnumerator` and `ActiveRecordBatchEnumerator`. This allows for using cursors with datetime columns where `ActiveRecord.default_timezone` is set to `:local` and Active Record is not using the same timezone as the database.
+
+## v1.6.0 (Sep 24, 2024)
+
+### Features
+
+- [464](https://github.com/Shopify/job-iteration/pull/464) - Add interruption adapter for [GoodJob](https://github.com/bensheldon/good_job).
+- [505](https://github.com/Shopify/job-iteration/pull/505) - Add interruption adapter for [Solid Queue](https://github.com/rails/solid_queue).
+
 ## v1.5.1 (May 29,2024)
-- [483](https://github.com/Shopify/job-iteration/pull/483) Reverts [#456 Use Arel instead of String for AR Enumerator conditionals](https://github.com/Shopify/job-iteration/pull/456)
+
+### Bug fixes
+
+- [483](https://github.com/Shopify/job-iteration/pull/483) - Reverts [#456 Use Arel instead of String for AR Enumerator conditionals](https://github.com/Shopify/job-iteration/pull/456)
 
 ## v1.5.0 (May 29, 2024)
+
 ### Changes
 
 - [437](https://github.com/Shopify/job-iteration/pull/437) - Use minimum between per-class `job_iteration_max_job_runtime` and `JobIteration.max_job_runtime`, instead of enforcing only setting decreasing values.
@@ -21,7 +40,7 @@ when generating position for cursor based on `:id` column (Rails 7.1 and above,
 primary models are now supported). This ensures we grab the value of the id column, rather than a
 potentially composite primary key value.
 - [456](https://github.com/Shopify/job-iteration/pull/431) - Use Arel to generate SQL that's type compatible for the
-  cursor pagination conditionals in ActiveRecord cursor. Previously, the cursor would coerce numeric ids to a string value 
+  cursor pagination conditionals in ActiveRecord cursor. Previously, the cursor would coerce numeric ids to a string value
   (e.g.: `... AND id > '1'`)
 
 ## v1.4.1 (Sep 5, 2023)

data/Gemfile.lock

@@ -1,73 +1,92 @@
 GIT
   remote: https://github.com/brianmario/mysql2
-  revision: 43ea8af635f5e23f054294ef7759320d47f30e5f
+  revision: f6a9b68b42a51d1a370403f11eb88527dcb42dc6
   specs:
     mysql2 (0.5.6)
+      bigdecimal
 
 PATH
   remote: .
   specs:
-    job-iteration (1.5.1)
+    job-iteration (1.7.0)
       activejob (>= 5.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (7.0.7)
-      activesupport (= 7.0.7)
+    activejob (7.1.3.4)
+      activesupport (= 7.1.3.4)
       globalid (>= 0.3.6)
-    activemodel (7.0.7)
-      activesupport (= 7.0.7)
-    activerecord (7.0.7)
-      activemodel (= 7.0.7)
-      activesupport (= 7.0.7)
-    activesupport (7.0.7)
+    activemodel (7.1.3.4)
+      activesupport (= 7.1.3.4)
+    activerecord (7.1.3.4)
+      activemodel (= 7.1.3.4)
+      activesupport (= 7.1.3.4)
+      timeout (>= 0.4.0)
+    activesupport (7.1.3.4)
+      base64
+      bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
+      connection_pool (>= 2.2.5)
+      drb
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
+      mutex_m
       tzinfo (~> 2.0)
     ast (2.4.2)
+    base64 (0.2.0)
+    bigdecimal (3.1.8)
     coderay (1.1.3)
-    concurrent-ruby (1.2.3)
+    concurrent-ruby (1.3.4)
     connection_pool (2.4.1)
     csv (3.3.0)
-    globalid (1.1.0)
-      activesupport (>= 5.0)
-    i18n (1.14.4)
+    drb (2.2.1)
+    globalid (1.2.1)
+      activesupport (>= 6.1)
+    i18n (1.14.6)
       concurrent-ruby (~> 1.0)
-    json (2.7.1)
+    json (2.7.2)
     language_server-protocol (3.17.0.3)
-    method_source (1.0.0)
-    minitest (5.19.0)
-    mocha (2.2.0)
+    method_source (1.1.0)
+    minitest (5.24.0)
+    mocha (2.4.5)
       ruby2_keywords (>= 0.0.5)
     mono_logger (1.1.2)
     multi_json (1.15.0)
-    parallel (1.24.0)
-    parser (3.3.0.5)
+    mustermann (3.0.0)
+      ruby2_keywords (~> 0.0.1)
+    mutex_m (0.2.0)
+    parallel (1.25.1)
+    parser (3.3.3.0)
       ast (~> 2.4.1)
       racc
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    racc (1.7.3)
-    rack (3.0.9.1)
+    racc (1.8.0)
+    rack (3.1.5)
+    rack-protection (4.0.0)
+      base64 (>= 0.1.0)
+      rack (>= 3.0.0, < 4)
+    rack-session (2.0.0)
+      rack (>= 3.0.0)
     rainbow (3.1.1)
     rake (13.2.1)
-    redis (5.2.0)
+    redis (5.3.0)
       redis-client (>= 0.22.0)
-    redis-client (0.22.1)
+    redis-client (0.22.2)
       connection_pool
     redis-namespace (1.11.0)
       redis (>= 4)
-    regexp_parser (2.9.0)
+    regexp_parser (2.9.2)
     resque (2.6.0)
       mono_logger (~> 1.0)
       multi_json (~> 1.0)
       redis-namespace (~> 1.6)
       sinatra (>= 0.9.2)
-    rexml (3.2.6)
-    rubocop (1.62.1)
+    rexml (3.3.6)
+      strscan
+    rubocop (1.64.1)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
@@ -78,24 +97,31 @@ GEM
       rubocop-ast (>= 1.31.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.31.2)
-      parser (>= 3.3.0.4)
-    rubocop-shopify (2.14.0)
+    rubocop-ast (1.31.3)
+      parser (>= 3.3.1.0)
+    rubocop-shopify (2.15.1)
       rubocop (~> 1.51)
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
-    sidekiq (7.1.2)
+    sidekiq (7.2.4)
       concurrent-ruby (< 2)
       connection_pool (>= 2.3.0)
       rack (>= 2.2.4)
-      redis-client (>= 0.14.0)
-    sinatra (1.0)
-      rack (>= 1.0)
-    sorbet-runtime (0.5.10978)
+      redis-client (>= 0.19.0)
+    sinatra (4.0.0)
+      mustermann (~> 3.0)
+      rack (>= 3.0.0, < 4)
+      rack-protection (= 4.0.0)
+      rack-session (>= 2.0.0, < 3)
+      tilt (~> 2.0)
+    sorbet-runtime (0.5.11460)
+    strscan (3.1.0)
+    tilt (2.4.0)
+    timeout (0.4.1)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.5.0)
-    yard (0.9.36)
+    yard (0.9.37)
 
 PLATFORMS
   ruby
@@ -118,4 +144,4 @@ DEPENDENCIES
   yard
 
 BUNDLED WITH
-   2.5.7
+   2.5.14

data/guides/iteration-how-it-works.md

@@ -29,6 +29,10 @@ In other words, if you are trying to process 100 records but the job consistentl
 
 If no retries are configured or retries are exhausted, Active Job 'bubbles up' the exception to the job backend. Retries by the backend (e.g. Sidekiq) are not supported, meaning that jobs retried by the job backend instead of Active Job will restart from the beginning.
 
+## Stopping a job
+
+Because jobs typically retry when exceptions are thrown, there is a special mechanism to fully stop a job that still has iterations remaining. To do this, you can `throw(:abort)`. This is then caught by job-iteration and signals that the job should complete now, regardless of its iteration state.
+
 ## Signals
 
 It's critical to know [UNIX signals](https://www.tutorialspoint.com/unix/unix-signals-traps.htm) in order to understand how interruption works. There are two main signals that Sidekiq and Resque use: `SIGTERM` and `SIGKILL`. `SIGTERM` is the graceful termination signal which means that the process should exit _soon_, not immediately. For Iteration, it means that we have time to wait for the last iteration to finish and to push job back to the queue with the last cursor position.

data/lib/job-iteration/active_record_batch_enumerator.rb

@@ -8,8 +8,9 @@ module JobIteration
 
     SQL_DATETIME_WITH_NSEC = "%Y-%m-%d %H:%M:%S.%N"
 
-    def initialize(relation, columns: nil, batch_size: 100, cursor: nil)
+    def initialize(relation, columns: nil, batch_size: 100, timezone: nil, cursor: nil)
       @batch_size = batch_size
+      @timezone = timezone
       @primary_key = "#{relation.table_name}.#{relation.primary_key}"
       @columns = Array(columns&.map(&:to_s) || @primary_key)
       @primary_key_index = @columns.index(@primary_key) || @columns.index(relation.primary_key)
@@ -114,7 +115,10 @@ module JobIteration
     end
 
     def column_value(value)
-      value.is_a?(Time) ? value.strftime(SQL_DATETIME_WITH_NSEC) : value
+      return value unless value.is_a?(Time)
+
+      value = value.in_time_zone(@timezone) unless @timezone.nil?
+      value.strftime(SQL_DATETIME_WITH_NSEC)
     end
   end
 end

data/lib/job-iteration/active_record_enumerator.rb

@@ -1,15 +1,16 @@
 # frozen_string_literal: true
 
-require_relative "./active_record_cursor"
+require_relative "active_record_cursor"
 module JobIteration
   # Builds Enumerator based on ActiveRecord Relation. Supports enumerating on rows and batches.
   # @see EnumeratorBuilder
   class ActiveRecordEnumerator
     SQL_DATETIME_WITH_NSEC = "%Y-%m-%d %H:%M:%S.%N"
 
-    def initialize(relation, columns: nil, batch_size: 100, cursor: nil)
+    def initialize(relation, columns: nil, batch_size: 100, timezone: nil, cursor: nil)
       @relation = relation
       @batch_size = batch_size
+      @timezone = timezone
       @columns = if columns
         Array(columns)
       else
@@ -61,6 +62,7 @@ module JobIteration
       value = record.read_attribute(attribute.to_sym)
       case record.class.columns_hash.fetch(attribute).type
       when :datetime
+        value = value.in_time_zone(@timezone) unless @timezone.nil?
         value.strftime(SQL_DATETIME_WITH_NSEC)
       else
         value

data/lib/job-iteration/enumerator_builder.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
-require_relative "./active_record_batch_enumerator"
-require_relative "./active_record_enumerator"
-require_relative "./csv_enumerator"
-require_relative "./throttle_enumerator"
-require_relative "./nested_enumerator"
+require_relative "active_record_batch_enumerator"
+require_relative "active_record_enumerator"
+require_relative "csv_enumerator"
+require_relative "throttle_enumerator"
+require_relative "nested_enumerator"
 require "forwardable"
 
 module JobIteration
@@ -144,6 +144,10 @@ module JobIteration
       CsvEnumerator.new(enumerable).rows(cursor: cursor)
     end
 
+    def build_csv_enumerator_on_batches(enumerable, cursor:, batch_size: 100)
+      CsvEnumerator.new(enumerable).batches(cursor: cursor, batch_size: batch_size)
+    end
+
     # Builds Enumerator for nested iteration.
     #
     # @param enums [Array<Proc>] an Array of Procs, each should return an Enumerator.
@@ -186,6 +190,7 @@ module JobIteration
     alias_method :active_record_on_batch_relations, :build_active_record_enumerator_on_batch_relations
     alias_method :throttle, :build_throttle_enumerator
     alias_method :csv, :build_csv_enumerator
+    alias_method :csv_on_batches, :build_csv_enumerator_on_batches
     alias_method :nested, :build_nested_enumerator
 
     private

data/lib/job-iteration/interruption_adapters/good_job_adapter.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+begin
+  require "good_job"
+rescue LoadError
+  # GoodJob is not available, no need to load the adapter
+  return
+end
+
+begin
+  # GoodJob.current_thread_shutting_down? was introduced in GoodJob 3.26
+  gem("good_job", ">= 3.26")
+rescue Gem::LoadError
+  warn("job-iteration's interruption adapter for GoodJob requires GoodJob 3.26 or newer")
+  return
+end
+
+module JobIteration
+  module InterruptionAdapters
+    module GoodJobAdapter
+      class << self
+        def call
+          !!::GoodJob.current_thread_shutting_down?
+        end
+      end
+    end
+
+    register(:good_job, GoodJobAdapter)
+  end
+end

data/lib/job-iteration/interruption_adapters/solid_queue_adapter.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+begin
+  require "solid_queue"
+rescue LoadError
+  # SolidQueue is not available, no need to load the adapter
+  return
+end
+
+begin
+  # SolidQueue.on_worker_stop was introduced in SolidQueue 0.7.1
+  gem("solid_queue", ">= 0.7.1")
+rescue Gem::LoadError
+  warn("job-iteration's interruption adapter for SolidQueue requires SolidQueue 0.7.1 or newer")
+  return
+end
+
+module JobIteration
+  module InterruptionAdapters
+    module SolidQueueAdapter
+      class << self
+        attr_accessor :stopping
+
+        def call
+          stopping
+        end
+      end
+
+      SolidQueue.on_worker_stop do
+        SolidQueueAdapter.stopping = true
+      end
+    end
+
+    register(:solid_queue, SolidQueueAdapter)
+  end
+end

data/lib/job-iteration/interruption_adapters.rb

@@ -4,7 +4,7 @@ require_relative "interruption_adapters/null_adapter"
 
 module JobIteration
   module InterruptionAdapters
-    BUNDLED_ADAPTERS = [:resque, :sidekiq].freeze # @api private
+    BUNDLED_ADAPTERS = [:good_job, :resque, :sidekiq, :solid_queue].freeze # @api private
 
     class << self
       # Returns adapter for specified name.

data/lib/job-iteration/iteration.rb

@@ -14,7 +14,7 @@ module JobIteration
     # The time when the job starts running. If the job is interrupted and runs again, the value is updated.
     attr_accessor :start_time
 
-    # The total time the job has been running, including multiple iterations.
+    # The total time (in seconds) the job has been running, including multiple iterations.
     # The time isn't reset if the job is interrupted.
     attr_accessor :total_time
 

data/lib/job-iteration/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JobIteration
-  VERSION = "1.5.1"
+  VERSION = "1.7.0"
 end

data/lib/job-iteration.rb

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
 require "active_job"
-require_relative "./job-iteration/version"
-require_relative "./job-iteration/enumerator_builder"
-require_relative "./job-iteration/interruption_adapters"
-require_relative "./job-iteration/iteration"
-require_relative "./job-iteration/log_subscriber"
-require_relative "./job-iteration/railtie"
+require_relative "job-iteration/version"
+require_relative "job-iteration/enumerator_builder"
+require_relative "job-iteration/interruption_adapters"
+require_relative "job-iteration/iteration"
+require_relative "job-iteration/log_subscriber"
+require_relative "job-iteration/railtie"
 
 module JobIteration
   Deprecation = ActiveSupport::Deprecation.new("2.0", "JobIteration")
@@ -62,9 +62,9 @@ module JobIteration
   # Overrides interruption checks based on queue adapter.
   # @deprecated - Use JobIteration::InterruptionAdapters.register(:foo, callable) instead.
   def interruption_adapter=(adapter)
-    Deprecation.warn("Setting JobIteration.interruption_adapter is deprecated. "\
-      "Use JobIteration::InterruptionAdapters.register(:foo, callable) instead "\
-      "to register the callable (a proc, method, or other object responding to #call) "\
+    Deprecation.warn("Setting JobIteration.interruption_adapter is deprecated. " \
+      "Use JobIteration::InterruptionAdapters.register(:foo, callable) instead " \
+      "to register the callable (a proc, method, or other object responding to #call) " \
       "as the interruption adapter for queue adapter :foo.")
     @interruption_adapter = adapter
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: job-iteration
 version: !ruby/object:Gem::Version
-  version: 1.5.1
+  version: 1.7.0
 platform: ruby
 authors:
 - Shopify
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-05-29 00:00:00.000000000 Z
+date: 2024-10-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activejob
@@ -54,7 +54,6 @@ files:
 - guides/custom-enumerator.md
 - guides/iteration-how-it-works.md
 - guides/throttling.md
-- isogun.yml
 - job-iteration.gemspec
 - lib/job-iteration.rb
 - lib/job-iteration/active_record_batch_enumerator.rb
@@ -63,9 +62,11 @@ files:
 - lib/job-iteration/csv_enumerator.rb
 - lib/job-iteration/enumerator_builder.rb
 - lib/job-iteration/interruption_adapters.rb
+- lib/job-iteration/interruption_adapters/good_job_adapter.rb
 - lib/job-iteration/interruption_adapters/null_adapter.rb
 - lib/job-iteration/interruption_adapters/resque_adapter.rb
 - lib/job-iteration/interruption_adapters/sidekiq_adapter.rb
+- lib/job-iteration/interruption_adapters/solid_queue_adapter.rb
 - lib/job-iteration/iteration.rb
 - lib/job-iteration/log_subscriber.rb
 - lib/job-iteration/nested_enumerator.rb
@@ -79,7 +80,7 @@ licenses:
 metadata:
   changelog_uri: https://github.com/Shopify/job-iteration/blob/main/CHANGELOG.md
   allowed_push_host: https://rubygems.org
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -94,8 +95,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.10
-signing_key: 
+rubygems_version: 3.5.21
+signing_key:
 specification_version: 4
 summary: Makes your background jobs interruptible and resumable.
 test_files: []

data/isogun.yml

@@ -1,11 +0,0 @@
-name: job-iteration
-
-vm:
-  ip_address: 192.168.64.142
-  memory:     1G
-  cores:      2
-
-services:
-  - redis
-  - mysql
-