job-iteration 1.6.0 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 55bbe81f1fe209219d6dbb5aa469f027834d6b09dbdd6485b305fb4e3aa9bcb1
-  data.tar.gz: cf611985f1b8b6dc2cb1d00026fb53d6716ff353d0e59a96235f0e8538e1878b
+  metadata.gz: a38399b1ad6f7c81b2dd731e39abc75b4febcbac50ea9c310c6c96ada5f802ff
+  data.tar.gz: 96182f4dcdd4594c9fde855b2394b96a516e7134671a18905ebd3ae6b13ddcfa
 SHA512:
-  metadata.gz: e8857b87a8a999ec88b584896ff7c384310badbf7c2e7c553b9b64fed108cb7756fdc365a0008766d9179c40279ed221c96eca0b363fd5283f6c7d3d676ab82f
-  data.tar.gz: 89e9e6d584fef095c6c04b96a369abbf710283dffe9856091eef3fc7492222bd7591f22a8ebaaa7ea76f5d3281cd2c8163ba57d37ead43b095fb08fb377aea3a
+  metadata.gz: 515d1e0106c1b7c20ce3058df4df1c0c2a29dfb0ebe297bc3ac81da5365b7c8d68cf1058dc5de9c1c37331570260065e59aa86b1801f3b821a70d504aa3b25ac
+  data.tar.gz: 27303a6ea6910734d7d23f6ef0570cf7b2e6f011b1776731829e8366d93e2ad1032c1fce4bf44dee42e1c253f209f97cda8c761824ab7b81f8bfa51a339b0218

data/.github/workflows/ci.yml

@@ -15,29 +15,37 @@ jobs:
     strategy:
       matrix:
         ruby: ["2.6", "2.7", "3.0", "3.1", "3.2", "3.3"]
-        rails: ["5.2", "6.0", "6.1", "7.0", "7.1", "edge"]
+        rails: ["5.2", "6.0", "6.1", "7.0", "7.1", "7.2", "edge"]
         gemfile: [rails_gems]
         exclude:
           - ruby: "2.6"
             rails: "7.0"
           - ruby: "2.6"
             rails: "7.1"
+          - ruby: "2.6"
+            rails: "7.2"
           - ruby: "2.6"
             rails: "edge"
           - ruby: "2.7"
             rails: "7.1"
+          - ruby: "2.7"
+            rails: "7.2"
           - ruby: "2.7"
             rails: "edge"
           - ruby: "3.0"
             rails: "5.2"
           - ruby: "3.0"
             rails: "7.1"
+          - ruby: "3.0"
+            rails: "7.2"
           - ruby: "3.0"
             rails: "edge"
           - ruby: "3.1"
             rails: "5.2"
           - ruby: "3.1"
             rails: "6.0"
+          - ruby: "3.1"
+            rails: "edge"
           - ruby: "3.2"
             rails: "5.2"
           - ruby: "3.2"

data/CHANGELOG.md

@@ -1,6 +1,38 @@
 ### Main (unreleased)
 
-Nil
+### Changes
+
+nil
+
+### Features
+
+nil
+
+### Bug fixes
+
+nil
+
+## v1.8.0 (Dec 10, 2024)
+
+### Changes
+
+- [513](https://github.com/Shopify/job-iteration/pull/513) Deprecate returning enumerators from `build_enumerator` that are not wrapped with `enumerator_builder.wrap`. The built-in enumerator builders now always wrap.
+
+### Features
+
+- [340](https://github.com/Shopify/job-iteration/pull/340) Add `cursor.iteration` instrumentation for the query to fetch the next batch of records for the Active Record cursor.
+- [523](https://github.com/Shopify/job-iteration/pull/523) Add interruption adapter for [aws-activejob-sqs](https://github.com/aws/aws-activejob-sqs-ruby).
+
+### Bug fixes
+
+- [515](https://github.com/Shopify/job-iteration/pull/515) Fix size of array enumerators.
+
+## v1.7.0 (Oct 11, 2024)
+
+### Features
+
+- [509](https://github.com/Shopify/job-iteration/pull/509) - Added CSV batching functionality to `EnumeratorBuilder` with `build_csv_enumerator_on_batches` method and `csv_on_batches` alias.
+- [512](https://github.com/Shopify/job-iteration/pull/512) - Added support for custom timezones on `ActiveRecordEnumerator` and `ActiveRecordBatchEnumerator`. This allows for using cursors with datetime columns where `ActiveRecord.default_timezone` is set to `:local` and Active Record is not using the same timezone as the database.
 
 ## v1.6.0 (Sep 24, 2024)
 
@@ -10,9 +42,13 @@ Nil
 - [505](https://github.com/Shopify/job-iteration/pull/505) - Add interruption adapter for [Solid Queue](https://github.com/rails/solid_queue).
 
 ## v1.5.1 (May 29,2024)
+
+### Bug fixes
+
 - [483](https://github.com/Shopify/job-iteration/pull/483) - Reverts [#456 Use Arel instead of String for AR Enumerator conditionals](https://github.com/Shopify/job-iteration/pull/456)
 
 ## v1.5.0 (May 29, 2024)
+
 ### Changes
 
 - [437](https://github.com/Shopify/job-iteration/pull/437) - Use minimum between per-class `job_iteration_max_job_runtime` and `JobIteration.max_job_runtime`, instead of enforcing only setting decreasing values.
@@ -29,7 +65,7 @@ when generating position for cursor based on `:id` column (Rails 7.1 and above,
 primary models are now supported). This ensures we grab the value of the id column, rather than a
 potentially composite primary key value.
 - [456](https://github.com/Shopify/job-iteration/pull/431) - Use Arel to generate SQL that's type compatible for the
-  cursor pagination conditionals in ActiveRecord cursor. Previously, the cursor would coerce numeric ids to a string value 
+  cursor pagination conditionals in ActiveRecord cursor. Previously, the cursor would coerce numeric ids to a string value
   (e.g.: `... AND id > '1'`)
 
 ## v1.4.1 (Sep 5, 2023)

data/Gemfile.lock

@@ -1,6 +1,6 @@
 GIT
   remote: https://github.com/brianmario/mysql2
-  revision: f6a9b68b42a51d1a370403f11eb88527dcb42dc6
+  revision: 58f8d009a0d107776dd6d24c9906426fe0f7b856
   specs:
     mysql2 (0.5.6)
       bigdecimal
@@ -8,7 +8,7 @@ GIT
 PATH
   remote: .
   specs:
-    job-iteration (1.6.0)
+    job-iteration (1.8.0)
       activejob (>= 5.2)
 
 GEM
@@ -49,7 +49,7 @@ GEM
     language_server-protocol (3.17.0.3)
     method_source (1.1.0)
     minitest (5.24.0)
-    mocha (2.4.5)
+    mocha (2.7.0)
       ruby2_keywords (>= 0.0.5)
     mono_logger (1.1.2)
     multi_json (1.15.0)
@@ -60,7 +60,7 @@ GEM
     parser (3.3.3.0)
       ast (~> 2.4.1)
       racc
-    pry (0.14.2)
+    pry (0.15.0)
       coderay (~> 1.1)
       method_source (~> 1.0)
     racc (1.8.0)
@@ -84,8 +84,7 @@ GEM
       multi_json (~> 1.0)
       redis-namespace (~> 1.6)
       sinatra (>= 0.9.2)
-    rexml (3.3.6)
-      strscan
+    rexml (3.3.9)
     rubocop (1.64.1)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
@@ -115,7 +114,6 @@ GEM
       rack-session (>= 2.0.0, < 3)
       tilt (~> 2.0)
     sorbet-runtime (0.5.11460)
-    strscan (3.1.0)
     tilt (2.4.0)
     timeout (0.4.1)
     tzinfo (2.0.6)

data/README.md

@@ -161,7 +161,7 @@ Iteration hooks into Sidekiq and Resque out of the box to support graceful inter
 * [Writing custom enumerator](guides/custom-enumerator.md)
 * [Throttling](guides/throttling.md)
 
-For more detailed documentation, see [rubydoc](https://www.rubydoc.info/github/Shopify/job-iteration).
+For more detailed documentation, see [rubydoc](https://www.rubydoc.info/gems/job-iteration).
 
 ## Requirements
 

data/guides/custom-enumerator.md

@@ -78,12 +78,14 @@ class LoadRefundsForChargeJob < ActiveJob::Base
   # Use an exponential back-off strategy when Stripe's API returns errors.
 
   def build_enumerator(charge_id, cursor:)
-    StripeListEnumerator.new(
-      Stripe::Refund,
-      params: { charge: charge_id}, # "charge_id" will be a prefixed Stripe ID such as "chrg_123"
-      options: { api_key: "sk_test_123", stripe_version: "2018-01-18" },
-      cursor: cursor
-    ).to_enumerator
+    enumerator_builder.wrap(
+      StripeListEnumerator.new(
+        Stripe::Refund,
+        params: { charge: charge_id}, # "charge_id" will be a prefixed Stripe ID such as "chrg_123"
+        options: { api_key: "sk_test_123", stripe_version: "2018-01-18" },
+        cursor: cursor
+      ).to_enumerator
+    )
   end
 
   # Note that in this case `each_iteration` will only receive one positional argument per iteration.
@@ -114,9 +116,11 @@ class RedisPopListJob < ActiveJob::Base
   # @see https://redis.io/commands/lpop/
   def build_enumerator(*)
     @redis = Redis.new
-    Enumerator.new do |yielder|
-      yielder.yield @redis.lpop(key), nil
-    end
+    enumerator_builder.wrap(
+      Enumerator.new do |yielder|
+        yielder.yield @redis.lpop(key), nil
+      end
+    )
   end
 
   def each_iteration(item_from_redis)

data/guides/throttling.md

@@ -38,9 +38,31 @@ def build_enumerator(_params, cursor:)
 end
 ```
 
+If you want to apply throttling on all jobs, you can subclass your own EnumeratorBuilder and override the default
+enumerator builder. The builder always wraps the returned enumerators from `build_enumerator`
+
+```ruby
+class MyOwnBuilder < JobIteration::EnumeratorBuilder
+  class Wrapper < Enumerator
+    class << self
+      def wrap(_builder, enum)
+        ThrottleEnumerator.new(
+          enum,
+          nil,
+          throttle_on: -> { DatabaseStatus.unhealthy? },
+          backoff: 30.seconds
+        )
+      end
+    end
+  end
+end
+
+JobIteration.enumerator_builder = MyOwnBuilder
+```
+
 Note that it's up to you to implement `DatabaseStatus.unhealthy?` that works for your database choice. At Shopify, a helper like `DatabaseStatus` checks the following MySQL metrics:
 
 * Replication lag across all regions
 * DB threads
 * DB is available for writes (otherwise indicates a failover happening)
-* [Semian](https://github.com/shopify/semian) open circuits
+* [Semian](https://github.com/shopify/semian) open circuits

data/lib/job-iteration/active_record_batch_enumerator.rb

@@ -8,8 +8,9 @@ module JobIteration
 
     SQL_DATETIME_WITH_NSEC = "%Y-%m-%d %H:%M:%S.%N"
 
-    def initialize(relation, columns: nil, batch_size: 100, cursor: nil)
+    def initialize(relation, columns: nil, batch_size: 100, timezone: nil, cursor: nil)
       @batch_size = batch_size
+      @timezone = timezone
       @primary_key = "#{relation.table_name}.#{relation.primary_key}"
       @columns = Array(columns&.map(&:to_s) || @primary_key)
       @primary_key_index = @columns.index(@primary_key) || @columns.index(relation.primary_key)
@@ -114,7 +115,10 @@ module JobIteration
     end
 
     def column_value(value)
-      value.is_a?(Time) ? value.strftime(SQL_DATETIME_WITH_NSEC) : value
+      return value unless value.is_a?(Time)
+
+      value = value.in_time_zone(@timezone) unless @timezone.nil?
+      value.strftime(SQL_DATETIME_WITH_NSEC)
     end
   end
 end

data/lib/job-iteration/active_record_enumerator.rb

@@ -7,9 +7,10 @@ module JobIteration
   class ActiveRecordEnumerator
     SQL_DATETIME_WITH_NSEC = "%Y-%m-%d %H:%M:%S.%N"
 
-    def initialize(relation, columns: nil, batch_size: 100, cursor: nil)
+    def initialize(relation, columns: nil, batch_size: 100, timezone: nil, cursor: nil)
       @relation = relation
       @batch_size = batch_size
+      @timezone = timezone
       @columns = if columns
         Array(columns)
       else
@@ -31,7 +32,7 @@ module JobIteration
     def batches
       cursor = finder_cursor
       Enumerator.new(method(:size)) do |yielder|
-        while (records = cursor.next_batch(@batch_size))
+        while (records = instrument_next_batch(cursor))
           yielder.yield(records, cursor_value(records.last)) if records.any?
         end
       end
@@ -43,6 +44,12 @@ module JobIteration
 
     private
 
+    def instrument_next_batch(cursor)
+      ActiveSupport::Notifications.instrument("active_record_cursor.iteration") do
+        cursor.next_batch(@batch_size)
+      end
+    end
+
     def cursor_value(record)
       positions = @columns.map do |column|
         attribute_name = column.to_s.split(".").last
@@ -61,6 +68,7 @@ module JobIteration
       value = record.read_attribute(attribute.to_sym)
       case record.class.columns_hash.fetch(attribute).type
       when :datetime
+        value = value.in_time_zone(@timezone) unless @timezone.nil?
         value.strftime(SQL_DATETIME_WITH_NSEC)
       else
         value

data/lib/job-iteration/enumerator_builder.rb

@@ -16,8 +16,7 @@ module JobIteration
     # `enumerator_builder` is _always_ the type that is returned from
     # `build_enumerator`. This prevents people from implementing custom
     # Enumerators without wrapping them in
-    # `enumerator_builder.wrap(custom_enum)`. We don't do this yet for backwards
-    # compatibility with raw calls to EnumeratorBuilder. Think of these wrappers
+    # `enumerator_builder.wrap(custom_enum)`. Think of these wrappers
     # the way you should a middleware.
     class Wrapper < Enumerator
       class << self
@@ -63,7 +62,7 @@ module JobIteration
           cursor + 1
         end
 
-      wrap(self, enumerable.each_with_index.drop(drop).to_enum { enumerable.size })
+      wrap(self, enumerable.each_with_index.drop(drop).to_enum { enumerable.size - drop })
     end
 
     # Builds Enumerator from Active Record Relation. Each Enumerator tick moves the cursor one row forward.
@@ -131,17 +130,24 @@ module JobIteration
       enum
     end
 
-    def build_throttle_enumerator(enum, throttle_on:, backoff:)
-      JobIteration::ThrottleEnumerator.new(
-        enum,
+    def build_throttle_enumerator(enumerable, throttle_on:, backoff:)
+      enum = JobIteration::ThrottleEnumerator.new(
+        enumerable,
         @job,
         throttle_on: throttle_on,
         backoff: backoff,
       ).to_enum
+      wrap(self, enum)
     end
 
     def build_csv_enumerator(enumerable, cursor:)
-      CsvEnumerator.new(enumerable).rows(cursor: cursor)
+      enum = CsvEnumerator.new(enumerable).rows(cursor: cursor)
+      wrap(self, enum)
+    end
+
+    def build_csv_enumerator_on_batches(enumerable, cursor:, batch_size: 100)
+      enum = CsvEnumerator.new(enumerable).batches(cursor: cursor, batch_size: batch_size)
+      wrap(self, enum)
     end
 
     # Builds Enumerator for nested iteration.
@@ -175,7 +181,8 @@ module JobIteration
     #   end
     #
     def build_nested_enumerator(enums, cursor:)
-      NestedEnumerator.new(enums, cursor: cursor).each
+      enum = NestedEnumerator.new(enums, cursor: cursor).each
+      wrap(self, enum)
     end
 
     alias_method :once, :build_once_enumerator
@@ -186,6 +193,7 @@ module JobIteration
     alias_method :active_record_on_batch_relations, :build_active_record_enumerator_on_batch_relations
     alias_method :throttle, :build_throttle_enumerator
     alias_method :csv, :build_csv_enumerator
+    alias_method :csv_on_batches, :build_csv_enumerator_on_batches
     alias_method :nested, :build_nested_enumerator
 
     private

data/lib/job-iteration/interruption_adapters/sqs_adapter.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+begin
+  require "aws-activejob-sqs"
+rescue LoadError
+  # Aws::ActiveJob::SQS is not available, no need to load the adapter
+  return
+end
+
+begin
+  # Aws::ActiveJob::SQS.on_worker_stop was introduced in Aws::ActiveJob::SQS 0.1.1
+  gem("aws-activejob-sqs", ">= 0.1.1")
+rescue Gem::LoadError
+  warn("job-iteration's interruption adapter for SQS requires aws-activejob-sqs 0.1.1 or newer")
+  return
+end
+
+module JobIteration
+  module InterruptionAdapters
+    module SqsAdapter
+      class << self
+        attr_accessor :stopping
+
+        def call
+          stopping
+        end
+      end
+
+      Aws::ActiveJob::SQS.on_worker_stop do
+        SqsAdapter.stopping = true
+      end
+    end
+
+    register(:sqs, SqsAdapter)
+  end
+end

data/lib/job-iteration/interruption_adapters.rb

@@ -4,7 +4,7 @@ require_relative "interruption_adapters/null_adapter"
 
 module JobIteration
   module InterruptionAdapters
-    BUNDLED_ADAPTERS = [:good_job, :resque, :sidekiq, :solid_queue].freeze # @api private
+    BUNDLED_ADAPTERS = [:good_job, :resque, :sidekiq, :solid_queue, :sqs].freeze # @api private
 
     class << self
       # Returns adapter for specified name.

data/lib/job-iteration/iteration.rb

@@ -219,7 +219,14 @@ module JobIteration
     end
 
     def assert_enumerator!(enum)
-      return if enum.is_a?(Enumerator)
+      if enum.is_a?(Enumerator)
+        unless enum.is_a?(JobIteration.enumerator_builder::Wrapper)
+          JobIteration::Deprecation.warn("Returning an unwrapped enumerator from build_enumerator is deprecated. " \
+            "Wrap the enumerator using enumerator_builder.wrap(my_enumerator) instead.")
+        end
+
+        return
+      end
 
       raise ArgumentError, <<~EOS
         #build_enumerator is expected to return Enumerator object, but returned #{enum.class}.

data/lib/job-iteration/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JobIteration
-  VERSION = "1.6.0"
+  VERSION = "1.8.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: job-iteration
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 1.8.0
 platform: ruby
 authors:
 - Shopify
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-09-25 00:00:00.000000000 Z
+date: 2024-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activejob
@@ -67,6 +67,7 @@ files:
 - lib/job-iteration/interruption_adapters/resque_adapter.rb
 - lib/job-iteration/interruption_adapters/sidekiq_adapter.rb
 - lib/job-iteration/interruption_adapters/solid_queue_adapter.rb
+- lib/job-iteration/interruption_adapters/sqs_adapter.rb
 - lib/job-iteration/iteration.rb
 - lib/job-iteration/log_subscriber.rb
 - lib/job-iteration/nested_enumerator.rb
@@ -95,7 +96,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.18
+rubygems_version: 3.5.23
 signing_key:
 specification_version: 4
 summary: Makes your background jobs interruptible and resumable.