job-iteration 1.7.0 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0c3d42266a1de05ed85c35d99e76b55a131144054db3e3b83026aa3e180488c6
-  data.tar.gz: c272d3a7e0316cbb0a9b34326f8b1cdff5cb43c2ff58828dea1767a7f47eecb9
+  metadata.gz: a38399b1ad6f7c81b2dd731e39abc75b4febcbac50ea9c310c6c96ada5f802ff
+  data.tar.gz: 96182f4dcdd4594c9fde855b2394b96a516e7134671a18905ebd3ae6b13ddcfa
 SHA512:
-  metadata.gz: 500cefbc1ab8e7f02a4679b13f8c1401662e63021d20b2b4b5209d979f70c208d5b2a91ed5765d5dc5739666772fbe86e6f1a3e658bd9b1dad03f4b42d97b270
-  data.tar.gz: 37e100ba000a3ab67a07e78937ac12c58822a13e8f5641e6a3312a49a51733da3a1d4d3c8bf1a1f1928fc45c3aa026caf003fb97565a412e4c371b389513c7bb
+  metadata.gz: 515d1e0106c1b7c20ce3058df4df1c0c2a29dfb0ebe297bc3ac81da5365b7c8d68cf1058dc5de9c1c37331570260065e59aa86b1801f3b821a70d504aa3b25ac
+  data.tar.gz: 27303a6ea6910734d7d23f6ef0570cf7b2e6f011b1776731829e8366d93e2ad1032c1fce4bf44dee42e1c253f209f97cda8c761824ab7b81f8bfa51a339b0218

data/CHANGELOG.md

@@ -1,6 +1,31 @@
 ### Main (unreleased)
 
-Nil
+### Changes
+
+nil
+
+### Features
+
+nil
+
+### Bug fixes
+
+nil
+
+## v1.8.0 (Dec 10, 2024)
+
+### Changes
+
+- [513](https://github.com/Shopify/job-iteration/pull/513) Deprecate returning enumerators from `build_enumerator` that are not wrapped with `enumerator_builder.wrap`. The built-in enumerator builders now always wrap.
+
+### Features
+
+- [340](https://github.com/Shopify/job-iteration/pull/340) Add `cursor.iteration` instrumentation for the query to fetch the next batch of records for the Active Record cursor.
+- [523](https://github.com/Shopify/job-iteration/pull/523) Add interruption adapter for [aws-activejob-sqs](https://github.com/aws/aws-activejob-sqs-ruby).
+
+### Bug fixes
+
+- [515](https://github.com/Shopify/job-iteration/pull/515) Fix size of array enumerators.
 
 ## v1.7.0 (Oct 11, 2024)
 

data/Gemfile.lock

@@ -1,6 +1,6 @@
 GIT
   remote: https://github.com/brianmario/mysql2
-  revision: f6a9b68b42a51d1a370403f11eb88527dcb42dc6
+  revision: 58f8d009a0d107776dd6d24c9906426fe0f7b856
   specs:
     mysql2 (0.5.6)
       bigdecimal
@@ -8,7 +8,7 @@ GIT
 PATH
   remote: .
   specs:
-    job-iteration (1.7.0)
+    job-iteration (1.8.0)
       activejob (>= 5.2)
 
 GEM
@@ -49,7 +49,7 @@ GEM
     language_server-protocol (3.17.0.3)
     method_source (1.1.0)
     minitest (5.24.0)
-    mocha (2.4.5)
+    mocha (2.7.0)
       ruby2_keywords (>= 0.0.5)
     mono_logger (1.1.2)
     multi_json (1.15.0)
@@ -60,7 +60,7 @@ GEM
     parser (3.3.3.0)
       ast (~> 2.4.1)
       racc
-    pry (0.14.2)
+    pry (0.15.0)
       coderay (~> 1.1)
       method_source (~> 1.0)
     racc (1.8.0)
@@ -84,8 +84,7 @@ GEM
       multi_json (~> 1.0)
       redis-namespace (~> 1.6)
       sinatra (>= 0.9.2)
-    rexml (3.3.6)
-      strscan
+    rexml (3.3.9)
     rubocop (1.64.1)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
@@ -115,7 +114,6 @@ GEM
       rack-session (>= 2.0.0, < 3)
       tilt (~> 2.0)
     sorbet-runtime (0.5.11460)
-    strscan (3.1.0)
     tilt (2.4.0)
     timeout (0.4.1)
     tzinfo (2.0.6)

data/README.md

@@ -161,7 +161,7 @@ Iteration hooks into Sidekiq and Resque out of the box to support graceful inter
 * [Writing custom enumerator](guides/custom-enumerator.md)
 * [Throttling](guides/throttling.md)
 
-For more detailed documentation, see [rubydoc](https://www.rubydoc.info/github/Shopify/job-iteration).
+For more detailed documentation, see [rubydoc](https://www.rubydoc.info/gems/job-iteration).
 
 ## Requirements
 

data/guides/custom-enumerator.md

@@ -78,12 +78,14 @@ class LoadRefundsForChargeJob < ActiveJob::Base
   # Use an exponential back-off strategy when Stripe's API returns errors.
 
   def build_enumerator(charge_id, cursor:)
-    StripeListEnumerator.new(
-      Stripe::Refund,
-      params: { charge: charge_id}, # "charge_id" will be a prefixed Stripe ID such as "chrg_123"
-      options: { api_key: "sk_test_123", stripe_version: "2018-01-18" },
-      cursor: cursor
-    ).to_enumerator
+    enumerator_builder.wrap(
+      StripeListEnumerator.new(
+        Stripe::Refund,
+        params: { charge: charge_id}, # "charge_id" will be a prefixed Stripe ID such as "chrg_123"
+        options: { api_key: "sk_test_123", stripe_version: "2018-01-18" },
+        cursor: cursor
+      ).to_enumerator
+    )
   end
 
   # Note that in this case `each_iteration` will only receive one positional argument per iteration.
@@ -114,9 +116,11 @@ class RedisPopListJob < ActiveJob::Base
   # @see https://redis.io/commands/lpop/
   def build_enumerator(*)
     @redis = Redis.new
-    Enumerator.new do |yielder|
-      yielder.yield @redis.lpop(key), nil
-    end
+    enumerator_builder.wrap(
+      Enumerator.new do |yielder|
+        yielder.yield @redis.lpop(key), nil
+      end
+    )
   end
 
   def each_iteration(item_from_redis)

data/guides/throttling.md

@@ -38,9 +38,31 @@ def build_enumerator(_params, cursor:)
 end
 ```
 
+If you want to apply throttling on all jobs, you can subclass your own EnumeratorBuilder and override the default
+enumerator builder. The builder always wraps the returned enumerators from `build_enumerator`
+
+```ruby
+class MyOwnBuilder < JobIteration::EnumeratorBuilder
+  class Wrapper < Enumerator
+    class << self
+      def wrap(_builder, enum)
+        ThrottleEnumerator.new(
+          enum,
+          nil,
+          throttle_on: -> { DatabaseStatus.unhealthy? },
+          backoff: 30.seconds
+        )
+      end
+    end
+  end
+end
+
+JobIteration.enumerator_builder = MyOwnBuilder
+```
+
 Note that it's up to you to implement `DatabaseStatus.unhealthy?` that works for your database choice. At Shopify, a helper like `DatabaseStatus` checks the following MySQL metrics:
 
 * Replication lag across all regions
 * DB threads
 * DB is available for writes (otherwise indicates a failover happening)
-* [Semian](https://github.com/shopify/semian) open circuits
+* [Semian](https://github.com/shopify/semian) open circuits

data/lib/job-iteration/active_record_enumerator.rb

@@ -32,7 +32,7 @@ module JobIteration
     def batches
       cursor = finder_cursor
       Enumerator.new(method(:size)) do |yielder|
-        while (records = cursor.next_batch(@batch_size))
+        while (records = instrument_next_batch(cursor))
           yielder.yield(records, cursor_value(records.last)) if records.any?
         end
       end
@@ -44,6 +44,12 @@ module JobIteration
 
     private
 
+    def instrument_next_batch(cursor)
+      ActiveSupport::Notifications.instrument("active_record_cursor.iteration") do
+        cursor.next_batch(@batch_size)
+      end
+    end
+
     def cursor_value(record)
       positions = @columns.map do |column|
         attribute_name = column.to_s.split(".").last

data/lib/job-iteration/enumerator_builder.rb

@@ -16,8 +16,7 @@ module JobIteration
     # `enumerator_builder` is _always_ the type that is returned from
     # `build_enumerator`. This prevents people from implementing custom
     # Enumerators without wrapping them in
-    # `enumerator_builder.wrap(custom_enum)`. We don't do this yet for backwards
-    # compatibility with raw calls to EnumeratorBuilder. Think of these wrappers
+    # `enumerator_builder.wrap(custom_enum)`. Think of these wrappers
     # the way you should a middleware.
     class Wrapper < Enumerator
       class << self
@@ -63,7 +62,7 @@ module JobIteration
           cursor + 1
         end
 
-      wrap(self, enumerable.each_with_index.drop(drop).to_enum { enumerable.size })
+      wrap(self, enumerable.each_with_index.drop(drop).to_enum { enumerable.size - drop })
     end
 
     # Builds Enumerator from Active Record Relation. Each Enumerator tick moves the cursor one row forward.
@@ -131,21 +130,24 @@ module JobIteration
       enum
     end
 
-    def build_throttle_enumerator(enum, throttle_on:, backoff:)
-      JobIteration::ThrottleEnumerator.new(
-        enum,
+    def build_throttle_enumerator(enumerable, throttle_on:, backoff:)
+      enum = JobIteration::ThrottleEnumerator.new(
+        enumerable,
         @job,
         throttle_on: throttle_on,
         backoff: backoff,
       ).to_enum
+      wrap(self, enum)
     end
 
     def build_csv_enumerator(enumerable, cursor:)
-      CsvEnumerator.new(enumerable).rows(cursor: cursor)
+      enum = CsvEnumerator.new(enumerable).rows(cursor: cursor)
+      wrap(self, enum)
     end
 
     def build_csv_enumerator_on_batches(enumerable, cursor:, batch_size: 100)
-      CsvEnumerator.new(enumerable).batches(cursor: cursor, batch_size: batch_size)
+      enum = CsvEnumerator.new(enumerable).batches(cursor: cursor, batch_size: batch_size)
+      wrap(self, enum)
     end
 
     # Builds Enumerator for nested iteration.
@@ -179,7 +181,8 @@ module JobIteration
     #   end
     #
     def build_nested_enumerator(enums, cursor:)
-      NestedEnumerator.new(enums, cursor: cursor).each
+      enum = NestedEnumerator.new(enums, cursor: cursor).each
+      wrap(self, enum)
     end
 
     alias_method :once, :build_once_enumerator

data/lib/job-iteration/interruption_adapters/sqs_adapter.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+begin
+  require "aws-activejob-sqs"
+rescue LoadError
+  # Aws::ActiveJob::SQS is not available, no need to load the adapter
+  return
+end
+
+begin
+  # Aws::ActiveJob::SQS.on_worker_stop was introduced in Aws::ActiveJob::SQS 0.1.1
+  gem("aws-activejob-sqs", ">= 0.1.1")
+rescue Gem::LoadError
+  warn("job-iteration's interruption adapter for SQS requires aws-activejob-sqs 0.1.1 or newer")
+  return
+end
+
+module JobIteration
+  module InterruptionAdapters
+    module SqsAdapter
+      class << self
+        attr_accessor :stopping
+
+        def call
+          stopping
+        end
+      end
+
+      Aws::ActiveJob::SQS.on_worker_stop do
+        SqsAdapter.stopping = true
+      end
+    end
+
+    register(:sqs, SqsAdapter)
+  end
+end

data/lib/job-iteration/interruption_adapters.rb

@@ -4,7 +4,7 @@ require_relative "interruption_adapters/null_adapter"
 
 module JobIteration
   module InterruptionAdapters
-    BUNDLED_ADAPTERS = [:good_job, :resque, :sidekiq, :solid_queue].freeze # @api private
+    BUNDLED_ADAPTERS = [:good_job, :resque, :sidekiq, :solid_queue, :sqs].freeze # @api private
 
     class << self
       # Returns adapter for specified name.

data/lib/job-iteration/iteration.rb

@@ -219,7 +219,14 @@ module JobIteration
     end
 
     def assert_enumerator!(enum)
-      return if enum.is_a?(Enumerator)
+      if enum.is_a?(Enumerator)
+        unless enum.is_a?(JobIteration.enumerator_builder::Wrapper)
+          JobIteration::Deprecation.warn("Returning an unwrapped enumerator from build_enumerator is deprecated. " \
+            "Wrap the enumerator using enumerator_builder.wrap(my_enumerator) instead.")
+        end
+
+        return
+      end
 
       raise ArgumentError, <<~EOS
         #build_enumerator is expected to return Enumerator object, but returned #{enum.class}.

data/lib/job-iteration/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JobIteration
-  VERSION = "1.7.0"
+  VERSION = "1.8.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: job-iteration
 version: !ruby/object:Gem::Version
-  version: 1.7.0
+  version: 1.8.0
 platform: ruby
 authors:
 - Shopify
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-10-11 00:00:00.000000000 Z
+date: 2024-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activejob
@@ -67,6 +67,7 @@ files:
 - lib/job-iteration/interruption_adapters/resque_adapter.rb
 - lib/job-iteration/interruption_adapters/sidekiq_adapter.rb
 - lib/job-iteration/interruption_adapters/solid_queue_adapter.rb
+- lib/job-iteration/interruption_adapters/sqs_adapter.rb
 - lib/job-iteration/iteration.rb
 - lib/job-iteration/log_subscriber.rb
 - lib/job-iteration/nested_enumerator.rb
@@ -95,7 +96,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.21
+rubygems_version: 3.5.23
 signing_key:
 specification_version: 4
 summary: Makes your background jobs interruptible and resumable.