job-iteration 1.1.3 → 1.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3321e73fd83420a90163ec8963f381897cd1bd6e06717ce3c1e5db645e3aa823
-  data.tar.gz: bae3d1af0d3d803423c42c0085d3b5c9487c804e7362a9a70fe845e6d1abc406
+  metadata.gz: 99e26756140f1346b646a3931dc6cc8de1181a67a27916d9a0d85deb1d4dddeb
+  data.tar.gz: '058d65ec35e8cebbf74ba37c2227aa63cb1b5fbf183c81ee4c1b4f1cc3a9d033'
 SHA512:
-  metadata.gz: f78e4bbc18af7343762de3dbfbb4fc958512b7e62806a8adb372f5d94d3fa37b2e16fa0228f343e37679516ddb477099b8d9c3f4517493eb9fb9594ffba86e9e
-  data.tar.gz: d1593c8350cf85a78754375f808fd467421a9b04349fb776b3e7471a4ab12d3df14e104021c17ffc80f9595248b2e99659a6747954296dd0d15afefa244bc4ec
+  metadata.gz: 86a6a0f8ef55ab7502215fc4efcd6bc6e7b7a0b4737decf470db964d2302705d42ab1d16eff904e0d2c914f7ac30ca66cd2b99fffcb6aeaf1db904619fd89418
+  data.tar.gz: 7d9c9d5023f56365e2a1040fc7ccbb57f01c73f50c7addeaaedcf64edc2dab5cf2383d56cda46b832cdc6acce367abf87757dbd1113971335f51d675883e4141

data/.rubocop.yml

@@ -5,7 +5,7 @@ AllCops:
   TargetRubyVersion: 2.4.4
   Exclude:
     - 'vendor/bundle/**/*'
-Lint/HandleExceptions:
+Lint/SuppressedException:
   Exclude:
     - lib/job-iteration.rb
 Style/GlobalVars:

data/.travis.yml

@@ -3,8 +3,9 @@ services:
   - redis-server
 language: ruby
 rvm:
-  - 2.5.5
-  - 2.6.2
+  - 2.5
+  - 2.6
+  - 2.7
 before_install:
   - mysql -e 'CREATE DATABASE job_iteration_test;'
 script:
@@ -14,4 +15,5 @@ script:
 
 gemfile:
   - 'gemfiles/rails_5_2.gemfile'
+  - 'gemfiles/rails_6_0.gemfile'
   - 'gemfiles/rails_edge.gemfile'

data/CHANGELOG.md

@@ -4,6 +4,28 @@
 
 #### Bug fix
 
+## v1.1.8 (June 8, 2020)
+
+- Preserve ruby2_keywords tags in arguments on Ruby 2.7
+
+## v1.1.7 (June 4, 2020)
+
+- [54](https://github.com/Shopify/job-iteration/pull/54) - Fix warnings on Ruby 2.7
+
+## v1.1.6 (May 22, 2020)
+
+- [49](https://github.com/Shopify/job-iteration/pull/49) -  Log when enumerator has nothing to iterate
+- [52](https://github.com/Shopify/job-iteration/pull/52) -  Fix CSVEnumerator cursor to properly remove already processed rows
+
+## v1.1.5 (February 27, 2020)
+
+- [47](https://github.com/Shopify/job-iteration/pull/47) -  Optional `sorbet-runtime` support for `JobIteration::Iteration` interface validation
+
+## v1.1.4 (December 13, 2019)
+
+- [45](https://github.com/Shopify/job-iteration/pull/45) -  Add Throttle enumerator
+
+
 ### v1.1.3 (August 20, 2019)
 
 - [36](https://github.com/shopify/job-iteration/pull/39) -  Check method validation at job initialization step

data/Gemfile

@@ -11,7 +11,7 @@ gemspec
 gem 'sidekiq'
 gem 'resque'
 
-gem 'mysql2', '~> 0.4.4'
+gem 'mysql2', '~> 0.5'
 gem 'globalid'
 gem 'i18n'
 gem 'redis'
@@ -20,6 +20,9 @@ gem 'database_cleaner'
 gem 'pry'
 gem 'mocha'
 
-gem 'rubocop'
+gem 'rubocop', '~> 0.77.0'
 gem 'yard'
 gem 'rake'
+
+# for unit testing optional sorbet support
+gem 'sorbet-runtime'

data/README.md

@@ -112,6 +112,7 @@ Iteration hooks into Sidekiq and Resque out of the box to support graceful inter
 * [Iteration: how it works](guides/iteration-how-it-works.md)
 * [Best practices](guides/best-practices.md)
 * [Writing custom enumerator](guides/custom-enumerator.md)
+* [Throttling](guides/throttling.md)
 
 For more detailed documentation, see [rubydoc](https://www.rubydoc.info/github/Shopify/job-iteration).
 

data/dev.yml

@@ -7,7 +7,7 @@ up:
     - mysql-client:
         or:        [mysql@5.7]
   - ruby:
-      version: 2.6.2
+      version: 2.6.5
   - railgun
   - bundler
   - custom:

data/gemfiles/rails_6_0.gemfile

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+eval_gemfile '../Gemfile'
+
+gem 'activejob', '~> 6.0.0'
+gem 'activerecord', '~> 6.0.0'

data/guides/custom-enumerator.md

@@ -8,7 +8,7 @@ class ListJob < ActiveJob::Base
 
   def build_enumerator(*)
     @redis = Redis.new
-    Enumerator.new |yielder|
+    Enumerator.new do |yielder|
       yielder.yield @redis.lpop(key), nil
     end
   end

data/guides/throttling.md

@@ -0,0 +1,46 @@
+Iteration comes with a special wrapper enumerator that allows you to throttle iterations based on external signal (e.g. database health).
+
+Consider this example:
+
+```ruby
+class InactiveAccountDeleteJob < ActiveJob::Base
+  include JobIteration::Iteration
+
+  def build_enumerator(_params, cursor:)
+    enumerator_builder.active_record_on_batches(
+      Account.inactive,
+      cursor: cursor
+    )
+  end
+
+  def each_iteration(batch, _params)
+    Account.where(id: batch.map(&:id)).delete_all
+  end
+end
+```
+
+For an app that keeps track of customer accounts, it's typical to purge old data that's no longer relevant for storage.
+
+At the same time, if you've got a lot of DB writes to perform, this can cause extra load on the database and slow down other parts of your service.
+
+You can change `build_enumerator` to wrap enumeration on DB rows into a throttle enumerator, which takes signal as a proc and enqueues the job for later in case the proc returned `true`.
+
+```ruby
+def build_enumerator(_params, cursor:)
+  enumerator_builder.build_throttle_enumerator(
+    enumerator_builder.active_record_on_batches(
+      Account.inactive,
+      cursor: cursor
+    ),
+    throttle_on: -> { DatabaseStatus.unhealthy? },
+    backoff: 30.seconds
+  )
+end
+```
+
+Note that it's up to you to implement `DatabaseStatus.unhealthy?` that works for your database choice. At Shopify, a helper like `DatabaseStatus` checks the following MySQL metrics:
+
+* Replication lag across all regions
+* DB threads
+* DB is available for writes (otherwise indicates a failover happening)
+* [Semian](https://github.com/shopify/semian) open circuits

data/job-iteration.gemspec

@@ -23,6 +23,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = %w(lib)
 
   spec.metadata["changelog_uri"] = "https://github.com/Shopify/job-iteration/blob/master/CHANGELOG.md"
+  spec.metadata["allowed_push_host"] = "https://rubygems.org"
 
   spec.add_development_dependency("activerecord")
   spec.add_dependency("activejob", ">= 5.2")

data/lib/job-iteration/active_record_cursor.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 module JobIteration
+  # Curious about how this works from the SQL perspective?
+  # Check "Pagination Done the Right way": https://bit.ly/2Rq7iPF
   class ActiveRecordCursor # @private
     include Comparable
 

data/lib/job-iteration/csv_enumerator.rb

@@ -32,8 +32,8 @@ module JobIteration
     def rows(cursor:)
       @csv.lazy
         .each_with_index
-        .drop(cursor.to_i)
-        .to_enum { count_rows_in_file }
+        .drop(count_of_processed_rows(cursor))
+        .to_enum { count_of_rows_in_file }
     end
 
     # Constructs a enumerator on batches of CSV rows
@@ -42,13 +42,13 @@ module JobIteration
       @csv.lazy
         .each_slice(batch_size)
         .each_with_index
-        .drop(cursor.to_i)
-        .to_enum { (count_rows_in_file.to_f / batch_size).ceil }
+        .drop(count_of_processed_rows(cursor))
+        .to_enum { (count_of_rows_in_file.to_f / batch_size).ceil }
     end
 
     private
 
-    def count_rows_in_file
+    def count_of_rows_in_file
       # TODO: Remove rescue for NoMethodError when Ruby 2.6 is no longer supported.
       begin
         filepath = @csv.path
@@ -63,5 +63,9 @@ module JobIteration
       count -= 1 if @csv.headers
       count
     end
+
+    def count_of_processed_rows(cursor)
+      cursor.nil? ? 0 : cursor + 1
+    end
   end
 end

data/lib/job-iteration/enumerator_builder.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 require_relative "./active_record_enumerator"
 require_relative "./csv_enumerator"
+require_relative "./throttle_enumerator"
 require "forwardable"
 
 module JobIteration
@@ -61,16 +62,6 @@ module JobIteration
       wrap(self, enumerable.each_with_index.drop(drop).to_enum { enumerable.size })
     end
 
-    # Builds Enumerator from a lock queue instance that belongs to a job.
-    # The helper is only to be used from jobs that use LockQueue module.
-    def build_lock_queue_enumerator(lock_queue, at_most_once:)
-      unless lock_queue.is_a?(BackgroundQueue::LockQueue::RedisQueue) ||
-          lock_queue.is_a?(BackgroundQueue::LockQueue::RolloutRedisQueue)
-        raise ArgumentError, "an argument to #build_lock_queue_enumerator must be a LockQueue"
-      end
-      wrap(self, BackgroundQueue::LockQueueEnumerator.new(lock_queue, at_most_once: at_most_once).to_enum)
-    end
-
     # Builds Enumerator from Active Record Relation. Each Enumerator tick moves the cursor one row forward.
     #
     # +columns:+ argument is used to build the actual query for iteration. +columns+: defaults to primary key:
@@ -119,11 +110,21 @@ module JobIteration
       wrap(self, enum)
     end
 
+    def build_throttle_enumerator(enum, throttle_on:, backoff:)
+      JobIteration::ThrottleEnumerator.new(
+        enum,
+        @job,
+        throttle_on: throttle_on,
+        backoff: backoff
+      ).to_enum
+    end
+
     alias_method :once, :build_once_enumerator
     alias_method :times, :build_times_enumerator
     alias_method :array, :build_array_enumerator
     alias_method :active_record_on_records, :build_active_record_enumerator_on_records
     alias_method :active_record_on_batches, :build_active_record_enumerator_on_batches
+    alias_method :throttle, :build_throttle_enumerator
 
     private
 

data/lib/job-iteration/iteration.rb

@@ -49,6 +49,7 @@ module JobIteration
       self.total_time = 0.0
       assert_implements_methods!
     end
+    ruby2_keywords(:initialize) if respond_to?(:ruby2_keywords, true)
 
     def serialize # @private
       super.merge(
@@ -69,7 +70,7 @@ module JobIteration
       interruptible_perform(*params)
     end
 
-    def retry_job(*)
+    def retry_job(*, **)
       super unless defined?(@retried) && @retried
       @retried = true
     end
@@ -116,8 +117,10 @@ module JobIteration
 
     def iterate_with_enumerator(enumerator, arguments)
       arguments = arguments.dup.freeze
+      found_record = false
       enumerator.each do |object_from_enumerator, index|
         record_unit_of_work do
+          found_record = true
           each_iteration(object_from_enumerator, *arguments)
           self.cursor_position = index
         end
@@ -128,6 +131,11 @@ module JobIteration
         return false
       end
 
+      logger.info(
+        "[JobIteration::Iteration] Enumerator found nothing to iterate! " \
+        "times_interrupted=#{times_interrupted} cursor_position=#{cursor_position}"
+      ) unless found_record
+
       true
     end
 
@@ -176,7 +184,7 @@ module JobIteration
       end
 
       if respond_to?(:build_enumerator, true)
-        parameters = method(:build_enumerator).parameters
+        parameters = method_parameters(:build_enumerator)
         unless valid_cursor_parameter?(parameters)
           raise ArgumentError, "Iteration job (#{self.class}) #build_enumerator " \
           "expects the keyword argument `cursor`"
@@ -187,6 +195,17 @@ module JobIteration
       end
     end
 
+    def method_parameters(method_name)
+      method = method(method_name)
+
+      if defined?(T::Private::Methods)
+        signature = T::Private::Methods.signature_for_method(method)
+        method = signature.method if signature
+      end
+
+      method.parameters
+    end
+
     def iteration_instrumentation_tags
       { job_class: self.class.name }
     end

data/lib/job-iteration/throttle_enumerator.rb

@@ -0,0 +1,46 @@
+# typed: true
+# frozen_string_literal: true
+module JobIteration
+  # ThrottleEnumerator allows you to throttle iterations
+  # based on external signal (e.g. database health).
+  # @example
+  #   def build_enumerator(_params, cursor:)
+  #     enumerator_builder.build_throttle_enumerator(
+  #       enumerator_builder.active_record_on_batches(
+  #         Account.inactive,
+  #         cursor: cursor
+  #       ),
+  #       throttle_on: -> { DatabaseStatus.unhealthy? },
+  #       backoff: 30.seconds
+  #     )
+  #   end
+  # The enumerator from above will mimic +active_record_on_batches+,
+  # except when +DatabaseStatus.unhealthy?+ starts to return true.
+  # In that case, it will re-enqueue the job with a specified backoff.
+  class ThrottleEnumerator
+    def initialize(enum, job, throttle_on:, backoff:)
+      @enum = enum
+      @job = job
+      @throttle_on = throttle_on
+      @backoff = backoff
+    end
+
+    def to_enum
+      Enumerator.new(-> { @enum.size }) do |yielder|
+        @enum.each do |*val|
+          if should_throttle?
+            ActiveSupport::Notifications.instrument("throttled.iteration", job_class: @job.class.name)
+            @job.retry_job(wait: @backoff)
+            throw(:abort, :skip_complete_callbacks)
+          end
+
+          yielder.yield(*val)
+        end
+      end
+    end
+
+    def should_throttle?
+      @throttle_on.call
+    end
+  end
+end

data/lib/job-iteration/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JobIteration
-  VERSION = "1.1.3"
+  VERSION = "1.1.8"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: job-iteration
 version: !ruby/object:Gem::Version
-  version: 1.1.3
+  version: 1.1.8
 platform: ruby
 authors:
 - Shopify
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-08-20 00:00:00.000000000 Z
+date: 2020-06-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -58,10 +58,12 @@ files:
 - bin/setup
 - dev.yml
 - gemfiles/rails_5_2.gemfile
+- gemfiles/rails_6_0.gemfile
 - gemfiles/rails_edge.gemfile
 - guides/best-practices.md
 - guides/custom-enumerator.md
 - guides/iteration-how-it-works.md
+- guides/throttling.md
 - job-iteration.gemspec
 - lib/job-iteration.rb
 - lib/job-iteration/active_record_cursor.rb
@@ -72,6 +74,7 @@ files:
 - lib/job-iteration/integrations/sidekiq.rb
 - lib/job-iteration/iteration.rb
 - lib/job-iteration/test_helper.rb
+- lib/job-iteration/throttle_enumerator.rb
 - lib/job-iteration/version.rb
 - railgun.yml
 homepage: https://github.com/shopify/job-iteration
@@ -79,6 +82,7 @@ licenses:
 - MIT
 metadata:
   changelog_uri: https://github.com/Shopify/job-iteration/blob/master/CHANGELOG.md
+  allowed_push_host: https://rubygems.org
 post_install_message: 
 rdoc_options: []
 require_paths: