job-iteration 1.1.3 → 1.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3321e73fd83420a90163ec8963f381897cd1bd6e06717ce3c1e5db645e3aa823
-  data.tar.gz: bae3d1af0d3d803423c42c0085d3b5c9487c804e7362a9a70fe845e6d1abc406
+  metadata.gz: 21dc74805b967b6082859b0c9a07d955f9f77ca3e82f250ceaf4f2aecf4afd7b
+  data.tar.gz: 23ea1a3e03c8b0578dcf852449204f60e5391bdb92f26987da7d42f45b052595
 SHA512:
-  metadata.gz: f78e4bbc18af7343762de3dbfbb4fc958512b7e62806a8adb372f5d94d3fa37b2e16fa0228f343e37679516ddb477099b8d9c3f4517493eb9fb9594ffba86e9e
-  data.tar.gz: d1593c8350cf85a78754375f808fd467421a9b04349fb776b3e7471a4ab12d3df14e104021c17ffc80f9595248b2e99659a6747954296dd0d15afefa244bc4ec
+  metadata.gz: b3b5b27d999f8ea00a443d01857f0483fef67edcd0b0525d8ef8003c8814b9efe1e34713d6f45d0aa2de68c79c2517be141595f37c2501265f04ea6a4da7e0af
+  data.tar.gz: e47797081fc96034f2fe21746528047cba4f2873a0e6a36a7a38f2265d38a1dcb045f66216c3e9579699d63a9b6416136cf456e630404bb6d5381c433ef7232c

data/.rubocop.yml

@@ -5,7 +5,7 @@ AllCops:
   TargetRubyVersion: 2.4.4
   Exclude:
     - 'vendor/bundle/**/*'
-Lint/HandleExceptions:
+Lint/SuppressedException:
   Exclude:
     - lib/job-iteration.rb
 Style/GlobalVars:

data/CHANGELOG.md

@@ -4,6 +4,11 @@
 
 #### Bug fix
 
+## v1.1.4 (December 13, 2019)
+
+- [45](https://github.com/Shopify/job-iteration/pull/45) -  Add Throttle enumerator
+
+
 ### v1.1.3 (August 20, 2019)
 
 - [36](https://github.com/shopify/job-iteration/pull/39) -  Check method validation at job initialization step

data/Gemfile

@@ -11,7 +11,7 @@ gemspec
 gem 'sidekiq'
 gem 'resque'
 
-gem 'mysql2', '~> 0.4.4'
+gem 'mysql2', '~> 0.5'
 gem 'globalid'
 gem 'i18n'
 gem 'redis'
@@ -20,6 +20,6 @@ gem 'database_cleaner'
 gem 'pry'
 gem 'mocha'
 
-gem 'rubocop'
+gem 'rubocop', '~> 0.77.0'
 gem 'yard'
 gem 'rake'

data/README.md

@@ -112,6 +112,7 @@ Iteration hooks into Sidekiq and Resque out of the box to support graceful inter
 * [Iteration: how it works](guides/iteration-how-it-works.md)
 * [Best practices](guides/best-practices.md)
 * [Writing custom enumerator](guides/custom-enumerator.md)
+* [Throttling](guides/throttling.md)
 
 For more detailed documentation, see [rubydoc](https://www.rubydoc.info/github/Shopify/job-iteration).
 

data/guides/throttling.md

@@ -0,0 +1,46 @@
+Iteration comes with a special wrapper enumerator that allows you to throttle iterations based on external signal (e.g. database health).
+
+Consider this example:
+
+```ruby
+class InactiveAccountDeleteJob < ActiveJob::Base
+  include JobIteration::Iteration
+
+  def build_enumerator(_params, cursor:)
+    enumerator_builder.active_record_on_batches(
+      Account.inactive,
+      cursor: cursor
+    )
+  end
+
+  def each_iteration(batch, _params)
+    Account.where(id: batch.map(&:id)).delete_all
+  end
+end
+```
+
+For an app that keeps track of customer accounts, it's typical to purge old data that's no longer relevant for storage.
+
+At the same time, if you've got a lot of DB writes to perform, this can cause extra load on the database and slow down other parts of your service.
+
+You can change `build_enumerator` to wrap enumeration on DB rows into a throttle enumerator, which takes signal as a proc and enqueues the job for later in case the proc returned `true`.
+
+```ruby
+def build_enumerator(_params, cursor:)
+  enumerator_builder.build_throttle_enumerator(
+    enumerator_builder.active_record_on_batches(
+      Account.inactive,
+      cursor: cursor
+    ),
+    throttle_on: -> { DatabaseStatus.unhealthy? },
+    backoff: 30.seconds
+  )
+end
+```
+
+Note that it's up to you to implement `DatabaseStatus.unhealthy?` that works for your database choice. At Shopify, a helper like `DatabaseStatus` checks the following MySQL metrics:
+
+* Replication lag across all regions
+* DB threads
+* DB is available for writes (otherwise indicates a failover happening)
+* [Semian](https://github.com/shopify/semian) open circuits

data/lib/job-iteration/enumerator_builder.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 require_relative "./active_record_enumerator"
 require_relative "./csv_enumerator"
+require_relative "./throttle_enumerator"
 require "forwardable"
 
 module JobIteration
@@ -61,16 +62,6 @@ module JobIteration
       wrap(self, enumerable.each_with_index.drop(drop).to_enum { enumerable.size })
     end
 
-    # Builds Enumerator from a lock queue instance that belongs to a job.
-    # The helper is only to be used from jobs that use LockQueue module.
-    def build_lock_queue_enumerator(lock_queue, at_most_once:)
-      unless lock_queue.is_a?(BackgroundQueue::LockQueue::RedisQueue) ||
-          lock_queue.is_a?(BackgroundQueue::LockQueue::RolloutRedisQueue)
-        raise ArgumentError, "an argument to #build_lock_queue_enumerator must be a LockQueue"
-      end
-      wrap(self, BackgroundQueue::LockQueueEnumerator.new(lock_queue, at_most_once: at_most_once).to_enum)
-    end
-
     # Builds Enumerator from Active Record Relation. Each Enumerator tick moves the cursor one row forward.
     #
     # +columns:+ argument is used to build the actual query for iteration. +columns+: defaults to primary key:
@@ -119,11 +110,21 @@ module JobIteration
       wrap(self, enum)
     end
 
+    def build_throttle_enumerator(enum, throttle_on:, backoff:)
+      JobIteration::ThrottleEnumerator.new(
+        enum,
+        @job,
+        throttle_on: throttle_on,
+        backoff: backoff
+      ).to_enum
+    end
+
     alias_method :once, :build_once_enumerator
     alias_method :times, :build_times_enumerator
     alias_method :array, :build_array_enumerator
     alias_method :active_record_on_records, :build_active_record_enumerator_on_records
     alias_method :active_record_on_batches, :build_active_record_enumerator_on_batches
+    alias_method :throttle, :build_throttle_enumerator
 
     private
 

data/lib/job-iteration/throttle_enumerator.rb

@@ -0,0 +1,46 @@
+# typed: true
+# frozen_string_literal: true
+module JobIteration
+  # ThrottleEnumerator allows you to throttle iterations
+  # based on external signal (e.g. database health).
+  # @example
+  #   def build_enumerator(_params, cursor:)
+  #     enumerator_builder.build_throttle_enumerator(
+  #       enumerator_builder.active_record_on_batches(
+  #         Account.inactive,
+  #         cursor: cursor
+  #       ),
+  #       throttle_on: -> { DatabaseStatus.unhealthy? },
+  #       backoff: 30.seconds
+  #     )
+  #   end
+  # The enumerator from above will mimic +active_record_on_batches+,
+  # except when +DatabaseStatus.unhealthy?+ starts to return true.
+  # In that case, it will re-enqueue the job with a specified backoff.
+  class ThrottleEnumerator
+    def initialize(enum, job, throttle_on:, backoff:)
+      @enum = enum
+      @job = job
+      @throttle_on = throttle_on
+      @backoff = backoff
+    end
+
+    def to_enum
+      Enumerator.new(-> { @enum.size }) do |yielder|
+        @enum.each do |*val|
+          if should_throttle?
+            ActiveSupport::Notifications.instrument("throttled.iteration", job_class: @job.class.name)
+            @job.retry_job(wait: @backoff)
+            throw(:abort, :skip_complete_callbacks)
+          end
+
+          yielder.yield(*val)
+        end
+      end
+    end
+
+    def should_throttle?
+      @throttle_on.call
+    end
+  end
+end

data/lib/job-iteration/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JobIteration
-  VERSION = "1.1.3"
+  VERSION = "1.1.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: job-iteration
 version: !ruby/object:Gem::Version
-  version: 1.1.3
+  version: 1.1.4
 platform: ruby
 authors:
 - Shopify
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-08-20 00:00:00.000000000 Z
+date: 2019-12-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -62,6 +62,7 @@ files:
 - guides/best-practices.md
 - guides/custom-enumerator.md
 - guides/iteration-how-it-works.md
+- guides/throttling.md
 - job-iteration.gemspec
 - lib/job-iteration.rb
 - lib/job-iteration/active_record_cursor.rb
@@ -72,6 +73,7 @@ files:
 - lib/job-iteration/integrations/sidekiq.rb
 - lib/job-iteration/iteration.rb
 - lib/job-iteration/test_helper.rb
+- lib/job-iteration/throttle_enumerator.rb
 - lib/job-iteration/version.rb
 - railgun.yml
 homepage: https://github.com/shopify/job-iteration