job-iteration 1.3.6 → 1.4.1

data/lib/job-iteration/iteration.rb

@@ -8,11 +8,16 @@ module JobIteration
 
     attr_accessor(
       :cursor_position,
-      :start_time,
       :times_interrupted,
-      :total_time,
     )
 
+    # The time when the job starts running. If the job is interrupted and runs again, the value is updated.
+    attr_accessor :start_time
+
+    # The total time the job has been running, including multiple iterations.
+    # The time isn't reset if the job is interrupted.
+    attr_accessor :total_time
+
     class CursorError < ArgumentError
       attr_reader :cursor
 
@@ -39,6 +44,33 @@ module JobIteration
       define_callbacks :start
       define_callbacks :shutdown
       define_callbacks :complete
+
+      class_attribute(
+        :job_iteration_max_job_runtime,
+        instance_writer: false,
+        instance_predicate: false,
+        default: JobIteration.max_job_runtime,
+      )
+
+      singleton_class.prepend(PrependedClassMethods)
+    end
+
+    module PrependedClassMethods
+      def job_iteration_max_job_runtime=(new)
+        existing = job_iteration_max_job_runtime
+
+        if existing && (!new || new > existing)
+          existing_label = existing.inspect
+          new_label = new ? new.inspect : "#{new.inspect} (no limit)"
+          raise(
+            ArgumentError,
+            "job_iteration_max_job_runtime may only decrease; " \
+              "#{self} tried to increase it from #{existing_label} to #{new_label}",
+          )
+        end
+
+        super
+      end
     end
 
     module ClassMethods
@@ -68,7 +100,7 @@ module JobIteration
 
     def initialize(*arguments)
       super
-      @job_iteration_retry_backoff = nil
+      @job_iteration_retry_backoff = JobIteration.default_retry_backoff
       @needs_reenqueue = false
       self.times_interrupted = 0
       self.total_time = 0.0
@@ -87,12 +119,14 @@ module JobIteration
     def deserialize(job_data) # @private
       super
       self.cursor_position = job_data["cursor_position"]
-      self.times_interrupted = job_data["times_interrupted"] || 0
-      self.total_time = job_data["total_time"] || 0
+      self.times_interrupted = Integer(job_data["times_interrupted"] || 0)
+      self.total_time = Float(job_data["total_time"] || 0.0)
     end
 
     def perform(*params) # @private
       interruptible_perform(*params)
+
+      nil
     end
 
     def retry_job(*, **)
@@ -110,13 +144,12 @@ module JobIteration
       self.start_time = Time.now.utc
 
       enumerator = nil
-      ActiveSupport::Notifications.instrument("build_enumerator.iteration", iteration_instrumentation_tags) do
+      ActiveSupport::Notifications.instrument("build_enumerator.iteration", instrumentation_tags) do
         enumerator = build_enumerator(*arguments, cursor: cursor_position)
       end
 
       unless enumerator
-        logger.info("[JobIteration::Iteration] `build_enumerator` returned nil. " \
-          "Skipping the job.")
+        ActiveSupport::Notifications.instrument("nil_enumerator.iteration", instrumentation_tags)
         return
       end
 
@@ -125,7 +158,10 @@ module JobIteration
       if executions == 1 && times_interrupted == 0
         run_callbacks(:start)
       else
-        ActiveSupport::Notifications.instrument("resumed.iteration", iteration_instrumentation_tags)
+        ActiveSupport::Notifications.instrument(
+          "resumed.iteration",
+          instrumentation_tags.merge(times_interrupted: times_interrupted, total_time: total_time),
+        )
       end
 
       completed = catch(:abort) do
@@ -139,7 +175,10 @@ module JobIteration
         reenqueue_iteration_job
       elsif completed
         run_callbacks(:complete)
-        output_interrupt_summary
+        ActiveSupport::Notifications.instrument(
+          "completed.iteration",
+          instrumentation_tags.merge(times_interrupted: times_interrupted, total_time: total_time),
+        )
       end
     end
 
@@ -148,14 +187,15 @@ module JobIteration
       found_record = false
       @needs_reenqueue = false
 
-      enumerator.each do |object_from_enumerator, index|
+      enumerator.each do |object_from_enumerator, cursor_from_enumerator|
         # Deferred until 2.0.0
-        # assert_valid_cursor!(index)
+        # assert_valid_cursor!(cursor_from_enumerator)
 
-        record_unit_of_work do
+        tags = instrumentation_tags.merge(cursor_position: cursor_from_enumerator)
+        ActiveSupport::Notifications.instrument("each_iteration.iteration", tags) do
           found_record = true
           each_iteration(object_from_enumerator, *arguments)
-          self.cursor_position = index
+          self.cursor_position = cursor_from_enumerator
         end
 
         next unless job_should_exit?
@@ -165,25 +205,19 @@ module JobIteration
         return false
       end
 
-      logger.info(
-        "[JobIteration::Iteration] Enumerator found nothing to iterate! " \
-          "times_interrupted=#{times_interrupted} cursor_position=#{cursor_position}"
+      ActiveSupport::Notifications.instrument(
+        "not_found.iteration",
+        instrumentation_tags.merge(times_interrupted: times_interrupted),
       ) unless found_record
 
-      adjust_total_time
-
       true
-    end
-
-    def record_unit_of_work(&block)
-      ActiveSupport::Notifications.instrument("each_iteration.iteration", iteration_instrumentation_tags, &block)
+    ensure
+      adjust_total_time
     end
 
     def reenqueue_iteration_job
-      ActiveSupport::Notifications.instrument("interrupted.iteration", iteration_instrumentation_tags)
-      logger.info("[JobIteration::Iteration] Interrupting and re-enqueueing the job cursor_position=#{cursor_position}")
+      ActiveSupport::Notifications.instrument("interrupted.iteration", instrumentation_tags)
 
-      adjust_total_time
       self.times_interrupted += 1
 
       self.already_in_queue = true if respond_to?(:already_in_queue=)
@@ -225,7 +259,7 @@ module JobIteration
       unless respond_to?(:each_iteration, true)
         raise(
           ArgumentError,
-          "Iteration job (#{self.class}) must implement #each_iteration method"
+          "Iteration job (#{self.class}) must implement #each_iteration method",
         )
       end
 
@@ -252,17 +286,12 @@ module JobIteration
       method.parameters
     end
 
-    def iteration_instrumentation_tags
-      { job_class: self.class.name }
-    end
-
-    def output_interrupt_summary
-      message = "[JobIteration::Iteration] Completed iterating. times_interrupted=%d total_time=%.3f"
-      logger.info(Kernel.format(message, times_interrupted, total_time))
+    def instrumentation_tags
+      { job_class: self.class.name, cursor_position: cursor_position }
     end
 
     def job_should_exit?
-      if ::JobIteration.max_job_runtime && start_time && (Time.now.utc - start_time) > ::JobIteration.max_job_runtime
+      if job_iteration_max_job_runtime && start_time && (Time.now.utc - start_time) > job_iteration_max_job_runtime
         return true
       end
 

data/lib/job-iteration/log_subscriber.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module JobIteration
+  class LogSubscriber < ActiveSupport::LogSubscriber
+    def logger
+      JobIteration.logger
+    end
+
+    def nil_enumerator(event)
+      info do
+        "[JobIteration::Iteration] `build_enumerator` returned nil. Skipping the job."
+      end
+    end
+
+    def not_found(event)
+      info do
+        "[JobIteration::Iteration] Enumerator found nothing to iterate! " \
+          "times_interrupted=#{event.payload[:times_interrupted]} cursor_position=#{event.payload[:cursor_position]}"
+      end
+    end
+
+    def interrupted(event)
+      info do
+        "[JobIteration::Iteration] Interrupting and re-enqueueing the job " \
+          "cursor_position=#{event.payload[:cursor_position]}"
+      end
+    end
+
+    def completed(event)
+      info do
+        message = "[JobIteration::Iteration] Completed iterating. times_interrupted=%d total_time=%.3f"
+        Kernel.format(message, event.payload[:times_interrupted], event.payload[:total_time])
+      end
+    end
+  end
+end
+
+JobIteration::LogSubscriber.attach_to(:iteration)

data/lib/job-iteration/nested_enumerator.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module JobIteration
+  # @private
+  class NestedEnumerator
+    def initialize(enums, cursor: nil)
+      unless enums.all?(Proc)
+        raise ArgumentError, "enums must contain only procs/lambdas"
+      end
+
+      if cursor && enums.size != cursor.size
+        raise ArgumentError, "cursor should have one object per enum"
+      end
+
+      @enums = enums
+      @cursors = cursor || Array.new(enums.size)
+    end
+
+    def each(&block)
+      return to_enum unless block_given?
+
+      iterate([], 0, &block)
+    end
+
+    private
+
+    def iterate(current_objects, index, &block)
+      enumerator = @enums[index].call(*current_objects, @cursors[index])
+
+      enumerator.each do |object_from_enumerator, cursor_from_enumerator|
+        if index == @cursors.size - 1
+          # we've reached the innermost enumerator, yield for `iterate_with_enumerator`
+          updated_cursor = @cursors.dup
+          updated_cursor[index] = cursor_from_enumerator
+          yield object_from_enumerator, updated_cursor
+        else
+          # we need to go deeper
+          next_index = index + 1
+          iterate(current_objects + [object_from_enumerator], next_index, &block)
+          # reset cursor at the index of the nested enumerator that just finished, so we don't skip items when that
+          # index is reused in the next nested iteration
+          @cursors[next_index] = nil
+          @cursors[index] = cursor_from_enumerator
+        end
+      end
+    end
+  end
+end

data/lib/job-iteration/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JobIteration
-  VERSION = "1.3.6"
+  VERSION = "1.4.1"
 end

data/lib/job-iteration.rb

@@ -1,8 +1,10 @@
 # frozen_string_literal: true
 
+require "active_job"
 require_relative "./job-iteration/version"
 require_relative "./job-iteration/enumerator_builder"
 require_relative "./job-iteration/iteration"
+require_relative "./job-iteration/log_subscriber"
 
 module JobIteration
   IntegrationLoadError = Class.new(StandardError)
@@ -11,6 +13,14 @@ module JobIteration
 
   extend self
 
+  attr_writer :logger
+
+  class << self
+    def logger
+      @logger || ActiveJob::Base.logger
+    end
+  end
+
   # Use this to _always_ interrupt the job after it's been running for more than N seconds.
   # @example
   #
@@ -18,8 +28,27 @@ module JobIteration
   #
   # This setting will make it to always interrupt a job after it's been iterating for 5 minutes.
   # Defaults to nil which means that jobs will not be interrupted except on termination signal.
+  #
+  # This setting can be further reduced (but not increased) by using the inheritable per-class
+  # job_iteration_max_job_runtime setting.
+  # @example
+  #
+  #   class MyJob < ActiveJob::Base
+  #     include JobIteration::Iteration
+  #     self.job_iteration_max_job_runtime = 1.minute
+  #     # ...
   attr_accessor :max_job_runtime
 
+  # Configures a delay duration to wait before resuming an interrupted job.
+  # @example
+  #
+  #   JobIteration.default_retry_backoff = 10.seconds
+  #
+  # Defaults to nil which means interrupted jobs will be retried immediately.
+  # This value will be ignored when an interruption is raised by a throttle enumerator,
+  # where the throttle backoff value will take precedence over this setting.
+  attr_accessor :default_retry_backoff
+
   # Used internally for hooking into job processing frameworks like Sidekiq and Resque.
   attr_accessor :interruption_adapter
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: job-iteration
 version: !ruby/object:Gem::Version
-  version: 1.3.6
+  version: 1.4.1
 platform: ruby
 authors:
 - Shopify
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-03-10 00:00:00.000000000 Z
+date: 2023-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -47,6 +47,7 @@ extra_rdoc_files: []
 files:
 - ".github/dependabot.yml"
 - ".github/workflows/ci.yml"
+- ".github/workflows/cla.yml"
 - ".gitignore"
 - ".rubocop.yml"
 - ".yardopts"
@@ -65,10 +66,12 @@ files:
 - gemfiles/rails_6_1.gemfile
 - gemfiles/rails_7_0.gemfile
 - gemfiles/rails_edge.gemfile
+- guides/argument-semantics.md
 - guides/best-practices.md
 - guides/custom-enumerator.md
 - guides/iteration-how-it-works.md
 - guides/throttling.md
+- isogun.yml
 - job-iteration.gemspec
 - lib/job-iteration.rb
 - lib/job-iteration/active_record_batch_enumerator.rb
@@ -79,10 +82,11 @@ files:
 - lib/job-iteration/integrations/resque.rb
 - lib/job-iteration/integrations/sidekiq.rb
 - lib/job-iteration/iteration.rb
+- lib/job-iteration/log_subscriber.rb
+- lib/job-iteration/nested_enumerator.rb
 - lib/job-iteration/test_helper.rb
 - lib/job-iteration/throttle_enumerator.rb
 - lib/job-iteration/version.rb
-- railgun.yml
 homepage: https://github.com/shopify/job-iteration
 licenses:
 - MIT
@@ -104,7 +108,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.20
+rubygems_version: 3.4.19
 signing_key: 
 specification_version: 4
 summary: Makes your background jobs interruptible and resumable.