job-iteration 0.9.0

data/lib/job-iteration/integrations/resque.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'resque'
+
+module JobIteration
+  module Integrations
+    # The trick is required in order to call shutdown? on a Resque::Worker instance
+    module ResqueIterationExtension
+      def initialize(*)
+        $resque_worker = self
+        super
+      end
+    end
+    Resque::Worker.prepend(ResqueIterationExtension)
+
+    module ResqueInterruptionAdapter
+      extend self
+
+      def shutdown?
+        $resque_worker.try!(:shutdown?)
+      end
+    end
+
+    JobIteration.interruption_adapter = ResqueInterruptionAdapter
+  end
+end

data/lib/job-iteration/integrations/sidekiq.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'sidekiq'
+
+module JobIteration
+  module Integrations
+    module SidekiqInterruptionAdapter
+      extend self
+
+      def shutdown?
+        if defined?(Sidekiq::CLI) && Sidekiq::CLI.instance
+          Sidekiq::CLI.instance.launcher.stopping?
+        else
+          false
+        end
+      end
+    end
+
+    JobIteration.interruption_adapter = SidekiqInterruptionAdapter
+  end
+end

data/lib/job-iteration/iteration.rb

@@ -0,0 +1,204 @@
+# frozen_string_literal: true
+
+require 'active_support/all'
+
+module JobIteration
+  module Iteration
+    extend ActiveSupport::Concern
+
+    included do |_base|
+      attr_accessor(
+        :cursor_position,
+        :start_time,
+        :times_interrupted,
+        :total_time,
+      )
+
+      define_callbacks :start
+      define_callbacks :shutdown
+      define_callbacks :complete
+    end
+
+    module ClassMethods
+      def method_added(method_name)
+        ban_perform_definition if method_name.to_sym == :perform
+      end
+
+      def on_start(*filters, &blk)
+        set_callback(:start, :after, *filters, &blk)
+      end
+
+      def on_shutdown(*filters, &blk)
+        set_callback(:shutdown, :after, *filters, &blk)
+      end
+
+      def on_complete(*filters, &blk)
+        set_callback(:complete, :after, *filters, &blk)
+      end
+
+      def supports_interruption?
+        true
+      end
+
+      private
+
+      def ban_perform_definition
+        raise "Job that is using Iteration (#{self}) cannot redefine #perform"
+      end
+    end
+
+    def initialize(*arguments)
+      super
+      self.times_interrupted = 0
+      self.total_time = 0.0
+    end
+
+    def serialize
+      super.merge(
+        'cursor_position' => cursor_position,
+        'times_interrupted' => times_interrupted,
+        'total_time' => total_time,
+      )
+    end
+
+    def deserialize(job_data)
+      super
+      self.cursor_position = job_data['cursor_position']
+      self.times_interrupted = job_data['times_interrupted'] || 0
+      self.total_time = job_data['total_time'] || 0
+    end
+
+    def perform(*params)
+      interruptible_perform(*params)
+    end
+
+    private
+
+    def enumerator_builder
+      JobIteration::EnumeratorBuilder.new(self)
+    end
+
+    def interruptible_perform(*arguments)
+      assert_implements_methods!
+
+      self.start_time = Time.now.utc
+
+      enumerator = nil
+      ActiveSupport::Notifications.instrument("build_enumerator.iteration", iteration_instrumentation_tags) do
+        enumerator = build_enumerator(*arguments, cursor: cursor_position)
+      end
+
+      unless enumerator
+        logger.info "[JobIteration::Iteration] `build_enumerator` returned nil. " \
+          "Skipping the job."
+        return
+      end
+
+      assert_enumerator!(enumerator)
+
+      if executions == 1 && times_interrupted == 0
+        run_callbacks :start
+      else
+        ActiveSupport::Notifications.instrument("resumed.iteration", iteration_instrumentation_tags)
+      end
+
+      catch(:abort) do
+        return unless iterate_with_enumerator(enumerator, arguments)
+      end
+
+      run_callbacks :shutdown
+      run_callbacks :complete
+
+      output_interrupt_summary
+    end
+
+    def iterate_with_enumerator(enumerator, arguments)
+      arguments = arguments.dup.freeze
+      enumerator.each do |iteration, index|
+        record_unit_of_work do
+          each_iteration(iteration, *arguments)
+          self.cursor_position = index
+        end
+
+        next unless job_should_exit?
+        self.executions -= 1 if executions > 1
+        shutdown_and_reenqueue
+        return false
+      end
+
+      true
+    end
+
+    def record_unit_of_work
+      ActiveSupport::Notifications.instrument("each_iteration.iteration", iteration_instrumentation_tags) do
+        yield
+      end
+    end
+
+    def shutdown_and_reenqueue
+      ActiveSupport::Notifications.instrument("interrupted.iteration", iteration_instrumentation_tags)
+      logger.info "[JobIteration::Iteration] Interrupting and re-enqueueing the job cursor_position=#{cursor_position}"
+
+      adjust_total_time
+      self.times_interrupted += 1
+
+      self.already_in_queue = true if respond_to?(:already_in_queue=)
+      run_callbacks :shutdown
+      enqueue
+
+      true
+    end
+
+    def adjust_total_time
+      self.total_time += (Time.now.utc.to_f - start_time.to_f).round(6)
+    end
+
+    def assert_enumerator!(enum)
+      return if enum.is_a?(Enumerator)
+
+      raise ArgumentError, <<~EOS
+        #build_enumerator is expected to return Enumerator object, but returned #{enum.class}.
+        Example:
+           def build_enumerator(params, cursor:)
+            enumerator_builder.active_record_on_records(
+              Shop.find(params[:shop_id]).products,
+              cursor: cursor
+            )
+          end
+      EOS
+    end
+
+    def assert_implements_methods!
+      unless respond_to?(:each_iteration, true)
+        raise(
+          ArgumentError,
+          "Iteration job (#{self.class}) must implement #each_iteration method"
+        )
+      end
+
+      unless respond_to?(:build_enumerator, true)
+        raise ArgumentError, "Iteration job (#{self.class}) must implement #build_enumerator " \
+          "to provide a collection to iterate"
+      end
+    end
+
+    def iteration_instrumentation_tags
+      { job_class: self.class.name }
+    end
+
+    def output_interrupt_summary
+      adjust_total_time
+
+      message = "[JobIteration::Iteration] Completed iterating. times_interrupted=%d total_time=%.3f"
+      logger.info Kernel.format(message, times_interrupted, total_time)
+    end
+
+    def job_should_exit?
+      if ::JobIteration.max_job_runtime && start_time && (Time.now.utc - start_time) > ::JobIteration.max_job_runtime
+        return true
+      end
+
+      JobIteration.interruption_adapter.shutdown?
+    end
+  end
+end

data/lib/job-iteration/test_helper.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module JobIteration
+  module TestHelper
+    class StoppingSupervisor
+      def initialize(stop_after_count)
+        @stop_after_count = stop_after_count
+        @calls = 0
+      end
+
+      def shutdown?
+        @calls += 1
+        (@calls % @stop_after_count) == 0
+      end
+    end
+
+    private
+
+    def iterate_exact_times(n_times)
+      JobIteration.stubs(:interruption_adapter).returns(StoppingSupervisor.new(n_times.size))
+    end
+
+    def iterate_once
+      iterate_exact_times(1.times)
+    end
+
+    def continue_iterating
+      stub_shutdown_adapter_to_return(false)
+    end
+
+    def mark_job_worker_as_interrupted
+      stub_shutdown_adapter_to_return(true)
+    end
+
+    def stub_shutdown_adapter_to_return(_value)
+      adapter = mock
+      adapter.stubs(:shutdown?).returns(false)
+      JobIteration.stubs(:interruption_adapter).returns(adapter)
+    end
+  end
+end

data/lib/job-iteration/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module JobIteration
+  VERSION = "0.9.0"
+end

metadata

@@ -0,0 +1,122 @@
+--- !ruby/object:Gem::Specification
+name: job-iteration
+version: !ruby/object:Gem::Version
+  version: 0.9.0
+platform: ruby
+authors:
+- Shopify
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-07-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activejob
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+description: Makes your background jobs interruptible and resumable.
+email:
+- ops-accounts+shipit@shopify.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rubocop.yml"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- guides/best-practices.md
+- guides/iteration-how-it-works.md
+- job-iteration.gemspec
+- lib/job-iteration.rb
+- lib/job-iteration/active_record_cursor.rb
+- lib/job-iteration/active_record_enumerator.rb
+- lib/job-iteration/csv_enumerator.rb
+- lib/job-iteration/enumerator_builder.rb
+- lib/job-iteration/integrations/resque.rb
+- lib/job-iteration/integrations/sidekiq.rb
+- lib/job-iteration/iteration.rb
+- lib/job-iteration/test_helper.rb
+- lib/job-iteration/version.rb
+homepage: https://github.com/shopify/job-iteration
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14
+signing_key: 
+specification_version: 4
+summary: Makes your background jobs interruptible and resumable.
+test_files: []