maintenance_tasks 1.2.2 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 45bd2df064845d494e9bfd74589131d6b77caf345b3d5f4c852403dfa05fd5ac
-  data.tar.gz: a48816bc76477c58717f273a1318249dd638daf5b186c8b196df57d41b2e50ec
+  metadata.gz: 649f66cc11a303666134c75aad8575b81eb376e358f69312180e21a14528ce50
+  data.tar.gz: b564b46e647c467c5d93a6198ab112729623059cfe194d1115d095c8ab8d9953
 SHA512:
-  metadata.gz: d96b17402de319cef50393e133100317cbb480180e6c1041e998e24fa1ccd224a4f252832b60c068f489217c4c1ea60121e15e015559746bf6dfc967bceb4d12
-  data.tar.gz: 5fe898977ebda4b13829df10917466ab4c7474cf705600f5574b561073b9191cf3b200b36bc32d34f9477be51b80b70335e2ed759b241de62771500ff7e74ed5
+  metadata.gz: f313350c5a80fa8840b23976eb7cde8776f825bc25a9b6a8fb7ce5d6f1abe8148d547ae7b2c1368a72caf980a2a12377e33b7c325e1447de3c0a0b0686d87a8d
+  data.tar.gz: '084950e7be06503e12d437294c784d02e2f0449c76db6297c5f95fcfd40845e4ce7e3354e16fbe12dd99b82554946f16fbc3415c047c17109aab1303c6b81b9b'

data/README.md

@@ -112,6 +112,46 @@ title,content
 My Title,Hello World!
 ```
 
+### Throttling
+
+Maintenance Tasks often modify a lot of data and can be taxing on your database.
+The gem provides a throttling mechanism that can be used to throttle a Task when
+a given condition is met. If a Task is throttled, it will be interrupted and
+retried after a backoff period has passed. The default backoff is 30 seconds.
+Specify the throttle condition as a block:
+
+```ruby
+# app/tasks/maintenance/update_posts_throttled_task.rb
+module Maintenance
+  class UpdatePostsThrottledTask < MaintenanceTasks::Task
+    throttle_on(backoff: 1.minute) do
+      DatabaseStatus.unhealthy?
+    end
+
+    def collection
+      Post.all
+    end
+
+    def count
+      collection.count
+    end
+
+    def process(post)
+      post.update!(content: "New content added on #{Time.now.utc}")
+    end
+  end
+end
+```
+
+Note that it's up to you to define a throttling condition that makes sense for
+your app. Shopify implements `DatabaseStatus.healthy?` to check various MySQL
+metrics such as replication lag, DB threads, whether DB writes are available, 
+etc.
+
+Tasks can define multiple throttle conditions. Throttle conditions are inherited
+by descendants, and new conditions will be appended without impacting existing
+conditions.
+
 ### Considerations when writing Tasks
 
 MaintenanceTasks relies on the queue adapter configured for your application to

data/app/jobs/concerns/maintenance_tasks/task_job_concern.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+module MaintenanceTasks
+  # Concern that holds the behaviour of the job that runs the tasks. It is
+  # included in {TaskJob} and if MaintenanceTasks.job is overridden, it must be
+  # included in the job.
+  module TaskJobConcern
+    extend ActiveSupport::Concern
+    include JobIteration::Iteration
+
+    included do
+      before_perform(:before_perform)
+
+      on_start(:on_start)
+      on_complete(:on_complete)
+      on_shutdown(:on_shutdown)
+
+      after_perform(:after_perform)
+
+      rescue_from StandardError, with: :on_error
+    end
+
+    class_methods do
+      # Overrides ActiveJob::Exceptions.retry_on to declare it unsupported.
+      # The use of rescue_from prevents retry_on from being usable.
+      def retry_on(*, **)
+        raise NotImplementedError, "retry_on is not supported"
+      end
+    end
+
+    private
+
+    def build_enumerator(_run, cursor:)
+      cursor ||= @run.cursor
+      collection = @task.collection
+
+      collection_enum = case collection
+      when ActiveRecord::Relation
+        enumerator_builder.active_record_on_records(collection, cursor: cursor)
+      when Array
+        enumerator_builder.build_array_enumerator(collection, cursor: cursor)
+      when CSV
+        JobIteration::CsvEnumerator.new(collection).rows(cursor: cursor)
+      else
+        raise ArgumentError, "#{@task.class.name}#collection must be either "\
+          "an Active Record Relation, Array, or CSV."
+      end
+
+      @task.throttle_conditions.reduce(collection_enum) do |enum, condition|
+        enumerator_builder.build_throttle_enumerator(enum, **condition)
+      end
+    end
+
+    # Performs task iteration logic for the current input returned by the
+    # enumerator.
+    #
+    # @param input [Object] the current element from the enumerator.
+    # @param _run [Run] the current Run, passed as an argument by Job Iteration.
+    def each_iteration(input, _run)
+      throw(:abort, :skip_complete_callbacks) if @run.stopping?
+      task_iteration(input)
+      @ticker.tick
+      @run.reload_status
+    end
+
+    def task_iteration(input)
+      @task.process(input)
+    rescue => error
+      @errored_element = input
+      raise error
+    end
+
+    def before_perform
+      @run = arguments.first
+      @task = Task.named(@run.task_name).new
+      if @task.respond_to?(:csv_content=)
+        @task.csv_content = @run.csv_file.download
+      end
+      @run.job_id = job_id
+
+      @run.running! unless @run.stopping?
+
+      @ticker = Ticker.new(MaintenanceTasks.ticker_delay) do |ticks, duration|
+        @run.persist_progress(ticks, duration)
+      end
+    end
+
+    def on_start
+      @run.update!(started_at: Time.now, tick_total: @task.count)
+    end
+
+    def on_complete
+      @run.status = :succeeded
+      @run.ended_at = Time.now
+    end
+
+    def on_shutdown
+      if @run.cancelling?
+        @run.status = :cancelled
+        @run.ended_at = Time.now
+      else
+        @run.status = @run.pausing? ? :paused : :interrupted
+        @run.cursor = cursor_position
+      end
+
+      @ticker.persist
+    end
+
+    # We are reopening a private part of Job Iteration's API here, so we should
+    # ensure the method is still defined upstream. This way, in the case where
+    # the method changes upstream, we catch it at load time instead of at
+    # runtime while calling `super`.
+    unless JobIteration::Iteration
+        .private_method_defined?(:reenqueue_iteration_job)
+      error_message = <<~HEREDOC
+        JobIteration::Iteration#reenqueue_iteration_job is expected to be
+        defined. Upgrading the maintenance_tasks gem should solve this problem.
+      HEREDOC
+      raise error_message
+    end
+    def reenqueue_iteration_job(should_ignore: true)
+      super() unless should_ignore
+      @reenqueue_iteration_job = true
+    end
+
+    def after_perform
+      @run.save!
+      if defined?(@reenqueue_iteration_job) && @reenqueue_iteration_job
+        reenqueue_iteration_job(should_ignore: false)
+      end
+    end
+
+    def on_error(error)
+      @ticker.persist if defined?(@ticker)
+
+      if defined?(@run)
+        @run.persist_error(error)
+
+        task_context = {
+          task_name: @run.task_name,
+          started_at: @run.started_at,
+          ended_at: @run.ended_at,
+        }
+      else
+        task_context = {}
+      end
+      errored_element = @errored_element if defined?(@errored_element)
+      MaintenanceTasks.error_handler.call(error, task_context, errored_element)
+    end
+  end
+end

data/app/jobs/maintenance_tasks/task_job.rb

@@ -3,139 +3,6 @@
 module MaintenanceTasks
   # Base class that is inherited by the host application's task classes.
   class TaskJob < ActiveJob::Base
-    include JobIteration::Iteration
-
-    before_perform(:before_perform)
-
-    on_start(:on_start)
-    on_complete(:on_complete)
-    on_shutdown(:on_shutdown)
-
-    after_perform(:after_perform)
-
-    rescue_from StandardError, with: :on_error
-
-    class << self
-      # Overrides ActiveJob::Exceptions.retry_on to declare it unsupported.
-      # The use of rescue_from prevents retry_on from being usable.
-      def retry_on(*, **)
-        raise NotImplementedError, "retry_on is not supported"
-      end
-    end
-
-    private
-
-    def build_enumerator(_run, cursor:)
-      cursor ||= @run.cursor
-      collection = @task.collection
-
-      case collection
-      when ActiveRecord::Relation
-        enumerator_builder.active_record_on_records(collection, cursor: cursor)
-      when Array
-        enumerator_builder.build_array_enumerator(collection, cursor: cursor)
-      when CSV
-        JobIteration::CsvEnumerator.new(collection).rows(cursor: cursor)
-      else
-        raise ArgumentError, "#{@task.class.name}#collection must be either "\
-          "an Active Record Relation, Array, or CSV."
-      end
-    end
-
-    # Performs task iteration logic for the current input returned by the
-    # enumerator.
-    #
-    # @param input [Object] the current element from the enumerator.
-    # @param _run [Run] the current Run, passed as an argument by Job Iteration.
-    def each_iteration(input, _run)
-      throw(:abort, :skip_complete_callbacks) if @run.stopping?
-      task_iteration(input)
-      @ticker.tick
-      @run.reload_status
-    end
-
-    def task_iteration(input)
-      @task.process(input)
-    rescue => error
-      @errored_element = input
-      raise error
-    end
-
-    def before_perform
-      @run = arguments.first
-      @task = Task.named(@run.task_name).new
-      if @task.respond_to?(:csv_content=)
-        @task.csv_content = @run.csv_file.download
-      end
-      @run.job_id = job_id
-
-      @run.running! unless @run.stopping?
-
-      @ticker = Ticker.new(MaintenanceTasks.ticker_delay) do |ticks, duration|
-        @run.persist_progress(ticks, duration)
-      end
-    end
-
-    def on_start
-      @run.update!(started_at: Time.now, tick_total: @task.count)
-    end
-
-    def on_complete
-      @run.status = :succeeded
-      @run.ended_at = Time.now
-    end
-
-    def on_shutdown
-      if @run.cancelling?
-        @run.status = :cancelled
-        @run.ended_at = Time.now
-      else
-        @run.status = @run.pausing? ? :paused : :interrupted
-        @run.cursor = cursor_position
-      end
-
-      @ticker.persist
-    end
-
-    # We are reopening a private part of Job Iteration's API here, so we should
-    # ensure the method is still defined upstream. This way, in the case where
-    # the method changes upstream, we catch it at load time instead of at
-    # runtime while calling `super`.
-    unless private_method_defined?(:reenqueue_iteration_job)
-      error_message = <<~HEREDOC
-        JobIteration::Iteration#reenqueue_iteration_job is expected to be
-        defined. Upgrading the maintenance_tasks gem should solve this problem.
-      HEREDOC
-      raise error_message
-    end
-    def reenqueue_iteration_job(should_ignore: true)
-      super() unless should_ignore
-      @reenqueue_iteration_job = true
-    end
-
-    def after_perform
-      @run.save!
-      if defined?(@reenqueue_iteration_job) && @reenqueue_iteration_job
-        reenqueue_iteration_job(should_ignore: false)
-      end
-    end
-
-    def on_error(error)
-      @ticker.persist if defined?(@ticker)
-
-      if defined?(@run)
-        @run.persist_error(error)
-
-        task_context = {
-          task_name: @run.task_name,
-          started_at: @run.started_at,
-          ended_at: @run.ended_at,
-        }
-      else
-        task_context = {}
-      end
-      errored_element = @errored_element if defined?(@errored_element)
-      MaintenanceTasks.error_handler.call(error, task_context, errored_element)
-    end
+    include TaskJobConcern
   end
 end

data/app/tasks/maintenance_tasks/task.rb

@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-
 module MaintenanceTasks
   # Base class that is inherited by the host application's task classes.
   class Task
@@ -7,6 +6,13 @@ module MaintenanceTasks
 
     class NotFoundError < NameError; end
 
+    # The throttle conditions for a given Task. This is provided as an array of
+    # hashes, with each hash specifying two keys: throttle_condition and
+    # backoff. Note that Tasks inherit conditions from their superclasses.
+    #
+    # @api private
+    class_attribute :throttle_conditions, default: []
+
     class << self
       # Finds a Task with the given name.
       #
@@ -72,6 +78,19 @@ module MaintenanceTasks
         new.count
       end
 
+      # Add a condition under which this Task will be throttled.
+      #
+      # @param backoff [ActiveSupport::Duration] optionally, a custom backoff
+      #   can be specified. This is the time to wait before retrying the Task.
+      #   If no value is specified, it defaults to 30 seconds.
+      # @yieldreturn [Boolean] where the throttle condition is being met,
+      #   indicating that the Task should throttle.
+      def throttle_on(backoff: 30.seconds, &condition)
+        self.throttle_conditions += [
+          { throttle_on: condition, backoff: backoff },
+        ]
+      end
+
       private
 
       def load_constants

data/lib/generators/maintenance_tasks/templates/task_spec.rb.tt

@@ -4,11 +4,13 @@ require 'rails_helper'
 module <%= tasks_module %>
 <% module_namespacing do -%>
   RSpec.describe <%= class_name %>Task do
-    # describe '#process' do
-      # it 'performs a task iteration' do
-        # <%= tasks_module %>::<%= class_name %>Task.process(element)
-      # end
-    # end
+    describe "#process" do
+      subject(:process) { described_class.process(element) }
+      let(:element) { 
+        # Object to be processed in a single iteration of this task
+      }
+      pending "add some examples to (or delete) #{__FILE__}"
+    end
   end
 <% end -%>
 end

data/lib/maintenance_tasks.rb

@@ -7,46 +7,56 @@ require "active_record"
 require "job-iteration"
 require "maintenance_tasks/engine"
 
-# Force the TaskJob class to load so we can verify upstream compatibility with
-# the JobIteration gem
-require_relative "../app/jobs/maintenance_tasks/task_job"
-
 # The engine's namespace module. It provides isolation between the host
 # application's code and the engine-specific code. Top-level engine constants
 # and variables are defined under this module.
 module MaintenanceTasks
-  # The module to namespace Tasks in, as a String. Defaults to 'Maintenance'.
-  # @param [String] the tasks_module value.
+  # @!attribute tasks_module
+  #   @scope class
+  #
+  #   The module to namespace Tasks in, as a String. Defaults to 'Maintenance'.
+  #   @return [String] the name of the module.
   mattr_accessor :tasks_module, default: "Maintenance"
 
-  # Defines the job to be used to perform Tasks. This job must be either
-  # `MaintenanceTasks::TaskJob` or a class that inherits from it.
+  # @!attribute job
+  #   @scope class
   #
-  # @param [String] the name of the job class.
+  #   The name of the job to be used to perform Tasks. Defaults to
+  #   `"MaintenanceTasks::TaskJob"`. This job must be either a class that
+  #   inherits from {TaskJob} or a class that includes {TaskJobConcern}.
+  #
+  #   @return [String] the name of the job class.
   mattr_accessor :job, default: "MaintenanceTasks::TaskJob"
 
-  # After each iteration, the progress of the task may be updated. This duration
-  # in seconds limits these updates, skipping if the duration since the last
-  # update is lower than this value, except if the job is interrupted, in which
-  # case the progress will always be recorded.
+  # @!attribute ticker_delay
+  #   @scope class
+  #
+  #   The delay between updates to the tick count. After each iteration, the
+  #   progress of the Task may be updated. This duration in seconds limits
+  #   these updates, skipping if the duration since the last update is lower
+  #   than this value, except if the job is interrupted, in which case the
+  #   progress will always be recorded.
   #
-  # @param [ActiveSupport::Duration, Numeric] Duration of the delay to update
-  #   the ticker during Task iterations.
+  #   @return [ActiveSupport::Duration, Numeric] duration of the delay between
+  #     updates to the tick count during Task iterations.
   mattr_accessor :ticker_delay, default: 1.second
 
-  # Specifies which Active Storage service to use for uploading CSV file blobs.
+  # @!attribute active_storage_service
+  #   @scope class
   #
-  # @param [Symbol] the key for the storage service, as specified in the app's
-  #   config/storage.yml.
+  #   The Active Storage service to use for uploading CSV file blobs.
+  #
+  #   @return [Symbol] the key for the storage service, as specified in the
+  #     app's config/storage.yml.
   mattr_accessor :active_storage_service
 
-  # Retrieves the callback to be performed when an error occurs in the task.
+  # @private
   def self.error_handler
     return @error_handler if defined?(@error_handler)
     @error_handler = ->(_error, _task_context, _errored_element) {}
   end
 
-  # Defines a callback to be performed when an error occurs in the task.
+  # @private
   def self.error_handler=(error_handler)
     unless error_handler.arity == 3
       ActiveSupport::Deprecation.warn(
@@ -59,4 +69,12 @@ module MaintenanceTasks
     end
     @error_handler = error_handler
   end
+
+  # @!attribute error_handler
+  #   @scope class
+  #
+  #   The callback to perform when an error occurs in the Task.  See the
+  #   {file:README#label-Customizing+the+error+handler} for details.
+  #
+  #   @return [Proc] the callback to perform when an error occurs in the Task.
 end

data/lib/maintenance_tasks/engine.rb

@@ -12,6 +12,7 @@ module MaintenanceTasks
     end
 
     config.to_prepare do
+      _ = TaskJobConcern # load this for JobIteration compatibility check
       unless Rails.autoloaders.zeitwerk_enabled?
         tasks_module = MaintenanceTasks.tasks_module.underscore
         Dir["#{Rails.root}/app/tasks/#{tasks_module}/*.rb"].each do |file|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: maintenance_tasks
 version: !ruby/object:Gem::Version
-  version: 1.2.2
+  version: 1.3.0
 platform: ruby
 authors:
 - Shopify Engineering
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-04-16 00:00:00.000000000 Z
+date: 2021-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: actionpack
@@ -94,6 +94,7 @@ files:
 - app/controllers/maintenance_tasks/tasks_controller.rb
 - app/helpers/maintenance_tasks/application_helper.rb
 - app/helpers/maintenance_tasks/tasks_helper.rb
+- app/jobs/concerns/maintenance_tasks/task_job_concern.rb
 - app/jobs/maintenance_tasks/task_job.rb
 - app/models/maintenance_tasks/application_record.rb
 - app/models/maintenance_tasks/csv_collection.rb
@@ -139,10 +140,10 @@ homepage: https://github.com/Shopify/maintenance_tasks
 licenses:
 - MIT
 metadata:
-  source_code_uri: https://github.com/Shopify/maintenance_tasks/tree/v1.2.2
+  source_code_uri: https://github.com/Shopify/maintenance_tasks/tree/v1.3.0
   allowed_push_host: https://rubygems.org
 post_install_message: |-
-  Thank you for installing Maintenance Tasks 1.2.2. To complete, please run:
+  Thank you for installing Maintenance Tasks 1.3.0. To complete, please run:
 
   rails generate maintenance_tasks:install
 rdoc_options: []
@@ -159,7 +160,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.2.17
 signing_key: 
 specification_version: 4
 summary: A Rails engine for queuing and managing maintenance tasks