maintenance_tasks 1.6.0 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8affacb1fe5aa5298899bb27875132a4b299b1bd89e239b8fc234df1d00e171d
-  data.tar.gz: cd9351e3abed12d1c1bede76cea7ac1d664159983b3ece57fe2707a0de971db5
+  metadata.gz: 6019b1623a8cc72e877154f52e8437339598f0cab121189d78692e0989e86e61
+  data.tar.gz: 21563dc6dffadd2e58dd551fe2b7d71297b2da36500722f5d9fed91892b1fd6e
 SHA512:
-  metadata.gz: c0459cf005532667f4d5bbc8a1b05adc5e4aa57bccb4b4a3e9b9338589ff73043272201e5cd0a86ed0abfa38430b18ae5021ec1991a81f064444665be8de4e3f
-  data.tar.gz: d1f9b671f215a2d4a908a9a2353f02d6800511ea8d3337b4074712ec3aed7bdcd3a6b243960915a11e4ee31ca1b756d1992ce3af5a5cd27457a9016b42136842
+  metadata.gz: 93228d08b49fab144297cb44a0aa4b9be53144ff13b88c2eba384ef29fdb47b9a8caefaf94aaf1fbfbf4584f4d544396720c17d86f3bad31508fd66dfd7507b8
+  data.tar.gz: cab4cebb685fddf978eeebd62a5c16867215f849d424b9ba948beba0e93f42a5fe11ee1eccc15174c1494fa16a2fa100d3a7745c5eacef2527d425895805cf29

data/README.md

@@ -114,8 +114,9 @@ title,content
 My Title,Hello World!
 ```
 
-The files uploaded to your Active Storage service provider will be renamed first
+The files uploaded to your Active Storage service provider will be renamed
 to include an ISO8601 timestamp and the Task name in snake case format.
+The CSV is expected to have a trailing newline at the end of the file.
 
 ### Processing Batch Collections
 
@@ -200,6 +201,20 @@ Tasks can define multiple throttle conditions. Throttle conditions are inherited
 by descendants, and new conditions will be appended without impacting existing
 conditions.
 
+The backoff can also be specified as a proc:
+
+```ruby
+# app/tasks/maintenance/update_posts_throttled_task.rb
+
+module Maintenance
+  class UpdatePostsThrottledTask < MaintenanceTasks::Task
+    throttle_on(backoff: -> { RandomBackoffGenerator.generate_duration } ) do
+      DatabaseStatus.unhealthy?
+    end
+    ...
+  end
+end
+```
 ### Custom Task Parameters
 
 Tasks may need additional information, supplied via parameters, to run.
@@ -241,12 +256,12 @@ The Task provides callbacks that hook into its life cycle.
 
 Available callbacks are:
 
-`after_start`
-`after_pause`
-`after_interrupt`
-`after_cancel`
-`after_complete`
-`after_error`
+* `after_start`
+* `after_pause`
+* `after_interrupt`
+* `after_cancel`
+* `after_complete`
+* `after_error`
 
 ```ruby
 module Maintenance
@@ -542,7 +557,7 @@ you can define an error handler:
 
 MaintenanceTasks.error_handler = ->(error, task_context, _errored_element) do
   Bugsnag.notify(error) do |notification|
-    notification.add_tab(:task, task_context)
+    notification.add_metadata(:task, task_context)
   end
 end
 ```

data/app/controllers/maintenance_tasks/tasks_controller.rb

@@ -24,11 +24,12 @@ module MaintenanceTasks
     end
 
     # Runs a given Task and redirects to the Task page.
-    def run
+    def run(&block)
       task = Runner.run(
         name: params.fetch(:id),
         csv_file: params[:csv_file],
         arguments: params.fetch(:task_arguments, {}).permit!.to_h,
+        &block
       )
       redirect_to(task_path(task))
     rescue ActiveRecord::RecordInvalid => error

data/app/helpers/maintenance_tasks/tasks_helper.rb

@@ -100,5 +100,24 @@ module MaintenanceTasks
         only_path: true
       )
     end
+
+    # Return the appropriate field tag for the parameter
+    def parameter_field(form_builder, parameter_name)
+      case form_builder.object.class.attribute_types[parameter_name]
+      when ActiveModel::Type::Integer, ActiveModel::Type::Decimal,
+           ActiveModel::Type::Float
+        form_builder.number_field(parameter_name)
+      when ActiveModel::Type::DateTime
+        form_builder.datetime_field(parameter_name)
+      when ActiveModel::Type::Date
+        form_builder.date_field(parameter_name)
+      when ActiveModel::Type::Time
+        form_builder.time_field(parameter_name)
+      when ActiveModel::Type::Boolean
+        form_builder.check_box(parameter_name)
+      else
+        form_builder.text_area(parameter_name, class: "textarea")
+      end
+    end
   end
 end

data/app/jobs/concerns/maintenance_tasks/task_job_concern.rb

@@ -12,8 +12,8 @@ module MaintenanceTasks
       before_perform(:before_perform)
 
       on_start(:on_start)
-      on_complete(:on_complete)
       on_shutdown(:on_shutdown)
+      on_complete(:on_complete)
 
       after_perform(:after_perform)
 
@@ -63,9 +63,16 @@ module MaintenanceTasks
           Array, or CSV.
         MSG
       end
+      throttle_enumerator(collection_enum)
+    end
 
+    def throttle_enumerator(collection_enum)
       @task.throttle_conditions.reduce(collection_enum) do |enum, condition|
-        enumerator_builder.build_throttle_enumerator(enum, **condition)
+        enumerator_builder.build_throttle_enumerator(
+          enum,
+          throttle_on: condition[:throttle_on],
+          backoff: condition[:backoff].call
+        )
       end
     end
 
@@ -105,33 +112,19 @@ module MaintenanceTasks
     def on_start
       count = @task.count
       count = @enumerator&.size if count == :no_count
-      @run.update!(started_at: Time.now, tick_total: count)
-      @task.run_callbacks(:start)
-    end
-
-    def on_complete
-      @run.status = :succeeded
-      @run.ended_at = Time.now
-      @task.run_callbacks(:complete)
+      @run.start(count)
     end
 
     def on_shutdown
-      if @run.cancelling?
-        @run.status = :cancelled
-        @task.run_callbacks(:cancel)
-        @run.ended_at = Time.now
-      elsif @run.pausing?
-        @run.status = :paused
-        @task.run_callbacks(:pause)
-      else
-        @run.status = :interrupted
-        @task.run_callbacks(:interrupt)
-      end
-
+      @run.job_shutdown
       @run.cursor = cursor_position
       @ticker.persist
     end
 
+    def on_complete
+      @run.complete
+    end
+
     # We are reopening a private part of Job Iteration's API here, so we should
     # ensure the method is still defined upstream. This way, in the case where
     # the method changes upstream, we catch it at load time instead of at
@@ -150,7 +143,7 @@ module MaintenanceTasks
     end
 
     def after_perform
-      @run.save!
+      @run.persist_transition
       if defined?(@reenqueue_iteration_job) && @reenqueue_iteration_job
         reenqueue_iteration_job(should_ignore: false)
       end
@@ -171,15 +164,8 @@ module MaintenanceTasks
         task_context = {}
       end
       errored_element = @errored_element if defined?(@errored_element)
-      run_error_callback
     ensure
       MaintenanceTasks.error_handler.call(error, task_context, errored_element)
     end
-
-    def run_error_callback
-      @task.run_callbacks(:error) if defined?(@task)
-    rescue
-      nil
-    end
   end
 end

data/app/models/maintenance_tasks/csv_collection_builder.rb

@@ -15,9 +15,10 @@ module MaintenanceTasks
       CSV.new(task.csv_content, headers: true)
     end
 
-    # The number of rows to be processed. Excludes the header row from the count
-    # and assumed a trailing new line in the CSV file. Note that this number is
-    # an approximation based on the number of new lines.
+    # The number of rows to be processed. Excludes the header row from the
+    # count and assumes a trailing newline is at the end of the CSV file.
+    # Note that this number is an approximation based on the number of
+    # newlines.
     #
     # @return [Integer] the approximate number of rows to process.
     def count(task)

data/app/models/maintenance_tasks/run.rb

@@ -66,9 +66,40 @@ module MaintenanceTasks
 
     # Sets the run status to enqueued, making sure the transition is validated
     # in case it's already enqueued.
+    #
+    # Rescues and retries status transition if an ActiveRecord::StaleObjectError
+    # is encountered.
     def enqueued!
       status_will_change!
       super
+    rescue ActiveRecord::StaleObjectError
+      reload_status
+      retry
+    end
+
+    CALLBACKS_TRANSITION = {
+      cancelled: :cancel,
+      interrupted: :interrupt,
+      paused: :pause,
+      succeeded: :complete,
+    }.transform_keys(&:to_s)
+    private_constant :CALLBACKS_TRANSITION
+
+    # Saves the run, persisting the transition of its status, and all other
+    # changes to the object.
+    def persist_transition
+      save!
+      callback = CALLBACKS_TRANSITION[status]
+      run_task_callbacks(callback) if callback
+    rescue ActiveRecord::StaleObjectError
+      success = succeeded?
+      reload_status
+      if success
+        self.status = :succeeded
+      else
+        job_shutdown
+      end
+      retry
     end
 
     # Increments +tick_count+ by +number_of_ticks+ and +time_running+ by
@@ -86,6 +117,11 @@ module MaintenanceTasks
         time_running: duration,
         touch: true
       )
+      if locking_enabled?
+        locking_column = self.class.locking_column
+        self[locking_column] += 1
+        clear_attribute_change(locking_column)
+      end
     end
 
     # Marks the run as errored and persists the error data.
@@ -100,20 +136,34 @@ module MaintenanceTasks
         backtrace: MaintenanceTasks.backtrace_cleaner.clean(error.backtrace),
         ended_at: Time.now,
       )
+      run_task_callbacks(:error)
+    rescue ActiveRecord::StaleObjectError
+      reload_status
+      retry
     end
 
-    # Refreshes just the status attribute on the Active Record object, and
-    # ensures ActiveModel::Dirty does not mark the object as changed.
+    # Refreshes the status and lock version attributes on the Active Record
+    # object, and ensures ActiveModel::Dirty doesn't mark the object as changed.
+    #
     # This allows us to get the Run's most up-to-date status without needing
     # to reload the entire record.
     #
     # @return [MaintenanceTasks::Run] the Run record with its updated status.
     def reload_status
-      updated_status = self.class.uncached do
-        self.class.where(id: id).pluck(:status).first
+      columns_to_reload = if locking_enabled?
+        [:status, self.class.locking_column]
+      else
+        [:status]
+      end
+      updated_status, updated_lock_version = self.class.uncached do
+        self.class.where(id: id).pluck(*columns_to_reload).first
       end
+
       self.status = updated_status
-      clear_attribute_changes([:status])
+      if updated_lock_version
+        self[self.class.locking_column] = updated_lock_version
+      end
+      clear_attribute_changes(columns_to_reload)
       self
     end
 
@@ -173,21 +223,65 @@ module MaintenanceTasks
       seconds_to_finished.seconds
     end
 
-    # Mark a Run as running.
+    # Marks a Run as running.
     #
     # If the run is stopping already, it will not transition to running.
+    # Rescues and retries status transition if an ActiveRecord::StaleObjectError
+    # is encountered.
     def running
-      return if stopping?
-      updated = self.class.where(id: id).where.not(status: STOPPING_STATUSES)
-        .update_all(status: :running, updated_at: Time.now) > 0
-      if updated
-        self.status = :running
-        clear_attribute_changes([:status])
+      if locking_enabled?
+        begin
+          running! unless stopping?
+        rescue ActiveRecord::StaleObjectError
+          reload_status
+          retry
+        end
+      else
+        # Preserve swap-and-replace solution for data races until users
+        # run migration to upgrade to optimistic locking solution
+        return if stopping?
+        updated = self.class.where(id: id).where.not(status: STOPPING_STATUSES)
+          .update_all(status: :running, updated_at: Time.now) > 0
+        if updated
+          self.status = :running
+          clear_attribute_changes([:status])
+        else
+          reload_status
+        end
+      end
+    end
+
+    # Starts a Run, setting its started_at timestamp and tick_total.
+    #
+    # @param count [Integer] the total iterations to be performed, as
+    #   specified by the Task.
+    def start(count)
+      update!(started_at: Time.now, tick_total: count)
+      run_task_callbacks(:start)
+    rescue ActiveRecord::StaleObjectError
+      reload_status
+      retry
+    end
+
+    # Handles transitioning the status on a Run when the job shuts down.
+    def job_shutdown
+      if cancelling?
+        self.status = :cancelled
+        self.ended_at = Time.now
+      elsif pausing?
+        self.status = :paused
       else
-        reload_status
+        self.status = :interrupted
       end
     end
 
+    # Handles the completion of a Run, setting a status of succeeded and the
+    # ended_at timestamp.
+    def complete
+      self.status = :succeeded
+      self.ended_at = Time.now
+    end
+
     # Cancels a Run.
     #
     # If the Run is paused, it will transition directly to cancelled, since the
@@ -201,10 +295,26 @@ module MaintenanceTasks
     # will be updated.
     def cancel
       if paused? || stuck?
-        update!(status: :cancelled, ended_at: Time.now)
+        self.status = :cancelled
+        self.ended_at = Time.now
+        persist_transition
       else
         cancelling!
       end
+    rescue ActiveRecord::StaleObjectError
+      reload_status
+      retry
+    end
+
+    # Marks a Run as pausing.
+    #
+    # Rescues and retries status transition if an ActiveRecord::StaleObjectError
+    # is encountered.
+    def pausing!
+      super
+    rescue ActiveRecord::StaleObjectError
+      reload_status
+      retry
     end
 
     # Returns whether a Run is stuck, which is defined as having a status of
@@ -297,6 +407,12 @@ module MaintenanceTasks
 
     private
 
+    def run_task_callbacks(callback)
+      task.run_callbacks(callback)
+    rescue
+      nil
+    end
+
     def arguments_match_task_attributes
       invalid_argument_keys = arguments.keys - task.attribute_names
       if invalid_argument_keys.any?

data/app/models/maintenance_tasks/task.rb

@@ -59,7 +59,8 @@ module MaintenanceTasks
             "To resolve this issue run: bin/rails active_storage:install"
         end
 
-        self.collection_builder_strategy = CsvCollectionBuilder.new
+        self.collection_builder_strategy =
+          MaintenanceTasks::CsvCollectionBuilder.new
       end
 
       # Returns whether the Task handles CSV.
@@ -98,14 +99,19 @@ module MaintenanceTasks
 
       # Add a condition under which this Task will be throttled.
       #
-      # @param backoff [ActiveSupport::Duration] optionally, a custom backoff
-      #   can be specified. This is the time to wait before retrying the Task.
-      #   If no value is specified, it defaults to 30 seconds.
+      # @param backoff [ActiveSupport::Duration, #call] a custom backoff
+      #   can be specified. This is the time to wait before retrying the Task,
+      #   defaulting to 30 seconds. If provided as a Duration, the backoff is
+      #   wrapped in a proc. Alternatively,an object responding to call can be
+      #   used. It must return an ActiveSupport::Duration.
       # @yieldreturn [Boolean] where the throttle condition is being met,
       #   indicating that the Task should throttle.
       def throttle_on(backoff: 30.seconds, &condition)
+        backoff_as_proc = backoff
+        backoff_as_proc = -> { backoff } unless backoff.respond_to?(:call)
+
         self.throttle_conditions += [
-          { throttle_on: condition, backoff: backoff },
+          { throttle_on: condition, backoff: backoff_as_proc },
         ]
       end
 

data/app/models/maintenance_tasks/task_data.rb

@@ -74,7 +74,11 @@ module MaintenanceTasks
     def code
       return if deleted?
       task = Task.named(name)
-      file = task.instance_method(:process).source_location.first
+      file = if Object.respond_to?(:const_source_location)
+        Object.const_source_location(task.name).first
+      else
+        task.instance_method(:process).source_location.first
+      end
       File.read(file)
     end
 
@@ -142,6 +146,13 @@ module MaintenanceTasks
       end
     end
 
+    # @return [MaintenanceTasks::Task, nil] an instance of the Task class.
+    # @return [nil] if the Task file was deleted.
+    def new
+      return if deleted?
+      MaintenanceTasks::Task.named(name).new
+    end
+
     private
 
     def runs

data/app/views/maintenance_tasks/runs/_run.html.erb

@@ -10,6 +10,10 @@
     <%= render "maintenance_tasks/runs/info/#{run.status}", run: run %>
   </div>
 
+  <div class="content" id="custom-content">
+    <%= render "maintenance_tasks/runs/info/custom", run: run %>
+  </div>
+
   <%= render "maintenance_tasks/runs/csv", run: run %>
   <%= tag.hr if run.csv_file.present? && run.arguments.present? %>
   <%= render "maintenance_tasks/runs/arguments", run: run %>

data/app/views/maintenance_tasks/tasks/_task.html.erb

@@ -15,6 +15,10 @@
       <%= render "maintenance_tasks/runs/info/#{run.status}", run: run %>
     </div>
 
+    <div class="content" id="custom-content">
+      <%= render "maintenance_tasks/runs/info/custom", run: run %>
+    </div>
+
     <%= render "maintenance_tasks/runs/csv", run: run %>
     <%= tag.hr if run.csv_file.present? && run.arguments.present? %>
     <%= render "maintenance_tasks/runs/arguments", run: run %>

data/app/views/maintenance_tasks/tasks/show.html.erb

@@ -16,6 +16,10 @@
     <%= render "maintenance_tasks/runs/info/#{last_run.status}", run: last_run %>
   </div>
 
+  <div class="content" id="custom-content">
+    <%= render "maintenance_tasks/runs/info/custom", run: last_run %>
+  </div>
+
   <%= render "maintenance_tasks/runs/csv", run: last_run %>
   <%= tag.hr if last_run.csv_file.present? %>
   <%= render "maintenance_tasks/runs/arguments", run: last_run %>
@@ -31,20 +35,22 @@
           <%= form.file_field :csv_file %>
         </div>
       <% end %>
-      <% if @task.parameter_names.any? %>
+      <% parameter_names = @task.parameter_names %>
+      <% if parameter_names.any? %>
         <div class="block">
-          <%= form.fields_for :task_arguments do |ff| %>
-            <% @task.parameter_names.each do |parameter| %>
+          <%= form.fields_for :task_arguments, @task.new do |ff| %>
+            <% parameter_names.each do |parameter_name| %>
               <div class="field">
-                <%= ff.label parameter, parameter, class: "label is-family-monospace" %>
+                <%= ff.label parameter_name, parameter_name, class: "label is-family-monospace" %>
                 <div class="control">
-                  <%= ff.text_area parameter, class: "textarea" %>
+                  <%= parameter_field(ff, parameter_name) %>
                 </div>
               </div>
             <% end %>
           <% end %>
         </div>
       <% end %>
+      <%= render "maintenance_tasks/tasks/custom", form: form %>
       <div class="block">
         <%= form.submit 'Run', class: "button is-success", disabled: @task.deleted? %>
       </div>

data/db/migrate/20211210152329_add_lock_version_to_maintenance_tasks_runs.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class AddLockVersionToMaintenanceTasksRuns < ActiveRecord::Migration[6.0]
+  def change
+    add_column(:maintenance_tasks_runs, :lock_version, :integer,
+      default: 0, null: false)
+  end
+end

data/lib/maintenance_tasks/engine.rb

@@ -8,7 +8,16 @@ module MaintenanceTasks
   class Engine < ::Rails::Engine
     isolate_namespace MaintenanceTasks
 
-    initializer "eager_load_for_classic_autoloader" do
+    initializer "maintenance_tasks.warn_classic_autoloader" do
+      unless Rails.autoloaders.zeitwerk_enabled?
+        ActiveSupport::Deprecation.warn(<<~MSG.squish)
+          Autoloading in classic mode is deprecated and support will be removed in the next
+          release of Maintenance Tasks. Please use Zeitwerk to autoload your application.
+        MSG
+      end
+    end
+
+    initializer "maintenance_tasks.eager_load_for_classic_autoloader" do
       eager_load! unless Rails.autoloaders.zeitwerk_enabled?
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: maintenance_tasks
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 1.7.0
 platform: ruby
 authors:
 - Shopify Engineering
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-11-05 00:00:00.000000000 Z
+date: 2022-01-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: actionpack
@@ -114,6 +114,7 @@ files:
 - app/views/maintenance_tasks/runs/_run.html.erb
 - app/views/maintenance_tasks/runs/info/_cancelled.html.erb
 - app/views/maintenance_tasks/runs/info/_cancelling.html.erb
+- app/views/maintenance_tasks/runs/info/_custom.html.erb
 - app/views/maintenance_tasks/runs/info/_enqueued.html.erb
 - app/views/maintenance_tasks/runs/info/_errored.html.erb
 - app/views/maintenance_tasks/runs/info/_interrupted.html.erb
@@ -121,6 +122,7 @@ files:
 - app/views/maintenance_tasks/runs/info/_pausing.html.erb
 - app/views/maintenance_tasks/runs/info/_running.html.erb
 - app/views/maintenance_tasks/runs/info/_succeeded.html.erb
+- app/views/maintenance_tasks/tasks/_custom.html.erb
 - app/views/maintenance_tasks/tasks/_task.html.erb
 - app/views/maintenance_tasks/tasks/index.html.erb
 - app/views/maintenance_tasks/tasks/show.html.erb
@@ -128,6 +130,7 @@ files:
 - db/migrate/20201211151756_create_maintenance_tasks_runs.rb
 - db/migrate/20210225152418_remove_index_on_task_name.rb
 - db/migrate/20210517131953_add_arguments_to_maintenance_tasks_runs.rb
+- db/migrate/20211210152329_add_lock_version_to_maintenance_tasks_runs.rb
 - exe/maintenance_tasks
 - lib/generators/maintenance_tasks/install_generator.rb
 - lib/generators/maintenance_tasks/task_generator.rb
@@ -144,7 +147,7 @@ homepage: https://github.com/Shopify/maintenance_tasks
 licenses:
 - MIT
 metadata:
-  source_code_uri: https://github.com/Shopify/maintenance_tasks/tree/v1.6.0
+  source_code_uri: https://github.com/Shopify/maintenance_tasks/tree/v1.7.0
   allowed_push_host: https://rubygems.org
 post_install_message: 
 rdoc_options: []