maintenance_tasks 2.4.0 → 2.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1bca60024506654676fc6c094e6c4a4a1af4c87fc3a80c81f5a7be58a73b6061
-  data.tar.gz: 351ba1281e24013a2cbdd43302498769d398e38f0a695850d988d1d4aa4987bb
+  metadata.gz: f26c6afe0c4b45b6d191162ab2c37e3b058e8b6ae473b17cfc3724e29e638fce
+  data.tar.gz: 4308f3356279fa5879429077cb482d91033336bfa62c1c4afb51fc9d957b1785
 SHA512:
-  metadata.gz: c4cb3cfe2d3ddeb5f614976f6b2c04ce655e3b4acdf739ea133e5a303a224e170e3f6b9ea0aa58c32b71e72f30017524c172750dd86308a29bcd46d309cb45ed
-  data.tar.gz: 1c990f1658545d4bd5e622598c5ce7a08f0dbdda9e1d8c9e0bd56c46d491fe5a440ce53100c976d196b7cc2535a839e16253a278d6e9e1ae1dc6a1121e25848e
+  metadata.gz: 383bc985f1a465e00889b20997d5718ff2039d5cdc6ca6b14a427a183fef4df3ff445eda2893d10b41d0bf243a6150ad5699afdfcbe3beaf280e4d35f130c1f5
+  data.tar.gz: d2b54c567a3d89e78f3538ca22b35d83dcb150e7520fc3014aa2040836f2e2741eb2bda700211c6411b8355b54ee19814655cf79827358d2d2f922fa855934af

data/README.md

@@ -299,6 +299,35 @@ module Maintenance
 end
 ```
 
+### Tasks with Custom Enumerators
+
+If you have a special use case requiring iteration over an unsupported
+collection type, such as external resources fetched from some API, you can
+implement the `enumerator_builder(cursor:)` method in your task.
+
+This method should return an `Enumerator`, yielding pairs of
+`[item, cursor]`. Maintenance Tasks takes care of persisting the current
+cursor position and will provide it as the `cursor` argument if your task is
+interrupted or resumed. The `cursor` is stored as a `String`, so your custom
+enumerator should handle serializing/deserializing the value if required.
+
+```ruby
+# app/tasks/maintenance/custom_enumerator_task.rb
+
+module Maintenance
+  class CustomEnumeratorTask < MaintenanceTasks::Task
+    def enumerator_builder(cursor:)
+      after_id = cursor&.to_i
+      PostAPI.index(after_id: after_id).map { |post| [post, post.id] }.to_enum
+    end
+
+    def process(post)
+      Post.create!(post)
+    end
+  end
+end
+```
+
 ### Throttling
 
 Maintenance tasks often modify a lot of data and can be taxing on your database.
@@ -897,6 +926,15 @@ controller class which **must inherit** from `ActionController::Base`.
 
 If no value is specified, it will default to `"ActionController::Base"`.
 
+#### Configure time after which the task will be considered stuck
+
+To specify a time duration after which a task is considered stuck if it has not been updated,
+you can configure `MaintenanceTasks.stuck_task_duration`. This duration should account for
+job infrastructure events that may prevent the maintenance tasks job from being executed and cancelling the task.
+
+The value for `MaintenanceTasks.stuck_task_duration` must be an `ActiveSupport::Duration`.
+If no value is specified, it will default to 5 minutes.
+
 ### Metadata
 
 `MaintenanceTasks.metadata` can be configured to specify a proc from which to

data/app/helpers/maintenance_tasks/tasks_helper.rb

@@ -101,8 +101,17 @@ module MaintenanceTasks
       )
     end
 
-    # Return the appropriate field tag for the parameter
+    # Return the appropriate field tag for the parameter, based on its type.
+    # If the parameter has a `validates_inclusion_of` validator, return a dropdown list of options instead.
     def parameter_field(form_builder, parameter_name)
+      inclusion_validator = form_builder.object.class.validators_on(parameter_name).find do |validator|
+        validator.kind == :inclusion
+      end
+
+      return form_builder.select(
+        parameter_name, inclusion_validator.options[:in], prompt: "Select a value"
+      ) if inclusion_validator
+
       case form_builder.object.class.attribute_types[parameter_name]
       when ActiveModel::Type::Integer
         form_builder.number_field(parameter_name)

data/app/jobs/concerns/maintenance_tasks/task_job_concern.rb

@@ -32,45 +32,7 @@ module MaintenanceTasks
 
     def build_enumerator(_run, cursor:)
       cursor ||= @run.cursor
-      collection = @task.collection
-      @enumerator = nil
-
-      @collection_enum = case collection
-      when :no_collection
-        enumerator_builder.build_once_enumerator(cursor: nil)
-      when ActiveRecord::Relation
-        enumerator_builder.active_record_on_records(collection, cursor: cursor)
-      when ActiveRecord::Batches::BatchEnumerator
-        if collection.start || collection.finish
-          raise ArgumentError, <<~MSG.squish
-            #{@task.class.name}#collection cannot support
-            a batch enumerator with the "start" or "finish" options.
-          MSG
-        end
-
-        # For now, only support automatic count based on the enumerator for
-        # batches
-        enumerator_builder.active_record_on_batch_relations(
-          collection.relation,
-          cursor: cursor,
-          batch_size: collection.batch_size,
-        )
-      when Array
-        enumerator_builder.build_array_enumerator(collection, cursor: cursor&.to_i)
-      when BatchCsvCollectionBuilder::BatchCsv
-        JobIteration::CsvEnumerator.new(collection.csv).batches(
-          batch_size: collection.batch_size,
-          cursor: cursor&.to_i,
-        )
-      when CSV
-        JobIteration::CsvEnumerator.new(collection).rows(cursor: cursor&.to_i)
-      else
-        raise ArgumentError, <<~MSG.squish
-          #{@task.class.name}#collection must be either an
-          Active Record Relation, ActiveRecord::Batches::BatchEnumerator,
-          Array, or CSV.
-        MSG
-      end
+      @collection_enum = @task.enumerator_builder(cursor: cursor)
       throttle_enumerator(@collection_enum)
     end
 
@@ -165,6 +127,7 @@ module MaintenanceTasks
       @ticker.persist if defined?(@ticker)
 
       if defined?(@run)
+        @run.cursor = cursor_position
         @run.persist_error(error)
 
         task_context = {

data/app/models/maintenance_tasks/run.rb

@@ -32,7 +32,6 @@ module MaintenanceTasks
       :cancelled,
     ]
     COMPLETED_STATUSES = [:succeeded, :errored, :cancelled]
-    STUCK_TASK_TIMEOUT = 5.minutes
 
     enum status: STATUSES.to_h { |status| [status, status.to_s] }
 
@@ -342,7 +341,7 @@ module MaintenanceTasks
     #
     # @return [Boolean] whether the Run is stuck.
     def stuck?
-      (cancelling? || pausing?) && updated_at <= STUCK_TASK_TIMEOUT.ago
+      (cancelling? || pausing?) && updated_at <= MaintenanceTasks.stuck_task_duration.ago
     end
 
     # Performs validation on the task_name attribute.

data/app/models/maintenance_tasks/task.rb

@@ -247,5 +247,55 @@ module MaintenanceTasks
     def count
       self.class.collection_builder_strategy.count(self)
     end
+
+    # Default enumeration builder. You may override this method to return any
+    # Enumerator yielding pairs of `[item, item_cursor]`.
+    #
+    # @param cursor [String, nil] cursor position to resume from, or nil on
+    #   initial call.
+    #
+    # @return [Enumerator]
+    def enumerator_builder(cursor:)
+      collection = self.collection
+
+      job_iteration_builder = JobIteration::EnumeratorBuilder.new(nil)
+
+      case collection
+      when :no_collection
+        job_iteration_builder.build_once_enumerator(cursor: nil)
+      when ActiveRecord::Relation
+        job_iteration_builder.active_record_on_records(collection, cursor: cursor)
+      when ActiveRecord::Batches::BatchEnumerator
+        if collection.start || collection.finish
+          raise ArgumentError, <<~MSG.squish
+            #{self.class.name}#collection cannot support
+            a batch enumerator with the "start" or "finish" options.
+          MSG
+        end
+
+        # For now, only support automatic count based on the enumerator for
+        # batches
+        job_iteration_builder.active_record_on_batch_relations(
+          collection.relation,
+          cursor: cursor,
+          batch_size: collection.batch_size,
+        )
+      when Array
+        job_iteration_builder.build_array_enumerator(collection, cursor: cursor&.to_i)
+      when BatchCsvCollectionBuilder::BatchCsv
+        JobIteration::CsvEnumerator.new(collection.csv).batches(
+          batch_size: collection.batch_size,
+          cursor: cursor&.to_i,
+        )
+      when CSV
+        JobIteration::CsvEnumerator.new(collection).rows(cursor: cursor&.to_i)
+      else
+        raise ArgumentError, <<~MSG.squish
+          #{self.class.name}#collection must be either an
+          Active Record Relation, ActiveRecord::Batches::BatchEnumerator,
+          Array, or CSV.
+        MSG
+      end
+    end
   end
 end

data/app/validators/maintenance_tasks/run_status_validator.rb

@@ -60,6 +60,9 @@ module MaintenanceTasks
       # interrupted -> errored occurs when the task is deleted while it is
       #   interrupted.
       "interrupted" => ["running", "pausing", "cancelling", "errored"],
+      # errored -> enqueued occurs when the task is retried after encounting an
+      #   error.
+      "errored" => ["enqueued"],
     }
 
     # Validate whether a transition from one Run status

data/app/views/maintenance_tasks/runs/_arguments.html.erb

@@ -1,22 +1,4 @@
-<% if run.arguments.present? %>
-  <div class="table-container">
-    <h6 class="title is-6">Arguments:</h6>
-    <table class="table">
-      <tbody>
-        <% run.arguments.transform_values(&:to_s).each do |key, value| %>
-          <tr>
-            <td class="is-family-monospace"><%= key %></td>
-            <td>
-              <% next if value.empty? %>
-              <% if value.include?("\n") %>
-                <pre><%= value %></pre>
-              <% else %>
-                <code><%= value %></code>
-              <% end %>
-            </td>
-          </tr>
-          <% end %>
-      </tbody>
-    </table>
-  </div>
+<% if arguments.present? %>
+  <h6 class="title is-6">Arguments:</h6>
+  <%= render "maintenance_tasks/runs/serializable", serializable: arguments %>
 <% end %>

data/app/views/maintenance_tasks/runs/_metadata.html.erb

@@ -0,0 +1,4 @@
+<% if metadata.present? %>
+  <h6 class="title is-6">Metadata:</h6>
+  <%= render "maintenance_tasks/runs/serializable", serializable: metadata %>
+<% end %>

data/app/views/maintenance_tasks/runs/_run.html.erb

@@ -2,6 +2,7 @@
   <h5 class="title is-5">
     <%= time_tag run.created_at, title: run.created_at.utc.iso8601 %>
     <%= status_tag run.status %>
+    <span class="is-pulled-right" title="Run ID">#<%= run.id %></span>
   </h5>
 
   <%= progress run %>
@@ -16,12 +17,16 @@
 
   <%= render "maintenance_tasks/runs/csv", run: run %>
   <%= tag.hr if run.csv_file.present? && run.arguments.present? %>
-  <%= render "maintenance_tasks/runs/arguments", run: run %>
+  <%= render "maintenance_tasks/runs/arguments", arguments: run.arguments %>
+  <%= tag.hr if run.csv_file.present? || run.arguments.present? && run.metadata.present? %>
+  <%= render "maintenance_tasks/runs/metadata", metadata: run.metadata %>
 
   <div class="buttons">
     <% if run.paused? %>
       <%= button_to 'Resume', resume_task_run_path(@task, run), method: :put, class: 'button is-primary', disabled: @task.deleted? %>
       <%= button_to 'Cancel', cancel_task_run_path(@task, run), method: :put, class: 'button is-danger' %>
+    <% elsif run.errored? %>
+      <%= button_to 'Resume', resume_task_run_path(@task, run), method: :put, class: 'button is-primary', disabled: @task.deleted? %>
     <% elsif run.cancelling? %>
       <% if run.stuck? %>
         <%= button_to 'Cancel', cancel_task_run_path(@task, run), method: :put, class: 'button is-danger', disabled: @task.deleted? %>

data/app/views/maintenance_tasks/runs/_serializable.html.erb

@@ -0,0 +1,26 @@
+<% if serializable.present? %>
+  <% case serializable %>
+  <% when Hash %>
+    <div class="table-container">
+      <table class="table">
+        <tbody>
+          <% serializable.transform_values(&:to_s).each do |key, value| %>
+            <tr>
+              <td class="is-family-monospace"><%= key %></td>
+              <td>
+                <% next if value.empty? %>
+                <% if value.include?("\n") %>
+                  <pre><%= value %></pre>
+                <% else %>
+                  <code><%= value %></code>
+                <% end %>
+              </td>
+            </tr>
+          <% end %>
+        </tbody>
+      </table>
+    </div>
+  <% else %>
+    <code><%= serializable.inspect %></code>
+  <% end %>
+<% end %>

data/app/views/maintenance_tasks/tasks/_task.html.erb

@@ -21,6 +21,8 @@
 
     <%= render "maintenance_tasks/runs/csv", run: run %>
     <%= tag.hr if run.csv_file.present? && run.arguments.present? %>
-    <%= render "maintenance_tasks/runs/arguments", run: run %>
+    <%= render "maintenance_tasks/runs/arguments", arguments: run.arguments %>
+    <%= tag.hr if run.csv_file.present? || run.arguments.present? && run.metadata.present? %>
+    <%= render "maintenance_tasks/runs/metadata", metadata: run.metadata %>
   <% end %>
 </div>

data/lib/maintenance_tasks.rb

@@ -89,4 +89,11 @@ module MaintenanceTasks
   #
   #   @return [Proc] generates a hash containing the metadata to be stored on the Run
   mattr_accessor :metadata, default: nil
+
+  # @!attribute stuck_task_duration
+  #  @scope class
+  #  The duration after which a task is considered stuck and can be force cancelled.
+  #
+  #  @return [ActiveSupport::Duration] the threshold in seconds after which a task is considered stuck.
+  mattr_accessor :stuck_task_duration, default: 5.minutes
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: maintenance_tasks
 version: !ruby/object:Gem::Version
-  version: 2.4.0
+  version: 2.5.0
 platform: ruby
 authors:
 - Shopify Engineering
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-12-20 00:00:00.000000000 Z
+date: 2024-01-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: actionpack
@@ -128,7 +128,9 @@ files:
 - app/views/layouts/maintenance_tasks/application.html.erb
 - app/views/maintenance_tasks/runs/_arguments.html.erb
 - app/views/maintenance_tasks/runs/_csv.html.erb
+- app/views/maintenance_tasks/runs/_metadata.html.erb
 - app/views/maintenance_tasks/runs/_run.html.erb
+- app/views/maintenance_tasks/runs/_serializable.html.erb
 - app/views/maintenance_tasks/runs/info/_cancelled.html.erb
 - app/views/maintenance_tasks/runs/info/_cancelling.html.erb
 - app/views/maintenance_tasks/runs/info/_custom.html.erb
@@ -169,7 +171,7 @@ homepage: https://github.com/Shopify/maintenance_tasks
 licenses:
 - MIT
 metadata:
-  source_code_uri: https://github.com/Shopify/maintenance_tasks/tree/v2.4.0
+  source_code_uri: https://github.com/Shopify/maintenance_tasks/tree/v2.5.0
   allowed_push_host: https://rubygems.org
 post_install_message: 
 rdoc_options: []
@@ -186,7 +188,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.22
+rubygems_version: 3.5.5
 signing_key: 
 specification_version: 4
 summary: A Rails engine for queuing and managing maintenance tasks