maintenance_tasks 2.6.0 → 2.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2dca6b60a4a6d6366eaf05b43d1e7d1244c59baef10fe9d25561cab727a70638
-  data.tar.gz: c9827e001e131747ea2cb20301ef9749f394bd603a1d657afac0024e1feefc79
+  metadata.gz: dac65eae0e9b5f8066a994408059eb42689792fed29f36321632ff42a804a7cd
+  data.tar.gz: a8df371985f28c8d97a2e8363f263e93fa40c2291370c63da8bfa115bad0e21a
 SHA512:
-  metadata.gz: 269fe1ab563047cf368cb552b91305c0a99b25affa6bea95553f55541c6e5a947726e441b3487181146d90d75bf9e9e1e853fc3302fda483ce0d00032ad7bceb
-  data.tar.gz: 7b1ae78d3049cb3c28c17bcf068acc735441e4b4a3de6babeb8107294408a1ee139c03b6bad12e1004f0bef59c834314429079df0a11c6384148f007bc3a44a9
+  metadata.gz: 5144b9f8ab67ae9ea05b16c74178a96239cf22611627822e6ca00205c145e1767bc51015f796fd95dd40b973f3e5b6b1e1e67a950c72c628995602c631e0e49a
+  data.tar.gz: caeefe52b00431e3ba41297cc10f3dfb17c184b57237e74e6825f50b683be58431c079bf4921ac2529286a0b926fdbbe2ee6e61dc4350e3457d477ce18dc1ba1

data/README.md

@@ -10,8 +10,8 @@ engine helps with the second part of this process, backfilling.
 
 Maintenance tasks are collection-based tasks, usually using Active Record, that
 update the data in your database. They can be paused or interrupted. Maintenance
-tasks can operate [in batches][#processing-batch-collections] and use
-[throttling][#throttling] to control the load on your database.
+tasks can operate [in batches](#processing-batch-collections) and use
+[throttling](#throttling) to control the load on your database.
 
 Maintenance tasks aren't meant to happen on a regular basis. They're used as
 needed, or as one-offs. Normally maintenance tasks are ephemeral, so they are
@@ -191,15 +191,57 @@ module Maintenance
 end
 ```
 
+`posts.csv`:
 ```csv
-# posts.csv
 title,content
 My Title,Hello World!
 ```
 
 The files uploaded to your Active Storage service provider will be renamed to
-include an ISO 8601 timestamp and the Task name in snake case format. The CSV is
-expected to have a trailing newline at the end of the file.
+include an ISO 8601 timestamp and the Task name in snake case format.
+
+The implicit `#count` method loads and parses the entire file to determine the
+accurate number of rows. With files with millions of rows, it takes several
+seconds to process. Consider skipping the count (defining a `count` that returns
+`nil`) or use an approximation, eg: count the number of new lines:
+
+```ruby
+def count(task)
+  task.csv_content.count("\n") - 1
+end
+```
+
+#### CSV options
+
+Tasks can pass [options for Ruby's CSV parser][csv-parse-options] by adding
+keyword arguments to `csv_collection`:
+
+[csv-parse-options]: https://ruby-doc.org/3.3.0/stdlibs/csv/CSV.html#class-CSV-label-Options+for+Parsing
+
+```ruby
+# app/tasks/maintenance/import_posts_task.rb
+
+module Maintenance
+  class ImportPosts
+    csv_collection(skip_lines: /^#/, converters: ->(field) { field.strip })
+
+    def process(row)
+      Post.create!(title: row["title"], content: row["content"])
+    end
+  end
+end
+```
+
+These options instruct Ruby's CSV parser to skip lines that start with a `#`,
+and removes the leading and trailing spaces from any field, so that the
+following file will be processed identically as the previous example:
+
+`posts.csv`:
+```csv
+# A comment
+title,content
+ My Title ,Hello World!
+```
 
 #### Batch CSV Tasks
 
@@ -453,6 +495,68 @@ module Maintenance
 end
 ```
 
+### Subscribing to instrumentation events
+
+If you are interested in actioning a specific task event, please refer to the
+[Using Task Callbacks](#using-task-callbacks) section below. However, if you
+want to subscribe to all events, irrespective of the task, you can use the
+following Active Support notifications:
+
+```ruby
+enqueued.maintenance_tasks    # This event is published when a task has been enqueued by the user.
+succeeded.maintenance_tasks   # This event is published when a task has finished without any errors.
+cancelled.maintenance_tasks   # This event is published when the user explicitly halts the execution of a task.
+paused.maintenance_tasks      # This event is published when a task is paused by the user in the middle of its run.
+errored.maintenance_tasks     # This event is published when the task's code produces an unhandled exception.
+```
+
+These notifications offer a way to monitor the lifecycle of maintenance tasks in
+your application.
+
+Usage example:
+
+ ```ruby
+ ActiveSupport::Notifications.subscribe("succeeded.maintenance_tasks") do |*, payload|
+  task_name = payload[:task_name]
+  arguments = payload[:arguments]
+  metadata = payload[:metadata]
+  job_id = payload[:job_id]
+  run_id = payload[:run_id]
+  time_running = payload[:time_running]
+  started_at = payload[:started_at]
+  ended_at = payload[:ended_at]
+rescue => e
+  Rails.logger.error(e)
+end
+
+ActiveSupport::Notifications.subscribe("errored.maintenance_tasks") do |*, payload|
+  task_name = payload[:task_name]
+  error = payload[:error]
+  error_message = error[:message]
+  error_class = error[:class]
+  error_backtrace = error[:backtrace]
+rescue => e
+  Rails.logger.error(e)
+end
+
+# or
+
+class MaintenanceTasksInstrumenter < ActiveSupport::Subscriber
+  attach_to :maintenance_tasks
+
+  def enqueued(event)
+    task_name = event.payload[:task_name]
+    arguments = event.payload[:arguments]
+    metadata = event.payload[:metadata]
+
+    SlackNotifier.broadcast(SLACK_CHANNEL,
+      "Job #{task_name} was started by #{metadata[:user_email]}} with arguments #{arguments.to_s.truncate(255)}")
+  rescue => e
+    Rails.logger.error(e)
+  end
+end
+```
+
 ### Using Task Callbacks
 
 The Task provides callbacks that hook into its life cycle.
@@ -503,21 +607,6 @@ end
 If any of the other callbacks cause an exception, it will be handled by the
 error handler, and will cause the task to stop running.
 
-Callback behaviour can be shared across all tasks using an initializer.
-
-```ruby
-# config/initializer/maintenance_tasks.rb
-Rails.autoloaders.main.on_load("MaintenanceTasks::Task") do
-  MaintenanceTasks::Task.class_eval do
-    after_start(:notify)
-
-    private
-
-    def notify; end
-  end
-end
-```
-
 ### Considerations when writing Tasks
 
 Maintenance Tasks relies on the queue adapter configured for your application to

data/app/jobs/concerns/maintenance_tasks/task_job_concern.rb

@@ -32,6 +32,7 @@ module MaintenanceTasks
 
     def build_enumerator(_run, cursor:)
       cursor ||= @run.cursor
+      self.cursor_position = cursor
       @collection_enum = @task.enumerator_builder(cursor: cursor)
 
       @collection_enum ||= case (collection = @task.collection)

data/app/models/maintenance_tasks/batch_csv_collection_builder.rb

@@ -12,16 +12,17 @@ module MaintenanceTasks
     # Initialize a BatchCsvCollectionBuilder with a batch size.
     #
     # @param batch_size [Integer] the number of CSV rows in a batch.
-    def initialize(batch_size)
+    # @param csv_options [Hash] options to pass to the CSV parser.
+    def initialize(batch_size, **csv_options)
       @batch_size = batch_size
-      super()
+      super(**csv_options)
     end
 
     # Defines the collection to be iterated over, based on the provided CSV.
     # Includes the CSV and the batch size.
     def collection(task)
       BatchCsv.new(
-        csv: CSV.new(task.csv_content, headers: true),
+        csv: CSV.new(task.csv_content, **@csv_options),
         batch_size: @batch_size,
       )
     end

data/app/models/maintenance_tasks/csv_collection_builder.rb

@@ -5,24 +5,27 @@ require "csv"
 module MaintenanceTasks
   # Strategy for building a Task that processes CSV files.
   #
+  # @param csv_options [Hash] options to pass to the CSV parser.
   # @api private
   class CsvCollectionBuilder
+    def initialize(**csv_options)
+      @csv_options = csv_options
+    end
+
     # Defines the collection to be iterated over, based on the provided CSV.
     #
-    # @return [CSV] the CSV object constructed from the specified CSV content,
-    #   with headers.
+    # @return [CSV] the CSV object constructed from the specified CSV content.
     def collection(task)
-      CSV.new(task.csv_content, headers: true)
+      CSV.new(task.csv_content, **@csv_options)
     end
 
-    # The number of rows to be processed. Excludes the header row from the
-    # count and assumes a trailing newline is at the end of the CSV file.
-    # Note that this number is an approximation based on the number of
-    # newlines.
+    # The number of rows to be processed.
+    # It uses the CSV library for an accurate row count.
+    # Note that the entire file is loaded. It will take several seconds with files with millions of rows.
     #
     # @return [Integer] the approximate number of rows to process.
     def count(task)
-      task.csv_content.count("\n") - 1
+      CSV.new(task.csv_content, **@csv_options).count
     end
 
     # Return that the Task processes CSV content.

data/app/models/maintenance_tasks/progress.rb

@@ -52,17 +52,17 @@ module MaintenanceTasks
       total = @run.tick_total
 
       if !total?
-        "Processed #{number_to_delimited(count)} "\
+        "Processed #{number_to_delimited(count)} " \
           "#{"item".pluralize(count)}."
       elsif over_total?
-        "Processed #{number_to_delimited(count)} "\
-          "#{"item".pluralize(count)} "\
+        "Processed #{number_to_delimited(count)} " \
+          "#{"item".pluralize(count)} " \
           "(expected #{number_to_delimited(total)})."
       else
         percentage = 100.0 * count / total
 
-        "Processed #{number_to_delimited(count)} out of "\
-          "#{number_to_delimited(total)} #{"item".pluralize(total)} "\
+        "Processed #{number_to_delimited(count)} out of " \
+          "#{number_to_delimited(total)} #{"item".pluralize(total)} " \
           "(#{number_to_percentage(percentage, precision: 0)})."
       end
     end

data/app/models/maintenance_tasks/run.rb

@@ -39,6 +39,8 @@ module MaintenanceTasks
       enum status: STATUSES.to_h { |status| [status, status.to_s] }
     end
 
+    after_save :instrument_status_change
+
     validate :task_name_belongs_to_a_valid_task, on: :create
     validate :csv_attachment_presence, on: :create
     validate :csv_content_type, on: :create
@@ -452,6 +454,30 @@ module MaintenanceTasks
 
     private
 
+    def instrument_status_change
+      return unless status_previously_changed? || id_previously_changed?
+      return if running? || pausing? || cancelling? || interrupted?
+
+      attr = {
+        run_id: id,
+        job_id: job_id,
+        task_name: task_name,
+        arguments: arguments,
+        metadata: metadata,
+        time_running: time_running,
+        started_at: started_at,
+        ended_at: ended_at,
+      }
+
+      attr[:error] = {
+        message: error_message,
+        class: error_class,
+        backtrace: backtrace,
+      } if errored?
+
+      ActiveSupport::Notifications.instrument("#{status}.maintenance_tasks", attr)
+    end
+
     def run_task_callbacks(callback)
       task.run_callbacks(callback)
     rescue Task::NotFoundError

data/app/models/maintenance_tasks/runner.rb

@@ -74,7 +74,7 @@ module MaintenanceTasks
 
     def enqueue(run, job)
       unless job.enqueue
-        raise "The job to perform #{run.task_name} could not be enqueued. "\
+        raise "The job to perform #{run.task_name} could not be enqueued. " \
           "Enqueuing has been prevented by a callback."
       end
     rescue => error

data/app/models/maintenance_tasks/task.rb

@@ -65,20 +65,25 @@ module MaintenanceTasks
       # Make this Task a task that handles CSV.
       #
       # @param in_batches [Integer] optionally, supply a batch size if the CSV
-      # should be processed in batches.
+      #   should be processed in batches.
+      # @param csv_options [Hash] optionally, supply options for the CSV parser.
+      #   If not given, defaults to: <code>{ headers: true }</code>
+      # @see https://ruby-doc.org/3.3.0/stdlibs/csv/CSV.html#class-CSV-label-Options+for+Parsing
       #
       # An input to upload a CSV will be added in the form to start a Run. The
       # collection and count method are implemented.
-      def csv_collection(in_batches: nil)
+      def csv_collection(in_batches: nil, **csv_options)
         unless defined?(ActiveStorage)
-          raise NotImplementedError, "Active Storage needs to be installed\n"\
+          raise NotImplementedError, "Active Storage needs to be installed\n" \
             "To resolve this issue run: bin/rails active_storage:install"
         end
 
+        csv_options[:headers] = true unless csv_options.key?(:headers)
+        csv_options[:encoding] ||= Encoding.default_external
         self.collection_builder_strategy = if in_batches
-          BatchCsvCollectionBuilder.new(in_batches)
+          BatchCsvCollectionBuilder.new(in_batches, **csv_options)
         else
-          CsvCollectionBuilder.new
+          CsvCollectionBuilder.new(**csv_options)
         end
       end
 

data/lib/generators/maintenance_tasks/task_generator.rb

@@ -6,7 +6,7 @@ module MaintenanceTasks
   # @api private
   class TaskGenerator < Rails::Generators::NamedBase
     source_root File.expand_path("templates", __dir__)
-    desc "This generator creates a task file at app/tasks and a corresponding "\
+    desc "This generator creates a task file at app/tasks and a corresponding " \
       "test."
 
     class_option :csv,
@@ -24,7 +24,7 @@ module MaintenanceTasks
     # Creates the Task file.
     def create_task_file
       if options[:csv] && options[:no_collection]
-        raise "Multiple Task type options provided. Please use either "\
+        raise "Multiple Task type options provided. Please use either " \
           "--csv or --no-collection."
       end
       template_file = File.join(

data/lib/maintenance_tasks/cli.rb

@@ -23,11 +23,11 @@ module MaintenanceTasks
     DESC
 
     # Specify the CSV file to process for CSV Tasks
-    desc = "Supply a CSV file to be processed by a CSV Task, "\
+    desc = "Supply a CSV file to be processed by a CSV Task, " \
       "--csv path/to/csv/file.csv"
     option :csv, lazy_default: :stdin, desc: desc
     # Specify arguments to supply to a Task supporting parameters
-    desc = "Supply arguments for a Task that accepts parameters as a set of "\
+    desc = "Supply arguments for a Task that accepts parameters as a set of " \
       "<key>:<value> pairs."
     option :arguments, type: :hash, desc: desc
 

data/lib/tasks/maintenance_tasks_tasks.rake

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 # desc "Explaining what the task does"
 # task :maintenance_tasks do
 #   # Task goes here

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: maintenance_tasks
 version: !ruby/object:Gem::Version
-  version: 2.6.0
+  version: 2.7.1
 platform: ruby
 authors:
 - Shopify Engineering
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-02-12 00:00:00.000000000 Z
+date: 2024-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: actionpack
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '6.0'
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: job-iteration
   requirement: !ruby/object:Gem::Requirement
@@ -171,7 +185,7 @@ homepage: https://github.com/Shopify/maintenance_tasks
 licenses:
 - MIT
 metadata:
-  source_code_uri: https://github.com/Shopify/maintenance_tasks/tree/v2.6.0
+  source_code_uri: https://github.com/Shopify/maintenance_tasks/tree/v2.7.1
   allowed_push_host: https://rubygems.org
 post_install_message: 
 rdoc_options: []
@@ -188,7 +202,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.6
+rubygems_version: 3.5.11
 signing_key: 
 specification_version: 4
 summary: A Rails engine for queuing and managing maintenance tasks