maintenance_tasks 1.8.1 → 1.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6dbf3120a28de1ce88571561ba3be52d160743d365f862a65573b24fcc64cc24
-  data.tar.gz: 87fda9aaf48c7643a85cc542cbda6d89721ea0b656396329abd9a2d4a42cab68
+  metadata.gz: cd4ace45bec0d57e14080ddf832419d1daa1a06fc8cb7d7e79af2c14b2e9c26c
+  data.tar.gz: 698945e60d17b43eeab9da5a3f43aa68fb0d9841f2aad1f66fc0c1f2994a4bc5
 SHA512:
-  metadata.gz: 77cc4fad4f591683cb3b6062265c487efb8f15b7f31aca2620730af3b74dd57296f8451d7077256fe2f5718762b8ba3de76e7f1affe2c888c202c585a781a570
-  data.tar.gz: 15e9c05f39cc0cc6216b66081ec051eebd4a669e5c5e002672f2028d5a160fa8dd4a9144e726bc455aa7a60b96018c2fe1a572079a9d705d7b299b7762e483df
+  metadata.gz: 5a4c862c0eddbf87fd0196b390eb6cc01b7dbe56d08772c4afb23e7cbaf351a6a9328e4c8bdf66057b4e871c099983649b6acddb6371fc7f216bb226b9ba372c
+  data.tar.gz: f7da9ab1b727fc9be10900da568a59c8c7a6a9620e7619f879942a5489b56ab673c8efcfbfc0551d54f018df0baef1638220854a2f88c7c0824bcfb6ccbaf222

data/README.md

@@ -8,9 +8,9 @@ A Rails engine for queuing and managing maintenance tasks.
 
 To install the gem and run the install generator, execute:
 
-```bash
-$ bundle add maintenance_tasks
-$ bin/rails generate maintenance_tasks:install
+```sh-session
+bundle add maintenance_tasks
+bin/rails generate maintenance_tasks:install
 ```
 
 The generator creates and runs a migration to add the necessary table to your
@@ -39,8 +39,8 @@ take a look at the [Active Job documentation][active-job-docs].
 
 A generator is provided to create tasks. Generate a new task by running:
 
-```bash
-$ bin/rails generate maintenance_tasks:task update_posts
+```sh-session
+bin/rails generate maintenance_tasks:task update_posts
 ```
 
 This creates the task file `app/tasks/maintenance/update_posts_task.rb`.
@@ -86,8 +86,8 @@ instuctions][setup].
 
 Generate a CSV Task by running:
 
-```bash
-$ bin/rails generate maintenance_tasks:task import_posts --csv
+```sh-session
+bin/rails generate maintenance_tasks:task import_posts --csv
 ```
 
 The generated task is a subclass of `MaintenanceTasks::Task` that implements:
@@ -118,6 +118,33 @@ The files uploaded to your Active Storage service provider will be renamed
 to include an ISO8601 timestamp and the Task name in snake case format.
 The CSV is expected to have a trailing newline at the end of the file.
 
+#### Batch CSV Tasks
+
+Tasks can process CSVs in batches. Add the `in_batches` option to your task's
+`csv_collection` macro:
+
+```ruby
+# app/tasks/maintenance/batch_import_posts_task.rb
+
+module Maintenance
+  class BatchImportPostsTask < MaintenanceTasks::Task
+    csv_collection(in_batches: 50)
+
+    def process(batch_of_rows)
+      Post.insert_all(post_rows.map(&:to_h))
+    end
+  end
+end
+```
+
+As with a regular CSV task, ensure you've implemented the following method:
+
+* `process`: do the work of your Task on a batch (array of `CSV::Row` objects).
+
+Note that `#count` is calculated automatically based on the number of batches in
+your collection, and your Task's progress will be displayed in terms of batches
+(not the total number of rows in your CSV).
+
 ### Processing Batch Collections
 
 The Maintenance Tasks gem supports processing Active Records in batches. This
@@ -168,8 +195,8 @@ collection-less tasks.
 
 Generate a collection-less Task by running:
 
-```bash
-$ bin/rails generate maintenance_tasks:task no_collection_task --no-collection
+```sh-session
+bin/rails generate maintenance_tasks:task no_collection_task --no-collection
 ```
 
 The generated task is a subclass of `MaintenanceTasks::Task` that implements:
@@ -456,21 +483,21 @@ You can run your new Task by accessing the Web UI and clicking on "Run".
 
 Alternatively, you can run your Task in the command line:
 
-```bash
-$ bundle exec maintenance_tasks perform Maintenance::UpdatePostsTask
+```sh-session
+bundle exec maintenance_tasks perform Maintenance::UpdatePostsTask
 ```
 
 To run a Task that processes CSVs from the command line, use the --csv option:
 
-```bash
-$ bundle exec maintenance_tasks perform Maintenance::ImportPostsTask --csv "path/to/my_csv.csv"
+```sh-session
+bundle exec maintenance_tasks perform Maintenance::ImportPostsTask --csv "path/to/my_csv.csv"
 ```
 
 To run a Task that takes arguments from the command line, use the --arguments
 option, passing arguments as a set of \<key>:\<value> pairs:
 
-```bash
-$ bundle exec maintenance_tasks perform Maintenance::ParamsTask \
+```sh-session
+bundle exec maintenance_tasks perform Maintenance::ParamsTask \
   --arguments post_ids:1,2,3 content:"Hello, World!"
 ```
 
@@ -718,8 +745,8 @@ clean backtraces.
 Use bundler to check for and upgrade to newer versions. After installing a new
 version, re-run the install command:
 
-```bash
-$ bin/rails generate maintenance_tasks:install
+```sh-session
+bin/rails generate maintenance_tasks:install
 ```
 
 This ensures that new migrations are installed and run as well.

data/app/controllers/maintenance_tasks/application_controller.rb

@@ -8,7 +8,15 @@ module MaintenanceTasks
     BULMA_CDN = "https://cdn.jsdelivr.net"
 
     content_security_policy do |policy|
-      policy.style_src(BULMA_CDN)
+      policy.style_src(
+        BULMA_CDN,
+        # ruby syntax highlighting
+        "'sha256-y9V0na/WU44EUNI/HDP7kZ7mfEci4PAOIjYOOan6JMA='",
+      )
+      policy.script_src(
+        # page refresh script
+        "'sha256-2RPaBS4XCMLp0JJ/sW407W9l4qjC+WQAHmTOFJTGfqo='",
+      )
       policy.frame_ancestors(:self)
     end
 

data/app/controllers/maintenance_tasks/tasks_controller.rb

@@ -44,7 +44,7 @@ module MaintenanceTasks
     private
 
     def set_refresh
-      @refresh = 3
+      @refresh = true
     end
   end
 end

data/app/jobs/concerns/maintenance_tasks/task_job_concern.rb

@@ -47,6 +47,7 @@ module MaintenanceTasks
             a batch enumerator with the "start" or "finish" options.
           MSG
         end
+
         # For now, only support automatic count based on the enumerator for
         # batches
         @enumerator = enumerator_builder.active_record_on_batch_relations(
@@ -56,6 +57,11 @@ module MaintenanceTasks
         )
       when Array
         enumerator_builder.build_array_enumerator(collection, cursor: cursor)
+      when BatchCsvCollectionBuilder::BatchCsv
+        JobIteration::CsvEnumerator.new(collection.csv).batches(
+          batch_size: collection.batch_size,
+          cursor: cursor,
+        )
       when CSV
         JobIteration::CsvEnumerator.new(collection).rows(cursor: cursor)
       else
@@ -151,7 +157,7 @@ module MaintenanceTasks
     def after_perform
       @run.persist_transition
       if defined?(@reenqueue_iteration_job) && @reenqueue_iteration_job
-        reenqueue_iteration_job(should_ignore: false)
+        reenqueue_iteration_job(should_ignore: false) unless @run.stopped?
       end
     end
 

data/app/models/maintenance_tasks/batch_csv_collection_builder.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module MaintenanceTasks
+  # Strategy for building a Task that processes CSV files in batches.
+  #
+  # @api private
+  class BatchCsvCollectionBuilder < CsvCollectionBuilder
+    BatchCsv = Struct.new(:csv, :batch_size, keyword_init: true)
+
+    # Initialize a BatchCsvCollectionBuilder with a batch size.
+    #
+    # @param batch_size [Integer] the number of CSV rows in a batch.
+    def initialize(batch_size)
+      @batch_size = batch_size
+      super()
+    end
+
+    # Defines the collection to be iterated over, based on the provided CSV.
+    # Includes the CSV and the batch size.
+    def collection(task)
+      BatchCsv.new(
+        csv: CSV.new(task.csv_content, headers: true),
+        batch_size: @batch_size
+      )
+    end
+
+    # The number of batches to be processed. Excludes the header row from the
+    # count and assumes a trailing newline is at the end of the CSV file.
+    # Note that this number is an approximation based on the number of
+    # newlines.
+    #
+    # @return [Integer] the approximate number of batches to process.
+    def count(task)
+      (task.csv_content.count("\n") + @batch_size - 1) / @batch_size
+    end
+  end
+end

data/app/models/maintenance_tasks/run.rb

@@ -52,6 +52,7 @@ module MaintenanceTasks
     # Ensure ActiveStorage is in use before preloading the attachments
     scope :with_attached_csv, -> do
       return unless defined?(ActiveStorage)
+
       with_attached_csv_file if ActiveStorage::Attachment.table_exists?
     end
 
@@ -240,6 +241,7 @@ module MaintenanceTasks
         # Preserve swap-and-replace solution for data races until users
         # run migration to upgrade to optimistic locking solution
         return if stopping?
+
         updated = self.class.where(id: id).where.not(status: STOPPING_STATUSES)
           .update_all(status: :running, updated_at: Time.now) > 0
         if updated
@@ -384,6 +386,7 @@ module MaintenanceTasks
     def csv_file
       return unless defined?(ActiveStorage)
       return unless ActiveStorage::Attachment.table_exists?
+
       super
     end
 
@@ -426,6 +429,7 @@ module MaintenanceTasks
     def truncate(attribute_name, value)
       limit = self.class.column_for_attribute(attribute_name).limit
       return value unless limit
+
       value&.first(limit)
     end
   end

data/app/models/maintenance_tasks/task.rb

@@ -38,6 +38,7 @@ module MaintenanceTasks
         unless task.is_a?(Class) && task < Task
           raise NotFoundError.new("#{name} is not a Task.", name)
         end
+
         task
       end
 
@@ -52,16 +53,23 @@ module MaintenanceTasks
 
       # Make this Task a task that handles CSV.
       #
+      # @param in_batches [Integer] optionally, supply a batch size if the CSV
+      # should be processed in batches.
+      #
       # An input to upload a CSV will be added in the form to start a Run. The
       # collection and count method are implemented.
-      def csv_collection
+      def csv_collection(in_batches: nil)
         unless defined?(ActiveStorage)
           raise NotImplementedError, "Active Storage needs to be installed\n"\
             "To resolve this issue run: bin/rails active_storage:install"
         end
 
-        self.collection_builder_strategy =
-          MaintenanceTasks::CsvCollectionBuilder.new
+        if in_batches
+          self.collection_builder_strategy =
+            BatchCsvCollectionBuilder.new(in_batches)
+        else
+          self.collection_builder_strategy = CsvCollectionBuilder.new
+        end
       end
 
       # Make this a Task that calls #process once, instead of iterating over
@@ -172,6 +180,7 @@ module MaintenanceTasks
       def load_constants
         namespace = MaintenanceTasks.tasks_module.safe_constantize
         return unless namespace
+
         namespace.constants.map { |constant| namespace.const_get(constant) }
       end
     end

data/app/models/maintenance_tasks/task_data.rb

@@ -73,6 +73,7 @@ module MaintenanceTasks
     # @return [nil] if the Task file was deleted.
     def code
       return if deleted?
+
       task = Task.named(name)
       file = if Object.respond_to?(:const_source_location)
         Object.const_source_location(task.name).first
@@ -88,6 +89,7 @@ module MaintenanceTasks
     # @return [nil] if there are no Runs associated with the Task.
     def last_run
       return @last_run if defined?(@last_run)
+
       @last_run = runs.first
     end
 
@@ -100,6 +102,7 @@ module MaintenanceTasks
     #   record previous to the last Run.
     def previous_runs
       return Run.none unless last_run
+
       runs.where.not(id: last_run.id)
     end
 
@@ -150,6 +153,7 @@ module MaintenanceTasks
     # @return [nil] if the Task file was deleted.
     def new
       return if deleted?
+
       MaintenanceTasks::Task.named(name).new
     end
 

data/app/models/maintenance_tasks/ticker.rb

@@ -41,6 +41,7 @@ module MaintenanceTasks
     # row will call the block at most once (if it had been throttled).
     def persist
       return if @ticks_recorded == 0
+
       now = Time.now
       duration = now - @last_persisted
       @last_persisted = now

data/app/views/layouts/maintenance_tasks/application.html.erb

@@ -15,13 +15,13 @@
     <%= csrf_meta_tags %>
 
     <%=
-      stylesheet_link_tag URI.join(controller.class::BULMA_CDN, 'npm/bulma@0.9.1/css/bulma.css'),
+      stylesheet_link_tag(URI.join(controller.class::BULMA_CDN, 'npm/bulma@0.9.3/css/bulma.css'),
         media: :all,
-        integrity: 'sha256-67AR2JVjhMZCLVxapLuBSMap5RrXbksv4vlllenHBSE=',
-        crossorigin: 'anonymous'
+        integrity: 'sha384-Zkr9rpl37lclZu6AwYQZZm0CxiMqLZFiibodW+UXLnAWPBr6qgIzPpcmHkpwnyWD',
+        crossorigin: 'anonymous') unless request.xhr?
     %>
 
-    <style nonce="<%= content_security_policy_nonce %>">
+    <style>
       .ruby-comment { color: #6a737d;}
       .ruby-const { color: #e36209; }
       .ruby-embexpr-beg, .ruby-embexpr-end, .ruby-period { color: #24292e; }
@@ -31,12 +31,30 @@
       .ruby-label, .ruby-tstring-beg, .ruby-tstring-content, .ruby-tstring-end { color: #032f62; }
     </style>
 
-    <% if defined?(@refresh) %>
-      <meta http-equiv="refresh" content="<%= @refresh %>">
-    <% end %>
+    <script>
+      function refresh() {
+        if (!("refresh" in document.body.dataset)) return
+        window.setTimeout(() => {
+          document.body.style.cursor = "wait"
+          fetch(document.location, { headers: { "X-Requested-With": "XMLHttpRequest" } }).then(
+            async response => {
+              const text = await response.text()
+              const newDocument = new DOMParser().parseFromString(text, "text/html")
+              document.body.replaceWith(newDocument.body)
+              <%# force a redraw for Safari %>
+              window.scrollTo({ top: document.documentElement.scrollTop + 1 })
+              window.scrollTo({ top: document.documentElement.scrollTop - 1 })
+              refresh()
+            },
+            error => location.reload()
+          )
+        }, 3000)
+      }
+      document.addEventListener('DOMContentLoaded', refresh)
+    </script>
   </head>
 
-  <body>
+  <body <%= "data-refresh" if defined?(@refresh) && @refresh %>>
     <%= render 'layouts/maintenance_tasks/navbar' %>
 
     <section class="section">

data/app/views/maintenance_tasks/runs/_arguments.html.erb

@@ -3,11 +3,11 @@
     <h6 class="title is-6">Arguments:</h6>
     <table class="table">
       <tbody>
-        <% run.arguments.each do |key, value| %>
+        <% run.arguments.transform_values(&:to_s).each do |key, value| %>
           <tr>
             <td class="is-family-monospace"><%= key %></td>
             <td>
-              <% next if value.nil? || value.empty? %>
+              <% next if value.empty? %>
               <% if value.include?("\n") %>
                 <pre><%= value %><pre>
               <% else %>

data/lib/generators/maintenance_tasks/templates/no_collection_task_test.rb.tt

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-require 'test_helper'
+require "test_helper"
 
 module <%= tasks_module %>
 <% module_namespacing do -%>

data/lib/generators/maintenance_tasks/templates/task_spec.rb.tt

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
-require 'rails_helper'
+require "rails_helper"
 
 module <%= tasks_module %>
 <% module_namespacing do -%>
   RSpec.describe <%= class_name %>Task do
     describe "#process" do
       subject(:process) { described_class.process(element) }
-      let(:element) { 
+      let(:element) {
         # Object to be processed in a single iteration of this task
       }
       pending "add some examples to (or delete) #{__FILE__}"

data/lib/generators/maintenance_tasks/templates/task_test.rb.tt

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-require 'test_helper'
+require "test_helper"
 
 module <%= tasks_module %>
 <% module_namespacing do -%>

data/lib/maintenance_tasks.rb

@@ -66,6 +66,7 @@ module MaintenanceTasks
   # @private
   def self.error_handler
     return @error_handler if defined?(@error_handler)
+
     @error_handler = ->(_error, _task_context, _errored_element) {}
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: maintenance_tasks
 version: !ruby/object:Gem::Version
-  version: 1.8.1
+  version: 1.10.0
 platform: ruby
 authors:
 - Shopify Engineering
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-02-25 00:00:00.000000000 Z
+date: 2022-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: actionpack
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: 1.3.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: 1.3.6
 - !ruby/object:Gem::Dependency
   name: railties
   requirement: !ruby/object:Gem::Requirement
@@ -97,6 +97,7 @@ files:
 - app/jobs/concerns/maintenance_tasks/task_job_concern.rb
 - app/jobs/maintenance_tasks/task_job.rb
 - app/models/maintenance_tasks/application_record.rb
+- app/models/maintenance_tasks/batch_csv_collection_builder.rb
 - app/models/maintenance_tasks/csv_collection_builder.rb
 - app/models/maintenance_tasks/no_collection_builder.rb
 - app/models/maintenance_tasks/null_collection_builder.rb
@@ -150,7 +151,7 @@ homepage: https://github.com/Shopify/maintenance_tasks
 licenses:
 - MIT
 metadata:
-  source_code_uri: https://github.com/Shopify/maintenance_tasks/tree/v1.8.1
+  source_code_uri: https://github.com/Shopify/maintenance_tasks/tree/v1.10.0
   allowed_push_host: https://rubygems.org
 post_install_message: 
 rdoc_options: []