maintenance_tasks 1.8.2 → 1.9.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +27 -0
- data/app/jobs/concerns/maintenance_tasks/task_job_concern.rb +5 -0
- data/app/models/maintenance_tasks/batch_csv_collection_builder.rb +39 -0
- data/app/models/maintenance_tasks/task.rb +10 -3
- data/app/views/maintenance_tasks/runs/_arguments.html.erb +2 -2
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0a050fbc2d9dbc3eab88b027ee68163f09cae1472df9e236a7151f5fa1f4464b
|
4
|
+
data.tar.gz: c6cf0f55b868b7810c7361669d4383d0fd159dc337b270396e3123a795e75d24
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 903da6a69576ffcf022a09248fd7945ec816150505d36c65d96f70cfeec969cb3315e820317c5bff8c882d4e0c09ad33758f9221b2d4d2cf08292a0415219c3a
|
7
|
+
data.tar.gz: 811d79fa5d58252649faea6ed0bc82b6f8fa471d77ae02535cf73fb873a9e663deb893fc81d2528863f68194123db7f7dc55c1a9dc299143da9586fccb4570a6
|
data/README.md
CHANGED
@@ -118,6 +118,33 @@ The files uploaded to your Active Storage service provider will be renamed
|
|
118
118
|
to include an ISO8601 timestamp and the Task name in snake case format.
|
119
119
|
The CSV is expected to have a trailing newline at the end of the file.
|
120
120
|
|
121
|
+
#### Batch CSV Tasks
|
122
|
+
|
123
|
+
Tasks can process CSVs in batches. Add the `in_batches` option to your task's
|
124
|
+
`csv_collection` macro:
|
125
|
+
|
126
|
+
```ruby
|
127
|
+
# app/tasks/maintenance/batch_import_posts_task.rb
|
128
|
+
|
129
|
+
module Maintenance
|
130
|
+
class BatchImportPostsTask < MaintenanceTasks::Task
|
131
|
+
csv_collection(in_batches: 50)
|
132
|
+
|
133
|
+
def process(batch_of_rows)
|
134
|
+
Post.insert_all(post_rows.map(&:to_h))
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
138
|
+
```
|
139
|
+
|
140
|
+
As with a regular CSV task, ensure you've implemented the following method:
|
141
|
+
|
142
|
+
* `process`: do the work of your Task on a batch (array of `CSV::Row` objects).
|
143
|
+
|
144
|
+
Note that `#count` is calculated automatically based on the number of batches in
|
145
|
+
your collection, and your Task's progress will be displayed in terms of batches
|
146
|
+
(not the total number of rows in your CSV).
|
147
|
+
|
121
148
|
### Processing Batch Collections
|
122
149
|
|
123
150
|
The Maintenance Tasks gem supports processing Active Records in batches. This
|
@@ -57,6 +57,11 @@ module MaintenanceTasks
|
|
57
57
|
)
|
58
58
|
when Array
|
59
59
|
enumerator_builder.build_array_enumerator(collection, cursor: cursor)
|
60
|
+
when BatchCsvCollectionBuilder::BatchCsv
|
61
|
+
JobIteration::CsvEnumerator.new(collection.csv).batches(
|
62
|
+
batch_size: collection.batch_size,
|
63
|
+
cursor: cursor,
|
64
|
+
)
|
60
65
|
when CSV
|
61
66
|
JobIteration::CsvEnumerator.new(collection).rows(cursor: cursor)
|
62
67
|
else
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "csv"
|
4
|
+
|
5
|
+
module MaintenanceTasks
|
6
|
+
# Strategy for building a Task that processes CSV files in batches.
|
7
|
+
#
|
8
|
+
# @api private
|
9
|
+
class BatchCsvCollectionBuilder < CsvCollectionBuilder
|
10
|
+
BatchCsv = Struct.new(:csv, :batch_size, keyword_init: true)
|
11
|
+
|
12
|
+
# Initialize a BatchCsvCollectionBuilder with a batch size.
|
13
|
+
#
|
14
|
+
# @param batch_size [Integer] the number of CSV rows in a batch.
|
15
|
+
def initialize(batch_size)
|
16
|
+
@batch_size = batch_size
|
17
|
+
super()
|
18
|
+
end
|
19
|
+
|
20
|
+
# Defines the collection to be iterated over, based on the provided CSV.
|
21
|
+
# Includes the CSV and the batch size.
|
22
|
+
def collection(task)
|
23
|
+
BatchCsv.new(
|
24
|
+
csv: CSV.new(task.csv_content, headers: true),
|
25
|
+
batch_size: @batch_size
|
26
|
+
)
|
27
|
+
end
|
28
|
+
|
29
|
+
# The number of batches to be processed. Excludes the header row from the
|
30
|
+
# count and assumes a trailing newline is at the end of the CSV file.
|
31
|
+
# Note that this number is an approximation based on the number of
|
32
|
+
# newlines.
|
33
|
+
#
|
34
|
+
# @return [Integer] the approximate number of batches to process.
|
35
|
+
def count(task)
|
36
|
+
(task.csv_content.count("\n") + @batch_size - 1) / @batch_size
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -53,16 +53,23 @@ module MaintenanceTasks
|
|
53
53
|
|
54
54
|
# Make this Task a task that handles CSV.
|
55
55
|
#
|
56
|
+
# @param in_batches [Integer] optionally, supply a batch size if the CSV
|
57
|
+
# should be processed in batches.
|
58
|
+
#
|
56
59
|
# An input to upload a CSV will be added in the form to start a Run. The
|
57
60
|
# collection and count method are implemented.
|
58
|
-
def csv_collection
|
61
|
+
def csv_collection(in_batches: nil)
|
59
62
|
unless defined?(ActiveStorage)
|
60
63
|
raise NotImplementedError, "Active Storage needs to be installed\n"\
|
61
64
|
"To resolve this issue run: bin/rails active_storage:install"
|
62
65
|
end
|
63
66
|
|
64
|
-
|
65
|
-
|
67
|
+
if in_batches
|
68
|
+
self.collection_builder_strategy =
|
69
|
+
BatchCsvCollectionBuilder.new(in_batches)
|
70
|
+
else
|
71
|
+
self.collection_builder_strategy = CsvCollectionBuilder.new
|
72
|
+
end
|
66
73
|
end
|
67
74
|
|
68
75
|
# Make this a Task that calls #process once, instead of iterating over
|
@@ -3,11 +3,11 @@
|
|
3
3
|
<h6 class="title is-6">Arguments:</h6>
|
4
4
|
<table class="table">
|
5
5
|
<tbody>
|
6
|
-
<% run.arguments.each do |key, value| %>
|
6
|
+
<% run.arguments.transform_values(&:to_s).each do |key, value| %>
|
7
7
|
<tr>
|
8
8
|
<td class="is-family-monospace"><%= key %></td>
|
9
9
|
<td>
|
10
|
-
<% next if value.
|
10
|
+
<% next if value.empty? %>
|
11
11
|
<% if value.include?("\n") %>
|
12
12
|
<pre><%= value %><pre>
|
13
13
|
<% else %>
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: maintenance_tasks
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.9.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shopify Engineering
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-04-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: actionpack
|
@@ -97,6 +97,7 @@ files:
|
|
97
97
|
- app/jobs/concerns/maintenance_tasks/task_job_concern.rb
|
98
98
|
- app/jobs/maintenance_tasks/task_job.rb
|
99
99
|
- app/models/maintenance_tasks/application_record.rb
|
100
|
+
- app/models/maintenance_tasks/batch_csv_collection_builder.rb
|
100
101
|
- app/models/maintenance_tasks/csv_collection_builder.rb
|
101
102
|
- app/models/maintenance_tasks/no_collection_builder.rb
|
102
103
|
- app/models/maintenance_tasks/null_collection_builder.rb
|
@@ -150,7 +151,7 @@ homepage: https://github.com/Shopify/maintenance_tasks
|
|
150
151
|
licenses:
|
151
152
|
- MIT
|
152
153
|
metadata:
|
153
|
-
source_code_uri: https://github.com/Shopify/maintenance_tasks/tree/v1.
|
154
|
+
source_code_uri: https://github.com/Shopify/maintenance_tasks/tree/v1.9.0
|
154
155
|
allowed_push_host: https://rubygems.org
|
155
156
|
post_install_message:
|
156
157
|
rdoc_options: []
|