maintenance_tasks 1.8.2 → 1.10.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +44 -17
- data/app/controllers/maintenance_tasks/application_controller.rb +9 -1
- data/app/controllers/maintenance_tasks/tasks_controller.rb +1 -1
- data/app/jobs/concerns/maintenance_tasks/task_job_concern.rb +5 -0
- data/app/models/maintenance_tasks/batch_csv_collection_builder.rb +39 -0
- data/app/models/maintenance_tasks/task.rb +10 -3
- data/app/views/layouts/maintenance_tasks/application.html.erb +27 -9
- data/app/views/maintenance_tasks/runs/_arguments.html.erb +2 -2
- data/lib/generators/maintenance_tasks/templates/no_collection_task_test.rb.tt +1 -1
- data/lib/generators/maintenance_tasks/templates/task_spec.rb.tt +2 -2
- data/lib/generators/maintenance_tasks/templates/task_test.rb.tt +1 -1
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 89ca6938b40f972d96333415fdeb3ec433b4d14701731b5cfcffb81354bb5a4d
|
4
|
+
data.tar.gz: 01c307846ba7d76f8553c6cb2b270e6773719e9276d147a97a6bd39220bd5459
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fcf9d547baf0b48bb0b08654c4e5d7f91fb6ec27ad58e62a9a20c60e19a2a45512e56f225353ecc9be47144aaeea11f8b94afb8a1052cd6e456eba19f3672c1a
|
7
|
+
data.tar.gz: b950bf5c74429db618e09e5849dac8649919a6df4c812a0252af755eae7b0f198bc9944d008740e8491116c2ef85549dfbc709f9db74f0b805a5360531d86961
|
data/README.md
CHANGED
@@ -8,9 +8,9 @@ A Rails engine for queuing and managing maintenance tasks.
|
|
8
8
|
|
9
9
|
To install the gem and run the install generator, execute:
|
10
10
|
|
11
|
-
```
|
12
|
-
|
13
|
-
|
11
|
+
```sh-session
|
12
|
+
bundle add maintenance_tasks
|
13
|
+
bin/rails generate maintenance_tasks:install
|
14
14
|
```
|
15
15
|
|
16
16
|
The generator creates and runs a migration to add the necessary table to your
|
@@ -39,8 +39,8 @@ take a look at the [Active Job documentation][active-job-docs].
|
|
39
39
|
|
40
40
|
A generator is provided to create tasks. Generate a new task by running:
|
41
41
|
|
42
|
-
```
|
43
|
-
|
42
|
+
```sh-session
|
43
|
+
bin/rails generate maintenance_tasks:task update_posts
|
44
44
|
```
|
45
45
|
|
46
46
|
This creates the task file `app/tasks/maintenance/update_posts_task.rb`.
|
@@ -86,8 +86,8 @@ instuctions][setup].
|
|
86
86
|
|
87
87
|
Generate a CSV Task by running:
|
88
88
|
|
89
|
-
```
|
90
|
-
|
89
|
+
```sh-session
|
90
|
+
bin/rails generate maintenance_tasks:task import_posts --csv
|
91
91
|
```
|
92
92
|
|
93
93
|
The generated task is a subclass of `MaintenanceTasks::Task` that implements:
|
@@ -118,6 +118,33 @@ The files uploaded to your Active Storage service provider will be renamed
|
|
118
118
|
to include an ISO8601 timestamp and the Task name in snake case format.
|
119
119
|
The CSV is expected to have a trailing newline at the end of the file.
|
120
120
|
|
121
|
+
#### Batch CSV Tasks
|
122
|
+
|
123
|
+
Tasks can process CSVs in batches. Add the `in_batches` option to your task's
|
124
|
+
`csv_collection` macro:
|
125
|
+
|
126
|
+
```ruby
|
127
|
+
# app/tasks/maintenance/batch_import_posts_task.rb
|
128
|
+
|
129
|
+
module Maintenance
|
130
|
+
class BatchImportPostsTask < MaintenanceTasks::Task
|
131
|
+
csv_collection(in_batches: 50)
|
132
|
+
|
133
|
+
def process(batch_of_rows)
|
134
|
+
Post.insert_all(post_rows.map(&:to_h))
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
138
|
+
```
|
139
|
+
|
140
|
+
As with a regular CSV task, ensure you've implemented the following method:
|
141
|
+
|
142
|
+
* `process`: do the work of your Task on a batch (array of `CSV::Row` objects).
|
143
|
+
|
144
|
+
Note that `#count` is calculated automatically based on the number of batches in
|
145
|
+
your collection, and your Task's progress will be displayed in terms of batches
|
146
|
+
(not the total number of rows in your CSV).
|
147
|
+
|
121
148
|
### Processing Batch Collections
|
122
149
|
|
123
150
|
The Maintenance Tasks gem supports processing Active Records in batches. This
|
@@ -168,8 +195,8 @@ collection-less tasks.
|
|
168
195
|
|
169
196
|
Generate a collection-less Task by running:
|
170
197
|
|
171
|
-
```
|
172
|
-
|
198
|
+
```sh-session
|
199
|
+
bin/rails generate maintenance_tasks:task no_collection_task --no-collection
|
173
200
|
```
|
174
201
|
|
175
202
|
The generated task is a subclass of `MaintenanceTasks::Task` that implements:
|
@@ -456,21 +483,21 @@ You can run your new Task by accessing the Web UI and clicking on "Run".
|
|
456
483
|
|
457
484
|
Alternatively, you can run your Task in the command line:
|
458
485
|
|
459
|
-
```
|
460
|
-
|
486
|
+
```sh-session
|
487
|
+
bundle exec maintenance_tasks perform Maintenance::UpdatePostsTask
|
461
488
|
```
|
462
489
|
|
463
490
|
To run a Task that processes CSVs from the command line, use the --csv option:
|
464
491
|
|
465
|
-
```
|
466
|
-
|
492
|
+
```sh-session
|
493
|
+
bundle exec maintenance_tasks perform Maintenance::ImportPostsTask --csv "path/to/my_csv.csv"
|
467
494
|
```
|
468
495
|
|
469
496
|
To run a Task that takes arguments from the command line, use the --arguments
|
470
497
|
option, passing arguments as a set of \<key>:\<value> pairs:
|
471
498
|
|
472
|
-
```
|
473
|
-
|
499
|
+
```sh-session
|
500
|
+
bundle exec maintenance_tasks perform Maintenance::ParamsTask \
|
474
501
|
--arguments post_ids:1,2,3 content:"Hello, World!"
|
475
502
|
```
|
476
503
|
|
@@ -718,8 +745,8 @@ clean backtraces.
|
|
718
745
|
Use bundler to check for and upgrade to newer versions. After installing a new
|
719
746
|
version, re-run the install command:
|
720
747
|
|
721
|
-
```
|
722
|
-
|
748
|
+
```sh-session
|
749
|
+
bin/rails generate maintenance_tasks:install
|
723
750
|
```
|
724
751
|
|
725
752
|
This ensures that new migrations are installed and run as well.
|
@@ -8,7 +8,15 @@ module MaintenanceTasks
|
|
8
8
|
BULMA_CDN = "https://cdn.jsdelivr.net"
|
9
9
|
|
10
10
|
content_security_policy do |policy|
|
11
|
-
policy.style_src(
|
11
|
+
policy.style_src(
|
12
|
+
BULMA_CDN,
|
13
|
+
# ruby syntax highlighting
|
14
|
+
"'sha256-y9V0na/WU44EUNI/HDP7kZ7mfEci4PAOIjYOOan6JMA='",
|
15
|
+
)
|
16
|
+
policy.script_src(
|
17
|
+
# page refresh script
|
18
|
+
"'sha256-2RPaBS4XCMLp0JJ/sW407W9l4qjC+WQAHmTOFJTGfqo='",
|
19
|
+
)
|
12
20
|
policy.frame_ancestors(:self)
|
13
21
|
end
|
14
22
|
|
@@ -57,6 +57,11 @@ module MaintenanceTasks
|
|
57
57
|
)
|
58
58
|
when Array
|
59
59
|
enumerator_builder.build_array_enumerator(collection, cursor: cursor)
|
60
|
+
when BatchCsvCollectionBuilder::BatchCsv
|
61
|
+
JobIteration::CsvEnumerator.new(collection.csv).batches(
|
62
|
+
batch_size: collection.batch_size,
|
63
|
+
cursor: cursor,
|
64
|
+
)
|
60
65
|
when CSV
|
61
66
|
JobIteration::CsvEnumerator.new(collection).rows(cursor: cursor)
|
62
67
|
else
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "csv"
|
4
|
+
|
5
|
+
module MaintenanceTasks
|
6
|
+
# Strategy for building a Task that processes CSV files in batches.
|
7
|
+
#
|
8
|
+
# @api private
|
9
|
+
class BatchCsvCollectionBuilder < CsvCollectionBuilder
|
10
|
+
BatchCsv = Struct.new(:csv, :batch_size, keyword_init: true)
|
11
|
+
|
12
|
+
# Initialize a BatchCsvCollectionBuilder with a batch size.
|
13
|
+
#
|
14
|
+
# @param batch_size [Integer] the number of CSV rows in a batch.
|
15
|
+
def initialize(batch_size)
|
16
|
+
@batch_size = batch_size
|
17
|
+
super()
|
18
|
+
end
|
19
|
+
|
20
|
+
# Defines the collection to be iterated over, based on the provided CSV.
|
21
|
+
# Includes the CSV and the batch size.
|
22
|
+
def collection(task)
|
23
|
+
BatchCsv.new(
|
24
|
+
csv: CSV.new(task.csv_content, headers: true),
|
25
|
+
batch_size: @batch_size
|
26
|
+
)
|
27
|
+
end
|
28
|
+
|
29
|
+
# The number of batches to be processed. Excludes the header row from the
|
30
|
+
# count and assumes a trailing newline is at the end of the CSV file.
|
31
|
+
# Note that this number is an approximation based on the number of
|
32
|
+
# newlines.
|
33
|
+
#
|
34
|
+
# @return [Integer] the approximate number of batches to process.
|
35
|
+
def count(task)
|
36
|
+
(task.csv_content.count("\n") + @batch_size - 1) / @batch_size
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -53,16 +53,23 @@ module MaintenanceTasks
|
|
53
53
|
|
54
54
|
# Make this Task a task that handles CSV.
|
55
55
|
#
|
56
|
+
# @param in_batches [Integer] optionally, supply a batch size if the CSV
|
57
|
+
# should be processed in batches.
|
58
|
+
#
|
56
59
|
# An input to upload a CSV will be added in the form to start a Run. The
|
57
60
|
# collection and count method are implemented.
|
58
|
-
def csv_collection
|
61
|
+
def csv_collection(in_batches: nil)
|
59
62
|
unless defined?(ActiveStorage)
|
60
63
|
raise NotImplementedError, "Active Storage needs to be installed\n"\
|
61
64
|
"To resolve this issue run: bin/rails active_storage:install"
|
62
65
|
end
|
63
66
|
|
64
|
-
|
65
|
-
|
67
|
+
if in_batches
|
68
|
+
self.collection_builder_strategy =
|
69
|
+
BatchCsvCollectionBuilder.new(in_batches)
|
70
|
+
else
|
71
|
+
self.collection_builder_strategy = CsvCollectionBuilder.new
|
72
|
+
end
|
66
73
|
end
|
67
74
|
|
68
75
|
# Make this a Task that calls #process once, instead of iterating over
|
@@ -15,13 +15,13 @@
|
|
15
15
|
<%= csrf_meta_tags %>
|
16
16
|
|
17
17
|
<%=
|
18
|
-
stylesheet_link_tag
|
18
|
+
stylesheet_link_tag(URI.join(controller.class::BULMA_CDN, 'npm/bulma@0.9.3/css/bulma.css'),
|
19
19
|
media: :all,
|
20
|
-
integrity: '
|
21
|
-
crossorigin: 'anonymous'
|
20
|
+
integrity: 'sha384-Zkr9rpl37lclZu6AwYQZZm0CxiMqLZFiibodW+UXLnAWPBr6qgIzPpcmHkpwnyWD',
|
21
|
+
crossorigin: 'anonymous') unless request.xhr?
|
22
22
|
%>
|
23
23
|
|
24
|
-
<style
|
24
|
+
<style>
|
25
25
|
.ruby-comment { color: #6a737d;}
|
26
26
|
.ruby-const { color: #e36209; }
|
27
27
|
.ruby-embexpr-beg, .ruby-embexpr-end, .ruby-period { color: #24292e; }
|
@@ -31,12 +31,30 @@
|
|
31
31
|
.ruby-label, .ruby-tstring-beg, .ruby-tstring-content, .ruby-tstring-end { color: #032f62; }
|
32
32
|
</style>
|
33
33
|
|
34
|
-
|
35
|
-
|
36
|
-
|
34
|
+
<script>
|
35
|
+
function refresh() {
|
36
|
+
if (!("refresh" in document.body.dataset)) return
|
37
|
+
window.setTimeout(() => {
|
38
|
+
document.body.style.cursor = "wait"
|
39
|
+
fetch(document.location, { headers: { "X-Requested-With": "XMLHttpRequest" } }).then(
|
40
|
+
async response => {
|
41
|
+
const text = await response.text()
|
42
|
+
const newDocument = new DOMParser().parseFromString(text, "text/html")
|
43
|
+
document.body.replaceWith(newDocument.body)
|
44
|
+
<%# force a redraw for Safari %>
|
45
|
+
window.scrollTo({ top: document.documentElement.scrollTop + 1 })
|
46
|
+
window.scrollTo({ top: document.documentElement.scrollTop - 1 })
|
47
|
+
refresh()
|
48
|
+
},
|
49
|
+
error => location.reload()
|
50
|
+
)
|
51
|
+
}, 3000)
|
52
|
+
}
|
53
|
+
document.addEventListener('DOMContentLoaded', refresh)
|
54
|
+
</script>
|
37
55
|
</head>
|
38
56
|
|
39
|
-
|
57
|
+
<%= tag.body(data: { refresh: defined?(@refresh) && @refresh }) do %>
|
40
58
|
<%= render 'layouts/maintenance_tasks/navbar' %>
|
41
59
|
|
42
60
|
<section class="section">
|
@@ -50,5 +68,5 @@
|
|
50
68
|
<%= yield %>
|
51
69
|
</div>
|
52
70
|
</div>
|
53
|
-
|
71
|
+
<% end %>
|
54
72
|
</html>
|
@@ -3,11 +3,11 @@
|
|
3
3
|
<h6 class="title is-6">Arguments:</h6>
|
4
4
|
<table class="table">
|
5
5
|
<tbody>
|
6
|
-
<% run.arguments.each do |key, value| %>
|
6
|
+
<% run.arguments.transform_values(&:to_s).each do |key, value| %>
|
7
7
|
<tr>
|
8
8
|
<td class="is-family-monospace"><%= key %></td>
|
9
9
|
<td>
|
10
|
-
<% next if value.
|
10
|
+
<% next if value.empty? %>
|
11
11
|
<% if value.include?("\n") %>
|
12
12
|
<pre><%= value %><pre>
|
13
13
|
<% else %>
|
@@ -1,12 +1,12 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
require
|
2
|
+
require "rails_helper"
|
3
3
|
|
4
4
|
module <%= tasks_module %>
|
5
5
|
<% module_namespacing do -%>
|
6
6
|
RSpec.describe <%= class_name %>Task do
|
7
7
|
describe "#process" do
|
8
8
|
subject(:process) { described_class.process(element) }
|
9
|
-
let(:element) {
|
9
|
+
let(:element) {
|
10
10
|
# Object to be processed in a single iteration of this task
|
11
11
|
}
|
12
12
|
pending "add some examples to (or delete) #{__FILE__}"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: maintenance_tasks
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.10.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shopify Engineering
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-05-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: actionpack
|
@@ -97,6 +97,7 @@ files:
|
|
97
97
|
- app/jobs/concerns/maintenance_tasks/task_job_concern.rb
|
98
98
|
- app/jobs/maintenance_tasks/task_job.rb
|
99
99
|
- app/models/maintenance_tasks/application_record.rb
|
100
|
+
- app/models/maintenance_tasks/batch_csv_collection_builder.rb
|
100
101
|
- app/models/maintenance_tasks/csv_collection_builder.rb
|
101
102
|
- app/models/maintenance_tasks/no_collection_builder.rb
|
102
103
|
- app/models/maintenance_tasks/null_collection_builder.rb
|
@@ -150,7 +151,7 @@ homepage: https://github.com/Shopify/maintenance_tasks
|
|
150
151
|
licenses:
|
151
152
|
- MIT
|
152
153
|
metadata:
|
153
|
-
source_code_uri: https://github.com/Shopify/maintenance_tasks/tree/v1.
|
154
|
+
source_code_uri: https://github.com/Shopify/maintenance_tasks/tree/v1.10.1
|
154
155
|
allowed_push_host: https://rubygems.org
|
155
156
|
post_install_message:
|
156
157
|
rdoc_options: []
|