maintenance_tasks 1.6.0 → 1.8.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +53 -8
- data/app/controllers/maintenance_tasks/tasks_controller.rb +2 -1
- data/app/helpers/maintenance_tasks/tasks_helper.rb +19 -0
- data/app/jobs/concerns/maintenance_tasks/task_job_concern.rb +23 -31
- data/app/models/maintenance_tasks/csv_collection_builder.rb +9 -3
- data/app/models/maintenance_tasks/no_collection_builder.rb +29 -0
- data/app/models/maintenance_tasks/null_collection_builder.rb +7 -0
- data/app/models/maintenance_tasks/run.rb +138 -16
- data/app/models/maintenance_tasks/task.rb +30 -13
- data/app/models/maintenance_tasks/task_data.rb +12 -1
- data/app/views/maintenance_tasks/runs/_run.html.erb +4 -0
- data/app/views/maintenance_tasks/runs/info/_custom.html.erb +0 -0
- data/app/views/maintenance_tasks/tasks/_custom.html.erb +0 -0
- data/app/views/maintenance_tasks/tasks/_task.html.erb +4 -0
- data/app/views/maintenance_tasks/tasks/show.html.erb +11 -5
- data/db/migrate/20211210152329_add_lock_version_to_maintenance_tasks_runs.rb +8 -0
- data/lib/generators/maintenance_tasks/task_generator.rb +13 -0
- data/lib/generators/maintenance_tasks/templates/no_collection_task.rb.tt +13 -0
- data/lib/generators/maintenance_tasks/templates/no_collection_task_test.rb.tt +12 -0
- data/lib/generators/maintenance_tasks/templates/task_test.rb.tt +4 -0
- data/lib/maintenance_tasks/engine.rb +10 -1
- metadata +9 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6dbf3120a28de1ce88571561ba3be52d160743d365f862a65573b24fcc64cc24
|
4
|
+
data.tar.gz: 87fda9aaf48c7643a85cc542cbda6d89721ea0b656396329abd9a2d4a42cab68
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 77cc4fad4f591683cb3b6062265c487efb8f15b7f31aca2620730af3b74dd57296f8451d7077256fe2f5718762b8ba3de76e7f1affe2c888c202c585a781a570
|
7
|
+
data.tar.gz: 15e9c05f39cc0cc6216b66081ec051eebd4a669e5c5e002672f2028d5a160fa8dd4a9144e726bc455aa7a60b96018c2fe1a572079a9d705d7b299b7762e483df
|
data/README.md
CHANGED
@@ -114,8 +114,9 @@ title,content
|
|
114
114
|
My Title,Hello World!
|
115
115
|
```
|
116
116
|
|
117
|
-
The files uploaded to your Active Storage service provider will be renamed
|
117
|
+
The files uploaded to your Active Storage service provider will be renamed
|
118
118
|
to include an ISO8601 timestamp and the Task name in snake case format.
|
119
|
+
The CSV is expected to have a trailing newline at the end of the file.
|
119
120
|
|
120
121
|
### Processing Batch Collections
|
121
122
|
|
@@ -159,6 +160,36 @@ primary keys of the records of the batch first, and then perform an additional
|
|
159
160
|
query to load the records when calling `each` (or any `Enumerable` method)
|
160
161
|
inside `#process`.
|
161
162
|
|
163
|
+
### Tasks that don't need a Collection
|
164
|
+
|
165
|
+
Sometimes, you might want to run a Task that performs a single operation, such
|
166
|
+
as enqueuing another background job or hitting an external API. The gem supports
|
167
|
+
collection-less tasks.
|
168
|
+
|
169
|
+
Generate a collection-less Task by running:
|
170
|
+
|
171
|
+
```bash
|
172
|
+
$ bin/rails generate maintenance_tasks:task no_collection_task --no-collection
|
173
|
+
```
|
174
|
+
|
175
|
+
The generated task is a subclass of `MaintenanceTasks::Task` that implements:
|
176
|
+
|
177
|
+
* `process`: do the work of your maintenance task
|
178
|
+
|
179
|
+
```ruby
|
180
|
+
# app/tasks/maintenance/no_collection_task.rb
|
181
|
+
|
182
|
+
module Maintenance
|
183
|
+
class NoCollectionTask < MaintenanceTasks::Task
|
184
|
+
no_collection
|
185
|
+
|
186
|
+
def process
|
187
|
+
SomeAsyncJob.perform_later
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|
191
|
+
```
|
192
|
+
|
162
193
|
### Throttling
|
163
194
|
|
164
195
|
Maintenance Tasks often modify a lot of data and can be taxing on your database.
|
@@ -200,6 +231,20 @@ Tasks can define multiple throttle conditions. Throttle conditions are inherited
|
|
200
231
|
by descendants, and new conditions will be appended without impacting existing
|
201
232
|
conditions.
|
202
233
|
|
234
|
+
The backoff can also be specified as a proc:
|
235
|
+
|
236
|
+
```ruby
|
237
|
+
# app/tasks/maintenance/update_posts_throttled_task.rb
|
238
|
+
|
239
|
+
module Maintenance
|
240
|
+
class UpdatePostsThrottledTask < MaintenanceTasks::Task
|
241
|
+
throttle_on(backoff: -> { RandomBackoffGenerator.generate_duration } ) do
|
242
|
+
DatabaseStatus.unhealthy?
|
243
|
+
end
|
244
|
+
...
|
245
|
+
end
|
246
|
+
end
|
247
|
+
```
|
203
248
|
### Custom Task Parameters
|
204
249
|
|
205
250
|
Tasks may need additional information, supplied via parameters, to run.
|
@@ -241,12 +286,12 @@ The Task provides callbacks that hook into its life cycle.
|
|
241
286
|
|
242
287
|
Available callbacks are:
|
243
288
|
|
244
|
-
`after_start`
|
245
|
-
`after_pause`
|
246
|
-
`after_interrupt`
|
247
|
-
`after_cancel`
|
248
|
-
`after_complete`
|
249
|
-
`after_error`
|
289
|
+
* `after_start`
|
290
|
+
* `after_pause`
|
291
|
+
* `after_interrupt`
|
292
|
+
* `after_cancel`
|
293
|
+
* `after_complete`
|
294
|
+
* `after_error`
|
250
295
|
|
251
296
|
```ruby
|
252
297
|
module Maintenance
|
@@ -542,7 +587,7 @@ you can define an error handler:
|
|
542
587
|
|
543
588
|
MaintenanceTasks.error_handler = ->(error, task_context, _errored_element) do
|
544
589
|
Bugsnag.notify(error) do |notification|
|
545
|
-
notification.
|
590
|
+
notification.add_metadata(:task, task_context)
|
546
591
|
end
|
547
592
|
end
|
548
593
|
```
|
@@ -24,11 +24,12 @@ module MaintenanceTasks
|
|
24
24
|
end
|
25
25
|
|
26
26
|
# Runs a given Task and redirects to the Task page.
|
27
|
-
def run
|
27
|
+
def run(&block)
|
28
28
|
task = Runner.run(
|
29
29
|
name: params.fetch(:id),
|
30
30
|
csv_file: params[:csv_file],
|
31
31
|
arguments: params.fetch(:task_arguments, {}).permit!.to_h,
|
32
|
+
&block
|
32
33
|
)
|
33
34
|
redirect_to(task_path(task))
|
34
35
|
rescue ActiveRecord::RecordInvalid => error
|
@@ -100,5 +100,24 @@ module MaintenanceTasks
|
|
100
100
|
only_path: true
|
101
101
|
)
|
102
102
|
end
|
103
|
+
|
104
|
+
# Return the appropriate field tag for the parameter
|
105
|
+
def parameter_field(form_builder, parameter_name)
|
106
|
+
case form_builder.object.class.attribute_types[parameter_name]
|
107
|
+
when ActiveModel::Type::Integer, ActiveModel::Type::Decimal,
|
108
|
+
ActiveModel::Type::Float
|
109
|
+
form_builder.number_field(parameter_name)
|
110
|
+
when ActiveModel::Type::DateTime
|
111
|
+
form_builder.datetime_field(parameter_name)
|
112
|
+
when ActiveModel::Type::Date
|
113
|
+
form_builder.date_field(parameter_name)
|
114
|
+
when ActiveModel::Type::Time
|
115
|
+
form_builder.time_field(parameter_name)
|
116
|
+
when ActiveModel::Type::Boolean
|
117
|
+
form_builder.check_box(parameter_name)
|
118
|
+
else
|
119
|
+
form_builder.text_area(parameter_name, class: "textarea")
|
120
|
+
end
|
121
|
+
end
|
103
122
|
end
|
104
123
|
end
|
@@ -12,8 +12,8 @@ module MaintenanceTasks
|
|
12
12
|
before_perform(:before_perform)
|
13
13
|
|
14
14
|
on_start(:on_start)
|
15
|
-
on_complete(:on_complete)
|
16
15
|
on_shutdown(:on_shutdown)
|
16
|
+
on_complete(:on_complete)
|
17
17
|
|
18
18
|
after_perform(:after_perform)
|
19
19
|
|
@@ -36,6 +36,8 @@ module MaintenanceTasks
|
|
36
36
|
@enumerator = nil
|
37
37
|
|
38
38
|
collection_enum = case collection
|
39
|
+
when :no_collection
|
40
|
+
enumerator_builder.build_once_enumerator(cursor: nil)
|
39
41
|
when ActiveRecord::Relation
|
40
42
|
enumerator_builder.active_record_on_records(collection, cursor: cursor)
|
41
43
|
when ActiveRecord::Batches::BatchEnumerator
|
@@ -63,9 +65,16 @@ module MaintenanceTasks
|
|
63
65
|
Array, or CSV.
|
64
66
|
MSG
|
65
67
|
end
|
68
|
+
throttle_enumerator(collection_enum)
|
69
|
+
end
|
66
70
|
|
71
|
+
def throttle_enumerator(collection_enum)
|
67
72
|
@task.throttle_conditions.reduce(collection_enum) do |enum, condition|
|
68
|
-
enumerator_builder.build_throttle_enumerator(
|
73
|
+
enumerator_builder.build_throttle_enumerator(
|
74
|
+
enum,
|
75
|
+
throttle_on: condition[:throttle_on],
|
76
|
+
backoff: condition[:backoff].call
|
77
|
+
)
|
69
78
|
end
|
70
79
|
end
|
71
80
|
|
@@ -82,7 +91,11 @@ module MaintenanceTasks
|
|
82
91
|
end
|
83
92
|
|
84
93
|
def task_iteration(input)
|
85
|
-
@task.
|
94
|
+
if @task.no_collection?
|
95
|
+
@task.process
|
96
|
+
else
|
97
|
+
@task.process(input)
|
98
|
+
end
|
86
99
|
rescue => error
|
87
100
|
@errored_element = input
|
88
101
|
raise error
|
@@ -105,33 +118,19 @@ module MaintenanceTasks
|
|
105
118
|
def on_start
|
106
119
|
count = @task.count
|
107
120
|
count = @enumerator&.size if count == :no_count
|
108
|
-
@run.
|
109
|
-
@task.run_callbacks(:start)
|
110
|
-
end
|
111
|
-
|
112
|
-
def on_complete
|
113
|
-
@run.status = :succeeded
|
114
|
-
@run.ended_at = Time.now
|
115
|
-
@task.run_callbacks(:complete)
|
121
|
+
@run.start(count)
|
116
122
|
end
|
117
123
|
|
118
124
|
def on_shutdown
|
119
|
-
|
120
|
-
@run.status = :cancelled
|
121
|
-
@task.run_callbacks(:cancel)
|
122
|
-
@run.ended_at = Time.now
|
123
|
-
elsif @run.pausing?
|
124
|
-
@run.status = :paused
|
125
|
-
@task.run_callbacks(:pause)
|
126
|
-
else
|
127
|
-
@run.status = :interrupted
|
128
|
-
@task.run_callbacks(:interrupt)
|
129
|
-
end
|
130
|
-
|
125
|
+
@run.job_shutdown
|
131
126
|
@run.cursor = cursor_position
|
132
127
|
@ticker.persist
|
133
128
|
end
|
134
129
|
|
130
|
+
def on_complete
|
131
|
+
@run.complete
|
132
|
+
end
|
133
|
+
|
135
134
|
# We are reopening a private part of Job Iteration's API here, so we should
|
136
135
|
# ensure the method is still defined upstream. This way, in the case where
|
137
136
|
# the method changes upstream, we catch it at load time instead of at
|
@@ -150,7 +149,7 @@ module MaintenanceTasks
|
|
150
149
|
end
|
151
150
|
|
152
151
|
def after_perform
|
153
|
-
@run.
|
152
|
+
@run.persist_transition
|
154
153
|
if defined?(@reenqueue_iteration_job) && @reenqueue_iteration_job
|
155
154
|
reenqueue_iteration_job(should_ignore: false)
|
156
155
|
end
|
@@ -171,15 +170,8 @@ module MaintenanceTasks
|
|
171
170
|
task_context = {}
|
172
171
|
end
|
173
172
|
errored_element = @errored_element if defined?(@errored_element)
|
174
|
-
run_error_callback
|
175
173
|
ensure
|
176
174
|
MaintenanceTasks.error_handler.call(error, task_context, errored_element)
|
177
175
|
end
|
178
|
-
|
179
|
-
def run_error_callback
|
180
|
-
@task.run_callbacks(:error) if defined?(@task)
|
181
|
-
rescue
|
182
|
-
nil
|
183
|
-
end
|
184
176
|
end
|
185
177
|
end
|
@@ -15,9 +15,10 @@ module MaintenanceTasks
|
|
15
15
|
CSV.new(task.csv_content, headers: true)
|
16
16
|
end
|
17
17
|
|
18
|
-
# The number of rows to be processed. Excludes the header row from the
|
19
|
-
# and
|
20
|
-
# an approximation based on the number of
|
18
|
+
# The number of rows to be processed. Excludes the header row from the
|
19
|
+
# count and assumes a trailing newline is at the end of the CSV file.
|
20
|
+
# Note that this number is an approximation based on the number of
|
21
|
+
# newlines.
|
21
22
|
#
|
22
23
|
# @return [Integer] the approximate number of rows to process.
|
23
24
|
def count(task)
|
@@ -28,5 +29,10 @@ module MaintenanceTasks
|
|
28
29
|
def has_csv_content?
|
29
30
|
true
|
30
31
|
end
|
32
|
+
|
33
|
+
# Returns that the Task processes a collection.
|
34
|
+
def no_collection?
|
35
|
+
false
|
36
|
+
end
|
31
37
|
end
|
32
38
|
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module MaintenanceTasks
|
4
|
+
# Strategy for building a Task that has no collection. These Tasks
|
5
|
+
# consist of a single iteration.
|
6
|
+
#
|
7
|
+
# @api private
|
8
|
+
class NoCollectionBuilder
|
9
|
+
# Specifies that this task does not process a collection.
|
10
|
+
def collection(_task)
|
11
|
+
:no_collection
|
12
|
+
end
|
13
|
+
|
14
|
+
# The number of rows to be processed. Always returns 1.
|
15
|
+
def count(_task)
|
16
|
+
1
|
17
|
+
end
|
18
|
+
|
19
|
+
# Return that the Task does not process CSV content.
|
20
|
+
def has_csv_content?
|
21
|
+
false
|
22
|
+
end
|
23
|
+
|
24
|
+
# Returns that the Task is collection-less.
|
25
|
+
def no_collection?
|
26
|
+
true
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -2,6 +2,8 @@
|
|
2
2
|
|
3
3
|
module MaintenanceTasks
|
4
4
|
# Base strategy for building a collection-based Task to be performed.
|
5
|
+
#
|
6
|
+
# @api private
|
5
7
|
class NullCollectionBuilder
|
6
8
|
# Placeholder method to raise in case a subclass fails to implement the
|
7
9
|
# expected instance method.
|
@@ -27,5 +29,10 @@ module MaintenanceTasks
|
|
27
29
|
def has_csv_content?
|
28
30
|
false
|
29
31
|
end
|
32
|
+
|
33
|
+
# Returns that the Task processes a collection.
|
34
|
+
def no_collection?
|
35
|
+
false
|
36
|
+
end
|
30
37
|
end
|
31
38
|
end
|
@@ -66,9 +66,40 @@ module MaintenanceTasks
|
|
66
66
|
|
67
67
|
# Sets the run status to enqueued, making sure the transition is validated
|
68
68
|
# in case it's already enqueued.
|
69
|
+
#
|
70
|
+
# Rescues and retries status transition if an ActiveRecord::StaleObjectError
|
71
|
+
# is encountered.
|
69
72
|
def enqueued!
|
70
73
|
status_will_change!
|
71
74
|
super
|
75
|
+
rescue ActiveRecord::StaleObjectError
|
76
|
+
reload_status
|
77
|
+
retry
|
78
|
+
end
|
79
|
+
|
80
|
+
CALLBACKS_TRANSITION = {
|
81
|
+
cancelled: :cancel,
|
82
|
+
interrupted: :interrupt,
|
83
|
+
paused: :pause,
|
84
|
+
succeeded: :complete,
|
85
|
+
}.transform_keys(&:to_s)
|
86
|
+
private_constant :CALLBACKS_TRANSITION
|
87
|
+
|
88
|
+
# Saves the run, persisting the transition of its status, and all other
|
89
|
+
# changes to the object.
|
90
|
+
def persist_transition
|
91
|
+
save!
|
92
|
+
callback = CALLBACKS_TRANSITION[status]
|
93
|
+
run_task_callbacks(callback) if callback
|
94
|
+
rescue ActiveRecord::StaleObjectError
|
95
|
+
success = succeeded?
|
96
|
+
reload_status
|
97
|
+
if success
|
98
|
+
self.status = :succeeded
|
99
|
+
else
|
100
|
+
job_shutdown
|
101
|
+
end
|
102
|
+
retry
|
72
103
|
end
|
73
104
|
|
74
105
|
# Increments +tick_count+ by +number_of_ticks+ and +time_running+ by
|
@@ -86,6 +117,11 @@ module MaintenanceTasks
|
|
86
117
|
time_running: duration,
|
87
118
|
touch: true
|
88
119
|
)
|
120
|
+
if locking_enabled?
|
121
|
+
locking_column = self.class.locking_column
|
122
|
+
self[locking_column] += 1
|
123
|
+
clear_attribute_change(locking_column)
|
124
|
+
end
|
89
125
|
end
|
90
126
|
|
91
127
|
# Marks the run as errored and persists the error data.
|
@@ -95,25 +131,39 @@ module MaintenanceTasks
|
|
95
131
|
self.started_at ||= Time.now
|
96
132
|
update!(
|
97
133
|
status: :errored,
|
98
|
-
error_class: error.class.
|
99
|
-
error_message: error.message,
|
134
|
+
error_class: truncate(:error_class, error.class.name),
|
135
|
+
error_message: truncate(:error_message, error.message),
|
100
136
|
backtrace: MaintenanceTasks.backtrace_cleaner.clean(error.backtrace),
|
101
137
|
ended_at: Time.now,
|
102
138
|
)
|
139
|
+
run_task_callbacks(:error)
|
140
|
+
rescue ActiveRecord::StaleObjectError
|
141
|
+
reload_status
|
142
|
+
retry
|
103
143
|
end
|
104
144
|
|
105
|
-
# Refreshes
|
106
|
-
# ensures ActiveModel::Dirty
|
145
|
+
# Refreshes the status and lock version attributes on the Active Record
|
146
|
+
# object, and ensures ActiveModel::Dirty doesn't mark the object as changed.
|
147
|
+
#
|
107
148
|
# This allows us to get the Run's most up-to-date status without needing
|
108
149
|
# to reload the entire record.
|
109
150
|
#
|
110
151
|
# @return [MaintenanceTasks::Run] the Run record with its updated status.
|
111
152
|
def reload_status
|
112
|
-
|
113
|
-
self.class.
|
153
|
+
columns_to_reload = if locking_enabled?
|
154
|
+
[:status, self.class.locking_column]
|
155
|
+
else
|
156
|
+
[:status]
|
157
|
+
end
|
158
|
+
updated_status, updated_lock_version = self.class.uncached do
|
159
|
+
self.class.where(id: id).pluck(*columns_to_reload).first
|
114
160
|
end
|
161
|
+
|
115
162
|
self.status = updated_status
|
116
|
-
|
163
|
+
if updated_lock_version
|
164
|
+
self[self.class.locking_column] = updated_lock_version
|
165
|
+
end
|
166
|
+
clear_attribute_changes(columns_to_reload)
|
117
167
|
self
|
118
168
|
end
|
119
169
|
|
@@ -173,21 +223,65 @@ module MaintenanceTasks
|
|
173
223
|
seconds_to_finished.seconds
|
174
224
|
end
|
175
225
|
|
176
|
-
#
|
226
|
+
# Marks a Run as running.
|
177
227
|
#
|
178
228
|
# If the run is stopping already, it will not transition to running.
|
229
|
+
# Rescues and retries status transition if an ActiveRecord::StaleObjectError
|
230
|
+
# is encountered.
|
179
231
|
def running
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
232
|
+
if locking_enabled?
|
233
|
+
begin
|
234
|
+
running! unless stopping?
|
235
|
+
rescue ActiveRecord::StaleObjectError
|
236
|
+
reload_status
|
237
|
+
retry
|
238
|
+
end
|
239
|
+
else
|
240
|
+
# Preserve swap-and-replace solution for data races until users
|
241
|
+
# run migration to upgrade to optimistic locking solution
|
242
|
+
return if stopping?
|
243
|
+
updated = self.class.where(id: id).where.not(status: STOPPING_STATUSES)
|
244
|
+
.update_all(status: :running, updated_at: Time.now) > 0
|
245
|
+
if updated
|
246
|
+
self.status = :running
|
247
|
+
clear_attribute_changes([:status])
|
248
|
+
else
|
249
|
+
reload_status
|
250
|
+
end
|
251
|
+
end
|
252
|
+
end
|
253
|
+
|
254
|
+
# Starts a Run, setting its started_at timestamp and tick_total.
|
255
|
+
#
|
256
|
+
# @param count [Integer] the total iterations to be performed, as
|
257
|
+
# specified by the Task.
|
258
|
+
def start(count)
|
259
|
+
update!(started_at: Time.now, tick_total: count)
|
260
|
+
run_task_callbacks(:start)
|
261
|
+
rescue ActiveRecord::StaleObjectError
|
262
|
+
reload_status
|
263
|
+
retry
|
264
|
+
end
|
265
|
+
|
266
|
+
# Handles transitioning the status on a Run when the job shuts down.
|
267
|
+
def job_shutdown
|
268
|
+
if cancelling?
|
269
|
+
self.status = :cancelled
|
270
|
+
self.ended_at = Time.now
|
271
|
+
elsif pausing?
|
272
|
+
self.status = :paused
|
186
273
|
else
|
187
|
-
|
274
|
+
self.status = :interrupted
|
188
275
|
end
|
189
276
|
end
|
190
277
|
|
278
|
+
# Handles the completion of a Run, setting a status of succeeded and the
|
279
|
+
# ended_at timestamp.
|
280
|
+
def complete
|
281
|
+
self.status = :succeeded
|
282
|
+
self.ended_at = Time.now
|
283
|
+
end
|
284
|
+
|
191
285
|
# Cancels a Run.
|
192
286
|
#
|
193
287
|
# If the Run is paused, it will transition directly to cancelled, since the
|
@@ -201,10 +295,26 @@ module MaintenanceTasks
|
|
201
295
|
# will be updated.
|
202
296
|
def cancel
|
203
297
|
if paused? || stuck?
|
204
|
-
|
298
|
+
self.status = :cancelled
|
299
|
+
self.ended_at = Time.now
|
300
|
+
persist_transition
|
205
301
|
else
|
206
302
|
cancelling!
|
207
303
|
end
|
304
|
+
rescue ActiveRecord::StaleObjectError
|
305
|
+
reload_status
|
306
|
+
retry
|
307
|
+
end
|
308
|
+
|
309
|
+
# Marks a Run as pausing.
|
310
|
+
#
|
311
|
+
# Rescues and retries status transition if an ActiveRecord::StaleObjectError
|
312
|
+
# is encountered.
|
313
|
+
def pausing!
|
314
|
+
super
|
315
|
+
rescue ActiveRecord::StaleObjectError
|
316
|
+
reload_status
|
317
|
+
retry
|
208
318
|
end
|
209
319
|
|
210
320
|
# Returns whether a Run is stuck, which is defined as having a status of
|
@@ -297,6 +407,12 @@ module MaintenanceTasks
|
|
297
407
|
|
298
408
|
private
|
299
409
|
|
410
|
+
def run_task_callbacks(callback)
|
411
|
+
task.run_callbacks(callback)
|
412
|
+
rescue
|
413
|
+
nil
|
414
|
+
end
|
415
|
+
|
300
416
|
def arguments_match_task_attributes
|
301
417
|
invalid_argument_keys = arguments.keys - task.attribute_names
|
302
418
|
if invalid_argument_keys.any?
|
@@ -306,5 +422,11 @@ module MaintenanceTasks
|
|
306
422
|
errors.add(:base, error_message)
|
307
423
|
end
|
308
424
|
end
|
425
|
+
|
426
|
+
def truncate(attribute_name, value)
|
427
|
+
limit = self.class.column_for_attribute(attribute_name).limit
|
428
|
+
return value unless limit
|
429
|
+
value&.first(limit)
|
430
|
+
end
|
309
431
|
end
|
310
432
|
end
|
@@ -18,6 +18,7 @@ module MaintenanceTasks
|
|
18
18
|
# @api private
|
19
19
|
class_attribute :throttle_conditions, default: []
|
20
20
|
|
21
|
+
# @api private
|
21
22
|
class_attribute :collection_builder_strategy,
|
22
23
|
default: NullCollectionBuilder.new
|
23
24
|
|
@@ -59,23 +60,27 @@ module MaintenanceTasks
|
|
59
60
|
"To resolve this issue run: bin/rails active_storage:install"
|
60
61
|
end
|
61
62
|
|
62
|
-
self.collection_builder_strategy =
|
63
|
+
self.collection_builder_strategy =
|
64
|
+
MaintenanceTasks::CsvCollectionBuilder.new
|
63
65
|
end
|
64
66
|
|
65
|
-
#
|
66
|
-
#
|
67
|
-
|
68
|
-
|
69
|
-
|
67
|
+
# Make this a Task that calls #process once, instead of iterating over
|
68
|
+
# a collection.
|
69
|
+
def no_collection
|
70
|
+
self.collection_builder_strategy =
|
71
|
+
MaintenanceTasks::NoCollectionBuilder.new
|
70
72
|
end
|
71
73
|
|
74
|
+
delegate :has_csv_content?, :no_collection?,
|
75
|
+
to: :collection_builder_strategy
|
76
|
+
|
72
77
|
# Processes one item.
|
73
78
|
#
|
74
79
|
# Especially useful for tests.
|
75
80
|
#
|
76
|
-
# @param
|
77
|
-
def process(
|
78
|
-
new.process(
|
81
|
+
# @param args [Object, nil] the item to process
|
82
|
+
def process(*args)
|
83
|
+
new.process(*args)
|
79
84
|
end
|
80
85
|
|
81
86
|
# Returns the collection for this Task.
|
@@ -98,14 +103,19 @@ module MaintenanceTasks
|
|
98
103
|
|
99
104
|
# Add a condition under which this Task will be throttled.
|
100
105
|
#
|
101
|
-
# @param backoff [ActiveSupport::Duration]
|
102
|
-
# can be specified. This is the time to wait before retrying the Task
|
103
|
-
# If
|
106
|
+
# @param backoff [ActiveSupport::Duration, #call] a custom backoff
|
107
|
+
# can be specified. This is the time to wait before retrying the Task,
|
108
|
+
# defaulting to 30 seconds. If provided as a Duration, the backoff is
|
109
|
+
# wrapped in a proc. Alternatively,an object responding to call can be
|
110
|
+
# used. It must return an ActiveSupport::Duration.
|
104
111
|
# @yieldreturn [Boolean] where the throttle condition is being met,
|
105
112
|
# indicating that the Task should throttle.
|
106
113
|
def throttle_on(backoff: 30.seconds, &condition)
|
114
|
+
backoff_as_proc = backoff
|
115
|
+
backoff_as_proc = -> { backoff } unless backoff.respond_to?(:call)
|
116
|
+
|
107
117
|
self.throttle_conditions += [
|
108
|
-
{ throttle_on: condition, backoff:
|
118
|
+
{ throttle_on: condition, backoff: backoff_as_proc },
|
109
119
|
]
|
110
120
|
end
|
111
121
|
|
@@ -191,6 +201,13 @@ module MaintenanceTasks
|
|
191
201
|
self.class.has_csv_content?
|
192
202
|
end
|
193
203
|
|
204
|
+
# Returns whether the Task is collection-less.
|
205
|
+
#
|
206
|
+
# @return [Boolean] whether the Task is collection-less.
|
207
|
+
def no_collection?
|
208
|
+
self.class.no_collection?
|
209
|
+
end
|
210
|
+
|
194
211
|
# The collection to be processed, delegated to the strategy.
|
195
212
|
#
|
196
213
|
# @return the collection.
|
@@ -74,7 +74,11 @@ module MaintenanceTasks
|
|
74
74
|
def code
|
75
75
|
return if deleted?
|
76
76
|
task = Task.named(name)
|
77
|
-
file =
|
77
|
+
file = if Object.respond_to?(:const_source_location)
|
78
|
+
Object.const_source_location(task.name).first
|
79
|
+
else
|
80
|
+
task.instance_method(:process).source_location.first
|
81
|
+
end
|
78
82
|
File.read(file)
|
79
83
|
end
|
80
84
|
|
@@ -142,6 +146,13 @@ module MaintenanceTasks
|
|
142
146
|
end
|
143
147
|
end
|
144
148
|
|
149
|
+
# @return [MaintenanceTasks::Task, nil] an instance of the Task class.
|
150
|
+
# @return [nil] if the Task file was deleted.
|
151
|
+
def new
|
152
|
+
return if deleted?
|
153
|
+
MaintenanceTasks::Task.named(name).new
|
154
|
+
end
|
155
|
+
|
145
156
|
private
|
146
157
|
|
147
158
|
def runs
|
@@ -10,6 +10,10 @@
|
|
10
10
|
<%= render "maintenance_tasks/runs/info/#{run.status}", run: run %>
|
11
11
|
</div>
|
12
12
|
|
13
|
+
<div class="content" id="custom-content">
|
14
|
+
<%= render "maintenance_tasks/runs/info/custom", run: run %>
|
15
|
+
</div>
|
16
|
+
|
13
17
|
<%= render "maintenance_tasks/runs/csv", run: run %>
|
14
18
|
<%= tag.hr if run.csv_file.present? && run.arguments.present? %>
|
15
19
|
<%= render "maintenance_tasks/runs/arguments", run: run %>
|
File without changes
|
File without changes
|
@@ -15,6 +15,10 @@
|
|
15
15
|
<%= render "maintenance_tasks/runs/info/#{run.status}", run: run %>
|
16
16
|
</div>
|
17
17
|
|
18
|
+
<div class="content" id="custom-content">
|
19
|
+
<%= render "maintenance_tasks/runs/info/custom", run: run %>
|
20
|
+
</div>
|
21
|
+
|
18
22
|
<%= render "maintenance_tasks/runs/csv", run: run %>
|
19
23
|
<%= tag.hr if run.csv_file.present? && run.arguments.present? %>
|
20
24
|
<%= render "maintenance_tasks/runs/arguments", run: run %>
|
@@ -16,6 +16,10 @@
|
|
16
16
|
<%= render "maintenance_tasks/runs/info/#{last_run.status}", run: last_run %>
|
17
17
|
</div>
|
18
18
|
|
19
|
+
<div class="content" id="custom-content">
|
20
|
+
<%= render "maintenance_tasks/runs/info/custom", run: last_run %>
|
21
|
+
</div>
|
22
|
+
|
19
23
|
<%= render "maintenance_tasks/runs/csv", run: last_run %>
|
20
24
|
<%= tag.hr if last_run.csv_file.present? %>
|
21
25
|
<%= render "maintenance_tasks/runs/arguments", run: last_run %>
|
@@ -31,20 +35,22 @@
|
|
31
35
|
<%= form.file_field :csv_file %>
|
32
36
|
</div>
|
33
37
|
<% end %>
|
34
|
-
<%
|
38
|
+
<% parameter_names = @task.parameter_names %>
|
39
|
+
<% if parameter_names.any? %>
|
35
40
|
<div class="block">
|
36
|
-
<%= form.fields_for :task_arguments do |ff| %>
|
37
|
-
<%
|
41
|
+
<%= form.fields_for :task_arguments, @task.new do |ff| %>
|
42
|
+
<% parameter_names.each do |parameter_name| %>
|
38
43
|
<div class="field">
|
39
|
-
<%= ff.label
|
44
|
+
<%= ff.label parameter_name, parameter_name, class: "label is-family-monospace" %>
|
40
45
|
<div class="control">
|
41
|
-
<%= ff
|
46
|
+
<%= parameter_field(ff, parameter_name) %>
|
42
47
|
</div>
|
43
48
|
</div>
|
44
49
|
<% end %>
|
45
50
|
<% end %>
|
46
51
|
</div>
|
47
52
|
<% end %>
|
53
|
+
<%= render "maintenance_tasks/tasks/custom", form: form %>
|
48
54
|
<div class="block">
|
49
55
|
<%= form.submit 'Run', class: "button is-success", disabled: @task.deleted? %>
|
50
56
|
</div>
|
@@ -12,10 +12,17 @@ module MaintenanceTasks
|
|
12
12
|
class_option :csv, type: :boolean, default: false,
|
13
13
|
desc: "Generate a CSV Task."
|
14
14
|
|
15
|
+
class_option :no_collection, type: :boolean, default: false,
|
16
|
+
desc: "Generate a collection-less Task."
|
17
|
+
|
15
18
|
check_class_collision suffix: "Task"
|
16
19
|
|
17
20
|
# Creates the Task file.
|
18
21
|
def create_task_file
|
22
|
+
if options[:csv] && options[:no_collection]
|
23
|
+
raise "Multiple Task type options provided. Please use either "\
|
24
|
+
"--csv or --no-collection."
|
25
|
+
end
|
19
26
|
template_file = File.join(
|
20
27
|
"app/tasks/#{tasks_module_file_path}",
|
21
28
|
class_path,
|
@@ -23,6 +30,8 @@ module MaintenanceTasks
|
|
23
30
|
)
|
24
31
|
if options[:csv]
|
25
32
|
template("csv_task.rb", template_file)
|
33
|
+
elsif no_collection?
|
34
|
+
template("no_collection_task.rb", template_file)
|
26
35
|
else
|
27
36
|
template("task.rb", template_file)
|
28
37
|
end
|
@@ -76,5 +85,9 @@ module MaintenanceTasks
|
|
76
85
|
def test_framework
|
77
86
|
Rails.application.config.generators.options[:rails][:test_framework]
|
78
87
|
end
|
88
|
+
|
89
|
+
def no_collection?
|
90
|
+
options[:no_collection]
|
91
|
+
end
|
79
92
|
end
|
80
93
|
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'test_helper'
|
3
|
+
|
4
|
+
module <%= tasks_module %>
|
5
|
+
<% module_namespacing do -%>
|
6
|
+
class <%= class_name %>TaskTest < ActiveSupport::TestCase
|
7
|
+
# test "#process performs a task iteration" do
|
8
|
+
# <%= tasks_module %>::<%= class_name %>Task.process
|
9
|
+
# end
|
10
|
+
end
|
11
|
+
<% end -%>
|
12
|
+
end
|
@@ -5,7 +5,11 @@ module <%= tasks_module %>
|
|
5
5
|
<% module_namespacing do -%>
|
6
6
|
class <%= class_name %>TaskTest < ActiveSupport::TestCase
|
7
7
|
# test "#process performs a task iteration" do
|
8
|
+
<%- if no_collection? -%>
|
9
|
+
# <%= tasks_module %>::<%= class_name %>Task.process
|
10
|
+
<%- else -%>
|
8
11
|
# <%= tasks_module %>::<%= class_name %>Task.process(element)
|
12
|
+
<%- end -%>
|
9
13
|
# end
|
10
14
|
end
|
11
15
|
<% end -%>
|
@@ -8,7 +8,16 @@ module MaintenanceTasks
|
|
8
8
|
class Engine < ::Rails::Engine
|
9
9
|
isolate_namespace MaintenanceTasks
|
10
10
|
|
11
|
-
initializer "
|
11
|
+
initializer "maintenance_tasks.warn_classic_autoloader" do
|
12
|
+
unless Rails.autoloaders.zeitwerk_enabled?
|
13
|
+
ActiveSupport::Deprecation.warn(<<~MSG.squish)
|
14
|
+
Autoloading in classic mode is deprecated and support will be removed in the next
|
15
|
+
release of Maintenance Tasks. Please use Zeitwerk to autoload your application.
|
16
|
+
MSG
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
initializer "maintenance_tasks.eager_load_for_classic_autoloader" do
|
12
21
|
eager_load! unless Rails.autoloaders.zeitwerk_enabled?
|
13
22
|
end
|
14
23
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: maintenance_tasks
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.8.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shopify Engineering
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-02-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: actionpack
|
@@ -98,6 +98,7 @@ files:
|
|
98
98
|
- app/jobs/maintenance_tasks/task_job.rb
|
99
99
|
- app/models/maintenance_tasks/application_record.rb
|
100
100
|
- app/models/maintenance_tasks/csv_collection_builder.rb
|
101
|
+
- app/models/maintenance_tasks/no_collection_builder.rb
|
101
102
|
- app/models/maintenance_tasks/null_collection_builder.rb
|
102
103
|
- app/models/maintenance_tasks/progress.rb
|
103
104
|
- app/models/maintenance_tasks/run.rb
|
@@ -114,6 +115,7 @@ files:
|
|
114
115
|
- app/views/maintenance_tasks/runs/_run.html.erb
|
115
116
|
- app/views/maintenance_tasks/runs/info/_cancelled.html.erb
|
116
117
|
- app/views/maintenance_tasks/runs/info/_cancelling.html.erb
|
118
|
+
- app/views/maintenance_tasks/runs/info/_custom.html.erb
|
117
119
|
- app/views/maintenance_tasks/runs/info/_enqueued.html.erb
|
118
120
|
- app/views/maintenance_tasks/runs/info/_errored.html.erb
|
119
121
|
- app/views/maintenance_tasks/runs/info/_interrupted.html.erb
|
@@ -121,6 +123,7 @@ files:
|
|
121
123
|
- app/views/maintenance_tasks/runs/info/_pausing.html.erb
|
122
124
|
- app/views/maintenance_tasks/runs/info/_running.html.erb
|
123
125
|
- app/views/maintenance_tasks/runs/info/_succeeded.html.erb
|
126
|
+
- app/views/maintenance_tasks/tasks/_custom.html.erb
|
124
127
|
- app/views/maintenance_tasks/tasks/_task.html.erb
|
125
128
|
- app/views/maintenance_tasks/tasks/index.html.erb
|
126
129
|
- app/views/maintenance_tasks/tasks/show.html.erb
|
@@ -128,10 +131,13 @@ files:
|
|
128
131
|
- db/migrate/20201211151756_create_maintenance_tasks_runs.rb
|
129
132
|
- db/migrate/20210225152418_remove_index_on_task_name.rb
|
130
133
|
- db/migrate/20210517131953_add_arguments_to_maintenance_tasks_runs.rb
|
134
|
+
- db/migrate/20211210152329_add_lock_version_to_maintenance_tasks_runs.rb
|
131
135
|
- exe/maintenance_tasks
|
132
136
|
- lib/generators/maintenance_tasks/install_generator.rb
|
133
137
|
- lib/generators/maintenance_tasks/task_generator.rb
|
134
138
|
- lib/generators/maintenance_tasks/templates/csv_task.rb.tt
|
139
|
+
- lib/generators/maintenance_tasks/templates/no_collection_task.rb.tt
|
140
|
+
- lib/generators/maintenance_tasks/templates/no_collection_task_test.rb.tt
|
135
141
|
- lib/generators/maintenance_tasks/templates/task.rb.tt
|
136
142
|
- lib/generators/maintenance_tasks/templates/task_spec.rb.tt
|
137
143
|
- lib/generators/maintenance_tasks/templates/task_test.rb.tt
|
@@ -144,7 +150,7 @@ homepage: https://github.com/Shopify/maintenance_tasks
|
|
144
150
|
licenses:
|
145
151
|
- MIT
|
146
152
|
metadata:
|
147
|
-
source_code_uri: https://github.com/Shopify/maintenance_tasks/tree/v1.
|
153
|
+
source_code_uri: https://github.com/Shopify/maintenance_tasks/tree/v1.8.1
|
148
154
|
allowed_push_host: https://rubygems.org
|
149
155
|
post_install_message:
|
150
156
|
rdoc_options: []
|