maintenance_tasks 1.6.0 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +53 -8
- data/app/controllers/maintenance_tasks/tasks_controller.rb +2 -1
- data/app/helpers/maintenance_tasks/tasks_helper.rb +19 -0
- data/app/jobs/concerns/maintenance_tasks/task_job_concern.rb +23 -31
- data/app/models/maintenance_tasks/csv_collection_builder.rb +9 -3
- data/app/models/maintenance_tasks/no_collection_builder.rb +29 -0
- data/app/models/maintenance_tasks/null_collection_builder.rb +7 -0
- data/app/models/maintenance_tasks/run.rb +138 -16
- data/app/models/maintenance_tasks/task.rb +30 -13
- data/app/models/maintenance_tasks/task_data.rb +12 -1
- data/app/views/maintenance_tasks/runs/_run.html.erb +4 -0
- data/app/views/maintenance_tasks/runs/info/_custom.html.erb +0 -0
- data/app/views/maintenance_tasks/tasks/_custom.html.erb +0 -0
- data/app/views/maintenance_tasks/tasks/_task.html.erb +4 -0
- data/app/views/maintenance_tasks/tasks/show.html.erb +11 -5
- data/db/migrate/20211210152329_add_lock_version_to_maintenance_tasks_runs.rb +8 -0
- data/lib/generators/maintenance_tasks/task_generator.rb +13 -0
- data/lib/generators/maintenance_tasks/templates/no_collection_task.rb.tt +13 -0
- data/lib/generators/maintenance_tasks/templates/no_collection_task_test.rb.tt +12 -0
- data/lib/generators/maintenance_tasks/templates/task_test.rb.tt +4 -0
- data/lib/maintenance_tasks/engine.rb +10 -1
- metadata +9 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6dbf3120a28de1ce88571561ba3be52d160743d365f862a65573b24fcc64cc24
|
4
|
+
data.tar.gz: 87fda9aaf48c7643a85cc542cbda6d89721ea0b656396329abd9a2d4a42cab68
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 77cc4fad4f591683cb3b6062265c487efb8f15b7f31aca2620730af3b74dd57296f8451d7077256fe2f5718762b8ba3de76e7f1affe2c888c202c585a781a570
|
7
|
+
data.tar.gz: 15e9c05f39cc0cc6216b66081ec051eebd4a669e5c5e002672f2028d5a160fa8dd4a9144e726bc455aa7a60b96018c2fe1a572079a9d705d7b299b7762e483df
|
data/README.md
CHANGED
@@ -114,8 +114,9 @@ title,content
|
|
114
114
|
My Title,Hello World!
|
115
115
|
```
|
116
116
|
|
117
|
-
The files uploaded to your Active Storage service provider will be renamed
|
117
|
+
The files uploaded to your Active Storage service provider will be renamed
|
118
118
|
to include an ISO8601 timestamp and the Task name in snake case format.
|
119
|
+
The CSV is expected to have a trailing newline at the end of the file.
|
119
120
|
|
120
121
|
### Processing Batch Collections
|
121
122
|
|
@@ -159,6 +160,36 @@ primary keys of the records of the batch first, and then perform an additional
|
|
159
160
|
query to load the records when calling `each` (or any `Enumerable` method)
|
160
161
|
inside `#process`.
|
161
162
|
|
163
|
+
### Tasks that don't need a Collection
|
164
|
+
|
165
|
+
Sometimes, you might want to run a Task that performs a single operation, such
|
166
|
+
as enqueuing another background job or hitting an external API. The gem supports
|
167
|
+
collection-less tasks.
|
168
|
+
|
169
|
+
Generate a collection-less Task by running:
|
170
|
+
|
171
|
+
```bash
|
172
|
+
$ bin/rails generate maintenance_tasks:task no_collection_task --no-collection
|
173
|
+
```
|
174
|
+
|
175
|
+
The generated task is a subclass of `MaintenanceTasks::Task` that implements:
|
176
|
+
|
177
|
+
* `process`: do the work of your maintenance task
|
178
|
+
|
179
|
+
```ruby
|
180
|
+
# app/tasks/maintenance/no_collection_task.rb
|
181
|
+
|
182
|
+
module Maintenance
|
183
|
+
class NoCollectionTask < MaintenanceTasks::Task
|
184
|
+
no_collection
|
185
|
+
|
186
|
+
def process
|
187
|
+
SomeAsyncJob.perform_later
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|
191
|
+
```
|
192
|
+
|
162
193
|
### Throttling
|
163
194
|
|
164
195
|
Maintenance Tasks often modify a lot of data and can be taxing on your database.
|
@@ -200,6 +231,20 @@ Tasks can define multiple throttle conditions. Throttle conditions are inherited
|
|
200
231
|
by descendants, and new conditions will be appended without impacting existing
|
201
232
|
conditions.
|
202
233
|
|
234
|
+
The backoff can also be specified as a proc:
|
235
|
+
|
236
|
+
```ruby
|
237
|
+
# app/tasks/maintenance/update_posts_throttled_task.rb
|
238
|
+
|
239
|
+
module Maintenance
|
240
|
+
class UpdatePostsThrottledTask < MaintenanceTasks::Task
|
241
|
+
throttle_on(backoff: -> { RandomBackoffGenerator.generate_duration } ) do
|
242
|
+
DatabaseStatus.unhealthy?
|
243
|
+
end
|
244
|
+
...
|
245
|
+
end
|
246
|
+
end
|
247
|
+
```
|
203
248
|
### Custom Task Parameters
|
204
249
|
|
205
250
|
Tasks may need additional information, supplied via parameters, to run.
|
@@ -241,12 +286,12 @@ The Task provides callbacks that hook into its life cycle.
|
|
241
286
|
|
242
287
|
Available callbacks are:
|
243
288
|
|
244
|
-
`after_start`
|
245
|
-
`after_pause`
|
246
|
-
`after_interrupt`
|
247
|
-
`after_cancel`
|
248
|
-
`after_complete`
|
249
|
-
`after_error`
|
289
|
+
* `after_start`
|
290
|
+
* `after_pause`
|
291
|
+
* `after_interrupt`
|
292
|
+
* `after_cancel`
|
293
|
+
* `after_complete`
|
294
|
+
* `after_error`
|
250
295
|
|
251
296
|
```ruby
|
252
297
|
module Maintenance
|
@@ -542,7 +587,7 @@ you can define an error handler:
|
|
542
587
|
|
543
588
|
MaintenanceTasks.error_handler = ->(error, task_context, _errored_element) do
|
544
589
|
Bugsnag.notify(error) do |notification|
|
545
|
-
notification.
|
590
|
+
notification.add_metadata(:task, task_context)
|
546
591
|
end
|
547
592
|
end
|
548
593
|
```
|
@@ -24,11 +24,12 @@ module MaintenanceTasks
|
|
24
24
|
end
|
25
25
|
|
26
26
|
# Runs a given Task and redirects to the Task page.
|
27
|
-
def run
|
27
|
+
def run(&block)
|
28
28
|
task = Runner.run(
|
29
29
|
name: params.fetch(:id),
|
30
30
|
csv_file: params[:csv_file],
|
31
31
|
arguments: params.fetch(:task_arguments, {}).permit!.to_h,
|
32
|
+
&block
|
32
33
|
)
|
33
34
|
redirect_to(task_path(task))
|
34
35
|
rescue ActiveRecord::RecordInvalid => error
|
@@ -100,5 +100,24 @@ module MaintenanceTasks
|
|
100
100
|
only_path: true
|
101
101
|
)
|
102
102
|
end
|
103
|
+
|
104
|
+
# Return the appropriate field tag for the parameter
|
105
|
+
def parameter_field(form_builder, parameter_name)
|
106
|
+
case form_builder.object.class.attribute_types[parameter_name]
|
107
|
+
when ActiveModel::Type::Integer, ActiveModel::Type::Decimal,
|
108
|
+
ActiveModel::Type::Float
|
109
|
+
form_builder.number_field(parameter_name)
|
110
|
+
when ActiveModel::Type::DateTime
|
111
|
+
form_builder.datetime_field(parameter_name)
|
112
|
+
when ActiveModel::Type::Date
|
113
|
+
form_builder.date_field(parameter_name)
|
114
|
+
when ActiveModel::Type::Time
|
115
|
+
form_builder.time_field(parameter_name)
|
116
|
+
when ActiveModel::Type::Boolean
|
117
|
+
form_builder.check_box(parameter_name)
|
118
|
+
else
|
119
|
+
form_builder.text_area(parameter_name, class: "textarea")
|
120
|
+
end
|
121
|
+
end
|
103
122
|
end
|
104
123
|
end
|
@@ -12,8 +12,8 @@ module MaintenanceTasks
|
|
12
12
|
before_perform(:before_perform)
|
13
13
|
|
14
14
|
on_start(:on_start)
|
15
|
-
on_complete(:on_complete)
|
16
15
|
on_shutdown(:on_shutdown)
|
16
|
+
on_complete(:on_complete)
|
17
17
|
|
18
18
|
after_perform(:after_perform)
|
19
19
|
|
@@ -36,6 +36,8 @@ module MaintenanceTasks
|
|
36
36
|
@enumerator = nil
|
37
37
|
|
38
38
|
collection_enum = case collection
|
39
|
+
when :no_collection
|
40
|
+
enumerator_builder.build_once_enumerator(cursor: nil)
|
39
41
|
when ActiveRecord::Relation
|
40
42
|
enumerator_builder.active_record_on_records(collection, cursor: cursor)
|
41
43
|
when ActiveRecord::Batches::BatchEnumerator
|
@@ -63,9 +65,16 @@ module MaintenanceTasks
|
|
63
65
|
Array, or CSV.
|
64
66
|
MSG
|
65
67
|
end
|
68
|
+
throttle_enumerator(collection_enum)
|
69
|
+
end
|
66
70
|
|
71
|
+
def throttle_enumerator(collection_enum)
|
67
72
|
@task.throttle_conditions.reduce(collection_enum) do |enum, condition|
|
68
|
-
enumerator_builder.build_throttle_enumerator(
|
73
|
+
enumerator_builder.build_throttle_enumerator(
|
74
|
+
enum,
|
75
|
+
throttle_on: condition[:throttle_on],
|
76
|
+
backoff: condition[:backoff].call
|
77
|
+
)
|
69
78
|
end
|
70
79
|
end
|
71
80
|
|
@@ -82,7 +91,11 @@ module MaintenanceTasks
|
|
82
91
|
end
|
83
92
|
|
84
93
|
def task_iteration(input)
|
85
|
-
@task.
|
94
|
+
if @task.no_collection?
|
95
|
+
@task.process
|
96
|
+
else
|
97
|
+
@task.process(input)
|
98
|
+
end
|
86
99
|
rescue => error
|
87
100
|
@errored_element = input
|
88
101
|
raise error
|
@@ -105,33 +118,19 @@ module MaintenanceTasks
|
|
105
118
|
def on_start
|
106
119
|
count = @task.count
|
107
120
|
count = @enumerator&.size if count == :no_count
|
108
|
-
@run.
|
109
|
-
@task.run_callbacks(:start)
|
110
|
-
end
|
111
|
-
|
112
|
-
def on_complete
|
113
|
-
@run.status = :succeeded
|
114
|
-
@run.ended_at = Time.now
|
115
|
-
@task.run_callbacks(:complete)
|
121
|
+
@run.start(count)
|
116
122
|
end
|
117
123
|
|
118
124
|
def on_shutdown
|
119
|
-
|
120
|
-
@run.status = :cancelled
|
121
|
-
@task.run_callbacks(:cancel)
|
122
|
-
@run.ended_at = Time.now
|
123
|
-
elsif @run.pausing?
|
124
|
-
@run.status = :paused
|
125
|
-
@task.run_callbacks(:pause)
|
126
|
-
else
|
127
|
-
@run.status = :interrupted
|
128
|
-
@task.run_callbacks(:interrupt)
|
129
|
-
end
|
130
|
-
|
125
|
+
@run.job_shutdown
|
131
126
|
@run.cursor = cursor_position
|
132
127
|
@ticker.persist
|
133
128
|
end
|
134
129
|
|
130
|
+
def on_complete
|
131
|
+
@run.complete
|
132
|
+
end
|
133
|
+
|
135
134
|
# We are reopening a private part of Job Iteration's API here, so we should
|
136
135
|
# ensure the method is still defined upstream. This way, in the case where
|
137
136
|
# the method changes upstream, we catch it at load time instead of at
|
@@ -150,7 +149,7 @@ module MaintenanceTasks
|
|
150
149
|
end
|
151
150
|
|
152
151
|
def after_perform
|
153
|
-
@run.
|
152
|
+
@run.persist_transition
|
154
153
|
if defined?(@reenqueue_iteration_job) && @reenqueue_iteration_job
|
155
154
|
reenqueue_iteration_job(should_ignore: false)
|
156
155
|
end
|
@@ -171,15 +170,8 @@ module MaintenanceTasks
|
|
171
170
|
task_context = {}
|
172
171
|
end
|
173
172
|
errored_element = @errored_element if defined?(@errored_element)
|
174
|
-
run_error_callback
|
175
173
|
ensure
|
176
174
|
MaintenanceTasks.error_handler.call(error, task_context, errored_element)
|
177
175
|
end
|
178
|
-
|
179
|
-
def run_error_callback
|
180
|
-
@task.run_callbacks(:error) if defined?(@task)
|
181
|
-
rescue
|
182
|
-
nil
|
183
|
-
end
|
184
176
|
end
|
185
177
|
end
|
@@ -15,9 +15,10 @@ module MaintenanceTasks
|
|
15
15
|
CSV.new(task.csv_content, headers: true)
|
16
16
|
end
|
17
17
|
|
18
|
-
# The number of rows to be processed. Excludes the header row from the
|
19
|
-
# and
|
20
|
-
# an approximation based on the number of
|
18
|
+
# The number of rows to be processed. Excludes the header row from the
|
19
|
+
# count and assumes a trailing newline is at the end of the CSV file.
|
20
|
+
# Note that this number is an approximation based on the number of
|
21
|
+
# newlines.
|
21
22
|
#
|
22
23
|
# @return [Integer] the approximate number of rows to process.
|
23
24
|
def count(task)
|
@@ -28,5 +29,10 @@ module MaintenanceTasks
|
|
28
29
|
def has_csv_content?
|
29
30
|
true
|
30
31
|
end
|
32
|
+
|
33
|
+
# Returns that the Task processes a collection.
|
34
|
+
def no_collection?
|
35
|
+
false
|
36
|
+
end
|
31
37
|
end
|
32
38
|
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module MaintenanceTasks
|
4
|
+
# Strategy for building a Task that has no collection. These Tasks
|
5
|
+
# consist of a single iteration.
|
6
|
+
#
|
7
|
+
# @api private
|
8
|
+
class NoCollectionBuilder
|
9
|
+
# Specifies that this task does not process a collection.
|
10
|
+
def collection(_task)
|
11
|
+
:no_collection
|
12
|
+
end
|
13
|
+
|
14
|
+
# The number of rows to be processed. Always returns 1.
|
15
|
+
def count(_task)
|
16
|
+
1
|
17
|
+
end
|
18
|
+
|
19
|
+
# Return that the Task does not process CSV content.
|
20
|
+
def has_csv_content?
|
21
|
+
false
|
22
|
+
end
|
23
|
+
|
24
|
+
# Returns that the Task is collection-less.
|
25
|
+
def no_collection?
|
26
|
+
true
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -2,6 +2,8 @@
|
|
2
2
|
|
3
3
|
module MaintenanceTasks
|
4
4
|
# Base strategy for building a collection-based Task to be performed.
|
5
|
+
#
|
6
|
+
# @api private
|
5
7
|
class NullCollectionBuilder
|
6
8
|
# Placeholder method to raise in case a subclass fails to implement the
|
7
9
|
# expected instance method.
|
@@ -27,5 +29,10 @@ module MaintenanceTasks
|
|
27
29
|
def has_csv_content?
|
28
30
|
false
|
29
31
|
end
|
32
|
+
|
33
|
+
# Returns that the Task processes a collection.
|
34
|
+
def no_collection?
|
35
|
+
false
|
36
|
+
end
|
30
37
|
end
|
31
38
|
end
|
@@ -66,9 +66,40 @@ module MaintenanceTasks
|
|
66
66
|
|
67
67
|
# Sets the run status to enqueued, making sure the transition is validated
|
68
68
|
# in case it's already enqueued.
|
69
|
+
#
|
70
|
+
# Rescues and retries status transition if an ActiveRecord::StaleObjectError
|
71
|
+
# is encountered.
|
69
72
|
def enqueued!
|
70
73
|
status_will_change!
|
71
74
|
super
|
75
|
+
rescue ActiveRecord::StaleObjectError
|
76
|
+
reload_status
|
77
|
+
retry
|
78
|
+
end
|
79
|
+
|
80
|
+
CALLBACKS_TRANSITION = {
|
81
|
+
cancelled: :cancel,
|
82
|
+
interrupted: :interrupt,
|
83
|
+
paused: :pause,
|
84
|
+
succeeded: :complete,
|
85
|
+
}.transform_keys(&:to_s)
|
86
|
+
private_constant :CALLBACKS_TRANSITION
|
87
|
+
|
88
|
+
# Saves the run, persisting the transition of its status, and all other
|
89
|
+
# changes to the object.
|
90
|
+
def persist_transition
|
91
|
+
save!
|
92
|
+
callback = CALLBACKS_TRANSITION[status]
|
93
|
+
run_task_callbacks(callback) if callback
|
94
|
+
rescue ActiveRecord::StaleObjectError
|
95
|
+
success = succeeded?
|
96
|
+
reload_status
|
97
|
+
if success
|
98
|
+
self.status = :succeeded
|
99
|
+
else
|
100
|
+
job_shutdown
|
101
|
+
end
|
102
|
+
retry
|
72
103
|
end
|
73
104
|
|
74
105
|
# Increments +tick_count+ by +number_of_ticks+ and +time_running+ by
|
@@ -86,6 +117,11 @@ module MaintenanceTasks
|
|
86
117
|
time_running: duration,
|
87
118
|
touch: true
|
88
119
|
)
|
120
|
+
if locking_enabled?
|
121
|
+
locking_column = self.class.locking_column
|
122
|
+
self[locking_column] += 1
|
123
|
+
clear_attribute_change(locking_column)
|
124
|
+
end
|
89
125
|
end
|
90
126
|
|
91
127
|
# Marks the run as errored and persists the error data.
|
@@ -95,25 +131,39 @@ module MaintenanceTasks
|
|
95
131
|
self.started_at ||= Time.now
|
96
132
|
update!(
|
97
133
|
status: :errored,
|
98
|
-
error_class: error.class.
|
99
|
-
error_message: error.message,
|
134
|
+
error_class: truncate(:error_class, error.class.name),
|
135
|
+
error_message: truncate(:error_message, error.message),
|
100
136
|
backtrace: MaintenanceTasks.backtrace_cleaner.clean(error.backtrace),
|
101
137
|
ended_at: Time.now,
|
102
138
|
)
|
139
|
+
run_task_callbacks(:error)
|
140
|
+
rescue ActiveRecord::StaleObjectError
|
141
|
+
reload_status
|
142
|
+
retry
|
103
143
|
end
|
104
144
|
|
105
|
-
# Refreshes
|
106
|
-
# ensures ActiveModel::Dirty
|
145
|
+
# Refreshes the status and lock version attributes on the Active Record
|
146
|
+
# object, and ensures ActiveModel::Dirty doesn't mark the object as changed.
|
147
|
+
#
|
107
148
|
# This allows us to get the Run's most up-to-date status without needing
|
108
149
|
# to reload the entire record.
|
109
150
|
#
|
110
151
|
# @return [MaintenanceTasks::Run] the Run record with its updated status.
|
111
152
|
def reload_status
|
112
|
-
|
113
|
-
self.class.
|
153
|
+
columns_to_reload = if locking_enabled?
|
154
|
+
[:status, self.class.locking_column]
|
155
|
+
else
|
156
|
+
[:status]
|
157
|
+
end
|
158
|
+
updated_status, updated_lock_version = self.class.uncached do
|
159
|
+
self.class.where(id: id).pluck(*columns_to_reload).first
|
114
160
|
end
|
161
|
+
|
115
162
|
self.status = updated_status
|
116
|
-
|
163
|
+
if updated_lock_version
|
164
|
+
self[self.class.locking_column] = updated_lock_version
|
165
|
+
end
|
166
|
+
clear_attribute_changes(columns_to_reload)
|
117
167
|
self
|
118
168
|
end
|
119
169
|
|
@@ -173,21 +223,65 @@ module MaintenanceTasks
|
|
173
223
|
seconds_to_finished.seconds
|
174
224
|
end
|
175
225
|
|
176
|
-
#
|
226
|
+
# Marks a Run as running.
|
177
227
|
#
|
178
228
|
# If the run is stopping already, it will not transition to running.
|
229
|
+
# Rescues and retries status transition if an ActiveRecord::StaleObjectError
|
230
|
+
# is encountered.
|
179
231
|
def running
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
232
|
+
if locking_enabled?
|
233
|
+
begin
|
234
|
+
running! unless stopping?
|
235
|
+
rescue ActiveRecord::StaleObjectError
|
236
|
+
reload_status
|
237
|
+
retry
|
238
|
+
end
|
239
|
+
else
|
240
|
+
# Preserve swap-and-replace solution for data races until users
|
241
|
+
# run migration to upgrade to optimistic locking solution
|
242
|
+
return if stopping?
|
243
|
+
updated = self.class.where(id: id).where.not(status: STOPPING_STATUSES)
|
244
|
+
.update_all(status: :running, updated_at: Time.now) > 0
|
245
|
+
if updated
|
246
|
+
self.status = :running
|
247
|
+
clear_attribute_changes([:status])
|
248
|
+
else
|
249
|
+
reload_status
|
250
|
+
end
|
251
|
+
end
|
252
|
+
end
|
253
|
+
|
254
|
+
# Starts a Run, setting its started_at timestamp and tick_total.
|
255
|
+
#
|
256
|
+
# @param count [Integer] the total iterations to be performed, as
|
257
|
+
# specified by the Task.
|
258
|
+
def start(count)
|
259
|
+
update!(started_at: Time.now, tick_total: count)
|
260
|
+
run_task_callbacks(:start)
|
261
|
+
rescue ActiveRecord::StaleObjectError
|
262
|
+
reload_status
|
263
|
+
retry
|
264
|
+
end
|
265
|
+
|
266
|
+
# Handles transitioning the status on a Run when the job shuts down.
|
267
|
+
def job_shutdown
|
268
|
+
if cancelling?
|
269
|
+
self.status = :cancelled
|
270
|
+
self.ended_at = Time.now
|
271
|
+
elsif pausing?
|
272
|
+
self.status = :paused
|
186
273
|
else
|
187
|
-
|
274
|
+
self.status = :interrupted
|
188
275
|
end
|
189
276
|
end
|
190
277
|
|
278
|
+
# Handles the completion of a Run, setting a status of succeeded and the
|
279
|
+
# ended_at timestamp.
|
280
|
+
def complete
|
281
|
+
self.status = :succeeded
|
282
|
+
self.ended_at = Time.now
|
283
|
+
end
|
284
|
+
|
191
285
|
# Cancels a Run.
|
192
286
|
#
|
193
287
|
# If the Run is paused, it will transition directly to cancelled, since the
|
@@ -201,10 +295,26 @@ module MaintenanceTasks
|
|
201
295
|
# will be updated.
|
202
296
|
def cancel
|
203
297
|
if paused? || stuck?
|
204
|
-
|
298
|
+
self.status = :cancelled
|
299
|
+
self.ended_at = Time.now
|
300
|
+
persist_transition
|
205
301
|
else
|
206
302
|
cancelling!
|
207
303
|
end
|
304
|
+
rescue ActiveRecord::StaleObjectError
|
305
|
+
reload_status
|
306
|
+
retry
|
307
|
+
end
|
308
|
+
|
309
|
+
# Marks a Run as pausing.
|
310
|
+
#
|
311
|
+
# Rescues and retries status transition if an ActiveRecord::StaleObjectError
|
312
|
+
# is encountered.
|
313
|
+
def pausing!
|
314
|
+
super
|
315
|
+
rescue ActiveRecord::StaleObjectError
|
316
|
+
reload_status
|
317
|
+
retry
|
208
318
|
end
|
209
319
|
|
210
320
|
# Returns whether a Run is stuck, which is defined as having a status of
|
@@ -297,6 +407,12 @@ module MaintenanceTasks
|
|
297
407
|
|
298
408
|
private
|
299
409
|
|
410
|
+
def run_task_callbacks(callback)
|
411
|
+
task.run_callbacks(callback)
|
412
|
+
rescue
|
413
|
+
nil
|
414
|
+
end
|
415
|
+
|
300
416
|
def arguments_match_task_attributes
|
301
417
|
invalid_argument_keys = arguments.keys - task.attribute_names
|
302
418
|
if invalid_argument_keys.any?
|
@@ -306,5 +422,11 @@ module MaintenanceTasks
|
|
306
422
|
errors.add(:base, error_message)
|
307
423
|
end
|
308
424
|
end
|
425
|
+
|
426
|
+
def truncate(attribute_name, value)
|
427
|
+
limit = self.class.column_for_attribute(attribute_name).limit
|
428
|
+
return value unless limit
|
429
|
+
value&.first(limit)
|
430
|
+
end
|
309
431
|
end
|
310
432
|
end
|
@@ -18,6 +18,7 @@ module MaintenanceTasks
|
|
18
18
|
# @api private
|
19
19
|
class_attribute :throttle_conditions, default: []
|
20
20
|
|
21
|
+
# @api private
|
21
22
|
class_attribute :collection_builder_strategy,
|
22
23
|
default: NullCollectionBuilder.new
|
23
24
|
|
@@ -59,23 +60,27 @@ module MaintenanceTasks
|
|
59
60
|
"To resolve this issue run: bin/rails active_storage:install"
|
60
61
|
end
|
61
62
|
|
62
|
-
self.collection_builder_strategy =
|
63
|
+
self.collection_builder_strategy =
|
64
|
+
MaintenanceTasks::CsvCollectionBuilder.new
|
63
65
|
end
|
64
66
|
|
65
|
-
#
|
66
|
-
#
|
67
|
-
|
68
|
-
|
69
|
-
|
67
|
+
# Make this a Task that calls #process once, instead of iterating over
|
68
|
+
# a collection.
|
69
|
+
def no_collection
|
70
|
+
self.collection_builder_strategy =
|
71
|
+
MaintenanceTasks::NoCollectionBuilder.new
|
70
72
|
end
|
71
73
|
|
74
|
+
delegate :has_csv_content?, :no_collection?,
|
75
|
+
to: :collection_builder_strategy
|
76
|
+
|
72
77
|
# Processes one item.
|
73
78
|
#
|
74
79
|
# Especially useful for tests.
|
75
80
|
#
|
76
|
-
# @param
|
77
|
-
def process(
|
78
|
-
new.process(
|
81
|
+
# @param args [Object, nil] the item to process
|
82
|
+
def process(*args)
|
83
|
+
new.process(*args)
|
79
84
|
end
|
80
85
|
|
81
86
|
# Returns the collection for this Task.
|
@@ -98,14 +103,19 @@ module MaintenanceTasks
|
|
98
103
|
|
99
104
|
# Add a condition under which this Task will be throttled.
|
100
105
|
#
|
101
|
-
# @param backoff [ActiveSupport::Duration]
|
102
|
-
# can be specified. This is the time to wait before retrying the Task
|
103
|
-
# If
|
106
|
+
# @param backoff [ActiveSupport::Duration, #call] a custom backoff
|
107
|
+
# can be specified. This is the time to wait before retrying the Task,
|
108
|
+
# defaulting to 30 seconds. If provided as a Duration, the backoff is
|
109
|
+
# wrapped in a proc. Alternatively,an object responding to call can be
|
110
|
+
# used. It must return an ActiveSupport::Duration.
|
104
111
|
# @yieldreturn [Boolean] where the throttle condition is being met,
|
105
112
|
# indicating that the Task should throttle.
|
106
113
|
def throttle_on(backoff: 30.seconds, &condition)
|
114
|
+
backoff_as_proc = backoff
|
115
|
+
backoff_as_proc = -> { backoff } unless backoff.respond_to?(:call)
|
116
|
+
|
107
117
|
self.throttle_conditions += [
|
108
|
-
{ throttle_on: condition, backoff:
|
118
|
+
{ throttle_on: condition, backoff: backoff_as_proc },
|
109
119
|
]
|
110
120
|
end
|
111
121
|
|
@@ -191,6 +201,13 @@ module MaintenanceTasks
|
|
191
201
|
self.class.has_csv_content?
|
192
202
|
end
|
193
203
|
|
204
|
+
# Returns whether the Task is collection-less.
|
205
|
+
#
|
206
|
+
# @return [Boolean] whether the Task is collection-less.
|
207
|
+
def no_collection?
|
208
|
+
self.class.no_collection?
|
209
|
+
end
|
210
|
+
|
194
211
|
# The collection to be processed, delegated to the strategy.
|
195
212
|
#
|
196
213
|
# @return the collection.
|
@@ -74,7 +74,11 @@ module MaintenanceTasks
|
|
74
74
|
def code
|
75
75
|
return if deleted?
|
76
76
|
task = Task.named(name)
|
77
|
-
file =
|
77
|
+
file = if Object.respond_to?(:const_source_location)
|
78
|
+
Object.const_source_location(task.name).first
|
79
|
+
else
|
80
|
+
task.instance_method(:process).source_location.first
|
81
|
+
end
|
78
82
|
File.read(file)
|
79
83
|
end
|
80
84
|
|
@@ -142,6 +146,13 @@ module MaintenanceTasks
|
|
142
146
|
end
|
143
147
|
end
|
144
148
|
|
149
|
+
# @return [MaintenanceTasks::Task, nil] an instance of the Task class.
|
150
|
+
# @return [nil] if the Task file was deleted.
|
151
|
+
def new
|
152
|
+
return if deleted?
|
153
|
+
MaintenanceTasks::Task.named(name).new
|
154
|
+
end
|
155
|
+
|
145
156
|
private
|
146
157
|
|
147
158
|
def runs
|
@@ -10,6 +10,10 @@
|
|
10
10
|
<%= render "maintenance_tasks/runs/info/#{run.status}", run: run %>
|
11
11
|
</div>
|
12
12
|
|
13
|
+
<div class="content" id="custom-content">
|
14
|
+
<%= render "maintenance_tasks/runs/info/custom", run: run %>
|
15
|
+
</div>
|
16
|
+
|
13
17
|
<%= render "maintenance_tasks/runs/csv", run: run %>
|
14
18
|
<%= tag.hr if run.csv_file.present? && run.arguments.present? %>
|
15
19
|
<%= render "maintenance_tasks/runs/arguments", run: run %>
|
File without changes
|
File without changes
|
@@ -15,6 +15,10 @@
|
|
15
15
|
<%= render "maintenance_tasks/runs/info/#{run.status}", run: run %>
|
16
16
|
</div>
|
17
17
|
|
18
|
+
<div class="content" id="custom-content">
|
19
|
+
<%= render "maintenance_tasks/runs/info/custom", run: run %>
|
20
|
+
</div>
|
21
|
+
|
18
22
|
<%= render "maintenance_tasks/runs/csv", run: run %>
|
19
23
|
<%= tag.hr if run.csv_file.present? && run.arguments.present? %>
|
20
24
|
<%= render "maintenance_tasks/runs/arguments", run: run %>
|
@@ -16,6 +16,10 @@
|
|
16
16
|
<%= render "maintenance_tasks/runs/info/#{last_run.status}", run: last_run %>
|
17
17
|
</div>
|
18
18
|
|
19
|
+
<div class="content" id="custom-content">
|
20
|
+
<%= render "maintenance_tasks/runs/info/custom", run: last_run %>
|
21
|
+
</div>
|
22
|
+
|
19
23
|
<%= render "maintenance_tasks/runs/csv", run: last_run %>
|
20
24
|
<%= tag.hr if last_run.csv_file.present? %>
|
21
25
|
<%= render "maintenance_tasks/runs/arguments", run: last_run %>
|
@@ -31,20 +35,22 @@
|
|
31
35
|
<%= form.file_field :csv_file %>
|
32
36
|
</div>
|
33
37
|
<% end %>
|
34
|
-
<%
|
38
|
+
<% parameter_names = @task.parameter_names %>
|
39
|
+
<% if parameter_names.any? %>
|
35
40
|
<div class="block">
|
36
|
-
<%= form.fields_for :task_arguments do |ff| %>
|
37
|
-
<%
|
41
|
+
<%= form.fields_for :task_arguments, @task.new do |ff| %>
|
42
|
+
<% parameter_names.each do |parameter_name| %>
|
38
43
|
<div class="field">
|
39
|
-
<%= ff.label
|
44
|
+
<%= ff.label parameter_name, parameter_name, class: "label is-family-monospace" %>
|
40
45
|
<div class="control">
|
41
|
-
<%= ff
|
46
|
+
<%= parameter_field(ff, parameter_name) %>
|
42
47
|
</div>
|
43
48
|
</div>
|
44
49
|
<% end %>
|
45
50
|
<% end %>
|
46
51
|
</div>
|
47
52
|
<% end %>
|
53
|
+
<%= render "maintenance_tasks/tasks/custom", form: form %>
|
48
54
|
<div class="block">
|
49
55
|
<%= form.submit 'Run', class: "button is-success", disabled: @task.deleted? %>
|
50
56
|
</div>
|
@@ -12,10 +12,17 @@ module MaintenanceTasks
|
|
12
12
|
class_option :csv, type: :boolean, default: false,
|
13
13
|
desc: "Generate a CSV Task."
|
14
14
|
|
15
|
+
class_option :no_collection, type: :boolean, default: false,
|
16
|
+
desc: "Generate a collection-less Task."
|
17
|
+
|
15
18
|
check_class_collision suffix: "Task"
|
16
19
|
|
17
20
|
# Creates the Task file.
|
18
21
|
def create_task_file
|
22
|
+
if options[:csv] && options[:no_collection]
|
23
|
+
raise "Multiple Task type options provided. Please use either "\
|
24
|
+
"--csv or --no-collection."
|
25
|
+
end
|
19
26
|
template_file = File.join(
|
20
27
|
"app/tasks/#{tasks_module_file_path}",
|
21
28
|
class_path,
|
@@ -23,6 +30,8 @@ module MaintenanceTasks
|
|
23
30
|
)
|
24
31
|
if options[:csv]
|
25
32
|
template("csv_task.rb", template_file)
|
33
|
+
elsif no_collection?
|
34
|
+
template("no_collection_task.rb", template_file)
|
26
35
|
else
|
27
36
|
template("task.rb", template_file)
|
28
37
|
end
|
@@ -76,5 +85,9 @@ module MaintenanceTasks
|
|
76
85
|
def test_framework
|
77
86
|
Rails.application.config.generators.options[:rails][:test_framework]
|
78
87
|
end
|
88
|
+
|
89
|
+
def no_collection?
|
90
|
+
options[:no_collection]
|
91
|
+
end
|
79
92
|
end
|
80
93
|
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'test_helper'
|
3
|
+
|
4
|
+
module <%= tasks_module %>
|
5
|
+
<% module_namespacing do -%>
|
6
|
+
class <%= class_name %>TaskTest < ActiveSupport::TestCase
|
7
|
+
# test "#process performs a task iteration" do
|
8
|
+
# <%= tasks_module %>::<%= class_name %>Task.process
|
9
|
+
# end
|
10
|
+
end
|
11
|
+
<% end -%>
|
12
|
+
end
|
@@ -5,7 +5,11 @@ module <%= tasks_module %>
|
|
5
5
|
<% module_namespacing do -%>
|
6
6
|
class <%= class_name %>TaskTest < ActiveSupport::TestCase
|
7
7
|
# test "#process performs a task iteration" do
|
8
|
+
<%- if no_collection? -%>
|
9
|
+
# <%= tasks_module %>::<%= class_name %>Task.process
|
10
|
+
<%- else -%>
|
8
11
|
# <%= tasks_module %>::<%= class_name %>Task.process(element)
|
12
|
+
<%- end -%>
|
9
13
|
# end
|
10
14
|
end
|
11
15
|
<% end -%>
|
@@ -8,7 +8,16 @@ module MaintenanceTasks
|
|
8
8
|
class Engine < ::Rails::Engine
|
9
9
|
isolate_namespace MaintenanceTasks
|
10
10
|
|
11
|
-
initializer "
|
11
|
+
initializer "maintenance_tasks.warn_classic_autoloader" do
|
12
|
+
unless Rails.autoloaders.zeitwerk_enabled?
|
13
|
+
ActiveSupport::Deprecation.warn(<<~MSG.squish)
|
14
|
+
Autoloading in classic mode is deprecated and support will be removed in the next
|
15
|
+
release of Maintenance Tasks. Please use Zeitwerk to autoload your application.
|
16
|
+
MSG
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
initializer "maintenance_tasks.eager_load_for_classic_autoloader" do
|
12
21
|
eager_load! unless Rails.autoloaders.zeitwerk_enabled?
|
13
22
|
end
|
14
23
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: maintenance_tasks
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.8.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shopify Engineering
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-02-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: actionpack
|
@@ -98,6 +98,7 @@ files:
|
|
98
98
|
- app/jobs/maintenance_tasks/task_job.rb
|
99
99
|
- app/models/maintenance_tasks/application_record.rb
|
100
100
|
- app/models/maintenance_tasks/csv_collection_builder.rb
|
101
|
+
- app/models/maintenance_tasks/no_collection_builder.rb
|
101
102
|
- app/models/maintenance_tasks/null_collection_builder.rb
|
102
103
|
- app/models/maintenance_tasks/progress.rb
|
103
104
|
- app/models/maintenance_tasks/run.rb
|
@@ -114,6 +115,7 @@ files:
|
|
114
115
|
- app/views/maintenance_tasks/runs/_run.html.erb
|
115
116
|
- app/views/maintenance_tasks/runs/info/_cancelled.html.erb
|
116
117
|
- app/views/maintenance_tasks/runs/info/_cancelling.html.erb
|
118
|
+
- app/views/maintenance_tasks/runs/info/_custom.html.erb
|
117
119
|
- app/views/maintenance_tasks/runs/info/_enqueued.html.erb
|
118
120
|
- app/views/maintenance_tasks/runs/info/_errored.html.erb
|
119
121
|
- app/views/maintenance_tasks/runs/info/_interrupted.html.erb
|
@@ -121,6 +123,7 @@ files:
|
|
121
123
|
- app/views/maintenance_tasks/runs/info/_pausing.html.erb
|
122
124
|
- app/views/maintenance_tasks/runs/info/_running.html.erb
|
123
125
|
- app/views/maintenance_tasks/runs/info/_succeeded.html.erb
|
126
|
+
- app/views/maintenance_tasks/tasks/_custom.html.erb
|
124
127
|
- app/views/maintenance_tasks/tasks/_task.html.erb
|
125
128
|
- app/views/maintenance_tasks/tasks/index.html.erb
|
126
129
|
- app/views/maintenance_tasks/tasks/show.html.erb
|
@@ -128,10 +131,13 @@ files:
|
|
128
131
|
- db/migrate/20201211151756_create_maintenance_tasks_runs.rb
|
129
132
|
- db/migrate/20210225152418_remove_index_on_task_name.rb
|
130
133
|
- db/migrate/20210517131953_add_arguments_to_maintenance_tasks_runs.rb
|
134
|
+
- db/migrate/20211210152329_add_lock_version_to_maintenance_tasks_runs.rb
|
131
135
|
- exe/maintenance_tasks
|
132
136
|
- lib/generators/maintenance_tasks/install_generator.rb
|
133
137
|
- lib/generators/maintenance_tasks/task_generator.rb
|
134
138
|
- lib/generators/maintenance_tasks/templates/csv_task.rb.tt
|
139
|
+
- lib/generators/maintenance_tasks/templates/no_collection_task.rb.tt
|
140
|
+
- lib/generators/maintenance_tasks/templates/no_collection_task_test.rb.tt
|
135
141
|
- lib/generators/maintenance_tasks/templates/task.rb.tt
|
136
142
|
- lib/generators/maintenance_tasks/templates/task_spec.rb.tt
|
137
143
|
- lib/generators/maintenance_tasks/templates/task_test.rb.tt
|
@@ -144,7 +150,7 @@ homepage: https://github.com/Shopify/maintenance_tasks
|
|
144
150
|
licenses:
|
145
151
|
- MIT
|
146
152
|
metadata:
|
147
|
-
source_code_uri: https://github.com/Shopify/maintenance_tasks/tree/v1.
|
153
|
+
source_code_uri: https://github.com/Shopify/maintenance_tasks/tree/v1.8.1
|
148
154
|
allowed_push_host: https://rubygems.org
|
149
155
|
post_install_message:
|
150
156
|
rdoc_options: []
|