sidekiq-tasks 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +4 -0
- data/CHANGELOG.md +14 -0
- data/README.md +96 -0
- data/docs/custom_storage.md +44 -11
- data/lib/sidekiq/tasks/config.rb +28 -9
- data/lib/sidekiq/tasks/job.rb +3 -2
- data/lib/sidekiq/tasks/magic_comments/comment.rb +28 -0
- data/lib/sidekiq/tasks/magic_comments/comments.rb +31 -0
- data/lib/sidekiq/tasks/magic_comments/handlers/base.rb +26 -0
- data/lib/sidekiq/tasks/magic_comments/handlers/disable.rb +17 -0
- data/lib/sidekiq/tasks/magic_comments/handlers/dry_run.rb +17 -0
- data/lib/sidekiq/tasks/magic_comments/handlers/enable.rb +17 -0
- data/lib/sidekiq/tasks/magic_comments/handlers/sidekiq_options.rb +32 -0
- data/lib/sidekiq/tasks/magic_comments/registration.rb +11 -0
- data/lib/sidekiq/tasks/magic_comments/registry.rb +19 -0
- data/lib/sidekiq/tasks/magic_comments/scanner.rb +83 -0
- data/lib/sidekiq/tasks/magic_comments.rb +20 -0
- data/lib/sidekiq/tasks/run.rb +161 -0
- data/lib/sidekiq/tasks/sidekiq_options_validator.rb +25 -0
- data/lib/sidekiq/tasks/storage/base.rb +22 -0
- data/lib/sidekiq/tasks/storage/redis.rb +47 -26
- data/lib/sidekiq/tasks/strategies/base.rb +5 -2
- data/lib/sidekiq/tasks/strategies/rake_task.rb +26 -2
- data/lib/sidekiq/tasks/strategies/rules/disable_with_comment.rb +2 -2
- data/lib/sidekiq/tasks/strategies/rules/enable_with_comment.rb +8 -56
- data/lib/sidekiq/tasks/task.rb +24 -7
- data/lib/sidekiq/tasks/task_metadata.rb +18 -2
- data/lib/sidekiq/tasks/version.rb +1 -1
- data/lib/sidekiq/tasks/web/extension.rb +2 -1
- data/lib/sidekiq/tasks/web/helpers/tag_helper.rb +1 -1
- data/lib/sidekiq/tasks/web/helpers/task_helper.rb +19 -0
- data/lib/sidekiq/tasks.rb +18 -1
- data/web/assets/tasks/css/components/alerts.css +25 -0
- data/web/assets/tasks/css/components/report.css +70 -0
- data/web/assets/tasks/css/ext.css +2 -0
- data/web/locales/en.yml +10 -0
- data/web/locales/fr.yml +10 -0
- data/web/views/task.erb +87 -3
- metadata +31 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 4298eb4b8e1e6e6a3136940f0dc23aba41777faa48046aa9f6e8011656b0f1c3
|
|
4
|
+
data.tar.gz: 24b88855eea1cb680d68e553ea33b3727dbc7840de45e201fbce7c835750d3b3
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: fc95a609c74c0e0cdf36b40cdaa052e0049ee13f368ea34541bab6c60c163b76aa064cf16f2d7004a69c577d123ea5617f09c1dd933f2fa53c2756c1ffcfba85
|
|
7
|
+
data.tar.gz: 44dcf4152c96020b7001c72333d331fc063e9c10e28decd60c1c9d7f6455037ebae10700571e2cb84800088b0220e09cdb16c0842089c987d656b146cb58c3ef
|
data/.rubocop.yml
CHANGED
|
@@ -26,6 +26,7 @@ Metrics/MethodLength:
|
|
|
26
26
|
Metrics/ParameterLists:
|
|
27
27
|
Exclude:
|
|
28
28
|
- spec/**/*
|
|
29
|
+
- lib/sidekiq/tasks/task_metadata.rb
|
|
29
30
|
|
|
30
31
|
Metrics/PerceivedComplexity:
|
|
31
32
|
Exclude:
|
|
@@ -41,6 +42,9 @@ Naming/MemoizedInstanceVariableName:
|
|
|
41
42
|
Style/Documentation:
|
|
42
43
|
Enabled: false
|
|
43
44
|
|
|
45
|
+
Style/EmptyMethod:
|
|
46
|
+
EnforcedStyle: expanded
|
|
47
|
+
|
|
44
48
|
Style/FrozenStringLiteralComment:
|
|
45
49
|
Enabled: false
|
|
46
50
|
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
## Changelog
|
|
2
2
|
|
|
3
|
+
### [1.2.0] - 2026-10-28
|
|
4
|
+
|
|
5
|
+
- Add per-task `sidekiq_options` magic comment to override the global options at enqueue time, displayed on the task detail page.
|
|
6
|
+
- Render tasks with an invalid magic comment as a broken state in the web UI instead of crashing the tasks list.
|
|
7
|
+
- Validate the values of the supported keys of the `sidekiq_options` magic comment, other keys are passed as is to Sidekiq.
|
|
8
|
+
- Add dry run mode: tasks with the `sidekiq-tasks:dry_run` magic comment get a "Dry run" checkbox, checked by default, and read it with `Sidekiq::Tasks.dry_run?`.
|
|
9
|
+
- Add structured execution reports (`Sidekiq::Tasks.report`) displayed in the task history.
|
|
10
|
+
- Add a configurable `report_max_bytes` option (default: 64 KB, maximum: 1 MB) bounding the report size. The Redis storage also keeps the reports of a task history within 10 MB.
|
|
11
|
+
- Fix detection of stacked magic comments placed before a `desc` or a `namespace`.
|
|
12
|
+
- Fix magic comments leaking to a neighbour task or a similarly named namespace, and support nested namespaces.
|
|
13
|
+
- Magic comments must now be alone on their line, right above the `task`, `desc` or `namespace` call, with no blank line in between.
|
|
14
|
+
- Escape task arguments, enqueuing user and error message in the task history.
|
|
15
|
+
- Fix execution times lost in the history when the job starts before the history entry is stored.
|
|
16
|
+
|
|
3
17
|
### [1.1.1] - 2026-04-12
|
|
4
18
|
|
|
5
19
|
- Transfer repository ownership to Capsens organization.
|
data/README.md
CHANGED
|
@@ -197,6 +197,23 @@ end
|
|
|
197
197
|
|
|
198
198
|
All standard [Sidekiq job options](https://github.com/sidekiq/sidekiq/wiki/Advanced-Options#jobs) are supported.
|
|
199
199
|
|
|
200
|
+
### Per-task overrides via magic comment
|
|
201
|
+
|
|
202
|
+
You can override `sidekiq_options` for a single Rake task with the `sidekiq-tasks:sidekiq_options` magic comment placed right before the task:
|
|
203
|
+
|
|
204
|
+
```ruby
|
|
205
|
+
# sidekiq-tasks:enable
|
|
206
|
+
# sidekiq-tasks:sidekiq_options: queue: low
|
|
207
|
+
task :generate_sales_report do
|
|
208
|
+
# ...
|
|
209
|
+
end
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
Placed before a namespace, it applies to all its tasks, and a task level comment takes precedence.
|
|
213
|
+
|
|
214
|
+
The value is parsed as inline YAML and merged on top of the global `config.sidekiq_options` at enqueue time. Supported keys: `queue`, `retry`, `retry_for`, `dead`, `backtrace`, `pool`, `tags`.
|
|
215
|
+
These keys are validated: an invalid value (e.g. `retry: abc`) displays the task as broken in the web interface, and it cannot be enqueued. Other keys are passed as is to Sidekiq, e.g. middleware options such as sidekiq-unique-jobs `lock: until_executed`.
|
|
216
|
+
|
|
200
217
|
You can also override the `enqueue_task` method to implement your own enqueuing logic for your strategy:
|
|
201
218
|
|
|
202
219
|
```ruby
|
|
@@ -243,6 +260,85 @@ end
|
|
|
243
260
|
|
|
244
261
|
You can also use a custom storage backend. See the [Custom Storage Guide](docs/custom_storage.md) for more details.
|
|
245
262
|
|
|
263
|
+
## Dry run and reports
|
|
264
|
+
|
|
265
|
+
Inside a task, `Sidekiq::Tasks.dry_run?` tells whether the task runs in dry run mode and `Sidekiq::Tasks.report` gives access to a structured report:
|
|
266
|
+
|
|
267
|
+
```ruby
|
|
268
|
+
# sidekiq-tasks:enable
|
|
269
|
+
# sidekiq-tasks:dry_run
|
|
270
|
+
task :sync_users => :environment do
|
|
271
|
+
User.find_each do |user|
|
|
272
|
+
Sync.call(user) unless Sidekiq::Tasks.dry_run?
|
|
273
|
+
Sidekiq::Tasks.report.increment(:success)
|
|
274
|
+
rescue => e
|
|
275
|
+
Sidekiq::Tasks.report.increment(:errors)
|
|
276
|
+
Sidekiq::Tasks.report.add(:failed, id: user.id, email: user.email, error: e.message)
|
|
277
|
+
end
|
|
278
|
+
end
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
Both are shortcuts for `Sidekiq::Tasks.current_run.dry_run?` and `Sidekiq::Tasks.current_run.report`.
|
|
282
|
+
|
|
283
|
+
### Dry run
|
|
284
|
+
|
|
285
|
+
Dry run mode lets a user check what a task would do before running it for real, typically together with a [report](#report) of the rows it would process.
|
|
286
|
+
|
|
287
|
+
**What the gem does:**
|
|
288
|
+
|
|
289
|
+
- detects tasks declaring the `sidekiq-tasks:dry_run` magic comment. Like `enable`, it can be placed before the task, its `desc` or its `namespace`, stacked with other magic comments,
|
|
290
|
+
- displays a **Dry run** checkbox on these tasks, checked by default: the user has to uncheck it to execute the task for real. Other tasks display a warning stating that they will be executed for real,
|
|
291
|
+
- passes the choice to the Sidekiq job, where `Sidekiq::Tasks.dry_run?` returns it,
|
|
292
|
+
- flags dry runs with a badge in the task history.
|
|
293
|
+
|
|
294
|
+
**What the developer does:**
|
|
295
|
+
|
|
296
|
+
- adds the `sidekiq-tasks:dry_run` magic comment to the tasks that support dry run mode,
|
|
297
|
+
- skips every side effect when `Sidekiq::Tasks.dry_run?` is `true`: database writes, emails, API calls, file uploads, enqueued jobs...,
|
|
298
|
+
- keeps the rest of the task running (queries, validations, computations), so that the dry run reflects what the real run would do,
|
|
299
|
+
- optionally fills the report, so that the dry run result shows up in the web interface.
|
|
300
|
+
|
|
301
|
+
> [!WARNING]
|
|
302
|
+
> The gem never skips anything by itself: a task declaring the magic comment without checking `Sidekiq::Tasks.dry_run?` executes all of its side effects, even when the checkbox is checked.
|
|
303
|
+
|
|
304
|
+
Outside of the web interface (e.g. `rake sync_users` in a terminal), `dry_run?` is `false`.
|
|
305
|
+
Both `dry_run?` and `report` are optional: a task can use the report without dry run mode, and existing tasks keep working unchanged.
|
|
306
|
+
|
|
307
|
+
### Report
|
|
308
|
+
|
|
309
|
+
- `report[:key] = value` sets a summary value.
|
|
310
|
+
- `report.increment(:key)` increments a counter.
|
|
311
|
+
- `report.add(:key, item)` adds an item (a scalar or a flat Hash) to a list. Hash items are displayed as a table whose columns are the hash keys.
|
|
312
|
+
|
|
313
|
+
Keys are free: use as many lists as needed (`:synced`, `:skipped`, `:failed`...). Values are displayed in the history table, lists in a collapsible section below it (the latest execution is expanded). The report is stored whether the task runs in dry run mode or not, and also when it fails.
|
|
314
|
+
|
|
315
|
+
Numbers, booleans and `nil` are kept as is, other values are converted to strings: pass the useful attributes of an object (`{id: user.id, email: user.email}`) rather than the object itself, and prefer one list per key to nested structures.
|
|
316
|
+
|
|
317
|
+
The report is stored with the execution history, so its size is bounded by `report_max_bytes` (default: 64 KB, maximum: 1 MB):
|
|
318
|
+
|
|
319
|
+
```ruby
|
|
320
|
+
Sidekiq::Tasks.configure do |config|
|
|
321
|
+
config.report_max_bytes = 256 * 1024
|
|
322
|
+
end
|
|
323
|
+
```
|
|
324
|
+
|
|
325
|
+
- a value above the maximum does not prevent the application from starting: it is lowered to the maximum and a warning is logged,
|
|
326
|
+
- when the report exceeds it, the last items of the longest lists are dropped and the report is flagged as truncated in the web interface, the counts remain exact,
|
|
327
|
+
- with the Redis storage, the reports of a task history never exceed 10 MB: with a large `history_limit`, the size of each report is lowered accordingly (`10 MB / history_limit`),
|
|
328
|
+
- strings are truncated to 255 characters, so that a single long message does not fill the whole report,
|
|
329
|
+
- at most 50 values, 50 lists and 50 keys per item, guarding against keys built from the data,
|
|
330
|
+
- while the task runs, items stop being kept beyond 4 times the report size, so that the worker memory stays bounded.
|
|
331
|
+
|
|
332
|
+
The report never makes a task fail. Outside of a Sidekiq::Tasks job, it is printed as JSON when the process exits.
|
|
333
|
+
|
|
334
|
+
> [!TIP]
|
|
335
|
+
> For large or complex reports, prefer a database storage (see the [Custom Storage Guide](docs/custom_storage.md)) over Redis: reports are kept in the database instead of Redis memory, and are not lowered by the 10 MB budget of the Redis storage.
|
|
336
|
+
|
|
337
|
+
> [!NOTE]
|
|
338
|
+
> Custom strategies enable dry run mode by building their metadata with `dry_run: true`.
|
|
339
|
+
> Those overriding `enqueue_task` then receive a `dry_run: true` keyword argument, to pass to `Sidekiq::Tasks::Job` as its third argument.
|
|
340
|
+
> Custom storages display the dry run badge and reports by implementing `store_dry_run` and `store_execution_report`, see the [Custom Storage Guide](docs/custom_storage.md).
|
|
341
|
+
|
|
246
342
|
## Current user
|
|
247
343
|
|
|
248
344
|
You can configure a `current_user` proc to track who enqueued each task.
|
data/docs/custom_storage.md
CHANGED
|
@@ -7,7 +7,7 @@ This guide walks you through implementing a custom storage backend using ActiveR
|
|
|
7
7
|
```bash
|
|
8
8
|
rails generate model TaskExecution \
|
|
9
9
|
task_name:string:index \
|
|
10
|
-
jid:string:
|
|
10
|
+
jid:string:uniq \
|
|
11
11
|
args:jsonb \
|
|
12
12
|
enqueued_at:datetime \
|
|
13
13
|
executed_at:datetime \
|
|
@@ -38,22 +38,16 @@ class ActiveRecordStorage < Sidekiq::Tasks::Storage::Base
|
|
|
38
38
|
end
|
|
39
39
|
|
|
40
40
|
def store_enqueue(jid, args, user: nil)
|
|
41
|
-
TaskExecution.
|
|
42
|
-
task_name: task_name,
|
|
43
|
-
jid: jid,
|
|
44
|
-
args: args,
|
|
45
|
-
enqueued_at: Time.now,
|
|
46
|
-
user: user
|
|
47
|
-
)
|
|
41
|
+
TaskExecution.create_or_find_by!(jid: jid).update!(task_name: task_name, args: args, enqueued_at: Time.now, user: user)
|
|
48
42
|
end
|
|
49
43
|
|
|
50
44
|
def store_execution(jid, time_key)
|
|
51
|
-
TaskExecution.
|
|
45
|
+
TaskExecution.create_or_find_by!(jid: jid).update!(time_key => Time.now)
|
|
52
46
|
end
|
|
53
47
|
|
|
54
48
|
def store_execution_error(jid, error)
|
|
55
49
|
message = truncate_message("#{error.class}: #{error.message}", ERROR_MESSAGE_MAX_LENGTH)
|
|
56
|
-
TaskExecution.
|
|
50
|
+
TaskExecution.create_or_find_by!(jid: jid).update!(error: message)
|
|
57
51
|
end
|
|
58
52
|
end
|
|
59
53
|
|
|
@@ -80,9 +74,48 @@ The `history` method must return an array of hashes with the following keys:
|
|
|
80
74
|
| `"error"` | String \| nil | Error message if execution failed |
|
|
81
75
|
| `"user"` | Hash \| nil | User who enqueued the task |
|
|
82
76
|
|
|
77
|
+
Two optional keys are displayed when present:
|
|
78
|
+
|
|
79
|
+
| Key | Type | Description |
|
|
80
|
+
|-------------|---------------|--------------------------------------------------------------|
|
|
81
|
+
| `"dry_run"` | Boolean | Whether the task was executed in dry run mode |
|
|
82
|
+
| `"report"` | Hash \| nil | The execution report, see `Sidekiq::Tasks::Report#to_h` |
|
|
83
|
+
|
|
84
|
+
They are stored through `store_dry_run(jid)` and `store_execution_report(jid, report)`, which do nothing by default.
|
|
85
|
+
Override them to support dry run badges and reports, e.g. with `dry_run:boolean` and `report:jsonb` columns, also added to the `history` select:
|
|
86
|
+
|
|
87
|
+
```ruby
|
|
88
|
+
def store_dry_run(jid)
|
|
89
|
+
TaskExecution.create_or_find_by!(jid: jid).update!(dry_run: true)
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def store_execution_report(jid, report)
|
|
93
|
+
TaskExecution.create_or_find_by!(jid: jid).update!(report: report)
|
|
94
|
+
end
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
Reports are bounded by `config.report_max_bytes` (default: 64 KB, maximum: 1 MB). The 10 MB budget of the Redis storage for the reports of a task history does not apply to custom storages.
|
|
98
|
+
|
|
83
99
|
## Migrating from Redis
|
|
84
100
|
|
|
85
|
-
|
|
101
|
+
To keep the history, add this one-off task in `lib/tasks/sidekiq_tasks_copy_redis_history.rake`, run it once after switching to the custom storage with `bin/rails sidekiq_tasks:copy_redis_history`, then delete it:
|
|
102
|
+
|
|
103
|
+
```ruby
|
|
104
|
+
namespace :sidekiq_tasks do
|
|
105
|
+
desc "Copy the sidekiq-tasks history from Redis to the database"
|
|
106
|
+
task copy_redis_history: :environment do
|
|
107
|
+
Sidekiq.redis { |conn| conn.keys("task:*:history") }.each do |history_key|
|
|
108
|
+
task_name = history_key.delete_prefix("task:").delete_suffix(":history")
|
|
109
|
+
|
|
110
|
+
Sidekiq::Tasks::Storage::Redis.new(task_name).history.each do |entry|
|
|
111
|
+
TaskExecution.create_or_find_by!(jid: entry["jid"]).update!(entry.slice(*TaskExecution.column_names))
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
Then clean up the old Redis history:
|
|
86
119
|
|
|
87
120
|
```bash
|
|
88
121
|
redis-cli --scan --pattern "task:*" | xargs redis-cli del
|
data/lib/sidekiq/tasks/config.rb
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
require "sidekiq"
|
|
2
|
+
|
|
1
3
|
module Sidekiq
|
|
2
4
|
module Tasks
|
|
3
5
|
class Config
|
|
@@ -19,9 +21,14 @@ module Sidekiq
|
|
|
19
21
|
|
|
20
22
|
DEFAULT_HISTORY_LIMIT = 10
|
|
21
23
|
|
|
24
|
+
DEFAULT_REPORT_MAX_BYTES = 64 * 1024
|
|
25
|
+
|
|
26
|
+
REPORT_MAX_BYTES_LIMIT = 1024 * 1024
|
|
27
|
+
|
|
22
28
|
include Sidekiq::Tasks::Validations
|
|
23
29
|
|
|
24
|
-
attr_reader :strategies, :sidekiq_options, :authorization, :history_limit, :current_user, :storage
|
|
30
|
+
attr_reader :strategies, :sidekiq_options, :authorization, :history_limit, :current_user, :storage,
|
|
31
|
+
:report_max_bytes
|
|
25
32
|
|
|
26
33
|
def initialize
|
|
27
34
|
@sidekiq_options = DEFAULT_SIDEKIQ_OPTIONS
|
|
@@ -30,18 +37,12 @@ module Sidekiq
|
|
|
30
37
|
@authorization = ->(_env) { true }
|
|
31
38
|
@history_limit = DEFAULT_HISTORY_LIMIT
|
|
32
39
|
@current_user = nil
|
|
40
|
+
@report_max_bytes = DEFAULT_REPORT_MAX_BYTES
|
|
33
41
|
end
|
|
34
42
|
|
|
35
43
|
# @see https://github.com/sidekiq/sidekiq/wiki/Advanced-Options#jobs
|
|
36
44
|
def sidekiq_options=(options)
|
|
37
|
-
|
|
38
|
-
validate_hash_option!(options, :queue, [String])
|
|
39
|
-
validate_hash_option!(options, :retry, [NilClass, TrueClass, FalseClass, Integer])
|
|
40
|
-
validate_hash_option!(options, :retry_for, [NilClass, Integer, Float])
|
|
41
|
-
validate_hash_option!(options, :dead, [NilClass, TrueClass, FalseClass])
|
|
42
|
-
validate_hash_option!(options, :backtrace, [NilClass, TrueClass, FalseClass, Integer])
|
|
43
|
-
validate_hash_option!(options, :pool, [NilClass, String])
|
|
44
|
-
validate_hash_option!(options, :tags, [NilClass, Array])
|
|
45
|
+
Sidekiq::Tasks::SidekiqOptionsValidator.validate!(options)
|
|
45
46
|
|
|
46
47
|
@sidekiq_options = options
|
|
47
48
|
end
|
|
@@ -66,6 +67,14 @@ module Sidekiq
|
|
|
66
67
|
@history_limit = limit
|
|
67
68
|
end
|
|
68
69
|
|
|
70
|
+
def report_max_bytes=(max_bytes)
|
|
71
|
+
validate_class!(max_bytes, [Integer], "report_max_bytes")
|
|
72
|
+
|
|
73
|
+
raise Sidekiq::Tasks::ArgumentError, "'report_max_bytes' must be greater than 0" if max_bytes <= 0
|
|
74
|
+
|
|
75
|
+
@report_max_bytes = bounded(max_bytes, REPORT_MAX_BYTES_LIMIT, "report_max_bytes")
|
|
76
|
+
end
|
|
77
|
+
|
|
69
78
|
def authorization=(authorization_proc)
|
|
70
79
|
validate_class!(authorization_proc, [Proc], "authorization")
|
|
71
80
|
|
|
@@ -77,6 +86,16 @@ module Sidekiq
|
|
|
77
86
|
|
|
78
87
|
@current_user = current_user_proc
|
|
79
88
|
end
|
|
89
|
+
|
|
90
|
+
private
|
|
91
|
+
|
|
92
|
+
# A too large value never prevents the application from starting: it is lowered to the limit.
|
|
93
|
+
def bounded(value, limit, name)
|
|
94
|
+
return value if value <= limit
|
|
95
|
+
|
|
96
|
+
Sidekiq.logger.warn("[sidekiq-tasks] '#{name}' (#{value}) exceeds the maximum, using #{limit}")
|
|
97
|
+
limit
|
|
98
|
+
end
|
|
80
99
|
end
|
|
81
100
|
end
|
|
82
101
|
end
|
data/lib/sidekiq/tasks/job.rb
CHANGED
|
@@ -9,10 +9,11 @@ module Sidekiq
|
|
|
9
9
|
|
|
10
10
|
# @param name [String] The name of the task to execute.
|
|
11
11
|
# @param args [Hash] The arguments to pass to the task.
|
|
12
|
+
# @param dry_run [Boolean] Whether the task is executed in dry run mode.
|
|
12
13
|
# @raise [Sidekiq::Tasks::TaskNotFoundError] If the task is not found.
|
|
13
|
-
def perform(name, args)
|
|
14
|
+
def perform(name, args, dry_run = false) # rubocop:disable Style/OptionalBooleanParameter -- Sidekiq args are positional
|
|
14
15
|
symbolized_args = JSON.parse(args, symbolize_names: true)
|
|
15
|
-
Sidekiq::Tasks.tasks.find_by!(name: name).execute(symbolized_args, jid: jid)
|
|
16
|
+
Sidekiq::Tasks.tasks.find_by!(name: name).execute(symbolized_args, jid: jid, dry_run: dry_run)
|
|
16
17
|
end
|
|
17
18
|
end
|
|
18
19
|
end
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
module Sidekiq
|
|
2
|
+
module Tasks
|
|
3
|
+
module MagicComments
|
|
4
|
+
class Comment
|
|
5
|
+
VALID_LOCATIONS = %i[task desc namespace].freeze
|
|
6
|
+
|
|
7
|
+
attr_reader :name, :raw_value, :location
|
|
8
|
+
|
|
9
|
+
def initialize(name:, location:, raw_value: nil)
|
|
10
|
+
@name = name.to_s
|
|
11
|
+
@raw_value = raw_value
|
|
12
|
+
@location = location
|
|
13
|
+
|
|
14
|
+
validate!
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
private
|
|
18
|
+
|
|
19
|
+
def validate!
|
|
20
|
+
return if VALID_LOCATIONS.include?(location)
|
|
21
|
+
|
|
22
|
+
raise Sidekiq::Tasks::ArgumentError,
|
|
23
|
+
"'location' must be one of #{VALID_LOCATIONS.inspect} but received #{location.inspect}"
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
module Sidekiq
|
|
2
|
+
module Tasks
|
|
3
|
+
module MagicComments
|
|
4
|
+
class Comments
|
|
5
|
+
include Enumerable
|
|
6
|
+
|
|
7
|
+
def initialize(list = [])
|
|
8
|
+
@list = list
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def each(&block)
|
|
12
|
+
@list.each(&block)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def any?(name)
|
|
16
|
+
@list.any? { |comment| comment.name == name.to_s }
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def fetch(name, default: nil)
|
|
20
|
+
comment = @list.find { |c| c.name == name.to_s }
|
|
21
|
+
return default unless comment
|
|
22
|
+
|
|
23
|
+
handler = Registry.lookup(name)
|
|
24
|
+
return comment.raw_value unless handler
|
|
25
|
+
|
|
26
|
+
handler.cast(comment.raw_value)
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
module Sidekiq
|
|
2
|
+
module Tasks
|
|
3
|
+
module MagicComments
|
|
4
|
+
module Handlers
|
|
5
|
+
class Base
|
|
6
|
+
# Returns the magic comment token used after `sidekiq-tasks:`.
|
|
7
|
+
#
|
|
8
|
+
# @abstract Subclasses must implement this method.
|
|
9
|
+
# @return [String]
|
|
10
|
+
def self.name_token
|
|
11
|
+
raise Sidekiq::Tasks::NotImplementedError, "Handler must implement .name_token"
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# Casts the raw string value extracted from the magic comment into a typed value.
|
|
15
|
+
#
|
|
16
|
+
# @abstract Subclasses must implement this method.
|
|
17
|
+
# @param raw_value [String, NilClass]
|
|
18
|
+
# @return [Object]
|
|
19
|
+
def self.cast(_raw_value)
|
|
20
|
+
raise Sidekiq::Tasks::NotImplementedError, "Handler must implement .cast"
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
require "yaml"
|
|
2
|
+
|
|
3
|
+
module Sidekiq
|
|
4
|
+
module Tasks
|
|
5
|
+
module MagicComments
|
|
6
|
+
module Handlers
|
|
7
|
+
class SidekiqOptions < Base
|
|
8
|
+
def self.name_token
|
|
9
|
+
"sidekiq_options"
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def self.cast(raw_value)
|
|
13
|
+
return {} if raw_value.nil? || raw_value.strip.empty?
|
|
14
|
+
|
|
15
|
+
options = parse_yaml(raw_value)
|
|
16
|
+
# Unknown keys are passed as is to Sidekiq, for middleware options such as sidekiq-unique-jobs locks.
|
|
17
|
+
Sidekiq::Tasks::SidekiqOptionsValidator.validate!(options, allow_partial: true)
|
|
18
|
+
options
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def self.parse_yaml(raw_value)
|
|
22
|
+
YAML.safe_load("{#{raw_value}}", permitted_classes: [Symbol], symbolize_names: true)
|
|
23
|
+
rescue Psych::SyntaxError => e
|
|
24
|
+
raise Sidekiq::Tasks::ArgumentError,
|
|
25
|
+
"'sidekiq_options' magic comment is not valid YAML: #{e.message}"
|
|
26
|
+
end
|
|
27
|
+
private_class_method :parse_yaml
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
require_relative "registry"
|
|
2
|
+
require_relative "handlers/base"
|
|
3
|
+
require_relative "handlers/enable"
|
|
4
|
+
require_relative "handlers/disable"
|
|
5
|
+
require_relative "handlers/dry_run"
|
|
6
|
+
require_relative "handlers/sidekiq_options"
|
|
7
|
+
|
|
8
|
+
Sidekiq::Tasks::MagicComments::Registry.register(Sidekiq::Tasks::MagicComments::Handlers::Enable)
|
|
9
|
+
Sidekiq::Tasks::MagicComments::Registry.register(Sidekiq::Tasks::MagicComments::Handlers::Disable)
|
|
10
|
+
Sidekiq::Tasks::MagicComments::Registry.register(Sidekiq::Tasks::MagicComments::Handlers::SidekiqOptions)
|
|
11
|
+
Sidekiq::Tasks::MagicComments::Registry.register(Sidekiq::Tasks::MagicComments::Handlers::DryRun)
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
module Sidekiq
|
|
2
|
+
module Tasks
|
|
3
|
+
module MagicComments
|
|
4
|
+
module Registry
|
|
5
|
+
@handlers = {}
|
|
6
|
+
|
|
7
|
+
class << self
|
|
8
|
+
def register(handler)
|
|
9
|
+
@handlers[handler.name_token.to_s] = handler
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def lookup(name)
|
|
13
|
+
@handlers[name.to_s]
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
require "prism"
|
|
2
|
+
|
|
3
|
+
module Sidekiq
|
|
4
|
+
module Tasks
|
|
5
|
+
module MagicComments
|
|
6
|
+
# A magic comment only applies to the task, desc or namespace call right below it.
|
|
7
|
+
class Scanner
|
|
8
|
+
MAGIC_COMMENT_REGEX = /\A#\s*sidekiq-tasks:([a-z_]+)(?::\s*(.+?))?\s*\z/
|
|
9
|
+
|
|
10
|
+
def initialize
|
|
11
|
+
@comments_by_file = {}
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# @return [Comments] From the task to the outermost namespace, so that the closest comment wins.
|
|
15
|
+
def scan(task)
|
|
16
|
+
file, task_line = task.locations.first.split(":")
|
|
17
|
+
@comments_by_file[file] ||= comments_by_task_line(file)
|
|
18
|
+
|
|
19
|
+
Comments.new(@comments_by_file[file].fetch(task_line.to_i, []))
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
private
|
|
23
|
+
|
|
24
|
+
def comments_by_task_line(file)
|
|
25
|
+
parse_result = Prism.parse(read_file(file))
|
|
26
|
+
comment_lines =
|
|
27
|
+
parse_result
|
|
28
|
+
.comments
|
|
29
|
+
.reject(&:trailing?)
|
|
30
|
+
.to_h { |comment| [comment.location.start_line, comment.slice] }
|
|
31
|
+
|
|
32
|
+
annotated_calls_by_task_line(parse_result.value).transform_values do |annotated_calls|
|
|
33
|
+
annotated_calls.flat_map { |call, location| magic_comments_above(call, location, comment_lines) }
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def read_file(file)
|
|
38
|
+
File.read(file)
|
|
39
|
+
rescue Errno::ENOENT
|
|
40
|
+
raise Sidekiq::Tasks::ArgumentError, "File '#{file}' not found"
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Keyed by the line where the task call starts, as recorded by Rake.
|
|
44
|
+
def annotated_calls_by_task_line(node, namespace_calls = [], annotated_calls = {})
|
|
45
|
+
[nil, *node.compact_child_nodes].each_cons(2) do |previous_child, child|
|
|
46
|
+
if rake_call?(child, :task)
|
|
47
|
+
annotated_calls[child.location.start_line] = task_annotated_calls(child, previous_child, namespace_calls)
|
|
48
|
+
elsif rake_call?(child, :namespace)
|
|
49
|
+
annotated_calls_by_task_line(child, [[child, :namespace], *namespace_calls], annotated_calls)
|
|
50
|
+
else
|
|
51
|
+
annotated_calls_by_task_line(child, namespace_calls, annotated_calls)
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
annotated_calls
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def task_annotated_calls(task_call, previous_child, namespace_calls)
|
|
59
|
+
desc_calls = rake_call?(previous_child, :desc) ? [[previous_child, :desc]] : []
|
|
60
|
+
|
|
61
|
+
[[task_call, :task], *desc_calls, *namespace_calls]
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def rake_call?(node, method_name)
|
|
65
|
+
node.is_a?(Prism::CallNode) && node.receiver.nil? && node.name == method_name
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def magic_comments_above(call, location, comment_lines)
|
|
69
|
+
line = call.location.start_line - 1
|
|
70
|
+
magic_comments = []
|
|
71
|
+
|
|
72
|
+
while comment_lines.key?(line)
|
|
73
|
+
match = comment_lines[line].match(MAGIC_COMMENT_REGEX)
|
|
74
|
+
magic_comments.unshift(Comment.new(name: match[1], raw_value: match[2], location: location)) if match
|
|
75
|
+
line -= 1
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
magic_comments
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
module Sidekiq
|
|
2
|
+
module Tasks
|
|
3
|
+
module MagicComments
|
|
4
|
+
autoload :Comment, "sidekiq/tasks/magic_comments/comment"
|
|
5
|
+
autoload :Comments, "sidekiq/tasks/magic_comments/comments"
|
|
6
|
+
autoload :Registry, "sidekiq/tasks/magic_comments/registry"
|
|
7
|
+
autoload :Scanner, "sidekiq/tasks/magic_comments/scanner"
|
|
8
|
+
|
|
9
|
+
module Handlers
|
|
10
|
+
autoload :Base, "sidekiq/tasks/magic_comments/handlers/base"
|
|
11
|
+
autoload :Enable, "sidekiq/tasks/magic_comments/handlers/enable"
|
|
12
|
+
autoload :Disable, "sidekiq/tasks/magic_comments/handlers/disable"
|
|
13
|
+
autoload :DryRun, "sidekiq/tasks/magic_comments/handlers/dry_run"
|
|
14
|
+
autoload :SidekiqOptions, "sidekiq/tasks/magic_comments/handlers/sidekiq_options"
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
require_relative "magic_comments/registration"
|