sidekiq 7.2.4 → 7.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Changes.md +35 -0
- data/README.md +1 -1
- data/lib/sidekiq/api.rb +1 -1
- data/lib/sidekiq/capsule.rb +3 -0
- data/lib/sidekiq/cli.rb +1 -0
- data/lib/sidekiq/client.rb +2 -2
- data/lib/sidekiq/config.rb +5 -1
- data/lib/sidekiq/fetch.rb +1 -1
- data/lib/sidekiq/iterable_job.rb +53 -0
- data/lib/sidekiq/job/interrupt_handler.rb +22 -0
- data/lib/sidekiq/job/iterable/active_record_enumerator.rb +53 -0
- data/lib/sidekiq/job/iterable/csv_enumerator.rb +47 -0
- data/lib/sidekiq/job/iterable/enumerators.rb +135 -0
- data/lib/sidekiq/job/iterable.rb +231 -0
- data/lib/sidekiq/job.rb +13 -2
- data/lib/sidekiq/job_logger.rb +23 -10
- data/lib/sidekiq/job_retry.rb +6 -1
- data/lib/sidekiq/middleware/current_attributes.rb +27 -11
- data/lib/sidekiq/processor.rb +11 -1
- data/lib/sidekiq/redis_client_adapter.rb +8 -5
- data/lib/sidekiq/redis_connection.rb +25 -1
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +2 -1
- data/lib/sidekiq/web/application.rb +8 -4
- data/lib/sidekiq/web/helpers.rb +53 -7
- data/lib/sidekiq/web.rb +46 -1
- data/lib/sidekiq.rb +2 -1
- data/sidekiq.gemspec +2 -1
- data/web/assets/javascripts/application.js +6 -1
- data/web/assets/javascripts/dashboard-charts.js +22 -12
- data/web/assets/javascripts/dashboard.js +1 -1
- data/web/assets/stylesheets/application.css +13 -1
- data/web/locales/tr.yml +101 -0
- data/web/views/dashboard.erb +6 -6
- data/web/views/layout.erb +6 -6
- data/web/views/metrics.erb +4 -4
- data/web/views/metrics_for_job.erb +4 -4
- metadata +26 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 34279eae9b8159c48dd31c7b088341136d8b785e7b853389e29e179443af1577
|
4
|
+
data.tar.gz: 520c6f705995df5c8e50e4e607956bcd8fe78d332e9a33fdf7a4ff25b7534f60
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 939c535c352b1c9390e7675d75cfc3728685282187c0edd8475373c6bad1ee67391477a46e26ea3de06724e8cf2570ef759862eb37234c1f9d53f79adafd5947
|
7
|
+
data.tar.gz: 1a127bf11d514789f6ec808f964d47e9d459687a5b2d91ebe468d6a59a28c5f9dbf01203e1aee583311c776ca9caa41831d74a15557adf5bb0bd71cce27cffdd
|
data/Changes.md
CHANGED
@@ -2,6 +2,41 @@
|
|
2
2
|
|
3
3
|
[Sidekiq Changes](https://github.com/sidekiq/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/sidekiq/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/sidekiq/sidekiq/blob/main/Ent-Changes.md)
|
4
4
|
|
5
|
+
7.3.0
|
6
|
+
----------
|
7
|
+
|
8
|
+
- **NEW FEATURE** Add `Sidekiq::IterableJob`, iteration support for long-running jobs. [#6286, fatkodima]
|
9
|
+
Iterable jobs are interruptible and can restart quickly if
|
10
|
+
running during a deploy. You must ensure that `each_iteration`
|
11
|
+
doesn't take more than Sidekiq's `-t` timeout (default: 25 seconds). Iterable jobs must not implement `perform`.
|
12
|
+
```ruby
|
13
|
+
class ProcessArrayJob
|
14
|
+
include Sidekiq::IterableJob
|
15
|
+
def build_enumerator(*args, **kwargs)
|
16
|
+
array_enumerator(args, **kwargs)
|
17
|
+
end
|
18
|
+
def each_iteration(arg)
|
19
|
+
puts arg
|
20
|
+
end
|
21
|
+
end
|
22
|
+
ProcessArrayJob.perform_async(1, 2, 3)
|
23
|
+
```
|
24
|
+
See the [Iteration](//github.com/sidekiq/sidekiq/wiki/Iteration) wiki page and the RDoc in `Sidekiq::IterableJob`.
|
25
|
+
This feature should be considered BETA until the next minor release.
|
26
|
+
- **SECURITY** The Web UI no longer allows extensions to use `<script>`.
|
27
|
+
Adjust CSP to disallow inline scripts within the Web UI. Please see
|
28
|
+
`examples/webui-ext` for how to register Web UI extensions and use
|
29
|
+
dynamic CSS and JS. This will make Sidekiq immune to XSS attacks. [#6270]
|
30
|
+
- Add config option, `:skip_default_job_logging` to disable Sidekiq's default
|
31
|
+
start/finish job logging. [#6200]
|
32
|
+
- Allow `Sidekiq::Limiter.redis` to use Redis Cluster [#6288]
|
33
|
+
- Retain CurrentAttributeѕ after inline execution [#6307]
|
34
|
+
- Ignore non-existent CurrentAttributes attributes when restoring [#6341]
|
35
|
+
- Raise default Redis {read,write,connect} timeouts from 1 to 3 seconds
|
36
|
+
to minimize ReadTimeoutErrors [#6162]
|
37
|
+
- Add `logger` as a dependency since it will become bundled in Ruby 3.5 [#6320]
|
38
|
+
- Ignore unsupported locales in the Web UI [#6313]
|
39
|
+
|
5
40
|
7.2.4
|
6
41
|
----------
|
7
42
|
|
data/README.md
CHANGED
@@ -86,7 +86,7 @@ Useful resources:
|
|
86
86
|
* Occasional announcements are made to the [@sidekiq](https://ruby.social/@sidekiq) Mastodon account.
|
87
87
|
* The [Sidekiq tag](https://stackoverflow.com/questions/tagged/sidekiq) on Stack Overflow has lots of useful Q & A.
|
88
88
|
|
89
|
-
Every
|
89
|
+
Every Thursday morning is Sidekiq office hour: I video chat and answer questions.
|
90
90
|
See the [Sidekiq support page](https://sidekiq.org/support.html) for details.
|
91
91
|
|
92
92
|
Contributing
|
data/lib/sidekiq/api.rb
CHANGED
data/lib/sidekiq/capsule.rb
CHANGED
@@ -17,6 +17,7 @@ module Sidekiq
|
|
17
17
|
# end
|
18
18
|
class Capsule
|
19
19
|
include Sidekiq::Component
|
20
|
+
extend Forwardable
|
20
21
|
|
21
22
|
attr_reader :name
|
22
23
|
attr_reader :queues
|
@@ -24,6 +25,8 @@ module Sidekiq
|
|
24
25
|
attr_reader :mode
|
25
26
|
attr_reader :weights
|
26
27
|
|
28
|
+
def_delegators :@config, :[], :[]=, :fetch, :key?, :has_key?, :merge!, :dig
|
29
|
+
|
27
30
|
def initialize(name, config)
|
28
31
|
@name = name
|
29
32
|
@config = config
|
data/lib/sidekiq/cli.rb
CHANGED
data/lib/sidekiq/client.rb
CHANGED
@@ -248,10 +248,10 @@ module Sidekiq
|
|
248
248
|
def atomic_push(conn, payloads)
|
249
249
|
if payloads.first.key?("at")
|
250
250
|
conn.zadd("schedule", payloads.flat_map { |hash|
|
251
|
-
at = hash
|
251
|
+
at = hash["at"].to_s
|
252
252
|
# ActiveJob sets this but the job has not been enqueued yet
|
253
253
|
hash.delete("enqueued_at")
|
254
|
-
[at, Sidekiq.dump_json(hash)]
|
254
|
+
[at, Sidekiq.dump_json(hash.except("at"))]
|
255
255
|
})
|
256
256
|
else
|
257
257
|
queue = payloads.first["queue"]
|
data/lib/sidekiq/config.rb
CHANGED
@@ -17,6 +17,10 @@ module Sidekiq
|
|
17
17
|
poll_interval_average: nil,
|
18
18
|
average_scheduled_poll_interval: 5,
|
19
19
|
on_complex_arguments: :raise,
|
20
|
+
iteration: {
|
21
|
+
max_job_runtime: nil,
|
22
|
+
retry_backoff: 0
|
23
|
+
},
|
20
24
|
error_handlers: [],
|
21
25
|
death_handlers: [],
|
22
26
|
lifecycle_events: {
|
@@ -52,7 +56,7 @@ module Sidekiq
|
|
52
56
|
@capsules = {}
|
53
57
|
end
|
54
58
|
|
55
|
-
def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge
|
59
|
+
def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!, :dig
|
56
60
|
attr_reader :capsules
|
57
61
|
|
58
62
|
def to_json(*)
|
data/lib/sidekiq/fetch.rb
CHANGED
@@ -44,7 +44,7 @@ module Sidekiq # :nodoc:
|
|
44
44
|
return nil
|
45
45
|
end
|
46
46
|
|
47
|
-
queue, job = redis { |conn| conn.blocking_call(
|
47
|
+
queue, job = redis { |conn| conn.blocking_call(TIMEOUT, "brpop", *qs, TIMEOUT) }
|
48
48
|
UnitOfWork.new(queue, job, config) if queue
|
49
49
|
end
|
50
50
|
|
@@ -0,0 +1,53 @@
|
|
1
|
+
require "sidekiq/job/iterable"
|
2
|
+
|
3
|
+
# Iterable jobs are ones which provide a sequence to process using
|
4
|
+
# `build_enumerator(*args, cursor: cursor)` and then process each
|
5
|
+
# element of that sequence in `each_iteration(item, *args)`.
|
6
|
+
#
|
7
|
+
# The job is kicked off as normal:
|
8
|
+
#
|
9
|
+
# ProcessUserSet.perform_async(123)
|
10
|
+
#
|
11
|
+
# but instead of calling `perform`, Sidekiq will call:
|
12
|
+
#
|
13
|
+
# enum = ProcessUserSet#build_enumerator(123, cursor:nil)
|
14
|
+
#
|
15
|
+
# Your Enumerator must yield `(object, updated_cursor)` and
|
16
|
+
# Sidekiq will call your `each_iteration` method:
|
17
|
+
#
|
18
|
+
# ProcessUserSet#each_iteration(object, 123)
|
19
|
+
#
|
20
|
+
# After every iteration, Sidekiq will check for shutdown. If we are
|
21
|
+
# stopping, the cursor will be saved to Redis and the job re-queued
|
22
|
+
# to pick up the rest of the work upon restart. Your job will get
|
23
|
+
# the updated_cursor so it can pick up right where it stopped.
|
24
|
+
#
|
25
|
+
# enum = ProcessUserSet#build_enumerator(123, cursor: updated_cursor)
|
26
|
+
#
|
27
|
+
# The cursor object must be serializable to JSON.
|
28
|
+
#
|
29
|
+
# Note there are several APIs to help you build enumerators for
|
30
|
+
# ActiveRecord Relations, CSV files, etc. See sidekiq/job/iterable/*.rb.
|
31
|
+
module Sidekiq
|
32
|
+
module IterableJob
|
33
|
+
def self.included(base)
|
34
|
+
base.include Sidekiq::Job
|
35
|
+
base.include Sidekiq::Job::Iterable
|
36
|
+
end
|
37
|
+
|
38
|
+
# def build_enumerator(*args, cursor:)
|
39
|
+
# def each_iteration(item, *args)
|
40
|
+
|
41
|
+
# Your job can also define several callbacks during points
|
42
|
+
# in each job's lifecycle.
|
43
|
+
#
|
44
|
+
# def on_start
|
45
|
+
# def on_resume
|
46
|
+
# def on_stop
|
47
|
+
# def on_complete
|
48
|
+
# def around_iteration
|
49
|
+
#
|
50
|
+
# To keep things simple and compatible, this is the same
|
51
|
+
# API as the `sidekiq-iteration` gem.
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module Sidekiq
|
2
|
+
module Job
|
3
|
+
class InterruptHandler
|
4
|
+
include Sidekiq::ServerMiddleware
|
5
|
+
|
6
|
+
def call(instance, hash, queue)
|
7
|
+
yield
|
8
|
+
rescue Interrupted
|
9
|
+
logger.debug "Interrupted, re-queueing..."
|
10
|
+
c = Sidekiq::Client.new
|
11
|
+
c.push(hash)
|
12
|
+
raise Sidekiq::JobRetry::Skip
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
Sidekiq.configure_server do |config|
|
19
|
+
config.server_middleware do |chain|
|
20
|
+
chain.add Sidekiq::Job::InterruptHandler
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
module Job
|
5
|
+
module Iterable
|
6
|
+
# @api private
|
7
|
+
class ActiveRecordEnumerator
|
8
|
+
def initialize(relation, cursor: nil, **options)
|
9
|
+
@relation = relation
|
10
|
+
@cursor = cursor
|
11
|
+
@options = options
|
12
|
+
end
|
13
|
+
|
14
|
+
def records
|
15
|
+
Enumerator.new(-> { @relation.count }) do |yielder|
|
16
|
+
@relation.find_each(**@options, start: @cursor) do |record|
|
17
|
+
yielder.yield(record, record.id)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def batches
|
23
|
+
Enumerator.new(-> { @relation.count }) do |yielder|
|
24
|
+
@relation.find_in_batches(**@options, start: @cursor) do |batch|
|
25
|
+
yielder.yield(batch, batch.last.id)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def relations
|
31
|
+
Enumerator.new(-> { relations_size }) do |yielder|
|
32
|
+
# Convenience to use :batch_size for all the
|
33
|
+
# ActiveRecord batching methods.
|
34
|
+
options = @options.dup
|
35
|
+
options[:of] ||= options.delete(:batch_size)
|
36
|
+
|
37
|
+
@relation.in_batches(**options, start: @cursor) do |relation|
|
38
|
+
last_record = relation.last
|
39
|
+
yielder.yield(relation, last_record.id)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
private
|
45
|
+
|
46
|
+
def relations_size
|
47
|
+
batch_size = @options[:batch_size] || 1000
|
48
|
+
(@relation.count + batch_size - 1) / batch_size # ceiling division
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
module Job
|
5
|
+
module Iterable
|
6
|
+
# @api private
|
7
|
+
class CsvEnumerator
|
8
|
+
def initialize(csv)
|
9
|
+
unless defined?(CSV) && csv.instance_of?(CSV)
|
10
|
+
raise ArgumentError, "CsvEnumerator.new takes CSV object"
|
11
|
+
end
|
12
|
+
|
13
|
+
@csv = csv
|
14
|
+
end
|
15
|
+
|
16
|
+
def rows(cursor:)
|
17
|
+
@csv.lazy
|
18
|
+
.each_with_index
|
19
|
+
.drop(cursor || 0)
|
20
|
+
.to_enum { count_of_rows_in_file }
|
21
|
+
end
|
22
|
+
|
23
|
+
def batches(cursor:, batch_size: 100)
|
24
|
+
@csv.lazy
|
25
|
+
.each_slice(batch_size)
|
26
|
+
.with_index
|
27
|
+
.drop(cursor || 0)
|
28
|
+
.to_enum { (count_of_rows_in_file.to_f / batch_size).ceil }
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def count_of_rows_in_file
|
34
|
+
filepath = @csv.path
|
35
|
+
return unless filepath
|
36
|
+
|
37
|
+
count = IO.popen(["wc", "-l", filepath]) do |out|
|
38
|
+
out.read.strip.to_i
|
39
|
+
end
|
40
|
+
|
41
|
+
count -= 1 if @csv.headers
|
42
|
+
count
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,135 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "active_record_enumerator"
|
4
|
+
require_relative "csv_enumerator"
|
5
|
+
|
6
|
+
module Sidekiq
|
7
|
+
module Job
|
8
|
+
module Iterable
|
9
|
+
module Enumerators
|
10
|
+
# Builds Enumerator object from a given array, using +cursor+ as an offset.
|
11
|
+
#
|
12
|
+
# @param array [Array]
|
13
|
+
# @param cursor [Integer] offset to start iteration from
|
14
|
+
#
|
15
|
+
# @return [Enumerator]
|
16
|
+
#
|
17
|
+
# @example
|
18
|
+
# array_enumerator(['build', 'enumerator', 'from', 'any', 'array'], cursor: cursor)
|
19
|
+
#
|
20
|
+
def array_enumerator(array, cursor:)
|
21
|
+
raise ArgumentError, "array must be an Array" unless array.is_a?(Array)
|
22
|
+
|
23
|
+
x = array.each_with_index.drop(cursor || 0)
|
24
|
+
x.to_enum { x.size }
|
25
|
+
end
|
26
|
+
|
27
|
+
# Builds Enumerator from `ActiveRecord::Relation`.
|
28
|
+
# Each Enumerator tick moves the cursor one row forward.
|
29
|
+
#
|
30
|
+
# @param relation [ActiveRecord::Relation] relation to iterate
|
31
|
+
# @param cursor [Object] offset id to start iteration from
|
32
|
+
# @param options [Hash] additional options that will be passed to relevant
|
33
|
+
# ActiveRecord batching methods
|
34
|
+
#
|
35
|
+
# @return [ActiveRecordEnumerator]
|
36
|
+
#
|
37
|
+
# @example
|
38
|
+
# def build_enumerator(cursor:)
|
39
|
+
# active_record_records_enumerator(User.all, cursor: cursor)
|
40
|
+
# end
|
41
|
+
#
|
42
|
+
# def each_iteration(user)
|
43
|
+
# user.notify_about_something
|
44
|
+
# end
|
45
|
+
#
|
46
|
+
def active_record_records_enumerator(relation, cursor:, **options)
|
47
|
+
ActiveRecordEnumerator.new(relation, cursor: cursor, **options).records
|
48
|
+
end
|
49
|
+
|
50
|
+
# Builds Enumerator from `ActiveRecord::Relation` and enumerates on batches of records.
|
51
|
+
# Each Enumerator tick moves the cursor `:batch_size` rows forward.
|
52
|
+
# @see #active_record_records_enumerator
|
53
|
+
#
|
54
|
+
# @example
|
55
|
+
# def build_enumerator(product_id, cursor:)
|
56
|
+
# active_record_batches_enumerator(
|
57
|
+
# Comment.where(product_id: product_id).select(:id),
|
58
|
+
# cursor: cursor,
|
59
|
+
# batch_size: 100
|
60
|
+
# )
|
61
|
+
# end
|
62
|
+
#
|
63
|
+
# def each_iteration(batch_of_comments, product_id)
|
64
|
+
# comment_ids = batch_of_comments.map(&:id)
|
65
|
+
# CommentService.call(comment_ids: comment_ids)
|
66
|
+
# end
|
67
|
+
#
|
68
|
+
def active_record_batches_enumerator(relation, cursor:, **options)
|
69
|
+
ActiveRecordEnumerator.new(relation, cursor: cursor, **options).batches
|
70
|
+
end
|
71
|
+
|
72
|
+
# Builds Enumerator from `ActiveRecord::Relation` and enumerates on batches,
|
73
|
+
# yielding `ActiveRecord::Relation`s.
|
74
|
+
# @see #active_record_records_enumerator
|
75
|
+
#
|
76
|
+
# @example
|
77
|
+
# def build_enumerator(product_id, cursor:)
|
78
|
+
# active_record_relations_enumerator(
|
79
|
+
# Product.find(product_id).comments,
|
80
|
+
# cursor: cursor,
|
81
|
+
# batch_size: 100,
|
82
|
+
# )
|
83
|
+
# end
|
84
|
+
#
|
85
|
+
# def each_iteration(batch_of_comments, product_id)
|
86
|
+
# # batch_of_comments will be a Comment::ActiveRecord_Relation
|
87
|
+
# batch_of_comments.update_all(deleted: true)
|
88
|
+
# end
|
89
|
+
#
|
90
|
+
def active_record_relations_enumerator(relation, cursor:, **options)
|
91
|
+
ActiveRecordEnumerator.new(relation, cursor: cursor, **options).relations
|
92
|
+
end
|
93
|
+
|
94
|
+
# Builds Enumerator from a CSV file.
|
95
|
+
#
|
96
|
+
# @param csv [CSV] an instance of CSV object
|
97
|
+
# @param cursor [Integer] offset to start iteration from
|
98
|
+
#
|
99
|
+
# @example
|
100
|
+
# def build_enumerator(import_id, cursor:)
|
101
|
+
# import = Import.find(import_id)
|
102
|
+
# csv_enumerator(import.csv, cursor: cursor)
|
103
|
+
# end
|
104
|
+
#
|
105
|
+
# def each_iteration(csv_row)
|
106
|
+
# # insert csv_row into database
|
107
|
+
# end
|
108
|
+
#
|
109
|
+
def csv_enumerator(csv, cursor:)
|
110
|
+
CsvEnumerator.new(csv).rows(cursor: cursor)
|
111
|
+
end
|
112
|
+
|
113
|
+
# Builds Enumerator from a CSV file and enumerates on batches of records.
|
114
|
+
#
|
115
|
+
# @param csv [CSV] an instance of CSV object
|
116
|
+
# @param cursor [Integer] offset to start iteration from
|
117
|
+
# @option options :batch_size [Integer] (100) size of the batch
|
118
|
+
#
|
119
|
+
# @example
|
120
|
+
# def build_enumerator(import_id, cursor:)
|
121
|
+
# import = Import.find(import_id)
|
122
|
+
# csv_batches_enumerator(import.csv, cursor: cursor)
|
123
|
+
# end
|
124
|
+
#
|
125
|
+
# def each_iteration(batch_of_csv_rows)
|
126
|
+
# # ...
|
127
|
+
# end
|
128
|
+
#
|
129
|
+
def csv_batches_enumerator(csv, cursor:, **options)
|
130
|
+
CsvEnumerator.new(csv).batches(cursor: cursor, **options)
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
@@ -0,0 +1,231 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "iterable/enumerators"
|
4
|
+
|
5
|
+
module Sidekiq
|
6
|
+
module Job
|
7
|
+
class Interrupted < ::RuntimeError; end
|
8
|
+
|
9
|
+
module Iterable
|
10
|
+
include Enumerators
|
11
|
+
|
12
|
+
# @api private
|
13
|
+
def self.included(base)
|
14
|
+
base.extend(ClassMethods)
|
15
|
+
end
|
16
|
+
|
17
|
+
# @api private
|
18
|
+
module ClassMethods
|
19
|
+
def method_added(method_name)
|
20
|
+
raise "#{self} is an iterable job and must not define #perform" if method_name == :perform
|
21
|
+
super
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
# @api private
|
26
|
+
def initialize
|
27
|
+
super
|
28
|
+
|
29
|
+
@_executions = 0
|
30
|
+
@_cursor = nil
|
31
|
+
@_start_time = nil
|
32
|
+
@_runtime = 0
|
33
|
+
end
|
34
|
+
|
35
|
+
# A hook to override that will be called when the job starts iterating.
|
36
|
+
#
|
37
|
+
# It is called only once, for the first time.
|
38
|
+
#
|
39
|
+
def on_start
|
40
|
+
end
|
41
|
+
|
42
|
+
# A hook to override that will be called around each iteration.
|
43
|
+
#
|
44
|
+
# Can be useful for some metrics collection, performance tracking etc.
|
45
|
+
#
|
46
|
+
def around_iteration
|
47
|
+
yield
|
48
|
+
end
|
49
|
+
|
50
|
+
# A hook to override that will be called when the job resumes iterating.
|
51
|
+
#
|
52
|
+
def on_resume
|
53
|
+
end
|
54
|
+
|
55
|
+
# A hook to override that will be called each time the job is interrupted.
|
56
|
+
#
|
57
|
+
# This can be due to interruption or sidekiq stopping.
|
58
|
+
#
|
59
|
+
def on_stop
|
60
|
+
end
|
61
|
+
|
62
|
+
# A hook to override that will be called when the job finished iterating.
|
63
|
+
#
|
64
|
+
def on_complete
|
65
|
+
end
|
66
|
+
|
67
|
+
# The enumerator to be iterated over.
|
68
|
+
#
|
69
|
+
# @return [Enumerator]
|
70
|
+
#
|
71
|
+
# @raise [NotImplementedError] with a message advising subclasses to
|
72
|
+
# implement an override for this method.
|
73
|
+
#
|
74
|
+
def build_enumerator(*)
|
75
|
+
raise NotImplementedError, "#{self.class.name} must implement a '#build_enumerator' method"
|
76
|
+
end
|
77
|
+
|
78
|
+
# The action to be performed on each item from the enumerator.
|
79
|
+
#
|
80
|
+
# @return [void]
|
81
|
+
#
|
82
|
+
# @raise [NotImplementedError] with a message advising subclasses to
|
83
|
+
# implement an override for this method.
|
84
|
+
#
|
85
|
+
def each_iteration(*)
|
86
|
+
raise NotImplementedError, "#{self.class.name} must implement an '#each_iteration' method"
|
87
|
+
end
|
88
|
+
|
89
|
+
def iteration_key
|
90
|
+
"it-#{jid}"
|
91
|
+
end
|
92
|
+
|
93
|
+
# @api private
|
94
|
+
def perform(*arguments)
|
95
|
+
fetch_previous_iteration_state
|
96
|
+
|
97
|
+
@_executions += 1
|
98
|
+
@_start_time = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
99
|
+
|
100
|
+
enumerator = build_enumerator(*arguments, cursor: @_cursor)
|
101
|
+
unless enumerator
|
102
|
+
logger.info("'#build_enumerator' returned nil, skipping the job.")
|
103
|
+
return
|
104
|
+
end
|
105
|
+
|
106
|
+
assert_enumerator!(enumerator)
|
107
|
+
|
108
|
+
if @_executions == 1
|
109
|
+
on_start
|
110
|
+
else
|
111
|
+
on_resume
|
112
|
+
end
|
113
|
+
|
114
|
+
completed = catch(:abort) do
|
115
|
+
iterate_with_enumerator(enumerator, arguments)
|
116
|
+
end
|
117
|
+
|
118
|
+
on_stop
|
119
|
+
completed = handle_completed(completed)
|
120
|
+
|
121
|
+
if completed
|
122
|
+
on_complete
|
123
|
+
cleanup
|
124
|
+
else
|
125
|
+
reenqueue_iteration_job
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
private
|
130
|
+
|
131
|
+
def fetch_previous_iteration_state
|
132
|
+
state = Sidekiq.redis { |conn| conn.hgetall(iteration_key) }
|
133
|
+
|
134
|
+
unless state.empty?
|
135
|
+
@_executions = state["ex"].to_i
|
136
|
+
@_cursor = Sidekiq.load_json(state["c"])
|
137
|
+
@_runtime = state["rt"].to_f
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
STATE_FLUSH_INTERVAL = 5 # seconds
|
142
|
+
# we need to keep the state around as long as the job
|
143
|
+
# might be retrying
|
144
|
+
STATE_TTL = 30 * 24 * 60 * 60 # one month
|
145
|
+
|
146
|
+
def iterate_with_enumerator(enumerator, arguments)
|
147
|
+
found_record = false
|
148
|
+
state_flushed_at = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
149
|
+
|
150
|
+
enumerator.each do |object, cursor|
|
151
|
+
found_record = true
|
152
|
+
@_cursor = cursor
|
153
|
+
|
154
|
+
is_interrupted = interrupted?
|
155
|
+
if ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - state_flushed_at >= STATE_FLUSH_INTERVAL || is_interrupted
|
156
|
+
flush_state
|
157
|
+
state_flushed_at = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
158
|
+
end
|
159
|
+
|
160
|
+
return false if is_interrupted
|
161
|
+
|
162
|
+
around_iteration do
|
163
|
+
each_iteration(object, *arguments)
|
164
|
+
end
|
165
|
+
end
|
166
|
+
|
167
|
+
logger.debug("Enumerator found nothing to iterate!") unless found_record
|
168
|
+
true
|
169
|
+
ensure
|
170
|
+
@_runtime += (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @_start_time)
|
171
|
+
end
|
172
|
+
|
173
|
+
def reenqueue_iteration_job
|
174
|
+
flush_state
|
175
|
+
logger.debug { "Interrupting job (cursor=#{@_cursor.inspect})" }
|
176
|
+
|
177
|
+
raise Interrupted
|
178
|
+
end
|
179
|
+
|
180
|
+
def assert_enumerator!(enum)
|
181
|
+
unless enum.is_a?(Enumerator)
|
182
|
+
raise ArgumentError, <<~MSG
|
183
|
+
#build_enumerator must return an Enumerator, but returned #{enum.class}.
|
184
|
+
Example:
|
185
|
+
def build_enumerator(params, cursor:)
|
186
|
+
active_record_records_enumerator(
|
187
|
+
Shop.find(params["shop_id"]).products,
|
188
|
+
cursor: cursor
|
189
|
+
)
|
190
|
+
end
|
191
|
+
MSG
|
192
|
+
end
|
193
|
+
end
|
194
|
+
|
195
|
+
def flush_state
|
196
|
+
key = iteration_key
|
197
|
+
state = {
|
198
|
+
"ex" => @_executions,
|
199
|
+
"c" => Sidekiq.dump_json(@_cursor),
|
200
|
+
"rt" => @_runtime
|
201
|
+
}
|
202
|
+
|
203
|
+
Sidekiq.redis do |conn|
|
204
|
+
conn.multi do |pipe|
|
205
|
+
pipe.hset(key, state)
|
206
|
+
pipe.expire(key, STATE_TTL)
|
207
|
+
end
|
208
|
+
end
|
209
|
+
end
|
210
|
+
|
211
|
+
def cleanup
|
212
|
+
logger.debug {
|
213
|
+
format("Completed iteration. executions=%d runtime=%.3f", @_executions, @_runtime)
|
214
|
+
}
|
215
|
+
Sidekiq.redis { |conn| conn.unlink(iteration_key) }
|
216
|
+
end
|
217
|
+
|
218
|
+
def handle_completed(completed)
|
219
|
+
case completed
|
220
|
+
when nil, # someone aborted the job but wants to call the on_complete callback
|
221
|
+
true
|
222
|
+
true
|
223
|
+
when false
|
224
|
+
false
|
225
|
+
else
|
226
|
+
raise "Unexpected thrown value: #{completed.inspect}"
|
227
|
+
end
|
228
|
+
end
|
229
|
+
end
|
230
|
+
end
|
231
|
+
end
|