job-iteration 1.7.0 → 1.9.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +1 -0
- data/CHANGELOG.md +32 -1
- data/Gemfile +3 -0
- data/Gemfile.lock +100 -55
- data/README.md +1 -1
- data/guides/custom-enumerator.md +13 -9
- data/guides/throttling.md +23 -1
- data/lib/job-iteration/active_record_enumerator.rb +7 -1
- data/lib/job-iteration/enumerator_builder.rb +12 -9
- data/lib/job-iteration/interruption_adapters/sqs_adapter.rb +36 -0
- data/lib/job-iteration/interruption_adapters.rb +1 -1
- data/lib/job-iteration/iteration.rb +8 -1
- data/lib/job-iteration/version.rb +1 -1
- data/lib/tapioca/dsl/compilers/job_iteration.rb +112 -0
- metadata +5 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b32eb2fd6eb2dc54f59eff462a3cde3882192134c2964ffc05b737957e3d4abb
|
4
|
+
data.tar.gz: 255864c2fac9d3cc0f75da1929d18c1aeb8e63ff83ea3dc6c19038c0e4894e54
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7c241d626e4c92c4424dd08745feab35f1606fafbf6f0d610f34a704d16702a52623f646e5715925b331509257c737053b25bf90091f61a948b6fba2315cc659
|
7
|
+
data.tar.gz: 19b821e84f93954cea21b0709455ac5ad12b2633f70c2182ae93e75d2e883a0985dda1cf7c56ac89295caab1720ef3508857201d0c33f18b35ffad76996f093b
|
data/.github/workflows/ci.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,37 @@
|
|
1
1
|
### Main (unreleased)
|
2
2
|
|
3
|
-
|
3
|
+
### Changes
|
4
|
+
|
5
|
+
nil
|
6
|
+
|
7
|
+
### Features
|
8
|
+
|
9
|
+
nil
|
10
|
+
|
11
|
+
### Bug fixes
|
12
|
+
|
13
|
+
nil
|
14
|
+
|
15
|
+
## v1.9.0 (Feb 3, 2024)
|
16
|
+
|
17
|
+
### Features
|
18
|
+
|
19
|
+
- [533](https://github.com/Shopify/job-iteration/pull/533) Added a custom compiler for [Tapioca](https://github.com/Shopify/tapioca) that generates Sorbet types for `MyJob.perform_later` if `MyJob` includes `JobIteration::Iteration` and has defined a type for `build_enumerator`.
|
20
|
+
|
21
|
+
## v1.8.0 (Dec 10, 2024)
|
22
|
+
|
23
|
+
### Changes
|
24
|
+
|
25
|
+
- [513](https://github.com/Shopify/job-iteration/pull/513) Deprecate returning enumerators from `build_enumerator` that are not wrapped with `enumerator_builder.wrap`. The built-in enumerator builders now always wrap.
|
26
|
+
|
27
|
+
### Features
|
28
|
+
|
29
|
+
- [340](https://github.com/Shopify/job-iteration/pull/340) Add `cursor.iteration` instrumentation for the query to fetch the next batch of records for the Active Record cursor.
|
30
|
+
- [523](https://github.com/Shopify/job-iteration/pull/523) Add interruption adapter for [aws-activejob-sqs](https://github.com/aws/aws-activejob-sqs-ruby).
|
31
|
+
|
32
|
+
### Bug fixes
|
33
|
+
|
34
|
+
- [515](https://github.com/Shopify/job-iteration/pull/515) Fix size of array enumerators.
|
4
35
|
|
5
36
|
## v1.7.0 (Oct 11, 2024)
|
6
37
|
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
GIT
|
2
2
|
remote: https://github.com/brianmario/mysql2
|
3
|
-
revision:
|
3
|
+
revision: 57b8df188c963ae0e4d4e1123d3e9de2bbcab637
|
4
4
|
specs:
|
5
5
|
mysql2 (0.5.6)
|
6
6
|
bigdecimal
|
@@ -8,123 +8,166 @@ GIT
|
|
8
8
|
PATH
|
9
9
|
remote: .
|
10
10
|
specs:
|
11
|
-
job-iteration (1.
|
11
|
+
job-iteration (1.9.0)
|
12
12
|
activejob (>= 5.2)
|
13
13
|
|
14
14
|
GEM
|
15
15
|
remote: https://rubygems.org/
|
16
16
|
specs:
|
17
|
-
activejob (
|
18
|
-
activesupport (=
|
17
|
+
activejob (8.0.1)
|
18
|
+
activesupport (= 8.0.1)
|
19
19
|
globalid (>= 0.3.6)
|
20
|
-
activemodel (
|
21
|
-
activesupport (=
|
22
|
-
activerecord (
|
23
|
-
activemodel (=
|
24
|
-
activesupport (=
|
20
|
+
activemodel (8.0.1)
|
21
|
+
activesupport (= 8.0.1)
|
22
|
+
activerecord (8.0.1)
|
23
|
+
activemodel (= 8.0.1)
|
24
|
+
activesupport (= 8.0.1)
|
25
25
|
timeout (>= 0.4.0)
|
26
|
-
activesupport (
|
26
|
+
activesupport (8.0.1)
|
27
27
|
base64
|
28
|
+
benchmark (>= 0.3)
|
28
29
|
bigdecimal
|
29
|
-
concurrent-ruby (~> 1.0, >= 1.
|
30
|
+
concurrent-ruby (~> 1.0, >= 1.3.1)
|
30
31
|
connection_pool (>= 2.2.5)
|
31
32
|
drb
|
32
33
|
i18n (>= 1.6, < 2)
|
34
|
+
logger (>= 1.4.2)
|
33
35
|
minitest (>= 5.1)
|
34
|
-
|
35
|
-
tzinfo (~> 2.0)
|
36
|
+
securerandom (>= 0.3)
|
37
|
+
tzinfo (~> 2.0, >= 2.0.5)
|
38
|
+
uri (>= 0.13.1)
|
36
39
|
ast (2.4.2)
|
37
40
|
base64 (0.2.0)
|
38
|
-
|
41
|
+
benchmark (0.4.0)
|
42
|
+
bigdecimal (3.1.9)
|
39
43
|
coderay (1.1.3)
|
40
|
-
concurrent-ruby (1.3.
|
41
|
-
connection_pool (2.
|
42
|
-
csv (3.3.
|
44
|
+
concurrent-ruby (1.3.5)
|
45
|
+
connection_pool (2.5.0)
|
46
|
+
csv (3.3.2)
|
43
47
|
drb (2.2.1)
|
48
|
+
erubi (1.13.1)
|
44
49
|
globalid (1.2.1)
|
45
50
|
activesupport (>= 6.1)
|
46
|
-
i18n (1.14.
|
51
|
+
i18n (1.14.7)
|
47
52
|
concurrent-ruby (~> 1.0)
|
48
|
-
json (2.
|
49
|
-
language_server-protocol (3.17.0.
|
53
|
+
json (2.9.1)
|
54
|
+
language_server-protocol (3.17.0.4)
|
55
|
+
logger (1.6.5)
|
50
56
|
method_source (1.1.0)
|
51
|
-
minitest (5.
|
52
|
-
mocha (2.
|
57
|
+
minitest (5.25.4)
|
58
|
+
mocha (2.7.1)
|
53
59
|
ruby2_keywords (>= 0.0.5)
|
54
60
|
mono_logger (1.1.2)
|
55
61
|
multi_json (1.15.0)
|
56
|
-
mustermann (3.0.
|
62
|
+
mustermann (3.0.3)
|
57
63
|
ruby2_keywords (~> 0.0.1)
|
58
|
-
|
59
|
-
parallel (1.
|
60
|
-
parser (3.3.
|
64
|
+
netrc (0.11.0)
|
65
|
+
parallel (1.26.3)
|
66
|
+
parser (3.3.7.0)
|
61
67
|
ast (~> 2.4.1)
|
62
68
|
racc
|
63
|
-
|
69
|
+
prism (1.3.0)
|
70
|
+
pry (0.15.2)
|
64
71
|
coderay (~> 1.1)
|
65
72
|
method_source (~> 1.0)
|
66
|
-
racc (1.8.
|
67
|
-
rack (3.1.
|
68
|
-
rack-protection (4.
|
73
|
+
racc (1.8.1)
|
74
|
+
rack (3.1.8)
|
75
|
+
rack-protection (4.1.1)
|
69
76
|
base64 (>= 0.1.0)
|
77
|
+
logger (>= 1.6.0)
|
70
78
|
rack (>= 3.0.0, < 4)
|
71
|
-
rack-session (2.
|
79
|
+
rack-session (2.1.0)
|
80
|
+
base64 (>= 0.1.0)
|
72
81
|
rack (>= 3.0.0)
|
73
82
|
rainbow (3.1.1)
|
74
83
|
rake (13.2.1)
|
84
|
+
rbi (0.2.4)
|
85
|
+
prism (~> 1.0)
|
86
|
+
sorbet-runtime (>= 0.5.9204)
|
75
87
|
redis (5.3.0)
|
76
88
|
redis-client (>= 0.22.0)
|
77
|
-
redis-client (0.
|
89
|
+
redis-client (0.23.2)
|
78
90
|
connection_pool
|
79
91
|
redis-namespace (1.11.0)
|
80
92
|
redis (>= 4)
|
81
|
-
regexp_parser (2.
|
82
|
-
resque (2.
|
83
|
-
mono_logger (~> 1
|
93
|
+
regexp_parser (2.10.0)
|
94
|
+
resque (2.7.0)
|
95
|
+
mono_logger (~> 1)
|
84
96
|
multi_json (~> 1.0)
|
85
97
|
redis-namespace (~> 1.6)
|
86
98
|
sinatra (>= 0.9.2)
|
87
|
-
|
88
|
-
strscan
|
89
|
-
rubocop (1.64.1)
|
99
|
+
rubocop (1.71.0)
|
90
100
|
json (~> 2.3)
|
91
101
|
language_server-protocol (>= 3.17.0)
|
92
102
|
parallel (~> 1.10)
|
93
103
|
parser (>= 3.3.0.2)
|
94
104
|
rainbow (>= 2.2.2, < 4.0)
|
95
|
-
regexp_parser (>=
|
96
|
-
|
97
|
-
rubocop-ast (>= 1.31.1, < 2.0)
|
105
|
+
regexp_parser (>= 2.9.3, < 3.0)
|
106
|
+
rubocop-ast (>= 1.36.2, < 2.0)
|
98
107
|
ruby-progressbar (~> 1.7)
|
99
|
-
unicode-display_width (>= 2.4.0, <
|
100
|
-
rubocop-ast (1.
|
108
|
+
unicode-display_width (>= 2.4.0, < 4.0)
|
109
|
+
rubocop-ast (1.38.0)
|
101
110
|
parser (>= 3.3.1.0)
|
102
111
|
rubocop-shopify (2.15.1)
|
103
112
|
rubocop (~> 1.51)
|
104
113
|
ruby-progressbar (1.13.0)
|
105
114
|
ruby2_keywords (0.0.5)
|
106
|
-
|
107
|
-
|
115
|
+
securerandom (0.4.1)
|
116
|
+
sidekiq (7.3.8)
|
117
|
+
base64
|
108
118
|
connection_pool (>= 2.3.0)
|
119
|
+
logger
|
109
120
|
rack (>= 2.2.4)
|
110
|
-
redis-client (>= 0.
|
111
|
-
sinatra (4.
|
121
|
+
redis-client (>= 0.22.2)
|
122
|
+
sinatra (4.1.1)
|
123
|
+
logger (>= 1.6.0)
|
112
124
|
mustermann (~> 3.0)
|
113
125
|
rack (>= 3.0.0, < 4)
|
114
|
-
rack-protection (= 4.
|
126
|
+
rack-protection (= 4.1.1)
|
115
127
|
rack-session (>= 2.0.0, < 3)
|
116
128
|
tilt (~> 2.0)
|
117
|
-
sorbet
|
118
|
-
|
119
|
-
|
120
|
-
|
129
|
+
sorbet (0.5.11787)
|
130
|
+
sorbet-static (= 0.5.11787)
|
131
|
+
sorbet-runtime (0.5.11787)
|
132
|
+
sorbet-static (0.5.11787-universal-darwin)
|
133
|
+
sorbet-static (0.5.11787-x86_64-linux)
|
134
|
+
sorbet-static-and-runtime (0.5.11787)
|
135
|
+
sorbet (= 0.5.11787)
|
136
|
+
sorbet-runtime (= 0.5.11787)
|
137
|
+
spoom (1.5.2)
|
138
|
+
erubi (>= 1.10.0)
|
139
|
+
prism (>= 0.28.0)
|
140
|
+
rbi (>= 0.2.3)
|
141
|
+
sorbet-static-and-runtime (>= 0.5.10187)
|
142
|
+
thor (>= 0.19.2)
|
143
|
+
tapioca (0.16.8)
|
144
|
+
benchmark
|
145
|
+
bundler (>= 2.2.25)
|
146
|
+
netrc (>= 0.11.0)
|
147
|
+
parallel (>= 1.21.0)
|
148
|
+
rbi (~> 0.2)
|
149
|
+
sorbet-static-and-runtime (>= 0.5.11087)
|
150
|
+
spoom (>= 1.2.0)
|
151
|
+
thor (>= 1.2.0)
|
152
|
+
yard-sorbet
|
153
|
+
thor (1.3.2)
|
154
|
+
tilt (2.6.0)
|
155
|
+
timeout (0.4.3)
|
121
156
|
tzinfo (2.0.6)
|
122
157
|
concurrent-ruby (~> 1.0)
|
123
|
-
unicode-display_width (
|
158
|
+
unicode-display_width (3.1.4)
|
159
|
+
unicode-emoji (~> 4.0, >= 4.0.4)
|
160
|
+
unicode-emoji (4.0.4)
|
161
|
+
uri (1.0.2)
|
124
162
|
yard (0.9.37)
|
163
|
+
yard-sorbet (0.9.0)
|
164
|
+
sorbet-runtime
|
165
|
+
yard
|
125
166
|
|
126
167
|
PLATFORMS
|
127
|
-
|
168
|
+
arm64-darwin
|
169
|
+
x86_64-darwin
|
170
|
+
x86_64-linux
|
128
171
|
|
129
172
|
DEPENDENCIES
|
130
173
|
activerecord
|
@@ -132,6 +175,7 @@ DEPENDENCIES
|
|
132
175
|
globalid
|
133
176
|
i18n
|
134
177
|
job-iteration!
|
178
|
+
logger
|
135
179
|
mocha
|
136
180
|
mysql2!
|
137
181
|
pry
|
@@ -141,7 +185,8 @@ DEPENDENCIES
|
|
141
185
|
rubocop-shopify
|
142
186
|
sidekiq
|
143
187
|
sorbet-runtime
|
188
|
+
tapioca
|
144
189
|
yard
|
145
190
|
|
146
191
|
BUNDLED WITH
|
147
|
-
2.
|
192
|
+
2.6.1
|
data/README.md
CHANGED
@@ -161,7 +161,7 @@ Iteration hooks into Sidekiq and Resque out of the box to support graceful inter
|
|
161
161
|
* [Writing custom enumerator](guides/custom-enumerator.md)
|
162
162
|
* [Throttling](guides/throttling.md)
|
163
163
|
|
164
|
-
For more detailed documentation, see [rubydoc](https://www.rubydoc.info/
|
164
|
+
For more detailed documentation, see [rubydoc](https://www.rubydoc.info/gems/job-iteration).
|
165
165
|
|
166
166
|
## Requirements
|
167
167
|
|
data/guides/custom-enumerator.md
CHANGED
@@ -78,12 +78,14 @@ class LoadRefundsForChargeJob < ActiveJob::Base
|
|
78
78
|
# Use an exponential back-off strategy when Stripe's API returns errors.
|
79
79
|
|
80
80
|
def build_enumerator(charge_id, cursor:)
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
81
|
+
enumerator_builder.wrap(
|
82
|
+
StripeListEnumerator.new(
|
83
|
+
Stripe::Refund,
|
84
|
+
params: { charge: charge_id}, # "charge_id" will be a prefixed Stripe ID such as "chrg_123"
|
85
|
+
options: { api_key: "sk_test_123", stripe_version: "2018-01-18" },
|
86
|
+
cursor: cursor
|
87
|
+
).to_enumerator
|
88
|
+
)
|
87
89
|
end
|
88
90
|
|
89
91
|
# Note that in this case `each_iteration` will only receive one positional argument per iteration.
|
@@ -114,9 +116,11 @@ class RedisPopListJob < ActiveJob::Base
|
|
114
116
|
# @see https://redis.io/commands/lpop/
|
115
117
|
def build_enumerator(*)
|
116
118
|
@redis = Redis.new
|
117
|
-
|
118
|
-
|
119
|
-
|
119
|
+
enumerator_builder.wrap(
|
120
|
+
Enumerator.new do |yielder|
|
121
|
+
yielder.yield @redis.lpop(key), nil
|
122
|
+
end
|
123
|
+
)
|
120
124
|
end
|
121
125
|
|
122
126
|
def each_iteration(item_from_redis)
|
data/guides/throttling.md
CHANGED
@@ -38,9 +38,31 @@ def build_enumerator(_params, cursor:)
|
|
38
38
|
end
|
39
39
|
```
|
40
40
|
|
41
|
+
If you want to apply throttling on all jobs, you can subclass your own EnumeratorBuilder and override the default
|
42
|
+
enumerator builder. The builder always wraps the returned enumerators from `build_enumerator`
|
43
|
+
|
44
|
+
```ruby
|
45
|
+
class MyOwnBuilder < JobIteration::EnumeratorBuilder
|
46
|
+
class Wrapper < Enumerator
|
47
|
+
class << self
|
48
|
+
def wrap(_builder, enum)
|
49
|
+
ThrottleEnumerator.new(
|
50
|
+
enum,
|
51
|
+
nil,
|
52
|
+
throttle_on: -> { DatabaseStatus.unhealthy? },
|
53
|
+
backoff: 30.seconds
|
54
|
+
)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
JobIteration.enumerator_builder = MyOwnBuilder
|
61
|
+
```
|
62
|
+
|
41
63
|
Note that it's up to you to implement `DatabaseStatus.unhealthy?` that works for your database choice. At Shopify, a helper like `DatabaseStatus` checks the following MySQL metrics:
|
42
64
|
|
43
65
|
* Replication lag across all regions
|
44
66
|
* DB threads
|
45
67
|
* DB is available for writes (otherwise indicates a failover happening)
|
46
|
-
* [Semian](https://github.com/shopify/semian) open circuits
|
68
|
+
* [Semian](https://github.com/shopify/semian) open circuits
|
@@ -32,7 +32,7 @@ module JobIteration
|
|
32
32
|
def batches
|
33
33
|
cursor = finder_cursor
|
34
34
|
Enumerator.new(method(:size)) do |yielder|
|
35
|
-
while (records = cursor
|
35
|
+
while (records = instrument_next_batch(cursor))
|
36
36
|
yielder.yield(records, cursor_value(records.last)) if records.any?
|
37
37
|
end
|
38
38
|
end
|
@@ -44,6 +44,12 @@ module JobIteration
|
|
44
44
|
|
45
45
|
private
|
46
46
|
|
47
|
+
def instrument_next_batch(cursor)
|
48
|
+
ActiveSupport::Notifications.instrument("active_record_cursor.iteration") do
|
49
|
+
cursor.next_batch(@batch_size)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
47
53
|
def cursor_value(record)
|
48
54
|
positions = @columns.map do |column|
|
49
55
|
attribute_name = column.to_s.split(".").last
|
@@ -16,8 +16,7 @@ module JobIteration
|
|
16
16
|
# `enumerator_builder` is _always_ the type that is returned from
|
17
17
|
# `build_enumerator`. This prevents people from implementing custom
|
18
18
|
# Enumerators without wrapping them in
|
19
|
-
# `enumerator_builder.wrap(custom_enum)`.
|
20
|
-
# compatibility with raw calls to EnumeratorBuilder. Think of these wrappers
|
19
|
+
# `enumerator_builder.wrap(custom_enum)`. Think of these wrappers
|
21
20
|
# the way you should a middleware.
|
22
21
|
class Wrapper < Enumerator
|
23
22
|
class << self
|
@@ -63,7 +62,7 @@ module JobIteration
|
|
63
62
|
cursor + 1
|
64
63
|
end
|
65
64
|
|
66
|
-
wrap(self, enumerable.each_with_index.drop(drop).to_enum { enumerable.size })
|
65
|
+
wrap(self, enumerable.each_with_index.drop(drop).to_enum { enumerable.size - drop })
|
67
66
|
end
|
68
67
|
|
69
68
|
# Builds Enumerator from Active Record Relation. Each Enumerator tick moves the cursor one row forward.
|
@@ -131,21 +130,24 @@ module JobIteration
|
|
131
130
|
enum
|
132
131
|
end
|
133
132
|
|
134
|
-
def build_throttle_enumerator(
|
135
|
-
JobIteration::ThrottleEnumerator.new(
|
136
|
-
|
133
|
+
def build_throttle_enumerator(enumerable, throttle_on:, backoff:)
|
134
|
+
enum = JobIteration::ThrottleEnumerator.new(
|
135
|
+
enumerable,
|
137
136
|
@job,
|
138
137
|
throttle_on: throttle_on,
|
139
138
|
backoff: backoff,
|
140
139
|
).to_enum
|
140
|
+
wrap(self, enum)
|
141
141
|
end
|
142
142
|
|
143
143
|
def build_csv_enumerator(enumerable, cursor:)
|
144
|
-
CsvEnumerator.new(enumerable).rows(cursor: cursor)
|
144
|
+
enum = CsvEnumerator.new(enumerable).rows(cursor: cursor)
|
145
|
+
wrap(self, enum)
|
145
146
|
end
|
146
147
|
|
147
148
|
def build_csv_enumerator_on_batches(enumerable, cursor:, batch_size: 100)
|
148
|
-
CsvEnumerator.new(enumerable).batches(cursor: cursor, batch_size: batch_size)
|
149
|
+
enum = CsvEnumerator.new(enumerable).batches(cursor: cursor, batch_size: batch_size)
|
150
|
+
wrap(self, enum)
|
149
151
|
end
|
150
152
|
|
151
153
|
# Builds Enumerator for nested iteration.
|
@@ -179,7 +181,8 @@ module JobIteration
|
|
179
181
|
# end
|
180
182
|
#
|
181
183
|
def build_nested_enumerator(enums, cursor:)
|
182
|
-
NestedEnumerator.new(enums, cursor: cursor).each
|
184
|
+
enum = NestedEnumerator.new(enums, cursor: cursor).each
|
185
|
+
wrap(self, enum)
|
183
186
|
end
|
184
187
|
|
185
188
|
alias_method :once, :build_once_enumerator
|
@@ -0,0 +1,36 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
begin
|
4
|
+
require "aws-activejob-sqs"
|
5
|
+
rescue LoadError
|
6
|
+
# Aws::ActiveJob::SQS is not available, no need to load the adapter
|
7
|
+
return
|
8
|
+
end
|
9
|
+
|
10
|
+
begin
|
11
|
+
# Aws::ActiveJob::SQS.on_worker_stop was introduced in Aws::ActiveJob::SQS 0.1.1
|
12
|
+
gem("aws-activejob-sqs", ">= 0.1.1")
|
13
|
+
rescue Gem::LoadError
|
14
|
+
warn("job-iteration's interruption adapter for SQS requires aws-activejob-sqs 0.1.1 or newer")
|
15
|
+
return
|
16
|
+
end
|
17
|
+
|
18
|
+
module JobIteration
|
19
|
+
module InterruptionAdapters
|
20
|
+
module SqsAdapter
|
21
|
+
class << self
|
22
|
+
attr_accessor :stopping
|
23
|
+
|
24
|
+
def call
|
25
|
+
stopping
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
Aws::ActiveJob::SQS.on_worker_stop do
|
30
|
+
SqsAdapter.stopping = true
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
register(:sqs, SqsAdapter)
|
35
|
+
end
|
36
|
+
end
|
@@ -4,7 +4,7 @@ require_relative "interruption_adapters/null_adapter"
|
|
4
4
|
|
5
5
|
module JobIteration
|
6
6
|
module InterruptionAdapters
|
7
|
-
BUNDLED_ADAPTERS = [:good_job, :resque, :sidekiq, :solid_queue].freeze # @api private
|
7
|
+
BUNDLED_ADAPTERS = [:good_job, :resque, :sidekiq, :solid_queue, :sqs].freeze # @api private
|
8
8
|
|
9
9
|
class << self
|
10
10
|
# Returns adapter for specified name.
|
@@ -219,7 +219,14 @@ module JobIteration
|
|
219
219
|
end
|
220
220
|
|
221
221
|
def assert_enumerator!(enum)
|
222
|
-
|
222
|
+
if enum.is_a?(Enumerator)
|
223
|
+
unless enum.is_a?(JobIteration.enumerator_builder::Wrapper)
|
224
|
+
JobIteration::Deprecation.warn("Returning an unwrapped enumerator from build_enumerator is deprecated. " \
|
225
|
+
"Wrap the enumerator using enumerator_builder.wrap(my_enumerator) instead.")
|
226
|
+
end
|
227
|
+
|
228
|
+
return
|
229
|
+
end
|
223
230
|
|
224
231
|
raise ArgumentError, <<~EOS
|
225
232
|
#build_enumerator is expected to return Enumerator object, but returned #{enum.class}.
|
@@ -0,0 +1,112 @@
|
|
1
|
+
# typed: strict
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
return unless defined?(JobIteration::Iteration)
|
5
|
+
|
6
|
+
module Tapioca
|
7
|
+
module Dsl
|
8
|
+
module Compilers
|
9
|
+
class JobIteration < Compiler
|
10
|
+
extend T::Sig
|
11
|
+
|
12
|
+
ConstantType = type_member { { fixed: T.class_of(::JobIteration::Iteration) } }
|
13
|
+
PARAM_TYPES_IN_ORDER = [
|
14
|
+
RBI::Param,
|
15
|
+
RBI::OptParam,
|
16
|
+
RBI::RestParam,
|
17
|
+
RBI::KwParam,
|
18
|
+
RBI::KwOptParam,
|
19
|
+
RBI::KwRestParam,
|
20
|
+
RBI::BlockParam,
|
21
|
+
].freeze
|
22
|
+
|
23
|
+
sig { override.void }
|
24
|
+
def decorate
|
25
|
+
return unless constant.instance_methods(false).include?(:build_enumerator)
|
26
|
+
|
27
|
+
root.create_path(constant) do |job|
|
28
|
+
method = constant.instance_method(:build_enumerator)
|
29
|
+
constant_name = name_of(constant)
|
30
|
+
signature = signature_of(method)
|
31
|
+
|
32
|
+
parameters = compile_method_parameters_to_rbi(method).reject do |typed_param|
|
33
|
+
typed_param.param.name == "cursor"
|
34
|
+
end
|
35
|
+
|
36
|
+
if signature
|
37
|
+
fixed_hash_args = signature.arg_types.select { |arg_type| T::Types::FixedHash === arg_type[1] }.to_h
|
38
|
+
expanded_parameters = parameters.flat_map do |typed_param|
|
39
|
+
if (hash_type = fixed_hash_args[typed_param.param.name.to_sym])
|
40
|
+
hash_type.types.map do |key, value|
|
41
|
+
if value.name.start_with?("T.nilable")
|
42
|
+
create_kw_opt_param(key.to_s, type: value.to_s, default: "nil")
|
43
|
+
else
|
44
|
+
create_kw_param(key.to_s, type: value.to_s)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
else
|
48
|
+
typed_param
|
49
|
+
end
|
50
|
+
end
|
51
|
+
else
|
52
|
+
expanded_parameters = parameters
|
53
|
+
end
|
54
|
+
|
55
|
+
# Sorbet expects optional keyword arguments to be after required keyword arguments.
|
56
|
+
expanded_parameters.sort_by! { |typed_param| PARAM_TYPES_IN_ORDER.index(typed_param.param.class) }
|
57
|
+
|
58
|
+
job.create_method(
|
59
|
+
"perform_later",
|
60
|
+
parameters: perform_later_parameters(expanded_parameters, constant_name),
|
61
|
+
return_type: "T.any(#{constant_name}, FalseClass)",
|
62
|
+
class_method: true,
|
63
|
+
)
|
64
|
+
|
65
|
+
job.create_method(
|
66
|
+
"perform_now",
|
67
|
+
parameters: expanded_parameters,
|
68
|
+
return_type: "T.any(NilClass, Exception)",
|
69
|
+
class_method: true,
|
70
|
+
)
|
71
|
+
|
72
|
+
job.create_method(
|
73
|
+
"perform",
|
74
|
+
parameters: expanded_parameters,
|
75
|
+
return_type: "void",
|
76
|
+
class_method: false,
|
77
|
+
)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
private
|
82
|
+
|
83
|
+
sig do
|
84
|
+
params(
|
85
|
+
parameters: T::Array[RBI::TypedParam],
|
86
|
+
constant_name: T.nilable(String),
|
87
|
+
).returns(T::Array[RBI::TypedParam])
|
88
|
+
end
|
89
|
+
def perform_later_parameters(parameters, constant_name)
|
90
|
+
if ::Gem::Requirement.new(">= 7.0").satisfied_by?(::ActiveJob.gem_version)
|
91
|
+
parameters.reject! { |typed_param| RBI::BlockParam === typed_param.param }
|
92
|
+
parameters + [create_block_param(
|
93
|
+
"block",
|
94
|
+
type: "T.nilable(T.proc.params(job: #{constant_name}).void)",
|
95
|
+
)]
|
96
|
+
else
|
97
|
+
parameters
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
class << self
|
102
|
+
extend T::Sig
|
103
|
+
|
104
|
+
sig { override.returns(T::Enumerable[Module]) }
|
105
|
+
def gather_constants
|
106
|
+
all_classes.select { |c| ::JobIteration::Iteration > c }
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: job-iteration
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.9.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shopify
|
8
|
-
autorequire:
|
9
8
|
bindir: exe
|
10
9
|
cert_chain: []
|
11
|
-
date:
|
10
|
+
date: 2025-02-03 00:00:00.000000000 Z
|
12
11
|
dependencies:
|
13
12
|
- !ruby/object:Gem::Dependency
|
14
13
|
name: activejob
|
@@ -67,6 +66,7 @@ files:
|
|
67
66
|
- lib/job-iteration/interruption_adapters/resque_adapter.rb
|
68
67
|
- lib/job-iteration/interruption_adapters/sidekiq_adapter.rb
|
69
68
|
- lib/job-iteration/interruption_adapters/solid_queue_adapter.rb
|
69
|
+
- lib/job-iteration/interruption_adapters/sqs_adapter.rb
|
70
70
|
- lib/job-iteration/iteration.rb
|
71
71
|
- lib/job-iteration/log_subscriber.rb
|
72
72
|
- lib/job-iteration/nested_enumerator.rb
|
@@ -74,13 +74,13 @@ files:
|
|
74
74
|
- lib/job-iteration/test_helper.rb
|
75
75
|
- lib/job-iteration/throttle_enumerator.rb
|
76
76
|
- lib/job-iteration/version.rb
|
77
|
+
- lib/tapioca/dsl/compilers/job_iteration.rb
|
77
78
|
homepage: https://github.com/shopify/job-iteration
|
78
79
|
licenses:
|
79
80
|
- MIT
|
80
81
|
metadata:
|
81
82
|
changelog_uri: https://github.com/Shopify/job-iteration/blob/main/CHANGELOG.md
|
82
83
|
allowed_push_host: https://rubygems.org
|
83
|
-
post_install_message:
|
84
84
|
rdoc_options: []
|
85
85
|
require_paths:
|
86
86
|
- lib
|
@@ -95,8 +95,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '0'
|
97
97
|
requirements: []
|
98
|
-
rubygems_version: 3.
|
99
|
-
signing_key:
|
98
|
+
rubygems_version: 3.6.3
|
100
99
|
specification_version: 4
|
101
100
|
summary: Makes your background jobs interruptible and resumable.
|
102
101
|
test_files: []
|