job-iteration 1.3.3 → 1.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +8 -3
- data/CHANGELOG.md +4 -0
- data/Gemfile +1 -1
- data/Gemfile.lock +29 -24
- data/LICENSE.txt +1 -1
- data/README.md +2 -2
- data/guides/best-practices.md +8 -0
- data/job-iteration.gemspec +1 -1
- data/lib/job-iteration/active_record_batch_enumerator.rb +2 -1
- data/lib/job-iteration/enumerator_builder.rb +4 -2
- data/lib/job-iteration/integrations/resque.rb +8 -2
- data/lib/job-iteration/iteration.rb +1 -1
- data/lib/job-iteration/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 5d279ff6eabe074dc75f9907a5e733d5bac2da3b97a6657fe2b1bc9eb6f03554
|
|
4
|
+
data.tar.gz: 59c7b39c5351e21ae50cee2234d169a8c0e86014f4294c3b942f76402056eabf
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: '021873185a818abe4f4c42744404239e8b0516c27f12fe028b0ed82920d5a1f595caf72a57955354c847575dc18671b8b5c47a2acd2c4b9cd24de153f916ebb2'
|
|
7
|
+
data.tar.gz: 1cbf2f0da940592a227ca922af2bc956be2c7f9ae68fb995f0f56d2a14f071983b37dc3726d364b10108e4344ae9f3e89ccaf88dae08b16b912ebf07ee110ddf
|
data/.github/workflows/ci.yml
CHANGED
|
@@ -13,13 +13,18 @@ jobs:
|
|
|
13
13
|
- 6379:6379
|
|
14
14
|
strategy:
|
|
15
15
|
matrix:
|
|
16
|
-
ruby: [2.6, 2.7, 3.0]
|
|
16
|
+
ruby: ["2.6", "2.7", "3.0"]
|
|
17
17
|
gemfile: [rails_5_2, rails_6_0, rails_edge]
|
|
18
18
|
exclude:
|
|
19
|
-
- ruby: 2.6
|
|
19
|
+
- ruby: "2.6"
|
|
20
20
|
gemfile: rails_edge
|
|
21
|
-
- ruby: 3.0
|
|
21
|
+
- ruby: "3.0"
|
|
22
22
|
gemfile: rails_5_2
|
|
23
|
+
include:
|
|
24
|
+
- ruby: "3.1"
|
|
25
|
+
gemfile: rails_edge
|
|
26
|
+
- ruby: head
|
|
27
|
+
gemfile: rails_edge
|
|
23
28
|
env:
|
|
24
29
|
BUNDLE_GEMFILE: gemfiles/${{ matrix.gemfile }}.gemfile
|
|
25
30
|
steps:
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
### Master (unreleased)
|
|
2
2
|
|
|
3
|
+
## v.1.3.4 (Jan 18, 2022)
|
|
4
|
+
|
|
5
|
+
- [174](https://github.com/Shopify/job-iteration/pull/174) - Fix Ruby 3.2 compatibility
|
|
6
|
+
|
|
3
7
|
## v.1.3.3 (Nov 17, 2021)
|
|
4
8
|
- [153](https://github.com/Shopify/job-iteration/pull/153) - Re-enqueue jobs only after shutdown hooks have run
|
|
5
9
|
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
|
@@ -1,21 +1,27 @@
|
|
|
1
|
+
GIT
|
|
2
|
+
remote: https://github.com/brianmario/mysql2
|
|
3
|
+
revision: 25c42c712118b046eb9df7a0f50ffde1a04ee6d1
|
|
4
|
+
specs:
|
|
5
|
+
mysql2 (0.5.3)
|
|
6
|
+
|
|
1
7
|
PATH
|
|
2
8
|
remote: .
|
|
3
9
|
specs:
|
|
4
|
-
job-iteration (1.3.
|
|
10
|
+
job-iteration (1.3.4)
|
|
5
11
|
activejob (>= 5.2)
|
|
6
12
|
|
|
7
13
|
GEM
|
|
8
14
|
remote: https://rubygems.org/
|
|
9
15
|
specs:
|
|
10
|
-
activejob (6.1.4.
|
|
11
|
-
activesupport (= 6.1.4.
|
|
16
|
+
activejob (6.1.4.4)
|
|
17
|
+
activesupport (= 6.1.4.4)
|
|
12
18
|
globalid (>= 0.3.6)
|
|
13
|
-
activemodel (6.1.4.
|
|
14
|
-
activesupport (= 6.1.4.
|
|
15
|
-
activerecord (6.1.4.
|
|
16
|
-
activemodel (= 6.1.4.
|
|
17
|
-
activesupport (= 6.1.4.
|
|
18
|
-
activesupport (6.1.4.
|
|
19
|
+
activemodel (6.1.4.4)
|
|
20
|
+
activesupport (= 6.1.4.4)
|
|
21
|
+
activerecord (6.1.4.4)
|
|
22
|
+
activemodel (= 6.1.4.4)
|
|
23
|
+
activesupport (= 6.1.4.4)
|
|
24
|
+
activesupport (6.1.4.4)
|
|
19
25
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
|
20
26
|
i18n (>= 1.6, < 2)
|
|
21
27
|
minitest (>= 5.1)
|
|
@@ -31,20 +37,19 @@ GEM
|
|
|
31
37
|
activerecord (>= 5.a)
|
|
32
38
|
database_cleaner-core (~> 2.0.0)
|
|
33
39
|
database_cleaner-core (2.0.1)
|
|
34
|
-
globalid (0.
|
|
40
|
+
globalid (1.0.0)
|
|
35
41
|
activesupport (>= 5.0)
|
|
36
42
|
i18n (1.8.11)
|
|
37
43
|
concurrent-ruby (~> 1.0)
|
|
38
44
|
method_source (1.0.0)
|
|
39
|
-
minitest (5.
|
|
45
|
+
minitest (5.15.0)
|
|
40
46
|
mocha (1.13.0)
|
|
41
47
|
mono_logger (1.1.1)
|
|
42
48
|
multi_json (1.15.0)
|
|
43
49
|
mustermann (1.1.1)
|
|
44
50
|
ruby2_keywords (~> 0.0.1)
|
|
45
|
-
mysql2 (0.5.3)
|
|
46
51
|
parallel (1.21.0)
|
|
47
|
-
parser (3.0.
|
|
52
|
+
parser (3.1.0.0)
|
|
48
53
|
ast (~> 2.4.1)
|
|
49
54
|
pry (0.14.1)
|
|
50
55
|
coderay (~> 1.1)
|
|
@@ -52,12 +57,12 @@ GEM
|
|
|
52
57
|
rack (2.2.3)
|
|
53
58
|
rack-protection (2.1.0)
|
|
54
59
|
rack
|
|
55
|
-
rainbow (3.
|
|
60
|
+
rainbow (3.1.1)
|
|
56
61
|
rake (13.0.6)
|
|
57
62
|
redis (4.5.1)
|
|
58
63
|
redis-namespace (1.8.1)
|
|
59
64
|
redis (>= 3.0.4)
|
|
60
|
-
regexp_parser (2.
|
|
65
|
+
regexp_parser (2.2.0)
|
|
61
66
|
resque (2.2.0)
|
|
62
67
|
mono_logger (~> 1.0)
|
|
63
68
|
multi_json (~> 1.0)
|
|
@@ -65,19 +70,19 @@ GEM
|
|
|
65
70
|
sinatra (>= 0.9.2)
|
|
66
71
|
vegas (~> 0.1.2)
|
|
67
72
|
rexml (3.2.5)
|
|
68
|
-
rubocop (1.
|
|
73
|
+
rubocop (1.24.1)
|
|
69
74
|
parallel (~> 1.10)
|
|
70
75
|
parser (>= 3.0.0.0)
|
|
71
76
|
rainbow (>= 2.2.2, < 4.0)
|
|
72
77
|
regexp_parser (>= 1.8, < 3.0)
|
|
73
78
|
rexml
|
|
74
|
-
rubocop-ast (>= 1.
|
|
79
|
+
rubocop-ast (>= 1.15.1, < 2.0)
|
|
75
80
|
ruby-progressbar (~> 1.7)
|
|
76
81
|
unicode-display_width (>= 1.4.0, < 3.0)
|
|
77
|
-
rubocop-ast (1.
|
|
82
|
+
rubocop-ast (1.15.1)
|
|
78
83
|
parser (>= 3.0.1.1)
|
|
79
|
-
rubocop-shopify (2.
|
|
80
|
-
rubocop (~> 1.
|
|
84
|
+
rubocop-shopify (2.4.0)
|
|
85
|
+
rubocop (~> 1.24)
|
|
81
86
|
ruby-progressbar (1.11.0)
|
|
82
87
|
ruby2_keywords (0.0.5)
|
|
83
88
|
sidekiq (6.3.1)
|
|
@@ -89,7 +94,7 @@ GEM
|
|
|
89
94
|
rack (~> 2.2)
|
|
90
95
|
rack-protection (= 2.1.0)
|
|
91
96
|
tilt (~> 2.0)
|
|
92
|
-
sorbet-runtime (0.5.
|
|
97
|
+
sorbet-runtime (0.5.9426)
|
|
93
98
|
tilt (2.0.10)
|
|
94
99
|
tzinfo (2.0.4)
|
|
95
100
|
concurrent-ruby (~> 1.0)
|
|
@@ -97,7 +102,7 @@ GEM
|
|
|
97
102
|
vegas (0.1.11)
|
|
98
103
|
rack (>= 1.0.0)
|
|
99
104
|
yard (0.9.26)
|
|
100
|
-
zeitwerk (2.
|
|
105
|
+
zeitwerk (2.5.1)
|
|
101
106
|
|
|
102
107
|
PLATFORMS
|
|
103
108
|
ruby
|
|
@@ -109,7 +114,7 @@ DEPENDENCIES
|
|
|
109
114
|
i18n
|
|
110
115
|
job-iteration!
|
|
111
116
|
mocha
|
|
112
|
-
mysql2
|
|
117
|
+
mysql2!
|
|
113
118
|
pry
|
|
114
119
|
rake
|
|
115
120
|
redis
|
|
@@ -120,4 +125,4 @@ DEPENDENCIES
|
|
|
120
125
|
yard
|
|
121
126
|
|
|
122
127
|
BUNDLED WITH
|
|
123
|
-
2.
|
|
128
|
+
2.3.5
|
data/LICENSE.txt
CHANGED
data/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
[](https://github.com/Shopify/job-iteration/actions/workflows/ci.yml)
|
|
4
4
|
|
|
5
|
-
Meet Iteration, an extension for [ActiveJob](https://github.com/rails/rails/tree/
|
|
5
|
+
Meet Iteration, an extension for [ActiveJob](https://github.com/rails/rails/tree/main/activejob) that makes your jobs interruptible and resumable, saving all progress that the job has made (aka checkpoint for jobs).
|
|
6
6
|
|
|
7
7
|
## Background
|
|
8
8
|
|
|
@@ -223,4 +223,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
|
|
|
223
223
|
|
|
224
224
|
## Code of Conduct
|
|
225
225
|
|
|
226
|
-
Everyone interacting in the Job::Iteration project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/Shopify/job-iteration/blob/
|
|
226
|
+
Everyone interacting in the Job::Iteration project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/Shopify/job-iteration/blob/main/CODE_OF_CONDUCT.md).
|
data/guides/best-practices.md
CHANGED
|
@@ -41,6 +41,14 @@ ActiveSupport::Notifications.subscribe('interrupted.iteration') do |_, _, _, _,
|
|
|
41
41
|
tags: { job_class: tags[:job_class]&.underscore }
|
|
42
42
|
)
|
|
43
43
|
end
|
|
44
|
+
|
|
45
|
+
# If you're using ThrottleEnumerator
|
|
46
|
+
ActiveSupport::Notifications.subscribe('throttled.iteration') do |_, _, _, _, tags|
|
|
47
|
+
StatsD.increment(
|
|
48
|
+
"iteration.throttled",
|
|
49
|
+
tags: { job_class: tags[:job_class]&.underscore }
|
|
50
|
+
)
|
|
51
|
+
end
|
|
44
52
|
```
|
|
45
53
|
|
|
46
54
|
## Max iteration time
|
data/job-iteration.gemspec
CHANGED
|
@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
|
|
|
23
23
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
24
24
|
spec.require_paths = ["lib"]
|
|
25
25
|
|
|
26
|
-
spec.metadata["changelog_uri"] = "https://github.com/Shopify/job-iteration/blob/
|
|
26
|
+
spec.metadata["changelog_uri"] = "https://github.com/Shopify/job-iteration/blob/main/CHANGELOG.md"
|
|
27
27
|
spec.metadata["allowed_push_host"] = "https://rubygems.org"
|
|
28
28
|
|
|
29
29
|
spec.add_development_dependency("activerecord")
|
|
@@ -65,7 +65,8 @@ module JobIteration
|
|
|
65
65
|
@cursor = Array.wrap(cursor)
|
|
66
66
|
|
|
67
67
|
# Yields relations by selecting the primary keys of records in the batch.
|
|
68
|
-
# Post.where(published: nil) results in an enumerator of relations like:
|
|
68
|
+
# Post.where(published: nil) results in an enumerator of relations like:
|
|
69
|
+
# Post.where(published: nil, ids: batch_of_ids)
|
|
69
70
|
@base_relation.where(@primary_key => ids)
|
|
70
71
|
end
|
|
71
72
|
|
|
@@ -119,12 +119,14 @@ module JobIteration
|
|
|
119
119
|
|
|
120
120
|
# Builds Enumerator from Active Record Relation and enumerates on batches, yielding Active Record Relations.
|
|
121
121
|
# See documentation for #build_active_record_enumerator_on_batches.
|
|
122
|
-
def build_active_record_enumerator_on_batch_relations(scope, cursor:, **args)
|
|
123
|
-
JobIteration::ActiveRecordBatchEnumerator.new(
|
|
122
|
+
def build_active_record_enumerator_on_batch_relations(scope, wrap: true, cursor:, **args)
|
|
123
|
+
enum = JobIteration::ActiveRecordBatchEnumerator.new(
|
|
124
124
|
scope,
|
|
125
125
|
cursor: cursor,
|
|
126
126
|
**args
|
|
127
127
|
).each
|
|
128
|
+
enum = wrap(self, enum) if wrap
|
|
129
|
+
enum
|
|
128
130
|
end
|
|
129
131
|
|
|
130
132
|
def build_throttle_enumerator(enum, throttle_on:, backoff:)
|
|
@@ -10,8 +10,14 @@ module JobIteration
|
|
|
10
10
|
super
|
|
11
11
|
end
|
|
12
12
|
end
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
|
|
14
|
+
# @private
|
|
15
|
+
module ::Resque
|
|
16
|
+
class Worker
|
|
17
|
+
# The patch is required in order to call shutdown? on a Resque::Worker instance
|
|
18
|
+
prepend(ResqueIterationExtension)
|
|
19
|
+
end
|
|
20
|
+
end
|
|
15
21
|
|
|
16
22
|
JobIteration.interruption_adapter = -> { $resque_worker.try!(:shutdown?) }
|
|
17
23
|
end
|
|
@@ -283,7 +283,7 @@ module JobIteration
|
|
|
283
283
|
def valid_cursor_parameter?(parameters)
|
|
284
284
|
# this condition is when people use the splat operator.
|
|
285
285
|
# def build_enumerator(*)
|
|
286
|
-
return true if parameters == [[:rest]]
|
|
286
|
+
return true if parameters == [[:rest]] || parameters == [[:rest, :*]]
|
|
287
287
|
|
|
288
288
|
parameters.each do |parameter_type, parameter_name|
|
|
289
289
|
next unless parameter_name == :cursor
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: job-iteration
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.3.
|
|
4
|
+
version: 1.3.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Shopify
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2022-01-18 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activerecord
|
|
@@ -85,7 +85,7 @@ homepage: https://github.com/shopify/job-iteration
|
|
|
85
85
|
licenses:
|
|
86
86
|
- MIT
|
|
87
87
|
metadata:
|
|
88
|
-
changelog_uri: https://github.com/Shopify/job-iteration/blob/
|
|
88
|
+
changelog_uri: https://github.com/Shopify/job-iteration/blob/main/CHANGELOG.md
|
|
89
89
|
allowed_push_host: https://rubygems.org
|
|
90
90
|
post_install_message:
|
|
91
91
|
rdoc_options: []
|