job-iteration 1.1.6 → 1.1.11
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +47 -0
- data/.rubocop.yml +2 -2
- data/CHANGELOG.md +23 -0
- data/Gemfile +13 -13
- data/README.md +12 -6
- data/dev.yml +1 -1
- data/gemfiles/rails_5_2.gemfile +3 -3
- data/gemfiles/rails_6_0.gemfile +6 -0
- data/gemfiles/rails_edge.gemfile +3 -3
- data/guides/custom-enumerator.md +2 -0
- data/job-iteration.gemspec +1 -1
- data/lib/job-iteration.rb +2 -2
- data/lib/job-iteration/active_record_cursor.rb +6 -4
- data/lib/job-iteration/active_record_enumerator.rb +1 -1
- data/lib/job-iteration/integrations/resque.rb +1 -1
- data/lib/job-iteration/integrations/sidekiq.rb +1 -1
- data/lib/job-iteration/iteration.rb +62 -8
- data/lib/job-iteration/version.rb +1 -1
- metadata +4 -3
- data/.travis.yml +0 -17
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 18324439fe7b98c7f1bca543c078ddbfdff18af5a7e97a87b21b48890e919769
|
4
|
+
data.tar.gz: 3d912ea06a5a66ee841fbd605e8dfc414bd1760c61d9edf74445e79ae16a6466
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 622368d1208ea23014188c028f832f51f899b6bdfbca8938c6396dcca48f913679ee7f41a035c5cbe0376346ea40a583c4615bd665251cf7dc36a30aeb37904a
|
7
|
+
data.tar.gz: 51931d565cffb4600141e1e96dc9a1a6b8a522589c8b0b38dc4cd1846e8d59a9f150d0d0bfcde4aaee587478af57418093a9e313b2f17c9f2c54ffb6cf18382a
|
@@ -0,0 +1,47 @@
|
|
1
|
+
name: CI
|
2
|
+
|
3
|
+
on: [push]
|
4
|
+
|
5
|
+
jobs:
|
6
|
+
build:
|
7
|
+
runs-on: ubuntu-latest
|
8
|
+
name: Ruby ${{ matrix.ruby }} | Gemfile ${{ matrix.gemfile }}
|
9
|
+
services:
|
10
|
+
redis:
|
11
|
+
image: redis
|
12
|
+
ports:
|
13
|
+
- 6379:6379
|
14
|
+
strategy:
|
15
|
+
matrix:
|
16
|
+
ruby: [2.5, 2.6, 2.7, 3.0]
|
17
|
+
gemfile: [rails_5_2, rails_6_0, rails_edge]
|
18
|
+
exclude:
|
19
|
+
- ruby: 2.5
|
20
|
+
gemfile: rails_edge
|
21
|
+
- ruby: 2.6
|
22
|
+
gemfile: rails_edge
|
23
|
+
- ruby: 3.0
|
24
|
+
gemfile: rails_5_2
|
25
|
+
env:
|
26
|
+
BUNDLE_GEMFILE: gemfiles/${{ matrix.gemfile }}.gemfile
|
27
|
+
steps:
|
28
|
+
- name: Check out code
|
29
|
+
uses: actions/checkout@v2
|
30
|
+
- name: Set up Ruby ${{ matrix.ruby }}
|
31
|
+
uses: ruby/setup-ruby@v1
|
32
|
+
with:
|
33
|
+
ruby-version: ${{ matrix.ruby }}
|
34
|
+
bundler-cache: true
|
35
|
+
- name: Start MySQL and create DB
|
36
|
+
run: |
|
37
|
+
sudo systemctl start mysql.service
|
38
|
+
mysql -uroot -h localhost -proot -e "CREATE DATABASE job_iteration_test;"
|
39
|
+
- name: Rubocop
|
40
|
+
run: bundle exec rubocop
|
41
|
+
- name: Ruby tests
|
42
|
+
run: bundle exec rake test
|
43
|
+
env:
|
44
|
+
REDIS_HOST: localhost
|
45
|
+
REDIS_PORT: ${{ job.services.redis.ports[6379] }}
|
46
|
+
- name: Documentation correctly written
|
47
|
+
run: bundle exec yardoc --no-output --no-save --no-stats --fail-on-warning
|
data/.rubocop.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -4,6 +4,29 @@
|
|
4
4
|
|
5
5
|
#### Bug fix
|
6
6
|
|
7
|
+
|
8
|
+
## v1.1.11 (April 19, 2021)
|
9
|
+
|
10
|
+
#### Bug fix
|
11
|
+
|
12
|
+
- [73](https://github.com/Shopify/job-iteration/pull/73) - Enforce cursor be serializable
|
13
|
+
|
14
|
+
## v1.1.10 (March 30, 2021)
|
15
|
+
|
16
|
+
- [69](https://github.com/Shopify/job-iteration/pull/69) - Fix memory leak in ActiveRecordCursor
|
17
|
+
|
18
|
+
## v1.1.9 (January 6, 2021)
|
19
|
+
|
20
|
+
- [61](https://github.com/Shopify/job-iteration/pull/61) - Call `super` in `method_added`
|
21
|
+
|
22
|
+
## v1.1.8 (June 8, 2020)
|
23
|
+
|
24
|
+
- Preserve ruby2_keywords tags in arguments on Ruby 2.7
|
25
|
+
|
26
|
+
## v1.1.7 (June 4, 2020)
|
27
|
+
|
28
|
+
- [54](https://github.com/Shopify/job-iteration/pull/54) - Fix warnings on Ruby 2.7
|
29
|
+
|
7
30
|
## v1.1.6 (May 22, 2020)
|
8
31
|
|
9
32
|
- [49](https://github.com/Shopify/job-iteration/pull/49) - Log when enumerator has nothing to iterate
|
data/Gemfile
CHANGED
@@ -8,21 +8,21 @@ git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
|
|
8
8
|
gemspec
|
9
9
|
|
10
10
|
# for integration testing
|
11
|
-
gem
|
12
|
-
gem
|
11
|
+
gem "sidekiq"
|
12
|
+
gem "resque"
|
13
13
|
|
14
|
-
gem
|
15
|
-
gem
|
16
|
-
gem
|
17
|
-
gem
|
18
|
-
gem
|
14
|
+
gem "mysql2", "~> 0.5"
|
15
|
+
gem "globalid"
|
16
|
+
gem "i18n"
|
17
|
+
gem "redis"
|
18
|
+
gem "database_cleaner"
|
19
19
|
|
20
|
-
gem
|
21
|
-
gem
|
20
|
+
gem "pry"
|
21
|
+
gem "mocha"
|
22
22
|
|
23
|
-
gem
|
24
|
-
gem
|
25
|
-
gem
|
23
|
+
gem "rubocop-shopify", require: false
|
24
|
+
gem "yard"
|
25
|
+
gem "rake"
|
26
26
|
|
27
27
|
# for unit testing optional sorbet support
|
28
|
-
gem
|
28
|
+
gem "sorbet-runtime"
|
data/README.md
CHANGED
@@ -9,7 +9,7 @@ Meet Iteration, an extension for [ActiveJob](https://github.com/rails/rails/tree
|
|
9
9
|
Imagine the following job:
|
10
10
|
|
11
11
|
```ruby
|
12
|
-
class SimpleJob <
|
12
|
+
class SimpleJob < ApplicationJob
|
13
13
|
def perform
|
14
14
|
User.find_each do |user|
|
15
15
|
user.notify_about_something
|
@@ -43,7 +43,7 @@ And then execute:
|
|
43
43
|
In the job, include `JobIteration::Iteration` module and start describing the job with two methods (`build_enumerator` and `each_iteration`) instead of `perform`:
|
44
44
|
|
45
45
|
```ruby
|
46
|
-
class NotifyUsersJob <
|
46
|
+
class NotifyUsersJob < ApplicationJob
|
47
47
|
include JobIteration::Iteration
|
48
48
|
|
49
49
|
def build_enumerator(cursor:)
|
@@ -64,7 +64,9 @@ end
|
|
64
64
|
Check out more examples of Iterations:
|
65
65
|
|
66
66
|
```ruby
|
67
|
-
class BatchesJob <
|
67
|
+
class BatchesJob < ApplicationJob
|
68
|
+
include JobIteration::Iteration
|
69
|
+
|
68
70
|
def build_enumerator(product_id, cursor:)
|
69
71
|
enumerator_builder.active_record_on_batches(
|
70
72
|
Product.find(product_id).comments,
|
@@ -81,7 +83,9 @@ end
|
|
81
83
|
```
|
82
84
|
|
83
85
|
```ruby
|
84
|
-
class ArrayJob <
|
86
|
+
class ArrayJob < ApplicationJob
|
87
|
+
include JobIteration::Iteration
|
88
|
+
|
85
89
|
def build_enumerator(cursor:)
|
86
90
|
enumerator_builder.array(['build', 'enumerator', 'from', 'any', 'array'], cursor: cursor)
|
87
91
|
end
|
@@ -93,7 +97,9 @@ end
|
|
93
97
|
```
|
94
98
|
|
95
99
|
```ruby
|
96
|
-
class CsvJob <
|
100
|
+
class CsvJob < ApplicationJob
|
101
|
+
include JobIteration::Iteration
|
102
|
+
|
97
103
|
def build_enumerator(import_id, cursor:)
|
98
104
|
import = Import.find(import_id)
|
99
105
|
JobIteration::CsvEnumerator.new(import.csv).rows(cursor: cursor)
|
@@ -153,7 +159,7 @@ There a few configuration assumptions that are required for Iteration to work wi
|
|
153
159
|
**My job has a complex flow. How do I write my own Enumerator?** Iteration API takes care of persisting the cursor (that you may use to calculate an offset) and controlling the job state. The power of Enumerator object is that you can use the cursor in any way you want. One example is a cursorless job that pops records from a datastore until the job is interrupted:
|
154
160
|
|
155
161
|
```ruby
|
156
|
-
class MyJob <
|
162
|
+
class MyJob < ApplicationJob
|
157
163
|
include JobIteration::Iteration
|
158
164
|
|
159
165
|
def build_enumerator(cursor:)
|
data/dev.yml
CHANGED
data/gemfiles/rails_5_2.gemfile
CHANGED
data/gemfiles/rails_edge.gemfile
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
eval_gemfile
|
3
|
+
eval_gemfile "../Gemfile"
|
4
4
|
|
5
|
-
gem
|
6
|
-
gem
|
5
|
+
gem "activejob", github: "rails/rails", branch: "main"
|
6
|
+
gem "activerecord", github: "rails/rails", branch: "main"
|
data/guides/custom-enumerator.md
CHANGED
@@ -72,3 +72,5 @@ end
|
|
72
72
|
```
|
73
73
|
|
74
74
|
We recommend that you read the implementation of the other enumerators that come with the library (`CsvEnumerator`, `ActiveRecordEnumerator`) to gain a better understanding of building Enumerator objects.
|
75
|
+
|
76
|
+
Code that is written after the `yield` in a custom enumerator is not guaranteed to execute. In the case that a job is forced to exit ie `job_should_exit?` is true, then the job is re-enqueued during the yield and the rest of the code in the enumerator does not run. You can follow that logic [here](https://github.com/Shopify/job-iteration/blob/9641f455b9126efff2214692c0bef423e0d12c39/lib/job-iteration/iteration.rb#L128-L131).
|
data/job-iteration.gemspec
CHANGED
@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
|
|
10
10
|
spec.authors = %w(Shopify)
|
11
11
|
spec.email = ["ops-accounts+shipit@shopify.com"]
|
12
12
|
|
13
|
-
spec.summary =
|
13
|
+
spec.summary = "Makes your background jobs interruptible and resumable."
|
14
14
|
spec.description = spec.summary
|
15
15
|
spec.homepage = "https://github.com/shopify/job-iteration"
|
16
16
|
spec.license = "MIT"
|
data/lib/job-iteration.rb
CHANGED
@@ -54,11 +54,11 @@ module JobIteration
|
|
54
54
|
def load_integration(integration)
|
55
55
|
unless INTEGRATIONS.include?(integration)
|
56
56
|
raise IntegrationLoadError,
|
57
|
-
"#{integration} integration is not supported. Available integrations: #{INTEGRATIONS.join(
|
57
|
+
"#{integration} integration is not supported. Available integrations: #{INTEGRATIONS.join(", ")}"
|
58
58
|
end
|
59
59
|
|
60
60
|
require_relative "./job-iteration/integrations/#{integration}"
|
61
61
|
end
|
62
62
|
end
|
63
63
|
|
64
|
-
JobIteration.load_integrations unless ENV[
|
64
|
+
JobIteration.load_integrations unless ENV["ITERATION_DISABLE_AUTOCONFIGURE"]
|
@@ -23,7 +23,7 @@ module JobIteration
|
|
23
23
|
@columns = Array.wrap(columns)
|
24
24
|
self.position = Array.wrap(position)
|
25
25
|
raise ArgumentError, "Must specify at least one column" if columns.empty?
|
26
|
-
if relation.joins_values.present? && !@columns.all? { |column| column.to_s.include?(
|
26
|
+
if relation.joins_values.present? && !@columns.all? { |column| column.to_s.include?(".") }
|
27
27
|
raise ArgumentError, "You need to specify fully-qualified columns if you join a table"
|
28
28
|
end
|
29
29
|
|
@@ -31,7 +31,7 @@ module JobIteration
|
|
31
31
|
raise ConditionNotSupportedError
|
32
32
|
end
|
33
33
|
|
34
|
-
@base_relation = relation.reorder(@columns.join(
|
34
|
+
@base_relation = relation.reorder(@columns.join(","))
|
35
35
|
@reached_end = false
|
36
36
|
end
|
37
37
|
|
@@ -50,7 +50,7 @@ module JobIteration
|
|
50
50
|
|
51
51
|
def update_from_record(record)
|
52
52
|
self.position = @columns.map do |column|
|
53
|
-
method = column.to_s.split(
|
53
|
+
method = column.to_s.split(".").last
|
54
54
|
record.send(method.to_sym)
|
55
55
|
end
|
56
56
|
end
|
@@ -64,7 +64,9 @@ module JobIteration
|
|
64
64
|
relation = relation.where(*conditions)
|
65
65
|
end
|
66
66
|
|
67
|
-
records = relation.
|
67
|
+
records = relation.uncached do
|
68
|
+
relation.to_a
|
69
|
+
end
|
68
70
|
|
69
71
|
update_from_record(records.last) unless records.empty?
|
70
72
|
@reached_end = records.size < batch_size
|
@@ -40,7 +40,7 @@ module JobIteration
|
|
40
40
|
|
41
41
|
def cursor_value(record)
|
42
42
|
positions = @columns.map do |column|
|
43
|
-
attribute_name = column.to_s.split(
|
43
|
+
attribute_name = column.to_s.split(".").last
|
44
44
|
column_value(record, attribute_name)
|
45
45
|
end
|
46
46
|
return positions.first if positions.size == 1
|
@@ -1,11 +1,33 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "active_support/all"
|
4
4
|
|
5
5
|
module JobIteration
|
6
6
|
module Iteration
|
7
7
|
extend ActiveSupport::Concern
|
8
8
|
|
9
|
+
class CursorError < ArgumentError
|
10
|
+
attr_reader :cursor
|
11
|
+
|
12
|
+
def initialize(message, cursor:)
|
13
|
+
super(message)
|
14
|
+
@cursor = cursor
|
15
|
+
end
|
16
|
+
|
17
|
+
def message
|
18
|
+
"#{super} (#{inspected_cursor})"
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
def inspected_cursor
|
24
|
+
cursor.inspect
|
25
|
+
rescue NoMethodError
|
26
|
+
# For those brave enough to try to use BasicObject as cursor. Nice try.
|
27
|
+
Object.instance_method(:inspect).bind(cursor).call
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
9
31
|
included do |_base|
|
10
32
|
attr_accessor(
|
11
33
|
:cursor_position,
|
@@ -22,6 +44,7 @@ module JobIteration
|
|
22
44
|
module ClassMethods
|
23
45
|
def method_added(method_name)
|
24
46
|
ban_perform_definition if method_name.to_sym == :perform
|
47
|
+
super
|
25
48
|
end
|
26
49
|
|
27
50
|
def on_start(*filters, &blk)
|
@@ -49,27 +72,28 @@ module JobIteration
|
|
49
72
|
self.total_time = 0.0
|
50
73
|
assert_implements_methods!
|
51
74
|
end
|
75
|
+
ruby2_keywords(:initialize) if respond_to?(:ruby2_keywords, true)
|
52
76
|
|
53
77
|
def serialize # @private
|
54
78
|
super.merge(
|
55
|
-
|
56
|
-
|
57
|
-
|
79
|
+
"cursor_position" => cursor_position,
|
80
|
+
"times_interrupted" => times_interrupted,
|
81
|
+
"total_time" => total_time,
|
58
82
|
)
|
59
83
|
end
|
60
84
|
|
61
85
|
def deserialize(job_data) # @private
|
62
86
|
super
|
63
|
-
self.cursor_position = job_data[
|
64
|
-
self.times_interrupted = job_data[
|
65
|
-
self.total_time = job_data[
|
87
|
+
self.cursor_position = job_data["cursor_position"]
|
88
|
+
self.times_interrupted = job_data["times_interrupted"] || 0
|
89
|
+
self.total_time = job_data["total_time"] || 0
|
66
90
|
end
|
67
91
|
|
68
92
|
def perform(*params) # @private
|
69
93
|
interruptible_perform(*params)
|
70
94
|
end
|
71
95
|
|
72
|
-
def retry_job(
|
96
|
+
def retry_job(*, **)
|
73
97
|
super unless defined?(@retried) && @retried
|
74
98
|
@retried = true
|
75
99
|
end
|
@@ -118,6 +142,8 @@ module JobIteration
|
|
118
142
|
arguments = arguments.dup.freeze
|
119
143
|
found_record = false
|
120
144
|
enumerator.each do |object_from_enumerator, index|
|
145
|
+
assert_valid_cursor!(index)
|
146
|
+
|
121
147
|
record_unit_of_work do
|
122
148
|
found_record = true
|
123
149
|
each_iteration(object_from_enumerator, *arguments)
|
@@ -174,6 +200,18 @@ module JobIteration
|
|
174
200
|
EOS
|
175
201
|
end
|
176
202
|
|
203
|
+
# The adapter must be able to serialize and deserialize the cursor back into an equivalent object.
|
204
|
+
# https://github.com/mperham/sidekiq/wiki/Best-Practices#1-make-your-job-parameters-small-and-simple
|
205
|
+
def assert_valid_cursor!(cursor)
|
206
|
+
return if serializable?(cursor)
|
207
|
+
|
208
|
+
raise CursorError.new(
|
209
|
+
"Cursor must be composed of objects capable of built-in (de)serialization: " \
|
210
|
+
"Strings, Integers, Floats, Arrays, Hashes, true, false, or nil.",
|
211
|
+
cursor: cursor,
|
212
|
+
)
|
213
|
+
end
|
214
|
+
|
177
215
|
def assert_implements_methods!
|
178
216
|
unless respond_to?(:each_iteration, true)
|
179
217
|
raise(
|
@@ -249,5 +287,21 @@ module JobIteration
|
|
249
287
|
end
|
250
288
|
false
|
251
289
|
end
|
290
|
+
|
291
|
+
SIMPLE_SERIALIZABLE_CLASSES = [String, Integer, Float, NilClass, TrueClass, FalseClass].freeze
|
292
|
+
private_constant :SIMPLE_SERIALIZABLE_CLASSES
|
293
|
+
def serializable?(object)
|
294
|
+
# Subclasses must be excluded, hence not using is_a? or ===.
|
295
|
+
if object.instance_of?(Array)
|
296
|
+
object.all? { |element| serializable?(element) }
|
297
|
+
elsif object.instance_of?(Hash)
|
298
|
+
object.all? { |key, value| serializable?(key) && serializable?(value) }
|
299
|
+
else
|
300
|
+
SIMPLE_SERIALIZABLE_CLASSES.any? { |klass| object.instance_of?(klass) }
|
301
|
+
end
|
302
|
+
rescue NoMethodError
|
303
|
+
# BasicObject doesn't respond to instance_of, but we can't serialize it anyway
|
304
|
+
false
|
305
|
+
end
|
252
306
|
end
|
253
307
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: job-iteration
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.11
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shopify
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-04-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -45,9 +45,9 @@ executables: []
|
|
45
45
|
extensions: []
|
46
46
|
extra_rdoc_files: []
|
47
47
|
files:
|
48
|
+
- ".github/workflows/ci.yml"
|
48
49
|
- ".gitignore"
|
49
50
|
- ".rubocop.yml"
|
50
|
-
- ".travis.yml"
|
51
51
|
- ".yardopts"
|
52
52
|
- CHANGELOG.md
|
53
53
|
- CODE_OF_CONDUCT.md
|
@@ -58,6 +58,7 @@ files:
|
|
58
58
|
- bin/setup
|
59
59
|
- dev.yml
|
60
60
|
- gemfiles/rails_5_2.gemfile
|
61
|
+
- gemfiles/rails_6_0.gemfile
|
61
62
|
- gemfiles/rails_edge.gemfile
|
62
63
|
- guides/best-practices.md
|
63
64
|
- guides/custom-enumerator.md
|
data/.travis.yml
DELETED
@@ -1,17 +0,0 @@
|
|
1
|
-
services:
|
2
|
-
- mysql
|
3
|
-
- redis-server
|
4
|
-
language: ruby
|
5
|
-
rvm:
|
6
|
-
- 2.5.5
|
7
|
-
- 2.6.2
|
8
|
-
before_install:
|
9
|
-
- mysql -e 'CREATE DATABASE job_iteration_test;'
|
10
|
-
script:
|
11
|
-
- bundle exec rake test
|
12
|
-
- bundle exec rubocop
|
13
|
-
- bundle exec yardoc --no-output --no-save --no-stats --fail-on-warning
|
14
|
-
|
15
|
-
gemfile:
|
16
|
-
- 'gemfiles/rails_5_2.gemfile'
|
17
|
-
- 'gemfiles/rails_edge.gemfile'
|