sidekiq-bulk 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.travis.yml +15 -1
- data/Appraisals +8 -0
- data/README.md +16 -6
- data/gemfiles/sidekiq_2_x.gemfile +0 -2
- data/gemfiles/sidekiq_3_x.gemfile +4 -2
- data/gemfiles/sidekiq_4_x.gemfile +9 -0
- data/gemfiles/sidekiq_5_x.gemfile +5 -0
- data/gemfiles/sidekiq_edge.gemfile +7 -3
- data/lib/sidekiq/bulk.rb +3 -2
- data/sidekiq-bulk.gemspec +4 -3
- data/spec/examples.txt +19 -19
- data/spec/sidekiq_bulk_spec.rb +21 -13
- metadata +19 -17
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: da7a63fe002a3933351125609b04a760e5336a3a1c448425d0b2864216ed2718
|
4
|
+
data.tar.gz: c523da8e96d9b7215c2d13bcb69b4996569f13e8cb8228fb3707a6cdf4401780
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3be6a538d179025cd4056db28ab9e14d7b39932bf73fface7f6e1bd38f4eb4062403987fc8f6a082ad2149708779a78af3a62562d8bde665044d756b18d9718d
|
7
|
+
data.tar.gz: caed99e6c353b0c6f9cccae14af8f2f255e68bb3bc85128912e1b987e3f3b4a393b2e59b4e3289522ecd91988237afcdc6d4342eb64e2cbdd900150b3442822a
|
data/.travis.yml
CHANGED
@@ -4,11 +4,16 @@ cache: bundler
|
|
4
4
|
rvm:
|
5
5
|
- 2.0.0
|
6
6
|
- 2.1
|
7
|
-
- 2.2
|
7
|
+
- 2.2.2
|
8
|
+
- 2.3.0
|
9
|
+
- 2.4.0
|
10
|
+
- 2.5.0
|
8
11
|
- ruby-head
|
9
12
|
gemfile:
|
10
13
|
- gemfiles/sidekiq_2_x.gemfile
|
11
14
|
- gemfiles/sidekiq_3_x.gemfile
|
15
|
+
- gemfiles/sidekiq_4_x.gemfile
|
16
|
+
- gemfiles/sidekiq_5_x.gemfile
|
12
17
|
- gemfiles/sidekiq_edge.gemfile
|
13
18
|
script:
|
14
19
|
- bundle exec rspec
|
@@ -16,3 +21,12 @@ matrix:
|
|
16
21
|
allow_failures:
|
17
22
|
- rvm: ruby-head
|
18
23
|
- gemfile: gemfiles/sidekiq_edge.gemfile
|
24
|
+
exclude:
|
25
|
+
- rvm: 2.0.0
|
26
|
+
gemfile: gemfiles/sidekiq_5_x.gemfile
|
27
|
+
- rvm: 2.1
|
28
|
+
gemfile: gemfiles/sidekiq_5_x.gemfile
|
29
|
+
- rvm: 2.0.0
|
30
|
+
gemfile: gemfiles/sidekiq_edge.gemfile
|
31
|
+
- rvm: 2.1
|
32
|
+
gemfile: gemfiles/sidekiq_edge.gemfile
|
data/Appraisals
CHANGED
@@ -6,6 +6,14 @@ appraise "sidekiq-3-x" do
|
|
6
6
|
gem "sidekiq", "~> 3.0"
|
7
7
|
end
|
8
8
|
|
9
|
+
appraise "sidekiq-4-x" do
|
10
|
+
gem "sidekiq", "~> 4.0"
|
11
|
+
end
|
12
|
+
|
13
|
+
appraise "sidekiq-5-x" do
|
14
|
+
gem "sidekiq", "~> 5.0"
|
15
|
+
end
|
16
|
+
|
9
17
|
appraise "sidekiq-edge" do
|
10
18
|
gem "sidekiq", github: "mperham/sidekiq"
|
11
19
|
end
|
data/README.md
CHANGED
@@ -4,23 +4,33 @@
|
|
4
4
|
|
5
5
|
Give your workers more to do!
|
6
6
|
|
7
|
-
Augments Sidekiq job classes with a `push_bulk` method for easier bulk pushing.
|
7
|
+
Augments Sidekiq job classes with a `push_bulk` method for easier and faster bulk pushing.
|
8
8
|
|
9
|
-
|
9
|
+
Let's say you want to enqueue a bunch of jobs. You might do this:
|
10
10
|
|
11
|
-
|
11
|
+
```ruby
|
12
|
+
big_array.each do |e|
|
13
|
+
FooJob.perform_async(e)
|
14
|
+
end
|
15
|
+
```
|
16
|
+
|
17
|
+
If `big_array` has lots of elements, this can be quite slow because of repeated Redis calls per job.
|
18
|
+
|
19
|
+
If, instead, you can push all of `big_array` to Redis in one go, it's more efficient.
|
20
|
+
|
21
|
+
Sidekiq comes with `Sidekiq::Client.push_bulk` which does let you push in bulk. This gem provides a wrapper around that method so that instead of:
|
12
22
|
|
13
23
|
```ruby
|
14
24
|
Sidekiq::Client.push_bulk("class" => FooJob, "args" => [[1], [2], [3]])
|
15
25
|
```
|
16
26
|
|
17
|
-
You can write
|
27
|
+
You can write:
|
18
28
|
|
19
29
|
```ruby
|
20
30
|
FooJob.push_bulk([1, 2, 3])
|
21
31
|
```
|
22
32
|
|
23
|
-
|
33
|
+
By default, jobs are sent in batches of 10,000 as a trade-off. Pushing N jobs is not O(1).
|
24
34
|
|
25
35
|
### Installing
|
26
36
|
|
@@ -69,7 +79,7 @@ end
|
|
69
79
|
|
70
80
|
## Job count splitting
|
71
81
|
|
72
|
-
`push_bulk` will only enqueue at most 10,000 jobs at a time. That is, if `items` has 20,000 elements, `push_bulk(items)` will push the first 10,000, then the second
|
82
|
+
`push_bulk` will only enqueue at most 10,000 jobs at a time. That is, if `items` has 20,000 elements, `push_bulk(items)` will push the first 10,000, then the second 10,000. You can control the threshold with `limit:`.
|
73
83
|
|
74
84
|
```ruby
|
75
85
|
# push in groups of 50,000 jobs at a time
|
@@ -1,7 +1,11 @@
|
|
1
|
-
# This file was generated by Appraisal
|
2
|
-
|
3
1
|
source "https://rubygems.org/"
|
4
2
|
|
3
|
+
# TODO: this is temporary until bundler 2.0
|
4
|
+
git_source(:github) do |repo_name|
|
5
|
+
repo_name = "#{repo_name}/#{repo_name}" unless repo_name.include?("/")
|
6
|
+
"https://github.com/#{repo_name}.git"
|
7
|
+
end
|
8
|
+
|
5
9
|
gemspec :path => "../"
|
6
10
|
|
7
|
-
gem "sidekiq", github
|
11
|
+
gem "sidekiq", :github => "mperham/sidekiq"
|
data/lib/sidekiq/bulk.rb
CHANGED
@@ -1,11 +1,12 @@
|
|
1
1
|
require "sidekiq"
|
2
|
-
require "active_support/core_ext/array/grouping"
|
3
2
|
|
4
3
|
module SidekiqBulk
|
5
4
|
def push_bulk(items, limit: 10_000, &block)
|
6
|
-
items.
|
5
|
+
job_ids = items.each_slice(limit).map do |group|
|
7
6
|
push_bulk!(group, &block)
|
8
7
|
end
|
8
|
+
|
9
|
+
job_ids.flatten
|
9
10
|
end
|
10
11
|
|
11
12
|
def push_bulk!(items, &block)
|
data/sidekiq-bulk.gemspec
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = "sidekiq-bulk"
|
3
|
-
s.version = "0.
|
3
|
+
s.version = "0.2.0"
|
4
4
|
s.authors = ["Adam Prescott"]
|
5
5
|
s.email = ["adam@aprescott.com"]
|
6
6
|
s.homepage = "https://github.com/aprescott/sidekiq-bulk"
|
@@ -13,9 +13,10 @@ Gem::Specification.new do |s|
|
|
13
13
|
s.licenses = ["MIT"]
|
14
14
|
|
15
15
|
s.add_dependency("sidekiq")
|
16
|
-
s.add_dependency("activesupport")
|
17
16
|
s.add_development_dependency("rspec", ">= 3.3")
|
18
|
-
s.add_development_dependency("rspec-sidekiq")
|
17
|
+
s.add_development_dependency("rspec-sidekiq", ">= 3.0.2")
|
19
18
|
s.add_development_dependency("pry-byebug")
|
19
|
+
# byebug is locked while ruby < 2.2.0 is still tested
|
20
|
+
s.add_development_dependency("byebug", "~> 9.0.6")
|
20
21
|
s.add_development_dependency("appraisal")
|
21
22
|
end
|
data/spec/examples.txt
CHANGED
@@ -1,21 +1,21 @@
|
|
1
1
|
example_id | status | run_time |
|
2
2
|
-------------------------------------- | ------ | --------------- |
|
3
|
-
./spec/sidekiq_bulk_spec.rb[1:1:1] | passed | 0.
|
4
|
-
./spec/sidekiq_bulk_spec.rb[1:1:2] | passed | 0.
|
5
|
-
./spec/sidekiq_bulk_spec.rb[1:1:3] | passed | 0.
|
6
|
-
./spec/sidekiq_bulk_spec.rb[1:1:4] | passed | 0.
|
7
|
-
./spec/sidekiq_bulk_spec.rb[1:1:5] | passed | 0.
|
8
|
-
./spec/sidekiq_bulk_spec.rb[1:1:6] | passed | 0.
|
9
|
-
./spec/sidekiq_bulk_spec.rb[1:1:7] | passed | 0.
|
10
|
-
./spec/sidekiq_bulk_spec.rb[1:1:8] | passed | 0.
|
11
|
-
./spec/sidekiq_bulk_spec.rb[1:1:9:1:1] | passed | 0.
|
12
|
-
./spec/sidekiq_bulk_spec.rb[1:1:9:2:1] | passed | 0.
|
13
|
-
./spec/sidekiq_bulk_spec.rb[1:1:9:3:1] | passed | 0.
|
14
|
-
./spec/sidekiq_bulk_spec.rb[1:2:1] | passed | 0.
|
15
|
-
./spec/sidekiq_bulk_spec.rb[1:2:2] | passed | 0.
|
16
|
-
./spec/sidekiq_bulk_spec.rb[1:2:3] | passed | 0.
|
17
|
-
./spec/sidekiq_bulk_spec.rb[1:2:4] | passed | 0.
|
18
|
-
./spec/sidekiq_bulk_spec.rb[1:2:5] | passed | 0.
|
19
|
-
./spec/sidekiq_bulk_spec.rb[1:2:6] | passed | 0.
|
20
|
-
./spec/sidekiq_bulk_spec.rb[1:3:1] | passed | 0.
|
21
|
-
./spec/sidekiq_bulk_spec.rb[1:3:2] | passed | 0.
|
3
|
+
./spec/sidekiq_bulk_spec.rb[1:1:1] | passed | 0.00134 seconds |
|
4
|
+
./spec/sidekiq_bulk_spec.rb[1:1:2] | passed | 0.00232 seconds |
|
5
|
+
./spec/sidekiq_bulk_spec.rb[1:1:3] | passed | 0.00053 seconds |
|
6
|
+
./spec/sidekiq_bulk_spec.rb[1:1:4] | passed | 0.00068 seconds |
|
7
|
+
./spec/sidekiq_bulk_spec.rb[1:1:5] | passed | 0.0024 seconds |
|
8
|
+
./spec/sidekiq_bulk_spec.rb[1:1:6] | passed | 0.00122 seconds |
|
9
|
+
./spec/sidekiq_bulk_spec.rb[1:1:7] | passed | 0.00061 seconds |
|
10
|
+
./spec/sidekiq_bulk_spec.rb[1:1:8] | passed | 0.00748 seconds |
|
11
|
+
./spec/sidekiq_bulk_spec.rb[1:1:9:1:1] | passed | 0.01438 seconds |
|
12
|
+
./spec/sidekiq_bulk_spec.rb[1:1:9:2:1] | passed | 0.0065 seconds |
|
13
|
+
./spec/sidekiq_bulk_spec.rb[1:1:9:3:1] | passed | 0.01214 seconds |
|
14
|
+
./spec/sidekiq_bulk_spec.rb[1:2:1] | passed | 0.00016 seconds |
|
15
|
+
./spec/sidekiq_bulk_spec.rb[1:2:2] | passed | 0.0008 seconds |
|
16
|
+
./spec/sidekiq_bulk_spec.rb[1:2:3] | passed | 0.00036 seconds |
|
17
|
+
./spec/sidekiq_bulk_spec.rb[1:2:4] | passed | 0.00038 seconds |
|
18
|
+
./spec/sidekiq_bulk_spec.rb[1:2:5] | passed | 0.00046 seconds |
|
19
|
+
./spec/sidekiq_bulk_spec.rb[1:2:6] | passed | 0.18121 seconds |
|
20
|
+
./spec/sidekiq_bulk_spec.rb[1:3:1] | passed | 0.00127 seconds |
|
21
|
+
./spec/sidekiq_bulk_spec.rb[1:3:2] | passed | 0.00076 seconds |
|
data/spec/sidekiq_bulk_spec.rb
CHANGED
@@ -19,9 +19,9 @@ RSpec.describe SidekiqBulk do
|
|
19
19
|
FooJob.public_send(method_name, [1, 2, 3]) { |el| [2*el, "some-value"] }
|
20
20
|
|
21
21
|
expect(FooJob.jobs.length).to eq(3)
|
22
|
-
expect(FooJob).to
|
23
|
-
expect(FooJob).to
|
24
|
-
expect(FooJob).to
|
22
|
+
expect(FooJob).to have_enqueued_sidekiq_job(2, "some-value")
|
23
|
+
expect(FooJob).to have_enqueued_sidekiq_job(4, "some-value")
|
24
|
+
expect(FooJob).to have_enqueued_sidekiq_job(6, "some-value")
|
25
25
|
end
|
26
26
|
|
27
27
|
it "goes through the Sidekiq::Client interface" do
|
@@ -40,9 +40,17 @@ RSpec.describe SidekiqBulk do
|
|
40
40
|
FooJob.public_send(method_name, [10, -6.1, "a thing"])
|
41
41
|
|
42
42
|
expect(FooJob.jobs.length).to eq(3)
|
43
|
-
expect(FooJob).to
|
44
|
-
expect(FooJob).to
|
45
|
-
expect(FooJob).to
|
43
|
+
expect(FooJob).to have_enqueued_sidekiq_job(10)
|
44
|
+
expect(FooJob).to have_enqueued_sidekiq_job(-6.1)
|
45
|
+
expect(FooJob).to have_enqueued_sidekiq_job("a thing")
|
46
|
+
end
|
47
|
+
|
48
|
+
it "returns the enqueued job ids" do
|
49
|
+
allow(Sidekiq::Client).to receive(:push_bulk).and_return(["jid-1", "jid-2", "jid-3"])
|
50
|
+
|
51
|
+
job_ids = FooJob.public_send(method_name, [1, 2, 3])
|
52
|
+
|
53
|
+
expect(job_ids).to match_array(["jid-1", "jid-2", "jid-3"])
|
46
54
|
end
|
47
55
|
end
|
48
56
|
|
@@ -53,13 +61,13 @@ RSpec.describe SidekiqBulk do
|
|
53
61
|
FooJob.push_bulk([1, 2, 3, 4, 5, 6, 7], limit: 3)
|
54
62
|
|
55
63
|
expect(FooJob.jobs.length).to eq(7)
|
56
|
-
expect(FooJob).to
|
57
|
-
expect(FooJob).to
|
58
|
-
expect(FooJob).to
|
59
|
-
expect(FooJob).to
|
60
|
-
expect(FooJob).to
|
61
|
-
expect(FooJob).to
|
62
|
-
expect(FooJob).to
|
64
|
+
expect(FooJob).to have_enqueued_sidekiq_job(1)
|
65
|
+
expect(FooJob).to have_enqueued_sidekiq_job(2)
|
66
|
+
expect(FooJob).to have_enqueued_sidekiq_job(3)
|
67
|
+
expect(FooJob).to have_enqueued_sidekiq_job(4)
|
68
|
+
expect(FooJob).to have_enqueued_sidekiq_job(5)
|
69
|
+
expect(FooJob).to have_enqueued_sidekiq_job(6)
|
70
|
+
expect(FooJob).to have_enqueued_sidekiq_job(7)
|
63
71
|
end
|
64
72
|
|
65
73
|
it "limits with the item transformation" do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq-bulk
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Adam Prescott
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-11-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sidekiq
|
@@ -25,35 +25,35 @@ dependencies:
|
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
28
|
+
name: rspec
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
34
|
-
type: :
|
33
|
+
version: '3.3'
|
34
|
+
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
40
|
+
version: '3.3'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name: rspec
|
42
|
+
name: rspec-sidekiq
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - ">="
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
47
|
+
version: 3.0.2
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version:
|
54
|
+
version: 3.0.2
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
|
-
name:
|
56
|
+
name: pry-byebug
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
59
|
- - ">="
|
@@ -67,19 +67,19 @@ dependencies:
|
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
|
-
name:
|
70
|
+
name: byebug
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- - "
|
73
|
+
- - "~>"
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version:
|
75
|
+
version: 9.0.6
|
76
76
|
type: :development
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
|
-
- - "
|
80
|
+
- - "~>"
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version:
|
82
|
+
version: 9.0.6
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
84
|
name: appraisal
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
@@ -110,6 +110,8 @@ files:
|
|
110
110
|
- README.md
|
111
111
|
- gemfiles/sidekiq_2_x.gemfile
|
112
112
|
- gemfiles/sidekiq_3_x.gemfile
|
113
|
+
- gemfiles/sidekiq_4_x.gemfile
|
114
|
+
- gemfiles/sidekiq_5_x.gemfile
|
113
115
|
- gemfiles/sidekiq_edge.gemfile
|
114
116
|
- lib/sidekiq-bulk.rb
|
115
117
|
- lib/sidekiq/bulk.rb
|
@@ -138,12 +140,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
138
140
|
version: '0'
|
139
141
|
requirements: []
|
140
142
|
rubyforge_project:
|
141
|
-
rubygems_version: 2.
|
143
|
+
rubygems_version: 2.7.6
|
142
144
|
signing_key:
|
143
145
|
specification_version: 4
|
144
146
|
summary: Give your workers more to do!
|
145
147
|
test_files:
|
148
|
+
- spec/spec_helper.rb
|
146
149
|
- spec/examples.txt
|
147
150
|
- spec/re_run_friendly_formatter.rb
|
148
151
|
- spec/sidekiq_bulk_spec.rb
|
149
|
-
- spec/spec_helper.rb
|