sidekiq-bulk 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: f1693e23620b5bd3a969bf7585a1912501fb5bf0
-  data.tar.gz: 2e05f32f17c51b721a177f7fed44dcf09a75ffc4
+SHA256:
+  metadata.gz: da7a63fe002a3933351125609b04a760e5336a3a1c448425d0b2864216ed2718
+  data.tar.gz: c523da8e96d9b7215c2d13bcb69b4996569f13e8cb8228fb3707a6cdf4401780
 SHA512:
-  metadata.gz: 06bdf4398f1fd1b7546fe9868a122c0a027c866206b8e40c37967433ab30cef299cb2181bbbf5610611d90d26bebc8371bf85046a2400de88f9080c4a765cbd8
-  data.tar.gz: 75e2d5be6ac87607d7541531f6c97f3c8b4413075079f20176c5aeb057faada439b4411cfa8645b50716979f47fada0646c0da846fc28bd8569401e01db64326
+  metadata.gz: 3be6a538d179025cd4056db28ab9e14d7b39932bf73fface7f6e1bd38f4eb4062403987fc8f6a082ad2149708779a78af3a62562d8bde665044d756b18d9718d
+  data.tar.gz: caed99e6c353b0c6f9cccae14af8f2f255e68bb3bc85128912e1b987e3f3b4a393b2e59b4e3289522ecd91988237afcdc6d4342eb64e2cbdd900150b3442822a

data/.travis.yml

@@ -4,11 +4,16 @@ cache: bundler
 rvm:
   - 2.0.0
   - 2.1
-  - 2.2
+  - 2.2.2
+  - 2.3.0
+  - 2.4.0
+  - 2.5.0
   - ruby-head
 gemfile:
   - gemfiles/sidekiq_2_x.gemfile
   - gemfiles/sidekiq_3_x.gemfile
+  - gemfiles/sidekiq_4_x.gemfile
+  - gemfiles/sidekiq_5_x.gemfile
   - gemfiles/sidekiq_edge.gemfile
 script:
   - bundle exec rspec
@@ -16,3 +21,12 @@ matrix:
   allow_failures:
     - rvm: ruby-head
     - gemfile: gemfiles/sidekiq_edge.gemfile
+  exclude:
+    - rvm: 2.0.0
+      gemfile: gemfiles/sidekiq_5_x.gemfile
+    - rvm: 2.1
+      gemfile: gemfiles/sidekiq_5_x.gemfile
+    - rvm: 2.0.0
+      gemfile: gemfiles/sidekiq_edge.gemfile
+    - rvm: 2.1
+      gemfile: gemfiles/sidekiq_edge.gemfile

data/Appraisals

@@ -6,6 +6,14 @@ appraise "sidekiq-3-x" do
   gem "sidekiq", "~> 3.0"
 end
 
+appraise "sidekiq-4-x" do
+  gem "sidekiq", "~> 4.0"
+end
+
+appraise "sidekiq-5-x" do
+  gem "sidekiq", "~> 5.0"
+end
+
 appraise "sidekiq-edge" do
   gem "sidekiq", github: "mperham/sidekiq"
 end

data/README.md

@@ -4,23 +4,33 @@
 
 Give your workers more to do!
 
-Augments Sidekiq job classes with a `push_bulk` method for easier bulk pushing.
+Augments Sidekiq job classes with a `push_bulk` method for easier and faster bulk pushing.
 
-Sidekiq comes with `Sidekiq::Client.push_bulk` which can be faster than `perform_async` if you have lots and lots of jobs to enqueue.
+Let's say you want to enqueue a bunch of jobs. You might do this:
 
-This gem provides a wrapper around `Sidekiq::Client.push_bulk` so that instead of
+```ruby
+big_array.each do |e|
+  FooJob.perform_async(e)
+end
+```
+
+If `big_array` has lots of elements, this can be quite slow because of repeated Redis calls per job.
+
+If, instead, you can push all of `big_array` to Redis in one go, it's more efficient.
+
+Sidekiq comes with `Sidekiq::Client.push_bulk` which does let you push in bulk. This gem provides a wrapper around that method so that instead of:
 
 ```ruby
 Sidekiq::Client.push_bulk("class" => FooJob, "args" => [[1], [2], [3]])
 ```
 
-You can write
+You can write:
 
 ```ruby
 FooJob.push_bulk([1, 2, 3])
 ```
 
-More stuff is supported!
+By default, jobs are sent in batches of 10,000 as a trade-off. Pushing N jobs is not O(1).
 
 ### Installing
 
@@ -69,7 +79,7 @@ end
 
 ## Job count splitting
 
-`push_bulk` will only enqueue at most 10,000 jobs at a time. That is, if `items` has 20,000 elements, `push_bulk(items)` will push the first 10,000, then the second 20,000. You can control the threshold with `limit:`.
+`push_bulk` will only enqueue at most 10,000 jobs at a time. That is, if `items` has 20,000 elements, `push_bulk(items)` will push the first 10,000, then the second 10,000. You can control the threshold with `limit:`.
 
 ```ruby
 # push in groups of 50,000 jobs at a time

data/gemfiles/sidekiq_2_x.gemfile

@@ -1,5 +1,3 @@
-# This file was generated by Appraisal
-
 source "https://rubygems.org/"
 
 gemspec :path => "../"

data/gemfiles/sidekiq_3_x.gemfile

@@ -1,7 +1,9 @@
-# This file was generated by Appraisal
-
 source "https://rubygems.org/"
 
 gemspec :path => "../"
 
 gem "sidekiq", "~> 3.0"
+
+if RUBY_VERSION < "2.2.2"
+  gem "rack", "~> 1.6.5"
+end

data/gemfiles/sidekiq_4_x.gemfile

@@ -0,0 +1,9 @@
+source "https://rubygems.org/"
+
+gemspec :path => "../"
+
+gem "sidekiq", "~> 4.0"
+
+if RUBY_VERSION < "2.2.2"
+  gem "rack", "~> 1.6.5"
+end

data/gemfiles/sidekiq_5_x.gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org/"
+
+gemspec :path => "../"
+
+gem "sidekiq", "~> 5.0"

data/gemfiles/sidekiq_edge.gemfile

@@ -1,7 +1,11 @@
-# This file was generated by Appraisal
-
 source "https://rubygems.org/"
 
+# TODO: this is temporary until bundler 2.0
+git_source(:github) do |repo_name|
+  repo_name = "#{repo_name}/#{repo_name}" unless repo_name.include?("/")
+  "https://github.com/#{repo_name}.git"
+end
+
 gemspec :path => "../"
 
-gem "sidekiq", github: "mperham/sidekiq"
+gem "sidekiq", :github => "mperham/sidekiq"

data/lib/sidekiq/bulk.rb

@@ -1,11 +1,12 @@
 require "sidekiq"
-require "active_support/core_ext/array/grouping"
 
 module SidekiqBulk
   def push_bulk(items, limit: 10_000, &block)
-    items.in_groups_of(limit, false).each do |group|
+    job_ids = items.each_slice(limit).map do |group|
       push_bulk!(group, &block)
     end
+
+    job_ids.flatten
   end
 
   def push_bulk!(items, &block)

data/sidekiq-bulk.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name         = "sidekiq-bulk"
-  s.version      = "0.1.1"
+  s.version      = "0.2.0"
   s.authors      = ["Adam Prescott"]
   s.email        = ["adam@aprescott.com"]
   s.homepage     = "https://github.com/aprescott/sidekiq-bulk"
@@ -13,9 +13,10 @@ Gem::Specification.new do |s|
   s.licenses = ["MIT"]
 
   s.add_dependency("sidekiq")
-  s.add_dependency("activesupport")
   s.add_development_dependency("rspec", ">= 3.3")
-  s.add_development_dependency("rspec-sidekiq")
+  s.add_development_dependency("rspec-sidekiq", ">= 3.0.2")
   s.add_development_dependency("pry-byebug")
+  # byebug is locked while ruby < 2.2.0 is still tested
+  s.add_development_dependency("byebug", "~> 9.0.6")
   s.add_development_dependency("appraisal")
 end

data/spec/examples.txt

@@ -1,21 +1,21 @@
 example_id                             | status | run_time        |
 -------------------------------------- | ------ | --------------- |
-./spec/sidekiq_bulk_spec.rb[1:1:1]     | passed | 0.00013 seconds |
-./spec/sidekiq_bulk_spec.rb[1:1:2]     | passed | 0.00045 seconds |
-./spec/sidekiq_bulk_spec.rb[1:1:3]     | passed | 0.00036 seconds |
-./spec/sidekiq_bulk_spec.rb[1:1:4]     | passed | 0.00034 seconds |
-./spec/sidekiq_bulk_spec.rb[1:1:5]     | passed | 0.0004 seconds  |
-./spec/sidekiq_bulk_spec.rb[1:1:6]     | passed | 0.00073 seconds |
-./spec/sidekiq_bulk_spec.rb[1:1:7]     | passed | 0.00144 seconds |
-./spec/sidekiq_bulk_spec.rb[1:1:8]     | passed | 0.00052 seconds |
-./spec/sidekiq_bulk_spec.rb[1:1:9:1:1] | passed | 0.00223 seconds |
-./spec/sidekiq_bulk_spec.rb[1:1:9:2:1] | passed | 0.00207 seconds |
-./spec/sidekiq_bulk_spec.rb[1:1:9:3:1] | passed | 0.00761 seconds |
-./spec/sidekiq_bulk_spec.rb[1:2:1]     | passed | 0.0008 seconds  |
-./spec/sidekiq_bulk_spec.rb[1:2:2]     | passed | 0.00064 seconds |
-./spec/sidekiq_bulk_spec.rb[1:2:3]     | passed | 0.0077 seconds  |
-./spec/sidekiq_bulk_spec.rb[1:2:4]     | passed | 0.00043 seconds |
-./spec/sidekiq_bulk_spec.rb[1:2:5]     | passed | 0.0024 seconds  |
-./spec/sidekiq_bulk_spec.rb[1:2:6]     | passed | 0.21426 seconds |
-./spec/sidekiq_bulk_spec.rb[1:3:1]     | passed | 0.00166 seconds |
-./spec/sidekiq_bulk_spec.rb[1:3:2]     | passed | 0.00032 seconds |
+./spec/sidekiq_bulk_spec.rb[1:1:1]     | passed | 0.00134 seconds |
+./spec/sidekiq_bulk_spec.rb[1:1:2]     | passed | 0.00232 seconds |
+./spec/sidekiq_bulk_spec.rb[1:1:3]     | passed | 0.00053 seconds |
+./spec/sidekiq_bulk_spec.rb[1:1:4]     | passed | 0.00068 seconds |
+./spec/sidekiq_bulk_spec.rb[1:1:5]     | passed | 0.0024 seconds  |
+./spec/sidekiq_bulk_spec.rb[1:1:6]     | passed | 0.00122 seconds |
+./spec/sidekiq_bulk_spec.rb[1:1:7]     | passed | 0.00061 seconds |
+./spec/sidekiq_bulk_spec.rb[1:1:8]     | passed | 0.00748 seconds |
+./spec/sidekiq_bulk_spec.rb[1:1:9:1:1] | passed | 0.01438 seconds |
+./spec/sidekiq_bulk_spec.rb[1:1:9:2:1] | passed | 0.0065 seconds  |
+./spec/sidekiq_bulk_spec.rb[1:1:9:3:1] | passed | 0.01214 seconds |
+./spec/sidekiq_bulk_spec.rb[1:2:1]     | passed | 0.00016 seconds |
+./spec/sidekiq_bulk_spec.rb[1:2:2]     | passed | 0.0008 seconds  |
+./spec/sidekiq_bulk_spec.rb[1:2:3]     | passed | 0.00036 seconds |
+./spec/sidekiq_bulk_spec.rb[1:2:4]     | passed | 0.00038 seconds |
+./spec/sidekiq_bulk_spec.rb[1:2:5]     | passed | 0.00046 seconds |
+./spec/sidekiq_bulk_spec.rb[1:2:6]     | passed | 0.18121 seconds |
+./spec/sidekiq_bulk_spec.rb[1:3:1]     | passed | 0.00127 seconds |
+./spec/sidekiq_bulk_spec.rb[1:3:2]     | passed | 0.00076 seconds |

data/spec/sidekiq_bulk_spec.rb

@@ -19,9 +19,9 @@ RSpec.describe SidekiqBulk do
       FooJob.public_send(method_name, [1, 2, 3]) { |el| [2*el, "some-value"] }
 
       expect(FooJob.jobs.length).to eq(3)
-      expect(FooJob).to have_enqueued_job(2, "some-value")
-      expect(FooJob).to have_enqueued_job(4, "some-value")
-      expect(FooJob).to have_enqueued_job(6, "some-value")
+      expect(FooJob).to have_enqueued_sidekiq_job(2, "some-value")
+      expect(FooJob).to have_enqueued_sidekiq_job(4, "some-value")
+      expect(FooJob).to have_enqueued_sidekiq_job(6, "some-value")
     end
 
     it "goes through the Sidekiq::Client interface" do
@@ -40,9 +40,17 @@ RSpec.describe SidekiqBulk do
       FooJob.public_send(method_name, [10, -6.1, "a thing"])
 
       expect(FooJob.jobs.length).to eq(3)
-      expect(FooJob).to have_enqueued_job(10)
-      expect(FooJob).to have_enqueued_job(-6.1)
-      expect(FooJob).to have_enqueued_job("a thing")
+      expect(FooJob).to have_enqueued_sidekiq_job(10)
+      expect(FooJob).to have_enqueued_sidekiq_job(-6.1)
+      expect(FooJob).to have_enqueued_sidekiq_job("a thing")
+    end
+
+    it "returns the enqueued job ids" do
+      allow(Sidekiq::Client).to receive(:push_bulk).and_return(["jid-1", "jid-2", "jid-3"])
+
+      job_ids = FooJob.public_send(method_name, [1, 2, 3])
+
+      expect(job_ids).to match_array(["jid-1", "jid-2", "jid-3"])
     end
   end
 
@@ -53,13 +61,13 @@ RSpec.describe SidekiqBulk do
       FooJob.push_bulk([1, 2, 3, 4, 5, 6, 7], limit: 3)
 
       expect(FooJob.jobs.length).to eq(7)
-      expect(FooJob).to have_enqueued_job(1)
-      expect(FooJob).to have_enqueued_job(2)
-      expect(FooJob).to have_enqueued_job(3)
-      expect(FooJob).to have_enqueued_job(4)
-      expect(FooJob).to have_enqueued_job(5)
-      expect(FooJob).to have_enqueued_job(6)
-      expect(FooJob).to have_enqueued_job(7)
+      expect(FooJob).to have_enqueued_sidekiq_job(1)
+      expect(FooJob).to have_enqueued_sidekiq_job(2)
+      expect(FooJob).to have_enqueued_sidekiq_job(3)
+      expect(FooJob).to have_enqueued_sidekiq_job(4)
+      expect(FooJob).to have_enqueued_sidekiq_job(5)
+      expect(FooJob).to have_enqueued_sidekiq_job(6)
+      expect(FooJob).to have_enqueued_sidekiq_job(7)
     end
 
     it "limits with the item transformation" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-bulk
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Adam Prescott
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-22 00:00:00.000000000 Z
+date: 2018-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -25,35 +25,35 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: activesupport
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
+        version: '3.3'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.3'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: rspec-sidekiq
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '3.3'
+        version: 3.0.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '3.3'
+        version: 3.0.2
 - !ruby/object:Gem::Dependency
-  name: rspec-sidekiq
+  name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -67,19 +67,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pry-byebug
+  name: byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 9.0.6
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 9.0.6
 - !ruby/object:Gem::Dependency
   name: appraisal
   requirement: !ruby/object:Gem::Requirement
@@ -110,6 +110,8 @@ files:
 - README.md
 - gemfiles/sidekiq_2_x.gemfile
 - gemfiles/sidekiq_3_x.gemfile
+- gemfiles/sidekiq_4_x.gemfile
+- gemfiles/sidekiq_5_x.gemfile
 - gemfiles/sidekiq_edge.gemfile
 - lib/sidekiq-bulk.rb
 - lib/sidekiq/bulk.rb
@@ -138,12 +140,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Give your workers more to do!
 test_files:
+- spec/spec_helper.rb
 - spec/examples.txt
 - spec/re_run_friendly_formatter.rb
 - spec/sidekiq_bulk_spec.rb
-- spec/spec_helper.rb