sidekiq-bulk 0.0.2 → 0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1ab5533e949a3c0fd54cc150325b0b6fa0c58e18
-  data.tar.gz: 3734f891f92c8e9668779c8e8ecdb29f118e8456
+  metadata.gz: 6e466d4ca76140df65784cbe9c529e50c506e806
+  data.tar.gz: b56858162f9045a52b429ab18a42a052be4f5267
 SHA512:
-  metadata.gz: 0a84969c10529edd072482cac04682223b0738f3f81a757acf7efa4f42da9b70e814b865de30db8516836891031b739cbb4c3b65abac0c8b1ddb18b2009fe016
-  data.tar.gz: 048773246bf5ef94b82cbdac5e6e69df7596acf3dc00d219407cc199f1a166b527ea0a80aad0ecf9523bb10028aae2f229b825cdcc2aec3afd312cc555e2bcad
+  metadata.gz: 9789acdf3dd12f7113bd2edeeafe57d63d9f2920b6a5072728d0e8bad8c6439adf2cacba9fe1457acab521ac66cd88b165169f858989d8e0d44fc580ea13c4ae
+  data.tar.gz: 30fd6e17e8893ab0fcf2ec6367114ad8a18ab00db4643fa117c8a3b3480be7a6a7d12f729da245a5dea67fe285162952218765d7a8babd7f84db8828ab6ea320

data/README.md

@@ -1,6 +1,6 @@
 # sidekiq-bulk
 
-[![Build Status](https://travis-ci.org/aprescott/sidekiq-bulk.svg?branch=master)](https://travis-ci.org/aprescott/sidekiq-bulk)
+[![Build Status](https://travis-ci.org/aprescott/sidekiq-bulk.svg?branch=master)](https://travis-ci.org/aprescott/sidekiq-bulk) [![Code Climate](https://codeclimate.com/github/aprescott/sidekiq-bulk/badges/gpa.svg)](https://codeclimate.com/github/aprescott/sidekiq-bulk)
 
 Give your workers more to do!
 
@@ -11,7 +11,7 @@ Sidekiq comes with `Sidekiq::Client.push_bulk` which can be faster than `perform
 This gem provides a wrapper around `Sidekiq::Client.push_bulk` so that instead of
 
 ```ruby
-Sidekiq::Client.push_bulk("class" => FooJob, "args" => [[1], [2], [3])
+Sidekiq::Client.push_bulk("class" => FooJob, "args" => [[1], [2], [3]])
 ```
 
 You can write
@@ -67,6 +67,39 @@ all_users.each do |user|
 end
 ```
 
+## Job count splitting
+
+`push_bulk` will only enqueue at most 10,000 jobs at a time. That is, if `items` has 20,000 elements, `push_bulk(items)` will push the first 10,000, then the second 20,000. You can control the threshold with `limit:`.
+
+```ruby
+# push in groups of 50,000 jobs at a time
+FooJob.push_bulk(items, limit: 50_000)
+
+# equivalent to FooJob.push_bulk(items, limit: 10_000)
+FooJob.push_bulk(items)
+```
+
+This also works with a block.
+
+```ruby
+# this results in 5 pushes
+
+users.length # => 100_000
+FooJob.push_bulk(users, limit: 20_000) do |user|
+  [user.id, "some-value"]
+end
+```
+
+And to disable push splitting, use `push_bulk!`.
+
+```ruby
+# one single push of 500,000 jobs, no splitting
+
+FooJob.push_bulk!(users) do |user|
+  [user.id, "some-value"]
+end
+```
+
 ### License
 
 Copyright (c) 2015 Adam Prescott, licensed under the MIT license. See LICENSE.

data/lib/sidekiq/bulk.rb

@@ -1,5 +1,13 @@
+require "active_support/core_ext/array/grouping"
+
 module SidekiqBulk
-  def push_bulk(items, &block)
+  def push_bulk(items, limit: 10_000, &block)
+    items.in_groups_of(limit, false).each do |group|
+      push_bulk!(group, &block)
+    end
+  end
+
+  def push_bulk!(items, &block)
     if block
       args = items.map(&block)
     else

data/sidekiq-bulk.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name         = "sidekiq-bulk"
-  s.version      = "0.0.2"
+  s.version      = "0.1"
   s.authors      = ["Adam Prescott"]
   s.email        = ["adam@aprescott.com"]
   s.homepage     = "https://github.com/aprescott/sidekiq-bulk"
@@ -13,6 +13,7 @@ Gem::Specification.new do |s|
   s.licenses = ["MIT"]
 
   s.add_dependency("sidekiq")
+  s.add_dependency("activesupport")
   s.add_development_dependency("rspec", ">= 3.3")
   s.add_development_dependency("rspec-sidekiq")
   s.add_development_dependency("pry-byebug")

data/spec/examples.txt

@@ -1,8 +1,21 @@
-example_id                         | status | run_time        |
----------------------------------- | ------ | --------------- |
-./spec/sidekiq_bulk_spec.rb[1:1]   | passed | 0.00181 seconds |
-./spec/sidekiq_bulk_spec.rb[1:2]   | passed | 0.00131 seconds |
-./spec/sidekiq_bulk_spec.rb[1:3]   | passed | 0.00118 seconds |
-./spec/sidekiq_bulk_spec.rb[1:4]   | passed | 0.00685 seconds |
-./spec/sidekiq_bulk_spec.rb[1:5]   | passed | 0.00366 seconds |
-./spec/sidekiq_bulk_spec.rb[1:6:1] | passed | 0.00135 seconds |
+example_id                             | status | run_time        |
+-------------------------------------- | ------ | --------------- |
+./spec/sidekiq_bulk_spec.rb[1:1:1]     | passed | 0.00012 seconds |
+./spec/sidekiq_bulk_spec.rb[1:1:2]     | passed | 0.00047 seconds |
+./spec/sidekiq_bulk_spec.rb[1:1:3]     | passed | 0.00036 seconds |
+./spec/sidekiq_bulk_spec.rb[1:1:4]     | passed | 0.00037 seconds |
+./spec/sidekiq_bulk_spec.rb[1:1:5]     | passed | 0.00035 seconds |
+./spec/sidekiq_bulk_spec.rb[1:1:6]     | passed | 0.00074 seconds |
+./spec/sidekiq_bulk_spec.rb[1:1:7]     | passed | 0.00044 seconds |
+./spec/sidekiq_bulk_spec.rb[1:1:8]     | passed | 0.00139 seconds |
+./spec/sidekiq_bulk_spec.rb[1:1:9:1:1] | passed | 0.00193 seconds |
+./spec/sidekiq_bulk_spec.rb[1:1:9:2:1] | passed | 0.00221 seconds |
+./spec/sidekiq_bulk_spec.rb[1:1:9:3:1] | passed | 0.01202 seconds |
+./spec/sidekiq_bulk_spec.rb[1:2:1]     | passed | 0.00074 seconds |
+./spec/sidekiq_bulk_spec.rb[1:2:2]     | passed | 0.00123 seconds |
+./spec/sidekiq_bulk_spec.rb[1:2:3]     | passed | 0.00047 seconds |
+./spec/sidekiq_bulk_spec.rb[1:2:4]     | passed | 0.00692 seconds |
+./spec/sidekiq_bulk_spec.rb[1:2:5]     | passed | 0.00237 seconds |
+./spec/sidekiq_bulk_spec.rb[1:2:6]     | passed | 0.23908 seconds |
+./spec/sidekiq_bulk_spec.rb[1:3:1]     | passed | 0.00145 seconds |
+./spec/sidekiq_bulk_spec.rb[1:3:2]     | passed | 0.00037 seconds |

data/spec/sidekiq_bulk_spec.rb

@@ -10,41 +10,122 @@ RSpec.describe SidekiqBulk do
   class BarJob < FooJob
   end
 
-  it "provides a push_bulk method on job classes" do
-    expect(FooJob).to respond_to(:push_bulk)
-  end
+  shared_examples "a bulk push method" do |method_name|
+    it "provides a push_bulk method on job classes" do
+      expect(FooJob).to respond_to(method_name)
+    end
 
-  it "enqueues the job" do
-    FooJob.push_bulk([1, 2, 3]) { |el| [el, "some-value"] }
+    it "enqueues the job" do
+      FooJob.public_send(method_name, [1, 2, 3]) { |el| [2*el, "some-value"] }
 
-    expect(FooJob.jobs.length).to eq(3)
-    expect(FooJob).to have_enqueued_job(1, "some-value")
-    expect(FooJob).to have_enqueued_job(2, "some-value")
-    expect(FooJob).to have_enqueued_job(3, "some-value")
-  end
+      expect(FooJob.jobs.length).to eq(3)
+      expect(FooJob).to have_enqueued_job(2, "some-value")
+      expect(FooJob).to have_enqueued_job(4, "some-value")
+      expect(FooJob).to have_enqueued_job(6, "some-value")
+    end
+
+    it "goes through the Sidekiq::Client interface" do
+      expect(Sidekiq::Client).to receive(:push_bulk).once.with("class" => FooJob, "args" => [[1], [2], [3]])
 
-  it "goes through the Sidekiq::Client interface" do
-    expect(Sidekiq::Client).to receive(:push_bulk).with("class" => FooJob, "args" => [[1], [2], [3]])
+      FooJob.public_send(method_name, [1, 2, 3])
+    end
+
+    it "uses the correct class name for subclasses" do
+      expect(Sidekiq::Client).to receive(:push_bulk).once.with("class" => BarJob, "args" => [[1], [2], [3]])
+
+      BarJob.push_bulk([1, 2, 3])
+    end
 
-    FooJob.push_bulk([1, 2, 3])
+    it "defaults to the identity function with no block given" do
+      FooJob.public_send(method_name, [10, -6.1, "a thing"])
+
+      expect(FooJob.jobs.length).to eq(3)
+      expect(FooJob).to have_enqueued_job(10)
+      expect(FooJob).to have_enqueued_job(-6.1)
+      expect(FooJob).to have_enqueued_job("a thing")
+    end
   end
 
-  it "uses the correct class name for subclasses" do
-    expect(Sidekiq::Client).to receive(:push_bulk).with("class" => BarJob, "args" => [[1], [2], [3]])
+  describe "#push_bulk" do
+    include_examples "a bulk push method", :push_bulk
+
+    it "limits the size of groups" do
+      FooJob.push_bulk([1, 2, 3, 4, 5, 6, 7], limit: 3)
+
+      expect(FooJob.jobs.length).to eq(7)
+      expect(FooJob).to have_enqueued_job(1)
+      expect(FooJob).to have_enqueued_job(2)
+      expect(FooJob).to have_enqueued_job(3)
+      expect(FooJob).to have_enqueued_job(4)
+      expect(FooJob).to have_enqueued_job(5)
+      expect(FooJob).to have_enqueued_job(6)
+      expect(FooJob).to have_enqueued_job(7)
+    end
+
+    it "limits with the item transformation" do
+      allow(Sidekiq::Client).to receive(:push_bulk)
 
-    BarJob.push_bulk([1, 2, 3])
+      FooJob.push_bulk([1, 2, 3, 4, 5, 6, 7], limit: 4) do |item|
+        [2*item, "some-value"]
+      end
+
+      expect(Sidekiq::Client).to have_received(:push_bulk).exactly(2).times
+      expect(Sidekiq::Client).to have_received(:push_bulk).with("class" => FooJob, "args" => [[2, "some-value"], [4, "some-value"], [6, "some-value"], [8, "some-value"]])
+      expect(Sidekiq::Client).to have_received(:push_bulk).with("class" => FooJob, "args" => [[10, "some-value"], [12, "some-value"], [14, "some-value"]])
+    end
+
+    it "goes through the Sidekiq::Client interface" do
+      allow(Sidekiq::Client).to receive(:push_bulk)
+
+      FooJob.push_bulk([1, 2, 3, 4, 5, 6, 7], limit: 3)
+
+      expect(Sidekiq::Client).to have_received(:push_bulk).exactly(3).times
+      expect(Sidekiq::Client).to have_received(:push_bulk).with("class" => FooJob, "args" => [[1], [2], [3]])
+      expect(Sidekiq::Client).to have_received(:push_bulk).with("class" => FooJob, "args" => [[4], [5], [6]])
+      expect(Sidekiq::Client).to have_received(:push_bulk).with("class" => FooJob, "args" => [[7]])
+    end
+
+    context "when no limit is specified" do
+      let(:item_count) { 9_999 }
+
+      before do
+        allow(Sidekiq::Client).to receive(:push_bulk)
+
+        FooJob.push_bulk((1..item_count).to_a)
+      end
+
+      context "when the item count is 10,000" do
+        let(:item_count) { 10_000 }
+
+        specify { expect(Sidekiq::Client).to have_received(:push_bulk).exactly(1).times }
+      end
+
+      context "when the item count is 10,001" do
+        let(:item_count) { 10_001 }
+
+        specify { expect(Sidekiq::Client).to have_received(:push_bulk).exactly(2).times }
+      end
+
+      context "when the item count is 40,000" do
+        let(:item_count) { 40_000 }
+
+        specify { expect(Sidekiq::Client).to have_received(:push_bulk).exactly(4).times }
+      end
+    end
   end
 
-  it "defaults to the identity function with no block given" do
-    FooJob.push_bulk([1, 2, 3])
+  describe "#push_bulk!" do
+    include_examples "a bulk push method", :push_bulk!
 
-    expect(FooJob.jobs.length).to eq(3)
-    expect(FooJob).to have_enqueued_job(1)
-    expect(FooJob).to have_enqueued_job(2)
-    expect(FooJob).to have_enqueued_job(3)
+    it "does not limit the number of jobs in one push" do
+      expect(Sidekiq::Client).to receive(:push_bulk).once.with("class" => FooJob, "args" => (1..100_000).map { |e| [e] })
+
+      FooJob.push_bulk!((1..100_000).to_a)
+    end
   end
 
   describe "inline test", sidekiq: :inline do
     specify { expect { FooJob.push_bulk([1, 2, 3]) }.to raise_error(RuntimeError, "1") }
+    specify { expect { FooJob.push_bulk!([1, 2, 3]) }.to raise_error(RuntimeError, "1") }
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-bulk
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: '0.1'
 platform: ruby
 authors:
 - Adam Prescott
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-20 00:00:00.000000000 Z
+date: 2015-07-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement