sidekiq-batching 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b41e2102a596f97cf7dbc3e6a078c12d41085a66
-  data.tar.gz: 4d5563a28bf4b753ff984ee58172bbfce43938f6
+  metadata.gz: e6d0c4e5564c036527dcc48e5336efc2193bcbab
+  data.tar.gz: 228c68f5f5005a261c5cf1c90398575628a8a482
 SHA512:
-  metadata.gz: 2ea999d0b2ecb210c0e8bfe62fbce383156da92f9ec117c5a1b7382f56ee87f71b63938db59f64c9b6c4c6a869d55816320747ae286a01129ac5323de89107af
-  data.tar.gz: 363c5fa098b66647c6ef60da7277d589f3767bb8afc1bfef394ce0a7574a85fd68eebba8998b9f5240b1d437306eb25fe9419bc5961ae639001f9e8b3cdb882e
+  metadata.gz: e1ebf0255ad1c14f6fd7b6242b934a0ef6a92ed064256991da94fb988596baf265b9f9d4b32f197326ee2d2f0781c80efca049c3fc757fda55d3d1c5c8ee535d
+  data.tar.gz: 6208a1a7020cfc52799051efda54a92daed302bdda620925c6668aba03d225301fab07724cf7f89b8044d1e4dbc93d61f53408d70b337105a1edf85ba79261f7

data/README.md

@@ -6,6 +6,8 @@ Useful for:
 * Grouping asynchronous API index calls into bulks for bulk updating/indexing.
 * Periodical batch updating of recently changing database counters.
 
+Sponsored by [Evil Martians](http://evilmartians.com)
+
 ## Usage
 
 Create a worker:
@@ -50,6 +52,10 @@ This jobs will be grouped into a single job which will be performed with the sin
 
 This will happen for every 30 jobs in a row or every 60 seconds.
 
+## Web UI
+
+![Web UI](web.png)
+
 Add this line to your `config/routes.rb` to activate web UI:
 
 ```ruby

data/lib/sidekiq/batching/actor.rb

@@ -2,7 +2,6 @@ module Sidekiq
   module Batching
     class Actor
       include Sidekiq::Batching::Logging
-      include Celluloid
 
       def initialize
         link_to_sidekiq_manager
@@ -44,4 +43,4 @@ module Sidekiq
       end
     end
   end
-end
+end

data/lib/sidekiq/batching/batch.rb

@@ -12,7 +12,13 @@ module Sidekiq
       attr_reader :name, :worker_class, :queue
 
       def add(msg)
-        @redis.push_msg(@name, msg.to_json)
+        msg = msg.to_json
+        @redis.push_msg(@name, msg, enqueue_similar_once?) if should_add? msg
+      end
+
+      def should_add? msg
+        return true unless enqueue_similar_once?
+        !@redis.enqueued?(@name, msg)
       end
 
       def size
@@ -94,6 +100,10 @@ module Sidekiq
         end
       end
 
+      def enqueue_similar_once?
+        worker_class_options['batch_unique'] == true
+      end
+
       def set_current_time_as_last
         @redis.set_last_execution_time(@name, Time.now)
       end

data/lib/sidekiq/batching/redis.rb

@@ -1,10 +1,29 @@
 module Sidekiq
   module Batching
     class Redis
-      def push_msg(name, msg)
+
+      PLUCK_SCRIPT = <<-SCRIPT
+        local pluck_values = redis.call('lrange', KEYS[1], 0, ARGV[1] - 1)
+        redis.call('ltrim', KEYS[1], ARGV[1], -1)
+        for k, v in pairs(pluck_values) do
+          redis.call('srem', KEYS[2], v)
+        end
+        return pluck_values
+      SCRIPT
+
+      def push_msg(name, msg, remember_unique = false)
+        redis do |conn|
+          conn.multi do
+            conn.sadd(ns('batches'), name)
+            conn.rpush(ns(name), msg)
+            conn.sadd(unique_messages_key(name), msg) if remember_unique
+          end
+        end
+      end
+
+      def enqueued?(name, msg)
         redis do |conn|
-          conn.sadd(ns('batches'), name)
-          conn.rpush(ns(name), msg)
+          conn.sismember(unique_messages_key(name), msg)
         end
       end
 
@@ -17,14 +36,9 @@ module Sidekiq
       end
 
       def pluck(name, limit)
-        redis do |conn|
-          result = conn.pipelined do
-            conn.lrange(ns(name), 0, limit - 1)
-            conn.ltrim(ns(name), limit, -1)
-          end
-
-          result.first
-        end
+        keys = [ns(name), unique_messages_key(name)]
+        args = [limit]
+        redis { |conn| conn.eval PLUCK_SCRIPT, keys, args }
       end
 
       def get_last_execution_time(name)
@@ -55,6 +69,11 @@ module Sidekiq
       end
 
       private
+
+      def unique_messages_key name
+        ns("#{name}:unique_messages")
+      end
+
       def ns(key = nil)
         "batching:#{key}"
       end
@@ -64,4 +83,4 @@ module Sidekiq
       end
     end
   end
-end
+end

data/lib/sidekiq/batching/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   module Batching
-    VERSION = "0.0.3"
+    VERSION = '0.0.4'
   end
 end

data/spec/modules/batch_spec.rb

@@ -29,34 +29,34 @@ describe Sidekiq::Batching::Batch do
     it 'must flush if limit exceeds for limit worker' do
       batch = subject.new(BatchedSizeWorker.name, 'batched_size')
 
-      expect(batch.could_flush?).to be_false
+      expect(batch.could_flush?).to be_falsy
       BatchedSizeWorker.perform_async('bar')
-      expect(batch.could_flush?).to be_false
+      expect(batch.could_flush?).to be_falsy
       4.times { BatchedSizeWorker.perform_async('bar') }
-      expect(batch.could_flush?).to be_true
+      expect(batch.could_flush?).to be_truthy
     end
 
     it 'must flush if limit exceeds for both worker' do
       batch = subject.new(BatchedBothWorker.name, 'batched_both')
 
-      expect(batch.could_flush?).to be_false
+      expect(batch.could_flush?).to be_falsy
       BatchedBothWorker.perform_async('bar')
-      expect(batch.could_flush?).to be_false
+      expect(batch.could_flush?).to be_falsy
       4.times { BatchedBothWorker.perform_async('bar') }
-      expect(batch.could_flush?).to be_true
+      expect(batch.could_flush?).to be_truthy
     end
 
     it 'must flush if limit okay but time came' do
       batch = subject.new(BatchedIntervalWorker.name, 'batched_interval')
 
-      expect(batch.could_flush?).to be_false
+      expect(batch.could_flush?).to be_falsy
       BatchedIntervalWorker.perform_async('bar')
-      expect(batch.could_flush?).to be_false
+      expect(batch.could_flush?).to be_falsy
       expect(batch.size).to eq(1)
 
       Timecop.travel(2.hours.since)
 
-      expect(batch.could_flush?).to be_true
+      expect(batch.could_flush?).to be_truthy
     end
   end
 
@@ -64,7 +64,7 @@ describe Sidekiq::Batching::Batch do
     it 'must put wokrer to queue on flush' do
       batch = subject.new(BatchedSizeWorker.name, 'batched_size')
 
-      expect(batch.could_flush?).to be_false
+      expect(batch.could_flush?).to be_falsy
       10.times { BatchedSizeWorker.perform_async('bar') }
       batch.flush
       expect(BatchedSizeWorker).to have_enqueued_job([["bar"], ["bar"], ["bar"]])
@@ -72,6 +72,56 @@ describe Sidekiq::Batching::Batch do
     end
   end
 
+  context 'with similar args' do
+    context 'option batch_unique = true' do
+      it 'enqueues once' do
+        batch = subject.new(BatchedUniqueArgsWorker.name, 'batched_unique_args')
+        3.times { BatchedUniqueArgsWorker.perform_async('bar', 1) }
+        expect(batch.size).to eq(1)
+      end
+
+      it 'enqueues once each unique set of args' do
+        batch = subject.new(BatchedUniqueArgsWorker.name, 'batched_unique_args')
+        3.times { BatchedUniqueArgsWorker.perform_async('bar', 1) }
+        6.times { BatchedUniqueArgsWorker.perform_async('baz', 1) }
+        3.times { BatchedUniqueArgsWorker.perform_async('bar', 1) }
+        2.times { BatchedUniqueArgsWorker.perform_async('baz', 3) }
+        7.times { BatchedUniqueArgsWorker.perform_async('bar', 1) }
+        expect(batch.size).to eq(3)
+      end
+
+      context 'flushing' do
+
+        it 'works' do
+          batch = subject.new(BatchedUniqueArgsWorker.name, 'batched_unique_args')
+          2.times { BatchedUniqueArgsWorker.perform_async('bar', 1) }
+          2.times { BatchedUniqueArgsWorker.perform_async('baz', 1) }
+          batch.flush
+          expect(batch.size).to eq(0)
+        end
+
+        it 'allows to enqueue again after flush' do
+          batch = subject.new(BatchedUniqueArgsWorker.name, 'batched_unique_args')
+          2.times { BatchedUniqueArgsWorker.perform_async('bar', 1) }
+          2.times { BatchedUniqueArgsWorker.perform_async('baz', 1) }
+          batch.flush
+          BatchedUniqueArgsWorker.perform_async('bar', 1)
+          BatchedUniqueArgsWorker.perform_async('baz', 1)
+          expect(batch.size).to eq(2)
+        end
+      end
+
+    end
+
+    context 'batch_unique is not specified' do
+      it 'enqueues all' do
+        batch = subject.new(BatchedSizeWorker.name, 'batched_size')
+        3.times { BatchedSizeWorker.perform_async('bar', 1) }
+        expect(batch.size).to eq(3)
+      end
+    end
+  end
+
   private
   def expect_batch(klass, queue)
     expect(klass).to_not have_enqueued_job('bar')
@@ -83,4 +133,4 @@ describe Sidekiq::Batching::Batch do
     expect(stats.first.queue).to eq(queue)
     expect(batch.pluck).to eq [['bar']]
   end
-end
+end

data/spec/modules/redis_spec.rb

@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+describe Sidekiq::Batching::Redis do
+  subject { Sidekiq::Batching::Redis.new }
+
+  let(:queue_name)    { "my_queue" }
+  let(:key)           { "batching:#{queue_name}" }
+  let(:unique_key)    { "batching:#{queue_name}:unique_messages" }
+
+  describe "#push_msg" do
+    it "adds message to queue" do
+      subject.push_msg(queue_name, 'My message')
+      expect(redis { |c| c.llen key }).to eq 1
+      expect(redis { |c| c.lrange key, 0, 1 }).to eq ['My message']
+      expect(redis { |c| c.smembers unique_key}).to eq []
+    end
+
+    it "remembers unique message if specified" do
+      subject.push_msg(queue_name, 'My message', true)
+      expect(redis { |c| c.smembers unique_key}).to eq ['My message']
+    end
+  end
+
+  describe "#pluck" do
+    it "removes messages from queue" do
+      subject.push_msg(queue_name, "Message 1")
+      subject.push_msg(queue_name, "Message 2")
+      subject.pluck(queue_name, 2)
+      expect(redis { |c| c.llen key }).to eq 0
+    end
+
+    it "forgets unique messages" do
+      subject.push_msg(queue_name, "Message 1", true)
+      subject.push_msg(queue_name, "Message 2", true)
+      expect(redis { |c| c.scard unique_key }).to eq 2
+      subject.pluck(queue_name, 2)
+      expect(redis { |c| c.smembers unique_key }).to eq []
+    end
+  end
+
+  private
+
+  def redis(&block)
+    Sidekiq.redis(&block)
+  end
+
+end

data/spec/spec_helper.rb

@@ -41,4 +41,4 @@ RSpec.configure do |config|
   end
 end
 
-$: << File.join(File.dirname(__FILE__), '..', 'lib')
+$: << File.join(File.dirname(__FILE__), '..', 'lib')

data/spec/support/test_workers.rb

@@ -30,4 +30,13 @@ class BatchedBothWorker
 
   def perform(foo)
   end
-end
+end
+
+class BatchedUniqueArgsWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: :batched_unique_args, batch_size: 3, batch_unique: true
+
+  def perform(foo)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-batching
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Victor Sokolov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-14 00:00:00.000000000 Z
+date: 2015-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -147,8 +147,10 @@ files:
 - lib/sidekiq/batching/web.rb
 - sidekiq-batching.gemspec
 - spec/modules/batch_spec.rb
+- spec/modules/redis_spec.rb
 - spec/spec_helper.rb
 - spec/support/test_workers.rb
+- web.png
 homepage: http://github.com/gzigzigzeo/sidekiq-batching
 licenses:
 - MIT
@@ -169,11 +171,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.3
 signing_key: 
 specification_version: 4
 summary: Allows identical sidekiq jobs to be processed with a single background call
 test_files:
 - spec/modules/batch_spec.rb
+- spec/modules/redis_spec.rb
 - spec/spec_helper.rb
 - spec/support/test_workers.rb