sidekiq-merger 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4bed498597132701a71bf77a583f97668ce5264c
-  data.tar.gz: 66244be154087fca1647ba21d7b12727167d5106
+  metadata.gz: 2333d3953e27f0ba0d3f0efc23a8bc48747b2a6d
+  data.tar.gz: ed6230c813938f38f4092ee9a27589a774c55029
 SHA512:
-  metadata.gz: 01887441761e31726a0c4cb6feff20e55be5202c3fa9c3d00a43ff28b54589c484ddd95f305ac0c589972e27c05c4ae46201ca408af32ee0904aae2b80827e16
-  data.tar.gz: 762fca326555feada8466a1eeda5acf2189a8f9e1b2b979e2568b90ccddbc5371d840833f106640889f1f69a744ce79bfa29ee932cf1f905370586375a74830b
+  metadata.gz: 271adc650104013b83c9fa5b99672c995c10b7e2d3518485daa9b5f2d62a9fc14333cd29014d8024f0e17a7b6bcfea393010d544f1ca899bab876a9a14647f7d
+  data.tar.gz: 1e647aa5449188f1649bd57a84fb34629f348996991e28085aa19a83139cd31eaa2ec089aebb82f6e6cfcd4ec57ffc0738eb52118a07498d72c851d8aeefb9a0

data/README.md

@@ -6,7 +6,7 @@
 [![Coverage Status][cov-image]][cov-link]
 [![Code Climate][gpa-image]][gpa-link]
 
-Merge sidekiq jobs occurring within specific period. This sidekiq middleware is inspired by [sidekiq-grouping](https://github.com/gzigzigzeo/sidekiq-grouping).
+Merge sidekiq jobs occurring before the execution time. Inspired by [sidekiq-grouping](https://github.com/gzigzigzeo/sidekiq-grouping).
 
 ## Installation
 
@@ -34,8 +34,10 @@ class SomeWorker
 
   sidekiq_options merger: { key: -> (args) { args[0] } }
 
-  def perform(*ids)
-    # Do something
+  def perform(*merged_args)
+    merged_args.each do |args|
+      # Do something
+    end
   end
 end
 ```
@@ -46,9 +48,12 @@ Then, enqueue jobs by `perform_in` or `perform_at`.
 SomeWorker.perform_in 100, 4
 SomeWorker.perform_in 100, 3
 SomeWorker.perform_in 100, 5
+# Passed 100 seconds from the first enqueue.
+SomeWorker.perform_in 100, 6
+SomeWorker.perform_in 100, 1
 ```
 
-`SomeWorker` will be executed in 100 seconds with args of `[4], [3], [5]`.
+`SomeWorker` will be executed in 100 seconds with args of `[4], [3], [5]`, then with args of `[6], [1]`.
 
 `perform_async` works without merging args.
 

data/lib/sidekiq/merger/merge.rb

@@ -6,7 +6,7 @@ class Sidekiq::Merger::Merge
     def all
       redis = Sidekiq::Merger::Redis.new
 
-      redis.all.map { |full_merge_key| initialize_with_full_merge_key(full_merge_key, redis: redis) }
+      redis.all_merges.map { |full_merge_key| initialize_with_full_merge_key(full_merge_key, redis: redis) }
     end
 
     def initialize_with_full_merge_key(full_merge_key, options = {})
@@ -47,17 +47,17 @@ class Sidekiq::Merger::Merge
   end
 
   def add(args, execution_time)
-    if !options[:unique] || !@redis.exists?(full_merge_key, args)
-      @redis.push(full_merge_key, args, execution_time)
+    if !options[:unique] || !@redis.merge_exists?(full_merge_key, args)
+      @redis.push_message(full_merge_key, args, execution_time)
     end
   end
 
   def delete(args)
-    @redis.delete(full_merge_key, args)
+    @redis.delete_message(full_merge_key, args)
   end
 
   def delete_all
-    @redis.delete_all(full_merge_key)
+    @redis.delete_merge(full_merge_key)
   end
 
   def size
@@ -67,8 +67,8 @@ class Sidekiq::Merger::Merge
   def flush
     msgs = []
 
-    if @redis.lock(full_merge_key, Sidekiq::Merger::Config.lock_ttl)
-      msgs = @redis.pluck(full_merge_key)
+    if @redis.lock_merge(full_merge_key, Sidekiq::Merger::Config.lock_ttl)
+      msgs = @redis.pluck_merge(full_merge_key)
     end
 
     unless msgs.empty?
@@ -90,11 +90,11 @@ class Sidekiq::Merger::Merge
   end
 
   def all_args
-    @redis.get(full_merge_key)
+    @redis.get_merge(full_merge_key)
   end
 
   def execution_time
-    @execution_time ||= @redis.execution_time(full_merge_key)
+    @execution_time ||= @redis.merge_execution_time(full_merge_key)
   end
 
   def ==(other)

data/lib/sidekiq/merger/redis.rb

@@ -42,7 +42,7 @@ class Sidekiq::Merger::Redis
     end
   end
 
-  def push(key, msg, execution_time)
+  def push_message(key, msg, execution_time)
     msg_json = msg.to_json
     redis do |conn|
       conn.multi do
@@ -54,17 +54,17 @@ class Sidekiq::Merger::Redis
     end
   end
 
-  def delete(key, msg)
+  def delete_message(key, msg)
     msg_json = msg.to_json
     redis do |conn|
       conn.multi do
         conn.srem(unique_msg_key(key), msg_json)
-        conn.lrem(msg_key(key), msg_json)
+        conn.lrem(msg_key(key), 0, msg_json)
       end
     end
   end
 
-  def execution_time(key)
+  def merge_execution_time(key)
     redis do |conn|
       t = conn.get(time_key(key))
       Time.at(t.to_i) unless t.nil?
@@ -75,26 +75,26 @@ class Sidekiq::Merger::Redis
     redis { |conn| conn.llen(msg_key(key)) }
   end
 
-  def exists?(key, msg)
+  def merge_exists?(key, msg)
     msg_json = msg.to_json
     redis { |conn| conn.sismember(unique_msg_key(key), msg_json) }
   end
 
-  def all
+  def all_merges
     redis { |conn| conn.smembers(merges_key) }
   end
 
-  def lock(key, ttl)
+  def lock_merge(key, ttl)
     redis { |conn| conn.set(lock_key(key), true, nx: true, ex: ttl) }
   end
 
-  def get(key)
+  def get_merge(key)
     msgs = []
     redis { |conn| msgs = conn.lrange(msg_key(key), 0, -1) }
     msgs.map { |msg| JSON.parse(msg) }
   end
 
-  def pluck(key)
+  def pluck_merge(key)
     msgs = []
     redis do |conn|
       conn.multi do
@@ -108,7 +108,7 @@ class Sidekiq::Merger::Redis
     extract_future_value(msgs).map { |msg| JSON.parse(msg) }
   end
 
-  def delete_all(key)
+  def delete_merge(key)
     redis do |conn|
       conn.multi do
         conn.del(unique_msg_key(key))

data/lib/sidekiq/merger/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   module Merger
-    VERSION = "0.0.4".freeze
+    VERSION = "0.0.5".freeze
   end
 end

data/spec/sidekiq/merger/merge_spec.rb

@@ -63,19 +63,19 @@ describe Sidekiq::Merger::Merge do
 
   describe "#add" do
     it "adds the args in lazy merge" do
-      expect(redis).to receive(:push).with("name:queue:foo", [1, 2, 3], execution_time)
+      expect(redis).to receive(:push_message).with("name:queue:foo", [1, 2, 3], execution_time)
       subject.add([1, 2, 3], execution_time)
     end
     context "with unique option" do
       let(:options) { { key: -> (args) { args.to_json }, unique: true } }
       it "adds the args in lazy merge" do
-        expect(redis).to receive(:push).with("name:queue:foo", [1, 2, 3], execution_time)
+        expect(redis).to receive(:push_message).with("name:queue:foo", [1, 2, 3], execution_time)
         subject.add([1, 2, 3], execution_time)
       end
       context "the args has alredy been added" do
         before { subject.add([1, 2, 3], execution_time) }
         it "adds the args in lazy merge" do
-          expect(redis).not_to receive(:push)
+          expect(redis).not_to receive(:push_message)
           subject.add([1, 2, 3], execution_time)
         end
       end
@@ -84,7 +84,7 @@ describe Sidekiq::Merger::Merge do
 
   describe "#delete" do
     it "adds the args in lazy merge" do
-      expect(redis).to receive(:delete).with("name:queue:foo", [1, 2, 3])
+      expect(redis).to receive(:delete_message).with("name:queue:foo", [1, 2, 3])
       subject.delete([1, 2, 3])
     end
   end

data/spec/sidekiq/merger/redis_spec.rb

@@ -26,10 +26,10 @@ describe Sidekiq::Merger::Redis do
     end
   end
 
-  describe "#push" do
-    shared_examples_for "push spec" do
-      it "pushes the args" do
-        subject.push(pushing_key, pushing_msg, pushing_execution_time)
+  describe "#push_message" do
+    shared_examples_for "push_message spec" do
+      it "pushes the msg" do
+        subject.push_message(pushing_key, pushing_msg, pushing_execution_time)
         described_class.redis do |conn|
           expect(conn.smembers("sidekiq-merger:merges")).to contain_exactly(*merge_keys)
           expect(conn.keys("sidekiq-merger:time:*")).to contain_exactly(*times.keys)
@@ -44,7 +44,7 @@ describe Sidekiq::Merger::Redis do
         end
       end
       it "sets the execution time" do
-        subject.push(pushing_key, pushing_msg, pushing_execution_time)
+        subject.push_message(pushing_key, pushing_msg, pushing_execution_time)
         described_class.redis do |conn|
           merge_keys.each do |key, time|
             expect(conn.get(key)).to eq time
@@ -57,7 +57,7 @@ describe Sidekiq::Merger::Redis do
     let(:pushing_msg) { [1, 2, 3] }
     let(:pushing_execution_time) { execution_time }
 
-    include_examples "push spec" do
+    include_examples "push_message spec" do
       let(:merge_keys) { ["foo"] }
       let(:times) { {
         "sidekiq-merger:time:foo" => execution_time.to_i.to_s,
@@ -72,8 +72,8 @@ describe Sidekiq::Merger::Redis do
 
     context "the merge key already exists" do
       let(:pushing_msg) { [2, 3, 4] }
-      before { subject.push("foo", [1, 2, 3], execution_time) }
-      include_examples "push spec" do
+      before { subject.push_message("foo", [1, 2, 3], execution_time) }
+      include_examples "push_message spec" do
         let(:merge_keys) { ["foo"] }
         let(:times) { {
           "sidekiq-merger:time:foo" => execution_time.to_i.to_s,
@@ -87,9 +87,9 @@ describe Sidekiq::Merger::Redis do
       end
     end
 
-    context "the args has already ben pushed" do
-      before { subject.push("foo", [1, 2, 3], execution_time) }
-      include_examples "push spec" do
+    context "the msg has already ben pushed" do
+      before { subject.push_message("foo", [1, 2, 3], execution_time) }
+      include_examples "push_message spec" do
         let(:merge_keys) { ["foo"] }
         let(:times) { {
           "sidekiq-merger:time:foo" => execution_time.to_i.to_s,
@@ -107,8 +107,8 @@ describe Sidekiq::Merger::Redis do
       let(:pushing_key) { "bar" }
       let(:pushing_msg) { [2, 3, 4] }
       let(:pushing_execution_time) { execution_time + 1.hour }
-      before { subject.push("foo", [1, 2, 3], execution_time) }
-      include_examples "push spec" do
+      before { subject.push_message("foo", [1, 2, 3], execution_time) }
+      include_examples "push_message spec" do
         let(:merge_keys) { ["foo", "bar"] }
         let(:times) { {
           "sidekiq-merger:time:foo" => execution_time.to_i.to_s,
@@ -126,45 +126,112 @@ describe Sidekiq::Merger::Redis do
     end
   end
 
-  describe "#delete" do
+  describe "#delete_message" do
+    before do
+      subject.redis do |conn|
+        conn.sadd("sidekiq-merger:unique_msg:foo", "[1,2,3]")
+        conn.sadd("sidekiq-merger:unique_msg:foo", "[2,3,4]")
+        conn.lpush("sidekiq-merger:msg:foo", "[1,2,3]")
+        conn.lpush("sidekiq-merger:msg:foo", "[2,3,4]")
+      end
+    end
+    it "deletes the msg" do
+      subject.delete_message("foo", [1, 2, 3])
+      subject.redis do |conn|
+        expect(conn.smembers("sidekiq-merger:unique_msg:foo")).to contain_exactly "[2,3,4]"
+        expect(conn.lrange("sidekiq-merger:msg:foo", 0, -1)).to contain_exactly "[2,3,4]"
+      end
+    end
+    context "with duplicate msgs" do
+      it "deletes the msg" do
+        subject.redis do |conn|
+          conn.lpush("sidekiq-merger:msg:foo", "[1,2,3]")
+        end
+        subject.delete_message("foo", [1, 2, 3])
+        subject.redis do |conn|
+          expect(conn.smembers("sidekiq-merger:unique_msg:foo")).to contain_exactly "[2,3,4]"
+          expect(conn.lrange("sidekiq-merger:msg:foo", 0, -1)).to contain_exactly "[2,3,4]"
+        end
+      end
+    end
   end
 
   describe "#merge_size" do
+    before do
+      subject.redis do |conn|
+        conn.lpush("sidekiq-merger:msg:foo", "[1,2,3]")
+        conn.lpush("sidekiq-merger:msg:foo", "[2,3,4]")
+      end
+    end
+    it "returns the size" do
+      expect(subject.merge_size("foo")).to eq 2
+    end
   end
 
-  describe "#exists?" do
+  describe "#merge_exists?" do
     context "unique key exists" do
       it "returns true" do
         described_class.redis { |conn| conn.sadd("sidekiq-merger:unique_msg:foo", "\"test\"") }
-        expect(subject.exists?("foo", "test")).to eq true
+        expect(subject.merge_exists?("foo", "test")).to eq true
       end
     end
     context "unique key does not exists" do
       it "returns false" do
-        expect(subject.exists?("foo", "test")).to eq false
+        expect(subject.merge_exists?("foo", "test")).to eq false
       end
     end
   end
 
-  describe "#all" do
+  describe "#all_merges" do
+    before do
+      subject.push_message("foo", [1, 2, 3], execution_time)
+      subject.push_message("bar", [2, 3, 4], execution_time)
+    end
+    it "gets all the merges" do
+      expect(subject.all_merges).to contain_exactly "foo", "bar"
+    end
   end
 
-  describe "#lock" do
+  describe "#lock_merge" do
+    it "locks the key" do
+      subject.lock_merge("foo", 3)
+      subject.redis do |conn|
+        expect(conn.exists("sidekiq-merger:lock:foo")).to eq true
+      end
+    end
   end
 
-  describe "#get" do
+  describe "#get_merge" do
+    before do
+      subject.push_message("bar", [1, 2, 3], execution_time)
+      subject.push_message("bar", [2, 3, 4], execution_time)
+    end
+    it "gets all the msg" do
+      expect(subject.get_merge("bar")).to contain_exactly [1, 2, 3], [2, 3, 4]
+      expect(subject.merge_size("bar")).to eq 2
+    end
   end
 
-  describe "#pluck" do
+  describe "#pluck_merge" do
     before do
-      subject.push("bar", [1, 2, 3], execution_time)
-      subject.push("bar", [2, 3, 4], execution_time)
+      subject.push_message("bar", [1, 2, 3], execution_time)
+      subject.push_message("bar", [2, 3, 4], execution_time)
     end
-    it "plucks all the args" do
-      expect(subject.pluck("bar")).to contain_exactly [1, 2, 3], [2, 3, 4]
+    it "plucks all the msg" do
+      expect(subject.pluck_merge("bar")).to contain_exactly [1, 2, 3], [2, 3, 4]
+      expect(subject.merge_size("bar")).to eq 0
     end
   end
 
-  describe "#delete_all" do
+  describe "#delete_merge" do
+    before do
+      subject.push_message("foo", [1, 2, 3], execution_time)
+      subject.push_message("foo", [1, 2, 3], execution_time)
+    end
+    it "deletes the merge" do
+      expect {
+        subject.delete_merge("foo")
+      }.to change { subject.merge_size("foo") }.from(2).to(0)
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-merger
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - dtaniwaki