sidekiq-merger 0.0.1 → 0.0.4

data/lib/sidekiq/merger/merge.rb

@@ -0,0 +1,113 @@
+require_relative "redis"
+require "active_support/core_ext/hash/indifferent_access"
+
+class Sidekiq::Merger::Merge
+  class << self
+    def all
+      redis = Sidekiq::Merger::Redis.new
+
+      redis.all.map { |full_merge_key| initialize_with_full_merge_key(full_merge_key, redis: redis) }
+    end
+
+    def initialize_with_full_merge_key(full_merge_key, options = {})
+      keys = full_merge_key.split(":")
+      raise "Invalid merge key" if keys.size < 3
+      worker_class = keys[0].camelize.constantize
+      queue = keys[1]
+      merge_key = keys[2]
+      new(worker_class, queue, merge_key, options)
+    end
+
+    def initialize_with_args(worker_class, queue, args, options = {})
+      new(worker_class, queue, merge_key(worker_class, args), options)
+    end
+
+    def merge_key(worker_class, args)
+      options = get_options(worker_class)
+      merge_key = options["key"]
+      if merge_key.respond_to?(:call)
+        merge_key.call(args)
+      else
+        merge_key
+      end
+    end
+
+    def get_options(worker_class)
+      (worker_class.get_sidekiq_options["merger"] || {}).with_indifferent_access
+    end
+  end
+
+  attr_reader :worker_class, :queue, :merge_key
+
+  def initialize(worker_class, queue, merge_key, redis: Sidekiq::Merger::Redis.new)
+    @worker_class = worker_class
+    @queue = queue
+    @merge_key = merge_key
+    @redis = redis
+  end
+
+  def add(args, execution_time)
+    if !options[:unique] || !@redis.exists?(full_merge_key, args)
+      @redis.push(full_merge_key, args, execution_time)
+    end
+  end
+
+  def delete(args)
+    @redis.delete(full_merge_key, args)
+  end
+
+  def delete_all
+    @redis.delete_all(full_merge_key)
+  end
+
+  def size
+    @redis.merge_size(full_merge_key)
+  end
+
+  def flush
+    msgs = []
+
+    if @redis.lock(full_merge_key, Sidekiq::Merger::Config.lock_ttl)
+      msgs = @redis.pluck(full_merge_key)
+    end
+
+    unless msgs.empty?
+      Sidekiq::Client.push(
+        "class" => worker_class,
+        "queue" => queue,
+        "args" => msgs,
+        "merged" => true
+      )
+    end
+  end
+
+  def can_flush?
+    !execution_time.nil? && execution_time < Time.now
+  end
+
+  def full_merge_key
+    @full_merge_key ||= [worker_class.name.to_s.underscore, queue, merge_key].join(":")
+  end
+
+  def all_args
+    @redis.get(full_merge_key)
+  end
+
+  def execution_time
+    @execution_time ||= @redis.execution_time(full_merge_key)
+  end
+
+  def ==(other)
+    self.worker_class == other.worker_class &&
+    self.queue == other.queue &&
+    self.merge_key == other.merge_key
+  end
+
+  private
+
+  def options
+    @options ||= self.class.get_options(worker_class)
+  rescue NameError
+    {}
+  end
+end

data/lib/sidekiq/merger/middleware.rb

@@ -1,18 +1,20 @@
-require_relative "batch"
+require_relative "merge"
 
 class Sidekiq::Merger::Middleware
-  def call(worker_class, msg, queue, _redis_pool = nil)
+  def call(worker_class, msg, queue, redis_pool = nil)
     return yield if defined?(Sidekiq::Testing) && Sidekiq::Testing.inline?
 
     worker_class = worker_class.camelize.constantize if worker_class.is_a?(String)
     options = worker_class.get_sidekiq_options
 
     if !msg["at"].nil? && options.key?("merger")
-      Sidekiq::Merger::Batch
+      Sidekiq::Merger::Merge
         .initialize_with_args(worker_class, queue, msg["args"])
         .add(msg["args"], msg["at"])
+      false
     else
-      yield
+      msg["args"] = [msg["args"]] unless msg.delete("merged")
+      yield(worker_class, msg, queue, redis_pool)
     end
   end
 end

data/lib/sidekiq/merger/redis.rb

@@ -13,12 +13,16 @@ class Sidekiq::Merger::Redis
           end
           return true
         SCRIPT
-        conn.eval(script, [], [batches_key, msg_key("*"), lock_key("*")])
+        conn.eval(script, [], [merges_key, unique_msg_key("*"), msg_key("*"), lock_key("*")])
       end
     end
 
-    def batches_key
-      "#{KEY_PREFIX}:batches"
+    def merges_key
+      "#{KEY_PREFIX}:merges"
+    end
+
+    def unique_msg_key(key)
+      "#{KEY_PREFIX}:unique_msg:#{key}"
     end
 
     def msg_key(key)
@@ -39,33 +43,45 @@ class Sidekiq::Merger::Redis
   end
 
   def push(key, msg, execution_time)
+    msg_json = msg.to_json
     redis do |conn|
       conn.multi do
-        conn.sadd(batches_key, key)
-        conn.setnx(time_key(key), execution_time.to_json)
-        conn.sadd(msg_key(key), msg.to_json)
+        conn.sadd(merges_key, key)
+        conn.setnx(time_key(key), execution_time.to_i)
+        conn.lpush(msg_key(key), msg_json)
+        conn.sadd(unique_msg_key(key), msg_json)
       end
     end
   end
 
   def delete(key, msg)
-    redis { |conn| conn.srem(msg_key(key), msg.to_json) }
+    msg_json = msg.to_json
+    redis do |conn|
+      conn.multi do
+        conn.srem(unique_msg_key(key), msg_json)
+        conn.lrem(msg_key(key), msg_json)
+      end
+    end
   end
 
   def execution_time(key)
-    redis { |conn| Time.parse(conn.get(time_key(key))) rescue nil }
+    redis do |conn|
+      t = conn.get(time_key(key))
+      Time.at(t.to_i) unless t.nil?
+    end
   end
 
-  def batch_size(key)
-    redis { |conn| conn.scard(msg_key(key)) }
+  def merge_size(key)
+    redis { |conn| conn.llen(msg_key(key)) }
   end
 
   def exists?(key, msg)
-    redis { |conn| conn.sismember(msg_key(key), msg.to_json) }
+    msg_json = msg.to_json
+    redis { |conn| conn.sismember(unique_msg_key(key), msg_json) }
   end
 
   def all
-    redis { |conn| conn.smembers(batches_key) }
+    redis { |conn| conn.smembers(merges_key) }
   end
 
   def lock(key, ttl)
@@ -74,33 +90,44 @@ class Sidekiq::Merger::Redis
 
   def get(key)
     msgs = []
-    redis do |conn|
-      msgs = conn.smembers(msg_key(key))
-    end
+    redis { |conn| msgs = conn.lrange(msg_key(key), 0, -1) }
     msgs.map { |msg| JSON.parse(msg) }
   end
 
   def pluck(key)
     msgs = []
     redis do |conn|
-      msgs = conn.smembers(msg_key(key))
-      conn.del(msg_key(key))
-      conn.del(time_key(key))
-      conn.srem(batches_key, key)
+      conn.multi do
+        msgs = conn.lrange(msg_key(key), 0, -1)
+        conn.del(unique_msg_key(key))
+        conn.del(msg_key(key))
+        conn.del(time_key(key))
+        conn.srem(merges_key, key)
+      end
     end
-    msgs.map { |msg| JSON.parse(msg) }
+    extract_future_value(msgs).map { |msg| JSON.parse(msg) }
   end
 
   def delete_all(key)
     redis do |conn|
-      conn.del(msg_key(key))
-      conn.del(time_key(key))
-      conn.del(lock_key(key))
-      conn.srem(batches_key, key)
+      conn.multi do
+        conn.del(unique_msg_key(key))
+        conn.del(msg_key(key))
+        conn.del(time_key(key))
+        conn.del(lock_key(key))
+        conn.srem(merges_key, key)
+      end
     end
   end
 
   private
 
-  delegate :batches_key, :msg_key, :time_key, :lock_key, :redis, to: "self.class"
+  delegate :merges_key, :msg_key, :unique_msg_key, :time_key, :lock_key, :redis, to: "self.class"
+
+  def extract_future_value(future)
+    while future.value.is_a?(Redis::FutureNotReady)
+      sleep(0.001)
+    end
+    future.value
+  end
 end

data/lib/sidekiq/merger/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   module Merger
-    VERSION = "0.0.1".freeze
+    VERSION = "0.0.4".freeze
   end
 end

data/lib/sidekiq/merger/views/index.erb

@@ -0,0 +1,41 @@
+<header class="row">
+  <div class="col-sm-5">
+    <h3>Merged jobs</h3>
+  </div>
+</header>
+
+<div class="container">
+  <div class="row">
+    <div class="col-sm-12">
+      <% if true %>
+      <table class="table table-striped table-bordered table-white" style="width: 100%; margin: 0; table-layout:fixed;">
+        <thead>
+          <th style="width: 25%">Worker</th>
+          <th style="width: 15%">Queue</th>
+          <th style="width: 10%">Count</th>
+          <th style="width: 20%">All Args</th>
+          <th style="width: 20%">Execution time</th>
+          <th style="width: 10%">Actions</th>
+        </thead>
+        <% @merges.each do |merge| %>
+        <tr>
+          <td><%= merge.worker_class %></td>
+          <td><%= merge.queue %></td>
+          <td><%= merge.size %></td>
+          <td><%= merge.all_args %></td>
+          <td><%= merge.execution_time || "&ndash;"%></td>
+          <td>
+            <form action="<%= "#{root_path}merger/#{URI.encode_www_form_component merge.full_merge_key}/delete" %>" method="post">
+              <%= csrf_tag %>
+              <input class="btn btn-danger btn-xs" type="submit" name="delete" value="Delete" data-confirm="Are you sure you want to delete this merge?" />
+            </form>
+          </td>
+        </tr>
+        <% end %>
+      </table>
+      <% else %>
+      <div class="alert alert-success">No recurring jobs found.</div>
+      <% end %>
+    </div>
+  </div>
+</div>

data/lib/sidekiq/merger/web.rb

@@ -0,0 +1,22 @@
+require "sidekiq/web"
+
+module Sidekiq::Merger::Web
+  VIEWS = File.expand_path("views", File.dirname(__FILE__))
+
+  def self.registered(app)
+    app.get "/merger" do
+      @merges = Sidekiq::Merger::Merge.all
+      erb File.read(File.join(VIEWS, "index.erb")), locals: { view_path: VIEWS }
+    end
+
+    app.post "/merger/:full_merge_key/delete" do
+      full_merge_key = URI.decode_www_form_component params[:full_merge_key]
+      merge = Sidekiq::Merger::Merge.initialize_with_full_merge_key(full_merge_key)
+      merge.delete_all
+      redirect "#{root_path}/merger"
+    end
+  end
+end
+
+Sidekiq::Web.register(Sidekiq::Merger::Web)
+Sidekiq::Web.tabs["Merger"] = "merger"

data/sidekiq-merger.gemspec

@@ -24,16 +24,14 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = [">= 2.2.2", "< 2.5"]
 
-  spec.add_development_dependency "bundler", "~> 1.13"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "simplecov"
-  spec.add_development_dependency "timecop"
-  spec.add_development_dependency "rubocop"
-  spec.add_development_dependency "pry"
-  spec.add_development_dependency "coveralls"
+  spec.add_development_dependency "rake", ">= 10.0", "< 13"
+  spec.add_development_dependency "rspec", ">= 3.0", "< 4"
+  spec.add_development_dependency "simplecov", "~> 0.12"
+  spec.add_development_dependency "timecop", "~> 0.8"
+  spec.add_development_dependency "rubocop", "~> 0.47"
+  spec.add_development_dependency "coveralls", "~> 0.8"
 
-  spec.add_dependency "sidekiq", ">= 3.4.0"
-  spec.add_dependency "concurrent-ruby"
-  spec.add_dependency "activesupport", ">= 3.2.0"
+  spec.add_runtime_dependency "sidekiq", ">= 3.4", "< 5"
+  spec.add_runtime_dependency "concurrent-ruby", "~> 1.0"
+  spec.add_runtime_dependency "activesupport", ">= 3.2", "< 6"
 end

data/spec/sidekiq/merger/flusher_spec.rb

@@ -4,13 +4,13 @@ describe Sidekiq::Merger::Flusher do
   subject { described_class.new(Sidekiq.logger) }
 
   describe "#call" do
-    let(:active_batch) { double(full_batch_key: "active", can_flush?: true, flush: nil) }
-    let(:inactive_batch) { double(full_batch_key: "inactive", can_flush?: false, flush: nil) }
-    let(:batches) { [active_batch, inactive_batch] }
-    it "adds the args to the batch" do
-      allow(Sidekiq::Merger::Batch).to receive(:all).and_return batches
-      expect(active_batch).to receive(:flush)
-      expect(inactive_batch).not_to receive(:flush)
+    let(:active_merge) { double(full_merge_key: "active", can_flush?: true, flush: nil) }
+    let(:inactive_merge) { double(full_merge_key: "inactive", can_flush?: false, flush: nil) }
+    let(:merges) { [active_merge, inactive_merge] }
+    it "adds the args to the merge" do
+      allow(Sidekiq::Merger::Merge).to receive(:all).and_return merges
+      expect(active_merge).to receive(:flush)
+      expect(inactive_merge).not_to receive(:flush)
 
       subject.flush
     end

data/spec/sidekiq/merger/{batch_spec.rb → merge_spec.rb}

@@ -1,6 +1,6 @@
 require "spec_helper"
 
-describe Sidekiq::Merger::Batch do
+describe Sidekiq::Merger::Merge do
   subject { described_class.new(worker_class, queue, "foo", redis: redis) }
   let(:redis) { Sidekiq::Merger::Redis.new }
   let(:queue) { "queue" }
@@ -27,8 +27,8 @@ describe Sidekiq::Merger::Batch do
   describe ".all" do
     it "returns all the keys" do
       redis.redis do |conn|
-        conn.sadd("sidekiq-merger:batches", "string:foo:xxx")
-        conn.sadd("sidekiq-merger:batches", "numeric:bar:yyy")
+        conn.sadd("sidekiq-merger:merges", "string:foo:xxx")
+        conn.sadd("sidekiq-merger:merges", "numeric:bar:yyy")
       end
 
       expect(described_class.all).to contain_exactly(
@@ -40,18 +40,18 @@ describe Sidekiq::Merger::Batch do
     context "including invalid key" do
       it "raises an error" do
         redis.redis do |conn|
-          conn.sadd("sidekiq-merger:batches", "string:foo:xxx")
-          conn.sadd("sidekiq-merger:batches", "invalid")
+          conn.sadd("sidekiq-merger:merges", "string:foo:xxx")
+          conn.sadd("sidekiq-merger:merges", "invalid")
         end
         expect {
           described_class.all
-        }.to raise_error RuntimeError, "Invalid batch key"
+        }.to raise_error RuntimeError, "Invalid merge key"
       end
     end
   end
 
   describe ".initialize_with_args" do
-    it "provides batch_key from args" do
+    it "provides merge_key from args" do
       expect(described_class).to receive(:new).with(worker_class, queue, "[1,2,3]", anything)
       described_class.initialize_with_args(worker_class, queue, [1, 2, 3])
     end
@@ -62,14 +62,28 @@ describe Sidekiq::Merger::Batch do
   end
 
   describe "#add" do
-    it "adds the args in lazy batch" do
+    it "adds the args in lazy merge" do
       expect(redis).to receive(:push).with("name:queue:foo", [1, 2, 3], execution_time)
       subject.add([1, 2, 3], execution_time)
     end
+    context "with unique option" do
+      let(:options) { { key: -> (args) { args.to_json }, unique: true } }
+      it "adds the args in lazy merge" do
+        expect(redis).to receive(:push).with("name:queue:foo", [1, 2, 3], execution_time)
+        subject.add([1, 2, 3], execution_time)
+      end
+      context "the args has alredy been added" do
+        before { subject.add([1, 2, 3], execution_time) }
+        it "adds the args in lazy merge" do
+          expect(redis).not_to receive(:push)
+          subject.add([1, 2, 3], execution_time)
+        end
+      end
+    end
   end
 
   describe "#delete" do
-    it "adds the args in lazy batch" do
+    it "adds the args in lazy merge" do
       expect(redis).to receive(:delete).with("name:queue:foo", [1, 2, 3])
       subject.delete([1, 2, 3])
     end
@@ -87,7 +101,8 @@ describe Sidekiq::Merger::Batch do
       expect(Sidekiq::Client).to receive(:push).with(
         "class" => worker_class,
         "queue" => queue,
-        "args" => [[1, 2, 3], [2, 3, 4]]
+        "args" => a_collection_containing_exactly([1, 2, 3], [2, 3, 4]),
+        "merged" => true
       )
 
       subject.flush
@@ -95,19 +110,18 @@ describe Sidekiq::Merger::Batch do
   end
 
   describe "#can_flush?" do
-    let(:options) { { flush_interval: 10.seconds } }
-    context "it has not get anything in batch" do
+    context "it has not get anything in merge" do
       it "returns false" do
         expect(subject.can_flush?).to eq false
       end
     end
-    context "it has not passed the interval time" do
+    context "it has not passed the execution time" do
       it "returns false" do
         subject.add([], execution_time)
         expect(subject.can_flush?).to eq false
       end
     end
-    context "it has passed the interval time" do
+    context "it has passed the execution time" do
       it "returns true" do
         subject.add([], execution_time)
         Timecop.travel(10.seconds)
@@ -116,9 +130,9 @@ describe Sidekiq::Merger::Batch do
     end
   end
 
-  describe "#full_batch_key" do
-    it "returns full batch key" do
-      expect(subject.full_batch_key).to eq "name:queue:foo"
+  describe "#full_merge_key" do
+    it "returns full merge key" do
+      expect(subject.full_merge_key).to eq "name:queue:foo"
     end
   end
 end