sidekiq-merger 0.0.1 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 668c85fb3075147efdfbfacd83b1f6cad5766249
-  data.tar.gz: ef35056d67ce049cf7b0c3c328cd04f91b01a77c
+  metadata.gz: 4bed498597132701a71bf77a583f97668ce5264c
+  data.tar.gz: 66244be154087fca1647ba21d7b12727167d5106
 SHA512:
-  metadata.gz: 031c84fa7cbf1cebe5e21244a1e7bfa85aa3d3c0a496652b825490aa2a24716cbf4bde327d4ef3e19bd099a403ef7104aa7c2d23e6d60a680ad908b7ea875628
-  data.tar.gz: ab858f4ad9cfafe67f7a837f42df0770b329d14b5a756d2d3c0e0109e6bb10aac23694a24522cead863ea09430a8a3ad7a2e195bf23b90ec5fb0c1e0f5daac05
+  metadata.gz: 01887441761e31726a0c4cb6feff20e55be5202c3fa9c3d00a43ff28b54589c484ddd95f305ac0c589972e27c05c4ae46201ca408af32ee0904aae2b80827e16
+  data.tar.gz: 762fca326555feada8466a1eeda5acf2189a8f9e1b2b979e2568b90ccddbc5371d840833f106640889f1f69a744ce79bfa29ee932cf1f905370586375a74830b

data/.dockerignore

@@ -0,0 +1,21 @@
+.bundle/
+.yardoc
+Gemfile.lock
+_yardoc/
+coverage/
+doc/
+pkg/
+spec/reports/
+tmp/
+rdoc/
+
+*.gem
+*.rbc
+
+.ruby-version
+.ruby-gemset
+.rvmrc
+
+.env
+
+.dockerignore

data/.gemrelease

@@ -0,0 +1,2 @@
+bump:
+  tag: true

data/.gitignore

@@ -1,9 +1,19 @@
-/.bundle/
-/.yardoc
-/Gemfile.lock
-/_yardoc/
-/coverage/
-/doc/
-/pkg/
-/spec/reports/
-/tmp/
+.bundle/
+.yardoc
+Gemfile.lock
+_yardoc/
+coverage/
+doc/
+pkg/
+spec/reports/
+tmp/
+rdoc/
+
+*.gem
+*.rbc
+
+.ruby-version
+.ruby-gemset
+.rvmrc
+
+.env

data/.travis.yml

@@ -4,7 +4,11 @@ rvm:
   - 2.2.6
   - 2.3.3
   - 2.4.0
-before_install: gem install bundler -v 1.13.6
+services:
+  - redis-server
+before_install:
+  - gem update --system
+  - gem --version
 script:
   - "bundle exec rake spec"
   - "bundle exec rubocop -D"

data/Dockerfile

@@ -0,0 +1,10 @@
+FROM ruby:2.3.3
+MAINTAINER dtaniwaki
+
+ENV PORT ${PORT:-3000}
+RUN gem install bundler
+ADD . /gem
+WORKDIR /gem/app
+RUN bundle install -j4
+
+EXPOSE 3000

data/Gemfile

@@ -1,4 +1,6 @@
 source "https://rubygems.org"
 
-# Specify your gem's dependencies in sidekiq-lazy-batch.gemspec
 gemspec
+
+gem "gem-release"
+gem "pry"

data/README.md

@@ -6,7 +6,7 @@
 [![Coverage Status][cov-image]][cov-link]
 [![Code Climate][gpa-image]][gpa-link]
 
-Merge sidekiq jobs occurring within specific period.
+Merge sidekiq jobs occurring within specific period. This sidekiq middleware is inspired by [sidekiq-grouping](https://github.com/gzigzigzeo/sidekiq-grouping).
 
 ## Installation
 
@@ -18,28 +18,90 @@ gem 'sidekiq-merger'
 
 And then execute:
 
-    $ bundle
+  $ bundle
 
 Or install it yourself as:
 
-    $ gem install sidekiq-merger
+  $ gem install sidekiq-merger
 
 ## Usage
 
 Add merger option into your workers.
 
-```
-class YourWorker
+```ruby
+class SomeWorker
   include Sidekiq::Worker
 
   sidekiq_options merger: { key: -> (args) { args[0] } }
 
-  def perform(all_args)
+  def perform(*ids)
     # Do something
   end
 end
 ```
 
+Then, enqueue jobs by `perform_in` or `perform_at`.
+
+```ruby
+SomeWorker.perform_in 100, 4
+SomeWorker.perform_in 100, 3
+SomeWorker.perform_in 100, 5
+```
+
+`SomeWorker` will be executed in 100 seconds with args of `[4], [3], [5]`.
+
+`perform_async` works without merging args.
+
+```ruby
+SomeWorker.perform_async 4
+SomeWorker.perform_async 3
+SomeWorker.perform_async 5
+```
+
+In this case, `SomeWorker` will be executed 3 times with args of `[4]`, `[3]` and `[5]`.
+
+## Options
+
+### `key` (optional, default: `nil`)
+
+Defines merge key so different arguments can be merged.
+
+Format: `String` or `Proc`
+e.g. `sidekiq_options merger: { key: -> (args) { args[0..1] } }`
+
+### `unique` (optional, default: `false`)
+
+Prevents enqueue of jobs with identical arguments.
+
+Format: `Boolean`
+e.g. `true`
+
+## Web UI
+
+![Web UI](misc/web_ui.png)
+
+Add this line to your `config/routes.rb` to activate web UI:
+
+```ruby
+require "sidekiq/merger/web"
+```
+
+## Test
+
+  $ bundle exec rspec
+
+The test coverage is available at `./coverage/index.html`.
+
+To check the behavior of this plugin, you can run docker containers.
+
+  $ docker-compose up
+
+Then, open `http://localhost:3000/`.
+
+## Lint
+
+  $ bundle exec rubocop
+
 ## Contributing
 
 1. Fork it

data/app/Gemfile

@@ -0,0 +1,8 @@
+source "https://rubygems.org"
+
+gem "rack-flash3"
+gem "sinatra"
+gem "sidekiq"
+gem "sidekiq-status"
+
+gem "sidekiq-merger", path: "../"

data/app/app.rb

@@ -0,0 +1,43 @@
+require_relative "./sidekiq"
+require "sinatra/base"
+require "rack/flash"
+require "sidekiq/web"
+require "sidekiq-status/web"
+require "sidekiq/merger/web"
+
+class App < Sinatra::Application
+  enable :sessions
+  use Rack::Flash
+
+  get "/" do
+    erb :index
+  end
+
+  post "/some_worker/perform_in" do
+    n = rand(10)
+    SomeWorker.perform_in((params[:in] || 60).to_i, n)
+    flash[:notice] = "Added #{n} to SomeWorker"
+    redirect "/"
+  end
+
+  post "/some_worker/perform_async" do
+    n = rand(10)
+    SomeWorker.perform_async(n)
+    flash[:notice] = "Added #{n} to SomeWorker"
+    redirect "/"
+  end
+
+  post "/unique_worker/perform_in" do
+    n = rand(10)
+    UniqueWorker.perform_in((params[:in] || 60).to_i, n)
+    flash[:notice] = "Added #{n} to UniqueWorker"
+    redirect "/"
+  end
+
+  post "/unique_worker/perform_async" do
+    n = rand(10)
+    UniqueWorker.perform_async(n)
+    flash[:notice] = "Added #{n} to UniqueWorker"
+    redirect "/"
+  end
+end

data/app/config.ru

@@ -0,0 +1,3 @@
+require "./app"
+
+run Rack::URLMap.new("/" => App, "/sidekiq" => Sidekiq::Web)

data/app/sidekiq.rb

@@ -0,0 +1,25 @@
+require "active_support/core_ext/numeric/time"
+require "sidekiq"
+require "sidekiq-status"
+require "sidekiq-merger"
+require_relative "./some_worker"
+require_relative "./unique_worker"
+
+expiration = 30.minutes
+
+Sidekiq.configure_client do |config|
+  config.redis = { url: "redis://#{ENV["REDIS_HOST"]}:#{ENV["REDIS_PORT"]}" }
+  config.client_middleware do |chain|
+    chain.add Sidekiq::Status::ClientMiddleware, expiration: expiration
+  end
+end
+
+Sidekiq.configure_server do |config|
+  config.redis = { url: "redis://#{ENV["REDIS_HOST"]}:#{ENV["REDIS_PORT"]}" }
+  config.server_middleware do |chain|
+    chain.add Sidekiq::Status::ServerMiddleware, expiration: expiration
+  end
+  config.client_middleware do |chain|
+    chain.add Sidekiq::Status::ClientMiddleware, expiration: expiration
+  end
+end

data/app/some_worker.rb

@@ -0,0 +1,9 @@
+class SomeWorker
+  include Sidekiq::Worker
+
+  sidekiq_options merger: { key: "foo" }
+
+  def perform(*ids)
+    puts "Get IDs: #{ids.inspect}"
+  end
+end

data/app/unique_worker.rb

@@ -0,0 +1,9 @@
+class UniqueWorker
+  include Sidekiq::Worker
+
+  sidekiq_options merger: { key: "foo", unique: true }
+
+  def perform(*ids)
+    puts "Get IDs: #{ids.inspect}"
+  end
+end

data/app/views/index.erb

@@ -0,0 +1,35 @@
+<html>
+<head>
+<meta charset="UTF-8">
+</head>
+<body>
+  <h1>Sidekiq Merger</h1>
+  <% if flash[:notice] %>
+    <p style="color: green; font-weight: bold;"><%= flash[:notice] %></p>
+  <% end %>
+  <p>
+    <a href="/sidekiq" target="_blank">Open sidekiq console</a>
+  </p>
+  <h2>Workers</h2>
+  <div style="margin-left: 20px;">
+    <h3>SomeWorker</h3>
+    <div>
+      <form action="/some_worker/perform_in" method="post" style="display: inline-block;">
+        <input type="submit" name="perform_in" value="perform_in">
+      </form>
+      <form action="/some_worker/perform_in" method="post" style="display: inline-block;">
+        <input type="submit" name="perform_async" value="perform_async">
+      </form>
+    </div>
+    <h3>UniqueWorker</h3>
+    <div>
+      <form action="/some_worker/perform_in" method="post" style="display: inline-block;">
+        <input type="submit" name="perform_in" value="perform_in">
+      </form>
+      <form action="/some_worker/perform_in" method="post" style="display: inline-block;">
+        <input type="submit" name="perform_async" value="perform_async">
+      </form>
+    </div>
+  </div>
+</body>
+</html>

data/docker-compose-common.yml

@@ -0,0 +1,7 @@
+version: '2.1'
+services:
+  app:
+    build: .
+    environment:
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379

data/docker-compose.yml

@@ -0,0 +1,26 @@
+version: '2.1'
+services:
+  worker:
+    extends:
+      file: 'docker-compose-common.yml'
+      service: app
+    command: bundle exec sidekiq -r ./sidekiq.rb
+    links:
+      - redis
+    environment:
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
+  web:
+    extends:
+      file: 'docker-compose-common.yml'
+      service: app
+    command: bundle exec rackup -p 3000 --host 0.0.0.0
+    ports:
+      - 3000:3000
+    links:
+      - redis
+    environment:
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
+  redis:
+    image: redis:latest

data/lib/sidekiq/merger.rb

@@ -9,25 +9,21 @@ require_relative "merger/logging_observer"
 module Sidekiq::Merger
   class << self
     attr_accessor :logger
-  end
-
-  self.logger ||= Sidekiq.logger
 
-  def logger
-    self.class.logger
+    def start!
+      interval = Sidekiq::Merger::Config.poll_interval
+      observer = Sidekiq::Merger::LoggingObserver.new(logger)
+      flusher = Sidekiq::Merger::Flusher.new(logger)
+      task = Concurrent::TimerTask.new(
+        execution_interval: interval
+      ) { flusher.flush }
+      task.add_observer(observer)
+      logger.info(
+        "[Sidekiq::Merger] Started polling merges every #{interval} seconds"
+      )
+      task.execute
+    end
   end
 
-  def start!
-    interval = Sidekiq::Merger::Config.poll_interval
-    observer = Sidekiq::Merger::LoggingObserver.new(logger)
-    flusher = Sidekiq::Merger::Flusher.new(logger)
-    task = Concurrent::TimerTask.new(
-      execution_interval: interval
-    ) { flusher.flush }
-    task.add_observer(observer)
-    logger.info(
-      "[Sidekiq::Merger] Started polling batches every #{interval} seconds"
-    )
-    task.execute
-  end
+  self.logger = Sidekiq.logger
 end

data/lib/sidekiq/merger/flusher.rb

@@ -4,12 +4,12 @@ class Sidekiq::Merger::Flusher
   end
 
   def flush
-    batches = Sidekiq::Merger::Batch.all.select(&:can_flush?)
-    unless batches.empty?
+    merges = Sidekiq::Merger::Merge.all.select(&:can_flush?)
+    unless merges.empty?
       @logger.info(
-        "[Sidekiq::Merger] Trying to flush batched queues: #{batches.map(&:full_batch_key).join(",")}"
+        "[Sidekiq::Merger] Trying to flush merged queues: #{merges.map(&:full_merge_key).join(",")}"
       )
-      batches.each(&:flush)
+      merges.each(&:flush)
     end
   end
 end