sidekiq-merger 0.0.9 → 0.0.10

Sign up to get free protection for your applications and to get access to all the features.
Files changed (20) hide show
  1. checksums.yaml +4 -4
  2. data/.codeclimate.yml +3 -0
  3. data/.travis.yml +9 -0
  4. data/Dockerfile +5 -2
  5. data/Gemfile +2 -0
  6. data/README.md +14 -1
  7. data/app/Gemfile.lock +2 -2
  8. data/app/views/index.erb +53 -26
  9. data/docker-compose.yml +2 -1
  10. data/lib/sidekiq/merger.rb +8 -0
  11. data/lib/sidekiq/merger/config.rb +1 -1
  12. data/lib/sidekiq/merger/version.rb +1 -1
  13. data/misc/bulk_notification_flow.png +0 -0
  14. data/misc/cancel_task_flow.png +0 -0
  15. data/spec/sidekiq/merger/merge_spec.rb +12 -27
  16. data/spec/sidekiq/merger/middleware_spec.rb +1 -18
  17. data/spec/sidekiq/merger_spec.rb +16 -1
  18. data/spec/spec_helper.rb +6 -19
  19. data/spec/support/worker_class.rb +34 -0
  20. metadata +7 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 7acc8bcc7c43fe634d4be54f9a4ce5c09135da27
4
- data.tar.gz: ae19824415ac8fe3a38cd876977334f7e41ffaa2
3
+ metadata.gz: c2966632f9d19e7a52b7394eb98d0fe8b114e9ba
4
+ data.tar.gz: 2ade2314686d38e8b93c62be6aee28878ef3eac9
5
5
  SHA512:
6
- metadata.gz: 8fe75f849916a6c70177f68bc9f4326b873d7715e43efdc42db5f26206f7561dd2db4a986c229e32fbe43e40b915ac769db94c6d03b10aaab0a78d27a777e9e5
7
- data.tar.gz: 6a0ef14b3ebac1b36ff959da8d35a28454757b5cbe4ea89efb0799fec10d2c34f4d7e324e67b482c432f24812758f79d640401ad1bec877419abcb2e0dc1cf78
6
+ metadata.gz: b4eb0461a1a436e3a1a5b96afafa36a094e1b6c5a658dd40727766283ec6d6a5f2a8b19d5a93fad528e9ec6ce138bf756b2a80ec2d7f4afe9f17af293c8cff63
7
+ data.tar.gz: bdcc4958eaba9693a28e2a6707280a9e7a563e72b00c339b2e36cbd8549eb43026668ad3248ef8f956beef520403a157102321f9579b548f8020d663334a0419
data/.codeclimate.yml ADDED
@@ -0,0 +1,3 @@
1
+ exclude_paths:
2
+ - "app/"
3
+ - "spec/"
data/.travis.yml CHANGED
@@ -4,6 +4,13 @@ rvm:
4
4
  - 2.2.6
5
5
  - 2.3.3
6
6
  - 2.4.0
7
+ env:
8
+ -
9
+ - SIDEKIQ_VERSION=3.4.0
10
+ - SIDEKIQ_VERSION=3.5.0
11
+ - SIDEKIQ_VERSION=4.0.0
12
+ - SIDEKIQ_VERSION=4.1.0
13
+ - SIDEKIQ_VERSION=4.2.0
7
14
  services:
8
15
  - redis-server
9
16
  before_install:
@@ -12,3 +19,5 @@ before_install:
12
19
  script:
13
20
  - "bundle exec rake spec"
14
21
  - "bundle exec rubocop -D"
22
+ notifications:
23
+ email: false
data/Dockerfile CHANGED
@@ -1,10 +1,13 @@
1
1
  FROM ruby:2.3.3
2
2
  MAINTAINER dtaniwaki
3
3
 
4
- ENV PORT ${PORT:-3000}
4
+ ENV PORT 3000
5
+ ENV REDIS_HOST 127.0.0.1
6
+ ENV REDIS_PORT 6379
7
+
5
8
  RUN gem install bundler
6
9
  ADD . /gem
7
10
  WORKDIR /gem/app
8
11
  RUN bundle install -j4
9
12
 
10
- EXPOSE 3000
13
+ EXPOSE $PORT
data/Gemfile CHANGED
@@ -4,3 +4,5 @@ gemspec
4
4
 
5
5
  gem "gem-release"
6
6
  gem "pry"
7
+
8
+ gem "sidekiq", "~> #{ENV["SIDEKIQ_VERSION"]}" unless ENV["SIDEKIQ_VERSION"].nil?
data/README.md CHANGED
@@ -6,8 +6,20 @@
6
6
  [![Coverage Status][cov-image]][cov-link]
7
7
  [![Code Climate][gpa-image]][gpa-link]
8
8
 
9
+ [![Docker][docker-hub-image]][docker-hub-link]
10
+
9
11
  Merge [sidekiq](http://sidekiq.org/) jobs occurring before the execution times. Inspired by [sidekiq-grouping](https://github.com/gzigzigzeo/sidekiq-grouping).
10
12
 
13
+ ## Use Case
14
+
15
+ ### Cancel Task
16
+
17
+ ![Cancel Task](misc/cancel_task_flow.png)
18
+
19
+ ### Bulk Notification
20
+
21
+ ![Bulk Notification](misc/bulk_notification_flow.png)
22
+
11
23
  ## Installation
12
24
 
13
25
  Add this line to your application's Gemfile:
@@ -133,4 +145,5 @@ Copyright (c) 2017 dtaniwaki. See [LICENSE](LICENSE) for details.
133
145
  [cov-link]: https://coveralls.io/r/dtaniwaki/sidekiq-merger
134
146
  [gpa-image]: https://codeclimate.com/github/dtaniwaki/sidekiq-merger.svg
135
147
  [gpa-link]: https://codeclimate.com/github/dtaniwaki/sidekiq-merger
136
-
148
+ [docker-hub-image]: http://dockeri.co/image/dtaniwaki/sidekiq-merger
149
+ [docker-hub-link]: https://hub.docker.com/r/dtaniwaki/sidekiq-merger/
data/app/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: ../
3
3
  specs:
4
- sidekiq-merger (0.0.6)
4
+ sidekiq-merger (0.0.9)
5
5
  activesupport (>= 3.2, < 6)
6
6
  concurrent-ruby (~> 1.0)
7
7
  sidekiq (>= 3.4, < 5)
@@ -16,7 +16,7 @@ GEM
16
16
  tzinfo (~> 1.1)
17
17
  concurrent-ruby (1.0.4)
18
18
  connection_pool (2.2.1)
19
- i18n (0.7.0)
19
+ i18n (0.8.0)
20
20
  minitest (5.10.1)
21
21
  rack (1.6.5)
22
22
  rack-flash3 (1.0.5)
data/app/views/index.erb CHANGED
@@ -1,35 +1,62 @@
1
- <html>
1
+ <!DOCTYPE html>
2
+ <html lang="en">
2
3
  <head>
3
- <meta charset="UTF-8">
4
+ <meta charset="UTF-8">
5
+ <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css">
4
6
  </head>
5
7
  <body>
6
- <h1>Sidekiq Merger</h1>
7
- <% if flash[:notice] %>
8
- <p style="color: green; font-weight: bold;"><%= flash[:notice] %></p>
9
- <% end %>
10
- <p>
11
- <a href="/sidekiq" target="_blank">Open sidekiq console</a>
12
- </p>
13
- <h2>Workers</h2>
14
- <div style="margin-left: 20px;">
15
- <h3>SomeWorker</h3>
16
- <div>
17
- <form action="/some_worker/perform_in" method="post" style="display: inline-block;">
18
- <input type="submit" name="perform_in" value="perform_in">
19
- </form>
20
- <form action="/some_worker/perform_in" method="post" style="display: inline-block;">
21
- <input type="submit" name="perform_async" value="perform_async">
22
- </form>
8
+ <header class="navbar">
9
+ <div class="container">
10
+ <h1>Sidekiq Merger</h1>
11
+ <a href="https://github.com/dtaniwaki/sidekiq-merger">View Source on GitHub →</a>
23
12
  </div>
24
- <h3>UniqueWorker</h3>
13
+ </header>
14
+ <div class="container">
15
+ <% if flash[:notice] %>
16
+ <div class="alert alert-info">
17
+ <a href="#" class="close" data-dismiss="alert">&times;</a>
18
+ <p><%= flash[:notice] %></p>
19
+ </div>
20
+ <% end %>
21
+ <p class="lead">
22
+ Click the `perform_in` buttons to create or merge tasks until the execution time (in 60s).<br>
23
+ Click the `perform_async` buttons to execute a single task.<br><br>
24
+ Open <a href="/sidekiq/merges?poll=true" target="_blank">sidekiq console</a> to check what happens.
25
+ </p>
25
26
  <div>
26
- <form action="/unique_worker/perform_in" method="post" style="display: inline-block;">
27
- <input type="submit" name="perform_in" value="perform_in">
28
- </form>
29
- <form action="/unique_worker/perform_in" method="post" style="display: inline-block;">
30
- <input type="submit" name="perform_async" value="perform_async">
31
- </form>
27
+ <h2>SomeWorker</h2>
28
+ <p>
29
+ <code>sidekiq_options merger: { unique: false }</code>
30
+ </p>
31
+ <p class="lead">
32
+ <small>Tasks will be merged regardless of uniqueness.</small>
33
+ </p>
34
+ <div>
35
+ <form action="/some_worker/perform_in" method="post" style="display: inline-block;">
36
+ <input type="submit" name="perform_in" value="perform_in" class="btn btn-primary">
37
+ </form>
38
+ <form action="/some_worker/perform_in" method="post" style="display: inline-block;">
39
+ <input type="submit" name="perform_async" value="perform_async" class="btn btn-default">
40
+ </form>
41
+ </div>
42
+ <h2>UniqueWorker</h2>
43
+ <p>
44
+ <code>sidekiq_options merger: { unique: true }</code>
45
+ </p>
46
+ <p class="lead">
47
+ <small>Tasks will be merged if they haven't added already.</small>
48
+ </p>
49
+ <div>
50
+ <form action="/unique_worker/perform_in" method="post" style="display: inline-block;">
51
+ <input type="submit" name="perform_in" value="perform_in" class="btn btn-primary">
52
+ </form>
53
+ <form action="/unique_worker/perform_in" method="post" style="display: inline-block;">
54
+ <input type="submit" name="perform_async" value="perform_async" class="btn btn-default">
55
+ </form>
56
+ </div>
32
57
  </div>
33
58
  </div>
59
+ <script type="text/javascript" src="https://code.jquery.com/jquery-3.1.1.min.js"></script>
60
+ <script type="text/javascript" src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js"></script>
34
61
  </body>
35
62
  </html>
data/docker-compose.yml CHANGED
@@ -20,7 +20,8 @@ services:
20
20
  links:
21
21
  - redis
22
22
  environment:
23
+ - PORT=3000
23
24
  - REDIS_HOST=redis
24
25
  - REDIS_PORT=6379
25
26
  redis:
26
- image: redis:latest
27
+ image: redis:3.2.7
data/lib/sidekiq/merger.rb CHANGED
@@ -22,6 +22,14 @@ module Sidekiq::Merger
22
22
  task.add_observer(observer)
23
23
  task
24
24
  end
25
+
26
+ def configure(&block)
27
+ yield config
28
+ end
29
+
30
+ def config
31
+ @config ||= Config.new
32
+ end
25
33
  end
26
34
 
27
35
  self.logger = Sidekiq.logger
data/lib/sidekiq/merger/config.rb CHANGED
@@ -1,6 +1,6 @@
1
1
  require "active_support/configurable"
2
2
 
3
- module Sidekiq::Merger::Config
3
+ class Sidekiq::Merger::Config
4
4
  include ActiveSupport::Configurable
5
5
 
6
6
  def self.options
data/lib/sidekiq/merger/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  module Sidekiq
2
2
  module Merger
3
- VERSION = "0.0.9".freeze
3
+ VERSION = "0.0.10".freeze
4
4
  end
5
5
  end
data/misc/bulk_notification_flow.png ADDED
Binary file
data/misc/cancel_task_flow.png ADDED
Binary file
data/spec/sidekiq/merger/merge_spec.rb CHANGED
@@ -1,28 +1,13 @@
1
1
  require "spec_helper"
2
2
 
3
- describe Sidekiq::Merger::Merge do
3
+ describe Sidekiq::Merger::Merge, worker_class: true do
4
4
  subject { described_class.new(worker_class, queue, args, redis: redis) }
5
5
  let(:args) { "foo" }
6
6
  let(:redis) { Sidekiq::Merger::Redis.new }
7
7
  let(:queue) { "queue" }
8
8
  let(:now) { Time.now }
9
9
  let(:execution_time) { now + 10.seconds }
10
- let(:options) { { key: -> (args) { args.to_json } } }
11
- let(:worker_class) do
12
- local_options = options
13
- Class.new do
14
- include Sidekiq::Worker
15
-
16
- sidekiq_options merger: local_options
17
-
18
- def self.name
19
- "name"
20
- end
21
-
22
- def perform(args)
23
- end
24
- end
25
- end
10
+ let(:worker_options) { { key: -> (args) { args.to_json } } }
26
11
  before { Timecop.freeze(now) }
27
12
 
28
13
  describe ".all" do
@@ -64,30 +49,30 @@ describe Sidekiq::Merger::Merge do
64
49
 
65
50
  describe ".merge_key" do
66
51
  let(:args) { "foo" }
67
- let(:options) { {} }
52
+ let(:worker_options) { {} }
68
53
  it "returns an empty string" do
69
54
  expect(described_class.merge_key(worker_class, args)).to eq ""
70
55
  end
71
56
  context "string key" do
72
- let(:options) { { key: "bar" } }
57
+ let(:worker_options) { { key: "bar" } }
73
58
  it "returns the string" do
74
59
  expect(described_class.merge_key(worker_class, args)).to eq "bar"
75
60
  end
76
61
  end
77
62
  context "other type key" do
78
- let(:options) { { key: [1, 2, 3] } }
63
+ let(:worker_options) { { key: [1, 2, 3] } }
79
64
  it "returns nil" do
80
65
  expect(described_class.merge_key(worker_class, args)).to eq "[1,2,3]"
81
66
  end
82
67
  end
83
68
  context "proc key" do
84
69
  let(:args) { [1, 2, 3] }
85
- let(:options) { { key: -> (args) { args[0].to_s } } }
70
+ let(:worker_options) { { key: -> (args) { args[0].to_s } } }
86
71
  it "returns the result of the proc" do
87
72
  expect(described_class.merge_key(worker_class, args)).to eq "1"
88
73
  end
89
74
  context "non-string result" do
90
- let(:options) { { key: -> (args) { args[0] } } }
75
+ let(:worker_options) { { key: -> (args) { args[0] } } }
91
76
  it "returns nil" do
92
77
  expect(described_class.merge_key(worker_class, args)).to eq "1"
93
78
  end
@@ -97,13 +82,13 @@ describe Sidekiq::Merger::Merge do
97
82
 
98
83
  describe "#add" do
99
84
  it "adds the args in lazy merge" do
100
- expect(redis).to receive(:push_message).with("name:queue:foo", [1, 2, 3], execution_time)
85
+ expect(redis).to receive(:push_message).with("some_worker:queue:foo", [1, 2, 3], execution_time)
101
86
  subject.add([1, 2, 3], execution_time)
102
87
  end
103
88
  context "with unique option" do
104
- let(:options) { { key: -> (args) { args.to_json }, unique: true } }
89
+ let(:worker_options) { { key: -> (args) { args.to_json }, unique: true } }
105
90
  it "adds the args in lazy merge" do
106
- expect(redis).to receive(:push_message).with("name:queue:foo", [1, 2, 3], execution_time)
91
+ expect(redis).to receive(:push_message).with("some_worker:queue:foo", [1, 2, 3], execution_time)
107
92
  subject.add([1, 2, 3], execution_time)
108
93
  end
109
94
  context "the args has alredy been added" do
@@ -118,7 +103,7 @@ describe Sidekiq::Merger::Merge do
118
103
 
119
104
  describe "#delete" do
120
105
  it "adds the args in lazy merge" do
121
- expect(redis).to receive(:delete_message).with("name:queue:foo", [1, 2, 3])
106
+ expect(redis).to receive(:delete_message).with("some_worker:queue:foo", [1, 2, 3])
122
107
  subject.delete([1, 2, 3])
123
108
  end
124
109
  end
@@ -195,7 +180,7 @@ describe Sidekiq::Merger::Merge do
195
180
 
196
181
  describe "#full_merge_key" do
197
182
  it "returns full merge key" do
198
- expect(subject.full_merge_key).to eq "name:queue:foo"
183
+ expect(subject.full_merge_key).to eq "some_worker:queue:foo"
199
184
  end
200
185
  end
201
186
  end
data/spec/sidekiq/merger/middleware_spec.rb CHANGED
@@ -1,28 +1,11 @@
1
1
  require "spec_helper"
2
2
 
3
- describe Sidekiq::Merger::Middleware do
3
+ describe Sidekiq::Merger::Middleware, worker_class: true do
4
4
  subject { described_class.new }
5
5
  let(:flusher) { Sidekiq::Merger::Flusher.new(Sidekiq.logger) }
6
6
  let(:queue) { "queue" }
7
7
  let(:now) { Time.now }
8
- let(:options) { { key: -> (args) { "key" } } }
9
- let(:worker_class) do
10
- local_options = options
11
- Class.new do
12
- include Sidekiq::Worker
13
-
14
- sidekiq_options merger: local_options
15
-
16
- def self.name
17
- "name"
18
- end
19
-
20
- def perform(*args)
21
- end
22
- end
23
- end
24
8
  before :example do
25
- allow(Object).to receive(:const_get).with("Name").and_return worker_class
26
9
  Timecop.freeze(now)
27
10
  end
28
11
 
data/spec/sidekiq/merger_spec.rb CHANGED
@@ -4,11 +4,26 @@ describe Sidekiq::Merger do
4
4
  it "has a version number" do
5
5
  expect(described_class::VERSION).not_to be nil
6
6
  end
7
- describe "#create_task" do
7
+ describe ".create_task" do
8
8
  it "starts a monitoring task" do
9
9
  task = described_class.create_task
10
10
  expect(task).to be_a Concurrent::TimerTask
11
11
  task.shutdown
12
12
  end
13
13
  end
14
+ describe ".configure" do
15
+ it "yields to the config" do
16
+ expect { |b| described_class.configure(&b) }.to yield_with_args(described_class.config)
17
+ end
18
+ end
19
+ describe ".config" do
20
+ it "returns a config" do
21
+ expect(described_class.config).to be_a Sidekiq::Merger::Config
22
+ end
23
+ context "called twice" do
24
+ it "returns the same config instance" do
25
+ expect(described_class.config).to be described_class.config
26
+ end
27
+ end
28
+ end
14
29
  end
data/spec/spec_helper.rb CHANGED
@@ -21,31 +21,15 @@ require "sidekiq/merger"
21
21
  Dir[File.join(__dir__, "support", "**", "*.rb")].each { |f| require f }
22
22
 
23
23
  RSpec.configure do |config|
24
- # rspec-expectations config goes here. You can use an alternate
25
- # assertion/expectation library such as wrong or the stdlib/minitest
26
- # assertions if you prefer.
27
24
  config.expect_with :rspec do |expectations|
28
- # This option will default to `true` in RSpec 4. It makes the `description`
29
- # and `failure_message` of custom matchers include text for helper methods
30
- # defined using `chain`, e.g.:
31
- # be_bigger_than(2).and_smaller_than(4).description
32
- # # => "be bigger than 2 and smaller than 4"
33
- # ...rather than:
34
- # # => "be bigger than 2"
35
25
  expectations.include_chain_clauses_in_custom_matcher_descriptions = true
36
26
  end
37
27
 
38
28
  config.mock_with :rspec do |mocks|
39
- # Prevents you from mocking or stubbing a method that does not exist on
40
- # a real object. This is generally recommended, and will default to
41
- # `true` in RSpec 4.
42
29
  mocks.verify_partial_doubles = true
43
30
  end
44
31
 
45
32
  if config.files_to_run.one?
46
- # Use the documentation formatter for detailed output,
47
- # unless a formatter has already been configured
48
- # (e.g. via a command-line flag).
49
33
  config.default_formatter = "doc"
50
34
  end
51
35
 
@@ -59,9 +43,12 @@ RSpec.configure do |config|
59
43
  Sidekiq.logger = nil
60
44
  end
61
45
 
62
- config.before :example do
63
- Sidekiq::Merger::Redis.redis do |conn|
64
- conn.flushall
46
+ config.around :example do |example|
47
+ Sidekiq::Merger::Redis.redis { |conn| conn.flushall }
48
+ begin
49
+ example.run
50
+ ensure
51
+ Sidekiq::Merger::Redis.redis { |conn| conn.flushall }
65
52
  end
66
53
  end
67
54
 
data/spec/support/worker_class.rb ADDED
@@ -0,0 +1,34 @@
1
+ RSpec.shared_context "worker class", worker_class: true do
2
+ let(:worker_options) { { key: -> (args) { "key" } } }
3
+ let(:worker_class) do
4
+ local_options = worker_options
5
+ Class.new do
6
+ include Sidekiq::Worker
7
+
8
+ sidekiq_options merger: local_options
9
+
10
+ def self.name
11
+ "SomeWorker"
12
+ end
13
+
14
+ def self.to_s
15
+ "SomeWorker"
16
+ end
17
+
18
+ def perform(*args)
19
+ end
20
+ end
21
+ end
22
+ before :example do
23
+ allow(Object).to receive(:const_get).with(anything).and_call_original
24
+ allow(Object).to receive(:const_get).with("SomeWorker").and_return worker_class
25
+ end
26
+ around :example do |example|
27
+ worker_class.jobs.clear
28
+ begin
29
+ example.run
30
+ ensure
31
+ worker_class.jobs.clear
32
+ end
33
+ end
34
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sidekiq-merger
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.9
4
+ version: 0.0.10
5
5
  platform: ruby
6
6
  authors:
7
7
  - dtaniwaki
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-02-01 00:00:00.000000000 Z
11
+ date: 2017-02-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -167,6 +167,7 @@ executables: []
167
167
  extensions: []
168
168
  extra_rdoc_files: []
169
169
  files:
170
+ - ".codeclimate.yml"
170
171
  - ".dockerignore"
171
172
  - ".gemrelease"
172
173
  - ".gitignore"
@@ -199,6 +200,8 @@ files:
199
200
  - lib/sidekiq/merger/version.rb
200
201
  - lib/sidekiq/merger/views/index.erb
201
202
  - lib/sidekiq/merger/web.rb
203
+ - misc/bulk_notification_flow.png
204
+ - misc/cancel_task_flow.png
202
205
  - misc/web_ui.png
203
206
  - sidekiq-merger.gemspec
204
207
  - spec/sidekiq/merger/flusher_spec.rb
@@ -209,6 +212,7 @@ files:
209
212
  - spec/sidekiq/merger_spec.rb
210
213
  - spec/spec_helper.rb
211
214
  - spec/support/matchers.rb
215
+ - spec/support/worker_class.rb
212
216
  homepage: https://github.com/dtaniwaki/sidekiq-merger
213
217
  licenses:
214
218
  - MIT
@@ -245,3 +249,4 @@ test_files:
245
249
  - spec/sidekiq/merger_spec.rb
246
250
  - spec/spec_helper.rb
247
251
  - spec/support/matchers.rb
252
+ - spec/support/worker_class.rb