sidekiq-merger 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 668c85fb3075147efdfbfacd83b1f6cad5766249
4
+ data.tar.gz: ef35056d67ce049cf7b0c3c328cd04f91b01a77c
5
+ SHA512:
6
+ metadata.gz: 031c84fa7cbf1cebe5e21244a1e7bfa85aa3d3c0a496652b825490aa2a24716cbf4bde327d4ef3e19bd099a403ef7104aa7c2d23e6d60a680ad908b7ea875628
7
+ data.tar.gz: ab858f4ad9cfafe67f7a837f42df0770b329d14b5a756d2d3c0e0109e6bb10aac23694a24522cead863ea09430a8a3ad7a2e195bf23b90ec5fb0c1e0f5daac05
data/.gitignore ADDED
@@ -0,0 +1,9 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --format documentation
2
+ --color
data/.rubocop.yml ADDED
@@ -0,0 +1,90 @@
1
+ AllCops:
2
+ DisabledByDefault: true
3
+
4
+ Style/StringLiterals:
5
+ Enabled: true
6
+ EnforcedStyle: double_quotes
7
+
8
+ Style/StringLiteralsInInterpolation:
9
+ Enabled: true
10
+ EnforcedStyle: double_quotes
11
+
12
+ Style/SpaceBeforeBlockBraces:
13
+ Enabled: true
14
+ EnforcedStyle: 'space'
15
+
16
+ Style/SpaceInsideBrackets:
17
+ Enabled: true
18
+
19
+ Style/SpaceInsideHashLiteralBraces:
20
+ Enabled: true
21
+
22
+ Style/SpaceInsideBlockBraces:
23
+ Enabled: true
24
+
25
+ Style/SpaceAroundEqualsInParameterDefault:
26
+ Enabled: true
27
+
28
+ Style/SpaceBeforeComma:
29
+ Enabled: false
30
+
31
+ Style/SpaceAroundOperators:
32
+ Enabled: true
33
+
34
+ Style/SpaceAfterComma:
35
+ Enabled: true
36
+
37
+ Style/ExtraSpacing:
38
+ Enabled: true
39
+ AllowForAlignment: true
40
+
41
+ Lint/DuplicateMethods:
42
+ Enabled: true
43
+
44
+ Metrics/LineLength:
45
+ Enabled: true
46
+ Max: 200
47
+ AllowHeredoc: true
48
+ AllowURI: true
49
+ URISchemes: http, https
50
+
51
+ Metrics/MethodLength:
52
+ Enabled: true
53
+ Max: 25
54
+
55
+ Metrics/ClassLength:
56
+ Enabled: true
57
+ Max: 160
58
+
59
+ Metrics/ModuleLength:
60
+ Enabled: true
61
+ Max: 160
62
+
63
+ Style/ClassAndModuleChildren:
64
+ EnforcedStyle: compact
65
+ Exclude:
66
+ - lib/sidekiq/merger/version.rb
67
+
68
+ Rails/PluralizationGrammar:
69
+ Enabled: true
70
+
71
+ Style/AlignArray:
72
+ Enabled: true
73
+
74
+ Style/AlignHash:
75
+ Enabled: true
76
+
77
+ Style/BlockEndNewline:
78
+ Enabled: false
79
+
80
+ Style/DoubleNegation:
81
+ Enabled: false
82
+
83
+ Metrics/AbcSize:
84
+ Max: 60
85
+
86
+ Metrics/CyclomaticComplexity:
87
+ Max: 12
88
+
89
+ Metrics/PerceivedComplexity:
90
+ Max: 12
data/.travis.yml ADDED
@@ -0,0 +1,10 @@
1
+ sudo: false
2
+ language: ruby
3
+ rvm:
4
+ - 2.2.6
5
+ - 2.3.3
6
+ - 2.4.0
7
+ before_install: gem install bundler -v 1.13.6
8
+ script:
9
+ - "bundle exec rake spec"
10
+ - "bundle exec rubocop -D"
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source "https://rubygems.org"
2
+
3
+ # Specify your gem's dependencies in sidekiq-lazy-batch.gemspec
4
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,20 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2017 dtaniwaki
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy of
6
+ this software and associated documentation files (the "Software"), to deal in
7
+ the Software without restriction, including without limitation the rights to
8
+ use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
9
+ the Software, and to permit persons to whom the Software is furnished to do so,
10
+ subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
17
+ FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
18
+ COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
19
+ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
20
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,65 @@
1
+ # sidekiq-merger
2
+
3
+ [![Gem Version][gem-image]][gem-link]
4
+ [![Dependency Status][deps-image]][deps-link]
5
+ [![Build Status][build-image]][build-link]
6
+ [![Coverage Status][cov-image]][cov-link]
7
+ [![Code Climate][gpa-image]][gpa-link]
8
+
9
+ Merge sidekiq jobs occurring within specific period.
10
+
11
+ ## Installation
12
+
13
+ Add this line to your application's Gemfile:
14
+
15
+ ```ruby
16
+ gem 'sidekiq-merger'
17
+ ```
18
+
19
+ And then execute:
20
+
21
+ $ bundle
22
+
23
+ Or install it yourself as:
24
+
25
+ $ gem install sidekiq-merger
26
+
27
+ ## Usage
28
+
29
+ Add merger option into your workers.
30
+
31
+ ```
32
+ class YourWorker
33
+ include Sidekiq::Worker
34
+
35
+ sidekiq_options merger: { key: -> (args) { args[0] } }
36
+
37
+ def perform(all_args)
38
+ # Do something
39
+ end
40
+ end
41
+ ```
42
+
43
+ ## Contributing
44
+
45
+ 1. Fork it
46
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
47
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
48
+ 4. Push to the branch (`git push origin my-new-feature`)
49
+ 5. Create new [Pull Request](../../pull/new/master)
50
+
51
+ ## Copyright
52
+
53
+ Copyright (c) 2017 dtaniwaki. See [LICENSE](LICENSE) for details.
54
+
55
+ [gem-image]: https://badge.fury.io/rb/sidekiq-merger.svg
56
+ [gem-link]: http://badge.fury.io/rb/sidekiq-merger
57
+ [build-image]: https://secure.travis-ci.org/dtaniwaki/sidekiq-merger.svg
58
+ [build-link]: http://travis-ci.org/dtaniwaki/sidekiq-merger
59
+ [deps-image]: https://gemnasium.com/dtaniwaki/sidekiq-merger.svg
60
+ [deps-link]: https://gemnasium.com/dtaniwaki/sidekiq-merger
61
+ [cov-image]: https://coveralls.io/repos/dtaniwaki/sidekiq-merger/badge.png
62
+ [cov-link]: https://coveralls.io/r/dtaniwaki/sidekiq-merger
63
+ [gpa-image]: https://codeclimate.com/github/dtaniwaki/sidekiq-merger.svg
64
+ [gpa-link]: https://codeclimate.com/github/dtaniwaki/sidekiq-merger
65
+
data/Rakefile ADDED
@@ -0,0 +1,6 @@
1
+ require "bundler/gem_tasks"
2
+ require "rspec/core/rake_task"
3
+
4
+ RSpec::Core::RakeTask.new(:spec)
5
+
6
+ task default: :spec
data/bin/console ADDED
@@ -0,0 +1,7 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "sidekiq-merger"
5
+
6
+ require "pry"
7
+ Pry.start
data/bin/setup ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,15 @@
1
+ require_relative "sidekiq/merger"
2
+
3
+ Sidekiq.configure_client do |config|
4
+ config.client_middleware do |chain|
5
+ chain.add Sidekiq::Merger::Middleware
6
+ end
7
+ end
8
+
9
+ Sidekiq.configure_server do |config|
10
+ config.client_middleware do |chain|
11
+ chain.add Sidekiq::Merger::Middleware
12
+ end
13
+ end
14
+
15
+ Sidekiq::Merger.start! if Sidekiq.server?
@@ -0,0 +1,33 @@
1
+ require "sidekiq"
2
+ require "concurrent"
3
+ require_relative "merger/version"
4
+ require_relative "merger/middleware"
5
+ require_relative "merger/config"
6
+ require_relative "merger/flusher"
7
+ require_relative "merger/logging_observer"
8
+
9
+ module Sidekiq::Merger
10
+ class << self
11
+ attr_accessor :logger
12
+ end
13
+
14
+ self.logger ||= Sidekiq.logger
15
+
16
+ def logger
17
+ self.class.logger
18
+ end
19
+
20
+ def start!
21
+ interval = Sidekiq::Merger::Config.poll_interval
22
+ observer = Sidekiq::Merger::LoggingObserver.new(logger)
23
+ flusher = Sidekiq::Merger::Flusher.new(logger)
24
+ task = Concurrent::TimerTask.new(
25
+ execution_interval: interval
26
+ ) { flusher.flush }
27
+ task.add_observer(observer)
28
+ logger.info(
29
+ "[Sidekiq::Merger] Started polling batches every #{interval} seconds"
30
+ )
31
+ task.execute
32
+ end
33
+ end
@@ -0,0 +1,100 @@
1
+ require_relative "redis"
2
+ require "active_support/core_ext/hash/indifferent_access"
3
+
4
+ class Sidekiq::Merger::Batch
5
+ class << self
6
+ def all
7
+ redis = Sidekiq::Merger::Redis.new
8
+
9
+ redis.all.map do |full_batch_key|
10
+ keys = full_batch_key.split(":")
11
+ raise "Invalid batch key" if keys.size < 3
12
+ worker_class = keys[0].camelize.constantize
13
+ queue = keys[1]
14
+ batch_key = keys[2]
15
+ new(worker_class, queue, batch_key, redis: redis)
16
+ end
17
+ end
18
+
19
+ def initialize_with_args(worker_class, queue, args, options = {})
20
+ new(worker_class, queue, batch_key(worker_class, args), options)
21
+ end
22
+
23
+ def batch_key(worker_class, args)
24
+ options = get_options(worker_class)
25
+ batch_key = options["key"]
26
+ if batch_key.respond_to?(:call)
27
+ batch_key.call(args)
28
+ else
29
+ batch_key
30
+ end
31
+ end
32
+
33
+ def get_options(worker_class)
34
+ (worker_class.get_sidekiq_options["merger"] || {}).with_indifferent_access
35
+ end
36
+ end
37
+
38
+ attr_reader :worker_class, :queue, :batch_key
39
+
40
+ def initialize(worker_class, queue, batch_key, redis: Sidekiq::Merger::Redis.new)
41
+ @worker_class = worker_class
42
+ @queue = queue
43
+ @batch_key = batch_key
44
+ @redis = redis
45
+ end
46
+
47
+ def add(args, execution_time)
48
+ @redis.push(full_batch_key, args, execution_time)
49
+ end
50
+
51
+ def delete(args)
52
+ @redis.delete(full_batch_key, args)
53
+ end
54
+
55
+ def size
56
+ @redis.batch_size(full_batch_key)
57
+ end
58
+
59
+ def flush
60
+ msgs = []
61
+
62
+ if @redis.lock(full_batch_key, Sidekiq::Merger::Config.lock_ttl)
63
+ msgs = @redis.pluck(full_batch_key)
64
+ end
65
+
66
+ unless msgs.empty?
67
+ Sidekiq::Client.push(
68
+ "class" => worker_class,
69
+ "queue" => queue,
70
+ "args" => msgs
71
+ )
72
+ end
73
+ end
74
+
75
+ def can_flush?
76
+ !execution_time.nil? && execution_time < Time.now
77
+ end
78
+
79
+ def full_batch_key
80
+ @full_batch_key ||= [worker_class.name.to_s.underscore, queue, batch_key].join(":")
81
+ end
82
+
83
+ def execution_time
84
+ @execution_time ||= @redis.execution_time(full_batch_key)
85
+ end
86
+
87
+ def ==(other)
88
+ self.worker_class == other.worker_class &&
89
+ self.queue == other.queue &&
90
+ self.batch_key == other.batch_key
91
+ end
92
+
93
+ private
94
+
95
+ def options
96
+ @options ||= self.class.get_options(worker_class)
97
+ rescue NameError
98
+ {}
99
+ end
100
+ end
@@ -0,0 +1,17 @@
1
+ require "active_support/configurable"
2
+
3
+ module Sidekiq::Merger::Config
4
+ include ActiveSupport::Configurable
5
+
6
+ def self.options
7
+ Sidekiq.options["merger"] || {}
8
+ end
9
+
10
+ config_accessor :poll_interval do
11
+ options[:poll_interval] || 5
12
+ end
13
+
14
+ config_accessor :lock_ttl do
15
+ options[:lock_ttl] || 1
16
+ end
17
+ end
@@ -0,0 +1,15 @@
1
+ class Sidekiq::Merger::Flusher
2
+ def initialize(logger)
3
+ @logger = logger
4
+ end
5
+
6
+ def flush
7
+ batches = Sidekiq::Merger::Batch.all.select(&:can_flush?)
8
+ unless batches.empty?
9
+ @logger.info(
10
+ "[Sidekiq::Merger] Trying to flush batched queues: #{batches.map(&:full_batch_key).join(",")}"
11
+ )
12
+ batches.each(&:flush)
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,17 @@
1
+ class Sidekiq::Merger::LoggingObserver
2
+ def initialize(logger)
3
+ @logger = logger
4
+ end
5
+
6
+ def update(time, _result, ex)
7
+ if ex.is_a?(Concurrent::TimeoutError)
8
+ @logger.error(
9
+ "[Sidekiq::Grouping] (#{time}) Execution timed out\n"
10
+ )
11
+ elsif ex.present?
12
+ @logger.error(
13
+ "[Sidekiq::Grouping] Execution failed with error #{ex}\n"
14
+ )
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,18 @@
1
+ require_relative "batch"
2
+
3
+ class Sidekiq::Merger::Middleware
4
+ def call(worker_class, msg, queue, _redis_pool = nil)
5
+ return yield if defined?(Sidekiq::Testing) && Sidekiq::Testing.inline?
6
+
7
+ worker_class = worker_class.camelize.constantize if worker_class.is_a?(String)
8
+ options = worker_class.get_sidekiq_options
9
+
10
+ if !msg["at"].nil? && options.key?("merger")
11
+ Sidekiq::Merger::Batch
12
+ .initialize_with_args(worker_class, queue, msg["args"])
13
+ .add(msg["args"], msg["at"])
14
+ else
15
+ yield
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,106 @@
1
+ require "active_support/core_ext/module/delegation"
2
+
3
+ class Sidekiq::Merger::Redis
4
+ class << self
5
+ KEY_PREFIX = "sidekiq-merger".freeze
6
+
7
+ def purge!
8
+ redis do |conn|
9
+ conn.eval
10
+ script = <<-SCRIPT
11
+ for i=1, #ARGV do
12
+ redis.call('del', unpack(redis.call('keys', ARGV[i])))
13
+ end
14
+ return true
15
+ SCRIPT
16
+ conn.eval(script, [], [batches_key, msg_key("*"), lock_key("*")])
17
+ end
18
+ end
19
+
20
+ def batches_key
21
+ "#{KEY_PREFIX}:batches"
22
+ end
23
+
24
+ def msg_key(key)
25
+ "#{KEY_PREFIX}:msg:#{key}"
26
+ end
27
+
28
+ def time_key(key)
29
+ "#{KEY_PREFIX}:time:#{key}"
30
+ end
31
+
32
+ def lock_key(key)
33
+ "#{KEY_PREFIX}:lock:#{key}"
34
+ end
35
+
36
+ def redis(&block)
37
+ Sidekiq.redis(&block)
38
+ end
39
+ end
40
+
41
+ def push(key, msg, execution_time)
42
+ redis do |conn|
43
+ conn.multi do
44
+ conn.sadd(batches_key, key)
45
+ conn.setnx(time_key(key), execution_time.to_json)
46
+ conn.sadd(msg_key(key), msg.to_json)
47
+ end
48
+ end
49
+ end
50
+
51
+ def delete(key, msg)
52
+ redis { |conn| conn.srem(msg_key(key), msg.to_json) }
53
+ end
54
+
55
+ def execution_time(key)
56
+ redis { |conn| Time.parse(conn.get(time_key(key))) rescue nil }
57
+ end
58
+
59
+ def batch_size(key)
60
+ redis { |conn| conn.scard(msg_key(key)) }
61
+ end
62
+
63
+ def exists?(key, msg)
64
+ redis { |conn| conn.sismember(msg_key(key), msg.to_json) }
65
+ end
66
+
67
+ def all
68
+ redis { |conn| conn.smembers(batches_key) }
69
+ end
70
+
71
+ def lock(key, ttl)
72
+ redis { |conn| conn.set(lock_key(key), true, nx: true, ex: ttl) }
73
+ end
74
+
75
+ def get(key)
76
+ msgs = []
77
+ redis do |conn|
78
+ msgs = conn.smembers(msg_key(key))
79
+ end
80
+ msgs.map { |msg| JSON.parse(msg) }
81
+ end
82
+
83
+ def pluck(key)
84
+ msgs = []
85
+ redis do |conn|
86
+ msgs = conn.smembers(msg_key(key))
87
+ conn.del(msg_key(key))
88
+ conn.del(time_key(key))
89
+ conn.srem(batches_key, key)
90
+ end
91
+ msgs.map { |msg| JSON.parse(msg) }
92
+ end
93
+
94
+ def delete_all(key)
95
+ redis do |conn|
96
+ conn.del(msg_key(key))
97
+ conn.del(time_key(key))
98
+ conn.del(lock_key(key))
99
+ conn.srem(batches_key, key)
100
+ end
101
+ end
102
+
103
+ private
104
+
105
+ delegate :batches_key, :msg_key, :time_key, :lock_key, :redis, to: "self.class"
106
+ end
@@ -0,0 +1,5 @@
1
+ module Sidekiq
2
+ module Merger
3
+ VERSION = "0.0.1".freeze
4
+ end
5
+ end
@@ -0,0 +1,39 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path("../lib", __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require "sidekiq/merger/version"
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "sidekiq-merger"
8
+ spec.version = Sidekiq::Merger::VERSION
9
+ spec.platform = Gem::Platform::RUBY
10
+ spec.authors = ["dtaniwaki"]
11
+ spec.email = ["daisuketaniwaki@gmail.com"]
12
+
13
+ spec.summary = "Sidekiq merger plugin"
14
+ spec.description = "Merge sidekiq jobs."
15
+ spec.homepage = "https://github.com/dtaniwaki/sidekiq-merger"
16
+ spec.license = "MIT"
17
+
18
+ spec.files = `git ls-files -z`.split("\x0").reject do |f|
19
+ f.match(%r{^(test|spec|features)/})
20
+ end
21
+ spec.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
22
+ spec.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
23
+ spec.require_paths = ["lib"]
24
+
25
+ spec.required_ruby_version = [">= 2.2.2", "< 2.5"]
26
+
27
+ spec.add_development_dependency "bundler", "~> 1.13"
28
+ spec.add_development_dependency "rake", "~> 10.0"
29
+ spec.add_development_dependency "rspec", "~> 3.0"
30
+ spec.add_development_dependency "simplecov"
31
+ spec.add_development_dependency "timecop"
32
+ spec.add_development_dependency "rubocop"
33
+ spec.add_development_dependency "pry"
34
+ spec.add_development_dependency "coveralls"
35
+
36
+ spec.add_dependency "sidekiq", ">= 3.4.0"
37
+ spec.add_dependency "concurrent-ruby"
38
+ spec.add_dependency "activesupport", ">= 3.2.0"
39
+ end
@@ -0,0 +1,124 @@
1
+ require "spec_helper"
2
+
3
+ describe Sidekiq::Merger::Batch do
4
+ subject { described_class.new(worker_class, queue, "foo", redis: redis) }
5
+ let(:redis) { Sidekiq::Merger::Redis.new }
6
+ let(:queue) { "queue" }
7
+ let(:now) { Time.now }
8
+ let(:execution_time) { now + 10.seconds }
9
+ let(:options) { { key: -> (args) { args.to_json } } }
10
+ let(:worker_class) do
11
+ local_options = options
12
+ Class.new do
13
+ include Sidekiq::Worker
14
+
15
+ sidekiq_options merger: local_options
16
+
17
+ def self.name
18
+ "name"
19
+ end
20
+
21
+ def perform(args)
22
+ end
23
+ end
24
+ end
25
+ before { Timecop.freeze(now) }
26
+
27
+ describe ".all" do
28
+ it "returns all the keys" do
29
+ redis.redis do |conn|
30
+ conn.sadd("sidekiq-merger:batches", "string:foo:xxx")
31
+ conn.sadd("sidekiq-merger:batches", "numeric:bar:yyy")
32
+ end
33
+
34
+ expect(described_class.all).to contain_exactly(
35
+ described_class.new(String, "foo", "xxx"),
36
+ described_class.new(Numeric, "bar", "yyy")
37
+ )
38
+ end
39
+
40
+ context "including invalid key" do
41
+ it "raises an error" do
42
+ redis.redis do |conn|
43
+ conn.sadd("sidekiq-merger:batches", "string:foo:xxx")
44
+ conn.sadd("sidekiq-merger:batches", "invalid")
45
+ end
46
+ expect {
47
+ described_class.all
48
+ }.to raise_error RuntimeError, "Invalid batch key"
49
+ end
50
+ end
51
+ end
52
+
53
+ describe ".initialize_with_args" do
54
+ it "provides batch_key from args" do
55
+ expect(described_class).to receive(:new).with(worker_class, queue, "[1,2,3]", anything)
56
+ described_class.initialize_with_args(worker_class, queue, [1, 2, 3])
57
+ end
58
+ it "passes options" do
59
+ expect(described_class).to receive(:new).with(worker_class, queue, anything, { redis: 1 })
60
+ described_class.initialize_with_args(worker_class, queue, anything, redis: 1)
61
+ end
62
+ end
63
+
64
+ describe "#add" do
65
+ it "adds the args in lazy batch" do
66
+ expect(redis).to receive(:push).with("name:queue:foo", [1, 2, 3], execution_time)
67
+ subject.add([1, 2, 3], execution_time)
68
+ end
69
+ end
70
+
71
+ describe "#delete" do
72
+ it "adds the args in lazy batch" do
73
+ expect(redis).to receive(:delete).with("name:queue:foo", [1, 2, 3])
74
+ subject.delete([1, 2, 3])
75
+ end
76
+ end
77
+
78
+ describe "#size" do
79
+ end
80
+
81
+ describe "#flush" do
82
+ before do
83
+ subject.add([1, 2, 3], execution_time)
84
+ subject.add([2, 3, 4], execution_time)
85
+ end
86
+ it "flushes all the args" do
87
+ expect(Sidekiq::Client).to receive(:push).with(
88
+ "class" => worker_class,
89
+ "queue" => queue,
90
+ "args" => [[1, 2, 3], [2, 3, 4]]
91
+ )
92
+
93
+ subject.flush
94
+ end
95
+ end
96
+
97
+ describe "#can_flush?" do
98
+ let(:options) { { flush_interval: 10.seconds } }
99
+ context "it has not get anything in batch" do
100
+ it "returns false" do
101
+ expect(subject.can_flush?).to eq false
102
+ end
103
+ end
104
+ context "it has not passed the interval time" do
105
+ it "returns false" do
106
+ subject.add([], execution_time)
107
+ expect(subject.can_flush?).to eq false
108
+ end
109
+ end
110
+ context "it has passed the interval time" do
111
+ it "returns true" do
112
+ subject.add([], execution_time)
113
+ Timecop.travel(10.seconds)
114
+ expect(subject.can_flush?).to eq true
115
+ end
116
+ end
117
+ end
118
+
119
+ describe "#full_batch_key" do
120
+ it "returns full batch key" do
121
+ expect(subject.full_batch_key).to eq "name:queue:foo"
122
+ end
123
+ end
124
+ end
@@ -0,0 +1,18 @@
1
+ require "spec_helper"
2
+
3
+ describe Sidekiq::Merger::Flusher do
4
+ subject { described_class.new(Sidekiq.logger) }
5
+
6
+ describe "#call" do
7
+ let(:active_batch) { double(full_batch_key: "active", can_flush?: true, flush: nil) }
8
+ let(:inactive_batch) { double(full_batch_key: "inactive", can_flush?: false, flush: nil) }
9
+ let(:batches) { [active_batch, inactive_batch] }
10
+ it "adds the args to the batch" do
11
+ allow(Sidekiq::Merger::Batch).to receive(:all).and_return batches
12
+ expect(active_batch).to receive(:flush)
13
+ expect(inactive_batch).not_to receive(:flush)
14
+
15
+ subject.flush
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,50 @@
1
+ require "spec_helper"
2
+
3
+ describe Sidekiq::Merger::Middleware do
4
+ subject { described_class.new }
5
+ let(:flusher) { Sidekiq::Merger::Flusher.new(Sidekiq.logger) }
6
+ let(:queue) { "queue" }
7
+ let(:now) { Time.now }
8
+ let(:options) { { key: -> (args) { "key" } } }
9
+ let(:worker_class) do
10
+ local_options = options
11
+ Class.new do
12
+ include Sidekiq::Worker
13
+
14
+ sidekiq_options merger: local_options
15
+
16
+ def self.name
17
+ "name"
18
+ end
19
+
20
+ def perform(args)
21
+ end
22
+ end
23
+ end
24
+ before :example do
25
+ allow(Object).to receive(:const_get).with("Name").and_return worker_class
26
+ end
27
+
28
+ describe "#call" do
29
+ it "adds the args to the batch" do
30
+ subject.call(worker_class, { "args" => [1, 2, 3], "at" => now + 10.seconds }, queue) {}
31
+ subject.call(worker_class, { "args" => [2, 3, 4], "at" => now + 15.seconds }, queue) {}
32
+ flusher.flush
33
+ expect(worker_class.jobs.size).to eq 0
34
+ Timecop.travel(10.seconds)
35
+ flusher.flush
36
+ expect(worker_class.jobs.size).to eq 1
37
+ job = worker_class.jobs[0]
38
+ expect(job["queue"]).to eq queue
39
+ expect(job["args"]).to eq [[1, 2, 3], [2, 3, 4]]
40
+ end
41
+ context "without at msg" do
42
+ it "does not add the args to the batch" do
43
+ subject.call(worker_class, { "args" => [1, 2, 3] }, queue) {}
44
+ subject.call(worker_class, { "args" => [2, 3, 4] }, queue) {}
45
+ flusher.flush
46
+ expect(worker_class.jobs.size).to eq 0
47
+ end
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,140 @@
1
+ require "spec_helper"
2
+
3
+ describe Sidekiq::Merger::Redis do
4
+ subject { described_class.new }
5
+ let(:now) { Time.now }
6
+ let(:execution_time) { now + 10.seconds }
7
+ before { Timecop.freeze(now) }
8
+
9
+ describe ".purge" do
10
+ it "cleans up all the keys" do
11
+ described_class.redis do |conn|
12
+ conn.sadd("sidekiq-merger:batches", "test")
13
+ conn.set("sidekiq-merger:msg:foo", "test")
14
+ conn.set("sidekiq-merger:lock:foo", "test")
15
+ end
16
+
17
+ described_class.purge!
18
+
19
+ described_class.redis do |conn|
20
+ expect(conn.smembers("sidekiq-merger:batches")).to be_empty
21
+ expect(conn.keys("sidekiq-merger:msg:*")).to be_empty
22
+ expect(conn.keys("sidekiq-merger:lock:*")).to be_empty
23
+ end
24
+ end
25
+ end
26
+
27
+ describe "#push" do
28
+ it "pushes the args" do
29
+ subject.push("foo", [1, 2, 3], execution_time)
30
+ described_class.redis do |conn|
31
+ expect(conn.smembers("sidekiq-merger:batches")).to contain_exactly "foo"
32
+ expect(conn.keys("sidekiq-merger:time:*")).to contain_exactly "sidekiq-merger:time:foo"
33
+ expect(conn.keys("sidekiq-merger:msg:*")).to contain_exactly "sidekiq-merger:msg:foo"
34
+ expect(conn.smembers("sidekiq-merger:msg:foo")).to contain_exactly "[1,2,3]"
35
+ end
36
+ end
37
+ it "sets the execution time" do
38
+ subject.push("foo", [1, 2, 3], execution_time)
39
+ described_class.redis do |conn|
40
+ expect(conn.get("sidekiq-merger:time:foo")).to eq execution_time.to_json
41
+ end
42
+ end
43
+
44
+ context "the batch key already exists" do
45
+ before do
46
+ subject.push("foo", [1, 2, 3], execution_time)
47
+ end
48
+ it "pushes the args" do
49
+ subject.push("foo", [2, 3, 4], execution_time + 1.hour)
50
+ described_class.redis do |conn|
51
+ expect(conn.smembers("sidekiq-merger:batches")).to contain_exactly "foo"
52
+ expect(conn.keys("sidekiq-merger:time:*")).to contain_exactly "sidekiq-merger:time:foo"
53
+ expect(conn.keys("sidekiq-merger:msg:*")).to contain_exactly "sidekiq-merger:msg:foo"
54
+ expect(conn.smembers("sidekiq-merger:msg:foo")).to contain_exactly "[1,2,3]", "[2,3,4]"
55
+ end
56
+ end
57
+ it "does not update the execution time" do
58
+ subject.push("foo", [2, 3, 4], execution_time + 1.hour)
59
+ described_class.redis do |conn|
60
+ expect(conn.get("sidekiq-merger:time:foo")).to eq execution_time.to_json
61
+ end
62
+ end
63
+ end
64
+
65
+ context "the args has already been pushed" do
66
+ before do
67
+ subject.push("foo", [1, 2, 3], execution_time)
68
+ end
69
+ it "does not push the args" do
70
+ subject.push("foo", [1, 2, 3], execution_time + 1.hour)
71
+ described_class.redis do |conn|
72
+ expect(conn.smembers("sidekiq-merger:batches")).to contain_exactly "foo"
73
+ expect(conn.keys("sidekiq-merger:time:*")).to contain_exactly "sidekiq-merger:time:foo"
74
+ expect(conn.keys("sidekiq-merger:msg:*")).to contain_exactly "sidekiq-merger:msg:foo"
75
+ expect(conn.smembers("sidekiq-merger:msg:foo")).to contain_exactly "[1,2,3]"
76
+ end
77
+ end
78
+ it "does not update the execution time" do
79
+ subject.push("foo", [1, 2, 3], execution_time + 1.hour)
80
+ described_class.redis do |conn|
81
+ expect(conn.get("sidekiq-merger:time:foo")).to eq execution_time.to_json
82
+ end
83
+ end
84
+ end
85
+
86
+ context "other batch key already exists" do
87
+ before do
88
+ subject.push("foo", [1, 2, 3], execution_time)
89
+ end
90
+ it "does not interfere the other batch" do
91
+ subject.push("bar", [2, 3, 4], execution_time + 1.hour)
92
+ described_class.redis do |conn|
93
+ expect(conn.smembers("sidekiq-merger:batches")).to contain_exactly "foo", "bar"
94
+ expect(conn.keys("sidekiq-merger:time:*")).to contain_exactly "sidekiq-merger:time:foo", "sidekiq-merger:time:bar"
95
+ expect(conn.keys("sidekiq-merger:msg:*")).to contain_exactly "sidekiq-merger:msg:foo", "sidekiq-merger:msg:bar"
96
+ expect(conn.smembers("sidekiq-merger:msg:foo")).to contain_exactly "[1,2,3]"
97
+ expect(conn.smembers("sidekiq-merger:msg:bar")).to contain_exactly "[2,3,4]"
98
+ end
99
+ end
100
+ it "sets the execution time" do
101
+ subject.push("bar", [2, 3, 4], execution_time + 1.hour)
102
+ described_class.redis do |conn|
103
+ expect(conn.get("sidekiq-merger:time:foo")).to eq execution_time.to_json
104
+ expect(conn.get("sidekiq-merger:time:bar")).to eq (execution_time + 1.hour).to_json
105
+ end
106
+ end
107
+ end
108
+ end
109
+
110
+ describe "#delete" do
111
+ end
112
+
113
+ describe "#batch_size" do
114
+ end
115
+
116
+ describe "#exists?" do
117
+ end
118
+
119
+ describe "#all" do
120
+ end
121
+
122
+ describe "#lock" do
123
+ end
124
+
125
+ describe "#get" do
126
+ end
127
+
128
+ describe "#pluck" do
129
+ before do
130
+ subject.push("bar", [1, 2, 3], execution_time)
131
+ subject.push("bar", [2, 3, 4], execution_time)
132
+ end
133
+ it "plucks all the args" do
134
+ expect(subject.pluck("bar")).to eq [[1, 2, 3], [2, 3, 4]]
135
+ end
136
+ end
137
+
138
+ describe "#delete_all" do
139
+ end
140
+ end
@@ -0,0 +1,7 @@
1
+ require "spec_helper"
2
+
3
+ describe Sidekiq::Merger do
4
+ it "has a version number" do
5
+ expect(Sidekiq::Merger::VERSION).not_to be nil
6
+ end
7
+ end
@@ -0,0 +1,66 @@
1
+ $LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
2
+ require "rubygems"
3
+ require "sidekiq/testing"
4
+ require "active_support/core_ext/numeric/time"
5
+ require "timecop"
6
+ require "simplecov"
7
+ require "coveralls"
8
+
9
+ pid = Process.pid
10
+ SimpleCov.at_exit do
11
+ SimpleCov.result.format! if Process.pid == pid
12
+ end
13
+ SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
14
+ SimpleCov::Formatter::HTMLFormatter,
15
+ Coveralls::SimpleCov::Formatter
16
+ ]
17
+ SimpleCov.start
18
+
19
+ require "sidekiq/merger"
20
+
21
+ Dir[File.join(__dir__, "support", "**", "*.rb")].each { |f| require f }
22
+
23
+ RSpec.configure do |config|
24
+ # rspec-expectations config goes here. You can use an alternate
25
+ # assertion/expectation library such as wrong or the stdlib/minitest
26
+ # assertions if you prefer.
27
+ config.expect_with :rspec do |expectations|
28
+ # This option will default to `true` in RSpec 4. It makes the `description`
29
+ # and `failure_message` of custom matchers include text for helper methods
30
+ # defined using `chain`, e.g.:
31
+ # be_bigger_than(2).and_smaller_than(4).description
32
+ # # => "be bigger than 2 and smaller than 4"
33
+ # ...rather than:
34
+ # # => "be bigger than 2"
35
+ expectations.include_chain_clauses_in_custom_matcher_descriptions = true
36
+ end
37
+
38
+ config.mock_with :rspec do |mocks|
39
+ # Prevents you from mocking or stubbing a method that does not exist on
40
+ # a real object. This is generally recommended, and will default to
41
+ # `true` in RSpec 4.
42
+ mocks.verify_partial_doubles = true
43
+ end
44
+
45
+ if config.files_to_run.one?
46
+ # Use the documentation formatter for detailed output,
47
+ # unless a formatter has already been configured
48
+ # (e.g. via a command-line flag).
49
+ config.default_formatter = "doc"
50
+ end
51
+
52
+ config.order = :random
53
+
54
+ Kernel.srand config.seed
55
+
56
+ config.before :suite do
57
+ Sidekiq.logger.level = Logger::ERROR
58
+ end
59
+
60
+ config.before :example do
61
+ Timecop.return
62
+ Sidekiq::Merger::Redis.redis do |conn|
63
+ conn.flushall
64
+ end
65
+ end
66
+ end
File without changes
metadata ADDED
@@ -0,0 +1,237 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: sidekiq-merger
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - dtaniwaki
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2017-01-30 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.13'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.13'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '10.0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '10.0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rspec
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '3.0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '3.0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: simplecov
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: timecop
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: rubocop
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ - !ruby/object:Gem::Dependency
98
+ name: pry
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: '0'
111
+ - !ruby/object:Gem::Dependency
112
+ name: coveralls
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ">="
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - ">="
123
+ - !ruby/object:Gem::Version
124
+ version: '0'
125
+ - !ruby/object:Gem::Dependency
126
+ name: sidekiq
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - ">="
130
+ - !ruby/object:Gem::Version
131
+ version: 3.4.0
132
+ type: :runtime
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - ">="
137
+ - !ruby/object:Gem::Version
138
+ version: 3.4.0
139
+ - !ruby/object:Gem::Dependency
140
+ name: concurrent-ruby
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - ">="
144
+ - !ruby/object:Gem::Version
145
+ version: '0'
146
+ type: :runtime
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - ">="
151
+ - !ruby/object:Gem::Version
152
+ version: '0'
153
+ - !ruby/object:Gem::Dependency
154
+ name: activesupport
155
+ requirement: !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - ">="
158
+ - !ruby/object:Gem::Version
159
+ version: 3.2.0
160
+ type: :runtime
161
+ prerelease: false
162
+ version_requirements: !ruby/object:Gem::Requirement
163
+ requirements:
164
+ - - ">="
165
+ - !ruby/object:Gem::Version
166
+ version: 3.2.0
167
+ description: Merge sidekiq jobs.
168
+ email:
169
+ - daisuketaniwaki@gmail.com
170
+ executables:
171
+ - console
172
+ - setup
173
+ extensions: []
174
+ extra_rdoc_files: []
175
+ files:
176
+ - ".gitignore"
177
+ - ".rspec"
178
+ - ".rubocop.yml"
179
+ - ".travis.yml"
180
+ - Gemfile
181
+ - LICENSE
182
+ - README.md
183
+ - Rakefile
184
+ - bin/console
185
+ - bin/setup
186
+ - lib/sidekiq-merger.rb
187
+ - lib/sidekiq/merger.rb
188
+ - lib/sidekiq/merger/batch.rb
189
+ - lib/sidekiq/merger/config.rb
190
+ - lib/sidekiq/merger/flusher.rb
191
+ - lib/sidekiq/merger/logging_observer.rb
192
+ - lib/sidekiq/merger/middleware.rb
193
+ - lib/sidekiq/merger/redis.rb
194
+ - lib/sidekiq/merger/version.rb
195
+ - sidekiq-merger.gemspec
196
+ - spec/sidekiq/merger/batch_spec.rb
197
+ - spec/sidekiq/merger/flusher_spec.rb
198
+ - spec/sidekiq/merger/middleware_spec.rb
199
+ - spec/sidekiq/merger/redis_spec.rb
200
+ - spec/sidekiq/merger_spec.rb
201
+ - spec/spec_helper.rb
202
+ - spec/support/matchers.rb
203
+ homepage: https://github.com/dtaniwaki/sidekiq-merger
204
+ licenses:
205
+ - MIT
206
+ metadata: {}
207
+ post_install_message:
208
+ rdoc_options: []
209
+ require_paths:
210
+ - lib
211
+ required_ruby_version: !ruby/object:Gem::Requirement
212
+ requirements:
213
+ - - ">="
214
+ - !ruby/object:Gem::Version
215
+ version: 2.2.2
216
+ - - "<"
217
+ - !ruby/object:Gem::Version
218
+ version: '2.5'
219
+ required_rubygems_version: !ruby/object:Gem::Requirement
220
+ requirements:
221
+ - - ">="
222
+ - !ruby/object:Gem::Version
223
+ version: '0'
224
+ requirements: []
225
+ rubyforge_project:
226
+ rubygems_version: 2.5.1
227
+ signing_key:
228
+ specification_version: 4
229
+ summary: Sidekiq merger plugin
230
+ test_files:
231
+ - spec/sidekiq/merger/batch_spec.rb
232
+ - spec/sidekiq/merger/flusher_spec.rb
233
+ - spec/sidekiq/merger/middleware_spec.rb
234
+ - spec/sidekiq/merger/redis_spec.rb
235
+ - spec/sidekiq/merger_spec.rb
236
+ - spec/spec_helper.rb
237
+ - spec/support/matchers.rb