sidekiq-merger 0.0.1 → 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.dockerignore +21 -0
- data/.gemrelease +2 -0
- data/.gitignore +19 -9
- data/.travis.yml +5 -1
- data/Dockerfile +10 -0
- data/Gemfile +3 -1
- data/README.md +68 -6
- data/app/Gemfile +8 -0
- data/app/app.rb +43 -0
- data/app/config.ru +3 -0
- data/app/sidekiq.rb +25 -0
- data/app/some_worker.rb +9 -0
- data/app/unique_worker.rb +9 -0
- data/app/views/index.erb +35 -0
- data/docker-compose-common.yml +7 -0
- data/docker-compose.yml +26 -0
- data/lib/sidekiq/merger.rb +14 -18
- data/lib/sidekiq/merger/flusher.rb +4 -4
- data/lib/sidekiq/merger/merge.rb +113 -0
- data/lib/sidekiq/merger/middleware.rb +6 -4
- data/lib/sidekiq/merger/redis.rb +52 -25
- data/lib/sidekiq/merger/version.rb +1 -1
- data/lib/sidekiq/merger/views/index.erb +41 -0
- data/lib/sidekiq/merger/web.rb +22 -0
- data/misc/web_ui.png +0 -0
- data/sidekiq-merger.gemspec +9 -11
- data/spec/sidekiq/merger/flusher_spec.rb +7 -7
- data/spec/sidekiq/merger/{batch_spec.rb → merge_spec.rb} +31 -17
- data/spec/sidekiq/merger/middleware_spec.rb +12 -6
- data/spec/sidekiq/merger/redis_spec.rb +99 -69
- data/spec/spec_helper.rb +7 -2
- metadata +73 -66
- data/bin/console +0 -7
- data/bin/setup +0 -8
- data/lib/sidekiq/merger/batch.rb +0 -100
data/bin/console
DELETED
data/bin/setup
DELETED
data/lib/sidekiq/merger/batch.rb
DELETED
@@ -1,100 +0,0 @@
|
|
1
|
-
require_relative "redis"
|
2
|
-
require "active_support/core_ext/hash/indifferent_access"
|
3
|
-
|
4
|
-
class Sidekiq::Merger::Batch
|
5
|
-
class << self
|
6
|
-
def all
|
7
|
-
redis = Sidekiq::Merger::Redis.new
|
8
|
-
|
9
|
-
redis.all.map do |full_batch_key|
|
10
|
-
keys = full_batch_key.split(":")
|
11
|
-
raise "Invalid batch key" if keys.size < 3
|
12
|
-
worker_class = keys[0].camelize.constantize
|
13
|
-
queue = keys[1]
|
14
|
-
batch_key = keys[2]
|
15
|
-
new(worker_class, queue, batch_key, redis: redis)
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
def initialize_with_args(worker_class, queue, args, options = {})
|
20
|
-
new(worker_class, queue, batch_key(worker_class, args), options)
|
21
|
-
end
|
22
|
-
|
23
|
-
def batch_key(worker_class, args)
|
24
|
-
options = get_options(worker_class)
|
25
|
-
batch_key = options["key"]
|
26
|
-
if batch_key.respond_to?(:call)
|
27
|
-
batch_key.call(args)
|
28
|
-
else
|
29
|
-
batch_key
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
def get_options(worker_class)
|
34
|
-
(worker_class.get_sidekiq_options["merger"] || {}).with_indifferent_access
|
35
|
-
end
|
36
|
-
end
|
37
|
-
|
38
|
-
attr_reader :worker_class, :queue, :batch_key
|
39
|
-
|
40
|
-
def initialize(worker_class, queue, batch_key, redis: Sidekiq::Merger::Redis.new)
|
41
|
-
@worker_class = worker_class
|
42
|
-
@queue = queue
|
43
|
-
@batch_key = batch_key
|
44
|
-
@redis = redis
|
45
|
-
end
|
46
|
-
|
47
|
-
def add(args, execution_time)
|
48
|
-
@redis.push(full_batch_key, args, execution_time)
|
49
|
-
end
|
50
|
-
|
51
|
-
def delete(args)
|
52
|
-
@redis.delete(full_batch_key, args)
|
53
|
-
end
|
54
|
-
|
55
|
-
def size
|
56
|
-
@redis.batch_size(full_batch_key)
|
57
|
-
end
|
58
|
-
|
59
|
-
def flush
|
60
|
-
msgs = []
|
61
|
-
|
62
|
-
if @redis.lock(full_batch_key, Sidekiq::Merger::Config.lock_ttl)
|
63
|
-
msgs = @redis.pluck(full_batch_key)
|
64
|
-
end
|
65
|
-
|
66
|
-
unless msgs.empty?
|
67
|
-
Sidekiq::Client.push(
|
68
|
-
"class" => worker_class,
|
69
|
-
"queue" => queue,
|
70
|
-
"args" => msgs
|
71
|
-
)
|
72
|
-
end
|
73
|
-
end
|
74
|
-
|
75
|
-
def can_flush?
|
76
|
-
!execution_time.nil? && execution_time < Time.now
|
77
|
-
end
|
78
|
-
|
79
|
-
def full_batch_key
|
80
|
-
@full_batch_key ||= [worker_class.name.to_s.underscore, queue, batch_key].join(":")
|
81
|
-
end
|
82
|
-
|
83
|
-
def execution_time
|
84
|
-
@execution_time ||= @redis.execution_time(full_batch_key)
|
85
|
-
end
|
86
|
-
|
87
|
-
def ==(other)
|
88
|
-
self.worker_class == other.worker_class &&
|
89
|
-
self.queue == other.queue &&
|
90
|
-
self.batch_key == other.batch_key
|
91
|
-
end
|
92
|
-
|
93
|
-
private
|
94
|
-
|
95
|
-
def options
|
96
|
-
@options ||= self.class.get_options(worker_class)
|
97
|
-
rescue NameError
|
98
|
-
{}
|
99
|
-
end
|
100
|
-
end
|