sidekiq-merger 0.0.1 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.dockerignore +21 -0
- data/.gemrelease +2 -0
- data/.gitignore +19 -9
- data/.travis.yml +5 -1
- data/Dockerfile +10 -0
- data/Gemfile +3 -1
- data/README.md +68 -6
- data/app/Gemfile +8 -0
- data/app/app.rb +43 -0
- data/app/config.ru +3 -0
- data/app/sidekiq.rb +25 -0
- data/app/some_worker.rb +9 -0
- data/app/unique_worker.rb +9 -0
- data/app/views/index.erb +35 -0
- data/docker-compose-common.yml +7 -0
- data/docker-compose.yml +26 -0
- data/lib/sidekiq/merger.rb +14 -18
- data/lib/sidekiq/merger/flusher.rb +4 -4
- data/lib/sidekiq/merger/merge.rb +113 -0
- data/lib/sidekiq/merger/middleware.rb +6 -4
- data/lib/sidekiq/merger/redis.rb +52 -25
- data/lib/sidekiq/merger/version.rb +1 -1
- data/lib/sidekiq/merger/views/index.erb +41 -0
- data/lib/sidekiq/merger/web.rb +22 -0
- data/misc/web_ui.png +0 -0
- data/sidekiq-merger.gemspec +9 -11
- data/spec/sidekiq/merger/flusher_spec.rb +7 -7
- data/spec/sidekiq/merger/{batch_spec.rb → merge_spec.rb} +31 -17
- data/spec/sidekiq/merger/middleware_spec.rb +12 -6
- data/spec/sidekiq/merger/redis_spec.rb +99 -69
- data/spec/spec_helper.rb +7 -2
- metadata +73 -66
- data/bin/console +0 -7
- data/bin/setup +0 -8
- data/lib/sidekiq/merger/batch.rb +0 -100
data/bin/console
DELETED
data/bin/setup
DELETED
data/lib/sidekiq/merger/batch.rb
DELETED
|
@@ -1,100 +0,0 @@
|
|
|
1
|
-
require_relative "redis"
|
|
2
|
-
require "active_support/core_ext/hash/indifferent_access"
|
|
3
|
-
|
|
4
|
-
class Sidekiq::Merger::Batch
|
|
5
|
-
class << self
|
|
6
|
-
def all
|
|
7
|
-
redis = Sidekiq::Merger::Redis.new
|
|
8
|
-
|
|
9
|
-
redis.all.map do |full_batch_key|
|
|
10
|
-
keys = full_batch_key.split(":")
|
|
11
|
-
raise "Invalid batch key" if keys.size < 3
|
|
12
|
-
worker_class = keys[0].camelize.constantize
|
|
13
|
-
queue = keys[1]
|
|
14
|
-
batch_key = keys[2]
|
|
15
|
-
new(worker_class, queue, batch_key, redis: redis)
|
|
16
|
-
end
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
def initialize_with_args(worker_class, queue, args, options = {})
|
|
20
|
-
new(worker_class, queue, batch_key(worker_class, args), options)
|
|
21
|
-
end
|
|
22
|
-
|
|
23
|
-
def batch_key(worker_class, args)
|
|
24
|
-
options = get_options(worker_class)
|
|
25
|
-
batch_key = options["key"]
|
|
26
|
-
if batch_key.respond_to?(:call)
|
|
27
|
-
batch_key.call(args)
|
|
28
|
-
else
|
|
29
|
-
batch_key
|
|
30
|
-
end
|
|
31
|
-
end
|
|
32
|
-
|
|
33
|
-
def get_options(worker_class)
|
|
34
|
-
(worker_class.get_sidekiq_options["merger"] || {}).with_indifferent_access
|
|
35
|
-
end
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
attr_reader :worker_class, :queue, :batch_key
|
|
39
|
-
|
|
40
|
-
def initialize(worker_class, queue, batch_key, redis: Sidekiq::Merger::Redis.new)
|
|
41
|
-
@worker_class = worker_class
|
|
42
|
-
@queue = queue
|
|
43
|
-
@batch_key = batch_key
|
|
44
|
-
@redis = redis
|
|
45
|
-
end
|
|
46
|
-
|
|
47
|
-
def add(args, execution_time)
|
|
48
|
-
@redis.push(full_batch_key, args, execution_time)
|
|
49
|
-
end
|
|
50
|
-
|
|
51
|
-
def delete(args)
|
|
52
|
-
@redis.delete(full_batch_key, args)
|
|
53
|
-
end
|
|
54
|
-
|
|
55
|
-
def size
|
|
56
|
-
@redis.batch_size(full_batch_key)
|
|
57
|
-
end
|
|
58
|
-
|
|
59
|
-
def flush
|
|
60
|
-
msgs = []
|
|
61
|
-
|
|
62
|
-
if @redis.lock(full_batch_key, Sidekiq::Merger::Config.lock_ttl)
|
|
63
|
-
msgs = @redis.pluck(full_batch_key)
|
|
64
|
-
end
|
|
65
|
-
|
|
66
|
-
unless msgs.empty?
|
|
67
|
-
Sidekiq::Client.push(
|
|
68
|
-
"class" => worker_class,
|
|
69
|
-
"queue" => queue,
|
|
70
|
-
"args" => msgs
|
|
71
|
-
)
|
|
72
|
-
end
|
|
73
|
-
end
|
|
74
|
-
|
|
75
|
-
def can_flush?
|
|
76
|
-
!execution_time.nil? && execution_time < Time.now
|
|
77
|
-
end
|
|
78
|
-
|
|
79
|
-
def full_batch_key
|
|
80
|
-
@full_batch_key ||= [worker_class.name.to_s.underscore, queue, batch_key].join(":")
|
|
81
|
-
end
|
|
82
|
-
|
|
83
|
-
def execution_time
|
|
84
|
-
@execution_time ||= @redis.execution_time(full_batch_key)
|
|
85
|
-
end
|
|
86
|
-
|
|
87
|
-
def ==(other)
|
|
88
|
-
self.worker_class == other.worker_class &&
|
|
89
|
-
self.queue == other.queue &&
|
|
90
|
-
self.batch_key == other.batch_key
|
|
91
|
-
end
|
|
92
|
-
|
|
93
|
-
private
|
|
94
|
-
|
|
95
|
-
def options
|
|
96
|
-
@options ||= self.class.get_options(worker_class)
|
|
97
|
-
rescue NameError
|
|
98
|
-
{}
|
|
99
|
-
end
|
|
100
|
-
end
|