qs 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bench/config.qs +4 -27
- data/bench/dispatcher.qs +24 -0
- data/bench/report.rb +80 -10
- data/bench/report.txt +10 -3
- data/bench/setup.rb +55 -0
- data/lib/qs.rb +75 -15
- data/lib/qs/client.rb +73 -22
- data/lib/qs/daemon.rb +21 -21
- data/lib/qs/daemon_data.rb +4 -4
- data/lib/qs/dispatch_job.rb +36 -0
- data/lib/qs/dispatch_job_handler.rb +79 -0
- data/lib/qs/dispatcher_queue.rb +19 -0
- data/lib/qs/error_handler.rb +12 -12
- data/lib/qs/event.rb +82 -0
- data/lib/qs/event_handler.rb +34 -0
- data/lib/qs/event_handler_test_helpers.rb +17 -0
- data/lib/qs/job.rb +19 -31
- data/lib/qs/job_handler.rb +6 -63
- data/lib/qs/{test_helpers.rb → job_handler_test_helpers.rb} +2 -2
- data/lib/qs/message.rb +29 -0
- data/lib/qs/message_handler.rb +84 -0
- data/lib/qs/payload.rb +98 -0
- data/lib/qs/payload_handler.rb +106 -54
- data/lib/qs/queue.rb +39 -6
- data/lib/qs/queue_item.rb +33 -0
- data/lib/qs/route.rb +7 -7
- data/lib/qs/runner.rb +6 -5
- data/lib/qs/test_runner.rb +41 -13
- data/lib/qs/version.rb +1 -1
- data/qs.gemspec +1 -1
- data/test/helper.rb +1 -1
- data/test/support/app_daemon.rb +77 -11
- data/test/support/factory.rb +34 -0
- data/test/system/daemon_tests.rb +146 -77
- data/test/system/queue_tests.rb +87 -0
- data/test/unit/client_tests.rb +184 -45
- data/test/unit/daemon_data_tests.rb +4 -4
- data/test/unit/daemon_tests.rb +32 -32
- data/test/unit/dispatch_job_handler_tests.rb +163 -0
- data/test/unit/dispatch_job_tests.rb +75 -0
- data/test/unit/dispatcher_queue_tests.rb +42 -0
- data/test/unit/error_handler_tests.rb +9 -9
- data/test/unit/event_handler_test_helpers_tests.rb +55 -0
- data/test/unit/event_handler_tests.rb +63 -0
- data/test/unit/event_tests.rb +162 -0
- data/test/unit/{test_helper_tests.rb → job_handler_test_helper_tests.rb} +13 -19
- data/test/unit/job_handler_tests.rb +17 -210
- data/test/unit/job_tests.rb +49 -79
- data/test/unit/message_handler_tests.rb +235 -0
- data/test/unit/message_tests.rb +64 -0
- data/test/unit/payload_handler_tests.rb +285 -86
- data/test/unit/payload_tests.rb +139 -0
- data/test/unit/qs_runner_tests.rb +6 -6
- data/test/unit/qs_tests.rb +167 -28
- data/test/unit/queue_item_tests.rb +51 -0
- data/test/unit/queue_tests.rb +126 -18
- data/test/unit/route_tests.rb +12 -13
- data/test/unit/runner_tests.rb +10 -10
- data/test/unit/test_runner_tests.rb +117 -24
- metadata +51 -21
- data/bench/queue.rb +0 -8
- data/lib/qs/redis_item.rb +0 -33
- data/test/unit/redis_item_tests.rb +0 -49
data/bench/config.qs
CHANGED
@@ -1,20 +1,7 @@
|
|
1
1
|
require 'qs'
|
2
|
-
require 'bench/queue'
|
3
2
|
|
4
|
-
|
5
|
-
|
6
|
-
LOGGER = if ENV['BENCH_REPORT']
|
7
|
-
Logger.new(ROOT_PATH.join('log/bench_daemon.log').to_s)
|
8
|
-
else
|
9
|
-
Logger.new(STDOUT)
|
10
|
-
end
|
11
|
-
LOGGER.datetime_format = "" # turn off the datetime in the logs
|
12
|
-
|
13
|
-
PROGRESS_IO = if ENV['BENCH_PROGRESS_IO']
|
14
|
-
::IO.for_fd(ENV['BENCH_PROGRESS_IO'].to_i)
|
15
|
-
else
|
16
|
-
File.open('/dev/null', 'w')
|
17
|
-
end
|
3
|
+
ENV['LOG_NAME'] = 'log/bench_daemon.log'
|
4
|
+
require 'bench/setup'
|
18
5
|
|
19
6
|
class BenchDaemon
|
20
7
|
include Qs::Daemon
|
@@ -27,21 +14,11 @@ class BenchDaemon
|
|
27
14
|
|
28
15
|
queue BenchQueue
|
29
16
|
|
30
|
-
# if
|
31
|
-
error do |exception,
|
17
|
+
# if fails notify the bench report so it doesn't hang forever on IO.select
|
18
|
+
error do |exception, context|
|
32
19
|
PROGRESS_IO.write_nonblock('F')
|
33
20
|
end
|
34
21
|
|
35
|
-
class Multiply
|
36
|
-
include Qs::JobHandler
|
37
|
-
|
38
|
-
after{ PROGRESS_IO.write_nonblock('.') }
|
39
|
-
|
40
|
-
def run!
|
41
|
-
'a' * params['size']
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
22
|
end
|
46
23
|
|
47
24
|
run BenchDaemon.new
|
data/bench/dispatcher.qs
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'qs'
|
2
|
+
|
3
|
+
ENV['LOG_NAME'] = 'log/bench_dispatcher_daemon.log'
|
4
|
+
require 'bench/setup'
|
5
|
+
|
6
|
+
class DispatcherDaemon
|
7
|
+
include Qs::Daemon
|
8
|
+
|
9
|
+
name 'bench-dispatcher'
|
10
|
+
pid_file ROOT_PATH.join('tmp/bench_dispatcher_daemon.pid').to_s
|
11
|
+
|
12
|
+
logger LOGGER
|
13
|
+
verbose_logging false
|
14
|
+
|
15
|
+
queue Qs.dispatcher_queue
|
16
|
+
|
17
|
+
# if fails notify the bench report so it doesn't hang forever on IO.select
|
18
|
+
error do |exception, context|
|
19
|
+
PROGRESS_IO.write_nonblock('F')
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
|
24
|
+
run DispatcherDaemon.new
|
data/bench/report.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'benchmark'
|
2
2
|
require 'scmd'
|
3
|
-
require 'bench/
|
3
|
+
require 'bench/setup'
|
4
4
|
|
5
5
|
class BenchRunner
|
6
6
|
|
@@ -18,23 +18,37 @@ class BenchRunner
|
|
18
18
|
@job_name = 'multiply'
|
19
19
|
@job_params = { 'size' => 100_000 }
|
20
20
|
|
21
|
+
@number_of_events = ENV['NUM_EVENTS'] || 10_000
|
22
|
+
@event_channel = 'something'
|
23
|
+
@event_name = 'happened'
|
24
|
+
@event_params = { 'size' => 100_000 }
|
25
|
+
|
21
26
|
@progress_reader, @progress_writer = IO.pipe
|
22
27
|
|
23
28
|
@results = {}
|
24
|
-
|
25
|
-
Qs.init
|
26
29
|
end
|
27
30
|
|
28
31
|
def run
|
29
32
|
output "Running benchmark report..."
|
30
33
|
output("\n", false)
|
31
34
|
|
32
|
-
|
35
|
+
Qs.client.clear(BenchQueue.redis_key)
|
36
|
+
Qs.client.clear(Qs.dispatcher_queue.redis_key)
|
37
|
+
|
38
|
+
benchmark_enqueueing_jobs
|
33
39
|
benchmark_running_jobs
|
34
40
|
|
41
|
+
benchmark_publishing_events
|
42
|
+
benchmark_running_events
|
43
|
+
|
35
44
|
size = @results.values.map(&:size).max
|
36
|
-
output "
|
37
|
-
output "
|
45
|
+
output "\n", false
|
46
|
+
output "Enqueueing #{@number_of_jobs} Jobs Time: #{@results[:enqueueing_jobs].rjust(size)}"
|
47
|
+
output "Running #{@number_of_jobs} Jobs Time: #{@results[:running_jobs].rjust(size)}"
|
48
|
+
|
49
|
+
output "\n", false
|
50
|
+
output "Publishing #{@number_of_events} Events Time: #{@results[:publishing_events].rjust(size)}"
|
51
|
+
output "Running #{@number_of_events} Events Time: #{@results[:running_events].rjust(size)}"
|
38
52
|
|
39
53
|
output "\n"
|
40
54
|
output "Done running benchmark report"
|
@@ -42,15 +56,15 @@ class BenchRunner
|
|
42
56
|
|
43
57
|
private
|
44
58
|
|
45
|
-
def
|
46
|
-
output "
|
59
|
+
def benchmark_enqueueing_jobs
|
60
|
+
output "Enqueuing jobs"
|
47
61
|
benchmark = Benchmark.measure do
|
48
62
|
(1..@number_of_jobs).each do |n|
|
49
63
|
BenchQueue.add(@job_name, @job_params)
|
50
64
|
output('.', false) if ((n - 1) % 100 == 0)
|
51
65
|
end
|
52
66
|
end
|
53
|
-
@results[:
|
67
|
+
@results[:enqueueing_jobs] = round_and_display(benchmark.real)
|
54
68
|
output("\n", false)
|
55
69
|
end
|
56
70
|
|
@@ -78,7 +92,6 @@ class BenchRunner
|
|
78
92
|
end
|
79
93
|
end
|
80
94
|
@results[:running_jobs] = round_and_display(benchmark.real)
|
81
|
-
output("\n", false)
|
82
95
|
ensure
|
83
96
|
cmd.kill('TERM')
|
84
97
|
cmd.wait(5)
|
@@ -87,6 +100,63 @@ class BenchRunner
|
|
87
100
|
output("\n", false)
|
88
101
|
end
|
89
102
|
|
103
|
+
def benchmark_publishing_events
|
104
|
+
output "Publishing events"
|
105
|
+
benchmark = Benchmark.measure do
|
106
|
+
(1..@number_of_events).each do |n|
|
107
|
+
Qs.publish(@event_channel, @event_name, @event_params)
|
108
|
+
output('.', false) if ((n - 1) % 100 == 0)
|
109
|
+
end
|
110
|
+
end
|
111
|
+
@results[:publishing_events] = round_and_display(benchmark.real)
|
112
|
+
output("\n", false)
|
113
|
+
end
|
114
|
+
|
115
|
+
def benchmark_running_events
|
116
|
+
bench_queue_cmd_str = "bundle exec ./bin/qs bench/config.qs"
|
117
|
+
bench_queue_cmd = Scmd.new(bench_queue_cmd_str, {
|
118
|
+
'BENCH_REPORT' => 'yes',
|
119
|
+
'BENCH_PROGRESS_IO' => @progress_writer.fileno
|
120
|
+
})
|
121
|
+
|
122
|
+
dispatcher_queue_cmd_str = "bundle exec ./bin/qs bench/dispatcher.qs"
|
123
|
+
dispatcher_queue_cmd = Scmd.new(dispatcher_queue_cmd_str, {
|
124
|
+
'BENCH_REPORT' => 'yes',
|
125
|
+
'BENCH_PROGRESS_IO' => @progress_writer.fileno
|
126
|
+
})
|
127
|
+
|
128
|
+
output "Running events"
|
129
|
+
begin
|
130
|
+
benchmark = Benchmark.measure do
|
131
|
+
bench_queue_cmd.start
|
132
|
+
if !bench_queue_cmd.running?
|
133
|
+
raise "failed to start qs process: #{bench_queue_cmd_str.inspect}"
|
134
|
+
end
|
135
|
+
|
136
|
+
dispatcher_queue_cmd.start
|
137
|
+
if !dispatcher_queue_cmd.running?
|
138
|
+
raise "failed to start qs process: #{dispatcher_queue_cmd_str.inspect}"
|
139
|
+
end
|
140
|
+
|
141
|
+
progress = 0
|
142
|
+
while progress < @number_of_jobs
|
143
|
+
::IO.select([@progress_reader])
|
144
|
+
result = @progress_reader.read_nonblock(1)
|
145
|
+
progress += 1
|
146
|
+
output(result, false) if ((progress - 1) % 100 == 0)
|
147
|
+
end
|
148
|
+
end
|
149
|
+
@results[:running_events] = round_and_display(benchmark.real)
|
150
|
+
ensure
|
151
|
+
dispatcher_queue_cmd.kill('TERM')
|
152
|
+
bench_queue_cmd.kill('TERM')
|
153
|
+
dispatcher_queue_cmd.wait(5)
|
154
|
+
bench_queue_cmd.wait(5)
|
155
|
+
end
|
156
|
+
|
157
|
+
output("\n", false)
|
158
|
+
end
|
159
|
+
|
90
160
|
private
|
91
161
|
|
92
162
|
def output(message, puts = true)
|
data/bench/report.txt
CHANGED
@@ -1,11 +1,18 @@
|
|
1
1
|
Running benchmark report...
|
2
2
|
|
3
|
-
|
3
|
+
Enqueuing jobs
|
4
4
|
....................................................................................................
|
5
5
|
Running jobs
|
6
6
|
....................................................................................................
|
7
|
+
Publishing events
|
8
|
+
....................................................................................................
|
9
|
+
Running events
|
10
|
+
....................................................................................................
|
11
|
+
|
12
|
+
Enqueueing 10000 Jobs Time: 1.6858
|
13
|
+
Running 10000 Jobs Time: 11.3011
|
7
14
|
|
8
|
-
|
9
|
-
Running 10000
|
15
|
+
Publishing 10000 Events Time: 2.4596
|
16
|
+
Running 10000 Events Time: 16.0522
|
10
17
|
|
11
18
|
Done running benchmark report
|
data/bench/setup.rb
ADDED
@@ -0,0 +1,55 @@
|
|
1
|
+
require 'qs'
|
2
|
+
require 'json'
|
3
|
+
|
4
|
+
ROOT_PATH = Pathname.new(File.expand_path('../..', __FILE__))
|
5
|
+
|
6
|
+
LOGGER = if ENV['BENCH_REPORT']
|
7
|
+
Logger.new(ROOT_PATH.join(ENV['LOG_NAME']).to_s)
|
8
|
+
else
|
9
|
+
Logger.new(STDOUT)
|
10
|
+
end
|
11
|
+
LOGGER.datetime_format = "" # turn off the datetime in the logs
|
12
|
+
|
13
|
+
PROGRESS_IO = if ENV['BENCH_PROGRESS_IO']
|
14
|
+
::IO.for_fd(ENV['BENCH_PROGRESS_IO'].to_i)
|
15
|
+
else
|
16
|
+
File.open('/dev/null', 'w')
|
17
|
+
end
|
18
|
+
|
19
|
+
Qs.config.dispatcher.queue_name = 'bench-dispatcher'
|
20
|
+
Qs.config.event_publisher = 'Bench Script'
|
21
|
+
Qs.init
|
22
|
+
|
23
|
+
BenchQueue = Qs::Queue.new do
|
24
|
+
name 'bench'
|
25
|
+
|
26
|
+
job 'multiply', 'BenchHandlers::Multiply'
|
27
|
+
|
28
|
+
event 'something', 'happened', 'BenchHandlers::SomethingHappened'
|
29
|
+
|
30
|
+
end
|
31
|
+
BenchQueue.sync_subscriptions
|
32
|
+
|
33
|
+
module BenchHandlers
|
34
|
+
|
35
|
+
class Multiply
|
36
|
+
include Qs::JobHandler
|
37
|
+
|
38
|
+
after{ PROGRESS_IO.write_nonblock('.') }
|
39
|
+
|
40
|
+
def run!
|
41
|
+
'a' * params['size']
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
class SomethingHappened
|
46
|
+
include Qs::EventHandler
|
47
|
+
|
48
|
+
after{ PROGRESS_IO.write_nonblock('.') }
|
49
|
+
|
50
|
+
def run!
|
51
|
+
'a' * params['size']
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
data/lib/qs.rb
CHANGED
@@ -2,6 +2,8 @@ require 'ns-options'
|
|
2
2
|
require 'qs/version'
|
3
3
|
require 'qs/client'
|
4
4
|
require 'qs/daemon'
|
5
|
+
require 'qs/dispatcher_queue'
|
6
|
+
require 'qs/event_handler'
|
5
7
|
require 'qs/job_handler'
|
6
8
|
require 'qs/queue'
|
7
9
|
|
@@ -19,19 +21,27 @@ module Qs
|
|
19
21
|
self.config.redis.db
|
20
22
|
)
|
21
23
|
|
22
|
-
@
|
23
|
-
|
24
|
-
|
25
|
-
|
24
|
+
@dispatcher_queue ||= DispatcherQueue.new({
|
25
|
+
:queue_class => self.config.dispatcher_queue_class,
|
26
|
+
:queue_name => self.config.dispatcher.queue_name,
|
27
|
+
:job_name => self.config.dispatcher.job_name,
|
28
|
+
:job_handler_class_name => self.config.dispatcher.job_handler_class_name
|
29
|
+
})
|
30
|
+
|
31
|
+
@encoder ||= self.config.encoder
|
32
|
+
@decoder ||= self.config.decoder
|
33
|
+
@client ||= Client.new(self.redis_config)
|
34
|
+
@redis ||= @client.redis
|
26
35
|
true
|
27
36
|
end
|
28
37
|
|
29
38
|
def self.reset!
|
30
39
|
self.config.reset
|
31
|
-
@
|
32
|
-
@
|
33
|
-
@
|
34
|
-
@
|
40
|
+
@dispatcher_queue = nil
|
41
|
+
@encoder = nil
|
42
|
+
@decoder = nil
|
43
|
+
@client = nil
|
44
|
+
@redis = nil
|
35
45
|
true
|
36
46
|
end
|
37
47
|
|
@@ -39,16 +49,36 @@ module Qs
|
|
39
49
|
@client.enqueue(queue, job_name, params)
|
40
50
|
end
|
41
51
|
|
52
|
+
def self.publish(channel, name, params = nil)
|
53
|
+
@client.publish(channel, name, params)
|
54
|
+
end
|
55
|
+
|
56
|
+
def self.publish_as(publisher, channel, name, params = nil)
|
57
|
+
@client.publish_as(publisher, channel, name, params)
|
58
|
+
end
|
59
|
+
|
42
60
|
def self.push(queue_name, payload)
|
43
61
|
@client.push(queue_name, payload)
|
44
62
|
end
|
45
63
|
|
46
|
-
def self.
|
47
|
-
@
|
64
|
+
def self.encode(payload)
|
65
|
+
@encoder.call(payload)
|
66
|
+
end
|
67
|
+
|
68
|
+
def self.decode(encoded_payload)
|
69
|
+
@decoder.call(encoded_payload)
|
48
70
|
end
|
49
71
|
|
50
|
-
def self.
|
51
|
-
|
72
|
+
def self.sync_subscriptions(queue)
|
73
|
+
self.client.sync_subscriptions(queue)
|
74
|
+
end
|
75
|
+
|
76
|
+
def self.clear_subscriptions(queue)
|
77
|
+
self.client.clear_subscriptions(queue)
|
78
|
+
end
|
79
|
+
|
80
|
+
def self.event_subscribers(event)
|
81
|
+
self.client.event_subscribers(event)
|
52
82
|
end
|
53
83
|
|
54
84
|
def self.client
|
@@ -63,16 +93,40 @@ module Qs
|
|
63
93
|
self.config.redis.to_hash
|
64
94
|
end
|
65
95
|
|
96
|
+
def self.dispatcher_queue
|
97
|
+
@dispatcher_queue
|
98
|
+
end
|
99
|
+
|
100
|
+
def self.dispatcher_job_name
|
101
|
+
self.config.dispatcher.job_name
|
102
|
+
end
|
103
|
+
|
104
|
+
def self.event_publisher
|
105
|
+
self.config.event_publisher
|
106
|
+
end
|
107
|
+
|
108
|
+
def self.published_events
|
109
|
+
self.dispatcher_queue.published_events
|
110
|
+
end
|
111
|
+
|
66
112
|
class Config
|
67
113
|
include NsOptions::Proxy
|
68
114
|
|
69
|
-
option :
|
70
|
-
option :
|
115
|
+
option :encoder, Proc, :default => proc{ |p| ::JSON.dump(p) }
|
116
|
+
option :decoder, Proc, :default => proc{ |p| ::JSON.load(p) }
|
71
117
|
|
72
118
|
option :timeout, Float
|
73
119
|
|
120
|
+
option :event_publisher, String
|
121
|
+
|
122
|
+
namespace :dispatcher do
|
123
|
+
option :queue_name, String, :default => 'dispatcher'
|
124
|
+
option :job_name, String, :default => 'run_dispatch_job'
|
125
|
+
option :job_handler_class_name, String, :default => DispatcherQueue::RunDispatchJob.to_s
|
126
|
+
end
|
127
|
+
|
74
128
|
namespace :redis do
|
75
|
-
option :ip, :default => '
|
129
|
+
option :ip, :default => '127.0.0.1'
|
76
130
|
option :port, :default => 6379
|
77
131
|
option :db, :default => 0
|
78
132
|
|
@@ -83,6 +137,12 @@ module Qs
|
|
83
137
|
option :timeout, Integer, :default => 1
|
84
138
|
option :size, Integer, :default => 4
|
85
139
|
end
|
140
|
+
|
141
|
+
attr_accessor :dispatcher_queue_class
|
142
|
+
|
143
|
+
def initialize
|
144
|
+
self.dispatcher_queue_class = Queue
|
145
|
+
end
|
86
146
|
end
|
87
147
|
|
88
148
|
module RedisUrl
|
data/lib/qs/client.rb
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
require 'hella-redis'
|
2
2
|
require 'qs'
|
3
|
+
require 'qs/dispatch_job'
|
3
4
|
require 'qs/job'
|
5
|
+
require 'qs/payload'
|
4
6
|
require 'qs/queue'
|
5
7
|
|
6
8
|
module Qs
|
@@ -29,13 +31,24 @@ module Qs
|
|
29
31
|
@redis_config = redis_config
|
30
32
|
end
|
31
33
|
|
32
|
-
def enqueue(queue, job_name,
|
33
|
-
job = Qs::Job.new(job_name, params
|
34
|
+
def enqueue(queue, job_name, job_params = nil)
|
35
|
+
job = Qs::Job.new(job_name, :params => job_params)
|
34
36
|
enqueue!(queue, job)
|
35
37
|
job
|
36
38
|
end
|
37
39
|
|
38
|
-
def
|
40
|
+
def publish(channel, name, params = nil)
|
41
|
+
publish!(channel, name, :event_params => params)
|
42
|
+
end
|
43
|
+
|
44
|
+
def publish_as(publisher, channel, name, params = nil)
|
45
|
+
publish!(channel, name, {
|
46
|
+
:event_params => params,
|
47
|
+
:event_publisher => publisher,
|
48
|
+
})
|
49
|
+
end
|
50
|
+
|
51
|
+
def push(queue_name, payload_hash)
|
39
52
|
raise NotImplementedError
|
40
53
|
end
|
41
54
|
|
@@ -43,12 +56,12 @@ module Qs
|
|
43
56
|
self.redis.with{ |c| c.brpop(*args) }
|
44
57
|
end
|
45
58
|
|
46
|
-
def append(queue_redis_key,
|
47
|
-
self.redis.with{ |c| c.lpush(queue_redis_key,
|
59
|
+
def append(queue_redis_key, encoded_payload)
|
60
|
+
self.redis.with{ |c| c.lpush(queue_redis_key, encoded_payload) }
|
48
61
|
end
|
49
62
|
|
50
|
-
def prepend(queue_redis_key,
|
51
|
-
self.redis.with{ |c| c.rpush(queue_redis_key,
|
63
|
+
def prepend(queue_redis_key, encoded_payload)
|
64
|
+
self.redis.with{ |c| c.rpush(queue_redis_key, encoded_payload) }
|
52
65
|
end
|
53
66
|
|
54
67
|
def clear(redis_key)
|
@@ -59,6 +72,44 @@ module Qs
|
|
59
72
|
self.redis.with{ |c| c.ping }
|
60
73
|
end
|
61
74
|
|
75
|
+
def sync_subscriptions(queue)
|
76
|
+
pattern = Qs::Event::SubscribersRedisKey.new('*')
|
77
|
+
all_event_subs_keys = self.redis.with{ |c| c.keys(pattern) }
|
78
|
+
|
79
|
+
event_subs_keys = queue.event_route_names.map do |route_name|
|
80
|
+
Qs::Event::SubscribersRedisKey.new(route_name)
|
81
|
+
end
|
82
|
+
redis_transaction do |c|
|
83
|
+
all_event_subs_keys.each{ |key| c.srem(key, queue.name) }
|
84
|
+
event_subs_keys.each{ |key| c.sadd(key, queue.name) }
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
def clear_subscriptions(queue)
|
89
|
+
pattern = Qs::Event::SubscribersRedisKey.new('*')
|
90
|
+
event_subs_keys = self.redis.with{ |c| c.keys(pattern) }
|
91
|
+
|
92
|
+
redis_transaction do |c|
|
93
|
+
event_subs_keys.each{ |key| c.srem(key, queue.name) }
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
def event_subscribers(event)
|
98
|
+
self.redis.with{ |c| c.smembers(event.subscribers_redis_key) }
|
99
|
+
end
|
100
|
+
|
101
|
+
private
|
102
|
+
|
103
|
+
def publish!(channel, name, options = nil)
|
104
|
+
dispatch_job = DispatchJob.new(channel, name, options)
|
105
|
+
enqueue!(Qs.dispatcher_queue, dispatch_job)
|
106
|
+
dispatch_job.event
|
107
|
+
end
|
108
|
+
|
109
|
+
def redis_transaction
|
110
|
+
self.redis.with{ |c| c.pipelined{ c.multi{ yield c } } }
|
111
|
+
end
|
112
|
+
|
62
113
|
end
|
63
114
|
|
64
115
|
end
|
@@ -71,17 +122,17 @@ module Qs
|
|
71
122
|
@redis = HellaRedis::Connection.new(self.redis_config)
|
72
123
|
end
|
73
124
|
|
74
|
-
def push(queue_name,
|
75
|
-
queue_redis_key
|
76
|
-
|
77
|
-
self.append(queue_redis_key,
|
125
|
+
def push(queue_name, payload_hash)
|
126
|
+
queue_redis_key = Queue::RedisKey.new(queue_name)
|
127
|
+
encoded_payload = Qs.encode(payload_hash)
|
128
|
+
self.append(queue_redis_key, encoded_payload)
|
78
129
|
end
|
79
130
|
|
80
131
|
private
|
81
132
|
|
82
133
|
def enqueue!(queue, job)
|
83
|
-
|
84
|
-
self.append(queue.redis_key,
|
134
|
+
encoded_payload = Qs::Payload.serialize(job)
|
135
|
+
self.append(queue.redis_key, encoded_payload)
|
85
136
|
end
|
86
137
|
|
87
138
|
end
|
@@ -98,11 +149,11 @@ module Qs
|
|
98
149
|
@pushed_items = []
|
99
150
|
end
|
100
151
|
|
101
|
-
def push(queue_name,
|
102
|
-
# attempt to
|
103
|
-
# on the developer if it can't be
|
104
|
-
Qs.
|
105
|
-
@pushed_items << PushedItem.new(queue_name,
|
152
|
+
def push(queue_name, payload_hash)
|
153
|
+
# attempt to encode (and then throw away) the payload hash, this will
|
154
|
+
# error on the developer if it can't be encoded
|
155
|
+
Qs.encode(payload_hash)
|
156
|
+
@pushed_items << PushedItem.new(queue_name, payload_hash)
|
106
157
|
end
|
107
158
|
|
108
159
|
def reset!
|
@@ -112,13 +163,13 @@ module Qs
|
|
112
163
|
private
|
113
164
|
|
114
165
|
def enqueue!(queue, job)
|
115
|
-
# attempt to serialize (and then throw away) the job
|
116
|
-
#
|
117
|
-
Qs.serialize(job
|
166
|
+
# attempt to serialize (and then throw away) the job, this will error on
|
167
|
+
# the developer if it can't serialize the job
|
168
|
+
Qs::Payload.serialize(job)
|
118
169
|
queue.enqueued_jobs << job
|
119
170
|
end
|
120
171
|
|
121
|
-
PushedItem = Struct.new(:queue_name, :
|
172
|
+
PushedItem = Struct.new(:queue_name, :payload_hash)
|
122
173
|
|
123
174
|
end
|
124
175
|
|