mqrpc 0.0.4 → 0.0.6
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/mqrpc/agent.rb +85 -22
- data/lib/mqrpc/sizedhash.rb +31 -19
- metadata +2 -2
data/lib/mqrpc/agent.rb
CHANGED
@@ -6,6 +6,7 @@ require 'mqrpc/operation'
|
|
6
6
|
require 'mqrpc/sizedhash'
|
7
7
|
require 'thread'
|
8
8
|
require 'uuid'
|
9
|
+
require 'set'
|
9
10
|
|
10
11
|
# http://github.com/tmm1/amqp/issues/#issue/3
|
11
12
|
# This is our (lame) hack to at least notify the user that something is
|
@@ -14,7 +15,7 @@ module AMQP
|
|
14
15
|
module Client
|
15
16
|
alias :original_reconnect :reconnect
|
16
17
|
def reconnect(*args)
|
17
|
-
|
18
|
+
MQRPC::logger.warn "reconnecting to broker (bad MQ settings?)"
|
18
19
|
|
19
20
|
# some rate limiting
|
20
21
|
sleep(5)
|
@@ -30,19 +31,64 @@ module MQRPC
|
|
30
31
|
MAXBUF = 20
|
31
32
|
MAXMESSAGEWAIT = MAXBUF * 20
|
32
33
|
|
34
|
+
class << self
|
35
|
+
attr_accessor :message_handlers
|
36
|
+
attr_accessor :pipelines
|
37
|
+
end
|
38
|
+
|
39
|
+
# Subclasses use this to declare their support of
|
40
|
+
# any given message
|
41
|
+
def self.handle(messageclass, method)
|
42
|
+
if self.message_handlers == nil
|
43
|
+
self.message_handlers = Hash.new
|
44
|
+
end
|
45
|
+
self.message_handlers[messageclass] = method
|
46
|
+
end
|
47
|
+
|
48
|
+
def self.pipeline(source, destination)
|
49
|
+
if self.pipelines == nil
|
50
|
+
self.pipelines = Hash.new
|
51
|
+
end
|
52
|
+
|
53
|
+
self.pipelines[destination] = source
|
54
|
+
end
|
55
|
+
|
33
56
|
def initialize(config)
|
34
57
|
Thread::abort_on_exception = true
|
35
58
|
@config = config
|
36
59
|
@handler = self
|
37
60
|
@id = UUID::generate
|
38
61
|
@outbuffer = Hash.new { |h,k| h[k] = [] }
|
39
|
-
@queues =
|
40
|
-
@topics =
|
62
|
+
@queues = Set.new
|
63
|
+
@topics = Set.new
|
41
64
|
@receive_queue = Queue.new
|
42
65
|
@want_subscriptions = Queue.new
|
66
|
+
|
67
|
+
# figure out how to really do this correctly, see also def self.pipeline
|
68
|
+
if self.class.pipelines == nil
|
69
|
+
self.class.pipelines = Hash.new
|
70
|
+
end
|
71
|
+
|
43
72
|
@slidingwindow = Hash.new do |h,k|
|
44
|
-
MQRPC::logger.
|
45
|
-
h[k] = SizedThreadSafeHash.new(MAXMESSAGEWAIT)
|
73
|
+
MQRPC::logger.debug "New sliding window for #{k}"
|
74
|
+
h[k] = SizedThreadSafeHash.new(MAXMESSAGEWAIT) do |state|
|
75
|
+
if self.class.pipelines[k]
|
76
|
+
source = self.class.pipelines[k]
|
77
|
+
MQRPC::logger.debug "Got sizedhash callback for #{k}: #{state}"
|
78
|
+
case state
|
79
|
+
when :blocked
|
80
|
+
MQRPC::logger.info("Queue '#{k}' is full, unsubscribing from #{source}")
|
81
|
+
exchange = @mq.topic(@config.mqexchange, :durable => true)
|
82
|
+
mq_q = @mq.queue(source, :durable => true)
|
83
|
+
mq_q.bind(exchange, :key => "*")
|
84
|
+
mq_q.unsubscribe
|
85
|
+
@queues.delete(source)
|
86
|
+
when :ready
|
87
|
+
MQRPC::logger.info("Queue '#{k}' is ready, resubscribing to #{source}")
|
88
|
+
subscribe(source)
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
46
92
|
end
|
47
93
|
|
48
94
|
@mq = nil
|
@@ -84,9 +130,7 @@ module MQRPC
|
|
84
130
|
end
|
85
131
|
|
86
132
|
MQRPC::logger.info "Subscribing to main queue #{@id}"
|
87
|
-
|
88
|
-
mq_q.subscribe(:ack =>true) { |hdr, msg| @receive_queue << [hdr, msg] }
|
89
|
-
#handle_new_subscriptions
|
133
|
+
subscribe(@id)
|
90
134
|
|
91
135
|
# TODO(sissel): make this a deferred thread that reads from a Queue
|
92
136
|
#EM.add_periodic_timer(5) { handle_new_subscriptions }
|
@@ -119,27 +163,39 @@ module MQRPC
|
|
119
163
|
end # def subscribe_topic
|
120
164
|
|
121
165
|
def handle_message(hdr, msg_body)
|
166
|
+
queue = hdr.routing_key
|
167
|
+
|
168
|
+
# If we just unsubscribed from a queue, we may still have some
|
169
|
+
# messages buffered, so reject the message.
|
170
|
+
# Currently RabbitMQ doesn't support message rejection, so let's
|
171
|
+
# ack the message then push it back into the queue, unmodified.
|
172
|
+
if !@queues.include?(queue)
|
173
|
+
MQRPC::logger.warn("Got message on queue #{queue} that we are not "\
|
174
|
+
"subscribed to; rejecting")
|
175
|
+
hdr.ack
|
176
|
+
@mq.queue(queue, :durable => true).publish(msg_body, :persistent => true)
|
177
|
+
return
|
178
|
+
end
|
179
|
+
|
180
|
+
# TODO(sissel): handle any exceptions we might get from here.
|
122
181
|
obj = JSON::load(msg_body)
|
123
182
|
if !obj.is_a?(Array)
|
124
183
|
obj = [obj]
|
125
184
|
end
|
126
185
|
|
127
|
-
queue = hdr.routing_key
|
128
186
|
obj.each do |item|
|
129
187
|
message = Message.new_from_data(item)
|
130
188
|
slidingwindow = @slidingwindow[queue]
|
131
189
|
if message.respond_to?(:from_queue)
|
132
190
|
slidingwindow = @slidingwindow[message.from_queue]
|
133
191
|
end
|
134
|
-
MQRPC::logger.debug "
|
135
|
-
MQRPC::logger.debug "Received message: #{message.inspect}"
|
192
|
+
MQRPC::logger.debug "Got message #{message.class}##{message.id} on queue #{queue}"
|
193
|
+
#MQRPC::logger.debug "Received message: #{message.inspect}"
|
136
194
|
if (message.respond_to?(:in_reply_to) and
|
137
195
|
slidingwindow.include?(message.in_reply_to))
|
138
196
|
MQRPC::logger.debug "Got response to #{message.in_reply_to}"
|
139
197
|
slidingwindow.delete(message.in_reply_to)
|
140
198
|
end
|
141
|
-
name = message.class.name.split(":")[-1]
|
142
|
-
func = "#{name}Handler"
|
143
199
|
|
144
200
|
# Check if we have a specific operation looking for this
|
145
201
|
# message.
|
@@ -147,19 +203,20 @@ module MQRPC
|
|
147
203
|
@message_operations.has_key?(message.in_reply_to))
|
148
204
|
operation = @message_operations[message.in_reply_to]
|
149
205
|
operation.call(message)
|
150
|
-
elsif
|
151
|
-
|
206
|
+
elsif can_receive?(message.class)
|
207
|
+
func = self.class.message_handlers[message.class]
|
208
|
+
self.send(func, message) do |response|
|
152
209
|
reply_destination = message.reply_to
|
153
210
|
response.from_queue = queue
|
154
211
|
sendmsg(reply_destination, response)
|
155
212
|
end
|
156
213
|
|
157
|
-
# We should allow the message handler to defer acking
|
158
|
-
# For instance, if we want to index things, but only
|
159
|
-
# things once we actually flush to disk.
|
214
|
+
# TODO(sissel): We should allow the message handler to defer acking
|
215
|
+
# if they want For instance, if we want to index things, but only
|
216
|
+
# want to ack things once we actually flush to disk.
|
160
217
|
else
|
161
|
-
$stderr.puts "#{@handler.class.name} does not support #{
|
162
|
-
end
|
218
|
+
$stderr.puts "#{@handler.class.name} does not support #{message.class}"
|
219
|
+
end
|
163
220
|
end
|
164
221
|
hdr.ack
|
165
222
|
end # def handle_message
|
@@ -168,6 +225,10 @@ module MQRPC
|
|
168
225
|
@amqpthread.join
|
169
226
|
end # run
|
170
227
|
|
228
|
+
def can_receive?(message_class)
|
229
|
+
return self.class.message_handlers.include?(message_class)
|
230
|
+
end
|
231
|
+
|
171
232
|
def handle_subscriptions
|
172
233
|
while true do
|
173
234
|
type, name = @want_subscriptions.pop
|
@@ -175,12 +236,14 @@ module MQRPC
|
|
175
236
|
when :queue
|
176
237
|
next if @queues.include?(name)
|
177
238
|
MQRPC::logger.info "Subscribing to queue #{name}"
|
239
|
+
exchange = @mq.topic(@config.mqexchange, :durable => true)
|
178
240
|
mq_q = @mq.queue(name, :durable => true)
|
241
|
+
mq_q.bind(exchange, :key => "*")
|
179
242
|
mq_q.subscribe(:ack => true) { |hdr, msg| @receive_queue << [hdr, msg] }
|
180
243
|
@queues << name
|
181
244
|
when :topic
|
182
245
|
MQRPC::logger.info "Subscribing to topic #{name}"
|
183
|
-
exchange = @mq.topic(@config.mqexchange)
|
246
|
+
exchange = @mq.topic(@config.mqexchange, :durable => true)
|
184
247
|
mq_q = @mq.queue("#{@id}-#{name}",
|
185
248
|
:exclusive => true,
|
186
249
|
:auto_delete => true).bind(exchange, :key => name)
|
@@ -247,7 +310,7 @@ module MQRPC
|
|
247
310
|
flushout(destination)
|
248
311
|
end
|
249
312
|
else
|
250
|
-
MQRPC::logger.debug "
|
313
|
+
MQRPC::logger.debug "Sending to #{destination}: #{msg.inspect}"
|
251
314
|
@mq.queue(destination, :durable => true).publish([msg].to_json, :persistent => true)
|
252
315
|
end
|
253
316
|
|
data/lib/mqrpc/sizedhash.rb
CHANGED
@@ -1,15 +1,28 @@
|
|
1
1
|
require 'thread'
|
2
2
|
require 'mqrpc'
|
3
3
|
|
4
|
+
class TrackingMutex < Mutex
|
5
|
+
def synchronize(&blk)
|
6
|
+
#puts "Enter synchronize #{self} @ #{Thread.current} + #{caller[0]}"
|
7
|
+
super { blk.call }
|
8
|
+
#puts "Exit synchronize #{self} @ #{Thread.current} + #{caller[0]}"
|
9
|
+
end # def synchronize
|
10
|
+
end # clas TrackingMutex < Mutex
|
11
|
+
|
4
12
|
# Thread-safe sized hash similar to SizedQueue.
|
5
13
|
# The only time we will block execution is in setting new items.
|
6
14
|
# That is, SizedThreadSafeHash#[]=
|
7
15
|
class SizedThreadSafeHash
|
8
|
-
|
9
|
-
|
16
|
+
attr_reader :callback
|
17
|
+
attr_reader :size
|
18
|
+
|
19
|
+
def initialize(size, &callback)
|
20
|
+
@lock = TrackingMutex.new
|
10
21
|
@size = size
|
11
22
|
@condvar = ConditionVariable.new
|
12
23
|
@data = Hash.new
|
24
|
+
@callback = callback
|
25
|
+
@state = nil
|
13
26
|
end # def initialize
|
14
27
|
|
15
28
|
# set a key and value
|
@@ -17,11 +30,23 @@ class SizedThreadSafeHash
|
|
17
30
|
@lock.synchronize do
|
18
31
|
# If adding a new item, wait if the hash is full
|
19
32
|
if !@data.has_key?(key) and _withlock_full?
|
20
|
-
MQRPC::logger.info "#{self}: Waiting to add key #{key.inspect}, hash is full"
|
21
|
-
|
22
|
-
|
23
|
-
|
33
|
+
MQRPC::logger.info "#{self}: Waiting to add key #{key.inspect}, hash is full (thraed #{Thread.current})"
|
34
|
+
if @state != :blocked
|
35
|
+
@state = :blocked
|
36
|
+
@callback.call(@state) if @callback
|
37
|
+
else
|
38
|
+
puts "State is already #{@state} (want :blocked), skipping event call"
|
39
|
+
end
|
40
|
+
|
24
41
|
@condvar.wait(@lock)
|
42
|
+
|
43
|
+
if @state != :ready
|
44
|
+
@state = :ready
|
45
|
+
MQRPC::logger.info "#{self}: state => :ready"
|
46
|
+
@callback.call(@state) if @callback
|
47
|
+
else
|
48
|
+
puts "State is already #{@state} (want :ready), skipping event call"
|
49
|
+
end
|
25
50
|
end
|
26
51
|
@data[key] = value
|
27
52
|
end
|
@@ -82,16 +107,3 @@ class SizedThreadSafeHash
|
|
82
107
|
return @data.size >= @size
|
83
108
|
end
|
84
109
|
end # class SizedThreadSafeHash
|
85
|
-
|
86
|
-
#sh = SizedThreadSafeHash.new(10)
|
87
|
-
#Thread.new do
|
88
|
-
#0.upto(100) do |i|
|
89
|
-
#sh[i] = true
|
90
|
-
#end
|
91
|
-
#end
|
92
|
-
|
93
|
-
#0.upto(100) do |i|
|
94
|
-
#puts sh[i]
|
95
|
-
#sh.delete(i)
|
96
|
-
#sleep 1
|
97
|
-
#end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mqrpc
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jordan Sissel, Pete Fritchman
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2009-11-
|
12
|
+
date: 2009-11-14 00:00:00 -08:00
|
13
13
|
default_executable:
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|