reliable-msg 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/MIT-LICENSE +20 -0
- data/README +123 -0
- data/Rakefile +164 -0
- data/bin/queues +8 -0
- data/lib/reliable-msg.rb +12 -0
- data/lib/reliable-msg/cli.rb +170 -0
- data/lib/reliable-msg/message-store.rb +509 -0
- data/lib/reliable-msg/mysql.sql +8 -0
- data/lib/reliable-msg/queue-manager.rb +409 -0
- data/lib/reliable-msg/queue.rb +500 -0
- data/lib/reliable-msg/selector.rb +109 -0
- data/lib/uuid.rb +384 -0
- data/test/test-queue.rb +144 -0
- data/test/test-uuid.rb +48 -0
- metadata +63 -0
@@ -0,0 +1,8 @@
|
|
1
|
+
DROP TABLE IF EXISTS `reliable_msg_queues`;
|
2
|
+
CREATE TABLE `reliable_msg_queues` (
|
3
|
+
`id` varchar(255) NOT NULL default '',
|
4
|
+
`queue` varchar(255) NOT NULL default '',
|
5
|
+
`headers` text NOT NULL,
|
6
|
+
`object` blob NOT NULL,
|
7
|
+
PRIMARY KEY (`id`)
|
8
|
+
) ENGINE=InnoDB DEFAULT CHARSET=binary;
|
@@ -0,0 +1,409 @@
|
|
1
|
+
#
|
2
|
+
# = queue-manager.rb - Queue manager
|
3
|
+
#
|
4
|
+
# Author:: Assaf Arkin assaf@labnotes.org
|
5
|
+
# Documentation:: http://trac.labnotes.org/cgi-bin/trac.cgi/wiki/RubyReliableMessaging
|
6
|
+
# Copyright:: Copyright (c) 2005 Assaf Arkin
|
7
|
+
# License:: MIT and/or Creative Commons Attribution-ShareAlike
|
8
|
+
#
|
9
|
+
#--
|
10
|
+
# Changes:
|
11
|
+
#++
|
12
|
+
|
13
|
+
require 'singleton'
|
14
|
+
require 'drb'
|
15
|
+
require 'drb/acl'
|
16
|
+
require 'thread'
|
17
|
+
require 'yaml'
|
18
|
+
require 'uuid'
|
19
|
+
require 'reliable-msg/queue'
|
20
|
+
require 'reliable-msg/message-store'
|
21
|
+
|
22
|
+
module ReliableMsg
|
23
|
+
|
24
|
+
class Config #:nodoc:
|
25
|
+
|
26
|
+
CONFIG_FILE = "queues.cfg"
|
27
|
+
|
28
|
+
DEFAULT_STORE = MessageStore::Disk::DEFAULT_CONFIG
|
29
|
+
|
30
|
+
DEFAULT_DRB = {
|
31
|
+
"port"=>Queue::DRB_PORT,
|
32
|
+
"acl"=>"allow 127.0.0.1"
|
33
|
+
}
|
34
|
+
|
35
|
+
def initialize file, logger = nil
|
36
|
+
@logger = logger
|
37
|
+
# If no file specified, attempt to look for file in current directory.
|
38
|
+
# If not found in current directory, look for file in Gem directory.
|
39
|
+
unless file
|
40
|
+
file = if File.exist?(CONFIG_FILE)
|
41
|
+
CONFIG_FILE
|
42
|
+
else
|
43
|
+
file = File.expand_path(File.join(File.dirname(__FILE__), '..'))
|
44
|
+
File.basename(file) == 'lib' ? File.join(file, '..', CONFIG_FILE) : File.join(file, CONFIG_FILE)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
@file = File.expand_path(file)
|
48
|
+
@config = {}
|
49
|
+
end
|
50
|
+
|
51
|
+
def load_no_create
|
52
|
+
if File.exist?(@file)
|
53
|
+
@config= {}
|
54
|
+
File.open @file, "r" do |input|
|
55
|
+
YAML.load_documents input do |doc|
|
56
|
+
@config.merge! doc
|
57
|
+
end
|
58
|
+
end
|
59
|
+
true
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def load_or_create
|
64
|
+
if File.exist?(@file)
|
65
|
+
@config= {}
|
66
|
+
File.open @file, "r" do |input|
|
67
|
+
YAML.load_documents input do |doc|
|
68
|
+
@config.merge! doc
|
69
|
+
end
|
70
|
+
end
|
71
|
+
@logger.info "Loaded queues configuration from: #{@file}"
|
72
|
+
else
|
73
|
+
@config = {
|
74
|
+
"store" => DEFAULT_STORE,
|
75
|
+
"drb" => DEFAULT_DRB
|
76
|
+
}
|
77
|
+
save
|
78
|
+
@logger.info "Created queues configuration file in: #{@file}"
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
def create_if_none
|
83
|
+
if File.exist?(@file)
|
84
|
+
false
|
85
|
+
else
|
86
|
+
@config = {
|
87
|
+
"store" => DEFAULT_STORE,
|
88
|
+
"drb" => DEFAULT_DRB
|
89
|
+
}.merge(@config)
|
90
|
+
save
|
91
|
+
true
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
def exist?
|
96
|
+
File.exist?(@file)
|
97
|
+
end
|
98
|
+
|
99
|
+
def path
|
100
|
+
@file
|
101
|
+
end
|
102
|
+
|
103
|
+
def save
|
104
|
+
File.open @file, "w" do |output|
|
105
|
+
YAML::dump @config, output
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
def method_missing symbol, *args
|
110
|
+
if symbol.to_s[-1] == ?=
|
111
|
+
@config[symbol.to_s[0...-1]] = *args
|
112
|
+
else
|
113
|
+
@config[symbol.to_s]
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
end
|
118
|
+
|
119
|
+
|
120
|
+
class QueueManager
|
121
|
+
|
122
|
+
TX_TIMEOUT_CHECK_EVERY = 30
|
123
|
+
|
124
|
+
ERROR_SEND_MISSING_QUEUE = "You must specify a destination queue for the message" #:nodoc:
|
125
|
+
|
126
|
+
ERROR_RECEIVE_MISSING_QUEUE = "You must specify a queue to retrieve the message from" #:nodoc:
|
127
|
+
|
128
|
+
ERROR_INVALID_HEADER_NAME = "Invalid header '%s': expecting the name to be a symbol, found object of type %s" #:nodoc:
|
129
|
+
|
130
|
+
ERROR_INVALID_HEADER_VALUE = "Invalid header '%s': expecting the value to be %s, found object of type %s" #:nodoc:
|
131
|
+
|
132
|
+
ERROR_NO_TRANSACTION = "Transaction %s has completed, or was aborted" #:nodoc:
|
133
|
+
|
134
|
+
def initialize options = nil
|
135
|
+
options ||= {}
|
136
|
+
# Locks prevent two transactions from seeing the same message. We use a mutex
|
137
|
+
# to ensure that each transaction can determine the state of a lock before
|
138
|
+
# setting it.
|
139
|
+
@mutex = Mutex.new
|
140
|
+
@locks = {}
|
141
|
+
# Transactions use this hash to hold all inserted messages (:inserts), deleted
|
142
|
+
# messages (:deletes) and the transaction timeout (:timeout) until completion.
|
143
|
+
@transactions = {}
|
144
|
+
@logger = options[:logger] || Logger.new(STDOUT)
|
145
|
+
@config = Config.new options[:config], @logger
|
146
|
+
@config.load_or_create
|
147
|
+
end
|
148
|
+
|
149
|
+
def start
|
150
|
+
@mutex.synchronize do
|
151
|
+
return if @started
|
152
|
+
|
153
|
+
# Get the message store based on the configuration, or default store.
|
154
|
+
@store = MessageStore::Base.configure(@config.store || Config::DEFAULT_STORE, @logger)
|
155
|
+
@logger.info "Using message store #{@store.type}"
|
156
|
+
@store.activate
|
157
|
+
|
158
|
+
# Get the DRb URI from the configuration, or use the default. Create a DRb server.
|
159
|
+
drb = Config::DEFAULT_DRB
|
160
|
+
drb.merge(@config.drb) if @config.drb
|
161
|
+
drb_uri = "druby://localhost:#{drb['port']}"
|
162
|
+
@drb_server = DRb::DRbServer.new drb_uri, self, :tcp_acl=>ACL.new(drb["acl"].split(" "), ACL::ALLOW_DENY), :verbose=>true
|
163
|
+
@logger.info "Accepting requests at '#{drb_uri}'"
|
164
|
+
|
165
|
+
# Create a background thread to stop timed-out transactions.
|
166
|
+
@timeout_thread = Thread.new do
|
167
|
+
begin
|
168
|
+
while true
|
169
|
+
time = Time.new.to_i
|
170
|
+
@transactions.each_pair do |tid, tx|
|
171
|
+
if tx[:timeout] <= time
|
172
|
+
begin
|
173
|
+
@logger.warn "Timeout: aborting transaction #{tid}"
|
174
|
+
abort tid
|
175
|
+
rescue
|
176
|
+
end
|
177
|
+
end
|
178
|
+
end
|
179
|
+
sleep TX_TIMEOUT_CHECK_EVERY
|
180
|
+
end
|
181
|
+
rescue Exception=>error
|
182
|
+
retry
|
183
|
+
end
|
184
|
+
end
|
185
|
+
|
186
|
+
# Associate this queue manager with the local Queue class, instead of using DRb.
|
187
|
+
Queue.send :qm=, self
|
188
|
+
@started = true
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
def stop
|
193
|
+
@mutex.synchronize do
|
194
|
+
return unless @started
|
195
|
+
|
196
|
+
# Prevent transactions from timing out while we take down the server.
|
197
|
+
@timeout_thread.terminate
|
198
|
+
# Shutdown DRb server to prevent new requests from being processed.\
|
199
|
+
Queue.send :qm=, nil
|
200
|
+
drb_uri = @drb_server.uri
|
201
|
+
@drb_server.stop_service
|
202
|
+
# Deactivate the message store.
|
203
|
+
@store.deactivate
|
204
|
+
@store = nil
|
205
|
+
@drb_server = @store = @timeout_thread = nil
|
206
|
+
@logger.info "Stopped queue manager at '#{drb_uri}'"
|
207
|
+
end
|
208
|
+
end
|
209
|
+
|
210
|
+
def alive?
|
211
|
+
@drb_server && @drb_server.alive?
|
212
|
+
end
|
213
|
+
|
214
|
+
def queue args
|
215
|
+
# Get the arguments of this call.
|
216
|
+
message, headers, queue, tid = args[:message], args[:headers], args[:queue].downcase, args[:tid]
|
217
|
+
raise ArgumentError, ERROR_SEND_MISSING_QUEUE unless queue and queue.instance_of?(String) and !queue.empty?
|
218
|
+
time = Time.new.to_i
|
219
|
+
# TODO: change this to support the RM delivery protocol.
|
220
|
+
id = args[:id] || UUID.new
|
221
|
+
created = args[:created] || time
|
222
|
+
|
223
|
+
# Validate and freeze the headers. The cloning ensures that the headers we hold in memory
|
224
|
+
# are not modified by the caller. The validation ensures that the headers we hold in memory
|
225
|
+
# can be persisted safely. Basic types like string and integer are allowed, but application types
|
226
|
+
# may prevent us from restoring the index. Strings are cloned since strings may be replaced.
|
227
|
+
headers = if headers
|
228
|
+
copy = {}
|
229
|
+
headers.each_pair do |name, value|
|
230
|
+
raise ArgumentError, format(ERROR_INVALID_HEADER_NAME, name, name.class) unless name.instance_of?(Symbol)
|
231
|
+
case value
|
232
|
+
when String, Numeric, Symbol, true, false, nil
|
233
|
+
copy[name] = value.freeze
|
234
|
+
else
|
235
|
+
raise ArgumentError, format(ERROR_INVALID_HEADER_VALUE, name, "a string, numeric, symbol, true/false or nil", value.class)
|
236
|
+
end
|
237
|
+
end
|
238
|
+
copy
|
239
|
+
else
|
240
|
+
{}
|
241
|
+
end
|
242
|
+
|
243
|
+
# Set the message headers controlled by the queue.
|
244
|
+
headers[:id] = id
|
245
|
+
headers[:received] = time
|
246
|
+
headers[:delivery] ||= :best_effort
|
247
|
+
headers[:retry] = 0
|
248
|
+
headers[:max_retries] = integer headers[:max_retries], 0, Queue::DEFAULT_MAX_RETRIES
|
249
|
+
headers[:priority] = integer headers[:priority], 0, 0
|
250
|
+
if expires_at = headers[:expires_at]
|
251
|
+
raise ArgumentError, format(ERROR_INVALID_HEADER_VALUE, :expires_at, "an integer", expires_at.class) unless expires_at.is_a?(Integer)
|
252
|
+
elsif expires = headers[:expires]
|
253
|
+
raise ArgumentError, format(ERROR_INVALID_HEADER_VALUE, :expires, "an integer", expires.class) unless expires.is_a?(Integer)
|
254
|
+
headers[:expires_at] = Time.now.to_i + expires if expires > 0
|
255
|
+
end
|
256
|
+
# Create an insertion record for the new message.
|
257
|
+
insert = {:id=>id, :queue=>queue, :headers=>headers, :message=>message}
|
258
|
+
if tid
|
259
|
+
tx = @transactions[tid]
|
260
|
+
raise RuntimeError, format(ERROR_NO_TRANSACTION, tid) unless tx
|
261
|
+
tx[:inserts] << insert
|
262
|
+
else
|
263
|
+
@store.transaction do |inserts, deletes, dlqs|
|
264
|
+
inserts << insert
|
265
|
+
end
|
266
|
+
end
|
267
|
+
# Return the message identifier.
|
268
|
+
id
|
269
|
+
end
|
270
|
+
|
271
|
+
|
272
|
+
def enqueue args
|
273
|
+
# Get the arguments of this call.
|
274
|
+
queue, selector, tid = args[:queue].downcase, args[:selector], args[:tid]
|
275
|
+
id, headers = nil, nil
|
276
|
+
raise ArgumentError, ERROR_RECEIVE_MISSING_QUEUE unless queue and queue.instance_of?(String) and !queue.empty?
|
277
|
+
|
278
|
+
# We need to lock the selected message, before deleting, otherwise,
|
279
|
+
# we allow another transaction to see the message we're about to delete.
|
280
|
+
# This is true whether we delete the message inside or outside a client
|
281
|
+
# transaction. We can wrap everything with a mutex, but it's faster to
|
282
|
+
# release the locks mutex as fast as possibe.
|
283
|
+
message = @mutex.synchronize do
|
284
|
+
message = @store.select queue do |headers|
|
285
|
+
not @locks.has_key?(headers[:id]) and case selector
|
286
|
+
when nil
|
287
|
+
true
|
288
|
+
when String
|
289
|
+
headers[:id] == selector
|
290
|
+
when Hash
|
291
|
+
selector.all? { |name, value| headers[name] == value }
|
292
|
+
when Selector
|
293
|
+
selector.__evaluate__ headers
|
294
|
+
end
|
295
|
+
end
|
296
|
+
if message
|
297
|
+
@locks[message[:id]] = true
|
298
|
+
message
|
299
|
+
end
|
300
|
+
end
|
301
|
+
# Nothing to do if no message found.
|
302
|
+
return unless message
|
303
|
+
|
304
|
+
# If the message has expired, or maximum retry count elapsed, we either
|
305
|
+
# discard the message, or send it to the DLQ. Since we're out of a message,
|
306
|
+
# we call to get a new one. (This can be changed to repeat instead of recurse).
|
307
|
+
headers = message[:headers]
|
308
|
+
if queue != Queue::DLQ && ((headers[:expires_at] && headers[:expires_at] < Time.now.to_i) || (headers[:retry] > headers[:max_retries]))
|
309
|
+
expired = {:id=>message[:id], :queue=>queue, :headers=>headers}
|
310
|
+
if headers[:delivery] == :once || headers[:delivery] == :repeated
|
311
|
+
@store.transaction { |inserts, deletes, dlqs| dlqs << expired }
|
312
|
+
else # :best_effort
|
313
|
+
@store.transaction { |inserts, deletes, dlqs| deletes << expired }
|
314
|
+
end
|
315
|
+
@mutex.synchronize { @locks.delete message[:id] }
|
316
|
+
return enqueue(args)
|
317
|
+
end
|
318
|
+
|
319
|
+
delete = {:id=>message[:id], :queue=>queue, :headers=>headers}
|
320
|
+
begin
|
321
|
+
if tid
|
322
|
+
tx = @transactions[tid]
|
323
|
+
raise RuntimeError, format(ERROR_NO_TRANSACTION, tid) unless tx
|
324
|
+
if queue != Queue::DLQ && headers[:delivery] == :once
|
325
|
+
# Exactly once delivery: immediately move message to DLQ, so if
|
326
|
+
# transaction aborts, message is not retrieved again. Do not
|
327
|
+
# release lock here, to prevent message retrieved from DLQ.
|
328
|
+
# Change delete record so message removed from DLQ on commit.
|
329
|
+
@store.transaction do |inserts, deletes, dlqs|
|
330
|
+
dlqs << delete
|
331
|
+
end
|
332
|
+
delete[:queue] = Queue::DLQ
|
333
|
+
tx[:deletes] << delete
|
334
|
+
else
|
335
|
+
# At most once delivery: delete message if transaction commits.
|
336
|
+
# Best effort: we don't need to delete on commit, but it's more
|
337
|
+
# efficient this way.
|
338
|
+
# Exactly once: message never gets to expire in DLQ.
|
339
|
+
tx[:deletes] << delete
|
340
|
+
end
|
341
|
+
else
|
342
|
+
@store.transaction do |inserts, deletes, dlqs|
|
343
|
+
deletes << delete
|
344
|
+
end
|
345
|
+
@mutex.synchronize { @locks.delete message[:id] }
|
346
|
+
end
|
347
|
+
rescue Exception=>error
|
348
|
+
# Because errors do happen.
|
349
|
+
@mutex.synchronize { @locks.delete message[:id] }
|
350
|
+
raise error
|
351
|
+
end
|
352
|
+
|
353
|
+
# To prevent a transaction from modifying a message and then returning it to the
|
354
|
+
# queue by aborting, we instead clone the message by de-serializing (this happens
|
355
|
+
# in Queue, see there). The headers are also cloned (shallow, all values are frozen).
|
356
|
+
return :id=>message[:id], :headers=>message[:headers].clone, :message=>message[:message]
|
357
|
+
end
|
358
|
+
|
359
|
+
def begin timeout
|
360
|
+
tid = UUID.new
|
361
|
+
@transactions[tid] = {:inserts=>[], :deletes=>[], :timeout=>Time.new.to_i + timeout}
|
362
|
+
tid
|
363
|
+
end
|
364
|
+
|
365
|
+
def commit tid
|
366
|
+
tx = @transactions[tid]
|
367
|
+
raise RuntimeError, format(ERROR_NO_TRANSACTION, tid) unless tx
|
368
|
+
begin
|
369
|
+
@store.transaction do |inserts, deletes, dlqs|
|
370
|
+
inserts.concat tx[:inserts]
|
371
|
+
deletes.concat tx[:deletes]
|
372
|
+
end
|
373
|
+
# Release locks here, otherwise we expose messages before the
|
374
|
+
# transaction gets the chance to delete them from the queue.
|
375
|
+
@mutex.synchronize do
|
376
|
+
tx[:deletes].each { |delete| @locks.delete delete[:id] }
|
377
|
+
end
|
378
|
+
@transactions.delete tid
|
379
|
+
rescue Exception=>error
|
380
|
+
abort tid
|
381
|
+
raise error
|
382
|
+
end
|
383
|
+
end
|
384
|
+
|
385
|
+
def abort tid
|
386
|
+
tx = @transactions[tid]
|
387
|
+
raise RuntimeError, format(ERROR_NO_TRANSACTION, tid) unless tx
|
388
|
+
# Release locks here because we are no longer in posession of any
|
389
|
+
# retrieved messages.
|
390
|
+
@mutex.synchronize do
|
391
|
+
tx[:deletes].each do |delete|
|
392
|
+
@locks.delete delete[:id]
|
393
|
+
delete[:headers][:retry] += 1
|
394
|
+
end
|
395
|
+
end
|
396
|
+
@transactions.delete tid
|
397
|
+
@logger.warn "Transaction #{tid} aborted"
|
398
|
+
end
|
399
|
+
|
400
|
+
private
|
401
|
+
def integer value, minimum, default
|
402
|
+
return default unless value
|
403
|
+
value = value.to_i
|
404
|
+
value > minimum ? value : minimum
|
405
|
+
end
|
406
|
+
|
407
|
+
end
|
408
|
+
|
409
|
+
end
|
@@ -0,0 +1,500 @@
|
|
1
|
+
#
|
2
|
+
# = queue.rb - Reliable queue client API
|
3
|
+
#
|
4
|
+
# Author:: Assaf Arkin assaf@labnotes.org
|
5
|
+
# Documentation:: http://trac.labnotes.org/cgi-bin/trac.cgi/wiki/RubyReliableMessaging
|
6
|
+
# Copyright:: Copyright (c) 2005 Assaf Arkin
|
7
|
+
# License:: MIT and/or Creative Commons Attribution-ShareAlike
|
8
|
+
#
|
9
|
+
#--
|
10
|
+
# Changes:
|
11
|
+
#++
|
12
|
+
|
13
|
+
require 'drb'
|
14
|
+
require 'reliable-msg/selector'
|
15
|
+
|
16
|
+
|
17
|
+
module ReliableMsg
|
18
|
+
|
19
|
+
# == Reliable Messaging Client API
|
20
|
+
#
|
21
|
+
# Use the Queue object to put messages in queues, or get messages from queues.
|
22
|
+
#
|
23
|
+
# You can create a Queue object that connects to a single queue by passing the
|
24
|
+
# queue name to the initialized. You can also access other queues by specifying
|
25
|
+
# the destination queue when putting a message, or selecting from a queue when
|
26
|
+
# retrieving the message.
|
27
|
+
#
|
28
|
+
# For example:
|
29
|
+
# queue = Queue.new 'my-queue'
|
30
|
+
# # Put a message in the queue with priority 2, expiring in 30 seconds.
|
31
|
+
# msg = 'lorem ipsum'
|
32
|
+
# mid = queue.put msg, :priority=>2, :expires=>30
|
33
|
+
# # Retrieve and process a message from the queue.
|
34
|
+
# queue.get do |msg|
|
35
|
+
# if msg.id == mid
|
36
|
+
# print "Retrieved same message"
|
37
|
+
# end
|
38
|
+
# print "Message text: #{msg.object}"
|
39
|
+
# end
|
40
|
+
#
|
41
|
+
# See Queue.get and Queue.put for more examples.
|
42
|
+
class Queue
|
43
|
+
|
44
|
+
ERROR_INVALID_SELECTOR = 'Selector must be message identifier (String), set of header name/value pairs (Hash), or nil' # :nodoc:
|
45
|
+
|
46
|
+
ERROR_INVALID_TX_TIMEOUT = 'Invalid transaction timeout: must be a non-zero positive integer' # :nodoc:
|
47
|
+
|
48
|
+
ERROR_INVALID_CONNECT_COUNT = 'Invalid connection count: must be a non-zero positive integer' # :nodoc:
|
49
|
+
|
50
|
+
ERROR_SELECTOR_VALUE_OR_BLOCK = 'You can either pass a Selector object, or use a block' # :nodoc:
|
51
|
+
|
52
|
+
# The default DRb port used to connect to the queue manager.
|
53
|
+
DRB_PORT = 6438
|
54
|
+
|
55
|
+
DEFAULT_DRB_URI = "druby://localhost:#{DRB_PORT}" #:nodoc:
|
56
|
+
|
57
|
+
# The name of the dead letter queue (<tt>DLQ</tt>). Messages that expire or fail
|
58
|
+
# to process are automatically sent to the dead letter queue.
|
59
|
+
DLQ = DEAD_LETTER_QUEUE = 'dlq'
|
60
|
+
|
61
|
+
# Number of times to retry a connecting to the queue manager.
|
62
|
+
DEFAULT_CONNECT_RETRY = 5
|
63
|
+
|
64
|
+
# Default transaction timeout.
|
65
|
+
DEFAULT_TX_TIMEOUT = 120
|
66
|
+
|
67
|
+
# Default number of re-delivery attempts.
|
68
|
+
DEFAULT_MAX_RETRIES = 4;
|
69
|
+
|
70
|
+
# Thread.current entry for queue transaction.
|
71
|
+
THREAD_CURRENT_TX = :reliable_msg_tx #:nodoc:
|
72
|
+
|
73
|
+
# DRb URI for queue manager. You can override this to change the URI globally,
|
74
|
+
# for all Queue objects that are not instantiated with an alternative URI.
|
75
|
+
@@drb_uri = DEFAULT_DRB_URI
|
76
|
+
|
77
|
+
# Reference to the local queue manager. Defaults to a DRb object, unless
|
78
|
+
# the queue manager is running locally.
|
79
|
+
@@qm = nil #:nodoc:
|
80
|
+
|
81
|
+
# Cache of queue managers referenced by their URI.
|
82
|
+
@@qm_cache = {} #:nodoc:
|
83
|
+
|
84
|
+
# The optional argument +queue+ specifies the queue name. The application can
|
85
|
+
# still put messages in other queues by specifying the destination queue
|
86
|
+
# name in the header, or get from other queues by specifying the queue name
|
87
|
+
# in the selector.
|
88
|
+
#
|
89
|
+
# TODO: document options
|
90
|
+
# * :expires
|
91
|
+
# * :priority
|
92
|
+
# * :max_retries
|
93
|
+
# * :selector
|
94
|
+
# * :drb_uri
|
95
|
+
# * :tx_timeout
|
96
|
+
# * :connect_count
|
97
|
+
#
|
98
|
+
# :call-seq:
|
99
|
+
# Queue.new([name [,options]]) -> queue
|
100
|
+
#
|
101
|
+
def initialize queue = nil, options = nil
|
102
|
+
options.each do |name, value|
|
103
|
+
instance_variable_set "@#{name.to_s}".to_sym, value
|
104
|
+
end if options
|
105
|
+
@queue = queue
|
106
|
+
end
|
107
|
+
|
108
|
+
# Put a message in the queue.
|
109
|
+
#
|
110
|
+
# The +message+ argument is required, but may be +nil+
|
111
|
+
#
|
112
|
+
# Headers are optional. Headers are used to provide the application with additional
|
113
|
+
# information about the message, and can be used to retrieve messages (see Queue.put
|
114
|
+
# for discussion of selectors). Some headers are used to handle message processing
|
115
|
+
# internally (e.g. <tt>:priority</tt>, <tt>:expires</tt>).
|
116
|
+
#
|
117
|
+
# Each header uses a symbol for its name. The value may be string, numeric, true/false
|
118
|
+
# or nil. No other objects are allowed. To improve performance, keep headers as small
|
119
|
+
# as possible.
|
120
|
+
#
|
121
|
+
# The following headers have special meaning:
|
122
|
+
# * <tt>:delivery</tt> -- The message delivery mode.
|
123
|
+
# * <tt>:queue</tt> -- Puts the message in the named queue. Otherwise, uses the queue
|
124
|
+
# specified when creating the Queue object.
|
125
|
+
# * <tt>:priority</tt> -- The message priority. Messages with higher priority are
|
126
|
+
# retrieved first.
|
127
|
+
# * <tt>:expires</tt> -- Message expiration in seconds. Messages do not expire unless
|
128
|
+
# specified. Zero or +nil+ means no expiration.
|
129
|
+
# * <tt>:expires_at</tt> -- Specifies when the message expires (timestamp). Alternative
|
130
|
+
# to <tt>:expires</tt>.
|
131
|
+
# * <tt>:max_retries</tt> -- Maximum number of attempts to re-deliver message, afterwhich
|
132
|
+
# message moves to the DLQ. Minimum is 0 (deliver only once), default is 4 (deliver
|
133
|
+
# up to 5 times).
|
134
|
+
#
|
135
|
+
# Headers can be set on a per-queue basis when the Queue is created. This only affects
|
136
|
+
# messages put through that Queue object.
|
137
|
+
#
|
138
|
+
# Messages can be delivered using one of three delivery modes:
|
139
|
+
# * <tt>:best_effort</tt> -- Attempt to deliver the message once. If the message expires or
|
140
|
+
# cannot be delivered, discard the message. The is the default delivery mode.
|
141
|
+
# * <tt>:repeated</tt> -- Attempt to deliver until message expires, or up to maximum
|
142
|
+
# re-delivery count (see <tt>:max_retries</tt>). Afterwards, move message to dead-letter
|
143
|
+
# queue.
|
144
|
+
# * <tt>:once</tt> -- Attempt to deliver message exactly once. If message expires, or
|
145
|
+
# first delivery attempt fails, move message to dead-letter queue.
|
146
|
+
#
|
147
|
+
# For example:
|
148
|
+
# queue.put request
|
149
|
+
# queue.put notice, :expires=>10
|
150
|
+
# queue.put object, :queue=>'other-queue'
|
151
|
+
#
|
152
|
+
# :call-seq:
|
153
|
+
# queue.put(message[, headers]) -> id
|
154
|
+
#
|
155
|
+
def put message, headers = nil
|
156
|
+
tx = Thread.current[THREAD_CURRENT_TX]
|
157
|
+
# Use headers supplied by callers, or defaults for this queue.
|
158
|
+
headers ||= {}
|
159
|
+
headers.fetch(:priority, @priority || 0)
|
160
|
+
headers.fetch(:expires, @expires)
|
161
|
+
headers.fetch(:max_retries, @max_retries || DEFAULT_MAX_RETRIES)
|
162
|
+
# Serialize the message before sending to queue manager. We need the
|
163
|
+
# message to be serialized for storage, this just saves duplicate
|
164
|
+
# serialization when using DRb.
|
165
|
+
message = Marshal::dump message
|
166
|
+
# If inside a transaction, always send to the same queue manager, otherwise,
|
167
|
+
# allow repeated() to try and access multiple queue managers.
|
168
|
+
if tx
|
169
|
+
return tx[:qm].queue(:message=>message, :headers=>headers, :queue=>(headers[:queue] || @queue), :tid=>tx[:tid])
|
170
|
+
else
|
171
|
+
return repeated { |qm| qm.queue :message=>message, :headers=>headers, :queue=>(headers[:queue] || @queue) }
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
# Get a message from the queue.
|
176
|
+
#
|
177
|
+
# Call with no arguments to retrieve the next message in the queue. Call with a message
|
178
|
+
# identifier to retrieve that message. Call with selectors to retrieve the first message
|
179
|
+
# that matches.
|
180
|
+
#
|
181
|
+
# Selectors specify which headers to match. For example, to retrieve all messages in the
|
182
|
+
# queue 'my-queue' with priority 2:
|
183
|
+
# msg = queue.get :queue=>'my-queue', :priority=>2
|
184
|
+
# To put and get the same message:
|
185
|
+
# mid = queue.put obj
|
186
|
+
# msg = queue.get mid # or queue.get :id=>mid
|
187
|
+
# assert(msg.obj == obj)
|
188
|
+
#
|
189
|
+
# More complex selector expressions can be generated using Queue.selector. For example,
|
190
|
+
# to retrieve the next message with priority 2 or higher, received in the last 60 seconds:
|
191
|
+
# selector = Queue.selector { priority >= 2 and received > Time.new.to_i - 60 }
|
192
|
+
# msg = queue.get selector
|
193
|
+
# You can also specify selectors for a Queue to be used by default for all Queue.get calls
|
194
|
+
# on that Queue object. For example:
|
195
|
+
# queue.selector= { priority >= 2 and received > Time.new.to_i - 60 }
|
196
|
+
# msg = queue.get # default selector applies
|
197
|
+
#
|
198
|
+
# The following headers have special meaning:
|
199
|
+
# * <tt>:id</tt> -- The message identifier.
|
200
|
+
# * <tt>:queue</tt> -- Select a message originally delivered to the named queue. Only used
|
201
|
+
# when retrieving messages from the dead-letter queue.
|
202
|
+
# * <tt>:retry</tt> -- Specifies the retry count for the message. Zero when the message is
|
203
|
+
# first delivered, and incremented after each re-delivery attempt.
|
204
|
+
# * <tt>:created</tt> -- Indicates timestamp (in seconds) when the message was created.
|
205
|
+
# * <tt>:received</tt> -- Indicates timestamp (in seconds) when the message was received.
|
206
|
+
# * <tt>:expires_at</tt> -- Indicates timestamp (in seconds) when the message will expire,
|
207
|
+
# +nil+ if the message does not expire.
|
208
|
+
#
|
209
|
+
# Call this method without a block to return the message. The returned object is of type
|
210
|
+
# Message, or +nil+ if no message is found.
|
211
|
+
#
|
212
|
+
# Call this method in a block to retrieve and process the message. The block is called with
|
213
|
+
# the Message object, returning the result of the block. Returns +nil+ if no message is found.
|
214
|
+
#
|
215
|
+
# All operations performed on the queue inside the block are part of the same transaction.
|
216
|
+
# The transaction commits when the block completes. However, if the block raises an exception,
|
217
|
+
# the transaction aborts: the message along with any message retrieved through that Queue object
|
218
|
+
# are returned to the queue; messages put through that Queue object are discarded. You cannot
|
219
|
+
# put and get the same message inside a transaction.
|
220
|
+
#
|
221
|
+
# For example:
|
222
|
+
# queue.put obj
|
223
|
+
# while queue.get do |msg| # called for each message in the queue,
|
224
|
+
# # until the queue is empty
|
225
|
+
# ... do something with msg ...
|
226
|
+
# queue.put obj # puts another message in the queue
|
227
|
+
# true
|
228
|
+
# end
|
229
|
+
# This loop will only complete if it raises an exception, since it gets one message from
|
230
|
+
# the queue and puts another message in its place. After an exception, there will be at
|
231
|
+
# least one message in the queue.
|
232
|
+
#
|
233
|
+
# Each attempt to process a message increases its retry count. When the retry count
|
234
|
+
# (<tt>:retry</tt>) reaches the maximum allowed (<tt>:max_retry</tt>), the message is
|
235
|
+
# moved to the dead-letter queue.
|
236
|
+
#
|
237
|
+
# This method does not block and returns immediately if there is no message in the queue.
|
238
|
+
# To continue processing all messages in the queue:
|
239
|
+
# while true # repeat forever
|
240
|
+
# while true
|
241
|
+
# break unless queue.get do |msg|
|
242
|
+
# ... do something with msg ...
|
243
|
+
# true
|
244
|
+
# end
|
245
|
+
# end
|
246
|
+
# sleep 5 # no messages, wait
|
247
|
+
# end
|
248
|
+
#
|
249
|
+
# :call-seq:
|
250
|
+
# queue.get([selector]) -> msg or nil
|
251
|
+
# queue.get([selector]) {|msg| ... } -> obj
|
252
|
+
#
|
253
|
+
# See: Message
|
254
|
+
#
|
255
|
+
def get selector = nil, &block
|
256
|
+
tx = old_tx = Thread.current[THREAD_CURRENT_TX]
|
257
|
+
# If block, begin a new transaction.
|
258
|
+
if block
|
259
|
+
tx = {:qm=>qm}
|
260
|
+
tx[:tid] = tx[:qm].begin tx_timeout
|
261
|
+
Thread.current[THREAD_CURRENT_TX] = tx
|
262
|
+
end
|
263
|
+
result = begin
|
264
|
+
# Validate the selector: nil, string or hash.
|
265
|
+
selector = case selector
|
266
|
+
when String
|
267
|
+
{:id=>selector}
|
268
|
+
when Hash, Array, Selector
|
269
|
+
selector
|
270
|
+
when nil
|
271
|
+
@selector
|
272
|
+
else
|
273
|
+
raise ArgumentError, ERROR_INVALID_SELECTOR
|
274
|
+
end
|
275
|
+
# If inside a transaction, always retrieve from the same queue manager,
|
276
|
+
# otherwise, allow repeated() to try and access multiple queue managers.
|
277
|
+
message = if tx
|
278
|
+
tx[:qm].enqueue :queue=>@queue, :selector=>selector, :tid=>tx[:tid]
|
279
|
+
else
|
280
|
+
repeated { |qm| qm.enqueue :queue=>@queue, :selector=>selector }
|
281
|
+
end
|
282
|
+
# Result is either message, or result from processing block. Note that
|
283
|
+
# calling block may raise an exception. We deserialize the message here
|
284
|
+
# for two reasons:
|
285
|
+
# 1. It creates a distinct copy, so changing the message object and returning
|
286
|
+
# it to the queue (abort) does not affect other consumers.
|
287
|
+
# 2. The message may rely on classes known to the client but not available
|
288
|
+
# to the queue manager.
|
289
|
+
result = if message
|
290
|
+
message = Message.new(message[:id], message[:headers], Marshal::load(message[:message]))
|
291
|
+
block ? block.call(message) : message
|
292
|
+
end
|
293
|
+
rescue Exception=>error
|
294
|
+
# Abort the transaction if we started it. Propagate error.
|
295
|
+
qm.abort(tx[:tid]) if block
|
296
|
+
raise error
|
297
|
+
ensure
|
298
|
+
# Resume the old transaction.
|
299
|
+
Thread.current[THREAD_CURRENT_TX] = old_tx if block
|
300
|
+
end
|
301
|
+
# Commit the transaction and return the result. We do this outside the main
|
302
|
+
# block, since we don't abort in case of error (commit is one-phase) and we
|
303
|
+
# don't retain the transaction association, it completes by definition.
|
304
|
+
qm.commit(tx[:tid]) if block
|
305
|
+
result
|
306
|
+
end
|
307
|
+
|
308
|
+
# Returns the transaction timeout (in seconds).
|
309
|
+
#
|
310
|
+
# :call-seq:
|
311
|
+
# queue.tx_timeout -> numeric
|
312
|
+
#
|
313
|
+
def tx_timeout
|
314
|
+
@tx_timeout || DEFAULT_TX_TIMEOUT
|
315
|
+
end
|
316
|
+
|
317
|
+
# Sets the transaction timeout (in seconds). Affects future transactions started
|
318
|
+
# by Queue.get. Use +nil+ to restore the default timeout.
|
319
|
+
#
|
320
|
+
# :call-seq:
|
321
|
+
# queue.tx_timeout = timeout
|
322
|
+
# queue.tx_timeout = nil
|
323
|
+
#
|
324
|
+
def tx_timeout= timeout
|
325
|
+
if timeout
|
326
|
+
raise ArgumentError, ERROR_INVALID_TX_TIMEOUT unless timeout.instance_of?(Integer) and timeout > 0
|
327
|
+
@tx_timeout = timeout
|
328
|
+
else
|
329
|
+
@tx_timeout = nil
|
330
|
+
end
|
331
|
+
end
|
332
|
+
|
333
|
+
# Returns the number of connection attempts, before operations fail.
|
334
|
+
#
|
335
|
+
# :call-seq:
|
336
|
+
# queue.connect_count -> numeric
|
337
|
+
#
|
338
|
+
def connect_count
|
339
|
+
@connect_count || DEFAULT_CONNECT_RETRY
|
340
|
+
end
|
341
|
+
|
342
|
+
# Sets the number of connection attempts, before operations fail. The minimum is one.
|
343
|
+
# Use +nil+ to restore the default connection count.
|
344
|
+
#
|
345
|
+
# :call-seq:
|
346
|
+
# queue.connect_count = count
|
347
|
+
# queue.connect_count = nil
|
348
|
+
#
|
349
|
+
def connect_count= count
|
350
|
+
if count
|
351
|
+
raise ArgumentError, ERROR_INVALID_CONNECT_COUNT unless count.instance_of?(Integer) and count > 0
|
352
|
+
@connect_count = count
|
353
|
+
else
|
354
|
+
@connect_count = nil
|
355
|
+
end
|
356
|
+
end
|
357
|
+
|
358
|
+
# If called with no block, returns the selector associated with this Queue
|
359
|
+
# (see Queue.selector=). If called with a block, creates and returns a new
|
360
|
+
# selector (similar to Queue::selector).
|
361
|
+
#
|
362
|
+
# :call-seq:
|
363
|
+
# queue.selector -> selector
|
364
|
+
# queue.selector { ... } -> selector
|
365
|
+
#
|
366
|
+
def selector &block
|
367
|
+
block ? Selector.new(&block) : @selector
|
368
|
+
end
|
369
|
+
|
370
|
+
# Sets a default selector for this Queue. Affects all calls to Queue.get on this
|
371
|
+
# Queue object that do not specify a selector.
|
372
|
+
#
|
373
|
+
# You can pass a Selector object, a block expression, or +nil+ if you no longer
|
374
|
+
# want to use the default selector. For example:
|
375
|
+
# queue.selector= { priority >= 2 and received > Time.new.to_i - 60 }
|
376
|
+
# 10.times do
|
377
|
+
# p queue.get
|
378
|
+
# end
|
379
|
+
# queue.selector= nil
|
380
|
+
#
|
381
|
+
# :call-seq:
|
382
|
+
# queue.selector = selector
|
383
|
+
# queue.selector = { ... }
|
384
|
+
# queue.selector = nil
|
385
|
+
#
|
386
|
+
def selector= value = nil, &block
|
387
|
+
raise ArgumentError, ERROR_SELECTOR_VALUE_OR_BLOCK if (value && block)
|
388
|
+
if value
|
389
|
+
raise ArgumentError, ERROR_SELECTOR_VALUE_OR_BLOCK unless value.instance_of?(Selector)
|
390
|
+
@selector = value
|
391
|
+
elsif block
|
392
|
+
@selector = Selector.new &block
|
393
|
+
else
|
394
|
+
@selector = nil
|
395
|
+
end
|
396
|
+
end
|
397
|
+
|
398
|
+
# Create and return a new selector based on the block expression. For example:
|
399
|
+
# selector = Queue.selector { priority >= 2 and received > Time.new.to_i - 60 }
|
400
|
+
#
|
401
|
+
# :call-seq:
|
402
|
+
# Queue.selector { ... } -> selector
|
403
|
+
#
|
404
|
+
def self.selector &block
|
405
|
+
raise ArgumentError, ERROR_NO_SELECTOR_BLOCK unless block
|
406
|
+
Selector.new &block
|
407
|
+
end
|
408
|
+
|
409
|
+
private
|
410
|
+
|
411
|
+
# Returns the active queue manager. You can override this method to implement
|
412
|
+
# load balancing.
|
413
|
+
def qm
|
414
|
+
if uri = @drb_uri
|
415
|
+
# Queue specifies queue manager's URI: use that queue manager.
|
416
|
+
@@qm_cache[uri] ||= DRbObject.new(nil, uri)
|
417
|
+
else
|
418
|
+
# Use the same queue manager for all queues, and cache it.
|
419
|
+
# Create only the first time.
|
420
|
+
@@qm ||= DRbObject.new(nil, @@drb_uri || DEFAULT_DRB_URI)
|
421
|
+
end
|
422
|
+
end
|
423
|
+
|
424
|
+
# Called to execute the operation repeatedly and avoid connection failures. This only
|
425
|
+
# makes sense if we have a load balancing algorithm.
|
426
|
+
def repeated &block
|
427
|
+
count = connect_count
|
428
|
+
begin
|
429
|
+
block.call qm
|
430
|
+
rescue DRb::DRbConnError=>error
|
431
|
+
warn error
|
432
|
+
warn error.backtrace
|
433
|
+
retry if (count -= 1) > 0
|
434
|
+
raise error
|
435
|
+
end
|
436
|
+
end
|
437
|
+
|
438
|
+
class << self
|
439
|
+
private
|
440
|
+
# Sets the active queue manager. Used when the queue manager is running in the
|
441
|
+
# same process to bypass DRb calls.
|
442
|
+
def qm= qm
|
443
|
+
@@qm = qm
|
444
|
+
end
|
445
|
+
end
|
446
|
+
|
447
|
+
end
|
448
|
+
|
449
|
+
|
450
|
+
# == Retrieved Message
|
451
|
+
#
|
452
|
+
# Returned from Queue.get holding the last message retrieved from the
|
453
|
+
# queue and providing access to the message identifier, headers and object.
|
454
|
+
#
|
455
|
+
# For example:
|
456
|
+
# while queue.get do |msg|
|
457
|
+
# print "Message #{msg.id}"
|
458
|
+
# print "Headers: #{msg.headers.inspect}"
|
459
|
+
# print msg.object
|
460
|
+
# true
|
461
|
+
# end
|
462
|
+
class Message
|
463
|
+
|
464
|
+
|
465
|
+
def initialize id, headers, object # :nodoc:
|
466
|
+
@id, @object, @headers = id, object, headers
|
467
|
+
end
|
468
|
+
|
469
|
+
# Returns the message identifier.
|
470
|
+
#
|
471
|
+
# :call-seq:
|
472
|
+
# msg.id -> id
|
473
|
+
#
|
474
|
+
def id
|
475
|
+
@id
|
476
|
+
end
|
477
|
+
|
478
|
+
# Returns the message object.
|
479
|
+
#
|
480
|
+
# :call-seq:
|
481
|
+
# msg.object -> obj
|
482
|
+
#
|
483
|
+
def object
|
484
|
+
@object
|
485
|
+
end
|
486
|
+
|
487
|
+
# Returns the message headers.
|
488
|
+
#
|
489
|
+
# :call-seq:
|
490
|
+
# msg.headers -> hash
|
491
|
+
#
|
492
|
+
def headers
|
493
|
+
@headers
|
494
|
+
end
|
495
|
+
|
496
|
+
end
|
497
|
+
|
498
|
+
|
499
|
+
end
|
500
|
+
|