reliable-msg 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/MIT-LICENSE +20 -0
- data/README +123 -0
- data/Rakefile +164 -0
- data/bin/queues +8 -0
- data/lib/reliable-msg.rb +12 -0
- data/lib/reliable-msg/cli.rb +170 -0
- data/lib/reliable-msg/message-store.rb +509 -0
- data/lib/reliable-msg/mysql.sql +8 -0
- data/lib/reliable-msg/queue-manager.rb +409 -0
- data/lib/reliable-msg/queue.rb +500 -0
- data/lib/reliable-msg/selector.rb +109 -0
- data/lib/uuid.rb +384 -0
- data/test/test-queue.rb +144 -0
- data/test/test-uuid.rb +48 -0
- metadata +63 -0
@@ -0,0 +1,509 @@
|
|
1
|
+
#
|
2
|
+
# = message-store.rb - Queue manager storage adapters
|
3
|
+
#
|
4
|
+
# Author:: Assaf Arkin assaf@labnotes.org
|
5
|
+
# Documentation:: http://trac.labnotes.org/cgi-bin/trac.cgi/wiki/RubyReliableMessaging
|
6
|
+
# Copyright:: Copyright (c) 2005 Assaf Arkin
|
7
|
+
# License:: MIT and/or Creative Commons Attribution-ShareAlike
|
8
|
+
#
|
9
|
+
#--
|
10
|
+
# Changes:
|
11
|
+
#++
|
12
|
+
|
13
|
+
require 'thread'
|
14
|
+
require 'uuid'
|
15
|
+
require 'reliable-msg/queue'
|
16
|
+
|
17
|
+
module ReliableMsg
|
18
|
+
|
19
|
+
module MessageStore
|
20
|
+
|
21
|
+
ERROR_INVALID_MESSAGE_STORE = "No message store '%s' available (note: case is not important)" #:nodoc:
|
22
|
+
|
23
|
+
# Base class for message store.
|
24
|
+
class Base
|
25
|
+
|
26
|
+
ERROR_INVALID_MESSAGE_STORE = "No message store '%s' available (note: case is not important)" #:nodoc:
|
27
|
+
|
28
|
+
@@stores = {} #:nodoc:
|
29
|
+
|
30
|
+
def initialize logger
|
31
|
+
@logger = logger
|
32
|
+
end
|
33
|
+
|
34
|
+
# Returns the message store type name.
|
35
|
+
#
|
36
|
+
# :call-seq:
|
37
|
+
# store.type -> string
|
38
|
+
#
|
39
|
+
def type
|
40
|
+
raise RuntimeException, "Not implemented"
|
41
|
+
end
|
42
|
+
|
43
|
+
# Set up the message store. Create files, database tables, etc.
|
44
|
+
#
|
45
|
+
# :call-seq:
|
46
|
+
# store.setup
|
47
|
+
#
|
48
|
+
def setup
|
49
|
+
end
|
50
|
+
|
51
|
+
# Returns the message store configuration as a hash.
|
52
|
+
#
|
53
|
+
# :call-seq:
|
54
|
+
# store.configuration -> hash
|
55
|
+
#
|
56
|
+
def configuration
|
57
|
+
raise RuntimeException, "Not implemented"
|
58
|
+
end
|
59
|
+
|
60
|
+
# Activates the message store. Call this method before using the
|
61
|
+
# message store.
|
62
|
+
#
|
63
|
+
# :call-seq:
|
64
|
+
# store.activate
|
65
|
+
#
|
66
|
+
def activate
|
67
|
+
@mutex = Mutex.new
|
68
|
+
@queues = {Queue::DLQ=>[]}
|
69
|
+
@cache = {}
|
70
|
+
# TODO: add recovery logic
|
71
|
+
end
|
72
|
+
|
73
|
+
# Deactivates the message store. Call this method when done using
|
74
|
+
# the message store.
|
75
|
+
#
|
76
|
+
# :call-seq:
|
77
|
+
# store.deactivate
|
78
|
+
#
|
79
|
+
def deactivate
|
80
|
+
@mutex = @queues = @cache = nil
|
81
|
+
end
|
82
|
+
|
83
|
+
def transaction &block
|
84
|
+
result = block.call inserts = [], deletes = [], dlqs= []
|
85
|
+
begin
|
86
|
+
update inserts, deletes, dlqs unless inserts.empty? && deletes.empty? && dlqs.empty?
|
87
|
+
rescue Exception=>error
|
88
|
+
@logger.error error
|
89
|
+
# If an error occurs, the queue may be out of synch with the store.
|
90
|
+
# Empty the cache and reload the queue, before raising the error.
|
91
|
+
@cache = {}
|
92
|
+
@queues = {Queue::DLQ=>[]}
|
93
|
+
load_index
|
94
|
+
raise error
|
95
|
+
end
|
96
|
+
result
|
97
|
+
end
|
98
|
+
|
99
|
+
def select queue, &block
|
100
|
+
queue = @queues[queue]
|
101
|
+
return nil unless queue
|
102
|
+
queue.each do |headers|
|
103
|
+
selected = block.call(headers)
|
104
|
+
if selected
|
105
|
+
id = headers[:id]
|
106
|
+
message = @cache[id] || load(id, queue)
|
107
|
+
return {:id=>id, :headers=>headers, :message=>message}
|
108
|
+
end
|
109
|
+
end
|
110
|
+
return nil
|
111
|
+
end
|
112
|
+
|
113
|
+
# Returns a message store from the specified configuration (previously
|
114
|
+
# created with configure).
|
115
|
+
#
|
116
|
+
# :call-seq:
|
117
|
+
# Base::configure(config, logger) -> store
|
118
|
+
#
|
119
|
+
def self.configure config, logger
|
120
|
+
type = config["type"].downcase
|
121
|
+
cls = @@stores[type]
|
122
|
+
raise RuntimeError, format(ERROR_INVALID_MESSAGE_STORE, type) unless cls
|
123
|
+
cls.new config, logger
|
124
|
+
end
|
125
|
+
|
126
|
+
protected
|
127
|
+
|
128
|
+
def update inserts, deletes, dlqs
|
129
|
+
@mutex.synchronize do
|
130
|
+
inserts.each do |insert|
|
131
|
+
queue = @queues[insert[:queue]] ||= []
|
132
|
+
headers = insert[:headers]
|
133
|
+
# Add element based on priority, higher priority comes first.
|
134
|
+
priority = headers[:priority]
|
135
|
+
if priority > 0
|
136
|
+
queue.each_index do |idx|
|
137
|
+
if queue[idx][:priority] < priority
|
138
|
+
queue[idx, 0] = headers
|
139
|
+
break
|
140
|
+
end
|
141
|
+
end
|
142
|
+
queue << headers
|
143
|
+
else
|
144
|
+
queue << headers
|
145
|
+
end
|
146
|
+
@cache[insert[:id]] = insert[:message]
|
147
|
+
end
|
148
|
+
deletes.each do |delete|
|
149
|
+
queue = @queues[delete[:queue]]
|
150
|
+
id = delete[:id]
|
151
|
+
queue.delete_if { |headers| headers[:id] == id }
|
152
|
+
@cache.delete id
|
153
|
+
end
|
154
|
+
dlqs.each do |dlq|
|
155
|
+
queue = @queues[dlq[:queue]]
|
156
|
+
id = dlq[:id]
|
157
|
+
queue.delete_if do |headers|
|
158
|
+
if headers[:id] == id
|
159
|
+
@queues[Queue::DLQ] << headers
|
160
|
+
true
|
161
|
+
end
|
162
|
+
end
|
163
|
+
@cache.delete id
|
164
|
+
end
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
end
|
169
|
+
|
170
|
+
|
171
|
+
class Disk < Base #:nodoc:
|
172
|
+
|
173
|
+
TYPE = self.name.split('::').last.downcase
|
174
|
+
|
175
|
+
@@stores[TYPE] = self
|
176
|
+
|
177
|
+
# Default path where index and messages are stored.
|
178
|
+
DEFAULT_PATH = 'queues'
|
179
|
+
|
180
|
+
# Maximum number of open files.
|
181
|
+
MAX_OPEN_FILES = 20
|
182
|
+
|
183
|
+
DEFAULT_CONFIG = {
|
184
|
+
"type"=>TYPE,
|
185
|
+
"path"=>DEFAULT_PATH
|
186
|
+
}
|
187
|
+
|
188
|
+
def initialize config, logger
|
189
|
+
super logger
|
190
|
+
@fsync = config['fsync']
|
191
|
+
# file_map maps messages (by ID) to files. The value is a two-item array: the file
|
192
|
+
# name and, if opened, the File object. file_free keeps a list of all currently
|
193
|
+
# unused files, using the same two-item arrays.
|
194
|
+
@file_map = {}
|
195
|
+
@file_free = []
|
196
|
+
# Make sure the path points to the queue directory, and the master index is writeable.
|
197
|
+
@path = File.expand_path(config['path'] || DEFAULT_PATH)
|
198
|
+
end
|
199
|
+
|
200
|
+
def type
|
201
|
+
TYPE
|
202
|
+
end
|
203
|
+
|
204
|
+
def setup
|
205
|
+
if File.exist?(@path)
|
206
|
+
raise RuntimeError, "The path '#{@path}' is not a directory" unless File.directory?(@path)
|
207
|
+
false
|
208
|
+
else
|
209
|
+
Dir.mkdir @path
|
210
|
+
true
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
def configuration
|
215
|
+
{ "type"=>TYPE, "path"=>@path }
|
216
|
+
end
|
217
|
+
|
218
|
+
def activate
|
219
|
+
super
|
220
|
+
Dir.mkdir @path unless File.exist?(@path)
|
221
|
+
raise RuntimeError, "The path '#{@path}' is not a directory" unless File.directory?(@path)
|
222
|
+
index = "#{@path}/master.idx"
|
223
|
+
if File.exist? index
|
224
|
+
raise RuntimeError, "Cannot write to master index file '#{index}'" unless File.writable?(index)
|
225
|
+
@file = File.open index, "r+"
|
226
|
+
@file.flock File::LOCK_EX
|
227
|
+
@file.binmode # Things break if you forget binmode on Windows.
|
228
|
+
load_index
|
229
|
+
else
|
230
|
+
@file = File.open index, "w"
|
231
|
+
@file.flock File::LOCK_EX
|
232
|
+
@file.binmode # Things break if you forget binmode on Windows.
|
233
|
+
@last_block = @last_block_end = 8
|
234
|
+
# Save. This just prevents us from starting with an empty file, and to
|
235
|
+
# enable load_index().
|
236
|
+
update [], [], []
|
237
|
+
end
|
238
|
+
end
|
239
|
+
|
240
|
+
def deactivate
|
241
|
+
@file.close
|
242
|
+
@file_map.each_pair do |id, map|
|
243
|
+
map[1].close if map[1]
|
244
|
+
end
|
245
|
+
@file_free.each do |map|
|
246
|
+
map[1].close if map[1]
|
247
|
+
end
|
248
|
+
@file_map = @file_free = nil
|
249
|
+
super
|
250
|
+
end
|
251
|
+
|
252
|
+
protected
|
253
|
+
|
254
|
+
def update inserts, deletes, dlqs
|
255
|
+
inserts.each do |insert|
|
256
|
+
# Get an available open file, if none available, create a new file.
|
257
|
+
# This allows us to reuse previously opened files that no longer hold
|
258
|
+
# any referenced message to store new messages. The File object exists
|
259
|
+
# if the file was opened before, otherwise, we need to open it again.
|
260
|
+
free = @mutex.synchronize do
|
261
|
+
@file_free.shift
|
262
|
+
end
|
263
|
+
name = free ? free[0] : "#{@path}/#{UUID.new}.msg"
|
264
|
+
file = if free && free[1]
|
265
|
+
free[1]
|
266
|
+
else
|
267
|
+
file = File.open name, "w+"
|
268
|
+
file.binmode
|
269
|
+
end
|
270
|
+
# Store the message in the file, map the message to the file
|
271
|
+
# (message and file have different IDs).
|
272
|
+
file.sysseek 0, IO::SEEK_SET
|
273
|
+
file.syswrite insert[:message]
|
274
|
+
file.truncate file.pos
|
275
|
+
file.flush
|
276
|
+
@mutex.synchronize do
|
277
|
+
@file_map[insert[:id]] = [name, file]
|
278
|
+
end
|
279
|
+
end
|
280
|
+
super
|
281
|
+
@save = true
|
282
|
+
@mutex.synchronize do
|
283
|
+
deletes.each do |delete|
|
284
|
+
# Instead of deleting the file, we delete the mapping between the message
|
285
|
+
# and file and return the file (name and File) to the free list. But we
|
286
|
+
# only keep so many open files.
|
287
|
+
if @file_free.length < MAX_OPEN_FILES
|
288
|
+
@file_free << @file_map.delete(delete[:id])
|
289
|
+
else
|
290
|
+
free = @file_map.delete(delete[:id])
|
291
|
+
free[1].close
|
292
|
+
File.delete free[0]
|
293
|
+
end
|
294
|
+
end
|
295
|
+
end
|
296
|
+
@mutex.synchronize do
|
297
|
+
if @save
|
298
|
+
# Create an image of the index. We need that image in order to determine
|
299
|
+
# its length and therefore positioning in the file. The image contains the
|
300
|
+
# message map/free file map, but without the File objects.
|
301
|
+
file_map = {}
|
302
|
+
@file_map.each_pair { |id, name_file| file_map[id] = name_file[0] }
|
303
|
+
file_free = @file_free.collect { |name_file| name_file[0] }
|
304
|
+
image = Marshal::dump({:queues=>@queues, :file_map=>file_map, :file_free=>file_free})
|
305
|
+
length = image.length
|
306
|
+
# Determine if we can store the new image before the last one (must have
|
307
|
+
# enough space from header), or append it to the end of the last one.
|
308
|
+
next_block = length + 16 > @last_block ? @last_block_end : 8
|
309
|
+
# Seek to new position in file, write image length followed by image.
|
310
|
+
@file.sysseek next_block, IO::SEEK_SET
|
311
|
+
@file.syswrite sprintf("%08x", length)
|
312
|
+
@file.syswrite image
|
313
|
+
# Seek to beginning of file, write position of last block. Flush the
|
314
|
+
# updates to the O/S.
|
315
|
+
@file.sysseek 0, IO::SEEK_SET
|
316
|
+
@file.syswrite sprintf("%08x", next_block)
|
317
|
+
#@file.flush
|
318
|
+
@file.fsync if @fsync
|
319
|
+
# Note: the paranoids add fsync here
|
320
|
+
@last_block, @last_block_end = next_block, next_block + length + 8
|
321
|
+
@save = false
|
322
|
+
end
|
323
|
+
end
|
324
|
+
end
|
325
|
+
|
326
|
+
def load_index
|
327
|
+
@file.sysseek 0, IO::SEEK_SET
|
328
|
+
last_block = @file.sysread(8).hex
|
329
|
+
if last_block
|
330
|
+
# Seek to last block written, read length of image stored
|
331
|
+
# there and read that many bytes into memory, restoring the
|
332
|
+
# master index.
|
333
|
+
@file.sysseek last_block, IO::SEEK_SET
|
334
|
+
length = @file.sysread(8).hex
|
335
|
+
# Load the index image and create the queues, file_free and
|
336
|
+
# file_map structures.
|
337
|
+
image = Marshal::load @file.sysread(length)
|
338
|
+
@queues = image[:queues]
|
339
|
+
image[:file_free].each { |name| @file_free << [name, nil] }
|
340
|
+
image[:file_map].each_pair { |id, name| @file_map[id] = [name, nil] }
|
341
|
+
@last_block, @last_block_end = last_block, last_block + length + 8
|
342
|
+
end
|
343
|
+
end
|
344
|
+
|
345
|
+
def load id, queue
|
346
|
+
# Find the file from the message/file mapping.
|
347
|
+
map = @file_map[id]
|
348
|
+
return nil unless map # TODO: Error?
|
349
|
+
file = if map[1]
|
350
|
+
map[1]
|
351
|
+
else
|
352
|
+
file = File.open map[0], "r+"
|
353
|
+
file.binmode
|
354
|
+
map[1] = file
|
355
|
+
end
|
356
|
+
file.sysseek 0, IO::SEEK_SET
|
357
|
+
file.sysread file.stat.size
|
358
|
+
end
|
359
|
+
|
360
|
+
end
|
361
|
+
|
362
|
+
|
363
|
+
begin
|
364
|
+
|
365
|
+
# Make sure we have a MySQL library before creating this class,
|
366
|
+
# worst case we end up with a disk-based message store. Try the
|
367
|
+
# native MySQL library, followed by the Rails MySQL library.
|
368
|
+
begin
|
369
|
+
require 'mysql'
|
370
|
+
rescue LoadError
|
371
|
+
require 'active_record/vendor/mysql'
|
372
|
+
end
|
373
|
+
|
374
|
+
class MySQL < Base #:nodoc:
|
375
|
+
|
376
|
+
TYPE = self.name.split('::').last.downcase
|
377
|
+
|
378
|
+
@@stores[TYPE] = self
|
379
|
+
|
380
|
+
# Default prefix for tables in the database.
|
381
|
+
DEFAULT_PREFIX = 'reliable_msg_';
|
382
|
+
|
383
|
+
# Reference to an open MySQL connection held in the current thread.
|
384
|
+
THREAD_CURRENT_MYSQL = :reliable_msg_mysql #:nodoc:
|
385
|
+
|
386
|
+
def initialize config, logger
|
387
|
+
super logger
|
388
|
+
@config = { :host=>config['host'], :username=>config['username'], :password=>config['password'],
|
389
|
+
:database=>config['database'], :port=>config['port'], :socket=>config['socket'] }
|
390
|
+
@prefix = config['prefix'] || DEFAULT_PREFIX
|
391
|
+
@queues_table = "#{@prefix}queues"
|
392
|
+
end
|
393
|
+
|
394
|
+
def type
|
395
|
+
TYPE
|
396
|
+
end
|
397
|
+
|
398
|
+
def setup
|
399
|
+
mysql = connection
|
400
|
+
exists = false
|
401
|
+
mysql.query "SHOW TABLES" do |result|
|
402
|
+
while row = result.fetch_row
|
403
|
+
exists = true if row[0] == @queues_table
|
404
|
+
end
|
405
|
+
end
|
406
|
+
if exists
|
407
|
+
false
|
408
|
+
else
|
409
|
+
sql = File.open File.join(File.dirname(__FILE__), "mysql.sql"), "r" do |input|
|
410
|
+
input.readlines.join
|
411
|
+
end
|
412
|
+
sql.gsub! DEFAULT_PREFIX, @prefix
|
413
|
+
mysql.query sql
|
414
|
+
true
|
415
|
+
end
|
416
|
+
end
|
417
|
+
|
418
|
+
def configuration
|
419
|
+
config = { "type"=>TYPE, "host"=>@config[:host], "username"=>@config[:username],
|
420
|
+
"password"=>@config[:password], "database"=>@config[:database] }
|
421
|
+
config["port"] = @config[:port] if @config[:port]
|
422
|
+
config["socket"] = @config[:socket] if @config[:socket]
|
423
|
+
config["prefix"] = @config[:prefix] if @config[:prefix]
|
424
|
+
config
|
425
|
+
end
|
426
|
+
|
427
|
+
def activate
|
428
|
+
super
|
429
|
+
load_index
|
430
|
+
end
|
431
|
+
|
432
|
+
def deactivate
|
433
|
+
Thread.list.each do |thread|
|
434
|
+
if conn = thread[THREAD_CURRENT_MYSQL]
|
435
|
+
thread[THREAD_CURRENT_MYSQL] = nil
|
436
|
+
conn.close
|
437
|
+
end
|
438
|
+
end
|
439
|
+
super
|
440
|
+
end
|
441
|
+
|
442
|
+
protected
|
443
|
+
|
444
|
+
def update inserts, deletes, dlqs
|
445
|
+
mysql = connection
|
446
|
+
mysql.query "BEGIN"
|
447
|
+
begin
|
448
|
+
inserts.each do |insert|
|
449
|
+
mysql.query "INSERT INTO `#{@queues_table}` (id,queue,headers,object) VALUES('#{connection.quote insert[:id]}','#{connection.quote insert[:queue]}',BINARY '#{connection.quote Marshal::dump(insert[:headers])}',BINARY '#{connection.quote insert[:message]}')"
|
450
|
+
end
|
451
|
+
ids = deletes.collect {|delete| "'#{delete[:id]}'" }
|
452
|
+
if !ids.empty?
|
453
|
+
mysql.query "DELETE FROM `#{@queues_table}` WHERE id IN (#{ids.join ','})"
|
454
|
+
end
|
455
|
+
dlqs.each do |dlq|
|
456
|
+
mysql.query "UPDATE `#{@queues_table}` SET queue='#{Queue::DLQ}' WHERE id='#{connection.quote dlq[:id]}'"
|
457
|
+
end
|
458
|
+
mysql.query "COMMIT"
|
459
|
+
rescue Exception=>error
|
460
|
+
mysql.query "ROLLBACK"
|
461
|
+
raise error
|
462
|
+
end
|
463
|
+
super
|
464
|
+
end
|
465
|
+
|
466
|
+
def load_index
|
467
|
+
connection.query "SELECT id,queue,headers FROM `#{@queues_table}`" do |result|
|
468
|
+
while row = result.fetch_row
|
469
|
+
queue = @queues[row[1]] ||= []
|
470
|
+
headers = Marshal::load row[2]
|
471
|
+
# Add element based on priority, higher priority comes first.
|
472
|
+
priority = headers[:priority]
|
473
|
+
if priority > 0
|
474
|
+
queue.each_index do |idx|
|
475
|
+
if queue[idx][:priority] < priority
|
476
|
+
queue[idx, 0] = headers
|
477
|
+
break
|
478
|
+
end
|
479
|
+
end
|
480
|
+
else
|
481
|
+
queue << headers
|
482
|
+
end
|
483
|
+
end
|
484
|
+
end
|
485
|
+
end
|
486
|
+
|
487
|
+
def load id, queue
|
488
|
+
message = nil
|
489
|
+
connection.query "SELECT object FROM `#{@queues_table}` WHERE id='#{id}'" do |result|
|
490
|
+
message = if row = result.fetch_row
|
491
|
+
row[0]
|
492
|
+
end
|
493
|
+
end
|
494
|
+
message
|
495
|
+
end
|
496
|
+
|
497
|
+
def connection
|
498
|
+
Thread.current[THREAD_CURRENT_MYSQL] ||= Mysql.new @config[:host], @config[:username], @config[:password],
|
499
|
+
@config[:database], @config[:port], @config[:socket]
|
500
|
+
end
|
501
|
+
|
502
|
+
end
|
503
|
+
|
504
|
+
rescue LoadError
|
505
|
+
end
|
506
|
+
|
507
|
+
end
|
508
|
+
|
509
|
+
end
|