stompserver 0.9.7 → 0.9.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +8 -6
- data/Manifest.txt +25 -7
- data/README.txt +136 -19
- data/Rakefile +8 -8
- data/STATUS +5 -0
- data/bin/stompserver +43 -19
- data/client/README.txt +1 -0
- data/client/both.rb +25 -0
- data/client/consume.rb +14 -0
- data/client/send.rb +17 -0
- data/config/stompserver.conf +11 -0
- data/etc/passwd.example +3 -0
- data/lib/stomp_server.rb +132 -157
- data/lib/stomp_server/protocols/http.rb +128 -0
- data/lib/stomp_server/protocols/stomp.rb +186 -0
- data/lib/stomp_server/queue.rb +140 -0
- data/lib/stomp_server/queue/activerecord_queue.rb +104 -0
- data/lib/stomp_server/queue/ar_message.rb +5 -0
- data/lib/stomp_server/queue/dbm_queue.rb +68 -0
- data/lib/stomp_server/queue/file_queue.rb +47 -0
- data/lib/stomp_server/queue/memory_queue.rb +58 -0
- data/lib/stomp_server/queue_manager.rb +209 -0
- data/lib/stomp_server/stomp_auth.rb +22 -0
- data/lib/{stomp_frame.rb → stomp_server/stomp_frame.rb} +7 -7
- data/lib/stomp_server/stomp_id.rb +21 -0
- data/lib/stomp_server/stomp_user.rb +17 -0
- data/lib/stomp_server/test_server.rb +21 -0
- data/lib/{topic_manager.rb → stomp_server/topic_manager.rb} +14 -2
- data/test/tesly.rb +15 -0
- data/test/test_queue_manager.rb +39 -32
- data/test/test_stomp_frame.rb +3 -16
- data/test/test_topic_manager.rb +3 -4
- data/{test → test_todo}/test_stomp_server.rb +0 -27
- metadata +56 -23
- data/lib/frame_journal.rb +0 -135
- data/lib/queue_manager.rb +0 -81
- data/test/test_frame_journal.rb +0 -14
@@ -0,0 +1,104 @@
|
|
1
|
+
## Queue implementation using ActiveRecord
|
2
|
+
##
|
3
|
+
## all messages are stored in a single table
|
4
|
+
## they are indexed by 'stomp_id' which is the stomp 'message-id' header
|
5
|
+
## which must be unique accross all queues
|
6
|
+
##
|
7
|
+
require 'stomp_server/queue/ar_message'
|
8
|
+
require 'yaml'
|
9
|
+
|
10
|
+
module StompServer
|
11
|
+
class ActiveRecordQueue
|
12
|
+
|
13
|
+
def initialize(configdir, storagedir)
|
14
|
+
# Default configuration, use SQLite for simplicity
|
15
|
+
db_params = {
|
16
|
+
'adapter' => 'sqlite3',
|
17
|
+
'database' => "#{configdir}/stompserver_development"
|
18
|
+
}
|
19
|
+
# Load DB configuration
|
20
|
+
db_config = "#{configdir}/database.yml"
|
21
|
+
puts "reading from #{db_config}"
|
22
|
+
if File.exists? db_config
|
23
|
+
db_params.merge! YAML::load(File.open(db_config))
|
24
|
+
end
|
25
|
+
|
26
|
+
puts "using #{db_params['database']} DB"
|
27
|
+
|
28
|
+
# Setup activerecord
|
29
|
+
ActiveRecord::Base.establish_connection(db_params)
|
30
|
+
# Development <TODO> fix this
|
31
|
+
ActiveRecord::Base.logger = Logger.new(STDERR)
|
32
|
+
ActiveRecord::Base.logger.level = Logger::INFO
|
33
|
+
# we need the connection, it can't be done earlier
|
34
|
+
ArMessage.reset_column_information
|
35
|
+
reload_queues
|
36
|
+
@stompid = StompServer::StompId.new
|
37
|
+
end
|
38
|
+
|
39
|
+
# Add a frame to the queue
|
40
|
+
def enqueue(queue_name, frame)
|
41
|
+
unless @frames[queue_name]
|
42
|
+
@frames[queue_name] = {
|
43
|
+
:last_index => 0,
|
44
|
+
:frames => [],
|
45
|
+
}
|
46
|
+
end
|
47
|
+
affect_msgid_and_store(frame, queue_name)
|
48
|
+
@frames[queue_name][:frames] << frame
|
49
|
+
end
|
50
|
+
|
51
|
+
# Get and remove a frame from the queue
|
52
|
+
def dequeue(queue_name)
|
53
|
+
return nil unless @frames[queue_name] && !@frames[queue_name][:frames].empty?
|
54
|
+
frame = @frames[queue_name][:frames].shift
|
55
|
+
remove_from_store(frame.headers['message-id'])
|
56
|
+
return frame
|
57
|
+
end
|
58
|
+
|
59
|
+
# Requeue the frame previously pending
|
60
|
+
def requeue(queue_name, frame)
|
61
|
+
@frames[queue_name][:frames] << frame
|
62
|
+
ArMessage.create!(:stomp_id => frame.headers['message-id'],
|
63
|
+
:frame => frame)
|
64
|
+
end
|
65
|
+
|
66
|
+
# remove a frame from the store
|
67
|
+
def remove_from_store(message_id)
|
68
|
+
ArMessage.find_by_stomp_id(message_id).destroy
|
69
|
+
end
|
70
|
+
|
71
|
+
# store a frame (assigning it a message-id)
|
72
|
+
def affect_msgid_and_store(frame, queue_name)
|
73
|
+
msgid = assign_id(frame, queue_name)
|
74
|
+
ArMessage.create!(:stomp_id => msgid, :frame => frame)
|
75
|
+
end
|
76
|
+
|
77
|
+
def message_for?(queue_name)
|
78
|
+
@frames[queue_name] && !@frames[queue_name][:frames].empty?
|
79
|
+
end
|
80
|
+
|
81
|
+
def assign_id(frame, queue_name)
|
82
|
+
msgid = @stompid[@frames[queue_name][:last_index] += 1]
|
83
|
+
frame.headers['message-id'] = msgid
|
84
|
+
end
|
85
|
+
|
86
|
+
private
|
87
|
+
def reload_queues
|
88
|
+
@frames = Hash.new
|
89
|
+
ArMessage.find(:all).each { |message|
|
90
|
+
frame = message.frame
|
91
|
+
destination = frame.dest
|
92
|
+
msgid = message.stomp_id
|
93
|
+
@frames[destination] ||= Hash.new
|
94
|
+
@frames[destination][:frames] ||= Array.new
|
95
|
+
@frames[destination][:frames] << frame
|
96
|
+
}
|
97
|
+
# compute base index for each destination
|
98
|
+
@frames.each_pair { |destination,hash|
|
99
|
+
hash[:last_index] = hash[:frames].map{|f|
|
100
|
+
f.headers['message-id'].match(/(\d+)\Z/)[0].to_i}.max
|
101
|
+
}
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
|
2
|
+
module StompServer
|
3
|
+
class DBMQueue < Queue
|
4
|
+
|
5
|
+
def initialize *args
|
6
|
+
super
|
7
|
+
# Please don't use dbm files for storing large frames, it's problematic at best and uses large amounts of memory.
|
8
|
+
# sdbm croaks on marshalled data that contains certain characters, so we don't use it at all
|
9
|
+
@dbm = false
|
10
|
+
if RUBY_PLATFORM =~/linux|bsd/
|
11
|
+
types = ['bdb','dbm','gdbm']
|
12
|
+
else
|
13
|
+
types = ['bdb','gdbm']
|
14
|
+
end
|
15
|
+
types.each do |dbtype|
|
16
|
+
begin
|
17
|
+
require dbtype
|
18
|
+
@dbm = dbtype
|
19
|
+
puts "#{@dbm} loaded"
|
20
|
+
break
|
21
|
+
rescue LoadError => e
|
22
|
+
end
|
23
|
+
end
|
24
|
+
raise "No DBM library found. Tried bdb,dbm,gdbm" unless @dbm
|
25
|
+
@db = Hash.new
|
26
|
+
@queues.keys.each {|q| _open_queue(q)}
|
27
|
+
end
|
28
|
+
|
29
|
+
def dbmopen(dbname)
|
30
|
+
if @dbm == 'bdb'
|
31
|
+
BDB::Hash.new(dbname, nil, "a")
|
32
|
+
elsif @dbm == 'dbm'
|
33
|
+
DBM.open(dbname)
|
34
|
+
elsif @dbm == 'gdbm'
|
35
|
+
GDBM.open(dbname)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
|
40
|
+
def _open_queue(dest)
|
41
|
+
queue_name = dest.gsub('/', '_')
|
42
|
+
dbname = @directory + '/' + queue_name
|
43
|
+
@db[dest] = Hash.new
|
44
|
+
@db[dest][:dbh] = dbmopen(dbname)
|
45
|
+
@db[dest][:dbname] = dbname
|
46
|
+
end
|
47
|
+
|
48
|
+
|
49
|
+
def _close_queue(dest)
|
50
|
+
@db[dest][:dbh].close
|
51
|
+
dbname = @db[dest][:dbname]
|
52
|
+
File.delete(dbname) if File.exists?(dbname)
|
53
|
+
File.delete("#{dbname}.db") if File.exists?("#{dbname}.db")
|
54
|
+
File.delete("#{dbname}.pag") if File.exists?("#{dbname}.pag")
|
55
|
+
File.delete("#{dbname}.dir") if File.exists?("#{dbname}.dir")
|
56
|
+
end
|
57
|
+
|
58
|
+
def _writeframe(dest,frame,msgid)
|
59
|
+
@db[dest][:dbh][msgid] = Marshal::dump(frame)
|
60
|
+
end
|
61
|
+
|
62
|
+
def _readframe(dest,msgid)
|
63
|
+
frame_image = @db[dest][:dbh][msgid]
|
64
|
+
Marshal::load(frame_image)
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
68
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
|
2
|
+
module StompServer
|
3
|
+
class FileQueue < Queue
|
4
|
+
|
5
|
+
def _close_queue(dest)
|
6
|
+
Dir.delete(@queues[dest][:queue_dir]) if File.directory?(@queues[dest][:queue_dir])
|
7
|
+
end
|
8
|
+
|
9
|
+
def _open_queue(dest)
|
10
|
+
# handle clashes between _ and /
|
11
|
+
queue_name = dest.gsub('_','__')
|
12
|
+
queue_name = dest.gsub('/','_')
|
13
|
+
queue_dir = @directory + '/' + queue_name
|
14
|
+
@queues[dest][:queue_dir] = queue_dir
|
15
|
+
Dir.mkdir(queue_dir) unless File.directory?(queue_dir)
|
16
|
+
end
|
17
|
+
|
18
|
+
def _writeframe(dest,frame_todump,msgid)
|
19
|
+
filename = "#{@queues[dest][:queue_dir]}/#{msgid}"
|
20
|
+
frame = frame_todump.dup
|
21
|
+
frame_body = frame.body
|
22
|
+
frame.body = ''
|
23
|
+
frame_image = Marshal.dump(frame)
|
24
|
+
framelen = sprintf("%08x", frame_image.length)
|
25
|
+
bodylen = sprintf("%08x", frame_body.length)
|
26
|
+
File.open(filename,'wb') {|f| f.syswrite("#{framelen}#{bodylen}#{frame_image}#{frame_body}")}
|
27
|
+
return true
|
28
|
+
end
|
29
|
+
|
30
|
+
def _readframe(dest,msgid)
|
31
|
+
filename = "#{@queues[dest][:queue_dir]}/#{msgid}"
|
32
|
+
file = nil
|
33
|
+
File.open(filename,'rb') {|f| file = f.read}
|
34
|
+
frame_len = file[0,8].hex
|
35
|
+
body_len = file[8,8].hex
|
36
|
+
frame = Marshal::load(file[16,frame_len])
|
37
|
+
frame.body = file[(frame_len + 16),body_len]
|
38
|
+
if File.delete(filename)
|
39
|
+
result = frame
|
40
|
+
else
|
41
|
+
result = false
|
42
|
+
end
|
43
|
+
return result
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
@@ -0,0 +1,58 @@
|
|
1
|
+
|
2
|
+
module StompServer
|
3
|
+
class MemoryQueue
|
4
|
+
|
5
|
+
def initialize
|
6
|
+
@frame_index =0
|
7
|
+
@stompid = StompServer::StompId.new
|
8
|
+
@stats = Hash.new
|
9
|
+
@messages = Hash.new { Array.new }
|
10
|
+
puts "MemoryQueue initialized"
|
11
|
+
end
|
12
|
+
|
13
|
+
def stop
|
14
|
+
end
|
15
|
+
|
16
|
+
def monitor
|
17
|
+
stats = Hash.new
|
18
|
+
@messages.keys.each do |dest|
|
19
|
+
stats[dest] = {'size' => @messages[dest].size, 'enqueued' => @stats[dest][:enqueued], 'dequeued' => @stats[dest][:dequeued]}
|
20
|
+
end
|
21
|
+
stats
|
22
|
+
end
|
23
|
+
|
24
|
+
def dequeue(dest)
|
25
|
+
if frame = @messages[dest].shift
|
26
|
+
@stats[dest][:dequeued] += 1
|
27
|
+
return frame
|
28
|
+
else
|
29
|
+
return false
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def enqueue(dest,frame)
|
34
|
+
@frame_index += 1
|
35
|
+
if @stats[dest]
|
36
|
+
@stats[dest][:enqueued] += 1
|
37
|
+
else
|
38
|
+
@stats[dest] = Hash.new
|
39
|
+
@stats[dest][:enqueued] = 1
|
40
|
+
@stats[dest][:dequeued] = 0
|
41
|
+
end
|
42
|
+
assign_id(frame, dest)
|
43
|
+
requeue(dest, frame)
|
44
|
+
end
|
45
|
+
|
46
|
+
def requeue(dest,frame)
|
47
|
+
@messages[dest] += [frame]
|
48
|
+
end
|
49
|
+
|
50
|
+
def message_for?(dest)
|
51
|
+
!@messages[dest].empty?
|
52
|
+
end
|
53
|
+
|
54
|
+
def assign_id(frame, dest)
|
55
|
+
frame.headers['message-id'] = @stompid[@frame_index]
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,209 @@
|
|
1
|
+
# QueueManager is used in conjunction with a storage class. The storage class MUST implement the following two methods:
|
2
|
+
#
|
3
|
+
# - enqueue(queue name, frame)
|
4
|
+
# enqueue pushes a frame to the top of the queue in FIFO order. It's return value is ignored. enqueue must also set the
|
5
|
+
# message-id and add it to the frame header before inserting the frame into the queue.
|
6
|
+
#
|
7
|
+
# - dequeue(queue name)
|
8
|
+
# dequeue removes a frame from the bottom of the queue and returns it.
|
9
|
+
#
|
10
|
+
# - requeue(queue name,frame)
|
11
|
+
# does the same as enqueue, except it puts the from at the bottom of the queue
|
12
|
+
#
|
13
|
+
# The storage class MAY implement the stop() method which can be used to do any housekeeping that needs to be done before
|
14
|
+
# stompserver shuts down. stop() will be called when stompserver is shut down.
|
15
|
+
#
|
16
|
+
# The storage class MAY implement the monitor() method. monitor() should return a hash of hashes containing the queue statistics.
|
17
|
+
# See the file queue for an example. Statistics are available to clients in /queue/monitor.
|
18
|
+
#
|
19
|
+
|
20
|
+
module StompServer
|
21
|
+
class QueueMonitor
|
22
|
+
|
23
|
+
def initialize(qstore,queues)
|
24
|
+
@qstore = qstore
|
25
|
+
@queues = queues
|
26
|
+
@stompid = StompServer::StompId.new
|
27
|
+
puts "QueueManager initialized"
|
28
|
+
end
|
29
|
+
|
30
|
+
def start
|
31
|
+
count =0
|
32
|
+
EventMachine::add_periodic_timer 5, proc {count+=1; monitor(count) }
|
33
|
+
end
|
34
|
+
|
35
|
+
def monitor(count)
|
36
|
+
return unless @qstore.methods.include?('monitor')
|
37
|
+
users = @queues['/queue/monitor']
|
38
|
+
return if users.size == 0
|
39
|
+
stats = @qstore.monitor
|
40
|
+
return if stats.size == 0
|
41
|
+
body = ''
|
42
|
+
|
43
|
+
stats.each do |queue,qstats|
|
44
|
+
body << "Queue: #{queue}\n"
|
45
|
+
qstats.each {|stat,value| body << "#{stat}: #{value}\n"}
|
46
|
+
body << "\n"
|
47
|
+
end
|
48
|
+
|
49
|
+
headers = {
|
50
|
+
'message-id' => @stompid[count],
|
51
|
+
'destination' => '/queue/monitor',
|
52
|
+
'content-length' => body.size.to_s
|
53
|
+
}
|
54
|
+
|
55
|
+
frame = StompServer::StompFrame.new('MESSAGE', headers, body)
|
56
|
+
users.each {|user| user.user.stomp_send_data(frame)}
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
class QueueManager
|
61
|
+
Struct::new('QueueUser', :connection, :ack)
|
62
|
+
|
63
|
+
def initialize(qstore)
|
64
|
+
@qstore = qstore
|
65
|
+
@queues = Hash.new { Array.new }
|
66
|
+
@pending = Hash.new
|
67
|
+
if $STOMP_SERVER
|
68
|
+
monitor = StompServer::QueueMonitor.new(@qstore,@queues)
|
69
|
+
monitor.start
|
70
|
+
puts "Queue monitor started" if $DEBUG
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def stop
|
75
|
+
@qstore.stop if @qstore.methods.include?('stop')
|
76
|
+
end
|
77
|
+
|
78
|
+
def subscribe(dest, connection, use_ack=false)
|
79
|
+
puts "Subscribing to #{dest}"
|
80
|
+
user = Struct::QueueUser.new(connection, use_ack)
|
81
|
+
@queues[dest] += [user]
|
82
|
+
send_destination_backlog(dest,user) unless dest == '/queue/monitor'
|
83
|
+
end
|
84
|
+
|
85
|
+
# Send at most one frame to a connection
|
86
|
+
# used when use_ack == true
|
87
|
+
def send_a_backlog(connection)
|
88
|
+
puts "Sending a backlog" if $DEBUG
|
89
|
+
# lookup queues with data for this connection
|
90
|
+
possible_queues = @queues.select{ |destination,users|
|
91
|
+
@qstore.message_for?(destination) &&
|
92
|
+
users.detect{|u| u.connection == connection}
|
93
|
+
}
|
94
|
+
if possible_queues.empty?
|
95
|
+
puts "Nothing left" if $DEBUG
|
96
|
+
return
|
97
|
+
end
|
98
|
+
# Get a random one (avoid artificial priority between queues
|
99
|
+
# without coding a whole scheduler, which might be desirable later)
|
100
|
+
dest,users = possible_queues[rand(possible_queues.length)]
|
101
|
+
user = users.find{|u| u.connection == connection}
|
102
|
+
frame = @qstore.dequeue(dest)
|
103
|
+
puts "Chose #{dest}" if $DEBUG
|
104
|
+
send_to_user(frame, user)
|
105
|
+
end
|
106
|
+
|
107
|
+
def send_destination_backlog(dest,user)
|
108
|
+
puts "Sending destination backlog for #{dest}" if $DEBUG
|
109
|
+
if user.ack
|
110
|
+
# only send one message (waiting for ack)
|
111
|
+
frame = @qstore.dequeue(dest)
|
112
|
+
send_to_user(frame, user) if frame
|
113
|
+
else
|
114
|
+
while frame = @qstore.dequeue(dest)
|
115
|
+
send_to_user(frame, user)
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
def unsubscribe(dest, connection)
|
121
|
+
puts "Unsubscribing from #{dest}"
|
122
|
+
@queues.each do |d, queue|
|
123
|
+
queue.delete_if { |qu| qu.connection == connection and d == dest}
|
124
|
+
end
|
125
|
+
@queues.delete(dest) if @queues[dest].empty?
|
126
|
+
end
|
127
|
+
|
128
|
+
def ack(connection, frame)
|
129
|
+
puts "Acking #{frame.headers['message-id']}" if $DEBUG
|
130
|
+
unless @pending[connection]
|
131
|
+
puts "No message pending for connection!"
|
132
|
+
return
|
133
|
+
end
|
134
|
+
msgid = frame.headers['message-id']
|
135
|
+
p_msgid = @pending[connection].headers['message-id']
|
136
|
+
if p_msgid != msgid
|
137
|
+
# We don't know what happened, we requeue
|
138
|
+
# (probably a client connecting to a restarted server)
|
139
|
+
frame = @pending[connection]
|
140
|
+
@qstore.requeue(frame.headers['destination'],frame)
|
141
|
+
puts "Invalid message-id (received #{msgid} != #{p_msgid})"
|
142
|
+
end
|
143
|
+
@pending.delete connection
|
144
|
+
# We are free to work now, look if there's something for us
|
145
|
+
send_a_backlog(connection)
|
146
|
+
end
|
147
|
+
|
148
|
+
def disconnect(connection)
|
149
|
+
puts "Disconnecting"
|
150
|
+
frame = @pending[connection]
|
151
|
+
if frame
|
152
|
+
@qstore.requeue(frame.headers['destination'],frame)
|
153
|
+
@pending.delete connection
|
154
|
+
end
|
155
|
+
|
156
|
+
@queues.each do |dest, queue|
|
157
|
+
queue.delete_if { |qu| qu.connection == connection }
|
158
|
+
@queues.delete(dest) if queue.empty?
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
def send_to_user(frame, user)
|
163
|
+
connection = user.connection
|
164
|
+
if user.ack
|
165
|
+
raise "other connection's end already busy" if @pending[connection]
|
166
|
+
@pending[connection] = frame
|
167
|
+
end
|
168
|
+
connection.stomp_send_data(frame)
|
169
|
+
end
|
170
|
+
|
171
|
+
def sendmsg(frame)
|
172
|
+
frame.command = "MESSAGE"
|
173
|
+
dest = frame.headers['destination']
|
174
|
+
puts "Sending a message to #{dest}: #{frame}"
|
175
|
+
# Lookup a user willing to handle this destination
|
176
|
+
available_users = @queues[dest].reject{|user| @pending[user.connection]}
|
177
|
+
if available_users.empty?
|
178
|
+
@qstore.enqueue(dest,frame)
|
179
|
+
return
|
180
|
+
end
|
181
|
+
|
182
|
+
# Look for a user with ack (we favor reliability)
|
183
|
+
reliable_user = available_users.find{|u| u.ack}
|
184
|
+
|
185
|
+
if reliable_user
|
186
|
+
# give it a message-id
|
187
|
+
@qstore.assign_id(frame, dest)
|
188
|
+
send_to_user(frame, reliable_user)
|
189
|
+
else
|
190
|
+
random_user = available_users[rand(available_users.length)]
|
191
|
+
# Note message-id header isn't set but we won't need it anyway
|
192
|
+
# <TODO> could break some clients: fix this
|
193
|
+
send_to_user(frame, random_user)
|
194
|
+
end
|
195
|
+
end
|
196
|
+
|
197
|
+
# For protocol handlers that want direct access to the queue
|
198
|
+
def dequeue(dest)
|
199
|
+
@qstore.dequeue(dest)
|
200
|
+
end
|
201
|
+
|
202
|
+
def enqueue(frame)
|
203
|
+
frame.command = "MESSAGE"
|
204
|
+
dest = frame.headers['destination']
|
205
|
+
@qstore.enqueue(dest,frame)
|
206
|
+
end
|
207
|
+
|
208
|
+
end
|
209
|
+
end
|