stompserver 0.9.7 → 0.9.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,104 @@
1
+ ## Queue implementation using ActiveRecord
2
+ ##
3
+ ## all messages are stored in a single table
4
+ ## they are indexed by 'stomp_id' which is the stomp 'message-id' header
5
+ ## which must be unique accross all queues
6
+ ##
7
+ require 'stomp_server/queue/ar_message'
8
+ require 'yaml'
9
+
10
+ module StompServer
11
+ class ActiveRecordQueue
12
+
13
+ def initialize(configdir, storagedir)
14
+ # Default configuration, use SQLite for simplicity
15
+ db_params = {
16
+ 'adapter' => 'sqlite3',
17
+ 'database' => "#{configdir}/stompserver_development"
18
+ }
19
+ # Load DB configuration
20
+ db_config = "#{configdir}/database.yml"
21
+ puts "reading from #{db_config}"
22
+ if File.exists? db_config
23
+ db_params.merge! YAML::load(File.open(db_config))
24
+ end
25
+
26
+ puts "using #{db_params['database']} DB"
27
+
28
+ # Setup activerecord
29
+ ActiveRecord::Base.establish_connection(db_params)
30
+ # Development <TODO> fix this
31
+ ActiveRecord::Base.logger = Logger.new(STDERR)
32
+ ActiveRecord::Base.logger.level = Logger::INFO
33
+ # we need the connection, it can't be done earlier
34
+ ArMessage.reset_column_information
35
+ reload_queues
36
+ @stompid = StompServer::StompId.new
37
+ end
38
+
39
+ # Add a frame to the queue
40
+ def enqueue(queue_name, frame)
41
+ unless @frames[queue_name]
42
+ @frames[queue_name] = {
43
+ :last_index => 0,
44
+ :frames => [],
45
+ }
46
+ end
47
+ affect_msgid_and_store(frame, queue_name)
48
+ @frames[queue_name][:frames] << frame
49
+ end
50
+
51
+ # Get and remove a frame from the queue
52
+ def dequeue(queue_name)
53
+ return nil unless @frames[queue_name] && !@frames[queue_name][:frames].empty?
54
+ frame = @frames[queue_name][:frames].shift
55
+ remove_from_store(frame.headers['message-id'])
56
+ return frame
57
+ end
58
+
59
+ # Requeue the frame previously pending
60
+ def requeue(queue_name, frame)
61
+ @frames[queue_name][:frames] << frame
62
+ ArMessage.create!(:stomp_id => frame.headers['message-id'],
63
+ :frame => frame)
64
+ end
65
+
66
+ # remove a frame from the store
67
+ def remove_from_store(message_id)
68
+ ArMessage.find_by_stomp_id(message_id).destroy
69
+ end
70
+
71
+ # store a frame (assigning it a message-id)
72
+ def affect_msgid_and_store(frame, queue_name)
73
+ msgid = assign_id(frame, queue_name)
74
+ ArMessage.create!(:stomp_id => msgid, :frame => frame)
75
+ end
76
+
77
+ def message_for?(queue_name)
78
+ @frames[queue_name] && !@frames[queue_name][:frames].empty?
79
+ end
80
+
81
+ def assign_id(frame, queue_name)
82
+ msgid = @stompid[@frames[queue_name][:last_index] += 1]
83
+ frame.headers['message-id'] = msgid
84
+ end
85
+
86
+ private
87
+ def reload_queues
88
+ @frames = Hash.new
89
+ ArMessage.find(:all).each { |message|
90
+ frame = message.frame
91
+ destination = frame.dest
92
+ msgid = message.stomp_id
93
+ @frames[destination] ||= Hash.new
94
+ @frames[destination][:frames] ||= Array.new
95
+ @frames[destination][:frames] << frame
96
+ }
97
+ # compute base index for each destination
98
+ @frames.each_pair { |destination,hash|
99
+ hash[:last_index] = hash[:frames].map{|f|
100
+ f.headers['message-id'].match(/(\d+)\Z/)[0].to_i}.max
101
+ }
102
+ end
103
+ end
104
+ end
@@ -0,0 +1,5 @@
1
+ require 'active_record'
2
+
3
+ class ArMessage < ActiveRecord::Base
4
+ serialize :frame
5
+ end
@@ -0,0 +1,68 @@
1
+
2
+ module StompServer
3
+ class DBMQueue < Queue
4
+
5
+ def initialize *args
6
+ super
7
+ # Please don't use dbm files for storing large frames, it's problematic at best and uses large amounts of memory.
8
+ # sdbm croaks on marshalled data that contains certain characters, so we don't use it at all
9
+ @dbm = false
10
+ if RUBY_PLATFORM =~/linux|bsd/
11
+ types = ['bdb','dbm','gdbm']
12
+ else
13
+ types = ['bdb','gdbm']
14
+ end
15
+ types.each do |dbtype|
16
+ begin
17
+ require dbtype
18
+ @dbm = dbtype
19
+ puts "#{@dbm} loaded"
20
+ break
21
+ rescue LoadError => e
22
+ end
23
+ end
24
+ raise "No DBM library found. Tried bdb,dbm,gdbm" unless @dbm
25
+ @db = Hash.new
26
+ @queues.keys.each {|q| _open_queue(q)}
27
+ end
28
+
29
+ def dbmopen(dbname)
30
+ if @dbm == 'bdb'
31
+ BDB::Hash.new(dbname, nil, "a")
32
+ elsif @dbm == 'dbm'
33
+ DBM.open(dbname)
34
+ elsif @dbm == 'gdbm'
35
+ GDBM.open(dbname)
36
+ end
37
+ end
38
+
39
+
40
+ def _open_queue(dest)
41
+ queue_name = dest.gsub('/', '_')
42
+ dbname = @directory + '/' + queue_name
43
+ @db[dest] = Hash.new
44
+ @db[dest][:dbh] = dbmopen(dbname)
45
+ @db[dest][:dbname] = dbname
46
+ end
47
+
48
+
49
+ def _close_queue(dest)
50
+ @db[dest][:dbh].close
51
+ dbname = @db[dest][:dbname]
52
+ File.delete(dbname) if File.exists?(dbname)
53
+ File.delete("#{dbname}.db") if File.exists?("#{dbname}.db")
54
+ File.delete("#{dbname}.pag") if File.exists?("#{dbname}.pag")
55
+ File.delete("#{dbname}.dir") if File.exists?("#{dbname}.dir")
56
+ end
57
+
58
+ def _writeframe(dest,frame,msgid)
59
+ @db[dest][:dbh][msgid] = Marshal::dump(frame)
60
+ end
61
+
62
+ def _readframe(dest,msgid)
63
+ frame_image = @db[dest][:dbh][msgid]
64
+ Marshal::load(frame_image)
65
+ end
66
+
67
+ end
68
+ end
@@ -0,0 +1,47 @@
1
+
2
+ module StompServer
3
+ class FileQueue < Queue
4
+
5
+ def _close_queue(dest)
6
+ Dir.delete(@queues[dest][:queue_dir]) if File.directory?(@queues[dest][:queue_dir])
7
+ end
8
+
9
+ def _open_queue(dest)
10
+ # handle clashes between _ and /
11
+ queue_name = dest.gsub('_','__')
12
+ queue_name = dest.gsub('/','_')
13
+ queue_dir = @directory + '/' + queue_name
14
+ @queues[dest][:queue_dir] = queue_dir
15
+ Dir.mkdir(queue_dir) unless File.directory?(queue_dir)
16
+ end
17
+
18
+ def _writeframe(dest,frame_todump,msgid)
19
+ filename = "#{@queues[dest][:queue_dir]}/#{msgid}"
20
+ frame = frame_todump.dup
21
+ frame_body = frame.body
22
+ frame.body = ''
23
+ frame_image = Marshal.dump(frame)
24
+ framelen = sprintf("%08x", frame_image.length)
25
+ bodylen = sprintf("%08x", frame_body.length)
26
+ File.open(filename,'wb') {|f| f.syswrite("#{framelen}#{bodylen}#{frame_image}#{frame_body}")}
27
+ return true
28
+ end
29
+
30
+ def _readframe(dest,msgid)
31
+ filename = "#{@queues[dest][:queue_dir]}/#{msgid}"
32
+ file = nil
33
+ File.open(filename,'rb') {|f| file = f.read}
34
+ frame_len = file[0,8].hex
35
+ body_len = file[8,8].hex
36
+ frame = Marshal::load(file[16,frame_len])
37
+ frame.body = file[(frame_len + 16),body_len]
38
+ if File.delete(filename)
39
+ result = frame
40
+ else
41
+ result = false
42
+ end
43
+ return result
44
+ end
45
+ end
46
+ end
47
+
@@ -0,0 +1,58 @@
1
+
2
+ module StompServer
3
+ class MemoryQueue
4
+
5
+ def initialize
6
+ @frame_index =0
7
+ @stompid = StompServer::StompId.new
8
+ @stats = Hash.new
9
+ @messages = Hash.new { Array.new }
10
+ puts "MemoryQueue initialized"
11
+ end
12
+
13
+ def stop
14
+ end
15
+
16
+ def monitor
17
+ stats = Hash.new
18
+ @messages.keys.each do |dest|
19
+ stats[dest] = {'size' => @messages[dest].size, 'enqueued' => @stats[dest][:enqueued], 'dequeued' => @stats[dest][:dequeued]}
20
+ end
21
+ stats
22
+ end
23
+
24
+ def dequeue(dest)
25
+ if frame = @messages[dest].shift
26
+ @stats[dest][:dequeued] += 1
27
+ return frame
28
+ else
29
+ return false
30
+ end
31
+ end
32
+
33
+ def enqueue(dest,frame)
34
+ @frame_index += 1
35
+ if @stats[dest]
36
+ @stats[dest][:enqueued] += 1
37
+ else
38
+ @stats[dest] = Hash.new
39
+ @stats[dest][:enqueued] = 1
40
+ @stats[dest][:dequeued] = 0
41
+ end
42
+ assign_id(frame, dest)
43
+ requeue(dest, frame)
44
+ end
45
+
46
+ def requeue(dest,frame)
47
+ @messages[dest] += [frame]
48
+ end
49
+
50
+ def message_for?(dest)
51
+ !@messages[dest].empty?
52
+ end
53
+
54
+ def assign_id(frame, dest)
55
+ frame.headers['message-id'] = @stompid[@frame_index]
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,209 @@
1
+ # QueueManager is used in conjunction with a storage class. The storage class MUST implement the following two methods:
2
+ #
3
+ # - enqueue(queue name, frame)
4
+ # enqueue pushes a frame to the top of the queue in FIFO order. It's return value is ignored. enqueue must also set the
5
+ # message-id and add it to the frame header before inserting the frame into the queue.
6
+ #
7
+ # - dequeue(queue name)
8
+ # dequeue removes a frame from the bottom of the queue and returns it.
9
+ #
10
+ # - requeue(queue name,frame)
11
+ # does the same as enqueue, except it puts the from at the bottom of the queue
12
+ #
13
+ # The storage class MAY implement the stop() method which can be used to do any housekeeping that needs to be done before
14
+ # stompserver shuts down. stop() will be called when stompserver is shut down.
15
+ #
16
+ # The storage class MAY implement the monitor() method. monitor() should return a hash of hashes containing the queue statistics.
17
+ # See the file queue for an example. Statistics are available to clients in /queue/monitor.
18
+ #
19
+
20
+ module StompServer
21
+ class QueueMonitor
22
+
23
+ def initialize(qstore,queues)
24
+ @qstore = qstore
25
+ @queues = queues
26
+ @stompid = StompServer::StompId.new
27
+ puts "QueueManager initialized"
28
+ end
29
+
30
+ def start
31
+ count =0
32
+ EventMachine::add_periodic_timer 5, proc {count+=1; monitor(count) }
33
+ end
34
+
35
+ def monitor(count)
36
+ return unless @qstore.methods.include?('monitor')
37
+ users = @queues['/queue/monitor']
38
+ return if users.size == 0
39
+ stats = @qstore.monitor
40
+ return if stats.size == 0
41
+ body = ''
42
+
43
+ stats.each do |queue,qstats|
44
+ body << "Queue: #{queue}\n"
45
+ qstats.each {|stat,value| body << "#{stat}: #{value}\n"}
46
+ body << "\n"
47
+ end
48
+
49
+ headers = {
50
+ 'message-id' => @stompid[count],
51
+ 'destination' => '/queue/monitor',
52
+ 'content-length' => body.size.to_s
53
+ }
54
+
55
+ frame = StompServer::StompFrame.new('MESSAGE', headers, body)
56
+ users.each {|user| user.user.stomp_send_data(frame)}
57
+ end
58
+ end
59
+
60
+ class QueueManager
61
+ Struct::new('QueueUser', :connection, :ack)
62
+
63
+ def initialize(qstore)
64
+ @qstore = qstore
65
+ @queues = Hash.new { Array.new }
66
+ @pending = Hash.new
67
+ if $STOMP_SERVER
68
+ monitor = StompServer::QueueMonitor.new(@qstore,@queues)
69
+ monitor.start
70
+ puts "Queue monitor started" if $DEBUG
71
+ end
72
+ end
73
+
74
+ def stop
75
+ @qstore.stop if @qstore.methods.include?('stop')
76
+ end
77
+
78
+ def subscribe(dest, connection, use_ack=false)
79
+ puts "Subscribing to #{dest}"
80
+ user = Struct::QueueUser.new(connection, use_ack)
81
+ @queues[dest] += [user]
82
+ send_destination_backlog(dest,user) unless dest == '/queue/monitor'
83
+ end
84
+
85
+ # Send at most one frame to a connection
86
+ # used when use_ack == true
87
+ def send_a_backlog(connection)
88
+ puts "Sending a backlog" if $DEBUG
89
+ # lookup queues with data for this connection
90
+ possible_queues = @queues.select{ |destination,users|
91
+ @qstore.message_for?(destination) &&
92
+ users.detect{|u| u.connection == connection}
93
+ }
94
+ if possible_queues.empty?
95
+ puts "Nothing left" if $DEBUG
96
+ return
97
+ end
98
+ # Get a random one (avoid artificial priority between queues
99
+ # without coding a whole scheduler, which might be desirable later)
100
+ dest,users = possible_queues[rand(possible_queues.length)]
101
+ user = users.find{|u| u.connection == connection}
102
+ frame = @qstore.dequeue(dest)
103
+ puts "Chose #{dest}" if $DEBUG
104
+ send_to_user(frame, user)
105
+ end
106
+
107
+ def send_destination_backlog(dest,user)
108
+ puts "Sending destination backlog for #{dest}" if $DEBUG
109
+ if user.ack
110
+ # only send one message (waiting for ack)
111
+ frame = @qstore.dequeue(dest)
112
+ send_to_user(frame, user) if frame
113
+ else
114
+ while frame = @qstore.dequeue(dest)
115
+ send_to_user(frame, user)
116
+ end
117
+ end
118
+ end
119
+
120
+ def unsubscribe(dest, connection)
121
+ puts "Unsubscribing from #{dest}"
122
+ @queues.each do |d, queue|
123
+ queue.delete_if { |qu| qu.connection == connection and d == dest}
124
+ end
125
+ @queues.delete(dest) if @queues[dest].empty?
126
+ end
127
+
128
+ def ack(connection, frame)
129
+ puts "Acking #{frame.headers['message-id']}" if $DEBUG
130
+ unless @pending[connection]
131
+ puts "No message pending for connection!"
132
+ return
133
+ end
134
+ msgid = frame.headers['message-id']
135
+ p_msgid = @pending[connection].headers['message-id']
136
+ if p_msgid != msgid
137
+ # We don't know what happened, we requeue
138
+ # (probably a client connecting to a restarted server)
139
+ frame = @pending[connection]
140
+ @qstore.requeue(frame.headers['destination'],frame)
141
+ puts "Invalid message-id (received #{msgid} != #{p_msgid})"
142
+ end
143
+ @pending.delete connection
144
+ # We are free to work now, look if there's something for us
145
+ send_a_backlog(connection)
146
+ end
147
+
148
+ def disconnect(connection)
149
+ puts "Disconnecting"
150
+ frame = @pending[connection]
151
+ if frame
152
+ @qstore.requeue(frame.headers['destination'],frame)
153
+ @pending.delete connection
154
+ end
155
+
156
+ @queues.each do |dest, queue|
157
+ queue.delete_if { |qu| qu.connection == connection }
158
+ @queues.delete(dest) if queue.empty?
159
+ end
160
+ end
161
+
162
+ def send_to_user(frame, user)
163
+ connection = user.connection
164
+ if user.ack
165
+ raise "other connection's end already busy" if @pending[connection]
166
+ @pending[connection] = frame
167
+ end
168
+ connection.stomp_send_data(frame)
169
+ end
170
+
171
+ def sendmsg(frame)
172
+ frame.command = "MESSAGE"
173
+ dest = frame.headers['destination']
174
+ puts "Sending a message to #{dest}: #{frame}"
175
+ # Lookup a user willing to handle this destination
176
+ available_users = @queues[dest].reject{|user| @pending[user.connection]}
177
+ if available_users.empty?
178
+ @qstore.enqueue(dest,frame)
179
+ return
180
+ end
181
+
182
+ # Look for a user with ack (we favor reliability)
183
+ reliable_user = available_users.find{|u| u.ack}
184
+
185
+ if reliable_user
186
+ # give it a message-id
187
+ @qstore.assign_id(frame, dest)
188
+ send_to_user(frame, reliable_user)
189
+ else
190
+ random_user = available_users[rand(available_users.length)]
191
+ # Note message-id header isn't set but we won't need it anyway
192
+ # <TODO> could break some clients: fix this
193
+ send_to_user(frame, random_user)
194
+ end
195
+ end
196
+
197
+ # For protocol handlers that want direct access to the queue
198
+ def dequeue(dest)
199
+ @qstore.dequeue(dest)
200
+ end
201
+
202
+ def enqueue(frame)
203
+ frame.command = "MESSAGE"
204
+ dest = frame.headers['destination']
205
+ @qstore.enqueue(dest,frame)
206
+ end
207
+
208
+ end
209
+ end