mqrpc 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,232 @@
1
+ require 'rubygems'
2
+ require 'amqp'
3
+ require 'mq'
4
+ require 'mqrpc/logger'
5
+ require 'mqrpc/operation'
6
+ require 'thread'
7
+ require 'uuid'
8
+
9
+ # http://github.com/tmm1/amqp/issues/#issue/3
10
+ # This is our (lame) hack to at least notify the user that something is
11
+ # wrong.
12
+ module AMQP
13
+ module Client
14
+ alias :original_reconnect :reconnect
15
+ def reconnect(*args)
16
+ $logger.warn "reconnecting to broker (bad MQ settings?)"
17
+
18
+ # some rate limiting
19
+ sleep(5)
20
+
21
+ original_reconnect(*args)
22
+ end
23
+ end
24
+ end
25
+
26
+ module MQRPC
27
+ # TODO: document this class
28
+ class Agent
29
+ MAXBUF = 20
30
+
31
+ def initialize(config)
32
+ Thread::abort_on_exception = true
33
+ @config = config
34
+ @handler = self
35
+ @id = UUID::generate
36
+ @outbuffer = Hash.new { |h,k| h[k] = [] }
37
+ @queues = []
38
+ @receive_queue = Queue.new
39
+ @topics = []
40
+ @want_queues = []
41
+ @want_topics = []
42
+ #@slidingwindow = LogStash::SlidingWindowSet.new
43
+
44
+ @mq = nil
45
+ @message_operations = Hash.new
46
+
47
+ @startup_mutex = Mutex.new
48
+ @startup_condvar = ConditionVariable.new
49
+ @amqp_ready = false
50
+
51
+ start_amqp
52
+ @startup_mutex.synchronize do
53
+ MQRPC::logger.debug "Waiting for @mq ..."
54
+ @startup_condvar.wait(@startup_mutex) if !@amqp_ready
55
+ end
56
+
57
+ start_receiver
58
+ end # def initialize
59
+
60
+ def start_amqp
61
+ @amqpthread = Thread.new do
62
+ # Create connection to AMQP, and in turn, the main EventMachine loop.
63
+ amqp_config = {:host => @config.mqhost,
64
+ :port => @config.mqport,
65
+ :user => @config.mquser,
66
+ :pass => @config.mqpass,
67
+ :vhost => @config.mqvhost,
68
+ }
69
+ AMQP.start(amqp_config) do
70
+ @startup_mutex.synchronize do
71
+ @mq = MQ.new
72
+ # Notify the main calling thread (MessageSocket#initialize) that
73
+ # we can continue
74
+ @amqp_ready = true
75
+ @startup_condvar.signal
76
+ end
77
+
78
+ MQRPC::logger.info "Subscribing to main queue #{@id}"
79
+ mq_q = @mq.queue(@id, :auto_delete => true)
80
+ mq_q.subscribe(:ack =>true) { |hdr, msg| @receive_queue << [hdr, msg] }
81
+ handle_new_subscriptions
82
+
83
+ # TODO(sissel): make this a deferred thread that reads from a Queue
84
+ EM.add_periodic_timer(5) { handle_new_subscriptions }
85
+
86
+ EM.add_periodic_timer(1) do
87
+ # TODO(sissel): add locking
88
+ @outbuffer.each_key { |dest| flushout(dest) }
89
+ @outbuffer.clear
90
+ end
91
+ end # AMQP.start
92
+ end
93
+ end # def start_amqp
94
+
95
+ def start_receiver
96
+ Thread.new do
97
+ while true
98
+ header, message = @receive_queue.pop
99
+ handle_message(header, message)
100
+ end
101
+ end
102
+ end # def start_receiver
103
+
104
+ def subscribe(name)
105
+ @want_queues << name
106
+ end # def subscribe
107
+
108
+ def subscribe_topic(name)
109
+ @want_topics << name
110
+ end # def subscribe_topic
111
+
112
+ def handle_message(hdr, msg_body)
113
+ obj = JSON::load(msg_body)
114
+ if !obj.is_a?(Array)
115
+ obj = [obj]
116
+ end
117
+
118
+ obj.each do |item|
119
+ message = Message.new_from_data(item)
120
+ #if @slidingwindow.include?(message.id)
121
+ #puts "Removing ack for #{message.id}"
122
+ #@slidingwindow.delete(message.id)
123
+ #end
124
+ name = message.class.name.split(":")[-1]
125
+ func = "#{name}Handler"
126
+
127
+ # Check if we have a specific operation looking for this
128
+ # message.
129
+ if (message.respond_to?(:in_reply_to) and
130
+ @message_operations.has_key?(message.in_reply_to))
131
+ operation = @message_operations[message.in_reply_to]
132
+ operation.call message
133
+ elsif @handler.respond_to?(func)
134
+ @handler.send(func, message) do |response|
135
+ reply = message.reply_to
136
+ sendmsg(reply, response)
137
+ end
138
+
139
+ # We should allow the message handler to defer acking if they want
140
+ # For instance, if we want to index things, but only want to ack
141
+ # things once we actually flush to disk.
142
+ else
143
+ $stderr.puts "#{@handler.class.name} does not support #{func}"
144
+ end # if @handler.respond_to?(func)
145
+ end
146
+ hdr.ack
147
+ end # def handle_message
148
+
149
+ def run
150
+ @amqpthread.join
151
+ end # run
152
+
153
+ def handle_new_subscriptions
154
+ todo = @want_queues - @queues
155
+ todo.each do |queue|
156
+ MQRPC::logger.info "Subscribing to queue #{queue}"
157
+ mq_q = @mq.queue(queue, :durable => true)
158
+ mq_q.subscribe(:ack => true) { |hdr, msg| @receive_queue << [hdr, msg] }
159
+ @queues << queue
160
+ end # todo.each
161
+
162
+ todo = @want_topics - @topics
163
+ todo.each do |topic|
164
+ MQRPC::logger.info "Subscribing to topic #{topic}"
165
+ exchange = @mq.topic("amq.topic")
166
+ mq_q = @mq.queue("#{@id}-#{topic}",
167
+ :exclusive => true,
168
+ :auto_delete => true).bind(exchange, :key => topic)
169
+ mq_q.subscribe { |hdr, msg| @receive_queue << [hdr, msg] }
170
+ @topics << topic
171
+ end # todo.each
172
+ end # handle_new_subscriptions
173
+
174
+ def flushout(destination)
175
+ return unless @mq # wait until we are connected
176
+
177
+ msgs = @outbuffer[destination]
178
+ return if msgs.length == 0
179
+ data = msgs.to_json
180
+ @mq.queue(destination, :durable => true).publish(data, :persistent => true)
181
+ msgs.clear
182
+ end
183
+
184
+ def sendmsg_topic(key, msg)
185
+ return unless @mq # wait until we are connected
186
+ if (msg.is_a?(RequestMessage) and msg.id == nil)
187
+ msg.generate_id!
188
+ end
189
+ msg.timestamp = Time.now.to_f
190
+
191
+ data = msg.to_json
192
+ @mq.topic("amq.topic").publish(data, :key => key)
193
+ end
194
+
195
+ def sendmsg(destination, msg, &callback)
196
+ return unless @mq # wait until we are connected
197
+ if (msg.is_a?(RequestMessage) and msg.id == nil)
198
+ msg.generate_id!
199
+ end
200
+ msg.timestamp = Time.now.to_f
201
+ msg.reply_to = @id
202
+
203
+ #if (msg.is_a?(RequestMessage) and !msg.is_a?(ResponseMessage))
204
+ #MQRPC::logger.info "Tracking #{msg.class.name}##{msg.id}"
205
+ #@slidingwindow << msg.id
206
+ #end
207
+
208
+ if msg.buffer?
209
+ @outbuffer[destination] << msg
210
+ if @outbuffer[destination].length > MAXBUF
211
+ flushout(destination)
212
+ end
213
+ else
214
+ @mq.queue(destination, :durable => true).publish([msg].to_json, :persistent => true)
215
+ end
216
+
217
+ if block_given?
218
+ op = Operation.new(callback)
219
+ @message_operations[msg.id] = op
220
+ return op
221
+ end
222
+ end
223
+
224
+ def handler=(handler)
225
+ @handler = handler
226
+ end
227
+
228
+ def close
229
+ EM.stop_event_loop
230
+ end
231
+ end # class Agent
232
+ end # module MQRPC
@@ -0,0 +1,17 @@
1
+ module MQRPC
2
+ class Config
3
+ attr_reader :mqhost
4
+ attr_reader :mqport
5
+ attr_reader :mquser
6
+ attr_reader :mqpass
7
+ attr_reader :mqvhost
8
+
9
+ def initialize(options = {})
10
+ @mqhost = options["mqhost"] || "localhost"
11
+ @mqport = options["mqport"] || 5672
12
+ @mquser = options["mquser"] || "guest"
13
+ @mqpass = options["mqpass"] || "guest"
14
+ @mqvhost = options["mqvhost"] || "/"
15
+ end # def initialize
16
+ end # class Config
17
+ end # module MQRPC
@@ -0,0 +1,12 @@
1
+ require 'rubygems'
2
+ require 'mqrpc/messages/ping'
3
+
4
+ module MQRPC; module Functions; module Ping
5
+ def PingRequestHandler(request)
6
+ MQRPC::logger.debug "received PingRequest (#{request.pingdata})"
7
+ response = MQRPC::Messages::PingResponse.new
8
+ response.id = request.id
9
+ response.pingdata = request.pingdata
10
+ yield response
11
+ end
12
+ end; end; end
@@ -0,0 +1,15 @@
1
+ require 'rubygems'
2
+ require 'logger'
3
+
4
+ module MQRPC
5
+ @logger = Logger.new(STDOUT)
6
+ @logger.level = Logger::WARN
7
+
8
+ def self.logger
9
+ return @logger
10
+ end
11
+
12
+ def self.logger=(logger)
13
+ @logger = logger
14
+ end
15
+ end
@@ -0,0 +1,142 @@
1
+ require 'json'
2
+ require 'thread'
3
+ require 'mqrpc/logger'
4
+
5
+ module BindToHash
6
+ def header(method, key=nil)
7
+ key = method.to_s if key == nil
8
+ hashbind(method, "/#{key}")
9
+ end
10
+
11
+ def argument(method, key=nil)
12
+ key = method.to_s if key == nil
13
+ hashbind(method, "/args/#{key}")
14
+ end
15
+
16
+ private
17
+ def hashbind(method, key)
18
+ hashpath = __genhashpath(key)
19
+ self.class_eval %(
20
+ def #{method}
21
+ return #{hashpath}
22
+ end
23
+ def #{method}=(val)
24
+ #{hashpath} = val
25
+ end
26
+ )
27
+ end
28
+
29
+ def __genhashpath(key)
30
+ # TODO(sissel): enforce 'key' needs to be a string or symbol?
31
+ path = key.split("/").select { |x| x.length > 0 }\
32
+ .map { |x| "[#{x.inspect}]" }
33
+ return "@data#{path.join("")}"
34
+ end
35
+ end # modules BindToHash
36
+
37
+ module MQRPC
38
+ class Message
39
+ extend BindToHash
40
+
41
+ @@idseq = 0
42
+ @@idlock = Mutex.new
43
+ @@knowntypes = Hash.new
44
+ attr_accessor :data
45
+
46
+ # Message attributes
47
+ header :id
48
+ header :message_class
49
+ header :reply_to
50
+ header :timestamp
51
+
52
+ def initialize
53
+ generate_id!
54
+ end
55
+
56
+ def generate_id!
57
+ @@idlock.synchronize do
58
+ self.id = @@idseq
59
+ @@idseq += 1
60
+ end
61
+ end
62
+
63
+ def age
64
+ return Time.now.to_f - timestamp
65
+ end
66
+
67
+ def buffer?
68
+ return @buffer
69
+ end
70
+
71
+ def want_buffer(want_buffer=true)
72
+ @buffer = want_buffer
73
+ end
74
+
75
+ def self.inherited(subclass)
76
+ MQRPC::logger.debug "Message '#{subclass.name}' subclasses #{self.name}"
77
+ @@knowntypes[subclass.name] = subclass
78
+
79
+ # Call the class initializer if it has one.
80
+ if subclass.respond_to?(:class_initialize)
81
+ subclass.class_initialize
82
+ end
83
+ end # def self.inherited
84
+
85
+ def initialize
86
+ @data = Hash.new
87
+ want_buffer(false)
88
+ self.message_class = self.class.name
89
+ end
90
+
91
+ def self.new_from_data(data)
92
+ obj = nil
93
+ name = data["message_class"]
94
+ if @@knowntypes.has_key?(name)
95
+ obj = @@knowntypes[name].new
96
+ else
97
+ $stderr.puts "No known message class: #{name}, #{data.inspect}"
98
+ obj = Message.new
99
+ end
100
+ obj.data = data
101
+ return obj
102
+ end
103
+
104
+ def to_json(*args)
105
+ return @data.to_json(*args)
106
+ end
107
+
108
+ protected
109
+ attr :data
110
+ end # class Message
111
+
112
+ class RequestMessage < Message
113
+ header :args
114
+
115
+ def initialize
116
+ super
117
+ self.args = Hash.new
118
+ end
119
+
120
+ end # class RequestMessage
121
+
122
+ class ResponseMessage < Message
123
+ header :in_reply_to
124
+ header :args
125
+
126
+ def initialize(source_request=nil)
127
+ super()
128
+
129
+ # Copy the request id if we are given a source_request
130
+ if source_request.is_a?(RequestMessage)
131
+ self.in_reply_to = source_request.id
132
+ end
133
+ self.args = Hash.new
134
+ end
135
+
136
+ # Report the success of the request this response is for.
137
+ # Should be implemented by subclasses.
138
+ def success?
139
+ raise NotImplementedError
140
+ end
141
+ end # class ResponseMessage
142
+ end # module MQRPC
@@ -0,0 +1,17 @@
1
+ require 'rubygems'
2
+ require 'lib/mqrpc'
3
+
4
+ module MQRPC::Messages
5
+ class PingRequest < MQRPC::RequestMessage
6
+ def initialize
7
+ super
8
+ self.pingdata = Time.now.to_f
9
+ end
10
+
11
+ hashbind :pingdata, "/args/pingdata"
12
+ end # class PingRequest < RequestMessage
13
+
14
+ class PingResponse < MQRPC::ResponseMessage
15
+ hashbind :pingdata, "/args/pingdata"
16
+ end
17
+ end
@@ -0,0 +1,49 @@
1
+ require 'rubygems'
2
+ require 'thread'
3
+ require 'mqrpc/logger'
4
+
5
+ module MQRPC
6
+ # A single message operation
7
+ # * Takes a callback to call when a message is received
8
+ # * Allows you to wait for the operation to complete.
9
+ # * An operation is 'complete' when the callback returns :finished
10
+ class Operation
11
+ def initialize(callback)
12
+ @mutex = Mutex.new
13
+ @callback = callback
14
+ @cv = ConditionVariable.new
15
+ @finished = false
16
+ end # def initialize
17
+
18
+ def call(*args)
19
+ # TODO(sissel): allow the callback to simply invoke 'finished' on this
20
+ # operation rather than requiring it to emit ':finished'
21
+ @mutex.synchronize do
22
+ ret = @callback.call(*args)
23
+ if ret == :finished
24
+ MQRPC::logger.debug "operation #{self} finished"
25
+ @finished = true
26
+ @cv.signal
27
+ else
28
+ return ret
29
+ end
30
+ end
31
+ end # def call
32
+
33
+ # Block until the operation has finished.
34
+ # If the operation has already finished, this method will return
35
+ # immediately.
36
+ def wait_until_finished
37
+ @mutex.synchronize do
38
+ if !finished?
39
+ @cv.wait(@mutex)
40
+ end
41
+ end
42
+ end # def wait_until_finished
43
+
44
+ protected
45
+ def finished?
46
+ return @finished
47
+ end # def finished?
48
+ end # class Operation
49
+ end # module MQRPC
data/lib/mqrpc.rb ADDED
@@ -0,0 +1,4 @@
1
+ require 'mqrpc/agent'
2
+ require 'mqrpc/config'
3
+ require 'mqrpc/logger'
4
+ require 'mqrpc/message'
metadata ADDED
@@ -0,0 +1,92 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: mqrpc
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Jordan Sissel, Pete Fritchman
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+
12
+ date: 2009-11-05 00:00:00 -08:00
13
+ default_executable:
14
+ dependencies:
15
+ - !ruby/object:Gem::Dependency
16
+ name: amqp
17
+ type: :runtime
18
+ version_requirement:
19
+ version_requirements: !ruby/object:Gem::Requirement
20
+ requirements:
21
+ - - ">="
22
+ - !ruby/object:Gem::Version
23
+ version: 0.6.0
24
+ version:
25
+ - !ruby/object:Gem::Dependency
26
+ name: json
27
+ type: :runtime
28
+ version_requirement:
29
+ version_requirements: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: 1.1.7
34
+ version:
35
+ - !ruby/object:Gem::Dependency
36
+ name: uuid
37
+ type: :runtime
38
+ version_requirement:
39
+ version_requirements: !ruby/object:Gem::Requirement
40
+ requirements:
41
+ - - ">="
42
+ - !ruby/object:Gem::Version
43
+ version: 2.0.2
44
+ version:
45
+ description: RPC mechanism using AMQP as the transport
46
+ email: logstash-dev@googlegroups.com
47
+ executables: []
48
+
49
+ extensions: []
50
+
51
+ extra_rdoc_files: []
52
+
53
+ files:
54
+ - lib/mqrpc/agent.rb
55
+ - lib/mqrpc/config.rb
56
+ - lib/mqrpc/logger.rb
57
+ - lib/mqrpc/functions/ping.rb
58
+ - lib/mqrpc/message.rb
59
+ - lib/mqrpc/operation.rb
60
+ - lib/mqrpc/messages/ping.rb
61
+ - lib/mqrpc.rb
62
+ has_rdoc: true
63
+ homepage: http://code.google.com/p/logstash/wiki/MQRPC
64
+ licenses: []
65
+
66
+ post_install_message:
67
+ rdoc_options: []
68
+
69
+ require_paths:
70
+ - lib
71
+ - lib
72
+ required_ruby_version: !ruby/object:Gem::Requirement
73
+ requirements:
74
+ - - ">="
75
+ - !ruby/object:Gem::Version
76
+ version: "0"
77
+ version:
78
+ required_rubygems_version: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: "0"
83
+ version:
84
+ requirements: []
85
+
86
+ rubyforge_project:
87
+ rubygems_version: 1.3.5
88
+ signing_key:
89
+ specification_version: 3
90
+ summary: mqrpc - RPC over Message Queue (AMQP)
91
+ test_files: []
92
+