dripdrop 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,109 @@
1
+ require 'thin'
2
+ require 'json'
3
+
4
+ class DripDrop
5
+ class HTTPDeferrableBody
6
+ include EventMachine::Deferrable
7
+
8
+ def call(body)
9
+ body.each do |chunk|
10
+ @body_callback.call(chunk)
11
+ end
12
+ end
13
+
14
+ def each(&blk)
15
+ @body_callback = blk
16
+ end
17
+
18
+ def send_message(msg)
19
+ if msg.class == DripDrop::Message
20
+ json = msg.encode_json
21
+ self.call([json])
22
+ self.succeed
23
+ else
24
+ raise "Message Type not supported"
25
+ end
26
+ end
27
+ end
28
+
29
+ class HTTPApp
30
+
31
+ AsyncResponse = [-1, {}, []].freeze
32
+
33
+ def initialize(msg_format,&block)
34
+ @msg_format = msg_format
35
+ @recv_cbak = block
36
+ super
37
+ end
38
+
39
+ def call(env)
40
+ body = HTTPDeferrableBody.new
41
+
42
+ EM.next_tick do
43
+ env['async.callback'].call([200, {'Content-Type' => 'text/plain', 'Access-Control-Allow-Origin' => '*'}, body])
44
+ EM.next_tick do
45
+ case @msg_format
46
+ when :dripdrop_json
47
+ msg = DripDrop::Message.decode_json(env['rack.input'].read)
48
+ msg.head[:http_env] = env
49
+ @recv_cbak.call(body,msg)
50
+ else
51
+ raise "Unsupported message type #{@msg_format}"
52
+ end
53
+ end
54
+ end
55
+
56
+ AsyncResponse
57
+ end
58
+ end
59
+
60
+ class HTTPServerHandler
61
+ attr_reader :address, :opts
62
+
63
+ def initialize(address,opts={})
64
+ @address = address
65
+ @opts = opts
66
+ end
67
+
68
+ def on_recv(msg_format=:dripdrop_json,&block)
69
+ #Rack middleware was not meant to be used this way...
70
+ #Thin's error handling only rescues stuff w/o a backtrace
71
+ begin
72
+ Thin::Logging.debug = true
73
+ Thin::Logging.trace = true
74
+ Thin::Server.start(@address.host, @address.port) do
75
+ map '/' do
76
+ run HTTPApp.new(msg_format,&block)
77
+ end
78
+ end
79
+ rescue Exception => e
80
+ puts e.message; puts e.backtrace.join("\n");
81
+ end
82
+ end
83
+ end
84
+
85
+ class HTTPClientHandler
86
+ attr_reader :address, :opts
87
+
88
+ def initialize(address, opts={})
89
+ @address = address
90
+ @opts = opts
91
+ end
92
+
93
+ def send_message(msg,&block)
94
+ if msg.class == DripDrop::Message
95
+ req = EM::Protocols::HttpClient.request(
96
+ :host => address.host, :port => address.port,
97
+ :request => '/', :verb => 'POST',
98
+ :contenttype => 'application/json',
99
+ :content => msg.encode_json
100
+ )
101
+ req.callback do |response|
102
+ block.call(DripDrop::Message.decode_json(response[:content]))
103
+ end
104
+ else
105
+ raise "Unsupported message type '#{msg.class}'"
106
+ end
107
+ end
108
+ end
109
+ end
@@ -3,14 +3,14 @@ require 'ffi-rzmq'
3
3
  class DripDrop
4
4
  class ZMQBaseHandler
5
5
  attr_reader :address, :socket_ctype, :socket
6
-
6
+
7
7
  def initialize(address,zm_reactor,socket_ctype,opts={})
8
8
  @address = address
9
9
  @zm_reactor = zm_reactor
10
10
  @socket_ctype = socket_ctype # :bind or :connect
11
- @debug = opts[:debug]
11
+ @debug = opts[:debug] # TODO: Start actually using this
12
12
  end
13
-
13
+
14
14
  def on_attach(socket)
15
15
  @socket = socket
16
16
  if @socket_ctype == :bind
@@ -18,82 +18,70 @@ class DripDrop
18
18
  elsif @socket_ctype == :connect
19
19
  socket.connect(@address)
20
20
  else
21
- raise "Unsupported socket ctype '#{@socket_ctype}'"
21
+ raise "Unsupported socket ctype '#{@socket_ctype}'. Expected :bind or :connect"
22
22
  end
23
23
  end
24
-
24
+
25
25
  def on_recv(msg_format=:dripdrop,&block)
26
- @msg_format = msg_format
26
+ @msg_format = msg_format
27
27
  @recv_cbak = block
28
28
  self
29
29
  end
30
+
31
+ private
32
+
33
+ # Normalize Hash objs and DripDrop::Message objs into DripDrop::Message objs
34
+ def dd_messagify(message)
35
+ if message.is_a?(Hash)
36
+ return DripDrop::Message.new(message[:name], :head => message[:head],
37
+ :body => message[:body])
38
+ elsif message.is_a?(DripDrop::Message)
39
+ return message
40
+ else
41
+ return message
42
+ end
43
+ end
30
44
  end
31
45
 
32
- class ZMQWritableHandler < ZMQBaseHandler
46
+ module ZMQWritableHandler
33
47
  def initialize(*args)
34
48
  super(*args)
35
49
  @send_queue = []
36
50
  end
37
- end
38
-
39
- class ZMQReadableHandler < ZMQBaseHandler
40
- def initialize(*args,&block)
41
- super(*args)
42
- @recv_cbak = block
43
- end
44
- end
45
-
46
- class ZMQSubHandler < ZMQReadableHandler
47
- attr_reader :address, :socket_ctype
48
-
49
- def on_attach(socket)
50
- super(socket)
51
- socket.subscribe('')
52
- end
53
51
 
54
- def on_readable(socket, messages)
55
- if @msg_format == :raw
56
- @recv_cbak.call(messages)
57
- elsif @msg_format == :dripdrop
58
- unless messages.length == 2
59
- puts "Expected pub/sub message to come in two parts"
60
- return false
61
- end
62
- topic = messages.shift.copy_out_string
63
- body = messages.shift.copy_out_string
64
- msg = @recv_cbak.call(DripDrop::Message.decode(body))
65
- else
66
- raise "Unsupported message format '#{@msg_format}'"
67
- end
68
- end
69
- end
70
-
71
-
72
- class ZMQPubHandler < ZMQWritableHandler
73
- #Send any messages buffered in @send_queue
74
52
  def on_writable(socket)
75
53
  unless @send_queue.empty?
76
54
  message = @send_queue.shift
77
-
55
+
78
56
  num_parts = message.length
79
57
  message.each_with_index do |part,i|
80
- multipart = i + 1 < num_parts ? true : false
58
+ # Set the multi-part flag unless this is the last message
59
+ multipart_flag = i + 1 < num_parts ? true : false
60
+
81
61
  if part.class == ZMQ::Message
82
- socket.send_message(part, multipart)
62
+ socket.send_message(part, multipart_flag)
83
63
  else
84
- socket.send_message_string(part, multipart)
64
+ if part.class == String
65
+ socket.send_message_string(part, multipart_flag)
66
+ else
67
+ raise "Can only send Strings, not #{part.class}: #{part}"
68
+ end
85
69
  end
86
70
  end
87
71
  else
88
72
  @zm_reactor.deregister_writable(socket)
89
73
  end
90
74
  end
91
-
92
- #Sends a message along
75
+
76
+ # Sends a message, accepting either a DripDrop::Message,
77
+ # a hash that looks like a DripDrop::Message (has keys :name, :head, :body),
78
+ # or your own custom messages. Custom messages should either be a String, or
79
+ # for multipart messages, an Array of String objects.
93
80
  def send_message(message)
94
- if message.is_a?(DripDrop::Message)
95
- @send_queue.push([message.name, message.encoded])
96
- elsif message.is_a?(Array)
81
+ dd_message = dd_messagify(message)
82
+ if dd_message.is_a?(DripDrop::Message)
83
+ @send_queue.push([dd_message.encoded])
84
+ elsif message.class == Array
97
85
  @send_queue.push(message)
98
86
  else
99
87
  @send_queue.push([message])
@@ -102,35 +90,134 @@ class DripDrop
102
90
  end
103
91
  end
104
92
 
105
- class ZMQPullHandler < ZMQReadableHandler
93
+ module ZMQReadableHandler
94
+ def initialize(*args)
95
+ super(*args)
96
+ end
97
+
106
98
  def on_readable(socket, messages)
107
- if @msg_format == :raw
99
+ case @msg_format
100
+ when :raw
108
101
  @recv_cbak.call(messages)
109
- else
102
+ when :dripdrop
103
+ raise "Expected message in one part" if messages.length > 1
110
104
  body = messages.shift.copy_out_string
111
- msg = @recv_cbak.call(DripDrop::Message.decode(body))
105
+ @recv_cbak.call(DripDrop::Message.decode(body))
106
+ else
107
+ raise "Unknown message format '#{@msg_format}'"
112
108
  end
113
109
  end
114
110
  end
115
111
 
116
- class ZMQPushHandler < ZMQWritableHandler
117
- def on_writable(socket)
118
- unless @send_queue.empty?
119
- message = @send_queue.shift
120
- socket.send_message_string(message)
112
+ class ZMQSubHandler < ZMQBaseHandler
113
+ include ZMQReadableHandler
114
+
115
+ attr_reader :address, :socket_ctype
116
+ attr_accessor :topic_filter
117
+
118
+ def on_attach(socket)
119
+ super(socket)
120
+ socket.subscribe('')
121
+ end
122
+
123
+ def on_readable(socket, messages)
124
+ if @msg_format == :dripdrop
125
+ unless messages.length == 2
126
+ puts "Expected pub/sub message to come in two parts, not #{messages.length}: #{messages.inspect}"
127
+ return false
128
+ end
129
+ topic = messages.shift.copy_out_string
130
+ if @topic_filter.nil? || topic.match(@topic_filter)
131
+ body = messages.shift.copy_out_string
132
+ msg = @recv_cbak.call(DripDrop::Message.decode(body))
133
+ end
121
134
  else
122
- @zm_reactor.deregister_writable(socket)
135
+ super(socket,messages)
123
136
  end
124
137
  end
125
-
138
+ end
139
+
140
+ class ZMQPubHandler < ZMQBaseHandler
141
+ include ZMQWritableHandler
142
+
126
143
  #Sends a message along
127
144
  def send_message(message)
145
+ dd_message = dd_messagify(message)
146
+ if dd_message.is_a?(DripDrop::Message)
147
+ super([dd_message.name, dd_message.encoded])
148
+ else
149
+ super(message)
150
+ end
151
+ end
152
+ end
153
+
154
+ class ZMQPullHandler < ZMQBaseHandler
155
+ include ZMQReadableHandler
156
+
157
+
158
+ end
159
+
160
+ class ZMQPushHandler < ZMQBaseHandler
161
+ include ZMQWritableHandler
162
+ end
163
+
164
+ class ZMQXRepHandler < ZMQBaseHandler
165
+ include ZMQWritableHandler
166
+ include ZMQReadableHandler
167
+
168
+ def initialize(*args)
169
+ super(*args)
170
+ end
171
+
172
+ def on_readable(socket,messages)
173
+ if @msg_format == :dripdrop
174
+ identities = messages[0..-2].map {|m| m.copy_out_string}
175
+ body = messages.last.copy_out_string
176
+ message = DripDrop::Message.decode(body)
177
+ seq = message.head['_dripdrop/x_seq_counter']
178
+ @recv_cbak.call(identities,seq,message)
179
+ else
180
+ super(socket,messages)
181
+ end
182
+ end
183
+
184
+ def send_message(identities,seq,message)
128
185
  if message.is_a?(DripDrop::Message)
129
- @send_queue.push(message.encoded)
130
- @zm_reactor.register_writable(@socket)
186
+ message.head['_dripdrop/x_seq_counter'] = seq
187
+ super(identities + [message.encoded])
131
188
  else
132
- @send_queue.push(message)
189
+ super(message)
190
+ end
191
+ end
192
+ end
193
+
194
+ class ZMQXReqHandler < ZMQBaseHandler
195
+ include ZMQWritableHandler
196
+ include ZMQReadableHandler
197
+
198
+ def initialize(*args)
199
+ super(*args)
200
+ #Used to keep track of responses
201
+ @seq_counter = 0
202
+ @promises = {}
203
+
204
+ self.on_recv do |message|
205
+ seq = message.head['_dripdrop/x_seq_counter']
206
+ raise "Missing Seq Counter" unless seq
207
+ promise = @promises.delete(seq)
208
+ promise.call(message)
209
+ end
210
+ end
211
+
212
+ def send_message(message,&block)
213
+ dd_message = dd_messagify(message)
214
+ if dd_message.is_a?(DripDrop::Message)
215
+ @seq_counter += 1
216
+ dd_message.head['_dripdrop/x_seq_counter'] = @seq_counter
217
+ @promises[@seq_counter] = block if block
218
+ message = dd_message
133
219
  end
220
+ super(message)
134
221
  end
135
222
  end
136
223
  end
@@ -1,5 +1,6 @@
1
1
  require 'rubygems'
2
2
  require 'bert'
3
+ require 'json'
3
4
 
4
5
  class DripDrop
5
6
  #DripDrop::Message messages are exchanged between all tiers in the architecture
@@ -33,6 +34,10 @@ class DripDrop
33
34
  BERT.encode(self.to_hash)
34
35
  end
35
36
 
37
+ def encode_json
38
+ self.to_hash.to_json
39
+ end
40
+
36
41
  #Convert the Message to a hash like:
37
42
  #{:name => @name, :head => @head, :body => @body}
38
43
  def to_hash
@@ -44,7 +49,7 @@ class DripDrop
44
49
  def self.parse(msg)
45
50
  return nil if msg.nil? || msg.empty?
46
51
  #This makes parsing ZMQ messages less painful, even if its ugly here
47
- #We check the class name as a string if case we don't have ZMQ loaded
52
+ #We check the class name as a string in case we don't have ZMQ loaded
48
53
  if msg.class.to_s == 'ZMQ::Message'
49
54
  msg = msg.copy_out_string
50
55
  return nil if msg.empty?
@@ -52,7 +57,17 @@ class DripDrop
52
57
  decoded = BERT.decode(msg)
53
58
  self.new(decoded[:name], :head => decoded[:head], :body => decoded[:body])
54
59
  end
55
-
60
+
61
+ def self.decode_json(str)
62
+ begin
63
+ json_hash = JSON.parse(str)
64
+ rescue JSON::ParserError => e
65
+ puts "Could not parse msg '#{str}': #{e.message}"
66
+ return nil
67
+ end
68
+ self.new(json_hash['name'], :head => json_hash['head'], :body => json_hash['body'])
69
+ end
70
+
56
71
  private
57
72
 
58
73
  #Sanitize a string so it'll look good for JSON, BERT, and MongoDB
data/lib/dripdrop/node.rb CHANGED
@@ -7,6 +7,7 @@ require 'uri'
7
7
  require 'dripdrop/message'
8
8
  require 'dripdrop/handlers/zeromq'
9
9
  require 'dripdrop/handlers/websockets'
10
+ require 'dripdrop/handlers/http'
10
11
 
11
12
  class DripDrop
12
13
  class Node
@@ -19,15 +20,43 @@ class DripDrop
19
20
  @recipients_for = {}
20
21
  @handler_default_opts = {:debug => @debug}
21
22
  @zm_reactor = nil
22
-
23
- EM.run do
24
- ZM::Reactor.new(:my_reactor).run do |zm_reactor|
25
- @zm_reactor = zm_reactor
26
- block.call(self)
23
+ @block = block
24
+ @thread = nil
25
+ end
26
+
27
+ def start
28
+ @thread = Thread.new do
29
+ EM.run do
30
+ ZM::Reactor.new(:my_reactor).run do |zm_reactor|
31
+ @zm_reactor = zm_reactor
32
+ self.instance_eval(&@block)
33
+ end
27
34
  end
28
35
  end
29
36
  end
37
+
38
+ def join
39
+ if @thread
40
+ @thread.join
41
+ else
42
+ raise "Can't join on a node that isn't yet started"
43
+ end
44
+ end
45
+
46
+ #Blocking version of start, equivalent to +start+ then +join+
47
+ def start!
48
+ self.start
49
+ self.join
50
+ end
51
+
52
+ def stop
53
+ @zm_reactor.stop
54
+ EM.stop
55
+ end
56
+
57
+ #TODO: All these need to be majorly DRYed up
30
58
 
59
+ # Creates a ZMQ::SUB type socket. Can only receive messages via +on_recv+
31
60
  def zmq_subscribe(address,socket_ctype,opts={},&block)
32
61
  zm_addr = str_to_zm_address(address)
33
62
  h_opts = handler_opts_given(opts)
@@ -36,6 +65,7 @@ class DripDrop
36
65
  handler
37
66
  end
38
67
 
68
+ # Creates a ZMQ::PUB type socket, can only send messages via +send_message+
39
69
  def zmq_publish(address,socket_ctype,opts={})
40
70
  zm_addr = str_to_zm_address(address)
41
71
  h_opts = handler_opts_given(opts)
@@ -44,6 +74,7 @@ class DripDrop
44
74
  handler
45
75
  end
46
76
 
77
+ # Creates a ZMQ::PULL type socket. Can only receive messages via +on_recv+
47
78
  def zmq_pull(address,socket_ctype,opts={},&block)
48
79
  zm_addr = str_to_zm_address(address)
49
80
  h_opts = handler_opts_given(opts)
@@ -52,6 +83,7 @@ class DripDrop
52
83
  handler
53
84
  end
54
85
 
86
+ # Creates a ZMQ::PUSH type socket, can only send messages via +send_message+
55
87
  def zmq_push(address,socket_ctype,opts={})
56
88
  zm_addr = str_to_zm_address(address)
57
89
  h_opts = handler_opts_given(opts)
@@ -60,12 +92,54 @@ class DripDrop
60
92
  handler
61
93
  end
62
94
 
95
+ # Creates a ZMQ::XREP type socket, both sends and receivesc XREP sockets are extremely
96
+ # powerful, so their functionality is currently limited. XREP sockets in DripDrop can reply
97
+ # to the original source of the message.
98
+ #
99
+ # Receiving with XREP sockets in DripDrop is different than other types of sockets, on_recv
100
+ # passes 3 arguments to its callback, +identities+, +seq+, and +message+. Identities is the
101
+ # socket identity, seq is the sequence number of the message (all messages received at the socket
102
+ # get a monotonically incrementing +seq+, and +message+ is the message itself.
103
+ #
104
+ # To reply from an xrep handler, be sure to call send messages with the same +identities+ and +seq+
105
+ # arguments that +on_recv+ had. So, send_message takes +identities+, +seq+, and +message+.
106
+ def zmq_xrep(address,socket_ctype,opts={})
107
+ zm_addr = str_to_zm_address(address)
108
+ h_opts = handler_opts_given(opts)
109
+ handler = DripDrop::ZMQXRepHandler.new(zm_addr,@zm_reactor,socket_ctype,h_opts)
110
+ @zm_reactor.xrep_socket(handler)
111
+ handler
112
+ end
113
+
114
+ # See the documentation for +zmq_xrep+ for more info
115
+ def zmq_xreq(address,socket_ctype,opts={})
116
+ zm_addr = str_to_zm_address(address)
117
+ h_opts = handler_opts_given(opts)
118
+ handler = DripDrop::ZMQXReqHandler.new(zm_addr,@zm_reactor,socket_ctype,h_opts)
119
+ @zm_reactor.xreq_socket(handler)
120
+ handler
121
+ end
122
+
63
123
  def websocket(address,opts={},&block)
64
124
  uri = URI.parse(address)
65
125
  h_opts = handler_opts_given(opts)
66
126
  handler = DripDrop::WebSocketHandler.new(uri,h_opts)
67
127
  handler
68
128
  end
129
+
130
+ def http_server(address,opts={},&block)
131
+ uri = URI.parse(address)
132
+ h_opts = handler_opts_given(opts)
133
+ handler = DripDrop::HTTPServerHandler.new(uri, h_opts,&block)
134
+ handler
135
+ end
136
+
137
+ def http_client(address,opts={})
138
+ uri = URI.parse(address)
139
+ h_opts = handler_opts_given(opts)
140
+ handler = DripDrop::HTTPClientHandler.new(uri, h_opts)
141
+ handler
142
+ end
69
143
 
70
144
  def send_internal(dest,data)
71
145
  return false unless @recipients_for[dest]
data/lib/dripdrop.rb CHANGED
@@ -0,0 +1 @@
1
+ require 'dripdrop/node'
@@ -0,0 +1,56 @@
1
+ require 'spec_helper'
2
+
3
+ describe "zmq push/pull" do
4
+ def pp_send_messages(to_send)
5
+ responses = []
6
+ push = nil
7
+ pull = nil
8
+
9
+ @ddn = DripDrop::Node.new do
10
+ addr = rand_addr
11
+
12
+ push = zmq_push(addr, :bind)
13
+
14
+ pull1 = zmq_pull(addr, :connect)
15
+ pull2 = zmq_pull(addr, :connect)
16
+ pull = [pull1, pull2]
17
+
18
+ pull1.on_recv do |message|
19
+ message.head['recv_sock'] = 1
20
+ responses << message
21
+ end
22
+ pull2.on_recv do |message|
23
+ message.head['recv_sock'] = 2
24
+ responses << message
25
+ end
26
+
27
+ to_send.each {|message| push.send_message(message)}
28
+ end
29
+
30
+ @ddn.start
31
+ sleep 0.1
32
+ @ddn.stop
33
+
34
+ {:responses => responses, :handlers => { :push => push, :pull => [pull] }}
35
+ end
36
+ describe "basic sending and receiving" do
37
+ before(:all) do
38
+ @sent = []
39
+ 10.times {|i| @sent << DripDrop::Message.new("test-#{i}")}
40
+ pp_info = pp_send_messages(@sent)
41
+ @responses = pp_info[:responses]
42
+ @push_handler = pp_info[:handlers][:push]
43
+ @pull_handlers = pp_info[:handlers][:pull]
44
+ end
45
+
46
+ it "should receive all sent messages in order" do
47
+ @sent.zip(@responses).each do |sent,response|
48
+ sent.name.should == response.name
49
+ end
50
+ end
51
+
52
+ it "should receive messages on both pull sockets" do
53
+ @responses.map {|r| r.head['recv_sock']}.uniq.sort.should == [1,2]
54
+ end
55
+ end
56
+ end