dripdrop 0.1.0 → 0.2.0

data/lib/dripdrop/handlers/http.rb

@@ -0,0 +1,109 @@
+require 'thin'
+require 'json'
+
+class DripDrop
+  class HTTPDeferrableBody
+    include EventMachine::Deferrable
+    
+    def call(body)
+      body.each do |chunk|
+        @body_callback.call(chunk)
+      end
+    end
+
+    def each(&blk)
+      @body_callback = blk
+    end
+    
+    def send_message(msg)
+      if msg.class == DripDrop::Message
+        json = msg.encode_json
+        self.call([json])
+        self.succeed
+      else
+        raise "Message Type not supported"
+      end
+    end
+  end
+  
+  class HTTPApp
+    
+    AsyncResponse = [-1, {}, []].freeze
+    
+    def initialize(msg_format,&block)
+      @msg_format = msg_format
+      @recv_cbak  = block
+      super
+    end
+    
+    def call(env)
+      body = HTTPDeferrableBody.new
+      
+      EM.next_tick do
+        env['async.callback'].call([200, {'Content-Type' => 'text/plain', 'Access-Control-Allow-Origin' => '*'}, body])
+        EM.next_tick do
+          case @msg_format
+          when :dripdrop_json
+            msg = DripDrop::Message.decode_json(env['rack.input'].read)
+            msg.head[:http_env] = env
+            @recv_cbak.call(body,msg)
+          else
+            raise "Unsupported message type #{@msg_format}"
+          end
+        end
+      end
+       
+      AsyncResponse
+    end
+  end
+  
+  class HTTPServerHandler
+    attr_reader :address, :opts
+    
+    def initialize(address,opts={})
+      @address = address
+      @opts    = opts
+    end
+    
+    def on_recv(msg_format=:dripdrop_json,&block)
+      #Rack middleware was not meant to be used this way...
+      #Thin's error handling only rescues stuff w/o a backtrace
+      begin
+        Thin::Logging.debug = true
+        Thin::Logging.trace = true
+        Thin::Server.start(@address.host, @address.port) do
+          map '/' do
+            run HTTPApp.new(msg_format,&block)
+          end
+        end
+      rescue Exception => e
+        puts e.message; puts e.backtrace.join("\n");
+      end
+    end
+  end
+
+  class HTTPClientHandler
+    attr_reader :address, :opts
+    
+    def initialize(address, opts={})
+      @address = address
+      @opts    = opts
+    end
+    
+    def send_message(msg,&block)
+      if msg.class == DripDrop::Message
+        req = EM::Protocols::HttpClient.request(
+          :host => address.host, :port => address.port,
+          :request => '/', :verb => 'POST',
+          :contenttype => 'application/json',
+          :content => msg.encode_json
+        )
+        req.callback do |response|
+          block.call(DripDrop::Message.decode_json(response[:content]))
+        end
+      else
+        raise "Unsupported message type '#{msg.class}'"
+      end
+    end
+  end
+end

data/lib/dripdrop/handlers/zeromq.rb

@@ -3,14 +3,14 @@ require 'ffi-rzmq'
 class DripDrop
   class ZMQBaseHandler
     attr_reader :address, :socket_ctype, :socket
-    
+
     def initialize(address,zm_reactor,socket_ctype,opts={})
       @address      = address
       @zm_reactor   = zm_reactor
       @socket_ctype = socket_ctype # :bind or :connect
-      @debug        = opts[:debug]
+      @debug        = opts[:debug] # TODO: Start actually using this
     end
-    
+
     def on_attach(socket)
       @socket = socket
       if    @socket_ctype == :bind
@@ -18,82 +18,70 @@ class DripDrop
       elsif @socket_ctype == :connect
         socket.connect(@address)
       else
-        raise "Unsupported socket ctype '#{@socket_ctype}'"
+        raise "Unsupported socket ctype '#{@socket_ctype}'. Expected :bind or :connect"
       end
     end
-     
+
     def on_recv(msg_format=:dripdrop,&block)
-      @msg_format = msg_format 
+      @msg_format = msg_format
       @recv_cbak = block
       self
     end
+
+    private
+
+    # Normalize Hash objs and DripDrop::Message objs into DripDrop::Message objs
+    def dd_messagify(message)
+      if message.is_a?(Hash)
+        return DripDrop::Message.new(message[:name], :head => message[:head],
+                                                     :body => message[:body])
+      elsif message.is_a?(DripDrop::Message)
+        return message
+      else
+        return message
+      end
+    end
   end
 
-  class ZMQWritableHandler < ZMQBaseHandler
+  module ZMQWritableHandler
     def initialize(*args)
       super(*args)
       @send_queue = []
     end
-  end
-  
-  class ZMQReadableHandler < ZMQBaseHandler
-    def initialize(*args,&block)
-      super(*args)
-      @recv_cbak = block
-    end
-  end
-  
-  class ZMQSubHandler < ZMQReadableHandler
-    attr_reader :address, :socket_ctype
-    
-    def on_attach(socket)
-      super(socket)
-      socket.subscribe('')
-    end
 
-    def on_readable(socket, messages)
-      if    @msg_format == :raw
-        @recv_cbak.call(messages)
-      elsif @msg_format == :dripdrop
-        unless messages.length == 2
-          puts "Expected pub/sub message to come in two parts" 
-          return false
-        end
-        topic = messages.shift.copy_out_string
-        body  = messages.shift.copy_out_string
-        msg   = @recv_cbak.call(DripDrop::Message.decode(body))
-      else
-        raise "Unsupported message format '#{@msg_format}'"
-      end
-    end
-  end
-    
-  
-  class ZMQPubHandler < ZMQWritableHandler
-    #Send any messages buffered in @send_queue
     def on_writable(socket)
       unless @send_queue.empty?
         message = @send_queue.shift
-        
+
         num_parts = message.length
         message.each_with_index do |part,i|
-          multipart = i + 1 < num_parts ? true : false
+          # Set the multi-part flag unless this is the last message
+          multipart_flag = i + 1 < num_parts ? true : false
+
           if part.class == ZMQ::Message
-            socket.send_message(part, multipart)
+            socket.send_message(part, multipart_flag)
           else
-            socket.send_message_string(part, multipart)
+            if part.class == String
+              socket.send_message_string(part, multipart_flag)
+            else
+              raise "Can only send Strings, not #{part.class}: #{part}"
+            end
           end
         end
       else
         @zm_reactor.deregister_writable(socket)
       end
     end
-    
-    #Sends a message along
+
+    # Sends a message, accepting either a DripDrop::Message,
+    # a hash that looks like a DripDrop::Message (has keys :name, :head, :body),
+    # or your own custom messages. Custom messages should either be a String, or
+    # for multipart messages, an Array of String objects.
     def send_message(message)
-      if message.is_a?(DripDrop::Message)
-        @send_queue.push([message.name, message.encoded])
-      elsif message.is_a?(Array)
+      dd_message = dd_messagify(message)
+      if dd_message.is_a?(DripDrop::Message)
+        @send_queue.push([dd_message.encoded])
+      elsif message.class == Array
         @send_queue.push(message)
       else
         @send_queue.push([message])
@@ -102,35 +90,134 @@ class DripDrop
     end
   end
 
-  class ZMQPullHandler < ZMQReadableHandler
+  module ZMQReadableHandler
+    def initialize(*args)
+      super(*args)
+    end
+
     def on_readable(socket, messages)
-      if @msg_format == :raw
+      case @msg_format
+      when :raw
         @recv_cbak.call(messages)
-      else
+      when :dripdrop
+        raise "Expected message in one part" if messages.length > 1
         body  = messages.shift.copy_out_string
-        msg   = @recv_cbak.call(DripDrop::Message.decode(body))
+        @recv_cbak.call(DripDrop::Message.decode(body))
+      else
+        raise "Unknown message format '#{@msg_format}'"
       end
     end
   end
 
-  class ZMQPushHandler < ZMQWritableHandler
-    def on_writable(socket)
-      unless @send_queue.empty?
-        message = @send_queue.shift
-        socket.send_message_string(message)
+  class ZMQSubHandler < ZMQBaseHandler
+    include ZMQReadableHandler
+
+    attr_reader :address, :socket_ctype
+    attr_accessor :topic_filter
+
+    def on_attach(socket)
+      super(socket)
+      socket.subscribe('')
+    end
+
+    def on_readable(socket, messages)
+      if @msg_format == :dripdrop
+        unless messages.length == 2
+          puts "Expected pub/sub message to come in two parts, not #{messages.length}: #{messages.inspect}"
+          return false
+        end
+        topic = messages.shift.copy_out_string
+        if @topic_filter.nil? || topic.match(@topic_filter)
+          body  = messages.shift.copy_out_string
+          msg   = @recv_cbak.call(DripDrop::Message.decode(body))
+        end
       else
-        @zm_reactor.deregister_writable(socket)
+        super(socket,messages)
       end
     end
-    
+  end
+
+  class ZMQPubHandler < ZMQBaseHandler
+    include ZMQWritableHandler
+
     #Sends a message along
     def send_message(message)
+      dd_message = dd_messagify(message)
+      if dd_message.is_a?(DripDrop::Message)
+        super([dd_message.name, dd_message.encoded])
+      else
+        super(message)
+      end
+    end
+  end
+
+  class ZMQPullHandler < ZMQBaseHandler
+    include ZMQReadableHandler
+
+
+  end
+
+  class ZMQPushHandler < ZMQBaseHandler
+    include ZMQWritableHandler
+  end
+
+  class ZMQXRepHandler < ZMQBaseHandler
+    include ZMQWritableHandler
+    include ZMQReadableHandler
+
+    def initialize(*args)
+      super(*args)
+    end
+
+    def on_readable(socket,messages)
+      if @msg_format == :dripdrop
+        identities = messages[0..-2].map {|m| m.copy_out_string}
+        body  = messages.last.copy_out_string
+        message = DripDrop::Message.decode(body)
+        seq     = message.head['_dripdrop/x_seq_counter']
+        @recv_cbak.call(identities,seq,message)
+      else
+        super(socket,messages)
+      end
+    end
+
+    def send_message(identities,seq,message)
       if message.is_a?(DripDrop::Message)
-        @send_queue.push(message.encoded)
-        @zm_reactor.register_writable(@socket)
+        message.head['_dripdrop/x_seq_counter'] = seq
+        super(identities + [message.encoded])
       else
-        @send_queue.push(message)
+        super(message)
+      end
+    end
+  end
+
+  class ZMQXReqHandler < ZMQBaseHandler
+    include ZMQWritableHandler
+    include ZMQReadableHandler
+
+    def initialize(*args)
+      super(*args)
+      #Used to keep track of responses
+      @seq_counter = 0
+      @promises = {}
+
+      self.on_recv do |message|
+        seq = message.head['_dripdrop/x_seq_counter']
+        raise "Missing Seq Counter" unless seq
+        promise = @promises.delete(seq)
+        promise.call(message)
+      end
+    end
+
+    def send_message(message,&block)
+      dd_message = dd_messagify(message)
+      if dd_message.is_a?(DripDrop::Message)
+        @seq_counter += 1
+        dd_message.head['_dripdrop/x_seq_counter'] = @seq_counter
+        @promises[@seq_counter] = block if block
+        message = dd_message
       end
+      super(message)
     end
   end
 end

data/lib/dripdrop/message.rb

@@ -1,5 +1,6 @@
 require 'rubygems'
 require 'bert'
+require 'json'
 
 class DripDrop
   #DripDrop::Message messages are exchanged between all tiers in the architecture
@@ -33,6 +34,10 @@ class DripDrop
       BERT.encode(self.to_hash)
     end
     
+    def encode_json
+      self.to_hash.to_json
+    end
+
     #Convert the Message to a hash like:
     #{:name => @name, :head => @head, :body => @body}
     def to_hash
@@ -44,7 +49,7 @@ class DripDrop
     def self.parse(msg)
       return nil if msg.nil? || msg.empty?
       #This makes parsing ZMQ messages less painful, even if its ugly here
-      #We check the class name as a string if case we don't have ZMQ loaded
+      #We check the class name as a string in case we don't have ZMQ loaded
       if msg.class.to_s == 'ZMQ::Message'
         msg = msg.copy_out_string 
         return nil if msg.empty?
@@ -52,7 +57,17 @@ class DripDrop
       decoded = BERT.decode(msg)
       self.new(decoded[:name], :head => decoded[:head], :body => decoded[:body])
     end
-    
+
+    def self.decode_json(str)
+      begin
+        json_hash = JSON.parse(str)
+      rescue JSON::ParserError => e
+        puts "Could not parse msg '#{str}': #{e.message}"
+        return nil
+      end
+      self.new(json_hash['name'], :head => json_hash['head'], :body => json_hash['body'])
+    end
+
     private
     
     #Sanitize a string so it'll look good for JSON, BERT, and MongoDB

data/lib/dripdrop/node.rb

@@ -7,6 +7,7 @@ require 'uri'
 require 'dripdrop/message'
 require 'dripdrop/handlers/zeromq'
 require 'dripdrop/handlers/websockets'
+require 'dripdrop/handlers/http'
 
 class DripDrop
   class Node
@@ -19,15 +20,43 @@ class DripDrop
       @recipients_for = {}
       @handler_default_opts = {:debug => @debug}
       @zm_reactor = nil
-      
-      EM.run do
-        ZM::Reactor.new(:my_reactor).run do |zm_reactor|
-          @zm_reactor = zm_reactor
-          block.call(self)
+      @block = block
+      @thread = nil
+    end
+
+    def start
+      @thread = Thread.new do
+        EM.run do
+          ZM::Reactor.new(:my_reactor).run do |zm_reactor|
+            @zm_reactor = zm_reactor
+            self.instance_eval(&@block)
+          end
         end
       end
     end
+
+    def join
+      if @thread
+        @thread.join
+      else
+        raise "Can't join on a node that isn't yet started"
+      end
+    end
+
+    #Blocking version of start, equivalent to +start+ then +join+
+    def start!
+      self.start
+      self.join
+    end
+
+    def stop
+      @zm_reactor.stop
+      EM.stop
+    end
+
+    #TODO: All these need to be majorly DRYed up
      
+    # Creates a ZMQ::SUB type socket. Can only receive messages via +on_recv+
     def zmq_subscribe(address,socket_ctype,opts={},&block)
       zm_addr = str_to_zm_address(address)
       h_opts  = handler_opts_given(opts)
@@ -36,6 +65,7 @@ class DripDrop
       handler
     end
 
+    # Creates a ZMQ::PUB type socket, can only send messages via +send_message+
     def zmq_publish(address,socket_ctype,opts={})
       zm_addr = str_to_zm_address(address)
       h_opts  = handler_opts_given(opts)
@@ -44,6 +74,7 @@ class DripDrop
       handler
     end
 
+    # Creates a ZMQ::PULL type socket. Can only receive messages via +on_recv+
     def zmq_pull(address,socket_ctype,opts={},&block)
       zm_addr = str_to_zm_address(address)
       h_opts  = handler_opts_given(opts)
@@ -52,6 +83,7 @@ class DripDrop
       handler
     end
 
+    # Creates a ZMQ::PUSH type socket, can only send messages via +send_message+
     def zmq_push(address,socket_ctype,opts={})
       zm_addr = str_to_zm_address(address)
       h_opts  = handler_opts_given(opts)
@@ -60,12 +92,54 @@ class DripDrop
       handler
     end
     
+    # Creates a ZMQ::XREP type socket, both sends and receivesc XREP sockets are extremely
+    # powerful, so their functionality is currently limited. XREP sockets in DripDrop can reply
+    # to the original source of the message.
+    #
+    # Receiving with XREP sockets in DripDrop is different than other types of sockets, on_recv
+    # passes 3 arguments to its callback, +identities+, +seq+, and +message+. Identities is the 
+    # socket identity, seq is the sequence number of the message (all messages received at the socket
+    # get a monotonically incrementing +seq+, and +message+ is the message itself.
+    # 
+    # To reply from an xrep handler, be sure to call send messages with the same +identities+ and +seq+
+    # arguments that +on_recv+ had. So, send_message takes +identities+, +seq+, and +message+.
+    def zmq_xrep(address,socket_ctype,opts={})
+      zm_addr = str_to_zm_address(address)
+      h_opts  = handler_opts_given(opts)
+      handler = DripDrop::ZMQXRepHandler.new(zm_addr,@zm_reactor,socket_ctype,h_opts)
+      @zm_reactor.xrep_socket(handler)
+      handler
+    end
+ 
+    # See the documentation for +zmq_xrep+ for more info
+    def zmq_xreq(address,socket_ctype,opts={})
+      zm_addr = str_to_zm_address(address)
+      h_opts  = handler_opts_given(opts)
+      handler = DripDrop::ZMQXReqHandler.new(zm_addr,@zm_reactor,socket_ctype,h_opts)
+      @zm_reactor.xreq_socket(handler)
+      handler
+    end
+    
     def websocket(address,opts={},&block)
       uri     = URI.parse(address)
       h_opts  = handler_opts_given(opts)
       handler = DripDrop::WebSocketHandler.new(uri,h_opts)
       handler
     end
+    
+    def http_server(address,opts={},&block)
+      uri     = URI.parse(address)
+      h_opts  = handler_opts_given(opts)
+      handler = DripDrop::HTTPServerHandler.new(uri, h_opts,&block)
+      handler
+    end
+    
+    def http_client(address,opts={})
+      uri     = URI.parse(address)
+      h_opts  = handler_opts_given(opts)
+      handler = DripDrop::HTTPClientHandler.new(uri, h_opts)
+      handler
+    end
 
     def send_internal(dest,data)
       return false unless @recipients_for[dest]

data/lib/dripdrop.rb

@@ -0,0 +1 @@
+require 'dripdrop/node'

data/spec/node/zmq_pushpull.rb

@@ -0,0 +1,56 @@
+require 'spec_helper'
+
+describe "zmq push/pull" do
+  def pp_send_messages(to_send)
+    responses = []
+    push = nil
+    pull = nil
+    
+    @ddn = DripDrop::Node.new do
+      addr = rand_addr
+      
+      push = zmq_push(addr, :bind)
+      
+      pull1 = zmq_pull(addr, :connect)
+      pull2 = zmq_pull(addr, :connect)
+      pull = [pull1, pull2] 
+      
+      pull1.on_recv do |message|
+        message.head['recv_sock'] = 1
+        responses << message
+      end
+       pull2.on_recv do |message|
+        message.head['recv_sock'] = 2
+        responses << message
+      end
+       
+      to_send.each {|message| push.send_message(message)}
+    end
+    
+    @ddn.start
+    sleep 0.1
+    @ddn.stop
+    
+    {:responses => responses, :handlers => { :push => push, :pull => [pull] }}
+  end
+  describe "basic sending and receiving" do
+    before(:all) do
+      @sent = []
+      10.times {|i| @sent << DripDrop::Message.new("test-#{i}")}
+      pp_info = pp_send_messages(@sent)
+      @responses     = pp_info[:responses]
+      @push_handler  = pp_info[:handlers][:push]
+      @pull_handlers = pp_info[:handlers][:pull]
+    end
+
+    it "should receive all sent messages in order" do
+      @sent.zip(@responses).each do |sent,response|
+        sent.name.should == response.name
+      end
+    end
+    
+    it "should receive messages on both pull sockets" do
+      @responses.map {|r| r.head['recv_sock']}.uniq.sort.should == [1,2]
+    end
+  end
+end