arpie 0.0.3 → 0.0.4

data/README

@@ -6,6 +6,9 @@ your implementation details).
 
 Arpie also provides a robust replay-protected RPC framework.
 
+The useful core of arpie is a protocol stack that can be used to read/split/assemble/write
+any data stream, but is tailored for packeted streaming data.
+
 The Arpie server uses one ruby-thread per client, the client runs entirely in the
 calling thread; though an example implementation for evented callbacks is provided.
 
@@ -27,7 +30,7 @@ to get the newest version.
 
   server = TCPServer.new(51210)
 
-  e = Arpie::Server.new(Arpie::MarshalProtocol.new)
+  e = Arpie::Server.new(Arpie::MarshalProtocol.new, Arpie::SizedProtocol.new)
 
   e.handle do |server, ep, msg|
     ep.write_message msg.reverse
@@ -37,7 +40,7 @@ to get the newest version.
     server.accept
   end
 
-  c = Arpie::Client.new(Arpie::MarshalProtocol.new)
+  c = Arpie::Client.new(Arpie::MarshalProtocol.new, Arpie::SizedProtocol.new)
   c.connect do
     TCPSocket.new("127.0.0.1", 51210)
   end
@@ -60,7 +63,7 @@ to get the newest version.
 
   server = TCPServer.new(51210)
 
-  e = Arpie::ProxyServer.new(Arpie::MarshalProtocol.new)
+  e = Arpie::ProxyServer.new(Arpie::MarshalProtocol.new, Arpie::SizedProtocol.new)
 
   e.handle MyHandler.new
 
@@ -68,7 +71,7 @@ to get the newest version.
     server.accept
   end
 
-  p = Arpie::ProxyClient.new(Arpie::MarshalProtocol.new)
+  p = Arpie::ProxyClient.new(Arpie::MarshalProtocol.new, Arpie::SizedProtocol.new)
   p.connect do |transport|
     TCPSocket.new("127.0.0.1", 51210)
   end

data/Rakefile

@@ -9,7 +9,7 @@ include FileUtils
 # Configuration
 ##############################################################################
 NAME = "arpie"
-VERS = "0.0.3"
+VERS = "0.0.4"
 CLEAN.include ["**/.*.sw?", "pkg", ".config", "rdoc", "coverage"]
 RDOC_OPTS = ["--quiet", "--line-numbers", "--inline-source", '--title', \
   "#{NAME}: A high-performing layered networking protocol framework. Simple to use, simple to extend.", \

data/lib/arpie.rb

@@ -1,4 +1,5 @@
 require 'arpie/protocol'
+require 'arpie/xmlrpc'
 require 'arpie/server'
 require 'arpie/client'
 require 'arpie/proxy'

data/lib/arpie/client.rb

@@ -7,6 +7,7 @@ module Arpie
   #
   # See README for examples.
   class Client
+    # The protocol chain used.
     attr_reader :protocol
 
     # How often should this Client retry a connection.
@@ -22,8 +23,8 @@ module Arpie
     # unnecessary load in case of network failure.
     attr_accessor :connect_sleep
 
-    def initialize protocol
-      @protocol = protocol
+    def initialize *protocols
+      @protocol = Arpie::ProtocolChain.new(*protocols)
       @read_io = nil
       @write_io = nil
       @connector = lambda { raise ArgumentError, "No connector specified, cannot connect to Endpoint." }
@@ -69,7 +70,7 @@ module Arpie
     # Receive a message. Blocks until received.
     def read_message
       io_retry do
-        message = @protocol.read_message(@read_io)
+        return @protocol.read_message(@read_io)
       end
     end
 
@@ -144,8 +145,8 @@ module Arpie
   class RPCClient < Client
     private :read_message, :write_message
 
-    def initialize protocol
-      super(protocol)
+    def initialize *protocols
+      super(*protocols)
 
       @on_pre_call = lambda {|client, message| }
       @on_post_call = lambda {|client, message, reply| }

data/lib/arpie/protocol.rb

@@ -2,146 +2,305 @@ require 'shellwords'
 require 'yaml'
 
 module Arpie
+  MTU = 1024
+  # Raised by arpie when a Protocol thinks the stream got corrupted
+  # (by calling stream_error!).
+  # This usually results in a dropped connection.
+  class StreamError < IOError ; end
+  # Raised by arpie when a Protocol needs more data to parse a packet.
+  # Usually only of relevance to the programmer when using Protocol#from directly.
+  class EIncomplete < RuntimeError ; end
+
+  # :stopdoc:
+  # Used internally by arpie.
+  class ESwallow < RuntimeError ; end
+  class ETryAgain < RuntimeError ; end
+  class YieldResult < RuntimeError
+    attr_reader :result
+    def initialize result
+      @result = result
+    end
+  end
+  # :startdoc:
 
-  # A Protocol converts messages (which are arbitary objects)
-  # to a suitable on-the-wire format, and back.
-  class Protocol
-    MTU = 1024
+  # A RPC call. You need to wrap all calls sent over RPC protocols in this.
+  class RPCall < Struct.new(:ns, :meth, :argv, :uuid); end
 
-    private_class_method :new
+  # A ProtocolChain wraps one or more Protocols to provide a parser
+  # list, into which io data can be fed and parsed packets received; and
+  # vice versa.
+  class ProtocolChain
 
-    attr_reader :message
+    # Array of Protocols.
+    attr_reader :chain
 
-    def initialize
-      @message = nil
-      @buffer = ""
-      reset
-    end
+    # String holding all read, but yet unparsed bytes.
+    attr_reader :buffer
 
-    # Reads data from +io+. Returns true, if a whole
-    # message has been read, or false if more data is needed.
-    # The read message can be retrieved via Protocol#message.
-    def read_partial io
-      @buffer << io.readpartial(MTU)
+    # A buffer holding all parsed, but unreturned messages.
+    attr_reader :messages
 
-      if idx = complete?(@buffer)
-        @message = from @buffer[0, idx]
-        @buffer = @buffer[idx, -1] || ""
-        return true
-      end
+    # The endpoint class of this Protocol.
+    # Defaults to Arpie::Endpoint
+    attr_accessor :endpoint_class
+
+    # Create a new Chain. Supply an Array of Protocol
+    # instances, where the leftmost is the innermost.
+    #
+    # Example:
+    #   MarshalProtocol.new, SizedProtocol.new
+    # would wrap marshalled data inside SizedProtocol.
+    def initialize *protocols
+      protocols.size > 0 or raise ArgumentError, "Specify at least one protocol."
+      protocols[-1].class::CAN_SEPARATE_MESSAGES or
+        raise ArgumentError,
+          "The outermost protocol needs to be able to " +
+          "separate messages in a stream (#{protocols.inspect} does not)."
+
+      @endpoint_class = Arpie::Endpoint
 
-      return false
+      @chain = protocols
+      @buffer = ""
+      @messages = []
+    end
+
+    # Convert the given +message+ to wire format by
+    # passing it through all protocols in the chain.
+    def to message
+      ret = @chain.inject(message) {|msg, p|
+        p.to(msg)
+      }
+    end
+
+    # Convert the given +binary+ to message format
+    # by passing it through all protocols in the chain.
+    # May raise EStreamError or EIncomplete, in the case that
+    # +binary+ does not satisfy one of the protocols.
+    #
+    # Returns an array of messages, even if only one message
+    # was contained.
+    def from binary
+      r, w = IO.pipe
+      w.write(binary)
+      w.close
+      results = []
+      results << read_message(r) until false rescue begin
+        r.close
+        return results
+      end
+      raise "Interal error: should not reach this."
     end
 
-    # Read a message from +io+. Block until a message
-    # has been received.
+    # Read a message from +io+. Block until all protocols
+    # agree that a message has been received.
+    #
     # Returns the message.
     def read_message io
-      select([io]) until read_partial(io)
-      @message
-    end
+      return @messages.shift if @messages.size > 0
+
+      messages = [@buffer]
+      chain = @chain.reverse
+      p_index = 0
+
+      while p_index < chain.size do p = chain[p_index]
+        cut_to_index = nil
+        messages_for_next = []
+
+        messages.each do |message|
+          cut_to_index = p.from(message) do |object|
+            messages_for_next << object
+          end rescue case $!
+
+            when YieldResult
+              messages_for_next.concat($!.result)
+              next
+
+            when ESwallow
+              messages.delete(message)
+              messages_for_next = messages
+              p_index -= 1
+              break
+
+            when EIncomplete
+              # All protocols above the io one need to wait for each
+              # one above to yield more messages.
+              if p_index > 0
+                # Unwind to the parent protocol and let it read in some
+                # more messages ..
+                messages_for_next = messages
+                messages_for_next.shift
+                p_index -= 1
+                break
+
+              # The first protocol manages +io+.
+              else
+                select([io])
+                @buffer << io.readpartial(MTU) rescue raise $!.class,
+                  "#{$!.to_s}; unparseable bytes remaining in buffer: #{@buffer.size}"
+                retry
+              end
+
+            when ETryAgain
+              retry
 
-    def write_raw_partial io, message
-      io.write(message)
-    end
+            else
+              raise
+          end # rescue case
+        end # messages.each
 
-    # Write +message+ to +io+.
-    def write_message io, message
-      io.write(to message)
-    end
+         raise "BUG: #{p.class.to_s}#from did not yield a message." if
+           messages_for_next.size == 0
 
-    # Convert obj to on-the-wire format.
-    def to obj
-      obj
-    end
+        messages = messages_for_next
 
-    # Convert obj from on-the-wire-format.
-    def from obj
-      obj
+        if p_index == 0
+          if cut_to_index.nil? || cut_to_index < 0
+            raise "Protocol '#{p.class.to_s}'implementation faulty: " +
+              "from did return an invalid cut index: #{cut_to_index.inspect}."
+          else
+            @buffer[0, cut_to_index] = ""
+          end
+        end
+
+        p_index += 1
+      end # chain loop
+
+      message = messages.shift
+      @messages = messages
+      message
     end
 
-    # Returns a Fixnum if the given obj contains a complete message.
-    # The Fixnum is the index up to where the message runs; the rest
-    # is assumed to be (part of) the next message.
-    # Returns nil if obj does not describe a complete message (eg,
-    # more data needs to be read).
-    def complete? obj
-      nil
+    # Write +message+ to +io+.
+    def write_message io, message
+      io.write(to message)
     end
 
-    # Reset all state buffers. This is usually called
-    # when the underlying connection drops, and any half-read
-    # messages need to be discarded.
     def reset
-      @message = nil
       @buffer = ""
     end
-
-    def endpoint_klass
-      Arpie::Endpoint
-    end
   end
 
-  # A simple separator-based protocol. This can be used to implement
-  # newline-delimited communication.
-  class SeparatorProtocol < Protocol
-    public_class_method :new
-
-    attr_accessor :separator
-
-    def initialize separator = "\n"
-      super()
-      @separator = separator
-    end
-
-    def complete? obj
-      obj.index(@separator)
-    end
-
-    def from obj
-      obj.gsub(/#{Regexp.escape(@separator)}$/, "")
-    end
+  # A Protocol converts messages (which are arbitary objects)
+  # to a suitable on-the-wire format, and back.
+  class Protocol
 
-    def to obj
-      obj + @separator
-    end
-  end
+    # Set this to true in child classes which implement
+    # message separation within a stream.
+    CAN_SEPARATE_MESSAGES = false
 
-  # A linebased-protocol, which does shellwords-escaping/joining
-  # on the lines; messages sent are arrays of parameters.
-  # Note that all parameters are expected to be strings.
-  class ShellwordsProtocol < SeparatorProtocol
+    # Convert obj to on-the-wire format.
     def to obj
-      super Shellwords.join(obj)
+      obj
     end
 
-    def from obj
-      Shellwords.shellwords(super obj)
+    # Extract message(s) from +binary+.i
+    #
+    # Yields each message found, with all protocol-specifics stripped.
+    #
+    # Should call +incomplete+ when no message can be read yet.
+    #
+    # Must not block by waiting for multiple messages if a message
+    # can be yielded directly.
+    #
+    # Must not return without calling +incomplete+ or yielding a message.
+    #
+    # Must return the number of bytes these message(s) occupied in the stream,
+    # for truncating of the same.
+    # Mandatory when CAN_SEPARATE_MESSAGES is true for this class, but ignored
+    # otherwise.
+    def from binary, &block #:yields: message
+      yield binary
+      0
+    end
+
+    # Call this within Protocol#from to reparse the current
+    # message.
+    def again!
+      raise ETryAgain
+    end
+
+    # Tell the protocol chain that the given chunk of data
+    # is not enough to construct a whole message.
+    # This breaks out of Protocol#from.
+    def incomplete!
+      raise EIncomplete
+    end
+
+    # Swallow the complete message currently passed to Protocol#from.
+    # Not advised for the outermost protocol, which works on io buffer streams
+    # and may swallow more than intended.
+    def gulp!
+      raise ESwallow
+    end
+    alias_method :swallow!, :gulp!
+
+    # Stow away a message in this protocols buffer for later reassembly.
+    # Optional argument: a token if you are planning to reassemble multiple
+    # interleaved/fragmented message streams.
+    #
+    # +binary+   is the binary packet you want to add to the assembly
+    # +token+    is a object which can be used to re-identify multiple concurrent assemblies
+    # +meta+     is a hash containing meta-information for this assembly
+    #            each call to assemble! will merge these hashes, and pass them
+    #            on to Protocol#assemble
+    def assemble! binary, token = :default, meta = {}
+      @stowbuffer ||= {}
+      @stowbuffer[token] ||= []
+      @stowbuffer[token] << binary
+
+      @metabuffer ||= {}
+      @metabuffer[token] ||= {}
+      @metabuffer[token].merge(meta)
+
+      assembled = []
+
+      assemble @stowbuffer[token], token, meta do |a|
+        assembled << a
+      end  # rescue case $!
+        #when EIncomplet
+        #  puts "Cant reassemble, asking for moarh"
+        #  raise EIncomplete
+        #else
+        #  raise
+      #end
+
+      assembled.size > 0 or raise "assemble! did not return any results."
+      raise YieldResult, assembled
+    end
+
+    # Called when we're trying to reassemble a stream of packets.
+    #
+    # Call incomplete! when not enough data is here to reassemble this stream,
+    # and yield all results of the reassembled stream.
+    def assemble binaries, token
+      raise NotImplementedError, "Tried to assemble! something, but no assembler defined."
+    end
+
+    # Call this if you think the stream has been corrupted, or
+    # non-protocol data arrived.
+    # +message+ is the text to display.
+    # +data+ is the optional misbehaving data for printing.
+    # This breaks out of Protocol#from.
+    def bogon! data = nil, message = nil
+      raise StreamError, "#{self.class.to_s}#{message.nil? ? " thinks the data is bogus" : ": " + message }#{data.nil? ? "" : ": " + data.inspect}."
     end
   end
 
+
   # A sample binary protocol, which simply prefixes each message with the
   # size of the data to be expected.
   class SizedProtocol < Protocol
-    public_class_method :new
-
-    def initialize
-      super
-      @max_message_size = 1024 * 1024
-    end
+    CAN_SEPARATE_MESSAGES = true
 
-    def complete? obj
-      sz = obj.unpack("Q")[0]
-      obj.size == sz + 8 ? sz + 8 : nil
+    def from binary
+      sz = binary.unpack('Q')[0] or incomplete!
+      binary.size >= sz + 8 or incomplete!
+      yield binary[8, sz]
+      8 + sz
     end
 
-    def from obj
-      sz, data = obj.unpack("Qa*")
-      data
-    end
-
-    def to obj
-      [obj.size, obj].pack("Qa*")
+    def to object
+      [object.size, object].pack('Qa*')
     end
   end
 
@@ -149,132 +308,66 @@ module Arpie
   # this protocol. Served as an example, but a viable
   # choice for ruby-only production code.
   # Messages are arbitary objects.
-  class MarshalProtocol < SizedProtocol
-    def to obj
-      super Marshal.dump(obj)
+  class MarshalProtocol < Protocol
+    def to object
+      Marshal.dump(object)
     end
 
-    def from obj
-      Marshal.load(super obj)
+    def from binary
+      yield Marshal.load(binary)
     end
   end
 
-  # A protocol which encodes objects into YAML representation.
-  # Messages are arbitary yaml-encodable objects.
-  class YAMLProtocol < Arpie::Protocol
-    public_class_method :new
-
-    def complete? obj
-      obj =~ /\.\.\.$/
-    end
-
-    def to obj
-      YAML.dump(obj) + "...\n"
-    end
+  # A simple separator-based protocol. This can be used to implement
+  # newline-delimited communication.
+  class SeparatorProtocol < Protocol
+    CAN_SEPARATE_MESSAGES = true
+    attr_accessor :separator
 
-    def from obj
-      YAML.load(obj)
+    def initialize separator = "\n"
+      @separator = separator
     end
-  end
 
-  # A RPC Protocol encapsulates RPCProtocol::Call
-  # messages.
-  class RPCProtocol < Protocol
+    def from binary
+      idx = binary.index(@separator) or incomplete!
+      yield binary[0, idx]
 
-    # A RPC call.
-    class Call < Struct.new(:ns, :meth, :argv, :uuid); end
-  end
-
-  # A XMLRPC Protocol based on rubys xmlrpc stdlib.
-  # This does not encode HTTP headers; usage together with
-  # a real webserver is advised.
-  class XMLRPCProtocol < RPCProtocol
-    public_class_method :new
-
-    require 'xmlrpc/create'
-    require 'xmlrpc/parser'
-    require 'xmlrpc/config'
-
-    VALID_MODES = [:client, :server].freeze
-
-    attr_reader :mode
-    attr_accessor :writer
-    attr_accessor :parser
-
-    def initialize mode, writer = XMLRPC::Create, parser = XMLRPC::XMLParser::REXMLStreamParser
-      super()
-      raise ArgumentError, "Not a valid mode, expecting one of #{VALID_MODES.inspect}" unless
-        VALID_MODES.index(mode)
-
-      @mode = mode
-      @writer = writer.new
-      @parser = parser.new
+      @separator.size + idx
     end
 
-    def to obj
-      case @mode
-        when :client
-          @writer.methodCall(obj.ns + obj.meth, *obj.argv)
-
-        when :server
-          case obj
-            when Exception
-              # TODO: wrap XMLFault
-            else
-              @writer.methodResponse(true, obj)
-            end
-      end
+    def to object
+      object.to_s + @separator
     end
+  end
 
-    def from obj
-      case @mode
-        when :client
-          @parser.parseMethodResponse(obj)[1]
-
-        when :server
-          vv = @parser.parseMethodCall(obj)
-          RPCProtocol::Call.new('', vv[0], vv[1])
-      end
+  # A linebased-protocol, which does shellwords-escaping/joining
+  # on the lines; messages sent are arrays of parameters.
+  # Note that all parameters are expected to be strings.
+  class ShellwordsProtocol < Protocol
+    def to object
+      raise ArgumentError, "#{self.class.to_s} can only encode arrays." unless
+        object.is_a?(Array)
+      Shellwords.join(object.map {|x| x.to_s })
     end
 
-    def complete? obj
-      case @mode
-        when :client
-          obj.index("</methodResponse>")
-        when :server
-          obj.index("</methodCall>")
-      end
+    def from binary
+      yield Shellwords.shellwords(binary)
     end
   end
 
-  # This simulates a very basic HTTP XMLRPC client/server.
-  # It is not recommended to use this with production code.
-  class HTTPXMLRPCProtocol < XMLRPCProtocol
-    def to obj
-      r = super
-      case @mode
-        when :client
-          "GET / HTTP/1.[01]\r\nContent-Length: #{r.size}\r\n\r\n" + r
-        when :server
-          "HTTP/1.0 200 OK\r\nContent-Length: #{r.size}\r\n\r\n" + r
-      end
-    end
+  # A protocol which encodes objects into YAML representation.
+  # Messages are arbitary yaml-encodable objects.
+  class YAMLProtocol < Protocol
+    CAN_SEPARATE_MESSAGES = true
 
-    def from obj
-      # Simply strip all HTTP headers.
-      header, obj = obj.split(/\r\n\r\n/, 2)
-      super(obj)
+    def to object
+      YAML.dump(object) + "...\n"
     end
 
-
-    def complete? obj
-      # Complete if: has headers, has content-length, has data of content-length
-      header, body = obj.split(/\r\n\r\n/, 2)
-
-      header =~ /content-length:\s+(\d+)/i or return nil
-
-      content_length = $1.to_i
-      body.size == content_length ? header.size + 4 + body.size : nil
+    def from binary
+      index = binary =~ /^\.\.\.$/x or incomplete!
+      yield YAML.load(binary[0, index])
+      4 + index
     end
   end
 end