em-rserve 0.1.0

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in em-rserve.gemspec
+gemspec

data/README

@@ -0,0 +1,26 @@
+
+EM::Rserve is an attempt to bring RServe over EventMachine.
+Consider it a somewhat stable version.
+
+EM::Rserve is pure-ruby and should work wherever EventMachine is supported.
+
+So far it can:
+- connect to a server
+- detach and reattach a session
+- parse most low-level messages
+- parse most R' S-expressions to a ruby tree
+- translate several common R' S-expressions to "base" ruby objects such as arrays, strings, hashes ...
+- translate several Ruby objects (arrays of integers, strings etc.) to R' S-expression
+- evaluate strings of R code
+- handle pools of connections with ruby fibers
+
+Limitations:
+- large QAP1 messages (>8MB) are not supported, this limitation means that you cannot transfer large object directly over RServe protocol
+- no support for the connections with a password. somehow, password without encryption defeats the purpose. hence, if you need a password, you should setup an SSH proxy instead
+
+Links:
+- http://www.r-project.org/
+- http://www.rforge.net/Rserve/
+- http://rubyeventmachine.com/
+
+Contributions welcome.

data/Rakefile

@@ -0,0 +1 @@
+require 'bundler/gem_tasks'

data/TODO

@@ -0,0 +1,14 @@
+* missing types
+* special handling in ruby2r translator
+  - Factor
+  - Table
+* RESP_ERR as a flag and not an OR-ed value
+  -> will need to modify Header#error? and Header#error
+* support for long (>8M) messages
+* login command
+  support crypt and store salt from connection setup
+* better attach method
+* write doc
+* more spec once the RServe spec is better understood/have enough examples
+* subclass Request for each commmand instead of current quick'n dirty mechanism
+* better test requests stacking/unstacking

data/em-rserve.gemspec

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "em-rserve/version"
+
+Gem::Specification.new do |s|
+  s.name        = "em-rserve"
+  s.version     = EM::Rserve::VERSION
+  s.authors     = ["crapooze"]
+  s.email       = ["crapooze@gmail.com"]
+  s.homepage    = ""
+  s.summary     = %q{An EventMachine client for RServe}
+  s.description = %q{Do evented stats with EventMachine and RServe}
+
+  s.rubyforge_project = "em-rserve"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/lib/em-rserve/connection.rb

@@ -0,0 +1,84 @@
+
+require "em-rserve/protocol/connector"
+require "em-rserve/protocol/parser"
+require "em-rserve/protocol/request"
+require "em-rserve/qap1/constants"
+require "em-rserve/qap1/header"
+require "em-rserve/qap1/message"
+
+module EM::Rserve
+  class Connection < EM::Connection
+    include Protocol::Connector
+    include QAP1
+
+    def shutdown!(&blk)
+      header = Header.new(Constants::CMD_shutdown,0,0,0)
+      send_data header.to_bin
+
+      request(&blk) 
+    end
+
+    def r_eval(string, void=false,&blk)
+      data = Message.encode_string(string)
+      if void
+        header = Header.new(0x0002, data.length, 0, 0)
+      else
+        header = Header.new(0x0003, data.length, 0, 0)
+      end
+      send_data header.to_bin
+      send_data data
+
+      request(&blk)
+    end
+
+    def login(user,pwd, crypted=true, &blk)
+      raise NotImplementedError, "will come later"
+      #XXX need to read the salt during connection setup
+      cifer = crypted ? pwd : crypt(pwd, salt)
+      data = Message.encode_string([user, cifer].join("\n"))
+      header = Header.new(Constants::CMD_login, data.length, 0, 0)
+      send_data header.to_bin
+      send_data data
+
+      request(&blk)
+    end
+
+    def detach(&blk)
+      header = Header.new(Constants::CMD_detachSession, 0, 0, 0)
+      send_data header.to_bin #port, key of 20 bytes
+
+      request(&blk)
+    end
+
+    def attach(key, &blk)
+      #XXX it seems that there is no need to send a Header + Message, and
+      #raw_writing because the server does a read of 32 bytes on newly accepted
+      #connections
+      raise ArgumentError, "wrong key length, Rserve wants 32bytes" unless key.size == 32
+      send_data key
+
+      request(&blk)
+    end
+
+    def assign(symbol, sexp_node, parse_symbol_name=true, &blk)
+      data = Message.new([symbol.to_s, sexp_node]).to_bin
+      data << "\xFF" * data.length % 4
+      header = if parse_symbol_name
+                 Header.new(Constants::CMD_setSEXP, data.length, 0, 0)
+               else
+                 Header.new(Constants::CMD_assignSEXP, data.length, 0, 0)
+               end
+      send_data header.to_bin
+      send_data data
+
+      request(&blk)
+    end
+
+    # MISSING:
+    #   - open/close/delete/read/write files
+    #   - set encoding
+    #   - set buffer size
+    #   - control commands
+    #   - serial commands
+  end
+end

data/lib/em-rserve/fibered_connection.rb

@@ -0,0 +1,66 @@
+
+require "fiber"
+require "em-rserve/connection"
+
+module EM::Rserve
+class FiberedConnection < EM::Rserve::Connection
+  attr_accessor :fiber
+
+  def self.start(fiber=Fiber.current, *args)
+    conn = super(*args)
+    conn.fiber = fiber
+    Fiber.yield 
+  end
+
+  def ready
+    super
+    fiber.resume self if fiber
+  end
+
+  def call(script, *args)
+    r_eval(script.to_s, *args) do |req|
+      req.errback do |err|
+        fiber.resume nil
+      end
+
+      req.callback do |msg|
+        unless msg
+          fiber.resume nil
+          next
+        end
+        root = msg.parameters.first
+        if root
+          fiber.resume EM::Rserve::R::RtoRuby::Translator.r_to_ruby(root) 
+        else
+          fiber.resume nil
+        end
+      end
+    end
+
+    Fiber.yield
+  end
+
+  def set(sym, val)
+    root = EM::Rserve::R::RubytoR::Translator.ruby_to_r val
+
+    assign(sym, root) do |req|
+      req.errback do |err|
+        fiber.resume nil
+      end
+      req.callback do |msg|
+        fiber.resume val
+      end
+    end
+
+    Fiber.yield
+  end
+
+  # *WARNING* this method is unsafe always check your input parameter
+  def get(sym)
+    call(sym)
+  end
+
+  alias :[]= :set
+  alias :[]  :get
+end
+end

data/lib/em-rserve/pooler.rb

@@ -0,0 +1,76 @@
+
+require "em-rserve/fibered_connection"
+
+module EM::Rserve
+  class Pooler
+    class << self
+      def r(klass=FiberedConnection)
+        Fiber.new do
+          begin
+            conn = klass.start 
+            yield conn
+          ensure
+            conn.close_connection
+          end
+        end.resume
+      end
+    end
+
+    attr_reader :connections, :size, :connection_class
+    def initialize(size=10, klass=FiberedConnection)
+      @connections = []
+      @size = size
+      @connection_class = klass
+    end
+
+    def empty?
+      connections.empty?
+    end
+
+    def full?
+      connections.size >= size
+    end
+
+    def connection
+      #XXX duplicated code from Pooler.r to avoid proc-ing the blk
+      Fiber.new do
+        yield connection_class.start
+      end.resume
+    end
+
+    def r
+      conn = connections.shift
+      if conn
+        Fiber.new do
+          begin
+            conn.fiber = Fiber.current
+            yield conn
+          ensure
+            conn.close_connection
+          end
+          fill 1 unless full?
+        end.resume
+      else
+        fill size
+        connection do |conn| 
+          begin
+            yield conn
+          ensure
+            conn.close_connection
+          end
+        end
+      end
+    end
+
+    def preconnect!
+      connection do |conn|
+        conn.fiber = nil
+        connections << conn
+      end
+    end
+
+    def fill(n=size)
+      n.times{preconnect!}
+    end
+  end
+end

data/lib/em-rserve/protocol/connector.rb

@@ -0,0 +1,89 @@
+
+require "em-rserve/protocol/parser"
+require "em-rserve/protocol/request"
+
+module EM::Rserve
+  module Protocol
+    module Connector 
+      def self.included(obj)
+        obj.extend ClassMethods
+      end
+
+      module ClassMethods
+        def start(server='127.0.0.1', port=6311)
+          EM.connect(server, port, self)
+        end
+      end
+
+      extend ClassMethods
+
+      attr_accessor :request_queue
+
+      def post_init
+        super
+        #type of parser carries the state, no need to carry it internally and do
+        #zillions of state check
+        @parser = nil
+        replace_parser! IDParser
+        @request_queue = []
+      end
+
+      def request
+        r = Request.new
+        yield r if block_given?
+        request_queue << r
+        r
+      end
+
+      def replace_parser!(klass)
+        new_parser = klass.new(self)
+        if @parser
+          new_parser.replace(@parser)
+        end
+        @parser = new_parser
+      end
+
+      def receive_data(dat)
+        @parser << dat
+      end
+
+      def receive_id(id)
+        # on last line, the messaging can start
+        if id.last_one?
+          replace_parser! MessageParser
+          EM.next_tick do
+            ready
+          end
+          throw :stop
+        end
+      end
+
+      # HOOKS TO OVERRIDE, PLEASE CALL SUPER
+
+      def ready
+      end
+
+      def receive_message_header(head)
+        if head.error?
+          receive_error_message_header(head)
+        elsif head.ok?
+          receive_success_message_header(head)
+        else
+          raise RuntimeError, "nor OK, nor error message #{head}"
+        end
+      end
+
+      def receive_error_message_header(head)
+        request_queue.shift.error(head)
+      end
+
+      def receive_success_message_header(head)
+        request_queue.shift.success(nil) unless head.body?
+      end
+
+      def receive_message(msg)
+        request_queue.shift.success(msg)
+      end
+    end
+  end
+end

data/lib/em-rserve/protocol/id.rb

@@ -0,0 +1,14 @@
+
+module EM::Rserve
+  module Protocol
+    ID = Struct.new(:string) do
+      def ignorable?
+        string == '----' or string == "\r\n\r\n"
+      end
+
+      def last_one?
+        string == "--\r\n"
+      end
+    end
+  end
+end

data/lib/em-rserve/protocol/parser.rb

@@ -0,0 +1,93 @@
+
+require 'em-rserve/protocol/id'
+require 'em-rserve/qap1/header'
+require 'em-rserve/qap1/message'
+require 'em-rserve/r/sexp'
+
+module EM::Rserve
+  module Protocol
+    class Parser
+      attr_reader :handler, :buffer
+
+      def initialize(handler)
+        @handler = handler
+        @buffer  = ''
+      end
+
+      def replace(other)
+        @buffer.replace(other.buffer)
+        self
+      end
+
+      def << data
+        buffer << data if data
+        parse_loop!
+      end
+
+      def parse_loop!
+        catch :stop do
+          loop do
+            parse!
+          end
+        end
+      end
+
+      # should overload this method and throw :stop when more data is needed or any other
+      # reason to stop parsing
+      def parse!
+        raise NotImplementedError, "this class is intended to be a top class, not a useful parser"
+      end
+    end
+
+    # This parser is useful only at the beginning.
+    # Instead of carrying its dynamic all the time (e.g., keeping a state).
+    # We pop-it out as another parser.
+    class IDParser < Parser
+      def parse!
+        if buffer.size >= 4
+          dat = buffer.slice(0, 4)
+          @buffer = buffer.slice(4 .. -1)
+          handler.receive_id(Protocol::ID.new(dat))
+        else
+          throw :stop
+        end
+      end
+    end
+
+    # This message parser will parse qap1 headers and associated qap1 data.
+    class MessageParser < Parser
+      def initialize(handler)
+        super(handler)
+        @header  = nil
+      end
+
+      def parse!
+        if @header
+          #XXX here we have a header with the size of data
+          #to expect, out approach is to delay the message
+          #until we have all the data, not well suited for
+          #streams but for all other messages this is the
+          #good way of doing it
+          expected_length = @header.message_length
+          if expected_length > 0 and buffer.size >= expected_length
+            dat = buffer.slice(0, expected_length)
+            @buffer = buffer.slice(expected_length .. -1)
+            message = QAP1::Message.from_bin dat
+            handler.receive_message message
+            @header = nil
+          else
+            throw :stop
+          end
+        elsif buffer.size >= 16
+          dat = buffer.slice(0, 16)
+          @buffer = buffer.slice(16 .. -1)
+          @header = QAP1::Header.from_bin dat
+          handler.receive_message_header @header
+          @header = nil unless @header.body?
+        else
+          throw :stop
+        end
+      end
+    end
+  end
+end

data/lib/em-rserve/protocol/request.rb

@@ -0,0 +1,22 @@
+
+module EM::Rserve
+  module Protocol
+    Request = Struct.new(:callback_blk, :errback_blk) do
+      def callback(&blk)
+        self.callback_blk = blk
+      end
+
+      def errback(&blk)
+        self.errback_blk = blk
+      end
+
+      def error(val)
+        errback_blk.call(val) if errback_blk
+      end
+
+      def success(val)
+        callback_blk.call(val) if callback_blk
+      end
+    end
+  end
+end

data/lib/em-rserve/qap1/constants.rb

@@ -0,0 +1,111 @@
+
+module EM::Rserve
+  module QAP1 
+    module Constants
+      CMD_RESP=0x010000 # all responses have this flag set
+      RESP_OK=(CMD_RESP|0x0001) # command succeeded; returned parameters depend on the command issued
+      RESP_ERR=(CMD_RESP|0x0002) # command failed, check stats code attached string may describe the error
+
+      ERR_auth_failed=0x41 # auth.failed or auth.requested but no login came. in case of authentification failure due to name/pwd mismatch, server may send CMD_accessDenied instead
+      ERR_conn_broken=0x42 #  connection closed or broken packet killed it */
+
+      ERR_inv_cmd=0x43 #  unsupported/invalid command */
+      ERR_inv_par=0x44 #  some parameters are invalid */
+      ERR_Rerror=0x45 #  R-error occured, usually followed by connection shutdown */
+      ERR_IOerror=0x46 #  I/O error */
+
+
+      ERR_notOpen=0x47 #  attempt to perform fileRead/Write  on closed file */
+      ERR_accessDenied=0x48 #  this answer is also valid on CMD_login; otherwise it's sent if the server deosn;t allow the user to issue the specified command. (e.g. some server admins may block file I/O operations for some users)
+      ERR_unsupportedCmd=0x49 #  unsupported command */
+      ERR_unknownCmd=0x4a #  unknown command - the difference between unsupported and unknown is that unsupported commands are known to the server but for some reasons (e.g. platform dependent) it's not supported. unknown commands are simply not recognized by the server at all. */
+      ERR_data_overflow=0x4b #  incoming packet is too big. currently there is a limit as of the size of an incoming packet. */
+      ERR_object_too_big=0x4c #  the requested object is too big to be transported in that way. If received after CMD_eval then the evaluation itself was successful. optional parameter is the size of the object
+      ERR_out_of_mem=0x4d #  out of memory. the connection is usually closed after this error was sent
+      ERR_ctrl_closed=0x4e #  control pipe to the master process is closed or broken
+      ERR_session_busy=0x50 #  session is still busy */
+      ERR_detach_failed=0x51 #  unable to detach session (cannot determine peer IP or problems creating a listening socket for resume) */
+
+
+      CMD_login=0x001 #  "name\npwd" : - */
+      CMD_voidEval=0x002 #  string : - */
+      CMD_eval=0x003 #  string : encoded SEXP */
+      CMD_shutdown=0x004 #  [admin-pwd] : - */
+
+      #/* file I/O routines. server may answe */
+      CMD_openFile=0x010 #  fn : - */
+      CMD_createFile=0x011 #  fn : - */
+      CMD_closeFile=0x012 #  - : - */
+      CMD_readFile=0x013 #  [int size] : data... ; if size not present,
+      #server is free to choose any value - usually
+      #it uses the size of its static buffer */
+      CMD_writeFile=0x014 #  data : - */
+      CMD_removeFile=0x015 #  fn : - */
+
+      # /* object manipulation */
+      CMD_setSEXP=0x020 #  string(name), REXP : - */
+      CMD_assignSEXP=0x021 #  string(name), REXP : - ; same as setSEXP except that the name is parsed */
+
+      # /* session management (since 0.4-0) */
+      CMD_detachSession=0x030 #  : session key */
+      CMD_detachedVoidEval=0x031 #  string : session key; doesn't */
+      CMD_attachSession=0x032 #  session key : - */
+
+      # control commands (since 0.6-0) - passed on to the master process */
+      # Note: currently all control commands are asychronous, i.e. RESP_OK
+      # indicates that the command was enqueued in the master pipe, but there
+      # is no guarantee that it will be processed. Moreover non-forked
+      # connections (e.g. the default debug setup) don't process any
+      # control commands until the current client connection is closed so
+      # the connection issuing the control command will never see its
+      # result.
+      CMD_ctrl=0x40  #  -- not a command - just a constant -- */
+      CMD_ctrlEval=0x42  #  string : - */
+      CMD_ctrlSource=0x45  #  string : - */
+      CMD_ctrlShutdown=0x44  #  - : - */
+
+      # /* 'internal' commands (since 0.1-9) */
+      CMD_setBufferSize=0x081  #  [int sendBufSize]  this commad allow clients to request bigger buffer sizes if large data is to be transported from Rserve to the client. (incoming buffer is resized automatically) */
+      CMD_setEncoding=0x082  #  string (one of "native","latin1","utf8") : -; since 0.5-3 */
+
+      # /* special commands - the payload of packages with this mask does not contain defined parameters */
+
+      CMD_SPECIAL_MASK=0xf0
+
+      CMD_serEval=0xf5 #  serialized eval - the packets are raw serialized data without data header */
+      CMD_serAssign=0xf6 #  serialized assign - serialized list with [[1]]=name, [[2]]=value */
+      CMD_serEEval=0xf7 #  serialized expression eval - like serEval with one additional evaluation round */
+
+      #  data types for the transport protocol (QAP1)do NOT confuse with XT_.. values.
+
+      DT_INT=1  #  int */
+      DT_CHAR=2  #  char */
+      DT_DOUBLE=3  #  double */
+      DT_STRING=4  #  0 terminted string */
+      DT_BYTESTREAM=5  #  stream of bytes (unlike DT_STRING may contain 0) */
+      DT_SEXP=10 #  encoded SEXP */
+      DT_ARRAY=11 #  array of objects (i.e. first 4 bytes specify how many subsequent objects are part of the array; 0 is legitimate) */
+      DT_LARGE=64 #  new in 0102: if this flag is set then the length of the object is coded as 56-bit integer enlarging the header by 4 bytes */
+
+      ERROR_DESCRIPTIONS={
+        ERR_auth_failed=>'auth.failed or auth.requested but no login came',
+        ERR_conn_broken=>'connection closed or broken packet killed it',
+        ERR_inv_cmd=>"unsupported/invalid command",
+        ERR_inv_par=>"some parameters are invalid",
+        ERR_Rerror=>"R-error",
+        ERR_IOerror=>"I/O error",
+        ERR_notOpen=>"attempt to perform fileRead/Write  on closed file",
+        ERR_accessDenied=>"Access denied",
+        ERR_unsupportedCmd=>"unsupported command",
+        ERR_unknownCmd=>"unknown command",
+        ERR_data_overflow=>"data overflow",
+        ERR_object_too_big=>"requested object is too big",
+        ERR_out_of_mem=>"out of memory",
+        ERR_ctrl_closed=>"control pipe to the master process is closed or broken",
+        ERR_session_busy=>"session still busy",
+        ERR_detach_failed=>"unable to detach seesion"
+      } # error descriptions
+
+    end
+  end
+end

data/lib/em-rserve/qap1/header.rb

@@ -0,0 +1,45 @@
+
+require 'em-rserve/qap1/constants'
+
+module EM::Rserve
+  module QAP1
+    Header = Struct.new(:command, :length, :offset, :length2) do
+
+      def self.from_bin(dat)
+        raise unless dat.size == 16
+        self.new(* dat.unpack('VVVV'))
+      end
+
+      def message_length
+        length #TODO: use length2
+      end
+
+      def body?
+        message_length > 0
+      end
+
+      def to_bin
+        to_a.pack('VVVV')
+      end
+
+      def response?
+        command & Constants::CMD_RESP > 0
+      end
+
+      def ok?
+        command & Constants::RESP_OK > 0
+      end
+
+      def error?
+        ((command & Constants::RESP_ERR) & ~Constants::RESP_OK) > 0
+      end
+
+      def error
+        ((command & ~Constants::RESP_ERR) >> 24) & 0xff
+      end
+
+      #     -> self.for_message
+      #     -> prepare_message
+    end
+  end
+end