em-rserve 0.1.0 → 0.1.1

data/Gemfile

@@ -1,4 +1,7 @@
 source "http://rubygems.org"
 
-# Specify your gem's dependencies in em-rserve.gemspec
-gemspec
+gem "eventmachine"
+
+group :development, :test do
+  gem "rspec"
+end

data/README

@@ -15,7 +15,6 @@ So far it can:
 - handle pools of connections with ruby fibers
 
 Limitations:
-- large QAP1 messages (>8MB) are not supported, this limitation means that you cannot transfer large object directly over RServe protocol
 - no support for the connections with a password. somehow, password without encryption defeats the purpose. hence, if you need a password, you should setup an SSH proxy instead
 
 Links:

data/TODO

@@ -4,7 +4,7 @@
   - Table
 * RESP_ERR as a flag and not an OR-ed value
   -> will need to modify Header#error? and Header#error
-* support for long (>8M) messages
+* factorize headers parsing/dumping for SEXP and QAP1 parameters
 * login command
   support crypt and store salt from connection setup
 * better attach method

data/lib/em-rserve/pooler.rb

@@ -21,6 +21,7 @@ module EM::Rserve
       @connections = []
       @size = size
       @connection_class = klass
+      fill size
     end
 
     def empty?

data/lib/em-rserve/qap1/header.rb

@@ -11,7 +11,7 @@ module EM::Rserve
       end
 
       def message_length
-        length #TODO: use length2
+        length | length2 << 32 
       end
 
       def body?
@@ -37,9 +37,6 @@ module EM::Rserve
       def error
         ((command & ~Constants::RESP_ERR) >> 24) & 0xff
       end
-
-      #     -> self.for_message
-      #     -> prepare_message
     end
   end
 end

data/lib/em-rserve/qap1/rpack.rb

@@ -8,13 +8,21 @@ module EM::Rserve
       include Constants
       include R
 
+      LARGE_SIZE_TRESHOLD = 0xfffff0 
+
       def parameter_head(type, len)
-        #TODO: support long parameter, type: 0xbf + DT_LARGE if len overflows
-        (type & 0x000000ff) | ((len << 8) & 0xffffff00)
+        if len > LARGE_SIZE_TRESHOLD
+          head = (type & 0x000000bf) | ((len << 8) & 0xffffff00) | DT_LARGE
+          extra_len_bits = (len & ~0xffffff) >> 24
+          [head, extra_len_bits].pack('VV')
+        else
+          head = (type & 0x000000bf) | ((len << 8) & 0xffffff00)
+          [head].pack('V')
+        end
       end
 
       def pack_parameter(type, len, val, rule)
-        [parameter_head(type, len), val].flatten.pack('V'+rule)
+        [parameter_head(type, len), val].flatten.pack('a*'+rule)
       end
 
       def encode_parameter(param, type=nil)
@@ -76,9 +84,9 @@ module EM::Rserve
       end
 
       def head_parameter(head)
-        type = head & 0x000000bf 
+        type  = head & 0x000000bf 
         large = (head & DT_LARGE) > 0
-        len  = (head & 0xffffff00) >> 8
+        len   = (head & 0xffffff00) >> 8
         [type, len, large]
       end
 
@@ -112,7 +120,10 @@ module EM::Rserve
         until buffer.empty? do
           head, buffer = buffer.unpack('Va*')
           type, len, large = head_parameter(head)
-          raise NotImplementedError, "too long QAP1 message" if large
+          if large
+            long_len, buffer = buffer.unpack('Va*')
+            len = long_len << 24 | len
+          end
           if buffer.size < len
             raise RuntimeError, "cannot decode #{buffer} (not enough bytes)"
           end

data/lib/em-rserve/r/sexp.rb

@@ -57,6 +57,8 @@ module EM::Rserve
           end
         end
 
+        LARGE_SIZE_TRESHOLD = 0xfffff0 #recommandation
+
         def dump_sexp
           body = dumped_value_with_attribute
           children_data = children.map do |n|
@@ -64,8 +66,9 @@ module EM::Rserve
           end.join('')
           size = body.size + children_data.size
           flags = {:large => false, :attr => attribute && true}
+          flags[:large] = true if size > LARGE_SIZE_TRESHOLD
           head = self.class.parameter_head(self.class.code, flags, size)
-          [head, body + children_data].pack('Va*')
+          [head, body + children_data].join('')
         end
 
         def dumped_value_with_attribute
@@ -84,10 +87,16 @@ module EM::Rserve
           head_bits  = type & 0x3f #high bits are for flags
           len_bits   = (len & 0xffffff) << 8
           flags_bits = 0
-          flags_bits |= 0x40 if flags[:large]
           flags_bits |= 0x80 if flags[:attr]
-          ret = head_bits | len_bits | flags_bits
-          ret
+          if flags[:large]
+            flags_bits |= 0x40 
+            head = head_bits | len_bits | flags_bits
+            extra_len_bits = (len & ~0xffffff) >> 24
+            [head, extra_len_bits].pack('VV')
+          else
+            head = head_bits | len_bits | flags_bits
+            [head].pack('V')
+          end
         end
 
         class << self
@@ -392,6 +401,10 @@ module EM::Rserve
       def self.decode_nodes(buffer)
         head, buffer = buffer.unpack('Va*')
         type, flags, len = head_parameter(head)
+        if flags[:large]
+          long_len, buffer = buffer.unpack('Va*')
+          len = long_len << 24 | len
+        end
 
         # Cuts the buffer at the correct length and give away the remainder
         buffer, remainder = buffer.unpack("a#{len}a*")

data/lib/em-rserve/version.rb

@@ -1,5 +1,5 @@
 module EM
   module Rserve
-    VERSION = "0.1.0"
+    VERSION = "0.1.1"
   end
 end

data/samples/assign.rb

@@ -10,13 +10,25 @@ class DevelConnection < EM::Rserve::Connection
   def dump_sexp(msg)
     raise unless msg.parameters.size == 1
     root = msg.parameters.first
-    pp root
     val =  EM::Rserve::R::RtoRuby::Translator.r_to_ruby(root)
-    p val
+    if val.respond_to? :size
+      if val.size > 100
+        puts "too large to dump: #{val.class}"
+      else 
+        pp root
+        p val
+      end
+    else
+      pp root
+      p val
+    end
   end
 
   def dump_r_val(str) 
     r_eval(str) do |req|
+      req.errback do |msg|
+        puts "could not dump"
+      end
       req.callback do |msg|
         dump_sexp msg
       end
@@ -38,11 +50,16 @@ class DevelConnection < EM::Rserve::Connection
   def loop_parse_r_val(str)
     r_eval(str) do |req|
       req.callback do |msg|
+        puts "callback> #{str}"
         raise unless msg.parameters.size == 1
         root = msg.parameters.first
-        new_root = EM::Rserve::R::Sexp.parse(root.dump_sexp)
+        puts 'got parsed SEXP parameter'
         val1 =  EM::Rserve::R::RtoRuby::Translator.r_to_ruby(root)
+        puts 'translated SEXP'
+        new_root = EM::Rserve::R::Sexp.parse(root.dump_sexp)
+        puts 'dumped SEXP'
         val2 =  EM::Rserve::R::RtoRuby::Translator.r_to_ruby(new_root)
+        puts 're-translated dumped SEXP'
         if val1 == val2
           p "ok: #{str}" 
         else
@@ -155,9 +172,14 @@ class DevelConnection < EM::Rserve::Connection
   def ready
     puts "ready"
 
-    loop_parse_r_val '1+1i'
-    loop_parse_r_val 'c(1:1000)'
-    loop_parse_r_val 'c(1:10000000)'
+    #do_array_int((1 .. 10).to_a)
+    #do_array_int((1 .. 10000000).to_a)
+    loop_parse_r_val 'c(1:5)'
+    loop_parse_r_val "c(1:10000000)"
+    loop_parse_r_val 'table(c(1,2,3,2,2))'
+    loop_parse_r_val "data.frame(foo=c(1:8), bar=seq(100,800,100))"
+    loop_parse_r_val "data.frame(foo=c(1,2,3), bar=c(NA,FALSE,TRUE), row.names=c('foo','bar','baz'))" 
+
 
     return
 
@@ -173,10 +195,6 @@ class DevelConnection < EM::Rserve::Connection
 
     return
 
-    loop_parse_r_val 'c(1:5)'
-    loop_parse_r_val 'table(c(1,2,3,2,2))'
-    loop_parse_r_val "data.frame(foo=c(1:8), bar=seq(100,800,100))"
-    loop_parse_r_val "data.frame(foo=c(1,2,3), bar=c(NA,FALSE,TRUE), row.names=c('foo','bar','baz'))" 
     
     #do_int
     #do_double

metadata

@@ -1,34 +1,23 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: em-rserve
-version: !ruby/object:Gem::Version 
-  hash: 27
-  prerelease: false
-  segments: 
-  - 0
-  - 1
-  - 0
-  version: 0.1.0
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+  prerelease: 
 platform: ruby
-authors: 
+authors:
 - crapooze
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-10-10 00:00:00 +02:00
-default_executable: 
+date: 2011-10-21 00:00:00.000000000Z
 dependencies: []
-
 description: Do evented stats with EventMachine and RServe
-email: 
+email:
 - crapooze@gmail.com
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - .gitignore
 - Gemfile
 - README
@@ -63,39 +52,28 @@ files:
 - specs/parser_spec.rb
 - specs/ruby_to_r_translator_spec.rb
 - specs/spec_helper.rb
-has_rdoc: true
-homepage: ""
+homepage: ''
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: em-rserve
-rubygems_version: 1.3.7
+rubygems_version: 1.8.6
 signing_key: 
 specification_version: 3
 summary: An EventMachine client for RServe
 test_files: []
-