em-rserve 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +4 -0
- data/Gemfile +4 -0
- data/README +26 -0
- data/Rakefile +1 -0
- data/TODO +14 -0
- data/em-rserve.gemspec +20 -0
- data/lib/em-rserve/connection.rb +84 -0
- data/lib/em-rserve/fibered_connection.rb +66 -0
- data/lib/em-rserve/pooler.rb +76 -0
- data/lib/em-rserve/protocol/connector.rb +89 -0
- data/lib/em-rserve/protocol/id.rb +14 -0
- data/lib/em-rserve/protocol/parser.rb +93 -0
- data/lib/em-rserve/protocol/request.rb +22 -0
- data/lib/em-rserve/qap1/constants.rb +111 -0
- data/lib/em-rserve/qap1/header.rb +45 -0
- data/lib/em-rserve/qap1/message.rb +26 -0
- data/lib/em-rserve/qap1/rpack.rb +127 -0
- data/lib/em-rserve/r/r_to_ruby/translator.rb +152 -0
- data/lib/em-rserve/r/ruby_to_r/translator.rb +131 -0
- data/lib/em-rserve/r/sexp.rb +421 -0
- data/lib/em-rserve/version.rb +5 -0
- data/lib/em-rserve.rb +11 -0
- data/samples/assign.rb +211 -0
- data/samples/detach_attach.rb +70 -0
- data/samples/fibers.rb +32 -0
- data/samples/run.rb +66 -0
- data/samples/sql.rb +167 -0
- data/samples/views/plot.erb +6 -0
- data/samples/webplot.rb +37 -0
- data/specs/id_parser_spec.rb +45 -0
- data/specs/message_parser_spec.rb +70 -0
- data/specs/parser_spec.rb +80 -0
- data/specs/ruby_to_r_translator_spec.rb +88 -0
- data/specs/spec_helper.rb +3 -0
- metadata +101 -0
@@ -0,0 +1,26 @@
|
|
1
|
+
|
2
|
+
require 'em-rserve/qap1/rpack'
|
3
|
+
|
4
|
+
module EM::Rserve
|
5
|
+
module QAP1
|
6
|
+
class Message
|
7
|
+
extend Rpack
|
8
|
+
|
9
|
+
def self.from_bin(dat)
|
10
|
+
new decode_parameters(dat)
|
11
|
+
end
|
12
|
+
|
13
|
+
attr_reader :parameters
|
14
|
+
|
15
|
+
def initialize(params=[])
|
16
|
+
@parameters = params
|
17
|
+
end
|
18
|
+
|
19
|
+
def pack_parameters
|
20
|
+
parameters.map{|p| self.class.encode_parameter(p)}.join
|
21
|
+
end
|
22
|
+
|
23
|
+
alias :to_bin :pack_parameters
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,127 @@
|
|
1
|
+
|
2
|
+
require 'em-rserve/qap1/constants'
|
3
|
+
require 'em-rserve/r/sexp'
|
4
|
+
|
5
|
+
module EM::Rserve
|
6
|
+
module QAP1
|
7
|
+
module Rpack
|
8
|
+
include Constants
|
9
|
+
include R
|
10
|
+
|
11
|
+
def parameter_head(type, len)
|
12
|
+
#TODO: support long parameter, type: 0xbf + DT_LARGE if len overflows
|
13
|
+
(type & 0x000000ff) | ((len << 8) & 0xffffff00)
|
14
|
+
end
|
15
|
+
|
16
|
+
def pack_parameter(type, len, val, rule)
|
17
|
+
[parameter_head(type, len), val].flatten.pack('V'+rule)
|
18
|
+
end
|
19
|
+
|
20
|
+
def encode_parameter(param, type=nil)
|
21
|
+
type ||= param
|
22
|
+
case param
|
23
|
+
when Integer, :int
|
24
|
+
encode_int param
|
25
|
+
when :char
|
26
|
+
encode_char param
|
27
|
+
when Float, :double
|
28
|
+
encode_double param
|
29
|
+
when String, :string
|
30
|
+
encode_string param
|
31
|
+
when Sexp::Node
|
32
|
+
encode_sexp_node param
|
33
|
+
when :bytestream
|
34
|
+
encode_bytestream param
|
35
|
+
#XXX Array and rest
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def encode_int(val)
|
40
|
+
pack_parameter(DT_INT, 4, val, 'V')
|
41
|
+
end
|
42
|
+
|
43
|
+
def encode_char(val)
|
44
|
+
pack_parameter(DT_CHAR, 1, val, 'C')
|
45
|
+
end
|
46
|
+
|
47
|
+
def encode_double(val)
|
48
|
+
pack_parameter(DT_DOUBLE, 8, val, 'D')
|
49
|
+
end
|
50
|
+
|
51
|
+
def encode_string(val, len=nil, has_null=false)
|
52
|
+
if has_null
|
53
|
+
len ||= val.length
|
54
|
+
pack_parameter(DT_STRING, len, val, 'a*')
|
55
|
+
else
|
56
|
+
len ||= val.length + 1
|
57
|
+
pack_parameter(DT_STRING, len, [val, 0], 'a*C')
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def encode_bytestream(val, len=nil)
|
62
|
+
len ||= val.length
|
63
|
+
pack_parameter(DT_BYTESTREAM, len, val, 'a*')
|
64
|
+
end
|
65
|
+
|
66
|
+
def encode_large(val, len=nil)
|
67
|
+
end
|
68
|
+
|
69
|
+
def encode_sexp_node(node, len=nil)
|
70
|
+
dat = node.dump_sexp
|
71
|
+
len ||= dat.length
|
72
|
+
pack_parameter(DT_SEXP, len, dat, 'a*')
|
73
|
+
end
|
74
|
+
|
75
|
+
def encode_array(val, len=nil)
|
76
|
+
end
|
77
|
+
|
78
|
+
def head_parameter(head)
|
79
|
+
type = head & 0x000000bf
|
80
|
+
large = (head & DT_LARGE) > 0
|
81
|
+
len = (head & 0xffffff00) >> 8
|
82
|
+
[type, len, large]
|
83
|
+
end
|
84
|
+
|
85
|
+
def decode_sexp(dat)
|
86
|
+
Sexp.parse(dat)
|
87
|
+
end
|
88
|
+
|
89
|
+
def decode_int(dat)
|
90
|
+
dat.unpack('i').first
|
91
|
+
end
|
92
|
+
|
93
|
+
def decode_bytestream(dat)
|
94
|
+
dat
|
95
|
+
end
|
96
|
+
|
97
|
+
def decode_parameter(type, dat, len=nil)
|
98
|
+
case type
|
99
|
+
when DT_SEXP
|
100
|
+
decode_sexp dat
|
101
|
+
when DT_INT
|
102
|
+
decode_int dat
|
103
|
+
when DT_BYTESTREAM
|
104
|
+
decode_bytestream dat
|
105
|
+
else
|
106
|
+
p "missing decode: #{type}"
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
def decode_parameters(buffer)
|
111
|
+
params = []
|
112
|
+
until buffer.empty? do
|
113
|
+
head, buffer = buffer.unpack('Va*')
|
114
|
+
type, len, large = head_parameter(head)
|
115
|
+
raise NotImplementedError, "too long QAP1 message" if large
|
116
|
+
if buffer.size < len
|
117
|
+
raise RuntimeError, "cannot decode #{buffer} (not enough bytes)"
|
118
|
+
end
|
119
|
+
param_data = buffer.slice(0, len)
|
120
|
+
buffer = buffer.slice(len .. -1)
|
121
|
+
params << decode_parameter(type, param_data, len)
|
122
|
+
end
|
123
|
+
params
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
@@ -0,0 +1,152 @@
|
|
1
|
+
|
2
|
+
require 'em-rserve/r/sexp'
|
3
|
+
|
4
|
+
module EM::Rserve
|
5
|
+
module R
|
6
|
+
module RtoRuby
|
7
|
+
class Translator
|
8
|
+
def self.r_to_ruby(root)
|
9
|
+
node = root.children.first
|
10
|
+
attR = node.attribute
|
11
|
+
|
12
|
+
pair = [node.class, attR.class]
|
13
|
+
klass = case pair
|
14
|
+
when [Sexp::Node::ArrayInt, NilClass]
|
15
|
+
ArrayTranslator
|
16
|
+
when [Sexp::Node::ArrayBool, NilClass]
|
17
|
+
ArrayTranslator
|
18
|
+
when [Sexp::Node::ArrayString, NilClass]
|
19
|
+
ArrayTranslator
|
20
|
+
when [Sexp::Node::ArrayDouble, NilClass]
|
21
|
+
ArrayTranslator
|
22
|
+
when [Sexp::Node::ArrayComplex, NilClass]
|
23
|
+
ArrayTranslator
|
24
|
+
when [Sexp::Node::ArrayDouble, Sexp::Node::ListTag]
|
25
|
+
DateTranslator
|
26
|
+
when [Sexp::Node::ArrayInt, Sexp::Node::ListTag]
|
27
|
+
FactorTableTranslator
|
28
|
+
when [Sexp::Node::Vector, Sexp::Node::ListTag]
|
29
|
+
DataFrameTranslator
|
30
|
+
when [Sexp::Node::Closure, NilClass]
|
31
|
+
ClosureTranslator
|
32
|
+
else
|
33
|
+
DefaultTranslator
|
34
|
+
end
|
35
|
+
klass.new(node).translate
|
36
|
+
end
|
37
|
+
|
38
|
+
attr_reader :node
|
39
|
+
|
40
|
+
def initialize(node=nil)
|
41
|
+
@node = node
|
42
|
+
end
|
43
|
+
|
44
|
+
def translate
|
45
|
+
throw :cannot_translate
|
46
|
+
end
|
47
|
+
|
48
|
+
class DefaultTranslator < Translator
|
49
|
+
end
|
50
|
+
|
51
|
+
class ArrayTranslator < Translator
|
52
|
+
def translate
|
53
|
+
node.rb_val
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
class DateTranslator < Translator
|
58
|
+
def translate
|
59
|
+
case node.attribute.rb_val['class']
|
60
|
+
when 'Date'
|
61
|
+
Time.at(node.rb_val * 86400) # R gives days Ruby wants secs
|
62
|
+
else
|
63
|
+
super
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
class FactorTableTranslator < Translator
|
69
|
+
# Symbols represented by ints
|
70
|
+
class Factor < Array
|
71
|
+
attr_accessor :levels
|
72
|
+
end
|
73
|
+
|
74
|
+
# Frequency table
|
75
|
+
class Table < Hash
|
76
|
+
end
|
77
|
+
|
78
|
+
def translate_factor
|
79
|
+
levels = node.attribute.rb_val['levels']
|
80
|
+
levels = levels.map{|str| str.to_sym}
|
81
|
+
ret = Factor.new.replace(node.rb_val.map{|i| levels[i-1]})
|
82
|
+
end
|
83
|
+
|
84
|
+
def translate_table
|
85
|
+
keys = node.attribute.rb_val['dimnames'].first
|
86
|
+
vals = node.rb_val
|
87
|
+
Table[ keys.zip(vals) ]
|
88
|
+
end
|
89
|
+
|
90
|
+
def translate
|
91
|
+
case node.attribute.rb_val['class']
|
92
|
+
when 'factor'
|
93
|
+
translate_factor
|
94
|
+
when 'table'
|
95
|
+
translate_table
|
96
|
+
else
|
97
|
+
super
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
class DataFrameTranslator < Translator
|
103
|
+
class DataFrame < Hash
|
104
|
+
attr_accessor :rows, :r_class
|
105
|
+
|
106
|
+
def inspect
|
107
|
+
super.sub(/}$/," @r_class: #{r_class}, @rows: #{rows}")
|
108
|
+
end
|
109
|
+
|
110
|
+
def each_struct
|
111
|
+
if block_given?
|
112
|
+
all_keys = keys #not sure keys will always return the same order
|
113
|
+
struct = Struct.new(*all_keys.map(&:to_sym))
|
114
|
+
#XXX when we map and transpose, we actually do computation before they
|
115
|
+
#are needed, could improve with true-style iterators
|
116
|
+
all_keys.map{|k| self[k]}.transpose.each do |values|
|
117
|
+
yield struct.new(*values)
|
118
|
+
end
|
119
|
+
else
|
120
|
+
Enumerator.new(:self, :each_struct)
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
def translate_data_frame
|
126
|
+
attrs = node.attribute.rb_val
|
127
|
+
cols = [attrs['names']].flatten
|
128
|
+
rows = case attrs['row.names']
|
129
|
+
when [-2147483648, -8]
|
130
|
+
nil
|
131
|
+
else
|
132
|
+
[attrs['row.names']].flatten
|
133
|
+
end
|
134
|
+
|
135
|
+
dfrm = DataFrame[ cols.zip(node.rb_val) ]
|
136
|
+
dfrm.rows = rows if rows
|
137
|
+
dfrm
|
138
|
+
end
|
139
|
+
|
140
|
+
def translate
|
141
|
+
h = translate_data_frame
|
142
|
+
h.r_class = node.attribute.rb_val['class']
|
143
|
+
h
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
class ClosureTranslator < Translator
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
@@ -0,0 +1,131 @@
|
|
1
|
+
require 'em-rserve/r/sexp'
|
2
|
+
|
3
|
+
module EM::Rserve
|
4
|
+
module R
|
5
|
+
module RubytoR
|
6
|
+
class Translator
|
7
|
+
|
8
|
+
def self.translator_klass_for(obj)
|
9
|
+
case obj
|
10
|
+
when Array
|
11
|
+
ArrayTranslator
|
12
|
+
when Hash
|
13
|
+
HashTranslator
|
14
|
+
else
|
15
|
+
SingleObjectTranslator
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.ruby_to_r(obj)
|
20
|
+
translator_klass_for(obj).new(obj).translate
|
21
|
+
end
|
22
|
+
|
23
|
+
attr_reader :obj
|
24
|
+
|
25
|
+
def initialize(obj=nil)
|
26
|
+
@obj = obj
|
27
|
+
end
|
28
|
+
|
29
|
+
def translate
|
30
|
+
throw :cannot_translate
|
31
|
+
end
|
32
|
+
|
33
|
+
class SingleObjectTranslator < Translator
|
34
|
+
def translate
|
35
|
+
Translator.ruby_to_r [obj]
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
class ArrayTranslator < Translator
|
40
|
+
MAPPING = {String => EM::Rserve::R::Sexp::Node::ArrayString,
|
41
|
+
Integer => EM::Rserve::R::Sexp::Node::ArrayInt,
|
42
|
+
Float => EM::Rserve::R::Sexp::Node::ArrayDouble,
|
43
|
+
NilClass => EM::Rserve::R::Sexp::Node::ArrayBool,
|
44
|
+
TrueClass => EM::Rserve::R::Sexp::Node::ArrayBool,
|
45
|
+
FalseClass => EM::Rserve::R::Sexp::Node::ArrayBool,
|
46
|
+
}
|
47
|
+
|
48
|
+
def array_node_class
|
49
|
+
classes = obj.map(&:class).uniq
|
50
|
+
if classes.size == 1
|
51
|
+
obj_klass = classes.first
|
52
|
+
MAPPING.each_pair do |klass, node|
|
53
|
+
return node if obj_klass.ancestors.include?(klass)
|
54
|
+
end
|
55
|
+
elsif (classes - [TrueClass,NilClass,FalseClass]).empty?
|
56
|
+
EM::Rserve::R::Sexp::Node::ArrayBool
|
57
|
+
elsif (classes - [Float, Fixnum]).empty?
|
58
|
+
EM::Rserve::R::Sexp::Node::ArrayDouble
|
59
|
+
else
|
60
|
+
nil
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def translate
|
65
|
+
klass = array_node_class
|
66
|
+
throw :cannot_translate unless klass
|
67
|
+
EM::Rserve::R::Sexp::Node::Root.new do |root|
|
68
|
+
klass.new do |array|
|
69
|
+
array.rb_raw = obj
|
70
|
+
root.children << array
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
class HashTranslator < Translator
|
77
|
+
def list_node_class
|
78
|
+
EM::Rserve::R::Sexp::Node::Vector
|
79
|
+
end
|
80
|
+
|
81
|
+
def list_node_attribute_class
|
82
|
+
EM::Rserve::R::Sexp::Node::ListTag
|
83
|
+
end
|
84
|
+
|
85
|
+
def translate
|
86
|
+
klass = list_node_class
|
87
|
+
attr_klass = list_node_attribute_class
|
88
|
+
throw :cannot_translate unless klass and attr_klass
|
89
|
+
raise :cannot_translate if obj.empty?
|
90
|
+
pairs = obj.each_pair.to_a
|
91
|
+
size = pairs.first.last.size
|
92
|
+
#TODO: check if sizes differ and raise
|
93
|
+
EM::Rserve::R::Sexp::Node::Root.new do |root|
|
94
|
+
klass.new do |vector|
|
95
|
+
vector.attribute = attr_klass.new do |taglist|
|
96
|
+
# add ArrayString with keys as strings for column names
|
97
|
+
# add SymName "names"
|
98
|
+
taglist.children << ArrayTranslator.new(pairs.map(&:first).map(&:to_s)).translate.children.first
|
99
|
+
sym = EM::Rserve::R::Sexp::Node::SymName.new
|
100
|
+
sym.rb_raw = "names"
|
101
|
+
taglist.children << sym
|
102
|
+
# add ArrayInt -2147483648, -(size) for rows
|
103
|
+
# add SymName "row.names"
|
104
|
+
taglist.children << ArrayTranslator.new([-2147483648, 0 - size]).translate.children.first
|
105
|
+
sym = EM::Rserve::R::Sexp::Node::SymName.new
|
106
|
+
sym.rb_raw = "row.names"
|
107
|
+
taglist.children << sym
|
108
|
+
|
109
|
+
# add ArrayString "data.frame"
|
110
|
+
# add SymName "class"
|
111
|
+
taglist.children << ArrayTranslator.new(["data.frame"]).translate.children.first
|
112
|
+
sym = EM::Rserve::R::Sexp::Node::SymName.new
|
113
|
+
sym.rb_raw = "class"
|
114
|
+
taglist.children << sym
|
115
|
+
end #attribute
|
116
|
+
|
117
|
+
pairs.each do |k,values|
|
118
|
+
array_node = ArrayTranslator.new(values).translate.children.first
|
119
|
+
vector.children << array_node
|
120
|
+
end #children
|
121
|
+
root.children << vector
|
122
|
+
end #vector
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|