avro-jruby 1.7.5
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG +1 -0
- data/Manifest +23 -0
- data/Rakefile +63 -0
- data/avro-jruby.gemspec +34 -0
- data/interop/test_interop.rb +41 -0
- data/lib/avro.rb +42 -0
- data/lib/avro/collect_hash.rb +25 -0
- data/lib/avro/data_file.rb +342 -0
- data/lib/avro/io.rb +615 -0
- data/lib/avro/ipc.rb +550 -0
- data/lib/avro/protocol.rb +161 -0
- data/lib/avro/schema.rb +405 -0
- data/test/random_data.rb +90 -0
- data/test/sample_ipc_client.rb +85 -0
- data/test/sample_ipc_http_client.rb +84 -0
- data/test/sample_ipc_http_server.rb +79 -0
- data/test/sample_ipc_server.rb +92 -0
- data/test/test_datafile.rb +188 -0
- data/test/test_help.rb +23 -0
- data/test/test_io.rb +393 -0
- data/test/test_protocol.rb +199 -0
- data/test/test_schema.rb +134 -0
- data/test/test_socket_transport.rb +40 -0
- data/test/tool.rb +144 -0
- metadata +103 -0
data/test/test_schema.rb
ADDED
@@ -0,0 +1,134 @@
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
3
|
+
# distributed with this work for additional information
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
6
|
+
# "License"); you may not use this file except in compliance
|
7
|
+
# with the License. You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
14
|
+
# See the License for the specific language governing permissions and
|
15
|
+
# limitations under the License.
|
16
|
+
|
17
|
+
require 'test_help'
|
18
|
+
|
19
|
+
class TestSchema < Test::Unit::TestCase
|
20
|
+
def test_default_namespace
|
21
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
22
|
+
{"type": "record", "name": "OuterRecord", "fields": [
|
23
|
+
{"name": "field1", "type": {
|
24
|
+
"type": "record", "name": "InnerRecord", "fields": []
|
25
|
+
}},
|
26
|
+
{"name": "field2", "type": "InnerRecord"}
|
27
|
+
]}
|
28
|
+
SCHEMA
|
29
|
+
|
30
|
+
assert_equal schema.name, 'OuterRecord'
|
31
|
+
assert_equal schema.fullname, 'OuterRecord'
|
32
|
+
assert_nil schema.namespace
|
33
|
+
|
34
|
+
schema.fields.each do |field|
|
35
|
+
assert_equal field.type.name, 'InnerRecord'
|
36
|
+
assert_equal field.type.fullname, 'InnerRecord'
|
37
|
+
assert_nil field.type.namespace
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def test_inherited_namespace
|
42
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
43
|
+
{"type": "record", "name": "OuterRecord", "namespace": "my.name.space",
|
44
|
+
"fields": [
|
45
|
+
{"name": "definition", "type": {
|
46
|
+
"type": "record", "name": "InnerRecord", "fields": []
|
47
|
+
}},
|
48
|
+
{"name": "relativeReference", "type": "InnerRecord"},
|
49
|
+
{"name": "absoluteReference", "type": "my.name.space.InnerRecord"}
|
50
|
+
]}
|
51
|
+
SCHEMA
|
52
|
+
|
53
|
+
assert_equal schema.name, 'OuterRecord'
|
54
|
+
assert_equal schema.fullname, 'my.name.space.OuterRecord'
|
55
|
+
assert_equal schema.namespace, 'my.name.space'
|
56
|
+
schema.fields.each do |field|
|
57
|
+
assert_equal field.type.name, 'InnerRecord'
|
58
|
+
assert_equal field.type.fullname, 'my.name.space.InnerRecord'
|
59
|
+
assert_equal field.type.namespace, 'my.name.space'
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def test_inherited_namespace_from_dotted_name
|
64
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
65
|
+
{"type": "record", "name": "my.name.space.OuterRecord", "fields": [
|
66
|
+
{"name": "definition", "type": {
|
67
|
+
"type": "enum", "name": "InnerEnum", "symbols": ["HELLO", "WORLD"]
|
68
|
+
}},
|
69
|
+
{"name": "relativeReference", "type": "InnerEnum"},
|
70
|
+
{"name": "absoluteReference", "type": "my.name.space.InnerEnum"}
|
71
|
+
]}
|
72
|
+
SCHEMA
|
73
|
+
|
74
|
+
assert_equal schema.name, 'OuterRecord'
|
75
|
+
assert_equal schema.fullname, 'my.name.space.OuterRecord'
|
76
|
+
assert_equal schema.namespace, 'my.name.space'
|
77
|
+
schema.fields.each do |field|
|
78
|
+
assert_equal field.type.name, 'InnerEnum'
|
79
|
+
assert_equal field.type.fullname, 'my.name.space.InnerEnum'
|
80
|
+
assert_equal field.type.namespace, 'my.name.space'
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def test_nested_namespaces
|
85
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
86
|
+
{"type": "record", "name": "outer.OuterRecord", "fields": [
|
87
|
+
{"name": "middle", "type": {
|
88
|
+
"type": "record", "name": "middle.MiddleRecord", "fields": [
|
89
|
+
{"name": "inner", "type": {
|
90
|
+
"type": "record", "name": "InnerRecord", "fields": [
|
91
|
+
{"name": "recursive", "type": "MiddleRecord"}
|
92
|
+
]
|
93
|
+
}}
|
94
|
+
]
|
95
|
+
}}
|
96
|
+
]}
|
97
|
+
SCHEMA
|
98
|
+
|
99
|
+
assert_equal schema.name, 'OuterRecord'
|
100
|
+
assert_equal schema.fullname, 'outer.OuterRecord'
|
101
|
+
assert_equal schema.namespace, 'outer'
|
102
|
+
middle = schema.fields.first.type
|
103
|
+
assert_equal middle.name, 'MiddleRecord'
|
104
|
+
assert_equal middle.fullname, 'middle.MiddleRecord'
|
105
|
+
assert_equal middle.namespace, 'middle'
|
106
|
+
inner = middle.fields.first.type
|
107
|
+
assert_equal inner.name, 'InnerRecord'
|
108
|
+
assert_equal inner.fullname, 'middle.InnerRecord'
|
109
|
+
assert_equal inner.namespace, 'middle'
|
110
|
+
assert_equal inner.fields.first.type, middle
|
111
|
+
end
|
112
|
+
|
113
|
+
def test_to_avro_includes_namespaces
|
114
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
115
|
+
{"type": "record", "name": "my.name.space.OuterRecord", "fields": [
|
116
|
+
{"name": "definition", "type": {
|
117
|
+
"type": "fixed", "name": "InnerFixed", "size": 16
|
118
|
+
}},
|
119
|
+
{"name": "reference", "type": "InnerFixed"}
|
120
|
+
]}
|
121
|
+
SCHEMA
|
122
|
+
|
123
|
+
assert_equal schema.to_avro, {
|
124
|
+
'type' => 'record', 'name' => 'OuterRecord', 'namespace' => 'my.name.space',
|
125
|
+
'fields' => [
|
126
|
+
{'name' => 'definition', 'type' => {
|
127
|
+
'type' => 'fixed', 'name' => 'InnerFixed', 'namespace' => 'my.name.space',
|
128
|
+
'size' => 16
|
129
|
+
}},
|
130
|
+
{'name' => 'reference', 'type' => 'my.name.space.InnerFixed'}
|
131
|
+
]
|
132
|
+
}
|
133
|
+
end
|
134
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
3
|
+
# distributed with this work for additional information
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
6
|
+
# "License"); you may not use this file except in compliance
|
7
|
+
# with the License. You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
14
|
+
# See the License for the specific language governing permissions and
|
15
|
+
# limitations under the License.
|
16
|
+
|
17
|
+
require 'test_help'
|
18
|
+
|
19
|
+
class TestSocketTransport < Test::Unit::TestCase
|
20
|
+
def test_buffer_writing
|
21
|
+
io = StringIO.new
|
22
|
+
st = Avro::IPC::SocketTransport.new(io)
|
23
|
+
buffer_length = "\000\000\000\006" # 6 in big-endian
|
24
|
+
message = 'abcdef'
|
25
|
+
null_ending = "\000\000\000\000" # 0 in big-endian
|
26
|
+
full = buffer_length + message + null_ending
|
27
|
+
st.write_framed_message('abcdef')
|
28
|
+
assert_equal full, io.string
|
29
|
+
end
|
30
|
+
|
31
|
+
def test_buffer_reading
|
32
|
+
buffer_length = "\000\000\000\005" # 5 in big-endian
|
33
|
+
message = "hello"
|
34
|
+
null_ending = "\000\000\000\000" # 0 in big-endian
|
35
|
+
full = buffer_length + message + null_ending
|
36
|
+
io = StringIO.new(full)
|
37
|
+
st = Avro::IPC::SocketTransport.new(io)
|
38
|
+
assert_equal 'hello', st.read_framed_message
|
39
|
+
end
|
40
|
+
end
|
data/test/tool.rb
ADDED
@@ -0,0 +1,144 @@
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
3
|
+
# distributed with this work for additional information
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
6
|
+
# "License"); you may not use this file except in compliance
|
7
|
+
# with the License. You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
14
|
+
# See the License for the specific language governing permissions and
|
15
|
+
# limitations under the License.
|
16
|
+
|
17
|
+
require 'avro'
|
18
|
+
require 'webrick'
|
19
|
+
require 'uri'
|
20
|
+
require 'logger'
|
21
|
+
|
22
|
+
class GenericResponder < Avro::IPC::Responder
|
23
|
+
def initialize(proto, msg, datum)
|
24
|
+
proto_json = open(proto).read
|
25
|
+
super(Avro::Protocol.parse(proto_json))
|
26
|
+
@msg = msg
|
27
|
+
@datum = datum
|
28
|
+
end
|
29
|
+
|
30
|
+
def call(message, request)
|
31
|
+
if message.name == @msg
|
32
|
+
STDERR.puts "Message: #{message.name} Datum: #{@datum.inspect}"
|
33
|
+
@datum
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
class GenericHandler < WEBrick::HTTPServlet::AbstractServlet
|
39
|
+
def do_POST(req, resp)
|
40
|
+
call_request = Avro::IPC::FramedReader.new(StringIO.new(req.body)).read_framed_message
|
41
|
+
unframed_resp = $responder.respond(call_request)
|
42
|
+
writer = Avro::IPC::FramedWriter.new(StringIO.new)
|
43
|
+
writer.write_framed_message(unframed_resp)
|
44
|
+
resp.body = writer.to_s
|
45
|
+
@server.stop
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def run_server(uri, proto, msg, datum)
|
50
|
+
uri = URI.parse(uri)
|
51
|
+
$responder = GenericResponder.new(proto, msg, datum)
|
52
|
+
server = WEBrick::HTTPServer.new(:BindAddress => uri.host,
|
53
|
+
:Port => uri.port,
|
54
|
+
:Logger => Logger.new(StringIO.new))
|
55
|
+
server.mount '/', GenericHandler
|
56
|
+
puts "Port: #{server.config[:Port]}"
|
57
|
+
STDOUT.flush
|
58
|
+
trap("INT") { server.stop }
|
59
|
+
trap("TERM") { server.stop }
|
60
|
+
server.start
|
61
|
+
end
|
62
|
+
|
63
|
+
def send_message(uri, proto, msg, datum)
|
64
|
+
uri = URI.parse(uri)
|
65
|
+
trans = Avro::IPC::HTTPTransceiver.new(uri.host, uri.port)
|
66
|
+
proto_json = open(proto).read
|
67
|
+
requestor = Avro::IPC::Requestor.new(Avro::Protocol.parse(proto_json),
|
68
|
+
trans)
|
69
|
+
p requestor.request(msg, datum)
|
70
|
+
end
|
71
|
+
|
72
|
+
def file_or_stdin(f)
|
73
|
+
f == "-" ? STDIN : open(f)
|
74
|
+
end
|
75
|
+
|
76
|
+
def main
|
77
|
+
if ARGV.size == 0
|
78
|
+
puts "Usage: #{$0} [dump|rpcreceive|rpcsend]"
|
79
|
+
return 1
|
80
|
+
end
|
81
|
+
|
82
|
+
case ARGV[0]
|
83
|
+
when "dump"
|
84
|
+
if ARGV.size != 3
|
85
|
+
puts "Usage: #{$0} dump input_file"
|
86
|
+
return 1
|
87
|
+
end
|
88
|
+
d = Avro::DataFile.new(file_or_stdin(ARGV[1]), Avro::IO::DatumReader.new)
|
89
|
+
d.each{|o| puts o.inspect }
|
90
|
+
d.close
|
91
|
+
when "rpcreceive"
|
92
|
+
usage_str = "Usage: #{$0} rpcreceive uri protocol_file "
|
93
|
+
usage_str += "message_name (-data d | -file f)"
|
94
|
+
|
95
|
+
unless [4, 6].include?(ARGV.size)
|
96
|
+
puts usage_str
|
97
|
+
return 1
|
98
|
+
end
|
99
|
+
uri, proto, msg = ARGV[1,3]
|
100
|
+
datum = nil
|
101
|
+
if ARGV.size > 4
|
102
|
+
case ARGV[4]
|
103
|
+
when "-file"
|
104
|
+
Avro::DataFile.open(ARGV[5]) {|f|
|
105
|
+
f.each{|d| datum = d; break }
|
106
|
+
}
|
107
|
+
when "-data"
|
108
|
+
puts "JSON Decoder not yet implemented."
|
109
|
+
return 1
|
110
|
+
else
|
111
|
+
puts usage_str
|
112
|
+
return 1
|
113
|
+
end
|
114
|
+
end
|
115
|
+
run_server(uri, proto, msg, datum)
|
116
|
+
when "rpcsend"
|
117
|
+
usage_str = "Usage: #{$0} rpcsend uri protocol_file "
|
118
|
+
usage_str += "message_name (-data d | -file f)"
|
119
|
+
unless [4,6].include?(ARGV.size)
|
120
|
+
puts usage_str
|
121
|
+
return 1
|
122
|
+
end
|
123
|
+
uri, proto, msg = ARGV[1,3]
|
124
|
+
datum = nil
|
125
|
+
if ARGV.size > 4
|
126
|
+
case ARGV[4]
|
127
|
+
when "-file"
|
128
|
+
Avro::DataFile.open(ARGV[5]){|f| f.each{|d| datum = d; break } }
|
129
|
+
when "-data"
|
130
|
+
puts "JSON Decoder not yet implemented"
|
131
|
+
return 1
|
132
|
+
else
|
133
|
+
puts usage_str
|
134
|
+
return 1
|
135
|
+
end
|
136
|
+
end
|
137
|
+
send_message(uri, proto, msg, datum)
|
138
|
+
end
|
139
|
+
return 0
|
140
|
+
end
|
141
|
+
|
142
|
+
if __FILE__ == $0
|
143
|
+
exit(main)
|
144
|
+
end
|
metadata
ADDED
@@ -0,0 +1,103 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: avro-jruby
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.7.5
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Apache Software Foundation
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2013-10-25 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: multi_json
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
30
|
+
description: Avro is a data serialization and RPC format
|
31
|
+
email: avro-dev@hadoop.apache.org
|
32
|
+
executables: []
|
33
|
+
extensions: []
|
34
|
+
extra_rdoc_files:
|
35
|
+
- CHANGELOG
|
36
|
+
- lib/avro.rb
|
37
|
+
- lib/avro/collect_hash.rb
|
38
|
+
- lib/avro/data_file.rb
|
39
|
+
- lib/avro/io.rb
|
40
|
+
- lib/avro/ipc.rb
|
41
|
+
- lib/avro/protocol.rb
|
42
|
+
- lib/avro/schema.rb
|
43
|
+
files:
|
44
|
+
- CHANGELOG
|
45
|
+
- Manifest
|
46
|
+
- Rakefile
|
47
|
+
- avro-jruby.gemspec
|
48
|
+
- interop/test_interop.rb
|
49
|
+
- lib/avro.rb
|
50
|
+
- lib/avro/collect_hash.rb
|
51
|
+
- lib/avro/data_file.rb
|
52
|
+
- lib/avro/io.rb
|
53
|
+
- lib/avro/ipc.rb
|
54
|
+
- lib/avro/protocol.rb
|
55
|
+
- lib/avro/schema.rb
|
56
|
+
- test/random_data.rb
|
57
|
+
- test/sample_ipc_client.rb
|
58
|
+
- test/sample_ipc_http_client.rb
|
59
|
+
- test/sample_ipc_http_server.rb
|
60
|
+
- test/sample_ipc_server.rb
|
61
|
+
- test/test_datafile.rb
|
62
|
+
- test/test_help.rb
|
63
|
+
- test/test_io.rb
|
64
|
+
- test/test_protocol.rb
|
65
|
+
- test/test_socket_transport.rb
|
66
|
+
- test/tool.rb
|
67
|
+
- test/test_schema.rb
|
68
|
+
homepage: https://github.com/aia/avro-gem-jruby
|
69
|
+
licenses: []
|
70
|
+
post_install_message:
|
71
|
+
rdoc_options:
|
72
|
+
- --line-numbers
|
73
|
+
- --inline-source
|
74
|
+
- --title
|
75
|
+
- Avro
|
76
|
+
require_paths:
|
77
|
+
- lib
|
78
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
79
|
+
none: false
|
80
|
+
requirements:
|
81
|
+
- - ! '>='
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
version: '0'
|
84
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
85
|
+
none: false
|
86
|
+
requirements:
|
87
|
+
- - ! '>='
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '1.2'
|
90
|
+
requirements: []
|
91
|
+
rubyforge_project: avro-jruby
|
92
|
+
rubygems_version: 1.8.25
|
93
|
+
signing_key:
|
94
|
+
specification_version: 4
|
95
|
+
summary: Apache Avro for Ruby
|
96
|
+
test_files:
|
97
|
+
- test/test_datafile.rb
|
98
|
+
- test/test_help.rb
|
99
|
+
- test/test_io.rb
|
100
|
+
- test/test_protocol.rb
|
101
|
+
- test/test_schema.rb
|
102
|
+
- test/test_socket_transport.rb
|
103
|
+
has_rdoc:
|