avro 1.3.3 → 1.5.4

Sign up to get free protection for your applications and to get access to all the features.
data/Manifest CHANGED
@@ -12,7 +12,12 @@ lib/avro/protocol.rb
12
12
  lib/avro/schema.rb
13
13
  test/random_data.rb
14
14
  test/sample_ipc_client.rb
15
+ test/sample_ipc_http_client.rb
16
+ test/sample_ipc_http_server.rb
15
17
  test/sample_ipc_server.rb
18
+ test/test_datafile.rb
16
19
  test/test_help.rb
17
20
  test/test_io.rb
18
21
  test/test_protocol.rb
22
+ test/test_socket_transport.rb
23
+ test/tool.rb
data/Rakefile CHANGED
@@ -53,7 +53,7 @@ SHARE = HERE + '/../../share'
53
53
  SCHEMAS = SHARE + '/test/schemas'
54
54
  BUILD = HERE + '/../../build'
55
55
 
56
- task :dist => [:manifest, :gem] do
56
+ task :dist => [:gem] do
57
57
  mkdir_p "../../dist/ruby"
58
58
  cp "pkg/avro-#{VERSION}.gem", "../../dist/ruby"
59
59
  end
@@ -2,22 +2,22 @@
2
2
 
3
3
  Gem::Specification.new do |s|
4
4
  s.name = %q{avro}
5
- s.version = "1.3.3"
5
+ s.version = "1.5.4"
6
6
 
7
7
  s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
8
8
  s.authors = ["Apache Software Foundation"]
9
- s.date = %q{2010-08-18}
9
+ s.date = %q{2011-09-07}
10
10
  s.description = %q{Apache is a data serialization and RPC format}
11
11
  s.email = %q{avro-dev@hadoop.apache.org}
12
12
  s.extra_rdoc_files = ["CHANGELOG", "lib/avro.rb", "lib/avro/collect_hash.rb", "lib/avro/data_file.rb", "lib/avro/io.rb", "lib/avro/ipc.rb", "lib/avro/protocol.rb", "lib/avro/schema.rb"]
13
- s.files = ["CHANGELOG", "Manifest", "Rakefile", "avro.gemspec", "interop/test_interop.rb", "lib/avro.rb", "lib/avro/collect_hash.rb", "lib/avro/data_file.rb", "lib/avro/io.rb", "lib/avro/ipc.rb", "lib/avro/protocol.rb", "lib/avro/schema.rb", "test/random_data.rb", "test/sample_ipc_client.rb", "test/sample_ipc_server.rb", "test/test_help.rb", "test/test_io.rb", "test/test_protocol.rb", "test/test_datafile.rb", "test/test_socket_transport.rb"]
13
+ s.files = ["CHANGELOG", "Manifest", "Rakefile", "avro.gemspec", "interop/test_interop.rb", "lib/avro.rb", "lib/avro/collect_hash.rb", "lib/avro/data_file.rb", "lib/avro/io.rb", "lib/avro/ipc.rb", "lib/avro/protocol.rb", "lib/avro/schema.rb", "test/random_data.rb", "test/sample_ipc_client.rb", "test/sample_ipc_http_client.rb", "test/sample_ipc_http_server.rb", "test/sample_ipc_server.rb", "test/test_datafile.rb", "test/test_help.rb", "test/test_io.rb", "test/test_protocol.rb", "test/test_socket_transport.rb", "test/tool.rb"]
14
14
  s.homepage = %q{http://hadoop.apache.org/avro/}
15
15
  s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Avro"]
16
16
  s.require_paths = ["lib"]
17
17
  s.rubyforge_project = %q{avro}
18
18
  s.rubygems_version = %q{1.3.7}
19
19
  s.summary = %q{Apache Avro for Ruby}
20
- s.test_files = ["test/test_datafile.rb", "test/test_help.rb", "test/test_io.rb", "test/test_protocol.rb", "test/test_socket_transport.rb"]
20
+ s.test_files = ["test/test_help.rb", "test/test_datafile.rb", "test/test_socket_transport.rb", "test/test_io.rb", "test/test_protocol.rb"]
21
21
 
22
22
  if s.respond_to? :specification_version then
23
23
  current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
@@ -16,7 +16,7 @@
16
16
 
17
17
  require 'yajl'
18
18
  require 'set'
19
- require 'md5'
19
+ require 'digest/md5'
20
20
  require 'net/http'
21
21
  require 'stringio'
22
22
 
@@ -173,6 +173,7 @@ module Avro
173
173
 
174
174
  # reset buffer
175
175
  buffer_writer.truncate(0)
176
+ buffer_writer.rewind
176
177
  self.block_count = 0
177
178
  end
178
179
  end
@@ -43,9 +43,9 @@ module Avro
43
43
  end
44
44
 
45
45
  def byte!
46
- @reader.read(1)[0]
46
+ @reader.read(1).unpack('C').first
47
47
  end
48
-
48
+
49
49
  def read_null
50
50
  # null is written as zero byte's
51
51
  nil
@@ -500,6 +500,11 @@ module Avro
500
500
  decoder.skip_int
501
501
  end
502
502
 
503
+ def skip_union(writers_schema, decoder)
504
+ index = decoder.read_long
505
+ skip_data(writers_schema.schemas[index], decoder)
506
+ end
507
+
503
508
  def skip_array(writers_schema, decoder)
504
509
  skip_blocks(decoder) { skip_data(writers_schema.items, decoder) }
505
510
  end
@@ -14,6 +14,8 @@
14
14
  # See the License for the specific language governing permissions and
15
15
  # limitations under the License.
16
16
 
17
+ require "net/http"
18
+
17
19
  module Avro::IPC
18
20
 
19
21
  class AvroRemoteError < Avro::AvroError; end
@@ -397,7 +399,7 @@ module Avro::IPC
397
399
  message_length = message.size
398
400
  total_bytes_sent = 0
399
401
  while message_length - total_bytes_sent > 0
400
- if message_length - total_bytes_sent > BUFFER_SIZE:
402
+ if message_length - total_bytes_sent > BUFFER_SIZE
401
403
  buffer_length = BUFFER_SIZE
402
404
  else
403
405
  buffer_length = message_length - total_bytes_sent
@@ -521,14 +523,13 @@ module Avro::IPC
521
523
  def initialize(host, port)
522
524
  @host, @port = host, port
523
525
  @remote_name = "#{host}:#{port}"
526
+ @conn = Net::HTTP.start host, port
524
527
  end
525
528
 
526
529
  def transceive(message)
527
530
  writer = FramedWriter.new(StringIO.new)
528
531
  writer.write_framed_message(message)
529
- resp = Net::HTTP.start(host, port) do |http|
530
- http.post('/', writer.to_s, {'Content-Type' => 'avro/binary'})
531
- end
532
+ resp = @conn.post('/', writer.to_s, {'Content-Type' => 'avro/binary'})
532
533
  FramedReader.new(StringIO.new(resp.body)).read_framed_message
533
534
  end
534
535
  end
@@ -102,7 +102,7 @@ module Avro
102
102
  when 'array'
103
103
  datum.is_a?(Array) &&
104
104
  datum.all?{|d| validate(expected_schema.items, d) }
105
- when 'map':
105
+ when 'map'
106
106
  datum.keys.all?{|k| k.is_a? String } &&
107
107
  datum.values.all?{|v| validate(expected_schema.values, v) }
108
108
  when 'union'
@@ -0,0 +1,84 @@
1
+ #!/usr/bin/env ruby
2
+ # Licensed to the Apache Software Foundation (ASF) under one
3
+ # or more contributor license agreements. See the NOTICE file
4
+ # distributed with this work for additional information
5
+ # regarding copyright ownership. The ASF licenses this file
6
+ # to you under the Apache License, Version 2.0 (the
7
+ # "License"); you may not use this file except in compliance
8
+ # with the License. You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+
18
+ require 'socket'
19
+ require 'avro'
20
+
21
+ MAIL_PROTOCOL_JSON = <<-JSON
22
+ {"namespace": "example.proto",
23
+ "protocol": "Mail",
24
+
25
+ "types": [
26
+ {"name": "Message", "type": "record",
27
+ "fields": [
28
+ {"name": "to", "type": "string"},
29
+ {"name": "from", "type": "string"},
30
+ {"name": "body", "type": "string"}
31
+ ]
32
+ }
33
+ ],
34
+
35
+ "messages": {
36
+ "send": {
37
+ "request": [{"name": "message", "type": "Message"}],
38
+ "response": "string"
39
+ },
40
+ "replay": {
41
+ "request": [],
42
+ "response": "string"
43
+ }
44
+ }
45
+ }
46
+ JSON
47
+
48
+ MAIL_PROTOCOL = Avro::Protocol.parse(MAIL_PROTOCOL_JSON)
49
+
50
+ def make_requestor(server_address, port, protocol)
51
+ transport = Avro::IPC::HTTPTransceiver.new(server_address, port)
52
+ Avro::IPC::Requestor.new(protocol, transport)
53
+ end
54
+
55
+ if $0 == __FILE__
56
+ if ![3, 4].include?(ARGV.length)
57
+ raise "Usage: <to> <from> <body> [<count>]"
58
+ end
59
+
60
+ # client code - attach to the server and send a message
61
+ # fill in the Message record
62
+ message = {
63
+ 'to' => ARGV[0],
64
+ 'from' => ARGV[1],
65
+ 'body' => ARGV[2]
66
+ }
67
+
68
+ num_messages = (ARGV[3] || 1).to_i
69
+
70
+ # build the parameters for the request
71
+ params = {'message' => message}
72
+ # send the requests and print the result
73
+
74
+ num_messages.times do
75
+ requestor = make_requestor('localhost', 9090, MAIL_PROTOCOL)
76
+ result = requestor.request('send', params)
77
+ puts("Result: " + result)
78
+ end
79
+
80
+ # try out a replay message
81
+ requestor = make_requestor('localhost', 9090, MAIL_PROTOCOL)
82
+ result = requestor.request('replay', {})
83
+ puts("Replay Result: " + result)
84
+ end
@@ -0,0 +1,79 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ require 'avro'
17
+ require 'webrick'
18
+
19
+ MAIL_PROTOCOL_JSON = <<-JSON
20
+ {"namespace": "example.proto",
21
+ "protocol": "Mail",
22
+
23
+ "types": [
24
+ {"name": "Message", "type": "record",
25
+ "fields": [
26
+ {"name": "to", "type": "string"},
27
+ {"name": "from", "type": "string"},
28
+ {"name": "body", "type": "string"}
29
+ ]
30
+ }
31
+ ],
32
+
33
+ "messages": {
34
+ "send": {
35
+ "request": [{"name": "message", "type": "Message"}],
36
+ "response": "string"
37
+ },
38
+ "replay": {
39
+ "request": [],
40
+ "response": "string"
41
+ }
42
+ }
43
+ }
44
+ JSON
45
+
46
+ MAIL_PROTOCOL = Avro::Protocol.parse(MAIL_PROTOCOL_JSON)
47
+
48
+ class MailResponder < Avro::IPC::Responder
49
+ def initialize
50
+ super(MAIL_PROTOCOL)
51
+ end
52
+
53
+ def call(message, request)
54
+ if message.name == 'send'
55
+ request_content = request['message']
56
+ "Sent message to #{request_content['to']} from #{request_content['from']} with body #{request_content['body']}"
57
+ elsif message.name == 'replay'
58
+ 'replay'
59
+ end
60
+ end
61
+ end
62
+
63
+ class MailHandler < WEBrick::HTTPServlet::AbstractServlet
64
+ def do_POST(req, resp)
65
+ responder = MailResponder.new
66
+ call_request = Avro::IPC::FramedReader.new(StringIO.new(req.body)).read_framed_message
67
+ unframed_resp = responder.respond(call_request)
68
+ writer = Avro::IPC::FramedWriter.new(StringIO.new)
69
+ writer.write_framed_message(unframed_resp)
70
+ resp.body = writer.to_s
71
+ end
72
+ end
73
+
74
+ if $0 == __FILE__
75
+ server = WEBrick::HTTPServer.new(:Host => 'localhost', :Port => 9090)
76
+ server.mount '/', MailHandler
77
+ trap("INT") { server.shutdown }
78
+ server.start
79
+ end
@@ -118,4 +118,26 @@ JSON
118
118
  end
119
119
  end
120
120
  end
121
+
122
+ def test_data_writer_handles_sync_interval
123
+ writer_schema = <<-JSON
124
+ { "type": "record",
125
+ "name": "something",
126
+ "fields": [
127
+ {"name": "something_boolean", "type": "boolean"}
128
+ ]}
129
+ JSON
130
+
131
+ data = {"something_boolean" => true }
132
+
133
+ Avro::DataFile.open('data.avr', 'w', writer_schema) do |dw|
134
+ while dw.writer.tell < Avro::DataFile::SYNC_INTERVAL
135
+ dw << data
136
+ end
137
+ block_count = dw.block_count
138
+ dw << data
139
+ # ensure we didn't just write another block
140
+ assert_equal(block_count+1, dw.block_count)
141
+ end
142
+ end
121
143
  end
@@ -5,9 +5,9 @@
5
5
  # to you under the Apache License, Version 2.0 (the
6
6
  # "License"); you may not use this file except in compliance
7
7
  # with the License. You may obtain a copy of the License at
8
- #
8
+ #
9
9
  # http://www.apache.org/licenses/LICENSE-2.0
10
- #
10
+ #
11
11
  # Unless required by applicable law or agreed to in writing, software
12
12
  # distributed under the License is distributed on an "AS IS" BASIS,
13
13
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -145,7 +145,7 @@ EOS
145
145
  bytes = []
146
146
  current_byte = reader.read(1)
147
147
  bytes << hexlify(current_byte)
148
- while (current_byte[0] & 0x80) != 0
148
+ while (current_byte.unpack('C').first & 0x80) != 0
149
149
  current_byte = reader.read(1)
150
150
  bytes << hexlify(current_byte)
151
151
  end
@@ -153,7 +153,7 @@ EOS
153
153
  end
154
154
 
155
155
  def hexlify(msg)
156
- msg.split("").collect { |c| c[0].to_s(16).rjust(2, '0') }.join
156
+ msg.unpack("H*")
157
157
  end
158
158
 
159
159
  def test_binary_int_encoding
@@ -232,6 +232,28 @@ EOS
232
232
  end
233
233
  end
234
234
 
235
+ def test_skip_union
236
+ ["hello", -1, 32, nil].each do |value_to_skip|
237
+ value_to_read = 6253
238
+
239
+ schema = Avro::Schema.parse('["int", "string", "null"]')
240
+ writer = StringIO.new
241
+ encoder = Avro::IO::BinaryEncoder.new(writer)
242
+ datum_writer = Avro::IO::DatumWriter.new(schema)
243
+ datum_writer.write(value_to_skip, encoder)
244
+ datum_writer.write(value_to_read, encoder)
245
+
246
+ reader = StringIO.new(writer.string)
247
+ decoder = Avro::IO::BinaryDecoder.new(reader)
248
+ datum_reader = Avro::IO::DatumReader.new(schema)
249
+ datum_reader.skip_data(schema, decoder)
250
+ read_value = datum_reader.read(decoder)
251
+
252
+ assert_equal value_to_read, read_value
253
+ end
254
+ end
255
+
256
+
235
257
  def test_schema_promotion
236
258
  promotable_schemas = ['"int"', '"long"', '"float"', '"double"']
237
259
  incorrect = 0
@@ -310,7 +332,7 @@ EOS
310
332
  count = 10
311
333
  random_data = RandomData.new(schm, seed)
312
334
 
313
-
335
+
314
336
  f = File.open(DATAFILE, 'wb')
315
337
  dw = Avro::DataFile::Writer.new(f, datum_writer(schm), schm)
316
338
  count.times{ dw << random_data.next }
@@ -0,0 +1,144 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+
17
+ require 'avro'
18
+ require 'webrick'
19
+ require 'uri'
20
+ require 'logger'
21
+
22
+ class GenericResponder < Avro::IPC::Responder
23
+ def initialize(proto, msg, datum)
24
+ proto_json = open(proto).read
25
+ super(Avro::Protocol.parse(proto_json))
26
+ @msg = msg
27
+ @datum = datum
28
+ end
29
+
30
+ def call(message, request)
31
+ if message.name == @msg
32
+ STDERR.puts "Message: #{message.name} Datum: #{@datum.inspect}"
33
+ @datum
34
+ end
35
+ end
36
+ end
37
+
38
+ class GenericHandler < WEBrick::HTTPServlet::AbstractServlet
39
+ def do_POST(req, resp)
40
+ call_request = Avro::IPC::FramedReader.new(StringIO.new(req.body)).read_framed_message
41
+ unframed_resp = $responder.respond(call_request)
42
+ writer = Avro::IPC::FramedWriter.new(StringIO.new)
43
+ writer.write_framed_message(unframed_resp)
44
+ resp.body = writer.to_s
45
+ @server.stop
46
+ end
47
+ end
48
+
49
+ def run_server(uri, proto, msg, datum)
50
+ uri = URI.parse(uri)
51
+ $responder = GenericResponder.new(proto, msg, datum)
52
+ server = WEBrick::HTTPServer.new(:BindAddress => uri.host,
53
+ :Port => uri.port,
54
+ :Logger => Logger.new(StringIO.new))
55
+ server.mount '/', GenericHandler
56
+ puts "Port: #{server.config[:Port]}"
57
+ STDOUT.flush
58
+ trap("INT") { server.stop }
59
+ trap("TERM") { server.stop }
60
+ server.start
61
+ end
62
+
63
+ def send_message(uri, proto, msg, datum)
64
+ uri = URI.parse(uri)
65
+ trans = Avro::IPC::HTTPTransceiver.new(uri.host, uri.port)
66
+ proto_json = open(proto).read
67
+ requestor = Avro::IPC::Requestor.new(Avro::Protocol.parse(proto_json),
68
+ trans)
69
+ p requestor.request(msg, datum)
70
+ end
71
+
72
+ def file_or_stdin(f)
73
+ f == "-" ? STDIN : open(f)
74
+ end
75
+
76
+ def main
77
+ if ARGV.size == 0
78
+ puts "Usage: #{$0} [dump|rpcreceive|rpcsend]"
79
+ return 1
80
+ end
81
+
82
+ case ARGV[0]
83
+ when "dump"
84
+ if ARGV.size != 3
85
+ puts "Usage: #{$0} dump input_file"
86
+ return 1
87
+ end
88
+ d = Avro::DataFile.new(file_or_stdin(ARGV[1]), Avro::IO::DatumReader.new)
89
+ d.each{|o| puts o.inspect }
90
+ d.close
91
+ when "rpcreceive"
92
+ usage_str = "Usage: #{$0} rpcreceive uri protocol_file "
93
+ usage_str += "message_name (-data d | -file f)"
94
+
95
+ unless [4, 6].include?(ARGV.size)
96
+ puts usage_str
97
+ return 1
98
+ end
99
+ uri, proto, msg = ARGV[1,3]
100
+ datum = nil
101
+ if ARGV.size > 4
102
+ case ARGV[4]
103
+ when "-file"
104
+ Avro::DataFile.open(ARGV[5]) {|f|
105
+ f.each{|d| datum = d; break }
106
+ }
107
+ when "-data"
108
+ puts "JSON Decoder not yet implemented."
109
+ return 1
110
+ else
111
+ puts usage_str
112
+ return 1
113
+ end
114
+ end
115
+ run_server(uri, proto, msg, datum)
116
+ when "rpcsend"
117
+ usage_str = "Usage: #{$0} rpcsend uri protocol_file "
118
+ usage_str += "message_name (-data d | -file f)"
119
+ unless [4,6].include?(ARGV.size)
120
+ puts usage_str
121
+ return 1
122
+ end
123
+ uri, proto, msg = ARGV[1,3]
124
+ datum = nil
125
+ if ARGV.size > 4
126
+ case ARGV[4]
127
+ when "-file"
128
+ Avro::DataFile.open(ARGV[5]){|f| f.each{|d| datum = d; break } }
129
+ when "-data"
130
+ puts "JSON Decoder not yet implemented"
131
+ return 1
132
+ else
133
+ puts usage_str
134
+ return 1
135
+ end
136
+ end
137
+ send_message(uri, proto, msg, datum)
138
+ end
139
+ return 0
140
+ end
141
+
142
+ if __FILE__ == $0
143
+ exit(main)
144
+ end
metadata CHANGED
@@ -1,13 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: avro
3
3
  version: !ruby/object:Gem::Version
4
- hash: 29
4
+ hash: 11
5
5
  prerelease: false
6
6
  segments:
7
7
  - 1
8
- - 3
9
- - 3
10
- version: 1.3.3
8
+ - 5
9
+ - 4
10
+ version: 1.5.4
11
11
  platform: ruby
12
12
  authors:
13
13
  - Apache Software Foundation
@@ -15,7 +15,7 @@ autorequire:
15
15
  bindir: bin
16
16
  cert_chain: []
17
17
 
18
- date: 2010-08-18 00:00:00 -07:00
18
+ date: 2011-09-07 00:00:00 -07:00
19
19
  default_executable:
20
20
  dependencies:
21
21
  - !ruby/object:Gem::Dependency
@@ -62,12 +62,15 @@ files:
62
62
  - lib/avro/schema.rb
63
63
  - test/random_data.rb
64
64
  - test/sample_ipc_client.rb
65
+ - test/sample_ipc_http_client.rb
66
+ - test/sample_ipc_http_server.rb
65
67
  - test/sample_ipc_server.rb
68
+ - test/test_datafile.rb
66
69
  - test/test_help.rb
67
70
  - test/test_io.rb
68
71
  - test/test_protocol.rb
69
- - test/test_datafile.rb
70
72
  - test/test_socket_transport.rb
73
+ - test/tool.rb
71
74
  has_rdoc: true
72
75
  homepage: http://hadoop.apache.org/avro/
73
76
  licenses: []
@@ -107,8 +110,8 @@ signing_key:
107
110
  specification_version: 3
108
111
  summary: Apache Avro for Ruby
109
112
  test_files:
110
- - test/test_datafile.rb
111
113
  - test/test_help.rb
114
+ - test/test_datafile.rb
115
+ - test/test_socket_transport.rb
112
116
  - test/test_io.rb
113
117
  - test/test_protocol.rb
114
- - test/test_socket_transport.rb