avro 1.3.3 → 1.5.4

data/Manifest

@@ -12,7 +12,12 @@ lib/avro/protocol.rb
 lib/avro/schema.rb
 test/random_data.rb
 test/sample_ipc_client.rb
+test/sample_ipc_http_client.rb
+test/sample_ipc_http_server.rb
 test/sample_ipc_server.rb
+test/test_datafile.rb
 test/test_help.rb
 test/test_io.rb
 test/test_protocol.rb
+test/test_socket_transport.rb
+test/tool.rb

data/Rakefile

@@ -53,7 +53,7 @@ SHARE = HERE + '/../../share'
 SCHEMAS = SHARE + '/test/schemas'
 BUILD = HERE + '/../../build'
 
-task :dist => [:manifest, :gem] do
+task :dist => [:gem] do
   mkdir_p "../../dist/ruby"
   cp "pkg/avro-#{VERSION}.gem", "../../dist/ruby"
 end

data/avro.gemspec

@@ -2,22 +2,22 @@
 
 Gem::Specification.new do |s|
   s.name = %q{avro}
-  s.version = "1.3.3"
+  s.version = "1.5.4"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Apache Software Foundation"]
-  s.date = %q{2010-08-18}
+  s.date = %q{2011-09-07}
   s.description = %q{Apache is a data serialization and RPC format}
   s.email = %q{avro-dev@hadoop.apache.org}
   s.extra_rdoc_files = ["CHANGELOG", "lib/avro.rb", "lib/avro/collect_hash.rb", "lib/avro/data_file.rb", "lib/avro/io.rb", "lib/avro/ipc.rb", "lib/avro/protocol.rb", "lib/avro/schema.rb"]
-  s.files = ["CHANGELOG", "Manifest", "Rakefile", "avro.gemspec", "interop/test_interop.rb", "lib/avro.rb", "lib/avro/collect_hash.rb", "lib/avro/data_file.rb", "lib/avro/io.rb", "lib/avro/ipc.rb", "lib/avro/protocol.rb", "lib/avro/schema.rb", "test/random_data.rb", "test/sample_ipc_client.rb", "test/sample_ipc_server.rb", "test/test_help.rb", "test/test_io.rb", "test/test_protocol.rb", "test/test_datafile.rb", "test/test_socket_transport.rb"]
+  s.files = ["CHANGELOG", "Manifest", "Rakefile", "avro.gemspec", "interop/test_interop.rb", "lib/avro.rb", "lib/avro/collect_hash.rb", "lib/avro/data_file.rb", "lib/avro/io.rb", "lib/avro/ipc.rb", "lib/avro/protocol.rb", "lib/avro/schema.rb", "test/random_data.rb", "test/sample_ipc_client.rb", "test/sample_ipc_http_client.rb", "test/sample_ipc_http_server.rb", "test/sample_ipc_server.rb", "test/test_datafile.rb", "test/test_help.rb", "test/test_io.rb", "test/test_protocol.rb", "test/test_socket_transport.rb", "test/tool.rb"]
   s.homepage = %q{http://hadoop.apache.org/avro/}
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Avro"]
   s.require_paths = ["lib"]
   s.rubyforge_project = %q{avro}
   s.rubygems_version = %q{1.3.7}
   s.summary = %q{Apache Avro for Ruby}
-  s.test_files = ["test/test_datafile.rb", "test/test_help.rb", "test/test_io.rb", "test/test_protocol.rb", "test/test_socket_transport.rb"]
+  s.test_files = ["test/test_help.rb", "test/test_datafile.rb", "test/test_socket_transport.rb", "test/test_io.rb", "test/test_protocol.rb"]
 
   if s.respond_to? :specification_version then
     current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION

data/lib/avro.rb

@@ -16,7 +16,7 @@
 
 require 'yajl'
 require 'set'
-require 'md5'
+require 'digest/md5'
 require 'net/http'
 require 'stringio'
 

data/lib/avro/data_file.rb

@@ -173,6 +173,7 @@ module Avro
 
           # reset buffer
           buffer_writer.truncate(0)
+          buffer_writer.rewind
           self.block_count = 0
         end
       end

data/lib/avro/io.rb

@@ -43,9 +43,9 @@ module Avro
       end
 
       def byte!
-        @reader.read(1)[0]
+        @reader.read(1).unpack('C').first
       end
-
+      
       def read_null
         # null is written as zero byte's
         nil
@@ -500,6 +500,11 @@ module Avro
         decoder.skip_int
       end
 
+      def skip_union(writers_schema, decoder)
+        index = decoder.read_long
+        skip_data(writers_schema.schemas[index], decoder)
+      end
+
       def skip_array(writers_schema, decoder)
         skip_blocks(decoder) { skip_data(writers_schema.items, decoder) }
       end

data/lib/avro/ipc.rb

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+require "net/http"
+
 module Avro::IPC
 
   class AvroRemoteError < Avro::AvroError; end
@@ -397,7 +399,7 @@ module Avro::IPC
       message_length = message.size
       total_bytes_sent = 0
       while message_length - total_bytes_sent > 0
-        if message_length - total_bytes_sent > BUFFER_SIZE:
+        if message_length - total_bytes_sent > BUFFER_SIZE
           buffer_length = BUFFER_SIZE
         else
           buffer_length = message_length - total_bytes_sent
@@ -521,14 +523,13 @@ module Avro::IPC
     def initialize(host, port)
       @host, @port = host, port
       @remote_name = "#{host}:#{port}"
+      @conn = Net::HTTP.start host, port
     end
 
     def transceive(message)
       writer = FramedWriter.new(StringIO.new)
       writer.write_framed_message(message)
-      resp = Net::HTTP.start(host, port) do |http|
-        http.post('/', writer.to_s, {'Content-Type' => 'avro/binary'})
-      end
+      resp = @conn.post('/', writer.to_s, {'Content-Type' => 'avro/binary'})
       FramedReader.new(StringIO.new(resp.body)).read_framed_message
     end
   end

data/lib/avro/schema.rb

@@ -102,7 +102,7 @@ module Avro
       when 'array'
         datum.is_a?(Array) &&
           datum.all?{|d| validate(expected_schema.items, d) }
-      when 'map':
+      when 'map'
           datum.keys.all?{|k| k.is_a? String } &&
           datum.values.all?{|v| validate(expected_schema.values, v) }
       when 'union'

data/test/sample_ipc_http_client.rb

@@ -0,0 +1,84 @@
+#!/usr/bin/env ruby
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'socket'
+require 'avro'
+
+MAIL_PROTOCOL_JSON = <<-JSON
+{"namespace": "example.proto",
+ "protocol": "Mail",
+
+ "types": [
+     {"name": "Message", "type": "record",
+      "fields": [
+          {"name": "to",   "type": "string"},
+          {"name": "from", "type": "string"},
+          {"name": "body", "type": "string"}
+      ]
+     }
+ ],
+
+ "messages": {
+     "send": {
+         "request": [{"name": "message", "type": "Message"}],
+         "response": "string"
+     },
+     "replay": {
+         "request": [],
+         "response": "string"
+     }
+ }
+}
+JSON
+
+MAIL_PROTOCOL = Avro::Protocol.parse(MAIL_PROTOCOL_JSON)
+
+def make_requestor(server_address, port, protocol)
+  transport = Avro::IPC::HTTPTransceiver.new(server_address, port)
+  Avro::IPC::Requestor.new(protocol, transport)
+end
+
+if $0 == __FILE__
+  if ![3, 4].include?(ARGV.length)
+    raise "Usage: <to> <from> <body> [<count>]"
+  end
+
+  # client code - attach to the server and send a message
+  # fill in the Message record
+  message = {
+    'to'   => ARGV[0],
+    'from' => ARGV[1],
+    'body' => ARGV[2]
+  }
+
+  num_messages = (ARGV[3] || 1).to_i
+
+  # build the parameters for the request
+  params = {'message' => message}
+  # send the requests and print the result
+
+  num_messages.times do
+    requestor = make_requestor('localhost', 9090, MAIL_PROTOCOL)
+    result = requestor.request('send', params)
+    puts("Result: " + result)
+  end
+
+  # try out a replay message
+  requestor = make_requestor('localhost', 9090, MAIL_PROTOCOL)
+  result = requestor.request('replay', {})
+  puts("Replay Result: " + result)
+end

data/test/sample_ipc_http_server.rb

@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+require 'avro'
+require 'webrick'
+
+MAIL_PROTOCOL_JSON = <<-JSON
+{"namespace": "example.proto",
+ "protocol": "Mail",
+
+ "types": [
+     {"name": "Message", "type": "record",
+      "fields": [
+          {"name": "to",   "type": "string"},
+          {"name": "from", "type": "string"},
+          {"name": "body", "type": "string"}
+      ]
+     }
+ ],
+
+ "messages": {
+     "send": {
+         "request": [{"name": "message", "type": "Message"}],
+         "response": "string"
+     },
+     "replay": {
+         "request": [],
+         "response": "string"
+     }
+ }
+}
+JSON
+
+MAIL_PROTOCOL = Avro::Protocol.parse(MAIL_PROTOCOL_JSON)
+
+class MailResponder < Avro::IPC::Responder
+  def initialize
+    super(MAIL_PROTOCOL)
+  end
+
+  def call(message, request)
+    if message.name == 'send'
+      request_content = request['message']
+      "Sent message to #{request_content['to']} from #{request_content['from']} with body #{request_content['body']}"
+    elsif message.name == 'replay'
+      'replay'
+    end
+  end
+end
+
+class MailHandler < WEBrick::HTTPServlet::AbstractServlet
+  def do_POST(req, resp)
+    responder = MailResponder.new
+    call_request = Avro::IPC::FramedReader.new(StringIO.new(req.body)).read_framed_message
+    unframed_resp = responder.respond(call_request)
+    writer = Avro::IPC::FramedWriter.new(StringIO.new)
+    writer.write_framed_message(unframed_resp)
+    resp.body = writer.to_s
+  end
+end
+
+if $0 == __FILE__
+  server = WEBrick::HTTPServer.new(:Host => 'localhost', :Port => 9090)
+  server.mount '/', MailHandler
+  trap("INT") { server.shutdown }
+  server.start
+end

data/test/test_datafile.rb

@@ -118,4 +118,26 @@ JSON
       end
     end
   end
+
+  def test_data_writer_handles_sync_interval
+    writer_schema = <<-JSON
+{ "type": "record",
+  "name": "something",
+  "fields": [
+    {"name": "something_boolean", "type": "boolean"}
+]}
+JSON
+
+    data = {"something_boolean" => true }
+
+    Avro::DataFile.open('data.avr', 'w', writer_schema) do |dw|
+      while dw.writer.tell < Avro::DataFile::SYNC_INTERVAL
+        dw << data
+      end
+      block_count = dw.block_count
+      dw << data
+      # ensure we didn't just write another block
+      assert_equal(block_count+1, dw.block_count)
+    end
+  end
 end

data/test/test_io.rb

@@ -5,9 +5,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 # http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -145,7 +145,7 @@ EOS
     bytes = []
     current_byte = reader.read(1)
     bytes << hexlify(current_byte)
-    while (current_byte[0] & 0x80) != 0
+    while (current_byte.unpack('C').first & 0x80) != 0
       current_byte = reader.read(1)
       bytes << hexlify(current_byte)
     end
@@ -153,7 +153,7 @@ EOS
   end
 
   def hexlify(msg)
-    msg.split("").collect { |c| c[0].to_s(16).rjust(2, '0') }.join
+    msg.unpack("H*")
   end
 
   def test_binary_int_encoding
@@ -232,6 +232,28 @@ EOS
     end
   end
 
+  def test_skip_union
+    ["hello", -1, 32, nil].each do |value_to_skip|
+      value_to_read = 6253
+
+      schema = Avro::Schema.parse('["int", "string", "null"]')
+      writer = StringIO.new
+      encoder = Avro::IO::BinaryEncoder.new(writer)
+      datum_writer = Avro::IO::DatumWriter.new(schema)
+      datum_writer.write(value_to_skip, encoder)
+      datum_writer.write(value_to_read, encoder)
+
+      reader = StringIO.new(writer.string)
+      decoder = Avro::IO::BinaryDecoder.new(reader)
+      datum_reader = Avro::IO::DatumReader.new(schema)
+      datum_reader.skip_data(schema, decoder)
+      read_value = datum_reader.read(decoder)
+
+      assert_equal value_to_read, read_value
+    end
+  end
+
+
   def test_schema_promotion
     promotable_schemas = ['"int"', '"long"', '"float"', '"double"']
     incorrect = 0
@@ -310,7 +332,7 @@ EOS
     count = 10
     random_data = RandomData.new(schm, seed)
 
-   
+
     f = File.open(DATAFILE, 'wb')
     dw = Avro::DataFile::Writer.new(f, datum_writer(schm), schm)
     count.times{ dw << random_data.next }

data/test/tool.rb

@@ -0,0 +1,144 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'avro'
+require 'webrick'
+require 'uri'
+require 'logger'
+
+class GenericResponder < Avro::IPC::Responder
+  def initialize(proto, msg, datum)
+    proto_json = open(proto).read
+    super(Avro::Protocol.parse(proto_json))
+    @msg = msg
+    @datum = datum
+  end
+
+  def call(message, request)
+    if message.name == @msg
+      STDERR.puts "Message: #{message.name} Datum: #{@datum.inspect}"
+      @datum
+    end
+  end
+end
+
+class GenericHandler < WEBrick::HTTPServlet::AbstractServlet
+  def do_POST(req, resp)
+    call_request = Avro::IPC::FramedReader.new(StringIO.new(req.body)).read_framed_message
+    unframed_resp = $responder.respond(call_request)
+    writer = Avro::IPC::FramedWriter.new(StringIO.new)
+    writer.write_framed_message(unframed_resp)
+    resp.body = writer.to_s
+    @server.stop
+  end
+end
+
+def run_server(uri, proto, msg, datum)
+  uri = URI.parse(uri)
+  $responder = GenericResponder.new(proto, msg, datum)
+  server = WEBrick::HTTPServer.new(:BindAddress => uri.host,
+                                   :Port => uri.port,
+                                   :Logger => Logger.new(StringIO.new))
+  server.mount '/', GenericHandler
+  puts "Port: #{server.config[:Port]}"
+  STDOUT.flush
+  trap("INT") { server.stop }
+  trap("TERM") { server.stop }
+  server.start
+end
+
+def send_message(uri, proto, msg, datum)
+  uri = URI.parse(uri)
+  trans = Avro::IPC::HTTPTransceiver.new(uri.host, uri.port)
+  proto_json = open(proto).read
+  requestor = Avro::IPC::Requestor.new(Avro::Protocol.parse(proto_json),
+                                       trans)
+  p requestor.request(msg, datum)
+end
+
+def file_or_stdin(f)
+  f == "-" ? STDIN : open(f)
+end
+
+def main
+  if ARGV.size == 0
+    puts "Usage: #{$0} [dump|rpcreceive|rpcsend]"
+    return 1
+  end
+
+  case ARGV[0]
+  when "dump"
+    if ARGV.size != 3
+      puts "Usage: #{$0} dump input_file"
+      return 1
+    end
+    d = Avro::DataFile.new(file_or_stdin(ARGV[1]), Avro::IO::DatumReader.new)
+    d.each{|o| puts o.inspect }
+    d.close
+  when "rpcreceive"
+    usage_str = "Usage: #{$0} rpcreceive uri protocol_file "
+    usage_str += "message_name (-data d | -file f)"
+
+    unless [4, 6].include?(ARGV.size)
+      puts usage_str
+      return 1
+    end
+    uri, proto, msg = ARGV[1,3]
+    datum = nil
+    if ARGV.size > 4
+      case ARGV[4]
+      when "-file"
+        Avro::DataFile.open(ARGV[5]) {|f|
+          f.each{|d| datum = d; break }
+        }
+      when "-data"
+        puts "JSON Decoder not yet implemented."
+        return 1
+      else
+        puts usage_str
+        return 1
+      end
+    end
+    run_server(uri, proto, msg, datum)
+  when "rpcsend"
+    usage_str = "Usage: #{$0} rpcsend uri protocol_file "
+    usage_str += "message_name (-data d | -file f)"
+    unless [4,6].include?(ARGV.size)
+      puts usage_str
+      return 1
+    end
+    uri, proto, msg = ARGV[1,3]
+    datum = nil
+    if ARGV.size > 4
+      case ARGV[4]
+      when "-file"
+        Avro::DataFile.open(ARGV[5]){|f| f.each{|d| datum = d; break } }
+      when "-data"
+        puts "JSON Decoder not yet implemented"
+        return 1
+      else
+        puts usage_str
+        return 1
+      end
+    end
+    send_message(uri, proto, msg, datum)
+  end
+  return 0
+end
+
+if __FILE__ == $0
+  exit(main)
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: avro
 version: !ruby/object:Gem::Version 
-  hash: 29
+  hash: 11
   prerelease: false
   segments: 
   - 1
-  - 3
-  - 3
-  version: 1.3.3
+  - 5
+  - 4
+  version: 1.5.4
 platform: ruby
 authors: 
 - Apache Software Foundation
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-08-18 00:00:00 -07:00
+date: 2011-09-07 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -62,12 +62,15 @@ files:
 - lib/avro/schema.rb
 - test/random_data.rb
 - test/sample_ipc_client.rb
+- test/sample_ipc_http_client.rb
+- test/sample_ipc_http_server.rb
 - test/sample_ipc_server.rb
+- test/test_datafile.rb
 - test/test_help.rb
 - test/test_io.rb
 - test/test_protocol.rb
-- test/test_datafile.rb
 - test/test_socket_transport.rb
+- test/tool.rb
 has_rdoc: true
 homepage: http://hadoop.apache.org/avro/
 licenses: []
@@ -107,8 +110,8 @@ signing_key:
 specification_version: 3
 summary: Apache Avro for Ruby
 test_files: 
-- test/test_datafile.rb
 - test/test_help.rb
+- test/test_datafile.rb
+- test/test_socket_transport.rb
 - test/test_io.rb
 - test/test_protocol.rb
-- test/test_socket_transport.rb