avro 1.3.0 → 1.3.3

data/Manifest

@@ -16,4 +16,3 @@ test/sample_ipc_server.rb
 test/test_help.rb
 test/test_io.rb
 test/test_protocol.rb
-tmp/test.rb.avro

data/Rakefile

@@ -16,7 +16,7 @@
 
 require 'rubygems'
 require 'echoe'
-VERSION = File.open('../../share/VERSION.txt').read
+VERSION = File.open('../../share/VERSION.txt').read.sub('-SNAPSHOT', '.pre1').chomp
 Echoe.new('avro', VERSION) do |p|
   p.author = "Apache Software Foundation"
   p.email = "avro-dev@hadoop.apache.org"

data/avro.gemspec

@@ -2,28 +2,28 @@
 
 Gem::Specification.new do |s|
   s.name = %q{avro}
-  s.version = "1.3.0"
+  s.version = "1.3.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Apache Software Foundation"]
-  s.date = %q{2010-03-01}
+  s.date = %q{2010-08-18}
   s.description = %q{Apache is a data serialization and RPC format}
   s.email = %q{avro-dev@hadoop.apache.org}
   s.extra_rdoc_files = ["CHANGELOG", "lib/avro.rb", "lib/avro/collect_hash.rb", "lib/avro/data_file.rb", "lib/avro/io.rb", "lib/avro/ipc.rb", "lib/avro/protocol.rb", "lib/avro/schema.rb"]
-  s.files = ["CHANGELOG", "Manifest", "Rakefile", "avro.gemspec", "interop/test_interop.rb", "lib/avro.rb", "lib/avro/collect_hash.rb", "lib/avro/data_file.rb", "lib/avro/io.rb", "lib/avro/ipc.rb", "lib/avro/protocol.rb", "lib/avro/schema.rb", "test/random_data.rb", "test/sample_ipc_client.rb", "test/sample_ipc_server.rb", "test/test_help.rb", "test/test_io.rb", "test/test_protocol.rb", "tmp/test.rb.avro"]
+  s.files = ["CHANGELOG", "Manifest", "Rakefile", "avro.gemspec", "interop/test_interop.rb", "lib/avro.rb", "lib/avro/collect_hash.rb", "lib/avro/data_file.rb", "lib/avro/io.rb", "lib/avro/ipc.rb", "lib/avro/protocol.rb", "lib/avro/schema.rb", "test/random_data.rb", "test/sample_ipc_client.rb", "test/sample_ipc_server.rb", "test/test_help.rb", "test/test_io.rb", "test/test_protocol.rb", "test/test_datafile.rb", "test/test_socket_transport.rb"]
   s.homepage = %q{http://hadoop.apache.org/avro/}
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Avro"]
   s.require_paths = ["lib"]
   s.rubyforge_project = %q{avro}
-  s.rubygems_version = %q{1.3.5}
+  s.rubygems_version = %q{1.3.7}
   s.summary = %q{Apache Avro for Ruby}
-  s.test_files = ["test/test_help.rb", "test/test_io.rb", "test/test_protocol.rb"]
+  s.test_files = ["test/test_datafile.rb", "test/test_help.rb", "test/test_io.rb", "test/test_protocol.rb", "test/test_socket_transport.rb"]
 
   if s.respond_to? :specification_version then
     current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
-    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<yajl-ruby>, [">= 0"])
     else
       s.add_dependency(%q<yajl-ruby>, [">= 0"])

data/lib/avro.rb

@@ -17,6 +17,8 @@
 require 'yajl'
 require 'set'
 require 'md5'
+require 'net/http'
+require 'stringio'
 
 module Avro
   VERSION = "FIXME"

data/lib/avro/data_file.rb

@@ -29,6 +29,39 @@ module Avro
 
     class DataFileError < AvroError; end
 
+    def self.open(file_path, mode='r', schema=nil)
+      schema = Avro::Schema.parse(schema) if schema
+      case mode
+      when 'w'
+        unless schema
+          raise DataFileError, "Writing an Avro file requires a schema."
+        end
+        io = open_writer(File.open(file_path, 'wb'), schema)
+      when 'r'
+        io = open_reader(File.open(file_path, 'rb'), schema)
+      else
+        raise DataFileError, "Only modes 'r' and 'w' allowed. You gave #{mode.inspect}."
+      end
+
+      yield io if block_given?
+      io
+    ensure
+      io.close if block_given? && io
+    end
+
+    class << self
+      private
+      def open_writer(file, schema)
+        writer = Avro::IO::DatumWriter.new(schema)
+        Avro::DataFile::Writer.new(file, writer, schema)
+      end
+
+      def open_reader(file, schema)
+        reader = Avro::IO::DatumReader.new(nil, schema)
+        Avro::DataFile::Reader.new(file, reader)
+      end
+    end
+
     class Writer
       def self.generate_sync_marker
         OpenSSL::Random.random_bytes(16)

data/lib/avro/io.rb

@@ -76,12 +76,7 @@ module Avro
         # The float is converted into a 32-bit integer using a method
         # equivalent to Java's floatToIntBits and then encoded in
         # little-endian format.
-
-        bits = (byte! & 0xFF) |
-          ((byte! & 0xff) <<  8) |
-          ((byte! & 0xff) << 16) |
-          ((byte! & 0xff) << 24)
-        [bits].pack('i').unpack('e')[0]
+        @reader.read(4).unpack('e')[0]
       end
 
       def read_double
@@ -89,16 +84,7 @@ module Avro
         # The double is converted into a 64-bit integer using a method
         # equivalent to Java's doubleToLongBits and then encoded in
         # little-endian format.
-
-        bits = (byte! & 0xFF) |
-          ((byte! & 0xff) <<  8) |
-          ((byte! & 0xff) << 16) |
-          ((byte! & 0xff) << 24) |
-          ((byte! & 0xff) << 32) |
-          ((byte! & 0xff) << 40) |
-          ((byte! & 0xff) << 48) |
-          ((byte! & 0xff) << 56)
-        [bits].pack('Q').unpack('d')[0]
+        @reader.read(8).unpack('E')[0]
       end
 
       def read_bytes
@@ -202,11 +188,7 @@ module Avro
       # equivalent to Java's floatToIntBits and then encoded in
       # little-endian format.
       def write_float(datum)
-        bits = [datum].pack('e').unpack('i')[0]
-        @writer.write(((bits      ) & 0xFF).chr)
-        @writer.write(((bits >> 8 ) & 0xFF).chr)
-        @writer.write(((bits >> 16) & 0xFF).chr)
-        @writer.write(((bits >> 24) & 0xFF).chr)
+        @writer.write([datum].pack('e'))
       end
 
       # A double is written as 8 bytes.
@@ -214,15 +196,7 @@ module Avro
       # equivalent to Java's doubleToLongBits and then encoded in
       # little-endian format.
       def write_double(datum)
-        bits = [datum].pack('d').unpack('Q')[0]
-        @writer.write(((bits      ) & 0xFF).chr)
-        @writer.write(((bits >> 8 ) & 0xFF).chr)
-        @writer.write(((bits >> 16) & 0xFF).chr)
-        @writer.write(((bits >> 24) & 0xFF).chr)
-        @writer.write(((bits >> 32) & 0xFF).chr)
-        @writer.write(((bits >> 40) & 0xFF).chr)
-        @writer.write(((bits >> 48) & 0xFF).chr)
-        @writer.write(((bits >> 56) & 0xFF).chr)
+        @writer.write([datum].pack('E'))
       end
 
       # Bytes are encoded as a long followed by that many bytes of data.
@@ -247,7 +221,7 @@ module Avro
     class DatumReader
       def self.check_props(schema_one, schema_two, prop_list)
         prop_list.all? do |prop|
-          schema_one.to_hash[prop] == schema_two.to_hash[prop]
+          schema_one.send(prop) == schema_two.send(prop)
         end
       end
 
@@ -256,33 +230,34 @@ module Avro
         r_type = readers_schema.type
 
         # This conditional is begging for some OO love.
-        if [w_type, r_type].include? 'union'
-          return true
-        elsif Schema::PRIMITIVE_TYPES.include?(w_type) &&
-              Schema::PRIMITIVE_TYPES.include?(r_type) &&
-            w_type == r_type
-          return true
-        elsif (w_type == r_type) && (r_type == 'record') &&
-            check_props(writers_schema, readers_schema, ['fullname'])
-          return true
-        elsif w_type == r_type && r_type == 'error' && check_props(writers_scheam, readers_schema, ['fullname'])
-          return true
-        elsif w_type == r_type && r_type == 'request'
-          return true
-        elsif (w_type == r_type) && (r_type == 'fixed') &&
-            check_props(writers_schema, readers_schema, ['fullname', 'size'])
-          return true
-        elsif (w_type == r_type) && (r_type == 'enum') &&
-            check_props(writers_schema, readers_schema, ['fullname'])
-          return true
-        elsif (w_type == r_type) && (r_type == 'map') &&
-            check_props(writers_schema.values, readers_schema.values, ['type'])
-          return true
-        elsif (w_type == r_type) && (r_type == 'array') &&
-            check_props(writers_schema.items, readers_schema.items, ['type'])
+        if w_type == 'union' || r_type == 'union'
           return true
         end
 
+        if w_type == r_type
+          if Schema::PRIMITIVE_TYPES.include?(w_type) &&
+              Schema::PRIMITIVE_TYPES.include?(r_type)
+            return true
+          end
+
+          case r_type
+          when 'record'
+            return check_props(writers_schema, readers_schema, [:fullname])
+          when 'error'
+            return check_props(writers_scheam, readers_schema, [:fullname])
+          when 'request'
+            return true
+          when 'fixed'
+            return check_props(writers_schema, readers_schema, [:fullname, :size])
+          when 'enum'
+            return check_props(writers_schema, readers_schema, [:fullname])
+          when 'map'
+            return check_props(writers_schema.values, readers_schema.values, [:type])
+          when 'array'
+            return check_props(writers_schema.items, readers_schema.items, [:type])
+          end
+        end
+
         # Handle schema promotion
         if w_type == 'int' && ['long', 'float', 'double'].include?(r_type)
           return true
@@ -424,7 +399,6 @@ module Avro
         if readers_fields_hash.size > read_record.size
           writers_fields_hash = writers_schema.fields_hash
           readers_fields_hash.each do |field_name, field|
-            
             unless writers_fields_hash.has_key? field_name
               if !field.default.nil?
                 field_val = read_default_value(field.type, field.default)
@@ -482,6 +456,77 @@ module Avro
           raise AvroError(fail_msg)
         end
       end
+
+      def skip_data(writers_schema, decoder)
+        case writers_schema.type
+        when 'null'
+          decoder.skip_null
+        when 'boolean'
+          decoder.skip_boolean
+        when 'string'
+          decoder.skip_string
+        when 'int'
+          decoder.skip_int
+        when 'long'
+          decoder.skip_long
+        when 'float'
+          decoder.skip_float
+        when 'double'
+          decoder.skip_double
+        when 'bytes'
+          decoder.skip_bytes
+        when 'fixed'
+          skip_fixed(writers_schema, decoder)
+        when 'enum'
+          skip_enum(writers_schema, decoder)
+        when 'array'
+          skip_array(writers_schema, decoder)
+        when 'map'
+          skip_map(writers_schema, decoder)
+        when 'union'
+          skip_union(writers_schema, decoder)
+        when 'record', 'error', 'request'
+          skip_record(writers_schema, decoder)
+        else
+          raise AvroError, "Unknown schema type: #{schm.type}"
+        end
+      end
+
+      def skip_fixed(writers_schema, decoder)
+        decoder.skip(writers_schema.size)
+      end
+
+      def skip_enum(writers_schema, decoder)
+        decoder.skip_int
+      end
+
+      def skip_array(writers_schema, decoder)
+        skip_blocks(decoder) { skip_data(writers_schema.items, decoder) }
+      end
+
+      def skip_map(writers_schema, decoder)
+        skip_blocks(decoder) {
+          decoder.skip_string
+          skip_data(writers_schema.values, decoder)
+        }
+      end
+
+      def skip_record(writers_schema, decoder)
+        writers_schema.fields.each{|f| skip_data(f.type, decoder) }
+      end
+
+      private
+      def skip_blocks(decoder, &blk)
+        block_count = decoder.read_long
+        while block_count != 0
+          if block_count < 0
+            decoder.skip(decoder.read_long)
+          else
+            block_count.times &blk
+          end
+          block_count = decoder.read_long
+        end
+      end
     end # DatumReader
 
     # DatumWriter for generic ruby objects

data/lib/avro/ipc.rb

@@ -13,7 +13,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-require 'stringio'
 
 module Avro::IPC
 
@@ -162,25 +161,28 @@ module Avro::IPC
 
     def read_handshake_response(decoder)
       handshake_response = HANDSHAKE_REQUESTOR_READER.read(decoder)
-      case match = handshake_response['match']
+      we_have_matching_schema = false
+
+      case handshake_response['match']
       when 'BOTH'
         self.send_protocol = false
-        true
+        we_have_matching_schema = true
       when 'CLIENT'
         raise AvroError.new('Handshake failure. match == CLIENT') if send_protocol
-        self.remote_protocol = handshake_response['serverProtocol']
+        self.remote_protocol = Avro::Protocol.parse(handshake_response['serverProtocol'])
         self.remote_hash = handshake_response['serverHash']
         self.send_protocol = false
-        false
+        we_have_matching_schema = true
       when 'NONE'
         raise AvroError.new('Handshake failure. match == NONE') if send_protocol
-        self.remote_protocol = handshake_response['serverProtocol']
+        self.remote_protocol = Avro::Protocol.parse(handshake_response['serverProtocol'])
         self.remote_hash = handshake_response['serverHash']
         self.send_protocol = true
-        false
       else
         raise AvroError.new("Unexpected match: #{match}")
       end
+
+      return we_have_matching_schema
     end
 
     def read_call_response(message_name, decoder)
@@ -236,11 +238,9 @@ module Avro::IPC
       protocol_cache[local_hash] = local_protocol
     end
 
-    def respond(transport)
-      # Called by a server to deserialize a request, compute and serialize
-      # a response or error. Compare to 'handle()' in Thrift.
-
-      call_request = transport.read_framed_message
+    # Called by a server to deserialize a request, compute and serialize
+    # a response or error. Compare to 'handle()' in Thrift.
+    def respond(call_request)
       buffer_decoder = Avro::IO::BinaryDecoder.new(StringIO.new(call_request))
       buffer_writer = StringIO.new('', 'w+')
       buffer_encoder = Avro::IO::BinaryEncoder.new(buffer_writer)
@@ -248,7 +248,7 @@ module Avro::IPC
       response_metadata = {}
 
       begin
-        remote_protocol = process_handshake(transport, buffer_decoder, buffer_encoder)
+        remote_protocol = process_handshake(buffer_decoder, buffer_encoder)
         # handshake failure
         unless remote_protocol
           return buffer_writer.string
@@ -300,7 +300,7 @@ module Avro::IPC
       buffer_writer.string
     end
 
-    def process_handshake(transport, decoder, encoder)
+    def process_handshake(decoder, encoder)
       handshake_request = HANDSHAKE_RESPONDER_READER.read(decoder)
       handshake_response = {}
 
@@ -308,8 +308,9 @@ module Avro::IPC
       client_hash = handshake_request['clientHash']
       client_protocol = handshake_request['clientProtocol']
       remote_protocol = protocol_cache[client_hash]
+
       if !remote_protocol && client_protocol
-        remote_protocol = protocol.parse(client_protocol)
+        remote_protocol = Avro::Protocol.parse(client_protocol)
         protocol_cache[client_hash] = remote_protocol
       end
 
@@ -422,7 +423,7 @@ module Avro::IPC
     end
 
     def write_buffer_length(n)
-      bytes_sent = sock.write([n].pack('I'))
+      bytes_sent = sock.write([n].pack('N'))
       if bytes_sent == 0
         raise ConnectionClosedException.new("socket sent 0 bytes")
       end
@@ -433,11 +434,102 @@ module Avro::IPC
       if read == '' || read == nil
         raise ConnectionClosedException.new("Socket read 0 bytes.")
       end
-      read.unpack('I')[0]
+      read.unpack('N')[0]
     end
 
     def close
       sock.close
     end
   end
+
+  class ConnectionClosedError < StandardError; end
+
+  class FramedWriter
+    attr_reader :writer
+    def initialize(writer)
+      @writer = writer
+    end
+
+    def write_framed_message(message)
+      message_size = message.size
+      total_bytes_sent = 0
+      while message_size - total_bytes_sent > 0
+        if message_size - total_bytes_sent > BUFFER_SIZE
+          buffer_size = BUFFER_SIZE
+        else
+          buffer_size = message_size - total_bytes_sent
+        end
+        write_buffer(message[total_bytes_sent, buffer_size])
+        total_bytes_sent += buffer_size
+      end
+      write_buffer_size(0)
+    end
+
+    def to_s; writer.string; end
+
+    private
+    def write_buffer(chunk)
+      buffer_size = chunk.size
+      write_buffer_size(buffer_size)
+      writer << chunk
+    end
+
+    def write_buffer_size(n)
+      writer.write([n].pack('N'))
+    end
+  end
+
+  class FramedReader
+    attr_reader :reader
+
+    def initialize(reader)
+      @reader = reader
+    end
+
+    def read_framed_message
+      message = []
+      loop do
+        buffer = ""
+        buffer_size = read_buffer_size
+
+        return message.join if buffer_size == 0
+
+        while buffer.size < buffer_size
+          chunk = reader.read(buffer_size - buffer.size)
+          chunk_error?(chunk)
+          buffer << chunk
+        end
+        message << buffer
+      end
+    end
+
+    private
+    def read_buffer_size
+      header = reader.read(BUFFER_HEADER_LENGTH)
+      chunk_error?(header)
+      header.unpack('N')[0]
+    end
+
+    def chunk_error?(chunk)
+      raise ConnectionClosedError.new("Reader read 0 bytes") if chunk == ''
+    end
+  end
+
+  # Only works for clients. Sigh.
+  class HTTPTransceiver
+    attr_reader :remote_name, :host, :port
+    def initialize(host, port)
+      @host, @port = host, port
+      @remote_name = "#{host}:#{port}"
+    end
+
+    def transceive(message)
+      writer = FramedWriter.new(StringIO.new)
+      writer.write_framed_message(message)
+      resp = Net::HTTP.start(host, port) do |http|
+        http.post('/', writer.to_s, {'Content-Type' => 'avro/binary'})
+      end
+      FramedReader.new(StringIO.new(resp.body)).read_framed_message
+    end
+  end
 end

data/lib/avro/protocol.rb

@@ -57,11 +57,11 @@ module Avro
     end
 
     def to_s
-      Yajl.dump to_hash
+      Yajl.dump to_avro
     end
 
     def ==(other)
-      to_hash == Yajl.load(other.to_s)
+      to_avro == other.to_avro
     end
 
     private
@@ -96,13 +96,14 @@ module Avro
       message_objects
     end
 
-    def to_hash
+    protected
+    def to_avro
       hsh = {'protocol' => name}
       hsh['namespace'] = namespace if namespace
-      hsh['types'] = types.map{|t| Yajl.load(t.to_s) } if types
+      hsh['types'] = types.map{|t| t.to_avro } if types
 
       if messages
-        hsh['messages'] = messages.collect_hash{|k,t| [k, Yajl.load(t.to_s)] }
+        hsh['messages'] = messages.collect_hash{|k,t| [k, t.to_avro] }
       end
 
       hsh
@@ -119,18 +120,22 @@ module Avro
         @errors = parse_errors(errors, names) if errors
       end
 
-      def to_s
-        hsh = {'request' => Yajl.load(request.to_s)}
+      def to_avro
+        hsh = {'request' => request.to_avro}
         if response_from_names
           hsh['response'] = response.fullname
         else
-          hsh['response'] = Yajl.load(response.to_s)
+          hsh['response'] = response.to_avro
         end
 
         if errors
-          hsh['errors'] = Yajl.load(errors.to_s)
+          hsh['errors'] = errors.to_avro
         end
-        Yajl.dump hsh
+        hsh
+      end
+
+      def to_s
+        Yajl.dump to_avro
       end
 
       def parse_request(request, names)

data/lib/avro/schema.rb

@@ -51,7 +51,7 @@ module Avro
             fields = json_obj['fields']
             return RecordSchema.new(name, namespace, fields, names, type)
           else
-            raise SchemaParseError.new("Unknown Named Type: #{type}")
+            raise SchemaParseError.new("Unknown named type: #{type}")
           end
         elsif VALID_TYPES.include?(type)
           case type
@@ -73,7 +73,7 @@ module Avro
       elsif PRIMITIVE_TYPES.include? json_obj
         return PrimitiveSchema.new(json_obj)
       else
-        msg = "Could not make an Avro Schema object from #{json_obj}"
+        msg = "#{json_obj.inspect} is not a schema we know about."
         raise SchemaParseError.new(msg)
       end
     end
@@ -129,12 +129,21 @@ module Avro
       @type.hash
     end
 
-    def to_hash
+    def subparse(json_obj, names=nil)
+      begin
+        Schema.real_parse(json_obj, names)
+      rescue => e
+        raise e if e.is_a? SchemaParseError
+        raise SchemaParseError, "Sub-schema for #{self.class.name} not a valid Avro schema. Bad schema: #{json_obj}"
+      end
+    end
+
+    def to_avro
       {'type' => @type}
     end
 
     def to_s
-      Yajl.dump to_hash
+      Yajl.dump to_avro
     end
 
     class NamedSchema < Schema
@@ -145,7 +154,7 @@ module Avro
         names = Name.add_name(names, self)
       end
 
-      def to_hash
+      def to_avro
         props = {'name' => @name}
         props.merge!('namespace' => @namespace) if @namespace
         super.merge props
@@ -194,8 +203,8 @@ module Avro
         fields.inject({}){|hsh, field| hsh[field.name] = field; hsh }
       end
 
-      def to_hash
-        hsh = super.merge('fields' => @fields.map {|f|Yajl.load(f.to_s)} )
+      def to_avro
+        hsh = super.merge('fields' => @fields.map {|f| f.to_avro } )
         if type == 'request'
           hsh['fields']
         else
@@ -215,20 +224,15 @@ module Avro
           @items = names[items]
           @items_schema_from_names = true
         else
-          begin
-            @items = Schema.real_parse(items, names)
-          rescue => e
-            msg = "Items schema not a valid Avro schema" + e.to_s
-            raise SchemaParseError, msg
-          end
+          @items = subparse(items, names)
         end
       end
 
-      def to_hash
+      def to_avro
         name_or_json = if items_schema_from_names
                          items.fullname
                        else
-                         Yajl.load(items.to_s)
+                         items.to_avro
                        end
         super.merge('items' => name_or_json)
       end
@@ -244,21 +248,17 @@ module Avro
           values_schema = names[values]
           @values_schema_from_names = true
         else
-          begin
-            values_schema = Schema.real_parse(values, names)
-          rescue => e
-            raise SchemaParseError.new('Values schema not a valid Avro schema.' + e.to_s)
-          end
+          values_schema = subparse(values, names)
         end
         @values = values_schema
       end
 
-      def to_hash
+      def to_avro
         to_dump = super
         if values_schema_from_names
           to_dump['values'] = values
         else
-          to_dump['values'] = Yajl.load(values.to_s)
+          to_dump['values'] = values.to_avro
         end
         to_dump
       end
@@ -277,11 +277,7 @@ module Avro
             new_schema = names[schema]
             from_names = true
           else
-            begin
-              new_schema = Schema.real_parse(schema, names)
-            rescue
-              raise SchemaParseError, 'Union item must be a valid Avro schema'
-            end
+            new_schema = subparse(schema, names)
           end
 
           ns_type = new_schema.type
@@ -299,7 +295,7 @@ module Avro
         end
       end
 
-      def to_s
+      def to_avro
         # FIXME(jmhodges) this from_name pattern is really weird and
         # seems code-smelly.
         to_dump = []
@@ -307,10 +303,10 @@ module Avro
           if schema_from_names_indices.include?(i)
             to_dump << schema.fullname
           else
-            to_dump << Yajl.load(schema.to_s)
+            to_dump << schema.to_avro
           end
         end
-        Yajl.dump(to_dump)
+        to_dump
       end
     end
 
@@ -325,7 +321,7 @@ module Avro
         @symbols = symbols
       end
 
-      def to_hash
+      def to_avro
         super.merge('symbols' => symbols)
       end
     end
@@ -340,8 +336,9 @@ module Avro
         super(type)
       end
 
-      def to_s
-        to_hash.size == 1 ? type.inspect : Yajl.dump(to_hash)
+      def to_avro
+        hsh = super
+        hsh.size == 1 ? type : hsh
       end
     end
 
@@ -356,12 +353,12 @@ module Avro
         @size = size
       end
 
-      def to_hash
+      def to_avro
         super.merge('size' => @size)
       end
     end
 
-    class Field
+    class Field < Schema
       attr_reader :type, :name, :default, :order, :type_from_names
       def initialize(type, name, default=nil, order=nil, names=nil)
         @type_from_names = false
@@ -369,7 +366,7 @@ module Avro
           type_schema = names[type]
           @type_from_names = true
         else
-          type_schema = Schema.real_parse(type, names)
+          type_schema = subparse(type, names)
         end
         @type = type_schema
         @name = name
@@ -377,8 +374,8 @@ module Avro
         @order = order
       end
 
-      def to_hash
-        sigh_type = type_from_names ? type.fullname : Yajl.load(type.to_s)
+      def to_avro
+        sigh_type = type_from_names ? type.fullname : type.to_avro
         hsh = {
           'name' => name,
           'type' => sigh_type
@@ -387,10 +384,6 @@ module Avro
         hsh['order'] = order if order
         hsh
       end
-
-      def to_s
-        Yajl.dump(to_hash)
-      end
     end
   end
 

data/test/sample_ipc_client.rb

@@ -66,8 +66,7 @@ if $0 == __FILE__
     'body' => ARGV[2]
   }
 
-  num_messages = ARGV[3].to_i
-  num_message = 1 if num_messages == 0
+  num_messages = (ARGV[3] || 1).to_i
 
   # build the parameters for the request
   params = {'message' => message}
@@ -83,4 +82,4 @@ if $0 == __FILE__
   requestor = make_requestor('localhost', 9090, MAIL_PROTOCOL)
   result = requestor.request('replay', {})
   puts("Replay Result: " + result)
-end
+end

data/test/sample_ipc_server.rb

@@ -81,11 +81,12 @@ class MailHandler < RequestHandler
   def handle(request)
     responder = MailResponder.new()
     transport = Avro::IPC::SocketTransport.new(request)
-    transport.write_framed_message(responder.respond(transport))
+    str = transport.read_framed_message
+    transport.write_framed_message(responder.respond(str))
   end
 end
 
 if $0 == __FILE__
   handler = MailHandler.new('localhost', 9090)
   handler.run
-end
+end

data/test/test_datafile.rb

@@ -0,0 +1,121 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'test_help'
+
+class TestDataFile < Test::Unit::TestCase
+  HERE = File.expand_path File.dirname(__FILE__)
+  def setup
+    if File.exists?(HERE + '/data.avr')
+      File.unlink(HERE + '/data.avr')
+    end
+  end
+
+  def teardown
+    if File.exists?(HERE + '/data.avr')
+      File.unlink(HERE + '/data.avr')
+    end
+  end
+
+  def test_differing_schemas_with_primitives
+    writer_schema = <<-JSON
+{ "type": "record",
+  "name": "User",
+  "fields" : [
+    {"name": "username", "type": "string"},
+    {"name": "age", "type": "int"},
+    {"name": "verified", "type": "boolean", "default": "false"}
+  ]}
+JSON
+
+    data = [{"username" => "john", "age" => 25, "verified" => true},
+            {"username" => "ryan", "age" => 23, "verified" => false}]
+
+    Avro::DataFile.open('data.avr', 'w', writer_schema) do |dw|
+      data.each{|h| dw << h }
+    end
+
+    # extract the username only from the avro serialized file
+    reader_schema = <<-JSON
+{ "type": "record",
+  "name": "User",
+  "fields" : [
+    {"name": "username", "type": "string"}
+ ]}
+JSON
+
+    Avro::DataFile.open('data.avr', 'r', reader_schema) do |dr|
+      dr.each_with_index do |record, i|
+        assert_equal data[i]['username'], record['username']
+      end
+    end
+  end
+
+  def test_differing_schemas_with_complex_objects
+    writer_schema = <<-JSON
+{ "type": "record",
+  "name": "something",
+  "fields": [
+    {"name": "something_fixed", "type": {"name": "inner_fixed",
+                                         "type": "fixed", "size": 3}},
+    {"name": "something_enum", "type": {"name": "inner_enum",
+                                        "type": "enum",
+                                        "symbols": ["hello", "goodbye"]}},
+    {"name": "something_array", "type": {"type": "array", "items": "int"}},
+    {"name": "something_map", "type": {"type": "map", "values": "int"}},
+    {"name": "something_record", "type": {"name": "inner_record",
+                                          "type": "record",
+                                          "fields": [
+                                            {"name": "inner", "type": "int"}
+                                          ]}},
+    {"name": "username", "type": "string"}
+]}
+JSON
+
+    data = [{"username" => "john",
+              "something_fixed" => "foo",
+              "something_enum" => "hello",
+              "something_array" => [1,2,3],
+              "something_map" => {"a" => 1, "b" => 2},
+              "something_record" => {"inner" => 2},
+              "something_error" => {"code" => 403}
+            },
+            {"username" => "ryan",
+              "something_fixed" => "bar",
+              "something_enum" => "goodbye",
+              "something_array" => [1,2,3],
+              "something_map" => {"a" => 2, "b" => 6},
+              "something_record" => {"inner" => 1},
+              "something_error" => {"code" => 401}
+            }]
+
+    Avro::DataFile.open('data.avr', 'w', writer_schema) do |dw|
+      data.each{|d| dw << d }
+    end
+
+    %w[fixed enum record error array map union].each do |s|
+      reader = Yajl.load(writer_schema)
+      reader['fields'] = reader['fields'].reject{|f| f['type']['type'] == s}
+      Avro::DataFile.open('data.avr', 'r', Yajl.dump(reader)) do |dr|
+        dr.each_with_index do |obj, i|
+          reader['fields'].each do |field|
+            assert_equal data[i][field['name']], obj[field['name']]
+          end
+        end
+      end
+    end
+  end
+end

data/test/test_socket_transport.rb

@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'test_help'
+
+class TestSocketTransport < Test::Unit::TestCase
+  def test_buffer_writing
+    io = StringIO.new
+    st = Avro::IPC::SocketTransport.new(io)
+    buffer_length = "\000\000\000\006"  # 6 in big-endian
+    message = 'abcdef'
+    null_ending = "\000\000\000\000" # 0 in big-endian
+    full = buffer_length + message + null_ending
+    st.write_framed_message('abcdef')
+    assert_equal full, io.string
+  end
+
+  def test_buffer_reading
+    buffer_length = "\000\000\000\005" # 5 in big-endian
+    message = "hello"
+    null_ending = "\000\000\000\000" # 0 in big-endian
+    full = buffer_length + message + null_ending
+    io = StringIO.new(full)
+    st = Avro::IPC::SocketTransport.new(io)
+    assert_equal 'hello', st.read_framed_message
+  end
+end

metadata

@@ -1,7 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: avro
 version: !ruby/object:Gem::Version 
-  version: 1.3.0
+  hash: 29
+  prerelease: false
+  segments: 
+  - 1
+  - 3
+  - 3
+  version: 1.3.3
 platform: ruby
 authors: 
 - Apache Software Foundation
@@ -9,19 +15,23 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-03-01 00:00:00 -08:00
+date: 2010-08-18 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: yajl-ruby
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
         version: "0"
-    version: 
+  type: :runtime
+  version_requirements: *id001
 description: Apache is a data serialization and RPC format
 email: avro-dev@hadoop.apache.org
 executables: []
@@ -56,7 +66,8 @@ files:
 - test/test_help.rb
 - test/test_io.rb
 - test/test_protocol.rb
-- tmp/test.rb.avro
+- test/test_datafile.rb
+- test/test_socket_transport.rb
 has_rdoc: true
 homepage: http://hadoop.apache.org/avro/
 licenses: []
@@ -70,25 +81,34 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 11
+      segments: 
+      - 1
+      - 2
       version: "1.2"
-  version: 
 requirements: []
 
 rubyforge_project: avro
-rubygems_version: 1.3.5
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: Apache Avro for Ruby
 test_files: 
+- test/test_datafile.rb
 - test/test_help.rb
 - test/test_io.rb
 - test/test_protocol.rb
+- test/test_socket_transport.rb