avro 1.7.4 → 1.7.5

data/Rakefile

@@ -45,6 +45,10 @@ task :generate_interop do
   ensure
     writer.close
   end
+
+  Avro::DataFile.open(BUILD + '/interop/data/ruby_deflate.avro', 'w', schema.to_s, :deflate) do |writer|
+    20.times { writer << r.next }
+  end
 end
 
 

data/avro.gemspec

@@ -2,22 +2,22 @@
 
 Gem::Specification.new do |s|
   s.name = "avro"
-  s.version = "1.7.4"
+  s.version = "1.7.5"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Apache Software Foundation"]
-  s.date = "2013-02-21"
+  s.date = "2013-08-19"
   s.description = "Avro is a data serialization and RPC format"
   s.email = "avro-dev@hadoop.apache.org"
   s.extra_rdoc_files = ["CHANGELOG", "lib/avro.rb", "lib/avro/collect_hash.rb", "lib/avro/data_file.rb", "lib/avro/io.rb", "lib/avro/ipc.rb", "lib/avro/protocol.rb", "lib/avro/schema.rb"]
-  s.files = ["CHANGELOG", "Manifest", "Rakefile", "avro.gemspec", "interop/test_interop.rb", "lib/avro.rb", "lib/avro/collect_hash.rb", "lib/avro/data_file.rb", "lib/avro/io.rb", "lib/avro/ipc.rb", "lib/avro/protocol.rb", "lib/avro/schema.rb", "test/random_data.rb", "test/sample_ipc_client.rb", "test/sample_ipc_http_client.rb", "test/sample_ipc_http_server.rb", "test/sample_ipc_server.rb", "test/test_datafile.rb", "test/test_help.rb", "test/test_io.rb", "test/test_protocol.rb", "test/test_socket_transport.rb", "test/tool.rb"]
+  s.files = ["CHANGELOG", "Manifest", "Rakefile", "avro.gemspec", "interop/test_interop.rb", "lib/avro.rb", "lib/avro/collect_hash.rb", "lib/avro/data_file.rb", "lib/avro/io.rb", "lib/avro/ipc.rb", "lib/avro/protocol.rb", "lib/avro/schema.rb", "test/random_data.rb", "test/sample_ipc_client.rb", "test/sample_ipc_http_client.rb", "test/sample_ipc_http_server.rb", "test/sample_ipc_server.rb", "test/test_datafile.rb", "test/test_help.rb", "test/test_io.rb", "test/test_protocol.rb", "test/test_socket_transport.rb", "test/tool.rb", "test/test_schema.rb"]
   s.homepage = "http://hadoop.apache.org/avro/"
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Avro"]
   s.require_paths = ["lib"]
   s.rubyforge_project = "avro"
   s.rubygems_version = "1.8.15"
   s.summary = "Apache Avro for Ruby"
-  s.test_files = ["test/test_datafile.rb", "test/test_help.rb", "test/test_protocol.rb", "test/test_socket_transport.rb", "test/test_io.rb"]
+  s.test_files = ["test/test_datafile.rb", "test/test_help.rb", "test/test_protocol.rb", "test/test_socket_transport.rb", "test/test_schema.rb", "test/test_io.rb"]
 
   if s.respond_to? :specification_version then
     s.specification_version = 3

data/interop/test_interop.rb

@@ -23,7 +23,7 @@ class TestInterop < Test::Unit::TestCase
   HERE = File.expand_path(File.dirname(__FILE__))
   SHARE = HERE + '/../../../share'
   SCHEMAS = SHARE + '/test/schemas'
-  Dir[HERE + '/../../../build/interop/data/*'].each do |fn|  
+  Dir[HERE + '/../../../build/interop/data/*'].each do |fn|
     define_method("test_read_#{File.basename(fn, 'avro')}") do
       projection = Avro::Schema.parse(File.read(SCHEMAS+'/interop.avsc'))
 

data/lib/avro.rb

@@ -19,6 +19,7 @@ require 'set'
 require 'digest/md5'
 require 'net/http'
 require 'stringio'
+require 'zlib'
 
 module Avro
   VERSION = "FIXME"

data/lib/avro/data_file.rb

@@ -24,19 +24,18 @@ module Avro
     SYNC_SIZE = 16
     SYNC_INTERVAL = 1000 * SYNC_SIZE
     META_SCHEMA = Schema.parse('{"type": "map", "values": "bytes"}')
-    VALID_CODECS = ['null']
     VALID_ENCODINGS = ['binary'] # not used yet
 
     class DataFileError < AvroError; end
 
-    def self.open(file_path, mode='r', schema=nil)
+    def self.open(file_path, mode='r', schema=nil, codec=nil)
       schema = Avro::Schema.parse(schema) if schema
       case mode
       when 'w'
         unless schema
           raise DataFileError, "Writing an Avro file requires a schema."
         end
-        io = open_writer(File.open(file_path, 'wb'), schema)
+        io = open_writer(File.open(file_path, 'wb'), schema, codec)
       when 'r'
         io = open_reader(File.open(file_path, 'rb'), schema)
       else
@@ -49,11 +48,34 @@ module Avro
       io.close if block_given? && io
     end
 
+    def self.codecs
+      @codecs
+    end
+
+    def self.register_codec(codec)
+      @codecs ||= {}
+      codec = codec.new if !codec.respond_to?(:codec_name) && codec.is_a?(Class)
+      @codecs[codec.codec_name.to_s] = codec
+    end
+
+    def self.get_codec(codec)
+      codec ||= 'null'
+      if codec.respond_to?(:compress) && codec.respond_to?(:decompress)
+        codec # it's a codec instance
+      elsif codec.is_a?(Class)
+        codec.new # it's a codec class
+      elsif @codecs.include?(codec.to_s)
+        @codecs[codec.to_s] # it's a string or symbol (codec name)
+      else
+        raise DataFileError, "Unknown codec: #{codec.inspect}"
+      end
+    end
+
     class << self
       private
-      def open_writer(file, schema)
+      def open_writer(file, schema, codec=nil)
         writer = Avro::IO::DatumWriter.new(schema)
-        Avro::DataFile::Writer.new(file, writer, schema)
+        Avro::DataFile::Writer.new(file, writer, schema, codec)
       end
 
       def open_reader(file, schema)
@@ -67,10 +89,10 @@ module Avro
         OpenSSL::Random.random_bytes(16)
       end
 
-      attr_reader :writer, :encoder, :datum_writer, :buffer_writer, :buffer_encoder, :sync_marker, :meta
+      attr_reader :writer, :encoder, :datum_writer, :buffer_writer, :buffer_encoder, :sync_marker, :meta, :codec
       attr_accessor :block_count
 
-      def initialize(writer, datum_writer, writers_schema=nil)
+      def initialize(writer, datum_writer, writers_schema=nil, codec=nil)
         # If writers_schema is not present, presume we're appending
         @writer = writer
         @encoder = IO::BinaryEncoder.new(@writer)
@@ -83,7 +105,8 @@ module Avro
 
         if writers_schema
           @sync_marker = Writer.generate_sync_marker
-          meta['avro.codec'] = 'null'
+          @codec = DataFile.get_codec(codec)
+          meta['avro.codec'] = @codec.codec_name.to_s
           meta['avro.schema'] = writers_schema.to_s
           datum_writer.writers_schema = writers_schema
           write_header
@@ -95,6 +118,7 @@ module Avro
           # collect metadata
           @sync_marker = dfr.sync_marker
           meta['avro.codec'] = dfr.meta['avro.codec']
+          @codec = DataFile.get_codec(meta['avro.codec'])
 
           # get schema used to write existing file
           schema_from_file = dfr.meta['avro.schema']
@@ -152,21 +176,15 @@ module Avro
 
       # TODO(jmhodges): make a schema for blocks and use datum_writer
       # TODO(jmhodges): do we really need the number of items in the block?
-      # TODO(jmhodges): use codec when writing the block contents
       def write_block
         if block_count > 0
           # write number of items in block and block size in bytes
           encoder.write_long(block_count)
-          to_write = buffer_writer.string
+          to_write = codec.compress(buffer_writer.string)
           encoder.write_long(to_write.size)
 
           # write block contents
-          if meta['avro.codec'] == 'null'
-            writer.write(to_write)
-          else
-            msg = "#{meta['avro.codec'].inspect} coded is not supported"
-            raise DataFileError, msg
-          end
+          writer.write(to_write)
 
           # write sync marker
           writer.write(sync_marker)
@@ -183,8 +201,14 @@ module Avro
     class Reader
       include ::Enumerable
 
-      attr_reader :reader, :decoder, :datum_reader, :sync_marker, :meta, :file_length
-      attr_accessor :block_count
+      # The reader and binary decoder for the raw file stream
+      attr_reader :reader, :decoder
+
+      # The binary decoder for the contents of a block (after codec decompression)
+      attr_reader :block_decoder
+
+      attr_reader :datum_reader, :sync_marker, :meta, :file_length, :codec
+      attr_accessor :block_count # records remaining in current block
 
       def initialize(reader, datum_reader)
         @reader = reader
@@ -194,11 +218,7 @@ module Avro
         # read the header: magic, meta, sync
         read_header
 
-        # ensure the codec is valid
-        codec_from_file = meta['avro.codec']
-        if codec_from_file && ! VALID_CODECS.include?(codec_from_file)
-          raise DataFileError, "Unknown codec: #{codec_from_file}"
-        end
+        @codec = DataFile.get_codec(meta['avro.codec'])
 
         # get ready to read
         @block_count = 0
@@ -220,7 +240,7 @@ module Avro
             end
           end
 
-          datum = datum_reader.read(decoder)
+          datum = datum_reader.read(block_decoder)
           self.block_count -= 1
           yield(datum)
         end
@@ -257,7 +277,9 @@ module Avro
 
       def read_block_header
         self.block_count = decoder.read_long
-        decoder.read_long # not doing anything with length in bytes
+        block_bytes = decoder.read_long
+        data = codec.decompress(reader.read(block_bytes))
+        @block_decoder = IO::BinaryDecoder.new(StringIO.new(data))
       end
 
       # read the length of the sync marker; if it matches the sync
@@ -273,5 +295,48 @@ module Avro
         end
       end
     end
+
+
+    class NullCodec
+      def codec_name; 'null'; end
+      def decompress(data); data; end
+      def compress(data); data; end
+    end
+
+    class DeflateCodec
+      attr_reader :level
+
+      def initialize(level=Zlib::DEFAULT_COMPRESSION)
+        @level = level
+      end
+
+      def codec_name; 'deflate'; end
+
+      def decompress(compressed)
+        # Passing a negative number to Inflate puts it into "raw" RFC1951 mode
+        # (without the RFC1950 header & checksum). See the docs for
+        # inflateInit2 in http://www.zlib.net/manual.html
+        zstream = Zlib::Inflate.new(-Zlib::MAX_WBITS)
+        data = zstream.inflate(compressed)
+        data << zstream.finish
+      ensure
+        zstream.close
+      end
+
+      def compress(data)
+        zstream = Zlib::Deflate.new(level, -Zlib::MAX_WBITS)
+        compressed = zstream.deflate(data)
+        compressed << zstream.finish
+      ensure
+        zstream.close
+      end
+    end
+
+    DataFile.register_codec NullCodec
+    DataFile.register_codec DeflateCodec
+
+    # TODO this constant won't be updated if you register another codec.
+    # Deprecated in favor of Avro::DataFile::codecs
+    VALID_CODECS = DataFile.codecs.keys
   end
 end

data/lib/avro/io.rb

@@ -220,51 +220,43 @@ module Avro
     end
 
     class DatumReader
-      def self.check_props(schema_one, schema_two, prop_list)
-        prop_list.all? do |prop|
-          schema_one.send(prop) == schema_two.send(prop)
-        end
-      end
-
       def self.match_schemas(writers_schema, readers_schema)
-        w_type = writers_schema.type
-        r_type = readers_schema.type
+        w_type = writers_schema.type_sym
+        r_type = readers_schema.type_sym
 
         # This conditional is begging for some OO love.
-        if w_type == 'union' || r_type == 'union'
+        if w_type == :union || r_type == :union
           return true
         end
 
         if w_type == r_type
-          if Schema::PRIMITIVE_TYPES.include?(w_type) &&
-              Schema::PRIMITIVE_TYPES.include?(r_type)
-            return true
-          end
+          return true if Schema::PRIMITIVE_TYPES_SYM.include?(r_type)
 
           case r_type
-          when 'record'
-            return check_props(writers_schema, readers_schema, [:fullname])
-          when 'error'
-            return check_props(writers_schema, readers_schema, [:fullname])
-          when 'request'
+          when :record
+            return writers_schema.fullname == readers_schema.fullname
+          when :error
+            return writers_schema.fullname == readers_schema.fullname
+          when :request
             return true
-          when 'fixed'
-            return check_props(writers_schema, readers_schema, [:fullname, :size])
-          when 'enum'
-            return check_props(writers_schema, readers_schema, [:fullname])
-          when 'map'
-            return check_props(writers_schema.values, readers_schema.values, [:type])
-          when 'array'
-            return check_props(writers_schema.items, readers_schema.items, [:type])
+          when :fixed
+            return writers_schema.fullname == readers_schema.fullname &&
+                   writers_schema.size == readers_schema.size
+          when :enum
+            return writers_schema.fullname == readers_schema.fullname
+          when :map
+            return writers_schema.values.type == readers_schema.values.type
+          when :array
+            return writers_schema.items.type == readers_schema.items.type
           end
         end
 
         # Handle schema promotion
-        if w_type == 'int' && ['long', 'float', 'double'].include?(r_type)
+        if w_type == :int && [:long, :float, :double].include?(r_type)
           return true
-        elsif w_type == 'long' && ['float', 'double'].include?(r_type)
+        elsif w_type == :long && [:float, :double].include?(r_type)
           return true
-        elsif w_type == 'float' && r_type == 'double'
+        elsif w_type == :float && r_type == :double
           return true
         end
 
@@ -291,7 +283,7 @@ module Avro
 
         # schema resolution: reader's schema is a union, writer's
         # schema is not
-        if writers_schema.type != 'union' && readers_schema.type == 'union'
+        if writers_schema.type_sym != :union && readers_schema.type_sym == :union
           rs = readers_schema.schemas.find{|s|
             self.class.match_schemas(writers_schema, s)
           }
@@ -301,21 +293,21 @@ module Avro
 
         # function dispatch for reading data based on type of writer's
         # schema
-        case writers_schema.type
-        when 'null';    decoder.read_null
-        when 'boolean'; decoder.read_boolean
-        when 'string';  decoder.read_string
-        when 'int';     decoder.read_int
-        when 'long';    decoder.read_long
-        when 'float';   decoder.read_float
-        when 'double';  decoder.read_double
-        when 'bytes';   decoder.read_bytes
-        when 'fixed';   read_fixed(writers_schema, readers_schema, decoder)
-        when 'enum';    read_enum(writers_schema, readers_schema, decoder)
-        when 'array';   read_array(writers_schema, readers_schema, decoder)
-        when 'map';     read_map(writers_schema, readers_schema, decoder)
-        when 'union';   read_union(writers_schema, readers_schema, decoder)
-        when 'record', 'error', 'request';  read_record(writers_schema, readers_schema, decoder)
+        case writers_schema.type_sym
+        when :null;    decoder.read_null
+        when :boolean; decoder.read_boolean
+        when :string;  decoder.read_string
+        when :int;     decoder.read_int
+        when :long;    decoder.read_long
+        when :float;   decoder.read_float
+        when :double;  decoder.read_double
+        when :bytes;   decoder.read_bytes
+        when :fixed;   read_fixed(writers_schema, readers_schema, decoder)
+        when :enum;    read_enum(writers_schema, readers_schema, decoder)
+        when :array;   read_array(writers_schema, readers_schema, decoder)
+        when :map;     read_map(writers_schema, readers_schema, decoder)
+        when :union;   read_union(writers_schema, readers_schema, decoder)
+        when :record, :error, :request;  read_record(writers_schema, readers_schema, decoder)
         else
           raise AvroError, "Cannot read unknown schema type: #{writers_schema.type}"
         end
@@ -416,34 +408,34 @@ module Avro
 
       def read_default_value(field_schema, default_value)
         # Basically a JSON Decoder?
-        case field_schema.type
-        when 'null'
+        case field_schema.type_sym
+        when :null
           return nil
-        when 'boolean'
+        when :boolean
           return default_value
-        when 'int', 'long'
+        when :int, :long
           return Integer(default_value)
-        when 'float', 'double'
+        when :float, :double
           return Float(default_value)
-        when 'enum', 'fixed', 'string', 'bytes'
+        when :enum, :fixed, :string, :bytes
           return default_value
-        when 'array'
+        when :array
           read_array = []
           default_value.each do |json_val|
             item_val = read_default_value(field_schema.items, json_val)
             read_array << item_val
           end
           return read_array
-        when 'map'
+        when :map
           read_map = {}
           default_value.each do |key, json_val|
             map_val = read_default_value(field_schema.values, json_val)
             read_map[key] = map_val
           end
           return read_map
-        when 'union'
+        when :union
           return read_default_value(field_schema.schemas[0], default_value)
-        when 'record', 'error'
+        when :record, :error
           read_record = {}
           field_schema.fields.each do |field|
             json_val = default_value[field.name]
@@ -459,37 +451,37 @@ module Avro
       end
 
       def skip_data(writers_schema, decoder)
-        case writers_schema.type
-        when 'null'
+        case writers_schema.type_sym
+        when :null
           decoder.skip_null
-        when 'boolean'
+        when :boolean
           decoder.skip_boolean
-        when 'string'
+        when :string
           decoder.skip_string
-        when 'int'
+        when :int
           decoder.skip_int
-        when 'long'
+        when :long
           decoder.skip_long
-        when 'float'
+        when :float
           decoder.skip_float
-        when 'double'
+        when :double
           decoder.skip_double
-        when 'bytes'
+        when :bytes
           decoder.skip_bytes
-        when 'fixed'
+        when :fixed
           skip_fixed(writers_schema, decoder)
-        when 'enum'
+        when :enum
           skip_enum(writers_schema, decoder)
-        when 'array'
+        when :array
           skip_array(writers_schema, decoder)
-        when 'map'
+        when :map
           skip_map(writers_schema, decoder)
-        when 'union'
+        when :union
           skip_union(writers_schema, decoder)
-        when 'record', 'error', 'request'
+        when :record, :error, :request
           skip_record(writers_schema, decoder)
         else
-          raise AvroError, "Unknown schema type: #{schm.type}"
+          raise AvroError, "Unknown schema type: #{writers_schema.type}"
         end
       end
 
@@ -552,21 +544,21 @@ module Avro
         end
 
         # function dispatch to write datum
-        case writers_schema.type
-        when 'null';    encoder.write_null(datum)
-        when 'boolean'; encoder.write_boolean(datum)
-        when 'string';  encoder.write_string(datum)
-        when 'int';     encoder.write_int(datum)
-        when 'long';    encoder.write_long(datum)
-        when 'float';   encoder.write_float(datum)
-        when 'double';  encoder.write_double(datum)
-        when 'bytes';   encoder.write_bytes(datum)
-        when 'fixed';   write_fixed(writers_schema, datum, encoder)
-        when 'enum';    write_enum(writers_schema, datum, encoder)
-        when 'array';   write_array(writers_schema, datum, encoder)
-        when 'map';     write_map(writers_schema, datum, encoder)
-        when 'union';   write_union(writers_schema, datum, encoder)
-        when 'record', 'error', 'request';  write_record(writers_schema, datum, encoder)
+        case writers_schema.type_sym
+        when :null;    encoder.write_null(datum)
+        when :boolean; encoder.write_boolean(datum)
+        when :string;  encoder.write_string(datum)
+        when :int;     encoder.write_int(datum)
+        when :long;    encoder.write_long(datum)
+        when :float;   encoder.write_float(datum)
+        when :double;  encoder.write_double(datum)
+        when :bytes;   encoder.write_bytes(datum)
+        when :fixed;   write_fixed(writers_schema, datum, encoder)
+        when :enum;    write_enum(writers_schema, datum, encoder)
+        when :array;   write_array(writers_schema, datum, encoder)
+        when :map;     write_map(writers_schema, datum, encoder)
+        when :union;   write_union(writers_schema, datum, encoder)
+        when :record, :error, :request;  write_record(writers_schema, datum, encoder)
         else
           raise AvroError.new("Unknown type: #{writers_schema.type}")
         end