avro 1.8.1 → 1.8.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d432977a240aa9a9356c21483d0f1f601079131e
-  data.tar.gz: a3a7f480daee4ae503f2aa32a053efc4c387ff1f
+  metadata.gz: 2725b644b8e1c2579bb7aa83a4f60a3f14486122
+  data.tar.gz: ce3c78d318872ff111c58d9cfab710fa207006ad
 SHA512:
-  metadata.gz: 322bf3242f8e369ed7cbceb8b82ebeb526b68a00137e18a9834cb62ea14e01e3e885549fe0301721e46546f9c657eedf596bfeba39e86f3fdb8a2a3e6ceadc13
-  data.tar.gz: 95c50679ab7d3c68e47db5e8b32e323460623dffba5fc9c22a7d95cca9a11ecba60951bcbdb3b1cdb34d30d27225dede1dbb44635791b857538aa8fdd4d33c78
+  metadata.gz: 54a416f6e98d93cf33de8d6ac2d6fac40337793597cd010e98576501880ca4377398aefb4fc07f7e7d4dd7859eb062ec78e923ebf7d45b8c3084d25c4d389be8
+  data.tar.gz: 224c3f0ee1cc0f52bda339f446caf4c66c31fb1200bb9af4fc4ea1b4daa4e5087f5749a524140300ce9271cdf406cac94f33f1d31a3658583e9ff16ec168b995

data/avro.gemspec

@@ -1,14 +1,14 @@
 # -*- encoding: utf-8 -*-
-# stub: avro 1.8.1 ruby lib
+# stub: avro 1.8.2 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "avro"
-  s.version = "1.8.1"
+  s.version = "1.8.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Apache Software Foundation"]
-  s.date = "2016-05-15"
+  s.date = "2017-05-07"
   s.description = "Avro is a data serialization and RPC format"
   s.email = "dev@avro.apache.org"
   s.extra_rdoc_files = ["CHANGELOG", "LICENSE", "lib/avro.rb", "lib/avro/data_file.rb", "lib/avro/io.rb", "lib/avro/ipc.rb", "lib/avro/protocol.rb", "lib/avro/schema.rb", "lib/avro/schema_normalization.rb"]
@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
   s.rubyforge_project = "avro"
   s.rubygems_version = "2.2.2"
   s.summary = "Apache Avro for Ruby"
-  s.test_files = ["test/test_schema_normalization.rb", "test/test_io.rb", "test/test_socket_transport.rb", "test/test_schema.rb", "test/test_fingerprints.rb", "test/test_help.rb", "test/test_datafile.rb", "test/test_protocol.rb"]
+  s.test_files = ["test/test_datafile.rb", "test/test_fingerprints.rb", "test/test_help.rb", "test/test_io.rb", "test/test_protocol.rb", "test/test_schema.rb", "test/test_schema_normalization.rb", "test/test_socket_transport.rb"]
 
   if s.respond_to? :specification_version then
     s.specification_version = 4

data/lib/avro/data_file.rb

@@ -338,12 +338,29 @@ module Avro
 
       def decompress(data)
         load_snappy!
+        crc32 = data.slice(-4..-1).unpack('N').first
+        uncompressed = Snappy.inflate(data.slice(0..-5))
+
+        if crc32 == Zlib.crc32(uncompressed)
+          uncompressed
+        else
+          # older versions of avro-ruby didn't write the checksum, so if it
+          # doesn't match this must assume that it wasn't there and return
+          # the entire payload uncompressed.
+          Snappy.inflate(data)
+        end
+      rescue Snappy::Error
+        # older versions of avro-ruby didn't write the checksum, so removing
+        # the last 4 bytes may cause Snappy to fail. recover by assuming the
+        # payload is from an older file and uncompress the entire buffer.
         Snappy.inflate(data)
       end
 
       def compress(data)
         load_snappy!
-        Snappy.deflate(data)
+        crc32 = Zlib.crc32(data)
+        compressed = Snappy.deflate(data)
+        [compressed, crc32].pack('a*N')
       end
 
       private

data/lib/avro/io.rb

@@ -407,6 +407,10 @@ module Avro
       end
 
       def read_default_value(field_schema, default_value)
+        if default_value == :no_default
+          raise AvroError, "Missing data for #{field_schema} with no default"
+        end
+
         # Basically a JSON Decoder?
         case field_schema.type_sym
         when :null

data/lib/avro/schema.rb

@@ -211,7 +211,7 @@ module Avro
           if field.respond_to?(:[]) # TODO(jmhodges) wtffffff
             type = field['type']
             name = field['name']
-            default = field['default']
+            default = field.key?('default') ? field['default'] : :no_default
             order = field['order']
             new_field = Field.new(type, name, default, order, names, namespace)
             # make sure field name has not been used yet
@@ -363,7 +363,7 @@ module Avro
     class Field < Schema
       attr_reader :type, :name, :default, :order
 
-      def initialize(type, name, default=nil, order=nil, names=nil, namespace=nil)
+      def initialize(type, name, default=:no_default, order=nil, names=nil, namespace=nil)
         @type = subparse(type, names, namespace)
         @name = name
         @default = default
@@ -372,7 +372,7 @@ module Avro
 
       def to_avro(names=Set.new)
         {'name' => name, 'type' => type.to_avro(names)}.tap do |avro|
-          avro['default'] = default if default
+          avro['default'] = default unless default == :no_default
           avro['order'] = order if order
         end
       end

data/test/test_io.rb

@@ -340,8 +340,37 @@ EOS
       assert_equal(incorrect, 0)
     end
   end
+
+  def test_snappy_backward_compat
+    # a snappy-compressed block payload without the checksum
+    # this has no back-references, just one literal so the last 9
+    # bytes are the uncompressed payload.
+    old_snappy_bytes = "\x09\x20\x02\x06\x02\x0a\x67\x72\x65\x65\x6e"
+    uncompressed_bytes = "\x02\x06\x02\x0a\x67\x72\x65\x65\x6e"
+    snappy = Avro::DataFile::SnappyCodec.new
+    assert_equal(uncompressed_bytes, snappy.decompress(old_snappy_bytes))
+  end
+
   private
 
+  def check_no_default(schema_json)
+    actual_schema = '{"type": "record", "name": "Foo", "fields": []}'
+    actual = Avro::Schema.parse(actual_schema)
+
+    expected_schema = <<EOS
+      {"type": "record",
+       "name": "Foo",
+       "fields": [{"name": "f", "type": #{schema_json}}]}
+EOS
+    expected = Avro::Schema.parse(expected_schema)
+
+    reader = Avro::IO::DatumReader.new(actual, expected)
+    assert_raise Avro::AvroError do
+      value = reader.read(Avro::IO::BinaryDecoder.new(StringIO.new))
+      assert_not_equal(value, :no_default) # should never return this
+    end
+  end
+
   def check_default(schema_json, default_json, default_value)
     actual_schema = '{"type": "record", "name": "Foo", "fields": []}'
     actual = Avro::Schema.parse(actual_schema)
@@ -381,6 +410,9 @@ EOS
 
     # test writing of data to file
     check_datafile(schema)
+
+    # check that AvroError is raised when there is no default
+    check_no_default(str)
   end
 
   def checkser(schm, randomdata)

data/test/test_schema.rb

@@ -143,4 +143,21 @@ class TestSchema < Test::Unit::TestCase
 
     assert_equal '"MissingType" is not a schema we know about.', error.message
   end
+
+  def test_to_avro_handles_falsey_defaults
+    schema = Avro::Schema.parse <<-SCHEMA
+      {"type": "record", "name": "Record", "namespace": "my.name.space",
+        "fields": [
+          {"name": "is_usable", "type": "boolean", "default": false}
+        ]
+      }
+    SCHEMA
+
+    assert_equal schema.to_avro, {
+      'type' => 'record', 'name' => 'Record', 'namespace' => 'my.name.space',
+      'fields' => [
+        {'name' => 'is_usable', 'type' => 'boolean', 'default' => false}
+      ]
+    }
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: avro
 version: !ruby/object:Gem::Version
-  version: 1.8.1
+  version: 1.8.2
 platform: ruby
 authors:
 - Apache Software Foundation
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-15 00:00:00.000000000 Z
+date: 2017-05-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -96,11 +96,11 @@ signing_key:
 specification_version: 4
 summary: Apache Avro for Ruby
 test_files:
-- test/test_schema_normalization.rb
-- test/test_io.rb
-- test/test_socket_transport.rb
-- test/test_schema.rb
+- test/test_datafile.rb
 - test/test_fingerprints.rb
 - test/test_help.rb
-- test/test_datafile.rb
+- test/test_io.rb
 - test/test_protocol.rb
+- test/test_schema.rb
+- test/test_schema_normalization.rb
+- test/test_socket_transport.rb