avro 1.7.7 → 1.8.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 87cb7e254e68c38b88c8b88da666a8fa7e82a2cf
+  data.tar.gz: 0b2d18424b7186f1fb67408a0d123663f9d59878
+SHA512:
+  metadata.gz: 0ef8f99ff41b179b150a2c68be7428bc003d0b7d7a21cc43e5bd22604f437e67e18833425afb08378fb68a1be7f8baba741d0be990be904f58e5c62700968c5b
+  data.tar.gz: 6b3d8465d9d6a3699155c346bfab8c7bc2044adcfdc89ded920c0c2cff68038d3b4e81ddb615cc17f2112c0d8ec1a2634f590ef96511072c8fd1fd5b4e99b373

data/LICENSE

@@ -0,0 +1,203 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+

data/Manifest

@@ -1,4 +1,6 @@
 CHANGELOG
+LICENSE
+NOTICE
 Manifest
 Rakefile
 avro.gemspec
@@ -9,14 +11,19 @@ lib/avro/io.rb
 lib/avro/ipc.rb
 lib/avro/protocol.rb
 lib/avro/schema.rb
+lib/avro/schema_normalization.rb
+test/case_finder.rb
 test/random_data.rb
 test/sample_ipc_client.rb
 test/sample_ipc_http_client.rb
 test/sample_ipc_http_server.rb
 test/sample_ipc_server.rb
 test/test_datafile.rb
+test/test_fingerprints.rb
 test/test_help.rb
 test/test_io.rb
 test/test_protocol.rb
+test/test_schema.rb
+test/test_schema_normalization.rb
 test/test_socket_transport.rb
 test/tool.rb

data/NOTICE

@@ -0,0 +1,6 @@
+Apache Avro
+Copyright 2010-2015 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+

data/Rakefile

@@ -19,11 +19,12 @@ require 'echoe'
 VERSION = File.open('../../share/VERSION.txt').read.sub('-SNAPSHOT', '.pre1').chomp
 Echoe.new('avro', VERSION) do |p|
   p.author = "Apache Software Foundation"
-  p.email = "avro-dev@hadoop.apache.org"
+  p.email = "dev@avro.apache.org"
   p.summary = "Apache Avro for Ruby"
   p.description = "Avro is a data serialization and RPC format"
-  p.url = "http://hadoop.apache.org/avro/"
+  p.url = "http://avro.apache.org/"
   p.runtime_dependencies = %w[multi_json]
+  p.licenses = ["Apache License 2.0 (Apache-2.0)"]
 end
 
 t = Rake::TestTask.new(:interop)

data/avro.gemspec

@@ -1,26 +1,28 @@
 # -*- encoding: utf-8 -*-
+# stub: avro 1.8.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "avro"
-  s.version = "1.7.7"
+  s.version = "1.8.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
+  s.require_paths = ["lib"]
   s.authors = ["Apache Software Foundation"]
-  s.date = "2014-07-18"
+  s.date = "2016-01-22"
   s.description = "Avro is a data serialization and RPC format"
-  s.email = "avro-dev@hadoop.apache.org"
-  s.extra_rdoc_files = ["CHANGELOG", "lib/avro.rb", "lib/avro/data_file.rb", "lib/avro/io.rb", "lib/avro/ipc.rb", "lib/avro/protocol.rb", "lib/avro/schema.rb"]
-  s.files = ["CHANGELOG", "Manifest", "Rakefile", "avro.gemspec", "interop/test_interop.rb", "lib/avro.rb", "lib/avro/data_file.rb", "lib/avro/io.rb", "lib/avro/ipc.rb", "lib/avro/protocol.rb", "lib/avro/schema.rb", "test/random_data.rb", "test/sample_ipc_client.rb", "test/sample_ipc_http_client.rb", "test/sample_ipc_http_server.rb", "test/sample_ipc_server.rb", "test/test_datafile.rb", "test/test_help.rb", "test/test_io.rb", "test/test_protocol.rb", "test/test_socket_transport.rb", "test/tool.rb", "test/test_schema.rb"]
-  s.homepage = "http://hadoop.apache.org/avro/"
+  s.email = "dev@avro.apache.org"
+  s.extra_rdoc_files = ["CHANGELOG", "LICENSE", "lib/avro.rb", "lib/avro/data_file.rb", "lib/avro/io.rb", "lib/avro/ipc.rb", "lib/avro/protocol.rb", "lib/avro/schema.rb", "lib/avro/schema_normalization.rb"]
+  s.files = ["CHANGELOG", "LICENSE", "Manifest", "NOTICE", "Rakefile", "avro.gemspec", "interop/test_interop.rb", "lib/avro.rb", "lib/avro/data_file.rb", "lib/avro/io.rb", "lib/avro/ipc.rb", "lib/avro/protocol.rb", "lib/avro/schema.rb", "lib/avro/schema_normalization.rb", "test/case_finder.rb", "test/random_data.rb", "test/sample_ipc_client.rb", "test/sample_ipc_http_client.rb", "test/sample_ipc_http_server.rb", "test/sample_ipc_server.rb", "test/test_datafile.rb", "test/test_fingerprints.rb", "test/test_help.rb", "test/test_io.rb", "test/test_protocol.rb", "test/test_schema.rb", "test/test_schema_normalization.rb", "test/test_socket_transport.rb", "test/tool.rb"]
+  s.homepage = "http://avro.apache.org/"
+  s.licenses = ["Apache License 2.0 (Apache-2.0)"]
   s.rdoc_options = ["--line-numbers", "--title", "Avro"]
-  s.require_paths = ["lib"]
   s.rubyforge_project = "avro"
-  s.rubygems_version = "1.8.15"
+  s.rubygems_version = "2.2.2"
   s.summary = "Apache Avro for Ruby"
-  s.test_files = ["test/test_datafile.rb", "test/test_help.rb", "test/test_protocol.rb", "test/test_socket_transport.rb", "test/test_schema.rb", "test/test_io.rb"]
+  s.test_files = ["test/test_datafile.rb", "test/test_fingerprints.rb", "test/test_help.rb", "test/test_io.rb", "test/test_protocol.rb", "test/test_schema.rb", "test/test_schema_normalization.rb", "test/test_socket_transport.rb"]
 
   if s.respond_to? :specification_version then
-    s.specification_version = 3
+    s.specification_version = 4
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<multi_json>, [">= 0"])

data/lib/avro.rb

@@ -39,3 +39,4 @@ require 'avro/io'
 require 'avro/data_file'
 require 'avro/protocol'
 require 'avro/ipc'
+require 'avro/schema_normalization'

data/lib/avro/data_file.rb

@@ -93,23 +93,22 @@ module Avro
       attr_reader :writer, :encoder, :datum_writer, :buffer_writer, :buffer_encoder, :sync_marker, :meta, :codec
       attr_accessor :block_count
 
-      def initialize(writer, datum_writer, writers_schema=nil, codec=nil)
+      def initialize(writer, datum_writer, writers_schema=nil, codec=nil, meta={})
         # If writers_schema is not present, presume we're appending
         @writer = writer
         @encoder = IO::BinaryEncoder.new(@writer)
         @datum_writer = datum_writer
+        @meta = meta
         @buffer_writer = StringIO.new('', 'w')
         @buffer_writer.set_encoding('BINARY') if @buffer_writer.respond_to?(:set_encoding)
         @buffer_encoder = IO::BinaryEncoder.new(@buffer_writer)
         @block_count = 0
 
-        @meta = {}
-
         if writers_schema
           @sync_marker = Writer.generate_sync_marker
           @codec = DataFile.get_codec(codec)
-          meta['avro.codec'] = @codec.codec_name.to_s
-          meta['avro.schema'] = writers_schema.to_s
+          @meta['avro.codec'] = @codec.codec_name.to_s
+          @meta['avro.schema'] = writers_schema.to_s
           datum_writer.writers_schema = writers_schema
           write_header
         else
@@ -119,12 +118,12 @@ module Avro
           # FIXME(jmhodges): collect arbitrary metadata
           # collect metadata
           @sync_marker = dfr.sync_marker
-          meta['avro.codec'] = dfr.meta['avro.codec']
+          @meta['avro.codec'] = dfr.meta['avro.codec']
           @codec = DataFile.get_codec(meta['avro.codec'])
 
           # get schema used to write existing file
           schema_from_file = dfr.meta['avro.schema']
-          meta['avro.schema'] = schema_from_file
+          @meta['avro.schema'] = schema_from_file
           datum_writer.writers_schema = Schema.parse(schema_from_file)
 
           # seek to the end of the file and prepare for writing
@@ -334,8 +333,31 @@ module Avro
       end
     end
 
+    class SnappyCodec
+      def codec_name; 'snappy'; end
+
+      def decompress(data)
+        load_snappy!
+        Snappy.inflate(data)
+      end
+
+      def compress(data)
+        load_snappy!
+        Snappy.deflate(data)
+      end
+
+      private
+
+      def load_snappy!
+        require 'snappy' unless defined?(Snappy)
+      rescue LoadError
+        raise LoadError, "Snappy compression is not available, please install the `snappy` gem."
+      end
+    end
+
     DataFile.register_codec NullCodec
     DataFile.register_codec DeflateCodec
+    DataFile.register_codec SnappyCodec
 
     # TODO this constant won't be updated if you register another codec.
     # Deprecated in favor of Avro::DataFile::codecs

data/lib/avro/io.rb

@@ -209,7 +209,7 @@ module Avro
       # A string is encoded as a long followed by that many bytes of
       # UTF-8 encoded character data
       def write_string(datum)
-        # FIXME utf-8 encode this in 1.9
+        datum = datum.encode('utf-8') if datum.respond_to? :encode
         write_bytes(datum)
       end
 

data/lib/avro/ipc.rb

@@ -100,7 +100,7 @@ module Avro::IPC
     def request(message_name, request_datum)
       # Writes a request message and reads a response or error message.
       # build handshake and call request
-      buffer_writer = StringIO.new('', 'w+')
+      buffer_writer = StringIO.new(''.force_encoding('BINARY'))
       buffer_encoder = Avro::IO::BinaryEncoder.new(buffer_writer)
       write_handshake_request(buffer_encoder)
       write_call_request(message_name, request_datum, buffer_encoder)
@@ -244,7 +244,7 @@ module Avro::IPC
     # a response or error. Compare to 'handle()' in Thrift.
     def respond(call_request, transport=nil)
       buffer_decoder = Avro::IO::BinaryDecoder.new(StringIO.new(call_request))
-      buffer_writer = StringIO.new('', 'w+')
+      buffer_writer = StringIO.new(''.force_encoding('BINARY'))
       buffer_encoder = Avro::IO::BinaryEncoder.new(buffer_writer)
       error = nil
       response_metadata = {}
@@ -294,6 +294,7 @@ module Avro::IPC
         end
       rescue Avro::AvroError => e
         error = AvroRemoteException.new(e.to_s)
+        # TODO does the stuff written here ever get used?
         buffer_encoder = Avro::IO::BinaryEncoder.new(StringIO.new)
         META_WRITER.write(response_metadata, buffer_encoder)
         buffer_encoder.write_boolean(true)
@@ -393,7 +394,7 @@ module Avro::IPC
     def read_framed_message
       message = []
       loop do
-        buffer = StringIO.new
+        buffer = StringIO.new(''.force_encoding('BINARY'))
         buffer_length = read_buffer_length
         if buffer_length == 0
           return message.join
@@ -410,7 +411,7 @@ module Avro::IPC
     end
 
     def write_framed_message(message)
-      message_length = message.size
+      message_length = message.bytesize
       total_bytes_sent = 0
       while message_length - total_bytes_sent > 0
         if message_length - total_bytes_sent > BUFFER_SIZE
@@ -426,7 +427,7 @@ module Avro::IPC
     end
 
     def write_buffer(chunk)
-      buffer_length = chunk.size
+      buffer_length = chunk.bytesize
       write_buffer_length(buffer_length)
       total_bytes_sent = 0
       while total_bytes_sent < buffer_length
@@ -467,7 +468,7 @@ module Avro::IPC
     end
 
     def write_framed_message(message)
-      message_size = message.size
+      message_size = message.bytesize
       total_bytes_sent = 0
       while message_size - total_bytes_sent > 0
         if message_size - total_bytes_sent > BUFFER_SIZE
@@ -485,7 +486,7 @@ module Avro::IPC
 
     private
     def write_buffer(chunk)
-      buffer_size = chunk.size
+      buffer_size = chunk.bytesize
       write_buffer_size(buffer_size)
       writer << chunk
     end
@@ -505,13 +506,13 @@ module Avro::IPC
     def read_framed_message
       message = []
       loop do
-        buffer = ""
+        buffer = ''.force_encoding('BINARY')
         buffer_size = read_buffer_size
 
         return message.join if buffer_size == 0
 
-        while buffer.size < buffer_size
-          chunk = reader.read(buffer_size - buffer.size)
+        while buffer.bytesize < buffer_size
+          chunk = reader.read(buffer_size - buffer.bytesize)
           chunk_error?(chunk)
           buffer << chunk
         end
@@ -541,7 +542,7 @@ module Avro::IPC
     end
 
     def transceive(message)
-      writer = FramedWriter.new(StringIO.new)
+      writer = FramedWriter.new(StringIO.new(''.force_encoding('BINARY')))
       writer.write_framed_message(message)
       resp = @conn.post('/', writer.to_s, {'Content-Type' => 'avro/binary'})
       FramedReader.new(StringIO.new(resp.body)).read_framed_message

data/lib/avro/schema.rb

@@ -86,8 +86,7 @@ module Avro
       elsif PRIMITIVE_TYPES.include? json_obj
         return PrimitiveSchema.new(json_obj)
       else
-        msg = "#{json_obj.inspect} is not a schema we know about."
-        raise SchemaParseError.new(msg)
+        raise UnknownSchemaError.new(json_obj)
       end
     end
 
@@ -109,7 +108,7 @@ module Avro
       when :float, :double
         datum.is_a?(Float) || datum.is_a?(Fixnum) || datum.is_a?(Bignum)
       when :fixed
-        datum.is_a?(String) && datum.size == expected_schema.size
+        datum.is_a?(String) && datum.bytesize == expected_schema.size
       when :enum
         expected_schema.symbols.include? datum
       when :array
@@ -138,6 +137,18 @@ module Avro
     # Deprecated in favor of {#type_sym}.
     def type; @type_sym.to_s; end
 
+    # Returns the MD5 fingerprint of the schema as an Integer.
+    def md5_fingerprint
+      parsing_form = SchemaNormalization.to_parsing_form(self)
+      Digest::MD5.hexdigest(parsing_form).to_i(16)
+    end
+
+    # Returns the SHA-256 fingerprint of the schema as an Integer.
+    def sha256_fingerprint
+      parsing_form = SchemaNormalization.to_parsing_form(self)
+      Digest::SHA256.hexdigest(parsing_form).to_i(16)
+    end
+
     def ==(other, seen=nil)
       other.is_a?(Schema) && type_sym == other.type_sym
     end
@@ -370,6 +381,15 @@ module Avro
 
   class SchemaParseError < AvroError; end
 
+  class UnknownSchemaError < SchemaParseError
+    attr_reader :type_name
+
+    def initialize(type)
+      @type_name = type
+      super("#{type.inspect} is not a schema we know about.")
+    end
+  end
+
   module Name
     def self.extract_namespace(name, namespace)
       parts = name.split('.')

data/lib/avro/schema_normalization.rb

@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Avro
+  class SchemaNormalization
+    def self.to_parsing_form(schema)
+      new.to_parsing_form(schema)
+    end
+
+    def initialize
+      @processed_names = []
+    end
+
+    def to_parsing_form(schema)
+      MultiJson.dump(normalize_schema(schema))
+    end
+
+    private
+
+    def normalize_schema(schema)
+      type = schema.type_sym.to_s
+
+      if Schema::NAMED_TYPES.include?(type)
+        if @processed_names.include?(schema.name)
+          return schema.name
+        else
+          @processed_names << schema.name
+        end
+      end
+
+      case type
+      when *Schema::PRIMITIVE_TYPES
+        type
+      when "record"
+        fields = schema.fields.map {|field| normalize_field(field) }
+
+        normalize_named_type(schema, fields: fields)
+      when "enum"
+        normalize_named_type(schema, symbols: schema.symbols)
+      when "fixed"
+        normalize_named_type(schema, size: schema.size)
+      when "array"
+        { type: type, items: normalize_schema(schema.items) }
+      when "map"
+        { type: type, values: normalize_schema(schema.values) }
+      when "union"
+        if schema.schemas.nil?
+          []
+        else
+          schema.schemas.map {|s| normalize_schema(s) }
+        end
+      else
+        raise "unknown type #{type}"
+      end
+    end
+
+    def normalize_field(field)
+      {
+        name: field.name,
+        type: normalize_schema(field.type)
+      }
+    end
+
+    def normalize_named_type(schema, attributes = {})
+      name = Name.make_fullname(schema.name, schema.namespace)
+
+      { name: name, type: schema.type_sym.to_s }.merge(attributes)
+    end
+  end
+end

data/test/case_finder.rb

@@ -0,0 +1,87 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+require 'strscan'
+
+class CaseFinder
+  PATH = File.expand_path("../../../../share/test/data/schema-tests.txt", __FILE__)
+
+  Case = Struct.new(:id, :input, :canonical, :fingerprint)
+
+  def self.cases
+    new.cases
+  end
+
+  def initialize
+    @scanner = StringScanner.new(File.read(PATH))
+    @cases = []
+  end
+
+  def cases
+    until @scanner.eos?
+      test_case = scan_case
+      @cases << test_case if test_case
+    end
+
+    @cases
+  end
+
+  private
+
+  def scan_case
+    if id = @scanner.scan(/\/\/ \d+\n/)
+      while @scanner.skip(/\/\/ .*\n/); end
+
+      input = scan_input
+      canonical = scan_canonical
+      fingerprint = scan_fingerprint
+
+      Case.new(id, input, canonical, fingerprint)
+    else
+      @scanner.skip(/.*\n/)
+      nil
+    end
+  end
+
+  def scan_item(name)
+    if @scanner.scan(/<<#{name}\n/)
+      lines = []
+      while line = @scanner.scan(/.+\n/)
+        break if line.chomp == name
+        lines << line
+      end
+      lines.join
+    elsif @scanner.scan(/<<#{name} /)
+      input = @scanner.scan(/.+$/)
+      @scanner.skip(/\n/)
+      input
+    end
+  end
+
+  def scan_input
+    scan_item("INPUT")
+  end
+
+  def scan_canonical
+    scan_item("canonical")
+  end
+
+  def scan_fingerprint
+    scan_item("fingerprint")
+  end
+end

data/test/test_datafile.rb

@@ -167,6 +167,19 @@ JSON
     assert_equal records, ['a' * 10_000]
   end
 
+  def test_snappy
+    Avro::DataFile.open('data.avr', 'w', '"string"', :snappy) do |writer|
+      writer << 'a' * 10_000
+    end
+    assert(File.size('data.avr') < 600)
+
+    records = []
+    Avro::DataFile.open('data.avr') do |reader|
+      reader.each {|record| records << record }
+    end
+    assert_equal records, ['a' * 10_000]
+  end
+
   def test_append_to_deflated_file
     schema = Avro::Schema.parse('"string"')
     writer = Avro::IO::DatumWriter.new(schema)
@@ -185,4 +198,17 @@ JSON
     end
     assert_equal records, ['a' * 10_000, 'b' * 10_000]
   end
+
+  def test_custom_meta
+    meta = { 'x.greeting' => 'yo' }
+
+    schema = Avro::Schema.parse('"string"')
+    writer = Avro::IO::DatumWriter.new(schema)
+    file = Avro::DataFile::Writer.new(File.open('data.avr', 'wb'), writer, schema, nil, meta)
+    file.close
+
+    Avro::DataFile.open('data.avr') do |reader|
+      assert_equal 'yo', reader.meta['x.greeting']
+    end
+  end
 end

data/test/test_fingerprints.rb

@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'test_help'
+
+class TestFingerprints < Test::Unit::TestCase
+  def test_md5_fingerprint
+    schema = Avro::Schema.parse <<-SCHEMA
+      { "type": "int" }
+    SCHEMA
+
+    assert_equal 318112854175969537208795771590915775282,
+      schema.md5_fingerprint
+  end
+
+  def test_sha256_fingerprint
+    schema = Avro::Schema.parse <<-SCHEMA
+      { "type": "int" }
+    SCHEMA
+
+    assert_equal 28572620203319713300323544804233350633246234624932075150020181448463213378117,
+      schema.sha256_fingerprint
+  end
+end

data/test/test_io.rb

@@ -21,45 +21,55 @@ class TestIO < Test::Unit::TestCase
   Schema = Avro::Schema
 
   def test_null
+    check('"null"')
     check_default('"null"', "null", nil)
   end
 
   def test_boolean
+    check('"boolean"')
     check_default('"boolean"', "true", true)
     check_default('"boolean"', "false", false)
   end
 
   def test_string
+    check('"string"')
     check_default('"string"', '"foo"', "foo")
   end
 
   def test_bytes
+    check('"bytes"')
     check_default('"bytes"', '"foo"', "foo")
   end
 
   def test_int
+    check('"int"')
     check_default('"int"', "5", 5)
   end
 
   def test_long
+    check('"long"')
     check_default('"long"', "9", 9)
   end
 
   def test_float
+    check('"float"')
     check_default('"float"', "1.2", 1.2)
   end
 
   def test_double
+    check('"double"')
     check_default('"double"', "1.2", 1.2)
   end
 
   def test_array
     array_schema = '{"type": "array", "items": "long"}'
+    check(array_schema)
     check_default(array_schema, "[1]", [1])
   end
 
   def test_map
     map_schema = '{"type": "map", "values": "long"}'
+    check(map_schema)
     check_default(map_schema, '{"a": 1}', {"a" => 1})
   end
 
@@ -70,6 +80,7 @@ class TestIO < Test::Unit::TestCase
        "fields": [{"name": "f",
                    "type": "long"}]}
 EOS
+    check(record_schema)
     check_default(record_schema, '{"f": 11}', {"f" => 11})
   end
 
@@ -80,11 +91,13 @@ EOS
        "fields": [{"name": "message",
                    "type": "string"}]}
 EOS
+    check(error_schema)
     check_default(error_schema, '{"message": "boom"}', {"message" => "boom"})
   end
 
   def test_enum
     enum_schema = '{"type": "enum", "name": "Test","symbols": ["A", "B"]}'
+    check(enum_schema)
     check_default(enum_schema, '"B"', "B")
   end
 
@@ -129,6 +142,7 @@ EOS
 
   def test_fixed
     fixed_schema = '{"type": "fixed", "name": "Test", "size": 1}'
+    check(fixed_schema)
     check_default(fixed_schema, '"a"', "a")
   end
 
@@ -196,6 +210,51 @@ EOS
     end
   end
 
+  def test_utf8_string_encoding
+    [
+      "\xC3".force_encoding('ISO-8859-1'),
+      "\xC3\x83".force_encoding('UTF-8')
+    ].each do |value|
+      output = ''.force_encoding('BINARY')
+      encoder = Avro::IO::BinaryEncoder.new(StringIO.new(output))
+      datum_writer = Avro::IO::DatumWriter.new(Avro::Schema.parse('"string"'))
+      datum_writer.write(value, encoder)
+
+      assert_equal "\x04\xc3\x83".force_encoding('BINARY'), output
+    end
+  end
+
+  def test_bytes_encoding
+    [
+      "\xC3\x83".force_encoding('BINARY'),
+      "\xC3\x83".force_encoding('ISO-8859-1'),
+      "\xC3\x83".force_encoding('UTF-8')
+    ].each do |value|
+      output = ''.force_encoding('BINARY')
+      encoder = Avro::IO::BinaryEncoder.new(StringIO.new(output))
+      datum_writer = Avro::IO::DatumWriter.new(Avro::Schema.parse('"bytes"'))
+      datum_writer.write(value, encoder)
+
+      assert_equal "\x04\xc3\x83".force_encoding('BINARY'), output
+    end
+  end
+
+  def test_fixed_encoding
+    [
+      "\xC3\x83".force_encoding('BINARY'),
+      "\xC3\x83".force_encoding('ISO-8859-1'),
+      "\xC3\x83".force_encoding('UTF-8')
+    ].each do |value|
+      output = ''.force_encoding('BINARY')
+      encoder = Avro::IO::BinaryEncoder.new(StringIO.new(output))
+      schema = '{"type": "fixed", "name": "TwoBytes", "size": 2}'
+      datum_writer = Avro::IO::DatumWriter.new(Avro::Schema.parse(schema))
+      datum_writer.write(value, encoder)
+
+      assert_equal "\xc3\x83".force_encoding('BINARY'), output
+    end
+  end
+
   def test_skip_long
     for value_to_skip, hex_encoding in BINARY_INT_ENCODINGS
       value_to_read = 6253
@@ -284,7 +343,6 @@ EOS
   private
 
   def check_default(schema_json, default_json, default_value)
-    check(schema_json)
     actual_schema = '{"type": "record", "name": "Foo", "fields": []}'
     actual = Avro::Schema.parse(actual_schema)
 

data/test/test_schema.rb

@@ -131,4 +131,16 @@ class TestSchema < Test::Unit::TestCase
       ]
     }
   end
+
+  def test_unknown_named_type
+    error = assert_raise Avro::UnknownSchemaError do
+      Avro::Schema.parse <<-SCHEMA
+        {"type": "record", "name": "my.name.space.Record", "fields": [
+          {"name": "reference", "type": "MissingType"}
+        ]}
+      SCHEMA
+    end
+
+    assert_equal '"MissingType" is not a schema we know about.', error.message
+  end
 end

data/test/test_schema_normalization.rb

@@ -0,0 +1,171 @@
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'test_help'
+require 'case_finder'
+
+class TestSchemaNormalization < Test::Unit::TestCase
+  def test_primitives
+    %w[null boolean string bytes int long float double].each do |type|
+      schema = Avro::Schema.parse(<<-JSON)
+        { "type": "#{type}" }
+      JSON
+
+      canonical_form = Avro::SchemaNormalization.to_parsing_form(schema)
+
+      assert_equal %("#{type}"), canonical_form
+    end
+  end
+
+  def test_records
+    schema = Avro::Schema.parse(<<-JSON)
+      {
+        "type": "record",
+        "name": "test",
+        "namespace": "random",
+        "doc": "some record",
+        "fields": [
+          { "name": "height", "type": "int", "doc": "the height" }
+        ]
+      }
+    JSON
+
+    expected_type = <<-JSON.strip
+      {"name":"random.test","type":"record","fields":[{"name":"height","type":"int"}]}
+    JSON
+
+    canonical_form = Avro::SchemaNormalization.to_parsing_form(schema)
+
+    assert_equal expected_type, canonical_form
+  end
+
+  def test_recursive_records
+    schema = Avro::Schema.parse(<<-JSON)
+      {
+        "type": "record",
+        "name": "item",
+        "fields": [
+          { "name": "next", "type": "item" }
+        ]
+      }
+    JSON
+
+    expected_type = <<-JSON.strip
+      {"name":"item","type":"record","fields":[{"name":"next","type":"item"}]}
+    JSON
+
+    canonical_form = Avro::SchemaNormalization.to_parsing_form(schema)
+
+    assert_equal expected_type, canonical_form
+  end
+
+  def test_enums
+    schema = Avro::Schema.parse(<<-JSON)
+      {
+        "type": "enum",
+        "name": "suit",
+        "namespace": "cards",
+        "doc": "the different suits of cards",
+        "symbols": ["club", "hearts", "diamond", "spades"]
+      }
+    JSON
+
+    expected_type = <<-JSON.strip
+      {"name":"cards.suit","type":"enum","symbols":["club","hearts","diamond","spades"]}
+    JSON
+
+    canonical_form = Avro::SchemaNormalization.to_parsing_form(schema)
+
+    assert_equal expected_type, canonical_form
+  end
+
+  def test_fixed
+    schema = Avro::Schema.parse(<<-JSON)
+      {
+        "type": "fixed",
+        "name": "id",
+        "namespace": "db",
+        "size": 64
+      }
+    JSON
+
+    expected_type = <<-JSON.strip
+      {"name":"db.id","type":"fixed","size":64}
+    JSON
+
+    canonical_form = Avro::SchemaNormalization.to_parsing_form(schema)
+
+    assert_equal expected_type, canonical_form
+  end
+
+  def test_arrays
+    schema = Avro::Schema.parse(<<-JSON)
+      {
+        "type": "array",
+        "doc": "the items",
+        "items": "int"
+      }
+    JSON
+
+    expected_type = <<-JSON.strip
+      {"type":"array","items":"int"}
+    JSON
+
+    canonical_form = Avro::SchemaNormalization.to_parsing_form(schema)
+
+    assert_equal expected_type, canonical_form
+  end
+
+  def test_maps
+    schema = Avro::Schema.parse(<<-JSON)
+      {
+        "type": "map",
+        "doc": "the items",
+        "values": "int"
+      }
+    JSON
+
+    expected_type = <<-JSON.strip
+      {"type":"map","values":"int"}
+    JSON
+
+    canonical_form = Avro::SchemaNormalization.to_parsing_form(schema)
+
+    assert_equal expected_type, canonical_form
+  end
+
+  def test_unions
+    schema = Avro::Schema.parse(<<-JSON)
+      ["int", "string"]
+    JSON
+
+    expected_type = <<-JSON.strip
+      ["int","string"]
+    JSON
+
+    canonical_form = Avro::SchemaNormalization.to_parsing_form(schema)
+
+    assert_equal expected_type, canonical_form
+  end
+
+  def test_shared_dataset
+    CaseFinder.cases.each do |test_case|
+      schema = Avro::Schema.parse(test_case.input)
+      assert_equal test_case.canonical, Avro::SchemaNormalization.to_parsing_form(schema)
+    end
+  end
+end

metadata

@@ -1,53 +1,48 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: avro
-version: !ruby/object:Gem::Version 
-  hash: 5
-  prerelease: 
-  segments: 
-  - 1
-  - 7
-  - 7
-  version: 1.7.7
+version: !ruby/object:Gem::Version
+  version: 1.8.0
 platform: ruby
-authors: 
+authors:
 - Apache Software Foundation
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2014-07-18 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+date: 2016-01-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: multi_json
-  prerelease: false
-  requirement: *id001
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Avro is a data serialization and RPC format
-email: avro-dev@hadoop.apache.org
+email: dev@avro.apache.org
 executables: []
-
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - CHANGELOG
+- LICENSE
 - lib/avro.rb
 - lib/avro/data_file.rb
 - lib/avro/io.rb
 - lib/avro/ipc.rb
 - lib/avro/protocol.rb
 - lib/avro/schema.rb
-files: 
+- lib/avro/schema_normalization.rb
+files:
 - CHANGELOG
+- LICENSE
 - Manifest
+- NOTICE
 - Rakefile
 - avro.gemspec
 - interop/test_interop.rb
@@ -57,58 +52,55 @@ files:
 - lib/avro/ipc.rb
 - lib/avro/protocol.rb
 - lib/avro/schema.rb
+- lib/avro/schema_normalization.rb
+- test/case_finder.rb
 - test/random_data.rb
 - test/sample_ipc_client.rb
 - test/sample_ipc_http_client.rb
 - test/sample_ipc_http_server.rb
 - test/sample_ipc_server.rb
 - test/test_datafile.rb
+- test/test_fingerprints.rb
 - test/test_help.rb
 - test/test_io.rb
 - test/test_protocol.rb
+- test/test_schema.rb
+- test/test_schema_normalization.rb
 - test/test_socket_transport.rb
 - test/tool.rb
-- test/test_schema.rb
-homepage: http://hadoop.apache.org/avro/
-licenses: []
-
+homepage: http://avro.apache.org/
+licenses:
+- Apache License 2.0 (Apache-2.0)
+metadata: {}
 post_install_message: 
-rdoc_options: 
-- --line-numbers
-- --title
+rdoc_options:
+- "--line-numbers"
+- "--title"
 - Avro
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 11
-      segments: 
-      - 1
-      - 2
-      version: "1.2"
+    - !ruby/object:Gem::Version
+      version: '1.2'
 requirements: []
-
 rubyforge_project: avro
-rubygems_version: 1.8.15
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Apache Avro for Ruby
-test_files: 
+test_files:
 - test/test_datafile.rb
+- test/test_fingerprints.rb
 - test/test_help.rb
+- test/test_io.rb
 - test/test_protocol.rb
-- test/test_socket_transport.rb
 - test/test_schema.rb
-- test/test_io.rb
+- test/test_schema_normalization.rb
+- test/test_socket_transport.rb