avro-patches 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4501694b4e822ab5d2f9b08cd4d2406474b7efa6
-  data.tar.gz: 7cb86acf0bb9cb38486d59263904e1eac8eb4be7
+  metadata.gz: 28f04e46e3ccd4a1d9400d32fed8474f1d4b9edc
+  data.tar.gz: 412323d266beed3d397ee861775dc340e2d12020
 SHA512:
-  metadata.gz: 7d2aa4dc274fef6e4727ac3d3842c984223b0ab8ab2fdaed522490d4a4a28d15ae636fb7844aea8f5ff20f0983e698c9d8aec42c3e7e3ba7b2b08e9e1a692761
-  data.tar.gz: 03b0e275a36bfbf88e18638661bbc59bcad4a423a6767743a5560c9b2092e8a25a829128c941d8bb3b79693926b478361d78b47826e3b37939e0a94a7f2c172c
+  metadata.gz: d95df0967c75e5f5aba614785db7f7407e9e5b88f35c2c524ed230555425d4dc846cde3ae15cfe8f8923c6f1792d8724e01ca9e8dfda73a230304a6e56ad6714
+  data.tar.gz: 568746bb7f634a1168a1b109f59170f7ad79e3ed1cbcf99f6325913b559283c235c66d1c8e3e08ec5e7032bc610dcb62b4dc84e4ce3eb366e77fc78ba759003b

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # avro-patches
 
+## v0.3.0
+- Further performance improvements for `Avro::SchemaValidator` and encoding.
+- Ensure that strings are encoded as UTF-8.
+
 ## v0.2.0
 - Performance improvements for `Avro::SchemaValidator`.
 

data/lib/avro-patches.rb

@@ -1,6 +1,7 @@
 require 'avro-patches/version'
 
 require 'avro'
+require 'avro-patches/ensure_encoding'
 require 'avro-patches/schema_validator'
 require 'avro-patches/logical_types'
 require 'avro-patches/schema_compatibility'

data/lib/avro-patches/ensure_encoding.rb

@@ -0,0 +1,5 @@
+# Change from "AVRO-1783: Ruby: Ensure correct binary encoding for byte strings"
+# https://github.com/apache/avro/commit/315d842148d57590a58fafecf6e5ea378e9e0d74
+
+# Only part of the above commit is included as we are not using protocols and RPC
+require_relative 'ensure_encoding/io'

data/lib/avro-patches/ensure_encoding/io.rb

@@ -0,0 +1,12 @@
+Avro::IO::DatumWriter.class_eval do
+  # A string is encoded as a long followed by that many bytes of
+  # UTF-8 encoded character data
+  def write_string(datum)
+    # The original commit used:
+    #   datum = datum.encode('utf-8') if datum.respond_to? :encode
+    # This always allocated a new string even if the string was already UTF-8 encoded.
+    # The form below is slightly more efficient.
+    datum = datum.encode(Encoding::UTF_8) if datum.respond_to?(:encode) && datum.encoding != Encoding::UTF_8
+    write_bytes(datum)
+  end
+end

data/lib/avro-patches/logical_types/io.rb

@@ -2,7 +2,7 @@ Avro::IO::DatumWriter.class_eval do
   def write_data(writers_schema, logical_datum, encoder)
     datum = writers_schema.type_adapter.encode(logical_datum)
 
-    unless Avro::Schema.validate(writers_schema, datum, encoded = true)
+    unless Avro::Schema.validate(writers_schema, datum, { recursive: false, encoded: true })
       raise Avro::IO::AvroTypeError.new(writers_schema, datum)
     end
 

data/lib/avro-patches/logical_types/schema.rb

@@ -57,8 +57,8 @@ Avro::Schema.class_eval do
   end
 
   # Determine if a ruby datum is an instance of a schema
-  def self.validate(expected_schema, logical_datum, encoded = false)
-    Avro::SchemaValidator.validate!(expected_schema, logical_datum, encoded)
+  def self.validate(expected_schema, logical_datum, options = { recursive: true, encoded: false })
+    Avro::SchemaValidator.validate!(expected_schema, logical_datum, options)
     true
   rescue Avro::SchemaValidator::ValidationError
     false

data/lib/avro-patches/logical_types/schema_validator.rb

@@ -1,9 +1,16 @@
 module AvroPatches
   module LogicalTypes
     module SchemaValidatorPatch
-      def validate!(expected_schema, logical_datum, encoded = false)
+      def validate!(expected_schema, logical_datum, options = { recursive: true, encoded: false })
+        options ||= {}
+        options[:recursive] = true unless options.key?(:recursive)
+
         result = Avro::SchemaValidator::Result.new
-        validate_recursive(expected_schema, logical_datum, Avro::SchemaValidator::ROOT_IDENTIFIER, result, encoded)
+        if options[:recursive]
+          validate_recursive(expected_schema, logical_datum, Avro::SchemaValidator::ROOT_IDENTIFIER, result, options[:encoded])
+        else
+          validate_simple(expected_schema, logical_datum, Avro::SchemaValidator::ROOT_IDENTIFIER, result, options[:encoded])
+        end
         fail Avro::SchemaValidator::ValidationError, result if result.failure?
         result
       end
@@ -11,14 +18,41 @@ module AvroPatches
       private
 
       def validate_recursive(expected_schema, logical_datum, path, result, encoded = false)
-        datum = if encoded
-                  logical_datum
-                else
-                  expected_schema.type_adapter.encode(logical_datum) rescue nil
-                end
+        datum = resolve_datum(expected_schema, logical_datum, encoded)
+
+        # The entire method is overridden so that encoded: true can be passed here
+        validate_simple(expected_schema, datum, path, result, true)
+
+        case expected_schema.type_sym
+        when :array
+          validate_array(expected_schema, datum, path, result)
+        when :map
+          validate_map(expected_schema, datum, path, result)
+        when :union
+          validate_union(expected_schema, datum, path, result)
+        when :record, :error, :request
+          fail Avro::SchemaValidator::TypeMismatchError unless datum.is_a?(Hash)
+          expected_schema.fields.each do |field|
+            deeper_path = deeper_path_for_hash(field.name, path)
+            validate_recursive(field.type, datum[field.name], deeper_path, result)
+          end
+        end
+      rescue Avro::SchemaValidator::TypeMismatchError
+        result.add_error(path, "expected type #{expected_schema.type_sym}, got #{actual_value_message(datum)}")
+      end
 
+      def validate_simple(expected_schema, logical_datum, path, result, encoded = false)
+        datum = resolve_datum(expected_schema, logical_datum, encoded)
         super(expected_schema, datum, path, result)
       end
+
+      def resolve_datum(expected_schema, logical_datum, encoded)
+        if encoded
+          logical_datum
+        else
+          expected_schema.type_adapter.encode(logical_datum) rescue nil
+        end
+      end
     end
   end
 end

data/lib/avro-patches/schema_validator/schema.rb

@@ -1,7 +1,7 @@
 Avro::Schema.class_eval do
   # Determine if a ruby datum is an instance of a schema
-  def self.validate(expected_schema, datum)
-    Avro::SchemaValidator.validate!(expected_schema, datum)
+  def self.validate(expected_schema, datum, options = { recursive: true })
+    Avro::SchemaValidator.validate!(expected_schema, datum, options)
     true
   rescue Avro::SchemaValidator::ValidationError
     false

data/lib/avro-patches/schema_validator/schema_validator.rb

@@ -63,16 +63,56 @@ module Avro
     TypeMismatchError = Class.new(ValidationError)
 
     class << self
-      def validate!(expected_schema, datum)
-        result = Result.new
-        validate_recursive(expected_schema, datum, ROOT_IDENTIFIER, result)
-        fail ValidationError, result if result.failure?
+      # This method is replaced by code in AvroPatches::LogicalTypes::SchemaValidatorPatch.
+      def validate!(expected_schema, datum, options = { recursive: true })
+        options ||= {}
+        options[:recursive] = true unless options.key?(:recursive)
+
+        result = Avro::SchemaValidator::Result.new
+        if options[:recursive]
+          validate_recursive(expected_schema, datum, ROOT_IDENTIFIER, result)
+        else
+          validate_simple(expected_schema, datum, ROOT_IDENTIFIER, result)
+        end
+        fail Avro::SchemaValidator::ValidationError, result if result.failure?
         result
       end
 
       private
 
+      def validate_type(expected_schema)
+        unless Avro::Schema::VALID_TYPES_SYM.include?(expected_schema.type_sym)
+          fail "Unexpected schema type #{expected_schema.type_sym} #{expected_schema.inspect}"
+        end
+      end
+
+      # This method is replaced by code in AvroPatches::LogicalTypes::SchemaValidatorPatch.
+      # The patches are layered this way because SchemaValidator exists on
+      # avro's master branch but logical type support is still in PR.
       def validate_recursive(expected_schema, datum, path, result)
+        validate_simple(expected_schema, datum, path, result)
+
+        case expected_schema.type_sym
+        when :array
+          validate_array(expected_schema, datum, path, result)
+        when :map
+          validate_map(expected_schema, datum, path, result)
+        when :union
+          validate_union(expected_schema, datum, path, result)
+        when :record, :error, :request
+          fail TypeMismatchError unless datum.is_a?(Hash)
+          expected_schema.fields.each do |field|
+            deeper_path = deeper_path_for_hash(field.name, path)
+            validate_recursive(field.type, datum[field.name], deeper_path, result)
+          end
+        end
+      rescue TypeMismatchError
+        result.add_error(path, "expected type #{expected_schema.type_sym}, got #{actual_value_message(datum)}")
+      end
+
+      def validate_simple(expected_schema, datum, path, result)
+        validate_type(expected_schema)
+
         case expected_schema.type_sym
         when :null
           fail TypeMismatchError unless datum.nil?
@@ -96,20 +136,6 @@ module Avro
           end
         when :enum
           result.add_error(path, enum_message(expected_schema.symbols, datum)) unless expected_schema.symbols.include?(datum)
-        when :array
-          validate_array(expected_schema, datum, path, result)
-        when :map
-          validate_map(expected_schema, datum, path, result)
-        when :union
-          validate_union(expected_schema, datum, path, result)
-        when :record, :error, :request
-          fail TypeMismatchError unless datum.is_a?(Hash)
-          expected_schema.fields.each do |field|
-            deeper_path = deeper_path_for_hash(field.name, path)
-            validate_recursive(field.type, datum[field.name], deeper_path, result)
-          end
-        else
-          fail "Unexpected schema type #{expected_schema.type_sym} #{expected_schema.inspect}"
         end
       rescue TypeMismatchError
         result.add_error(path, "expected type #{expected_schema.type_sym}, got #{actual_value_message(datum)}")

data/lib/avro-patches/version.rb

@@ -1,3 +1,3 @@
 module AvroPatches
-  VERSION = '0.2.0'.freeze
+  VERSION = '0.3.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: avro-patches
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Salsify, Inc
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-05 00:00:00.000000000 Z
+date: 2017-06-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -102,6 +102,8 @@ files:
 - bin/console
 - bin/setup
 - lib/avro-patches.rb
+- lib/avro-patches/ensure_encoding.rb
+- lib/avro-patches/ensure_encoding/io.rb
 - lib/avro-patches/logical_types.rb
 - lib/avro-patches/logical_types/io.rb
 - lib/avro-patches/logical_types/logical_types.rb