avro-patches 0.3.4 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: ed60cb5d2757a33263b574ba6346fc9665b84548
-  data.tar.gz: 2b36545e5fc8dcb16c5aa93bb1e83fd1248e1b28
+SHA256:
+  metadata.gz: 3c6ca3081364705f70d24e2993fd79e3f0fd24f241f4a5fd9e432fa4a55b70d2
+  data.tar.gz: 4277a7ce247beeac5ee52ad75c6a5bd7854fe673c8298c1e3f3654890b30db8b
 SHA512:
-  metadata.gz: 6727d406d14b6767422de730149d85abd1a819c773267c90a467d89a4233da76d8140bcefffbd67a3049a464e4072e3f201d685d0033f032cdd86b821523b133
-  data.tar.gz: a5960d92f14696c5fbb22c3ec80f245cdc3da3a8b1b50a5b39eaf919b68cc7786499d48bdffb87feffa6715dd183f6584811ae4e09c659783b52ac68dab9c826
+  metadata.gz: 7f25cd04b62ba3f44f050d32b27e8e28fc32714acc930a369baff540c8ae306494cfaa75b82882db4013bf9b0d28168f8bd3d2d33b48065ff0345d581675e69b
+  data.tar.gz: ee430790f1dc828a3101d132501acf86e05dc55d21e9b613a4dad28fe4c480395c596af56627ab35dfa1dd1545a00808dfd58ebbdd23dfb76284b2bd79e0424f

data/.ruby-gemset

@@ -0,0 +1 @@
+avro-patches

data/.ruby-version

@@ -0,0 +1 @@
+2.5.1

data/.travis.yml

@@ -1,7 +1,8 @@
 language: ruby
 rvm:
-  - 2.3.4
-  - 2.4.1
-before_install: gem install bundler -v 1.15.0 --no-document
+  - 2.3.5
+  - 2.4.4
+  - 2.5.1
+before_install: gem install bundler -v 1.16.2 --no-document
 script:
   - bundle exec rake test

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # avro-patches
 
+## v0.4.0
+- Optionally fail validation when extra fields are present.
+- Check that field defaults have the correct type.
+- Support values for logical types that were already encoded.
+- Remove bin directory scripts from the release.
+
 ## v0.3.4
 - Allow promotion of nested records to optional 
 

data/README.md

@@ -11,6 +11,8 @@ The following pending or unreleased changes are included:
 - [AVRO-1695: Ruby support for logical types revisited](https://github.com/apache/avro/pull/116)
 - [AVRO-1969: Add schema compatibility checker for Ruby](https://github.com/apache/avro/pull/170)
 - [AVRO-2039: Ruby encoding performance improvements](https://github.com/apache/avro/pull/230)
+- [AVRO-2200: Option to fail when extra fields are in the payload](https://github.com/apache/avro/pull/321)
+- [AVRO-2199: Validate that field defaults have the correct type](https://github.com/apache/avro/pull/320)
 
 In addition, compatibility with Ruby 2.4 (https://github.com/apache/avro/pull/191)
 has been integrated with the changes above.
@@ -46,8 +48,7 @@ To install this gem onto your local machine, run `bundle exec rake install`.
 To release a new version, update the version number in `version.rb`, and then
 run `bundle exec rake release`, which will create a git tag for the version,
 push git commits and tags, and push the `.gem` file to
-[rubygems.org](https://rubygems.org)
-.
+[rubygems.org](https://rubygems.org).
 
 ## Contributing
 

data/avro-patches.gemspec

@@ -23,9 +23,7 @@ Gem::Specification.new do |spec|
     raise 'RubyGems 2.0 or newer is required to protect against public gem pushes.'
   end
 
-  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  spec.bindir        = 'bin'
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(bin|test|spec|features)/}) }
   spec.require_paths = ['lib']
 
   spec.add_development_dependency 'bundler', '~> 1.15'

data/lib/avro-patches.rb

@@ -33,6 +33,7 @@ require 'avro-patches/ensure_encoding'
 require 'avro-patches/schema_validator'
 require 'avro-patches/logical_types'
 require 'avro-patches/schema_compatibility'
+require 'avro-patches/default_validation'
 
 # Remaining requires from the official avro gem
 require 'avro/data_file'

data/lib/avro-patches/default_validation.rb

@@ -0,0 +1 @@
+require "avro-patches/default_validation/schema"

data/lib/avro-patches/default_validation/schema.rb

@@ -0,0 +1,27 @@
+module AvroPatches
+  module DefaultValidation
+    module FieldPatch
+      def initialize(type, name, default=:no_default, order=nil, names=nil, namespace=nil)
+        super
+
+        validate_default! if default?
+      end
+
+      private
+
+      def validate_default!
+        type_for_default = if type.type_sym == :union
+                             type.schemas.first
+                           else
+                             type
+                           end
+
+        Avro::SchemaValidator.validate!(type_for_default, default)
+      rescue Avro::SchemaValidator::ValidationError => e
+        raise Avro::SchemaParseError, "Error validating default for #{name}: #{e.message}"
+      end
+    end
+  end
+end
+
+Avro::Schema::Field.prepend(AvroPatches::DefaultValidation::FieldPatch)

data/lib/avro-patches/logical_types/logical_types.rb

@@ -6,6 +6,8 @@ module Avro
       EPOCH_START = Date.new(1970, 1, 1)
 
       def self.encode(date)
+        return date.to_i if date.is_a?(Numeric)
+
         (date - EPOCH_START).to_i
       end
 
@@ -16,6 +18,8 @@ module Avro
 
     module TimestampMillis
       def self.encode(value)
+        return value.to_i if value.is_a?(Numeric)
+
         time = value.to_time
         time.to_i * 1000 + time.usec / 1000
       end
@@ -28,6 +32,8 @@ module Avro
 
     module TimestampMicros
       def self.encode(value)
+        return value.to_i if value.is_a?(Numeric)
+
         time = value.to_time
         time.to_i * 1000_000 + time.usec
       end
@@ -61,7 +67,7 @@ module Avro
     def self.type_adapter(type, logical_type)
       return unless logical_type
 
-      TYPES.fetch(type, {}).fetch(logical_type, Identity)
+      TYPES.fetch(type, {}.freeze).fetch(logical_type, Identity)
     end
   end
 end

data/lib/avro-patches/logical_types/schema_validator.rb

@@ -1,15 +1,17 @@
 module AvroPatches
   module LogicalTypes
     module SchemaValidatorPatch
-      def validate!(expected_schema, logical_datum, options = { recursive: true, encoded: false })
+      def validate!(expected_schema, logical_datum, options = { recursive: true, encoded: false, fail_on_extra_fields: false})
         options ||= {}
         options[:recursive] = true unless options.key?(:recursive)
 
         result = Avro::SchemaValidator::Result.new
         if options[:recursive]
-          validate_recursive(expected_schema, logical_datum, Avro::SchemaValidator::ROOT_IDENTIFIER, result, options[:encoded])
+          validate_recursive(expected_schema, logical_datum,
+                             Avro::SchemaValidator::ROOT_IDENTIFIER, result, options)
         else
-          validate_simple(expected_schema, logical_datum, Avro::SchemaValidator::ROOT_IDENTIFIER, result, options[:encoded])
+          validate_simple(expected_schema, logical_datum,
+                          Avro::SchemaValidator::ROOT_IDENTIFIER, result, options)
         end
         fail Avro::SchemaValidator::ValidationError, result if result.failure?
         result
@@ -17,11 +19,11 @@ module AvroPatches
 
       private
 
-      def validate_recursive(expected_schema, logical_datum, path, result, encoded = false)
-        datum = resolve_datum(expected_schema, logical_datum, encoded)
+      def validate_recursive(expected_schema, logical_datum, path, result, options = {})
+        datum = resolve_datum(expected_schema, logical_datum, options[:encoded])
 
         # The entire method is overridden so that encoded: true can be passed here
-        validate_simple(expected_schema, datum, path, result, true)
+        validate_simple(expected_schema, datum, path, result, encoded: true)
 
         case expected_schema.type_sym
         when :array
@@ -36,13 +38,20 @@ module AvroPatches
             deeper_path = deeper_path_for_hash(field.name, path)
             validate_recursive(field.type, datum[field.name], deeper_path, result)
           end
+          if options[:fail_on_extra_fields]
+            datum_fields = datum.keys.map(&:to_s)
+            schema_fields = expected_schema.fields.map(&:name)
+            (datum_fields - schema_fields).each do |extra_field|
+              result.add_error(path, "extra field '#{extra_field}' - not in schema")
+            end
+          end
         end
       rescue Avro::SchemaValidator::TypeMismatchError
         result.add_error(path, "expected type #{expected_schema.type_sym}, got #{actual_value_message(datum)}")
       end
 
-      def validate_simple(expected_schema, logical_datum, path, result, encoded = false)
-        datum = resolve_datum(expected_schema, logical_datum, encoded)
+      def validate_simple(expected_schema, logical_datum, path, result, options = {})
+        datum = resolve_datum(expected_schema, logical_datum, options[:encoded])
         super(expected_schema, datum, path, result)
       end
 

data/lib/avro-patches/version.rb

@@ -1,3 +1,3 @@
 module AvroPatches
-  VERSION = '0.3.4'.freeze
+  VERSION = '0.4.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: avro-patches
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.4.0
 platform: ruby
 authors:
 - Salsify, Inc
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-29 00:00:00.000000000 Z
+date: 2018-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -83,15 +83,15 @@ dependencies:
 description: Patches for the official Apache Avro Ruby implementation
 email:
 - engineering@salsify.com
-executables:
-- console
-- setup
+executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".overcommit.yml"
 - ".rspec"
+- ".ruby-gemset"
+- ".ruby-version"
 - ".travis.yml"
 - CHANGELOG.md
 - Gemfile
@@ -99,9 +99,9 @@ files:
 - README.md
 - Rakefile
 - avro-patches.gemspec
-- bin/console
-- bin/setup
 - lib/avro-patches.rb
+- lib/avro-patches/default_validation.rb
+- lib/avro-patches/default_validation/schema.rb
 - lib/avro-patches/ensure_encoding.rb
 - lib/avro-patches/ensure_encoding/io.rb
 - lib/avro-patches/logical_types.rb
@@ -140,7 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.14
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Patches for the official Apache Avro Ruby implementation

data/bin/console

@@ -1,14 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'bundler/setup'
-require 'avro-patches'
-
-# You can add fixtures and/or initialization code here to make experimenting
-# with your gem easier. You can also use a different console, if you like.
-
-# (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
-
-require 'irb'
-IRB.start

data/bin/setup

@@ -1,9 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-IFS=$'\n\t'
-set -v
-
-gem install bundler
-bundle update
-
-overcommit --install