avro-patches 0.3.4 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/.travis.yml +4 -3
- data/CHANGELOG.md +6 -0
- data/README.md +3 -2
- data/avro-patches.gemspec +1 -3
- data/lib/avro-patches.rb +1 -0
- data/lib/avro-patches/default_validation.rb +1 -0
- data/lib/avro-patches/default_validation/schema.rb +27 -0
- data/lib/avro-patches/logical_types/logical_types.rb +7 -1
- data/lib/avro-patches/logical_types/schema_validator.rb +17 -8
- data/lib/avro-patches/version.rb +1 -1
- metadata +8 -8
- data/bin/console +0 -14
- data/bin/setup +0 -9
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 3c6ca3081364705f70d24e2993fd79e3f0fd24f241f4a5fd9e432fa4a55b70d2
|
4
|
+
data.tar.gz: 4277a7ce247beeac5ee52ad75c6a5bd7854fe673c8298c1e3f3654890b30db8b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7f25cd04b62ba3f44f050d32b27e8e28fc32714acc930a369baff540c8ae306494cfaa75b82882db4013bf9b0d28168f8bd3d2d33b48065ff0345d581675e69b
|
7
|
+
data.tar.gz: ee430790f1dc828a3101d132501acf86e05dc55d21e9b613a4dad28fe4c480395c596af56627ab35dfa1dd1545a00808dfd58ebbdd23dfb76284b2bd79e0424f
|
data/.ruby-gemset
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
avro-patches
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
2.5.1
|
data/.travis.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,11 @@
|
|
1
1
|
# avro-patches
|
2
2
|
|
3
|
+
## v0.4.0
|
4
|
+
- Optionally fail validation when extra fields are present.
|
5
|
+
- Check that field defaults have the correct type.
|
6
|
+
- Support values for logical types that were already encoded.
|
7
|
+
- Remove bin directory scripts from the release.
|
8
|
+
|
3
9
|
## v0.3.4
|
4
10
|
- Allow promotion of nested records to optional
|
5
11
|
|
data/README.md
CHANGED
@@ -11,6 +11,8 @@ The following pending or unreleased changes are included:
|
|
11
11
|
- [AVRO-1695: Ruby support for logical types revisited](https://github.com/apache/avro/pull/116)
|
12
12
|
- [AVRO-1969: Add schema compatibility checker for Ruby](https://github.com/apache/avro/pull/170)
|
13
13
|
- [AVRO-2039: Ruby encoding performance improvements](https://github.com/apache/avro/pull/230)
|
14
|
+
- [AVRO-2200: Option to fail when extra fields are in the payload](https://github.com/apache/avro/pull/321)
|
15
|
+
- [AVRO-2199: Validate that field defaults have the correct type](https://github.com/apache/avro/pull/320)
|
14
16
|
|
15
17
|
In addition, compatibility with Ruby 2.4 (https://github.com/apache/avro/pull/191)
|
16
18
|
has been integrated with the changes above.
|
@@ -46,8 +48,7 @@ To install this gem onto your local machine, run `bundle exec rake install`.
|
|
46
48
|
To release a new version, update the version number in `version.rb`, and then
|
47
49
|
run `bundle exec rake release`, which will create a git tag for the version,
|
48
50
|
push git commits and tags, and push the `.gem` file to
|
49
|
-
[rubygems.org](https://rubygems.org)
|
50
|
-
.
|
51
|
+
[rubygems.org](https://rubygems.org).
|
51
52
|
|
52
53
|
## Contributing
|
53
54
|
|
data/avro-patches.gemspec
CHANGED
@@ -23,9 +23,7 @@ Gem::Specification.new do |spec|
|
|
23
23
|
raise 'RubyGems 2.0 or newer is required to protect against public gem pushes.'
|
24
24
|
end
|
25
25
|
|
26
|
-
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
27
|
-
spec.bindir = 'bin'
|
28
|
-
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
26
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(bin|test|spec|features)/}) }
|
29
27
|
spec.require_paths = ['lib']
|
30
28
|
|
31
29
|
spec.add_development_dependency 'bundler', '~> 1.15'
|
data/lib/avro-patches.rb
CHANGED
@@ -33,6 +33,7 @@ require 'avro-patches/ensure_encoding'
|
|
33
33
|
require 'avro-patches/schema_validator'
|
34
34
|
require 'avro-patches/logical_types'
|
35
35
|
require 'avro-patches/schema_compatibility'
|
36
|
+
require 'avro-patches/default_validation'
|
36
37
|
|
37
38
|
# Remaining requires from the official avro gem
|
38
39
|
require 'avro/data_file'
|
@@ -0,0 +1 @@
|
|
1
|
+
require "avro-patches/default_validation/schema"
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module AvroPatches
|
2
|
+
module DefaultValidation
|
3
|
+
module FieldPatch
|
4
|
+
def initialize(type, name, default=:no_default, order=nil, names=nil, namespace=nil)
|
5
|
+
super
|
6
|
+
|
7
|
+
validate_default! if default?
|
8
|
+
end
|
9
|
+
|
10
|
+
private
|
11
|
+
|
12
|
+
def validate_default!
|
13
|
+
type_for_default = if type.type_sym == :union
|
14
|
+
type.schemas.first
|
15
|
+
else
|
16
|
+
type
|
17
|
+
end
|
18
|
+
|
19
|
+
Avro::SchemaValidator.validate!(type_for_default, default)
|
20
|
+
rescue Avro::SchemaValidator::ValidationError => e
|
21
|
+
raise Avro::SchemaParseError, "Error validating default for #{name}: #{e.message}"
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
Avro::Schema::Field.prepend(AvroPatches::DefaultValidation::FieldPatch)
|
@@ -6,6 +6,8 @@ module Avro
|
|
6
6
|
EPOCH_START = Date.new(1970, 1, 1)
|
7
7
|
|
8
8
|
def self.encode(date)
|
9
|
+
return date.to_i if date.is_a?(Numeric)
|
10
|
+
|
9
11
|
(date - EPOCH_START).to_i
|
10
12
|
end
|
11
13
|
|
@@ -16,6 +18,8 @@ module Avro
|
|
16
18
|
|
17
19
|
module TimestampMillis
|
18
20
|
def self.encode(value)
|
21
|
+
return value.to_i if value.is_a?(Numeric)
|
22
|
+
|
19
23
|
time = value.to_time
|
20
24
|
time.to_i * 1000 + time.usec / 1000
|
21
25
|
end
|
@@ -28,6 +32,8 @@ module Avro
|
|
28
32
|
|
29
33
|
module TimestampMicros
|
30
34
|
def self.encode(value)
|
35
|
+
return value.to_i if value.is_a?(Numeric)
|
36
|
+
|
31
37
|
time = value.to_time
|
32
38
|
time.to_i * 1000_000 + time.usec
|
33
39
|
end
|
@@ -61,7 +67,7 @@ module Avro
|
|
61
67
|
def self.type_adapter(type, logical_type)
|
62
68
|
return unless logical_type
|
63
69
|
|
64
|
-
TYPES.fetch(type, {}).fetch(logical_type, Identity)
|
70
|
+
TYPES.fetch(type, {}.freeze).fetch(logical_type, Identity)
|
65
71
|
end
|
66
72
|
end
|
67
73
|
end
|
@@ -1,15 +1,17 @@
|
|
1
1
|
module AvroPatches
|
2
2
|
module LogicalTypes
|
3
3
|
module SchemaValidatorPatch
|
4
|
-
def validate!(expected_schema, logical_datum, options = { recursive: true, encoded: false })
|
4
|
+
def validate!(expected_schema, logical_datum, options = { recursive: true, encoded: false, fail_on_extra_fields: false})
|
5
5
|
options ||= {}
|
6
6
|
options[:recursive] = true unless options.key?(:recursive)
|
7
7
|
|
8
8
|
result = Avro::SchemaValidator::Result.new
|
9
9
|
if options[:recursive]
|
10
|
-
validate_recursive(expected_schema, logical_datum,
|
10
|
+
validate_recursive(expected_schema, logical_datum,
|
11
|
+
Avro::SchemaValidator::ROOT_IDENTIFIER, result, options)
|
11
12
|
else
|
12
|
-
validate_simple(expected_schema, logical_datum,
|
13
|
+
validate_simple(expected_schema, logical_datum,
|
14
|
+
Avro::SchemaValidator::ROOT_IDENTIFIER, result, options)
|
13
15
|
end
|
14
16
|
fail Avro::SchemaValidator::ValidationError, result if result.failure?
|
15
17
|
result
|
@@ -17,11 +19,11 @@ module AvroPatches
|
|
17
19
|
|
18
20
|
private
|
19
21
|
|
20
|
-
def validate_recursive(expected_schema, logical_datum, path, result,
|
21
|
-
datum = resolve_datum(expected_schema, logical_datum, encoded)
|
22
|
+
def validate_recursive(expected_schema, logical_datum, path, result, options = {})
|
23
|
+
datum = resolve_datum(expected_schema, logical_datum, options[:encoded])
|
22
24
|
|
23
25
|
# The entire method is overridden so that encoded: true can be passed here
|
24
|
-
validate_simple(expected_schema, datum, path, result, true)
|
26
|
+
validate_simple(expected_schema, datum, path, result, encoded: true)
|
25
27
|
|
26
28
|
case expected_schema.type_sym
|
27
29
|
when :array
|
@@ -36,13 +38,20 @@ module AvroPatches
|
|
36
38
|
deeper_path = deeper_path_for_hash(field.name, path)
|
37
39
|
validate_recursive(field.type, datum[field.name], deeper_path, result)
|
38
40
|
end
|
41
|
+
if options[:fail_on_extra_fields]
|
42
|
+
datum_fields = datum.keys.map(&:to_s)
|
43
|
+
schema_fields = expected_schema.fields.map(&:name)
|
44
|
+
(datum_fields - schema_fields).each do |extra_field|
|
45
|
+
result.add_error(path, "extra field '#{extra_field}' - not in schema")
|
46
|
+
end
|
47
|
+
end
|
39
48
|
end
|
40
49
|
rescue Avro::SchemaValidator::TypeMismatchError
|
41
50
|
result.add_error(path, "expected type #{expected_schema.type_sym}, got #{actual_value_message(datum)}")
|
42
51
|
end
|
43
52
|
|
44
|
-
def validate_simple(expected_schema, logical_datum, path, result,
|
45
|
-
datum = resolve_datum(expected_schema, logical_datum, encoded)
|
53
|
+
def validate_simple(expected_schema, logical_datum, path, result, options = {})
|
54
|
+
datum = resolve_datum(expected_schema, logical_datum, options[:encoded])
|
46
55
|
super(expected_schema, datum, path, result)
|
47
56
|
end
|
48
57
|
|
data/lib/avro-patches/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: avro-patches
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Salsify, Inc
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-07-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -83,15 +83,15 @@ dependencies:
|
|
83
83
|
description: Patches for the official Apache Avro Ruby implementation
|
84
84
|
email:
|
85
85
|
- engineering@salsify.com
|
86
|
-
executables:
|
87
|
-
- console
|
88
|
-
- setup
|
86
|
+
executables: []
|
89
87
|
extensions: []
|
90
88
|
extra_rdoc_files: []
|
91
89
|
files:
|
92
90
|
- ".gitignore"
|
93
91
|
- ".overcommit.yml"
|
94
92
|
- ".rspec"
|
93
|
+
- ".ruby-gemset"
|
94
|
+
- ".ruby-version"
|
95
95
|
- ".travis.yml"
|
96
96
|
- CHANGELOG.md
|
97
97
|
- Gemfile
|
@@ -99,9 +99,9 @@ files:
|
|
99
99
|
- README.md
|
100
100
|
- Rakefile
|
101
101
|
- avro-patches.gemspec
|
102
|
-
- bin/console
|
103
|
-
- bin/setup
|
104
102
|
- lib/avro-patches.rb
|
103
|
+
- lib/avro-patches/default_validation.rb
|
104
|
+
- lib/avro-patches/default_validation/schema.rb
|
105
105
|
- lib/avro-patches/ensure_encoding.rb
|
106
106
|
- lib/avro-patches/ensure_encoding/io.rb
|
107
107
|
- lib/avro-patches/logical_types.rb
|
@@ -140,7 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
140
140
|
version: '0'
|
141
141
|
requirements: []
|
142
142
|
rubyforge_project:
|
143
|
-
rubygems_version: 2.6
|
143
|
+
rubygems_version: 2.7.6
|
144
144
|
signing_key:
|
145
145
|
specification_version: 4
|
146
146
|
summary: Patches for the official Apache Avro Ruby implementation
|
data/bin/console
DELETED
@@ -1,14 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
require 'bundler/setup'
|
4
|
-
require 'avro-patches'
|
5
|
-
|
6
|
-
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
-
# with your gem easier. You can also use a different console, if you like.
|
8
|
-
|
9
|
-
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
-
# require "pry"
|
11
|
-
# Pry.start
|
12
|
-
|
13
|
-
require 'irb'
|
14
|
-
IRB.start
|