avro 1.10.1 → 1.10.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 25662687b72649ae6bbc6a18a975ada73965b60d
4
- data.tar.gz: 8ffb726920396bc1644c499440a1f7b844deb404
3
+ metadata.gz: 38a2f2d15367014464b4613e0fc80724b345d3b3
4
+ data.tar.gz: 823aa3ebb6ea719cbd4c94060757a545c7d626a9
5
5
  SHA512:
6
- metadata.gz: 0542c4933a9cd95411c76b26cb88e456c1cd733749eccff6125058ecbf64fd184f64f7547f0b24376b80ba3b8349937926dfbd85a159d7b51e587ec439e59ef8
7
- data.tar.gz: 76e8c8d2f1f30199a343fccfda5a4f6b972c6b33cef99b324a61a97554baebae59b62b8eec14682e6f10015d48c4740fb00f3c58fefdff39a313bdbbebbc9066
6
+ metadata.gz: da4d7d3ecf02bb18c30c044635188ba2b8a34ff13f9450330e850198610ee87f4e9116a79a95366bc23b5e3fb7f70906736d1ff7834e9293825c5c3eba072b9b
7
+ data.tar.gz: 3194b9379e1ae396acd0a120f583ade7127628d5adf78ac382fcf7215643c69c5341ee0a5f234db56eec3e62070683340b4a21bcff4ba0e2208a832c571f8dc9
data/avro.gemspec CHANGED
@@ -1,14 +1,14 @@
1
1
  # -*- encoding: utf-8 -*-
2
- # stub: avro 1.10.1 ruby lib
2
+ # stub: avro 1.10.2 ruby lib
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = "avro".freeze
6
- s.version = "1.10.1"
6
+ s.version = "1.10.2"
7
7
 
8
8
  s.required_rubygems_version = Gem::Requirement.new(">= 1.2".freeze) if s.respond_to? :required_rubygems_version=
9
9
  s.require_paths = ["lib".freeze]
10
10
  s.authors = ["Apache Software Foundation".freeze]
11
- s.date = "2020-11-18"
11
+ s.date = "2021-03-09"
12
12
  s.description = "Avro is a data serialization and RPC format".freeze
13
13
  s.email = "dev@avro.apache.org".freeze
14
14
  s.extra_rdoc_files = ["CHANGELOG".freeze, "LICENSE".freeze, "lib/avro.rb".freeze, "lib/avro/VERSION.txt".freeze, "lib/avro/data_file.rb".freeze, "lib/avro/io.rb".freeze, "lib/avro/ipc.rb".freeze, "lib/avro/logical_types.rb".freeze, "lib/avro/protocol.rb".freeze, "lib/avro/schema.rb".freeze, "lib/avro/schema_compatibility.rb".freeze, "lib/avro/schema_normalization.rb".freeze, "lib/avro/schema_validator.rb".freeze]
data/lib/avro/VERSION.txt CHANGED
@@ -1 +1 @@
1
- 1.10.1
1
+ 1.10.2
data/lib/avro/io.rb CHANGED
@@ -510,6 +510,8 @@ module Avro
510
510
 
511
511
  # DatumWriter for generic ruby objects
512
512
  class DatumWriter
513
+ VALIDATION_OPTIONS = { recursive: false, encoded: true }.freeze
514
+
513
515
  attr_accessor :writers_schema
514
516
  def initialize(writers_schema=nil)
515
517
  @writers_schema = writers_schema
@@ -522,7 +524,7 @@ module Avro
522
524
  def write_data(writers_schema, logical_datum, encoder)
523
525
  datum = writers_schema.type_adapter.encode(logical_datum)
524
526
 
525
- unless Schema.validate(writers_schema, datum, { recursive: false, encoded: true })
527
+ unless Schema.validate(writers_schema, datum, VALIDATION_OPTIONS)
526
528
  raise AvroTypeError.new(writers_schema, datum)
527
529
  end
528
530
 
@@ -580,12 +582,15 @@ module Avro
580
582
  end
581
583
 
582
584
  def write_union(writers_schema, datum, encoder)
583
- index_of_schema = -1
584
- found = writers_schema.schemas.
585
- find{|e| index_of_schema += 1; found = Schema.validate(e, datum) }
586
- unless found # Because find_index doesn't exist in 1.8.6
585
+ index_of_schema = writers_schema.schemas.find_index do |schema|
586
+ # Optimize away expensive validation calls for the common null type
587
+ schema.type_sym == :null ? datum.nil? : Schema.validate(schema, datum)
588
+ end
589
+
590
+ unless index_of_schema
587
591
  raise AvroTypeError.new(writers_schema, datum)
588
592
  end
593
+
589
594
  encoder.write_long(index_of_schema)
590
595
  write_data(writers_schema.schemas[index_of_schema], datum, encoder)
591
596
  end
data/lib/avro/schema.rb CHANGED
@@ -36,6 +36,8 @@ module Avro
36
36
  LONG_MIN_VALUE = -(1 << 63)
37
37
  LONG_MAX_VALUE = (1 << 63) - 1
38
38
 
39
+ DEFAULT_VALIDATE_OPTIONS = { recursive: true, encoded: false }.freeze
40
+
39
41
  def self.parse(json_string)
40
42
  real_parse(MultiJson.load(json_string), {})
41
43
  end
@@ -109,7 +111,7 @@ module Avro
109
111
  end
110
112
 
111
113
  # Determine if a ruby datum is an instance of a schema
112
- def self.validate(expected_schema, logical_datum, options = { recursive: true, encoded: false })
114
+ def self.validate(expected_schema, logical_datum, options = DEFAULT_VALIDATE_OPTIONS)
113
115
  SchemaValidator.validate!(expected_schema, logical_datum, options)
114
116
  true
115
117
  rescue SchemaValidator::ValidationError
@@ -15,6 +15,9 @@
15
15
  # limitations under the License.
16
16
  module Avro
17
17
  module SchemaCompatibility
18
+ INT_COERCIBLE_TYPES_SYM = [:long, :float, :double].freeze
19
+ LONG_COERCIBLE_TYPES_SYM = [:float, :double].freeze
20
+
18
21
  # Perform a full, recursive check that a datum written using the writers_schema
19
22
  # can be read using the readers_schema.
20
23
  def self.can_read?(writers_schema, readers_schema)
@@ -31,6 +34,9 @@ module Avro
31
34
  # be read using the readers_schema. This check includes matching the types,
32
35
  # including schema promotion, and matching the full name (including aliases) for named types.
33
36
  def self.match_schemas(writers_schema, readers_schema)
37
+ # Bypass deeper checks if the schemas are the same Ruby objects
38
+ return true if writers_schema.equal?(readers_schema)
39
+
34
40
  w_type = writers_schema.type_sym
35
41
  r_type = readers_schema.type_sym
36
42
 
@@ -62,9 +68,9 @@ module Avro
62
68
  end
63
69
 
64
70
  # Handle schema promotion
65
- if w_type == :int && [:long, :float, :double].include?(r_type)
71
+ if w_type == :int && INT_COERCIBLE_TYPES_SYM.include?(r_type)
66
72
  return true
67
- elsif w_type == :long && [:float, :double].include?(r_type)
73
+ elsif w_type == :long && LONG_COERCIBLE_TYPES_SYM.include?(r_type)
68
74
  return true
69
75
  elsif w_type == :float && r_type == :double
70
76
  return true
@@ -22,16 +22,18 @@ module Avro
22
22
  LONG_RANGE = Schema::LONG_MIN_VALUE..Schema::LONG_MAX_VALUE
23
23
  COMPLEX_TYPES = [:array, :error, :map, :record, :request].freeze
24
24
  BOOLEAN_VALUES = [true, false].freeze
25
+ DEFAULT_VALIDATION_OPTIONS = { recursive: true, encoded: false, fail_on_extra_fields: false }.freeze
26
+ RECURSIVE_SIMPLE_VALIDATION_OPTIONS = { encoded: true }.freeze
27
+ RUBY_CLASS_TO_AVRO_TYPE = {
28
+ NilClass => 'null'.freeze,
29
+ String => 'string'.freeze,
30
+ Float => 'float'.freeze,
31
+ Hash => 'record'.freeze
32
+ }.freeze
25
33
 
26
34
  class Result
27
- attr_reader :errors
28
-
29
- def initialize
30
- @errors = []
31
- end
32
-
33
35
  def <<(error)
34
- @errors << error
36
+ errors << error
35
37
  end
36
38
 
37
39
  def add_error(path, message)
@@ -39,11 +41,16 @@ module Avro
39
41
  end
40
42
 
41
43
  def failure?
42
- @errors.any?
44
+ defined?(@errors) && errors.any?
43
45
  end
44
46
 
45
47
  def to_s
46
- errors.join("\n")
48
+ failure? ? errors.join("\n") : ''
49
+ end
50
+
51
+ def errors
52
+ # Use less memory for success results by lazily creating the errors array
53
+ @errors ||= []
47
54
  end
48
55
  end
49
56
 
@@ -63,12 +70,9 @@ module Avro
63
70
  TypeMismatchError = Class.new(ValidationError)
64
71
 
65
72
  class << self
66
- def validate!(expected_schema, logical_datum, options = { recursive: true, encoded: false, fail_on_extra_fields: false })
67
- options ||= {}
68
- options[:recursive] = true unless options.key?(:recursive)
69
-
73
+ def validate!(expected_schema, logical_datum, options = DEFAULT_VALIDATION_OPTIONS)
70
74
  result = Result.new
71
- if options[:recursive]
75
+ if options.fetch(:recursive, true)
72
76
  validate_recursive(expected_schema, logical_datum, ROOT_IDENTIFIER, result, options)
73
77
  else
74
78
  validate_simple(expected_schema, logical_datum, ROOT_IDENTIFIER, result, options)
@@ -79,10 +83,10 @@ module Avro
79
83
 
80
84
  private
81
85
 
82
- def validate_recursive(expected_schema, logical_datum, path, result, options = {})
86
+ def validate_recursive(expected_schema, logical_datum, path, result, options)
83
87
  datum = resolve_datum(expected_schema, logical_datum, options[:encoded])
84
88
 
85
- validate_simple(expected_schema, datum, path, result, encoded: true)
89
+ validate_simple(expected_schema, datum, path, result, RECURSIVE_SIMPLE_VALIDATION_OPTIONS)
86
90
 
87
91
  case expected_schema.type_sym
88
92
  when :array
@@ -95,7 +99,8 @@ module Avro
95
99
  fail TypeMismatchError unless datum.is_a?(Hash)
96
100
  expected_schema.fields.each do |field|
97
101
  deeper_path = deeper_path_for_hash(field.name, path)
98
- validate_recursive(field.type, datum[field.name], deeper_path, result, options)
102
+ nested_value = datum.key?(field.name) ? datum[field.name] : datum[field.name.to_sym]
103
+ validate_recursive(field.type, nested_value, deeper_path, result, options)
99
104
  end
100
105
  if options[:fail_on_extra_fields]
101
106
  datum_fields = datum.keys.map(&:to_s)
@@ -109,7 +114,7 @@ module Avro
109
114
  result.add_error(path, "expected type #{expected_schema.type_sym}, got #{actual_value_message(datum)}")
110
115
  end
111
116
 
112
- def validate_simple(expected_schema, logical_datum, path, result, options = {})
117
+ def validate_simple(expected_schema, logical_datum, path, result, options)
113
118
  datum = resolve_datum(expected_schema, logical_datum, options[:encoded])
114
119
  validate_type(expected_schema)
115
120
 
@@ -163,14 +168,14 @@ module Avro
163
168
  "expected enum with values #{symbols}, got #{actual_value_message(datum)}"
164
169
  end
165
170
 
166
- def validate_array(expected_schema, datum, path, result, options = {})
171
+ def validate_array(expected_schema, datum, path, result, options)
167
172
  fail TypeMismatchError unless datum.is_a?(Array)
168
173
  datum.each_with_index do |d, i|
169
- validate_recursive(expected_schema.items, d, path + "[#{i}]", result, options)
174
+ validate_recursive(expected_schema.items, d, "#{path}[#{i}]", result, options)
170
175
  end
171
176
  end
172
177
 
173
- def validate_map(expected_schema, datum, path, result, options = {})
178
+ def validate_map(expected_schema, datum, path, result, options)
174
179
  fail TypeMismatchError unless datum.is_a?(Hash)
175
180
  datum.keys.each do |k|
176
181
  result.add_error(path, "unexpected key type '#{ruby_to_avro_type(k.class)}' in map") unless k.is_a?(String)
@@ -181,7 +186,7 @@ module Avro
181
186
  end
182
187
  end
183
188
 
184
- def validate_union(expected_schema, datum, path, result, options = {})
189
+ def validate_union(expected_schema, datum, path, result, options)
185
190
  if expected_schema.schemas.size == 1
186
191
  validate_recursive(expected_schema.schemas.first, datum, path, result, options)
187
192
  return
@@ -201,6 +206,9 @@ module Avro
201
206
 
202
207
  def first_compatible_type(datum, expected_schema, path, failures, options = {})
203
208
  expected_schema.schemas.find do |schema|
209
+ # Avoid expensive validation if we're just validating a nil
210
+ next datum.nil? if schema.type_sym == :null
211
+
204
212
  result = Result.new
205
213
  validate_recursive(schema, datum, path, result, options)
206
214
  failures << { type: schema.type_sym, result: result } if result.failure?
@@ -209,7 +217,9 @@ module Avro
209
217
  end
210
218
 
211
219
  def deeper_path_for_hash(sub_key, path)
212
- "#{path}#{PATH_SEPARATOR}#{sub_key}".squeeze(PATH_SEPARATOR)
220
+ deeper_path = "#{path}#{PATH_SEPARATOR}#{sub_key}"
221
+ deeper_path.squeeze!(PATH_SEPARATOR)
222
+ deeper_path
213
223
  end
214
224
 
215
225
  def actual_value_message(value)
@@ -226,16 +236,11 @@ module Avro
226
236
  end
227
237
 
228
238
  def ruby_to_avro_type(ruby_class)
229
- {
230
- NilClass => 'null',
231
- String => 'string',
232
- Float => 'float',
233
- Hash => 'record'
234
- }.fetch(ruby_class, ruby_class)
239
+ RUBY_CLASS_TO_AVRO_TYPE.fetch(ruby_class, ruby_class)
235
240
  end
236
241
 
237
242
  def ruby_integer_to_avro_type(value)
238
- INT_RANGE.cover?(value) ? 'int' : 'long'
243
+ INT_RANGE.cover?(value) ? 'int'.freeze : 'long'.freeze
239
244
  end
240
245
  end
241
246
  end
@@ -25,7 +25,9 @@ class TestSchemaCompatibility < Test::Unit::TestCase
25
25
  end
26
26
 
27
27
  def test_compatible_reader_writer_pairs
28
+ cached_schema = a_int_record1_schema
28
29
  [
30
+ cached_schema, cached_schema,
29
31
  long_schema, int_schema,
30
32
  float_schema, int_schema,
31
33
  float_schema, long_schema,
@@ -17,7 +17,7 @@
17
17
  require 'test_help'
18
18
 
19
19
  class TestSchemaValidator < Test::Unit::TestCase
20
- def validate!(schema, value, options=nil)
20
+ def validate!(schema, value, options = {})
21
21
  Avro::SchemaValidator.validate!(schema, value, options)
22
22
  end
23
23
 
@@ -196,6 +196,12 @@ class TestSchemaValidator < Test::Unit::TestCase
196
196
  assert_valid_schema(schema, [{ 'sub' => nil }], [{ 'sub' => 1 }])
197
197
  end
198
198
 
199
+ def test_validate_record_with_symbol_keys
200
+ schema = hash_to_schema(type: 'record', name: 'name', fields: [{ type: 'int', name: 'sub' }])
201
+
202
+ assert_valid_schema(schema, [{ sub: 1 }], [{ sub: '1' }])
203
+ end
204
+
199
205
  def test_validate_shallow_record
200
206
  schema = hash_to_schema(
201
207
  type: 'record', name: 'name', fields: [{ type: 'int', name: 'sub' }]
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: avro
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.10.1
4
+ version: 1.10.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Apache Software Foundation
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-11-18 00:00:00.000000000 Z
11
+ date: 2021-03-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: multi_json