avro 1.8.2 → 1.9.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 2725b644b8e1c2579bb7aa83a4f60a3f14486122
4
- data.tar.gz: ce3c78d318872ff111c58d9cfab710fa207006ad
3
+ metadata.gz: 96c7660634a5f7e85c04d6895dc94ae20a2ea9c8
4
+ data.tar.gz: 4e68260b9070cc565aa152988d9809a3f7671772
5
5
  SHA512:
6
- metadata.gz: 54a416f6e98d93cf33de8d6ac2d6fac40337793597cd010e98576501880ca4377398aefb4fc07f7e7d4dd7859eb062ec78e923ebf7d45b8c3084d25c4d389be8
7
- data.tar.gz: 224c3f0ee1cc0f52bda339f446caf4c66c31fb1200bb9af4fc4ea1b4daa4e5087f5749a524140300ce9271cdf406cac94f33f1d31a3658583e9ff16ec168b995
6
+ metadata.gz: '064323997a22d0341e21d93bda5855a27d3859e8f247a85032854198e00330a278912f12cf6037fb374e1e322b009bd902fdff112ededb78b994f6978d6db020'
7
+ data.tar.gz: ecdec673bb86ca8d8571198eebf0bada327f71134ddc176dbc327bdea27be31b62838f37a5952edaf33398f21591677268f4b15bee28176cac88b839b8df44b6
data/LICENSE CHANGED
@@ -1,7 +1,7 @@
1
1
 
2
2
  Apache License
3
3
  Version 2.0, January 2004
4
- http://www.apache.org/licenses/
4
+ https://www.apache.org/licenses/
5
5
 
6
6
  TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7
7
 
@@ -193,7 +193,7 @@
193
193
  you may not use this file except in compliance with the License.
194
194
  You may obtain a copy of the License at
195
195
 
196
- http://www.apache.org/licenses/LICENSE-2.0
196
+ https://www.apache.org/licenses/LICENSE-2.0
197
197
 
198
198
  Unless required by applicable law or agreed to in writing, software
199
199
  distributed under the License is distributed on an "AS IS" BASIS,
data/Manifest CHANGED
@@ -9,9 +9,12 @@ lib/avro.rb
9
9
  lib/avro/data_file.rb
10
10
  lib/avro/io.rb
11
11
  lib/avro/ipc.rb
12
+ lib/avro/logical_types.rb
12
13
  lib/avro/protocol.rb
13
14
  lib/avro/schema.rb
15
+ lib/avro/schema_compatibility.rb
14
16
  lib/avro/schema_normalization.rb
17
+ lib/avro/schema_validator.rb
15
18
  test/case_finder.rb
16
19
  test/random_data.rb
17
20
  test/sample_ipc_client.rb
@@ -22,8 +25,11 @@ test/test_datafile.rb
22
25
  test/test_fingerprints.rb
23
26
  test/test_help.rb
24
27
  test/test_io.rb
28
+ test/test_logical_types.rb
25
29
  test/test_protocol.rb
26
30
  test/test_schema.rb
31
+ test/test_schema_compatibility.rb
27
32
  test/test_schema_normalization.rb
33
+ test/test_schema_validator.rb
28
34
  test/test_socket_transport.rb
29
35
  test/tool.rb
data/NOTICE CHANGED
@@ -2,5 +2,5 @@ Apache Avro
2
2
  Copyright 2010-2015 The Apache Software Foundation
3
3
 
4
4
  This product includes software developed at
5
- The Apache Software Foundation (http://www.apache.org/).
5
+ The Apache Software Foundation (https://www.apache.org/).
6
6
 
data/Rakefile CHANGED
@@ -6,7 +6,7 @@
6
6
  # "License"); you may not use this file except in compliance
7
7
  # with the License. You may obtain a copy of the License at
8
8
  #
9
- # http://www.apache.org/licenses/LICENSE-2.0
9
+ # https://www.apache.org/licenses/LICENSE-2.0
10
10
  #
11
11
  # Unless required by applicable law or agreed to in writing, software
12
12
  # distributed under the License is distributed on an "AS IS" BASIS,
@@ -22,7 +22,7 @@ Echoe.new('avro', VERSION) do |p|
22
22
  p.email = "dev@avro.apache.org"
23
23
  p.summary = "Apache Avro for Ruby"
24
24
  p.description = "Avro is a data serialization and RPC format"
25
- p.url = "http://avro.apache.org/"
25
+ p.url = "https://avro.apache.org/"
26
26
  p.runtime_dependencies = %w[multi_json]
27
27
  p.licenses = ["Apache License 2.0 (Apache-2.0)"]
28
28
  end
@@ -1,35 +1,35 @@
1
1
  # -*- encoding: utf-8 -*-
2
- # stub: avro 1.8.2 ruby lib
2
+ # stub: avro 1.9.1 ruby lib
3
3
 
4
4
  Gem::Specification.new do |s|
5
- s.name = "avro"
6
- s.version = "1.8.2"
5
+ s.name = "avro".freeze
6
+ s.version = "1.9.1"
7
7
 
8
- s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
9
- s.require_paths = ["lib"]
10
- s.authors = ["Apache Software Foundation"]
11
- s.date = "2017-05-07"
12
- s.description = "Avro is a data serialization and RPC format"
13
- s.email = "dev@avro.apache.org"
14
- s.extra_rdoc_files = ["CHANGELOG", "LICENSE", "lib/avro.rb", "lib/avro/data_file.rb", "lib/avro/io.rb", "lib/avro/ipc.rb", "lib/avro/protocol.rb", "lib/avro/schema.rb", "lib/avro/schema_normalization.rb"]
15
- s.files = ["CHANGELOG", "LICENSE", "Manifest", "NOTICE", "Rakefile", "avro.gemspec", "interop/test_interop.rb", "lib/avro.rb", "lib/avro/data_file.rb", "lib/avro/io.rb", "lib/avro/ipc.rb", "lib/avro/protocol.rb", "lib/avro/schema.rb", "lib/avro/schema_normalization.rb", "test/case_finder.rb", "test/random_data.rb", "test/sample_ipc_client.rb", "test/sample_ipc_http_client.rb", "test/sample_ipc_http_server.rb", "test/sample_ipc_server.rb", "test/test_datafile.rb", "test/test_fingerprints.rb", "test/test_help.rb", "test/test_io.rb", "test/test_protocol.rb", "test/test_schema.rb", "test/test_schema_normalization.rb", "test/test_socket_transport.rb", "test/tool.rb"]
16
- s.homepage = "http://avro.apache.org/"
17
- s.licenses = ["Apache License 2.0 (Apache-2.0)"]
18
- s.rdoc_options = ["--line-numbers", "--title", "Avro"]
19
- s.rubyforge_project = "avro"
20
- s.rubygems_version = "2.2.2"
21
- s.summary = "Apache Avro for Ruby"
22
- s.test_files = ["test/test_datafile.rb", "test/test_fingerprints.rb", "test/test_help.rb", "test/test_io.rb", "test/test_protocol.rb", "test/test_schema.rb", "test/test_schema_normalization.rb", "test/test_socket_transport.rb"]
8
+ s.required_rubygems_version = Gem::Requirement.new(">= 1.2".freeze) if s.respond_to? :required_rubygems_version=
9
+ s.require_paths = ["lib".freeze]
10
+ s.authors = ["Apache Software Foundation".freeze]
11
+ s.date = "2019-08-28"
12
+ s.description = "Avro is a data serialization and RPC format".freeze
13
+ s.email = "dev@avro.apache.org".freeze
14
+ s.extra_rdoc_files = ["CHANGELOG".freeze, "LICENSE".freeze, "lib/avro.rb".freeze, "lib/avro/data_file.rb".freeze, "lib/avro/io.rb".freeze, "lib/avro/ipc.rb".freeze, "lib/avro/logical_types.rb".freeze, "lib/avro/protocol.rb".freeze, "lib/avro/schema.rb".freeze, "lib/avro/schema_compatibility.rb".freeze, "lib/avro/schema_normalization.rb".freeze, "lib/avro/schema_validator.rb".freeze]
15
+ s.files = ["CHANGELOG".freeze, "LICENSE".freeze, "Manifest".freeze, "NOTICE".freeze, "Rakefile".freeze, "avro.gemspec".freeze, "interop/test_interop.rb".freeze, "lib/avro.rb".freeze, "lib/avro/data_file.rb".freeze, "lib/avro/io.rb".freeze, "lib/avro/ipc.rb".freeze, "lib/avro/logical_types.rb".freeze, "lib/avro/protocol.rb".freeze, "lib/avro/schema.rb".freeze, "lib/avro/schema_compatibility.rb".freeze, "lib/avro/schema_normalization.rb".freeze, "lib/avro/schema_validator.rb".freeze, "test/case_finder.rb".freeze, "test/random_data.rb".freeze, "test/sample_ipc_client.rb".freeze, "test/sample_ipc_http_client.rb".freeze, "test/sample_ipc_http_server.rb".freeze, "test/sample_ipc_server.rb".freeze, "test/test_datafile.rb".freeze, "test/test_fingerprints.rb".freeze, "test/test_help.rb".freeze, "test/test_io.rb".freeze, "test/test_logical_types.rb".freeze, "test/test_protocol.rb".freeze, "test/test_schema.rb".freeze, "test/test_schema_compatibility.rb".freeze, "test/test_schema_normalization.rb".freeze, "test/test_schema_validator.rb".freeze, "test/test_socket_transport.rb".freeze, "test/tool.rb".freeze]
16
+ s.homepage = "https://avro.apache.org/".freeze
17
+ s.licenses = ["Apache License 2.0 (Apache-2.0)".freeze]
18
+ s.rdoc_options = ["--line-numbers".freeze, "--title".freeze, "Avro".freeze]
19
+ s.rubyforge_project = "avro".freeze
20
+ s.rubygems_version = "2.5.2.1".freeze
21
+ s.summary = "Apache Avro for Ruby".freeze
22
+ s.test_files = ["test/test_schema_validator.rb".freeze, "test/test_help.rb".freeze, "test/test_schema_normalization.rb".freeze, "test/test_datafile.rb".freeze, "test/test_schema.rb".freeze, "test/test_io.rb".freeze, "test/test_socket_transport.rb".freeze, "test/test_schema_compatibility.rb".freeze, "test/test_logical_types.rb".freeze, "test/test_fingerprints.rb".freeze, "test/test_protocol.rb".freeze]
23
23
 
24
24
  if s.respond_to? :specification_version then
25
25
  s.specification_version = 4
26
26
 
27
27
  if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
28
- s.add_runtime_dependency(%q<multi_json>, [">= 0"])
28
+ s.add_runtime_dependency(%q<multi_json>.freeze, [">= 0"])
29
29
  else
30
- s.add_dependency(%q<multi_json>, [">= 0"])
30
+ s.add_dependency(%q<multi_json>.freeze, [">= 0"])
31
31
  end
32
32
  else
33
- s.add_dependency(%q<multi_json>, [">= 0"])
33
+ s.add_dependency(%q<multi_json>.freeze, [">= 0"])
34
34
  end
35
35
  end
@@ -7,7 +7,7 @@
7
7
  # "License"); you may not use this file except in compliance
8
8
  # with the License. You may obtain a copy of the License at
9
9
  #
10
- # http://www.apache.org/licenses/LICENSE-2.0
10
+ # https://www.apache.org/licenses/LICENSE-2.0
11
11
  #
12
12
  # Unless required by applicable law or agreed to in writing, software
13
13
  # distributed under the License is distributed on an "AS IS" BASIS,
@@ -6,7 +6,7 @@
6
6
  # "License"); you may not use this file except in compliance
7
7
  # with the License. You may obtain a copy of the License at
8
8
  #
9
- # http://www.apache.org/licenses/LICENSE-2.0
9
+ # https://www.apache.org/licenses/LICENSE-2.0
10
10
  #
11
11
  # Unless required by applicable law or agreed to in writing, software
12
12
  # distributed under the License is distributed on an "AS IS" BASIS,
@@ -32,6 +32,15 @@ module Avro
32
32
  super(msg)
33
33
  end
34
34
  end
35
+
36
+ class << self
37
+ attr_writer :disable_field_default_validation
38
+
39
+ def disable_field_default_validation
40
+ @disable_field_default_validation ||=
41
+ ENV.fetch('AVRO_DISABLE_FIELD_DEFAULT_VALIDATION', '') != ''
42
+ end
43
+ end
35
44
  end
36
45
 
37
46
  require 'avro/schema'
@@ -40,3 +49,5 @@ require 'avro/data_file'
40
49
  require 'avro/protocol'
41
50
  require 'avro/ipc'
42
51
  require 'avro/schema_normalization'
52
+ require 'avro/schema_validator'
53
+ require 'avro/schema_compatibility'
@@ -6,7 +6,7 @@
6
6
  # "License"); you may not use this file except in compliance
7
7
  # with the License. You may obtain a copy of the License at
8
8
  #
9
- # http://www.apache.org/licenses/LICENSE-2.0
9
+ # https://www.apache.org/licenses/LICENSE-2.0
10
10
  #
11
11
  # Unless required by applicable law or agreed to in writing, software
12
12
  # distributed under the License is distributed on an "AS IS" BASIS,
@@ -316,7 +316,7 @@ module Avro
316
316
  def decompress(compressed)
317
317
  # Passing a negative number to Inflate puts it into "raw" RFC1951 mode
318
318
  # (without the RFC1950 header & checksum). See the docs for
319
- # inflateInit2 in http://www.zlib.net/manual.html
319
+ # inflateInit2 in https://www.zlib.net/manual.html
320
320
  zstream = Zlib::Inflate.new(-Zlib::MAX_WBITS)
321
321
  data = zstream.inflate(compressed)
322
322
  data << zstream.finish
@@ -5,9 +5,9 @@
5
5
  # to you under the Apache License, Version 2.0 (the
6
6
  # "License"); you may not use this file except in compliance
7
7
  # with the License. You may obtain a copy of the License at
8
- #
9
- # http://www.apache.org/licenses/LICENSE-2.0
10
- #
8
+ #
9
+ # https://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
11
  # Unless required by applicable law or agreed to in writing, software
12
12
  # distributed under the License is distributed on an "AS IS" BASIS,
13
13
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -43,9 +43,9 @@ module Avro
43
43
  end
44
44
 
45
45
  def byte!
46
- @reader.read(1).unpack('C').first
46
+ @reader.readbyte
47
47
  end
48
-
48
+
49
49
  def read_null
50
50
  # null is written as zero byte's
51
51
  nil
@@ -76,7 +76,7 @@ module Avro
76
76
  # The float is converted into a 32-bit integer using a method
77
77
  # equivalent to Java's floatToIntBits and then encoded in
78
78
  # little-endian format.
79
- @reader.read(4).unpack('e')[0]
79
+ read_and_unpack(4, 'e'.freeze)
80
80
  end
81
81
 
82
82
  def read_double
@@ -84,7 +84,7 @@ module Avro
84
84
  # The double is converted into a 64-bit integer using a method
85
85
  # equivalent to Java's doubleToLongBits and then encoded in
86
86
  # little-endian format.
87
- @reader.read(8).unpack('E')[0]
87
+ read_and_unpack(8, 'E'.freeze)
88
88
  end
89
89
 
90
90
  def read_bytes
@@ -97,7 +97,7 @@ module Avro
97
97
  # A string is encoded as a long followed by that many bytes of
98
98
  # UTF-8 encoded character data.
99
99
  read_bytes.tap do |string|
100
- string.force_encoding("UTF-8") if string.respond_to? :force_encoding
100
+ string.force_encoding('UTF-8'.freeze) if string.respond_to? :force_encoding
101
101
  end
102
102
  end
103
103
 
@@ -144,6 +144,23 @@ module Avro
144
144
  def skip(n)
145
145
  reader.seek(reader.tell() + n)
146
146
  end
147
+
148
+ private
149
+
150
+ # Optimize unpacking strings when `unpack1` is available (ruby >= 2.4)
151
+ if String.instance_methods.include?(:unpack1)
152
+
153
+ def read_and_unpack(byte_count, format)
154
+ @reader.read(byte_count).unpack1(format)
155
+ end
156
+
157
+ else
158
+
159
+ def read_and_unpack(byte_count, format)
160
+ @reader.read(byte_count).unpack(format)[0]
161
+ end
162
+
163
+ end
147
164
  end
148
165
 
149
166
  # Write leaf values
@@ -159,7 +176,7 @@ module Avro
159
176
  nil
160
177
  end
161
178
 
162
- # a boolean is written as a single byte
179
+ # a boolean is written as a single byte
163
180
  # whose value is either 0 (false) or 1 (true).
164
181
  def write_boolean(datum)
165
182
  on_disk = datum ? 1.chr : 0.chr
@@ -175,7 +192,6 @@ module Avro
175
192
  # int and long values are written using variable-length,
176
193
  # zig-zag coding.
177
194
  def write_long(n)
178
- foo = n
179
195
  n = (n << 1) ^ (n >> 63)
180
196
  while (n & ~0x7F) != 0
181
197
  @writer.write(((n & 0x7f) | 0x80).chr)
@@ -189,7 +205,7 @@ module Avro
189
205
  # equivalent to Java's floatToIntBits and then encoded in
190
206
  # little-endian format.
191
207
  def write_float(datum)
192
- @writer.write([datum].pack('e'))
208
+ @writer.write([datum].pack('e'.freeze))
193
209
  end
194
210
 
195
211
  # A double is written as 8 bytes.
@@ -197,7 +213,7 @@ module Avro
197
213
  # equivalent to Java's doubleToLongBits and then encoded in
198
214
  # little-endian format.
199
215
  def write_double(datum)
200
- @writer.write([datum].pack('E'))
216
+ @writer.write([datum].pack('E'.freeze))
201
217
  end
202
218
 
203
219
  # Bytes are encoded as a long followed by that many bytes of data.
@@ -209,7 +225,7 @@ module Avro
209
225
  # A string is encoded as a long followed by that many bytes of
210
226
  # UTF-8 encoded character data
211
227
  def write_string(datum)
212
- datum = datum.encode('utf-8') if datum.respond_to? :encode
228
+ datum = datum.encode('utf-8'.freeze) if datum.respond_to? :encode
213
229
  write_bytes(datum)
214
230
  end
215
231
 
@@ -221,46 +237,7 @@ module Avro
221
237
 
222
238
  class DatumReader
223
239
  def self.match_schemas(writers_schema, readers_schema)
224
- w_type = writers_schema.type_sym
225
- r_type = readers_schema.type_sym
226
-
227
- # This conditional is begging for some OO love.
228
- if w_type == :union || r_type == :union
229
- return true
230
- end
231
-
232
- if w_type == r_type
233
- return true if Schema::PRIMITIVE_TYPES_SYM.include?(r_type)
234
-
235
- case r_type
236
- when :record
237
- return writers_schema.fullname == readers_schema.fullname
238
- when :error
239
- return writers_schema.fullname == readers_schema.fullname
240
- when :request
241
- return true
242
- when :fixed
243
- return writers_schema.fullname == readers_schema.fullname &&
244
- writers_schema.size == readers_schema.size
245
- when :enum
246
- return writers_schema.fullname == readers_schema.fullname
247
- when :map
248
- return writers_schema.values.type == readers_schema.values.type
249
- when :array
250
- return writers_schema.items.type == readers_schema.items.type
251
- end
252
- end
253
-
254
- # Handle schema promotion
255
- if w_type == :int && [:long, :float, :double].include?(r_type)
256
- return true
257
- elsif w_type == :long && [:float, :double].include?(r_type)
258
- return true
259
- elsif w_type == :float && r_type == :double
260
- return true
261
- end
262
-
263
- return false
240
+ Avro::SchemaCompatibility.match_schemas(writers_schema, readers_schema)
264
241
  end
265
242
 
266
243
  attr_accessor :writers_schema, :readers_schema
@@ -293,7 +270,7 @@ module Avro
293
270
 
294
271
  # function dispatch for reading data based on type of writer's
295
272
  # schema
296
- case writers_schema.type_sym
273
+ datum = case writers_schema.type_sym
297
274
  when :null; decoder.read_null
298
275
  when :boolean; decoder.read_boolean
299
276
  when :string; decoder.read_string
@@ -311,6 +288,8 @@ module Avro
311
288
  else
312
289
  raise AvroError, "Cannot read unknown schema type: #{writers_schema.type}"
313
290
  end
291
+
292
+ readers_schema.type_adapter.decode(datum)
314
293
  end
315
294
 
316
295
  def read_fixed(writers_schema, readers_schema, decoder)
@@ -336,7 +315,7 @@ module Avro
336
315
  while block_count != 0
337
316
  if block_count < 0
338
317
  block_count = -block_count
339
- block_size = decoder.read_long
318
+ _block_size = decoder.read_long
340
319
  end
341
320
  block_count.times do
342
321
  read_items << read_data(writers_schema.items,
@@ -355,7 +334,7 @@ module Avro
355
334
  while block_count != 0
356
335
  if block_count < 0
357
336
  block_count = -block_count
358
- block_size = decoder.read_long
337
+ _block_size = decoder.read_long
359
338
  end
360
339
  block_count.times do
361
340
  key = decoder.read_string
@@ -393,11 +372,11 @@ module Avro
393
372
  writers_fields_hash = writers_schema.fields_hash
394
373
  readers_fields_hash.each do |field_name, field|
395
374
  unless writers_fields_hash.has_key? field_name
396
- if !field.default.nil?
375
+ if field.default?
397
376
  field_val = read_default_value(field.type, field.default)
398
377
  read_record[field.name] = field_val
399
378
  else
400
- # FIXME(jmhodges) another 'unset' here
379
+ raise AvroError, "Missing data for #{field.type} with no default"
401
380
  end
402
381
  end
403
382
  end
@@ -407,10 +386,6 @@ module Avro
407
386
  end
408
387
 
409
388
  def read_default_value(field_schema, default_value)
410
- if default_value == :no_default
411
- raise AvroError, "Missing data for #{field_schema} with no default"
412
- end
413
-
414
389
  # Basically a JSON Decoder?
415
390
  case field_schema.type_sym
416
391
  when :null
@@ -524,7 +499,7 @@ module Avro
524
499
  if block_count < 0
525
500
  decoder.skip(decoder.read_long)
526
501
  else
527
- block_count.times &blk
502
+ block_count.times(&blk)
528
503
  end
529
504
  block_count = decoder.read_long
530
505
  end
@@ -542,8 +517,10 @@ module Avro
542
517
  write_data(writers_schema, datum, encoder)
543
518
  end
544
519
 
545
- def write_data(writers_schema, datum, encoder)
546
- unless Schema.validate(writers_schema, datum)
520
+ def write_data(writers_schema, logical_datum, encoder)
521
+ datum = writers_schema.type_adapter.encode(logical_datum)
522
+
523
+ unless Schema.validate(writers_schema, datum, { recursive: false, encoded: true })
547
524
  raise AvroTypeError.new(writers_schema, datum)
548
525
  end
549
526
 
@@ -578,6 +555,7 @@ module Avro
578
555
  end
579
556
 
580
557
  def write_array(writers_schema, datum, encoder)
558
+ raise AvroTypeError.new(writers_schema, datum) unless datum.is_a?(Array)
581
559
  if datum.size > 0
582
560
  encoder.write_long(datum.size)
583
561
  datum.each do |item|
@@ -588,6 +566,7 @@ module Avro
588
566
  end
589
567
 
590
568
  def write_map(writers_schema, datum, encoder)
569
+ raise AvroTypeError.new(writers_schema, datum) unless datum.is_a?(Hash)
591
570
  if datum.size > 0
592
571
  encoder.write_long(datum.size)
593
572
  datum.each do |k,v|
@@ -610,6 +589,7 @@ module Avro
610
589
  end
611
590
 
612
591
  def write_record(writers_schema, datum, encoder)
592
+ raise AvroTypeError.new(writers_schema, datum) unless datum.is_a?(Hash)
613
593
  writers_schema.fields.each do |field|
614
594
  write_data(field.type, datum[field.name], encoder)
615
595
  end