avro 1.8.1 → 1.8.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/avro.gemspec +4 -4
- data/lib/avro/data_file.rb +18 -1
- data/lib/avro/io.rb +4 -0
- data/lib/avro/schema.rb +3 -3
- data/test/test_io.rb +32 -0
- data/test/test_schema.rb +17 -0
- metadata +7 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2725b644b8e1c2579bb7aa83a4f60a3f14486122
|
4
|
+
data.tar.gz: ce3c78d318872ff111c58d9cfab710fa207006ad
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 54a416f6e98d93cf33de8d6ac2d6fac40337793597cd010e98576501880ca4377398aefb4fc07f7e7d4dd7859eb062ec78e923ebf7d45b8c3084d25c4d389be8
|
7
|
+
data.tar.gz: 224c3f0ee1cc0f52bda339f446caf4c66c31fb1200bb9af4fc4ea1b4daa4e5087f5749a524140300ce9271cdf406cac94f33f1d31a3658583e9ff16ec168b995
|
data/avro.gemspec
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
|
-
# stub: avro 1.8.
|
2
|
+
# stub: avro 1.8.2 ruby lib
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "avro"
|
6
|
-
s.version = "1.8.
|
6
|
+
s.version = "1.8.2"
|
7
7
|
|
8
8
|
s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
|
9
9
|
s.require_paths = ["lib"]
|
10
10
|
s.authors = ["Apache Software Foundation"]
|
11
|
-
s.date = "
|
11
|
+
s.date = "2017-05-07"
|
12
12
|
s.description = "Avro is a data serialization and RPC format"
|
13
13
|
s.email = "dev@avro.apache.org"
|
14
14
|
s.extra_rdoc_files = ["CHANGELOG", "LICENSE", "lib/avro.rb", "lib/avro/data_file.rb", "lib/avro/io.rb", "lib/avro/ipc.rb", "lib/avro/protocol.rb", "lib/avro/schema.rb", "lib/avro/schema_normalization.rb"]
|
@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
|
|
19
19
|
s.rubyforge_project = "avro"
|
20
20
|
s.rubygems_version = "2.2.2"
|
21
21
|
s.summary = "Apache Avro for Ruby"
|
22
|
-
s.test_files = ["test/
|
22
|
+
s.test_files = ["test/test_datafile.rb", "test/test_fingerprints.rb", "test/test_help.rb", "test/test_io.rb", "test/test_protocol.rb", "test/test_schema.rb", "test/test_schema_normalization.rb", "test/test_socket_transport.rb"]
|
23
23
|
|
24
24
|
if s.respond_to? :specification_version then
|
25
25
|
s.specification_version = 4
|
data/lib/avro/data_file.rb
CHANGED
@@ -338,12 +338,29 @@ module Avro
|
|
338
338
|
|
339
339
|
def decompress(data)
|
340
340
|
load_snappy!
|
341
|
+
crc32 = data.slice(-4..-1).unpack('N').first
|
342
|
+
uncompressed = Snappy.inflate(data.slice(0..-5))
|
343
|
+
|
344
|
+
if crc32 == Zlib.crc32(uncompressed)
|
345
|
+
uncompressed
|
346
|
+
else
|
347
|
+
# older versions of avro-ruby didn't write the checksum, so if it
|
348
|
+
# doesn't match this must assume that it wasn't there and return
|
349
|
+
# the entire payload uncompressed.
|
350
|
+
Snappy.inflate(data)
|
351
|
+
end
|
352
|
+
rescue Snappy::Error
|
353
|
+
# older versions of avro-ruby didn't write the checksum, so removing
|
354
|
+
# the last 4 bytes may cause Snappy to fail. recover by assuming the
|
355
|
+
# payload is from an older file and uncompress the entire buffer.
|
341
356
|
Snappy.inflate(data)
|
342
357
|
end
|
343
358
|
|
344
359
|
def compress(data)
|
345
360
|
load_snappy!
|
346
|
-
|
361
|
+
crc32 = Zlib.crc32(data)
|
362
|
+
compressed = Snappy.deflate(data)
|
363
|
+
[compressed, crc32].pack('a*N')
|
347
364
|
end
|
348
365
|
|
349
366
|
private
|
data/lib/avro/io.rb
CHANGED
@@ -407,6 +407,10 @@ module Avro
|
|
407
407
|
end
|
408
408
|
|
409
409
|
def read_default_value(field_schema, default_value)
|
410
|
+
if default_value == :no_default
|
411
|
+
raise AvroError, "Missing data for #{field_schema} with no default"
|
412
|
+
end
|
413
|
+
|
410
414
|
# Basically a JSON Decoder?
|
411
415
|
case field_schema.type_sym
|
412
416
|
when :null
|
data/lib/avro/schema.rb
CHANGED
@@ -211,7 +211,7 @@ module Avro
|
|
211
211
|
if field.respond_to?(:[]) # TODO(jmhodges) wtffffff
|
212
212
|
type = field['type']
|
213
213
|
name = field['name']
|
214
|
-
default = field['default']
|
214
|
+
default = field.key?('default') ? field['default'] : :no_default
|
215
215
|
order = field['order']
|
216
216
|
new_field = Field.new(type, name, default, order, names, namespace)
|
217
217
|
# make sure field name has not been used yet
|
@@ -363,7 +363,7 @@ module Avro
|
|
363
363
|
class Field < Schema
|
364
364
|
attr_reader :type, :name, :default, :order
|
365
365
|
|
366
|
-
def initialize(type, name, default
|
366
|
+
def initialize(type, name, default=:no_default, order=nil, names=nil, namespace=nil)
|
367
367
|
@type = subparse(type, names, namespace)
|
368
368
|
@name = name
|
369
369
|
@default = default
|
@@ -372,7 +372,7 @@ module Avro
|
|
372
372
|
|
373
373
|
def to_avro(names=Set.new)
|
374
374
|
{'name' => name, 'type' => type.to_avro(names)}.tap do |avro|
|
375
|
-
avro['default'] = default
|
375
|
+
avro['default'] = default unless default == :no_default
|
376
376
|
avro['order'] = order if order
|
377
377
|
end
|
378
378
|
end
|
data/test/test_io.rb
CHANGED
@@ -340,8 +340,37 @@ EOS
|
|
340
340
|
assert_equal(incorrect, 0)
|
341
341
|
end
|
342
342
|
end
|
343
|
+
|
344
|
+
def test_snappy_backward_compat
|
345
|
+
# a snappy-compressed block payload without the checksum
|
346
|
+
# this has no back-references, just one literal so the last 9
|
347
|
+
# bytes are the uncompressed payload.
|
348
|
+
old_snappy_bytes = "\x09\x20\x02\x06\x02\x0a\x67\x72\x65\x65\x6e"
|
349
|
+
uncompressed_bytes = "\x02\x06\x02\x0a\x67\x72\x65\x65\x6e"
|
350
|
+
snappy = Avro::DataFile::SnappyCodec.new
|
351
|
+
assert_equal(uncompressed_bytes, snappy.decompress(old_snappy_bytes))
|
352
|
+
end
|
353
|
+
|
343
354
|
private
|
344
355
|
|
356
|
+
def check_no_default(schema_json)
|
357
|
+
actual_schema = '{"type": "record", "name": "Foo", "fields": []}'
|
358
|
+
actual = Avro::Schema.parse(actual_schema)
|
359
|
+
|
360
|
+
expected_schema = <<EOS
|
361
|
+
{"type": "record",
|
362
|
+
"name": "Foo",
|
363
|
+
"fields": [{"name": "f", "type": #{schema_json}}]}
|
364
|
+
EOS
|
365
|
+
expected = Avro::Schema.parse(expected_schema)
|
366
|
+
|
367
|
+
reader = Avro::IO::DatumReader.new(actual, expected)
|
368
|
+
assert_raise Avro::AvroError do
|
369
|
+
value = reader.read(Avro::IO::BinaryDecoder.new(StringIO.new))
|
370
|
+
assert_not_equal(value, :no_default) # should never return this
|
371
|
+
end
|
372
|
+
end
|
373
|
+
|
345
374
|
def check_default(schema_json, default_json, default_value)
|
346
375
|
actual_schema = '{"type": "record", "name": "Foo", "fields": []}'
|
347
376
|
actual = Avro::Schema.parse(actual_schema)
|
@@ -381,6 +410,9 @@ EOS
|
|
381
410
|
|
382
411
|
# test writing of data to file
|
383
412
|
check_datafile(schema)
|
413
|
+
|
414
|
+
# check that AvroError is raised when there is no default
|
415
|
+
check_no_default(str)
|
384
416
|
end
|
385
417
|
|
386
418
|
def checkser(schm, randomdata)
|
data/test/test_schema.rb
CHANGED
@@ -143,4 +143,21 @@ class TestSchema < Test::Unit::TestCase
|
|
143
143
|
|
144
144
|
assert_equal '"MissingType" is not a schema we know about.', error.message
|
145
145
|
end
|
146
|
+
|
147
|
+
def test_to_avro_handles_falsey_defaults
|
148
|
+
schema = Avro::Schema.parse <<-SCHEMA
|
149
|
+
{"type": "record", "name": "Record", "namespace": "my.name.space",
|
150
|
+
"fields": [
|
151
|
+
{"name": "is_usable", "type": "boolean", "default": false}
|
152
|
+
]
|
153
|
+
}
|
154
|
+
SCHEMA
|
155
|
+
|
156
|
+
assert_equal schema.to_avro, {
|
157
|
+
'type' => 'record', 'name' => 'Record', 'namespace' => 'my.name.space',
|
158
|
+
'fields' => [
|
159
|
+
{'name' => 'is_usable', 'type' => 'boolean', 'default' => false}
|
160
|
+
]
|
161
|
+
}
|
162
|
+
end
|
146
163
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: avro
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.8.
|
4
|
+
version: 1.8.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Apache Software Foundation
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-05-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: multi_json
|
@@ -96,11 +96,11 @@ signing_key:
|
|
96
96
|
specification_version: 4
|
97
97
|
summary: Apache Avro for Ruby
|
98
98
|
test_files:
|
99
|
-
- test/
|
100
|
-
- test/test_io.rb
|
101
|
-
- test/test_socket_transport.rb
|
102
|
-
- test/test_schema.rb
|
99
|
+
- test/test_datafile.rb
|
103
100
|
- test/test_fingerprints.rb
|
104
101
|
- test/test_help.rb
|
105
|
-
- test/
|
102
|
+
- test/test_io.rb
|
106
103
|
- test/test_protocol.rb
|
104
|
+
- test/test_schema.rb
|
105
|
+
- test/test_schema_normalization.rb
|
106
|
+
- test/test_socket_transport.rb
|