avromatic 0.13.0 → 0.14.0.rc0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/lib/avromatic/io/datum_reader.rb +74 -0
- data/lib/avromatic/messaging.rb +32 -0
- data/lib/avromatic/model/attribute/union.rb +15 -6
- data/lib/avromatic/model/messaging_serialization.rb +0 -2
- data/lib/avromatic/model/raw_serialization.rb +6 -3
- data/lib/avromatic/version.rb +1 -1
- data/lib/avromatic.rb +6 -4
- metadata +6 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: dd0e7ff37e01961347eb5222e574d1e1942078ec
|
4
|
+
data.tar.gz: 3894c1ed617bc9525eac01c72969bed69702c015
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 87701549c8cb63b5636d2647f5b3025fef2f652ca886c876223b9d44457370827d8324536b7492fe786c5b9686f67991074130d69e29901f2739ca63b8098f43
|
7
|
+
data.tar.gz: 9da95256f8eb308390bcd64517525ac8e4410a812666efcc3eb92bca6865a7f0f481ca7a086062d8844db69fb4b09ce874e7b19f7d7608b33aaa74d3565e3fde
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,9 @@
|
|
1
1
|
# avromatic changelog
|
2
2
|
|
3
|
+
## v0.14.0
|
4
|
+
- Add `Avromatic::Messaging` and `Avromatic::IO::DatumReader` classes to
|
5
|
+
optimize the decoding of Avro unions to Avromatic models.
|
6
|
+
|
3
7
|
## v0.13.0
|
4
8
|
- Add interfaces to deserialize as a hash of attributes instead of a model.
|
5
9
|
|
@@ -0,0 +1,74 @@
|
|
1
|
+
# rubocop:disable Style/WhenThen
|
2
|
+
module Avromatic
|
3
|
+
module IO
|
4
|
+
# Subclass DatumReader to include additional information about the union
|
5
|
+
# index used.
|
6
|
+
class DatumReader < Avro::IO::DatumReader
|
7
|
+
|
8
|
+
UNION_MEMBER_INDEX = '__avromatic_member_index'.freeze
|
9
|
+
|
10
|
+
def read_data(writers_schema, readers_schema, decoder, initial_record = {})
|
11
|
+
# schema matching
|
12
|
+
unless self.class.match_schemas(writers_schema, readers_schema)
|
13
|
+
raise SchemaMatchException.new(writers_schema, readers_schema)
|
14
|
+
end
|
15
|
+
|
16
|
+
# schema resolution: reader's schema is a union, writer's schema is not
|
17
|
+
if writers_schema.type_sym != :union && readers_schema.type_sym == :union
|
18
|
+
rs_index = readers_schema.schemas.find_index do |s|
|
19
|
+
self.class.match_schemas(writers_schema, s)
|
20
|
+
end
|
21
|
+
|
22
|
+
union_info = { UNION_MEMBER_INDEX => rs_index }
|
23
|
+
|
24
|
+
return read_data(writers_schema, readers_schema.schemas[rs_index], decoder, union_info) if rs_index
|
25
|
+
raise SchemaMatchException.new(writers_schema, readers_schema)
|
26
|
+
end
|
27
|
+
|
28
|
+
# function dispatch for reading data based on type of writer's
|
29
|
+
# schema
|
30
|
+
datum = case writers_schema.type_sym
|
31
|
+
when :null; decoder.read_null
|
32
|
+
when :boolean; decoder.read_boolean
|
33
|
+
when :string; decoder.read_string
|
34
|
+
when :int; decoder.read_int
|
35
|
+
when :long; decoder.read_long
|
36
|
+
when :float; decoder.read_float
|
37
|
+
when :double; decoder.read_double
|
38
|
+
when :bytes; decoder.read_bytes
|
39
|
+
when :fixed; read_fixed(writers_schema, readers_schema, decoder)
|
40
|
+
when :enum; read_enum(writers_schema, readers_schema, decoder)
|
41
|
+
when :array; read_array(writers_schema, readers_schema, decoder)
|
42
|
+
when :map; read_map(writers_schema, readers_schema, decoder)
|
43
|
+
when :union; read_union(writers_schema, readers_schema, decoder)
|
44
|
+
when :record, :error, :request; read_record(writers_schema, readers_schema, decoder, initial_record)
|
45
|
+
else
|
46
|
+
raise AvroError.new("Cannot read unknown schema type: #{writers_schema.type}")
|
47
|
+
end
|
48
|
+
|
49
|
+
if readers_schema.respond_to?(:logical_type)
|
50
|
+
readers_schema.type_adapter.decode(datum)
|
51
|
+
else
|
52
|
+
datum
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
# Override to specify initial record that may contain union index
|
57
|
+
def read_record(writers_schema, readers_schema, decoder, initial_record = {})
|
58
|
+
readers_fields_hash = readers_schema.fields_hash
|
59
|
+
read_record = Avromatic.use_custom_datum_reader ? initial_record : {}
|
60
|
+
writers_schema.fields.each do |field|
|
61
|
+
readers_field = readers_fields_hash[field.name]
|
62
|
+
if readers_field
|
63
|
+
field_val = read_data(field.type, readers_field.type, decoder)
|
64
|
+
read_record[field.name] = field_val
|
65
|
+
else
|
66
|
+
skip_data(field.type, decoder)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
read_record
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'avro_turf/messaging'
|
2
|
+
require 'avromatic/io/datum_reader'
|
3
|
+
|
4
|
+
module Avromatic
|
5
|
+
# Subclass AvroTurf::Messaging to use a custom DatumReader
|
6
|
+
class Messaging < AvroTurf::Messaging
|
7
|
+
def decode(data, schema_name: nil, namespace: @namespace)
|
8
|
+
readers_schema = schema_name && @schema_store.find(schema_name, namespace)
|
9
|
+
stream = StringIO.new(data)
|
10
|
+
decoder = Avro::IO::BinaryDecoder.new(stream)
|
11
|
+
|
12
|
+
# The first byte is MAGIC!!!
|
13
|
+
magic_byte = decoder.read(1)
|
14
|
+
|
15
|
+
if magic_byte != MAGIC_BYTE
|
16
|
+
raise "Expected data to begin with a magic byte, got `#{magic_byte.inspect}`"
|
17
|
+
end
|
18
|
+
|
19
|
+
# The schema id is a 4-byte big-endian integer.
|
20
|
+
schema_id = decoder.read(4).unpack('N').first
|
21
|
+
|
22
|
+
writers_schema = @schemas_by_id.fetch(schema_id) do
|
23
|
+
schema_json = @registry.fetch(schema_id)
|
24
|
+
@schemas_by_id[schema_id] = Avro::Schema.parse(schema_json)
|
25
|
+
end
|
26
|
+
|
27
|
+
# The following line was changed to use a custom DatumReader
|
28
|
+
reader = Avromatic::IO::DatumReader.new(writers_schema, readers_schema)
|
29
|
+
reader.read(decoder)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'avromatic/model/attribute_type/union'
|
2
|
+
require 'avromatic/io/datum_reader'
|
2
3
|
|
3
4
|
module Avromatic
|
4
5
|
module Model
|
@@ -12,6 +13,8 @@ module Avromatic
|
|
12
13
|
class Union < Virtus::Attribute
|
13
14
|
primitive Avromatic::Model::AttributeType::Union
|
14
15
|
|
16
|
+
MEMBER_INDEX = ::Avromatic::IO::DatumReader::UNION_MEMBER_INDEX
|
17
|
+
|
15
18
|
def initialize(*)
|
16
19
|
super
|
17
20
|
|
@@ -24,12 +27,11 @@ module Avromatic
|
|
24
27
|
return input if value_coerced?(input)
|
25
28
|
|
26
29
|
result = nil
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
nil
|
30
|
+
if input.key?(MEMBER_INDEX)
|
31
|
+
result = safe_coerce(member_attributes[input.delete(MEMBER_INDEX)], input)
|
32
|
+
else
|
33
|
+
member_attributes.find do |union_attribute|
|
34
|
+
result = safe_coerce(union_attribute, input)
|
33
35
|
end
|
34
36
|
end
|
35
37
|
result
|
@@ -43,6 +45,13 @@ module Avromatic
|
|
43
45
|
|
44
46
|
private
|
45
47
|
|
48
|
+
def safe_coerce(member_attribute, input)
|
49
|
+
coerced = member_attribute.coerce(input)
|
50
|
+
coerced unless coerced.is_a?(Avromatic::Model::Attributes) && coerced.invalid?
|
51
|
+
rescue
|
52
|
+
nil
|
53
|
+
end
|
54
|
+
|
46
55
|
def member_attributes
|
47
56
|
@member_attributes ||= Array.new
|
48
57
|
end
|
@@ -88,11 +88,14 @@ module Avromatic
|
|
88
88
|
end
|
89
89
|
|
90
90
|
def custom_datum_reader(schema, key_or_value)
|
91
|
-
|
91
|
+
datum_reader_class.new(schema, send("#{key_or_value}_avro_schema"))
|
92
92
|
end
|
93
93
|
end
|
94
94
|
|
95
95
|
module ClassMethods
|
96
|
+
def datum_reader_class
|
97
|
+
Avromatic::IO::DatumReader
|
98
|
+
end
|
96
99
|
|
97
100
|
# Store a hash of Procs by field name (as a symbol) to convert
|
98
101
|
# the value before Avro serialization.
|
@@ -111,8 +114,8 @@ module Avromatic
|
|
111
114
|
|
112
115
|
def datum_reader
|
113
116
|
@datum_reader ||= begin
|
114
|
-
hash = { value:
|
115
|
-
hash[:key] =
|
117
|
+
hash = { value: datum_reader_class.new(value_avro_schema) }
|
118
|
+
hash[:key] = datum_reader_class.new(key_avro_schema) if key_avro_schema
|
116
119
|
hash
|
117
120
|
end
|
118
121
|
end
|
data/lib/avromatic/version.rb
CHANGED
data/lib/avromatic.rb
CHANGED
@@ -1,14 +1,15 @@
|
|
1
1
|
require 'avromatic/version'
|
2
|
+
require 'avro_turf'
|
2
3
|
require 'avromatic/model'
|
3
4
|
require 'avromatic/model_registry'
|
4
|
-
require '
|
5
|
-
require 'avro_turf/messaging'
|
5
|
+
require 'avromatic/messaging'
|
6
6
|
require 'active_support/core_ext/string/inflections'
|
7
7
|
|
8
8
|
module Avromatic
|
9
9
|
class << self
|
10
10
|
attr_accessor :schema_registry, :registry_url, :schema_store, :logger,
|
11
|
-
:messaging, :type_registry, :nested_models
|
11
|
+
:messaging, :type_registry, :nested_models,
|
12
|
+
:use_custom_datum_reader
|
12
13
|
|
13
14
|
delegate :register_type, to: :type_registry
|
14
15
|
end
|
@@ -16,6 +17,7 @@ module Avromatic
|
|
16
17
|
self.nested_models = ModelRegistry.new
|
17
18
|
self.logger = Logger.new($stdout)
|
18
19
|
self.type_registry = Avromatic::Model::TypeRegistry.new
|
20
|
+
self.use_custom_datum_reader = true
|
19
21
|
|
20
22
|
def self.configure
|
21
23
|
yield self
|
@@ -31,7 +33,7 @@ module Avromatic
|
|
31
33
|
|
32
34
|
def self.build_messaging
|
33
35
|
raise 'Avromatic must be configured with a schema_store' unless schema_store
|
34
|
-
|
36
|
+
Avromatic::Messaging.new(
|
35
37
|
registry: schema_registry || build_schema_registry,
|
36
38
|
schema_store: schema_store,
|
37
39
|
logger: logger
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: avromatic
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.14.0.rc0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Salsify Engineering
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-01-
|
11
|
+
date: 2017-01-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: avro
|
@@ -245,6 +245,8 @@ files:
|
|
245
245
|
- gemfiles/rails4_1.gemfile
|
246
246
|
- gemfiles/rails4_2.gemfile
|
247
247
|
- lib/avromatic.rb
|
248
|
+
- lib/avromatic/io/datum_reader.rb
|
249
|
+
- lib/avromatic/messaging.rb
|
248
250
|
- lib/avromatic/model.rb
|
249
251
|
- lib/avromatic/model/attribute/abstract_timestamp.rb
|
250
252
|
- lib/avromatic/model/attribute/record.rb
|
@@ -287,9 +289,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
287
289
|
version: '0'
|
288
290
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
289
291
|
requirements:
|
290
|
-
- - "
|
292
|
+
- - ">"
|
291
293
|
- !ruby/object:Gem::Version
|
292
|
-
version:
|
294
|
+
version: 1.3.1
|
293
295
|
requirements: []
|
294
296
|
rubyforge_project:
|
295
297
|
rubygems_version: 2.6.8
|