avromatic 2.2.5 → 3.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +89 -0
- data/.gitignore +3 -0
- data/.rubocop.yml +4 -1
- data/Appraisals +8 -14
- data/CHANGELOG.md +25 -0
- data/README.md +2 -19
- data/avromatic.gemspec +9 -8
- data/bin/console +4 -3
- data/gemfiles/avro1_10_rails6_1.gemfile +9 -0
- data/gemfiles/{avro1_8_rails5_2.gemfile → avro1_9_rails6_1.gemfile} +3 -3
- data/lib/avromatic.rb +0 -5
- data/lib/avromatic/io.rb +1 -7
- data/lib/avromatic/io/datum_reader.rb +18 -68
- data/lib/avromatic/io/datum_writer.rb +5 -17
- data/lib/avromatic/io/union_datum.rb +25 -0
- data/lib/avromatic/messaging.rb +4 -2
- data/lib/avromatic/model/attributes.rb +22 -3
- data/lib/avromatic/model/configurable.rb +30 -2
- data/lib/avromatic/model/configuration.rb +5 -0
- data/lib/avromatic/model/field_helper.rb +5 -1
- data/lib/avromatic/model/messaging_serialization.rb +2 -1
- data/lib/avromatic/model/nested_models.rb +4 -2
- data/lib/avromatic/model/raw_serialization.rb +67 -27
- data/lib/avromatic/model/types/record_type.rb +3 -6
- data/lib/avromatic/model/types/union_type.rb +11 -8
- data/lib/avromatic/model/validation.rb +2 -2
- data/lib/avromatic/model_registry.rb +11 -2
- data/lib/avromatic/version.rb +1 -1
- metadata +41 -29
- data/.travis.yml +0 -16
- data/gemfiles/avro_patches_rails5_2.gemfile +0 -9
- data/gemfiles/avro_patches_rails6_0.gemfile +0 -9
- data/lib/avromatic/patches.rb +0 -18
- data/lib/avromatic/patches/schema_validator_patch.rb +0 -39
@@ -0,0 +1,25 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Avromatic
|
4
|
+
module IO
|
5
|
+
class UnionDatum
|
6
|
+
attr_reader :member_index, :datum
|
7
|
+
|
8
|
+
def initialize(member_index, datum)
|
9
|
+
@member_index = member_index
|
10
|
+
@datum = datum
|
11
|
+
end
|
12
|
+
|
13
|
+
def ==(other)
|
14
|
+
other.is_a?(Avromatic::IO::UnionDatum) &&
|
15
|
+
member_index == other.member_index &&
|
16
|
+
datum == other.datum
|
17
|
+
end
|
18
|
+
alias_method :eql?, :==
|
19
|
+
|
20
|
+
def hash
|
21
|
+
31 * datum.hash + member_index
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
data/lib/avromatic/messaging.rb
CHANGED
@@ -29,7 +29,8 @@ module Avromatic
|
|
29
29
|
end
|
30
30
|
|
31
31
|
# The following line differs from the parent class to use a custom DatumReader
|
32
|
-
|
32
|
+
reader_class = Avromatic.use_custom_datum_reader ? Avromatic::IO::DatumReader : Avro::IO::DatumReader
|
33
|
+
reader = reader_class.new(writers_schema, readers_schema)
|
33
34
|
reader.read(decoder)
|
34
35
|
end
|
35
36
|
|
@@ -50,7 +51,8 @@ module Avromatic
|
|
50
51
|
encoder.write([schema_id].pack('N'))
|
51
52
|
|
52
53
|
# The following line differs from the parent class to use a custom DatumWriter
|
53
|
-
|
54
|
+
writer_class = Avromatic.use_custom_datum_writer ? Avromatic::IO::DatumWriter : Avro::IO::DatumWriter
|
55
|
+
writer = writer_class.new(schema)
|
54
56
|
|
55
57
|
# The actual message comes last.
|
56
58
|
writer.write(message, encoder)
|
@@ -27,7 +27,10 @@ module Avromatic
|
|
27
27
|
def initialize(owner:, field:, type:)
|
28
28
|
@owner = owner
|
29
29
|
@field = field
|
30
|
+
@required = FieldHelper.required?(field)
|
31
|
+
@nullable = FieldHelper.nullable?(field)
|
30
32
|
@type = type
|
33
|
+
@values_immutable = type.referenced_model_classes.all?(&:recursively_immutable?)
|
31
34
|
@name = field.name.to_sym
|
32
35
|
@name_string = field.name.to_s.dup.freeze
|
33
36
|
@setter_name = "#{field.name}=".to_sym
|
@@ -40,8 +43,16 @@ module Avromatic
|
|
40
43
|
end
|
41
44
|
end
|
42
45
|
|
46
|
+
def nullable?
|
47
|
+
@nullable
|
48
|
+
end
|
49
|
+
|
43
50
|
def required?
|
44
|
-
|
51
|
+
@required
|
52
|
+
end
|
53
|
+
|
54
|
+
def values_immutable?
|
55
|
+
@values_immutable
|
45
56
|
end
|
46
57
|
|
47
58
|
def coerce(input)
|
@@ -69,6 +80,8 @@ module Avromatic
|
|
69
80
|
included do
|
70
81
|
class_attribute :attribute_definitions, instance_writer: false
|
71
82
|
self.attribute_definitions = {}
|
83
|
+
|
84
|
+
delegate :recursively_immutable?, to: :class
|
72
85
|
end
|
73
86
|
|
74
87
|
def initialize(data = {})
|
@@ -130,6 +143,12 @@ module Avromatic
|
|
130
143
|
define_avro_attributes(avro_schema, generated_methods_module)
|
131
144
|
end
|
132
145
|
|
146
|
+
def recursively_immutable?
|
147
|
+
return @recursively_immutable if defined?(@recursively_immutable)
|
148
|
+
|
149
|
+
@recursively_immutable = immutable? && attribute_definitions.each_value.all?(&:values_immutable?)
|
150
|
+
end
|
151
|
+
|
133
152
|
private
|
134
153
|
|
135
154
|
def check_for_field_conflicts!
|
@@ -174,10 +193,10 @@ module Avromatic
|
|
174
193
|
generated_methods_module.send(:define_method, "#{field.name}?") { !!_attributes[symbolized_field_name] } if FieldHelper.boolean?(field)
|
175
194
|
|
176
195
|
generated_methods_module.send(:define_method, "#{field.name}=") do |value|
|
177
|
-
_attributes[symbolized_field_name] =
|
196
|
+
_attributes[symbolized_field_name] = attribute_definition.coerce(value)
|
178
197
|
end
|
179
198
|
|
180
|
-
unless
|
199
|
+
unless mutable? # rubocop:disable Style/Next
|
181
200
|
generated_methods_module.send(:private, "#{field.name}=")
|
182
201
|
generated_methods_module.send(:define_method, :clone) { self }
|
183
202
|
generated_methods_module.send(:define_method, :dup) { self }
|
@@ -8,9 +8,23 @@ module Avromatic
|
|
8
8
|
module Configurable
|
9
9
|
extend ActiveSupport::Concern
|
10
10
|
|
11
|
+
# Wraps a reference to a field so we can access both the string and symbolized versions of the name
|
12
|
+
# without repeated memory allocations.
|
13
|
+
class FieldReference
|
14
|
+
attr_reader :name, :name_sym
|
15
|
+
|
16
|
+
def initialize(name)
|
17
|
+
@name = -name
|
18
|
+
@name_sym = name.to_sym
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
included do
|
23
|
+
class_attribute :config, instance_accessor: false, instance_predicate: false
|
24
|
+
end
|
25
|
+
|
11
26
|
module ClassMethods
|
12
|
-
|
13
|
-
delegate :avro_schema, :value_avro_schema, :key_avro_schema, to: :config
|
27
|
+
delegate :avro_schema, :value_avro_schema, :key_avro_schema, :mutable?, :immutable?, to: :config
|
14
28
|
|
15
29
|
def value_avro_field_names
|
16
30
|
@value_avro_field_names ||= value_avro_schema.fields.map(&:name).map(&:to_sym).freeze
|
@@ -20,6 +34,18 @@ module Avromatic
|
|
20
34
|
@key_avro_field_names ||= key_avro_schema.fields.map(&:name).map(&:to_sym).freeze
|
21
35
|
end
|
22
36
|
|
37
|
+
def value_avro_field_references
|
38
|
+
@value_avro_field_references ||= value_avro_schema.fields.map do |field|
|
39
|
+
Avromatic::Model::Configurable::FieldReference.new(field.name)
|
40
|
+
end.freeze
|
41
|
+
end
|
42
|
+
|
43
|
+
def key_avro_field_references
|
44
|
+
@key_avro_field_references ||= key_avro_schema.fields.map do |field|
|
45
|
+
Avromatic::Model::Configurable::FieldReference.new(field.name)
|
46
|
+
end.freeze
|
47
|
+
end
|
48
|
+
|
23
49
|
def value_avro_fields_by_name
|
24
50
|
@value_avro_fields_by_name ||= mapped_by_name(value_avro_schema)
|
25
51
|
end
|
@@ -43,6 +69,8 @@ module Avromatic
|
|
43
69
|
|
44
70
|
delegate :avro_schema, :value_avro_schema, :key_avro_schema,
|
45
71
|
:value_avro_field_names, :key_avro_field_names,
|
72
|
+
:value_avro_field_references, :key_avro_field_references,
|
73
|
+
:mutable?, :immutable?,
|
46
74
|
to: :class
|
47
75
|
end
|
48
76
|
end
|
@@ -8,6 +8,7 @@ module Avromatic
|
|
8
8
|
|
9
9
|
attr_reader :avro_schema, :key_avro_schema, :nested_models, :mutable,
|
10
10
|
:allow_optional_key_fields
|
11
|
+
alias_method :mutable?, :mutable
|
11
12
|
delegate :schema_store, to: Avromatic
|
12
13
|
|
13
14
|
# Either schema(_name) or value_schema(_name), but not both, must be
|
@@ -34,6 +35,10 @@ module Avromatic
|
|
34
35
|
|
35
36
|
alias_method :value_avro_schema, :avro_schema
|
36
37
|
|
38
|
+
def immutable?
|
39
|
+
!mutable?
|
40
|
+
end
|
41
|
+
|
37
42
|
private
|
38
43
|
|
39
44
|
def find_avro_schema(**options)
|
@@ -16,9 +16,13 @@ module Avromatic
|
|
16
16
|
!optional?(field)
|
17
17
|
end
|
18
18
|
|
19
|
+
def nullable?(field)
|
20
|
+
optional?(field) || field.type.type_sym == :null
|
21
|
+
end
|
22
|
+
|
19
23
|
def boolean?(field)
|
20
24
|
field.type.type_sym == :boolean ||
|
21
|
-
(
|
25
|
+
(optional?(field) && field.type.schemas.last.type_sym == :boolean)
|
22
26
|
end
|
23
27
|
end
|
24
28
|
end
|
@@ -48,7 +48,8 @@ module Avromatic
|
|
48
48
|
value_attributes = avro_messaging
|
49
49
|
.decode(message_value, schema_name: avro_schema.fullname)
|
50
50
|
|
51
|
-
value_attributes.merge!(key_attributes
|
51
|
+
value_attributes.merge!(key_attributes) if key_attributes
|
52
|
+
value_attributes
|
52
53
|
end
|
53
54
|
end
|
54
55
|
|
@@ -14,12 +14,14 @@ module Avromatic
|
|
14
14
|
def register!
|
15
15
|
return unless key_avro_schema.nil? && value_avro_schema.type_sym == :record
|
16
16
|
|
17
|
+
processed = Set.new
|
17
18
|
roots = [self]
|
18
19
|
until roots.empty?
|
19
20
|
model = roots.shift
|
20
|
-
|
21
|
+
# Avoid any nested model dependency cycles by ignoring already processed models
|
22
|
+
next unless processed.add?(model)
|
21
23
|
|
22
|
-
nested_models.
|
24
|
+
nested_models.ensure_registered_model(model)
|
23
25
|
roots.concat(model.referenced_model_classes)
|
24
26
|
end
|
25
27
|
end
|
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'active_support/deprecation'
|
4
|
+
|
3
5
|
module Avromatic
|
4
6
|
module Model
|
5
7
|
|
@@ -11,55 +13,93 @@ module Avromatic
|
|
11
13
|
module Encode
|
12
14
|
extend ActiveSupport::Concern
|
13
15
|
|
16
|
+
UNSPECIFIED = Object.new
|
17
|
+
|
14
18
|
delegate :datum_writer, :datum_reader, to: :class
|
15
19
|
private :datum_writer, :datum_reader
|
16
20
|
|
17
|
-
def avro_raw_value(validate:
|
18
|
-
|
19
|
-
|
21
|
+
def avro_raw_value(validate: UNSPECIFIED)
|
22
|
+
unless validate == UNSPECIFIED
|
23
|
+
ActiveSupport::Deprecation.warn("The 'validate' argument to #{__method__} is deprecated.")
|
24
|
+
end
|
25
|
+
|
26
|
+
if self.class.recursively_immutable?
|
27
|
+
@avro_raw_value ||= avro_raw_encode(value_attributes_for_avro, :value)
|
20
28
|
else
|
21
|
-
|
29
|
+
avro_raw_encode(value_attributes_for_avro, :value)
|
22
30
|
end
|
23
31
|
end
|
24
32
|
|
25
|
-
def avro_raw_key(validate:
|
33
|
+
def avro_raw_key(validate: UNSPECIFIED)
|
34
|
+
unless validate == UNSPECIFIED
|
35
|
+
ActiveSupport::Deprecation.warn("The 'validate' argument to #{__method__} is deprecated.")
|
36
|
+
end
|
37
|
+
|
26
38
|
raise 'Model has no key schema' unless key_avro_schema
|
27
|
-
avro_raw_encode(key_attributes_for_avro
|
39
|
+
avro_raw_encode(key_attributes_for_avro, :key)
|
28
40
|
end
|
29
41
|
|
30
|
-
def value_attributes_for_avro(validate:
|
31
|
-
|
32
|
-
|
42
|
+
def value_attributes_for_avro(validate: UNSPECIFIED)
|
43
|
+
unless validate == UNSPECIFIED
|
44
|
+
ActiveSupport::Deprecation.warn("The 'validate' argument to #{__method__} is deprecated.")
|
45
|
+
end
|
46
|
+
|
47
|
+
if self.class.recursively_immutable?
|
48
|
+
@value_attributes_for_avro ||= avro_hash(value_avro_field_references)
|
33
49
|
else
|
34
|
-
|
50
|
+
avro_hash(value_avro_field_references)
|
35
51
|
end
|
36
52
|
end
|
37
53
|
|
38
|
-
def key_attributes_for_avro(validate:
|
39
|
-
|
54
|
+
def key_attributes_for_avro(validate: UNSPECIFIED)
|
55
|
+
unless validate == UNSPECIFIED
|
56
|
+
ActiveSupport::Deprecation.warn("The 'validate' argument to #{__method__} is deprecated.")
|
57
|
+
end
|
58
|
+
|
59
|
+
avro_hash(key_avro_field_references)
|
40
60
|
end
|
41
61
|
|
42
|
-
def avro_value_datum(validate:
|
43
|
-
|
44
|
-
|
62
|
+
def avro_value_datum(validate: UNSPECIFIED)
|
63
|
+
unless validate == UNSPECIFIED
|
64
|
+
ActiveSupport::Deprecation.warn("The 'validate' argument to #{__method__} is deprecated.")
|
65
|
+
end
|
66
|
+
|
67
|
+
if self.class.recursively_immutable?
|
68
|
+
@avro_value_datum ||= avro_hash(value_avro_field_references, strict: true)
|
45
69
|
else
|
46
|
-
|
70
|
+
avro_hash(value_avro_field_references, strict: true)
|
47
71
|
end
|
48
72
|
end
|
49
73
|
|
50
|
-
def avro_key_datum(validate:
|
51
|
-
|
74
|
+
def avro_key_datum(validate: UNSPECIFIED)
|
75
|
+
unless validate == UNSPECIFIED
|
76
|
+
ActiveSupport::Deprecation.warn("The 'validate' argument to #{__method__} is deprecated.")
|
77
|
+
end
|
78
|
+
|
79
|
+
avro_hash(key_avro_field_references, strict: true)
|
52
80
|
end
|
53
81
|
|
54
82
|
private
|
55
83
|
|
56
|
-
def avro_hash(
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
value
|
62
|
-
|
84
|
+
def avro_hash(field_references, strict: false)
|
85
|
+
field_references.each_with_object(Hash.new) do |field_reference, result|
|
86
|
+
attribute_definition = self.class.attribute_definitions[field_reference.name_sym]
|
87
|
+
value = _attributes[field_reference.name_sym]
|
88
|
+
|
89
|
+
if value.nil? && !attribute_definition.nullable?
|
90
|
+
# We're missing a required attribute so perform an explicit validation to generate
|
91
|
+
# a more complete error message
|
92
|
+
avro_validate!
|
93
|
+
elsif _attributes.include?(field_reference.name_sym)
|
94
|
+
begin
|
95
|
+
result[field_reference.name] = attribute_definition.serialize(value, strict)
|
96
|
+
rescue Avromatic::Model::ValidationError
|
97
|
+
# Perform an explicit validation to generate a more complete error message
|
98
|
+
avro_validate!
|
99
|
+
# We should never get here but just in case...
|
100
|
+
raise
|
101
|
+
end
|
102
|
+
end
|
63
103
|
end
|
64
104
|
end
|
65
105
|
|
@@ -81,8 +121,8 @@ module Avromatic
|
|
81
121
|
def avro_raw_decode(key: nil, value:, key_schema: nil, value_schema: nil)
|
82
122
|
key_attributes = key && decode_avro_datum(key, key_schema, :key)
|
83
123
|
value_attributes = decode_avro_datum(value, value_schema, :value)
|
84
|
-
|
85
|
-
new(value_attributes
|
124
|
+
value_attributes.merge!(key_attributes) if key_attributes
|
125
|
+
new(value_attributes)
|
86
126
|
end
|
87
127
|
|
88
128
|
private
|
@@ -42,13 +42,10 @@ module Avromatic
|
|
42
42
|
def serialize(value, strict)
|
43
43
|
if value.nil?
|
44
44
|
value
|
45
|
-
elsif
|
46
|
-
|
47
|
-
# hash but then we'd have no place to stash the union member index...
|
48
|
-
{ Avromatic::IO::ENCODING_PROVIDER => value }
|
45
|
+
elsif strict
|
46
|
+
value.avro_value_datum
|
49
47
|
else
|
50
|
-
|
51
|
-
strict ? value.avro_value_datum(validate: false) : value.value_attributes_for_avro(validate: false)
|
48
|
+
value.value_attributes_for_avro
|
52
49
|
end
|
53
50
|
end
|
54
51
|
|
@@ -7,7 +7,6 @@ module Avromatic
|
|
7
7
|
module Model
|
8
8
|
module Types
|
9
9
|
class UnionType < AbstractType
|
10
|
-
MEMBER_INDEX = ::Avromatic::IO::DatumReader::UNION_MEMBER_INDEX
|
11
10
|
attr_reader :member_types, :value_classes, :input_classes
|
12
11
|
|
13
12
|
def initialize(member_types:)
|
@@ -24,15 +23,15 @@ module Avromatic
|
|
24
23
|
return input if coerced?(input)
|
25
24
|
|
26
25
|
result = nil
|
27
|
-
if input.is_a?(
|
28
|
-
result = member_types[input.
|
26
|
+
if input.is_a?(Avromatic::IO::UnionDatum)
|
27
|
+
result = member_types[input.member_index].coerce(input.datum)
|
29
28
|
else
|
30
29
|
member_types.find do |member_type|
|
31
30
|
result = safe_coerce(member_type, input)
|
32
31
|
end
|
33
32
|
end
|
34
33
|
|
35
|
-
|
34
|
+
if result.nil?
|
36
35
|
raise ArgumentError.new("Could not coerce '#{input.inspect}' to #{name}")
|
37
36
|
end
|
38
37
|
|
@@ -40,12 +39,16 @@ module Avromatic
|
|
40
39
|
end
|
41
40
|
|
42
41
|
def coerced?(value)
|
42
|
+
return false if value.is_a?(Avromatic::IO::UnionDatum)
|
43
|
+
|
43
44
|
value.nil? || member_types.any? do |member_type|
|
44
45
|
member_type.coerced?(value)
|
45
46
|
end
|
46
47
|
end
|
47
48
|
|
48
49
|
def coercible?(input)
|
50
|
+
return true if value.is_a?(Avromatic::IO::UnionDatum)
|
51
|
+
|
49
52
|
coerced?(input) || member_types.any? do |member_type|
|
50
53
|
member_type.coercible?(input)
|
51
54
|
end
|
@@ -60,11 +63,11 @@ module Avromatic
|
|
60
63
|
raise ArgumentError.new("Expected #{value.inspect} to be one of #{value_classes.map(&:name)}")
|
61
64
|
end
|
62
65
|
|
63
|
-
|
64
|
-
if !strict && Avromatic.use_custom_datum_writer
|
65
|
-
|
66
|
+
serialized_value = member_types[member_index].serialize(value, strict)
|
67
|
+
if !strict && Avromatic.use_custom_datum_writer
|
68
|
+
serialized_value = Avromatic::IO::UnionDatum.new(member_index, serialized_value)
|
66
69
|
end
|
67
|
-
|
70
|
+
serialized_value
|
68
71
|
end
|
69
72
|
|
70
73
|
def referenced_model_classes
|