field_struct-avro 1.0.0.pre1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +33 -0
- data/LICENSE.txt +21 -0
- data/README.md +226 -0
- data/lib/field_struct/avro/bytes_reader.rb +37 -0
- data/lib/field_struct/avro/cache.rb +117 -0
- data/lib/field_struct/avro/check.rb +72 -0
- data/lib/field_struct/avro/datum_reader.rb +160 -0
- data/lib/field_struct/avro/datum_writer.rb +152 -0
- data/lib/field_struct/avro/defaults.rb +68 -0
- data/lib/field_struct/avro/entry.rb +48 -0
- data/lib/field_struct/avro/evolution.rb +52 -0
- data/lib/field_struct/avro/methods.rb +36 -0
- data/lib/field_struct/avro/names.rb +80 -0
- data/lib/field_struct/avro/options.rb +197 -0
- data/lib/field_struct/avro/projection.rb +59 -0
- data/lib/field_struct/avro/resolution.rb +110 -0
- data/lib/field_struct/avro/scalar_types.rb +83 -0
- data/lib/field_struct/avro/schema_builder.rb +203 -0
- data/lib/field_struct/avro/site.rb +40 -0
- data/lib/field_struct/avro/stamp.rb +48 -0
- data/lib/field_struct/avro/trail.rb +56 -0
- data/lib/field_struct/avro/union_rules.rb +99 -0
- data/lib/field_struct/avro/value_json.rb +47 -0
- data/lib/field_struct/avro/version.rb +10 -0
- data/lib/field_struct/avro/writer_schema.rb +33 -0
- data/lib/field_struct/avro/zone.rb +23 -0
- data/lib/field_struct/avro.rb +138 -0
- data/sig/field_struct-avro.rbs +97 -0
- metadata +149 -0
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FieldStruct
|
|
4
|
+
module Avro
|
|
5
|
+
# Walks a (writer, reader) schema pair the way Avro resolution pairs fields and reports the pairs the avro gem
|
|
6
|
+
# would silently misread (AV-B39), plus the reader fields the writer lacked (AV-B40).
|
|
7
|
+
class Resolution
|
|
8
|
+
# +problems+: the sentences of AV-B39. +lacking+: reader record full name to the Avro names of its fields the writer lacked.
|
|
9
|
+
Result = Struct.new(:problems, :lacking)
|
|
10
|
+
|
|
11
|
+
# The avro gem's identity adapter, which a type without a decoded logical type gets.
|
|
12
|
+
IDENTITY = ::Avro::LogicalTypes::Identity
|
|
13
|
+
|
|
14
|
+
# @param writer [::Avro::Schema]
|
|
15
|
+
# @param reader [::Avro::Schema]
|
|
16
|
+
# @return [Result]
|
|
17
|
+
def self.call(writer, reader)
|
|
18
|
+
new.call(writer, reader)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def initialize
|
|
22
|
+
@problems = []
|
|
23
|
+
@lacking = {}
|
|
24
|
+
@seen = {}
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# @param writer [::Avro::Schema]
|
|
28
|
+
# @param reader [::Avro::Schema]
|
|
29
|
+
# @return [Result]
|
|
30
|
+
def call(writer, reader)
|
|
31
|
+
walk(writer, reader, '')
|
|
32
|
+
Result.new(@problems, @lacking)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
private
|
|
36
|
+
|
|
37
|
+
def walk(writer, reader, path)
|
|
38
|
+
return union(writer, reader, path) if writer.type_sym == :union || reader.type_sym == :union
|
|
39
|
+
return unless ::Avro::SchemaCompatibility.match_schemas(writer, reader)
|
|
40
|
+
|
|
41
|
+
if writer.type_sym == reader.type_sym
|
|
42
|
+
same_type(writer, reader, path)
|
|
43
|
+
elsif decoded?(writer) || decoded?(reader)
|
|
44
|
+
problem(path, name(writer), name(reader))
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def same_type(writer, reader, path)
|
|
49
|
+
case reader.type_sym
|
|
50
|
+
when :record then record(writer, reader, path)
|
|
51
|
+
when :array then walk(writer.items, reader.items, "#{path}[]")
|
|
52
|
+
when :map then walk(writer.values, reader.values, "#{path}{}")
|
|
53
|
+
else problem(path, name(writer), name(reader)) unless name(writer) == name(reader)
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def record(writer, reader, path)
|
|
58
|
+
return if @seen[[writer.fullname, reader.fullname]]
|
|
59
|
+
|
|
60
|
+
@seen[[writer.fullname, reader.fullname]] = true
|
|
61
|
+
written = writer.fields_hash
|
|
62
|
+
@lacking[reader.fullname] = reader.fields.map(&:name).reject { |field| written.key?(field) }
|
|
63
|
+
writer.fields.each do |field|
|
|
64
|
+
other = reader.fields_hash[field.name]
|
|
65
|
+
walk(field.type, other.type, [path, field.name].reject(&:empty?).join('.')) if other
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def union(writer, reader, path)
|
|
70
|
+
if writer.type_sym == :union && reader.type_sym == :union
|
|
71
|
+
writer.schemas.each { |branch| pair(branch, reader, path) }
|
|
72
|
+
elsif writer.type_sym == :union
|
|
73
|
+
from_union(writer, reader, path)
|
|
74
|
+
else
|
|
75
|
+
pair(writer, reader, path)
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def pair(branch, reader, path)
|
|
80
|
+
match = reader.schemas.find { |candidate| ::Avro::SchemaCompatibility.match_schemas(branch, candidate) }
|
|
81
|
+
walk(branch, match, path) if match
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def from_union(writer, reader, path)
|
|
85
|
+
matched = writer.schemas.select { |branch| ::Avro::SchemaCompatibility.match_schemas(branch, reader) }
|
|
86
|
+
before = @problems.size
|
|
87
|
+
matched.each { |branch| walk(branch, reader, path) }
|
|
88
|
+
return unless @problems.size == before && decoded?(reader) && matched.any?
|
|
89
|
+
|
|
90
|
+
@problems << "#{path}: written as a union with #{matched.map { |branch| name(branch) }.join(', ')}, " \
|
|
91
|
+
"read as #{name(reader)} outside one, which the avro gem fails to decode (keep the field optional)"
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def decoded?(schema)
|
|
95
|
+
!schema.type_adapter.equal?(IDENTITY)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def name(schema)
|
|
99
|
+
return schema.type_sym.to_s unless decoded?(schema)
|
|
100
|
+
return "decimal (scale #{schema.scale})" if schema.logical_type == 'decimal'
|
|
101
|
+
|
|
102
|
+
schema.logical_type
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def problem(path, written, read)
|
|
106
|
+
@problems << "#{path}: written as #{written}, read as #{read}"
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FieldStruct
|
|
4
|
+
module Avro
|
|
5
|
+
# Maps a scalar node to its Avro type (AV-B4..AV-B6, AV-B9).
|
|
6
|
+
class ScalarTypes
|
|
7
|
+
DATE = { 'type' => 'int', 'logicalType' => 'date' }.freeze
|
|
8
|
+
private_constant :DATE
|
|
9
|
+
|
|
10
|
+
# Checked in order, so a subclass wins: a String or Hash is the Avro type, a Symbol names the method that builds it.
|
|
11
|
+
MAPPING = [
|
|
12
|
+
[Types::Binary, 'bytes'], [Types::UUID, :uuid], [Types::String, 'string'], [Types::Value, 'string'],
|
|
13
|
+
[Types::Boolean, 'boolean'], [Types::Integer, 'long'], [Types::Float, 'double'], [Types::BigDecimal, :decimal],
|
|
14
|
+
[Types::Date, DATE], [Types::Time, :timestamp], [Types::DateTime, :timestamp], [Types::Symbol, :symbol]
|
|
15
|
+
].freeze
|
|
16
|
+
private_constant :MAPPING
|
|
17
|
+
|
|
18
|
+
# @param timestamp [::Symbol] the call's timestamp precision
|
|
19
|
+
# @param names [Names] the schema's name claims (enums claim theirs)
|
|
20
|
+
def initialize(timestamp, names)
|
|
21
|
+
@timestamp = timestamp
|
|
22
|
+
@names = names
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# @param node [FieldStruct::Introspection::ScalarNode]
|
|
26
|
+
# @param site [Site]
|
|
27
|
+
# @param base [::String] the name an enum under this node takes
|
|
28
|
+
# @return [::String, ::Hash] the Avro type
|
|
29
|
+
# @raise [::ArgumentError] for a type with no mapping or options Avro cannot hold
|
|
30
|
+
def call(node, site, base)
|
|
31
|
+
_, mapped = MAPPING.find { |klass, _| node.type.is_a?(klass) }
|
|
32
|
+
case mapped
|
|
33
|
+
when nil then unmapped(node.type, site)
|
|
34
|
+
when ::Symbol then send(mapped, node, site, base)
|
|
35
|
+
else mapped.dup
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
private
|
|
40
|
+
|
|
41
|
+
def uuid(node, *)
|
|
42
|
+
format = node.options[:format]
|
|
43
|
+
return 'string' unless format.nil? || format == node.type.class.default_format
|
|
44
|
+
|
|
45
|
+
{ 'type' => 'string', 'logicalType' => 'uuid' }
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def decimal(node, site, *)
|
|
49
|
+
spec = Options.decimal(site.opts, node.options, node.type, site)
|
|
50
|
+
return 'string' if spec.nil?
|
|
51
|
+
|
|
52
|
+
{ 'type' => 'bytes', 'logicalType' => 'decimal', 'precision' => spec[0], 'scale' => spec[1] }
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def timestamp(_node, site, *)
|
|
56
|
+
{ 'type' => 'long', 'logicalType' => "timestamp-#{Options.timestamp(site.opts, @timestamp, site)}" }
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def symbol(node, site, base)
|
|
60
|
+
symbols = node.options[:enum]
|
|
61
|
+
return 'string' unless symbols.is_a?(::Array) && !symbols.empty? && site.opts[:enum] != false
|
|
62
|
+
|
|
63
|
+
names = symbols.map(&:to_s)
|
|
64
|
+
return 'string' unless names.all? { |name| Names::VALID.match?(name) }
|
|
65
|
+
|
|
66
|
+
enum(names, site, base)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def enum(names, site, base)
|
|
70
|
+
type = { 'type' => 'enum', 'name' => base, 'symbols' => names }
|
|
71
|
+
type['default'] = site.opts[:enum_default].to_s if site.opts.key?(:enum_default)
|
|
72
|
+
full = site.frame.namespace ? "#{site.frame.namespace}.#{base}" : base
|
|
73
|
+
status = @names.claim(full, type, 'an enum', 'rename one of the fields')
|
|
74
|
+
status == :seen ? full : type
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def unmapped(type, site)
|
|
78
|
+
raise ::ArgumentError, "FieldStruct::Avro has no mapping for #{type.class.name || type.class.inspect} (#{site.where}); " \
|
|
79
|
+
'give the field avro: {type: …}'
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'site'
|
|
4
|
+
|
|
5
|
+
module FieldStruct
|
|
6
|
+
module Avro
|
|
7
|
+
# Builds the Avro record schema of a struct class by walking it with FieldStruct::Introspection (AV-B4..AV-B30).
|
|
8
|
+
class SchemaBuilder
|
|
9
|
+
# @param klass [::Class] a named struct class
|
|
10
|
+
# @param serialization [::Symbol]
|
|
11
|
+
# @param timestamp [::Symbol]
|
|
12
|
+
# @return [Entry]
|
|
13
|
+
# @raise [::ArgumentError] for anything the schema cannot hold
|
|
14
|
+
def self.entry(klass, serialization, timestamp)
|
|
15
|
+
new(klass, serialization, timestamp).entry
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# @param klass [::Class]
|
|
19
|
+
# @param serialization [::Symbol]
|
|
20
|
+
# @param timestamp [::Symbol]
|
|
21
|
+
def initialize(klass, serialization, timestamp)
|
|
22
|
+
@klass = klass
|
|
23
|
+
@serialization = serialization
|
|
24
|
+
@names = Names.new
|
|
25
|
+
@scalars = ScalarTypes.new(timestamp, @names)
|
|
26
|
+
@unions = UnionRules.new(serialization)
|
|
27
|
+
@walked = []
|
|
28
|
+
@stamps = []
|
|
29
|
+
@building = {}
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# @return [Entry]
|
|
33
|
+
def entry
|
|
34
|
+
Check.named!(@klass)
|
|
35
|
+
hash = root
|
|
36
|
+
check_serialization
|
|
37
|
+
Entry.new(hash, parse(hash), @stamps)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def root
|
|
43
|
+
name, namespace = Names.split(@klass)
|
|
44
|
+
full = [namespace, name].compact.join('.')
|
|
45
|
+
@names.claim(full, @klass, @klass.name, "put #{@klass} in a module")
|
|
46
|
+
@names.remember(@klass, full)
|
|
47
|
+
record(@klass, name, namespace, Trail.root(@klass), explicit_namespace: namespace)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def record(klass, name, namespace, trail, explicit_namespace: nil)
|
|
51
|
+
stamp(klass)
|
|
52
|
+
frame = Frame.new(klass, name, namespace, trail)
|
|
53
|
+
@building[klass] = true
|
|
54
|
+
fields = FieldStruct::Introspection.properties(klass, serialization: @serialization).map { |prop| field(frame, prop) }
|
|
55
|
+
@building.delete(klass)
|
|
56
|
+
head = { 'type' => 'record', 'name' => name }
|
|
57
|
+
head['namespace'] = explicit_namespace if explicit_namespace
|
|
58
|
+
head.merge('fields' => fields)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def stamp(klass)
|
|
62
|
+
return if @walked.include?(klass)
|
|
63
|
+
|
|
64
|
+
@walked << klass
|
|
65
|
+
@stamps << Stamp.of(klass, @serialization)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def field(frame, prop)
|
|
69
|
+
field = prop.field
|
|
70
|
+
label = frame.trail.label(field)
|
|
71
|
+
Names.check_field(label, prop.key)
|
|
72
|
+
site = Site.new(frame, field, prop.key, Options.read(field) { label }, label)
|
|
73
|
+
override = Options.override(site.opts)
|
|
74
|
+
type = override.nil? ? map(site, prop.node, "#{frame.name}_#{prop.key}") : Options.stringify(override)
|
|
75
|
+
field_hash(site, type)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def field_hash(site, type)
|
|
79
|
+
field = site.field
|
|
80
|
+
hash = { 'name' => site.key }
|
|
81
|
+
hash['doc'] = field.description if field.description
|
|
82
|
+
hash['type'], default = typed(field, type)
|
|
83
|
+
hash['default'] = default unless default.equal?(Defaults::NONE)
|
|
84
|
+
hash['x-fs-sensitive'] = true if field.sensitive?
|
|
85
|
+
hash
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def typed(field, type)
|
|
89
|
+
literal = Defaults.literal(field, type)
|
|
90
|
+
return [type, literal] if field.required? && !field.nullable?
|
|
91
|
+
return [nullable(type), field.required? ? Defaults::NONE : nil] if literal.equal?(Defaults::NONE)
|
|
92
|
+
|
|
93
|
+
[null_last(type), literal]
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def null_last(type)
|
|
97
|
+
type.is_a?(::Array) ? (type - ['null']) + ['null'] : [type, 'null']
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def nullable(type)
|
|
101
|
+
type.is_a?(::Array) ? ['null', *(type - ['null'])] : ['null', type]
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def map(site, node, base)
|
|
105
|
+
case node
|
|
106
|
+
when FieldStruct::Introspection::ScalarNode then @scalars.call(node, site, base)
|
|
107
|
+
when FieldStruct::Introspection::ArrayNode then array(site, node, base)
|
|
108
|
+
when FieldStruct::Introspection::MapNode then string_map(site, node, base)
|
|
109
|
+
when FieldStruct::Introspection::UnionNode then union(site, node, base)
|
|
110
|
+
else struct(site, node.klass, base)
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def array(site, node, base)
|
|
115
|
+
element = node.element
|
|
116
|
+
if element.nil?
|
|
117
|
+
raise ::ArgumentError, "FieldStruct::Avro can't describe #{site.label}: an array of arrays or unions records no element type"
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
{ 'type' => 'array', 'items' => nullable(map(site, element, base)) }
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# A +:hash+ field: Avro's String-keyed map, values nullable like array items (AV-B10a).
|
|
124
|
+
def string_map(site, node, base)
|
|
125
|
+
element = node.element
|
|
126
|
+
if element.nil?
|
|
127
|
+
raise ::ArgumentError, "FieldStruct::Avro can't describe #{site.label}: a map of arrays or unions records no value type"
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
{ 'type' => 'map', 'values' => nullable(map(site, element, base)) }
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def union(site, node, base)
|
|
134
|
+
types = node.members.each_with_index.map { |member, index| map(site, member, "#{base}_#{index}") }
|
|
135
|
+
classes = node.members.grep(FieldStruct::Introspection::StructNode).map(&:klass)
|
|
136
|
+
@unions.check_shadowing(classes, site.label)
|
|
137
|
+
@unions.order(types, site.frame, site.label)
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def struct(site, klass, base)
|
|
141
|
+
named = FieldStruct::Introspection.named?(klass)
|
|
142
|
+
recursive!(site, klass) unless named
|
|
143
|
+
return reference(site, klass) if named && @names.full_name_of(klass)
|
|
144
|
+
|
|
145
|
+
define(site, klass, named, base)
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def define(site, klass, named, base)
|
|
149
|
+
name, namespace = named ? Names.split(klass) : [base, nil]
|
|
150
|
+
effective = namespace || site.frame.namespace
|
|
151
|
+
full = effective ? "#{effective}.#{name}" : name
|
|
152
|
+
return reference(site, klass) if @names.claim(full, klass, struct_label(klass, named), fix(klass, named)) == :seen
|
|
153
|
+
|
|
154
|
+
@names.remember(klass, full)
|
|
155
|
+
record(klass, name, effective, site.frame.trail.enter(klass, site.field), explicit_namespace: namespace)
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def struct_label(klass, named)
|
|
159
|
+
named ? klass.name : 'an anonymous struct'
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def fix(klass, named)
|
|
163
|
+
named ? "put #{klass} in a module" : 'rename one of the fields'
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def recursive!(site, klass)
|
|
167
|
+
return unless @building[klass]
|
|
168
|
+
|
|
169
|
+
raise ::ArgumentError, "FieldStruct::Avro cannot describe a recursive anonymous struct (at #{site.label}); assign it to a constant"
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
def reference(site, klass)
|
|
173
|
+
full = @names.full_name_of(klass)
|
|
174
|
+
check_bare(site, klass, full) if FieldStruct::Introspection.named?(klass)
|
|
175
|
+
full
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
def check_bare(site, klass, full)
|
|
179
|
+
namespace = site.frame.namespace
|
|
180
|
+
return if full.include?('.') || namespace.nil?
|
|
181
|
+
|
|
182
|
+
raise ::ArgumentError, "FieldStruct::Avro: #{klass} has no namespace and is used inside #{site.frame.class_label} " \
|
|
183
|
+
"(Avro namespace #{namespace}), where Avro would read #{full} as #{namespace}.#{full}; " \
|
|
184
|
+
"put #{klass} in a module"
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
def check_serialization
|
|
188
|
+
return if @serialization == :avro || @walked.any? { |klass| klass.metadata.serializations.key?(@serialization) }
|
|
189
|
+
|
|
190
|
+
declared = ([:avro] + @walked.flat_map { |klass| klass.metadata.serializations.keys }).uniq
|
|
191
|
+
raise ::ArgumentError, "FieldStruct::Avro: no struct in #{@klass.name} declares serialize #{@serialization.inspect} " \
|
|
192
|
+
"(declared: #{declared.inspect})"
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
def parse(hash)
|
|
196
|
+
::Avro::Schema.parse(Oj.dump(hash, mode: :compat))
|
|
197
|
+
rescue ::StandardError => e
|
|
198
|
+
raise ::ArgumentError, "FieldStruct::Avro: #{@klass.name || @klass.inspect} doesn't make a valid Avro schema " \
|
|
199
|
+
"(check the avro: options of its fields and nested structs): #{e.message}"
|
|
200
|
+
end
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FieldStruct
|
|
4
|
+
module Avro
|
|
5
|
+
# A record being built: its struct class, Avro name, effective namespace and trail.
|
|
6
|
+
Frame = Struct.new(:klass, :name, :namespace, :trail) do
|
|
7
|
+
# @return [::String] the struct as errors name it
|
|
8
|
+
def class_label
|
|
9
|
+
klass.name || klass.inspect
|
|
10
|
+
end
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
# One field being mapped: the record around it, the field, its Avro name, its +avro:+ options and its label.
|
|
14
|
+
Site = Struct.new(:frame, :field, :key, :opts, :label) do
|
|
15
|
+
# @return [::String] +Klass#field+ plus the Avro name, for the unmappable-type error
|
|
16
|
+
def where
|
|
17
|
+
"#{label} (Avro #{frame.name}_#{key})"
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# One field being converted: the trail of the struct around it, the field and its +avro:+ options.
|
|
22
|
+
Spot = Struct.new(:trail, :field, :opts) do
|
|
23
|
+
# @return [::String] +Klass#field+
|
|
24
|
+
def label
|
|
25
|
+
trail.label(field)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# @return [::Boolean] whether the field, or anything above it, is sensitive
|
|
29
|
+
def sensitive?
|
|
30
|
+
trail.sensitive?(field)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# @param klass [::Class] a struct held by the field
|
|
34
|
+
# @return [Trail] the trail inside it
|
|
35
|
+
def enter(klass)
|
|
36
|
+
trail.enter(klass, field)
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FieldStruct
|
|
4
|
+
module Avro
|
|
5
|
+
# What a cached result was built from, for one struct class: its fields (compared by identity, a redeclared
|
|
6
|
+
# field is a new object), the identity of its +serialize+ mapping for the format and its class name (AV-B52).
|
|
7
|
+
class Stamp
|
|
8
|
+
# @param klass [::Class]
|
|
9
|
+
# @param serialization [::Symbol]
|
|
10
|
+
# @return [Stamp] the class as it is now
|
|
11
|
+
def self.of(klass, serialization)
|
|
12
|
+
new(klass, serialization, klass.metadata.to_a, klass.metadata.serialization(serialization), klass.name)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# @param klass [::Class]
|
|
16
|
+
# @param serialization [::Symbol]
|
|
17
|
+
# @param fields [::Array<FieldStruct::Field>]
|
|
18
|
+
# @param mapping [::Hash]
|
|
19
|
+
# @param name [::String, nil]
|
|
20
|
+
def initialize(klass, serialization, fields, mapping, name)
|
|
21
|
+
@klass = klass
|
|
22
|
+
@serialization = serialization
|
|
23
|
+
@fields = fields
|
|
24
|
+
@mapping = mapping
|
|
25
|
+
@name = name
|
|
26
|
+
freeze
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# @return [::Boolean] whether the class still looks the way it did
|
|
30
|
+
def fresh?
|
|
31
|
+
now = @klass.metadata
|
|
32
|
+
@name == @klass.name && @mapping.equal?(now.serialization(@serialization)) && same_fields?(now)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
private
|
|
36
|
+
|
|
37
|
+
def same_fields?(metadata)
|
|
38
|
+
index = 0
|
|
39
|
+
metadata.each do |field|
|
|
40
|
+
return false unless @fields[index].equal?(field)
|
|
41
|
+
|
|
42
|
+
index += 1
|
|
43
|
+
end
|
|
44
|
+
index == @fields.size
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FieldStruct
|
|
4
|
+
module Avro
|
|
5
|
+
# Where a conversion is: the nearest named struct, the field path inside anonymous structs below it, and
|
|
6
|
+
# whether any field on the way is sensitive. Messages name fields by it and never echo a sensitive value (AV-I4).
|
|
7
|
+
class Trail
|
|
8
|
+
# @return [::String] the nearest named struct's name
|
|
9
|
+
attr_reader :base
|
|
10
|
+
# @return [::Array<::Symbol>] field names below the named struct
|
|
11
|
+
attr_reader :path
|
|
12
|
+
# @return [::Boolean] whether a field above is sensitive
|
|
13
|
+
attr_reader :sensitive
|
|
14
|
+
|
|
15
|
+
# @param klass [::Class]
|
|
16
|
+
# @return [Trail] the trail at the root struct
|
|
17
|
+
def self.root(klass)
|
|
18
|
+
new(klass.name || klass.inspect, [], false)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# @param base [::String]
|
|
22
|
+
# @param path [::Array<::Symbol>]
|
|
23
|
+
# @param sensitive [::Boolean]
|
|
24
|
+
def initialize(base, path, sensitive)
|
|
25
|
+
@base = base
|
|
26
|
+
@path = path
|
|
27
|
+
@sensitive = sensitive
|
|
28
|
+
freeze
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# The trail inside a struct held by a field.
|
|
32
|
+
#
|
|
33
|
+
# @param klass [::Class] the struct held
|
|
34
|
+
# @param field [FieldStruct::Field] the field that holds it
|
|
35
|
+
# @return [Trail]
|
|
36
|
+
def enter(klass, field)
|
|
37
|
+
sensitive = @sensitive || field.sensitive?
|
|
38
|
+
return Trail.new(klass.name, [], sensitive) if FieldStruct::Introspection.named?(klass)
|
|
39
|
+
|
|
40
|
+
Trail.new(@base, [*@path, field.name], sensitive)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# @param field [FieldStruct::Field]
|
|
44
|
+
# @return [::String] +Klass#field+ (or +Holder#inner.field+ inside an anonymous struct)
|
|
45
|
+
def label(field)
|
|
46
|
+
"#{@base}##{[*@path, field.name].join('.')}"
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# @param field [FieldStruct::Field]
|
|
50
|
+
# @return [::Boolean] whether the field, or anything above it, is sensitive
|
|
51
|
+
def sensitive?(field)
|
|
52
|
+
@sensitive || field.sensitive?
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FieldStruct
|
|
4
|
+
module Avro
|
|
5
|
+
# What Avro allows in a union, and what it would silently get wrong (AV-B11, AV-B12).
|
|
6
|
+
class UnionRules
|
|
7
|
+
# Avro types that are not named.
|
|
8
|
+
PRIMITIVES = %w[null boolean int long float double bytes string array map].freeze
|
|
9
|
+
|
|
10
|
+
# @param serialization [::Symbol]
|
|
11
|
+
def initialize(serialization)
|
|
12
|
+
@serialization = serialization
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# Refuses two members in one slot, then puts the +long+ slot first.
|
|
16
|
+
#
|
|
17
|
+
# @param types [::Array<::Object>] the members' Avro types, declared order
|
|
18
|
+
# @param frame [Frame] for the namespace of unqualified names
|
|
19
|
+
# @param label [::String]
|
|
20
|
+
# @return [::Array<::Object>]
|
|
21
|
+
# @raise [::ArgumentError] when two members share a slot
|
|
22
|
+
def order(types, frame, label)
|
|
23
|
+
slots = types.map { |type| slot(type, frame) }
|
|
24
|
+
clash = slots.find { |name| slots.count(name) > 1 }
|
|
25
|
+
raise ::ArgumentError, "FieldStruct::Avro can't describe #{label}: two union members both become Avro #{clash}" if clash
|
|
26
|
+
|
|
27
|
+
types.zip(slots).sort_by.with_index { |(_, name), index| [name == 'long' ? 0 : 1, index] }.map(&:first)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Refuses a struct member that Avro would pick instead of a later one (shape matching).
|
|
31
|
+
#
|
|
32
|
+
# @param classes [::Array<::Class>] the struct members, declared order
|
|
33
|
+
# @param label [::String]
|
|
34
|
+
# @return [void]
|
|
35
|
+
# @raise [::ArgumentError] when a later struct fits an earlier one
|
|
36
|
+
def check_shadowing(classes, label)
|
|
37
|
+
classes.each_with_index do |earlier, index|
|
|
38
|
+
classes.drop(index + 1).each { |later| check_pair(earlier, later, label) }
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
private
|
|
43
|
+
|
|
44
|
+
def slot(type, frame)
|
|
45
|
+
case type
|
|
46
|
+
when ::String then type == 'bytes' ? 'string' : qualified(type, frame, primitive: PRIMITIVES.include?(type))
|
|
47
|
+
when ::Hash then hash_slot(type, frame)
|
|
48
|
+
else type.to_s
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def hash_slot(type, frame)
|
|
53
|
+
kind = type['type']
|
|
54
|
+
return qualified(type['name'], frame, namespace: type['namespace']) if %w[record enum fixed].include?(kind)
|
|
55
|
+
|
|
56
|
+
kind == 'bytes' ? 'string' : kind.to_s
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def qualified(name, frame, primitive: false, namespace: nil)
|
|
60
|
+
return name if primitive || name.include?('.')
|
|
61
|
+
|
|
62
|
+
namespace ||= frame.namespace
|
|
63
|
+
namespace ? "#{namespace}.#{name}" : name
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def check_pair(earlier, later, label)
|
|
67
|
+
required = required_keys(earlier)
|
|
68
|
+
later_keys = keys(later)
|
|
69
|
+
return unless required.all? { |key| later_keys.include?(key) }
|
|
70
|
+
|
|
71
|
+
names = [later, earlier].map { |klass| klass.name || klass.inspect }
|
|
72
|
+
raise ::ArgumentError, shadow_message(label, names, required.empty?, required_keys(later).all? { |k| keys(earlier).include?(k) })
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def shadow_message(label, names, no_required, swap_useless)
|
|
76
|
+
later, earlier = names
|
|
77
|
+
why = no_required ? "#{earlier} has no required fields, so any record fits it" : "every required field of #{earlier} is in #{later}"
|
|
78
|
+
swap = swap_useless ? '' : ", or put #{later} first"
|
|
79
|
+
"FieldStruct::Avro can't describe #{label}: a #{later} would be written as #{earlier}, which Avro picks by shape (#{why}). " \
|
|
80
|
+
"Give the structs distinct required fields#{swap}"
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def keys(klass)
|
|
84
|
+
FieldStruct::Introspection.properties(klass, serialization: @serialization).map(&:key)
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def required_keys(klass)
|
|
88
|
+
FieldStruct::Introspection.properties(klass, serialization: @serialization).select do |prop|
|
|
89
|
+
prop.field.required? && !overridden?(prop.field)
|
|
90
|
+
end.map(&:key)
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def overridden?(field)
|
|
94
|
+
avro = field.options[:avro]
|
|
95
|
+
avro.is_a?(::Hash) && !(avro[:type] || avro['type']).nil?
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FieldStruct
|
|
4
|
+
module Avro
|
|
5
|
+
# The JSON text of a +:value+ field (AV-B24), through Oj in compat mode.
|
|
6
|
+
module ValueJson
|
|
7
|
+
module_function
|
|
8
|
+
|
|
9
|
+
# @param value [::Object]
|
|
10
|
+
# @return [::String] the JSON +to_json+ would write for the value
|
|
11
|
+
def dump(value)
|
|
12
|
+
Oj.dump(json_ready(value), mode: :compat)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# @param text [::String]
|
|
16
|
+
# @param where [Spot] the field being read; its label is built only for the error
|
|
17
|
+
# @return [::Object] numbers with a fraction are BigDecimal; times and decimals stay Strings
|
|
18
|
+
# @raise [::ArgumentError] when the text does not parse (the parse error is not kept: it quotes the text)
|
|
19
|
+
def load(text, where)
|
|
20
|
+
Oj.load(text, mode: :compat, decimal_class: ::BigDecimal)
|
|
21
|
+
rescue ::StandardError => e
|
|
22
|
+
raise ::ArgumentError, "FieldStruct::Avro (#{where.label}) holds :value JSON that doesn't parse (#{e.class.name})", cause: nil
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# @param value [::Object]
|
|
26
|
+
# @return [::Object] the value with Symbols, decimals and times in their JSON forms
|
|
27
|
+
def json_ready(value)
|
|
28
|
+
case value
|
|
29
|
+
when ::Array then value.map { |item| json_ready(item) }
|
|
30
|
+
when ::Hash then value.to_h { |key, item| [key.to_s, json_ready(item)] }
|
|
31
|
+
else scalar_ready(value)
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# @param value [::Object]
|
|
36
|
+
# @return [::Object]
|
|
37
|
+
def scalar_ready(value)
|
|
38
|
+
case value
|
|
39
|
+
when ::Symbol then value.to_s
|
|
40
|
+
when ::BigDecimal then value.to_s('F')
|
|
41
|
+
when ::Date, ::Time then value.iso8601
|
|
42
|
+
else value.respond_to?(:as_json) ? value.as_json : value
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|