field_struct-avro 1.0.0.pre1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +33 -0
- data/LICENSE.txt +21 -0
- data/README.md +226 -0
- data/lib/field_struct/avro/bytes_reader.rb +37 -0
- data/lib/field_struct/avro/cache.rb +117 -0
- data/lib/field_struct/avro/check.rb +72 -0
- data/lib/field_struct/avro/datum_reader.rb +160 -0
- data/lib/field_struct/avro/datum_writer.rb +152 -0
- data/lib/field_struct/avro/defaults.rb +68 -0
- data/lib/field_struct/avro/entry.rb +48 -0
- data/lib/field_struct/avro/evolution.rb +52 -0
- data/lib/field_struct/avro/methods.rb +36 -0
- data/lib/field_struct/avro/names.rb +80 -0
- data/lib/field_struct/avro/options.rb +197 -0
- data/lib/field_struct/avro/projection.rb +59 -0
- data/lib/field_struct/avro/resolution.rb +110 -0
- data/lib/field_struct/avro/scalar_types.rb +83 -0
- data/lib/field_struct/avro/schema_builder.rb +203 -0
- data/lib/field_struct/avro/site.rb +40 -0
- data/lib/field_struct/avro/stamp.rb +48 -0
- data/lib/field_struct/avro/trail.rb +56 -0
- data/lib/field_struct/avro/union_rules.rb +99 -0
- data/lib/field_struct/avro/value_json.rb +47 -0
- data/lib/field_struct/avro/version.rb +10 -0
- data/lib/field_struct/avro/writer_schema.rb +33 -0
- data/lib/field_struct/avro/zone.rb +23 -0
- data/lib/field_struct/avro.rb +138 -0
- data/sig/field_struct-avro.rbs +97 -0
- metadata +149 -0
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FieldStruct
|
|
4
|
+
module Avro
|
|
5
|
+
# Turns a struct instance into the Hash the avro gem writes (AV-B19..AV-B24).
|
|
6
|
+
class DatumWriter
|
|
7
|
+
# The node classes of the introspection walk.
|
|
8
|
+
NODES = FieldStruct::Introspection
|
|
9
|
+
|
|
10
|
+
# @param serialization [::Symbol]
|
|
11
|
+
# @param timestamp [::Symbol] the call's timestamp precision
|
|
12
|
+
def initialize(serialization, timestamp)
|
|
13
|
+
@serialization = serialization
|
|
14
|
+
@timestamp = timestamp
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# @param instance [FieldStruct::Base]
|
|
18
|
+
# @param trail [Trail]
|
|
19
|
+
# @return [::Hash{::String=>::Object}]
|
|
20
|
+
def struct(instance, trail)
|
|
21
|
+
Cache.properties(instance.class, @serialization).to_h do |prop|
|
|
22
|
+
[prop.key, property(instance.public_send(prop.field.name), prop, trail)]
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
private
|
|
27
|
+
|
|
28
|
+
def property(value, prop, trail)
|
|
29
|
+
field = prop.field
|
|
30
|
+
opts = Options.read(field) { trail.label(field) }
|
|
31
|
+
override = Options.override(opts)
|
|
32
|
+
return nil if value.nil?
|
|
33
|
+
return overridden(value, prop.node, Options.stringify(override), Spot.new(trail, field, opts)) unless override.nil?
|
|
34
|
+
|
|
35
|
+
node = prop.node
|
|
36
|
+
# Most fields are scalars that need neither the options nor an error label, so they skip building a Spot.
|
|
37
|
+
return plain_scalar(value, node) if plain_scalar?(node)
|
|
38
|
+
|
|
39
|
+
node_value(value, node, Spot.new(trail, field, opts))
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def plain_scalar?(node)
|
|
43
|
+
return false unless node.is_a?(NODES::ScalarNode)
|
|
44
|
+
|
|
45
|
+
type = node.type
|
|
46
|
+
!type.is_a?(Types::BigDecimal) && !type.is_a?(Types::Time) && !type.is_a?(Types::DateTime)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def plain_scalar(value, node)
|
|
50
|
+
case node.type
|
|
51
|
+
when Types::Symbol then value.to_s
|
|
52
|
+
when Types::Value then ValueJson.dump(value)
|
|
53
|
+
else value
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def node_value(value, node, site)
|
|
58
|
+
return nil if value.nil?
|
|
59
|
+
|
|
60
|
+
case node
|
|
61
|
+
when NODES::ScalarNode then scalar(value, node, site)
|
|
62
|
+
when NODES::ArrayNode then array(value, node, site)
|
|
63
|
+
when NODES::MapNode then entries(value, node, site)
|
|
64
|
+
when NODES::StructNode then nested(value, node.klass, site)
|
|
65
|
+
else union(value, node, site)
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def array(value, node, site)
|
|
70
|
+
element = node.element
|
|
71
|
+
return raw(value, site) if element.nil? || !value.is_a?(::Array)
|
|
72
|
+
|
|
73
|
+
value.map { |item| node_value(item, element, site) }
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def entries(value, node, site)
|
|
77
|
+
element = node.element
|
|
78
|
+
return raw(value, site) if element.nil? || !value.is_a?(::Hash)
|
|
79
|
+
|
|
80
|
+
value.transform_values { |item| node_value(item, element, site) }
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def nested(value, klass, site)
|
|
84
|
+
return value unless value.is_a?(klass)
|
|
85
|
+
|
|
86
|
+
struct(value, site.enter(klass))
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def scalar(value, node, site)
|
|
90
|
+
case node.type
|
|
91
|
+
when Types::BigDecimal then decimal(value, node, site)
|
|
92
|
+
when Types::Time, Types::DateTime then time(value, site)
|
|
93
|
+
else plain_scalar(value, node)
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def decimal(value, node, site)
|
|
98
|
+
spec = Options.decimal(site.opts, node.options, node.type, site)
|
|
99
|
+
spec.nil? && value.is_a?(::BigDecimal) ? value.to_s('F') : value
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def time(value, site)
|
|
103
|
+
value = value.to_time if value.is_a?(::DateTime)
|
|
104
|
+
return value unless value.is_a?(::Time)
|
|
105
|
+
|
|
106
|
+
precision = Options.timestamp(site.opts, @timestamp, site)
|
|
107
|
+
check_precision(value, precision, site)
|
|
108
|
+
value
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def check_precision(time, precision, site)
|
|
112
|
+
return if (time.nsec % Options::UNITS.fetch(precision)).zero?
|
|
113
|
+
|
|
114
|
+
shown = site.sensitive? ? 'a sensitive time' : time.iso8601(9)
|
|
115
|
+
raise ::ArgumentError, "FieldStruct::Avro: #{site.label} is #{shown}, which has more precision than " \
|
|
116
|
+
"timestamp-#{precision}; round it, or use timestamp: :nanos"
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def union(value, node, site)
|
|
120
|
+
members = node.members.partition { |member| !value_member?(member) }.flatten(1)
|
|
121
|
+
member = members.find { |candidate| matches?(candidate, value) }
|
|
122
|
+
member ? node_value(value, member, site) : value
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def value_member?(member)
|
|
126
|
+
member.is_a?(NODES::ScalarNode) && member.type.is_a?(Types::Value)
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def matches?(member, value)
|
|
130
|
+
return value.is_a?(member.klass) if member.is_a?(NODES::StructNode)
|
|
131
|
+
|
|
132
|
+
Array(member.type.ruby_type).any? { |klass| value.is_a?(klass) }
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def overridden(value, node, type, site)
|
|
136
|
+
return ValueJson.dump(value) if node.is_a?(NODES::ScalarNode) && node.type.is_a?(Types::Value) && Options.string_type?(type)
|
|
137
|
+
|
|
138
|
+
raw(value, site)
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def raw(value, site)
|
|
142
|
+
case value
|
|
143
|
+
when ::Symbol then value.to_s
|
|
144
|
+
when ::Hash then value.to_h { |key, item| [raw(key, site), raw(item, site)] }
|
|
145
|
+
when ::Array then value.map { |item| raw(item, site) }
|
|
146
|
+
when FieldStruct::Base then struct(value, site.enter(value.class))
|
|
147
|
+
else value
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
end
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FieldStruct
|
|
4
|
+
module Avro
|
|
5
|
+
# Which field defaults can be written into the schema (AV-B8): only a literal of the field's own Avro type.
|
|
6
|
+
module Defaults
|
|
7
|
+
# Returned when the default is not a usable literal (nil itself is a valid Avro default).
|
|
8
|
+
NONE = ::Object.new.freeze
|
|
9
|
+
|
|
10
|
+
# The method that judges a default for each Avro primitive.
|
|
11
|
+
BY_TYPE = { 'boolean' => :boolean, 'long' => :long, 'double' => :double, 'string' => :string }.freeze
|
|
12
|
+
|
|
13
|
+
module_function
|
|
14
|
+
|
|
15
|
+
# @param field [FieldStruct::Field]
|
|
16
|
+
# @param type [::Object] the field's Avro type
|
|
17
|
+
# @return [::Object] the literal to write, or +NONE+
|
|
18
|
+
def literal(field, type)
|
|
19
|
+
default = field.default
|
|
20
|
+
return NONE if default.nil?
|
|
21
|
+
return enum(type, default) if type.is_a?(::Hash)
|
|
22
|
+
|
|
23
|
+
method = BY_TYPE[type]
|
|
24
|
+
method ? send(method, field, default) : NONE
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# @param default [::Object]
|
|
28
|
+
# @return [::Boolean, NONE]
|
|
29
|
+
def boolean(_field, default)
|
|
30
|
+
[true, false].include?(default) ? default : NONE
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# @param default [::Object]
|
|
34
|
+
# @return [::Integer, NONE]
|
|
35
|
+
def long(_field, default)
|
|
36
|
+
default.is_a?(::Integer) ? default : NONE
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# @param default [::Object]
|
|
40
|
+
# @return [::Float, NONE]
|
|
41
|
+
def double(_field, default)
|
|
42
|
+
return default.to_f if default.is_a?(::Integer)
|
|
43
|
+
|
|
44
|
+
default.is_a?(::Float) && default.finite? ? default : NONE
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# @param field [FieldStruct::Field]
|
|
48
|
+
# @param default [::Object]
|
|
49
|
+
# @return [::String, NONE]
|
|
50
|
+
def string(field, default)
|
|
51
|
+
type = field.type_instance
|
|
52
|
+
return default.to_s if type.is_a?(Types::Symbol) && default.is_a?(::Symbol)
|
|
53
|
+
return NONE if type.is_a?(Types::Binary) || !type.is_a?(Types::String)
|
|
54
|
+
|
|
55
|
+
default.is_a?(::String) ? default : NONE
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# @param type [::Hash]
|
|
59
|
+
# @param default [::Object]
|
|
60
|
+
# @return [::String, NONE]
|
|
61
|
+
def enum(type, default)
|
|
62
|
+
return NONE unless type['type'] == 'enum' && default.is_a?(::Symbol) && type['symbols'].include?(default.to_s)
|
|
63
|
+
|
|
64
|
+
default.to_s
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FieldStruct
|
|
4
|
+
module Avro
|
|
5
|
+
# The cached result of building a class's schema: the frozen Hash, the schema the avro gem parsed from it, and
|
|
6
|
+
# the stamps of every struct class the walk read (AV-B50..AV-B52).
|
|
7
|
+
class Entry
|
|
8
|
+
# @return [::Hash{::String=>::Object}] deeply frozen, owned by the entry
|
|
9
|
+
attr_reader :schema
|
|
10
|
+
# @return [::Avro::Schema]
|
|
11
|
+
attr_reader :parsed
|
|
12
|
+
|
|
13
|
+
# @param schema [::Hash{::String=>::Object}] what the builder produced; copied, never kept
|
|
14
|
+
# @param parsed [::Avro::Schema]
|
|
15
|
+
# @param stamps [::Array<Stamp>]
|
|
16
|
+
def initialize(schema, parsed, stamps)
|
|
17
|
+
@schema = Entry.deep_copy(schema, freeze: true)
|
|
18
|
+
@parsed = parsed
|
|
19
|
+
@stamps = stamps.freeze
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# @return [::Boolean] whether no walked class changed since the build
|
|
23
|
+
def fresh?
|
|
24
|
+
@stamps.all?(&:fresh?)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# @return [::Hash{::String=>::Object}] a fresh, unfrozen deep copy of the schema
|
|
28
|
+
def schema_copy
|
|
29
|
+
Entry.deep_copy(@schema, freeze: false)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Copies Hashes, Arrays and Strings; other values are shared.
|
|
33
|
+
#
|
|
34
|
+
# @param value [::Object]
|
|
35
|
+
# @param freeze [::Boolean] freeze every copied container
|
|
36
|
+
# @return [::Object]
|
|
37
|
+
def self.deep_copy(value, freeze:)
|
|
38
|
+
copy = case value
|
|
39
|
+
when ::Hash then value.to_h { |key, item| [deep_copy(key, freeze: freeze), deep_copy(item, freeze: freeze)] }
|
|
40
|
+
when ::Array then value.map { |item| deep_copy(item, freeze: freeze) }
|
|
41
|
+
when ::String then value.dup
|
|
42
|
+
else return value
|
|
43
|
+
end
|
|
44
|
+
freeze ? copy.freeze : copy
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FieldStruct
|
|
4
|
+
module Avro
|
|
5
|
+
# Decoding bytes written with another schema: parses the writer schema, refuses the pairs the avro gem would
|
|
6
|
+
# misread (AV-B39) and gives the schema to read with, minus the fields the writer lacked (AV-B40).
|
|
7
|
+
module Evolution
|
|
8
|
+
module_function
|
|
9
|
+
|
|
10
|
+
# @param entry [Entry] the class's schema entry
|
|
11
|
+
# @param writer_schema [::Hash, ::String, ::Avro::Schema, nil]
|
|
12
|
+
# @return [Array(::Avro::Schema, ::Avro::Schema)] the schemas to write-resolve and read with
|
|
13
|
+
# @raise [::ArgumentError] for a bad writer schema or an incompatible change
|
|
14
|
+
def schemas(entry, writer_schema)
|
|
15
|
+
return [entry.parsed, entry.parsed] if writer_schema.nil?
|
|
16
|
+
|
|
17
|
+
Cache.writer(writer_schema, entry)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Parses and checks one writer schema text against a reader entry (cached by +Cache.writer+).
|
|
21
|
+
#
|
|
22
|
+
# @param text [::String] the writer schema as JSON
|
|
23
|
+
# @param entry [Entry]
|
|
24
|
+
# @return [Array(::Avro::Schema, ::Avro::Schema)]
|
|
25
|
+
# @raise [::ArgumentError] for an invalid writer schema or an incompatible change
|
|
26
|
+
def resolve(text, entry)
|
|
27
|
+
writer = WriterSchema.parse(text)
|
|
28
|
+
result = Resolution.call(writer, entry.parsed)
|
|
29
|
+
refuse!(result.problems)
|
|
30
|
+
[writer, reader(entry, result.lacking)]
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# @param entry [Entry]
|
|
34
|
+
# @param lacking [::Hash{::String=>::Array<::String>}]
|
|
35
|
+
# @return [::Avro::Schema]
|
|
36
|
+
def reader(entry, lacking)
|
|
37
|
+
return entry.parsed if lacking.values.all?(&:empty?)
|
|
38
|
+
|
|
39
|
+
::Avro::Schema.parse(Oj.dump(Projection.call(entry.schema, lacking), mode: :compat))
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# @param problems [::Array<::String>]
|
|
43
|
+
# @return [void]
|
|
44
|
+
def refuse!(problems)
|
|
45
|
+
return if problems.empty?
|
|
46
|
+
|
|
47
|
+
raise ::ArgumentError, "FieldStruct::Avro can't read these bytes: #{problems.join('; ')}. " \
|
|
48
|
+
'The avro gem would decode them as the wrong value, or fail; this is not a compatible schema change'
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FieldStruct
|
|
4
|
+
module Avro
|
|
5
|
+
# Opt-in: +include FieldStruct::Avro::Methods+ in a struct adds +to_avro_schema+, +to_avro+ and +from_avro+.
|
|
6
|
+
module Methods
|
|
7
|
+
# @param base [::Class]
|
|
8
|
+
# @return [void]
|
|
9
|
+
def self.included(base)
|
|
10
|
+
base.extend(ClassMethods)
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
# The class-level methods +Methods+ adds.
|
|
14
|
+
module ClassMethods
|
|
15
|
+
# @param options [::Object] the options of {FieldStruct::Avro.schema}
|
|
16
|
+
# @return [::Hash{::String=>::Object}] see {FieldStruct::Avro.schema}
|
|
17
|
+
def to_avro_schema(**options)
|
|
18
|
+
FieldStruct::Avro.schema(self, **options)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# @param bytes [::String] Avro binary
|
|
22
|
+
# @param options [::Object] the options of {FieldStruct::Avro.decode}
|
|
23
|
+
# @return [FieldStruct::Base] an instance of the class
|
|
24
|
+
def from_avro(bytes, **options)
|
|
25
|
+
FieldStruct::Avro.decode(self, bytes, **options)
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# @param options [::Object] the options of {FieldStruct::Avro.encode}
|
|
30
|
+
# @return [::String] Avro binary
|
|
31
|
+
def to_avro(**options)
|
|
32
|
+
FieldStruct::Avro.encode(self, **options)
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FieldStruct
|
|
4
|
+
module Avro
|
|
5
|
+
# The Avro full names claimed so far in one schema (AV-B13): what holds each name, so a second claim of the
|
|
6
|
+
# same name is either the same thing (a reference) or a clash.
|
|
7
|
+
class Names
|
|
8
|
+
# Avro's name rule.
|
|
9
|
+
# @return [::Regexp]
|
|
10
|
+
VALID = /\A[A-Za-z_][A-Za-z0-9_]*\z/
|
|
11
|
+
|
|
12
|
+
Claim = Struct.new(:token, :label)
|
|
13
|
+
private_constant :Claim
|
|
14
|
+
|
|
15
|
+
def initialize
|
|
16
|
+
@claims = {}
|
|
17
|
+
@struct_names = {}
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Claims a full name for +token+.
|
|
21
|
+
#
|
|
22
|
+
# @param full [::String]
|
|
23
|
+
# @param token [::Object] what holds the name (a class, or the content of an enum)
|
|
24
|
+
# @param label [::String] how an error names the holder
|
|
25
|
+
# @param fix [::String] the advice when the name is taken by something else
|
|
26
|
+
# @return [:new, :seen] +:seen+ when the same token already holds it
|
|
27
|
+
# @raise [::ArgumentError] when something else holds it
|
|
28
|
+
def claim(full, token, label, fix)
|
|
29
|
+
existing = @claims[full]
|
|
30
|
+
if existing
|
|
31
|
+
return :seen if existing.token == token
|
|
32
|
+
|
|
33
|
+
raise ::ArgumentError, "FieldStruct::Avro: #{label} and #{existing.label} both become Avro #{full}; #{fix}"
|
|
34
|
+
end
|
|
35
|
+
@claims[full] = Claim.new(token, label)
|
|
36
|
+
:new
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# @param klass [::Class]
|
|
40
|
+
# @return [::String, nil] the full name a struct was defined under
|
|
41
|
+
def full_name_of(klass)
|
|
42
|
+
@struct_names[klass]
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# @param klass [::Class]
|
|
46
|
+
# @param full [::String]
|
|
47
|
+
# @return [void]
|
|
48
|
+
def remember(klass, full)
|
|
49
|
+
@struct_names[klass] = full
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Splits a named struct's class name into Avro name and namespace and checks every segment.
|
|
53
|
+
#
|
|
54
|
+
# @param klass [::Class] a named struct
|
|
55
|
+
# @return [Array(::String, ::String, nil)] name and namespace (nil without a module)
|
|
56
|
+
# @raise [::ArgumentError] for a segment Avro cannot name
|
|
57
|
+
def self.split(klass)
|
|
58
|
+
segments = klass.name.split('::')
|
|
59
|
+
segments.each do |part|
|
|
60
|
+
next if VALID.match?(part)
|
|
61
|
+
|
|
62
|
+
raise ::ArgumentError, "FieldStruct::Avro: class #{klass.name} is named #{part.inspect}, " \
|
|
63
|
+
'which Avro names can\'t be ([A-Za-z_][A-Za-z0-9_]*)'
|
|
64
|
+
end
|
|
65
|
+
[segments.last, segments.size > 1 ? segments[0...-1].join('.') : nil]
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# @param label [::String] +Klass#field+
|
|
69
|
+
# @param key [::String] the Avro field name
|
|
70
|
+
# @return [void]
|
|
71
|
+
# @raise [::ArgumentError] when Avro cannot use the name
|
|
72
|
+
def self.check_field(label, key)
|
|
73
|
+
return if VALID.match?(key)
|
|
74
|
+
|
|
75
|
+
raise ::ArgumentError, "FieldStruct::Avro: field #{label} is named #{key.inspect}, " \
|
|
76
|
+
'which Avro names can\'t be ([A-Za-z_][A-Za-z0-9_]*)'
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FieldStruct
|
|
4
|
+
module Avro
|
|
5
|
+
# The one interpretation of a field's +avro:+ option, shared by the schema builder and the datum converters
|
|
6
|
+
# so a schema never promises what the datum does not write (AV-I2).
|
|
7
|
+
module Options
|
|
8
|
+
# Nanoseconds in one unit of each timestamp precision.
|
|
9
|
+
# @return [::Hash{::Symbol=>::Integer}]
|
|
10
|
+
UNITS = { nanos: 1, micros: 1_000, millis: 1_000_000 }.freeze
|
|
11
|
+
|
|
12
|
+
# Default decimal precision.
|
|
13
|
+
# @return [::Integer]
|
|
14
|
+
DEFAULT_PRECISION = 38
|
|
15
|
+
|
|
16
|
+
# The read options per Field object (by identity), so a field's +avro:+ is symbolized once, not per call.
|
|
17
|
+
# Changing a field's options in place is not noticed (AV-B53); redeclaring the field makes a new Field.
|
|
18
|
+
READ = ::ObjectSpace::WeakMap.new
|
|
19
|
+
private_constant :READ
|
|
20
|
+
|
|
21
|
+
# The result for a field with no +avro:+ option.
|
|
22
|
+
NONE = {}.freeze
|
|
23
|
+
private_constant :NONE
|
|
24
|
+
|
|
25
|
+
# The keys an +avro:+ Hash may hold (AV-B16).
|
|
26
|
+
KEYS = %i[type scale precision timestamp enum enum_default].freeze
|
|
27
|
+
private_constant :KEYS
|
|
28
|
+
|
|
29
|
+
module_function
|
|
30
|
+
|
|
31
|
+
# The field's +avro:+ options as a frozen Hash with Symbol keys, read once per Field object (AV-B53).
|
|
32
|
+
#
|
|
33
|
+
# @param field [FieldStruct::Field]
|
|
34
|
+
# @yieldreturn [::String] +Klass#field+, built only for the error
|
|
35
|
+
# @return [::Hash{::Symbol=>::Object}]
|
|
36
|
+
# @raise [::ArgumentError] when +avro:+ is not a Hash
|
|
37
|
+
def read(field)
|
|
38
|
+
value = field.options[:avro]
|
|
39
|
+
return NONE if value.nil?
|
|
40
|
+
|
|
41
|
+
cached = READ[field]
|
|
42
|
+
return cached if cached
|
|
43
|
+
|
|
44
|
+
raise ::ArgumentError, not_a_hash(yield, value) unless value.is_a?(::Hash)
|
|
45
|
+
|
|
46
|
+
READ[field] = checked(value) { block_given? ? yield : field.name.to_s }
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# The options with Symbol keys, frozen, after refusing any key this adapter doesn't read (AV-B16).
|
|
50
|
+
#
|
|
51
|
+
# @param value [::Hash{::Object=>::Object}]
|
|
52
|
+
# @yieldreturn [::String] +Klass#field+, built only for the error
|
|
53
|
+
# @return [::Hash{::Symbol=>::Object}]
|
|
54
|
+
# @raise [::ArgumentError] for an unknown key
|
|
55
|
+
def checked(value)
|
|
56
|
+
opts = value.transform_keys { |key| key.respond_to?(:to_sym) ? key.to_sym : key }.freeze
|
|
57
|
+
unknown = opts.keys.find { |key| !KEYS.include?(key) }
|
|
58
|
+
raise ::ArgumentError, unknown_key(yield, unknown) unless unknown.nil?
|
|
59
|
+
|
|
60
|
+
opts
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# @param label [::String]
|
|
64
|
+
# @param key [::Object] the first key +avro:+ does not take
|
|
65
|
+
# @return [::String]
|
|
66
|
+
def unknown_key(label, key)
|
|
67
|
+
hint = FieldStruct::Suggestion.clause(key, KEYS, sigil: '', suffix: ':')
|
|
68
|
+
"FieldStruct::Avro (#{label}) avro: has no key #{key}:#{hint}; it takes #{KEYS.map { |known| "#{known}:" }.join(', ')}"
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# @param label [::String]
|
|
72
|
+
# @param value [::Object]
|
|
73
|
+
# @return [::String]
|
|
74
|
+
def not_a_hash(label, value)
|
|
75
|
+
"FieldStruct::Avro (#{label}) avro: takes a Hash of options, like avro: {type: 'string'}, got #{value.inspect}"
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# @param opts [::Hash{::Symbol=>::Object}]
|
|
79
|
+
# @return [::Object, nil] the +type:+ override, or nil for none
|
|
80
|
+
def override(opts)
|
|
81
|
+
opts[:type]
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# The options that also apply to array elements and union members (everything but +type:+).
|
|
85
|
+
#
|
|
86
|
+
# @param opts [::Hash{::Symbol=>::Object}]
|
|
87
|
+
# @return [::Hash{::Symbol=>::Object}]
|
|
88
|
+
def nested(opts)
|
|
89
|
+
opts.except(:type)
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# The timestamp precision of a field: its own +avro: {timestamp:}+ or the call's.
|
|
93
|
+
#
|
|
94
|
+
# @param opts [::Hash{::Symbol=>::Object}]
|
|
95
|
+
# @param default [::Symbol] the call's precision
|
|
96
|
+
# @param site [Site, Spot] the field being handled; its label is built only for the error
|
|
97
|
+
# @return [::Symbol]
|
|
98
|
+
# @raise [::ArgumentError] for a precision that is not in +TIMESTAMPS+
|
|
99
|
+
def timestamp(opts, default, site)
|
|
100
|
+
value = opts[:timestamp]
|
|
101
|
+
return default if value.nil?
|
|
102
|
+
return value if TIMESTAMPS.include?(value)
|
|
103
|
+
|
|
104
|
+
raise ::ArgumentError,
|
|
105
|
+
"FieldStruct::Avro (#{site.label}) avro: {timestamp:} must be one of #{TIMESTAMPS.inspect}, got #{value.inspect}"
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# The +[precision, scale]+ of a decimal field, or nil when it is written as a string (AV-B5).
|
|
109
|
+
#
|
|
110
|
+
# @param opts [::Hash{::Symbol=>::Object}] the field's +avro:+ options
|
|
111
|
+
# @param node_options [::Hash{::Symbol=>::Object}] the options the value is checked with
|
|
112
|
+
# @param type [FieldStruct::Types::Base] the decimal type instance
|
|
113
|
+
# @param where [Site, Spot] the field being handled; its label is built only for the error
|
|
114
|
+
# @return [::Array(::Integer, ::Integer), nil]
|
|
115
|
+
# @raise [::ArgumentError] for a precision or scale Avro cannot hold
|
|
116
|
+
def decimal(opts, node_options, type, where)
|
|
117
|
+
scale = decimal_scale(opts, node_options, type)
|
|
118
|
+
return nil if scale.nil?
|
|
119
|
+
|
|
120
|
+
precision = opts[:precision].nil? ? DEFAULT_PRECISION : opts[:precision]
|
|
121
|
+
check_integers(precision, scale, where)
|
|
122
|
+
scale = 0 if scale.negative?
|
|
123
|
+
check_range(precision, scale, where)
|
|
124
|
+
[precision, scale]
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# @param precision [::Object]
|
|
128
|
+
# @param scale [::Object]
|
|
129
|
+
# @param where [Site, Spot]
|
|
130
|
+
# @return [void]
|
|
131
|
+
def check_integers(precision, scale, where)
|
|
132
|
+
return if precision.is_a?(::Integer) && scale.is_a?(::Integer)
|
|
133
|
+
|
|
134
|
+
decimal_error(where, "decimal precision and scale must be integers, got #{precision.inspect} and #{scale.inspect}")
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# @param precision [::Integer]
|
|
138
|
+
# @param scale [::Integer]
|
|
139
|
+
# @param where [Site, Spot]
|
|
140
|
+
# @return [void]
|
|
141
|
+
def check_range(precision, scale, where)
|
|
142
|
+
decimal_error(where, "precision #{precision} must be at least 1") if precision < 1
|
|
143
|
+
decimal_error(where, "scale #{scale} is more than precision #{precision}; raise avro: {precision:}") if scale > precision
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# @param opts [::Hash{::Symbol=>::Object}]
|
|
147
|
+
# @param node_options [::Hash{::Symbol=>::Object}]
|
|
148
|
+
# @param type [FieldStruct::Types::Base]
|
|
149
|
+
# @return [::Object, nil]
|
|
150
|
+
def decimal_scale(opts, node_options, type)
|
|
151
|
+
return opts[:scale] unless opts[:scale].nil?
|
|
152
|
+
return node_options[:round] unless node_options[:round].nil?
|
|
153
|
+
|
|
154
|
+
type.class.respond_to?(:default_round) ? type.class.default_round : nil
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# @param where [Site, Spot]
|
|
158
|
+
# @param message [::String]
|
|
159
|
+
# @return [void]
|
|
160
|
+
def decimal_error(where, message)
|
|
161
|
+
raise ::ArgumentError, "FieldStruct::Avro can't describe #{where.label}: #{message}"
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
# Symbols become Strings, deeply, in Hashes and Arrays (an +avro: {type:}+ override as written to the schema).
|
|
165
|
+
#
|
|
166
|
+
# @param value [::Object]
|
|
167
|
+
# @return [::Object]
|
|
168
|
+
def stringify(value)
|
|
169
|
+
case value
|
|
170
|
+
when ::Symbol then value.to_s
|
|
171
|
+
when ::Hash then value.to_h { |key, item| [stringify(key), stringify(item)] }
|
|
172
|
+
when ::Array then value.map { |item| stringify(item) }
|
|
173
|
+
else value
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
# Whether an (already stringified) override is a plain Avro string, or a union of null and string.
|
|
178
|
+
#
|
|
179
|
+
# @param type [::Object]
|
|
180
|
+
# @return [::Boolean]
|
|
181
|
+
def string_type?(type)
|
|
182
|
+
case type
|
|
183
|
+
when ::String then type == 'string'
|
|
184
|
+
when ::Hash then type['type'] == 'string' && !type.key?('logicalType')
|
|
185
|
+
when ::Array then string_union?(type)
|
|
186
|
+
else false
|
|
187
|
+
end
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
# @param types [::Array<::Object>]
|
|
191
|
+
# @return [::Boolean] whether the union is only null and string types, with at least one string
|
|
192
|
+
def string_union?(types)
|
|
193
|
+
types.any? { |item| string_type?(item) } && types.all? { |item| item == 'null' || string_type?(item) }
|
|
194
|
+
end
|
|
195
|
+
end
|
|
196
|
+
end
|
|
197
|
+
end
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FieldStruct
|
|
4
|
+
module Avro
|
|
5
|
+
# A copy of a schema Hash without the fields the writer lacked, so the avro gem needs no default for them and
|
|
6
|
+
# the datum has no key for them: the loader then lets +new+ apply the field_struct default (AV-B40).
|
|
7
|
+
class Projection
|
|
8
|
+
# @param schema [::Hash] a schema Hash (never mutated)
|
|
9
|
+
# @param lacking [::Hash{::String=>::Array<::String>}] record full name to the field names to leave out
|
|
10
|
+
# @return [::Hash] the projected copy
|
|
11
|
+
def self.call(schema, lacking)
|
|
12
|
+
new(lacking).fragment(schema, nil)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# @param lacking [::Hash{::String=>::Array<::String>}]
|
|
16
|
+
def initialize(lacking)
|
|
17
|
+
@lacking = lacking
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# @param value [::Object] a schema fragment
|
|
21
|
+
# @param namespace [::String, nil] the enclosing namespace
|
|
22
|
+
# @return [::Object] the fragment, projected
|
|
23
|
+
def fragment(value, namespace)
|
|
24
|
+
case value
|
|
25
|
+
when ::Array then value.map { |item| fragment(item, namespace) }
|
|
26
|
+
when ::Hash then hash(value, namespace)
|
|
27
|
+
else value
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
def hash(node, namespace)
|
|
34
|
+
case node['type']
|
|
35
|
+
when 'record' then record(node, namespace)
|
|
36
|
+
when 'array' then node.merge('items' => fragment(node['items'], namespace))
|
|
37
|
+
when 'map' then node.merge('values' => fragment(node['values'], namespace))
|
|
38
|
+
when ::Hash, ::Array then node.merge('type' => fragment(node['type'], namespace))
|
|
39
|
+
else node
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def record(node, namespace)
|
|
44
|
+
full, inner = names(node, namespace)
|
|
45
|
+
left_out = @lacking.fetch(full, [])
|
|
46
|
+
fields = node['fields'].reject { |field| left_out.include?(field['name']) }
|
|
47
|
+
node.merge('fields' => fields.map { |field| field.merge('type' => fragment(field['type'], inner)) })
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def names(node, namespace)
|
|
51
|
+
name = node['name']
|
|
52
|
+
return [name, name.rpartition('.').first] if name.include?('.')
|
|
53
|
+
|
|
54
|
+
space = node.key?('namespace') ? node['namespace'] : namespace
|
|
55
|
+
[space ? "#{space}.#{name}" : name, space]
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|