field_struct-avro 1.0.0.pre1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +33 -0
- data/LICENSE.txt +21 -0
- data/README.md +226 -0
- data/lib/field_struct/avro/bytes_reader.rb +37 -0
- data/lib/field_struct/avro/cache.rb +117 -0
- data/lib/field_struct/avro/check.rb +72 -0
- data/lib/field_struct/avro/datum_reader.rb +160 -0
- data/lib/field_struct/avro/datum_writer.rb +152 -0
- data/lib/field_struct/avro/defaults.rb +68 -0
- data/lib/field_struct/avro/entry.rb +48 -0
- data/lib/field_struct/avro/evolution.rb +52 -0
- data/lib/field_struct/avro/methods.rb +36 -0
- data/lib/field_struct/avro/names.rb +80 -0
- data/lib/field_struct/avro/options.rb +197 -0
- data/lib/field_struct/avro/projection.rb +59 -0
- data/lib/field_struct/avro/resolution.rb +110 -0
- data/lib/field_struct/avro/scalar_types.rb +83 -0
- data/lib/field_struct/avro/schema_builder.rb +203 -0
- data/lib/field_struct/avro/site.rb +40 -0
- data/lib/field_struct/avro/stamp.rb +48 -0
- data/lib/field_struct/avro/trail.rb +56 -0
- data/lib/field_struct/avro/union_rules.rb +99 -0
- data/lib/field_struct/avro/value_json.rb +47 -0
- data/lib/field_struct/avro/version.rb +10 -0
- data/lib/field_struct/avro/writer_schema.rb +33 -0
- data/lib/field_struct/avro/zone.rb +23 -0
- data/lib/field_struct/avro.rb +138 -0
- data/sig/field_struct-avro.rbs +97 -0
- metadata +149 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 29e0995e1357c2fa74d60ed253e2c83bd236e74175bfb4700a496226306a1395
|
|
4
|
+
data.tar.gz: 8924d3f56b43ac1f44821e79540f4399312a8d77cfcf033edac09a6505892666
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 2aec86e8f7c3548ad753c040940fd40be2cf4f08a732b46a12234c54d64e22d4708358c1b5e1814dd27f639844c2d5e2c26d2979e9f8aaaf9f88c4b07f8d303e
|
|
7
|
+
data.tar.gz: 3e1c85d3e015bfc54a8d2b62fb2abb3585c651ee1043e6f53fbf7c3784e3ce65115b72f61c4f78e5bf32594e2324435917d8ace38eebe729d74f251a398e46d9
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## [Unreleased]
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
|
|
7
|
+
- Requiring the adapter registers `avro:` with `FieldStruct.register_option`, so field_struct accepts the key and
|
|
8
|
+
refuses a non-Hash value when the field is declared.
|
|
9
|
+
- A required field declared `nullable: true` is the union `["null", T]` with no default, so a nil encodes and a reader
|
|
10
|
+
still needs the field; with a default it is `[T, "null"]` plus that default.
|
|
11
|
+
- `:hash, of:` is an Avro `map` with nullable values; values are written and read like array items, and struct values
|
|
12
|
+
become records.
|
|
13
|
+
|
|
14
|
+
### Fixed
|
|
15
|
+
|
|
16
|
+
- `avro:` refuses a key the adapter doesn't read (`avro: {scal: 2}` raises with "did you mean scale:?"), where it used
|
|
17
|
+
to be ignored; it takes `type:`, `scale:`, `precision:`, `timestamp:`, `enum:` and `enum_default:`.
|
|
18
|
+
|
|
19
|
+
### Changed
|
|
20
|
+
|
|
21
|
+
- Faster encoding and decoding of plain scalar fields; error messages are unchanged.
|
|
22
|
+
|
|
23
|
+
## 1.0.0.pre1
|
|
24
|
+
|
|
25
|
+
- First release: Apache Avro for field_struct classes.
|
|
26
|
+
- `FieldStruct::Avro.schema`: a record schema from a struct (lossless mapping: `long`, `decimal`, `timestamp-*`, enums, nested
|
|
27
|
+
and recursive records, unions), parsed by the avro gem so a bad `avro:` option fails at boot.
|
|
28
|
+
- `to_datum`, `from_datum`, `encode`, `decode`: datums and the Avro binary encoding of one value, with time precision,
|
|
29
|
+
`zone:` and `:value` handling and errors that never echo a sensitive field.
|
|
30
|
+
- `decode(writer_schema:)`: reads bytes written with an older schema after a resolution check that refuses the changes the
|
|
31
|
+
avro gem would silently misread (decimal scale, timestamp precision, optional to required); absent fields take the
|
|
32
|
+
field_struct default.
|
|
33
|
+
- Caches for schemas, property lists and checked writer schemas, with `clear_cache`; the opt-in `Methods` mixin.
|
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Adrian Madrid
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
# field_struct-avro
|
|
2
|
+
|
|
3
|
+
Apache Avro for [field_struct](../../gems/field_struct) classes. Generate a record schema from a struct, turn
|
|
4
|
+
instances into Avro datums (the Hash the `avro` gem reads and writes) and back, and read and write the Avro binary
|
|
5
|
+
encoding, including bytes written with an older schema.
|
|
6
|
+
|
|
7
|
+
Nothing is lost on the way: integers are `long`, decimals with a scale are Avro `decimal`, times keep their
|
|
8
|
+
sub-second digits and symbols with an `enum:` are Avro enums. Where Avro can't hold a value exactly, you get an error,
|
|
9
|
+
never a rounded value. It does not validate (`format:`, `in:`), and it handles one datum at a time: no container
|
|
10
|
+
files, no schema registry, no Avro JSON.
|
|
11
|
+
|
|
12
|
+
## Install
|
|
13
|
+
|
|
14
|
+
```ruby
|
|
15
|
+
gem 'field_struct-avro'
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
```ruby
|
|
19
|
+
require 'field_struct/avro'
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
## Quick start
|
|
23
|
+
|
|
24
|
+
```ruby run
|
|
25
|
+
require 'field_struct/avro'
|
|
26
|
+
|
|
27
|
+
module Harbor
|
|
28
|
+
class Beacon < FieldStruct::Base
|
|
29
|
+
include FieldStruct::Avro::Methods # opt in to Beacon.to_avro_schema, #to_avro and Beacon.from_avro
|
|
30
|
+
|
|
31
|
+
required :id, :uuid
|
|
32
|
+
required :reach, :big_decimal, round: 2
|
|
33
|
+
required :lit, :time, zone: 'America/Denver'
|
|
34
|
+
required :tower, :symbol, enum: %i[new used]
|
|
35
|
+
optional :codes, :array, of: :string
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
schema = FieldStruct::Avro.schema(Harbor::Beacon) # the same as Harbor::Beacon.to_avro_schema
|
|
40
|
+
schema['name'] # => "Beacon"
|
|
41
|
+
schema['namespace'] # => "Harbor"
|
|
42
|
+
schema['fields'][1]['type'] # => {"type"=>"bytes", "logicalType"=>"decimal", "precision"=>38, "scale"=>2}
|
|
43
|
+
schema['fields'][3]['type'] # => {"type"=>"enum", "name"=>"Beacon_tower", "symbols"=>["new", "used"]}
|
|
44
|
+
schema['fields'][4]['type'] # => ["null", {"type"=>"array", "items"=>["null", "string"]}]
|
|
45
|
+
|
|
46
|
+
beacon = Harbor::Beacon.new(id: '8b1f4d4a-0d0e-4b29-9f4d-6a1b3d3c9f00', reach: '19.90',
|
|
47
|
+
lit: '2026-07-01T08:00:00', tower: :new, codes: %w[fiction])
|
|
48
|
+
bytes = beacon.to_avro # the same as FieldStruct::Avro.encode(beacon)
|
|
49
|
+
Harbor::Beacon.from_avro(bytes) == beacon # => true
|
|
50
|
+
FieldStruct::Avro.decode(Harbor::Beacon, bytes).lit.utc_offset # => -21600
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
`schema` returns a fresh copy every time, so changing the result changes nothing else. `serialization:` picks the
|
|
54
|
+
`serialize` mapping whose names become Avro field names (default `:avro`); `timestamp:` is `:micros` (default),
|
|
55
|
+
`:nanos` or `:millis`.
|
|
56
|
+
|
|
57
|
+
## What the schema says
|
|
58
|
+
|
|
59
|
+
| Field type | Avro |
|
|
60
|
+
|---|---|
|
|
61
|
+
| `:integer` | `long`, never `int` |
|
|
62
|
+
| `:float` | `double` |
|
|
63
|
+
| `:string`, `:email`, `:url`, `:value` | `string` (`:value` holds JSON text) |
|
|
64
|
+
| `:uuid` | `string` with `logicalType` `uuid` |
|
|
65
|
+
| `:binary` | `bytes` |
|
|
66
|
+
| `:big_decimal` | `decimal` with the field's scale (`round:` or `avro: {scale:}`); a `string` when there is none |
|
|
67
|
+
| `:date` | `int` with `logicalType` `date` |
|
|
68
|
+
| `:time`, `:datetime` | `long` with `timestamp-micros`, `-nanos` or `-millis` |
|
|
69
|
+
| `:symbol` with `enum:` | a named Avro `enum`; otherwise `string` |
|
|
70
|
+
| arrays | `array`, items always nullable (field_struct accepts nil elements) |
|
|
71
|
+
| `:hash, of:` | `map` (String keys), values nullable like array items |
|
|
72
|
+
| nested structs | a named `record`, referenced by full name after its first use |
|
|
73
|
+
|
|
74
|
+
An optional field is `["null", T]` with a `null` default; a field with a usable default is not nullable and carries
|
|
75
|
+
that default. Defaults Avro can't write (a date, a decimal, a lambda) are left out of the schema and applied by
|
|
76
|
+
field_struct when the value is read. A required field declared `nullable: true` is `["null", T]` too, with no default.
|
|
77
|
+
|
|
78
|
+
```ruby run
|
|
79
|
+
class Reading < FieldStruct::Base
|
|
80
|
+
required :value, :big_decimal
|
|
81
|
+
required :count, :integer, default: 0
|
|
82
|
+
optional :note, :string, desc: 'free text'
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
fields = FieldStruct::Avro.schema(Reading)['fields']
|
|
86
|
+
fields[0] # => {"name"=>"value", "type"=>"string"}
|
|
87
|
+
fields[1] # => {"name"=>"count", "type"=>"long", "default"=>0}
|
|
88
|
+
fields[2] # => {"name"=>"note", "doc"=>"free text", "type"=>["null", "string"], "default"=>nil}
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## Field options: `avro:`
|
|
92
|
+
|
|
93
|
+
A field's `avro:` option is a Hash. `type:` replaces the whole mapping with an Avro type you write; `timestamp:`,
|
|
94
|
+
`precision:`, `scale:`, `enum: false` and `enum_default:` tune the generated one. On array, map and union fields, everything
|
|
95
|
+
but `type:` also applies to the items (a map's values).
|
|
96
|
+
|
|
97
|
+
Requiring the adapter registers `avro:` with `FieldStruct.register_option`, so require it before the classes that use
|
|
98
|
+
the option. A value that isn't a Hash is refused when the field is declared. An unknown key inside `avro:` raises when
|
|
99
|
+
the schema is built, with a did-you-mean: `avro: has no key timestmp: (did you mean timestamp:?)`.
|
|
100
|
+
|
|
101
|
+
```ruby run
|
|
102
|
+
class Dispatch < FieldStruct::Base
|
|
103
|
+
required :fee, :big_decimal, avro: { scale: 2, precision: 12 }
|
|
104
|
+
required :seen, :time, avro: { timestamp: :millis }
|
|
105
|
+
required :level, :symbol, enum: %i[low high], avro: { enum_default: :low }
|
|
106
|
+
required :ref, :string, avro: { type: { 'type' => 'string', 'x-origin' => 'crm' } }
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
types = FieldStruct::Avro.schema(Dispatch)['fields'].to_h { |f| [f['name'], f['type']] }
|
|
110
|
+
types['fee'] # => {"type"=>"bytes", "logicalType"=>"decimal", "precision"=>12, "scale"=>2}
|
|
111
|
+
types['seen'] # => {"type"=>"long", "logicalType"=>"timestamp-millis"}
|
|
112
|
+
types['level'] # => {"type"=>"enum", "name"=>"Dispatch_level", "symbols"=>["low", "high"], "default"=>"low"}
|
|
113
|
+
types['ref'] # => {"type"=>"string", "x-origin"=>"crm"}
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
## Datums and bytes
|
|
117
|
+
|
|
118
|
+
`to_datum` and `from_datum` stop at the Hash the `avro` gem reads and writes; `encode` and `decode` go to and from
|
|
119
|
+
binary (one datum, no header).
|
|
120
|
+
|
|
121
|
+
```ruby run
|
|
122
|
+
class Pair < FieldStruct::Base
|
|
123
|
+
required :first_name, :string
|
|
124
|
+
required :updated, :time
|
|
125
|
+
serialize :avro, first_name: 'firstName'
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
pair = Pair.new(first_name: 'Ada', updated: Time.utc(2026, 10, 25, 21, 55, 12))
|
|
129
|
+
datum = FieldStruct::Avro.to_datum(pair)
|
|
130
|
+
datum.keys # => ["firstName", "updated"]
|
|
131
|
+
FieldStruct::Avro.from_datum(Pair, datum) == pair # => true
|
|
132
|
+
|
|
133
|
+
late = Pair.new(first_name: 'Ada', updated: Time.utc(2026, 10, 25, 21, 55, 12, Rational(123_456_789, 1000)))
|
|
134
|
+
begin
|
|
135
|
+
FieldStruct::Avro.encode(late)
|
|
136
|
+
rescue ArgumentError => e
|
|
137
|
+
e.message # => "FieldStruct::Avro: Pair#updated is 2026-10-25T21:55:12.123456789Z, which has more precision than timestamp-micros; round it, or use timestamp: :nanos"
|
|
138
|
+
end
|
|
139
|
+
FieldStruct::Avro.decode(Pair, FieldStruct::Avro.encode(late, timestamp: :nanos), timestamp: :nanos) == late # => true
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
Errors are `ArgumentError`s that name the class and field (`Pair#updated`). They never echo the value of a
|
|
143
|
+
`sensitive: true` field. Bytes that can't be read at all raise one `ArgumentError` with the avro gem's error as the
|
|
144
|
+
`cause`. A few errors of the `avro` gem itself pass through as raised, and may quote the value they couldn't write: a
|
|
145
|
+
decimal past its precision, an integer past 64 bits, an `avro: {type:}` override the value doesn't fit.
|
|
146
|
+
|
|
147
|
+
## Reading bytes written with an older schema
|
|
148
|
+
|
|
149
|
+
Pass the schema the bytes were written with as `writer_schema:`: a Hash, a JSON String or a parsed `Avro::Schema`.
|
|
150
|
+
Avro matches record names, so the older schema carries the name of the class that reads it. Fields the writer lacked
|
|
151
|
+
take the field_struct default, even one Avro can't express.
|
|
152
|
+
|
|
153
|
+
```ruby run
|
|
154
|
+
module V1
|
|
155
|
+
class Account < FieldStruct::Base
|
|
156
|
+
required :id, :integer
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
module V2
|
|
161
|
+
class Account < FieldStruct::Base
|
|
162
|
+
required :id, :integer
|
|
163
|
+
optional :nickname, :string
|
|
164
|
+
required :opened, :date, default: Date.new(2026, 1, 1)
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
bytes = FieldStruct::Avro.encode(V1::Account.new(id: 7))
|
|
169
|
+
old = FieldStruct::Avro.schema(V1::Account).merge('namespace' => 'V2')
|
|
170
|
+
account = FieldStruct::Avro.decode(V2::Account, bytes, writer_schema: old)
|
|
171
|
+
[account.id, account.nickname, account.opened] # => [7, nil, Date.new(2026, 1, 1)]
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
The `avro` gem pairs fields by their base type only, so a decimal read at another scale, or a timestamp at another
|
|
175
|
+
precision, would decode to a wrong value without a word. Before reading, the pair of schemas is checked and the decode
|
|
176
|
+
is refused with every problem listed:
|
|
177
|
+
|
|
178
|
+
```ruby run
|
|
179
|
+
module Before
|
|
180
|
+
class Lamp < FieldStruct::Base
|
|
181
|
+
required :reach, :big_decimal, round: 2
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
module After
|
|
186
|
+
class Lamp < FieldStruct::Base
|
|
187
|
+
required :reach, :big_decimal, round: 3
|
|
188
|
+
end
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
bytes = FieldStruct::Avro.encode(Before::Lamp.new(reach: '1.25'))
|
|
192
|
+
old = FieldStruct::Avro.schema(Before::Lamp).merge('namespace' => 'After')
|
|
193
|
+
begin
|
|
194
|
+
FieldStruct::Avro.decode(After::Lamp, bytes, writer_schema: old)
|
|
195
|
+
rescue ArgumentError => e
|
|
196
|
+
e.message # => "FieldStruct::Avro can't read these bytes: reach: written as decimal (scale 2), read as decimal (scale 3). The avro gem would decode them as the wrong value, or fail; this is not a compatible schema change"
|
|
197
|
+
end
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
A symbol the reader's enum lacks raises, naming the field, unless the field says what to read instead with
|
|
201
|
+
`avro: {enum_default:}`.
|
|
202
|
+
|
|
203
|
+
## Caching
|
|
204
|
+
|
|
205
|
+
Schemas, parsed schemas, field lists and checked writer schemas are cached per class, so `encode` and `decode` don't
|
|
206
|
+
rebuild anything. Adding or redeclaring a field, declaring a `serialize` later or naming an anonymous struct is noticed
|
|
207
|
+
on the next call. Changing a field's options in place (the `avro:` Hash, the `enum:` Array) is not: redeclare the
|
|
208
|
+
field. The cache keeps classes alive, so call `FieldStruct::Avro.clear_cache` when code reloads.
|
|
209
|
+
|
|
210
|
+
## Reference
|
|
211
|
+
|
|
212
|
+
| Call | Returns |
|
|
213
|
+
|---|---|
|
|
214
|
+
| `FieldStruct::Avro.schema(klass, serialization:, timestamp:)` | the record schema, a Hash with String keys |
|
|
215
|
+
| `FieldStruct::Avro.to_datum(struct, serialization:, timestamp:)` | the datum Hash |
|
|
216
|
+
| `FieldStruct::Avro.from_datum(klass, datum, serialization:)` | a struct |
|
|
217
|
+
| `FieldStruct::Avro.encode(struct, serialization:, timestamp:)` | Avro binary |
|
|
218
|
+
| `FieldStruct::Avro.decode(klass, bytes, writer_schema:, serialization:, timestamp:)` | a struct |
|
|
219
|
+
| `FieldStruct::Avro.clear_cache` | `nil` |
|
|
220
|
+
| `FieldStruct::Avro::Methods` | opt-in `to_avro_schema`, `#to_avro`, `from_avro` |
|
|
221
|
+
|
|
222
|
+
Needs Ruby 3.3 or newer. MIT licensed.
|
|
223
|
+
|
|
224
|
+
## Docs
|
|
225
|
+
|
|
226
|
+
The guide on the docs site, with every example run in CI: https://gliders-rb.github.io/adapters/avro/
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FieldStruct
|
|
4
|
+
module Avro
|
|
5
|
+
# Reads one datum from Avro binary with the avro gem, turning every way the gem fails on bad bytes into one
|
|
6
|
+
# error (AV-B37). The conversion into a struct is not covered: its errors are the adapter's own (AV-B45).
|
|
7
|
+
module BytesReader
|
|
8
|
+
# The hint when no writer schema was given.
|
|
9
|
+
OWN_HINT = 'they end early, or were written with another schema (pass writer_schema: for bytes from an older one)'
|
|
10
|
+
|
|
11
|
+
# The hint when one was given.
|
|
12
|
+
WRITER_HINT = "they end early, or the writer schema given doesn't resolve against %<klass>s's (the cause says why)"
|
|
13
|
+
|
|
14
|
+
module_function
|
|
15
|
+
|
|
16
|
+
# @param klass [::Class] the struct class being decoded, for the message
|
|
17
|
+
# @param bytes [::String]
|
|
18
|
+
# @param writer [::Avro::Schema] the schema the bytes were written with
|
|
19
|
+
# @param reader [::Avro::Schema] the class's schema
|
|
20
|
+
# @param writer_given [::Boolean] whether the caller passed +writer_schema:+
|
|
21
|
+
# @return [::Object] the datum the avro gem read
|
|
22
|
+
# @raise [::ArgumentError] when the avro gem cannot read the bytes (the original error is the +cause+)
|
|
23
|
+
def read(klass, bytes, writer, reader, writer_given:)
|
|
24
|
+
decoder = ::Avro::IO::BinaryDecoder.new(StringIO.new(bytes))
|
|
25
|
+
::Avro::IO::DatumReader.new(writer, reader).read(decoder)
|
|
26
|
+
rescue ::StandardError => e
|
|
27
|
+
raise ::ArgumentError, refusal(klass, e, writer_given)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# The message for a failed read; the hint depends on whether the caller named a writer schema.
|
|
31
|
+
def refusal(klass, error, writer_given)
|
|
32
|
+
advice = writer_given ? format(WRITER_HINT, klass: klass) : OWN_HINT
|
|
33
|
+
"FieldStruct::Avro can't decode these bytes as #{klass} (#{error.class.name}): #{advice}"
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'monitor'
|
|
4
|
+
|
|
5
|
+
module FieldStruct
|
|
6
|
+
module Avro
|
|
7
|
+
# The process-wide cache (AV-B50..AV-B57): schema entries per (class, serialization, timestamp), property lists
|
|
8
|
+
# per (class, serialization) and checked writer schemas. One monitor guards every write.
|
|
9
|
+
module Cache
|
|
10
|
+
# The most checked writer schemas kept; the oldest inserted goes first (AV-B55).
|
|
11
|
+
WRITERS_MAX = 512
|
|
12
|
+
|
|
13
|
+
# A property list and the stamp of the class it was read from.
|
|
14
|
+
Props = Struct.new(:list, :stamp)
|
|
15
|
+
|
|
16
|
+
@lock = Monitor.new
|
|
17
|
+
@readers = {}
|
|
18
|
+
@props = {}
|
|
19
|
+
@writers = {}
|
|
20
|
+
|
|
21
|
+
class << self
|
|
22
|
+
# @param klass [::Class]
|
|
23
|
+
# @param serialization [::Symbol]
|
|
24
|
+
# @param timestamp [::Symbol]
|
|
25
|
+
# @return [Entry] the class's schema entry, rebuilt when a class it walked changed
|
|
26
|
+
def reader(klass, serialization, timestamp)
|
|
27
|
+
key = [klass, serialization, timestamp]
|
|
28
|
+
entry = @readers[key]
|
|
29
|
+
return entry if entry&.fresh?
|
|
30
|
+
|
|
31
|
+
@lock.synchronize do
|
|
32
|
+
entry = @readers[key]
|
|
33
|
+
return entry if entry&.fresh?
|
|
34
|
+
|
|
35
|
+
replace_reader(key, SchemaBuilder.entry(klass, serialization, timestamp))
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# @param klass [::Class]
|
|
40
|
+
# @param serialization [::Symbol]
|
|
41
|
+
# @return [::Array<FieldStruct::Introspection::Property>] the frozen property list
|
|
42
|
+
# @raise [::ArgumentError] when a field's +avro:+ option is not a Hash
|
|
43
|
+
def properties(klass, serialization)
|
|
44
|
+
cached = fresh_props(klass, serialization)
|
|
45
|
+
return cached.list if cached
|
|
46
|
+
|
|
47
|
+
@lock.synchronize do
|
|
48
|
+
cached = fresh_props(klass, serialization)
|
|
49
|
+
return cached.list if cached
|
|
50
|
+
|
|
51
|
+
built = build_properties(klass, serialization)
|
|
52
|
+
(@props[klass] ||= {})[serialization] = built
|
|
53
|
+
built.list
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# @param writer_schema [::Hash, ::String, ::Avro::Schema] as given to +decode+
|
|
58
|
+
# @param entry [Entry] the reader entry
|
|
59
|
+
# @return [Array(::Avro::Schema, ::Avro::Schema)] the schemas to read with (see +Evolution+)
|
|
60
|
+
def writer(writer_schema, entry)
|
|
61
|
+
text = WriterSchema.text(writer_schema)
|
|
62
|
+
key = [text.dup.freeze, entry]
|
|
63
|
+
found = @writers[key]
|
|
64
|
+
return found if found
|
|
65
|
+
|
|
66
|
+
@lock.synchronize do
|
|
67
|
+
@writers[key] ||= Evolution.resolve(text, entry)
|
|
68
|
+
@writers.shift while @writers.size > Cache::WRITERS_MAX
|
|
69
|
+
@writers.fetch(key) { Evolution.resolve(text, entry) }
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# @return [::Integer] how many checked writer schemas are held
|
|
74
|
+
def writers_count
|
|
75
|
+
@writers.size
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# @return [nil]
|
|
79
|
+
def clear
|
|
80
|
+
@lock.synchronize do
|
|
81
|
+
@readers.clear
|
|
82
|
+
@props.clear
|
|
83
|
+
@writers.clear
|
|
84
|
+
end
|
|
85
|
+
nil
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
private
|
|
89
|
+
|
|
90
|
+
# Keyed by class, then serialization, so a lookup builds no key Array.
|
|
91
|
+
def fresh_props(klass, serialization)
|
|
92
|
+
cached = @props[klass]&.[](serialization)
|
|
93
|
+
cached if cached&.stamp&.fresh?
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# @param key [::Array]
|
|
97
|
+
# @param entry [Entry]
|
|
98
|
+
# @return [Entry]
|
|
99
|
+
def replace_reader(key, entry)
|
|
100
|
+
old = @readers[key]
|
|
101
|
+
@writers.delete_if { |(_, reader), _| reader.equal?(old) } if old
|
|
102
|
+
@readers[key] = entry
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# @param klass [::Class]
|
|
106
|
+
# @param serialization [::Symbol]
|
|
107
|
+
# @return [Props]
|
|
108
|
+
def build_properties(klass, serialization)
|
|
109
|
+
stamp = Stamp.of(klass, serialization)
|
|
110
|
+
list = FieldStruct::Introspection.properties(klass, serialization: serialization)
|
|
111
|
+
list.each { |prop| Options.read(prop.field) { "#{klass.name || klass.inspect}##{prop.field.name}" } }
|
|
112
|
+
Props.new(list.freeze, stamp)
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FieldStruct
|
|
4
|
+
module Avro
|
|
5
|
+
# Argument checking shared by every public function (AV-B1, AV-B2).
|
|
6
|
+
module Check
|
|
7
|
+
module_function
|
|
8
|
+
|
|
9
|
+
# Validates in the documented order and returns the serialization as a Symbol.
|
|
10
|
+
#
|
|
11
|
+
# @param klass [::Object] the class to check
|
|
12
|
+
# @param timestamp [::Symbol, nil] nil skips the check (+from_datum+ has none)
|
|
13
|
+
# @param serialization [::Symbol, ::String]
|
|
14
|
+
# @param bytes [::Object] checked for +decode+ only (pass +NOT_GIVEN+ to skip)
|
|
15
|
+
# @return [::Symbol] the serialization
|
|
16
|
+
# @raise [::ArgumentError] for the first bad argument
|
|
17
|
+
def call(klass, serialization:, timestamp: nil, bytes: NOT_GIVEN)
|
|
18
|
+
struct_class!(klass)
|
|
19
|
+
timestamp!(timestamp) unless timestamp.nil?
|
|
20
|
+
serialization = serialization!(serialization)
|
|
21
|
+
bytes!(bytes) unless bytes.equal?(NOT_GIVEN)
|
|
22
|
+
serialization
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Marks "no bytes argument" (nil is not a valid +bytes+ either).
|
|
26
|
+
NOT_GIVEN = ::Object.new.freeze
|
|
27
|
+
|
|
28
|
+
# @param klass [::Object]
|
|
29
|
+
# @return [void]
|
|
30
|
+
def struct_class!(klass)
|
|
31
|
+
return if klass.is_a?(::Class) && klass < FieldStruct::Base
|
|
32
|
+
|
|
33
|
+
raise ::ArgumentError, "FieldStruct::Avro expects a FieldStruct::Base subclass, got #{klass.inspect}"
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# @param timestamp [::Object]
|
|
37
|
+
# @return [void]
|
|
38
|
+
def timestamp!(timestamp)
|
|
39
|
+
return if TIMESTAMPS.include?(timestamp)
|
|
40
|
+
|
|
41
|
+
raise ::ArgumentError, "FieldStruct::Avro timestamp: must be one of #{TIMESTAMPS.inspect}, got #{timestamp.inspect}"
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# @param serialization [::Object]
|
|
45
|
+
# @return [::Symbol]
|
|
46
|
+
def serialization!(serialization)
|
|
47
|
+
serialization = serialization.to_sym if serialization.is_a?(::String)
|
|
48
|
+
return serialization if serialization.is_a?(::Symbol)
|
|
49
|
+
|
|
50
|
+
raise ::ArgumentError, "FieldStruct::Avro expects serialization: to be a Symbol, got #{serialization.inspect}"
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# @param bytes [::Object]
|
|
54
|
+
# @return [void]
|
|
55
|
+
def bytes!(bytes)
|
|
56
|
+
return if bytes.is_a?(::String)
|
|
57
|
+
|
|
58
|
+
raise ::ArgumentError, "FieldStruct::Avro.decode expects the bytes as a String, got #{bytes.class}"
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# The root of a schema must be assigned to a constant (AV-B2).
|
|
62
|
+
#
|
|
63
|
+
# @param klass [::Class]
|
|
64
|
+
# @return [void]
|
|
65
|
+
def named!(klass)
|
|
66
|
+
return if FieldStruct::Introspection.named?(klass)
|
|
67
|
+
|
|
68
|
+
raise ::ArgumentError, 'FieldStruct::Avro needs a named class for the root record; assign it to a constant'
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FieldStruct
|
|
4
|
+
module Avro
|
|
5
|
+
# Builds a struct from the Hash the avro gem read (AV-B25..AV-B29).
|
|
6
|
+
class DatumReader
|
|
7
|
+
# The node classes of the introspection walk.
|
|
8
|
+
NODES = FieldStruct::Introspection
|
|
9
|
+
|
|
10
|
+
# @param serialization [::Symbol]
|
|
11
|
+
def initialize(serialization)
|
|
12
|
+
@serialization = serialization
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# @param klass [::Class]
|
|
16
|
+
# @param datum [::Hash{::String=>::Object}]
|
|
17
|
+
# @param trail [Trail]
|
|
18
|
+
# @return [FieldStruct::Base]
|
|
19
|
+
def struct(klass, datum, trail)
|
|
20
|
+
attributes = {}
|
|
21
|
+
Cache.properties(klass, @serialization).each do |prop|
|
|
22
|
+
attributes[prop.field.name] = property(datum[prop.key], prop, trail) if datum.key?(prop.key)
|
|
23
|
+
end
|
|
24
|
+
klass.new(attributes)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
private
|
|
28
|
+
|
|
29
|
+
def property(value, prop, trail)
|
|
30
|
+
field = prop.field
|
|
31
|
+
opts = Options.read(field) { trail.label(field) }
|
|
32
|
+
override = Options.override(opts)
|
|
33
|
+
return nil if value.nil?
|
|
34
|
+
return overridden(value, prop.node, Options.stringify(override), Spot.new(trail, field, opts)) unless override.nil?
|
|
35
|
+
|
|
36
|
+
node = prop.node
|
|
37
|
+
# Most fields are plain scalars, which never report an error, so they skip building a Spot.
|
|
38
|
+
return plain_scalar(value, node) if plain_scalar?(node)
|
|
39
|
+
|
|
40
|
+
node_value(value, node, Spot.new(trail, field, opts))
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def plain_scalar?(node)
|
|
44
|
+
node.is_a?(NODES::ScalarNode) && !node.type.is_a?(Types::Symbol) && !node.type.is_a?(Types::Value)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def plain_scalar(value, node)
|
|
48
|
+
case node.type
|
|
49
|
+
when Types::Time, Types::DateTime then Zone.apply(value, node.options[:zone])
|
|
50
|
+
else value
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def node_value(value, node, spot)
|
|
55
|
+
return nil if value.nil?
|
|
56
|
+
|
|
57
|
+
case node
|
|
58
|
+
when NODES::ScalarNode then scalar(value, node, spot)
|
|
59
|
+
when NODES::ArrayNode then array(value, node, spot)
|
|
60
|
+
when NODES::MapNode then entries(value, node, spot)
|
|
61
|
+
when NODES::StructNode then nested(value, node.klass, spot)
|
|
62
|
+
else union(value, node, spot)
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def array(value, node, spot)
|
|
67
|
+
element = node.element
|
|
68
|
+
return value if element.nil? || !value.is_a?(::Array)
|
|
69
|
+
|
|
70
|
+
value.map { |item| node_value(item, element, spot) }
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def entries(value, node, spot)
|
|
74
|
+
element = node.element
|
|
75
|
+
return value if element.nil? || !value.is_a?(::Hash)
|
|
76
|
+
|
|
77
|
+
value.transform_values { |item| node_value(item, element, spot) }
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def nested(value, klass, spot)
|
|
81
|
+
value.is_a?(::Hash) ? struct(klass, value, spot.enter(klass)) : value
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def scalar(value, node, spot)
|
|
85
|
+
case node.type
|
|
86
|
+
when Types::Symbol then symbol(value, node, spot)
|
|
87
|
+
when Types::Value then value.is_a?(::String) ? ValueJson.load(value, spot) : value
|
|
88
|
+
else plain_scalar(value, node)
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def symbol(value, node, spot)
|
|
93
|
+
symbols = node.options[:enum]
|
|
94
|
+
return value unless symbols.is_a?(::Array) && (value.is_a?(::String) || value.is_a?(::Symbol))
|
|
95
|
+
return value if symbols.any? { |symbol| symbol.to_s == value.to_s }
|
|
96
|
+
return spot.opts[:enum_default].to_s if spot.opts.key?(:enum_default)
|
|
97
|
+
|
|
98
|
+
unknown_symbol(value, spot)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def unknown_symbol(value, spot)
|
|
102
|
+
shown = spot.sensitive? ? 'a sensitive symbol' : value.to_sym.inspect
|
|
103
|
+
raise ::ArgumentError, "FieldStruct::Avro: #{spot.label} is #{shown}, which isn't one of the field's enum symbols; " \
|
|
104
|
+
'give the reader avro: {enum_default:} to read new ones'
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def union(value, node, spot)
|
|
108
|
+
case value
|
|
109
|
+
when ::Hash then union_struct(value, node, spot)
|
|
110
|
+
when ::String then union_string(value, node, spot)
|
|
111
|
+
else value
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def union_struct(value, node, spot)
|
|
116
|
+
keys = value.keys.sort
|
|
117
|
+
member = node.members.grep(NODES::StructNode).find do |candidate|
|
|
118
|
+
Cache.properties(candidate.klass, @serialization).map(&:key).sort == keys
|
|
119
|
+
end
|
|
120
|
+
member ? nested(value, member.klass, spot) : value
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def union_string(value, node, spot)
|
|
124
|
+
value_member = node.members.any? { |member| member.is_a?(NODES::ScalarNode) && member.type.is_a?(Types::Value) }
|
|
125
|
+
value_member ? ValueJson.load(value, spot) : value
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def overridden(value, node, type, spot)
|
|
129
|
+
case node
|
|
130
|
+
when NODES::StructNode then nested(value, node.klass, spot)
|
|
131
|
+
when NODES::ArrayNode then overridden_array(value, node, spot)
|
|
132
|
+
when NODES::MapNode then overridden_entries(value, node, spot)
|
|
133
|
+
when NODES::ScalarNode then overridden_scalar(value, node, type, spot)
|
|
134
|
+
else value
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def overridden_array(value, node, spot)
|
|
139
|
+
element = node.element
|
|
140
|
+
return value unless value.is_a?(::Array) && element.is_a?(NODES::StructNode)
|
|
141
|
+
|
|
142
|
+
value.map { |item| nested(item, element.klass, spot) }
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def overridden_entries(value, node, spot)
|
|
146
|
+
element = node.element
|
|
147
|
+
return value unless value.is_a?(::Hash) && element.is_a?(NODES::StructNode)
|
|
148
|
+
|
|
149
|
+
value.transform_values { |item| nested(item, element.klass, spot) }
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def overridden_scalar(value, node, type, spot)
|
|
153
|
+
return Zone.apply(value, node.options[:zone]) if value.is_a?(::Time)
|
|
154
|
+
return ValueJson.load(value, spot) if node.type.is_a?(Types::Value) && value.is_a?(::String) && Options.string_type?(type)
|
|
155
|
+
|
|
156
|
+
value
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
end
|