field_struct-avro-native 1.0.0.pre1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +5 -0
- data/Cargo.lock +286 -0
- data/Cargo.toml +7 -0
- data/LICENSE.txt +21 -0
- data/README.md +122 -0
- data/ext/field_struct_avro_kernel/Cargo.toml +9 -0
- data/ext/field_struct_avro_kernel/src/decimal.rs +150 -0
- data/ext/field_struct_avro_kernel/src/lib.rs +48 -0
- data/ext/field_struct_avro_native/Cargo.toml +13 -0
- data/ext/field_struct_avro_native/extconf.rb +6 -0
- data/ext/field_struct_avro_native/src/lib.rs +47 -0
- data/ext/field_struct_avro_native/src/plan.rs +157 -0
- data/ext/field_struct_avro_native/src/write.rs +435 -0
- data/lib/field_struct/avro/native/compiler.rb +124 -0
- data/lib/field_struct/avro/native/version.rb +12 -0
- data/lib/field_struct/avro/native.rb +76 -0
- data/sig/field_struct-avro-native.rbs +61 -0
- metadata +98 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 9eab91e47b544a96c8aa8ff7bbb6c7a55a49cd4d46776567cf1a616f41327dc3
|
|
4
|
+
data.tar.gz: 89d8b167da605fb1413fbe5aae1724cba90a0bcb113d4dbeff875447453cd66a
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: a3739510cd7b3e5ea234726ef99f8b2fc8bc8d0a1d17f92da2633b3e3738b9bff13aee23575fae82f022f8bc5059cf06353168c1209b0da532ca8864923802c0
|
|
7
|
+
data.tar.gz: 91ab669af9c9848537307c50ed298c356acd268db31edf01ba859085da2a1a73e5ed7673d726ad9a846b3b6f290f8f799c75708d34fc7cbcec85b2a604a46cd7
|
data/CHANGELOG.md
ADDED
data/Cargo.lock
ADDED
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
# This file is automatically @generated by Cargo.
|
|
2
|
+
# It is not intended for manual editing.
|
|
3
|
+
version = 4
|
|
4
|
+
|
|
5
|
+
[[package]]
|
|
6
|
+
name = "aho-corasick"
|
|
7
|
+
version = "1.1.5"
|
|
8
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
9
|
+
checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
|
|
10
|
+
dependencies = [
|
|
11
|
+
"memchr",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
[[package]]
|
|
15
|
+
name = "bindgen"
|
|
16
|
+
version = "0.72.1"
|
|
17
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
18
|
+
checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
|
|
19
|
+
dependencies = [
|
|
20
|
+
"bitflags",
|
|
21
|
+
"cexpr",
|
|
22
|
+
"clang-sys",
|
|
23
|
+
"itertools",
|
|
24
|
+
"proc-macro2",
|
|
25
|
+
"quote",
|
|
26
|
+
"regex",
|
|
27
|
+
"rustc-hash",
|
|
28
|
+
"shlex",
|
|
29
|
+
"syn",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
[[package]]
|
|
33
|
+
name = "bitflags"
|
|
34
|
+
version = "2.13.2"
|
|
35
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
36
|
+
checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
|
|
37
|
+
|
|
38
|
+
[[package]]
|
|
39
|
+
name = "cexpr"
|
|
40
|
+
version = "0.6.0"
|
|
41
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
42
|
+
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
|
|
43
|
+
dependencies = [
|
|
44
|
+
"nom",
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
[[package]]
|
|
48
|
+
name = "cfg-if"
|
|
49
|
+
version = "1.0.5"
|
|
50
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
51
|
+
checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
|
|
52
|
+
|
|
53
|
+
[[package]]
|
|
54
|
+
name = "clang-sys"
|
|
55
|
+
version = "1.9.1"
|
|
56
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
57
|
+
checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a"
|
|
58
|
+
dependencies = [
|
|
59
|
+
"glob",
|
|
60
|
+
"libc",
|
|
61
|
+
"libloading",
|
|
62
|
+
]
|
|
63
|
+
|
|
64
|
+
[[package]]
|
|
65
|
+
name = "either"
|
|
66
|
+
version = "1.19.0"
|
|
67
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
68
|
+
checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be"
|
|
69
|
+
|
|
70
|
+
[[package]]
|
|
71
|
+
name = "field_struct_avro_kernel"
|
|
72
|
+
version = "1.0.0"
|
|
73
|
+
|
|
74
|
+
[[package]]
|
|
75
|
+
name = "field_struct_avro_native"
|
|
76
|
+
version = "1.0.0"
|
|
77
|
+
dependencies = [
|
|
78
|
+
"field_struct_avro_kernel",
|
|
79
|
+
"magnus",
|
|
80
|
+
]
|
|
81
|
+
|
|
82
|
+
[[package]]
|
|
83
|
+
name = "glob"
|
|
84
|
+
version = "0.3.4"
|
|
85
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
86
|
+
checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b"
|
|
87
|
+
|
|
88
|
+
[[package]]
|
|
89
|
+
name = "itertools"
|
|
90
|
+
version = "0.13.0"
|
|
91
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
92
|
+
checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
|
|
93
|
+
dependencies = [
|
|
94
|
+
"either",
|
|
95
|
+
]
|
|
96
|
+
|
|
97
|
+
[[package]]
|
|
98
|
+
name = "lazy_static"
|
|
99
|
+
version = "1.5.1"
|
|
100
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
101
|
+
checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb"
|
|
102
|
+
|
|
103
|
+
[[package]]
|
|
104
|
+
name = "libc"
|
|
105
|
+
version = "0.2.191"
|
|
106
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
107
|
+
checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
|
|
108
|
+
|
|
109
|
+
[[package]]
|
|
110
|
+
name = "libloading"
|
|
111
|
+
version = "0.8.9"
|
|
112
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
113
|
+
checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
|
|
114
|
+
dependencies = [
|
|
115
|
+
"cfg-if",
|
|
116
|
+
"windows-link",
|
|
117
|
+
]
|
|
118
|
+
|
|
119
|
+
[[package]]
|
|
120
|
+
name = "magnus"
|
|
121
|
+
version = "0.9.3"
|
|
122
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
123
|
+
checksum = "40ef424404d166cbb71c8b8d19c9c589c748c4547c1afa1928c5504299f6c713"
|
|
124
|
+
dependencies = [
|
|
125
|
+
"magnus-macros",
|
|
126
|
+
"rb-sys",
|
|
127
|
+
"rb-sys-env",
|
|
128
|
+
"seq-macro",
|
|
129
|
+
]
|
|
130
|
+
|
|
131
|
+
[[package]]
|
|
132
|
+
name = "magnus-macros"
|
|
133
|
+
version = "0.9.1"
|
|
134
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
135
|
+
checksum = "9dafde2eeb66f090e0f326112323eb8324b12cb32b5d1210e541d3df6b0ce77e"
|
|
136
|
+
dependencies = [
|
|
137
|
+
"proc-macro2",
|
|
138
|
+
"quote",
|
|
139
|
+
"syn",
|
|
140
|
+
]
|
|
141
|
+
|
|
142
|
+
[[package]]
|
|
143
|
+
name = "memchr"
|
|
144
|
+
version = "2.8.3"
|
|
145
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
146
|
+
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
|
147
|
+
|
|
148
|
+
[[package]]
|
|
149
|
+
name = "minimal-lexical"
|
|
150
|
+
version = "0.2.1"
|
|
151
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
152
|
+
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
|
153
|
+
|
|
154
|
+
[[package]]
|
|
155
|
+
name = "nom"
|
|
156
|
+
version = "7.1.3"
|
|
157
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
158
|
+
checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
|
|
159
|
+
dependencies = [
|
|
160
|
+
"memchr",
|
|
161
|
+
"minimal-lexical",
|
|
162
|
+
]
|
|
163
|
+
|
|
164
|
+
[[package]]
|
|
165
|
+
name = "proc-macro2"
|
|
166
|
+
version = "1.0.107"
|
|
167
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
168
|
+
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
|
169
|
+
dependencies = [
|
|
170
|
+
"unicode-ident",
|
|
171
|
+
]
|
|
172
|
+
|
|
173
|
+
[[package]]
|
|
174
|
+
name = "quote"
|
|
175
|
+
version = "1.0.47"
|
|
176
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
177
|
+
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
|
178
|
+
dependencies = [
|
|
179
|
+
"proc-macro2",
|
|
180
|
+
]
|
|
181
|
+
|
|
182
|
+
[[package]]
|
|
183
|
+
name = "rb-sys"
|
|
184
|
+
version = "0.9.130"
|
|
185
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
186
|
+
checksum = "02faf625bb10ba893e3ae620f19c9fb1b5f8fcae0fe4eb86bb3f2230fad75edb"
|
|
187
|
+
dependencies = [
|
|
188
|
+
"rb-sys-build",
|
|
189
|
+
]
|
|
190
|
+
|
|
191
|
+
[[package]]
|
|
192
|
+
name = "rb-sys-build"
|
|
193
|
+
version = "0.9.130"
|
|
194
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
195
|
+
checksum = "05be6f9c86fe5482808162826f6c047d093b3670582296c770de1d94f8066694"
|
|
196
|
+
dependencies = [
|
|
197
|
+
"bindgen",
|
|
198
|
+
"lazy_static",
|
|
199
|
+
"proc-macro2",
|
|
200
|
+
"quote",
|
|
201
|
+
"regex",
|
|
202
|
+
"shell-words",
|
|
203
|
+
"syn",
|
|
204
|
+
]
|
|
205
|
+
|
|
206
|
+
[[package]]
|
|
207
|
+
name = "rb-sys-env"
|
|
208
|
+
version = "0.2.3"
|
|
209
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
210
|
+
checksum = "cca7ad6a7e21e72151d56fe2495a259b5670e204c3adac41ee7ef676ea08117a"
|
|
211
|
+
|
|
212
|
+
[[package]]
|
|
213
|
+
name = "regex"
|
|
214
|
+
version = "1.13.1"
|
|
215
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
216
|
+
checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
|
|
217
|
+
dependencies = [
|
|
218
|
+
"aho-corasick",
|
|
219
|
+
"memchr",
|
|
220
|
+
"regex-automata",
|
|
221
|
+
"regex-syntax",
|
|
222
|
+
]
|
|
223
|
+
|
|
224
|
+
[[package]]
|
|
225
|
+
name = "regex-automata"
|
|
226
|
+
version = "0.4.18"
|
|
227
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
228
|
+
checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
|
|
229
|
+
dependencies = [
|
|
230
|
+
"aho-corasick",
|
|
231
|
+
"memchr",
|
|
232
|
+
"regex-syntax",
|
|
233
|
+
]
|
|
234
|
+
|
|
235
|
+
[[package]]
|
|
236
|
+
name = "regex-syntax"
|
|
237
|
+
version = "0.8.11"
|
|
238
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
239
|
+
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
|
|
240
|
+
|
|
241
|
+
[[package]]
|
|
242
|
+
name = "rustc-hash"
|
|
243
|
+
version = "2.1.3"
|
|
244
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
245
|
+
checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
|
|
246
|
+
|
|
247
|
+
[[package]]
|
|
248
|
+
name = "seq-macro"
|
|
249
|
+
version = "0.3.6"
|
|
250
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
251
|
+
checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
|
|
252
|
+
|
|
253
|
+
[[package]]
|
|
254
|
+
name = "shell-words"
|
|
255
|
+
version = "1.1.1"
|
|
256
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
257
|
+
checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77"
|
|
258
|
+
|
|
259
|
+
[[package]]
|
|
260
|
+
name = "shlex"
|
|
261
|
+
version = "1.3.0"
|
|
262
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
263
|
+
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
|
264
|
+
|
|
265
|
+
[[package]]
|
|
266
|
+
name = "syn"
|
|
267
|
+
version = "2.0.119"
|
|
268
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
269
|
+
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
|
270
|
+
dependencies = [
|
|
271
|
+
"proc-macro2",
|
|
272
|
+
"quote",
|
|
273
|
+
"unicode-ident",
|
|
274
|
+
]
|
|
275
|
+
|
|
276
|
+
[[package]]
|
|
277
|
+
name = "unicode-ident"
|
|
278
|
+
version = "1.0.27"
|
|
279
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
280
|
+
checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
|
|
281
|
+
|
|
282
|
+
[[package]]
|
|
283
|
+
name = "windows-link"
|
|
284
|
+
version = "0.2.1"
|
|
285
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
286
|
+
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
data/Cargo.toml
ADDED
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Adrian Madrid
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
# field_struct-avro-native
|
|
2
|
+
|
|
3
|
+
An optional Rust encoder for [field_struct-avro](../field_struct-avro). It writes the same Avro bytes as the
|
|
4
|
+
pure-Ruby path, about 2.6x faster end to end on an Order-like struct.
|
|
5
|
+
|
|
6
|
+
```ruby run
|
|
7
|
+
require 'field_struct/avro'
|
|
8
|
+
|
|
9
|
+
class Beacon < FieldStruct::Base
|
|
10
|
+
required :id, :integer
|
|
11
|
+
required :tower, :string
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
pure = FieldStruct::Avro.encode(Beacon.new(id: 7, tower: 'north'))
|
|
15
|
+
|
|
16
|
+
require 'field_struct/avro/native'
|
|
17
|
+
|
|
18
|
+
FieldStruct::Avro::Native.enabled? # => true
|
|
19
|
+
FieldStruct::Avro.encode(Beacon.new(id: 7, tower: 'north')) == pure # => true
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Without the `require`, nothing changes. field_struct-avro doesn't depend on this gem.
|
|
23
|
+
|
|
24
|
+
## What it does
|
|
25
|
+
|
|
26
|
+
`FieldStruct::Avro.encode` builds a datum with `to_datum` and hands it to the avro gem's binary writer. That
|
|
27
|
+
writer is most of the cost: it validates the whole datum, then writes varints a byte at a time. This gem replaces
|
|
28
|
+
only that last step. It compiles each parsed writer schema once into an encoding plan, keeps it on the schema
|
|
29
|
+
object (so it follows the adapter's cache: a rebuilt or cleared entry gets a new schema and a new plan), and
|
|
30
|
+
walks the datum in Rust into one buffer.
|
|
31
|
+
|
|
32
|
+
The contract is byte-for-byte identical output. When the plan is not sure about a datum, it doesn't raise.
|
|
33
|
+
It hands the datum to the pure path, which then writes it or raises exactly the error field_struct-avro
|
|
34
|
+
would raise. It falls back for:
|
|
35
|
+
|
|
36
|
+
- a value of a class the plan doesn't expect, like an Integer in a `double` field or a DateTime in a `date` field
|
|
37
|
+
- a value out of range: an integer past `int`/`long`, a decimal past its precision or scale, a time past `long`
|
|
38
|
+
- a String in an encoding other than UTF-8 (7-bit US-ASCII and binary Strings are fine)
|
|
39
|
+
- a record datum that isn't exactly what `to_datum` builds (String keys, one per field, in field order)
|
|
40
|
+
- a union member it can't rule in or out the way the avro gem would
|
|
41
|
+
- schema shapes it doesn't cover: `fixed`, logical types other than `decimal` (on bytes), `date`,
|
|
42
|
+
`timestamp-millis/micros/nanos` and the identity ones like `uuid`
|
|
43
|
+
|
|
44
|
+
Two avro gem 1.12 quirks the plan copies by falling back: a decimal zero at scale 0 raises "Rounding
|
|
45
|
+
necessary", and a map with a key that is not valid UTF-8 raises inside a union.
|
|
46
|
+
|
|
47
|
+
## Install
|
|
48
|
+
|
|
49
|
+
```ruby
|
|
50
|
+
gem 'field_struct-avro'
|
|
51
|
+
gem 'field_struct-avro-native'
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Ruby 3.3 or newer. Precompiled gems aren't published yet, so installing builds the extension from source, and the
|
|
55
|
+
machine needs Rust (stable) and a C toolchain.
|
|
56
|
+
|
|
57
|
+
If the extension can't load, `require 'field_struct/avro/native'` raises `LoadError`. The per-datum fallback above
|
|
58
|
+
doesn't cover that case. To keep the native encoder optional, rescue it around the require:
|
|
59
|
+
|
|
60
|
+
```ruby
|
|
61
|
+
begin
|
|
62
|
+
require 'field_struct/avro/native'
|
|
63
|
+
rescue LoadError
|
|
64
|
+
# field_struct-avro keeps encoding in pure Ruby
|
|
65
|
+
end
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## Building
|
|
69
|
+
|
|
70
|
+
You need Rust (stable) and a C toolchain. From this directory:
|
|
71
|
+
|
|
72
|
+
```sh
|
|
73
|
+
bundle install
|
|
74
|
+
bundle exec rake compile # builds lib/field_struct/avro/field_struct_avro_native.bundle (.so on Linux)
|
|
75
|
+
bundle exec rake # compile, specs, field_struct-avro's suite with native, rubocop, clippy, cargo test
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
The Ruby-free kernels (zigzag varints, decimal bytes) live in their own crate, `ext/field_struct_avro_kernel`,
|
|
79
|
+
so `cargo test` runs without linking libruby.
|
|
80
|
+
|
|
81
|
+
## Release builds
|
|
82
|
+
|
|
83
|
+
`bundle exec rake native gem` builds a platform gem for the machine you are on. The other platforms are built in
|
|
84
|
+
containers: `bundle exec rb-sys-dock --platform aarch64-linux --ruby-versions 3.3,3.4,4.0 --build` (needs Docker; the
|
|
85
|
+
gem lands in `pkg/`). The tag workflow, `field_struct-avro-native-release.yml`, does this for every platform plus the
|
|
86
|
+
source gem, smoke-tests each, and uploads them as artifacts. It never publishes.
|
|
87
|
+
`support/smoke_install.sh <gem>` installs a built gem into a throwaway `GEM_HOME` and runs an encode.
|
|
88
|
+
|
|
89
|
+
## Proof
|
|
90
|
+
|
|
91
|
+
- `spec/field_struct/avro/native/datum_fuzz_spec.rb` - 9,000 random schema/datum pairs per run, edges and wrong
|
|
92
|
+
shapes included: whenever the plan writes, the pure path writes the same bytes, and it must take most datums
|
|
93
|
+
and leave both good and bad ones to the pure path
|
|
94
|
+
- `struct_differential_spec.rb` - 3,000 random structs over every field type the adapter maps, at each timestamp
|
|
95
|
+
precision: same bytes, and they decode and re-encode to the same bytes
|
|
96
|
+
- `fallback_spec.rb` - each error the adapter raises comes out the same with native on: a time too precise for
|
|
97
|
+
its timestamp, an invalid struct, a decimal past its precision or scale, an integer past 64 bits
|
|
98
|
+
- `rake upstream` - field_struct-avro's whole suite with `FIELD_STRUCT_AVRO_NATIVE=1`
|
|
99
|
+
|
|
100
|
+
## Numbers
|
|
101
|
+
|
|
102
|
+
Ruby 4.0.7 with YJIT on an M-series Mac, `adapters/field_struct-avro/benchmarks/perf.rb` with and without
|
|
103
|
+
`-rfield_struct/avro/native`, two runs each:
|
|
104
|
+
|
|
105
|
+
| workload | pure | native | speedup |
|
|
106
|
+
|---|---:|---:|---:|
|
|
107
|
+
| encode, 1 order | 23,476 i/s (42.6 us) | 61,468 i/s (16.3 us) | 2.6x |
|
|
108
|
+
| encode, 1000 orders | 23.2 i/s (43.2 us/order) | 61.6 i/s (16.2 us/order) | 2.7x |
|
|
109
|
+
| allocations per order | 417 | 155 | |
|
|
110
|
+
|
|
111
|
+
The write step alone goes from 26.4 us to 1.9 us. What is left is `to_datum` (12.6 us), so the next step, if
|
|
112
|
+
we want one, is a plan that reads the struct directly instead of the datum Hash.
|
|
113
|
+
|
|
114
|
+
## Not yet
|
|
115
|
+
|
|
116
|
+
- Published precompiled platform gems. The release workflow builds them (x86_64-linux, aarch64-linux, arm64-darwin)
|
|
117
|
+
but nothing is published yet, so installing compiles from source.
|
|
118
|
+
- `decode` stays pure Ruby. The avro gem's decimal reader is the hot spot there, ceiling about 1.6x.
|
|
119
|
+
|
|
120
|
+
## Docs
|
|
121
|
+
|
|
122
|
+
The guide on the docs site, with every example run in CI: https://gliders-rb.github.io/adapters/avro-native/
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
//! The unscaled integer of a decimal, from `BigDecimal#to_s` text (`"-0.12345e3"`), as minimal big-endian
|
|
2
|
+
//! two's-complement bytes: the Avro `decimal` logical type on `bytes`.
|
|
3
|
+
|
|
4
|
+
/// The most unscaled digits handled; anything longer goes to the pure path.
|
|
5
|
+
const MAX_DIGITS: i64 = 1_000;
|
|
6
|
+
|
|
7
|
+
#[derive(Debug, PartialEq, Eq)]
|
|
8
|
+
pub struct Unscaled {
|
|
9
|
+
negative: bool,
|
|
10
|
+
/// Little-endian base 2**32 limbs of the magnitude; empty is zero.
|
|
11
|
+
limbs: Vec<u32>,
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
impl Unscaled {
|
|
15
|
+
/// `None` when the text is not a finite decimal, for zero at scale 0, when scaling by `scale` would need rounding, or when the
|
|
16
|
+
/// unscaled value has more than `precision` digits (the cases the avro gem raises on).
|
|
17
|
+
pub fn parse(text: &[u8], scale: u32, precision: u32) -> Option<Self> {
|
|
18
|
+
let (negative, rest) = match text.split_first() {
|
|
19
|
+
Some((b'-', rest)) => (true, rest),
|
|
20
|
+
_ => (false, text),
|
|
21
|
+
};
|
|
22
|
+
let rest = rest.strip_prefix(b"0.")?;
|
|
23
|
+
let (digits, exponent) = match rest.iter().position(|&b| b == b'e') {
|
|
24
|
+
Some(i) => (&rest[..i], parse_exponent(&rest[i + 1..])?),
|
|
25
|
+
None => (rest, 0),
|
|
26
|
+
};
|
|
27
|
+
if digits.is_empty() || !digits.iter().all(u8::is_ascii_digit) {
|
|
28
|
+
return None;
|
|
29
|
+
}
|
|
30
|
+
let digits = match digits.iter().rposition(|&b| b != b'0') {
|
|
31
|
+
Some(last) => &digits[..=last],
|
|
32
|
+
// The avro gem 1.12 raises "Rounding necessary" for a zero at scale 0 (observed); it writes zero at any
|
|
33
|
+
// other scale.
|
|
34
|
+
None if scale == 0 => return None,
|
|
35
|
+
None => return Some(Unscaled { negative: false, limbs: Vec::new() }),
|
|
36
|
+
};
|
|
37
|
+
let first = digits.iter().position(|&b| b != b'0')?;
|
|
38
|
+
let digits = &digits[first..];
|
|
39
|
+
// value = 0.<digits> * 10**exponent = digits * 10**(exponent - len); leading zeros after "0." shift the point.
|
|
40
|
+
let len = digits.len() as i64 + first as i64;
|
|
41
|
+
let shift = exponent.checked_sub(len)?.checked_add(scale as i64)?;
|
|
42
|
+
if shift < 0 {
|
|
43
|
+
return None;
|
|
44
|
+
}
|
|
45
|
+
let total = digits.len() as i64 + shift;
|
|
46
|
+
if total > precision as i64 || total > MAX_DIGITS {
|
|
47
|
+
return None;
|
|
48
|
+
}
|
|
49
|
+
let mut limbs = Vec::with_capacity((total as usize) / 9 + 1);
|
|
50
|
+
for &d in digits {
|
|
51
|
+
mul_add(&mut limbs, 10, (d - b'0') as u32);
|
|
52
|
+
}
|
|
53
|
+
for _ in 0..shift {
|
|
54
|
+
mul_add(&mut limbs, 10, 0);
|
|
55
|
+
}
|
|
56
|
+
Some(Unscaled { negative, limbs })
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
pub fn twos_complement(&self) -> Vec<u8> {
|
|
60
|
+
let mut bytes: Vec<u8> = self.limbs.iter().rev().flat_map(|limb| limb.to_be_bytes()).collect();
|
|
61
|
+
let lead = bytes.iter().position(|&b| b != 0).unwrap_or(bytes.len());
|
|
62
|
+
bytes.drain(..lead);
|
|
63
|
+
if bytes.is_empty() {
|
|
64
|
+
return vec![0];
|
|
65
|
+
}
|
|
66
|
+
if !self.negative {
|
|
67
|
+
if bytes[0] & 0x80 != 0 {
|
|
68
|
+
bytes.insert(0, 0);
|
|
69
|
+
}
|
|
70
|
+
return bytes;
|
|
71
|
+
}
|
|
72
|
+
for b in bytes.iter_mut() {
|
|
73
|
+
*b = !*b;
|
|
74
|
+
}
|
|
75
|
+
for b in bytes.iter_mut().rev() {
|
|
76
|
+
let (sum, carry) = b.overflowing_add(1);
|
|
77
|
+
*b = sum;
|
|
78
|
+
if !carry {
|
|
79
|
+
break;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
if bytes[0] & 0x80 == 0 {
|
|
83
|
+
bytes.insert(0, 0xff);
|
|
84
|
+
}
|
|
85
|
+
bytes
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
fn parse_exponent(text: &[u8]) -> Option<i64> {
|
|
90
|
+
let s = std::str::from_utf8(text).ok()?;
|
|
91
|
+
let s = s.strip_prefix('+').unwrap_or(s);
|
|
92
|
+
s.parse::<i64>().ok()
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
fn mul_add(limbs: &mut Vec<u32>, mul: u32, add: u32) {
|
|
96
|
+
let mut carry = add as u64;
|
|
97
|
+
for limb in limbs.iter_mut() {
|
|
98
|
+
let v = (*limb as u64) * (mul as u64) + carry;
|
|
99
|
+
*limb = v as u32;
|
|
100
|
+
carry = v >> 32;
|
|
101
|
+
}
|
|
102
|
+
if carry > 0 {
|
|
103
|
+
limbs.push(carry as u32);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
#[cfg(test)]
|
|
108
|
+
mod tests {
|
|
109
|
+
use super::Unscaled;
|
|
110
|
+
|
|
111
|
+
fn bytes(text: &str, scale: u32, precision: u32) -> Option<Vec<u8>> {
|
|
112
|
+
Unscaled::parse(text.as_bytes(), scale, precision).map(|u| u.twos_complement())
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
#[test]
|
|
116
|
+
fn small_values() {
|
|
117
|
+
assert_eq!(bytes("0.0", 2, 10), Some(vec![0]));
|
|
118
|
+
assert_eq!(bytes("-0.0", 2, 10), Some(vec![0]));
|
|
119
|
+
assert_eq!(bytes("0.0", 0, 10), None);
|
|
120
|
+
assert_eq!(bytes("0.1e1", 2, 10), Some(vec![100]));
|
|
121
|
+
assert_eq!(bytes("-0.1e1", 2, 10), Some(vec![156]));
|
|
122
|
+
assert_eq!(bytes("0.127e1", 2, 10), Some(vec![127]));
|
|
123
|
+
assert_eq!(bytes("0.128e1", 2, 10), Some(vec![0, 128]));
|
|
124
|
+
assert_eq!(bytes("-0.128e1", 2, 10), Some(vec![128]));
|
|
125
|
+
assert_eq!(bytes("-0.129e1", 2, 10), Some(vec![255, 127]));
|
|
126
|
+
assert_eq!(bytes("-0.1e-1", 2, 10), Some(vec![255]));
|
|
127
|
+
assert_eq!(bytes("0.1e6", 2, 10), Some(vec![0, 152, 150, 128]));
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
#[test]
|
|
131
|
+
fn rounding_and_precision() {
|
|
132
|
+
assert_eq!(bytes("0.1234e1", 2, 10), None);
|
|
133
|
+
assert_eq!(bytes("0.1e3", 2, 4), None);
|
|
134
|
+
assert_eq!(bytes("0.9999e2", 2, 4), Some(vec![39, 15]));
|
|
135
|
+
assert_eq!(bytes("NaN", 2, 4), None);
|
|
136
|
+
assert_eq!(bytes("Infinity", 2, 4), None);
|
|
137
|
+
assert_eq!(bytes("-Infinity", 2, 4), None);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
#[test]
|
|
141
|
+
fn wide_values() {
|
|
142
|
+
let nines = format!("0.{}e40", "9".repeat(40));
|
|
143
|
+
let expected = vec![29, 99, 41, 241, 195, 92, 164, 191, 171, 185, 245, 96, 255, 255, 255, 255, 255];
|
|
144
|
+
assert_eq!(bytes(&nines, 0, 40), Some(expected));
|
|
145
|
+
let limit = "0.170141183460469231731687303715884105728e39";
|
|
146
|
+
let mut expected = vec![0, 128];
|
|
147
|
+
expected.extend(std::iter::repeat_n(0, 15));
|
|
148
|
+
assert_eq!(bytes(limit, 0, 40), Some(expected));
|
|
149
|
+
}
|
|
150
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
//! Ruby-free kernels of the Avro binary encoding, kept apart from the extension so `cargo test` needs no libruby.
|
|
2
|
+
|
|
3
|
+
pub mod decimal;
|
|
4
|
+
|
|
5
|
+
/// Appends `n` as an Avro `long` (and `int`): zigzag, then a little-endian base-128 varint.
|
|
6
|
+
pub fn write_long(n: i64, out: &mut Vec<u8>) {
|
|
7
|
+
let mut z = ((n << 1) ^ (n >> 63)) as u64;
|
|
8
|
+
while z >= 0x80 {
|
|
9
|
+
out.push((z as u8) | 0x80);
|
|
10
|
+
z >>= 7;
|
|
11
|
+
}
|
|
12
|
+
out.push(z as u8);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/// Appends a length-prefixed byte run: Avro `bytes`, and `string` once it is UTF-8.
|
|
16
|
+
pub fn write_bytes(bytes: &[u8], out: &mut Vec<u8>) {
|
|
17
|
+
write_long(bytes.len() as i64, out);
|
|
18
|
+
out.extend_from_slice(bytes);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
#[cfg(test)]
|
|
22
|
+
mod tests {
|
|
23
|
+
use super::{write_bytes, write_long};
|
|
24
|
+
|
|
25
|
+
fn long(n: i64) -> Vec<u8> {
|
|
26
|
+
let mut out = Vec::new();
|
|
27
|
+
write_long(n, &mut out);
|
|
28
|
+
out
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
#[test]
|
|
32
|
+
fn zigzag_varints() {
|
|
33
|
+
assert_eq!(long(0), [0]);
|
|
34
|
+
assert_eq!(long(-1), [1]);
|
|
35
|
+
assert_eq!(long(1), [2]);
|
|
36
|
+
assert_eq!(long(-64), [127]);
|
|
37
|
+
assert_eq!(long(64), [128, 1]);
|
|
38
|
+
assert_eq!(long(i64::MAX), [254, 255, 255, 255, 255, 255, 255, 255, 255, 1]);
|
|
39
|
+
assert_eq!(long(i64::MIN), [255, 255, 255, 255, 255, 255, 255, 255, 255, 1]);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
#[test]
|
|
43
|
+
fn byte_runs() {
|
|
44
|
+
let mut out = Vec::new();
|
|
45
|
+
write_bytes(b"ab", &mut out);
|
|
46
|
+
assert_eq!(out, [4, b'a', b'b']);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
[package]
|
|
2
|
+
name = "field_struct_avro_native"
|
|
3
|
+
version = "1.0.0"
|
|
4
|
+
edition = "2021"
|
|
5
|
+
license = "MIT"
|
|
6
|
+
publish = false
|
|
7
|
+
|
|
8
|
+
[lib]
|
|
9
|
+
crate-type = ["cdylib"]
|
|
10
|
+
|
|
11
|
+
[dependencies]
|
|
12
|
+
magnus = { version = "0.9", features = ["rb-sys"] }
|
|
13
|
+
field_struct_avro_kernel = { path = "../field_struct_avro_kernel" }
|