field_struct-avro-native 1.0.0.pre1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 9eab91e47b544a96c8aa8ff7bbb6c7a55a49cd4d46776567cf1a616f41327dc3
4
+ data.tar.gz: 89d8b167da605fb1413fbe5aae1724cba90a0bcb113d4dbeff875447453cd66a
5
+ SHA512:
6
+ metadata.gz: a3739510cd7b3e5ea234726ef99f8b2fc8bc8d0a1d17f92da2633b3e3738b9bff13aee23575fae82f022f8bc5059cf06353168c1209b0da532ca8864923802c0
7
+ data.tar.gz: 91ab669af9c9848537307c50ed298c356acd268db31edf01ba859085da2a1a73e5ed7673d726ad9a846b3b6f290f8f799c75708d34fc7cbcec85b2a604a46cd7
data/CHANGELOG.md ADDED
@@ -0,0 +1,5 @@
1
+ # Changelog
2
+
3
+ ## 1.0.0.pre1
4
+
5
+ - First release: an optional Rust encoder for `FieldStruct::Avro.encode`, loaded with `require 'field_struct/avro/native'`.
data/Cargo.lock ADDED
@@ -0,0 +1,286 @@
1
+ # This file is automatically @generated by Cargo.
2
+ # It is not intended for manual editing.
3
+ version = 4
4
+
5
+ [[package]]
6
+ name = "aho-corasick"
7
+ version = "1.1.5"
8
+ source = "registry+https://github.com/rust-lang/crates.io-index"
9
+ checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
10
+ dependencies = [
11
+ "memchr",
12
+ ]
13
+
14
+ [[package]]
15
+ name = "bindgen"
16
+ version = "0.72.1"
17
+ source = "registry+https://github.com/rust-lang/crates.io-index"
18
+ checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
19
+ dependencies = [
20
+ "bitflags",
21
+ "cexpr",
22
+ "clang-sys",
23
+ "itertools",
24
+ "proc-macro2",
25
+ "quote",
26
+ "regex",
27
+ "rustc-hash",
28
+ "shlex",
29
+ "syn",
30
+ ]
31
+
32
+ [[package]]
33
+ name = "bitflags"
34
+ version = "2.13.2"
35
+ source = "registry+https://github.com/rust-lang/crates.io-index"
36
+ checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
37
+
38
+ [[package]]
39
+ name = "cexpr"
40
+ version = "0.6.0"
41
+ source = "registry+https://github.com/rust-lang/crates.io-index"
42
+ checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
43
+ dependencies = [
44
+ "nom",
45
+ ]
46
+
47
+ [[package]]
48
+ name = "cfg-if"
49
+ version = "1.0.5"
50
+ source = "registry+https://github.com/rust-lang/crates.io-index"
51
+ checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
52
+
53
+ [[package]]
54
+ name = "clang-sys"
55
+ version = "1.9.1"
56
+ source = "registry+https://github.com/rust-lang/crates.io-index"
57
+ checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a"
58
+ dependencies = [
59
+ "glob",
60
+ "libc",
61
+ "libloading",
62
+ ]
63
+
64
+ [[package]]
65
+ name = "either"
66
+ version = "1.19.0"
67
+ source = "registry+https://github.com/rust-lang/crates.io-index"
68
+ checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be"
69
+
70
+ [[package]]
71
+ name = "field_struct_avro_kernel"
72
+ version = "1.0.0"
73
+
74
+ [[package]]
75
+ name = "field_struct_avro_native"
76
+ version = "1.0.0"
77
+ dependencies = [
78
+ "field_struct_avro_kernel",
79
+ "magnus",
80
+ ]
81
+
82
+ [[package]]
83
+ name = "glob"
84
+ version = "0.3.4"
85
+ source = "registry+https://github.com/rust-lang/crates.io-index"
86
+ checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b"
87
+
88
+ [[package]]
89
+ name = "itertools"
90
+ version = "0.13.0"
91
+ source = "registry+https://github.com/rust-lang/crates.io-index"
92
+ checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
93
+ dependencies = [
94
+ "either",
95
+ ]
96
+
97
+ [[package]]
98
+ name = "lazy_static"
99
+ version = "1.5.1"
100
+ source = "registry+https://github.com/rust-lang/crates.io-index"
101
+ checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb"
102
+
103
+ [[package]]
104
+ name = "libc"
105
+ version = "0.2.191"
106
+ source = "registry+https://github.com/rust-lang/crates.io-index"
107
+ checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
108
+
109
+ [[package]]
110
+ name = "libloading"
111
+ version = "0.8.9"
112
+ source = "registry+https://github.com/rust-lang/crates.io-index"
113
+ checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
114
+ dependencies = [
115
+ "cfg-if",
116
+ "windows-link",
117
+ ]
118
+
119
+ [[package]]
120
+ name = "magnus"
121
+ version = "0.9.3"
122
+ source = "registry+https://github.com/rust-lang/crates.io-index"
123
+ checksum = "40ef424404d166cbb71c8b8d19c9c589c748c4547c1afa1928c5504299f6c713"
124
+ dependencies = [
125
+ "magnus-macros",
126
+ "rb-sys",
127
+ "rb-sys-env",
128
+ "seq-macro",
129
+ ]
130
+
131
+ [[package]]
132
+ name = "magnus-macros"
133
+ version = "0.9.1"
134
+ source = "registry+https://github.com/rust-lang/crates.io-index"
135
+ checksum = "9dafde2eeb66f090e0f326112323eb8324b12cb32b5d1210e541d3df6b0ce77e"
136
+ dependencies = [
137
+ "proc-macro2",
138
+ "quote",
139
+ "syn",
140
+ ]
141
+
142
+ [[package]]
143
+ name = "memchr"
144
+ version = "2.8.3"
145
+ source = "registry+https://github.com/rust-lang/crates.io-index"
146
+ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
147
+
148
+ [[package]]
149
+ name = "minimal-lexical"
150
+ version = "0.2.1"
151
+ source = "registry+https://github.com/rust-lang/crates.io-index"
152
+ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
153
+
154
+ [[package]]
155
+ name = "nom"
156
+ version = "7.1.3"
157
+ source = "registry+https://github.com/rust-lang/crates.io-index"
158
+ checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
159
+ dependencies = [
160
+ "memchr",
161
+ "minimal-lexical",
162
+ ]
163
+
164
+ [[package]]
165
+ name = "proc-macro2"
166
+ version = "1.0.107"
167
+ source = "registry+https://github.com/rust-lang/crates.io-index"
168
+ checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
169
+ dependencies = [
170
+ "unicode-ident",
171
+ ]
172
+
173
+ [[package]]
174
+ name = "quote"
175
+ version = "1.0.47"
176
+ source = "registry+https://github.com/rust-lang/crates.io-index"
177
+ checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
178
+ dependencies = [
179
+ "proc-macro2",
180
+ ]
181
+
182
+ [[package]]
183
+ name = "rb-sys"
184
+ version = "0.9.130"
185
+ source = "registry+https://github.com/rust-lang/crates.io-index"
186
+ checksum = "02faf625bb10ba893e3ae620f19c9fb1b5f8fcae0fe4eb86bb3f2230fad75edb"
187
+ dependencies = [
188
+ "rb-sys-build",
189
+ ]
190
+
191
+ [[package]]
192
+ name = "rb-sys-build"
193
+ version = "0.9.130"
194
+ source = "registry+https://github.com/rust-lang/crates.io-index"
195
+ checksum = "05be6f9c86fe5482808162826f6c047d093b3670582296c770de1d94f8066694"
196
+ dependencies = [
197
+ "bindgen",
198
+ "lazy_static",
199
+ "proc-macro2",
200
+ "quote",
201
+ "regex",
202
+ "shell-words",
203
+ "syn",
204
+ ]
205
+
206
+ [[package]]
207
+ name = "rb-sys-env"
208
+ version = "0.2.3"
209
+ source = "registry+https://github.com/rust-lang/crates.io-index"
210
+ checksum = "cca7ad6a7e21e72151d56fe2495a259b5670e204c3adac41ee7ef676ea08117a"
211
+
212
+ [[package]]
213
+ name = "regex"
214
+ version = "1.13.1"
215
+ source = "registry+https://github.com/rust-lang/crates.io-index"
216
+ checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
217
+ dependencies = [
218
+ "aho-corasick",
219
+ "memchr",
220
+ "regex-automata",
221
+ "regex-syntax",
222
+ ]
223
+
224
+ [[package]]
225
+ name = "regex-automata"
226
+ version = "0.4.18"
227
+ source = "registry+https://github.com/rust-lang/crates.io-index"
228
+ checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
229
+ dependencies = [
230
+ "aho-corasick",
231
+ "memchr",
232
+ "regex-syntax",
233
+ ]
234
+
235
+ [[package]]
236
+ name = "regex-syntax"
237
+ version = "0.8.11"
238
+ source = "registry+https://github.com/rust-lang/crates.io-index"
239
+ checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
240
+
241
+ [[package]]
242
+ name = "rustc-hash"
243
+ version = "2.1.3"
244
+ source = "registry+https://github.com/rust-lang/crates.io-index"
245
+ checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
246
+
247
+ [[package]]
248
+ name = "seq-macro"
249
+ version = "0.3.6"
250
+ source = "registry+https://github.com/rust-lang/crates.io-index"
251
+ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
252
+
253
+ [[package]]
254
+ name = "shell-words"
255
+ version = "1.1.1"
256
+ source = "registry+https://github.com/rust-lang/crates.io-index"
257
+ checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77"
258
+
259
+ [[package]]
260
+ name = "shlex"
261
+ version = "1.3.0"
262
+ source = "registry+https://github.com/rust-lang/crates.io-index"
263
+ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
264
+
265
+ [[package]]
266
+ name = "syn"
267
+ version = "2.0.119"
268
+ source = "registry+https://github.com/rust-lang/crates.io-index"
269
+ checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
270
+ dependencies = [
271
+ "proc-macro2",
272
+ "quote",
273
+ "unicode-ident",
274
+ ]
275
+
276
+ [[package]]
277
+ name = "unicode-ident"
278
+ version = "1.0.27"
279
+ source = "registry+https://github.com/rust-lang/crates.io-index"
280
+ checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
281
+
282
+ [[package]]
283
+ name = "windows-link"
284
+ version = "0.2.1"
285
+ source = "registry+https://github.com/rust-lang/crates.io-index"
286
+ checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
data/Cargo.toml ADDED
@@ -0,0 +1,7 @@
1
+ [workspace]
2
+ members = ["ext/field_struct_avro_native", "ext/field_struct_avro_kernel"]
3
+ resolver = "2"
4
+
5
+ [profile.release]
6
+ codegen-units = 1
7
+ lto = true
data/LICENSE.txt ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Adrian Madrid
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,122 @@
1
+ # field_struct-avro-native
2
+
3
+ An optional Rust encoder for [field_struct-avro](../field_struct-avro). It writes the same Avro bytes as the
4
+ pure-Ruby path, about 2.6x faster end to end on an Order-like struct.
5
+
6
+ ```ruby run
7
+ require 'field_struct/avro'
8
+
9
+ class Beacon < FieldStruct::Base
10
+ required :id, :integer
11
+ required :tower, :string
12
+ end
13
+
14
+ pure = FieldStruct::Avro.encode(Beacon.new(id: 7, tower: 'north'))
15
+
16
+ require 'field_struct/avro/native'
17
+
18
+ FieldStruct::Avro::Native.enabled? # => true
19
+ FieldStruct::Avro.encode(Beacon.new(id: 7, tower: 'north')) == pure # => true
20
+ ```
21
+
22
+ Without the `require`, nothing changes. field_struct-avro doesn't depend on this gem.
23
+
24
+ ## What it does
25
+
26
+ `FieldStruct::Avro.encode` builds a datum with `to_datum` and hands it to the avro gem's binary writer. That
27
+ writer is most of the cost: it validates the whole datum, then writes varints a byte at a time. This gem replaces
28
+ only that last step. It compiles each parsed writer schema once into an encoding plan, keeps it on the schema
29
+ object (so it follows the adapter's cache: a rebuilt or cleared entry gets a new schema and a new plan), and
30
+ walks the datum in Rust into one buffer.
31
+
32
+ The contract is byte-for-byte identical output. When the plan is not sure about a datum, it doesn't raise.
33
+ It hands the datum to the pure path, which then writes it or raises exactly the error field_struct-avro
34
+ would raise. It falls back for:
35
+
36
+ - a value of a class the plan doesn't expect, like an Integer in a `double` field or a DateTime in a `date` field
37
+ - a value out of range: an integer past `int`/`long`, a decimal past its precision or scale, a time past `long`
38
+ - a String in an encoding other than UTF-8 (7-bit US-ASCII and binary Strings are fine)
39
+ - a record datum that isn't exactly what `to_datum` builds (String keys, one per field, in field order)
40
+ - a union member it can't rule in or out the way the avro gem would
41
+ - schema shapes it doesn't cover: `fixed`, logical types other than `decimal` (on bytes), `date`,
42
+ `timestamp-millis/micros/nanos` and the identity ones like `uuid`
43
+
44
+ Two avro gem 1.12 quirks the plan copies by falling back: a decimal zero at scale 0 raises "Rounding
45
+ necessary", and a map with a key that is not valid UTF-8 raises inside a union.
46
+
47
+ ## Install
48
+
49
+ ```ruby
50
+ gem 'field_struct-avro'
51
+ gem 'field_struct-avro-native'
52
+ ```
53
+
54
+ Ruby 3.3 or newer. Precompiled gems aren't published yet, so installing builds the extension from source, and the
55
+ machine needs Rust (stable) and a C toolchain.
56
+
57
+ If the extension can't load, `require 'field_struct/avro/native'` raises `LoadError`. The per-datum fallback above
58
+ doesn't cover that case. To keep the native encoder optional, rescue it around the require:
59
+
60
+ ```ruby
61
+ begin
62
+ require 'field_struct/avro/native'
63
+ rescue LoadError
64
+ # field_struct-avro keeps encoding in pure Ruby
65
+ end
66
+ ```
67
+
68
+ ## Building
69
+
70
+ You need Rust (stable) and a C toolchain. From this directory:
71
+
72
+ ```sh
73
+ bundle install
74
+ bundle exec rake compile # builds lib/field_struct/avro/field_struct_avro_native.bundle (.so on Linux)
75
+ bundle exec rake # compile, specs, field_struct-avro's suite with native, rubocop, clippy, cargo test
76
+ ```
77
+
78
+ The Ruby-free kernels (zigzag varints, decimal bytes) live in their own crate, `ext/field_struct_avro_kernel`,
79
+ so `cargo test` runs without linking libruby.
80
+
81
+ ## Release builds
82
+
83
+ `bundle exec rake native gem` builds a platform gem for the machine you are on. The other platforms are built in
84
+ containers: `bundle exec rb-sys-dock --platform aarch64-linux --ruby-versions 3.3,3.4,4.0 --build` (needs Docker; the
85
+ gem lands in `pkg/`). The tag workflow, `field_struct-avro-native-release.yml`, does this for every platform plus the
86
+ source gem, smoke-tests each, and uploads them as artifacts. It never publishes.
87
+ `support/smoke_install.sh <gem>` installs a built gem into a throwaway `GEM_HOME` and runs an encode.
88
+
89
+ ## Proof
90
+
91
+ - `spec/field_struct/avro/native/datum_fuzz_spec.rb` - 9,000 random schema/datum pairs per run, edges and wrong
92
+ shapes included: whenever the plan writes, the pure path writes the same bytes, and it must take most datums
93
+ and leave both good and bad ones to the pure path
94
+ - `struct_differential_spec.rb` - 3,000 random structs over every field type the adapter maps, at each timestamp
95
+ precision: same bytes, and they decode and re-encode to the same bytes
96
+ - `fallback_spec.rb` - each error the adapter raises comes out the same with native on: a time too precise for
97
+ its timestamp, an invalid struct, a decimal past its precision or scale, an integer past 64 bits
98
+ - `rake upstream` - field_struct-avro's whole suite with `FIELD_STRUCT_AVRO_NATIVE=1`
99
+
100
+ ## Numbers
101
+
102
+ Ruby 4.0.7 with YJIT on an M-series Mac, `adapters/field_struct-avro/benchmarks/perf.rb` with and without
103
+ `-rfield_struct/avro/native`, two runs each:
104
+
105
+ | workload | pure | native | speedup |
106
+ |---|---:|---:|---:|
107
+ | encode, 1 order | 23,476 i/s (42.6 us) | 61,468 i/s (16.3 us) | 2.6x |
108
+ | encode, 1000 orders | 23.2 i/s (43.2 us/order) | 61.6 i/s (16.2 us/order) | 2.7x |
109
+ | allocations per order | 417 | 155 | |
110
+
111
+ The write step alone goes from 26.4 us to 1.9 us. What is left is `to_datum` (12.6 us), so the next step, if
112
+ we want one, is a plan that reads the struct directly instead of the datum Hash.
113
+
114
+ ## Not yet
115
+
116
+ - Published precompiled platform gems. The release workflow builds them (x86_64-linux, aarch64-linux, arm64-darwin)
117
+ but nothing is published yet, so installing compiles from source.
118
+ - `decode` stays pure Ruby. The avro gem's decimal reader is the hot spot there, ceiling about 1.6x.
119
+
120
+ ## Docs
121
+
122
+ The guide on the docs site, with every example run in CI: https://gliders-rb.github.io/adapters/avro-native/
@@ -0,0 +1,9 @@
1
+ [package]
2
+ name = "field_struct_avro_kernel"
3
+ version = "1.0.0"
4
+ edition = "2021"
5
+ license = "MIT"
6
+ publish = false
7
+ description = "Ruby-free Avro binary kernels (zigzag varints, decimal bytes) for field_struct-avro-native."
8
+
9
+ [dependencies]
@@ -0,0 +1,150 @@
1
+ //! The unscaled integer of a decimal, from `BigDecimal#to_s` text (`"-0.12345e3"`), as minimal big-endian
2
+ //! two's-complement bytes: the Avro `decimal` logical type on `bytes`.
3
+
4
+ /// The most unscaled digits handled; anything longer goes to the pure path.
5
+ const MAX_DIGITS: i64 = 1_000;
6
+
7
+ #[derive(Debug, PartialEq, Eq)]
8
+ pub struct Unscaled {
9
+ negative: bool,
10
+ /// Little-endian base 2**32 limbs of the magnitude; empty is zero.
11
+ limbs: Vec<u32>,
12
+ }
13
+
14
+ impl Unscaled {
15
+ /// `None` when the text is not a finite decimal, for zero at scale 0, when scaling by `scale` would need rounding, or when the
16
+ /// unscaled value has more than `precision` digits (the cases the avro gem raises on).
17
+ pub fn parse(text: &[u8], scale: u32, precision: u32) -> Option<Self> {
18
+ let (negative, rest) = match text.split_first() {
19
+ Some((b'-', rest)) => (true, rest),
20
+ _ => (false, text),
21
+ };
22
+ let rest = rest.strip_prefix(b"0.")?;
23
+ let (digits, exponent) = match rest.iter().position(|&b| b == b'e') {
24
+ Some(i) => (&rest[..i], parse_exponent(&rest[i + 1..])?),
25
+ None => (rest, 0),
26
+ };
27
+ if digits.is_empty() || !digits.iter().all(u8::is_ascii_digit) {
28
+ return None;
29
+ }
30
+ let digits = match digits.iter().rposition(|&b| b != b'0') {
31
+ Some(last) => &digits[..=last],
32
+ // The avro gem 1.12 raises "Rounding necessary" for a zero at scale 0 (observed); it writes zero at any
33
+ // other scale.
34
+ None if scale == 0 => return None,
35
+ None => return Some(Unscaled { negative: false, limbs: Vec::new() }),
36
+ };
37
+ let first = digits.iter().position(|&b| b != b'0')?;
38
+ let digits = &digits[first..];
39
+ // value = 0.<digits> * 10**exponent = digits * 10**(exponent - len); leading zeros after "0." shift the point.
40
+ let len = digits.len() as i64 + first as i64;
41
+ let shift = exponent.checked_sub(len)?.checked_add(scale as i64)?;
42
+ if shift < 0 {
43
+ return None;
44
+ }
45
+ let total = digits.len() as i64 + shift;
46
+ if total > precision as i64 || total > MAX_DIGITS {
47
+ return None;
48
+ }
49
+ let mut limbs = Vec::with_capacity((total as usize) / 9 + 1);
50
+ for &d in digits {
51
+ mul_add(&mut limbs, 10, (d - b'0') as u32);
52
+ }
53
+ for _ in 0..shift {
54
+ mul_add(&mut limbs, 10, 0);
55
+ }
56
+ Some(Unscaled { negative, limbs })
57
+ }
58
+
59
+ pub fn twos_complement(&self) -> Vec<u8> {
60
+ let mut bytes: Vec<u8> = self.limbs.iter().rev().flat_map(|limb| limb.to_be_bytes()).collect();
61
+ let lead = bytes.iter().position(|&b| b != 0).unwrap_or(bytes.len());
62
+ bytes.drain(..lead);
63
+ if bytes.is_empty() {
64
+ return vec![0];
65
+ }
66
+ if !self.negative {
67
+ if bytes[0] & 0x80 != 0 {
68
+ bytes.insert(0, 0);
69
+ }
70
+ return bytes;
71
+ }
72
+ for b in bytes.iter_mut() {
73
+ *b = !*b;
74
+ }
75
+ for b in bytes.iter_mut().rev() {
76
+ let (sum, carry) = b.overflowing_add(1);
77
+ *b = sum;
78
+ if !carry {
79
+ break;
80
+ }
81
+ }
82
+ if bytes[0] & 0x80 == 0 {
83
+ bytes.insert(0, 0xff);
84
+ }
85
+ bytes
86
+ }
87
+ }
88
+
89
+ fn parse_exponent(text: &[u8]) -> Option<i64> {
90
+ let s = std::str::from_utf8(text).ok()?;
91
+ let s = s.strip_prefix('+').unwrap_or(s);
92
+ s.parse::<i64>().ok()
93
+ }
94
+
95
+ fn mul_add(limbs: &mut Vec<u32>, mul: u32, add: u32) {
96
+ let mut carry = add as u64;
97
+ for limb in limbs.iter_mut() {
98
+ let v = (*limb as u64) * (mul as u64) + carry;
99
+ *limb = v as u32;
100
+ carry = v >> 32;
101
+ }
102
+ if carry > 0 {
103
+ limbs.push(carry as u32);
104
+ }
105
+ }
106
+
107
+ #[cfg(test)]
108
+ mod tests {
109
+ use super::Unscaled;
110
+
111
+ fn bytes(text: &str, scale: u32, precision: u32) -> Option<Vec<u8>> {
112
+ Unscaled::parse(text.as_bytes(), scale, precision).map(|u| u.twos_complement())
113
+ }
114
+
115
+ #[test]
116
+ fn small_values() {
117
+ assert_eq!(bytes("0.0", 2, 10), Some(vec![0]));
118
+ assert_eq!(bytes("-0.0", 2, 10), Some(vec![0]));
119
+ assert_eq!(bytes("0.0", 0, 10), None);
120
+ assert_eq!(bytes("0.1e1", 2, 10), Some(vec![100]));
121
+ assert_eq!(bytes("-0.1e1", 2, 10), Some(vec![156]));
122
+ assert_eq!(bytes("0.127e1", 2, 10), Some(vec![127]));
123
+ assert_eq!(bytes("0.128e1", 2, 10), Some(vec![0, 128]));
124
+ assert_eq!(bytes("-0.128e1", 2, 10), Some(vec![128]));
125
+ assert_eq!(bytes("-0.129e1", 2, 10), Some(vec![255, 127]));
126
+ assert_eq!(bytes("-0.1e-1", 2, 10), Some(vec![255]));
127
+ assert_eq!(bytes("0.1e6", 2, 10), Some(vec![0, 152, 150, 128]));
128
+ }
129
+
130
+ #[test]
131
+ fn rounding_and_precision() {
132
+ assert_eq!(bytes("0.1234e1", 2, 10), None);
133
+ assert_eq!(bytes("0.1e3", 2, 4), None);
134
+ assert_eq!(bytes("0.9999e2", 2, 4), Some(vec![39, 15]));
135
+ assert_eq!(bytes("NaN", 2, 4), None);
136
+ assert_eq!(bytes("Infinity", 2, 4), None);
137
+ assert_eq!(bytes("-Infinity", 2, 4), None);
138
+ }
139
+
140
+ #[test]
141
+ fn wide_values() {
142
+ let nines = format!("0.{}e40", "9".repeat(40));
143
+ let expected = vec![29, 99, 41, 241, 195, 92, 164, 191, 171, 185, 245, 96, 255, 255, 255, 255, 255];
144
+ assert_eq!(bytes(&nines, 0, 40), Some(expected));
145
+ let limit = "0.170141183460469231731687303715884105728e39";
146
+ let mut expected = vec![0, 128];
147
+ expected.extend(std::iter::repeat_n(0, 15));
148
+ assert_eq!(bytes(limit, 0, 40), Some(expected));
149
+ }
150
+ }
@@ -0,0 +1,48 @@
1
+ //! Ruby-free kernels of the Avro binary encoding, kept apart from the extension so `cargo test` needs no libruby.
2
+
3
+ pub mod decimal;
4
+
5
+ /// Appends `n` as an Avro `long` (and `int`): zigzag, then a little-endian base-128 varint.
6
+ pub fn write_long(n: i64, out: &mut Vec<u8>) {
7
+ let mut z = ((n << 1) ^ (n >> 63)) as u64;
8
+ while z >= 0x80 {
9
+ out.push((z as u8) | 0x80);
10
+ z >>= 7;
11
+ }
12
+ out.push(z as u8);
13
+ }
14
+
15
+ /// Appends a length-prefixed byte run: Avro `bytes`, and `string` once it is UTF-8.
16
+ pub fn write_bytes(bytes: &[u8], out: &mut Vec<u8>) {
17
+ write_long(bytes.len() as i64, out);
18
+ out.extend_from_slice(bytes);
19
+ }
20
+
21
+ #[cfg(test)]
22
+ mod tests {
23
+ use super::{write_bytes, write_long};
24
+
25
+ fn long(n: i64) -> Vec<u8> {
26
+ let mut out = Vec::new();
27
+ write_long(n, &mut out);
28
+ out
29
+ }
30
+
31
+ #[test]
32
+ fn zigzag_varints() {
33
+ assert_eq!(long(0), [0]);
34
+ assert_eq!(long(-1), [1]);
35
+ assert_eq!(long(1), [2]);
36
+ assert_eq!(long(-64), [127]);
37
+ assert_eq!(long(64), [128, 1]);
38
+ assert_eq!(long(i64::MAX), [254, 255, 255, 255, 255, 255, 255, 255, 255, 1]);
39
+ assert_eq!(long(i64::MIN), [255, 255, 255, 255, 255, 255, 255, 255, 255, 1]);
40
+ }
41
+
42
+ #[test]
43
+ fn byte_runs() {
44
+ let mut out = Vec::new();
45
+ write_bytes(b"ab", &mut out);
46
+ assert_eq!(out, [4, b'a', b'b']);
47
+ }
48
+ }
@@ -0,0 +1,13 @@
1
+ [package]
2
+ name = "field_struct_avro_native"
3
+ version = "1.0.0"
4
+ edition = "2021"
5
+ license = "MIT"
6
+ publish = false
7
+
8
+ [lib]
9
+ crate-type = ["cdylib"]
10
+
11
+ [dependencies]
12
+ magnus = { version = "0.9", features = ["rb-sys"] }
13
+ field_struct_avro_kernel = { path = "../field_struct_avro_kernel" }
@@ -0,0 +1,6 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'mkmf'
4
+ require 'rb_sys/mkmf'
5
+
6
+ create_rust_makefile('field_struct/avro/field_struct_avro_native')