zlight_csv 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +128 -0
- data/LICENSE +21 -0
- data/README.md +225 -0
- data/VERSION +1 -0
- data/ext/zlight_csv/Cargo.toml +30 -0
- data/ext/zlight_csv/extconf.rb +16 -0
- data/ext/zlight_csv/src/converter.rs +133 -0
- data/ext/zlight_csv/src/error.rs +122 -0
- data/ext/zlight_csv/src/lib.rs +55 -0
- data/ext/zlight_csv/src/options.rs +95 -0
- data/ext/zlight_csv/src/parser.rs +77 -0
- data/ext/zlight_csv/src/stream.rs +410 -0
- data/ext/zlight_csv/src/writer.rs +268 -0
- data/lib/zlight_csv/version.rb +6 -0
- data/lib/zlight_csv.rb +305 -0
- metadata +86 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: be8e072a58943ca8b89b8f68c0943ad9daf80be6edb6274fad3f8fe7a20af34a
|
|
4
|
+
data.tar.gz: 5bf977f61545af25293a5e2b3560ee5e06df896b373a4771252d9a9dc482451a
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: fb006274ac24756d4336d27647343b64b6c753f6b6a5d4c35357b354558d379f8548dbef8825092c59ec9d8c4eb2431561177decb506816d22553f8d5a3eb10a
|
|
7
|
+
data.tar.gz: 88fff6de78e745e818cf22a3e92f3772550824e2453d93bc7e864a59d03a7131dafe1a4ccb7116086b30161559b65ce0f1696f236c0d339078dfa47acbd222c6
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
## [0.5.1] - 2026-09-12
|
|
11
|
+
|
|
12
|
+
### Fixed
|
|
13
|
+
|
|
14
|
+
- The extension now builds against a Ruby that rb-sys has no prebuilt bindings
|
|
15
|
+
for, via rb-sys `stable-api-compiled-fallback`, rather than failing outright.
|
|
16
|
+
This is the mechanism that breaks the build on every new Ruby release until
|
|
17
|
+
rb-sys catches up
|
|
18
|
+
- Windows CI could not build the extension at all: the runner paired a MinGW
|
|
19
|
+
Ruby with an MSVC Rust toolchain. Ruby and Rust are now set up together so
|
|
20
|
+
their ABIs match, and Windows is verified across Ruby 3.1-4.0
|
|
21
|
+
- Specs that asserted Unix-only error messages and paths now pass on Windows
|
|
22
|
+
- Dropped an unreachable credentials check from the release workflow: the
|
|
23
|
+
credentials action fails first, so the check never ran
|
|
24
|
+
|
|
25
|
+
### Note
|
|
26
|
+
|
|
27
|
+
- 0.5.0 was tagged but never reached RubyGems: its release run built every gem
|
|
28
|
+
and then failed to publish, because trusted publishing still named the
|
|
29
|
+
project's former GitHub account. 0.5.1 is the first release carrying the
|
|
30
|
+
0.5.0 changes below.
|
|
31
|
+
|
|
32
|
+
## [0.5.0] - 2026-09-12
|
|
33
|
+
|
|
34
|
+
### Added
|
|
35
|
+
|
|
36
|
+
- Ruby 4.0 support
|
|
37
|
+
- `docker/test-matrix.sh`, which builds and tests the extension against every
|
|
38
|
+
supported Ruby in containers
|
|
39
|
+
- Windows, Ruby 4.0 and a `ruby-head` early-warning job in CI, plus weekly
|
|
40
|
+
scheduled runs and Dependabot
|
|
41
|
+
|
|
42
|
+
### Changed
|
|
43
|
+
|
|
44
|
+
- Upgraded magnus from 0.6 to 0.8. magnus 0.6 read a field of Ruby's
|
|
45
|
+
RTypedData struct that Ruby 4.0 removed, so the extension could not compile
|
|
46
|
+
against it at all
|
|
47
|
+
- Minimum Ruby is now 3.1. Ruby 3.0 is end-of-life and no longer supported by
|
|
48
|
+
magnus
|
|
49
|
+
- Repository URLs now point at `codebyisaad/zlight`, which is where the code
|
|
50
|
+
is hosted
|
|
51
|
+
|
|
52
|
+
### Fixed
|
|
53
|
+
|
|
54
|
+
- The source gem shipped no extension and could not be compiled, so any Ruby
|
|
55
|
+
or platform without a prebuilt binary installed a gem that failed to load
|
|
56
|
+
- `ZLight::ParseError`, `ZLight::EncodingError` and `ZLight::StreamClosedError`
|
|
57
|
+
were documented but never raised
|
|
58
|
+
- Integers outside the i64 range were silently converted to lossy Floats
|
|
59
|
+
- `col_sep` and `quote_char` longer than one byte were silently truncated when
|
|
60
|
+
reading, while writing rejected them
|
|
61
|
+
- Releases no longer yank every previously published version
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
## [0.4.0] - 2026-06-06
|
|
65
|
+
|
|
66
|
+
### Added
|
|
67
|
+
|
|
68
|
+
- `ZLight.generate` — build a CSV string from an array of hashes or an array
|
|
69
|
+
of arrays, with `headers`, `col_sep`, `quote_char` and `force_quotes` options
|
|
70
|
+
- `ZLight.write` — generate a CSV and write it to a file in one call
|
|
71
|
+
|
|
72
|
+
### Changed
|
|
73
|
+
|
|
74
|
+
- Parsed fields are tagged with the encoding of the input string, matching
|
|
75
|
+
Ruby's stdlib CSV, so the parser and writer round-trip without
|
|
76
|
+
`force_encoding`
|
|
77
|
+
|
|
78
|
+
## [0.3.0] - 2026-05-21
|
|
79
|
+
|
|
80
|
+
### Changed
|
|
81
|
+
|
|
82
|
+
- Release pipeline updates
|
|
83
|
+
|
|
84
|
+
## [0.2.5] - 2026-05-21
|
|
85
|
+
|
|
86
|
+
### Added
|
|
87
|
+
|
|
88
|
+
- `ZLight.stream` and `ZLight.stream_file` — read rows one at a time without
|
|
89
|
+
loading the whole input into memory
|
|
90
|
+
- `ZLight.open` — streaming with a block, closing the reader automatically
|
|
91
|
+
- `ZLight::StreamReader`, including `Enumerable`, so `lazy`, `select`, `find`
|
|
92
|
+
and friends work over a stream
|
|
93
|
+
|
|
94
|
+
### Fixed
|
|
95
|
+
|
|
96
|
+
- Packaging and gemspec metadata corrections across 0.2.1–0.2.5
|
|
97
|
+
|
|
98
|
+
## [0.2.0] - 2026-05-21
|
|
99
|
+
|
|
100
|
+
### Added
|
|
101
|
+
|
|
102
|
+
- Benchmark suite comparing ZLight against Ruby's stdlib CSV
|
|
103
|
+
- Edge case coverage for quoting, encodings and flexible records
|
|
104
|
+
|
|
105
|
+
## [0.1.2] - 2026-05-19
|
|
106
|
+
|
|
107
|
+
### Added
|
|
108
|
+
|
|
109
|
+
- Initial release
|
|
110
|
+
- `ZLight.parse` — parse CSV strings, with `headers`, `converters`, `col_sep`,
|
|
111
|
+
`quote_char` and `flexible` options
|
|
112
|
+
- `ZLight.read` — read and parse a CSV file
|
|
113
|
+
- `ZLight.foreach` — iterate over parsed rows
|
|
114
|
+
- Cross-platform precompiled native gems
|
|
115
|
+
|
|
116
|
+
### Performance
|
|
117
|
+
|
|
118
|
+
- Substantially faster than Ruby's stdlib CSV; see the README for measured
|
|
119
|
+
figures
|
|
120
|
+
|
|
121
|
+
[Unreleased]: https://github.com/codebyisaad/zlight/compare/v0.5.1...HEAD
|
|
122
|
+
[0.5.1]: https://github.com/codebyisaad/zlight/compare/v0.5.0...v0.5.1
|
|
123
|
+
[0.5.0]: https://github.com/codebyisaad/zlight/compare/v0.4.0...v0.5.0
|
|
124
|
+
[0.4.0]: https://github.com/codebyisaad/zlight/compare/v0.3.0...v0.4.0
|
|
125
|
+
[0.3.0]: https://github.com/codebyisaad/zlight/compare/v0.2.5...v0.3.0
|
|
126
|
+
[0.2.5]: https://github.com/codebyisaad/zlight/compare/v0.2.0...v0.2.5
|
|
127
|
+
[0.2.0]: https://github.com/codebyisaad/zlight/compare/v0.1.2...v0.2.0
|
|
128
|
+
[0.1.2]: https://github.com/codebyisaad/zlight/releases/tag/v0.1.2
|
data/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 Zaidan Chaudhary (also known as Saad Chaudhary)
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
# ZLight CSV
|
|
2
|
+
|
|
3
|
+
[](https://badge.fury.io/rb/zlight_csv)
|
|
4
|
+
[](https://opensource.org/licenses/MIT)
|
|
5
|
+
|
|
6
|
+
A fast CSV parser for Ruby, powered by Rust.
|
|
7
|
+
|
|
8
|
+
## Why ZLight?
|
|
9
|
+
|
|
10
|
+
Ruby's built-in CSV library is slow. ZLight parses CSV files **up to 40x faster** by using Rust under the hood.
|
|
11
|
+
The gap is widest on small inputs and narrows as they grow; the measurements below show the range.
|
|
12
|
+
|
|
13
|
+
### Benchmark Results
|
|
14
|
+
|
|
15
|
+
Parsing with headers and numeric conversion (Apple M1):
|
|
16
|
+
|
|
17
|
+
| Dataset | Ruby CSV | ZLight | Speedup |
|
|
18
|
+
|---------|----------|--------|---------|
|
|
19
|
+
| 1K rows | 12.6ms | 0.3ms | **42x faster** |
|
|
20
|
+
| 10K rows | 133ms | 4.4ms | **30x faster** |
|
|
21
|
+
| 100K rows | 1,458ms | 78ms | **19x faster** |
|
|
22
|
+
|
|
23
|
+
File reading comparison (100K rows):
|
|
24
|
+
|
|
25
|
+
| Method | Ruby CSV | ZLight | Speedup |
|
|
26
|
+
|--------|----------|--------|---------|
|
|
27
|
+
| Read all | 1,596ms | 58ms | **27x faster** |
|
|
28
|
+
| Streaming | 1,104ms | 74ms | **15x faster** |
|
|
29
|
+
|
|
30
|
+
Iterations per second (10K rows):
|
|
31
|
+
|
|
32
|
+
```
|
|
33
|
+
ZLight.parse: 253 i/s
|
|
34
|
+
ZLight.stream: 159 i/s
|
|
35
|
+
Ruby CSV: 8 i/s — 33x slower
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## Installation
|
|
39
|
+
|
|
40
|
+
```ruby
|
|
41
|
+
gem 'zlight_csv'
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
No Rust toolchain required — prebuilt binaries are available for Linux, macOS, and Windows.
|
|
45
|
+
|
|
46
|
+
## Usage
|
|
47
|
+
|
|
48
|
+
```ruby
|
|
49
|
+
require 'zlight_csv'
|
|
50
|
+
|
|
51
|
+
# Parse a CSV string
|
|
52
|
+
data = ZLight.parse("name,age\nAlice,30\nBob,25")
|
|
53
|
+
# => [{:name=>"Alice", :age=>"30"}, {:name=>"Bob", :age=>"25"}]
|
|
54
|
+
|
|
55
|
+
# With automatic numeric conversion
|
|
56
|
+
data = ZLight.parse(csv_string, converters: :numeric)
|
|
57
|
+
# => [{:name=>"Alice", :age=>30}, {:name=>"Bob", :age=>25}]
|
|
58
|
+
|
|
59
|
+
# Read from a file
|
|
60
|
+
data = ZLight.read("users.csv")
|
|
61
|
+
|
|
62
|
+
# Iterate over rows
|
|
63
|
+
ZLight.foreach(csv_string) do |row|
|
|
64
|
+
puts row[:name]
|
|
65
|
+
end
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
### Streaming Large Files
|
|
69
|
+
|
|
70
|
+
For large files, use streaming to process rows one at a time without loading everything into memory:
|
|
71
|
+
|
|
72
|
+
```ruby
|
|
73
|
+
# Stream from a file (auto-closes when block exits)
|
|
74
|
+
ZLight.open("large_file.csv") do |reader|
|
|
75
|
+
reader.each do |row|
|
|
76
|
+
process(row)
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Lazy enumeration — stop early without loading remaining rows
|
|
81
|
+
ZLight.open("huge_file.csv", converters: :numeric) do |reader|
|
|
82
|
+
high_scores = reader.lazy
|
|
83
|
+
.select { |row| row[:score] > 90 }
|
|
84
|
+
.first(100)
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Manual control
|
|
88
|
+
reader = ZLight.stream_file("data.csv")
|
|
89
|
+
while row = reader.next_row
|
|
90
|
+
break if row[:id] > 1000
|
|
91
|
+
process(row)
|
|
92
|
+
end
|
|
93
|
+
reader.close
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
**Streaming is especially efficient for partial reads:**
|
|
97
|
+
|
|
98
|
+
```
|
|
99
|
+
Finding first 100 rows from 100K dataset:
|
|
100
|
+
|
|
101
|
+
Ruby CSV (full parse): 1,696ms
|
|
102
|
+
ZLight.parse (full): 73ms
|
|
103
|
+
ZLight.stream (lazy): 0.3ms ← 5,600x faster!
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
### Options (Reading)
|
|
107
|
+
|
|
108
|
+
| Option | Default | Description |
|
|
109
|
+
|--------|---------|-------------|
|
|
110
|
+
| `headers` | `true` | Use first row as headers (returns hashes). Set `false` for arrays. |
|
|
111
|
+
| `converters` | `nil` | Set to `:numeric` to convert numbers automatically |
|
|
112
|
+
| `col_sep` | `","` | Column separator (`"\t"` for TSV, `";"` for European CSV) |
|
|
113
|
+
| `quote_char` | `"` | Quote character |
|
|
114
|
+
| `flexible` | `true` | Allow rows with varying column counts |
|
|
115
|
+
|
|
116
|
+
### Writing CSV
|
|
117
|
+
|
|
118
|
+
Generate CSV strings or write directly to files:
|
|
119
|
+
|
|
120
|
+
```ruby
|
|
121
|
+
# Generate CSV from array of hashes
|
|
122
|
+
csv = ZLight.generate([
|
|
123
|
+
{ name: "Alice", age: 30 },
|
|
124
|
+
{ name: "Bob", age: 25 }
|
|
125
|
+
])
|
|
126
|
+
# => "name,age\nAlice,30\nBob,25\n"
|
|
127
|
+
|
|
128
|
+
# Generate from array of arrays (no headers)
|
|
129
|
+
csv = ZLight.generate([["Alice", 30], ["Bob", 25]])
|
|
130
|
+
# => "Alice,30\nBob,25\n"
|
|
131
|
+
|
|
132
|
+
# Write directly to a file
|
|
133
|
+
ZLight.write("users.csv", data)
|
|
134
|
+
|
|
135
|
+
# With options
|
|
136
|
+
ZLight.generate(data, col_sep: "\t", force_quotes: true)
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
### Options (Writing)
|
|
140
|
+
|
|
141
|
+
| Option | Default | Description |
|
|
142
|
+
|--------|---------|-------------|
|
|
143
|
+
| `headers` | `true` | Write header row (for hash input) |
|
|
144
|
+
| `col_sep` | `","` | Column separator |
|
|
145
|
+
| `quote_char` | `"` | Quote character |
|
|
146
|
+
| `force_quotes` | `false` | Quote all fields, not just those requiring it |
|
|
147
|
+
|
|
148
|
+
## Compatibility
|
|
149
|
+
|
|
150
|
+
Works as a drop-in replacement for common `CSV.parse` patterns:
|
|
151
|
+
|
|
152
|
+
```ruby
|
|
153
|
+
# Before
|
|
154
|
+
CSV.parse(data, headers: true, header_converters: :symbol, converters: :numeric)
|
|
155
|
+
|
|
156
|
+
# After
|
|
157
|
+
ZLight.parse(data, converters: :numeric)
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
### Differences from stdlib CSV
|
|
161
|
+
|
|
162
|
+
ZLight is not a complete reimplementation of `CSV`. The differences below are
|
|
163
|
+
deliberate, and are the ones most likely to matter when migrating.
|
|
164
|
+
|
|
165
|
+
**API shape**
|
|
166
|
+
|
|
167
|
+
- `ZLight.foreach` takes a **CSV string**, while `CSV.foreach` takes a **file
|
|
168
|
+
path**. Use `ZLight.open` to iterate a file.
|
|
169
|
+
- `ZLight.foreach` parses the whole input before yielding. For genuinely lazy
|
|
170
|
+
iteration use `ZLight.stream` or `ZLight.open`.
|
|
171
|
+
- `ZLight::StreamReader` is single-pass. It includes `Enumerable`, but each
|
|
172
|
+
row is consumed as it is read, so a second pass yields nothing and there is
|
|
173
|
+
no rewind. Call `ZLight.stream` again to re-read.
|
|
174
|
+
|
|
175
|
+
**Parsing**
|
|
176
|
+
|
|
177
|
+
- Duplicate headers collapse. Rows are `Hash`es, so `"a,a"` keeps only the
|
|
178
|
+
last `:a` column; `CSV` keeps both.
|
|
179
|
+
- Fields beyond the header count are dropped rather than collected.
|
|
180
|
+
- Headers become symbols always, equivalent to `header_converters: :symbol`.
|
|
181
|
+
|
|
182
|
+
**`converters: :numeric`**
|
|
183
|
+
|
|
184
|
+
| Input | ZLight | Ruby CSV |
|
|
185
|
+
|--------------|-------------------|------------|
|
|
186
|
+
| `""` | `""` | `nil` |
|
|
187
|
+
| `"0x10"` | `"0x10"` | `16` |
|
|
188
|
+
| `"1_000"` | `"1_000"` | `1000` |
|
|
189
|
+
| `"Infinity"` | `Float::INFINITY` | `"Infinity"` |
|
|
190
|
+
| `"NaN"` | `Float::NAN` | `"NaN"` |
|
|
191
|
+
|
|
192
|
+
Integers of any size are exact, as in `CSV`.
|
|
193
|
+
|
|
194
|
+
**Writing**
|
|
195
|
+
|
|
196
|
+
- `ZLight.generate` takes its column order from the keys of the **first**
|
|
197
|
+
hash. Keys that appear only in later rows are not written; keys missing
|
|
198
|
+
from a later row are written as empty fields.
|
|
199
|
+
|
|
200
|
+
## Requirements
|
|
201
|
+
|
|
202
|
+
- Ruby 3.1 or newer, including Ruby 4.0
|
|
203
|
+
- Linux (x86_64, aarch64), macOS (Intel, Apple Silicon), or Windows (x64)
|
|
204
|
+
|
|
205
|
+
Every supported Ruby is built and tested on each change; see
|
|
206
|
+
`docker/test-matrix.sh` to run that matrix yourself. Rubies without a
|
|
207
|
+
precompiled binary install from the source gem and need a Rust toolchain.
|
|
208
|
+
|
|
209
|
+
## Roadmap
|
|
210
|
+
|
|
211
|
+
- [x] Streaming/lazy parsing for large files
|
|
212
|
+
- [x] CSV writing support
|
|
213
|
+
- [ ] Custom converter procs
|
|
214
|
+
|
|
215
|
+
## Contributing
|
|
216
|
+
|
|
217
|
+
Bug reports and pull requests are welcome on [GitHub](https://github.com/codebyisaad/zlight).
|
|
218
|
+
|
|
219
|
+
## Author
|
|
220
|
+
|
|
221
|
+
Zaidan Chaudhary, who also publishes as Saad Chaudhary.
|
|
222
|
+
|
|
223
|
+
## License
|
|
224
|
+
|
|
225
|
+
MIT. See [LICENSE](LICENSE).
|
data/VERSION
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
0.5.1
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
[package]
|
|
2
|
+
name = "zlight_csv"
|
|
3
|
+
# Synced from ../../VERSION by extconf.rb — do not edit manually.
|
|
4
|
+
version = "0.4.0"
|
|
5
|
+
edition = "2021"
|
|
6
|
+
authors = ["Zaidan Chaudhary", "Saad Chaudhary"]
|
|
7
|
+
description = "High-performance CSV parser for Ruby, powered by Rust"
|
|
8
|
+
license = "MIT"
|
|
9
|
+
repository = "https://github.com/codebyisaad/zlight"
|
|
10
|
+
|
|
11
|
+
[lib]
|
|
12
|
+
crate-type = ["cdylib"]
|
|
13
|
+
path = "src/lib.rs"
|
|
14
|
+
|
|
15
|
+
[dependencies]
|
|
16
|
+
csv = "1"
|
|
17
|
+
magnus = "0.8"
|
|
18
|
+
# Depended on directly, only to turn on stable-api-compiled-fallback.
|
|
19
|
+
# rb-sys ships prebuilt bindings for Ruby versions it knows about and fails
|
|
20
|
+
# outright on ones it does not, which is what breaks the build on every new
|
|
21
|
+
# Ruby before rb-sys catches up. The fallback compiles the shim from the
|
|
22
|
+
# headers of whatever Ruby is present instead, so an unreleased Ruby builds.
|
|
23
|
+
rb-sys = { version = "0.9", features = ["stable-api-compiled-fallback"] }
|
|
24
|
+
thiserror = "1"
|
|
25
|
+
|
|
26
|
+
[profile.release]
|
|
27
|
+
lto = true
|
|
28
|
+
codegen-units = 1
|
|
29
|
+
opt-level = 3
|
|
30
|
+
strip = true
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "mkmf"
|
|
4
|
+
require "rb_sys/mkmf"
|
|
5
|
+
|
|
6
|
+
def sync_cargo_version!
|
|
7
|
+
version = File.read(File.expand_path("../../VERSION", __dir__), encoding: "UTF-8").strip
|
|
8
|
+
cargo_toml = File.join(__dir__, "Cargo.toml")
|
|
9
|
+
contents = File.read(cargo_toml, encoding: "UTF-8")
|
|
10
|
+
updated = contents.sub(/^version = ".*"$/, %(version = "#{version}"))
|
|
11
|
+
File.write(cargo_toml, updated) if contents != updated
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
sync_cargo_version!
|
|
15
|
+
|
|
16
|
+
create_rust_makefile("zlight_csv/zlight_csv")
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
use magnus::{encoding::Index, prelude::*, Ruby, Value};
|
|
2
|
+
|
|
3
|
+
/// Converts a byte slice field to a Ruby string value tagged with `encoding`.
|
|
4
|
+
///
|
|
5
|
+
/// Fields are labelled with the encoding of the original input string (or UTF-8
|
|
6
|
+
/// for file streams) so that parsed values match Ruby's stdlib CSV behavior and
|
|
7
|
+
/// round-trip cleanly through the writer.
|
|
8
|
+
#[inline(always)]
|
|
9
|
+
pub fn field_to_string(ruby: &Ruby, field: &[u8], encoding: Index) -> Value {
|
|
10
|
+
ruby.enc_str_new(field, encoding).as_value()
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/// Fast check if a byte slice looks like it could be a float.
|
|
14
|
+
/// Scans for '.', 'e', or 'E' in a single pass.
|
|
15
|
+
#[inline(always)]
|
|
16
|
+
fn looks_like_float(bytes: &[u8]) -> bool {
|
|
17
|
+
bytes.iter().any(|&b| b == b'.' || b == b'e' || b == b'E')
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/// Trim ASCII whitespace from both ends without allocation.
|
|
21
|
+
#[inline(always)]
|
|
22
|
+
fn trim_ascii(bytes: &[u8]) -> &[u8] {
|
|
23
|
+
let start = bytes.iter().position(|&b| !b.is_ascii_whitespace()).unwrap_or(bytes.len());
|
|
24
|
+
let end = bytes.iter().rposition(|&b| !b.is_ascii_whitespace()).map_or(start, |i| i + 1);
|
|
25
|
+
&bytes[start..end]
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/// Outcome of the integer fast path.
|
|
29
|
+
enum IntScan {
|
|
30
|
+
/// Parsed exactly into an i64.
|
|
31
|
+
Fits(i64),
|
|
32
|
+
/// A valid decimal integer, but too large in magnitude for an i64.
|
|
33
|
+
TooLarge,
|
|
34
|
+
/// Not a decimal integer at all.
|
|
35
|
+
NotAnInteger,
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/// Fast path: scan an optionally signed run of ASCII digits without full UTF-8
|
|
39
|
+
/// validation.
|
|
40
|
+
///
|
|
41
|
+
/// Values outside i64 are reported as `TooLarge` rather than rejected, so the
|
|
42
|
+
/// caller can fall back to an exact Ruby Integer instead of an inexact float.
|
|
43
|
+
#[inline(always)]
|
|
44
|
+
fn scan_int(bytes: &[u8]) -> IntScan {
|
|
45
|
+
if bytes.is_empty() {
|
|
46
|
+
return IntScan::NotAnInteger;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
let digits = match bytes[0] {
|
|
50
|
+
b'-' | b'+' => &bytes[1..],
|
|
51
|
+
_ => bytes,
|
|
52
|
+
};
|
|
53
|
+
let negative = bytes[0] == b'-';
|
|
54
|
+
|
|
55
|
+
if digits.is_empty() {
|
|
56
|
+
return IntScan::NotAnInteger;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
let mut magnitude: i64 = 0;
|
|
60
|
+
let mut overflowed = false;
|
|
61
|
+
|
|
62
|
+
for &b in digits {
|
|
63
|
+
if !b.is_ascii_digit() {
|
|
64
|
+
return IntScan::NotAnInteger;
|
|
65
|
+
}
|
|
66
|
+
if overflowed {
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
match magnitude
|
|
70
|
+
.checked_mul(10)
|
|
71
|
+
.and_then(|m| m.checked_add((b - b'0') as i64))
|
|
72
|
+
{
|
|
73
|
+
Some(next) => magnitude = next,
|
|
74
|
+
// Keep scanning to confirm the rest is digits too, but remember
|
|
75
|
+
// that the value needs Ruby's arbitrary-precision Integer.
|
|
76
|
+
None => overflowed = true,
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
if overflowed {
|
|
81
|
+
IntScan::TooLarge
|
|
82
|
+
} else if negative {
|
|
83
|
+
IntScan::Fits(-magnitude)
|
|
84
|
+
} else {
|
|
85
|
+
IntScan::Fits(magnitude)
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/// Builds an exact Ruby Integer from a decimal string too large for an i64.
|
|
90
|
+
///
|
|
91
|
+
/// Ruby Integers are arbitrary precision, so no digits are lost. `scan_int` has
|
|
92
|
+
/// already established that the slice is ASCII digits with an optional sign,
|
|
93
|
+
/// which is why `String#to_i` is safe here.
|
|
94
|
+
#[inline]
|
|
95
|
+
fn big_integer(ruby: &Ruby, digits: &[u8]) -> Option<Value> {
|
|
96
|
+
let text = std::str::from_utf8(digits).ok()?;
|
|
97
|
+
ruby.str_new(text).funcall("to_i", ()).ok()
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/// Attempts to parse a byte slice as a numeric value (integer or float).
|
|
101
|
+
#[inline]
|
|
102
|
+
fn try_parse_numeric(ruby: &Ruby, field: &[u8]) -> Option<Value> {
|
|
103
|
+
let trimmed = trim_ascii(field);
|
|
104
|
+
if trimmed.is_empty() {
|
|
105
|
+
return None;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// Fast path: try integer parsing without UTF-8 conversion.
|
|
109
|
+
if !looks_like_float(trimmed) {
|
|
110
|
+
match scan_int(trimmed) {
|
|
111
|
+
IntScan::Fits(i) => return Some(ruby.integer_from_i64(i).as_value()),
|
|
112
|
+
IntScan::TooLarge => return big_integer(ruby, trimmed),
|
|
113
|
+
IntScan::NotAnInteger => {}
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// Slow path: need UTF-8 for float parsing.
|
|
118
|
+
let text = std::str::from_utf8(trimmed).ok()?;
|
|
119
|
+
text.parse::<f64>()
|
|
120
|
+
.ok()
|
|
121
|
+
.map(|f| ruby.float_from_f64(f).as_value())
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/// Converts a byte slice field to an appropriate Ruby value.
|
|
125
|
+
/// If `convert_numeric` is true, attempts to parse as number first.
|
|
126
|
+
#[inline(always)]
|
|
127
|
+
pub fn field_to_value(ruby: &Ruby, field: &[u8], convert_numeric: bool, encoding: Index) -> Value {
|
|
128
|
+
if !convert_numeric {
|
|
129
|
+
return field_to_string(ruby, field, encoding);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
try_parse_numeric(ruby, field).unwrap_or_else(|| field_to_string(ruby, field, encoding))
|
|
133
|
+
}
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
use magnus::{prelude::*, Error as MagnusError, ExceptionClass, RModule, Ruby};
|
|
2
|
+
use thiserror::Error;
|
|
3
|
+
|
|
4
|
+
/// Enumerated error types for ZlightCsv operations.
|
|
5
|
+
#[derive(Debug, Error)]
|
|
6
|
+
pub enum ZlightError {
|
|
7
|
+
#[error("CSV parsing error: {0}")]
|
|
8
|
+
CsvParse(csv::Error),
|
|
9
|
+
|
|
10
|
+
#[error("Invalid UTF-8 encoding in header: {0}")]
|
|
11
|
+
InvalidHeaderEncoding(String),
|
|
12
|
+
|
|
13
|
+
#[error("Missing required argument: {0}")]
|
|
14
|
+
MissingArgument(&'static str),
|
|
15
|
+
|
|
16
|
+
#[error("Invalid option value for '{key}': expected {expected}, got {actual}")]
|
|
17
|
+
InvalidOption {
|
|
18
|
+
key: &'static str,
|
|
19
|
+
expected: &'static str,
|
|
20
|
+
actual: String,
|
|
21
|
+
},
|
|
22
|
+
|
|
23
|
+
#[error("IO error: {0}")]
|
|
24
|
+
Io(#[from] std::io::Error),
|
|
25
|
+
|
|
26
|
+
#[error("Stream reader has been closed")]
|
|
27
|
+
StreamClosed,
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
impl ZlightError {
|
|
31
|
+
/// Converts to a Ruby exception with the appropriate exception class.
|
|
32
|
+
///
|
|
33
|
+
/// Requires the Ruby handle, which is always available here: an error is
|
|
34
|
+
/// only converted on its way out of an extension method, and those run on
|
|
35
|
+
/// a Ruby thread holding the GVL. Building an exception needs a class, and
|
|
36
|
+
/// a class cannot be obtained without the handle, so there is no sensible
|
|
37
|
+
/// fallback for a VM we are not attached to.
|
|
38
|
+
#[inline]
|
|
39
|
+
pub fn to_magnus_error(&self) -> MagnusError {
|
|
40
|
+
let ruby = Ruby::get().expect("errors are only converted while a Ruby method is running");
|
|
41
|
+
MagnusError::new(self.exception_class(&ruby), self.to_string())
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/// Resolves the exception class to raise, preferring the gem's own
|
|
45
|
+
/// `ZLight::*` class over the built-in it descends from.
|
|
46
|
+
fn exception_class(&self, ruby: &Ruby) -> ExceptionClass {
|
|
47
|
+
self.zlight_class_name()
|
|
48
|
+
.and_then(|name| resolve_zlight_class(ruby, name))
|
|
49
|
+
.unwrap_or_else(|| self.builtin_class(ruby))
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/// Name of the `ZLight::*` class this error is documented to raise as, for
|
|
53
|
+
/// errors that have one.
|
|
54
|
+
fn zlight_class_name(&self) -> Option<&'static str> {
|
|
55
|
+
match self {
|
|
56
|
+
ZlightError::CsvParse(_) => Some("ParseError"),
|
|
57
|
+
ZlightError::InvalidHeaderEncoding(_) => Some("EncodingError"),
|
|
58
|
+
ZlightError::StreamClosed => Some("StreamClosedError"),
|
|
59
|
+
ZlightError::MissingArgument(_)
|
|
60
|
+
| ZlightError::InvalidOption { .. }
|
|
61
|
+
| ZlightError::Io(_) => None,
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/// Built-in class to raise when there is no `ZLight::*` equivalent, and the
|
|
66
|
+
/// fallback if one cannot be resolved.
|
|
67
|
+
fn builtin_class(&self, ruby: &Ruby) -> ExceptionClass {
|
|
68
|
+
match self {
|
|
69
|
+
ZlightError::CsvParse(_) => ruby.exception_runtime_error(),
|
|
70
|
+
ZlightError::InvalidHeaderEncoding(_) => ruby.exception_encoding_error(),
|
|
71
|
+
ZlightError::MissingArgument(_) | ZlightError::InvalidOption { .. } => {
|
|
72
|
+
ruby.exception_arg_error()
|
|
73
|
+
}
|
|
74
|
+
ZlightError::Io(_) | ZlightError::StreamClosed => ruby.exception_io_error(),
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/// Looks up `ZLight::<name>`.
|
|
80
|
+
///
|
|
81
|
+
/// The extension is loaded before lib/zlight_csv.rb defines these classes, so
|
|
82
|
+
/// the lookup happens when an error is raised rather than at init. It returns
|
|
83
|
+
/// `None` if the Ruby side was never loaded, leaving the caller to fall back to
|
|
84
|
+
/// a built-in class.
|
|
85
|
+
fn resolve_zlight_class(ruby: &Ruby, name: &'static str) -> Option<ExceptionClass> {
|
|
86
|
+
let zlight: RModule = ruby.class_object().const_get("ZLight").ok()?;
|
|
87
|
+
zlight.const_get(name).ok()
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
impl From<csv::Error> for ZlightError {
|
|
91
|
+
/// The csv crate wraps read failures in its own error type. A directory or
|
|
92
|
+
/// an unreadable file is an IO problem, not malformed CSV, so unwrap those
|
|
93
|
+
/// rather than reporting them as a parse failure.
|
|
94
|
+
fn from(err: csv::Error) -> Self {
|
|
95
|
+
if !err.is_io_error() {
|
|
96
|
+
return ZlightError::CsvParse(err);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
match err.into_kind() {
|
|
100
|
+
csv::ErrorKind::Io(io) => ZlightError::Io(io),
|
|
101
|
+
kind => unreachable!("is_io_error() guarantees ErrorKind::Io, got {kind:?}"),
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
impl From<ZlightError> for MagnusError {
|
|
107
|
+
#[inline]
|
|
108
|
+
fn from(err: ZlightError) -> Self {
|
|
109
|
+
err.to_magnus_error()
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
pub type Result<T> = std::result::Result<T, ZlightError>;
|
|
114
|
+
|
|
115
|
+
/// Converts a `csv` crate error into a Ruby exception.
|
|
116
|
+
///
|
|
117
|
+
/// `?` cannot bridge csv::Error to MagnusError on its own, because that would
|
|
118
|
+
/// need two chained `From` conversions.
|
|
119
|
+
#[inline]
|
|
120
|
+
pub fn csv_error(err: csv::Error) -> MagnusError {
|
|
121
|
+
ZlightError::from(err).into()
|
|
122
|
+
}
|