zlight_csv 0.4.0-x86_64-linux → 0.5.1-x86_64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6b96428d44c1acb33466a227063e7318cdfdd4233464f533a832accac4e0abf4
-  data.tar.gz: 633c2c530b5ddfd75dc1e006ba345e253c1f4f41e503f0094575421e6d92375f
+  metadata.gz: f2f876b5dca405c9cee64f60d93c27652f33ffb4a6b3e8c6eff6e7d1af1b3bf0
+  data.tar.gz: 58f1e880ec5e544ac0a64dcfb9f025829bcc90d4f7fcde3acec987a404c47014
 SHA512:
-  metadata.gz: 89b7b97b16cbae21d906f990cf5440ebb3438e403e0a20f02dda8a13a5c302e868de5e5d649e97c362623dbde050c225eea28ad20c424a511ed883d3fcc22d7b
-  data.tar.gz: f164fd92db98090061b241cb81da26fb9fda646d244d6d63afac992f5bfcc66f059aefb1e499883ce446d995f3e5597af3c927b8882043462f695b3e116495c1
+  metadata.gz: d088be4df8df53496eb25bc21640237c12d1db7d7df6b3a188d5624e5af1436f59295331d2407cddfde9da6f4059be9b01b7959d0ecd69e0272784d53f9c191d
+  data.tar.gz: d400dc067e043d99fc94d7ea2e3e72da9b6b9b1de5b6bed9b3e150e255fb7122136aad0ad576d77694f2be40c39d22db60d4a8e4064d0dda3798dc59ca164ba7

data/CHANGELOG.md

@@ -7,22 +7,122 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
-## [0.1.0] - 2024-XX-XX
+## [0.5.1] - 2026-09-12
+
+### Fixed
+
+- The extension now builds against a Ruby that rb-sys has no prebuilt bindings
+  for, via rb-sys `stable-api-compiled-fallback`, rather than failing outright.
+  This is the mechanism that breaks the build on every new Ruby release until
+  rb-sys catches up
+- Windows CI could not build the extension at all: the runner paired a MinGW
+  Ruby with an MSVC Rust toolchain. Ruby and Rust are now set up together so
+  their ABIs match, and Windows is verified across Ruby 3.1-4.0
+- Specs that asserted Unix-only error messages and paths now pass on Windows
+- Dropped an unreachable credentials check from the release workflow: the
+  credentials action fails first, so the check never ran
+
+### Note
+
+- 0.5.0 was tagged but never reached RubyGems: its release run built every gem
+  and then failed to publish, because trusted publishing still named the
+  project's former GitHub account. 0.5.1 is the first release carrying the
+  0.5.0 changes below.
+
+## [0.5.0] - 2026-09-12
+
+### Added
+
+- Ruby 4.0 support
+- `docker/test-matrix.sh`, which builds and tests the extension against every
+  supported Ruby in containers
+- Windows, Ruby 4.0 and a `ruby-head` early-warning job in CI, plus weekly
+  scheduled runs and Dependabot
+
+### Changed
+
+- Upgraded magnus from 0.6 to 0.8. magnus 0.6 read a field of Ruby's
+  RTypedData struct that Ruby 4.0 removed, so the extension could not compile
+  against it at all
+- Minimum Ruby is now 3.1. Ruby 3.0 is end-of-life and no longer supported by
+  magnus
+- Repository URLs now point at `codebyisaad/zlight`, which is where the code
+  is hosted
+
+### Fixed
+
+- The source gem shipped no extension and could not be compiled, so any Ruby
+  or platform without a prebuilt binary installed a gem that failed to load
+- `ZLight::ParseError`, `ZLight::EncodingError` and `ZLight::StreamClosedError`
+  were documented but never raised
+- Integers outside the i64 range were silently converted to lossy Floats
+- `col_sep` and `quote_char` longer than one byte were silently truncated when
+  reading, while writing rejected them
+- Releases no longer yank every previously published version
+
+
+## [0.4.0] - 2026-06-06
+
+### Added
+
+- `ZLight.generate` — build a CSV string from an array of hashes or an array
+  of arrays, with `headers`, `col_sep`, `quote_char` and `force_quotes` options
+- `ZLight.write` — generate a CSV and write it to a file in one call
+
+### Changed
+
+- Parsed fields are tagged with the encoding of the input string, matching
+  Ruby's stdlib CSV, so the parser and writer round-trip without
+  `force_encoding`
+
+## [0.3.0] - 2026-05-21
+
+### Changed
+
+- Release pipeline updates
+
+## [0.2.5] - 2026-05-21
+
+### Added
+
+- `ZLight.stream` and `ZLight.stream_file` — read rows one at a time without
+  loading the whole input into memory
+- `ZLight.open` — streaming with a block, closing the reader automatically
+- `ZLight::StreamReader`, including `Enumerable`, so `lazy`, `select`, `find`
+  and friends work over a stream
+
+### Fixed
+
+- Packaging and gemspec metadata corrections across 0.2.1–0.2.5
+
+## [0.2.0] - 2026-05-21
+
+### Added
+
+- Benchmark suite comparing ZLight against Ruby's stdlib CSV
+- Edge case coverage for quoting, encodings and flexible records
+
+## [0.1.2] - 2026-05-19
 
 ### Added
 
 - Initial release
-- `Zlight::Csv.parse` - Parse CSV strings with optional headers and numeric conversion
-- `Zlight::Csv.read` - Read and parse CSV files
-- `Zlight::Csv.foreach` - Iterate over CSV rows
-- `Zlight.parse` and `Zlight.read` convenience methods
-- Support for custom delimiters (`col_sep`)
-- Support for custom quote characters (`quote_char`)
-- Flexible record length support
-- Enum-based error handling with descriptive messages
-- Cross-platform native gem builds
+- `ZLight.parse` — parse CSV strings, with `headers`, `converters`, `col_sep`,
+  `quote_char` and `flexible` options
+- `ZLight.read` — read and parse a CSV file
+- `ZLight.foreach` — iterate over parsed rows
+- Cross-platform precompiled native gems
 
 ### Performance
 
-- Up to 10x faster than Ruby's standard CSV library
-- Efficient memory usage through Rust's zero-copy parsing
+- Substantially faster than Ruby's stdlib CSV; see the README for measured
+  figures
+
+[Unreleased]: https://github.com/codebyisaad/zlight/compare/v0.5.1...HEAD
+[0.5.1]: https://github.com/codebyisaad/zlight/compare/v0.5.0...v0.5.1
+[0.5.0]: https://github.com/codebyisaad/zlight/compare/v0.4.0...v0.5.0
+[0.4.0]: https://github.com/codebyisaad/zlight/compare/v0.3.0...v0.4.0
+[0.3.0]: https://github.com/codebyisaad/zlight/compare/v0.2.5...v0.3.0
+[0.2.5]: https://github.com/codebyisaad/zlight/compare/v0.2.0...v0.2.5
+[0.2.0]: https://github.com/codebyisaad/zlight/compare/v0.1.2...v0.2.0
+[0.1.2]: https://github.com/codebyisaad/zlight/releases/tag/v0.1.2

data/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2024 Zaidan Chaudhary
+Copyright (c) 2024 Zaidan Chaudhary (also known as Saad Chaudhary)
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -7,7 +7,8 @@ A fast CSV parser for Ruby, powered by Rust.
 
 ## Why ZLight?
 
-Ruby's built-in CSV library is slow. ZLight parses CSV files **up to 30x faster** by using Rust under the hood.
+Ruby's built-in CSV library is slow. ZLight parses CSV files **up to 40x faster** by using Rust under the hood.
+The gap is widest on small inputs and narrows as they grow; the measurements below show the range.
 
 ### Benchmark Results
 
@@ -156,11 +157,55 @@ CSV.parse(data, headers: true, header_converters: :symbol, converters: :numeric)
 ZLight.parse(data, converters: :numeric)
 ```
 
+### Differences from stdlib CSV
+
+ZLight is not a complete reimplementation of `CSV`. The differences below are
+deliberate, and are the ones most likely to matter when migrating.
+
+**API shape**
+
+- `ZLight.foreach` takes a **CSV string**, while `CSV.foreach` takes a **file
+  path**. Use `ZLight.open` to iterate a file.
+- `ZLight.foreach` parses the whole input before yielding. For genuinely lazy
+  iteration use `ZLight.stream` or `ZLight.open`.
+- `ZLight::StreamReader` is single-pass. It includes `Enumerable`, but each
+  row is consumed as it is read, so a second pass yields nothing and there is
+  no rewind. Call `ZLight.stream` again to re-read.
+
+**Parsing**
+
+- Duplicate headers collapse. Rows are `Hash`es, so `"a,a"` keeps only the
+  last `:a` column; `CSV` keeps both.
+- Fields beyond the header count are dropped rather than collected.
+- Headers become symbols always, equivalent to `header_converters: :symbol`.
+
+**`converters: :numeric`**
+
+| Input        | ZLight            | Ruby CSV   |
+|--------------|-------------------|------------|
+| `""`         | `""`              | `nil`      |
+| `"0x10"`     | `"0x10"`          | `16`       |
+| `"1_000"`    | `"1_000"`         | `1000`     |
+| `"Infinity"` | `Float::INFINITY` | `"Infinity"` |
+| `"NaN"`      | `Float::NAN`      | `"NaN"`    |
+
+Integers of any size are exact, as in `CSV`.
+
+**Writing**
+
+- `ZLight.generate` takes its column order from the keys of the **first**
+  hash. Keys that appear only in later rows are not written; keys missing
+  from a later row are written as empty fields.
+
 ## Requirements
 
-- Ruby 3.0+
+- Ruby 3.1 or newer, including Ruby 4.0
 - Linux (x86_64, aarch64), macOS (Intel, Apple Silicon), or Windows (x64)
 
+Every supported Ruby is built and tested on each change; see
+`docker/test-matrix.sh` to run that matrix yourself. Rubies without a
+precompiled binary install from the source gem and need a Rust toolchain.
+
 ## Roadmap
 
 - [x] Streaming/lazy parsing for large files
@@ -169,8 +214,12 @@ ZLight.parse(data, converters: :numeric)
 
 ## Contributing
 
-Bug reports and pull requests are welcome on [GitHub](https://github.com/zaidanch/zlight).
+Bug reports and pull requests are welcome on [GitHub](https://github.com/codebyisaad/zlight).
+
+## Author
+
+Zaidan Chaudhary, who also publishes as Saad Chaudhary.
 
 ## License
 
-MIT
+MIT. See [LICENSE](LICENSE).

data/VERSION

@@ -1 +1 @@
-0.4.0
+0.5.1

data/ext/zlight_csv/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+name = "zlight_csv"
+# Synced from ../../VERSION by extconf.rb — do not edit manually.
+version = "0.5.1"
+edition = "2021"
+authors = ["Zaidan Chaudhary", "Saad Chaudhary"]
+description = "High-performance CSV parser for Ruby, powered by Rust"
+license = "MIT"
+repository = "https://github.com/codebyisaad/zlight"
+
+[lib]
+crate-type = ["cdylib"]
+path = "src/lib.rs"
+
+[dependencies]
+csv = "1"
+magnus = "0.8"
+# Depended on directly, only to turn on stable-api-compiled-fallback.
+# rb-sys ships prebuilt bindings for Ruby versions it knows about and fails
+# outright on ones it does not, which is what breaks the build on every new
+# Ruby before rb-sys catches up. The fallback compiles the shim from the
+# headers of whatever Ruby is present instead, so an unreleased Ruby builds.
+rb-sys = { version = "0.9", features = ["stable-api-compiled-fallback"] }
+thiserror = "1"
+
+[profile.release]
+lto = true
+codegen-units = 1
+opt-level = 3
+strip = true

data/ext/zlight_csv/extconf.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require "mkmf"
+require "rb_sys/mkmf"
+
+def sync_cargo_version!
+  version = File.read(File.expand_path("../../VERSION", __dir__), encoding: "UTF-8").strip
+  cargo_toml = File.join(__dir__, "Cargo.toml")
+  contents = File.read(cargo_toml, encoding: "UTF-8")
+  updated = contents.sub(/^version = ".*"$/, %(version = "#{version}"))
+  File.write(cargo_toml, updated) if contents != updated
+end
+
+sync_cargo_version!
+
+create_rust_makefile("zlight_csv/zlight_csv")

data/ext/zlight_csv/src/converter.rs

@@ -0,0 +1,133 @@
+use magnus::{encoding::Index, prelude::*, Ruby, Value};
+
+/// Converts a byte slice field to a Ruby string value tagged with `encoding`.
+///
+/// Fields are labelled with the encoding of the original input string (or UTF-8
+/// for file streams) so that parsed values match Ruby's stdlib CSV behavior and
+/// round-trip cleanly through the writer.
+#[inline(always)]
+pub fn field_to_string(ruby: &Ruby, field: &[u8], encoding: Index) -> Value {
+    ruby.enc_str_new(field, encoding).as_value()
+}
+
+/// Fast check if a byte slice looks like it could be a float.
+/// Scans for '.', 'e', or 'E' in a single pass.
+#[inline(always)]
+fn looks_like_float(bytes: &[u8]) -> bool {
+    bytes.iter().any(|&b| b == b'.' || b == b'e' || b == b'E')
+}
+
+/// Trim ASCII whitespace from both ends without allocation.
+#[inline(always)]
+fn trim_ascii(bytes: &[u8]) -> &[u8] {
+    let start = bytes.iter().position(|&b| !b.is_ascii_whitespace()).unwrap_or(bytes.len());
+    let end = bytes.iter().rposition(|&b| !b.is_ascii_whitespace()).map_or(start, |i| i + 1);
+    &bytes[start..end]
+}
+
+/// Outcome of the integer fast path.
+enum IntScan {
+    /// Parsed exactly into an i64.
+    Fits(i64),
+    /// A valid decimal integer, but too large in magnitude for an i64.
+    TooLarge,
+    /// Not a decimal integer at all.
+    NotAnInteger,
+}
+
+/// Fast path: scan an optionally signed run of ASCII digits without full UTF-8
+/// validation.
+///
+/// Values outside i64 are reported as `TooLarge` rather than rejected, so the
+/// caller can fall back to an exact Ruby Integer instead of an inexact float.
+#[inline(always)]
+fn scan_int(bytes: &[u8]) -> IntScan {
+    if bytes.is_empty() {
+        return IntScan::NotAnInteger;
+    }
+
+    let digits = match bytes[0] {
+        b'-' | b'+' => &bytes[1..],
+        _ => bytes,
+    };
+    let negative = bytes[0] == b'-';
+
+    if digits.is_empty() {
+        return IntScan::NotAnInteger;
+    }
+
+    let mut magnitude: i64 = 0;
+    let mut overflowed = false;
+
+    for &b in digits {
+        if !b.is_ascii_digit() {
+            return IntScan::NotAnInteger;
+        }
+        if overflowed {
+            continue;
+        }
+        match magnitude
+            .checked_mul(10)
+            .and_then(|m| m.checked_add((b - b'0') as i64))
+        {
+            Some(next) => magnitude = next,
+            // Keep scanning to confirm the rest is digits too, but remember
+            // that the value needs Ruby's arbitrary-precision Integer.
+            None => overflowed = true,
+        }
+    }
+
+    if overflowed {
+        IntScan::TooLarge
+    } else if negative {
+        IntScan::Fits(-magnitude)
+    } else {
+        IntScan::Fits(magnitude)
+    }
+}
+
+/// Builds an exact Ruby Integer from a decimal string too large for an i64.
+///
+/// Ruby Integers are arbitrary precision, so no digits are lost. `scan_int` has
+/// already established that the slice is ASCII digits with an optional sign,
+/// which is why `String#to_i` is safe here.
+#[inline]
+fn big_integer(ruby: &Ruby, digits: &[u8]) -> Option<Value> {
+    let text = std::str::from_utf8(digits).ok()?;
+    ruby.str_new(text).funcall("to_i", ()).ok()
+}
+
+/// Attempts to parse a byte slice as a numeric value (integer or float).
+#[inline]
+fn try_parse_numeric(ruby: &Ruby, field: &[u8]) -> Option<Value> {
+    let trimmed = trim_ascii(field);
+    if trimmed.is_empty() {
+        return None;
+    }
+
+    // Fast path: try integer parsing without UTF-8 conversion.
+    if !looks_like_float(trimmed) {
+        match scan_int(trimmed) {
+            IntScan::Fits(i) => return Some(ruby.integer_from_i64(i).as_value()),
+            IntScan::TooLarge => return big_integer(ruby, trimmed),
+            IntScan::NotAnInteger => {}
+        }
+    }
+
+    // Slow path: need UTF-8 for float parsing.
+    let text = std::str::from_utf8(trimmed).ok()?;
+    text.parse::<f64>()
+        .ok()
+        .map(|f| ruby.float_from_f64(f).as_value())
+}
+
+/// Converts a byte slice field to an appropriate Ruby value.
+/// If `convert_numeric` is true, attempts to parse as number first.
+#[inline(always)]
+pub fn field_to_value(ruby: &Ruby, field: &[u8], convert_numeric: bool, encoding: Index) -> Value {
+    if !convert_numeric {
+        return field_to_string(ruby, field, encoding);
+    }
+
+    try_parse_numeric(ruby, field).unwrap_or_else(|| field_to_string(ruby, field, encoding))
+}

data/ext/zlight_csv/src/error.rs

@@ -0,0 +1,122 @@
+use magnus::{prelude::*, Error as MagnusError, ExceptionClass, RModule, Ruby};
+use thiserror::Error;
+
+/// Enumerated error types for ZlightCsv operations.
+#[derive(Debug, Error)]
+pub enum ZlightError {
+    #[error("CSV parsing error: {0}")]
+    CsvParse(csv::Error),
+
+    #[error("Invalid UTF-8 encoding in header: {0}")]
+    InvalidHeaderEncoding(String),
+
+    #[error("Missing required argument: {0}")]
+    MissingArgument(&'static str),
+
+    #[error("Invalid option value for '{key}': expected {expected}, got {actual}")]
+    InvalidOption {
+        key: &'static str,
+        expected: &'static str,
+        actual: String,
+    },
+
+    #[error("IO error: {0}")]
+    Io(#[from] std::io::Error),
+
+    #[error("Stream reader has been closed")]
+    StreamClosed,
+}
+
+impl ZlightError {
+    /// Converts to a Ruby exception with the appropriate exception class.
+    ///
+    /// Requires the Ruby handle, which is always available here: an error is
+    /// only converted on its way out of an extension method, and those run on
+    /// a Ruby thread holding the GVL. Building an exception needs a class, and
+    /// a class cannot be obtained without the handle, so there is no sensible
+    /// fallback for a VM we are not attached to.
+    #[inline]
+    pub fn to_magnus_error(&self) -> MagnusError {
+        let ruby = Ruby::get().expect("errors are only converted while a Ruby method is running");
+        MagnusError::new(self.exception_class(&ruby), self.to_string())
+    }
+
+    /// Resolves the exception class to raise, preferring the gem's own
+    /// `ZLight::*` class over the built-in it descends from.
+    fn exception_class(&self, ruby: &Ruby) -> ExceptionClass {
+        self.zlight_class_name()
+            .and_then(|name| resolve_zlight_class(ruby, name))
+            .unwrap_or_else(|| self.builtin_class(ruby))
+    }
+
+    /// Name of the `ZLight::*` class this error is documented to raise as, for
+    /// errors that have one.
+    fn zlight_class_name(&self) -> Option<&'static str> {
+        match self {
+            ZlightError::CsvParse(_) => Some("ParseError"),
+            ZlightError::InvalidHeaderEncoding(_) => Some("EncodingError"),
+            ZlightError::StreamClosed => Some("StreamClosedError"),
+            ZlightError::MissingArgument(_)
+            | ZlightError::InvalidOption { .. }
+            | ZlightError::Io(_) => None,
+        }
+    }
+
+    /// Built-in class to raise when there is no `ZLight::*` equivalent, and the
+    /// fallback if one cannot be resolved.
+    fn builtin_class(&self, ruby: &Ruby) -> ExceptionClass {
+        match self {
+            ZlightError::CsvParse(_) => ruby.exception_runtime_error(),
+            ZlightError::InvalidHeaderEncoding(_) => ruby.exception_encoding_error(),
+            ZlightError::MissingArgument(_) | ZlightError::InvalidOption { .. } => {
+                ruby.exception_arg_error()
+            }
+            ZlightError::Io(_) | ZlightError::StreamClosed => ruby.exception_io_error(),
+        }
+    }
+}
+
+/// Looks up `ZLight::<name>`.
+///
+/// The extension is loaded before lib/zlight_csv.rb defines these classes, so
+/// the lookup happens when an error is raised rather than at init. It returns
+/// `None` if the Ruby side was never loaded, leaving the caller to fall back to
+/// a built-in class.
+fn resolve_zlight_class(ruby: &Ruby, name: &'static str) -> Option<ExceptionClass> {
+    let zlight: RModule = ruby.class_object().const_get("ZLight").ok()?;
+    zlight.const_get(name).ok()
+}
+
+impl From<csv::Error> for ZlightError {
+    /// The csv crate wraps read failures in its own error type. A directory or
+    /// an unreadable file is an IO problem, not malformed CSV, so unwrap those
+    /// rather than reporting them as a parse failure.
+    fn from(err: csv::Error) -> Self {
+        if !err.is_io_error() {
+            return ZlightError::CsvParse(err);
+        }
+
+        match err.into_kind() {
+            csv::ErrorKind::Io(io) => ZlightError::Io(io),
+            kind => unreachable!("is_io_error() guarantees ErrorKind::Io, got {kind:?}"),
+        }
+    }
+}
+
+impl From<ZlightError> for MagnusError {
+    #[inline]
+    fn from(err: ZlightError) -> Self {
+        err.to_magnus_error()
+    }
+}
+
+pub type Result<T> = std::result::Result<T, ZlightError>;
+
+/// Converts a `csv` crate error into a Ruby exception.
+///
+/// `?` cannot bridge csv::Error to MagnusError on its own, because that would
+/// need two chained `From` conversions.
+#[inline]
+pub fn csv_error(err: csv::Error) -> MagnusError {
+    ZlightError::from(err).into()
+}

data/ext/zlight_csv/src/lib.rs

@@ -0,0 +1,55 @@
+mod converter;
+mod error;
+mod options;
+mod parser;
+mod stream;
+mod writer;
+
+use magnus::{function, prelude::*, Error, RArray, Ruby, Value};
+
+use options::ParseOptions;
+use parser::{build_reader, parse_as_arrays, parse_as_hashes};
+
+fn parse(ruby: &Ruby, args: &[Value]) -> Result<RArray, Error> {
+    let (input, options) = ParseOptions::scan(ruby, args, "csv_string")?;
+    // Capture the input's encoding so parsed fields are tagged consistently
+    // (matching Ruby's stdlib CSV) rather than defaulting to BINARY.
+    let encoding = input.enc_get();
+
+    // SAFETY: `as_slice` borrows Ruby's own buffer, so Ruby must neither free
+    // nor move the string while `bytes` is live. It cannot be freed: `input`
+    // came from the argument list and stays reachable for the whole call.
+    // Parsing it without copying is the point of this extension; a copy here
+    // would double peak memory on large inputs.
+    //
+    // `input` is deliberately kept alive past the parse below, rather than
+    // ending its borrow at this line, so that Ruby's conservative stack scan
+    // still sees the string while the parser holds a pointer into it.
+    let bytes = unsafe { input.as_slice() };
+
+    let mut reader = build_reader(bytes, &options);
+
+    let result = if options.has_headers {
+        parse_as_hashes(ruby, &mut reader, options.convert_numeric, encoding)
+    } else {
+        parse_as_arrays(ruby, &mut reader, options.convert_numeric, encoding)
+    };
+
+    // Keeps `input` live until parsing has finished; see the note above.
+    std::hint::black_box(input);
+
+    result
+}
+
+/// Initializes the Ruby extension.
+#[magnus::init]
+fn init(ruby: &Ruby) -> Result<(), Error> {
+    let module = ruby.define_module("ZLight")?;
+    module.define_singleton_method("parse", function!(parse, -1))?;
+    module.define_singleton_method("generate", function!(writer::generate, -1))?;
+
+    // Initialize streaming support
+    stream::init(ruby)?;
+
+    Ok(())
+}