zlight_csv 0.4.0-aarch64-linux → 0.6.0-aarch64-linux

data/ext/zlight_csv/src/convert.rs

@@ -0,0 +1,203 @@
+//! Turning a raw CSV field into a Ruby value.
+//!
+//! # Where new field types go
+//!
+//! `Converter` is the only thing that decides what a `&[u8]` field becomes. To
+//! support a new kind of value, add a `Step` variant here and map the option to
+//! it in `options.rs`; the read paths call `Converter::apply` and nothing else.
+
+use magnus::{
+    encoding::Index, gc, prelude::*, Error as MagnusError, RString, Ruby, Value,
+};
+
+/// One stage of a conversion chain.
+#[derive(Debug, Clone)]
+pub enum Step {
+    /// Recognise integers and floats, leaving anything else a String.
+    Numeric,
+    /// Any Ruby object answering `call`, given the field and returning its
+    /// replacement.
+    Callable(Value),
+}
+
+/// How each field is turned into a Ruby value.
+///
+/// An empty chain means every field stays a String, which is the default.
+#[derive(Debug, Clone, Default)]
+pub struct Converter {
+    steps: Vec<Step>,
+}
+
+impl Converter {
+    pub fn new(steps: Vec<Step>) -> Self {
+        Self { steps }
+    }
+
+    /// Converts one field, running each step in turn.
+    ///
+    /// Matching Ruby's CSV, the chain stops as soon as a step returns anything
+    /// other than a String: a converter that has already produced a typed value
+    /// is not handed to the next converter.
+    pub fn apply(
+        &self,
+        ruby: &Ruby,
+        field: &[u8],
+        encoding: Index,
+    ) -> Result<Value, MagnusError> {
+        let mut steps = self.steps.iter();
+
+        // The first step reads the bytes directly, so `converters: :numeric`
+        // never builds a Ruby String for a field it is about to replace.
+        let Some(first) = steps.next() else {
+            return Ok(string(ruby, field, encoding));
+        };
+
+        let mut value = match first {
+            Step::Numeric => numeric(ruby, field).unwrap_or_else(|| string(ruby, field, encoding)),
+            Step::Callable(callable) => {
+                callable.funcall("call", (string(ruby, field, encoding),))?
+            }
+        };
+
+        for step in steps {
+            let Some(text) = RString::from_value(value) else {
+                break;
+            };
+
+            value = match step {
+                Step::Numeric => {
+                    // Copied because the conversion below allocates, and an
+                    // embedded Ruby string may move when it does.
+                    let bytes = unsafe { text.as_slice() }.to_vec();
+                    numeric(ruby, &bytes).unwrap_or(value)
+                }
+                Step::Callable(callable) => callable.funcall("call", (value,))?,
+            };
+        }
+
+        Ok(value)
+    }
+
+    /// Keeps any Ruby callables in the chain alive.
+    ///
+    /// A `StreamReader` outlives the call that created it, so a proc passed as
+    /// a converter is reachable only through this struct. Without marking it,
+    /// Ruby could collect it and the reader would call freed memory.
+    pub fn mark(&self, marker: &gc::Marker) {
+        for step in &self.steps {
+            if let Step::Callable(value) = step {
+                marker.mark(*value);
+            }
+        }
+    }
+}
+
+fn string(ruby: &Ruby, field: &[u8], encoding: Index) -> Value {
+    ruby.enc_str_new(field, encoding).as_value()
+}
+
+/// Scans for `.`, `e` or `E` in one pass to decide whether the integer fast
+/// path is worth trying.
+#[inline(always)]
+fn looks_like_float(bytes: &[u8]) -> bool {
+    bytes.iter().any(|&b| b == b'.' || b == b'e' || b == b'E')
+}
+
+#[inline(always)]
+fn trim_ascii(bytes: &[u8]) -> &[u8] {
+    let start = bytes
+        .iter()
+        .position(|&b| !b.is_ascii_whitespace())
+        .unwrap_or(bytes.len());
+    let end = bytes
+        .iter()
+        .rposition(|&b| !b.is_ascii_whitespace())
+        .map_or(start, |i| i + 1);
+    &bytes[start..end]
+}
+
+enum IntScan {
+    Fits(i64),
+    /// A valid decimal integer, but too large in magnitude for an i64.
+    TooLarge,
+    NotAnInteger,
+}
+
+/// Reads an optionally signed run of ASCII digits without validating UTF-8.
+///
+/// Overflow is reported rather than rejected so the caller can fall back to an
+/// exact Ruby Integer instead of an inexact float.
+#[inline(always)]
+fn scan_int(bytes: &[u8]) -> IntScan {
+    if bytes.is_empty() {
+        return IntScan::NotAnInteger;
+    }
+
+    let digits = match bytes[0] {
+        b'-' | b'+' => &bytes[1..],
+        _ => bytes,
+    };
+    let negative = bytes[0] == b'-';
+
+    if digits.is_empty() {
+        return IntScan::NotAnInteger;
+    }
+
+    let mut magnitude: i64 = 0;
+    let mut overflowed = false;
+
+    for &b in digits {
+        if !b.is_ascii_digit() {
+            return IntScan::NotAnInteger;
+        }
+        if overflowed {
+            continue;
+        }
+        match magnitude
+            .checked_mul(10)
+            .and_then(|m| m.checked_add((b - b'0') as i64))
+        {
+            Some(next) => magnitude = next,
+            None => overflowed = true,
+        }
+    }
+
+    if overflowed {
+        IntScan::TooLarge
+    } else if negative {
+        IntScan::Fits(-magnitude)
+    } else {
+        IntScan::Fits(magnitude)
+    }
+}
+
+/// Builds an exact Ruby Integer from digits too large for an i64.
+///
+/// `scan_int` has already established the slice is ASCII digits with an
+/// optional sign, which is what makes `String#to_i` safe here.
+fn big_integer(ruby: &Ruby, digits: &[u8]) -> Option<Value> {
+    let text = std::str::from_utf8(digits).ok()?;
+    ruby.str_new(text).funcall("to_i", ()).ok()
+}
+
+/// Returns the numeric value of a field, or `None` if it is not a number.
+fn numeric(ruby: &Ruby, field: &[u8]) -> Option<Value> {
+    let trimmed = trim_ascii(field);
+    if trimmed.is_empty() {
+        return None;
+    }
+
+    if !looks_like_float(trimmed) {
+        match scan_int(trimmed) {
+            IntScan::Fits(i) => return Some(ruby.integer_from_i64(i).as_value()),
+            IntScan::TooLarge => return big_integer(ruby, trimmed),
+            IntScan::NotAnInteger => {}
+        }
+    }
+
+    std::str::from_utf8(trimmed)
+        .ok()?
+        .parse::<f64>()
+        .ok()
+        .map(|f| ruby.float_from_f64(f).as_value())
+}

data/ext/zlight_csv/src/error.rs

@@ -0,0 +1,128 @@
+use magnus::{prelude::*, Error as MagnusError, ExceptionClass, RModule, Ruby};
+use thiserror::Error;
+
+#[derive(Debug, Error)]
+pub enum ZlightError {
+    #[error("CSV parsing error: {0}")]
+    CsvParse(csv::Error),
+
+    #[error("Invalid UTF-8 encoding in header: {0}")]
+    InvalidHeaderEncoding(String),
+
+    #[error("Missing required argument: {0}")]
+    MissingArgument(&'static str),
+
+    #[error("Invalid option value for '{key}': expected {expected}, got {actual}")]
+    InvalidOption {
+        key: &'static str,
+        expected: &'static str,
+        actual: String,
+    },
+
+    #[error("IO error: {0}")]
+    Io(#[from] std::io::Error),
+
+    #[error("Stream reader has been closed")]
+    StreamClosed,
+
+    #[error(
+        "Stream reader is already in use. Either a converter used the reader it \
+         was converting for, or another thread used it while a converter was \
+         running. Give each thread its own reader."
+    )]
+    ReaderBusy,
+}
+
+impl ZlightError {
+    /// Requires the Ruby handle, which is always available here: an error is
+    /// only converted on its way out of an extension method, and those run on
+    /// a Ruby thread holding the GVL. Building an exception needs a class, and
+    /// a class cannot be obtained without the handle, so there is no sensible
+    /// fallback for a VM we are not attached to.
+    #[inline]
+    pub fn to_magnus_error(&self) -> MagnusError {
+        let ruby = Ruby::get().expect("errors are only converted while a Ruby method is running");
+        MagnusError::new(self.exception_class(&ruby), self.to_string())
+    }
+
+    /// Resolves the exception class to raise, preferring the gem's own
+    /// `ZLight::*` class over the built-in it descends from.
+    fn exception_class(&self, ruby: &Ruby) -> ExceptionClass {
+        self.zlight_class_name()
+            .and_then(|name| resolve_zlight_class(ruby, name))
+            .unwrap_or_else(|| self.builtin_class(ruby))
+    }
+
+    /// Name of the `ZLight::*` class this error is documented to raise as, for
+    /// errors that have one.
+    fn zlight_class_name(&self) -> Option<&'static str> {
+        match self {
+            ZlightError::CsvParse(_) => Some("ParseError"),
+            ZlightError::InvalidHeaderEncoding(_) => Some("EncodingError"),
+            ZlightError::StreamClosed => Some("StreamClosedError"),
+            ZlightError::ReaderBusy => Some("Error"),
+            ZlightError::MissingArgument(_)
+            | ZlightError::InvalidOption { .. }
+            | ZlightError::Io(_) => None,
+        }
+    }
+
+    /// Built-in class to raise when there is no `ZLight::*` equivalent, and the
+    /// fallback if one cannot be resolved.
+    fn builtin_class(&self, ruby: &Ruby) -> ExceptionClass {
+        match self {
+            ZlightError::CsvParse(_) => ruby.exception_runtime_error(),
+            ZlightError::InvalidHeaderEncoding(_) => ruby.exception_encoding_error(),
+            ZlightError::MissingArgument(_) | ZlightError::InvalidOption { .. } => {
+                ruby.exception_arg_error()
+            }
+            ZlightError::Io(_) | ZlightError::StreamClosed => ruby.exception_io_error(),
+            ZlightError::ReaderBusy => ruby.exception_runtime_error(),
+        }
+    }
+}
+
+/// Looks up `ZLight::<name>`.
+///
+/// The extension is loaded before lib/zlight_csv.rb defines these classes, so
+/// the lookup happens when an error is raised rather than at init. It returns
+/// `None` if the Ruby side was never loaded, leaving the caller to fall back to
+/// a built-in class.
+fn resolve_zlight_class(ruby: &Ruby, name: &'static str) -> Option<ExceptionClass> {
+    let zlight: RModule = ruby.class_object().const_get("ZLight").ok()?;
+    zlight.const_get(name).ok()
+}
+
+impl From<csv::Error> for ZlightError {
+    /// The csv crate wraps read failures in its own error type. A directory or
+    /// an unreadable file is an IO problem, not malformed CSV, so unwrap those
+    /// rather than reporting them as a parse failure.
+    fn from(err: csv::Error) -> Self {
+        if !err.is_io_error() {
+            return ZlightError::CsvParse(err);
+        }
+
+        match err.into_kind() {
+            csv::ErrorKind::Io(io) => ZlightError::Io(io),
+            kind => unreachable!("is_io_error() guarantees ErrorKind::Io, got {kind:?}"),
+        }
+    }
+}
+
+impl From<ZlightError> for MagnusError {
+    #[inline]
+    fn from(err: ZlightError) -> Self {
+        err.to_magnus_error()
+    }
+}
+
+pub type Result<T> = std::result::Result<T, ZlightError>;
+
+/// Converts a `csv` crate error into a Ruby exception.
+///
+/// `?` cannot bridge csv::Error to MagnusError on its own, because that would
+/// need two chained `From` conversions.
+#[inline]
+pub fn csv_error(err: csv::Error) -> MagnusError {
+    ZlightError::from(err).into()
+}

data/ext/zlight_csv/src/lib.rs

@@ -0,0 +1,57 @@
+//! ZLight — a CSV parser and writer for Ruby, implemented in Rust.
+//!
+//! # Reading this codebase
+//!
+//! Every file has one job, and the layering is deliberate so that adding a
+//! feature touches one place rather than several:
+//!
+//! | Module        | Owns                                             |
+//! |---------------|--------------------------------------------------|
+//! | `lib.rs`      | what Ruby can call — the registration map, below |
+//! | `options`     | Ruby keyword arguments → typed settings          |
+//! | `convert`     | one field → one Ruby value                       |
+//! | `row`         | one record → one Ruby row (Hash or Array)        |
+//! | `reader`      | building the underlying `csv::Reader`            |
+//! | `read::eager` | `ZLight.parse`, whole input at once              |
+//! | `read::stream`| `ZLight::StreamReader`, one row at a time        |
+//! | `write`       | `ZLight.generate`                                |
+//! | `error`       | Rust errors → Ruby exception classes             |
+//!
+//! The two read paths sit on top of the same `reader`, `row` and `convert`,
+//! which is what stops them drifting apart. If you are adding something:
+//!
+//! * a new field type (dates, booleans, a Ruby proc) → `convert.rs`
+//! * a new result shape (a Row object, a struct) → `row.rs`
+//! * a new option → `options.rs`
+//! * a new Ruby method → a module under `read`/`write`, plus one line in
+//!   [`init`] so Ruby can see it
+//!
+//! See ARCHITECTURE.md for the same map written for someone who does not read
+//! Rust.
+
+mod convert;
+mod error;
+mod options;
+mod read;
+mod reader;
+mod row;
+mod write;
+
+use magnus::{function, prelude::*, Error, Ruby};
+
+/// Registers everything Ruby can call.
+///
+/// This function is the complete list of the extension's public surface; if a
+/// method is not named here, Ruby cannot reach it.
+#[magnus::init]
+fn init(ruby: &Ruby) -> Result<(), Error> {
+    let zlight = ruby.define_module("ZLight")?;
+
+    zlight.define_singleton_method("parse", function!(read::eager::parse, -1))?;
+    zlight.define_singleton_method("generate", function!(write::generate, -1))?;
+
+    // Defines ZLight::StreamReader along with .stream and .stream_file.
+    read::stream::init(ruby)?;
+
+    Ok(())
+}

data/ext/zlight_csv/src/options.rs

@@ -0,0 +1,143 @@
+use magnus::{
+    prelude::*,
+    scan_args::{get_kwargs, scan_args},
+    RArray, RHash, RString, Ruby, Symbol, Value,
+};
+
+use crate::convert::{Converter, Step};
+use crate::error::{Result, ZlightError};
+
+#[derive(Debug, Clone)]
+pub struct ParseOptions {
+    pub has_headers: bool,
+    /// How each field is turned into a Ruby value.
+    pub converter: Converter,
+    pub delimiter: u8,
+    pub quote: u8,
+    pub flexible: bool,
+}
+
+impl Default for ParseOptions {
+    #[inline]
+    fn default() -> Self {
+        Self {
+            has_headers: true,
+            converter: Converter::default(),
+            delimiter: b',',
+            quote: b'"',
+            flexible: true,
+        }
+    }
+}
+
+impl ParseOptions {
+    /// Scans a `(String, **options)` Ruby argument list.
+    ///
+    /// Every reader entry point takes the same shape, differing only in what
+    /// the leading string means. `argument` names it so that a missing
+    /// argument is reported as "path" or "csv_string" as appropriate.
+    pub fn scan(ruby: &Ruby, args: &[Value], argument: &'static str) -> Result<(RString, Self)> {
+        let parsed = scan_args::<(RString,), (), (), (), RHash, ()>(args)
+            .map_err(|_| ZlightError::MissingArgument(argument))?;
+
+        let (input,) = parsed.required;
+
+        Ok((input, Self::from_kwargs(ruby, parsed.keywords)?))
+    }
+
+    fn from_kwargs(ruby: &Ruby, kwargs: RHash) -> Result<Self> {
+        let kw = get_kwargs::<_, (), (Option<bool>, Option<Value>, Option<RString>, Option<RString>, Option<bool>), ()>(
+            kwargs,
+            &[],
+            &["headers", "converters", "col_sep", "quote_char", "flexible"],
+        )
+        .map_err(|e| ZlightError::InvalidOption {
+            key: "keywords",
+            expected: "valid keyword arguments",
+            actual: e.to_string(),
+        })?;
+
+        let (headers, converters, col_sep, quote_char, flexible) = kw.optional;
+
+        Ok(ParseOptions {
+            has_headers: headers.unwrap_or(true),
+            converter: converter_from(ruby, converters)?,
+            delimiter: single_byte(col_sep, "col_sep")?.unwrap_or(b','),
+            quote: single_byte(quote_char, "quote_char")?.unwrap_or(b'"'),
+            flexible: flexible.unwrap_or(true),
+        })
+    }
+}
+
+/// Validates that an optional separator/quote option is exactly one byte.
+///
+/// The underlying `csv` crate only supports single-byte delimiters and quotes,
+/// so a longer value would be truncated to its first byte and then silently
+/// parse or write the wrong thing. Reject it explicitly instead.
+pub fn single_byte(opt: Option<RString>, key: &'static str) -> Result<Option<u8>> {
+    let Some(string) = opt else { return Ok(None) };
+
+    // SAFETY: the slice is only read for its length and first byte before
+    // this function returns, with no Ruby allocation in between.
+    let bytes = unsafe { string.as_slice() };
+    match bytes.len() {
+        1 => Ok(Some(bytes[0])),
+        n => Err(ZlightError::InvalidOption {
+            key,
+            expected: "a single-byte string",
+            actual: format!("{}-byte string", n),
+        }),
+    }
+}
+
+/// Builds a `Converter` from the `converters:` option.
+///
+/// Accepts a single converter or an Array of them, where each is either the
+/// name of a built-in or any object answering `call`. Ruby's CSV reports an
+/// unusable converter as a NoMethodError from deep inside itself; an
+/// ArgumentError naming the problem is more use.
+fn converter_from(ruby: &Ruby, requested: Option<Value>) -> Result<Converter> {
+    let Some(value) = requested.filter(|value| !value.is_nil()) else {
+        return Ok(Converter::default());
+    };
+
+    let steps = match RArray::from_value(value) {
+        Some(array) => array
+            .into_iter()
+            .map(|element| step_from(ruby, element))
+            .collect::<Result<Vec<_>>>()?,
+        None => vec![step_from(ruby, value)?],
+    };
+
+    Ok(Converter::new(steps))
+}
+
+fn step_from(ruby: &Ruby, value: Value) -> Result<Step> {
+    if let Some(symbol) = Symbol::from_value(value) {
+        return match symbol.name() {
+            Ok(name) if name == "numeric" => Ok(Step::Numeric),
+            _ => Err(ZlightError::InvalidOption {
+                key: "converters",
+                expected: "a known converter name (:numeric)",
+                actual: format!(":{}", symbol.name().unwrap_or("?".into())),
+            }),
+        };
+    }
+
+    if responds_to_call(value) {
+        return Ok(Step::Callable(value));
+    }
+
+    let _ = ruby;
+    Err(ZlightError::InvalidOption {
+        key: "converters",
+        expected: "a converter name, or an object responding to #call",
+        actual: format!("{}", value.class()),
+    })
+}
+
+fn responds_to_call(value: Value) -> bool {
+    value
+        .funcall::<_, _, bool>("respond_to?", ("call",))
+        .unwrap_or(false)
+}

data/ext/zlight_csv/src/reader.rs

@@ -0,0 +1,26 @@
+//! Constructing the underlying CSV reader.
+//!
+//! Both read paths build their `csv::Reader` here so that a parse option can
+//! never apply to one and not the other. `ZLight.parse` reads from a byte
+//! slice borrowed from Ruby; `ZLight::StreamReader` reads from a file or an
+//! owned buffer. Only the source differs, which is why this is generic.
+
+use std::io::Read;
+
+use csv::{Reader, ReaderBuilder};
+
+use crate::options::ParseOptions;
+
+/// Reading in 64KB chunks rather than the crate default, which measurably
+/// helps on the large files this extension exists for.
+const BUFFER_CAPACITY: usize = 64 * 1024;
+
+pub fn build<R: Read>(source: R, options: &ParseOptions) -> Reader<R> {
+    ReaderBuilder::new()
+        .has_headers(options.has_headers)
+        .delimiter(options.delimiter)
+        .quote(options.quote)
+        .flexible(options.flexible)
+        .buffer_capacity(BUFFER_CAPACITY)
+        .from_reader(source)
+}

data/ext/zlight_csv/src/row.rs

@@ -0,0 +1,85 @@
+//! Turning a parsed record into the Ruby value a caller sees.
+//!
+//! # Where new output shapes go
+//!
+//! This is the plug point for result shape. Both read paths — the eager
+//! `ZLight.parse` and the streaming `ZLight::StreamReader` — build every row
+//! through `RowShape`, so they cannot drift apart. To return something new (a
+//! `ZLight::Row` that keeps duplicate headers, a struct, a columnar batch),
+//! add a variant here and select it in `options.rs`.
+
+use csv::ByteRecord;
+use magnus::{prelude::*, encoding::Index, Error as MagnusError, Ruby, StaticSymbol, Value};
+
+use crate::convert::Converter;
+use crate::error::ZlightError;
+
+/// The shape each parsed row takes on the Ruby side.
+pub enum RowShape {
+    /// A Hash keyed by the header row's symbols, resolved once up front.
+    Hashes(Vec<StaticSymbol>),
+    /// A positional Array, used when `headers: false`.
+    Arrays,
+}
+
+impl RowShape {
+    #[inline]
+    pub fn build(
+        &self,
+        ruby: &Ruby,
+        record: &ByteRecord,
+        converter: &Converter,
+        encoding: Index,
+    ) -> Result<Value, MagnusError> {
+        match self {
+            RowShape::Hashes(headers) => {
+                let hash = ruby.hash_new();
+
+                // zip stops at the shorter side, so a row with more fields
+                // than there are headers drops the extras, matching Ruby CSV.
+                for (header, field) in headers.iter().zip(record.iter()) {
+                    hash.aset(*header, converter.apply(ruby, field, encoding)?)?;
+                }
+
+                Ok(hash.as_value())
+            }
+            RowShape::Arrays => {
+                let array = ruby.ary_new_capa(record.len());
+
+                for field in record.iter() {
+                    array.push(converter.apply(ruby, field, encoding)?)?;
+                }
+
+                Ok(array.as_value())
+            }
+        }
+    }
+
+    pub fn headers(&self) -> Option<&[StaticSymbol]> {
+        match self {
+            RowShape::Hashes(headers) => Some(headers),
+            RowShape::Arrays => None,
+        }
+    }
+}
+
+/// Interns a header record as Ruby symbols.
+///
+/// Headers become symbols once per parse rather than per row, which is why
+/// `RowShape::Hashes` carries them. A header that is not valid UTF-8 cannot
+/// become a symbol, so it is reported rather than lossily replaced.
+pub fn header_symbols(
+    ruby: &Ruby,
+    headers: &ByteRecord,
+) -> Result<Vec<StaticSymbol>, ZlightError> {
+    headers
+        .iter()
+        .map(|header| {
+            std::str::from_utf8(header)
+                .map(|name| ruby.sym_new(name))
+                .map_err(|_| {
+                    ZlightError::InvalidHeaderEncoding(String::from_utf8_lossy(header).to_string())
+                })
+        })
+        .collect()
+}

data/lib/zlight_csv/errors.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module ZLight
+  # Base class for everything this library raises.
+  #
+  # Rescuing ZLight::Error catches any failure originating in the gem. The
+  # subclasses are raised from the Rust extension, which looks them up by name
+  # at raise time; see ext/zlight_csv/src/error.rs.
+  class Error < StandardError; end
+
+  # Raised when the CSV itself is malformed, such as a record whose field count
+  # does not match the header under `flexible: false`.
+  class ParseError < Error; end
+
+  # Raised when a header is not valid UTF-8 and so cannot become a Symbol.
+  class EncodingError < Error; end
+
+  # Raised when a row is read from a StreamReader that has been closed.
+  class StreamClosedError < Error; end
+end