zlight_csv 0.5.1 → 0.6.0

data/ext/zlight_csv/src/options.rs

@@ -1,22 +1,19 @@
 use magnus::{
+    prelude::*,
     scan_args::{get_kwargs, scan_args},
-    RHash, RString, Ruby, Symbol, Value,
+    RArray, RHash, RString, Ruby, Symbol, Value,
 };
 
+use crate::convert::{Converter, Step};
 use crate::error::{Result, ZlightError};
 
-/// Configuration options for CSV parsing.
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone)]
 pub struct ParseOptions {
-    /// Whether the CSV has a header row.
     pub has_headers: bool,
-    /// Whether to convert numeric strings to integers/floats.
-    pub convert_numeric: bool,
-    /// Field delimiter character (defaults to comma).
+    /// How each field is turned into a Ruby value.
+    pub converter: Converter,
     pub delimiter: u8,
-    /// Quote character (defaults to double quote).
     pub quote: u8,
-    /// Whether to allow flexible record lengths.
     pub flexible: bool,
 }
 
@@ -25,7 +22,7 @@ impl Default for ParseOptions {
     fn default() -> Self {
         Self {
             has_headers: true,
-            convert_numeric: false,
+            converter: Converter::default(),
             delimiter: b',',
             quote: b'"',
             flexible: true,
@@ -48,9 +45,8 @@ impl ParseOptions {
         Ok((input, Self::from_kwargs(ruby, parsed.keywords)?))
     }
 
-    /// Builds options from a keyword arguments hash, applying the defaults.
     fn from_kwargs(ruby: &Ruby, kwargs: RHash) -> Result<Self> {
-        let kw = get_kwargs::<_, (), (Option<bool>, Option<Symbol>, Option<RString>, Option<RString>, Option<bool>), ()>(
+        let kw = get_kwargs::<_, (), (Option<bool>, Option<Value>, Option<RString>, Option<RString>, Option<bool>), ()>(
             kwargs,
             &[],
             &["headers", "converters", "col_sep", "quote_char", "flexible"],
@@ -65,7 +61,7 @@ impl ParseOptions {
 
         Ok(ParseOptions {
             has_headers: headers.unwrap_or(true),
-            convert_numeric: converters.is_some_and(|sym| sym == ruby.to_symbol("numeric")),
+            converter: converter_from(ruby, converters)?,
             delimiter: single_byte(col_sep, "col_sep")?.unwrap_or(b','),
             quote: single_byte(quote_char, "quote_char")?.unwrap_or(b'"'),
             flexible: flexible.unwrap_or(true),
@@ -93,3 +89,55 @@ pub fn single_byte(opt: Option<RString>, key: &'static str) -> Result<Option<u8>
         }),
     }
 }
+
+/// Builds a `Converter` from the `converters:` option.
+///
+/// Accepts a single converter or an Array of them, where each is either the
+/// name of a built-in or any object answering `call`. Ruby's CSV reports an
+/// unusable converter as a NoMethodError from deep inside itself; an
+/// ArgumentError naming the problem is more use.
+fn converter_from(ruby: &Ruby, requested: Option<Value>) -> Result<Converter> {
+    let Some(value) = requested.filter(|value| !value.is_nil()) else {
+        return Ok(Converter::default());
+    };
+
+    let steps = match RArray::from_value(value) {
+        Some(array) => array
+            .into_iter()
+            .map(|element| step_from(ruby, element))
+            .collect::<Result<Vec<_>>>()?,
+        None => vec![step_from(ruby, value)?],
+    };
+
+    Ok(Converter::new(steps))
+}
+
+fn step_from(ruby: &Ruby, value: Value) -> Result<Step> {
+    if let Some(symbol) = Symbol::from_value(value) {
+        return match symbol.name() {
+            Ok(name) if name == "numeric" => Ok(Step::Numeric),
+            _ => Err(ZlightError::InvalidOption {
+                key: "converters",
+                expected: "a known converter name (:numeric)",
+                actual: format!(":{}", symbol.name().unwrap_or("?".into())),
+            }),
+        };
+    }
+
+    if responds_to_call(value) {
+        return Ok(Step::Callable(value));
+    }
+
+    let _ = ruby;
+    Err(ZlightError::InvalidOption {
+        key: "converters",
+        expected: "a converter name, or an object responding to #call",
+        actual: format!("{}", value.class()),
+    })
+}
+
+fn responds_to_call(value: Value) -> bool {
+    value
+        .funcall::<_, _, bool>("respond_to?", ("call",))
+        .unwrap_or(false)
+}

data/ext/zlight_csv/src/reader.rs

@@ -0,0 +1,26 @@
+//! Constructing the underlying CSV reader.
+//!
+//! Both read paths build their `csv::Reader` here so that a parse option can
+//! never apply to one and not the other. `ZLight.parse` reads from a byte
+//! slice borrowed from Ruby; `ZLight::StreamReader` reads from a file or an
+//! owned buffer. Only the source differs, which is why this is generic.
+
+use std::io::Read;
+
+use csv::{Reader, ReaderBuilder};
+
+use crate::options::ParseOptions;
+
+/// Reading in 64KB chunks rather than the crate default, which measurably
+/// helps on the large files this extension exists for.
+const BUFFER_CAPACITY: usize = 64 * 1024;
+
+pub fn build<R: Read>(source: R, options: &ParseOptions) -> Reader<R> {
+    ReaderBuilder::new()
+        .has_headers(options.has_headers)
+        .delimiter(options.delimiter)
+        .quote(options.quote)
+        .flexible(options.flexible)
+        .buffer_capacity(BUFFER_CAPACITY)
+        .from_reader(source)
+}

data/ext/zlight_csv/src/row.rs

@@ -0,0 +1,85 @@
+//! Turning a parsed record into the Ruby value a caller sees.
+//!
+//! # Where new output shapes go
+//!
+//! This is the plug point for result shape. Both read paths — the eager
+//! `ZLight.parse` and the streaming `ZLight::StreamReader` — build every row
+//! through `RowShape`, so they cannot drift apart. To return something new (a
+//! `ZLight::Row` that keeps duplicate headers, a struct, a columnar batch),
+//! add a variant here and select it in `options.rs`.
+
+use csv::ByteRecord;
+use magnus::{prelude::*, encoding::Index, Error as MagnusError, Ruby, StaticSymbol, Value};
+
+use crate::convert::Converter;
+use crate::error::ZlightError;
+
+/// The shape each parsed row takes on the Ruby side.
+pub enum RowShape {
+    /// A Hash keyed by the header row's symbols, resolved once up front.
+    Hashes(Vec<StaticSymbol>),
+    /// A positional Array, used when `headers: false`.
+    Arrays,
+}
+
+impl RowShape {
+    #[inline]
+    pub fn build(
+        &self,
+        ruby: &Ruby,
+        record: &ByteRecord,
+        converter: &Converter,
+        encoding: Index,
+    ) -> Result<Value, MagnusError> {
+        match self {
+            RowShape::Hashes(headers) => {
+                let hash = ruby.hash_new();
+
+                // zip stops at the shorter side, so a row with more fields
+                // than there are headers drops the extras, matching Ruby CSV.
+                for (header, field) in headers.iter().zip(record.iter()) {
+                    hash.aset(*header, converter.apply(ruby, field, encoding)?)?;
+                }
+
+                Ok(hash.as_value())
+            }
+            RowShape::Arrays => {
+                let array = ruby.ary_new_capa(record.len());
+
+                for field in record.iter() {
+                    array.push(converter.apply(ruby, field, encoding)?)?;
+                }
+
+                Ok(array.as_value())
+            }
+        }
+    }
+
+    pub fn headers(&self) -> Option<&[StaticSymbol]> {
+        match self {
+            RowShape::Hashes(headers) => Some(headers),
+            RowShape::Arrays => None,
+        }
+    }
+}
+
+/// Interns a header record as Ruby symbols.
+///
+/// Headers become symbols once per parse rather than per row, which is why
+/// `RowShape::Hashes` carries them. A header that is not valid UTF-8 cannot
+/// become a symbol, so it is reported rather than lossily replaced.
+pub fn header_symbols(
+    ruby: &Ruby,
+    headers: &ByteRecord,
+) -> Result<Vec<StaticSymbol>, ZlightError> {
+    headers
+        .iter()
+        .map(|header| {
+            std::str::from_utf8(header)
+                .map(|name| ruby.sym_new(name))
+                .map_err(|_| {
+                    ZlightError::InvalidHeaderEncoding(String::from_utf8_lossy(header).to_string())
+                })
+        })
+        .collect()
+}

data/lib/zlight_csv/errors.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module ZLight
+  # Base class for everything this library raises.
+  #
+  # Rescuing ZLight::Error catches any failure originating in the gem. The
+  # subclasses are raised from the Rust extension, which looks them up by name
+  # at raise time; see ext/zlight_csv/src/error.rs.
+  class Error < StandardError; end
+
+  # Raised when the CSV itself is malformed, such as a record whose field count
+  # does not match the header under `flexible: false`.
+  class ParseError < Error; end
+
+  # Raised when a header is not valid UTF-8 and so cannot become a Symbol.
+  class EncodingError < Error; end
+
+  # Raised when a row is read from a StreamReader that has been closed.
+  class StreamClosedError < Error; end
+end

data/lib/zlight_csv/native.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module ZLight
+  # Loads the compiled extension.
+  #
+  # A precompiled gem ships one shared library per Ruby minor version, laid out
+  # as lib/zlight_csv/<major.minor>/zlight_csv.<so|bundle|dll>, because a C
+  # extension built against one Ruby's ABI cannot be loaded by another. A gem
+  # built from source has a single library at lib/zlight_csv instead, so try
+  # the versioned path first and fall back.
+  module Native
+    def self.load!
+      require_relative "#{RUBY_VERSION[/\d+\.\d+/]}/zlight_csv"
+    rescue LoadError
+      require_relative 'zlight_csv'
+    end
+  end
+end
+
+ZLight::Native.load!