zlight_csv 0.4.0-aarch64-linux → 0.5.1-aarch64-linux

data/ext/zlight_csv/src/options.rs

@@ -0,0 +1,95 @@
+use magnus::{
+    scan_args::{get_kwargs, scan_args},
+    RHash, RString, Ruby, Symbol, Value,
+};
+
+use crate::error::{Result, ZlightError};
+
+/// Configuration options for CSV parsing.
+#[derive(Debug, Clone, Copy)]
+pub struct ParseOptions {
+    /// Whether the CSV has a header row.
+    pub has_headers: bool,
+    /// Whether to convert numeric strings to integers/floats.
+    pub convert_numeric: bool,
+    /// Field delimiter character (defaults to comma).
+    pub delimiter: u8,
+    /// Quote character (defaults to double quote).
+    pub quote: u8,
+    /// Whether to allow flexible record lengths.
+    pub flexible: bool,
+}
+
+impl Default for ParseOptions {
+    #[inline]
+    fn default() -> Self {
+        Self {
+            has_headers: true,
+            convert_numeric: false,
+            delimiter: b',',
+            quote: b'"',
+            flexible: true,
+        }
+    }
+}
+
+impl ParseOptions {
+    /// Scans a `(String, **options)` Ruby argument list.
+    ///
+    /// Every reader entry point takes the same shape, differing only in what
+    /// the leading string means. `argument` names it so that a missing
+    /// argument is reported as "path" or "csv_string" as appropriate.
+    pub fn scan(ruby: &Ruby, args: &[Value], argument: &'static str) -> Result<(RString, Self)> {
+        let parsed = scan_args::<(RString,), (), (), (), RHash, ()>(args)
+            .map_err(|_| ZlightError::MissingArgument(argument))?;
+
+        let (input,) = parsed.required;
+
+        Ok((input, Self::from_kwargs(ruby, parsed.keywords)?))
+    }
+
+    /// Builds options from a keyword arguments hash, applying the defaults.
+    fn from_kwargs(ruby: &Ruby, kwargs: RHash) -> Result<Self> {
+        let kw = get_kwargs::<_, (), (Option<bool>, Option<Symbol>, Option<RString>, Option<RString>, Option<bool>), ()>(
+            kwargs,
+            &[],
+            &["headers", "converters", "col_sep", "quote_char", "flexible"],
+        )
+        .map_err(|e| ZlightError::InvalidOption {
+            key: "keywords",
+            expected: "valid keyword arguments",
+            actual: e.to_string(),
+        })?;
+
+        let (headers, converters, col_sep, quote_char, flexible) = kw.optional;
+
+        Ok(ParseOptions {
+            has_headers: headers.unwrap_or(true),
+            convert_numeric: converters.is_some_and(|sym| sym == ruby.to_symbol("numeric")),
+            delimiter: single_byte(col_sep, "col_sep")?.unwrap_or(b','),
+            quote: single_byte(quote_char, "quote_char")?.unwrap_or(b'"'),
+            flexible: flexible.unwrap_or(true),
+        })
+    }
+}
+
+/// Validates that an optional separator/quote option is exactly one byte.
+///
+/// The underlying `csv` crate only supports single-byte delimiters and quotes,
+/// so a longer value would be truncated to its first byte and then silently
+/// parse or write the wrong thing. Reject it explicitly instead.
+pub fn single_byte(opt: Option<RString>, key: &'static str) -> Result<Option<u8>> {
+    let Some(string) = opt else { return Ok(None) };
+
+    // SAFETY: the slice is only read for its length and first byte before
+    // this function returns, with no Ruby allocation in between.
+    let bytes = unsafe { string.as_slice() };
+    match bytes.len() {
+        1 => Ok(Some(bytes[0])),
+        n => Err(ZlightError::InvalidOption {
+            key,
+            expected: "a single-byte string",
+            actual: format!("{}-byte string", n),
+        }),
+    }
+}

data/ext/zlight_csv/src/parser.rs

@@ -0,0 +1,77 @@
+use csv::{ByteRecord, Reader, ReaderBuilder};
+use magnus::{encoding::Index, Error as MagnusError, RArray, Ruby};
+
+use crate::converter::field_to_value;
+use crate::error::{csv_error, ZlightError};
+use crate::options::ParseOptions;
+
+/// Parses CSV data into an array of arrays (no headers mode).
+pub fn parse_as_arrays(
+    ruby: &Ruby,
+    reader: &mut Reader<&[u8]>,
+    convert_numeric: bool,
+    encoding: Index,
+) -> Result<RArray, MagnusError> {
+    let result = ruby.ary_new();
+    let mut record = ByteRecord::new();
+
+    while reader.read_byte_record(&mut record).map_err(csv_error)? {
+        let row = ruby.ary_new_capa(record.len());
+
+        for field in record.iter() {
+            row.push(field_to_value(ruby, field, convert_numeric, encoding))?;
+        }
+        result.push(row)?;
+    }
+
+    Ok(result)
+}
+
+/// Parses CSV data into an array of hashes (with headers mode).
+pub fn parse_as_hashes(
+    ruby: &Ruby,
+    reader: &mut Reader<&[u8]>,
+    convert_numeric: bool,
+    encoding: Index,
+) -> Result<RArray, MagnusError> {
+    // Pre-compute header symbols once
+    let byte_headers = reader.byte_headers().map_err(csv_error)?;
+
+    let headers: Vec<_> = byte_headers
+        .iter()
+        .map(|h| {
+            std::str::from_utf8(h).map(|s| ruby.sym_new(s)).map_err(|_| {
+                ZlightError::InvalidHeaderEncoding(String::from_utf8_lossy(h).to_string())
+            })
+        })
+        .collect::<Result<Vec<_>, _>>()?;
+
+    let result = ruby.ary_new();
+    let mut record = ByteRecord::new();
+
+    while reader.read_byte_record(&mut record).map_err(csv_error)? {
+        let hash = ruby.hash_new();
+
+        // Zip stops at the shorter side, so a row with more fields than there
+        // are headers drops the extras, matching Ruby CSV.
+        for (header, field) in headers.iter().zip(record.iter()) {
+            hash.aset(*header, field_to_value(ruby, field, convert_numeric, encoding))?;
+        }
+
+        result.push(hash)?;
+    }
+
+    Ok(result)
+}
+
+/// Builds a CSV reader with the given options.
+#[inline]
+pub fn build_reader<'a>(data: &'a [u8], options: &ParseOptions) -> Reader<&'a [u8]> {
+    ReaderBuilder::new()
+        .has_headers(options.has_headers)
+        .delimiter(options.delimiter)
+        .quote(options.quote)
+        .flexible(options.flexible)
+        .buffer_capacity(64 * 1024) // 64KB buffer for better I/O
+        .from_reader(data)
+}

data/ext/zlight_csv/src/stream.rs

@@ -0,0 +1,410 @@
+//! Streaming CSV parser for lazy/incremental parsing.
+//!
+//! Provides `StreamReader` which allows reading CSV rows one at a time
+//! without loading the entire file into memory.
+
+use std::cell::RefCell;
+use std::fs::File;
+use std::io::{BufReader, Cursor, Read};
+
+use csv::{ByteRecord, Reader, ReaderBuilder};
+use magnus::{
+    encoding::Index, function, method,
+    prelude::*,
+    typed_data::Obj,
+    DataTypeFunctions, Error as MagnusError, RArray, RString, Ruby, StaticSymbol, TypedData, Value,
+};
+
+use crate::converter::field_to_value;
+use crate::error::{csv_error, ZlightError};
+use crate::options::ParseOptions;
+
+// ============================================================================
+// Type Aliases
+// ============================================================================
+
+type BoxedReader = Reader<Box<dyn Read + Send>>;
+
+// ============================================================================
+// Reader State
+// ============================================================================
+
+/// State of the stream reader.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum ReaderState {
+    /// Reader has not been initialized yet (lazy init).
+    Uninitialized,
+    /// Reader is active and can read rows.
+    Active,
+    /// Reader has been exhausted (no more rows).
+    Exhausted,
+    /// Reader has been closed.
+    Closed,
+}
+
+// ============================================================================
+// StreamReader
+// ============================================================================
+
+/// Internal mutable state for the stream reader.
+struct StreamReaderInner {
+    /// The CSV reader (lazily initialized).
+    reader: Option<BoxedReader>,
+    /// Pending data source before initialization.
+    pending_data: Option<Vec<u8>>,
+    /// Opened file awaiting initialization.
+    pending_file: Option<File>,
+    /// Pre-computed header symbols.
+    headers: Option<Vec<StaticSymbol>>,
+    /// Reusable record buffer.
+    record: ByteRecord,
+    /// Current state.
+    state: ReaderState,
+}
+
+/// A streaming CSV reader that yields rows one at a time.
+///
+/// Exposed to Ruby as `ZLight::StreamReader`. Implements Ruby's Enumerable
+/// interface for seamless integration with Ruby iteration patterns.
+///
+/// # Example (Ruby)
+///
+/// ```ruby
+/// reader = ZLight.stream_file("large_file.csv")
+/// reader.each do |row|
+///   puts row[:name]
+/// end
+/// reader.close
+/// ```
+#[derive(TypedData)]
+#[magnus(class = "ZLight::StreamReader", free_immediately)]
+pub struct StreamReader {
+    /// Mutable internal state wrapped in RefCell for interior mutability.
+    inner: RefCell<StreamReaderInner>,
+    /// Parsing options (immutable after creation).
+    options: ParseOptions,
+    /// Encoding used to tag parsed field strings.
+    encoding: Index,
+}
+
+// SAFETY: magnus requires `TypedData: Send`, but this struct holds a RefCell
+// and Ruby values (StaticSymbol), neither of which is Send on its own.
+//
+// It is sound because a StreamReader is only ever reached through the Ruby
+// methods defined at the bottom of this file, and those run holding the GVL,
+// so no two threads touch one concurrently. Nothing here may be moved to a
+// Ractor or handed to a Rust thread.
+unsafe impl Send for StreamReader {}
+
+// The only Ruby values held are StaticSymbols, which Ruby never collects,
+// so there is nothing for the GC to mark. Add a `mark` implementation (and
+// the `mark` attribute above) if a collectable value is ever stored here.
+impl DataTypeFunctions for StreamReader {}
+
+impl StreamReader {
+    /// Creates a new StreamReader from in-memory data.
+    pub fn from_string(data: Vec<u8>, options: ParseOptions, encoding: Index) -> Self {
+        Self {
+            inner: RefCell::new(StreamReaderInner {
+                reader: None,
+                pending_data: Some(data),
+                pending_file: None,
+                headers: None,
+                record: ByteRecord::new(),
+                state: ReaderState::Uninitialized,
+            }),
+            options,
+            encoding,
+        }
+    }
+
+    /// Creates a new StreamReader from an already-opened file.
+    pub fn from_file(file: File, options: ParseOptions, encoding: Index) -> Self {
+        Self {
+            inner: RefCell::new(StreamReaderInner {
+                reader: None,
+                pending_data: None,
+                pending_file: Some(file),
+                headers: None,
+                record: ByteRecord::new(),
+                state: ReaderState::Uninitialized,
+            }),
+            options,
+            encoding,
+        }
+    }
+
+    /// Ensures the CSV reader is initialized.
+    fn ensure_initialized(&self, ruby: &Ruby) -> Result<(), ZlightError> {
+        let mut inner = self.inner.borrow_mut();
+
+        if inner.state != ReaderState::Uninitialized {
+            return Ok(());
+        }
+
+        // Create the reader from pending data or file
+        let boxed_reader: Box<dyn Read + Send> = if let Some(data) = inner.pending_data.take() {
+            Box::new(Cursor::new(data))
+        } else if let Some(file) = inner.pending_file.take() {
+            Box::new(BufReader::with_capacity(64 * 1024, file))
+        } else {
+            return Err(ZlightError::StreamClosed);
+        };
+
+        let mut csv_reader = ReaderBuilder::new()
+            .has_headers(self.options.has_headers)
+            .delimiter(self.options.delimiter)
+            .quote(self.options.quote)
+            .flexible(self.options.flexible)
+            .buffer_capacity(64 * 1024)
+            .from_reader(boxed_reader);
+
+        // Extract headers if enabled
+        if self.options.has_headers {
+            let byte_headers = csv_reader.byte_headers()?;
+            let header_symbols: Vec<StaticSymbol> = byte_headers
+                .iter()
+                .map(|h| {
+                    std::str::from_utf8(h)
+                        .map(|name| ruby.sym_new(name))
+                        .map_err(|_| {
+                            ZlightError::InvalidHeaderEncoding(
+                                String::from_utf8_lossy(h).to_string(),
+                            )
+                        })
+                })
+                .collect::<Result<Vec<_>, _>>()?;
+            inner.headers = Some(header_symbols);
+        }
+
+        inner.reader = Some(csv_reader);
+        inner.state = ReaderState::Active;
+
+        Ok(())
+    }
+
+    /// Reads the next row from the CSV.
+    ///
+    /// Returns `Ok(Some(row))` for each row, `Ok(None)` when exhausted.
+    pub fn next_row(&self, ruby: &Ruby) -> Result<Option<Value>, MagnusError> {
+        self.ensure_initialized(ruby)?;
+
+        // Read into the reusable buffer, releasing the mutable borrow before
+        // any Ruby object is allocated below.
+        {
+            let mut guard = self.inner.borrow_mut();
+            // Reborrow so the reader and the record buffer can be borrowed as
+            // the separate fields they are.
+            let inner = &mut *guard;
+
+            match inner.state {
+                ReaderState::Closed => return Err(ZlightError::StreamClosed.into()),
+                ReaderState::Exhausted => return Ok(None),
+                ReaderState::Uninitialized => {
+                    unreachable!("ensure_initialized leaves no Uninitialized state")
+                }
+                ReaderState::Active => {}
+            }
+
+            let reader = inner.reader.as_mut().ok_or(ZlightError::StreamClosed)?;
+
+            if !reader.read_byte_record(&mut inner.record).map_err(csv_error)? {
+                inner.state = ReaderState::Exhausted;
+                return Ok(None);
+            }
+        }
+
+        // Now build the row value with a fresh borrow
+        let inner = self.inner.borrow();
+        let convert = self.options.convert_numeric;
+
+        let row = if self.options.has_headers {
+            let headers = inner.headers.as_ref().ok_or(ZlightError::StreamClosed)?;
+            let hash = ruby.hash_new();
+
+            // Zip stops at the shorter side, so a row with more fields than
+            // there are headers drops the extras, matching Ruby CSV.
+            for (header, field) in headers.iter().zip(inner.record.iter()) {
+                hash.aset(*header, field_to_value(ruby, field, convert, self.encoding))?;
+            }
+
+            hash.as_value()
+        } else {
+            let arr = ruby.ary_new_capa(inner.record.len());
+
+            for field in inner.record.iter() {
+                arr.push(field_to_value(ruby, field, convert, self.encoding))?;
+            }
+
+            arr.as_value()
+        };
+
+        Ok(Some(row))
+    }
+
+    /// Returns the headers as a Ruby array of symbols.
+    pub fn headers(&self, ruby: &Ruby) -> Result<Option<RArray>, MagnusError> {
+        self.ensure_initialized(ruby)?;
+
+        let inner = self.inner.borrow();
+
+        match &inner.headers {
+            Some(headers) => {
+                let arr = ruby.ary_new_capa(headers.len());
+                for sym in headers {
+                    arr.push(*sym)?;
+                }
+                Ok(Some(arr))
+            }
+            None => Ok(None),
+        }
+    }
+
+    /// Closes the reader and releases all resources.
+    pub fn close(&self) {
+        let mut inner = self.inner.borrow_mut();
+        inner.reader = None;
+        inner.headers = None;
+        inner.pending_data = None;
+        // Dropping the file closes its descriptor.
+        inner.pending_file = None;
+        inner.state = ReaderState::Closed;
+    }
+
+    /// Returns whether the reader has been closed.
+    pub fn is_closed(&self) -> bool {
+        self.inner.borrow().state == ReaderState::Closed
+    }
+
+    /// Returns whether there are more rows to read.
+    pub fn is_exhausted(&self) -> bool {
+        let state = self.inner.borrow().state;
+        matches!(state, ReaderState::Exhausted | ReaderState::Closed)
+    }
+}
+
+// ============================================================================
+// Ruby Method Wrappers
+// ============================================================================
+
+/// Ruby: ZLight.stream(csv_string, **options) -> StreamReader
+fn rb_stream_string(ruby: &Ruby, args: &[Value]) -> Result<Obj<StreamReader>, MagnusError> {
+    let (input, options) = ParseOptions::scan(ruby, args, "csv_string")?;
+    let encoding = input.enc_get();
+    // SAFETY: copied immediately, before anything can allocate. The stream
+    // outlives this call, so it owns its bytes rather than borrowing Ruby's.
+    let bytes = unsafe { input.as_slice().to_vec() };
+
+    let reader = StreamReader::from_string(bytes, options, encoding);
+    Ok(ruby.obj_wrap(reader))
+}
+
+/// Ruby: ZLight.stream_file(path, **options) -> StreamReader
+fn rb_stream_file(ruby: &Ruby, args: &[Value]) -> Result<Obj<StreamReader>, MagnusError> {
+    let (path_arg, options) = ParseOptions::scan(ruby, args, "path")?;
+    let path_str = path_to_string(path_arg)?;
+
+    // Open now rather than on the first read. Checking for existence and then
+    // opening later left a window for the file to change in between, and it
+    // reported a missing file from stream_file while an unreadable one only
+    // surfaced rows later. Opening here reports both from the same place, and
+    // the message comes from the OS instead of being reconstructed.
+    let file = File::open(&path_str).map_err(|e| {
+        ZlightError::Io(std::io::Error::new(e.kind(), format!("{} - {}", e, path_str)))
+    })?;
+
+    // File streams have no source Ruby string; default to UTF-8 to match
+    // `ZLight.read`, which reads files as UTF-8.
+    let reader = StreamReader::from_file(file, options, ruby.utf8_encindex());
+    Ok(ruby.obj_wrap(reader))
+}
+
+/// Converts the path argument to an owned Rust string.
+fn path_to_string(path: RString) -> Result<String, ZlightError> {
+    // SAFETY: the slice is copied into an owned String before returning, with
+    // no Ruby allocation in between.
+    let bytes = unsafe { path.as_slice() };
+    std::str::from_utf8(bytes)
+        .map(str::to_owned)
+        .map_err(|_| ZlightError::InvalidOption {
+            key: "path",
+            expected: "valid UTF-8 string",
+            actual: "invalid encoding".to_string(),
+        })
+}
+
+/// Ruby: reader.next_row -> Hash/Array or nil
+fn rb_next_row(ruby: &Ruby, rb_self: Obj<StreamReader>) -> Result<Value, MagnusError> {
+    Ok(rb_self.next_row(ruby)?.unwrap_or_else(|| ruby.qnil().as_value()))
+}
+
+/// Ruby: reader.each { |row| ... } -> self
+/// Ruby: reader.each -> Enumerator
+fn rb_each(ruby: &Ruby, rb_self: Obj<StreamReader>) -> Result<Value, MagnusError> {
+    if !ruby.block_given() {
+        // Return Enumerator for lazy iteration
+        return rb_self.funcall("to_enum", ("each",));
+    }
+
+    let block = ruby.block_proc()?;
+
+    while let Some(row) = rb_self.next_row(ruby)? {
+        block.call::<_, Value>((row,))?;
+    }
+
+    Ok(rb_self.as_value())
+}
+
+/// Ruby: reader.headers -> Array<Symbol> or nil
+fn rb_headers(ruby: &Ruby, rb_self: Obj<StreamReader>) -> Result<Value, MagnusError> {
+    Ok(rb_self
+        .headers(ruby)?
+        .map_or_else(|| ruby.qnil().as_value(), |arr| arr.as_value()))
+}
+
+/// Ruby: reader.close -> nil
+fn rb_close(ruby: &Ruby, rb_self: Obj<StreamReader>) -> Value {
+    rb_self.close();
+    ruby.qnil().as_value()
+}
+
+/// Ruby: reader.closed? -> true/false
+fn rb_is_closed(rb_self: Obj<StreamReader>) -> bool {
+    rb_self.is_closed()
+}
+
+/// Ruby: reader.eof? -> true/false
+fn rb_is_eof(rb_self: Obj<StreamReader>) -> bool {
+    rb_self.is_exhausted()
+}
+
+// ============================================================================
+// Module Initialization
+// ============================================================================
+
+/// Initializes the StreamReader class and streaming methods.
+pub fn init(ruby: &Ruby) -> Result<(), MagnusError> {
+    // Get the ZLight module
+    let zlight = ruby.define_module("ZLight")?;
+
+    // Define StreamReader class
+    let stream_reader_class = zlight.define_class("StreamReader", ruby.class_object())?;
+
+    // Include Enumerable for map, select, reduce, etc.
+    let enumerable: magnus::RModule = ruby.class_object().funcall("const_get", ("Enumerable",))?;
+    stream_reader_class.include_module(enumerable)?;
+
+    // Instance methods
+    stream_reader_class.define_method("next_row", method!(rb_next_row, 0))?;
+    stream_reader_class.define_method("each", method!(rb_each, 0))?;
+    stream_reader_class.define_method("headers", method!(rb_headers, 0))?;
+    stream_reader_class.define_method("close", method!(rb_close, 0))?;
+    stream_reader_class.define_method("closed?", method!(rb_is_closed, 0))?;
+    stream_reader_class.define_method("eof?", method!(rb_is_eof, 0))?;
+
+    // Module-level factory methods
+    zlight.define_singleton_method("stream", function!(rb_stream_string, -1))?;
+    zlight.define_singleton_method("stream_file", function!(rb_stream_file, -1))?;
+
+    Ok(())
+}