osv 0.3.13 → 0.3.15

data/ext/osv/src/csv/record_reader.rs

@@ -0,0 +1,172 @@
+use super::parser::RecordParser;
+use super::{header_cache::StringCache, ruby_reader::SeekableRead};
+use magnus::{Error, Ruby};
+use std::io::BufReader;
+use std::{borrow::Cow, io::Read, thread};
+
+pub(crate) const READ_BUFFER_SIZE: usize = 16384;
+
+pub struct RecordReader<'a, T: RecordParser<'a>> {
+    inner: ReaderImpl<'a, T>,
+}
+
+#[allow(clippy::large_enum_variant)]
+enum ReaderImpl<'a, T: RecordParser<'a>> {
+    SingleThreaded {
+        reader: csv::Reader<BufReader<Box<dyn SeekableRead>>>,
+        headers: Vec<&'static str>,
+        null_string: Option<String>,
+        flexible_default: Option<Cow<'a, str>>,
+        string_record: csv::StringRecord,
+    },
+    MultiThreaded {
+        headers: Vec<&'static str>,
+        receiver: kanal::Receiver<T::Output>,
+        handle: Option<thread::JoinHandle<()>>,
+    },
+}
+
+impl<'a, T: RecordParser<'a>> RecordReader<'a, T> {
+    #[inline]
+    pub(crate) fn get_headers(
+        ruby: &Ruby,
+        reader: &mut csv::Reader<impl Read>,
+        has_headers: bool,
+    ) -> Result<Vec<String>, Error> {
+        let first_row = reader.headers().map_err(|e| {
+            Error::new(
+                ruby.exception_runtime_error(),
+                format!("Failed to read headers: {e}"),
+            )
+        })?;
+
+        let mut headers = Vec::with_capacity(first_row.len());
+        if has_headers {
+            headers.extend(first_row.iter().map(String::from));
+        } else {
+            headers.extend((0..first_row.len()).map(|i| format!("c{i}")));
+        }
+        Ok(headers)
+    }
+
+    pub(crate) fn new_single_threaded(
+        reader: csv::Reader<BufReader<Box<dyn SeekableRead>>>,
+        headers: Vec<&'static str>,
+        null_string: Option<String>,
+        flexible_default: Option<&'a str>,
+    ) -> Self {
+        let headers_len = headers.len();
+        Self {
+            inner: ReaderImpl::SingleThreaded {
+                reader,
+                headers,
+                null_string,
+                flexible_default: flexible_default.map(Cow::Borrowed),
+                string_record: csv::StringRecord::with_capacity(READ_BUFFER_SIZE, headers_len),
+            },
+        }
+    }
+}
+
+impl<T: RecordParser<'static> + Send> RecordReader<'static, T> {
+    pub(crate) fn new_multi_threaded(
+        mut reader: csv::Reader<Box<dyn Read + Send + 'static>>,
+        headers: Vec<&'static str>,
+        buffer_size: usize,
+        null_string: Option<String>,
+        flexible_default: Option<&'static str>,
+    ) -> Self {
+        let (sender, receiver) = kanal::bounded(buffer_size);
+        let headers_for_thread = headers.clone();
+
+        let handle = thread::spawn(move || {
+            let mut record =
+                csv::StringRecord::with_capacity(READ_BUFFER_SIZE, headers_for_thread.len());
+            while let Ok(true) = reader.read_record(&mut record) {
+                let row = T::parse(
+                    &headers_for_thread,
+                    &record,
+                    null_string.as_deref(),
+                    flexible_default.map(Cow::Borrowed),
+                );
+                if sender.send(row).is_err() {
+                    break;
+                }
+            }
+        });
+
+        Self {
+            inner: ReaderImpl::MultiThreaded {
+                headers,
+                receiver,
+                handle: Some(handle),
+            },
+        }
+    }
+}
+
+impl<'a, T: RecordParser<'a>> Iterator for RecordReader<'a, T> {
+    type Item = T::Output;
+
+    #[inline]
+    fn next(&mut self) -> Option<Self::Item> {
+        match &mut self.inner {
+            ReaderImpl::MultiThreaded {
+                receiver, handle, ..
+            } => match receiver.recv() {
+                Ok(record) => Some(record),
+                Err(_) => {
+                    if let Some(handle) = handle.take() {
+                        let _ = handle.join();
+                    }
+                    None
+                }
+            },
+            ReaderImpl::SingleThreaded {
+                reader,
+                headers,
+                null_string,
+                flexible_default,
+                ref mut string_record,
+            } => match reader.read_record(string_record) {
+                Ok(true) => Some(T::parse(
+                    headers,
+                    string_record,
+                    null_string.as_deref(),
+                    flexible_default.clone(),
+                )),
+                Ok(false) => None,
+                Err(_e) => None,
+            },
+        }
+    }
+
+    #[inline]
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        // We can't know the exact size without reading the whole file
+        (0, None)
+    }
+}
+
+impl<'a, T: RecordParser<'a>> Drop for RecordReader<'a, T> {
+    #[inline]
+    fn drop(&mut self) {
+        match &mut self.inner {
+            ReaderImpl::MultiThreaded {
+                receiver,
+                handle,
+                headers,
+                ..
+            } => {
+                receiver.close();
+                if let Some(handle) = handle.take() {
+                    let _ = handle.join();
+                }
+                let _ = StringCache::clear(headers);
+            }
+            ReaderImpl::SingleThreaded { headers, .. } => {
+                let _ = StringCache::clear(headers);
+            }
+        }
+    }
+}

data/ext/osv/src/csv/ruby_integration.rs

@@ -0,0 +1,30 @@
+use std::{fs::File, io, mem::ManuallyDrop};
+
+pub struct ForgottenFileHandle(pub ManuallyDrop<File>);
+
+impl std::io::Read for ForgottenFileHandle {
+    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+        self.0.read(buf)
+    }
+
+    fn read_vectored(&mut self, bufs: &mut [std::io::IoSliceMut<'_>]) -> io::Result<usize> {
+        self.0.read_vectored(bufs)
+    }
+
+    // fn read_buf(&mut self, cursor: BorrowedCursor<'_>) -> io::Result<()> {
+    //     self.0.read_buf(cursor)
+    // }
+
+    // #[inline]
+    // fn is_read_vectored(&self) -> bool {
+    //     self.0.is_read_vectored()
+    // }
+
+    fn read_to_end(&mut self, buf: &mut Vec<u8>) -> io::Result<usize> {
+        self.0.read_to_end(buf)
+    }
+
+    fn read_to_string(&mut self, buf: &mut String) -> io::Result<usize> {
+        self.0.read_to_string(buf)
+    }
+}

data/ext/osv/src/csv/ruby_reader.rs

@@ -0,0 +1,174 @@
+use magnus::{
+    value::{Opaque, ReprValue},
+    RClass, RString, Ruby, Value,
+};
+use std::io::{self, Read, Seek, SeekFrom, Write};
+use std::sync::OnceLock;
+
+static STRING_IO_CLASS: OnceLock<Opaque<RClass>> = OnceLock::new();
+
+/// A reader that can handle various Ruby input types (String, StringIO, IO-like objects)
+/// and provide a standard Read implementation for them.
+pub struct RubyReader<T> {
+    inner: T,
+    offset: usize,
+}
+
+pub trait SeekableRead: std::io::Read + Seek {}
+impl SeekableRead for RubyReader<Value> {}
+impl SeekableRead for RubyReader<RString> {}
+
+pub fn build_ruby_reader(
+    ruby: &Ruby,
+    input: Value,
+) -> Result<Box<dyn SeekableRead>, magnus::Error> {
+    if RubyReader::is_string_io(ruby, &input) {
+        RubyReader::from_string_io(ruby, input)
+    } else if RubyReader::is_io_like(&input) {
+        RubyReader::from_io(input)
+    } else {
+        RubyReader::from_string_like(input)
+    }
+}
+
+impl Seek for RubyReader<Value> {
+    fn seek(&mut self, pos: io::SeekFrom) -> io::Result<u64> {
+        let (whence, offset) = match pos {
+            SeekFrom::Start(i) => (0, i as i64),
+            SeekFrom::Current(i) => (1, i),
+            SeekFrom::End(i) => (2, i),
+        };
+
+        let new_position = self
+            .inner
+            .funcall("seek", (offset, whence))
+            .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?;
+
+        Ok(new_position)
+    }
+}
+
+impl Write for RubyReader<Value> {
+    fn write(&mut self, buf: &[u8]) -> Result<usize, io::Error> {
+        let ruby_bytes = RString::from_slice(buf);
+
+        let bytes_written = self
+            .inner
+            .funcall::<_, _, usize>("write", (ruby_bytes,))
+            .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?;
+
+        Ok(bytes_written)
+    }
+
+    fn flush(&mut self) -> Result<(), io::Error> {
+        self.inner
+            .funcall::<_, _, Value>("flush", ())
+            .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?;
+
+        Ok(())
+    }
+}
+
+impl Seek for RubyReader<RString> {
+    fn seek(&mut self, pos: io::SeekFrom) -> io::Result<u64> {
+        match pos {
+            io::SeekFrom::Start(offset) => self.offset = offset as usize,
+            io::SeekFrom::Current(offset) => self.offset = (self.offset as i64 + offset) as usize,
+            io::SeekFrom::End(offset) => {
+                self.offset = self.inner.len() - offset as usize
+            }
+        }
+        Ok(self.offset as u64)
+    }
+}
+
+impl RubyReader<Value> {
+    fn from_io(input: Value) -> Result<Box<dyn SeekableRead>, magnus::Error> {
+        if Self::is_io_like(&input) {
+            Ok(Box::new(Self::from_io_like(input)))
+        } else {
+            Err(magnus::Error::new(
+                magnus::exception::type_error(),
+                "Input is not an IO-like object",
+            ))
+        }
+    }
+
+    fn is_io_like(input: &Value) -> bool {
+        input.respond_to("read", false).unwrap_or(false)
+    }
+
+    fn from_io_like(input: Value) -> Self {
+        Self {
+            inner: input,
+            offset: 0,
+        }
+    }
+}
+
+impl RubyReader<RString> {
+    pub fn from_string_io(
+        ruby: &Ruby,
+        input: Value,
+    ) -> Result<Box<dyn SeekableRead>, magnus::Error> {
+        if !Self::is_string_io(ruby, &input) {
+            return Err(magnus::Error::new(
+                magnus::exception::type_error(),
+                "Input is not a StringIO",
+            ));
+        }
+
+        let string_content = input.funcall::<_, _, RString>("string", ()).unwrap();
+        Ok(Box::new(Self {
+            inner: string_content,
+            offset: 0,
+        }))
+    }
+
+    fn is_string_io(ruby: &Ruby, input: &Value) -> bool {
+        let string_io_class = STRING_IO_CLASS.get_or_init(|| {
+            let class = RClass::from_value(ruby.eval("StringIO").unwrap()).unwrap();
+            Opaque::from(class)
+        });
+        input.is_kind_of(ruby.get_inner(*string_io_class))
+    }
+
+    fn from_string_like(input: Value) -> Result<Box<dyn SeekableRead>, magnus::Error> {
+        // Try calling `to_str`, and if that fails, try `to_s`
+        let string_content = input
+            .funcall::<_, _, RString>("to_str", ())
+            .or_else(|_| input.funcall::<_, _, RString>("to_s", ()))?;
+        Ok(Box::new(Self {
+            inner: string_content,
+            offset: 0,
+        }))
+    }
+}
+
+impl Read for RubyReader<Value> {
+    fn read(&mut self, mut buf: &mut [u8]) -> io::Result<usize> {
+        let bytes = self
+            .inner
+            .funcall::<_, _, RString>("read", (buf.len(),))
+            .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?;
+
+        buf.write_all(unsafe { bytes.as_slice() })?;
+
+        Ok(bytes.len())
+    }
+}
+
+impl Read for RubyReader<RString> {
+    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+        let string_buffer = unsafe { self.inner.as_slice() };
+        if self.offset >= string_buffer.len() {
+            return Ok(0); // EOF
+        }
+
+        let remaining = string_buffer.len() - self.offset;
+        let copy_size = remaining.min(buf.len());
+        buf[..copy_size].copy_from_slice(&string_buffer[self.offset..self.offset + copy_size]);
+        self.offset += copy_size;
+        Ok(copy_size)
+    }
+}

data/ext/osv/src/lib.rs

@@ -1,3 +1,4 @@
+mod allocator;
 mod csv;
 mod reader;
 mod utils;

data/ext/osv/src/reader.rs

@@ -1,18 +1,19 @@
-use crate::csv::{CsvRecord, RecordReaderBuilder};
+use crate::csv::{CowValue, CsvRecord, RecordReaderBuilder};
 use crate::utils::*;
+use ahash::RandomState;
 use csv::Trim;
 use magnus::value::ReprValue;
 use magnus::{block::Yield, Error, KwArgs, RHash, Ruby, Symbol, Value};
 use std::collections::HashMap;
-use xxhash_rust::xxh3::Xxh3Builder;
 
 pub fn parse_csv(
     rb_self: Value,
     args: &[Value],
-) -> Result<Yield<Box<dyn Iterator<Item = CsvRecord<Xxh3Builder>>>>, Error> {
-    let ruby = unsafe { Ruby::get_unchecked() };
+) -> Result<Yield<Box<dyn Iterator<Item = CsvRecord<'static, RandomState>>>>, Error> {
+    let original = unsafe { Ruby::get_unchecked() };
+    let ruby: &'static Ruby = Box::leak(Box::new(original));
 
-    let CsvArgs {
+    let ReadCsvArgs {
         to_read,
         has_headers,
         delimiter,
@@ -23,7 +24,11 @@ pub fn parse_csv(
         flexible,
         flexible_default,
         trim,
-    } = parse_csv_args(&ruby, args)?;
+    } = parse_read_csv_args(ruby, args)?;
+
+    let flexible_default: &'static Option<String> = Box::leak(Box::new(flexible_default));
+    let leaked_flexible_default: &'static Option<&str> =
+        Box::leak(Box::new(flexible_default.as_deref()));
 
     if !ruby.block_given() {
         return create_enumerator(EnumeratorArgs {
@@ -36,7 +41,7 @@ pub fn parse_csv(
             buffer_size,
             result_type,
             flexible,
-            flexible_default,
+            flexible_default: leaked_flexible_default.as_deref(),
             trim: match trim {
                 Trim::All => Some("all".to_string()),
                 Trim::Headers => Some("headers".to_string()),
@@ -46,33 +51,33 @@ pub fn parse_csv(
         });
     }
 
-    let iter: Box<dyn Iterator<Item = CsvRecord<Xxh3Builder>>> = match result_type.as_str() {
-        "hash" => Box::new(
-            RecordReaderBuilder::<HashMap<&'static str, Option<String>, Xxh3Builder>>::new(
-                &ruby, to_read,
-            )
+    let iter: Box<dyn Iterator<Item = CsvRecord<RandomState>>> = match result_type.as_str() {
+        "hash" => {
+            let builder = RecordReaderBuilder::<
+                HashMap<&'static str, Option<CowValue<'static>>, RandomState>,
+            >::new(ruby, to_read)
             .has_headers(has_headers)
             .flexible(flexible)
-            .flexible_default(flexible_default)
+            .flexible_default(flexible_default.as_deref())
             .trim(trim)
             .delimiter(delimiter)
             .quote_char(quote_char)
             .null_string(null_string)
-            .buffer(buffer_size)
-            .build()?
-            .map(CsvRecord::Map),
-        ),
+            .buffer(buffer_size);
+
+            Box::new(builder.build_threaded()?.map(CsvRecord::Map))
+        }
         "array" => Box::new(
-            RecordReaderBuilder::<Vec<Option<String>>>::new(&ruby, to_read)
+            RecordReaderBuilder::<Vec<Option<CowValue<'static>>>>::new(ruby, to_read)
                 .has_headers(has_headers)
                 .flexible(flexible)
-                .flexible_default(flexible_default)
+                .flexible_default(flexible_default.as_deref())
                 .trim(trim)
                 .delimiter(delimiter)
                 .quote_char(quote_char)
                 .null_string(null_string)
                 .buffer(buffer_size)
-                .build()?
+                .build_threaded()?
                 .map(CsvRecord::Vec),
         ),
         _ => {
@@ -96,13 +101,13 @@ struct EnumeratorArgs {
     buffer_size: usize,
     result_type: String,
     flexible: bool,
-    flexible_default: Option<String>,
+    flexible_default: Option<&'static str>,
     trim: Option<String>,
 }
 
 fn create_enumerator(
     args: EnumeratorArgs,
-) -> Result<Yield<Box<dyn Iterator<Item = CsvRecord<Xxh3Builder>>>>, Error> {
+) -> Result<Yield<Box<dyn Iterator<Item = CsvRecord<'static, RandomState>>>>, Error> {
     let kwargs = RHash::new();
     kwargs.aset(Symbol::new("has_headers"), args.has_headers)?;
     kwargs.aset(

data/ext/osv/src/utils.rs

@@ -13,12 +13,12 @@ fn parse_string_or_symbol(ruby: &Ruby, value: Value) -> Result<Option<String>, E
         RString::from_value(value)
             .ok_or_else(|| Error::new(magnus::exception::type_error(), "Invalid string value"))?
             .to_string()
-            .map(|s| Some(s))
+            .map(Some)
     } else if value.is_kind_of(ruby.class_symbol()) {
         Symbol::from_value(value)
             .ok_or_else(|| Error::new(magnus::exception::type_error(), "Invalid symbol value"))?
             .funcall("to_s", ())
-            .map(|s| Some(s))
+            .map(Some)
     } else {
         Err(Error::new(
             magnus::exception::type_error(),
@@ -28,7 +28,7 @@ fn parse_string_or_symbol(ruby: &Ruby, value: Value) -> Result<Option<String>, E
 }
 
 #[derive(Debug)]
-pub struct CsvArgs {
+pub struct ReadCsvArgs {
     pub to_read: Value,
     pub has_headers: bool,
     pub delimiter: u8,
@@ -42,7 +42,7 @@ pub struct CsvArgs {
 }
 
 /// Parse common arguments for CSV parsing
-pub fn parse_csv_args(ruby: &Ruby, args: &[Value]) -> Result<CsvArgs, Error> {
+pub fn parse_read_csv_args(ruby: &Ruby, args: &[Value]) -> Result<ReadCsvArgs, Error> {
     let parsed_args = scan_args::<(Value,), (), (), (), _, ()>(args)?;
     let (to_read,) = parsed_args.required;
 
@@ -166,7 +166,7 @@ pub fn parse_csv_args(ruby: &Ruby, args: &[Value]) -> Result<CsvArgs, Error> {
         None => csv::Trim::None,
     };
 
-    Ok(CsvArgs {
+    Ok(ReadCsvArgs {
         to_read,
         has_headers,
         delimiter,

data/lib/osv/version.rb

@@ -1,3 +1,3 @@
 module OSV
-  VERSION = "0.3.13"
+  VERSION = "0.3.15"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: osv
 version: !ruby/object:Gem::Version
-  version: 0.3.13
+  version: 0.3.15
 platform: ruby
 authors:
 - Nathan Jaremko
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-12-24 00:00:00.000000000 Z
+date: 2025-01-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -38,16 +38,12 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.2.0
-description: |
-  OSV is a high-performance CSV parser for Ruby, implemented in Rust. It wraps BurntSushi's csv-rs crate to provide fast CSV parsing with support for both hash-based and array-based row formats.
-
-  Features include:
-  - Flexible input sources (file paths, gzipped files, IO objects, strings)
-  - Configurable parsing options (headers, separators, quote chars)
-  - Support for both hash and array output formats
-  - Whitespace trimming options
-  - Strict or flexible parsing modes
-  - Significantly faster than Ruby's standard CSV library
+description: |2
+      OSV is a high-performance CSV parser for Ruby, implemented in Rust.
+      It wraps BurntSushi's csv-rs crate to provide fast CSV parsing with support for both hash-based and array-based row formats.
+      Features include: Flexible input sources (file paths, gzipped files, IO objects, strings),
+      configurable parsing options (headers, separators, quote chars), support for both hash and array output formats,
+      whitespace trimming options, strict or flexible parsing modes, and is significantly faster than Ruby's standard CSV library.
 email:
 - nathan@jaremko.ca
 executables: []
@@ -63,13 +59,15 @@ files:
 - Rakefile
 - ext/osv/Cargo.toml
 - ext/osv/extconf.rb
+- ext/osv/src/allocator.rs
 - ext/osv/src/csv/builder.rs
 - ext/osv/src/csv/header_cache.rs
 - ext/osv/src/csv/mod.rs
 - ext/osv/src/csv/parser.rs
-- ext/osv/src/csv/read_impl.rs
-- ext/osv/src/csv/reader.rs
 - ext/osv/src/csv/record.rs
+- ext/osv/src/csv/record_reader.rs
+- ext/osv/src/csv/ruby_integration.rs
+- ext/osv/src/csv/ruby_reader.rs
 - ext/osv/src/lib.rs
 - ext/osv/src/reader.rs
 - ext/osv/src/utils.rs
@@ -84,8 +82,8 @@ metadata:
   source_code_uri: https://github.com/njaremko/osv
   readme_uri: https://github.com/njaremko/osv/blob/main/README.md
   changelog_uri: https://github.com/njaremko/osv/blob/main/CHANGELOG.md
-  rubygems_mfa_required: 'true'
   documentation_uri: https://www.rubydoc.info/gems/osv
+  funding_uri: https://github.com/sponsors/njaremko
 post_install_message:
 rdoc_options: []
 require_paths:

data/ext/osv/src/csv/read_impl.rs

@@ -1,75 +0,0 @@
-use super::{header_cache::StringCache, parser::RecordParser};
-use std::{io::Read, thread};
-
-pub(crate) const READ_BUFFER_SIZE: usize = 8192;
-
-pub enum ReadImpl<T: RecordParser> {
-    SingleThreaded {
-        reader: csv::Reader<Box<dyn Read>>,
-        headers: Vec<&'static str>,
-        null_string: Option<String>,
-        flexible_default: Option<String>,
-    },
-    MultiThreaded {
-        headers: Vec<&'static str>,
-        receiver: kanal::Receiver<T::Output>,
-        handle: Option<thread::JoinHandle<()>>,
-    },
-}
-
-impl<T: RecordParser> ReadImpl<T> {
-    #[inline]
-    pub fn next(&mut self) -> Option<T::Output> {
-        match self {
-            Self::MultiThreaded {
-                receiver, handle, ..
-            } => match receiver.recv() {
-                Ok(record) => Some(record),
-                Err(_) => {
-                    if let Some(handle) = handle.take() {
-                        let _ = handle.join();
-                    }
-                    None
-                }
-            },
-            Self::SingleThreaded {
-                reader,
-                headers,
-                null_string,
-                flexible_default,
-            } => {
-                let mut record = csv::StringRecord::with_capacity(READ_BUFFER_SIZE, headers.len());
-                match reader.read_record(&mut record) {
-                    Ok(true) => Some(T::parse(
-                        headers,
-                        &record,
-                        null_string.as_deref(),
-                        flexible_default.as_deref(),
-                    )),
-                    _ => None,
-                }
-            }
-        }
-    }
-
-    #[inline]
-    pub fn cleanup(&mut self) {
-        match self {
-            Self::MultiThreaded {
-                receiver,
-                handle,
-                headers,
-                ..
-            } => {
-                receiver.close();
-                if let Some(handle) = handle.take() {
-                    let _ = handle.join();
-                }
-                let _ = StringCache::clear(headers);
-            }
-            Self::SingleThreaded { headers, .. } => {
-                let _ = StringCache::clear(headers);
-            }
-        }
-    }
-}