osv 0.3.15 → 0.3.17

data/ext/osv/src/csv/parser.rs

@@ -2,77 +2,78 @@ use std::borrow::Cow;
 use std::collections::HashMap;
 use std::hash::BuildHasher;
 
-use super::CowValue;
+use super::header_cache::StringCacheKey;
+use super::CowStr;
 
 pub trait RecordParser<'a> {
-    type Output: 'a;
+    type Output;
 
     fn parse(
-        headers: &[&'static str],
+        headers: &[StringCacheKey],
         record: &csv::StringRecord,
-        null_string: Option<&str>,
+        null_string: Option<Cow<'a, str>>,
         flexible_default: Option<Cow<'a, str>>,
     ) -> Self::Output;
 }
 
-impl<'a, S: BuildHasher + Default + 'a> RecordParser<'a>
-    for HashMap<&'static str, Option<CowValue<'a>>, S>
+impl<'a, S: BuildHasher + Default> RecordParser<'a>
+    for HashMap<StringCacheKey, Option<CowStr<'a>>, S>
 {
     type Output = Self;
 
     #[inline]
     fn parse(
-        headers: &[&'static str],
+        headers: &[StringCacheKey],
         record: &csv::StringRecord,
-        null_string: Option<&str>,
+        null_string: Option<Cow<'a, str>>,
         flexible_default: Option<Cow<'a, str>>,
     ) -> Self::Output {
         let mut map = HashMap::with_capacity_and_hasher(headers.len(), S::default());
 
         let shared_empty = Cow::Borrowed("");
-        let shared_default = flexible_default.map(CowValue);
-        headers.iter().enumerate().for_each(|(i, &header)| {
+        let shared_default = flexible_default.map(CowStr);
+        headers.iter().enumerate().for_each(|(i, ref header)| {
             let value = record.get(i).map_or_else(
                 || shared_default.clone(),
                 |field| {
-                    if null_string == Some(field) {
+                    if null_string.as_deref() == Some(field) {
                         None
                     } else if field.is_empty() {
-                        Some(CowValue(shared_empty.clone()))
+                        Some(CowStr(shared_empty.clone()))
                     } else {
-                        Some(CowValue(Cow::Owned(field.to_string())))
+                        Some(CowStr(Cow::Owned(field.to_string())))
                     }
                 },
             );
-            map.insert(header, value);
+            map.insert((*header).clone(), value);
         });
         map
     }
 }
 
-impl<'a> RecordParser<'a> for Vec<Option<CowValue<'a>>> {
+impl<'a> RecordParser<'a> for Vec<Option<CowStr<'a>>> {
     type Output = Self;
 
     #[inline]
     fn parse(
-        headers: &[&'static str],
+        headers: &[StringCacheKey],
         record: &csv::StringRecord,
-        null_string: Option<&str>,
+        null_string: Option<Cow<'a, str>>,
         flexible_default: Option<Cow<'a, str>>,
     ) -> Self::Output {
         let target_len = headers.len();
         let mut vec = Vec::with_capacity(target_len);
 
         let shared_empty = Cow::Borrowed("");
-        let shared_default = flexible_default.map(CowValue);
+        let shared_default = flexible_default.map(CowStr);
 
         for field in record.iter() {
-            let value = if Some(field) == null_string {
+            let value = if Some(field) == null_string.as_deref() {
                 None
             } else if field.is_empty() {
-                Some(CowValue(shared_empty.clone()))
+                Some(CowStr(shared_empty.clone()))
             } else {
-                Some(CowValue(Cow::Owned(field.to_string())))
+                Some(CowStr(Cow::Owned(field.to_string())))
             };
             vec.push(value);
         }
@@ -85,67 +86,3 @@ impl<'a> RecordParser<'a> for Vec<Option<CowValue<'a>>> {
         vec
     }
 }
-
-// impl<'a, S: BuildHasher + Default + 'a> RecordParser<'a>
-//     for HashMap<&'static str, Option<String>, S>
-// {
-//     type Output = Self;
-
-//     #[inline]
-//     fn parse(
-//         headers: &[&'static str],
-//         record: &csv::StringRecord,
-//         null_string: Option<&str>,
-//         flexible_default: Option<Cow<'a, str>>,
-//     ) -> Self::Output {
-//         let mut map = HashMap::with_capacity_and_hasher(headers.len(), S::default());
-//         headers.iter().enumerate().for_each(|(i, &header)| {
-//             let value = record.get(i).map_or_else(
-//                 || flexible_default.clone(),
-//                 |field| {
-//                     if null_string == Some(field) {
-//                         None
-//                     } else if field.is_empty() {
-//                         Some(String::new())
-//                     } else {
-//                         Some(field.into())
-//                     }
-//                 },
-//             );
-//             map.insert(header, value);
-//         });
-//         map
-//     }
-// }
-
-// impl<'a> RecordParser<'a> for Vec<Option<String>> {
-//     type Output = Self;
-
-//     #[inline]
-//     fn parse(
-//         headers: &[&'static str],
-//         record: &csv::StringRecord,
-//         null_string: Option<&str>,
-//         flexible_default: Option<Cow<'a, str>>,
-//     ) -> Self::Output {
-//         let target_len = headers.len();
-//         let mut vec = Vec::with_capacity(target_len);
-//         for field in record.iter() {
-//             let value = if Some(field) == null_string {
-//                 None
-//             } else if field.is_empty() {
-//                 Some(String::new())
-//             } else {
-//                 Some(field.into())
-//             };
-//             vec.push(value);
-//         }
-
-//         if vec.len() < target_len {
-//             if let Some(default) = flexible_default {
-//                 vec.resize_with(target_len, || Some(default.to_string()));
-//             }
-//         }
-//         vec
-//     }
-// }

data/ext/osv/src/csv/record.rs

@@ -1,10 +1,13 @@
-use magnus::{IntoValue, Ruby, Value};
+use itertools::Itertools;
+use magnus::{value::ReprValue, IntoValue, Ruby, Value};
 use std::{borrow::Cow, collections::HashMap, hash::BuildHasher};
 
+use super::StringCacheKey;
+
 #[derive(Debug)]
 pub enum CsvRecord<'a, S: BuildHasher + Default> {
-    Vec(Vec<Option<CowValue<'a>>>),
-    Map(HashMap<&'static str, Option<CowValue<'a>>, S>),
+    Vec(Vec<Option<CowStr<'a>>>),
+    Map(HashMap<StringCacheKey, Option<CowStr<'a>>, S>),
 }
 
 impl<S: BuildHasher + Default> IntoValue for CsvRecord<'_, S> {
@@ -19,9 +22,23 @@ impl<S: BuildHasher + Default> IntoValue for CsvRecord<'_, S> {
             CsvRecord::Map(map) => {
                 // Pre-allocate the hash with the known size
                 let hash = handle.hash_new_capa(map.len());
-                map.into_iter()
-                    .try_for_each(|(k, v)| hash.aset(k, v))
-                    .unwrap();
+
+                let mut values: [Value; 128] = [handle.qnil().as_value(); 128];
+                let mut i = 0;
+
+                for chunk in &map.into_iter().chunks(128) {
+                    for (k, v) in chunk {
+                        values[i] = handle.into_value(k);
+                        values[i + 1] = handle.into_value(v);
+                        i += 2;
+                    }
+                    hash.bulk_insert(&values[..i]).unwrap();
+
+                    // Zero out used values
+                    values[..i].fill(handle.qnil().as_value());
+                    i = 0;
+                }
+
                 hash.into_value_with(handle)
             }
         }
@@ -29,9 +46,9 @@ impl<S: BuildHasher + Default> IntoValue for CsvRecord<'_, S> {
 }
 
 #[derive(Debug, Clone)]
-pub struct CowValue<'a>(pub Cow<'a, str>);
+pub struct CowStr<'a>(pub Cow<'a, str>);
 
-impl IntoValue for CowValue<'_> {
+impl IntoValue for CowStr<'_> {
     fn into_value_with(self, handle: &Ruby) -> Value {
         self.0.into_value_with(handle)
     }

data/ext/osv/src/csv/record_reader.rs

@@ -1,32 +1,35 @@
+use super::header_cache::StringCacheKey;
 use super::parser::RecordParser;
 use super::{header_cache::StringCache, ruby_reader::SeekableRead};
 use magnus::{Error, Ruby};
-use std::io::BufReader;
-use std::{borrow::Cow, io::Read, thread};
+use std::borrow::Cow;
+use std::io::{BufReader, Read};
 
+/// Size of the internal buffer used for reading CSV records
 pub(crate) const READ_BUFFER_SIZE: usize = 16384;
 
+/// A reader that processes CSV records using a specified parser.
+///
+/// This struct implements Iterator to provide a streaming interface for CSV records.
 pub struct RecordReader<'a, T: RecordParser<'a>> {
-    inner: ReaderImpl<'a, T>,
-}
-
-#[allow(clippy::large_enum_variant)]
-enum ReaderImpl<'a, T: RecordParser<'a>> {
-    SingleThreaded {
-        reader: csv::Reader<BufReader<Box<dyn SeekableRead>>>,
-        headers: Vec<&'static str>,
-        null_string: Option<String>,
-        flexible_default: Option<Cow<'a, str>>,
-        string_record: csv::StringRecord,
-    },
-    MultiThreaded {
-        headers: Vec<&'static str>,
-        receiver: kanal::Receiver<T::Output>,
-        handle: Option<thread::JoinHandle<()>>,
-    },
+    reader: csv::Reader<BufReader<Box<dyn SeekableRead>>>,
+    headers: Vec<StringCacheKey>,
+    null_string: Option<Cow<'a, str>>,
+    flexible_default: Option<Cow<'a, str>>,
+    string_record: csv::StringRecord,
+    parser: std::marker::PhantomData<T>,
 }
 
 impl<'a, T: RecordParser<'a>> RecordReader<'a, T> {
+    /// Reads and processes headers from a CSV reader.
+    ///
+    /// # Arguments
+    /// * `ruby` - Ruby VM context for error handling
+    /// * `reader` - CSV reader instance
+    /// * `has_headers` - Whether the CSV file contains headers
+    ///
+    /// # Returns
+    /// A vector of header strings or generated column names if `has_headers` is false
     #[inline]
     pub(crate) fn get_headers(
         ruby: &Ruby,
@@ -40,67 +43,41 @@ impl<'a, T: RecordParser<'a>> RecordReader<'a, T> {
             )
         })?;
 
-        let mut headers = Vec::with_capacity(first_row.len());
-        if has_headers {
-            headers.extend(first_row.iter().map(String::from));
+        Ok(if has_headers {
+            first_row.iter().map(String::from).collect()
         } else {
-            headers.extend((0..first_row.len()).map(|i| format!("c{i}")));
-        }
-        Ok(headers)
+            (0..first_row.len()).map(|i| format!("c{i}")).collect()
+        })
     }
 
-    pub(crate) fn new_single_threaded(
+    /// Creates a new RecordReader instance.
+    pub(crate) fn new(
         reader: csv::Reader<BufReader<Box<dyn SeekableRead>>>,
-        headers: Vec<&'static str>,
-        null_string: Option<String>,
-        flexible_default: Option<&'a str>,
+        headers: Vec<StringCacheKey>,
+        null_string: Option<Cow<'a, str>>,
+        flexible_default: Option<Cow<'a, str>>,
     ) -> Self {
         let headers_len = headers.len();
         Self {
-            inner: ReaderImpl::SingleThreaded {
-                reader,
-                headers,
-                null_string,
-                flexible_default: flexible_default.map(Cow::Borrowed),
-                string_record: csv::StringRecord::with_capacity(READ_BUFFER_SIZE, headers_len),
-            },
+            reader,
+            headers,
+            null_string,
+            flexible_default,
+            string_record: csv::StringRecord::with_capacity(READ_BUFFER_SIZE, headers_len),
+            parser: std::marker::PhantomData,
         }
     }
-}
 
-impl<T: RecordParser<'static> + Send> RecordReader<'static, T> {
-    pub(crate) fn new_multi_threaded(
-        mut reader: csv::Reader<Box<dyn Read + Send + 'static>>,
-        headers: Vec<&'static str>,
-        buffer_size: usize,
-        null_string: Option<String>,
-        flexible_default: Option<&'static str>,
-    ) -> Self {
-        let (sender, receiver) = kanal::bounded(buffer_size);
-        let headers_for_thread = headers.clone();
-
-        let handle = thread::spawn(move || {
-            let mut record =
-                csv::StringRecord::with_capacity(READ_BUFFER_SIZE, headers_for_thread.len());
-            while let Ok(true) = reader.read_record(&mut record) {
-                let row = T::parse(
-                    &headers_for_thread,
-                    &record,
-                    null_string.as_deref(),
-                    flexible_default.map(Cow::Borrowed),
-                );
-                if sender.send(row).is_err() {
-                    break;
-                }
-            }
-        });
-
-        Self {
-            inner: ReaderImpl::MultiThreaded {
-                headers,
-                receiver,
-                handle: Some(handle),
-            },
+    /// Attempts to read the next record, returning any errors encountered.
+    fn try_next(&mut self) -> csv::Result<Option<T::Output>> {
+        match self.reader.read_record(&mut self.string_record)? {
+            true => Ok(Some(T::parse(
+                &self.headers,
+                &self.string_record,
+                self.null_string.clone(),
+                self.flexible_default.clone(),
+            ))),
+            false => Ok(None),
         }
     }
 }
@@ -110,63 +87,21 @@ impl<'a, T: RecordParser<'a>> Iterator for RecordReader<'a, T> {
 
     #[inline]
     fn next(&mut self) -> Option<Self::Item> {
-        match &mut self.inner {
-            ReaderImpl::MultiThreaded {
-                receiver, handle, ..
-            } => match receiver.recv() {
-                Ok(record) => Some(record),
-                Err(_) => {
-                    if let Some(handle) = handle.take() {
-                        let _ = handle.join();
-                    }
-                    None
-                }
-            },
-            ReaderImpl::SingleThreaded {
-                reader,
-                headers,
-                null_string,
-                flexible_default,
-                ref mut string_record,
-            } => match reader.read_record(string_record) {
-                Ok(true) => Some(T::parse(
-                    headers,
-                    string_record,
-                    null_string.as_deref(),
-                    flexible_default.clone(),
-                )),
-                Ok(false) => None,
-                Err(_e) => None,
-            },
-        }
+        // Note: We intentionally swallow errors here to maintain Iterator contract.
+        // Errors can be handled by using try_next() directly if needed.
+        self.try_next().ok().flatten()
     }
 
     #[inline]
     fn size_hint(&self) -> (usize, Option<usize>) {
-        // We can't know the exact size without reading the whole file
-        (0, None)
+        (0, None) // Cannot determine size without reading entire file
     }
 }
 
 impl<'a, T: RecordParser<'a>> Drop for RecordReader<'a, T> {
     #[inline]
     fn drop(&mut self) {
-        match &mut self.inner {
-            ReaderImpl::MultiThreaded {
-                receiver,
-                handle,
-                headers,
-                ..
-            } => {
-                receiver.close();
-                if let Some(handle) = handle.take() {
-                    let _ = handle.join();
-                }
-                let _ = StringCache::clear(headers);
-            }
-            ReaderImpl::SingleThreaded { headers, .. } => {
-                let _ = StringCache::clear(headers);
-            }
-        }
+        // Intentionally ignore errors during cleanup as there's no meaningful way to handle them
+        let _ = StringCache::clear(&self.headers);
     }
 }

data/ext/osv/src/csv/ruby_integration.rs

@@ -1,30 +1,19 @@
-use std::{fs::File, io, mem::ManuallyDrop};
+use std::{
+    fs::File,
+    io::{self, Read, Seek, SeekFrom},
+    mem::ManuallyDrop,
+};
 
 pub struct ForgottenFileHandle(pub ManuallyDrop<File>);
 
-impl std::io::Read for ForgottenFileHandle {
+impl Read for ForgottenFileHandle {
     fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
         self.0.read(buf)
     }
+}
 
-    fn read_vectored(&mut self, bufs: &mut [std::io::IoSliceMut<'_>]) -> io::Result<usize> {
-        self.0.read_vectored(bufs)
-    }
-
-    // fn read_buf(&mut self, cursor: BorrowedCursor<'_>) -> io::Result<()> {
-    //     self.0.read_buf(cursor)
-    // }
-
-    // #[inline]
-    // fn is_read_vectored(&self) -> bool {
-    //     self.0.is_read_vectored()
-    // }
-
-    fn read_to_end(&mut self, buf: &mut Vec<u8>) -> io::Result<usize> {
-        self.0.read_to_end(buf)
-    }
-
-    fn read_to_string(&mut self, buf: &mut String) -> io::Result<usize> {
-        self.0.read_to_string(buf)
+impl Seek for ForgottenFileHandle {
+    fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
+        self.0.seek(pos)
     }
 }

data/ext/osv/src/csv/ruby_reader.rs

@@ -2,9 +2,12 @@ use magnus::{
     value::{Opaque, ReprValue},
     RClass, RString, Ruby, Value,
 };
-use std::io::{self, Read, Seek, SeekFrom, Write};
+use std::fs::File;
+use std::io::{self, BufReader, Read, Seek, SeekFrom, Write};
 use std::sync::OnceLock;
 
+use super::ForgottenFileHandle;
+
 static STRING_IO_CLASS: OnceLock<Opaque<RClass>> = OnceLock::new();
 
 /// A reader that can handle various Ruby input types (String, StringIO, IO-like objects)
@@ -17,6 +20,10 @@ pub struct RubyReader<T> {
 pub trait SeekableRead: std::io::Read + Seek {}
 impl SeekableRead for RubyReader<Value> {}
 impl SeekableRead for RubyReader<RString> {}
+impl SeekableRead for File {}
+impl<T: Read + Seek> SeekableRead for BufReader<T> {}
+impl SeekableRead for std::io::Cursor<Vec<u8>> {}
+impl SeekableRead for ForgottenFileHandle {}
 
 pub fn build_ruby_reader(
     ruby: &Ruby,
@@ -74,9 +81,7 @@ impl Seek for RubyReader<RString> {
         match pos {
             io::SeekFrom::Start(offset) => self.offset = offset as usize,
             io::SeekFrom::Current(offset) => self.offset = (self.offset as i64 + offset) as usize,
-            io::SeekFrom::End(offset) => {
-                self.offset = self.inner.len() - offset as usize
-            }
+            io::SeekFrom::End(offset) => self.offset = self.inner.len() - offset as usize,
         }
         Ok(self.offset as u64)
     }