osv 0.3.8 → 0.3.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 02205de8cef4d5f7633c06720a9e925a2b608116354da4a1678d4746d2197d23
-  data.tar.gz: 3e1d63323fdaad1b6a60e0a0a63801f98710615d6616c882f0cdce00e36c6e2e
+  metadata.gz: 935cf4c277ef52eb1b1c4a4c27d2fe54461489b38b4203a1e574ac1a9e3298df
+  data.tar.gz: aa13944483b0f9fa8963a830d5f2a2932110e86bb32c722565c2b4d5f9dbef3c
 SHA512:
-  metadata.gz: df6a4a4b86c41010ea671ac0e98c2ee6307e62ceff35dab125868f0ee7edb6d14984348ecd4ac9f913489e5a6be0b364240b461334554385aabe5b3374fe798d
-  data.tar.gz: d931b888ce9d0ad1cdb1fa3d0be8cd0e526292206742f5adde718f414e9feca97eff3af6d4139d144c18a50e4807650ea9f7582153bcee80cea1e6ed4ce4ef49
+  metadata.gz: ea38b823a0423ef8883c04f6b677b16ba5d55f9a23faf434fc758788d238972b17b6e2435f0ec92661a32ee31fafb39170c81933d67a201af8cd528678e6261a
+  data.tar.gz: 77a62c7d62f36bd4143166d65c64eecccece56f1ebb969295cf7af6ff8639af9df71b3923d3ba13590824db68604e90cdc61e22490e81a3d0e24bf1af5f0be47

data/Cargo.lock

@@ -274,6 +274,7 @@ dependencies = [
  "serde",
  "serde_magnus",
  "thiserror",
+ "xxhash-rust",
 ]
 
 [[package]]
@@ -526,3 +527,9 @@ name = "windows_x86_64_msvc"
 version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+
+[[package]]
+name = "xxhash-rust"
+version = "0.8.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7d48f1b18be023c95e7b75f481cac649d74be7c507ff4a407c55cfb957f7934"

data/README.md

@@ -112,45 +112,50 @@ Here's some unscientific benchmarks. You can find the code in the [benchmark/com
 ### 10,000 lines
 
 ```
-Benchmarking with 10001 lines of data
+Benchmarking with 100001 lines of data
 
-ruby 3.3.3 (2024-06-12 revision f1c7b6f435) [arm64-darwin23]
+ruby 3.3.6 (2024-11-05 revision 75015d4c1f) [arm64-darwin24]
 Warming up --------------------------------------
-   OSV - Hash output     6.000 i/100ms
+   OSV - Hash output     1.000 i/100ms
    CSV - Hash output     1.000 i/100ms
-  OSV - Array output    18.000 i/100ms
-  CSV - Array output     2.000 i/100ms
+  OSV - Array output     1.000 i/100ms
+OSV - Direct Open Array output
+                        12.719M i/100ms
+  CSV - Array output     1.000 i/100ms
 FastCSV - Array output
-                         9.000 i/100ms
-      OSV - StringIO     7.000 i/100ms
+                         1.000 i/100ms
+      OSV - StringIO     1.000 i/100ms
       CSV - StringIO     1.000 i/100ms
-  FastCSV - StringIO    20.000 i/100ms
-       OSV - Gzipped     6.000 i/100ms
+  FastCSV - StringIO     1.000 i/100ms
+       OSV - Gzipped     1.000 i/100ms
        CSV - Gzipped     1.000 i/100ms
 Calculating -------------------------------------
-   OSV - Hash output     73.360 (± 4.1%) i/s   (13.63 ms/i) -    366.000 in   5.000390s
-   CSV - Hash output     11.937 (±25.1%) i/s   (83.78 ms/i) -     52.000 in   5.036297s
-  OSV - Array output    189.738 (± 8.4%) i/s    (5.27 ms/i) -    954.000 in   5.071018s
-  CSV - Array output     25.471 (±11.8%) i/s   (39.26 ms/i) -    120.000 in   5.015289s
+   OSV - Hash output      6.722 (±14.9%) i/s  (148.77 ms/i) -     59.000 in  10.074753s
+   CSV - Hash output      1.223 (± 0.0%) i/s  (817.62 ms/i) -     13.000 in  10.788284s
+  OSV - Array output     17.284 (±11.6%) i/s   (57.86 ms/i) -    171.000 in  10.007321s
+OSV - Direct Open Array output
+                        213.629M (±13.5%) i/s    (4.68 ns/i) -      1.921B in  10.005506s
+  CSV - Array output      2.193 (± 0.0%) i/s  (455.93 ms/i) -     22.000 in  10.052607s
 FastCSV - Array output
-                         97.867 (± 2.0%) i/s   (10.22 ms/i) -    495.000 in   5.060957s
-      OSV - StringIO     80.784 (± 6.2%) i/s   (12.38 ms/i) -    406.000 in   5.046696s
-      CSV - StringIO     15.872 (± 0.0%) i/s   (63.01 ms/i) -     80.000 in   5.043361s
-  FastCSV - StringIO    200.511 (± 2.0%) i/s    (4.99 ms/i) -      1.020k in   5.088592s
-       OSV - Gzipped     55.220 (±12.7%) i/s   (18.11 ms/i) -    258.000 in   5.030928s
-       CSV - Gzipped     12.591 (±15.9%) i/s   (79.42 ms/i) -     59.000 in   5.039709s
+                          7.993 (± 0.0%) i/s  (125.11 ms/i) -     80.000 in  10.053729s
+      OSV - StringIO      6.626 (±15.1%) i/s  (150.91 ms/i) -     66.000 in  10.103646s
+      CSV - StringIO      1.478 (± 0.0%) i/s  (676.78 ms/i) -     15.000 in  10.158640s
+  FastCSV - StringIO     17.074 (± 5.9%) i/s   (58.57 ms/i) -    171.000 in  10.059266s
+       OSV - Gzipped      5.639 (± 0.0%) i/s  (177.32 ms/i) -     57.000 in  10.152487s
+       CSV - Gzipped      1.176 (± 0.0%) i/s  (850.19 ms/i) -     12.000 in  10.233398s
 
 Comparison:
-  FastCSV - StringIO:      200.5 i/s
-  OSV - Array output:      189.7 i/s - same-ish: difference falls within error
-FastCSV - Array output:       97.9 i/s - 2.05x  slower
-      OSV - StringIO:       80.8 i/s - 2.48x  slower
-   OSV - Hash output:       73.4 i/s - 2.73x  slower
-       OSV - Gzipped:       55.2 i/s - 3.63x  slower
-  CSV - Array output:       25.5 i/s - 7.87x  slower
-      CSV - StringIO:       15.9 i/s - 12.63x  slower
-       CSV - Gzipped:       12.6 i/s - 15.92x  slower
-   CSV - Hash output:       11.9 i/s - 16.80x  slower
+OSV - Direct Open Array output: 213629268.6 i/s
+  OSV - Array output:       17.3 i/s - 12360250.79x  slower
+  FastCSV - StringIO:       17.1 i/s - 12511956.50x  slower
+FastCSV - Array output:        8.0 i/s - 26727225.72x  slower
+   OSV - Hash output:        6.7 i/s - 31780615.83x  slower
+      OSV - StringIO:        6.6 i/s - 32239620.60x  slower
+       OSV - Gzipped:        5.6 i/s - 37881517.48x  slower
+  CSV - Array output:        2.2 i/s - 97400427.87x  slower
+      CSV - StringIO:        1.5 i/s - 144580048.04x  slower
+   CSV - Hash output:        1.2 i/s - 174666591.31x  slower
+       CSV - Gzipped:        1.2 i/s - 181626018.23x  slower
 ```
 
 ### 1,000,000 lines
@@ -158,11 +163,13 @@ FastCSV - Array output:       97.9 i/s - 2.05x  slower
 ```
 Benchmarking with 1000001 lines of data
 
-ruby 3.3.3 (2024-06-12 revision f1c7b6f435) [arm64-darwin23]
+ruby 3.3.6 (2024-11-05 revision 75015d4c1f) [arm64-darwin24]
 Warming up --------------------------------------
    OSV - Hash output     1.000 i/100ms
    CSV - Hash output     1.000 i/100ms
   OSV - Array output     1.000 i/100ms
+OSV - Direct Open Array output
+                         1.000 i/100ms
   CSV - Array output     1.000 i/100ms
 FastCSV - Array output
                          1.000 i/100ms
@@ -172,27 +179,30 @@ FastCSV - Array output
        OSV - Gzipped     1.000 i/100ms
        CSV - Gzipped     1.000 i/100ms
 Calculating -------------------------------------
-   OSV - Hash output      0.578 (± 0.0%) i/s     (1.73 s/i) -      3.000 in   5.287845s
-   CSV - Hash output      0.117 (± 0.0%) i/s     (8.57 s/i) -      1.000 in   8.571770s
-  OSV - Array output      1.142 (± 0.0%) i/s  (875.97 ms/i) -      5.000 in   5.234694s
-  CSV - Array output      0.235 (± 0.0%) i/s     (4.25 s/i) -      2.000 in   8.561144s
+   OSV - Hash output      0.492 (± 0.0%) i/s     (2.03 s/i) -      5.000 in  10.463278s
+   CSV - Hash output      0.114 (± 0.0%) i/s     (8.75 s/i) -      2.000 in  17.573877s
+  OSV - Array output      1.502 (± 0.0%) i/s  (665.58 ms/i) -     14.000 in  10.217551s
+OSV - Direct Open Array output
+                          1.626 (± 0.0%) i/s  (614.90 ms/i) -     16.000 in  10.190323s
+  CSV - Array output      0.183 (± 0.0%) i/s     (5.46 s/i) -      2.000 in  10.951943s
 FastCSV - Array output
-                          0.768 (± 0.0%) i/s     (1.30 s/i) -      4.000 in   6.924574s
-      OSV - StringIO      0.522 (± 0.0%) i/s     (1.91 s/i) -      3.000 in   5.803969s
-      CSV - StringIO      0.132 (± 0.0%) i/s     (7.59 s/i) -      1.000 in   7.593243s
-  FastCSV - StringIO      1.039 (± 0.0%) i/s  (962.53 ms/i) -      6.000 in   5.806644s
-       OSV - Gzipped      0.437 (± 0.0%) i/s     (2.29 s/i) -      3.000 in   6.885125s
-       CSV - Gzipped      0.115 (± 0.0%) i/s     (8.68 s/i) -      1.000 in   8.684069s
+                          0.326 (± 0.0%) i/s     (3.07 s/i) -      4.000 in  12.340605s
+      OSV - StringIO      0.567 (± 0.0%) i/s     (1.76 s/i) -      6.000 in  10.698027s
+      CSV - StringIO      0.141 (± 0.0%) i/s     (7.10 s/i) -      2.000 in  14.237144s
+  FastCSV - StringIO      0.923 (± 0.0%) i/s     (1.08 s/i) -     10.000 in  11.567775s
+       OSV - Gzipped      0.437 (± 0.0%) i/s     (2.29 s/i) -      5.000 in  11.452764s
+       CSV - Gzipped      0.104 (± 0.0%) i/s     (9.64 s/i) -      2.000 in  19.373423s
 
 Comparison:
-  OSV - Array output:        1.1 i/s
-  FastCSV - StringIO:        1.0 i/s - 1.10x  slower
-FastCSV - Array output:        0.8 i/s - 1.49x  slower
-   OSV - Hash output:        0.6 i/s - 1.98x  slower
-      OSV - StringIO:        0.5 i/s - 2.19x  slower
-       OSV - Gzipped:        0.4 i/s - 2.61x  slower
-  CSV - Array output:        0.2 i/s - 4.86x  slower
-      CSV - StringIO:        0.1 i/s - 8.67x  slower
-   CSV - Hash output:        0.1 i/s - 9.79x  slower
-       CSV - Gzipped:        0.1 i/s - 9.91x  slower
+OSV - Direct Open Array output:        1.6 i/s
+  OSV - Array output:        1.5 i/s - 1.08x  slower
+  FastCSV - StringIO:        0.9 i/s - 1.76x  slower
+      OSV - StringIO:        0.6 i/s - 2.87x  slower
+   OSV - Hash output:        0.5 i/s - 3.30x  slower
+       OSV - Gzipped:        0.4 i/s - 3.72x  slower
+FastCSV - Array output:        0.3 i/s - 4.99x  slower
+  CSV - Array output:        0.2 i/s - 8.88x  slower
+      CSV - StringIO:        0.1 i/s - 11.55x  slower
+   CSV - Hash output:        0.1 i/s - 14.24x  slower
+       CSV - Gzipped:        0.1 i/s - 15.68x  slower
 ```

data/Rakefile

@@ -11,7 +11,7 @@ Rake::ExtensionTask.new("osv") do |c|
 end
 
 task :dev do
-  ENV["RB_SYS_CARGO_PROFILE"] = "dev"
+  ENV["RB_SYS_CARGO_PROFILE"] = "release"
 end
 
 Rake::TestTask.new do |t|

data/ext/osv/Cargo.toml

@@ -15,3 +15,4 @@ rb-sys = "^0.9"
 serde = { version = "1.0", features = ["derive"] }
 serde_magnus = "0.8.1"
 thiserror = "2.0"
+xxhash-rust = { version = "0.8.12", features = ["xxh3"] }

data/ext/osv/src/csv/builder.rs

@@ -3,18 +3,21 @@ use super::{
     parser::RecordParser,
     read_impl::ReadImpl,
     reader::RecordReader,
+    READ_BUFFER_SIZE,
 };
 use flate2::read::GzDecoder;
 use magnus::{rb_sys::AsRawValue, value::ReprValue, Error as MagnusError, RString, Ruby, Value};
 use std::{
     fs::File,
-    io::{self, Read},
+    io::{self, BufReader, Read},
     marker::PhantomData,
     os::fd::FromRawFd,
     thread,
 };
 use thiserror::Error;
 
+pub(crate) static BUFFER_CHANNEL_SIZE: usize = 1024;
+
 #[derive(Error, Debug)]
 pub enum ReaderError {
     #[error("Failed to get file descriptor: {0}")]
@@ -68,7 +71,7 @@ impl<'a, T: RecordParser + Send + 'static> RecordReaderBuilder<'a, T> {
             delimiter: b',',
             quote_char: b'"',
             null_string: None,
-            buffer: 1000,
+            buffer: BUFFER_CHANNEL_SIZE,
             flexible: false,
             flexible_default: None,
             _phantom: PhantomData,
@@ -128,7 +131,7 @@ impl<'a, T: RecordParser + Send + 'static> RecordReaderBuilder<'a, T> {
         }
 
         let file = unsafe { File::from_raw_fd(fd) };
-        Ok(Box::new(file))
+        Ok(Box::new(BufReader::with_capacity(READ_BUFFER_SIZE, file)))
     }
 
     fn handle_file_path(&self) -> Result<Box<dyn Read + Send + 'static>, ReaderError> {
@@ -136,24 +139,27 @@ impl<'a, T: RecordParser + Send + 'static> RecordReaderBuilder<'a, T> {
         let file = File::open(&path)?;
 
         Ok(if path.ends_with(".gz") {
-            Box::new(GzDecoder::new(file))
+            Box::new(GzDecoder::new(BufReader::with_capacity(
+                READ_BUFFER_SIZE,
+                file,
+            )))
         } else {
-            Box::new(file)
+            Box::new(BufReader::with_capacity(READ_BUFFER_SIZE, file))
         })
     }
 
-    fn get_reader(&self) -> Result<Box<dyn Read + Send + 'static>, ReaderError> {
+    fn get_reader(&self) -> Result<(Box<dyn Read + Send + 'static>, bool), ReaderError> {
         let string_io: magnus::RClass = self.ruby.eval("StringIO")?;
         let gzip_reader_class: magnus::RClass = self.ruby.eval("Zlib::GzipReader")?;
 
         if self.to_read.is_kind_of(string_io) {
-            self.handle_string_io()
+            self.handle_string_io().map(|r| (r, false))
         } else if self.to_read.is_kind_of(gzip_reader_class) {
             Err(ReaderError::UnsupportedGzipReader)
         } else if self.to_read.is_kind_of(self.ruby.class_io()) {
-            self.handle_file_descriptor()
+            self.handle_file_descriptor().map(|r| (r, true))
         } else {
-            self.handle_file_path()
+            self.handle_file_path().map(|r| (r, false))
         }
     }
 
@@ -175,7 +181,7 @@ impl<'a, T: RecordParser + Send + 'static> RecordReaderBuilder<'a, T> {
 
     pub fn build(self) -> Result<RecordReader<T>, ReaderError> {
         match self.get_reader() {
-            Ok(readable) => self.build_multi_threaded(readable),
+            Ok((readable, should_forget)) => self.build_multi_threaded(readable, should_forget),
             Err(_) => {
                 let readable = self.get_single_threaded_reader()?;
                 self.build_single_threaded(readable)
@@ -186,6 +192,7 @@ impl<'a, T: RecordParser + Send + 'static> RecordReaderBuilder<'a, T> {
     fn build_multi_threaded(
         self,
         readable: Box<dyn Read + Send + 'static>,
+        should_forget: bool,
     ) -> Result<RecordReader<T>, ReaderError> {
         let flexible = self.flexible || self.flexible_default.is_some();
         let mut reader = csv::ReaderBuilder::new()
@@ -204,7 +211,7 @@ impl<'a, T: RecordParser + Send + 'static> RecordReaderBuilder<'a, T> {
 
         let flexible_default = self.flexible_default.clone();
         let handle = thread::spawn(move || {
-            let mut record = csv::StringRecord::new();
+            let mut record = csv::StringRecord::with_capacity(READ_BUFFER_SIZE, headers.len());
             while let Ok(true) = reader.read_record(&mut record) {
                 let row = T::parse(
                     &static_headers,
@@ -216,8 +223,10 @@ impl<'a, T: RecordParser + Send + 'static> RecordReaderBuilder<'a, T> {
                     break;
                 }
             }
-            let file_to_forget = reader.into_inner();
-            std::mem::forget(file_to_forget);
+            if should_forget {
+                let file_to_forget = reader.into_inner();
+                std::mem::forget(file_to_forget);
+            }
         });
 
         Ok(RecordReader {
@@ -257,30 +266,55 @@ impl<'a, T: RecordParser + Send + 'static> RecordReaderBuilder<'a, T> {
 
 struct RubyReader {
     inner: Value,
+    buffer: Option<Vec<u8>>,
+    offset: usize,
 }
 
 impl RubyReader {
     fn new(inner: Value) -> Self {
-        Self { inner }
+        Self {
+            inner,
+            buffer: None,
+            offset: 0,
+        }
     }
 }
 
+// Read the entire inner into a vector and then read future reads from that vector with offset
 impl Read for RubyReader {
     fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
-        let result = self.inner.funcall::<_, _, Value>("read", (buf.len(),));
+        // If we have an existing buffer, read from it
+        if let Some(buffer) = self.buffer.as_ref() {
+            let remaining = buffer.len() - self.offset;
+            let copy_size = remaining.min(buf.len());
+            buf[..copy_size].copy_from_slice(&buffer[self.offset..self.offset + copy_size]);
+            self.offset += copy_size;
+            return Ok(copy_size);
+        }
+
+        // No buffer yet - read the entire content from Ruby
+        let result = self.inner.funcall::<_, _, Value>("read", ());
         match result {
             Ok(data) => {
                 if data.is_nil() {
-                    return Ok(0);
+                    return Ok(0); // EOF
                 }
 
                 let string = RString::from_value(data).ok_or_else(|| {
                     io::Error::new(io::ErrorKind::Other, "Failed to convert to RString")
                 })?;
                 let bytes = unsafe { string.as_slice() };
-                let len = bytes.len().min(buf.len());
-                buf[..len].copy_from_slice(&bytes[..len]);
-                Ok(len)
+
+                // Store the entire content in the buffer
+                self.buffer = Some(bytes.to_vec());
+                self.offset = 0;
+
+                // Read initial chunk
+                let copy_size = bytes.len().min(buf.len());
+                buf[..copy_size].copy_from_slice(&bytes[..copy_size]);
+                self.offset = copy_size;
+
+                Ok(copy_size)
             }
             Err(e) => Err(io::Error::new(io::ErrorKind::Other, e.to_string())),
         }

data/ext/osv/src/csv/mod.rs

@@ -6,4 +6,6 @@ mod reader;
 mod record;
 
 pub use builder::RecordReaderBuilder;
+pub(crate) use builder::BUFFER_CHANNEL_SIZE;
+pub(crate) use read_impl::READ_BUFFER_SIZE;
 pub use record::CsvRecord;

data/ext/osv/src/csv/parser.rs

@@ -1,4 +1,5 @@
 use std::collections::HashMap;
+use std::hash::BuildHasher;
 
 pub trait RecordParser {
     type Output;
@@ -11,7 +12,7 @@ pub trait RecordParser {
     ) -> Self::Output;
 }
 
-impl RecordParser for HashMap<&'static str, Option<String>> {
+impl<S: BuildHasher + Default> RecordParser for HashMap<&'static str, Option<String>, S> {
     type Output = Self;
 
     #[inline]
@@ -21,21 +22,21 @@ impl RecordParser for HashMap<&'static str, Option<String>> {
         null_string: Option<&str>,
         flexible_default: Option<&str>,
     ) -> Self::Output {
-        let mut map = HashMap::with_capacity(headers.len());
-        headers.iter().enumerate().for_each(|(i, header)| {
+        let mut map = HashMap::with_capacity_and_hasher(headers.len(), S::default());
+        headers.iter().enumerate().for_each(|(i, &header)| {
             let value = record.get(i).map_or_else(
-                || flexible_default.map(|s| s.to_string()),
+                || flexible_default.map(ToString::to_string),
                 |field| {
                     if null_string == Some(field) {
                         None
                     } else if field.is_empty() {
                         Some(String::new())
                     } else {
-                        Some(field.to_string())
+                        Some(field.into())
                     }
                 },
             );
-            map.insert(*header, value);
+            map.insert(header, value);
         });
         map
     }
@@ -53,20 +54,20 @@ impl RecordParser for Vec<Option<String>> {
     ) -> Self::Output {
         let target_len = headers.len();
         let mut vec = Vec::with_capacity(target_len);
-        vec.extend(record.iter().map(|field| {
-            if null_string == Some(field) {
+        for field in record.iter() {
+            let value = if Some(field) == null_string {
                 None
             } else if field.is_empty() {
                 Some(String::new())
             } else {
-                Some(field.to_string())
-            }
-        }));
+                Some(field.into())
+            };
+            vec.push(value);
+        }
 
-        // Fill remaining slots with flexible_default if needed
-        if let Some(default) = flexible_default {
-            while vec.len() < target_len {
-                vec.push(Some(default.to_string()));
+        if vec.len() < target_len {
+            if let Some(default) = flexible_default {
+                vec.resize_with(target_len, || Some(default.to_string()));
             }
         }
         vec

data/ext/osv/src/csv/read_impl.rs

@@ -1,6 +1,8 @@
 use super::{header_cache::StringCache, parser::RecordParser};
 use std::{io::Read, thread};
 
+pub(crate) const READ_BUFFER_SIZE: usize = 8192;
+
 pub enum ReadImpl<T: RecordParser> {
     SingleThreaded {
         reader: csv::Reader<Box<dyn Read>>,
@@ -36,7 +38,7 @@ impl<T: RecordParser> ReadImpl<T> {
                 null_string,
                 flexible_default,
             } => {
-                let mut record = csv::StringRecord::new();
+                let mut record = csv::StringRecord::with_capacity(READ_BUFFER_SIZE, headers.len());
                 match reader.read_record(&mut record) {
                     Ok(true) => Some(T::parse(
                         headers,

data/ext/osv/src/csv/record.rs

@@ -1,13 +1,13 @@
 use magnus::{IntoValue, Ruby, Value};
-use std::collections::HashMap;
+use std::{collections::HashMap, hash::BuildHasher};
 
 #[derive(Debug)]
-pub enum CsvRecord {
+pub enum CsvRecord<S: BuildHasher + Default> {
     Vec(Vec<Option<String>>),
-    Map(HashMap<&'static str, Option<String>>),
+    Map(HashMap<&'static str, Option<String>, S>),
 }
 
-impl IntoValue for CsvRecord {
+impl<S: BuildHasher + Default> IntoValue for CsvRecord<S> {
     #[inline]
     fn into_value_with(self, handle: &Ruby) -> Value {
         match self {

data/ext/osv/src/reader.rs

@@ -3,11 +3,12 @@ use crate::utils::*;
 use magnus::value::ReprValue;
 use magnus::{block::Yield, Error, KwArgs, RHash, Ruby, Symbol, Value};
 use std::collections::HashMap;
+use xxhash_rust::xxh3::Xxh3Builder;
 
 pub fn parse_csv(
     rb_self: Value,
     args: &[Value],
-) -> Result<Yield<Box<dyn Iterator<Item = CsvRecord>>>, Error> {
+) -> Result<Yield<Box<dyn Iterator<Item = CsvRecord<Xxh3Builder>>>>, Error> {
     let ruby = unsafe { Ruby::get_unchecked() };
 
     let CsvArgs {
@@ -37,18 +38,20 @@ pub fn parse_csv(
         });
     }
 
-    let iter: Box<dyn Iterator<Item = CsvRecord>> = match result_type.as_str() {
+    let iter: Box<dyn Iterator<Item = CsvRecord<Xxh3Builder>>> = match result_type.as_str() {
         "hash" => Box::new(
-            RecordReaderBuilder::<HashMap<&'static str, Option<String>>>::new(&ruby, to_read)
-                .has_headers(has_headers)
-                .flexible(flexible)
-                .flexible_default(flexible_default)
-                .delimiter(delimiter)
-                .quote_char(quote_char)
-                .null_string(null_string)
-                .buffer(buffer_size)
-                .build()?
-                .map(CsvRecord::Map),
+            RecordReaderBuilder::<HashMap<&'static str, Option<String>, Xxh3Builder>>::new(
+                &ruby, to_read,
+            )
+            .has_headers(has_headers)
+            .flexible(flexible)
+            .flexible_default(flexible_default)
+            .delimiter(delimiter)
+            .quote_char(quote_char)
+            .null_string(null_string)
+            .buffer(buffer_size)
+            .build()?
+            .map(CsvRecord::Map),
         ),
         "array" => Box::new(
             RecordReaderBuilder::<Vec<Option<String>>>::new(&ruby, to_read)
@@ -88,7 +91,7 @@ struct EnumeratorArgs {
 
 fn create_enumerator(
     args: EnumeratorArgs,
-) -> Result<Yield<Box<dyn Iterator<Item = CsvRecord>>>, Error> {
+) -> Result<Yield<Box<dyn Iterator<Item = CsvRecord<Xxh3Builder>>>>, Error> {
     let kwargs = RHash::new();
     kwargs.aset(Symbol::new("has_headers"), args.has_headers)?;
     kwargs.aset(

data/ext/osv/src/utils.rs

@@ -4,6 +4,8 @@ use magnus::{
     Error, RString, Ruby, Symbol, Value,
 };
 
+use crate::csv::BUFFER_CHANNEL_SIZE;
+
 #[derive(Debug)]
 pub struct CsvArgs {
     pub to_read: Value,
@@ -81,7 +83,7 @@ pub fn parse_csv_args(ruby: &Ruby, args: &[Value]) -> Result<CsvArgs, Error> {
 
     let null_string = kwargs.optional.3.unwrap_or_default();
 
-    let buffer_size = kwargs.optional.4.unwrap_or(1000);
+    let buffer_size = kwargs.optional.4.unwrap_or(BUFFER_CHANNEL_SIZE);
 
     let result_type = match kwargs.optional.5 {
         Some(value) => {

data/lib/osv/version.rb

@@ -1,3 +1,3 @@
 module OSV
-  VERSION = "0.3.8"
+  VERSION = "0.3.9"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: osv
 version: !ruby/object:Gem::Version
-  version: 0.3.8
+  version: 0.3.9
 platform: ruby
 authors:
 - Nathan Jaremko