osv 0.3.12 → 0.3.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3158fed2ced03b0fd6bdc792198529e675a10656ec32d5d375a0fee88481d5e4
-  data.tar.gz: 0e2203377d114a6ee12cc7aa4abdb80baeea5adba3bd253825a6fa975cdd51aa
+  metadata.gz: 689f28c935746890aa680fd2f649076a36d6ce233d4cbf2717dc129174b593dc
+  data.tar.gz: 45ddaa6774a9a4e9391d000b30b6e92afb8560b81821d8fec363d54283bac6d9
 SHA512:
-  metadata.gz: 82ae4170b55db12c767ccff522f0415e6ff586455a9c10e09d603323a8d177e0d532fae45d20e5eef57de59a6e75e0e711930b2a5c881af8e8192f9c804fedc1
-  data.tar.gz: 3e00897d9e6ae42756e22e55a0d3e2450b9375a63af71618f9d866bf256783b1315f18d27632a7f94f0d4ef7cefd20210efef85f3a5bddc47b216f77e51c69f3
+  metadata.gz: 74c2052ea9cbc61ddef5d1c46abdd5e4cdf7c60c946c421e4b8da7c160ba3f3eb761842279cd9f066aa6a1aa2214d0ef9ba9ff11c46294e8e1d4ebbb95161d70
+  data.tar.gz: 5a795e5fa6d84b39082c2754dea655cd5b4f8a00558627fb64f661a14ec32daa8ea7b31a13724291e11531d03c8c5fa1bdb928c6d9422b87fcbb5b5aba7daad5

data/Gemfile

@@ -2,12 +2,12 @@ source "https://rubygems.org"
 
 gem "rb_sys", "~> 0.9.56"
 gem "rake"
-gem "csv"
 
 # Use local version of osv
 gemspec
 
 group :development, :test do
+  gem "csv"
   gem "minitest", "~> 5.0"
   gem "benchmark-ips", "~> 2.12"
   gem "fastcsv", "~> 0.0.7"

data/README.md

@@ -114,104 +114,62 @@ When `has_headers` is false, hash keys will be generated as `"c0"`, `"c1"`, etc.
 
 ## Performance
 
-This library is faster than the standard Ruby CSV library, and is comparable to the fastest CSV parser gems I've used.
+This library is faster than the standard Ruby CSV library. It's also faster than any other CSV gem I've been able to find.
 
 Here's some unscientific benchmarks. You can find the code in the [benchmark/comparison_benchmark.rb](benchmark/comparison_benchmark.rb) file.
 
-### 10,000 lines
+### 1,000,000 lines
 
 ```
-Benchmarking with 100001 lines of data
+🏃 Running benchmarks...
+Benchmarking with 3000001 lines of data
 
-ruby 3.3.6 (2024-11-05 revision 75015d4c1f) [arm64-darwin24]
+ruby 3.3.6 (2024-11-05 revision 75015d4c1f) +YJIT [arm64-darwin24]
 Warming up --------------------------------------
-   OSV - Hash output     1.000 i/100ms
-   CSV - Hash output     1.000 i/100ms
-  OSV - Array output     1.000 i/100ms
-OSV - Direct Open Array output
-                        12.719M i/100ms
-  CSV - Array output     1.000 i/100ms
-FastCSV - Array output
-                         1.000 i/100ms
-      OSV - StringIO     1.000 i/100ms
       CSV - StringIO     1.000 i/100ms
   FastCSV - StringIO     1.000 i/100ms
-       OSV - Gzipped     1.000 i/100ms
-       CSV - Gzipped     1.000 i/100ms
-Calculating -------------------------------------
-   OSV - Hash output      6.722 (±14.9%) i/s  (148.77 ms/i) -     59.000 in  10.074753s
-   CSV - Hash output      1.223 (± 0.0%) i/s  (817.62 ms/i) -     13.000 in  10.788284s
-  OSV - Array output     17.284 (±11.6%) i/s   (57.86 ms/i) -    171.000 in  10.007321s
-OSV - Direct Open Array output
-                        213.629M (±13.5%) i/s    (4.68 ns/i) -      1.921B in  10.005506s
-  CSV - Array output      2.193 (± 0.0%) i/s  (455.93 ms/i) -     22.000 in  10.052607s
-FastCSV - Array output
-                          7.993 (± 0.0%) i/s  (125.11 ms/i) -     80.000 in  10.053729s
-      OSV - StringIO      6.626 (±15.1%) i/s  (150.91 ms/i) -     66.000 in  10.103646s
-      CSV - StringIO      1.478 (± 0.0%) i/s  (676.78 ms/i) -     15.000 in  10.158640s
-  FastCSV - StringIO     17.074 (± 5.9%) i/s   (58.57 ms/i) -    171.000 in  10.059266s
-       OSV - Gzipped      5.639 (± 0.0%) i/s  (177.32 ms/i) -     57.000 in  10.152487s
-       CSV - Gzipped      1.176 (± 0.0%) i/s  (850.19 ms/i) -     12.000 in  10.233398s
-
-Comparison:
-OSV - Direct Open Array output: 213629268.6 i/s
-  OSV - Array output:       17.3 i/s - 12360250.79x  slower
-  FastCSV - StringIO:       17.1 i/s - 12511956.50x  slower
-FastCSV - Array output:        8.0 i/s - 26727225.72x  slower
-   OSV - Hash output:        6.7 i/s - 31780615.83x  slower
-      OSV - StringIO:        6.6 i/s - 32239620.60x  slower
-       OSV - Gzipped:        5.6 i/s - 37881517.48x  slower
-  CSV - Array output:        2.2 i/s - 97400427.87x  slower
-      CSV - StringIO:        1.5 i/s - 144580048.04x  slower
-   CSV - Hash output:        1.2 i/s - 174666591.31x  slower
-       CSV - Gzipped:        1.2 i/s - 181626018.23x  slower
-```
-
-### 1,000,000 lines
-
-```
-Benchmarking with 1000001 lines of data
-
-ruby 3.3.6 (2024-11-05 revision 75015d4c1f) [arm64-darwin24]
-Warming up --------------------------------------
-   OSV - Hash output     1.000 i/100ms
+      OSV - StringIO     1.000 i/100ms
    CSV - Hash output     1.000 i/100ms
-  OSV - Array output     1.000 i/100ms
-OSV - Direct Open Array output
-                         1.000 i/100ms
+   OSV - Hash output     1.000 i/100ms
   CSV - Array output     1.000 i/100ms
+  OSV - Array output     1.000 i/100ms
 FastCSV - Array output
                          1.000 i/100ms
-      OSV - StringIO     1.000 i/100ms
-      CSV - StringIO     1.000 i/100ms
-  FastCSV - StringIO     1.000 i/100ms
+OSV - Direct Open Array output
+                         1.000 i/100ms
        OSV - Gzipped     1.000 i/100ms
+OSV - Gzipped Direct     1.000 i/100ms
+   FastCSV - Gzipped     1.000 i/100ms
        CSV - Gzipped     1.000 i/100ms
 Calculating -------------------------------------
-   OSV - Hash output      0.492 (± 0.0%) i/s     (2.03 s/i) -      5.000 in  10.463278s
-   CSV - Hash output      0.114 (± 0.0%) i/s     (8.75 s/i) -      2.000 in  17.573877s
-  OSV - Array output      1.502 (± 0.0%) i/s  (665.58 ms/i) -     14.000 in  10.217551s
-OSV - Direct Open Array output
-                          1.626 (± 0.0%) i/s  (614.90 ms/i) -     16.000 in  10.190323s
-  CSV - Array output      0.183 (± 0.0%) i/s     (5.46 s/i) -      2.000 in  10.951943s
+      CSV - StringIO      0.083 (± 0.0%) i/s    (12.01 s/i) -      3.000 in  36.028672s
+  FastCSV - StringIO      0.366 (± 0.0%) i/s     (2.73 s/i) -     11.000 in  30.032350s
+      OSV - StringIO      0.522 (± 0.0%) i/s     (1.92 s/i) -     16.000 in  30.655768s
+   CSV - Hash output      0.062 (± 0.0%) i/s    (16.16 s/i) -      2.000 in  32.311990s
+   OSV - Hash output      0.273 (± 0.0%) i/s     (3.66 s/i) -      9.000 in  32.924970s
+  CSV - Array output      0.069 (± 0.0%) i/s    (14.50 s/i) -      3.000 in  43.488185s
+  OSV - Array output      0.601 (± 0.0%) i/s     (1.66 s/i) -     19.000 in  31.636782s
 FastCSV - Array output
-                          0.326 (± 0.0%) i/s     (3.07 s/i) -      4.000 in  12.340605s
-      OSV - StringIO      0.567 (± 0.0%) i/s     (1.76 s/i) -      6.000 in  10.698027s
-      CSV - StringIO      0.141 (± 0.0%) i/s     (7.10 s/i) -      2.000 in  14.237144s
-  FastCSV - StringIO      0.923 (± 0.0%) i/s     (1.08 s/i) -     10.000 in  11.567775s
-       OSV - Gzipped      0.437 (± 0.0%) i/s     (2.29 s/i) -      5.000 in  11.452764s
-       CSV - Gzipped      0.104 (± 0.0%) i/s     (9.64 s/i) -      2.000 in  19.373423s
+                          0.356 (± 0.0%) i/s     (2.81 s/i) -     11.000 in  30.871931s
+OSV - Direct Open Array output
+                          0.604 (± 0.0%) i/s     (1.66 s/i) -     19.000 in  31.469190s
+       OSV - Gzipped      0.424 (± 0.0%) i/s     (2.36 s/i) -     13.000 in  30.642322s
+OSV - Gzipped Direct      0.636 (± 0.0%) i/s     (1.57 s/i) -     20.000 in  31.424083s
+   FastCSV - Gzipped      0.323 (± 0.0%) i/s     (3.10 s/i) -     10.000 in  30.990648s
+       CSV - Gzipped      0.058 (± 0.0%) i/s    (17.11 s/i) -      2.000 in  34.228691s
 
 Comparison:
-OSV - Direct Open Array output:        1.6 i/s
-  OSV - Array output:        1.5 i/s - 1.08x  slower
-  FastCSV - StringIO:        0.9 i/s - 1.76x  slower
-      OSV - StringIO:        0.6 i/s - 2.87x  slower
-   OSV - Hash output:        0.5 i/s - 3.30x  slower
-       OSV - Gzipped:        0.4 i/s - 3.72x  slower
-FastCSV - Array output:        0.3 i/s - 4.99x  slower
-  CSV - Array output:        0.2 i/s - 8.88x  slower
-      CSV - StringIO:        0.1 i/s - 11.55x  slower
-   CSV - Hash output:        0.1 i/s - 14.24x  slower
-       CSV - Gzipped:        0.1 i/s - 15.68x  slower
+OSV - Gzipped Direct:        0.6 i/s
+OSV - Direct Open Array output:        0.6 i/s - 1.05x  slower
+  OSV - Array output:        0.6 i/s - 1.06x  slower
+      OSV - StringIO:        0.5 i/s - 1.22x  slower
+       OSV - Gzipped:        0.4 i/s - 1.50x  slower
+  FastCSV - StringIO:        0.4 i/s - 1.74x  slower
+FastCSV - Array output:        0.4 i/s - 1.79x  slower
+   FastCSV - Gzipped:        0.3 i/s - 1.97x  slower
+   OSV - Hash output:        0.3 i/s - 2.33x  slower
+      CSV - StringIO:        0.1 i/s - 7.64x  slower
+  CSV - Array output:        0.1 i/s - 9.23x  slower
+   CSV - Hash output:        0.1 i/s - 10.28x  slower
+       CSV - Gzipped:        0.1 i/s - 10.89x  slower
 ```

data/Rakefile

@@ -1,21 +1,19 @@
 # frozen_string_literal: true
 
 require "rake/testtask"
-require "rake/extensiontask"
+require "rb_sys/extensiontask"
 
 task default: :test
 
-Rake::ExtensionTask.new("osv") do |c|
-  c.lib_dir = "lib/osv"
-  c.ext_dir = "ext/osv"
-end
+GEMSPEC = Gem::Specification.load("osv.gemspec")
 
-task :dev do
-  ENV["RB_SYS_CARGO_PROFILE"] = "release"
+RbSys::ExtensionTask.new("osv", GEMSPEC) do |ext|
+  ext.lib_dir = "lib/osv"
+  ext.ext_dir = "ext/osv"
 end
 
 Rake::TestTask.new do |t|
-  t.deps << :dev << :compile
+  t.deps << :compile
   t.test_files = FileList[File.expand_path("test/*_test.rb", __dir__)]
   t.libs << "lib"
   t.libs << "test"

data/ext/osv/src/csv/builder.rs

@@ -1,19 +1,18 @@
 use super::{
     header_cache::{CacheError, StringCache},
     parser::RecordParser,
-    read_impl::ReadImpl,
-    reader::RecordReader,
-    READ_BUFFER_SIZE,
+    record_reader::{RecordReader, READ_BUFFER_SIZE},
+    ruby_reader::build_ruby_reader,
 };
 use flate2::read::GzDecoder;
-use magnus::{rb_sys::AsRawValue, value::ReprValue, Error as MagnusError, RString, Ruby, Value};
+use magnus::{rb_sys::AsRawValue, value::ReprValue, Error as MagnusError, Ruby, Value};
 use std::{
     fs::File,
     io::{self, BufReader, Read},
     marker::PhantomData,
     os::fd::FromRawFd,
-    thread,
 };
+
 use thiserror::Error;
 
 pub(crate) static BUFFER_CHANNEL_SIZE: usize = 1024;
@@ -28,8 +27,6 @@ pub enum ReaderError {
     FileOpen(#[from] io::Error),
     #[error("Failed to intern headers: {0}")]
     HeaderIntern(#[from] CacheError),
-    #[error("Unsupported GzipReader")]
-    UnsupportedGzipReader,
     #[error("Ruby error: {0}")]
     Ruby(String),
 }
@@ -49,7 +46,7 @@ impl From<ReaderError> for MagnusError {
     }
 }
 
-pub struct RecordReaderBuilder<'a, T: RecordParser + Send + 'static> {
+pub struct RecordReaderBuilder<'a, T: RecordParser<'a> + Send> {
     ruby: &'a Ruby,
     to_read: Value,
     has_headers: bool,
@@ -58,12 +55,55 @@ pub struct RecordReaderBuilder<'a, T: RecordParser + Send + 'static> {
     null_string: Option<String>,
     buffer: usize,
     flexible: bool,
-    flexible_default: Option<String>,
+    flexible_default: Option<&'a str>,
     trim: csv::Trim,
     _phantom: PhantomData<T>,
 }
 
-impl<'a, T: RecordParser + Send + 'static> RecordReaderBuilder<'a, T> {
+impl<T: RecordParser<'static> + Send + 'static> RecordReaderBuilder<'static, T> {
+    fn build_multi_threaded(
+        self,
+        readable: Box<dyn Read + Send + 'static>,
+        should_forget: bool,
+    ) -> Result<RecordReader<'static, T>, ReaderError> {
+        let flexible = self.flexible || self.flexible_default.is_some();
+        let mut reader = csv::ReaderBuilder::new()
+            .has_headers(self.has_headers)
+            .delimiter(self.delimiter)
+            .quote(self.quote_char)
+            .flexible(flexible)
+            .trim(self.trim)
+            .from_reader(readable);
+
+        let headers = RecordReader::<T>::get_headers(self.ruby, &mut reader, self.has_headers)?;
+        let static_headers = StringCache::intern_many(&headers)?;
+
+        Ok(RecordReader::new_multi_threaded(
+            reader,
+            static_headers,
+            self.buffer,
+            self.null_string,
+            self.flexible_default,
+            should_forget,
+        ))
+    }
+
+    pub fn build_threaded(self) -> Result<RecordReader<'static, T>, ReaderError> {
+        if self.to_read.is_kind_of(self.ruby.class_io()) {
+            let readable = self.handle_file_descriptor()?;
+            self.build_multi_threaded(readable, true)
+        } else if self.to_read.is_kind_of(self.ruby.class_string()) {
+            let readable = self.handle_file_path()?;
+            self.build_multi_threaded(readable, false)
+        } else {
+            let readable = build_ruby_reader(self.ruby, self.to_read)?;
+
+            self.build_single_threaded(readable)
+        }
+    }
+}
+
+impl<'a, T: RecordParser<'a> + Send> RecordReaderBuilder<'a, T> {
     pub fn new(ruby: &'a Ruby, to_read: Value) -> Self {
         Self {
             ruby,
@@ -110,7 +150,7 @@ impl<'a, T: RecordParser + Send + 'static> RecordReaderBuilder<'a, T> {
         self
     }
 
-    pub fn flexible_default(mut self, flexible_default: Option<String>) -> Self {
+    pub fn flexible_default(mut self, flexible_default: Option<&'a str>) -> Self {
         self.flexible_default = flexible_default;
         self
     }
@@ -120,12 +160,6 @@ impl<'a, T: RecordParser + Send + 'static> RecordReaderBuilder<'a, T> {
         self
     }
 
-    fn handle_string_io(&self) -> Result<Box<dyn Read + Send + 'static>, ReaderError> {
-        let string: RString = self.to_read.funcall("string", ())?;
-        let content = string.to_string()?;
-        Ok(Box::new(std::io::Cursor::new(content)))
-    }
-
     fn handle_file_descriptor(&self) -> Result<Box<dyn Read + Send + 'static>, ReaderError> {
         let raw_value = self.to_read.as_raw();
         let fd = std::panic::catch_unwind(|| unsafe { rb_sys::rb_io_descriptor(raw_value) })
@@ -155,101 +189,10 @@ impl<'a, T: RecordParser + Send + 'static> RecordReaderBuilder<'a, T> {
         })
     }
 
-    fn get_reader(&self) -> Result<(Box<dyn Read + Send + 'static>, bool), ReaderError> {
-        let string_io: magnus::RClass = self.ruby.eval("StringIO")?;
-        let gzip_reader_class: magnus::RClass = self.ruby.eval("Zlib::GzipReader")?;
-
-        if self.to_read.is_kind_of(string_io) {
-            self.handle_string_io().map(|r| (r, false))
-        } else if self.to_read.is_kind_of(gzip_reader_class) {
-            Err(ReaderError::UnsupportedGzipReader)
-        } else if self.to_read.is_kind_of(self.ruby.class_io()) {
-            self.handle_file_descriptor().map(|r| (r, true))
-        } else {
-            self.handle_file_path().map(|r| (r, false))
-        }
-    }
-
-    fn get_single_threaded_reader(&self) -> Result<Box<dyn Read>, ReaderError> {
-        let string_io: magnus::RClass = self.ruby.eval("StringIO")?;
-        let gzip_reader_class: magnus::RClass = self.ruby.eval("Zlib::GzipReader")?;
-
-        if self.to_read.is_kind_of(string_io) {
-            self.handle_string_io().map(|r| -> Box<dyn Read> { r })
-        } else if self.to_read.is_kind_of(gzip_reader_class) {
-            Ok(Box::new(RubyReader::new(self.to_read)))
-        } else if self.to_read.is_kind_of(self.ruby.class_io()) {
-            self.handle_file_descriptor()
-                .map(|r| -> Box<dyn Read> { r })
-        } else {
-            self.handle_file_path().map(|r| -> Box<dyn Read> { r })
-        }
-    }
-
-    pub fn build(self) -> Result<RecordReader<T>, ReaderError> {
-        match self.get_reader() {
-            Ok((readable, should_forget)) => self.build_multi_threaded(readable, should_forget),
-            Err(_) => {
-                let readable = self.get_single_threaded_reader()?;
-                self.build_single_threaded(readable)
-            }
-        }
-    }
-
-    fn build_multi_threaded(
-        self,
-        readable: Box<dyn Read + Send + 'static>,
-        should_forget: bool,
-    ) -> Result<RecordReader<T>, ReaderError> {
-        let flexible = self.flexible || self.flexible_default.is_some();
-        let mut reader = csv::ReaderBuilder::new()
-            .has_headers(self.has_headers)
-            .delimiter(self.delimiter)
-            .quote(self.quote_char)
-            .flexible(flexible)
-            .trim(self.trim)
-            .from_reader(readable);
-
-        let headers = RecordReader::<T>::get_headers(self.ruby, &mut reader, self.has_headers)?;
-        let static_headers = StringCache::intern_many(&headers)?;
-        let headers_for_cleanup = static_headers.clone();
-
-        let (sender, receiver) = kanal::bounded(self.buffer);
-        let null_string = self.null_string.clone();
-
-        let flexible_default = self.flexible_default.clone();
-        let handle = thread::spawn(move || {
-            let mut record = csv::StringRecord::with_capacity(READ_BUFFER_SIZE, headers.len());
-            while let Ok(true) = reader.read_record(&mut record) {
-                let row = T::parse(
-                    &static_headers,
-                    &record,
-                    null_string.as_deref(),
-                    flexible_default.as_deref(),
-                );
-                if sender.send(row).is_err() {
-                    break;
-                }
-            }
-            if should_forget {
-                let file_to_forget = reader.into_inner();
-                std::mem::forget(file_to_forget);
-            }
-        });
-
-        Ok(RecordReader {
-            reader: ReadImpl::MultiThreaded {
-                headers: headers_for_cleanup,
-                receiver,
-                handle: Some(handle),
-            },
-        })
-    }
-
     fn build_single_threaded(
         self,
-        readable: Box<dyn Read>,
-    ) -> Result<RecordReader<T>, ReaderError> {
+        readable: Box<dyn Read + 'a>,
+    ) -> Result<RecordReader<'a, T>, ReaderError> {
         let flexible = self.flexible || self.flexible_default.is_some();
         let mut reader = csv::ReaderBuilder::new()
             .has_headers(self.has_headers)
@@ -262,70 +205,11 @@ impl<'a, T: RecordParser + Send + 'static> RecordReaderBuilder<'a, T> {
         let headers = RecordReader::<T>::get_headers(self.ruby, &mut reader, self.has_headers)?;
         let static_headers = StringCache::intern_many(&headers)?;
 
-        Ok(RecordReader {
-            reader: ReadImpl::SingleThreaded {
-                reader,
-                headers: static_headers,
-                null_string: self.null_string,
-                flexible_default: self.flexible_default,
-            },
-        })
-    }
-}
-
-struct RubyReader {
-    inner: Value,
-    buffer: Option<Vec<u8>>,
-    offset: usize,
-}
-
-impl RubyReader {
-    fn new(inner: Value) -> Self {
-        Self {
-            inner,
-            buffer: None,
-            offset: 0,
-        }
-    }
-}
-
-// Read the entire inner into a vector and then read future reads from that vector with offset
-impl Read for RubyReader {
-    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
-        // If we have an existing buffer, read from it
-        if let Some(buffer) = self.buffer.as_ref() {
-            let remaining = buffer.len() - self.offset;
-            let copy_size = remaining.min(buf.len());
-            buf[..copy_size].copy_from_slice(&buffer[self.offset..self.offset + copy_size]);
-            self.offset += copy_size;
-            return Ok(copy_size);
-        }
-
-        // No buffer yet - read the entire content from Ruby
-        let result = self.inner.funcall::<_, _, Value>("read", ());
-        match result {
-            Ok(data) => {
-                if data.is_nil() {
-                    return Ok(0); // EOF
-                }
-
-                let string = RString::from_value(data).ok_or_else(|| {
-                    io::Error::new(io::ErrorKind::Other, "Failed to convert to RString")
-                })?;
-                let bytes = unsafe { string.as_slice() };
-
-                // Store the entire content in the buffer
-                self.buffer = Some(bytes.to_vec());
-                self.offset = 0;
-
-                // Read initial chunk
-                let copy_size = bytes.len().min(buf.len());
-                buf[..copy_size].copy_from_slice(&bytes[..copy_size]);
-                self.offset = copy_size;
-
-                Ok(copy_size)
-            }
-            Err(e) => Err(io::Error::new(io::ErrorKind::Other, e.to_string())),
-        }
+        Ok(RecordReader::new_single_threaded(
+            reader,
+            static_headers,
+            self.null_string,
+            self.flexible_default,
+        ))
     }
 }

data/ext/osv/src/csv/mod.rs

@@ -1,11 +1,12 @@
 mod builder;
 mod header_cache;
 mod parser;
-pub mod read_impl;
-mod reader;
 mod record;
+mod record_reader;
+mod ruby_reader;
 
 pub use builder::RecordReaderBuilder;
 pub(crate) use builder::BUFFER_CHANNEL_SIZE;
-pub(crate) use read_impl::READ_BUFFER_SIZE;
+pub use record::CowValue;
 pub use record::CsvRecord;
+pub(crate) use record_reader::READ_BUFFER_SIZE;