osv 0.3.14 → 0.3.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 689f28c935746890aa680fd2f649076a36d6ce233d4cbf2717dc129174b593dc
-  data.tar.gz: 45ddaa6774a9a4e9391d000b30b6e92afb8560b81821d8fec363d54283bac6d9
+  metadata.gz: 872cf06d1389f45f77b4eefc178cc8462ab165b833ab2c5bf4dc7f92e1c8308e
+  data.tar.gz: 84e6c5d0e03389966b8882a5a73f1698ddee3ed0edae24f2fd5b7f257935a98e
 SHA512:
-  metadata.gz: 74c2052ea9cbc61ddef5d1c46abdd5e4cdf7c60c946c421e4b8da7c160ba3f3eb761842279cd9f066aa6a1aa2214d0ef9ba9ff11c46294e8e1d4ebbb95161d70
-  data.tar.gz: 5a795e5fa6d84b39082c2754dea655cd5b4f8a00558627fb64f661a14ec32daa8ea7b31a13724291e11531d03c8c5fa1bdb928c6d9422b87fcbb5b5aba7daad5
+  metadata.gz: 445581447e8f5ec336da7843af715a5f5fbc298232a24f303a22eebb844f83f65ecc2e85d877a448119adae9e6a5529e377d87399a36e6f070562fa4ce0a11b7
+  data.tar.gz: '08f417b19b0549aa4a3db1538e4be413c5ec8faa3bd18e4c101a6fc3ea3e9496d04c30e39ea8eec9cc0cc3a38f8f83f7c2274e09c75259a26f3609620cf07a80'

data/Cargo.lock

@@ -1,6 +1,6 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
-version = 3
+version = 4
 
 [[package]]
 name = "adler2"
@@ -8,6 +8,19 @@ version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
 
+[[package]]
+name = "ahash"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
+dependencies = [
+ "cfg-if",
+ "getrandom",
+ "once_cell",
+ "version_check",
+ "zerocopy",
+]
+
 [[package]]
 name = "aho-corasick"
 version = "1.1.3"
@@ -49,6 +62,15 @@ version = "2.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
 
+[[package]]
+name = "cc"
+version = "1.2.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a012a0df96dd6d06ba9a1b29d6402d1a5d77c6befd2566afdc26e10603dc93d7"
+dependencies = [
+ "shlex",
+]
+
 [[package]]
 name = "cexpr"
 version = "0.6.0"
@@ -127,6 +149,17 @@ version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
 
+[[package]]
+name = "getrandom"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi",
+]
+
 [[package]]
 name = "glob"
 version = "0.3.1"
@@ -148,6 +181,26 @@ version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
 
+[[package]]
+name = "jemalloc-sys"
+version = "0.5.4+5.3.0-patched"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac6c1946e1cea1788cbfde01c993b52a10e2da07f4bac608228d1bed20bfebf2"
+dependencies = [
+ "cc",
+ "libc",
+]
+
+[[package]]
+name = "jemallocator"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0de374a9f8e63150e6f5e8a60cc14c668226d7a347d8aee1a45766e3c4dd3bc"
+dependencies = [
+ "jemalloc-sys",
+ "libc",
+]
+
 [[package]]
 name = "kanal"
 version = "0.1.0-pre8"
@@ -186,6 +239,16 @@ dependencies = [
  "windows-targets",
 ]
 
+[[package]]
+name = "libmimalloc-sys"
+version = "0.1.39"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23aa6811d3bd4deb8a84dde645f943476d13b248d818edcf8ce0b2f37f036b44"
+dependencies = [
+ "cc",
+ "libc",
+]
+
 [[package]]
 name = "lock_api"
 version = "0.4.12"
@@ -237,6 +300,15 @@ version = "2.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
 
+[[package]]
+name = "mimalloc"
+version = "0.1.43"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68914350ae34959d83f732418d51e2427a794055d0b9529f48259ac07af65633"
+dependencies = [
+ "libmimalloc-sys",
+]
+
 [[package]]
 name = "minimal-lexical"
 version = "0.2.1"
@@ -262,19 +334,27 @@ dependencies = [
  "minimal-lexical",
 ]
 
+[[package]]
+name = "once_cell"
+version = "1.20.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
+
 [[package]]
 name = "osv"
 version = "0.1.0"
 dependencies = [
+ "ahash",
  "csv",
  "flate2",
+ "jemallocator",
  "kanal",
  "magnus 0.7.1",
+ "mimalloc",
  "rb-sys",
  "serde",
  "serde_magnus",
  "thiserror",
- "xxhash-rust",
 ]
 
 [[package]]
@@ -464,6 +544,18 @@ version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83"
 
+[[package]]
+name = "version_check"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
+
+[[package]]
+name = "wasi"
+version = "0.11.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
+
 [[package]]
 name = "windows-targets"
 version = "0.52.6"
@@ -529,7 +621,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
 [[package]]
-name = "xxhash-rust"
-version = "0.8.14"
+name = "zerocopy"
+version = "0.7.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7d48f1b18be023c95e7b75f481cac649d74be7c507ff4a407c55cfb957f7934"
+checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
+dependencies = [
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.7.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]

data/README.md

@@ -118,10 +118,10 @@ This library is faster than the standard Ruby CSV library. It's also faster than
 
 Here's some unscientific benchmarks. You can find the code in the [benchmark/comparison_benchmark.rb](benchmark/comparison_benchmark.rb) file.
 
-### 1,000,000 lines
+### 1,000,000 records
 
 ```
-🏃 Running benchmarks...
+🏃  Running benchmarks...
 Benchmarking with 3000001 lines of data
 
 ruby 3.3.6 (2024-11-05 revision 75015d4c1f) +YJIT [arm64-darwin24]
@@ -142,34 +142,34 @@ OSV - Gzipped Direct     1.000 i/100ms
    FastCSV - Gzipped     1.000 i/100ms
        CSV - Gzipped     1.000 i/100ms
 Calculating -------------------------------------
-      CSV - StringIO      0.083 (± 0.0%) i/s    (12.01 s/i) -      3.000 in  36.028672s
-  FastCSV - StringIO      0.366 (± 0.0%) i/s     (2.73 s/i) -     11.000 in  30.032350s
-      OSV - StringIO      0.522 (± 0.0%) i/s     (1.92 s/i) -     16.000 in  30.655768s
-   CSV - Hash output      0.062 (± 0.0%) i/s    (16.16 s/i) -      2.000 in  32.311990s
-   OSV - Hash output      0.273 (± 0.0%) i/s     (3.66 s/i) -      9.000 in  32.924970s
-  CSV - Array output      0.069 (± 0.0%) i/s    (14.50 s/i) -      3.000 in  43.488185s
-  OSV - Array output      0.601 (± 0.0%) i/s     (1.66 s/i) -     19.000 in  31.636782s
+      CSV - StringIO      0.079 (± 0.0%) i/s    (12.69 s/i) -      3.000 in  38.139709s
+  FastCSV - StringIO      0.370 (± 0.0%) i/s     (2.71 s/i) -     12.000 in  32.474164s
+      OSV - StringIO      0.635 (± 0.0%) i/s     (1.58 s/i) -     19.000 in  30.772490s
+   CSV - Hash output      0.058 (± 0.0%) i/s    (17.11 s/i) -      2.000 in  34.212335s
+   OSV - Hash output      0.249 (± 0.0%) i/s     (4.01 s/i) -      8.000 in  32.124319s
+  CSV - Array output      0.066 (± 0.0%) i/s    (15.11 s/i) -      2.000 in  30.212137s
+  OSV - Array output      0.665 (± 0.0%) i/s     (1.50 s/i) -     20.000 in  30.813986s
 FastCSV - Array output
-                          0.356 (± 0.0%) i/s     (2.81 s/i) -     11.000 in  30.871931s
+                          0.351 (± 0.0%) i/s     (2.85 s/i) -     11.000 in  31.418786s
 OSV - Direct Open Array output
-                          0.604 (± 0.0%) i/s     (1.66 s/i) -     19.000 in  31.469190s
-       OSV - Gzipped      0.424 (± 0.0%) i/s     (2.36 s/i) -     13.000 in  30.642322s
-OSV - Gzipped Direct      0.636 (± 0.0%) i/s     (1.57 s/i) -     20.000 in  31.424083s
-   FastCSV - Gzipped      0.323 (± 0.0%) i/s     (3.10 s/i) -     10.000 in  30.990648s
-       CSV - Gzipped      0.058 (± 0.0%) i/s    (17.11 s/i) -      2.000 in  34.228691s
+                          0.713 (± 0.0%) i/s     (1.40 s/i) -     22.000 in  30.938525s
+       OSV - Gzipped      0.506 (± 0.0%) i/s     (1.98 s/i) -     16.000 in  31.709708s
+OSV - Gzipped Direct      0.685 (± 0.0%) i/s     (1.46 s/i) -     21.000 in  31.145435s
+   FastCSV - Gzipped      0.324 (± 0.0%) i/s     (3.09 s/i) -     10.000 in  30.983582s
+       CSV - Gzipped      0.057 (± 0.0%) i/s    (17.69 s/i) -      2.000 in  35.379009s
 
 Comparison:
-OSV - Gzipped Direct:        0.6 i/s
-OSV - Direct Open Array output:        0.6 i/s - 1.05x  slower
-  OSV - Array output:        0.6 i/s - 1.06x  slower
-      OSV - StringIO:        0.5 i/s - 1.22x  slower
-       OSV - Gzipped:        0.4 i/s - 1.50x  slower
-  FastCSV - StringIO:        0.4 i/s - 1.74x  slower
-FastCSV - Array output:        0.4 i/s - 1.79x  slower
-   FastCSV - Gzipped:        0.3 i/s - 1.97x  slower
-   OSV - Hash output:        0.3 i/s - 2.33x  slower
-      CSV - StringIO:        0.1 i/s - 7.64x  slower
-  CSV - Array output:        0.1 i/s - 9.23x  slower
-   CSV - Hash output:        0.1 i/s - 10.28x  slower
-       CSV - Gzipped:        0.1 i/s - 10.89x  slower
+OSV - Direct Open Array output:        0.7 i/s
+OSV - Gzipped Direct:        0.7 i/s - 1.04x  slower
+  OSV - Array output:        0.7 i/s - 1.07x  slower
+      OSV - StringIO:        0.6 i/s - 1.12x  slower
+       OSV - Gzipped:        0.5 i/s - 1.41x  slower
+  FastCSV - StringIO:        0.4 i/s - 1.93x  slower
+FastCSV - Array output:        0.4 i/s - 2.03x  slower
+   FastCSV - Gzipped:        0.3 i/s - 2.20x  slower
+   OSV - Hash output:        0.2 i/s - 2.86x  slower
+      CSV - StringIO:        0.1 i/s - 9.05x  slower
+  CSV - Array output:        0.1 i/s - 10.77x  slower
+   CSV - Hash output:        0.1 i/s - 12.20x  slower
+       CSV - Gzipped:        0.1 i/s - 12.61x  slower
 ```

data/ext/osv/Cargo.toml

@@ -7,6 +7,7 @@ edition = "2021"
 crate-type = ["cdylib"]
 
 [dependencies]
+ahash = "0.8"
 csv = "^1.3"
 flate2 = "1.0.35"
 kanal = "0.1.0-pre8"
@@ -15,4 +16,9 @@ rb-sys = "^0.9"
 serde = { version = "1.0", features = ["derive"] }
 serde_magnus = "0.8.1"
 thiserror = "2.0"
-xxhash-rust = { version = "0.8.12", features = ["xxh3"] }
+
+[target.'cfg(target_os = "linux")'.dependencies]
+jemallocator = { version = "0.5", features = ["disable_initial_exec_tls"] }
+
+[target.'cfg(not(any(target_os = "linux", target_os = "windows")))'.dependencies]
+mimalloc = { version = "0.1", default-features = false }

data/ext/osv/src/allocator.rs

@@ -0,0 +1,13 @@
+#[cfg(target_os = "linux")]
+use jemallocator::Jemalloc;
+
+#[cfg(not(any(target_os = "linux", target_os = "windows")))]
+use mimalloc::MiMalloc;
+
+#[global_allocator]
+#[cfg(target_os = "linux")]
+static ALLOC: Jemalloc = Jemalloc;
+
+#[global_allocator]
+#[cfg(not(any(target_os = "linux", target_os = "windows")))]
+static ALLOC: MiMalloc = MiMalloc;

data/ext/osv/src/csv/builder.rs

@@ -2,7 +2,8 @@ use super::{
     header_cache::{CacheError, StringCache},
     parser::RecordParser,
     record_reader::{RecordReader, READ_BUFFER_SIZE},
-    ruby_reader::build_ruby_reader,
+    ruby_reader::{build_ruby_reader, SeekableRead},
+    ForgottenFileHandle,
 };
 use flate2::read::GzDecoder;
 use magnus::{rb_sys::AsRawValue, value::ReprValue, Error as MagnusError, Ruby, Value};
@@ -10,6 +11,7 @@ use std::{
     fs::File,
     io::{self, BufReader, Read},
     marker::PhantomData,
+    mem::ManuallyDrop,
     os::fd::FromRawFd,
 };
 
@@ -64,7 +66,6 @@ impl<T: RecordParser<'static> + Send + 'static> RecordReaderBuilder<'static, T>
     fn build_multi_threaded(
         self,
         readable: Box<dyn Read + Send + 'static>,
-        should_forget: bool,
     ) -> Result<RecordReader<'static, T>, ReaderError> {
         let flexible = self.flexible || self.flexible_default.is_some();
         let mut reader = csv::ReaderBuilder::new()
@@ -84,21 +85,20 @@ impl<T: RecordParser<'static> + Send + 'static> RecordReaderBuilder<'static, T>
             self.buffer,
             self.null_string,
             self.flexible_default,
-            should_forget,
         ))
     }
 
     pub fn build_threaded(self) -> Result<RecordReader<'static, T>, ReaderError> {
         if self.to_read.is_kind_of(self.ruby.class_io()) {
             let readable = self.handle_file_descriptor()?;
-            self.build_multi_threaded(readable, true)
+            self.build_multi_threaded(readable)
         } else if self.to_read.is_kind_of(self.ruby.class_string()) {
             let readable = self.handle_file_path()?;
-            self.build_multi_threaded(readable, false)
+            self.build_multi_threaded(readable)
         } else {
             let readable = build_ruby_reader(self.ruby, self.to_read)?;
-
-            self.build_single_threaded(readable)
+            let buffered_reader = BufReader::with_capacity(READ_BUFFER_SIZE, readable);
+            self.build_single_threaded(buffered_reader)
         }
     }
 }
@@ -172,7 +172,11 @@ impl<'a, T: RecordParser<'a> + Send> RecordReaderBuilder<'a, T> {
         }
 
         let file = unsafe { File::from_raw_fd(fd) };
-        Ok(Box::new(BufReader::with_capacity(READ_BUFFER_SIZE, file)))
+        let forgotten = ForgottenFileHandle(ManuallyDrop::new(file));
+        Ok(Box::new(BufReader::with_capacity(
+            READ_BUFFER_SIZE,
+            forgotten,
+        )))
     }
 
     fn handle_file_path(&self) -> Result<Box<dyn Read + Send + 'static>, ReaderError> {
@@ -191,9 +195,10 @@ impl<'a, T: RecordParser<'a> + Send> RecordReaderBuilder<'a, T> {
 
     fn build_single_threaded(
         self,
-        readable: Box<dyn Read + 'a>,
+        readable: BufReader<Box<dyn SeekableRead>>,
     ) -> Result<RecordReader<'a, T>, ReaderError> {
         let flexible = self.flexible || self.flexible_default.is_some();
+
         let mut reader = csv::ReaderBuilder::new()
             .has_headers(self.has_headers)
             .delimiter(self.delimiter)

data/ext/osv/src/csv/mod.rs

@@ -3,10 +3,11 @@ mod header_cache;
 mod parser;
 mod record;
 mod record_reader;
+mod ruby_integration;
 mod ruby_reader;
 
 pub use builder::RecordReaderBuilder;
 pub(crate) use builder::BUFFER_CHANNEL_SIZE;
 pub use record::CowValue;
 pub use record::CsvRecord;
-pub(crate) use record_reader::READ_BUFFER_SIZE;
+pub use ruby_integration::*;

data/ext/osv/src/csv/parser.rs

@@ -30,7 +30,7 @@ impl<'a, S: BuildHasher + Default + 'a> RecordParser<'a>
         let mut map = HashMap::with_capacity_and_hasher(headers.len(), S::default());
 
         let shared_empty = Cow::Borrowed("");
-        let shared_default = flexible_default.map(|f| CowValue(f));
+        let shared_default = flexible_default.map(CowValue);
         headers.iter().enumerate().for_each(|(i, &header)| {
             let value = record.get(i).map_or_else(
                 || shared_default.clone(),
@@ -64,7 +64,7 @@ impl<'a> RecordParser<'a> for Vec<Option<CowValue<'a>>> {
         let mut vec = Vec::with_capacity(target_len);
 
         let shared_empty = Cow::Borrowed("");
-        let shared_default = flexible_default.map(|f| CowValue(f));
+        let shared_default = flexible_default.map(CowValue);
 
         for field in record.iter() {
             let value = if Some(field) == null_string {

data/ext/osv/src/csv/record.rs

@@ -7,7 +7,7 @@ pub enum CsvRecord<'a, S: BuildHasher + Default> {
     Map(HashMap<&'static str, Option<CowValue<'a>>, S>),
 }
 
-impl<'a, S: BuildHasher + Default> IntoValue for CsvRecord<'a, S> {
+impl<S: BuildHasher + Default> IntoValue for CsvRecord<'_, S> {
     #[inline]
     fn into_value_with(self, handle: &Ruby) -> Value {
         match self {
@@ -31,7 +31,7 @@ impl<'a, S: BuildHasher + Default> IntoValue for CsvRecord<'a, S> {
 #[derive(Debug, Clone)]
 pub struct CowValue<'a>(pub Cow<'a, str>);
 
-impl<'a> IntoValue for CowValue<'a> {
+impl IntoValue for CowValue<'_> {
     fn into_value_with(self, handle: &Ruby) -> Value {
         self.0.into_value_with(handle)
     }

data/ext/osv/src/csv/record_reader.rs

@@ -1,6 +1,7 @@
-use super::header_cache::StringCache;
 use super::parser::RecordParser;
+use super::{header_cache::StringCache, ruby_reader::SeekableRead};
 use magnus::{Error, Ruby};
+use std::io::BufReader;
 use std::{borrow::Cow, io::Read, thread};
 
 pub(crate) const READ_BUFFER_SIZE: usize = 16384;
@@ -9,9 +10,10 @@ pub struct RecordReader<'a, T: RecordParser<'a>> {
     inner: ReaderImpl<'a, T>,
 }
 
+#[allow(clippy::large_enum_variant)]
 enum ReaderImpl<'a, T: RecordParser<'a>> {
     SingleThreaded {
-        reader: csv::Reader<Box<dyn Read + 'a>>,
+        reader: csv::Reader<BufReader<Box<dyn SeekableRead>>>,
         headers: Vec<&'static str>,
         null_string: Option<String>,
         flexible_default: Option<Cow<'a, str>>,
@@ -48,7 +50,7 @@ impl<'a, T: RecordParser<'a>> RecordReader<'a, T> {
     }
 
     pub(crate) fn new_single_threaded(
-        reader: csv::Reader<Box<dyn Read + 'a>>,
+        reader: csv::Reader<BufReader<Box<dyn SeekableRead>>>,
         headers: Vec<&'static str>,
         null_string: Option<String>,
         flexible_default: Option<&'a str>,
@@ -59,7 +61,7 @@ impl<'a, T: RecordParser<'a>> RecordReader<'a, T> {
                 reader,
                 headers,
                 null_string,
-                flexible_default: flexible_default.map(|s| Cow::Borrowed(s)),
+                flexible_default: flexible_default.map(Cow::Borrowed),
                 string_record: csv::StringRecord::with_capacity(READ_BUFFER_SIZE, headers_len),
             },
         }
@@ -73,7 +75,6 @@ impl<T: RecordParser<'static> + Send> RecordReader<'static, T> {
         buffer_size: usize,
         null_string: Option<String>,
         flexible_default: Option<&'static str>,
-        should_forget: bool,
     ) -> Self {
         let (sender, receiver) = kanal::bounded(buffer_size);
         let headers_for_thread = headers.clone();
@@ -86,16 +87,12 @@ impl<T: RecordParser<'static> + Send> RecordReader<'static, T> {
                     &headers_for_thread,
                     &record,
                     null_string.as_deref(),
-                    flexible_default.map(|s| Cow::Borrowed(s)),
+                    flexible_default.map(Cow::Borrowed),
                 );
                 if sender.send(row).is_err() {
                     break;
                 }
             }
-            if should_forget {
-                let file_to_forget = reader.into_inner();
-                std::mem::forget(file_to_forget);
-            }
         });
 
         Self {
@@ -134,7 +131,7 @@ impl<'a, T: RecordParser<'a>> Iterator for RecordReader<'a, T> {
             } => match reader.read_record(string_record) {
                 Ok(true) => Some(T::parse(
                     headers,
-                    &string_record,
+                    string_record,
                     null_string.as_deref(),
                     flexible_default.clone(),
                 )),

data/ext/osv/src/csv/ruby_integration.rs

@@ -0,0 +1,30 @@
+use std::{fs::File, io, mem::ManuallyDrop};
+
+pub struct ForgottenFileHandle(pub ManuallyDrop<File>);
+
+impl std::io::Read for ForgottenFileHandle {
+    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+        self.0.read(buf)
+    }
+
+    fn read_vectored(&mut self, bufs: &mut [std::io::IoSliceMut<'_>]) -> io::Result<usize> {
+        self.0.read_vectored(bufs)
+    }
+
+    // fn read_buf(&mut self, cursor: BorrowedCursor<'_>) -> io::Result<()> {
+    //     self.0.read_buf(cursor)
+    // }
+
+    // #[inline]
+    // fn is_read_vectored(&self) -> bool {
+    //     self.0.is_read_vectored()
+    // }
+
+    fn read_to_end(&mut self, buf: &mut Vec<u8>) -> io::Result<usize> {
+        self.0.read_to_end(buf)
+    }
+
+    fn read_to_string(&mut self, buf: &mut String) -> io::Result<usize> {
+        self.0.read_to_string(buf)
+    }
+}

data/ext/osv/src/csv/ruby_reader.rs

@@ -1,43 +1,91 @@
-use super::READ_BUFFER_SIZE;
 use magnus::{
     value::{Opaque, ReprValue},
     RClass, RString, Ruby, Value,
 };
-use std::io::{self, Read};
+use std::io::{self, Read, Seek, SeekFrom, Write};
 use std::sync::OnceLock;
 
 static STRING_IO_CLASS: OnceLock<Opaque<RClass>> = OnceLock::new();
 
 /// A reader that can handle various Ruby input types (String, StringIO, IO-like objects)
 /// and provide a standard Read implementation for them.
-pub struct RubyReader<'a, T> {
-    #[allow(unused)]
-    ruby: &'a Ruby,
+pub struct RubyReader<T> {
     inner: T,
-    buffer: Option<Vec<u8>>,
     offset: usize,
-    // Number of bytes that have been read into the buffer
-    // Used as an upper bound for offset
-    buffered_bytes: usize,
 }
 
-pub fn build_ruby_reader<'a>(
-    ruby: &'a Ruby,
+pub trait SeekableRead: std::io::Read + Seek {}
+impl SeekableRead for RubyReader<Value> {}
+impl SeekableRead for RubyReader<RString> {}
+
+pub fn build_ruby_reader(
+    ruby: &Ruby,
     input: Value,
-) -> Result<Box<dyn Read + 'a>, magnus::Error> {
+) -> Result<Box<dyn SeekableRead>, magnus::Error> {
     if RubyReader::is_string_io(ruby, &input) {
         RubyReader::from_string_io(ruby, input)
     } else if RubyReader::is_io_like(&input) {
-        RubyReader::from_io(ruby, input)
+        RubyReader::from_io(input)
     } else {
-        RubyReader::from_string_like(ruby, input)
+        RubyReader::from_string_like(input)
     }
 }
 
-impl<'a> RubyReader<'a, Value> {
-    fn from_io(ruby: &'a Ruby, input: Value) -> Result<Box<dyn Read + 'a>, magnus::Error> {
+impl Seek for RubyReader<Value> {
+    fn seek(&mut self, pos: io::SeekFrom) -> io::Result<u64> {
+        let (whence, offset) = match pos {
+            SeekFrom::Start(i) => (0, i as i64),
+            SeekFrom::Current(i) => (1, i),
+            SeekFrom::End(i) => (2, i),
+        };
+
+        let new_position = self
+            .inner
+            .funcall("seek", (offset, whence))
+            .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?;
+
+        Ok(new_position)
+    }
+}
+
+impl Write for RubyReader<Value> {
+    fn write(&mut self, buf: &[u8]) -> Result<usize, io::Error> {
+        let ruby_bytes = RString::from_slice(buf);
+
+        let bytes_written = self
+            .inner
+            .funcall::<_, _, usize>("write", (ruby_bytes,))
+            .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?;
+
+        Ok(bytes_written)
+    }
+
+    fn flush(&mut self) -> Result<(), io::Error> {
+        self.inner
+            .funcall::<_, _, Value>("flush", ())
+            .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?;
+
+        Ok(())
+    }
+}
+
+impl Seek for RubyReader<RString> {
+    fn seek(&mut self, pos: io::SeekFrom) -> io::Result<u64> {
+        match pos {
+            io::SeekFrom::Start(offset) => self.offset = offset as usize,
+            io::SeekFrom::Current(offset) => self.offset = (self.offset as i64 + offset) as usize,
+            io::SeekFrom::End(offset) => {
+                self.offset = self.inner.len() - offset as usize
+            }
+        }
+        Ok(self.offset as u64)
+    }
+}
+
+impl RubyReader<Value> {
+    fn from_io(input: Value) -> Result<Box<dyn SeekableRead>, magnus::Error> {
         if Self::is_io_like(&input) {
-            Ok(Box::new(Self::from_io_like(ruby, input)))
+            Ok(Box::new(Self::from_io_like(input)))
         } else {
             Err(magnus::Error::new(
                 magnus::exception::type_error(),
@@ -50,70 +98,19 @@ impl<'a> RubyReader<'a, Value> {
         input.respond_to("read", false).unwrap_or(false)
     }
 
-    fn from_io_like(ruby: &'a Ruby, input: Value) -> Self {
+    fn from_io_like(input: Value) -> Self {
         Self {
-            ruby,
             inner: input,
-            buffer: Some(vec![0; READ_BUFFER_SIZE]),
             offset: 0,
-            buffered_bytes: 0,
         }
     }
-
-    fn read_from_buffer(&mut self, to_buf: &mut [u8]) -> Option<io::Result<usize>> {
-        if let Some(from_buf) = &self.buffer {
-            // If the offset is within the buffered bytes, copy the remaining bytes to the output buffer
-            if self.offset < self.buffered_bytes {
-                let remaining = self.buffered_bytes - self.offset;
-                let copy_size = remaining.min(to_buf.len());
-                to_buf[..copy_size]
-                    .copy_from_slice(&from_buf[self.offset..self.offset + copy_size]);
-                self.offset += copy_size;
-                Some(Ok(copy_size))
-            } else {
-                None
-            }
-        } else {
-            None
-        }
-    }
-
-    fn read_from_ruby(&mut self, buf: &mut [u8]) -> io::Result<usize> {
-        let buffer = self.buffer.as_mut().unwrap();
-        let result = self
-            .inner
-            .funcall::<_, _, RString>("read", (buffer.capacity(),))
-            .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?;
-
-        if result.is_nil() {
-            return Ok(0); // EOF
-        }
-
-        let bytes = unsafe { result.as_slice() };
-
-        // Update internal buffer
-        let bytes_len = bytes.len();
-        if bytes_len == 0 {
-            return Ok(0);
-        }
-
-        // Only copy what we actually read
-        buffer[..bytes_len].copy_from_slice(bytes);
-        self.buffered_bytes = bytes_len;
-
-        // Copy to output buffer
-        let copy_size = bytes_len.min(buf.len());
-        buf[..copy_size].copy_from_slice(&buffer[..copy_size]);
-        self.offset = copy_size;
-        Ok(copy_size)
-    }
 }
 
-impl<'a> RubyReader<'a, RString> {
+impl RubyReader<RString> {
     pub fn from_string_io(
-        ruby: &'a Ruby,
+        ruby: &Ruby,
         input: Value,
-    ) -> Result<Box<dyn Read + 'a>, magnus::Error> {
+    ) -> Result<Box<dyn SeekableRead>, magnus::Error> {
         if !Self::is_string_io(ruby, &input) {
             return Err(magnus::Error::new(
                 magnus::exception::type_error(),
@@ -123,11 +120,8 @@ impl<'a> RubyReader<'a, RString> {
 
         let string_content = input.funcall::<_, _, RString>("string", ()).unwrap();
         Ok(Box::new(Self {
-            ruby,
             inner: string_content,
-            buffer: None,
             offset: 0,
-            buffered_bytes: 0,
         }))
     }
 
@@ -139,33 +133,32 @@ impl<'a> RubyReader<'a, RString> {
         input.is_kind_of(ruby.get_inner(*string_io_class))
     }
 
-    fn from_string_like(ruby: &'a Ruby, input: Value) -> Result<Box<dyn Read + 'a>, magnus::Error> {
+    fn from_string_like(input: Value) -> Result<Box<dyn SeekableRead>, magnus::Error> {
         // Try calling `to_str`, and if that fails, try `to_s`
         let string_content = input
             .funcall::<_, _, RString>("to_str", ())
             .or_else(|_| input.funcall::<_, _, RString>("to_s", ()))?;
         Ok(Box::new(Self {
-            ruby,
             inner: string_content,
-            buffer: None,
             offset: 0,
-            buffered_bytes: 0,
         }))
     }
 }
 
-impl<'a> Read for RubyReader<'a, Value> {
-    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
-        if let Some(result) = self.read_from_buffer(buf) {
-            result
-        } else {
-            // If the buffer is empty, read from Ruby
-            self.read_from_ruby(buf)
-        }
+impl Read for RubyReader<Value> {
+    fn read(&mut self, mut buf: &mut [u8]) -> io::Result<usize> {
+        let bytes = self
+            .inner
+            .funcall::<_, _, RString>("read", (buf.len(),))
+            .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?;
+
+        buf.write_all(unsafe { bytes.as_slice() })?;
+
+        Ok(bytes.len())
     }
 }
 
-impl<'a> Read for RubyReader<'a, RString> {
+impl Read for RubyReader<RString> {
     fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
         let string_buffer = unsafe { self.inner.as_slice() };
         if self.offset >= string_buffer.len() {

data/ext/osv/src/lib.rs

@@ -1,3 +1,4 @@
+mod allocator;
 mod csv;
 mod reader;
 mod utils;

data/ext/osv/src/reader.rs

@@ -1,19 +1,19 @@
 use crate::csv::{CowValue, CsvRecord, RecordReaderBuilder};
 use crate::utils::*;
+use ahash::RandomState;
 use csv::Trim;
 use magnus::value::ReprValue;
 use magnus::{block::Yield, Error, KwArgs, RHash, Ruby, Symbol, Value};
 use std::collections::HashMap;
-use xxhash_rust::xxh3::Xxh3Builder;
 
-pub fn parse_csv<'a>(
+pub fn parse_csv(
     rb_self: Value,
     args: &[Value],
-) -> Result<Yield<Box<dyn Iterator<Item = CsvRecord<'static, Xxh3Builder>>>>, Error> {
+) -> Result<Yield<Box<dyn Iterator<Item = CsvRecord<'static, RandomState>>>>, Error> {
     let original = unsafe { Ruby::get_unchecked() };
     let ruby: &'static Ruby = Box::leak(Box::new(original));
 
-    let CsvArgs {
+    let ReadCsvArgs {
         to_read,
         has_headers,
         delimiter,
@@ -24,7 +24,7 @@ pub fn parse_csv<'a>(
         flexible,
         flexible_default,
         trim,
-    } = parse_csv_args(&ruby, args)?;
+    } = parse_read_csv_args(ruby, args)?;
 
     let flexible_default: &'static Option<String> = Box::leak(Box::new(flexible_default));
     let leaked_flexible_default: &'static Option<&str> =
@@ -51,11 +51,11 @@ pub fn parse_csv<'a>(
         });
     }
 
-    let iter: Box<dyn Iterator<Item = CsvRecord<Xxh3Builder>>> = match result_type.as_str() {
+    let iter: Box<dyn Iterator<Item = CsvRecord<RandomState>>> = match result_type.as_str() {
         "hash" => {
             let builder = RecordReaderBuilder::<
-                HashMap<&'static str, Option<CowValue<'static>>, Xxh3Builder>,
-            >::new(&ruby, to_read)
+                HashMap<&'static str, Option<CowValue<'static>>, RandomState>,
+            >::new(ruby, to_read)
             .has_headers(has_headers)
             .flexible(flexible)
             .flexible_default(flexible_default.as_deref())
@@ -68,7 +68,7 @@ pub fn parse_csv<'a>(
             Box::new(builder.build_threaded()?.map(CsvRecord::Map))
         }
         "array" => Box::new(
-            RecordReaderBuilder::<Vec<Option<CowValue<'static>>>>::new(&ruby, to_read)
+            RecordReaderBuilder::<Vec<Option<CowValue<'static>>>>::new(ruby, to_read)
                 .has_headers(has_headers)
                 .flexible(flexible)
                 .flexible_default(flexible_default.as_deref())
@@ -107,7 +107,7 @@ struct EnumeratorArgs {
 
 fn create_enumerator(
     args: EnumeratorArgs,
-) -> Result<Yield<Box<dyn Iterator<Item = CsvRecord<'static, Xxh3Builder>>>>, Error> {
+) -> Result<Yield<Box<dyn Iterator<Item = CsvRecord<'static, RandomState>>>>, Error> {
     let kwargs = RHash::new();
     kwargs.aset(Symbol::new("has_headers"), args.has_headers)?;
     kwargs.aset(

data/ext/osv/src/utils.rs

@@ -13,12 +13,12 @@ fn parse_string_or_symbol(ruby: &Ruby, value: Value) -> Result<Option<String>, E
         RString::from_value(value)
             .ok_or_else(|| Error::new(magnus::exception::type_error(), "Invalid string value"))?
             .to_string()
-            .map(|s| Some(s))
+            .map(Some)
     } else if value.is_kind_of(ruby.class_symbol()) {
         Symbol::from_value(value)
             .ok_or_else(|| Error::new(magnus::exception::type_error(), "Invalid symbol value"))?
             .funcall("to_s", ())
-            .map(|s| Some(s))
+            .map(Some)
     } else {
         Err(Error::new(
             magnus::exception::type_error(),
@@ -28,7 +28,7 @@ fn parse_string_or_symbol(ruby: &Ruby, value: Value) -> Result<Option<String>, E
 }
 
 #[derive(Debug)]
-pub struct CsvArgs {
+pub struct ReadCsvArgs {
     pub to_read: Value,
     pub has_headers: bool,
     pub delimiter: u8,
@@ -42,7 +42,7 @@ pub struct CsvArgs {
 }
 
 /// Parse common arguments for CSV parsing
-pub fn parse_csv_args(ruby: &Ruby, args: &[Value]) -> Result<CsvArgs, Error> {
+pub fn parse_read_csv_args(ruby: &Ruby, args: &[Value]) -> Result<ReadCsvArgs, Error> {
     let parsed_args = scan_args::<(Value,), (), (), (), _, ()>(args)?;
     let (to_read,) = parsed_args.required;
 
@@ -166,7 +166,7 @@ pub fn parse_csv_args(ruby: &Ruby, args: &[Value]) -> Result<CsvArgs, Error> {
         None => csv::Trim::None,
     };
 
-    Ok(CsvArgs {
+    Ok(ReadCsvArgs {
         to_read,
         has_headers,
         delimiter,

data/lib/osv/version.rb

@@ -1,3 +1,3 @@
 module OSV
-  VERSION = "0.3.14"
+  VERSION = "0.3.15"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: osv
 version: !ruby/object:Gem::Version
-  version: 0.3.14
+  version: 0.3.15
 platform: ruby
 authors:
 - Nathan Jaremko
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-01-02 00:00:00.000000000 Z
+date: 2025-01-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -59,12 +59,14 @@ files:
 - Rakefile
 - ext/osv/Cargo.toml
 - ext/osv/extconf.rb
+- ext/osv/src/allocator.rs
 - ext/osv/src/csv/builder.rs
 - ext/osv/src/csv/header_cache.rs
 - ext/osv/src/csv/mod.rs
 - ext/osv/src/csv/parser.rs
 - ext/osv/src/csv/record.rs
 - ext/osv/src/csv/record_reader.rs
+- ext/osv/src/csv/ruby_integration.rs
 - ext/osv/src/csv/ruby_reader.rs
 - ext/osv/src/lib.rs
 - ext/osv/src/reader.rs