osv 0.3.17 → 0.3.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4469c67b2a39d9ffa23923e36cd894eac415ca004a432e700102a334af11efd8
-  data.tar.gz: 8dee3117fe6511b9c5b6005ae37d991891e0f314508986743b659080c7885855
+  metadata.gz: 87b1080bdb84d6db09a077d9eacc6c4792cfe93851671e4ba0438fe27c3d3218
+  data.tar.gz: f1dc704d251d906e9d6011c288c37844af8ccc976eb90611c629b7bc9cc9353e
 SHA512:
-  metadata.gz: d8c94dc1c576cca0043c7501752bdd6dee0c8bf0523d9c99a0e8ab4d614a0eb4e6f087fa62be97bb5816f9998f2c414758ffcab260e90889afada8379fb03aec
-  data.tar.gz: c51ece65a713af0b351a183415816302fcdc35ad598d0e5ee9e5b693c1ef66826c5dfc3dab90f04499c90e994e590e6dd7121999b5dfe54ce20997e41df0ac02
+  metadata.gz: aa7a083aebc9f480528de32fea2890189857c2381811f1cd90a8115724c75640a6bd11424e186492ace41a5fed28ee74dd17172f983801b48b31b81a750e24e9
+  data.tar.gz: 214c20e5011660e2250364ca79bd91a6c812053b401e9a1266abd500cf1e9e7c045f02054144f46190e45b459480aae70bbfab4d19ab246d1e1f5a8556f616a1

data/README.md

@@ -142,34 +142,34 @@ OSV - Gzipped Direct     1.000 i/100ms
    FastCSV - Gzipped     1.000 i/100ms
        CSV - Gzipped     1.000 i/100ms
 Calculating -------------------------------------
-      CSV - StringIO      0.080 (± 0.0%) i/s    (12.43 s/i) -      3.000 in  37.301114s
-  FastCSV - StringIO      0.368 (± 0.0%) i/s     (2.72 s/i) -     12.000 in  32.619020s
-      OSV - StringIO      0.699 (± 0.0%) i/s     (1.43 s/i) -     21.000 in  30.091225s
-   CSV - Hash output      0.059 (± 0.0%) i/s    (16.95 s/i) -      2.000 in  33.908533s
-   OSV - Hash output      0.329 (± 0.0%) i/s     (3.04 s/i) -     10.000 in  30.551275s
-  CSV - Array output      0.066 (± 0.0%) i/s    (15.18 s/i) -      2.000 in  30.357327s
-  OSV - Array output      0.632 (± 0.0%) i/s     (1.58 s/i) -     19.000 in  30.150113s
+      CSV - StringIO      0.083 (± 0.0%) i/s    (12.06 s/i) -      3.000 in  36.304469s
+  FastCSV - StringIO      0.335 (± 0.0%) i/s     (2.98 s/i) -     10.000 in  31.019521s
+      OSV - StringIO      0.705 (± 0.0%) i/s     (1.42 s/i) -     21.000 in  30.629511s
+   CSV - Hash output      0.060 (± 0.0%) i/s    (16.74 s/i) -      2.000 in  33.475977s
+   OSV - Hash output      0.434 (± 0.0%) i/s     (2.30 s/i) -     13.000 in  30.071679s
+  CSV - Array output      0.063 (± 0.0%) i/s    (15.88 s/i) -      2.000 in  32.229906s
+  OSV - Array output      0.406 (± 0.0%) i/s     (2.47 s/i) -     12.000 in  31.072600s
 FastCSV - Array output
-                          0.350 (± 0.0%) i/s     (2.86 s/i) -     11.000 in  31.477268s
+                          0.321 (± 0.0%) i/s     (3.11 s/i) -     10.000 in  31.458966s
 OSV - Direct Open Array output
-                          0.641 (± 0.0%) i/s     (1.56 s/i) -     20.000 in  31.275201s
-       OSV - Gzipped      0.530 (± 0.0%) i/s     (1.89 s/i) -     16.000 in  30.183753s
-OSV - Gzipped Direct      0.727 (± 0.0%) i/s     (1.37 s/i) -     22.000 in  30.283991s
-   FastCSV - Gzipped      0.323 (± 0.0%) i/s     (3.09 s/i) -     10.000 in  30.949600s
-       CSV - Gzipped      0.056 (± 0.0%) i/s    (17.72 s/i) -      2.000 in  35.440473s
+                          0.686 (± 0.0%) i/s     (1.46 s/i) -     21.000 in  30.639715s
+       OSV - Gzipped      0.524 (± 0.0%) i/s     (1.91 s/i) -     16.000 in  30.695259s
+OSV - Gzipped Direct      0.519 (± 0.0%) i/s     (1.93 s/i) -     16.000 in  30.830005s
+   FastCSV - Gzipped      0.313 (± 0.0%) i/s     (3.20 s/i) -     10.000 in  32.031002s
+       CSV - Gzipped      0.057 (± 0.0%) i/s    (17.55 s/i) -      2.000 in  35.107808s
 
 Comparison:
-OSV - Gzipped Direct:        0.7 i/s
-      OSV - StringIO:        0.7 i/s - 1.04x  slower
-OSV - Direct Open Array output:        0.6 i/s - 1.14x  slower
-  OSV - Array output:        0.6 i/s - 1.15x  slower
-       OSV - Gzipped:        0.5 i/s - 1.37x  slower
-  FastCSV - StringIO:        0.4 i/s - 1.98x  slower
-FastCSV - Array output:        0.3 i/s - 2.08x  slower
-   OSV - Hash output:        0.3 i/s - 2.21x  slower
-   FastCSV - Gzipped:        0.3 i/s - 2.25x  slower
-      CSV - StringIO:        0.1 i/s - 9.04x  slower
-  CSV - Array output:        0.1 i/s - 11.04x  slower
-   CSV - Hash output:        0.1 i/s - 12.33x  slower
-       CSV - Gzipped:        0.1 i/s - 12.89x  slower
+      OSV - StringIO          : 0.7 i/s
+OSV - Direct Open Array output: 0.7 i/s - 1.03x  slower
+       OSV - Gzipped          : 0.5 i/s - 1.34x  slower
+OSV - Gzipped Direct          : 0.5 i/s - 1.36x  slower
+   OSV - Hash output          : 0.4 i/s - 1.62x  slower
+  OSV - Array output          : 0.4 i/s - 1.74x  slower
+  FastCSV - StringIO          : 0.3 i/s - 2.10x  slower
+FastCSV - Array output        : 0.3 i/s - 2.20x  slower
+   FastCSV - Gzipped          : 0.3 i/s - 2.26x  slower
+      CSV - StringIO          : 0.1 i/s - 8.50x  slower
+  CSV - Array output          : 0.1 i/s - 11.20x  slower
+   CSV - Hash output          : 0.1 i/s - 11.80x  slower
+       CSV - Gzipped          : 0.1 i/s - 12.37x  slower
 ```

data/ext/osv/src/csv/builder.rs

@@ -67,6 +67,7 @@ pub struct RecordReaderBuilder<'a, T: RecordParser<'a>> {
     flexible: bool,
     flexible_default: Option<String>,
     trim: csv::Trim,
+    ignore_null_bytes: bool,
     _phantom: PhantomData<T>,
     _phantom_a: PhantomData<&'a ()>,
 }
@@ -84,6 +85,7 @@ impl<'a, T: RecordParser<'a>> RecordReaderBuilder<'a, T> {
             flexible: false,
             flexible_default: None,
             trim: csv::Trim::None,
+            ignore_null_bytes: false,
             _phantom: PhantomData,
             _phantom_a: PhantomData,
         }
@@ -138,6 +140,12 @@ impl<'a, T: RecordParser<'a>> RecordReaderBuilder<'a, T> {
         self
     }
 
+    #[must_use]
+    pub fn ignore_null_bytes(mut self, ignore_null_bytes: bool) -> Self {
+        self.ignore_null_bytes = ignore_null_bytes;
+        self
+    }
+
     /// Handles reading from a file descriptor.
     fn handle_file_descriptor(&self) -> Result<Box<dyn SeekableRead>, ReaderError> {
         let raw_value = self.to_read.as_raw();
@@ -191,7 +199,10 @@ impl<'a, T: RecordParser<'a>> RecordReaderBuilder<'a, T> {
             .trim(self.trim)
             .from_reader(reader);
 
-        let headers = RecordReader::<T>::get_headers(&self.ruby, &mut reader, self.has_headers)?;
+        let mut headers = RecordReader::<T>::get_headers(&self.ruby, &mut reader, self.has_headers)?;
+        if self.ignore_null_bytes {
+            headers = headers.iter().map(|h| h.replace("\0", "")).collect();
+        }
         let static_headers = StringCache::intern_many(&headers)?;
 
         // We intern both of these to get static string references we can reuse throughout the parser.
@@ -204,7 +215,7 @@ impl<'a, T: RecordParser<'a>> RecordReaderBuilder<'a, T> {
                     .map_err(|e| ReaderError::InvalidFlexibleDefault(format!("{:?}", e)))
             })
             .transpose()?
-            .map(|s| Cow::Borrowed(s));
+            .map(Cow::Borrowed);
 
         let null_string = self
             .null_string
@@ -215,13 +226,14 @@ impl<'a, T: RecordParser<'a>> RecordReaderBuilder<'a, T> {
                     .map_err(|e| ReaderError::InvalidNullString(format!("{:?}", e)))
             })
             .transpose()?
-            .map(|s| Cow::Borrowed(s));
+            .map(Cow::Borrowed);
 
         Ok(RecordReader::new(
             reader,
             static_headers,
             null_string,
             flexible_default,
+            self.ignore_null_bytes,
         ))
     }
 }

data/ext/osv/src/csv/header_cache.rs

@@ -99,7 +99,7 @@ impl StringCache {
                 counter.fetch_add(1, Ordering::Relaxed);
                 result.push(*interned_string);
             } else {
-                let interned = StringCacheKey::new(&string);
+                let interned = StringCacheKey::new(string);
                 let leaked = Box::leak(string.clone().into_boxed_str());
                 cache.insert(leaked, (interned, AtomicU32::new(1)));
                 result.push(interned);
@@ -154,7 +154,7 @@ impl<I: Iterator> Iterator for HeaderCacheCleanupIter<I> {
 impl<I> Drop for HeaderCacheCleanupIter<I> {
     fn drop(&mut self) {
         if let Some(headers) = self.headers.get() {
-            StringCache::clear(&headers).unwrap();
+            StringCache::clear(headers).unwrap();
         }
     }
 }

data/ext/osv/src/csv/parser.rs

@@ -13,6 +13,7 @@ pub trait RecordParser<'a> {
         record: &csv::StringRecord,
         null_string: Option<Cow<'a, str>>,
         flexible_default: Option<Cow<'a, str>>,
+        ignore_null_bytes: bool,
     ) -> Self::Output;
 }
 
@@ -27,12 +28,13 @@ impl<'a, S: BuildHasher + Default> RecordParser<'a>
         record: &csv::StringRecord,
         null_string: Option<Cow<'a, str>>,
         flexible_default: Option<Cow<'a, str>>,
+        ignore_null_bytes: bool,
     ) -> Self::Output {
         let mut map = HashMap::with_capacity_and_hasher(headers.len(), S::default());
 
         let shared_empty = Cow::Borrowed("");
         let shared_default = flexible_default.map(CowStr);
-        headers.iter().enumerate().for_each(|(i, ref header)| {
+        headers.iter().enumerate().for_each(|(i, header)| {
             let value = record.get(i).map_or_else(
                 || shared_default.clone(),
                 |field| {
@@ -40,12 +42,15 @@ impl<'a, S: BuildHasher + Default> RecordParser<'a>
                         None
                     } else if field.is_empty() {
                         Some(CowStr(shared_empty.clone()))
-                    } else {
+                    } else if ignore_null_bytes  {
+                        Some(CowStr(Cow::Owned(field.replace("\0", "").to_string())))
+                    }
+                    else {
                         Some(CowStr(Cow::Owned(field.to_string())))
                     }
                 },
             );
-            map.insert((*header).clone(), value);
+            map.insert(*header, value);
         });
         map
     }
@@ -60,6 +65,7 @@ impl<'a> RecordParser<'a> for Vec<Option<CowStr<'a>>> {
         record: &csv::StringRecord,
         null_string: Option<Cow<'a, str>>,
         flexible_default: Option<Cow<'a, str>>,
+        ignore_null_bytes: bool,
     ) -> Self::Output {
         let target_len = headers.len();
         let mut vec = Vec::with_capacity(target_len);
@@ -72,7 +78,10 @@ impl<'a> RecordParser<'a> for Vec<Option<CowStr<'a>>> {
                 None
             } else if field.is_empty() {
                 Some(CowStr(shared_empty.clone()))
-            } else {
+            } else if ignore_null_bytes  {
+                Some(CowStr(Cow::Owned(field.replace("\0", "").to_string())))
+            }
+            else {
                 Some(CowStr(Cow::Owned(field.to_string())))
             };
             vec.push(value);

data/ext/osv/src/csv/record_reader.rs

@@ -18,6 +18,7 @@ pub struct RecordReader<'a, T: RecordParser<'a>> {
     flexible_default: Option<Cow<'a, str>>,
     string_record: csv::StringRecord,
     parser: std::marker::PhantomData<T>,
+    ignore_null_bytes: bool,
 }
 
 impl<'a, T: RecordParser<'a>> RecordReader<'a, T> {
@@ -56,6 +57,7 @@ impl<'a, T: RecordParser<'a>> RecordReader<'a, T> {
         headers: Vec<StringCacheKey>,
         null_string: Option<Cow<'a, str>>,
         flexible_default: Option<Cow<'a, str>>,
+        ignore_null_bytes: bool,
     ) -> Self {
         let headers_len = headers.len();
         Self {
@@ -65,6 +67,7 @@ impl<'a, T: RecordParser<'a>> RecordReader<'a, T> {
             flexible_default,
             string_record: csv::StringRecord::with_capacity(READ_BUFFER_SIZE, headers_len),
             parser: std::marker::PhantomData,
+            ignore_null_bytes,
         }
     }
 
@@ -76,6 +79,7 @@ impl<'a, T: RecordParser<'a>> RecordReader<'a, T> {
                 &self.string_record,
                 self.null_string.clone(),
                 self.flexible_default.clone(),
+                self.ignore_null_bytes
             ))),
             false => Ok(None),
         }

data/ext/osv/src/reader.rs

@@ -60,6 +60,7 @@ pub fn parse_csv(
         flexible,
         flexible_default,
         trim,
+        ignore_null_bytes,
     } = parse_read_csv_args(&ruby, args)?;
 
     if !ruby.block_given() {
@@ -70,9 +71,9 @@ pub fn parse_csv(
             delimiter,
             quote_char,
             null_string,
-            result_type: result_type,
+            result_type,
             flexible,
-            flexible_default: flexible_default,
+            flexible_default,
             trim: match trim {
                 Trim::All => Some("all".to_string()),
                 Trim::Headers => Some("headers".to_string()),
@@ -100,9 +101,11 @@ pub fn parse_csv(
             .trim(trim)
             .delimiter(delimiter)
             .quote_char(quote_char)
-            .null_string(null_string);
+            .null_string(null_string)
+            .ignore_null_bytes(ignore_null_bytes)
+            .build()?;
 
-            Box::new(builder.build()?.map(CsvRecord::Map))
+            Box::new(builder.map(CsvRecord::Map))
         }
         ResultType::Array => {
             let builder = RecordReaderBuilder::<Vec<Option<CowStr<'static>>>>::new(ruby, to_read)
@@ -113,6 +116,7 @@ pub fn parse_csv(
                 .delimiter(delimiter)
                 .quote_char(quote_char)
                 .null_string(null_string)
+                .ignore_null_bytes(ignore_null_bytes)
                 .build()?;
 
             Box::new(builder.map(CsvRecord::Vec))

data/ext/osv/src/utils.rs

@@ -36,6 +36,7 @@ pub struct ReadCsvArgs {
     pub flexible: bool,
     pub flexible_default: Option<String>,
     pub trim: csv::Trim,
+    pub ignore_null_bytes: bool,
 }
 
 /// Parse common arguments for CSV parsing
@@ -47,14 +48,15 @@ pub fn parse_read_csv_args(ruby: &Ruby, args: &[Value]) -> Result<ReadCsvArgs, E
         _,
         (),
         (
-            Option<bool>,
-            Option<String>,
-            Option<String>,
+            Option<Option<bool>>,
             Option<Option<String>>,
-            Option<Value>,
-            Option<bool>,
             Option<Option<String>>,
-            Option<Value>,
+            Option<Option<String>>,
+            Option<Option<Value>>,
+            Option<Option<bool>>,
+            Option<Option<Option<String>>>,
+            Option<Option<Value>>,
+            Option<Option<bool>>,
         ),
         (),
     >(
@@ -69,14 +71,16 @@ pub fn parse_read_csv_args(ruby: &Ruby, args: &[Value]) -> Result<ReadCsvArgs, E
             "flexible",
             "flexible_default",
             "trim",
+            "ignore_null_bytes",
         ],
     )?;
 
-    let has_headers = kwargs.optional.0.unwrap_or(true);
+    let has_headers = kwargs.optional.0.flatten().unwrap_or(true);
 
     let delimiter = *kwargs
         .optional
         .1
+        .flatten()
         .unwrap_or_else(|| ",".to_string())
         .as_bytes()
         .first()
@@ -90,6 +94,7 @@ pub fn parse_read_csv_args(ruby: &Ruby, args: &[Value]) -> Result<ReadCsvArgs, E
     let quote_char = *kwargs
         .optional
         .2
+        .flatten()
         .unwrap_or_else(|| "\"".to_string())
         .as_bytes()
         .first()
@@ -105,6 +110,7 @@ pub fn parse_read_csv_args(ruby: &Ruby, args: &[Value]) -> Result<ReadCsvArgs, E
     let result_type = match kwargs
         .optional
         .4
+        .flatten()
         .map(|value| parse_string_or_symbol(ruby, value))
     {
         Some(Ok(Some(parsed))) => match parsed.as_str() {
@@ -126,13 +132,14 @@ pub fn parse_read_csv_args(ruby: &Ruby, args: &[Value]) -> Result<ReadCsvArgs, E
         None => String::from("hash"),
     };
 
-    let flexible = kwargs.optional.5.unwrap_or_default();
+    let flexible = kwargs.optional.5.flatten().unwrap_or_default();
 
-    let flexible_default = kwargs.optional.6.unwrap_or_default();
+    let flexible_default = kwargs.optional.6.flatten().unwrap_or_default();
 
     let trim = match kwargs
         .optional
         .7
+        .flatten()
         .map(|value| parse_string_or_symbol(ruby, value))
     {
         Some(Ok(Some(parsed))) => match parsed.as_str() {
@@ -159,6 +166,8 @@ pub fn parse_read_csv_args(ruby: &Ruby, args: &[Value]) -> Result<ReadCsvArgs, E
         None => csv::Trim::None,
     };
 
+    let ignore_null_bytes = kwargs.optional.8.flatten().unwrap_or_default();
+
     Ok(ReadCsvArgs {
         to_read,
         has_headers,
@@ -169,5 +178,6 @@ pub fn parse_read_csv_args(ruby: &Ruby, args: &[Value]) -> Result<ReadCsvArgs, E
         flexible,
         flexible_default,
         trim,
+        ignore_null_bytes,
     })
 }

data/lib/osv/version.rb

@@ -1,3 +1,3 @@
 module OSV
-  VERSION = "0.3.17"
+  VERSION = "0.3.19"
 end

data/lib/osv.rbi

@@ -23,6 +23,8 @@ module OSV
   #   - `trim`: String specifying the trim mode
   #             ("all" or "headers" or "fields" or :all or :headers or :fields)
   #             (default: `nil`)
+  #   - `ignore_null_bytes`: Boolean specifying if null bytes should be ignored
+  #                         (default: false)
   sig do
     params(
       input: T.any(String, StringIO, IO),
@@ -34,6 +36,7 @@ module OSV
       result_type: T.nilable(T.any(String, Symbol)),
       flexible: T.nilable(T::Boolean),
       flexible_default: T.nilable(String),
+      ignore_null_bytes: T.nilable(T::Boolean),
       trim: T.nilable(T.any(String, Symbol)),
       blk: T.nilable(T.proc.params(row: T.any(T::Hash[String, T.nilable(String)], T::Array[T.nilable(String)])).void)
     ).returns(T.any(Enumerator, T.untyped))
@@ -48,6 +51,7 @@ module OSV
     result_type: nil,
     flexible: nil,
     flexible_default: nil,
+    ignore_null_bytes: nil,
     trim: nil,
     &blk
   )

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: osv
 version: !ruby/object:Gem::Version
-  version: 0.3.17
+  version: 0.3.19
 platform: ruby
 authors:
 - Nathan Jaremko
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-01-06 00:00:00.000000000 Z
+date: 2025-01-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys