osv 0.3.18 → 0.3.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: da944a5af1cc88630fe0952e6e710d2acb8ac420ae8708a107064f5ecf444dec
-  data.tar.gz: bd6de3860ff2f47eb03b9019d307d647fa8c2e8f366543fbe95604f284871b62
+  metadata.gz: e5d6f70d75ffe07595c68bf9c1d4c26d3c6f8fcb3dd1f6bcde44de036558584d
+  data.tar.gz: 70e7d68fe4e42755ed54fd900ef8eb871741ff9c17e683ae7aa7d3f20801c7da
 SHA512:
-  metadata.gz: 8a130687fb25aaae3734f2e69c2258ccf893c584cd0c2893b751282b393ee4d52b2317a338f1ef68a864222e4947614ffdca7e6b98d8d37dc934dfede61f7bc1
-  data.tar.gz: 332a5dc1c6ce6df721b22f9e66b54d48426da3a0148917f9ec13036edd63e1fb70a950a2971964d289e076536af47d090c89fd95961d8ca4b51f1f1b8a221a98
+  metadata.gz: 1b2817f12cef251ee9006ba93483b7c7a529e3302579148ef998513b5401fa8eeef924ba3553b1a95515437d905239ce08e5ab4117160d2aabd37ebf4c70fbd8
+  data.tar.gz: 5beb1952d332923bc4703dfdd7b8e234d785f548c0c5724706aad78302234f5f76b4c0094968354ff29eee5d03d5ed2e6ff633649c7a0e6092291b9824f77eac

data/README.md

@@ -88,6 +88,7 @@ OSV.for_each("data.csv",
                          # Implicitly enables flexible mode if set.
   trim: :all,            # Whether to trim whitespace. Options are :all, :headers, or :fields (default: nil)
   buffer_size: 1024,     # Number of rows to buffer in memory (default: 1024)
+  ignore_null_bytes: false, # Boolean specifying if null bytes should be ignored (default: false)
 )
 ```
 
@@ -104,6 +105,7 @@ OSV.for_each("data.csv",
 - `flexible`: Boolean specifying if the parser should be flexible (default: false)
 - `flexible_default`: String specifying the default value for missing fields. Implicitly enables flexible mode if set. (default: `nil`)
 - `trim`: String specifying the trim mode ("all" or "headers" or "fields" or :all or :headers or :fields)
+- `ignore_null_bytes`: Boolean specifying if null bytes should be ignored (default: false)
 
 When `has_headers` is false, hash keys will be generated as `"c0"`, `"c1"`, etc.
 

data/ext/osv/src/csv/builder.rs

@@ -67,6 +67,7 @@ pub struct RecordReaderBuilder<'a, T: RecordParser<'a>> {
     flexible: bool,
     flexible_default: Option<String>,
     trim: csv::Trim,
+    ignore_null_bytes: bool,
     _phantom: PhantomData<T>,
     _phantom_a: PhantomData<&'a ()>,
 }
@@ -84,6 +85,7 @@ impl<'a, T: RecordParser<'a>> RecordReaderBuilder<'a, T> {
             flexible: false,
             flexible_default: None,
             trim: csv::Trim::None,
+            ignore_null_bytes: false,
             _phantom: PhantomData,
             _phantom_a: PhantomData,
         }
@@ -138,6 +140,12 @@ impl<'a, T: RecordParser<'a>> RecordReaderBuilder<'a, T> {
         self
     }
 
+    #[must_use]
+    pub fn ignore_null_bytes(mut self, ignore_null_bytes: bool) -> Self {
+        self.ignore_null_bytes = ignore_null_bytes;
+        self
+    }
+
     /// Handles reading from a file descriptor.
     fn handle_file_descriptor(&self) -> Result<Box<dyn SeekableRead>, ReaderError> {
         let raw_value = self.to_read.as_raw();
@@ -191,7 +199,10 @@ impl<'a, T: RecordParser<'a>> RecordReaderBuilder<'a, T> {
             .trim(self.trim)
             .from_reader(reader);
 
-        let headers = RecordReader::<T>::get_headers(&self.ruby, &mut reader, self.has_headers)?;
+        let mut headers = RecordReader::<T>::get_headers(&self.ruby, &mut reader, self.has_headers)?;
+        if self.ignore_null_bytes {
+            headers = headers.iter().map(|h| h.replace("\0", "")).collect();
+        }
         let static_headers = StringCache::intern_many(&headers)?;
 
         // We intern both of these to get static string references we can reuse throughout the parser.
@@ -204,7 +215,7 @@ impl<'a, T: RecordParser<'a>> RecordReaderBuilder<'a, T> {
                     .map_err(|e| ReaderError::InvalidFlexibleDefault(format!("{:?}", e)))
             })
             .transpose()?
-            .map(|s| Cow::Borrowed(s));
+            .map(Cow::Borrowed);
 
         let null_string = self
             .null_string
@@ -215,13 +226,14 @@ impl<'a, T: RecordParser<'a>> RecordReaderBuilder<'a, T> {
                     .map_err(|e| ReaderError::InvalidNullString(format!("{:?}", e)))
             })
             .transpose()?
-            .map(|s| Cow::Borrowed(s));
+            .map(Cow::Borrowed);
 
         Ok(RecordReader::new(
             reader,
             static_headers,
             null_string,
             flexible_default,
+            self.ignore_null_bytes,
         ))
     }
 }

data/ext/osv/src/csv/header_cache.rs

@@ -99,7 +99,7 @@ impl StringCache {
                 counter.fetch_add(1, Ordering::Relaxed);
                 result.push(*interned_string);
             } else {
-                let interned = StringCacheKey::new(&string);
+                let interned = StringCacheKey::new(string);
                 let leaked = Box::leak(string.clone().into_boxed_str());
                 cache.insert(leaked, (interned, AtomicU32::new(1)));
                 result.push(interned);
@@ -154,7 +154,7 @@ impl<I: Iterator> Iterator for HeaderCacheCleanupIter<I> {
 impl<I> Drop for HeaderCacheCleanupIter<I> {
     fn drop(&mut self) {
         if let Some(headers) = self.headers.get() {
-            StringCache::clear(&headers).unwrap();
+            StringCache::clear(headers).unwrap();
         }
     }
 }

data/ext/osv/src/csv/parser.rs

@@ -13,6 +13,7 @@ pub trait RecordParser<'a> {
         record: &csv::StringRecord,
         null_string: Option<Cow<'a, str>>,
         flexible_default: Option<Cow<'a, str>>,
+        ignore_null_bytes: bool,
     ) -> Self::Output;
 }
 
@@ -27,12 +28,13 @@ impl<'a, S: BuildHasher + Default> RecordParser<'a>
         record: &csv::StringRecord,
         null_string: Option<Cow<'a, str>>,
         flexible_default: Option<Cow<'a, str>>,
+        ignore_null_bytes: bool,
     ) -> Self::Output {
         let mut map = HashMap::with_capacity_and_hasher(headers.len(), S::default());
 
         let shared_empty = Cow::Borrowed("");
         let shared_default = flexible_default.map(CowStr);
-        headers.iter().enumerate().for_each(|(i, ref header)| {
+        headers.iter().enumerate().for_each(|(i, header)| {
             let value = record.get(i).map_or_else(
                 || shared_default.clone(),
                 |field| {
@@ -40,12 +42,15 @@ impl<'a, S: BuildHasher + Default> RecordParser<'a>
                         None
                     } else if field.is_empty() {
                         Some(CowStr(shared_empty.clone()))
-                    } else {
+                    } else if ignore_null_bytes  {
+                        Some(CowStr(Cow::Owned(field.replace("\0", ""))))
+                    }
+                    else {
                         Some(CowStr(Cow::Owned(field.to_string())))
                     }
                 },
             );
-            map.insert((*header).clone(), value);
+            map.insert(*header, value);
         });
         map
     }
@@ -60,6 +65,7 @@ impl<'a> RecordParser<'a> for Vec<Option<CowStr<'a>>> {
         record: &csv::StringRecord,
         null_string: Option<Cow<'a, str>>,
         flexible_default: Option<Cow<'a, str>>,
+        ignore_null_bytes: bool,
     ) -> Self::Output {
         let target_len = headers.len();
         let mut vec = Vec::with_capacity(target_len);
@@ -72,7 +78,10 @@ impl<'a> RecordParser<'a> for Vec<Option<CowStr<'a>>> {
                 None
             } else if field.is_empty() {
                 Some(CowStr(shared_empty.clone()))
-            } else {
+            } else if ignore_null_bytes  {
+                Some(CowStr(Cow::Owned(field.replace("\0", ""))))
+            }
+            else {
                 Some(CowStr(Cow::Owned(field.to_string())))
             };
             vec.push(value);

data/ext/osv/src/csv/record_reader.rs

@@ -18,6 +18,7 @@ pub struct RecordReader<'a, T: RecordParser<'a>> {
     flexible_default: Option<Cow<'a, str>>,
     string_record: csv::StringRecord,
     parser: std::marker::PhantomData<T>,
+    ignore_null_bytes: bool,
 }
 
 impl<'a, T: RecordParser<'a>> RecordReader<'a, T> {
@@ -56,6 +57,7 @@ impl<'a, T: RecordParser<'a>> RecordReader<'a, T> {
         headers: Vec<StringCacheKey>,
         null_string: Option<Cow<'a, str>>,
         flexible_default: Option<Cow<'a, str>>,
+        ignore_null_bytes: bool,
     ) -> Self {
         let headers_len = headers.len();
         Self {
@@ -65,6 +67,7 @@ impl<'a, T: RecordParser<'a>> RecordReader<'a, T> {
             flexible_default,
             string_record: csv::StringRecord::with_capacity(READ_BUFFER_SIZE, headers_len),
             parser: std::marker::PhantomData,
+            ignore_null_bytes,
         }
     }
 
@@ -76,6 +79,7 @@ impl<'a, T: RecordParser<'a>> RecordReader<'a, T> {
                 &self.string_record,
                 self.null_string.clone(),
                 self.flexible_default.clone(),
+                self.ignore_null_bytes
             ))),
             false => Ok(None),
         }

data/ext/osv/src/reader.rs

@@ -60,6 +60,7 @@ pub fn parse_csv(
         flexible,
         flexible_default,
         trim,
+        ignore_null_bytes,
     } = parse_read_csv_args(&ruby, args)?;
 
     if !ruby.block_given() {
@@ -70,9 +71,9 @@ pub fn parse_csv(
             delimiter,
             quote_char,
             null_string,
-            result_type: result_type,
+            result_type,
             flexible,
-            flexible_default: flexible_default,
+            flexible_default,
             trim: match trim {
                 Trim::All => Some("all".to_string()),
                 Trim::Headers => Some("headers".to_string()),
@@ -100,9 +101,11 @@ pub fn parse_csv(
             .trim(trim)
             .delimiter(delimiter)
             .quote_char(quote_char)
-            .null_string(null_string);
+            .null_string(null_string)
+            .ignore_null_bytes(ignore_null_bytes)
+            .build()?;
 
-            Box::new(builder.build()?.map(CsvRecord::Map))
+            Box::new(builder.map(CsvRecord::Map))
         }
         ResultType::Array => {
             let builder = RecordReaderBuilder::<Vec<Option<CowStr<'static>>>>::new(ruby, to_read)
@@ -113,6 +116,7 @@ pub fn parse_csv(
                 .delimiter(delimiter)
                 .quote_char(quote_char)
                 .null_string(null_string)
+                .ignore_null_bytes(ignore_null_bytes)
                 .build()?;
 
             Box::new(builder.map(CsvRecord::Vec))

data/ext/osv/src/utils.rs

@@ -36,6 +36,7 @@ pub struct ReadCsvArgs {
     pub flexible: bool,
     pub flexible_default: Option<String>,
     pub trim: csv::Trim,
+    pub ignore_null_bytes: bool,
 }
 
 /// Parse common arguments for CSV parsing
@@ -55,6 +56,7 @@ pub fn parse_read_csv_args(ruby: &Ruby, args: &[Value]) -> Result<ReadCsvArgs, E
             Option<Option<bool>>,
             Option<Option<Option<String>>>,
             Option<Option<Value>>,
+            Option<Option<bool>>,
         ),
         (),
     >(
@@ -69,6 +71,7 @@ pub fn parse_read_csv_args(ruby: &Ruby, args: &[Value]) -> Result<ReadCsvArgs, E
             "flexible",
             "flexible_default",
             "trim",
+            "ignore_null_bytes",
         ],
     )?;
 
@@ -163,6 +166,8 @@ pub fn parse_read_csv_args(ruby: &Ruby, args: &[Value]) -> Result<ReadCsvArgs, E
         None => csv::Trim::None,
     };
 
+    let ignore_null_bytes = kwargs.optional.8.flatten().unwrap_or_default();
+
     Ok(ReadCsvArgs {
         to_read,
         has_headers,
@@ -173,5 +178,6 @@ pub fn parse_read_csv_args(ruby: &Ruby, args: &[Value]) -> Result<ReadCsvArgs, E
         flexible,
         flexible_default,
         trim,
+        ignore_null_bytes,
     })
 }

data/lib/osv/version.rb

@@ -1,3 +1,3 @@
 module OSV
-  VERSION = "0.3.18"
+  VERSION = "0.3.20"
 end

data/lib/osv.rbi

@@ -23,6 +23,8 @@ module OSV
   #   - `trim`: String specifying the trim mode
   #             ("all" or "headers" or "fields" or :all or :headers or :fields)
   #             (default: `nil`)
+  #   - `ignore_null_bytes`: Boolean specifying if null bytes should be ignored
+  #                         (default: false)
   sig do
     params(
       input: T.any(String, StringIO, IO),
@@ -34,6 +36,7 @@ module OSV
       result_type: T.nilable(T.any(String, Symbol)),
       flexible: T.nilable(T::Boolean),
       flexible_default: T.nilable(String),
+      ignore_null_bytes: T.nilable(T::Boolean),
       trim: T.nilable(T.any(String, Symbol)),
       blk: T.nilable(T.proc.params(row: T.any(T::Hash[String, T.nilable(String)], T::Array[T.nilable(String)])).void)
     ).returns(T.any(Enumerator, T.untyped))
@@ -48,6 +51,7 @@ module OSV
     result_type: nil,
     flexible: nil,
     flexible_default: nil,
+    ignore_null_bytes: nil,
     trim: nil,
     &blk
   )

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: osv
 version: !ruby/object:Gem::Version
-  version: 0.3.18
+  version: 0.3.20
 platform: ruby
 authors:
 - Nathan Jaremko
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-01-13 00:00:00.000000000 Z
+date: 2025-01-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys