osv 0.3.22 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 26bda7b8aed144013156dea4f4f68c322b0a2042d6478c225edde0c44f54452f
-  data.tar.gz: cd63b6b71c158d8a09196a4fff496c5c7e7a9ac2c9a64724bbf31c56ff9ee0c7
+  metadata.gz: 51e4a387f1ed43bddc9f1f7a118637953d04239b5324ef131b9c860577ed4d41
+  data.tar.gz: e42928a09656216bbadcc2458953a8c5f28401ddf27095fc05038e0960471854
 SHA512:
-  metadata.gz: 947a7cc0d9f644977d157d0424893daf10c62efee5b4d544f81731ca7f04056cc75bab464560f4aea0b793b4b4e6e63a964fed8adace0c232cd388182a709a3a
-  data.tar.gz: 78536bdbba174b441792e39dc3b1c2ca08d85bafe4dce7763156160abcc611bb38c0743e6cadfd5d3466410b42fa0e598dc32e4865393109b8c88ec9673bf44b
+  metadata.gz: 4100c50a629ba5803db883532cfbe547eb3091e421b0876595d91791d8952a7b0169477c9c6f31063eafa5b91d0a9b1a9f0a5ae016d70cdd101e284beebfaf22
+  data.tar.gz: 90a822c644fcb37dc1892ede85a54395bc9e62a4b0b0a1af838182d390702d0ee4253151faafcedbf734b0a381fe2acf5c1ab23b842059fbdd4d51570fe33e58

data/README.md

@@ -84,11 +84,10 @@ OSV.for_each("data.csv",
 
   # Parsing behavior
   flexible: false,       # Allow varying number of fields (default: false)
-  flexible_default: nil, # Default value for missing fields. If unset, we ignore missing fields.
-                         # Implicitly enables flexible mode if set.
   trim: :all,            # Whether to trim whitespace. Options are :all, :headers, or :fields (default: nil)
   buffer_size: 1024,     # Number of rows to buffer in memory (default: 1024)
   ignore_null_bytes: false, # Boolean specifying if null bytes should be ignored (default: false)
+  lossy: false,             # Boolean specifying if invalid UTF-8 characters should be replaced with a replacement character (default: false)
 )
 ```
 
@@ -103,9 +102,9 @@ OSV.for_each("data.csv",
 - `buffer_size`: Integer specifying the number of rows to buffer in memory (default: 1024)
 - `result_type`: String specifying the output format ("hash" or "array" or :hash or :array)
 - `flexible`: Boolean specifying if the parser should be flexible (default: false)
-- `flexible_default`: String specifying the default value for missing fields. Implicitly enables flexible mode if set. (default: `nil`)
 - `trim`: String specifying the trim mode ("all" or "headers" or "fields" or :all or :headers or :fields)
 - `ignore_null_bytes`: Boolean specifying if null bytes should be ignored (default: false)
+- `lossy`: Boolean specifying if invalid UTF-8 characters should be replaced with a replacement character (default: false)
 
 When `has_headers` is false, hash keys will be generated as `"c0"`, `"c1"`, etc.
 

data/ext/osv/src/csv/builder.rs

@@ -79,9 +79,9 @@ pub struct RecordReaderBuilder<'a, T: RecordParser<'a>> {
     quote_char: u8,
     null_string: Option<String>,
     flexible: bool,
-    flexible_default: Option<String>,
     trim: csv::Trim,
     ignore_null_bytes: bool,
+    lossy: bool,
     _phantom: PhantomData<T>,
     _phantom_a: PhantomData<&'a ()>,
 }
@@ -97,9 +97,9 @@ impl<'a, T: RecordParser<'a>> RecordReaderBuilder<'a, T> {
             quote_char: b'"',
             null_string: None,
             flexible: false,
-            flexible_default: None,
             trim: csv::Trim::None,
             ignore_null_bytes: false,
+            lossy: false,
             _phantom: PhantomData,
             _phantom_a: PhantomData,
         }
@@ -140,13 +140,6 @@ impl<'a, T: RecordParser<'a>> RecordReaderBuilder<'a, T> {
         self
     }
 
-    /// Sets the default value for missing fields when in flexible mode.
-    #[must_use]
-    pub fn flexible_default(mut self, flexible_default: Option<String>) -> Self {
-        self.flexible_default = flexible_default;
-        self
-    }
-
     /// Sets the trimming mode for fields.
     #[must_use]
     pub fn trim(mut self, trim: csv::Trim) -> Self {
@@ -160,6 +153,12 @@ impl<'a, T: RecordParser<'a>> RecordReaderBuilder<'a, T> {
         self
     }
 
+    #[must_use]
+    pub fn lossy(mut self, lossy: bool) -> Self {
+        self.lossy = lossy;
+        self
+    }
+
     /// Handles reading from a file descriptor.
     fn handle_file_descriptor(&self) -> Result<Box<dyn SeekableRead>, ReaderError> {
         let raw_value = self.to_read.as_raw();
@@ -202,7 +201,7 @@ impl<'a, T: RecordParser<'a>> RecordReaderBuilder<'a, T> {
             build_ruby_reader(&self.ruby, self.to_read)?
         };
 
-        let flexible = self.flexible || self.flexible_default.is_some();
+        let flexible = self.flexible;
         let reader = BufReader::with_capacity(READ_BUFFER_SIZE, readable);
 
         let mut reader = csv::ReaderBuilder::new()
@@ -220,18 +219,6 @@ impl<'a, T: RecordParser<'a>> RecordReaderBuilder<'a, T> {
         }
         let static_headers = StringCache::intern_many(&headers)?;
 
-        // We intern both of these to get static string references we can reuse throughout the parser.
-        let flexible_default = self
-            .flexible_default
-            .map(|s| {
-                RString::new(&s)
-                    .to_interned_str()
-                    .as_str()
-                    .map_err(|e| ReaderError::InvalidFlexibleDefault(format!("{:?}", e)))
-            })
-            .transpose()?
-            .map(Cow::Borrowed);
-
         let null_string = self
             .null_string
             .map(|s| {
@@ -247,8 +234,8 @@ impl<'a, T: RecordParser<'a>> RecordReaderBuilder<'a, T> {
             reader,
             static_headers,
             null_string,
-            flexible_default,
             self.ignore_null_bytes,
+            self.lossy,
         ))
     }
 }

data/ext/osv/src/csv/parser.rs

@@ -5,14 +5,18 @@ use std::hash::BuildHasher;
 use super::header_cache::StringCacheKey;
 use super::CowStr;
 
+pub enum CsvRecordType {
+    String(csv::StringRecord),
+    Byte(csv::ByteRecord),
+}
+
 pub trait RecordParser<'a> {
     type Output;
 
     fn parse(
         headers: &[StringCacheKey],
-        record: &csv::StringRecord,
+        record: &CsvRecordType,
         null_string: Option<Cow<'a, str>>,
-        flexible_default: Option<Cow<'a, str>>,
         ignore_null_bytes: bool,
     ) -> Self::Output;
 }
@@ -25,20 +29,18 @@ impl<'a, S: BuildHasher + Default> RecordParser<'a>
     #[inline]
     fn parse(
         headers: &[StringCacheKey],
-        record: &csv::StringRecord,
+        record: &CsvRecordType,
         null_string: Option<Cow<'a, str>>,
-        flexible_default: Option<Cow<'a, str>>,
         ignore_null_bytes: bool,
     ) -> Self::Output {
         let mut map = HashMap::with_capacity_and_hasher(headers.len(), S::default());
 
         let shared_empty = Cow::Borrowed("");
-        let shared_default = flexible_default.map(CowStr);
+
         headers.iter().enumerate().for_each(|(i, header)| {
-            let value = record.get(i).map_or_else(
-                || shared_default.clone(),
-                |field| {
-                    if null_string.as_deref() == Some(field) {
+            let value = match record {
+                CsvRecordType::String(s) => s.get(i).and_then(|field| {
+                    if null_string.as_deref() == Some(field.as_ref()) {
                         None
                     } else if field.is_empty() {
                         Some(CowStr(shared_empty.clone()))
@@ -47,8 +49,22 @@ impl<'a, S: BuildHasher + Default> RecordParser<'a>
                     } else {
                         Some(CowStr(Cow::Owned(field.to_string())))
                     }
-                },
-            );
+                }),
+
+                CsvRecordType::Byte(b) => b.get(i).and_then(|field| {
+                    let field = String::from_utf8_lossy(field);
+                    if null_string.as_deref() == Some(field.as_ref()) {
+                        None
+                    } else if field.is_empty() {
+                        Some(CowStr(shared_empty.clone()))
+                    } else if ignore_null_bytes {
+                        Some(CowStr(Cow::Owned(field.replace("\0", ""))))
+                    } else {
+                        Some(CowStr(Cow::Owned(field.to_string())))
+                    }
+                }),
+            };
+
             map.insert(*header, value);
         });
         map
@@ -61,35 +77,47 @@ impl<'a> RecordParser<'a> for Vec<Option<CowStr<'a>>> {
     #[inline]
     fn parse(
         headers: &[StringCacheKey],
-        record: &csv::StringRecord,
+        record: &CsvRecordType,
         null_string: Option<Cow<'a, str>>,
-        flexible_default: Option<Cow<'a, str>>,
         ignore_null_bytes: bool,
     ) -> Self::Output {
         let target_len = headers.len();
         let mut vec = Vec::with_capacity(target_len);
 
         let shared_empty = Cow::Borrowed("");
-        let shared_default = flexible_default.map(CowStr);
-
-        for field in record.iter() {
-            let value = if Some(field) == null_string.as_deref() {
-                None
-            } else if field.is_empty() {
-                Some(CowStr(shared_empty.clone()))
-            } else if ignore_null_bytes {
-                Some(CowStr(Cow::Owned(field.replace("\0", ""))))
-            } else {
-                Some(CowStr(Cow::Owned(field.to_string())))
-            };
-            vec.push(value);
-        }
 
-        if vec.len() < target_len {
-            if let Some(default) = shared_default {
-                vec.resize_with(target_len, || Some(default.clone()));
+        match record {
+            CsvRecordType::String(record) => {
+                for field in record.iter() {
+                    let value = if Some(field.as_ref()) == null_string.as_deref() {
+                        None
+                    } else if field.is_empty() {
+                        Some(CowStr(shared_empty.clone()))
+                    } else if ignore_null_bytes {
+                        Some(CowStr(Cow::Owned(field.replace("\0", ""))))
+                    } else {
+                        Some(CowStr(Cow::Owned(field.to_string())))
+                    };
+                    vec.push(value);
+                }
+            }
+            CsvRecordType::Byte(record) => {
+                for field in record.iter() {
+                    let field = String::from_utf8_lossy(field);
+                    let value = if Some(field.as_ref()) == null_string.as_deref() {
+                        None
+                    } else if field.is_empty() {
+                        Some(CowStr(shared_empty.clone()))
+                    } else if ignore_null_bytes {
+                        Some(CowStr(Cow::Owned(field.replace("\0", ""))))
+                    } else {
+                        Some(CowStr(Cow::Owned(field.to_string())))
+                    };
+                    vec.push(value);
+                }
             }
         }
+
         vec
     }
 }

data/ext/osv/src/csv/record_reader.rs

@@ -1,6 +1,6 @@
 use super::builder::ReaderError;
 use super::header_cache::StringCacheKey;
-use super::parser::RecordParser;
+use super::parser::{CsvRecordType, RecordParser};
 use super::{header_cache::StringCache, ruby_reader::SeekableRead};
 use magnus::{Error, Ruby};
 use std::borrow::Cow;
@@ -16,8 +16,7 @@ pub struct RecordReader<'a, T: RecordParser<'a>> {
     reader: csv::Reader<BufReader<Box<dyn SeekableRead>>>,
     headers: Vec<StringCacheKey>,
     null_string: Option<Cow<'a, str>>,
-    flexible_default: Option<Cow<'a, str>>,
-    string_record: csv::StringRecord,
+    string_record: CsvRecordType,
     parser: std::marker::PhantomData<T>,
     ignore_null_bytes: bool,
 }
@@ -57,16 +56,25 @@ impl<'a, T: RecordParser<'a>> RecordReader<'a, T> {
         reader: csv::Reader<BufReader<Box<dyn SeekableRead>>>,
         headers: Vec<StringCacheKey>,
         null_string: Option<Cow<'a, str>>,
-        flexible_default: Option<Cow<'a, str>>,
         ignore_null_bytes: bool,
+        lossy: bool,
     ) -> Self {
         let headers_len = headers.len();
         Self {
             reader,
             headers,
             null_string,
-            flexible_default,
-            string_record: csv::StringRecord::with_capacity(READ_BUFFER_SIZE, headers_len),
+            string_record: if lossy {
+                CsvRecordType::Byte(csv::ByteRecord::with_capacity(
+                    READ_BUFFER_SIZE,
+                    headers_len,
+                ))
+            } else {
+                CsvRecordType::String(csv::StringRecord::with_capacity(
+                    READ_BUFFER_SIZE,
+                    headers_len,
+                ))
+            },
             parser: std::marker::PhantomData,
             ignore_null_bytes,
         }
@@ -74,12 +82,15 @@ impl<'a, T: RecordParser<'a>> RecordReader<'a, T> {
 
     /// Attempts to read the next record, returning any errors encountered.
     fn try_next(&mut self) -> Result<Option<T::Output>, ReaderError> {
-        if self.reader.read_record(&mut self.string_record)? {
+        let record = match self.string_record {
+            CsvRecordType::String(ref mut record) => self.reader.read_record(record),
+            CsvRecordType::Byte(ref mut record) => self.reader.read_byte_record(record),
+        }?;
+        if record {
             Ok(Some(T::parse(
                 &self.headers,
                 &self.string_record,
                 self.null_string.clone(),
-                self.flexible_default.clone(),
                 self.ignore_null_bytes,
             )))
         } else {

data/ext/osv/src/reader.rs

@@ -34,9 +34,9 @@ struct EnumeratorArgs {
     null_string: Option<String>,
     result_type: String,
     flexible: bool,
-    flexible_default: Option<String>,
     trim: Option<String>,
     ignore_null_bytes: bool,
+    lossy: bool,
 }
 
 /// Parses a CSV file with the given configuration.
@@ -56,9 +56,9 @@ pub fn parse_csv(rb_self: Value, args: &[Value]) -> Result<Value, Error> {
         null_string,
         result_type,
         flexible,
-        flexible_default,
         trim,
         ignore_null_bytes,
+        lossy,
     } = parse_read_csv_args(&ruby, args)?;
 
     if !ruby.block_given() {
@@ -71,7 +71,6 @@ pub fn parse_csv(rb_self: Value, args: &[Value]) -> Result<Value, Error> {
             null_string,
             result_type,
             flexible,
-            flexible_default,
             trim: match trim {
                 Trim::All => Some("all".to_string()),
                 Trim::Headers => Some("headers".to_string()),
@@ -79,6 +78,7 @@ pub fn parse_csv(rb_self: Value, args: &[Value]) -> Result<Value, Error> {
                 _ => None,
             },
             ignore_null_bytes,
+            lossy,
         })
         .map(|yield_enum| yield_enum.into_value_with(&ruby));
     }
@@ -97,12 +97,12 @@ pub fn parse_csv(rb_self: Value, args: &[Value]) -> Result<Value, Error> {
             >::new(ruby, to_read)
             .has_headers(has_headers)
             .flexible(flexible)
-            .flexible_default(flexible_default)
             .trim(trim)
             .delimiter(delimiter)
             .quote_char(quote_char)
             .null_string(null_string)
             .ignore_null_bytes(ignore_null_bytes)
+            .lossy(lossy)
             .build()?;
 
             let ruby = unsafe { Ruby::get_unchecked() };
@@ -115,12 +115,12 @@ pub fn parse_csv(rb_self: Value, args: &[Value]) -> Result<Value, Error> {
             let builder = RecordReaderBuilder::<Vec<Option<CowStr<'static>>>>::new(ruby, to_read)
                 .has_headers(has_headers)
                 .flexible(flexible)
-                .flexible_default(flexible_default)
                 .trim(trim)
                 .delimiter(delimiter)
                 .quote_char(quote_char)
                 .null_string(null_string)
                 .ignore_null_bytes(ignore_null_bytes)
+                .lossy(lossy)
                 .build()?;
 
             let ruby = unsafe { Ruby::get_unchecked() };
@@ -150,10 +150,9 @@ fn create_enumerator(args: EnumeratorArgs) -> Result<magnus::Enumerator, Error>
     kwargs.aset(Symbol::new("nil_string"), args.null_string)?;
     kwargs.aset(Symbol::new("result_type"), Symbol::new(args.result_type))?;
     kwargs.aset(Symbol::new("flexible"), args.flexible)?;
-    kwargs.aset(Symbol::new("flexible_default"), args.flexible_default)?;
     kwargs.aset(Symbol::new("trim"), args.trim.map(Symbol::new))?;
     kwargs.aset(Symbol::new("ignore_null_bytes"), args.ignore_null_bytes)?;
-
+    kwargs.aset(Symbol::new("lossy"), args.lossy)?;
     Ok(args
         .rb_self
         .enumeratorize("for_each", (args.to_read, KwArgs(kwargs))))

data/ext/osv/src/utils.rs

@@ -34,9 +34,9 @@ pub struct ReadCsvArgs {
     pub null_string: Option<String>,
     pub result_type: String,
     pub flexible: bool,
-    pub flexible_default: Option<String>,
     pub trim: csv::Trim,
     pub ignore_null_bytes: bool,
+    pub lossy: bool,
 }
 
 /// Parse common arguments for CSV parsing
@@ -54,9 +54,9 @@ pub fn parse_read_csv_args(ruby: &Ruby, args: &[Value]) -> Result<ReadCsvArgs, E
             Option<Option<String>>,
             Option<Option<Value>>,
             Option<Option<bool>>,
-            Option<Option<Option<String>>>,
             Option<Option<Value>>,
             Option<Option<bool>>,
+            Option<Option<bool>>,
         ),
         (),
     >(
@@ -69,9 +69,9 @@ pub fn parse_read_csv_args(ruby: &Ruby, args: &[Value]) -> Result<ReadCsvArgs, E
             "nil_string",
             "result_type",
             "flexible",
-            "flexible_default",
             "trim",
             "ignore_null_bytes",
+            "lossy",
         ],
     )?;
 
@@ -134,11 +134,9 @@ pub fn parse_read_csv_args(ruby: &Ruby, args: &[Value]) -> Result<ReadCsvArgs, E
 
     let flexible = kwargs.optional.5.flatten().unwrap_or_default();
 
-    let flexible_default = kwargs.optional.6.flatten().unwrap_or_default();
-
     let trim = match kwargs
         .optional
-        .7
+        .6
         .flatten()
         .map(|value| parse_string_or_symbol(ruby, value))
     {
@@ -166,7 +164,9 @@ pub fn parse_read_csv_args(ruby: &Ruby, args: &[Value]) -> Result<ReadCsvArgs, E
         None => csv::Trim::None,
     };
 
-    let ignore_null_bytes = kwargs.optional.8.flatten().unwrap_or_default();
+    let ignore_null_bytes = kwargs.optional.7.flatten().unwrap_or_default();
+
+    let lossy = kwargs.optional.8.flatten().unwrap_or_default();
 
     Ok(ReadCsvArgs {
         to_read,
@@ -176,8 +176,8 @@ pub fn parse_read_csv_args(ruby: &Ruby, args: &[Value]) -> Result<ReadCsvArgs, E
         null_string,
         result_type,
         flexible,
-        flexible_default,
         trim,
         ignore_null_bytes,
+        lossy,
     })
 }

data/lib/osv/version.rb

@@ -1,3 +1,3 @@
 module OSV
-  VERSION = "0.3.22"
+  VERSION = "0.4.0"
 end

data/lib/osv.rbi

@@ -17,14 +17,12 @@ module OSV
   #                    ("hash" or "array" or :hash or :array)
   #   - `flexible`: Boolean specifying if the parser should be flexible
   #                 (default: false)
-  #   - `flexible_default`: String specifying the default value for missing fields.
-  #                         Implicitly enables flexible mode if set.
-  #                         (default: `nil`)
   #   - `trim`: String specifying the trim mode
   #             ("all" or "headers" or "fields" or :all or :headers or :fields)
   #             (default: `nil`)
   #   - `ignore_null_bytes`: Boolean specifying if null bytes should be ignored
   #                         (default: false)
+  #   - `lossy`: Boolean specifying if invalid UTF-8 characters should be replaced with a replacement character
   sig do
     params(
       input: T.any(String, StringIO, IO),
@@ -35,7 +33,6 @@ module OSV
       buffer_size: T.nilable(Integer),
       result_type: T.nilable(T.any(String, Symbol)),
       flexible: T.nilable(T::Boolean),
-      flexible_default: T.nilable(String),
       ignore_null_bytes: T.nilable(T::Boolean),
       trim: T.nilable(T.any(String, Symbol)),
       blk: T.nilable(T.proc.params(row: T.any(T::Hash[String, T.nilable(String)], T::Array[T.nilable(String)])).void)
@@ -50,9 +47,9 @@ module OSV
     buffer_size: nil,
     result_type: nil,
     flexible: nil,
-    flexible_default: nil,
     ignore_null_bytes: nil,
     trim: nil,
+    lossy: nil,
     &blk
   )
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: osv
 version: !ruby/object:Gem::Version
-  version: 0.3.22
+  version: 0.4.0
 platform: ruby
 authors:
 - Nathan Jaremko