osv 0.3.6 → 0.3.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6d51b65a56cd4a514b4e7746bc87d78be5abe033b390251d0e111ad4ad30813c
-  data.tar.gz: 59f4377530f2e5e54649b4c35a0ca78448f3a965279b358810ea9fcc17ddc5a8
+  metadata.gz: 02205de8cef4d5f7633c06720a9e925a2b608116354da4a1678d4746d2197d23
+  data.tar.gz: 3e1d63323fdaad1b6a60e0a0a63801f98710615d6616c882f0cdce00e36c6e2e
 SHA512:
-  metadata.gz: 397350596a09659e20fc9db840006649687db469cefcd5be09965ba147cc688957316cb9675d434594c3a19aa66efd2c734a41c71235dcc8d0b445c24e42f1a2
-  data.tar.gz: c225c5d016026a21afe8e90bf5a89e0465304236899d367b6484757073beefb1b75bb7381c37834bc2e344d1ee259d05e9ff82bba3d8214de098b3cf53eb571c
+  metadata.gz: df6a4a4b86c41010ea671ac0e98c2ee6307e62ceff35dab125868f0ee7edb6d14984348ecd4ac9f913489e5a6be0b364240b461334554385aabe5b3374fe798d
+  data.tar.gz: d931b888ce9d0ad1cdb1fa3d0be8cd0e526292206742f5adde718f414e9feca97eff3af6d4139d144c18a50e4807650ea9f7582153bcee80cea1e6ed4ce4ef49

data/README.md

@@ -71,6 +71,8 @@ Both methods support the following options:
   - if you want to interpret empty strings as nil, set this to an empty string
 - `buffer_size`: Integer specifying the read buffer size
 - `result_type`: String specifying the output format ("hash" or "array")
+- `flexible`: Boolean specifying if the parser should be flexible (default: false)
+- `flexible_default`: String specifying the default value for missing fields. Implicitly enables flexible mode if set. (default: `nil`)
 
 ### Input Sources
 

data/ext/osv/src/csv/builder.rs

@@ -54,6 +54,8 @@ pub struct RecordReaderBuilder<'a, T: RecordParser + Send + 'static> {
     quote_char: u8,
     null_string: Option<String>,
     buffer: usize,
+    flexible: bool,
+    flexible_default: Option<String>,
     _phantom: PhantomData<T>,
 }
 
@@ -67,6 +69,8 @@ impl<'a, T: RecordParser + Send + 'static> RecordReaderBuilder<'a, T> {
             quote_char: b'"',
             null_string: None,
             buffer: 1000,
+            flexible: false,
+            flexible_default: None,
             _phantom: PhantomData,
         }
     }
@@ -96,6 +100,16 @@ impl<'a, T: RecordParser + Send + 'static> RecordReaderBuilder<'a, T> {
         self
     }
 
+    pub fn flexible(mut self, flexible: bool) -> Self {
+        self.flexible = flexible;
+        self
+    }
+
+    pub fn flexible_default(mut self, flexible_default: Option<String>) -> Self {
+        self.flexible_default = flexible_default;
+        self
+    }
+
     fn handle_string_io(&self) -> Result<Box<dyn Read + Send + 'static>, ReaderError> {
         let string: RString = self.to_read.funcall("string", ())?;
         let content = string.to_string()?;
@@ -173,10 +187,12 @@ impl<'a, T: RecordParser + Send + 'static> RecordReaderBuilder<'a, T> {
         self,
         readable: Box<dyn Read + Send + 'static>,
     ) -> Result<RecordReader<T>, ReaderError> {
+        let flexible = self.flexible || self.flexible_default.is_some();
         let mut reader = csv::ReaderBuilder::new()
             .has_headers(self.has_headers)
             .delimiter(self.delimiter)
             .quote(self.quote_char)
+            .flexible(flexible)
             .from_reader(readable);
 
         let headers = RecordReader::<T>::get_headers(self.ruby, &mut reader, self.has_headers)?;
@@ -186,10 +202,16 @@ impl<'a, T: RecordParser + Send + 'static> RecordReaderBuilder<'a, T> {
         let (sender, receiver) = kanal::bounded(self.buffer);
         let null_string = self.null_string.clone();
 
+        let flexible_default = self.flexible_default.clone();
         let handle = thread::spawn(move || {
             let mut record = csv::StringRecord::new();
             while let Ok(true) = reader.read_record(&mut record) {
-                let row = T::parse(&static_headers, &record, null_string.as_deref());
+                let row = T::parse(
+                    &static_headers,
+                    &record,
+                    null_string.as_deref(),
+                    flexible_default.as_deref(),
+                );
                 if sender.send(row).is_err() {
                     break;
                 }
@@ -211,10 +233,12 @@ impl<'a, T: RecordParser + Send + 'static> RecordReaderBuilder<'a, T> {
         self,
         readable: Box<dyn Read>,
     ) -> Result<RecordReader<T>, ReaderError> {
+        let flexible = self.flexible || self.flexible_default.is_some();
         let mut reader = csv::ReaderBuilder::new()
             .has_headers(self.has_headers)
             .delimiter(self.delimiter)
             .quote(self.quote_char)
+            .flexible(flexible)
             .from_reader(readable);
 
         let headers = RecordReader::<T>::get_headers(self.ruby, &mut reader, self.has_headers)?;
@@ -225,6 +249,7 @@ impl<'a, T: RecordParser + Send + 'static> RecordReaderBuilder<'a, T> {
                 reader,
                 headers: static_headers,
                 null_string: self.null_string,
+                flexible_default: self.flexible_default,
             },
         })
     }

data/ext/osv/src/csv/parser.rs

@@ -7,6 +7,7 @@ pub trait RecordParser {
         headers: &[&'static str],
         record: &csv::StringRecord,
         null_string: Option<&str>,
+        flexible_default: Option<&str>,
     ) -> Self::Output;
 }
 
@@ -18,26 +19,24 @@ impl RecordParser for HashMap<&'static str, Option<String>> {
         headers: &[&'static str],
         record: &csv::StringRecord,
         null_string: Option<&str>,
+        flexible_default: Option<&str>,
     ) -> Self::Output {
         let mut map = HashMap::with_capacity(headers.len());
-        headers
-            .iter()
-            .zip(record.iter())
-            .for_each(|(header, field)| {
-                map.insert(
-                    *header,
+        headers.iter().enumerate().for_each(|(i, header)| {
+            let value = record.get(i).map_or_else(
+                || flexible_default.map(|s| s.to_string()),
+                |field| {
                     if null_string == Some(field) {
                         None
+                    } else if field.is_empty() {
+                        Some(String::new())
                     } else {
-                        // Avoid allocating for empty strings
-                        if field.is_empty() {
-                            Some(String::new())
-                        } else {
-                            Some(field.to_string())
-                        }
-                    },
-                );
-            });
+                        Some(field.to_string())
+                    }
+                },
+            );
+            map.insert(*header, value);
+        });
         map
     }
 }
@@ -47,23 +46,29 @@ impl RecordParser for Vec<Option<String>> {
 
     #[inline]
     fn parse(
-        _headers: &[&'static str],
+        headers: &[&'static str],
         record: &csv::StringRecord,
         null_string: Option<&str>,
+        flexible_default: Option<&str>,
     ) -> Self::Output {
-        let mut vec = Vec::with_capacity(record.len());
+        let target_len = headers.len();
+        let mut vec = Vec::with_capacity(target_len);
         vec.extend(record.iter().map(|field| {
             if null_string == Some(field) {
                 None
+            } else if field.is_empty() {
+                Some(String::new())
             } else {
-                // Avoid allocating for empty strings
-                if field.is_empty() {
-                    Some(String::new())
-                } else {
-                    Some(field.to_string())
-                }
+                Some(field.to_string())
             }
         }));
+
+        // Fill remaining slots with flexible_default if needed
+        if let Some(default) = flexible_default {
+            while vec.len() < target_len {
+                vec.push(Some(default.to_string()));
+            }
+        }
         vec
     }
 }

data/ext/osv/src/csv/read_impl.rs

@@ -6,6 +6,7 @@ pub enum ReadImpl<T: RecordParser> {
         reader: csv::Reader<Box<dyn Read>>,
         headers: Vec<&'static str>,
         null_string: Option<String>,
+        flexible_default: Option<String>,
     },
     MultiThreaded {
         headers: Vec<&'static str>,
@@ -33,10 +34,16 @@ impl<T: RecordParser> ReadImpl<T> {
                 reader,
                 headers,
                 null_string,
+                flexible_default,
             } => {
                 let mut record = csv::StringRecord::new();
                 match reader.read_record(&mut record) {
-                    Ok(true) => Some(T::parse(headers, &record, null_string.as_deref())),
+                    Ok(true) => Some(T::parse(
+                        headers,
+                        &record,
+                        null_string.as_deref(),
+                        flexible_default.as_deref(),
+                    )),
                     _ => None,
                 }
             }
@@ -50,6 +57,7 @@ impl<T: RecordParser> ReadImpl<T> {
                 receiver,
                 handle,
                 headers,
+                ..
             } => {
                 receiver.close();
                 if let Some(handle) = handle.take() {

data/ext/osv/src/reader.rs

@@ -18,6 +18,8 @@ pub fn parse_csv(
         null_string,
         buffer_size,
         result_type,
+        flexible,
+        flexible_default,
     } = parse_csv_args(&ruby, args)?;
 
     if !ruby.block_given() {
@@ -30,6 +32,8 @@ pub fn parse_csv(
             null_string,
             buffer_size,
             result_type,
+            flexible,
+            flexible_default,
         });
     }
 
@@ -37,6 +41,8 @@ pub fn parse_csv(
         "hash" => Box::new(
             RecordReaderBuilder::<HashMap<&'static str, Option<String>>>::new(&ruby, to_read)
                 .has_headers(has_headers)
+                .flexible(flexible)
+                .flexible_default(flexible_default)
                 .delimiter(delimiter)
                 .quote_char(quote_char)
                 .null_string(null_string)
@@ -47,6 +53,8 @@ pub fn parse_csv(
         "array" => Box::new(
             RecordReaderBuilder::<Vec<Option<String>>>::new(&ruby, to_read)
                 .has_headers(has_headers)
+                .flexible(flexible)
+                .flexible_default(flexible_default)
                 .delimiter(delimiter)
                 .quote_char(quote_char)
                 .null_string(null_string)
@@ -74,6 +82,8 @@ struct EnumeratorArgs {
     null_string: Option<String>,
     buffer_size: usize,
     result_type: String,
+    flexible: bool,
+    flexible_default: Option<String>,
 }
 
 fn create_enumerator(
@@ -92,7 +102,8 @@ fn create_enumerator(
     kwargs.aset(Symbol::new("nil_string"), args.null_string)?;
     kwargs.aset(Symbol::new("buffer_size"), args.buffer_size)?;
     kwargs.aset(Symbol::new("result_type"), Symbol::new(args.result_type))?;
-
+    kwargs.aset(Symbol::new("flexible"), args.flexible)?;
+    kwargs.aset(Symbol::new("flexible_default"), args.flexible_default)?;
     let enumerator = args
         .rb_self
         .enumeratorize("for_each", (args.to_read, KwArgs(kwargs)));

data/ext/osv/src/utils.rs

@@ -13,6 +13,8 @@ pub struct CsvArgs {
     pub null_string: Option<String>,
     pub buffer_size: usize,
     pub result_type: String,
+    pub flexible: bool,
+    pub flexible_default: Option<String>,
 }
 
 /// Parse common arguments for CSV parsing
@@ -30,6 +32,8 @@ pub fn parse_csv_args(ruby: &Ruby, args: &[Value]) -> Result<CsvArgs, Error> {
             Option<Option<String>>,
             Option<usize>,
             Option<Value>,
+            Option<bool>,
+            Option<Option<String>>,
         ),
         (),
     >(
@@ -42,6 +46,8 @@ pub fn parse_csv_args(ruby: &Ruby, args: &[Value]) -> Result<CsvArgs, Error> {
             "nil_string",
             "buffer_size",
             "result_type",
+            "flexible",
+            "flexible_default",
         ],
     )?;
 
@@ -111,6 +117,10 @@ pub fn parse_csv_args(ruby: &Ruby, args: &[Value]) -> Result<CsvArgs, Error> {
         None => String::from("hash"),
     };
 
+    let flexible = kwargs.optional.6.unwrap_or_default();
+
+    let flexible_default = kwargs.optional.7.unwrap_or_default();
+
     Ok(CsvArgs {
         to_read,
         has_headers,
@@ -119,5 +129,7 @@ pub fn parse_csv_args(ruby: &Ruby, args: &[Value]) -> Result<CsvArgs, Error> {
         null_string,
         buffer_size,
         result_type,
+        flexible,
+        flexible_default,
     })
 }

data/lib/osv/version.rb

@@ -1,3 +1,3 @@
 module OSV
-  VERSION = "0.3.6"
+  VERSION = "0.3.8"
 end

data/lib/osv.rbi

@@ -1,6 +1,25 @@
 # typed: strict
 
 module OSV
+  # Options:
+  #   - `has_headers`: Boolean indicating if the first row contains headers
+  #                    (default: true)
+  #   - `col_sep`: String specifying the field separator
+  #                (default: ",")
+  #   - `quote_char`: String specifying the quote character
+  #                   (default: "\"")
+  #   - `nil_string`: String that should be interpreted as nil
+  #                   By default, empty strings are interpreted as empty strings.
+  #                   If you want to interpret empty strings as nil, set this to
+  #                   an empty string.
+  #   - `buffer_size`: Integer specifying the read buffer size
+  #   - `result_type`: String specifying the output format
+  #                    ("hash" or "array")
+  #   - `flexible`: Boolean specifying if the parser should be flexible
+  #                 (default: false)
+  #   - `flexible_default`: String specifying the default value for missing fields.
+  #                         Implicitly enables flexible mode if set.
+  #                         (default: `nil`)
   sig do
     params(
       input: T.any(String, StringIO, IO),
@@ -10,6 +29,8 @@ module OSV
       nil_string: T.nilable(String),
       buffer_size: T.nilable(Integer),
       result_type: T.nilable(String),
+      flexible: T.nilable(T::Boolean),
+      flexible_default: T.nilable(String),
       blk: T.nilable(T.proc.params(row: T.any(T::Hash[String, T.nilable(String)], T::Array[T.nilable(String)])).void)
     ).returns(T.any(Enumerator, T.untyped))
   end
@@ -21,6 +42,8 @@ module OSV
     nil_string: nil,
     buffer_size: nil,
     result_type: nil,
+    flexible: nil,
+    flexible_default: nil,
     &blk
   )
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: osv
 version: !ruby/object:Gem::Version
-  version: 0.3.6
+  version: 0.3.8
 platform: ruby
 authors:
 - Nathan Jaremko