parquet 0.2.10 → 0.2.13

data/ext/parquet/src/types/record_types.rs

@@ -1,5 +1,7 @@
 use itertools::Itertools;
 
+use crate::reader::ReaderError;
+
 use super::*;
 
 #[derive(Debug)]
@@ -15,15 +17,16 @@ pub enum ColumnRecord<S: BuildHasher + Default> {
 }
 
 #[derive(Debug)]
-pub struct ParquetField(pub Field);
+pub struct ParquetField(pub Field, pub bool);
 
-impl<S: BuildHasher + Default> IntoValue for RowRecord<S> {
-    fn into_value_with(self, handle: &Ruby) -> Value {
+impl<S: BuildHasher + Default> TryIntoValue for RowRecord<S> {
+    fn try_into_value_with(self, handle: &Ruby) -> Result<Value, ReaderError> {
         match self {
             RowRecord::Vec(vec) => {
                 let ary = handle.ary_new_capa(vec.len());
-                vec.into_iter().try_for_each(|v| ary.push(v)).unwrap();
-                handle.into_value(ary)
+                vec.into_iter()
+                    .try_for_each(|v| ary.push(v.try_into_value_with(handle)?))?;
+                Ok(handle.into_value(ary))
             }
             RowRecord::Map(map) => {
                 let hash = handle.hash_new_capa(map.len());
@@ -36,41 +39,41 @@ impl<S: BuildHasher + Default> IntoValue for RowRecord<S> {
                     for (k, v) in chunk {
                         if i + 1 >= values.len() {
                             // Bulk insert current batch if array is full
-                            hash.bulk_insert(&values[..i]).unwrap();
+                            hash.bulk_insert(&values[..i])?;
                             values[..i].fill(handle.qnil().as_value());
                             i = 0;
                         }
                         values[i] = handle.into_value(k);
-                        values[i + 1] = handle.into_value(v);
+                        values[i + 1] = v.try_into_value_with(handle)?;
                         i += 2;
                     }
                     // Insert any remaining pairs
                     if i > 0 {
-                        hash.bulk_insert(&values[..i]).unwrap();
+                        hash.bulk_insert(&values[..i])?;
                         values[..i].fill(handle.qnil().as_value());
                         i = 0;
                     }
                 }
 
-                hash.into_value_with(handle)
+                Ok(hash.into_value_with(handle))
             }
         }
     }
 }
 
-impl<S: BuildHasher + Default> IntoValue for ColumnRecord<S> {
-    fn into_value_with(self, handle: &Ruby) -> Value {
+impl<S: BuildHasher + Default> TryIntoValue for ColumnRecord<S> {
+    fn try_into_value_with(self, handle: &Ruby) -> Result<Value, ReaderError> {
         match self {
             ColumnRecord::Vec(vec) => {
                 let ary = handle.ary_new_capa(vec.len());
-                vec.into_iter()
-                    .try_for_each(|v| {
-                        let nested_ary = handle.ary_new_capa(v.len());
-                        v.into_iter().try_for_each(|v| nested_ary.push(v)).unwrap();
-                        ary.push(nested_ary.into_value_with(handle))
-                    })
-                    .unwrap();
-                ary.into_value_with(handle)
+                vec.into_iter().try_for_each(|v| {
+                    let nested_ary = handle.ary_new_capa(v.len());
+                    v.into_iter()
+                        .try_for_each(|v| nested_ary.push(v.try_into_value_with(handle)?))?;
+                    ary.push(nested_ary.into_value_with(handle))?;
+                    Ok::<_, ReaderError>(())
+                })?;
+                Ok(ary.into_value_with(handle))
             }
             ColumnRecord::Map(map) => {
                 let hash = handle.hash_new_capa(map.len());
@@ -83,91 +86,98 @@ impl<S: BuildHasher + Default> IntoValue for ColumnRecord<S> {
                     for (k, v) in chunk {
                         if i + 1 >= values.len() {
                             // Bulk insert current batch if array is full
-                            hash.bulk_insert(&values[..i]).unwrap();
+                            hash.bulk_insert(&values[..i])?;
                             values[..i].fill(handle.qnil().as_value());
                             i = 0;
                         }
                         values[i] = handle.into_value(k);
                         let ary = handle.ary_new_capa(v.len());
-                        v.into_iter().try_for_each(|v| ary.push(v)).unwrap();
+                        v.into_iter()
+                            .try_for_each(|v| ary.push(v.try_into_value_with(handle)?))?;
                         values[i + 1] = handle.into_value(ary);
                         i += 2;
                     }
                     // Insert any remaining pairs
                     if i > 0 {
-                        hash.bulk_insert(&values[..i]).unwrap();
+                        hash.bulk_insert(&values[..i])?;
                         values[..i].fill(handle.qnil().as_value());
                         i = 0;
                     }
                 }
 
-                hash.into_value_with(handle)
+                Ok(hash.into_value_with(handle))
             }
         }
     }
 }
 
-impl IntoValue for ParquetField {
-    fn into_value_with(self, handle: &Ruby) -> Value {
+pub trait TryIntoValue {
+    fn try_into_value_with(self, handle: &Ruby) -> Result<Value, ReaderError>;
+}
+
+impl TryIntoValue for ParquetField {
+    fn try_into_value_with(self, handle: &Ruby) -> Result<Value, ReaderError> {
         match self.0 {
-            Field::Null => handle.qnil().as_value(),
-            Field::Bool(b) => b.into_value_with(handle),
-            Field::Short(s) => s.into_value_with(handle),
-            Field::Int(i) => i.into_value_with(handle),
-            Field::Long(l) => l.into_value_with(handle),
-            Field::UByte(ub) => ub.into_value_with(handle),
-            Field::UShort(us) => us.into_value_with(handle),
-            Field::UInt(ui) => ui.into_value_with(handle),
-            Field::ULong(ul) => ul.into_value_with(handle),
-            Field::Float16(f) => f32::from(f).into_value_with(handle),
-            Field::Float(f) => f.into_value_with(handle),
-            Field::Double(d) => d.into_value_with(handle),
-            Field::Str(s) => s.into_value_with(handle),
-            Field::Byte(b) => b.into_value_with(handle),
-            Field::Bytes(b) => handle.str_from_slice(b.data()).as_value(),
+            Field::Null => Ok(handle.qnil().as_value()),
+            Field::Bool(b) => Ok(b.into_value_with(handle)),
+            Field::Short(s) => Ok(s.into_value_with(handle)),
+            Field::Int(i) => Ok(i.into_value_with(handle)),
+            Field::Long(l) => Ok(l.into_value_with(handle)),
+            Field::UByte(ub) => Ok(ub.into_value_with(handle)),
+            Field::UShort(us) => Ok(us.into_value_with(handle)),
+            Field::UInt(ui) => Ok(ui.into_value_with(handle)),
+            Field::ULong(ul) => Ok(ul.into_value_with(handle)),
+            Field::Float16(f) => Ok(f32::from(f).into_value_with(handle)),
+            Field::Float(f) => Ok(f.into_value_with(handle)),
+            Field::Double(d) => Ok(d.into_value_with(handle)),
+            Field::Str(s) => {
+                if self.1 {
+                    Ok(simdutf8::basic::from_utf8(s.as_bytes())
+                        .map_err(|e| ReaderError::Utf8Error(e))
+                        .and_then(|s| Ok(s.into_value_with(handle)))?)
+                } else {
+                    Ok(handle.str_from_slice(s.as_bytes()).as_value())
+                }
+            }
+            Field::Byte(b) => Ok(b.into_value_with(handle)),
+            Field::Bytes(b) => Ok(handle.str_from_slice(b.data()).as_value()),
             Field::Date(d) => {
-                let ts = jiff::Timestamp::from_second((d as i64) * 86400).unwrap();
+                let ts = jiff::Timestamp::from_second((d as i64) * 86400)?;
                 let formatted = ts.strftime("%Y-%m-%d").to_string();
-                formatted.into_value_with(handle)
+                Ok(formatted.into_value_with(handle))
             }
             Field::TimestampMillis(ts) => {
-                let ts = jiff::Timestamp::from_millisecond(ts).unwrap();
+                let ts = jiff::Timestamp::from_millisecond(ts)?;
                 let time_class = handle.class_time();
-                time_class
-                    .funcall::<_, _, Value>("parse", (ts.to_string(),))
-                    .unwrap()
-                    .into_value_with(handle)
+                Ok(time_class
+                    .funcall::<_, _, Value>("parse", (ts.to_string(),))?
+                    .into_value_with(handle))
             }
             Field::TimestampMicros(ts) => {
-                let ts = jiff::Timestamp::from_microsecond(ts).unwrap();
+                let ts = jiff::Timestamp::from_microsecond(ts)?;
                 let time_class = handle.class_time();
-                time_class
-                    .funcall::<_, _, Value>("parse", (ts.to_string(),))
-                    .unwrap()
-                    .into_value_with(handle)
+                Ok(time_class
+                    .funcall::<_, _, Value>("parse", (ts.to_string(),))?
+                    .into_value_with(handle))
             }
             Field::ListInternal(list) => {
                 let elements = list.elements();
                 let ary = handle.ary_new_capa(elements.len());
-                elements
-                    .iter()
-                    .try_for_each(|e| ary.push(ParquetField(e.clone()).into_value_with(handle)))
-                    .unwrap();
-                ary.into_value_with(handle)
+                elements.iter().try_for_each(|e| {
+                    ary.push(ParquetField(e.clone(), self.1).try_into_value_with(handle)?)
+                })?;
+                Ok(ary.into_value_with(handle))
             }
             Field::MapInternal(map) => {
                 let entries = map.entries();
                 let hash = handle.hash_new_capa(entries.len());
-                entries
-                    .iter()
-                    .try_for_each(|(k, v)| {
-                        hash.aset(
-                            ParquetField(k.clone()).into_value_with(handle),
-                            ParquetField(v.clone()).into_value_with(handle),
-                        )
-                    })
-                    .unwrap();
-                hash.into_value_with(handle)
+                entries.iter().try_for_each(|(k, v)| {
+                    hash.aset(
+                        ParquetField(k.clone(), self.1).try_into_value_with(handle)?,
+                        ParquetField(v.clone(), self.1).try_into_value_with(handle)?,
+                    )
+                })?;
+                Ok(hash.into_value_with(handle))
             }
             Field::Decimal(d) => {
                 let value = match d {
@@ -185,20 +195,24 @@ impl IntoValue for ParquetField {
                         format!("{}e-{}", unscaled, scale)
                     }
                 };
-                handle.eval(&format!("BigDecimal(\"{value}\")")).unwrap()
+                Ok(handle.eval(&format!("BigDecimal(\"{value}\")"))?)
             }
             Field::Group(row) => {
                 let hash = handle.hash_new();
-                row.get_column_iter()
-                    .try_for_each(|(k, v)| {
-                        hash.aset(
-                            k.clone().into_value_with(handle),
-                            ParquetField(v.clone()).into_value_with(handle),
-                        )
-                    })
-                    .unwrap();
-                hash.into_value_with(handle)
+                row.get_column_iter().try_for_each(|(k, v)| {
+                    hash.aset(
+                        k.clone().into_value_with(handle),
+                        ParquetField(v.clone(), self.1).try_into_value_with(handle)?,
+                    )
+                })?;
+                Ok(hash.into_value_with(handle))
             }
         }
     }
 }
+
+// impl IntoValue for ParquetField {
+//     fn into_value_with(self, handle: &Ruby) -> Value {
+//         self.try_into_value_with(handle).unwrap()
+//     }
+// }

data/ext/parquet/src/types/timestamp.rs

@@ -64,10 +64,9 @@ macro_rules! impl_timestamp_conversion {
             ParquetValue::$unit(ts, tz) => {
                 let ts = parse_zoned_timestamp(&ParquetValue::$unit(ts, tz));
                 let time_class = $handle.class_time();
-                time_class
-                    .funcall::<_, _, Value>("parse", (ts.to_string(),))
-                    .unwrap()
-                    .into_value_with($handle)
+                Ok(time_class
+                    .funcall::<_, _, Value>("parse", (ts.to_string(),))?
+                    .into_value_with($handle))
             }
             _ => panic!("Invalid timestamp type"),
         }
@@ -80,6 +79,6 @@ macro_rules! impl_date_conversion {
     ($value:expr, $handle:expr) => {{
         let ts = jiff::Timestamp::from_second(($value as i64) * 86400).unwrap();
         let formatted = ts.strftime("%Y-%m-%d").to_string();
-        formatted.into_value_with($handle)
+        Ok(formatted.into_value_with($handle))
     }};
 }

data/ext/parquet/src/types/type_conversion.rs

@@ -419,7 +419,7 @@ macro_rules! impl_timestamp_to_arrow_conversion {
 macro_rules! impl_timestamp_array_conversion {
     ($column:expr, $array_type:ty, $variant:ident, $tz:expr) => {{
         let array = downcast_array::<$array_type>($column);
-        if array.is_nullable() {
+        Ok(ParquetValueVec(if array.is_nullable() {
             array
                 .values()
                 .iter()
@@ -438,7 +438,7 @@ macro_rules! impl_timestamp_array_conversion {
                 .iter()
                 .map(|x| ParquetValue::$variant(*x, $tz.clone().map(|s| s.into())))
                 .collect()
-        }
+        }))
     }};
 }
 

data/ext/parquet/src/utils.rs

@@ -32,6 +32,7 @@ pub struct ParquetRowsArgs {
     pub to_read: Value,
     pub result_type: ParserResultType,
     pub columns: Option<Vec<String>>,
+    pub strict: bool,
 }
 
 /// Parse common arguments for CSV parsing
@@ -39,10 +40,19 @@ pub fn parse_parquet_rows_args(ruby: &Ruby, args: &[Value]) -> Result<ParquetRow
     let parsed_args = scan_args::<(Value,), (), (), (), _, ()>(args)?;
     let (to_read,) = parsed_args.required;
 
-    let kwargs = get_kwargs::<_, (), (Option<Option<Value>>, Option<Option<Vec<String>>>), ()>(
+    let kwargs = get_kwargs::<
+        _,
+        (),
+        (
+            Option<Option<Value>>,
+            Option<Option<Vec<String>>>,
+            Option<Option<bool>>,
+        ),
+        (),
+    >(
         parsed_args.keywords,
         &[],
-        &["result_type", "columns"],
+        &["result_type", "columns", "strict"],
     )?;
 
     let result_type: ParserResultType = match kwargs
@@ -73,10 +83,13 @@ pub fn parse_parquet_rows_args(ruby: &Ruby, args: &[Value]) -> Result<ParquetRow
         None => ParserResultType::Hash,
     };
 
+    let strict = kwargs.optional.2.flatten().unwrap_or(false);
+
     Ok(ParquetRowsArgs {
         to_read,
         result_type,
         columns: kwargs.optional.1.flatten(),
+        strict,
     })
 }
 
@@ -86,6 +99,7 @@ pub struct ParquetColumnsArgs {
     pub result_type: ParserResultType,
     pub columns: Option<Vec<String>>,
     pub batch_size: Option<usize>,
+    pub strict: bool,
 }
 
 /// Parse common arguments for CSV parsing
@@ -103,12 +117,13 @@ pub fn parse_parquet_columns_args(
             Option<Option<Value>>,
             Option<Option<Vec<String>>>,
             Option<Option<usize>>,
+            Option<Option<bool>>,
         ),
         (),
     >(
         parsed_args.keywords,
         &[],
-        &["result_type", "columns", "batch_size"],
+        &["result_type", "columns", "batch_size", "strict"],
     )?;
 
     let result_type: ParserResultType = match kwargs
@@ -144,5 +159,6 @@ pub fn parse_parquet_columns_args(
         result_type,
         columns: kwargs.optional.1.flatten(),
         batch_size: kwargs.optional.2.flatten(),
+        strict: kwargs.optional.3.flatten().unwrap_or(false),
     })
 }

data/lib/parquet/version.rb

@@ -1,3 +1,3 @@
 module Parquet
-  VERSION = "0.2.10"
+  VERSION = "0.2.13"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parquet
 version: !ruby/object:Gem::Version
-  version: 0.2.10
+  version: 0.2.13
 platform: ruby
 authors:
 - Nathan Jaremko
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-01-30 00:00:00.000000000 Z
+date: 2025-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -76,6 +76,7 @@ files:
 - ext/parquet/src/writer/mod.rs
 - lib/parquet.rb
 - lib/parquet.rbi
+- lib/parquet/parquet.so
 - lib/parquet/version.rb
 homepage: https://github.com/njaremko/parquet-ruby
 licenses: