parquet 0.2.12 → 0.2.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7baa8799961bd4698da7c59c93cf8c36418553c29e4a56106a9338e1e00796d9
-  data.tar.gz: 84e6e87d4ea74a0be77e7cefa9ba21fd8c410b6a873108965294f41ac7443b04
+  metadata.gz: 0cf24938c23cee5bc8ed4049e2b3fee7794cb619755e26cf83d4bb8826ebccd7
+  data.tar.gz: 85f55738e3503729535de7854d7438bca69f0b82e648471c285a3eefdb51a69b
 SHA512:
-  metadata.gz: ff0aa33661944a72a69a31c287143b45d0c376fcba27ea4b5e416409702bb1acf896edeb5c2fb2bf485dd0083b5de21ca2ba9ee0cf619479b0f01f99b33a7c11
-  data.tar.gz: ed88efcc1e55a3c8b685f16e52dcdb9a378d64d2cf161ba27b6a613684bbbf13a60b532de556e21096558a0ca86a65ba201d11e793644214b2e015203531968f
+  metadata.gz: 17eaa053e7c05605d63c84786958f2980817509a6ba165654bfe50459cc30a37553671cc8b57a70831c255e463b26fc2768afcee3621664b443f9e0e67dc4460
+  data.tar.gz: e1a90f2683fce4a10b489eba3b0d98754ebeed9c418bd274e1af38c8fd9b5ad50f1e4ce72568f9c98001db408367c9610723f5c24796c801a1eaed4c23377d42

data/ext/parquet/src/enumerator.rs

@@ -6,6 +6,7 @@ pub struct RowEnumeratorArgs {
     pub to_read: Value,
     pub result_type: ParserResultType,
     pub columns: Option<Vec<String>>,
+    pub strict: bool,
 }
 
 /// Creates an enumerator for lazy Parquet row parsing
@@ -18,6 +19,9 @@ pub fn create_row_enumerator(args: RowEnumeratorArgs) -> Result<magnus::Enumerat
     if let Some(columns) = args.columns {
         kwargs.aset(Symbol::new("columns"), RArray::from_vec(columns))?;
     }
+    if args.strict {
+        kwargs.aset(Symbol::new("strict"), true)?;
+    }
     Ok(args
         .rb_self
         .enumeratorize("each_row", (args.to_read, KwArgs(kwargs))))
@@ -29,6 +33,7 @@ pub struct ColumnEnumeratorArgs {
     pub result_type: ParserResultType,
     pub columns: Option<Vec<String>>,
     pub batch_size: Option<usize>,
+    pub strict: bool,
 }
 
 #[inline]
@@ -46,6 +51,9 @@ pub fn create_column_enumerator(
     if let Some(batch_size) = args.batch_size {
         kwargs.aset(Symbol::new("batch_size"), batch_size)?;
     }
+    if args.strict {
+        kwargs.aset(Symbol::new("strict"), true)?;
+    }
     Ok(args
         .rb_self
         .enumeratorize("each_column", (args.to_read, KwArgs(kwargs))))

data/ext/parquet/src/header_cache.rs

@@ -16,7 +16,7 @@ use magnus::{IntoValue, RString, Ruby, Value};
 
 use thiserror::Error;
 
-#[derive(Debug, Error)]
+#[derive(Debug, Clone, Error)]
 pub enum CacheError {
     #[error("Failed to acquire lock: {0}")]
     LockError(String),

data/ext/parquet/src/reader/mod.rs

@@ -26,6 +26,10 @@ pub enum ReaderError {
     Parquet(#[from] parquet::errors::ParquetError),
     #[error("Arrow error: {0}")]
     Arrow(#[from] arrow_schema::ArrowError),
+    #[error("UTF-8 error: {0}")]
+    Utf8Error(#[from] simdutf8::basic::Utf8Error),
+    #[error("Jiff error: {0}")]
+    Jiff(#[from] jiff::Error),
 }
 
 impl From<MagnusError> for ReaderError {

data/ext/parquet/src/reader/parquet_column_reader.rs

@@ -1,4 +1,5 @@
 use crate::header_cache::StringCache;
+use crate::types::{ArrayWrapper, TryIntoValue};
 use crate::{
     create_column_enumerator, utils::*, ColumnEnumeratorArgs, ColumnRecord, ForgottenFileHandle,
     ParquetValueVec, ParserResultType, SeekableRubyValue,
@@ -27,6 +28,7 @@ pub fn parse_parquet_columns<'a>(rb_self: Value, args: &[Value]) -> Result<Value
         result_type,
         columns,
         batch_size,
+        strict,
     } = parse_parquet_columns_args(&ruby, args)?;
 
     if !ruby.block_given() {
@@ -36,6 +38,7 @@ pub fn parse_parquet_columns<'a>(rb_self: Value, args: &[Value]) -> Result<Value
             result_type,
             columns,
             batch_size,
+            strict,
         })
         .map(|yield_enum| yield_enum.into_value_with(&ruby));
     }
@@ -150,7 +153,7 @@ pub fn parse_parquet_columns<'a>(rb_self: Value, args: &[Value]) -> Result<Value
             map.insert(*field, vec![]);
         }
         let record = ColumnRecord::Map(map);
-        let _: Value = ruby.yield_value(record)?;
+        let _: Value = ruby.yield_value(record.try_into_value_with(&ruby)?)?;
         return Ok(ruby.qnil().into_value_with(&ruby));
     }
 
@@ -160,24 +163,37 @@ pub fn parse_parquet_columns<'a>(rb_self: Value, args: &[Value]) -> Result<Value
             let headers_clone = headers.clone();
             let iter = batch_reader.map(move |batch| {
                 batch.map_err(ReaderError::Arrow).and_then(|batch| {
-                    let headers = headers_clone.get_or_init(|| {
-                        let schema = batch.schema();
-                        let fields = schema.fields();
-                        let mut header_string = Vec::with_capacity(fields.len());
-                        for field in fields {
-                            header_string.push(field.name().to_owned());
-                        }
-                        StringCache::intern_many(&header_string).unwrap()
-                    });
-
-                    let mut map =
-                        HashMap::with_capacity_and_hasher(headers.len(), RandomState::default());
-
-                    batch.columns().iter().enumerate().for_each(|(i, column)| {
-                        let header = headers[i];
-                        let values = ParquetValueVec::try_from(column.clone()).unwrap();
-                        map.insert(header, values.into_inner());
-                    });
+                    let local_headers = headers_clone
+                        .get_or_init(|| {
+                            let schema = batch.schema();
+                            let fields = schema.fields();
+                            let mut header_string = Vec::with_capacity(fields.len());
+                            for field in fields {
+                                header_string.push(field.name().to_owned());
+                            }
+                            StringCache::intern_many(&header_string)
+                        })
+                        .as_ref()
+                        .map_err(|e| ReaderError::HeaderIntern(e.clone()))?;
+
+                    let mut map = HashMap::with_capacity_and_hasher(
+                        local_headers.len(),
+                        RandomState::default(),
+                    );
+
+                    batch
+                        .columns()
+                        .iter()
+                        .enumerate()
+                        .try_for_each(|(i, column)| {
+                            let header = local_headers[i];
+                            let values = ParquetValueVec::try_from(ArrayWrapper {
+                                array: &*column,
+                                strict: strict,
+                            })?;
+                            map.insert(header, values.into_inner());
+                            Ok::<_, ReaderError>(())
+                        })?;
 
                     Ok(ColumnRecord::Map::<RandomState>(map))
                 })
@@ -185,7 +201,7 @@ pub fn parse_parquet_columns<'a>(rb_self: Value, args: &[Value]) -> Result<Value
 
             for result in iter {
                 let record = result?;
-                let _: Value = ruby.yield_value(record)?;
+                let _: Value = ruby.yield_value(record.try_into_value_with(&ruby)?)?;
             }
         }
         ParserResultType::Array => {
@@ -195,17 +211,20 @@ pub fn parse_parquet_columns<'a>(rb_self: Value, args: &[Value]) -> Result<Value
                         .columns()
                         .into_iter()
                         .map(|column| {
-                            let values = ParquetValueVec::try_from(column.clone()).unwrap();
-                            values.into_inner()
+                            let values = ParquetValueVec::try_from(ArrayWrapper {
+                                array: &*column,
+                                strict: strict,
+                            })?;
+                            Ok::<_, ReaderError>(values.into_inner())
                         })
-                        .collect();
+                        .collect::<Result<Vec<_>, _>>()?;
                     Ok(ColumnRecord::Vec::<RandomState>(vec))
                 })
             });
 
             for result in iter {
                 let record = result?;
-                let _: Value = ruby.yield_value(record)?;
+                let _: Value = ruby.yield_value(record.try_into_value_with(&ruby)?)?;
             }
         }
     }

data/ext/parquet/src/reader/parquet_row_reader.rs

@@ -1,4 +1,5 @@
 use crate::header_cache::StringCache;
+use crate::types::TryIntoValue;
 use crate::{
     create_row_enumerator, utils::*, ForgottenFileHandle, ParquetField, ParserResultType,
     ReaderError, RowEnumeratorArgs, RowRecord, SeekableRubyValue,
@@ -25,6 +26,7 @@ pub fn parse_parquet_rows<'a>(rb_self: Value, args: &[Value]) -> Result<Value, M
         to_read,
         result_type,
         columns,
+        strict,
     } = parse_parquet_rows_args(&ruby, args)?;
 
     if !ruby.block_given() {
@@ -33,6 +35,7 @@ pub fn parse_parquet_rows<'a>(rb_self: Value, args: &[Value]) -> Result<Value, M
             to_read,
             result_type,
             columns,
+            strict,
         })
         .map(|yield_enum| yield_enum.into_value_with(&ruby));
     }
@@ -102,7 +105,7 @@ pub fn parse_parquet_rows<'a>(rb_self: Value, args: &[Value]) -> Result<Value, M
                     let mut map =
                         HashMap::with_capacity_and_hasher(headers.len(), RandomState::default());
                     row.get_column_iter().enumerate().for_each(|(i, (_, v))| {
-                        map.insert(headers[i], ParquetField(v.clone()));
+                        map.insert(headers[i], ParquetField(v.clone(), strict));
                     });
                     Ok(map)
                 })
@@ -112,7 +115,7 @@ pub fn parse_parquet_rows<'a>(rb_self: Value, args: &[Value]) -> Result<Value, M
 
             for result in iter {
                 let record = result?;
-                let _: Value = ruby.yield_value(record)?;
+                let _: Value = ruby.yield_value(record.try_into_value_with(&ruby)?)?;
             }
         }
         ParserResultType::Array => {
@@ -121,7 +124,7 @@ pub fn parse_parquet_rows<'a>(rb_self: Value, args: &[Value]) -> Result<Value, M
                     let column_count = row.get_column_iter().count();
                     let mut vec = Vec::with_capacity(column_count);
                     row.get_column_iter()
-                        .for_each(|(_, v)| vec.push(ParquetField(v.clone())));
+                        .for_each(|(_, v)| vec.push(ParquetField(v.clone(), strict)));
                     Ok(vec)
                 })
                 .and_then(|row| Ok(RowRecord::Vec::<RandomState>(row)))
@@ -130,7 +133,7 @@ pub fn parse_parquet_rows<'a>(rb_self: Value, args: &[Value]) -> Result<Value, M
 
             for result in iter {
                 let record = result?;
-                let _: Value = ruby.yield_value(record)?;
+                let _: Value = ruby.yield_value(record.try_into_value_with(&ruby)?)?;
             }
         }
     }

data/ext/parquet/src/types/parquet_value.rs

@@ -1,4 +1,7 @@
-use crate::{impl_date_conversion, impl_timestamp_array_conversion, impl_timestamp_conversion};
+use crate::{
+    impl_date_conversion, impl_timestamp_array_conversion, impl_timestamp_conversion,
+    reader::ReaderError,
+};
 
 use super::*;
 
@@ -103,23 +106,23 @@ impl std::hash::Hash for ParquetValue {
     }
 }
 
-impl IntoValue for ParquetValue {
-    fn into_value_with(self, handle: &Ruby) -> Value {
+impl TryIntoValue for ParquetValue {
+    fn try_into_value_with(self, handle: &Ruby) -> Result<Value, ReaderError> {
         match self {
-            ParquetValue::Int8(i) => i.into_value_with(handle),
-            ParquetValue::Int16(i) => i.into_value_with(handle),
-            ParquetValue::Int32(i) => i.into_value_with(handle),
-            ParquetValue::Int64(i) => i.into_value_with(handle),
-            ParquetValue::UInt8(i) => i.into_value_with(handle),
-            ParquetValue::UInt16(i) => i.into_value_with(handle),
-            ParquetValue::UInt32(i) => i.into_value_with(handle),
-            ParquetValue::UInt64(i) => i.into_value_with(handle),
-            ParquetValue::Float16(f) => f.into_value_with(handle),
-            ParquetValue::Float32(f) => f.into_value_with(handle),
-            ParquetValue::Float64(f) => f.into_value_with(handle),
-            ParquetValue::Boolean(b) => b.into_value_with(handle),
-            ParquetValue::String(s) => s.into_value_with(handle),
-            ParquetValue::Bytes(b) => handle.str_from_slice(&b).as_value(),
+            ParquetValue::Int8(i) => Ok(i.into_value_with(handle)),
+            ParquetValue::Int16(i) => Ok(i.into_value_with(handle)),
+            ParquetValue::Int32(i) => Ok(i.into_value_with(handle)),
+            ParquetValue::Int64(i) => Ok(i.into_value_with(handle)),
+            ParquetValue::UInt8(i) => Ok(i.into_value_with(handle)),
+            ParquetValue::UInt16(i) => Ok(i.into_value_with(handle)),
+            ParquetValue::UInt32(i) => Ok(i.into_value_with(handle)),
+            ParquetValue::UInt64(i) => Ok(i.into_value_with(handle)),
+            ParquetValue::Float16(f) => Ok(f.into_value_with(handle)),
+            ParquetValue::Float32(f) => Ok(f.into_value_with(handle)),
+            ParquetValue::Float64(f) => Ok(f.into_value_with(handle)),
+            ParquetValue::Boolean(b) => Ok(b.into_value_with(handle)),
+            ParquetValue::String(s) => Ok(s.into_value_with(handle)),
+            ParquetValue::Bytes(b) => Ok(handle.str_from_slice(&b).as_value()),
             ParquetValue::Date32(d) => impl_date_conversion!(d, handle),
             ParquetValue::Date64(d) => impl_date_conversion!(d, handle),
             timestamp @ ParquetValue::TimestampSecond(_, _) => {
@@ -136,21 +139,23 @@ impl IntoValue for ParquetValue {
             }
             ParquetValue::List(l) => {
                 let ary = handle.ary_new_capa(l.len());
-                l.into_iter()
-                    .try_for_each(|v| ary.push(v.into_value_with(handle)))
-                    .unwrap();
-                ary.into_value_with(handle)
+                l.into_iter().try_for_each(|v| {
+                    ary.push(v.try_into_value_with(handle)?)?;
+                    Ok::<_, ReaderError>(())
+                })?;
+                Ok(ary.into_value_with(handle))
             }
             ParquetValue::Map(m) => {
                 let hash = handle.hash_new_capa(m.len());
-                m.into_iter()
-                    .try_for_each(|(k, v)| {
-                        hash.aset(k.into_value_with(handle), v.into_value_with(handle))
-                    })
-                    .unwrap();
-                hash.into_value_with(handle)
-            }
-            ParquetValue::Null => handle.qnil().as_value(),
+                m.into_iter().try_for_each(|(k, v)| {
+                    hash.aset(
+                        k.try_into_value_with(handle)?,
+                        v.try_into_value_with(handle)?,
+                    )
+                })?;
+                Ok(hash.into_value_with(handle))
+            }
+            ParquetValue::Null => Ok(handle.qnil().as_value()),
         }
     }
 }
@@ -260,18 +265,10 @@ impl std::cmp::PartialEq for ParquetValueVec {
 
 impl std::cmp::Eq for ParquetValueVec {}
 
-impl TryFrom<Arc<dyn Array>> for ParquetValueVec {
-    type Error = String;
-
-    fn try_from(column: Arc<dyn Array>) -> Result<Self, Self::Error> {
-        ParquetValueVec::try_from(&*column)
-    }
-}
-
 macro_rules! impl_numeric_array_conversion {
     ($column:expr, $array_type:ty, $variant:ident) => {{
         let array = downcast_array::<$array_type>($column);
-        if array.is_nullable() {
+        Ok(ParquetValueVec(if array.is_nullable() {
             array
                 .values()
                 .iter()
@@ -290,13 +287,13 @@ macro_rules! impl_numeric_array_conversion {
                 .iter()
                 .map(|x| ParquetValue::$variant(*x))
                 .collect()
-        }
+        }))
     }};
 }
 macro_rules! impl_boolean_array_conversion {
     ($column:expr, $array_type:ty, $variant:ident) => {{
         let array = downcast_array::<$array_type>($column);
-        if array.is_nullable() {
+        Ok(ParquetValueVec(if array.is_nullable() {
             array
                 .values()
                 .iter()
@@ -315,34 +312,50 @@ macro_rules! impl_boolean_array_conversion {
                 .iter()
                 .map(|x| ParquetValue::$variant(x))
                 .collect()
-        }
+        }))
     }};
 }
 
-impl TryFrom<&dyn Array> for ParquetValueVec {
-    type Error = String;
+pub struct ArrayWrapper<'a> {
+    pub array: &'a dyn Array,
+    pub strict: bool,
+}
+
+impl<'a> TryFrom<ArrayWrapper<'a>> for ParquetValueVec {
+    type Error = ReaderError;
 
-    fn try_from(column: &dyn Array) -> Result<Self, Self::Error> {
-        let tmp_vec = match column.data_type() {
-            DataType::Boolean => impl_boolean_array_conversion!(column, BooleanArray, Boolean),
-            DataType::Int8 => impl_numeric_array_conversion!(column, Int8Array, Int8),
-            DataType::Int16 => impl_numeric_array_conversion!(column, Int16Array, Int16),
-            DataType::Int32 => impl_numeric_array_conversion!(column, Int32Array, Int32),
-            DataType::Int64 => impl_numeric_array_conversion!(column, Int64Array, Int64),
-            DataType::UInt8 => impl_numeric_array_conversion!(column, UInt8Array, UInt8),
-            DataType::UInt16 => impl_numeric_array_conversion!(column, UInt16Array, UInt16),
-            DataType::UInt32 => impl_numeric_array_conversion!(column, UInt32Array, UInt32),
-            DataType::UInt64 => impl_numeric_array_conversion!(column, UInt64Array, UInt64),
-            DataType::Float32 => impl_numeric_array_conversion!(column, Float32Array, Float32),
-            DataType::Float64 => impl_numeric_array_conversion!(column, Float64Array, Float64),
-            DataType::Date32 => impl_numeric_array_conversion!(column, Date32Array, Date32),
-            DataType::Date64 => impl_numeric_array_conversion!(column, Date64Array, Date64),
+    fn try_from(column: ArrayWrapper<'a>) -> Result<Self, Self::Error> {
+        match column.array.data_type() {
+            DataType::Boolean => {
+                impl_boolean_array_conversion!(column.array, BooleanArray, Boolean)
+            }
+            DataType::Int8 => impl_numeric_array_conversion!(column.array, Int8Array, Int8),
+            DataType::Int16 => impl_numeric_array_conversion!(column.array, Int16Array, Int16),
+            DataType::Int32 => impl_numeric_array_conversion!(column.array, Int32Array, Int32),
+            DataType::Int64 => impl_numeric_array_conversion!(column.array, Int64Array, Int64),
+            DataType::UInt8 => impl_numeric_array_conversion!(column.array, UInt8Array, UInt8),
+            DataType::UInt16 => impl_numeric_array_conversion!(column.array, UInt16Array, UInt16),
+            DataType::UInt32 => impl_numeric_array_conversion!(column.array, UInt32Array, UInt32),
+            DataType::UInt64 => impl_numeric_array_conversion!(column.array, UInt64Array, UInt64),
+            DataType::Float32 => {
+                impl_numeric_array_conversion!(column.array, Float32Array, Float32)
+            }
+            DataType::Float64 => {
+                impl_numeric_array_conversion!(column.array, Float64Array, Float64)
+            }
+            DataType::Date32 => impl_numeric_array_conversion!(column.array, Date32Array, Date32),
+            DataType::Date64 => impl_numeric_array_conversion!(column.array, Date64Array, Date64),
             DataType::Timestamp(TimeUnit::Second, tz) => {
-                impl_timestamp_array_conversion!(column, TimestampSecondArray, TimestampSecond, tz)
+                impl_timestamp_array_conversion!(
+                    column.array,
+                    TimestampSecondArray,
+                    TimestampSecond,
+                    tz
+                )
             }
             DataType::Timestamp(TimeUnit::Millisecond, tz) => {
                 impl_timestamp_array_conversion!(
-                    column,
+                    column.array,
                     TimestampMillisecondArray,
                     TimestampMillis,
                     tz
@@ -350,7 +363,7 @@ impl TryFrom<&dyn Array> for ParquetValueVec {
             }
             DataType::Timestamp(TimeUnit::Microsecond, tz) => {
                 impl_timestamp_array_conversion!(
-                    column,
+                    column.array,
                     TimestampMicrosecondArray,
                     TimestampMicros,
                     tz
@@ -358,72 +371,93 @@ impl TryFrom<&dyn Array> for ParquetValueVec {
             }
             DataType::Timestamp(TimeUnit::Nanosecond, tz) => {
                 impl_timestamp_array_conversion!(
-                    column,
+                    column.array,
                     TimestampNanosecondArray,
                     TimestampNanos,
                     tz
                 )
             }
             DataType::Float16 => {
-                let array = downcast_array::<Float16Array>(column);
+                let array = downcast_array::<Float16Array>(column.array);
                 if array.is_nullable() {
-                    array
-                        .values()
-                        .iter()
-                        .enumerate()
-                        .map(|(i, x)| {
-                            if array.is_null(i) {
-                                ParquetValue::Null
-                            } else {
-                                ParquetValue::Float16(f32::from(*x))
-                            }
-                        })
-                        .collect()
+                    Ok(ParquetValueVec(
+                        array
+                            .values()
+                            .iter()
+                            .enumerate()
+                            .map(|(i, x)| {
+                                if array.is_null(i) {
+                                    ParquetValue::Null
+                                } else {
+                                    ParquetValue::Float16(f32::from(*x))
+                                }
+                            })
+                            .collect(),
+                    ))
                 } else {
-                    array
-                        .values()
-                        .iter()
-                        .map(|x| ParquetValue::Float16(f32::from(*x)))
-                        .collect()
+                    Ok(ParquetValueVec(
+                        array
+                            .values()
+                            .iter()
+                            .map(|x| ParquetValue::Float16(f32::from(*x)))
+                            .collect(),
+                    ))
                 }
             }
             DataType::Utf8 => {
-                let array = downcast_array::<StringArray>(column);
-                array
-                    .iter()
-                    .map(|opt_x| match opt_x {
-                        Some(x) => ParquetValue::String(x.to_string()),
-                        None => ParquetValue::Null,
-                    })
-                    .collect()
+                let array = downcast_array::<StringArray>(column.array);
+                let mut tmp_vec = Vec::with_capacity(array.len());
+                let iter = array.iter().map(|opt_x| match opt_x {
+                    Some(x) => {
+                        if column.strict {
+                            Ok::<_, ReaderError>(ParquetValue::String(
+                                simdutf8::basic::from_utf8(x.as_bytes())?.to_string(),
+                            ))
+                        } else {
+                            Ok::<_, ReaderError>(ParquetValue::String(x.to_string()))
+                        }
+                    }
+                    None => Ok(ParquetValue::Null),
+                });
+                for x in iter {
+                    tmp_vec.push(x?);
+                }
+                Ok(ParquetValueVec(tmp_vec))
             }
             DataType::Binary => {
-                let array = downcast_array::<BinaryArray>(column);
-                array
-                    .iter()
-                    .map(|opt_x| match opt_x {
-                        Some(x) => ParquetValue::Bytes(x.to_vec()),
-                        None => ParquetValue::Null,
-                    })
-                    .collect()
+                let array = downcast_array::<BinaryArray>(column.array);
+                Ok(ParquetValueVec(
+                    array
+                        .iter()
+                        .map(|opt_x| match opt_x {
+                            Some(x) => ParquetValue::Bytes(x.to_vec()),
+                            None => ParquetValue::Null,
+                        })
+                        .collect(),
+                ))
             }
             DataType::List(_field) => {
-                let list_array = downcast_array::<ListArray>(column);
-                list_array
-                    .iter()
-                    .map(|x| match x {
-                        Some(values) => match ParquetValueVec::try_from(values) {
-                            Ok(vec) => ParquetValue::List(vec.into_inner()),
-                            Err(e) => {
-                                panic!("Error converting list array to ParquetValueVec: {}", e)
-                            }
-                        },
-                        None => ParquetValue::Null,
-                    })
-                    .collect()
+                let list_array = downcast_array::<ListArray>(column.array);
+                Ok(ParquetValueVec(
+                    list_array
+                        .iter()
+                        .map(|x| match x {
+                            Some(values) => match ParquetValueVec::try_from(ArrayWrapper {
+                                array: &*values,
+                                strict: column.strict,
+                            }) {
+                                Ok(vec) => ParquetValue::List(vec.into_inner()),
+                                Err(e) => {
+                                    panic!("Error converting list array to ParquetValueVec: {}", e)
+                                }
+                            },
+                            None => ParquetValue::Null,
+                        })
+                        .collect(),
+                ))
             }
             DataType::Struct(_) => {
-                let struct_array = downcast_array::<StructArray>(column);
+                let struct_array = downcast_array::<StructArray>(column.array);
                 let mut values = Vec::with_capacity(struct_array.len());
                 for i in 0..struct_array.len() {
                     if struct_array.is_null(i) {
@@ -433,8 +467,11 @@ impl TryFrom<&dyn Array> for ParquetValueVec {
 
                     let mut map = std::collections::HashMap::new();
                     for (field_idx, field) in struct_array.fields().iter().enumerate() {
-                        let column = struct_array.column(field_idx);
-                        let field_values = match ParquetValueVec::try_from(column.slice(i, 1)) {
+                        let c = struct_array.column(field_idx);
+                        let field_values = match ParquetValueVec::try_from(ArrayWrapper {
+                            array: &*c.slice(i, 1),
+                            strict: column.strict,
+                        }) {
                             Ok(vec) => vec.into_inner(),
                             Err(e) => {
                                 panic!("Error converting struct field to ParquetValueVec: {}", e)
@@ -447,16 +484,18 @@ impl TryFrom<&dyn Array> for ParquetValueVec {
                     }
                     values.push(ParquetValue::Map(map));
                 }
-                values
+                Ok(ParquetValueVec(values))
             }
             DataType::Null => {
-                let x = downcast_array::<NullArray>(column);
-                vec![ParquetValue::Null; x.len()]
+                let x = downcast_array::<NullArray>(column.array);
+                Ok(ParquetValueVec(vec![ParquetValue::Null; x.len()]))
             }
             _ => {
-                return Err(format!("Unsupported data type: {:?}", column.data_type()));
+                return Err(ReaderError::Ruby(format!(
+                    "Unsupported data type: {:?}",
+                    column.array.data_type()
+                )));
             }
-        };
-        Ok(ParquetValueVec(tmp_vec))
+        }
     }
 }

data/ext/parquet/src/types/record_types.rs

@@ -1,5 +1,7 @@
 use itertools::Itertools;
 
+use crate::reader::ReaderError;
+
 use super::*;
 
 #[derive(Debug)]
@@ -15,15 +17,16 @@ pub enum ColumnRecord<S: BuildHasher + Default> {
 }
 
 #[derive(Debug)]
-pub struct ParquetField(pub Field);
+pub struct ParquetField(pub Field, pub bool);
 
-impl<S: BuildHasher + Default> IntoValue for RowRecord<S> {
-    fn into_value_with(self, handle: &Ruby) -> Value {
+impl<S: BuildHasher + Default> TryIntoValue for RowRecord<S> {
+    fn try_into_value_with(self, handle: &Ruby) -> Result<Value, ReaderError> {
         match self {
             RowRecord::Vec(vec) => {
                 let ary = handle.ary_new_capa(vec.len());
-                vec.into_iter().try_for_each(|v| ary.push(v)).unwrap();
-                handle.into_value(ary)
+                vec.into_iter()
+                    .try_for_each(|v| ary.push(v.try_into_value_with(handle)?))?;
+                Ok(handle.into_value(ary))
             }
             RowRecord::Map(map) => {
                 let hash = handle.hash_new_capa(map.len());
@@ -36,41 +39,41 @@ impl<S: BuildHasher + Default> IntoValue for RowRecord<S> {
                     for (k, v) in chunk {
                         if i + 1 >= values.len() {
                             // Bulk insert current batch if array is full
-                            hash.bulk_insert(&values[..i]).unwrap();
+                            hash.bulk_insert(&values[..i])?;
                             values[..i].fill(handle.qnil().as_value());
                             i = 0;
                         }
                         values[i] = handle.into_value(k);
-                        values[i + 1] = handle.into_value(v);
+                        values[i + 1] = v.try_into_value_with(handle)?;
                         i += 2;
                     }
                     // Insert any remaining pairs
                     if i > 0 {
-                        hash.bulk_insert(&values[..i]).unwrap();
+                        hash.bulk_insert(&values[..i])?;
                         values[..i].fill(handle.qnil().as_value());
                         i = 0;
                     }
                 }
 
-                hash.into_value_with(handle)
+                Ok(hash.into_value_with(handle))
             }
         }
     }
 }
 
-impl<S: BuildHasher + Default> IntoValue for ColumnRecord<S> {
-    fn into_value_with(self, handle: &Ruby) -> Value {
+impl<S: BuildHasher + Default> TryIntoValue for ColumnRecord<S> {
+    fn try_into_value_with(self, handle: &Ruby) -> Result<Value, ReaderError> {
         match self {
             ColumnRecord::Vec(vec) => {
                 let ary = handle.ary_new_capa(vec.len());
-                vec.into_iter()
-                    .try_for_each(|v| {
-                        let nested_ary = handle.ary_new_capa(v.len());
-                        v.into_iter().try_for_each(|v| nested_ary.push(v)).unwrap();
-                        ary.push(nested_ary.into_value_with(handle))
-                    })
-                    .unwrap();
-                ary.into_value_with(handle)
+                vec.into_iter().try_for_each(|v| {
+                    let nested_ary = handle.ary_new_capa(v.len());
+                    v.into_iter()
+                        .try_for_each(|v| nested_ary.push(v.try_into_value_with(handle)?))?;
+                    ary.push(nested_ary.into_value_with(handle))?;
+                    Ok::<_, ReaderError>(())
+                })?;
+                Ok(ary.into_value_with(handle))
             }
             ColumnRecord::Map(map) => {
                 let hash = handle.hash_new_capa(map.len());
@@ -83,91 +86,98 @@ impl<S: BuildHasher + Default> IntoValue for ColumnRecord<S> {
                     for (k, v) in chunk {
                         if i + 1 >= values.len() {
                             // Bulk insert current batch if array is full
-                            hash.bulk_insert(&values[..i]).unwrap();
+                            hash.bulk_insert(&values[..i])?;
                             values[..i].fill(handle.qnil().as_value());
                             i = 0;
                         }
                         values[i] = handle.into_value(k);
                         let ary = handle.ary_new_capa(v.len());
-                        v.into_iter().try_for_each(|v| ary.push(v)).unwrap();
+                        v.into_iter()
+                            .try_for_each(|v| ary.push(v.try_into_value_with(handle)?))?;
                         values[i + 1] = handle.into_value(ary);
                         i += 2;
                     }
                     // Insert any remaining pairs
                     if i > 0 {
-                        hash.bulk_insert(&values[..i]).unwrap();
+                        hash.bulk_insert(&values[..i])?;
                         values[..i].fill(handle.qnil().as_value());
                         i = 0;
                     }
                 }
 
-                hash.into_value_with(handle)
+                Ok(hash.into_value_with(handle))
             }
         }
     }
 }
 
-impl IntoValue for ParquetField {
-    fn into_value_with(self, handle: &Ruby) -> Value {
+pub trait TryIntoValue {
+    fn try_into_value_with(self, handle: &Ruby) -> Result<Value, ReaderError>;
+}
+
+impl TryIntoValue for ParquetField {
+    fn try_into_value_with(self, handle: &Ruby) -> Result<Value, ReaderError> {
         match self.0 {
-            Field::Null => handle.qnil().as_value(),
-            Field::Bool(b) => b.into_value_with(handle),
-            Field::Short(s) => s.into_value_with(handle),
-            Field::Int(i) => i.into_value_with(handle),
-            Field::Long(l) => l.into_value_with(handle),
-            Field::UByte(ub) => ub.into_value_with(handle),
-            Field::UShort(us) => us.into_value_with(handle),
-            Field::UInt(ui) => ui.into_value_with(handle),
-            Field::ULong(ul) => ul.into_value_with(handle),
-            Field::Float16(f) => f32::from(f).into_value_with(handle),
-            Field::Float(f) => f.into_value_with(handle),
-            Field::Double(d) => d.into_value_with(handle),
-            Field::Str(s) => s.into_value_with(handle),
-            Field::Byte(b) => b.into_value_with(handle),
-            Field::Bytes(b) => handle.str_from_slice(b.data()).as_value(),
+            Field::Null => Ok(handle.qnil().as_value()),
+            Field::Bool(b) => Ok(b.into_value_with(handle)),
+            Field::Short(s) => Ok(s.into_value_with(handle)),
+            Field::Int(i) => Ok(i.into_value_with(handle)),
+            Field::Long(l) => Ok(l.into_value_with(handle)),
+            Field::UByte(ub) => Ok(ub.into_value_with(handle)),
+            Field::UShort(us) => Ok(us.into_value_with(handle)),
+            Field::UInt(ui) => Ok(ui.into_value_with(handle)),
+            Field::ULong(ul) => Ok(ul.into_value_with(handle)),
+            Field::Float16(f) => Ok(f32::from(f).into_value_with(handle)),
+            Field::Float(f) => Ok(f.into_value_with(handle)),
+            Field::Double(d) => Ok(d.into_value_with(handle)),
+            Field::Str(s) => {
+                if self.1 {
+                    Ok(simdutf8::basic::from_utf8(s.as_bytes())
+                        .map_err(|e| ReaderError::Utf8Error(e))
+                        .and_then(|s| Ok(s.into_value_with(handle)))?)
+                } else {
+                    Ok(handle.str_from_slice(s.as_bytes()).as_value())
+                }
+            }
+            Field::Byte(b) => Ok(b.into_value_with(handle)),
+            Field::Bytes(b) => Ok(handle.str_from_slice(b.data()).as_value()),
             Field::Date(d) => {
-                let ts = jiff::Timestamp::from_second((d as i64) * 86400).unwrap();
+                let ts = jiff::Timestamp::from_second((d as i64) * 86400)?;
                 let formatted = ts.strftime("%Y-%m-%d").to_string();
-                formatted.into_value_with(handle)
+                Ok(formatted.into_value_with(handle))
             }
             Field::TimestampMillis(ts) => {
-                let ts = jiff::Timestamp::from_millisecond(ts).unwrap();
+                let ts = jiff::Timestamp::from_millisecond(ts)?;
                 let time_class = handle.class_time();
-                time_class
-                    .funcall::<_, _, Value>("parse", (ts.to_string(),))
-                    .unwrap()
-                    .into_value_with(handle)
+                Ok(time_class
+                    .funcall::<_, _, Value>("parse", (ts.to_string(),))?
+                    .into_value_with(handle))
             }
             Field::TimestampMicros(ts) => {
-                let ts = jiff::Timestamp::from_microsecond(ts).unwrap();
+                let ts = jiff::Timestamp::from_microsecond(ts)?;
                 let time_class = handle.class_time();
-                time_class
-                    .funcall::<_, _, Value>("parse", (ts.to_string(),))
-                    .unwrap()
-                    .into_value_with(handle)
+                Ok(time_class
+                    .funcall::<_, _, Value>("parse", (ts.to_string(),))?
+                    .into_value_with(handle))
             }
             Field::ListInternal(list) => {
                 let elements = list.elements();
                 let ary = handle.ary_new_capa(elements.len());
-                elements
-                    .iter()
-                    .try_for_each(|e| ary.push(ParquetField(e.clone()).into_value_with(handle)))
-                    .unwrap();
-                ary.into_value_with(handle)
+                elements.iter().try_for_each(|e| {
+                    ary.push(ParquetField(e.clone(), self.1).try_into_value_with(handle)?)
+                })?;
+                Ok(ary.into_value_with(handle))
             }
             Field::MapInternal(map) => {
                 let entries = map.entries();
                 let hash = handle.hash_new_capa(entries.len());
-                entries
-                    .iter()
-                    .try_for_each(|(k, v)| {
-                        hash.aset(
-                            ParquetField(k.clone()).into_value_with(handle),
-                            ParquetField(v.clone()).into_value_with(handle),
-                        )
-                    })
-                    .unwrap();
-                hash.into_value_with(handle)
+                entries.iter().try_for_each(|(k, v)| {
+                    hash.aset(
+                        ParquetField(k.clone(), self.1).try_into_value_with(handle)?,
+                        ParquetField(v.clone(), self.1).try_into_value_with(handle)?,
+                    )
+                })?;
+                Ok(hash.into_value_with(handle))
             }
             Field::Decimal(d) => {
                 let value = match d {
@@ -185,20 +195,24 @@ impl IntoValue for ParquetField {
                         format!("{}e-{}", unscaled, scale)
                     }
                 };
-                handle.eval(&format!("BigDecimal(\"{value}\")")).unwrap()
+                Ok(handle.eval(&format!("BigDecimal(\"{value}\")"))?)
             }
             Field::Group(row) => {
                 let hash = handle.hash_new();
-                row.get_column_iter()
-                    .try_for_each(|(k, v)| {
-                        hash.aset(
-                            k.clone().into_value_with(handle),
-                            ParquetField(v.clone()).into_value_with(handle),
-                        )
-                    })
-                    .unwrap();
-                hash.into_value_with(handle)
+                row.get_column_iter().try_for_each(|(k, v)| {
+                    hash.aset(
+                        k.clone().into_value_with(handle),
+                        ParquetField(v.clone(), self.1).try_into_value_with(handle)?,
+                    )
+                })?;
+                Ok(hash.into_value_with(handle))
             }
         }
     }
 }
+
+// impl IntoValue for ParquetField {
+//     fn into_value_with(self, handle: &Ruby) -> Value {
+//         self.try_into_value_with(handle).unwrap()
+//     }
+// }

data/ext/parquet/src/types/timestamp.rs

@@ -64,10 +64,9 @@ macro_rules! impl_timestamp_conversion {
             ParquetValue::$unit(ts, tz) => {
                 let ts = parse_zoned_timestamp(&ParquetValue::$unit(ts, tz));
                 let time_class = $handle.class_time();
-                time_class
-                    .funcall::<_, _, Value>("parse", (ts.to_string(),))
-                    .unwrap()
-                    .into_value_with($handle)
+                Ok(time_class
+                    .funcall::<_, _, Value>("parse", (ts.to_string(),))?
+                    .into_value_with($handle))
             }
             _ => panic!("Invalid timestamp type"),
         }
@@ -80,6 +79,6 @@ macro_rules! impl_date_conversion {
     ($value:expr, $handle:expr) => {{
         let ts = jiff::Timestamp::from_second(($value as i64) * 86400).unwrap();
         let formatted = ts.strftime("%Y-%m-%d").to_string();
-        formatted.into_value_with($handle)
+        Ok(formatted.into_value_with($handle))
     }};
 }

data/ext/parquet/src/types/type_conversion.rs

@@ -419,7 +419,7 @@ macro_rules! impl_timestamp_to_arrow_conversion {
 macro_rules! impl_timestamp_array_conversion {
     ($column:expr, $array_type:ty, $variant:ident, $tz:expr) => {{
         let array = downcast_array::<$array_type>($column);
-        if array.is_nullable() {
+        Ok(ParquetValueVec(if array.is_nullable() {
             array
                 .values()
                 .iter()
@@ -438,7 +438,7 @@ macro_rules! impl_timestamp_array_conversion {
                 .iter()
                 .map(|x| ParquetValue::$variant(*x, $tz.clone().map(|s| s.into())))
                 .collect()
-        }
+        }))
     }};
 }
 

data/ext/parquet/src/utils.rs

@@ -32,6 +32,7 @@ pub struct ParquetRowsArgs {
     pub to_read: Value,
     pub result_type: ParserResultType,
     pub columns: Option<Vec<String>>,
+    pub strict: bool,
 }
 
 /// Parse common arguments for CSV parsing
@@ -39,10 +40,19 @@ pub fn parse_parquet_rows_args(ruby: &Ruby, args: &[Value]) -> Result<ParquetRow
     let parsed_args = scan_args::<(Value,), (), (), (), _, ()>(args)?;
     let (to_read,) = parsed_args.required;
 
-    let kwargs = get_kwargs::<_, (), (Option<Option<Value>>, Option<Option<Vec<String>>>), ()>(
+    let kwargs = get_kwargs::<
+        _,
+        (),
+        (
+            Option<Option<Value>>,
+            Option<Option<Vec<String>>>,
+            Option<Option<bool>>,
+        ),
+        (),
+    >(
         parsed_args.keywords,
         &[],
-        &["result_type", "columns"],
+        &["result_type", "columns", "strict"],
     )?;
 
     let result_type: ParserResultType = match kwargs
@@ -73,10 +83,13 @@ pub fn parse_parquet_rows_args(ruby: &Ruby, args: &[Value]) -> Result<ParquetRow
         None => ParserResultType::Hash,
     };
 
+    let strict = kwargs.optional.2.flatten().unwrap_or(false);
+
     Ok(ParquetRowsArgs {
         to_read,
         result_type,
         columns: kwargs.optional.1.flatten(),
+        strict,
     })
 }
 
@@ -86,6 +99,7 @@ pub struct ParquetColumnsArgs {
     pub result_type: ParserResultType,
     pub columns: Option<Vec<String>>,
     pub batch_size: Option<usize>,
+    pub strict: bool,
 }
 
 /// Parse common arguments for CSV parsing
@@ -103,12 +117,13 @@ pub fn parse_parquet_columns_args(
             Option<Option<Value>>,
             Option<Option<Vec<String>>>,
             Option<Option<usize>>,
+            Option<Option<bool>>,
         ),
         (),
     >(
         parsed_args.keywords,
         &[],
-        &["result_type", "columns", "batch_size"],
+        &["result_type", "columns", "batch_size", "strict"],
     )?;
 
     let result_type: ParserResultType = match kwargs
@@ -144,5 +159,6 @@ pub fn parse_parquet_columns_args(
         result_type,
         columns: kwargs.optional.1.flatten(),
         batch_size: kwargs.optional.2.flatten(),
+        strict: kwargs.optional.3.flatten().unwrap_or(false),
     })
 }

data/lib/parquet/version.rb

@@ -1,3 +1,3 @@
 module Parquet
-  VERSION = "0.2.12"
+  VERSION = "0.2.13"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parquet
 version: !ruby/object:Gem::Version
-  version: 0.2.12
+  version: 0.2.13
 platform: ruby
 authors:
 - Nathan Jaremko
@@ -76,6 +76,7 @@ files:
 - ext/parquet/src/writer/mod.rs
 - lib/parquet.rb
 - lib/parquet.rbi
+- lib/parquet/parquet.so
 - lib/parquet/version.rb
 homepage: https://github.com/njaremko/parquet-ruby
 licenses: