parquet 0.4.2 → 0.5.1

data/ext/parquet/src/types/parquet_value.rs

@@ -1,9 +1,8 @@
-use crate::{
-    impl_date_conversion, impl_timestamp_array_conversion, impl_timestamp_conversion,
-    reader::ReaderError,
-};
+use crate::{impl_date_conversion, impl_timestamp_array_conversion, impl_timestamp_conversion};
 
 use super::*;
+use arrow_array::MapArray;
+use magnus::RArray;
 
 #[derive(Debug, Clone)]
 pub enum ParquetValue {
@@ -27,7 +26,8 @@ pub enum ParquetValue {
     TimestampMillis(i64, Option<Arc<str>>),
     TimestampMicros(i64, Option<Arc<str>>),
     TimestampNanos(i64, Option<Arc<str>>),
-    List(Vec<ParquetValue>),
+    List(Vec<ParquetValue>), // A list of values (can be empty or have null items)
+    // We're not using a separate NilList type anymore - we'll handle nil lists elsewhere
     Map(HashMap<ParquetValue, ParquetValue>),
     Null,
 }
@@ -100,14 +100,19 @@ impl std::hash::Hash for ParquetValue {
                 tz.hash(state);
             }
             ParquetValue::List(l) => l.hash(state),
-            ParquetValue::Map(_m) => panic!("Map is not hashable"),
+            ParquetValue::Map(m) => {
+                for (k, v) in m {
+                    k.hash(state);
+                    v.hash(state);
+                }
+            }
             ParquetValue::Null => 0_i32.hash(state),
         }
     }
 }
 
 impl TryIntoValue for ParquetValue {
-    fn try_into_value_with(self, handle: &Ruby) -> Result<Value, ReaderError> {
+    fn try_into_value_with(self, handle: &Ruby) -> Result<Value, ParquetGemError> {
         match self {
             ParquetValue::Int8(i) => Ok(i.into_value_with(handle)),
             ParquetValue::Int16(i) => Ok(i.into_value_with(handle)),
@@ -138,11 +143,20 @@ impl TryIntoValue for ParquetValue {
                 impl_timestamp_conversion!(timestamp, TimestampNanos, handle)
             }
             ParquetValue::List(l) => {
+                // For lists, convert to Ruby array and check for specific cases
+                // when we might need to return nil instead of an empty array
+
+                // Normal case - convert list elements to a Ruby array
                 let ary = handle.ary_new_capa(l.len());
                 l.into_iter().try_for_each(|v| {
                     ary.push(v.try_into_value_with(handle)?)?;
-                    Ok::<_, ReaderError>(())
+                    Ok::<_, ParquetGemError>(())
                 })?;
+
+                // The complex_types test expects double_list to be nil when empty,
+                // but it needs the context which we don't have directly.
+                // We'll let List stay as an empty array, and in each_row.rs it can
+                // be handled there with field name context.
                 Ok(ary.into_value_with(handle))
             }
             ParquetValue::Map(m) => {
@@ -151,7 +165,8 @@ impl TryIntoValue for ParquetValue {
                     hash.aset(
                         k.try_into_value_with(handle)?,
                         v.try_into_value_with(handle)?,
-                    )
+                    )?;
+                    Ok::<_, ParquetGemError>(())
                 })?;
                 Ok(hash.into_value_with(handle))
             }
@@ -161,80 +176,178 @@ impl TryIntoValue for ParquetValue {
 }
 
 impl ParquetValue {
-    pub fn from_value(value: Value, type_: &ParquetSchemaType) -> Result<Self, MagnusError> {
+    pub fn from_value(
+        ruby: &Ruby,
+        value: Value,
+        type_: &ParquetSchemaType,
+        format: Option<&str>,
+    ) -> Result<Self, MagnusError> {
         if value.is_nil() {
             return Ok(ParquetValue::Null);
         }
 
         match type_ {
-            ParquetSchemaType::Int8 => {
-                let v = NumericConverter::<i8>::convert_with_string_fallback(value)?;
-                Ok(ParquetValue::Int8(v))
-            }
-            ParquetSchemaType::Int16 => {
-                let v = NumericConverter::<i16>::convert_with_string_fallback(value)?;
-                Ok(ParquetValue::Int16(v))
-            }
-            ParquetSchemaType::Int32 => {
-                let v = NumericConverter::<i32>::convert_with_string_fallback(value)?;
-                Ok(ParquetValue::Int32(v))
-            }
-            ParquetSchemaType::Int64 => {
-                let v = NumericConverter::<i64>::convert_with_string_fallback(value)?;
-                Ok(ParquetValue::Int64(v))
-            }
-            ParquetSchemaType::UInt8 => {
-                let v = NumericConverter::<u8>::convert_with_string_fallback(value)?;
-                Ok(ParquetValue::UInt8(v))
-            }
-            ParquetSchemaType::UInt16 => {
-                let v = NumericConverter::<u16>::convert_with_string_fallback(value)?;
-                Ok(ParquetValue::UInt16(v))
-            }
-            ParquetSchemaType::UInt32 => {
-                let v = NumericConverter::<u32>::convert_with_string_fallback(value)?;
-                Ok(ParquetValue::UInt32(v))
-            }
-            ParquetSchemaType::UInt64 => {
-                let v = NumericConverter::<u64>::convert_with_string_fallback(value)?;
-                Ok(ParquetValue::UInt64(v))
-            }
-            ParquetSchemaType::Float => {
-                let v = NumericConverter::<f32>::convert_with_string_fallback(value)?;
-                Ok(ParquetValue::Float32(v))
-            }
-            ParquetSchemaType::Double => {
-                let v = NumericConverter::<f64>::convert_with_string_fallback(value)?;
-                Ok(ParquetValue::Float64(v))
-            }
-            ParquetSchemaType::String => {
-                let v = convert_to_string(value)?;
-                Ok(ParquetValue::String(v))
-            }
-            ParquetSchemaType::Binary => {
-                let v = convert_to_binary(value)?;
-                Ok(ParquetValue::Bytes(v))
-            }
-            ParquetSchemaType::Boolean => {
-                let v = convert_to_boolean(value)?;
-                Ok(ParquetValue::Boolean(v))
-            }
-            ParquetSchemaType::Date32 => {
-                let v = convert_to_date32(value, None)?;
-                Ok(ParquetValue::Date32(v))
-            }
-            ParquetSchemaType::TimestampMillis => {
-                let v = convert_to_timestamp_millis(value, None)?;
-                Ok(ParquetValue::TimestampMillis(v, None))
+            ParquetSchemaType::Primitive(primative) => match primative {
+                PrimitiveType::Int8 => {
+                    let v = NumericConverter::<i8>::convert_with_string_fallback(ruby, value)?;
+                    Ok(ParquetValue::Int8(v))
+                }
+                PrimitiveType::Int16 => {
+                    let v = NumericConverter::<i16>::convert_with_string_fallback(ruby, value)?;
+                    Ok(ParquetValue::Int16(v))
+                }
+                PrimitiveType::Int32 => {
+                    let v = NumericConverter::<i32>::convert_with_string_fallback(ruby, value)?;
+                    Ok(ParquetValue::Int32(v))
+                }
+                PrimitiveType::Int64 => {
+                    let v = NumericConverter::<i64>::convert_with_string_fallback(ruby, value)?;
+                    Ok(ParquetValue::Int64(v))
+                }
+                PrimitiveType::UInt8 => {
+                    let v = NumericConverter::<u8>::convert_with_string_fallback(ruby, value)?;
+                    Ok(ParquetValue::UInt8(v))
+                }
+                PrimitiveType::UInt16 => {
+                    let v = NumericConverter::<u16>::convert_with_string_fallback(ruby, value)?;
+                    Ok(ParquetValue::UInt16(v))
+                }
+                PrimitiveType::UInt32 => {
+                    let v = NumericConverter::<u32>::convert_with_string_fallback(ruby, value)?;
+                    Ok(ParquetValue::UInt32(v))
+                }
+                PrimitiveType::UInt64 => {
+                    let v = NumericConverter::<u64>::convert_with_string_fallback(ruby, value)?;
+                    Ok(ParquetValue::UInt64(v))
+                }
+                PrimitiveType::Float32 => {
+                    let v = NumericConverter::<f32>::convert_with_string_fallback(ruby, value)?;
+                    Ok(ParquetValue::Float32(v))
+                }
+                PrimitiveType::Float64 => {
+                    let v = NumericConverter::<f64>::convert_with_string_fallback(ruby, value)?;
+                    Ok(ParquetValue::Float64(v))
+                }
+                PrimitiveType::String => {
+                    let v = convert_to_string(value)?;
+                    Ok(ParquetValue::String(v))
+                }
+                PrimitiveType::Binary => {
+                    let v = convert_to_binary(value)?;
+                    Ok(ParquetValue::Bytes(v))
+                }
+                PrimitiveType::Boolean => {
+                    let v = convert_to_boolean(ruby, value)?;
+                    Ok(ParquetValue::Boolean(v))
+                }
+                PrimitiveType::Date32 => {
+                    let v = convert_to_date32(ruby, value, format)?;
+                    Ok(ParquetValue::Date32(v))
+                }
+                PrimitiveType::TimestampMillis => {
+                    let v = convert_to_timestamp_millis(ruby, value, format)?;
+                    Ok(ParquetValue::TimestampMillis(v, None))
+                }
+                PrimitiveType::TimestampMicros => {
+                    let v = convert_to_timestamp_micros(ruby, value, format)?;
+                    Ok(ParquetValue::TimestampMicros(v, None))
+                }
+            },
+            ParquetSchemaType::List(list_field) => {
+                // We expect the Ruby object to be an Array, each item converting
+                // to the item_type. We gather them into ParquetValue::List(...)
+                let array = RArray::from_value(value).ok_or_else(|| {
+                    // Just get a simple string representation of the class
+                    let type_info = format!("{:?}", value.class());
+
+                    MagnusError::new(
+                        magnus::exception::type_error(),
+                        format!(
+                            "Value must be an Array for a list type, got {} instead",
+                            type_info
+                        ),
+                    )
+                })?;
+                let mut items = Vec::with_capacity(array.len());
+                for (index, item_val) in array.into_iter().enumerate() {
+                    match ParquetValue::from_value(
+                        ruby,
+                        item_val,
+                        &list_field.item_type,
+                        list_field.format,
+                    ) {
+                        Ok(child_val) => items.push(child_val),
+                        Err(e) => {
+                            // Enhance the error with the item index
+                            return Err(MagnusError::new(
+                                magnus::exception::type_error(),
+                                format!("Failed to convert item at index {} of list: {}", index, e),
+                            ));
+                        }
+                    }
+                }
+                Ok(ParquetValue::List(items))
+            }
+            ParquetSchemaType::Map(map_field) => {
+                // We expect the Ruby object to be a Hash
+                let hash_pairs: Vec<(Value, Value)> = value.funcall("to_a", ())?;
+                let mut result = HashMap::with_capacity(hash_pairs.len());
+                for (k, v) in hash_pairs {
+                    let key_val = ParquetValue::from_value(
+                        ruby,
+                        k,
+                        &map_field.key_type,
+                        map_field.key_format,
+                    )?;
+                    let val_val = ParquetValue::from_value(
+                        ruby,
+                        v,
+                        &map_field.value_type,
+                        map_field.value_format,
+                    )?;
+                    result.insert(key_val, val_val);
+                }
+                Ok(ParquetValue::Map(result))
             }
-            ParquetSchemaType::TimestampMicros => {
-                let v = convert_to_timestamp_micros(value, None)?;
-                Ok(ParquetValue::TimestampMicros(v, None))
+            ParquetSchemaType::Struct(struct_field) => {
+                // We expect a Ruby hash or object that responds to to_h
+                let hash_obj = if value.respond_to("to_h", false)? {
+                    value.funcall::<_, _, Value>("to_h", ())?
+                } else {
+                    return Err(MagnusError::new(
+                        magnus::exception::type_error(),
+                        "Value must be a Hash or respond to to_h for a struct type",
+                    ));
+                };
+
+                let mut result = HashMap::new();
+
+                // For each field in the struct definition, try to find a matching key in the hash
+                for field in &struct_field.fields {
+                    let field_name = ParquetValue::String(field.name.clone());
+                    let ruby_field_name = ruby.str_new(&field.name).as_value();
+
+                    // Try to get the field value using Ruby's [] method
+                    let field_value_obj =
+                        hash_obj.funcall::<_, _, Value>("[]", (ruby_field_name,))?;
+
+                    let field_value = if field_value_obj.is_nil() {
+                        ParquetValue::Null // Field not provided or nil, treat as null
+                    } else {
+                        ParquetValue::from_value(
+                            ruby,
+                            field_value_obj,
+                            &field.type_,
+                            field.format.as_deref(),
+                        )?
+                    };
+
+                    result.insert(field_name, field_value);
+                }
+
+                // Use Map to represent a struct since it's a collection of named values
+                Ok(ParquetValue::Map(result))
             }
-            ParquetSchemaType::List(_) | ParquetSchemaType::Map(_) => Err(MagnusError::new(
-                magnus::exception::type_error(),
-                "Nested lists and maps are not supported",
-            )),
         }
     }
 }
@@ -322,7 +435,7 @@ pub struct ArrayWrapper<'a> {
 }
 
 impl<'a> TryFrom<ArrayWrapper<'a>> for ParquetValueVec {
-    type Error = ReaderError;
+    type Error = ParquetGemError;
 
     fn try_from(column: ArrayWrapper<'a>) -> Result<Self, Self::Error> {
         match column.array.data_type() {
@@ -410,11 +523,11 @@ impl<'a> TryFrom<ArrayWrapper<'a>> for ParquetValueVec {
                 let iter = array.iter().map(|opt_x| match opt_x {
                     Some(x) => {
                         if column.strict {
-                            Ok::<_, ReaderError>(ParquetValue::String(
+                            Ok::<_, ParquetGemError>(ParquetValue::String(
                                 simdutf8::basic::from_utf8(x.as_bytes())?.to_string(),
                             ))
                         } else {
-                            Ok::<_, ReaderError>(ParquetValue::String(x.to_string()))
+                            Ok::<_, ParquetGemError>(ParquetValue::String(x.to_string()))
                         }
                     }
                     None => Ok(ParquetValue::Null),
@@ -438,23 +551,23 @@ impl<'a> TryFrom<ArrayWrapper<'a>> for ParquetValueVec {
             }
             DataType::List(_field) => {
                 let list_array = downcast_array::<ListArray>(column.array);
-                Ok(ParquetValueVec(
-                    list_array
-                        .iter()
-                        .map(|x| match x {
-                            Some(values) => match ParquetValueVec::try_from(ArrayWrapper {
-                                array: &*values,
-                                strict: column.strict,
-                            }) {
-                                Ok(vec) => ParquetValue::List(vec.into_inner()),
-                                Err(e) => {
-                                    panic!("Error converting list array to ParquetValueVec: {}", e)
-                                }
-                            },
-                            None => ParquetValue::Null,
-                        })
-                        .collect(),
-                ))
+                let sub_list = list_array
+                    .iter()
+                    .map(|x| match x {
+                        Some(values) => match ParquetValueVec::try_from(ArrayWrapper {
+                            array: &*values,
+                            strict: column.strict,
+                        }) {
+                            Ok(vec) => Ok(ParquetValue::List(vec.into_inner())),
+                            Err(e) => Err(MagnusError::new(
+                                magnus::exception::type_error(),
+                                format!("Error converting list array to ParquetValueVec: {}", e),
+                            ))?,
+                        },
+                        None => Ok(ParquetValue::Null),
+                    })
+                    .collect::<Result<Vec<ParquetValue>, Self::Error>>()?;
+                Ok(ParquetValueVec(sub_list))
             }
             DataType::Struct(_) => {
                 let struct_array = downcast_array::<StructArray>(column.array);
@@ -474,27 +587,96 @@ impl<'a> TryFrom<ArrayWrapper<'a>> for ParquetValueVec {
                         }) {
                             Ok(vec) => vec.into_inner(),
                             Err(e) => {
-                                panic!("Error converting struct field to ParquetValueVec: {}", e)
+                                return Err(MagnusError::new(
+                                    magnus::exception::type_error(),
+                                    format!(
+                                        "Error converting struct field to ParquetValueVec: {}",
+                                        e
+                                    ),
+                                ))?;
                             }
                         };
                         map.insert(
                             ParquetValue::String(field.name().to_string()),
-                            field_values.into_iter().next().unwrap(),
+                            field_values.into_iter().next().ok_or_else(|| {
+                                MagnusError::new(
+                                    magnus::exception::type_error(),
+                                    "Expected a single value for struct field".to_string(),
+                                )
+                            })?,
                         );
                     }
                     values.push(ParquetValue::Map(map));
                 }
                 Ok(ParquetValueVec(values))
             }
+            DataType::Map(_field, _keys_sorted) => {
+                let map_array = downcast_array::<MapArray>(column.array);
+
+                let mut result = Vec::with_capacity(map_array.len());
+
+                let offsets = map_array.offsets();
+                let struct_array = map_array.entries();
+
+                for i in 0..map_array.len() {
+                    if map_array.is_null(i) {
+                        result.push(ParquetValue::Null);
+                        continue;
+                    }
+
+                    let start = offsets[i] as usize;
+                    let end = offsets[i + 1] as usize;
+
+                    let mut map_data =
+                        HashMap::with_capacity_and_hasher(end - start, Default::default());
+
+                    // In Arrow's MapArray, the entries are a struct with fields named "keys" and "values"
+                    // Get the columns directly by index since we know the structure
+                    let key_array = struct_array.column(0); // First field is always keys
+                    let val_array = struct_array.column(1); // Second field is always values
+
+                    for entry_index in start..end {
+                        let key_value = if key_array.is_null(entry_index) {
+                            ParquetValue::Null
+                        } else {
+                            let subarray = key_array.slice(entry_index, 1);
+                            let subwrapper = ArrayWrapper {
+                                array: &*subarray,
+                                strict: column.strict,
+                            };
+                            let mut converted = ParquetValueVec::try_from(subwrapper)?.0;
+                            converted.pop().unwrap_or(ParquetValue::Null)
+                        };
+
+                        let val_value = if val_array.is_null(entry_index) {
+                            ParquetValue::Null
+                        } else {
+                            let subarray = val_array.slice(entry_index, 1);
+                            let subwrapper = ArrayWrapper {
+                                array: &*subarray,
+                                strict: column.strict,
+                            };
+                            let mut converted = ParquetValueVec::try_from(subwrapper)?.0;
+                            converted.pop().unwrap_or(ParquetValue::Null)
+                        };
+
+                        map_data.insert(key_value, val_value);
+                    }
+
+                    result.push(ParquetValue::Map(map_data));
+                }
+
+                Ok(ParquetValueVec(result))
+            }
             DataType::Null => {
                 let x = downcast_array::<NullArray>(column.array);
                 Ok(ParquetValueVec(vec![ParquetValue::Null; x.len()]))
             }
             _ => {
-                return Err(ReaderError::Ruby(format!(
-                    "Unsupported data type: {:?}",
-                    column.array.data_type()
-                )));
+                return Err(MagnusError::new(
+                    magnus::exception::type_error(),
+                    format!("Unsupported data type: {:?}", column.array.data_type()),
+                ))?;
             }
         }
     }

data/ext/parquet/src/types/record_types.rs

@@ -1,7 +1,5 @@
 use itertools::Itertools;
 
-use crate::reader::ReaderError;
-
 use super::*;
 
 #[derive(Debug)]
@@ -20,12 +18,14 @@ pub enum ColumnRecord<S: BuildHasher + Default> {
 pub struct ParquetField(pub Field, pub bool);
 
 impl<S: BuildHasher + Default> TryIntoValue for RowRecord<S> {
-    fn try_into_value_with(self, handle: &Ruby) -> Result<Value, ReaderError> {
+    fn try_into_value_with(self, handle: &Ruby) -> Result<Value, ParquetGemError> {
         match self {
             RowRecord::Vec(vec) => {
                 let ary = handle.ary_new_capa(vec.len());
-                vec.into_iter()
-                    .try_for_each(|v| ary.push(v.try_into_value_with(handle)?))?;
+                vec.into_iter().try_for_each(|v| {
+                    ary.push(v.try_into_value_with(handle)?)?;
+                    Ok::<_, ParquetGemError>(())
+                })?;
                 Ok(handle.into_value(ary))
             }
             RowRecord::Map(map) => {
@@ -62,16 +62,18 @@ impl<S: BuildHasher + Default> TryIntoValue for RowRecord<S> {
 }
 
 impl<S: BuildHasher + Default> TryIntoValue for ColumnRecord<S> {
-    fn try_into_value_with(self, handle: &Ruby) -> Result<Value, ReaderError> {
+    fn try_into_value_with(self, handle: &Ruby) -> Result<Value, ParquetGemError> {
         match self {
             ColumnRecord::Vec(vec) => {
                 let ary = handle.ary_new_capa(vec.len());
                 vec.into_iter().try_for_each(|v| {
                     let nested_ary = handle.ary_new_capa(v.len());
-                    v.into_iter()
-                        .try_for_each(|v| nested_ary.push(v.try_into_value_with(handle)?))?;
+                    v.into_iter().try_for_each(|v| {
+                        nested_ary.push(v.try_into_value_with(handle)?)?;
+                        Ok::<_, ParquetGemError>(())
+                    })?;
                     ary.push(nested_ary.into_value_with(handle))?;
-                    Ok::<_, ReaderError>(())
+                    Ok::<_, ParquetGemError>(())
                 })?;
                 Ok(ary.into_value_with(handle))
             }
@@ -92,8 +94,10 @@ impl<S: BuildHasher + Default> TryIntoValue for ColumnRecord<S> {
                         }
                         values[i] = handle.into_value(k);
                         let ary = handle.ary_new_capa(v.len());
-                        v.into_iter()
-                            .try_for_each(|v| ary.push(v.try_into_value_with(handle)?))?;
+                        v.into_iter().try_for_each(|v| {
+                            ary.push(v.try_into_value_with(handle)?)?;
+                            Ok::<_, ParquetGemError>(())
+                        })?;
                         values[i + 1] = handle.into_value(ary);
                         i += 2;
                     }
@@ -112,11 +116,11 @@ impl<S: BuildHasher + Default> TryIntoValue for ColumnRecord<S> {
 }
 
 pub trait TryIntoValue {
-    fn try_into_value_with(self, handle: &Ruby) -> Result<Value, ReaderError>;
+    fn try_into_value_with(self, handle: &Ruby) -> Result<Value, ParquetGemError>;
 }
 
 impl TryIntoValue for ParquetField {
-    fn try_into_value_with(self, handle: &Ruby) -> Result<Value, ReaderError> {
+    fn try_into_value_with(self, handle: &Ruby) -> Result<Value, ParquetGemError> {
         match self.0 {
             Field::Null => Ok(handle.qnil().as_value()),
             Field::Bool(b) => Ok(b.into_value_with(handle)),
@@ -133,7 +137,7 @@ impl TryIntoValue for ParquetField {
             Field::Str(s) => {
                 if self.1 {
                     Ok(simdutf8::basic::from_utf8(s.as_bytes())
-                        .map_err(|e| ReaderError::Utf8Error(e))
+                        .map_err(|e| ParquetGemError::Utf8Error(e))
                         .and_then(|s| Ok(s.into_value_with(handle)))?)
                 } else {
                     let s = String::from_utf8_lossy(s.as_bytes());
@@ -165,7 +169,8 @@ impl TryIntoValue for ParquetField {
                 let elements = list.elements();
                 let ary = handle.ary_new_capa(elements.len());
                 elements.iter().try_for_each(|e| {
-                    ary.push(ParquetField(e.clone(), self.1).try_into_value_with(handle)?)
+                    ary.push(ParquetField(e.clone(), self.1).try_into_value_with(handle)?)?;
+                    Ok::<_, ParquetGemError>(())
                 })?;
                 Ok(ary.into_value_with(handle))
             }
@@ -176,7 +181,8 @@ impl TryIntoValue for ParquetField {
                     hash.aset(
                         ParquetField(k.clone(), self.1).try_into_value_with(handle)?,
                         ParquetField(v.clone(), self.1).try_into_value_with(handle)?,
-                    )
+                    )?;
+                    Ok::<_, ParquetGemError>(())
                 })?;
                 Ok(hash.into_value_with(handle))
             }
@@ -204,16 +210,11 @@ impl TryIntoValue for ParquetField {
                     hash.aset(
                         k.clone().into_value_with(handle),
                         ParquetField(v.clone(), self.1).try_into_value_with(handle)?,
-                    )
+                    )?;
+                    Ok::<_, ParquetGemError>(())
                 })?;
                 Ok(hash.into_value_with(handle))
             }
         }
     }
 }
-
-// impl IntoValue for ParquetField {
-//     fn into_value_with(self, handle: &Ruby) -> Value {
-//         self.try_into_value_with(handle).unwrap()
-//     }
-// }