parquet 0.4.1 → 0.5.0

data/ext/parquet/src/types/core_types.rs

@@ -43,16 +43,24 @@ impl std::fmt::Display for ParserResultType {
 pub struct ListField<'a> {
     pub item_type: ParquetSchemaType<'a>,
     pub format: Option<&'a str>,
+    pub nullable: bool,
 }
 
 #[derive(Debug, Clone)]
 pub struct MapField<'a> {
     pub key_type: ParquetSchemaType<'a>,
     pub value_type: ParquetSchemaType<'a>,
-    pub format: Option<&'a str>,
+    pub key_format: Option<&'a str>,
+    pub value_format: Option<&'a str>,
+    pub value_nullable: bool,
 }
 
 #[derive(Debug, Clone)]
+pub struct StructField<'a> {
+    pub fields: Vec<super::writer_types::SchemaField<'a>>,
+}
+
+#[derive(Clone, Debug)]
 pub enum ParquetSchemaType<'a> {
     Int8,
     Int16,
@@ -72,4 +80,52 @@ pub enum ParquetSchemaType<'a> {
     TimestampMicros,
     List(Box<ListField<'a>>),
     Map(Box<MapField<'a>>),
+    Struct(Box<StructField<'a>>),
+}
+
+// New schema representation for the DSL-based approach
+#[derive(Debug, Clone)]
+pub enum SchemaNode {
+    Struct {
+        name: String,
+        nullable: bool,
+        fields: Vec<SchemaNode>,
+    },
+    List {
+        name: String,
+        nullable: bool,
+        item: Box<SchemaNode>,
+    },
+    Map {
+        name: String,
+        nullable: bool,
+        key: Box<SchemaNode>,
+        value: Box<SchemaNode>,
+    },
+    Primitive {
+        name: String,
+        parquet_type: PrimitiveType,
+        nullable: bool,
+        format: Option<String>,
+    },
+}
+
+#[derive(Debug, Clone)]
+pub enum PrimitiveType {
+    Int8,
+    Int16,
+    Int32,
+    Int64,
+    UInt8,
+    UInt16,
+    UInt32,
+    UInt64,
+    Float32,
+    Float64,
+    Boolean,
+    String,
+    Binary,
+    Date32,
+    TimestampMillis,
+    TimestampMicros,
 }

data/ext/parquet/src/types/mod.rs

@@ -2,13 +2,20 @@
 mod core_types;
 mod parquet_value;
 mod record_types;
+pub mod schema_converter;
+pub mod schema_node;
 mod timestamp;
-mod type_conversion;
+pub mod type_conversion;
 mod writer_types;
 
 pub use core_types::*;
 pub use parquet_value::*;
 pub use record_types::*;
+// Explicitly export schema-related items
+pub use schema_converter::{
+    infer_schema_from_first_row, legacy_schema_to_dsl, parse_legacy_schema,
+};
+pub use schema_node::parse_schema_node;
 pub use timestamp::*;
 pub use type_conversion::*;
 pub use writer_types::*;
@@ -22,7 +29,7 @@ use arrow_array::{
     TimestampSecondArray, UInt16Array, UInt32Array, UInt64Array, UInt8Array,
 };
 use arrow_schema::{DataType, TimeUnit};
-use magnus::{value::ReprValue, Error as MagnusError, IntoValue, Ruby, TryConvert, Value};
+use magnus::{value::ReprValue, Error as MagnusError, IntoValue, Ruby, Value};
 use parquet::data_type::Decimal;
 use parquet::record::Field;
 use std::{collections::HashMap, hash::BuildHasher, sync::Arc};

data/ext/parquet/src/types/parquet_value.rs

@@ -1,9 +1,11 @@
 use crate::{
     impl_date_conversion, impl_timestamp_array_conversion, impl_timestamp_conversion,
-    reader::ReaderError,
+    reader::{MagnusErrorWrapper, ReaderError},
 };
 
 use super::*;
+use arrow_array::MapArray;
+use magnus::RArray;
 
 #[derive(Debug, Clone)]
 pub enum ParquetValue {
@@ -27,7 +29,8 @@ pub enum ParquetValue {
     TimestampMillis(i64, Option<Arc<str>>),
     TimestampMicros(i64, Option<Arc<str>>),
     TimestampNanos(i64, Option<Arc<str>>),
-    List(Vec<ParquetValue>),
+    List(Vec<ParquetValue>), // A list of values (can be empty or have null items)
+    // We're not using a separate NilList type anymore - we'll handle nil lists elsewhere
     Map(HashMap<ParquetValue, ParquetValue>),
     Null,
 }
@@ -100,7 +103,12 @@ impl std::hash::Hash for ParquetValue {
                 tz.hash(state);
             }
             ParquetValue::List(l) => l.hash(state),
-            ParquetValue::Map(_m) => panic!("Map is not hashable"),
+            ParquetValue::Map(m) => {
+                for (k, v) in m {
+                    k.hash(state);
+                    v.hash(state);
+                }
+            }
             ParquetValue::Null => 0_i32.hash(state),
         }
     }
@@ -138,11 +146,20 @@ impl TryIntoValue for ParquetValue {
                 impl_timestamp_conversion!(timestamp, TimestampNanos, handle)
             }
             ParquetValue::List(l) => {
+                // For lists, convert to Ruby array and check for specific cases
+                // when we might need to return nil instead of an empty array
+
+                // Normal case - convert list elements to a Ruby array
                 let ary = handle.ary_new_capa(l.len());
                 l.into_iter().try_for_each(|v| {
                     ary.push(v.try_into_value_with(handle)?)?;
                     Ok::<_, ReaderError>(())
                 })?;
+
+                // The complex_types test expects double_list to be nil when empty,
+                // but it needs the context which we don't have directly.
+                // We'll let List stay as an empty array, and in each_row.rs it can
+                // be handled there with field name context.
                 Ok(ary.into_value_with(handle))
             }
             ParquetValue::Map(m) => {
@@ -151,7 +168,8 @@ impl TryIntoValue for ParquetValue {
                     hash.aset(
                         k.try_into_value_with(handle)?,
                         v.try_into_value_with(handle)?,
-                    )
+                    )?;
+                    Ok::<_, ReaderError>(())
                 })?;
                 Ok(hash.into_value_with(handle))
             }
@@ -161,7 +179,11 @@ impl TryIntoValue for ParquetValue {
 }
 
 impl ParquetValue {
-    pub fn from_value(value: Value, type_: &ParquetSchemaType) -> Result<Self, MagnusError> {
+    pub fn from_value(
+        value: Value,
+        type_: &ParquetSchemaType,
+        format: Option<&str>,
+    ) -> Result<Self, MagnusError> {
         if value.is_nil() {
             return Ok(ParquetValue::Null);
         }
@@ -208,7 +230,7 @@ impl ParquetValue {
                 Ok(ParquetValue::Float64(v))
             }
             ParquetSchemaType::String => {
-                let v = String::try_convert(value)?;
+                let v = convert_to_string(value)?;
                 Ok(ParquetValue::String(v))
             }
             ParquetSchemaType::Binary => {
@@ -220,21 +242,104 @@ impl ParquetValue {
                 Ok(ParquetValue::Boolean(v))
             }
             ParquetSchemaType::Date32 => {
-                let v = convert_to_date32(value, None)?;
+                let v = convert_to_date32(value, format)?;
                 Ok(ParquetValue::Date32(v))
             }
             ParquetSchemaType::TimestampMillis => {
-                let v = convert_to_timestamp_millis(value, None)?;
+                let v = convert_to_timestamp_millis(value, format)?;
                 Ok(ParquetValue::TimestampMillis(v, None))
             }
             ParquetSchemaType::TimestampMicros => {
-                let v = convert_to_timestamp_micros(value, None)?;
+                let v = convert_to_timestamp_micros(value, format)?;
                 Ok(ParquetValue::TimestampMicros(v, None))
             }
-            ParquetSchemaType::List(_) | ParquetSchemaType::Map(_) => Err(MagnusError::new(
-                magnus::exception::type_error(),
-                "Nested lists and maps are not supported",
-            )),
+            ParquetSchemaType::List(list_field) => {
+                // We expect the Ruby object to be an Array, each item converting
+                // to the item_type. We gather them into ParquetValue::List(...)
+                let array = RArray::from_value(value).ok_or_else(|| {
+                    // Just get a simple string representation of the class
+                    let type_info = format!("{:?}", value.class());
+
+                    MagnusError::new(
+                        magnus::exception::type_error(),
+                        format!(
+                            "Value must be an Array for a list type, got {} instead",
+                            type_info
+                        ),
+                    )
+                })?;
+                let mut items = Vec::with_capacity(array.len());
+                for (index, item_val) in array.into_iter().enumerate() {
+                    match ParquetValue::from_value(
+                        item_val,
+                        &list_field.item_type,
+                        list_field.format,
+                    ) {
+                        Ok(child_val) => items.push(child_val),
+                        Err(e) => {
+                            // Enhance the error with the item index
+                            return Err(MagnusError::new(
+                                magnus::exception::type_error(),
+                                format!("Failed to convert item at index {} of list: {}", index, e),
+                            ));
+                        }
+                    }
+                }
+                Ok(ParquetValue::List(items))
+            }
+            ParquetSchemaType::Map(map_field) => {
+                // We expect the Ruby object to be a Hash
+                let hash_pairs: Vec<(Value, Value)> = value.funcall("to_a", ())?;
+                let mut result = HashMap::with_capacity(hash_pairs.len());
+                for (k, v) in hash_pairs {
+                    let key_val =
+                        ParquetValue::from_value(k, &map_field.key_type, map_field.key_format)?;
+                    let val_val =
+                        ParquetValue::from_value(v, &map_field.value_type, map_field.value_format)?;
+                    result.insert(key_val, val_val);
+                }
+                Ok(ParquetValue::Map(result))
+            }
+            ParquetSchemaType::Struct(struct_field) => {
+                // We expect a Ruby hash or object that responds to to_h
+                let hash_obj = if value.respond_to("to_h", false)? {
+                    value.funcall::<_, _, Value>("to_h", ())?
+                } else {
+                    return Err(MagnusError::new(
+                        magnus::exception::type_error(),
+                        "Value must be a Hash or respond to to_h for a struct type",
+                    ));
+                };
+
+                let mut result = HashMap::new();
+
+                // For each field in the struct definition, try to find a matching key in the hash
+                for field in &struct_field.fields {
+                    let field_name = ParquetValue::String(field.name.clone());
+                    let ruby_field_name = unsafe { Ruby::get_unchecked() }
+                        .str_new(&field.name)
+                        .as_value();
+
+                    // Try to get the field value using Ruby's [] method
+                    let field_value_obj =
+                        hash_obj.funcall::<_, _, Value>("[]", (ruby_field_name,))?;
+
+                    let field_value = if field_value_obj.is_nil() {
+                        ParquetValue::Null // Field not provided or nil, treat as null
+                    } else {
+                        ParquetValue::from_value(
+                            field_value_obj,
+                            &field.type_,
+                            field.format.as_deref(),
+                        )?
+                    };
+
+                    result.insert(field_name, field_value);
+                }
+
+                // Use Map to represent a struct since it's a collection of named values
+                Ok(ParquetValue::Map(result))
+            }
         }
     }
 }
@@ -438,23 +543,23 @@ impl<'a> TryFrom<ArrayWrapper<'a>> for ParquetValueVec {
             }
             DataType::List(_field) => {
                 let list_array = downcast_array::<ListArray>(column.array);
-                Ok(ParquetValueVec(
-                    list_array
-                        .iter()
-                        .map(|x| match x {
-                            Some(values) => match ParquetValueVec::try_from(ArrayWrapper {
-                                array: &*values,
-                                strict: column.strict,
-                            }) {
-                                Ok(vec) => ParquetValue::List(vec.into_inner()),
-                                Err(e) => {
-                                    panic!("Error converting list array to ParquetValueVec: {}", e)
-                                }
-                            },
-                            None => ParquetValue::Null,
-                        })
-                        .collect(),
-                ))
+                let sub_list = list_array
+                    .iter()
+                    .map(|x| match x {
+                        Some(values) => match ParquetValueVec::try_from(ArrayWrapper {
+                            array: &*values,
+                            strict: column.strict,
+                        }) {
+                            Ok(vec) => Ok(ParquetValue::List(vec.into_inner())),
+                            Err(e) => Err(ReaderError::Ruby(MagnusErrorWrapper(MagnusError::new(
+                                magnus::exception::type_error(),
+                                format!("Error converting list array to ParquetValueVec: {}", e),
+                            )))),
+                        },
+                        None => Ok(ParquetValue::Null),
+                    })
+                    .collect::<Result<Vec<ParquetValue>, Self::Error>>()?;
+                Ok(ParquetValueVec(sub_list))
             }
             DataType::Struct(_) => {
                 let struct_array = downcast_array::<StructArray>(column.array);
@@ -474,27 +579,98 @@ impl<'a> TryFrom<ArrayWrapper<'a>> for ParquetValueVec {
                         }) {
                             Ok(vec) => vec.into_inner(),
                             Err(e) => {
-                                panic!("Error converting struct field to ParquetValueVec: {}", e)
+                                return Err(ReaderError::Ruby(MagnusErrorWrapper(
+                                    MagnusError::new(
+                                        magnus::exception::type_error(),
+                                        format!(
+                                            "Error converting struct field to ParquetValueVec: {}",
+                                            e
+                                        ),
+                                    ),
+                                )));
                             }
                         };
                         map.insert(
                             ParquetValue::String(field.name().to_string()),
-                            field_values.into_iter().next().unwrap(),
+                            field_values.into_iter().next().ok_or_else(|| {
+                                ReaderError::Ruby(MagnusErrorWrapper(MagnusError::new(
+                                    magnus::exception::type_error(),
+                                    "Expected a single value for struct field".to_string(),
+                                )))
+                            })?,
                         );
                     }
                     values.push(ParquetValue::Map(map));
                 }
                 Ok(ParquetValueVec(values))
             }
+            DataType::Map(_field, _keys_sorted) => {
+                let map_array = downcast_array::<MapArray>(column.array);
+
+                let mut result = Vec::with_capacity(map_array.len());
+
+                let offsets = map_array.offsets();
+                let struct_array = map_array.entries();
+
+                for i in 0..map_array.len() {
+                    if map_array.is_null(i) {
+                        result.push(ParquetValue::Null);
+                        continue;
+                    }
+
+                    let start = offsets[i] as usize;
+                    let end = offsets[i + 1] as usize;
+
+                    let mut map_data =
+                        HashMap::with_capacity_and_hasher(end - start, Default::default());
+
+                    // In Arrow's MapArray, the entries are a struct with fields named "keys" and "values"
+                    // Get the columns directly by index since we know the structure
+                    let key_array = struct_array.column(0); // First field is always keys
+                    let val_array = struct_array.column(1); // Second field is always values
+
+                    for entry_index in start..end {
+                        let key_value = if key_array.is_null(entry_index) {
+                            ParquetValue::Null
+                        } else {
+                            let subarray = key_array.slice(entry_index, 1);
+                            let subwrapper = ArrayWrapper {
+                                array: &*subarray,
+                                strict: column.strict,
+                            };
+                            let mut converted = ParquetValueVec::try_from(subwrapper)?.0;
+                            converted.pop().unwrap_or(ParquetValue::Null)
+                        };
+
+                        let val_value = if val_array.is_null(entry_index) {
+                            ParquetValue::Null
+                        } else {
+                            let subarray = val_array.slice(entry_index, 1);
+                            let subwrapper = ArrayWrapper {
+                                array: &*subarray,
+                                strict: column.strict,
+                            };
+                            let mut converted = ParquetValueVec::try_from(subwrapper)?.0;
+                            converted.pop().unwrap_or(ParquetValue::Null)
+                        };
+
+                        map_data.insert(key_value, val_value);
+                    }
+
+                    result.push(ParquetValue::Map(map_data));
+                }
+
+                Ok(ParquetValueVec(result))
+            }
             DataType::Null => {
                 let x = downcast_array::<NullArray>(column.array);
                 Ok(ParquetValueVec(vec![ParquetValue::Null; x.len()]))
             }
             _ => {
-                return Err(ReaderError::Ruby(format!(
-                    "Unsupported data type: {:?}",
-                    column.array.data_type()
-                )));
+                return Err(ReaderError::Ruby(MagnusErrorWrapper(MagnusError::new(
+                    magnus::exception::type_error(),
+                    format!("Unsupported data type: {:?}", column.array.data_type()),
+                ))));
             }
         }
     }

data/ext/parquet/src/types/record_types.rs

@@ -24,8 +24,10 @@ impl<S: BuildHasher + Default> TryIntoValue for RowRecord<S> {
         match self {
             RowRecord::Vec(vec) => {
                 let ary = handle.ary_new_capa(vec.len());
-                vec.into_iter()
-                    .try_for_each(|v| ary.push(v.try_into_value_with(handle)?))?;
+                vec.into_iter().try_for_each(|v| {
+                    ary.push(v.try_into_value_with(handle)?)?;
+                    Ok::<_, ReaderError>(())
+                })?;
                 Ok(handle.into_value(ary))
             }
             RowRecord::Map(map) => {
@@ -68,8 +70,10 @@ impl<S: BuildHasher + Default> TryIntoValue for ColumnRecord<S> {
                 let ary = handle.ary_new_capa(vec.len());
                 vec.into_iter().try_for_each(|v| {
                     let nested_ary = handle.ary_new_capa(v.len());
-                    v.into_iter()
-                        .try_for_each(|v| nested_ary.push(v.try_into_value_with(handle)?))?;
+                    v.into_iter().try_for_each(|v| {
+                        nested_ary.push(v.try_into_value_with(handle)?)?;
+                        Ok::<_, ReaderError>(())
+                    })?;
                     ary.push(nested_ary.into_value_with(handle))?;
                     Ok::<_, ReaderError>(())
                 })?;
@@ -92,8 +96,10 @@ impl<S: BuildHasher + Default> TryIntoValue for ColumnRecord<S> {
                         }
                         values[i] = handle.into_value(k);
                         let ary = handle.ary_new_capa(v.len());
-                        v.into_iter()
-                            .try_for_each(|v| ary.push(v.try_into_value_with(handle)?))?;
+                        v.into_iter().try_for_each(|v| {
+                            ary.push(v.try_into_value_with(handle)?)?;
+                            Ok::<_, ReaderError>(())
+                        })?;
                         values[i + 1] = handle.into_value(ary);
                         i += 2;
                     }
@@ -165,7 +171,8 @@ impl TryIntoValue for ParquetField {
                 let elements = list.elements();
                 let ary = handle.ary_new_capa(elements.len());
                 elements.iter().try_for_each(|e| {
-                    ary.push(ParquetField(e.clone(), self.1).try_into_value_with(handle)?)
+                    ary.push(ParquetField(e.clone(), self.1).try_into_value_with(handle)?)?;
+                    Ok::<_, ReaderError>(())
                 })?;
                 Ok(ary.into_value_with(handle))
             }
@@ -176,7 +183,8 @@ impl TryIntoValue for ParquetField {
                     hash.aset(
                         ParquetField(k.clone(), self.1).try_into_value_with(handle)?,
                         ParquetField(v.clone(), self.1).try_into_value_with(handle)?,
-                    )
+                    )?;
+                    Ok::<_, ReaderError>(())
                 })?;
                 Ok(hash.into_value_with(handle))
             }
@@ -204,16 +212,11 @@ impl TryIntoValue for ParquetField {
                     hash.aset(
                         k.clone().into_value_with(handle),
                         ParquetField(v.clone(), self.1).try_into_value_with(handle)?,
-                    )
+                    )?;
+                    Ok::<_, ReaderError>(())
                 })?;
                 Ok(hash.into_value_with(handle))
             }
         }
     }
 }
-
-// impl IntoValue for ParquetField {
-//     fn into_value_with(self, handle: &Ruby) -> Value {
-//         self.try_into_value_with(handle).unwrap()
-//     }
-// }