parquet-tyfoom 0.8.0

data/ext/parquet-core/src/arrow_conversion.rs

@@ -0,0 +1,1243 @@
+//! Bidirectional conversion between Arrow arrays and ParquetValue
+//!
+//! This module provides a unified interface for converting between Arrow's
+//! columnar format and Parquet's value representation. It consolidates
+//! the conversion logic that was previously duplicated between the reader
+//! and writer modules.
+
+use crate::{ParquetError, ParquetValue, Result};
+use arrow_array::{builder::*, Array, ArrayRef, ListArray, MapArray, StructArray};
+use arrow_schema::extension::Uuid as ArrowUuid;
+use arrow_schema::{DataType, Field};
+use bytes::Bytes;
+use indexmap::IndexMap;
+use ordered_float::OrderedFloat;
+use parquet::basic::LogicalType;
+use parquet::schema::types::Type;
+use std::sync::Arc as StdArc;
+use triomphe::Arc;
+
+/// Convert a single value from an Arrow array at the given index to a ParquetValue
+pub fn arrow_to_parquet_value(
+    arrow_field: &Field,
+    parquet_field: &Type,
+    array: &dyn Array,
+    index: usize,
+) -> Result<ParquetValue> {
+    use arrow_array::*;
+
+    if array.is_null(index) {
+        return Ok(ParquetValue::Null);
+    }
+
+    match array.data_type() {
+        // Primitive types
+        DataType::Boolean => {
+            let array = downcast_array::<BooleanArray>(array)?;
+            Ok(ParquetValue::Boolean(array.value(index)))
+        }
+        DataType::Int8 => {
+            let array = downcast_array::<Int8Array>(array)?;
+            Ok(ParquetValue::Int8(array.value(index)))
+        }
+        DataType::Int16 => {
+            let array = downcast_array::<Int16Array>(array)?;
+            Ok(ParquetValue::Int16(array.value(index)))
+        }
+        DataType::Int32 => {
+            let array = downcast_array::<Int32Array>(array)?;
+            Ok(ParquetValue::Int32(array.value(index)))
+        }
+        DataType::Int64 => {
+            let array = downcast_array::<Int64Array>(array)?;
+            Ok(ParquetValue::Int64(array.value(index)))
+        }
+        DataType::UInt8 => {
+            let array = downcast_array::<UInt8Array>(array)?;
+            Ok(ParquetValue::UInt8(array.value(index)))
+        }
+        DataType::UInt16 => {
+            let array = downcast_array::<UInt16Array>(array)?;
+            Ok(ParquetValue::UInt16(array.value(index)))
+        }
+        DataType::UInt32 => {
+            let array = downcast_array::<UInt32Array>(array)?;
+            Ok(ParquetValue::UInt32(array.value(index)))
+        }
+        DataType::UInt64 => {
+            let array = downcast_array::<UInt64Array>(array)?;
+            Ok(ParquetValue::UInt64(array.value(index)))
+        }
+        DataType::Float16 => {
+            let array = downcast_array::<Float16Array>(array)?;
+            let value = array.value(index);
+            Ok(ParquetValue::Float16(OrderedFloat(value.to_f32())))
+        }
+        DataType::Float32 => {
+            let array = downcast_array::<Float32Array>(array)?;
+            Ok(ParquetValue::Float32(OrderedFloat(array.value(index))))
+        }
+        DataType::Float64 => {
+            let array = downcast_array::<Float64Array>(array)?;
+            Ok(ParquetValue::Float64(OrderedFloat(array.value(index))))
+        }
+        // String and binary types
+        DataType::Utf8 => {
+            let array = downcast_array::<StringArray>(array)?;
+            Ok(ParquetValue::String(Arc::from(array.value(index))))
+        }
+        DataType::Binary => {
+            let array = downcast_array::<BinaryArray>(array)?;
+            Ok(ParquetValue::Bytes(Bytes::copy_from_slice(
+                array.value(index),
+            )))
+        }
+        DataType::FixedSizeBinary(_) => {
+            let array = downcast_array::<FixedSizeBinaryArray>(array)?;
+            let value = array.value(index);
+            if let Some(LogicalType::Uuid) = parquet_field.get_basic_info().logical_type_ref() {
+                let uuid = uuid::Uuid::from_slice(value)
+                    .map_err(|e| ParquetError::Conversion(format!("Invalid UUID: {}", e)))?;
+                Ok(ParquetValue::Uuid(uuid))
+            } else {
+                match arrow_field.try_extension_type::<ArrowUuid>() {
+                    Ok(_) => {
+                        let uuid = uuid::Uuid::from_slice(value).map_err(|e| {
+                            ParquetError::Conversion(format!("Invalid UUID: {}", e))
+                        })?;
+                        Ok(ParquetValue::Uuid(uuid))
+                    }
+                    Err(_) => Ok(ParquetValue::Bytes(Bytes::copy_from_slice(value))),
+                }
+            }
+        }
+
+        // Date and time types
+        DataType::Date32 => {
+            let array = downcast_array::<Date32Array>(array)?;
+            Ok(ParquetValue::Date32(array.value(index)))
+        }
+        DataType::Date64 => {
+            let array = downcast_array::<Date64Array>(array)?;
+            Ok(ParquetValue::Date64(array.value(index)))
+        }
+
+        // Timestamp types
+        DataType::Timestamp(unit, timezone) => {
+            let timezone = timezone.as_ref().map(|s| Arc::from(s.as_ref()));
+            match unit {
+                arrow_schema::TimeUnit::Millisecond => {
+                    let array = downcast_array::<TimestampMillisecondArray>(array)?;
+                    Ok(ParquetValue::TimestampMillis(array.value(index), timezone))
+                }
+                arrow_schema::TimeUnit::Microsecond => {
+                    let array = downcast_array::<TimestampMicrosecondArray>(array)?;
+                    Ok(ParquetValue::TimestampMicros(array.value(index), timezone))
+                }
+                arrow_schema::TimeUnit::Second => {
+                    let array = downcast_array::<TimestampSecondArray>(array)?;
+                    Ok(ParquetValue::TimestampSecond(array.value(index), timezone))
+                }
+                arrow_schema::TimeUnit::Nanosecond => {
+                    let array = downcast_array::<TimestampNanosecondArray>(array)?;
+                    Ok(ParquetValue::TimestampNanos(array.value(index), timezone))
+                }
+            }
+        }
+
+        // Time types
+        DataType::Time32(unit) => match unit {
+            arrow_schema::TimeUnit::Millisecond => {
+                let array = downcast_array::<Time32MillisecondArray>(array)?;
+                Ok(ParquetValue::TimeMillis(array.value(index)))
+            }
+            _ => Err(ParquetError::Conversion(format!(
+                "Unsupported time32 unit: {:?}",
+                unit
+            ))),
+        },
+        DataType::Time64(unit) => match unit {
+            arrow_schema::TimeUnit::Microsecond => {
+                let array = downcast_array::<Time64MicrosecondArray>(array)?;
+                Ok(ParquetValue::TimeMicros(array.value(index)))
+            }
+            arrow_schema::TimeUnit::Nanosecond => {
+                let array = downcast_array::<Time64NanosecondArray>(array)?;
+                Ok(ParquetValue::TimeNanos(array.value(index)))
+            }
+            _ => Err(ParquetError::Conversion(format!(
+                "Unsupported time64 unit: {:?}",
+                unit
+            ))),
+        },
+
+        // Decimal types
+        DataType::Decimal128(_precision, scale) => {
+            let array = downcast_array::<Decimal128Array>(array)?;
+            let value = array.value(index);
+            Ok(ParquetValue::Decimal128(value, *scale))
+        }
+        DataType::Decimal256(_precision, scale) => {
+            let array = downcast_array::<Decimal256Array>(array)?;
+            let bytes = array.value(index).to_le_bytes();
+
+            // Convert to BigInt
+            let bigint = if bytes[31] & 0x80 != 0 {
+                // Negative number - convert from two's complement
+                let mut inverted = [0u8; 32];
+                for (i, &b) in bytes.iter().enumerate() {
+                    inverted[i] = !b;
+                }
+                let positive = num::BigInt::from_bytes_le(num::bigint::Sign::Plus, &inverted);
+                -(positive + num::BigInt::from(1))
+            } else {
+                num::BigInt::from_bytes_le(num::bigint::Sign::Plus, &bytes)
+            };
+
+            Ok(ParquetValue::Decimal256(bigint, *scale))
+        }
+
+        // Complex types
+        DataType::List(item_field) => {
+            let array = downcast_array::<ListArray>(array)?;
+            let list_values = array.value(index);
+
+            let mut values = Vec::with_capacity(list_values.len());
+
+            // Get the list's element type from parquet schema
+            let element_type = match parquet_field {
+                parquet::schema::types::Type::GroupType { fields, .. } => {
+                    // List has a repeated group containing the element
+                    // The structure is: LIST -> repeated group -> element
+                    if let Some(repeated_group) = fields.first() {
+                        match repeated_group.as_ref() {
+                            parquet::schema::types::Type::GroupType {
+                                fields: inner_fields,
+                                ..
+                            } => {
+                                // This is the repeated group, get the actual element
+                                inner_fields.first().ok_or_else(|| {
+                                    ParquetError::Conversion(
+                                        "List repeated group missing element field".to_string(),
+                                    )
+                                })?
+                            }
+                            _ => repeated_group, // If it's not a group, use it directly
+                        }
+                    } else {
+                        return Err(ParquetError::Conversion(
+                            "List type missing fields".to_string(),
+                        ));
+                    }
+                }
+                _ => parquet_field, // Fallback for cases where it's not a proper list structure
+            };
+
+            for i in 0..list_values.len() {
+                values.push(arrow_to_parquet_value(
+                    item_field,
+                    element_type,
+                    &list_values,
+                    i,
+                )?);
+            }
+
+            Ok(ParquetValue::List(values))
+        }
+        DataType::Map(_, _) => {
+            let array = downcast_array::<MapArray>(array)?;
+            let map_value = array.value(index);
+
+            // The Arrow `MapArray` entries struct is always (key, value) by
+            // position — `MapArray::keys()`/`values()` are `column(0)`/`column(1)`
+            // and `try_new` enforces exactly two columns — so we index by position
+            // and never depend on the entry field names (which the Parquet spec
+            // does not fix).
+            debug_assert_eq!(map_value.num_columns(), 2);
+            let keys = map_value.column(0);
+            let values = map_value.column(1);
+
+            let key_field = map_value
+                .fields()
+                .get(0)
+                .ok_or_else(|| ParquetError::Conversion("No key field found".to_string()))?;
+
+            let value_field = map_value
+                .fields()
+                .get(1)
+                .ok_or_else(|| ParquetError::Conversion("No value field found".to_string()))?;
+
+            let mut map_vec = Vec::with_capacity(keys.len());
+
+            // Get key and value types from parquet schema
+            // Map structure is: MAP -> key_value (repeated group) -> key, value
+            let (key_type, value_type) = match parquet_field {
+                parquet::schema::types::Type::GroupType { fields, .. } => {
+                    // Get the key_value repeated group
+                    match fields.first() {
+                        Some(key_value_group) => match key_value_group.as_ref() {
+                            parquet::schema::types::Type::GroupType {
+                                fields: kv_fields, ..
+                            } => {
+                                let key_field = kv_fields.first().ok_or_else(|| {
+                                    ParquetError::Conversion("Map missing key field".to_string())
+                                })?;
+                                let value_field = kv_fields.get(1).ok_or_else(|| {
+                                    ParquetError::Conversion("Map missing value field".to_string())
+                                })?;
+                                (key_field.as_ref(), value_field.as_ref())
+                            }
+                            _ => {
+                                return Err(ParquetError::Conversion(
+                                    "Map key_value should be a group".to_string(),
+                                ))
+                            }
+                        },
+                        None => {
+                            return Err(ParquetError::Conversion(
+                                "Map type missing key_value field".to_string(),
+                            ))
+                        }
+                    }
+                }
+                _ => {
+                    return Err(ParquetError::Conversion(
+                        "Map type must be a group".to_string(),
+                    ))
+                }
+            };
+
+            for i in 0..keys.len() {
+                let key = arrow_to_parquet_value(key_field, key_type, keys, i)?;
+                let value = arrow_to_parquet_value(value_field, value_type, values, i)?;
+                map_vec.push((key, value));
+            }
+
+            Ok(ParquetValue::Map(map_vec))
+        }
+        DataType::Struct(_) => {
+            let array = downcast_array::<StructArray>(array)?;
+
+            let mut map = IndexMap::new();
+
+            // Get struct fields from parquet schema
+            let parquet_fields = match parquet_field {
+                parquet::schema::types::Type::GroupType { fields, .. } => fields,
+                _ => {
+                    return Err(ParquetError::Conversion(
+                        "Struct type must be a group".to_string(),
+                    ))
+                }
+            };
+
+            for (col_idx, arrow_field) in array.fields().iter().enumerate() {
+                let column = array.column(col_idx);
+
+                // Find matching parquet field by name
+                let nested_parquet_field = parquet_fields
+                    .iter()
+                    .find(|f| f.name() == arrow_field.name())
+                    .ok_or_else(|| {
+                        ParquetError::Conversion(format!(
+                            "No matching parquet field for struct field '{}'",
+                            arrow_field.name()
+                        ))
+                    })?;
+
+                let value =
+                    arrow_to_parquet_value(arrow_field, nested_parquet_field, column, index)?;
+                map.insert(Arc::from(arrow_field.name().as_str()), value);
+            }
+
+            Ok(ParquetValue::Record(map))
+        }
+
+        dt => Err(ParquetError::Conversion(format!(
+            "Unsupported data type for conversion: {:?}",
+            dt
+        ))),
+    }
+}
+
+/// Convert a slice of ParquetValues to an Arrow array
+pub fn parquet_values_to_arrow_array(values: &[ParquetValue], field: &Field) -> Result<ArrayRef> {
+    let value_refs = values.iter().collect::<Vec<_>>();
+    parquet_value_refs_to_arrow_array(&value_refs, field)
+}
+
+fn parquet_value_refs_to_arrow_array(values: &[&ParquetValue], field: &Field) -> Result<ArrayRef> {
+    match field.data_type() {
+        // Boolean
+        DataType::Boolean => {
+            let mut builder = BooleanBuilder::with_capacity(values.len());
+            for value in values {
+                match *value {
+                    ParquetValue::Boolean(b) => builder.append_value(*b),
+                    ParquetValue::Null => builder.append_null(),
+                    _ => {
+                        return Err(ParquetError::Conversion(format!(
+                            "Expected Boolean, got {:?}",
+                            value.type_name()
+                        )))
+                    }
+                }
+            }
+            Ok(StdArc::new(builder.finish()))
+        }
+
+        // Integer types with automatic upcasting
+        DataType::Int8 => build_int8_array(values),
+        DataType::Int16 => build_int16_array(values),
+        DataType::Int32 => build_int32_array(values),
+        DataType::Int64 => build_int64_array(values),
+        DataType::UInt8 => build_uint8_array(values),
+        DataType::UInt16 => build_uint16_array(values),
+        DataType::UInt32 => build_uint32_array(values),
+        DataType::UInt64 => build_uint64_array(values),
+
+        // Float types
+        DataType::Float32 => build_float32_array(values),
+        DataType::Float64 => build_float64_array(values),
+
+        // String and binary
+        DataType::Utf8 => build_string_array(values),
+        DataType::Binary => build_binary_array(values),
+        DataType::FixedSizeBinary(size) => build_fixed_binary_array(values, *size),
+
+        // Date and time
+        DataType::Date32 => build_date32_array(values),
+        DataType::Date64 => build_date64_array(values),
+        DataType::Time32(unit) => build_time32_array(values, unit),
+        DataType::Time64(unit) => build_time64_array(values, unit),
+
+        // Timestamp
+        DataType::Timestamp(unit, tz) => build_timestamp_array(values, unit, tz.as_deref()),
+
+        // Decimal
+        DataType::Decimal128(precision, scale) => {
+            build_decimal128_array(values, *precision, *scale)
+        }
+        DataType::Decimal256(precision, scale) => {
+            build_decimal256_array(values, *precision, *scale)
+        }
+
+        // Complex types
+        DataType::List(item_field) => build_list_array(values, item_field),
+        DataType::Map(entries_field, sorted) => build_map_array(values, entries_field, *sorted),
+        DataType::Struct(fields) => build_struct_array(values, fields),
+
+        dt => Err(ParquetError::Conversion(format!(
+            "Unsupported data type for conversion: {:?}",
+            dt
+        ))),
+    }
+}
+
+/// Helper function to downcast an array with better error messages
+fn downcast_array<T: 'static>(array: &dyn Array) -> Result<&T> {
+    array.as_any().downcast_ref::<T>().ok_or_else(|| {
+        ParquetError::Conversion(format!("Failed to cast to {}", std::any::type_name::<T>()))
+    })
+}
+
+/// Build Int8 array
+fn build_int8_array(values: &[&ParquetValue]) -> Result<ArrayRef> {
+    let mut builder = Int8Builder::with_capacity(values.len());
+    for value in values {
+        match *value {
+            ParquetValue::Int8(i) => builder.append_value(*i),
+            ParquetValue::Null => builder.append_null(),
+            _ => {
+                return Err(ParquetError::Conversion(format!(
+                    "Expected Int8, got {:?}",
+                    value.type_name()
+                )))
+            }
+        }
+    }
+    Ok(StdArc::new(builder.finish()))
+}
+
+/// Build Int16 array
+fn build_int16_array(values: &[&ParquetValue]) -> Result<ArrayRef> {
+    let mut builder = Int16Builder::with_capacity(values.len());
+    for value in values {
+        match *value {
+            ParquetValue::Int16(i) => builder.append_value(*i),
+            ParquetValue::Int8(i) => builder.append_value(*i as i16),
+            ParquetValue::Null => builder.append_null(),
+            _ => {
+                return Err(ParquetError::Conversion(format!(
+                    "Expected Int16, got {:?}",
+                    value.type_name()
+                )))
+            }
+        }
+    }
+    Ok(StdArc::new(builder.finish()))
+}
+
+/// Build Int32 array
+fn build_int32_array(values: &[&ParquetValue]) -> Result<ArrayRef> {
+    let mut builder = Int32Builder::with_capacity(values.len());
+    for value in values {
+        match *value {
+            ParquetValue::Int32(i) => builder.append_value(*i),
+            ParquetValue::Int16(i) => builder.append_value(*i as i32),
+            ParquetValue::Int8(i) => builder.append_value(*i as i32),
+            ParquetValue::Null => builder.append_null(),
+            _ => {
+                return Err(ParquetError::Conversion(format!(
+                    "Expected Int32, got {:?}",
+                    value.type_name()
+                )))
+            }
+        }
+    }
+    Ok(StdArc::new(builder.finish()))
+}
+
+/// Build Int64 array
+fn build_int64_array(values: &[&ParquetValue]) -> Result<ArrayRef> {
+    let mut builder = Int64Builder::with_capacity(values.len());
+    for value in values {
+        match *value {
+            ParquetValue::Int64(i) => builder.append_value(*i),
+            ParquetValue::Int32(i) => builder.append_value(*i as i64),
+            ParquetValue::Int16(i) => builder.append_value(*i as i64),
+            ParquetValue::Int8(i) => builder.append_value(*i as i64),
+            ParquetValue::Null => builder.append_null(),
+            _ => {
+                return Err(ParquetError::Conversion(format!(
+                    "Expected Int64, got {:?}",
+                    value.type_name()
+                )))
+            }
+        }
+    }
+    Ok(StdArc::new(builder.finish()))
+}
+
+/// Build UInt8 array
+fn build_uint8_array(values: &[&ParquetValue]) -> Result<ArrayRef> {
+    let mut builder = UInt8Builder::with_capacity(values.len());
+    for value in values {
+        match *value {
+            ParquetValue::UInt8(i) => builder.append_value(*i),
+            ParquetValue::Null => builder.append_null(),
+            _ => {
+                return Err(ParquetError::Conversion(format!(
+                    "Expected UInt8, got {:?}",
+                    value.type_name()
+                )))
+            }
+        }
+    }
+    Ok(StdArc::new(builder.finish()))
+}
+
+/// Build UInt16 array
+fn build_uint16_array(values: &[&ParquetValue]) -> Result<ArrayRef> {
+    let mut builder = UInt16Builder::with_capacity(values.len());
+    for value in values {
+        match *value {
+            ParquetValue::UInt16(i) => builder.append_value(*i),
+            ParquetValue::UInt8(i) => builder.append_value(*i as u16),
+            ParquetValue::Null => builder.append_null(),
+            _ => {
+                return Err(ParquetError::Conversion(format!(
+                    "Expected UInt16, got {:?}",
+                    value.type_name()
+                )))
+            }
+        }
+    }
+    Ok(StdArc::new(builder.finish()))
+}
+
+/// Build UInt32 array
+fn build_uint32_array(values: &[&ParquetValue]) -> Result<ArrayRef> {
+    let mut builder = UInt32Builder::with_capacity(values.len());
+    for value in values {
+        match *value {
+            ParquetValue::UInt32(i) => builder.append_value(*i),
+            ParquetValue::UInt16(i) => builder.append_value(*i as u32),
+            ParquetValue::UInt8(i) => builder.append_value(*i as u32),
+            ParquetValue::Null => builder.append_null(),
+            _ => {
+                return Err(ParquetError::Conversion(format!(
+                    "Expected UInt32, got {:?}",
+                    value.type_name()
+                )))
+            }
+        }
+    }
+    Ok(StdArc::new(builder.finish()))
+}
+
+/// Build UInt64 array
+fn build_uint64_array(values: &[&ParquetValue]) -> Result<ArrayRef> {
+    let mut builder = UInt64Builder::with_capacity(values.len());
+    for value in values {
+        match *value {
+            ParquetValue::UInt64(i) => builder.append_value(*i),
+            ParquetValue::UInt32(i) => builder.append_value(*i as u64),
+            ParquetValue::UInt16(i) => builder.append_value(*i as u64),
+            ParquetValue::UInt8(i) => builder.append_value(*i as u64),
+            ParquetValue::Null => builder.append_null(),
+            _ => {
+                return Err(ParquetError::Conversion(format!(
+                    "Expected UInt64, got {:?}",
+                    value.type_name()
+                )))
+            }
+        }
+    }
+    Ok(StdArc::new(builder.finish()))
+}
+
+/// Build Float32 array with Float16 support
+fn build_float32_array(values: &[&ParquetValue]) -> Result<ArrayRef> {
+    let mut builder = Float32Builder::with_capacity(values.len());
+    for value in values {
+        match *value {
+            ParquetValue::Float32(OrderedFloat(f)) => builder.append_value(*f),
+            ParquetValue::Float16(OrderedFloat(f)) => builder.append_value(*f),
+            ParquetValue::Null => builder.append_null(),
+            _ => {
+                return Err(ParquetError::Conversion(format!(
+                    "Expected Float32, got {:?}",
+                    value.type_name()
+                )))
+            }
+        }
+    }
+    Ok(StdArc::new(builder.finish()))
+}
+
+/// Build Float64 array with Float32 and Float16 support
+fn build_float64_array(values: &[&ParquetValue]) -> Result<ArrayRef> {
+    let mut builder = Float64Builder::with_capacity(values.len());
+    for value in values {
+        match *value {
+            ParquetValue::Float64(OrderedFloat(f)) => builder.append_value(*f),
+            ParquetValue::Float32(OrderedFloat(f)) => builder.append_value(*f as f64),
+            ParquetValue::Float16(OrderedFloat(f)) => builder.append_value(*f as f64),
+            ParquetValue::Null => builder.append_null(),
+            _ => {
+                return Err(ParquetError::Conversion(format!(
+                    "Expected Float64, got {:?}",
+                    value.type_name()
+                )))
+            }
+        }
+    }
+    Ok(StdArc::new(builder.finish()))
+}
+
+/// Build string array
+fn build_string_array(values: &[&ParquetValue]) -> Result<ArrayRef> {
+    // Pre-size the data buffer exactly: growing it by doubling would
+    // transiently hold up to 3x the payload during the final realloc.
+    let data_capacity: usize = values
+        .iter()
+        .map(|value| match value {
+            ParquetValue::String(s) => s.len(),
+            _ => 0,
+        })
+        .sum();
+    let mut builder = StringBuilder::with_capacity(values.len(), data_capacity);
+    for value in values {
+        match *value {
+            ParquetValue::String(s) => builder.append_value(s.as_ref()),
+            ParquetValue::Null => builder.append_null(),
+            _ => {
+                return Err(ParquetError::Conversion(format!(
+                    "Expected String, got {:?}",
+                    value.type_name()
+                )))
+            }
+        }
+    }
+    Ok(StdArc::new(builder.finish()))
+}
+
+/// Build binary array
+fn build_binary_array(values: &[&ParquetValue]) -> Result<ArrayRef> {
+    // Pre-size the data buffer exactly, as in build_string_array.
+    let data_capacity: usize = values
+        .iter()
+        .map(|value| match value {
+            ParquetValue::Bytes(b) => b.len(),
+            _ => 0,
+        })
+        .sum();
+    let mut builder = BinaryBuilder::with_capacity(values.len(), data_capacity);
+    for value in values {
+        match *value {
+            ParquetValue::Bytes(b) => builder.append_value(b.as_ref()),
+            ParquetValue::Null => builder.append_null(),
+            _ => {
+                return Err(ParquetError::Conversion(format!(
+                    "Expected Bytes, got {:?}",
+                    value.type_name()
+                )))
+            }
+        }
+    }
+    Ok(StdArc::new(builder.finish()))
+}
+
+/// Build fixed size binary array
+fn build_fixed_binary_array(values: &[&ParquetValue], size: i32) -> Result<ArrayRef> {
+    let mut builder = FixedSizeBinaryBuilder::with_capacity(values.len(), size);
+    for value in values {
+        match *value {
+            ParquetValue::Bytes(b) => {
+                if b.len() != size as usize {
+                    return Err(ParquetError::Conversion(format!(
+                        "Fixed size binary expected {} bytes, got {}",
+                        size,
+                        b.len()
+                    )));
+                }
+                builder.append_value(b.as_ref())?;
+            }
+            ParquetValue::Null => builder.append_null(),
+            _ => {
+                return Err(ParquetError::Conversion(format!(
+                    "Expected Bytes, got {:?}",
+                    value.type_name()
+                )))
+            }
+        }
+    }
+    Ok(StdArc::new(builder.finish()))
+}
+
+/// Build Date32 array
+fn build_date32_array(values: &[&ParquetValue]) -> Result<ArrayRef> {
+    let mut builder = Date32Builder::with_capacity(values.len());
+    for value in values {
+        match *value {
+            ParquetValue::Date32(d) => builder.append_value(*d),
+            ParquetValue::Null => builder.append_null(),
+            _ => {
+                return Err(ParquetError::Conversion(format!(
+                    "Expected Date32, got {:?}",
+                    value.type_name()
+                )))
+            }
+        }
+    }
+    Ok(StdArc::new(builder.finish()))
+}
+
+/// Build Date64 array
+fn build_date64_array(values: &[&ParquetValue]) -> Result<ArrayRef> {
+    let mut builder = Date64Builder::with_capacity(values.len());
+    for value in values {
+        match *value {
+            ParquetValue::Date64(d) => builder.append_value(*d),
+            ParquetValue::Null => builder.append_null(),
+            _ => {
+                return Err(ParquetError::Conversion(format!(
+                    "Expected Date64, got {:?}",
+                    value.type_name()
+                )))
+            }
+        }
+    }
+    Ok(StdArc::new(builder.finish()))
+}
+
+/// Build Time32 array
+fn build_time32_array(values: &[&ParquetValue], unit: &arrow_schema::TimeUnit) -> Result<ArrayRef> {
+    match unit {
+        arrow_schema::TimeUnit::Millisecond => {
+            let mut builder = Time32MillisecondBuilder::with_capacity(values.len());
+            for value in values {
+                match *value {
+                    ParquetValue::TimeMillis(t) => builder.append_value(*t),
+                    ParquetValue::Null => builder.append_null(),
+                    _ => {
+                        return Err(ParquetError::Conversion(format!(
+                            "Expected TimeMillis, got {:?}",
+                            value.type_name()
+                        )))
+                    }
+                }
+            }
+            Ok(StdArc::new(builder.finish()))
+        }
+        _ => Err(ParquetError::Conversion(format!(
+            "Unsupported time32 unit: {:?}",
+            unit
+        ))),
+    }
+}
+
+/// Build Time64 array
+fn build_time64_array(values: &[&ParquetValue], unit: &arrow_schema::TimeUnit) -> Result<ArrayRef> {
+    match unit {
+        arrow_schema::TimeUnit::Microsecond => {
+            let mut builder = Time64MicrosecondBuilder::with_capacity(values.len());
+            for value in values {
+                match *value {
+                    ParquetValue::TimeMicros(t) => builder.append_value(*t),
+                    ParquetValue::Null => builder.append_null(),
+                    _ => {
+                        return Err(ParquetError::Conversion(format!(
+                            "Expected TimeMicros, got {:?}",
+                            value.type_name()
+                        )))
+                    }
+                }
+            }
+            Ok(StdArc::new(builder.finish()))
+        }
+        arrow_schema::TimeUnit::Nanosecond => {
+            let mut builder = Time64NanosecondBuilder::with_capacity(values.len());
+            for value in values {
+                match *value {
+                    ParquetValue::TimeNanos(t) => builder.append_value(*t),
+                    ParquetValue::Null => builder.append_null(),
+                    _ => {
+                        return Err(ParquetError::Conversion(format!(
+                            "Expected TimeNanos, got {:?}",
+                            value.type_name()
+                        )))
+                    }
+                }
+            }
+            Ok(StdArc::new(builder.finish()))
+        }
+        _ => Err(ParquetError::Conversion(format!(
+            "Unsupported time64 unit: {:?}",
+            unit
+        ))),
+    }
+}
+
+/// Build timestamp array
+fn build_timestamp_array(
+    values: &[&ParquetValue],
+    unit: &arrow_schema::TimeUnit,
+    timezone: Option<&str>,
+) -> Result<ArrayRef> {
+    let tz = timezone.map(StdArc::from);
+
+    match unit {
+        arrow_schema::TimeUnit::Second => {
+            let mut builder =
+                TimestampSecondBuilder::with_capacity(values.len()).with_timezone_opt(tz.clone());
+            for value in values {
+                match *value {
+                    ParquetValue::TimestampSecond(t, _) => builder.append_value(*t),
+                    ParquetValue::Null => builder.append_null(),
+                    _ => {
+                        return Err(ParquetError::Conversion(format!(
+                            "Expected TimestampSecond, got {:?}",
+                            value.type_name()
+                        )))
+                    }
+                }
+            }
+            Ok(StdArc::new(builder.finish()))
+        }
+        arrow_schema::TimeUnit::Millisecond => {
+            let mut builder = TimestampMillisecondBuilder::with_capacity(values.len())
+                .with_timezone_opt(tz.clone());
+            for value in values {
+                match *value {
+                    ParquetValue::TimestampMillis(t, _) => builder.append_value(*t),
+                    ParquetValue::Null => builder.append_null(),
+                    _ => {
+                        return Err(ParquetError::Conversion(format!(
+                            "Expected TimestampMillis, got {:?}",
+                            value.type_name()
+                        )))
+                    }
+                }
+            }
+            Ok(StdArc::new(builder.finish()))
+        }
+        arrow_schema::TimeUnit::Microsecond => {
+            let mut builder = TimestampMicrosecondBuilder::with_capacity(values.len())
+                .with_timezone_opt(tz.clone());
+            for value in values {
+                match *value {
+                    ParquetValue::TimestampMicros(t, _) => builder.append_value(*t),
+                    ParquetValue::Null => builder.append_null(),
+                    _ => {
+                        return Err(ParquetError::Conversion(format!(
+                            "Expected TimestampMicros, got {:?}",
+                            value.type_name()
+                        )))
+                    }
+                }
+            }
+            Ok(StdArc::new(builder.finish()))
+        }
+        arrow_schema::TimeUnit::Nanosecond => {
+            let mut builder = TimestampNanosecondBuilder::with_capacity(values.len())
+                .with_timezone_opt(tz.clone());
+            for value in values {
+                match *value {
+                    ParquetValue::TimestampNanos(t, _) => builder.append_value(*t),
+                    ParquetValue::Null => builder.append_null(),
+                    _ => {
+                        return Err(ParquetError::Conversion(format!(
+                            "Expected TimestampNanos, got {:?}",
+                            value.type_name()
+                        )))
+                    }
+                }
+            }
+            Ok(StdArc::new(builder.finish()))
+        }
+    }
+}
+
+/// Build Decimal128 array
+fn build_decimal128_array(values: &[&ParquetValue], precision: u8, scale: i8) -> Result<ArrayRef> {
+    let mut builder = Decimal128Builder::with_capacity(values.len())
+        .with_precision_and_scale(precision, scale)?;
+    for (idx, value) in values.iter().enumerate() {
+        match *value {
+            ParquetValue::Decimal128(d, value_scale) => {
+                validate_decimal128_array_value(*d, *value_scale, precision, scale, idx)?;
+                builder.append_value(*d);
+            }
+            ParquetValue::Null => builder.append_null(),
+            _ => {
+                return Err(ParquetError::Conversion(format!(
+                    "Expected Decimal128, got {:?}",
+                    value.type_name()
+                )))
+            }
+        }
+    }
+    Ok(StdArc::new(builder.finish()))
+}
+
+/// Build Decimal256 array
+fn build_decimal256_array(values: &[&ParquetValue], precision: u8, scale: i8) -> Result<ArrayRef> {
+    let mut builder = Decimal256Builder::with_capacity(values.len())
+        .with_precision_and_scale(precision, scale)?;
+    for (idx, value) in values.iter().enumerate() {
+        match *value {
+            ParquetValue::Decimal256(bigint, value_scale) => {
+                validate_decimal256_array_value(bigint, *value_scale, precision, scale, idx)?;
+                let bytes = decimal256_from_bigint(bigint)?;
+                builder.append_value(bytes);
+            }
+            ParquetValue::Null => builder.append_null(),
+            _ => {
+                return Err(ParquetError::Conversion(format!(
+                    "Expected Decimal256, got {:?}",
+                    value.type_name()
+                )))
+            }
+        }
+    }
+    Ok(StdArc::new(builder.finish()))
+}
+
+fn validate_decimal128_array_value(
+    value: i128,
+    value_scale: i8,
+    precision: u8,
+    scale: i8,
+    index: usize,
+) -> Result<()> {
+    if value_scale != scale {
+        return Err(ParquetError::Conversion(format!(
+            "Decimal scale mismatch at value[{}]: array scale {}, value scale {}",
+            index, scale, value_scale
+        )));
+    }
+
+    validate_decimal_array_precision(decimal128_digit_count(value), precision, index)
+}
+
+fn validate_decimal256_array_value(
+    value: &num::BigInt,
+    value_scale: i8,
+    precision: u8,
+    scale: i8,
+    index: usize,
+) -> Result<()> {
+    if value_scale != scale {
+        return Err(ParquetError::Conversion(format!(
+            "Decimal scale mismatch at value[{}]: array scale {}, value scale {}",
+            index, scale, value_scale
+        )));
+    }
+
+    validate_decimal_array_precision(decimal256_digit_count(value), precision, index)
+}
+
+fn validate_decimal_array_precision(
+    value_digits: usize,
+    precision: u8,
+    index: usize,
+) -> Result<()> {
+    if value_digits > precision as usize {
+        return Err(ParquetError::Conversion(format!(
+            "Decimal precision overflow at value[{}]: array precision {}, value has {} digits",
+            index, precision, value_digits
+        )));
+    }
+
+    Ok(())
+}
+
+fn decimal128_digit_count(value: i128) -> usize {
+    value.unsigned_abs().to_string().len()
+}
+
+fn decimal256_digit_count(value: &num::BigInt) -> usize {
+    value.to_str_radix(10).trim_start_matches('-').len()
+}
+
+/// Convert BigInt to i256 (32-byte array)
+fn decimal256_from_bigint(bigint: &num::BigInt) -> Result<arrow_buffer::i256> {
+    // Get bytes in little-endian format
+    let (sign, mut bytes) = bigint.to_bytes_le();
+
+    // Ensure we have exactly 32 bytes
+    if bytes.len() > 32 {
+        return Err(ParquetError::Conversion(
+            "Decimal256 value too large".to_string(),
+        ));
+    }
+
+    // Pad with zeros or ones (for negative numbers) to reach 32 bytes
+    bytes.resize(32, 0);
+
+    // If negative, convert to two's complement
+    if sign == num::bigint::Sign::Minus {
+        // Invert all bits
+        for byte in &mut bytes {
+            *byte = !*byte;
+        }
+        // Add 1
+        let mut carry = true;
+        for byte in &mut bytes {
+            if carry {
+                let (new_byte, new_carry) = byte.overflowing_add(1);
+                *byte = new_byte;
+                carry = new_carry;
+            } else {
+                break;
+            }
+        }
+    }
+
+    let byte_array: [u8; 32] = bytes
+        .try_into()
+        .map_err(|_| ParquetError::Conversion("Failed to convert bytes to i256".to_string()))?;
+    Ok(arrow_buffer::i256::from_le_bytes(byte_array))
+}
+
+/// Build list array
+fn build_list_array(values: &[&ParquetValue], item_field: &StdArc<Field>) -> Result<ArrayRef> {
+    let mut all_items = Vec::new();
+    let mut offsets = Vec::with_capacity(values.len() + 1);
+    let mut null_buffer_builder = arrow_buffer::BooleanBufferBuilder::new(values.len());
+    offsets.push(0i32);
+
+    for value in values {
+        match *value {
+            ParquetValue::List(items) => {
+                all_items.extend(items.iter());
+                offsets.push(all_items.len() as i32);
+                null_buffer_builder.append(true);
+            }
+            ParquetValue::Null => {
+                offsets.push(all_items.len() as i32);
+                null_buffer_builder.append(false);
+            }
+            _ => {
+                return Err(ParquetError::Conversion(format!(
+                    "Expected List, got {:?}",
+                    value.type_name()
+                )))
+            }
+        }
+    }
+
+    let item_array = parquet_value_refs_to_arrow_array(&all_items, item_field)?;
+    let offset_buffer = arrow_buffer::OffsetBuffer::new(offsets.into());
+    let null_buffer = null_buffer_builder.finish();
+
+    Ok(StdArc::new(ListArray::new(
+        item_field.clone(),
+        offset_buffer,
+        item_array,
+        Some(null_buffer.into()),
+    )))
+}
+
+/// Build map array
+fn build_map_array(
+    values: &[&ParquetValue],
+    entries_field: &StdArc<Field>,
+    _sorted: bool,
+) -> Result<ArrayRef> {
+    // Extract the key and value fields from the entries struct
+    let (key_field, value_field) = match entries_field.data_type() {
+        DataType::Struct(fields) if fields.len() == 2 => (&fields[0], &fields[1]),
+        _ => {
+            return Err(ParquetError::Conversion(
+                "Map entries field must be a struct with exactly 2 fields".to_string(),
+            ))
+        }
+    };
+
+    let mut all_keys = Vec::new();
+    let mut all_values = Vec::new();
+    let mut offsets = Vec::with_capacity(values.len() + 1);
+    let mut null_buffer_builder = arrow_buffer::BooleanBufferBuilder::new(values.len());
+    offsets.push(0i32);
+
+    for value in values {
+        match *value {
+            ParquetValue::Map(entries) => {
+                for (k, v) in entries {
+                    all_keys.push(k);
+                    all_values.push(v);
+                }
+                offsets.push(all_keys.len() as i32);
+                null_buffer_builder.append(true);
+            }
+            ParquetValue::Null => {
+                offsets.push(all_keys.len() as i32);
+                null_buffer_builder.append(false);
+            }
+            _ => {
+                return Err(ParquetError::Conversion(format!(
+                    "Expected Map, got {:?}",
+                    value.type_name()
+                )))
+            }
+        }
+    }
+
+    let key_array = parquet_value_refs_to_arrow_array(&all_keys, key_field)?;
+    let value_array = parquet_value_refs_to_arrow_array(&all_values, value_field)?;
+
+    // Create struct array for entries
+    let struct_fields = match entries_field.data_type() {
+        DataType::Struct(fields) => fields.clone(),
+        _ => unreachable!("Map entries field must be a struct"),
+    };
+
+    let struct_array = StructArray::new(struct_fields, vec![key_array, value_array], None);
+
+    let offset_buffer = arrow_buffer::OffsetBuffer::new(offsets.into());
+    let null_buffer = null_buffer_builder.finish();
+
+    Ok(StdArc::new(MapArray::new(
+        entries_field.clone(),
+        offset_buffer,
+        struct_array,
+        Some(null_buffer.into()),
+        false, // sorted
+    )))
+}
+
+/// Build struct array
+fn build_struct_array(values: &[&ParquetValue], fields: &arrow_schema::Fields) -> Result<ArrayRef> {
+    let num_rows = values.len();
+    let mut field_arrays = Vec::with_capacity(fields.len());
+    let mut null_buffer_builder = arrow_buffer::BooleanBufferBuilder::new(num_rows);
+    let null_value = ParquetValue::Null;
+
+    // Prepare columns for each field
+    let mut field_columns: Vec<Vec<&ParquetValue>> =
+        vec![Vec::with_capacity(num_rows); fields.len()];
+
+    for value in values {
+        match *value {
+            ParquetValue::Record(map) => {
+                null_buffer_builder.append(true);
+                for (idx, field) in fields.iter().enumerate() {
+                    let field_value = map.get(field.name().as_str()).unwrap_or(&null_value);
+                    field_columns[idx].push(field_value);
+                }
+            }
+            ParquetValue::Null => {
+                null_buffer_builder.append(false);
+                for field_column in field_columns.iter_mut().take(fields.len()) {
+                    field_column.push(&null_value);
+                }
+            }
+            _ => {
+                return Err(ParquetError::Conversion(format!(
+                    "Expected Record, got {:?}",
+                    value.type_name()
+                )))
+            }
+        }
+    }
+
+    // Build arrays for each field
+    for (column, field) in field_columns.iter().zip(fields.iter()) {
+        let array = parquet_value_refs_to_arrow_array(column, field)?;
+        field_arrays.push(array);
+    }
+
+    let null_buffer = null_buffer_builder.finish();
+    Ok(StdArc::new(StructArray::new(
+        fields.clone(),
+        field_arrays,
+        Some(null_buffer.into()),
+    )))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use arrow_array::*;
+    use parquet::basic::Type as PhysicalType;
+
+    #[test]
+    fn test_primitive_conversion_roundtrip() {
+        // Test boolean
+        let values = vec![
+            ParquetValue::Boolean(true),
+            ParquetValue::Boolean(false),
+            ParquetValue::Null,
+        ];
+        let field = Field::new("test", DataType::Boolean, true);
+        let array = parquet_values_to_arrow_array(&values, &field).unwrap();
+        let type_ = Type::primitive_type_builder("test", PhysicalType::BOOLEAN)
+            .build()
+            .unwrap();
+
+        for (i, expected) in values.iter().enumerate() {
+            let actual = arrow_to_parquet_value(&field, &type_, array.as_ref(), i).unwrap();
+            assert_eq!(&actual, expected);
+        }
+    }
+
+    #[test]
+    fn test_integer_upcasting() {
+        // Test that smaller integers can be upcast to larger ones
+        let values = vec![
+            ParquetValue::Int8(42),
+            ParquetValue::Int16(1000),
+            ParquetValue::Int32(100000),
+        ];
+        let field = Field::new("test", DataType::Int64, false);
+        let array = parquet_values_to_arrow_array(&values, &field).unwrap();
+
+        assert_eq!(array.len(), 3);
+        let int64_array = array.as_any().downcast_ref::<Int64Array>().unwrap();
+        assert_eq!(int64_array.value(0), 42);
+        assert_eq!(int64_array.value(1), 1000);
+        assert_eq!(int64_array.value(2), 100000);
+    }
+}