parquet 0.5.1 → 0.5.3

data/ext/parquet/src/types/mod.rs

@@ -32,6 +32,7 @@ use arrow_schema::{DataType, TimeUnit};
 use magnus::{value::ReprValue, Error as MagnusError, IntoValue, Ruby, Value};
 use parquet::data_type::Decimal;
 use parquet::record::Field;
+use std::array::TryFromSliceError;
 use std::{collections::HashMap, hash::BuildHasher, sync::Arc};
 
 use crate::header_cache::StringCacheKey;
@@ -58,6 +59,8 @@ pub enum ParquetGemError {
     Utf8Error(#[from] simdutf8::basic::Utf8Error),
     #[error("Jiff error: {0}")]
     Jiff(#[from] jiff::Error),
+    #[error("Failed to cast slice to array: {0}")]
+    InvalidDecimal(#[from] TryFromSliceError),
 }
 
 #[derive(Debug)]
@@ -83,11 +86,11 @@ impl From<MagnusError> for ParquetGemError {
     }
 }
 
-impl Into<MagnusError> for ParquetGemError {
-    fn into(self) -> MagnusError {
-        match self {
-            Self::Ruby(MagnusErrorWrapper(err)) => err.into(),
-            _ => MagnusError::new(magnus::exception::runtime_error(), self.to_string()),
+impl From<ParquetGemError> for MagnusError {
+    fn from(val: ParquetGemError) -> Self {
+        match val {
+            ParquetGemError::Ruby(MagnusErrorWrapper(err)) => err,
+            _ => MagnusError::new(magnus::exception::runtime_error(), val.to_string()),
         }
     }
 }

data/ext/parquet/src/types/parquet_value.rs

@@ -2,7 +2,7 @@ use crate::{impl_date_conversion, impl_timestamp_array_conversion, impl_timestam
 
 use super::*;
 use arrow_array::MapArray;
-use magnus::RArray;
+use magnus::{RArray, RString};
 
 #[derive(Debug, Clone)]
 pub enum ParquetValue {
@@ -22,6 +22,7 @@ pub enum ParquetValue {
     Bytes(Vec<u8>),
     Date32(i32),
     Date64(i64),
+    Decimal128(i128),
     TimestampSecond(i64, Option<Arc<str>>),
     TimestampMillis(i64, Option<Arc<str>>),
     TimestampMicros(i64, Option<Arc<str>>),
@@ -51,6 +52,7 @@ impl PartialEq for ParquetValue {
             (ParquetValue::Bytes(a), ParquetValue::Bytes(b)) => a == b,
             (ParquetValue::Date32(a), ParquetValue::Date32(b)) => a == b,
             (ParquetValue::Date64(a), ParquetValue::Date64(b)) => a == b,
+            (ParquetValue::Decimal128(a), ParquetValue::Decimal128(b)) => a == b,
             (ParquetValue::TimestampSecond(a, _), ParquetValue::TimestampSecond(b, _)) => a == b,
             (ParquetValue::TimestampMillis(a, _), ParquetValue::TimestampMillis(b, _)) => a == b,
             (ParquetValue::TimestampMicros(a, _), ParquetValue::TimestampMicros(b, _)) => a == b,
@@ -83,6 +85,7 @@ impl std::hash::Hash for ParquetValue {
             ParquetValue::Bytes(b) => b.hash(state),
             ParquetValue::Date32(d) => d.hash(state),
             ParquetValue::Date64(d) => d.hash(state),
+            ParquetValue::Decimal128(d) => d.hash(state),
             ParquetValue::TimestampSecond(ts, tz) => {
                 ts.hash(state);
                 tz.hash(state);
@@ -128,6 +131,7 @@ impl TryIntoValue for ParquetValue {
             ParquetValue::Boolean(b) => Ok(b.into_value_with(handle)),
             ParquetValue::String(s) => Ok(s.into_value_with(handle)),
             ParquetValue::Bytes(b) => Ok(handle.str_from_slice(&b).as_value()),
+            ParquetValue::Decimal128(d) => Ok(d.to_string().into_value_with(handle)),
             ParquetValue::Date32(d) => impl_date_conversion!(d, handle),
             ParquetValue::Date64(d) => impl_date_conversion!(d, handle),
             timestamp @ ParquetValue::TimestampSecond(_, _) => {
@@ -160,7 +164,12 @@ impl TryIntoValue for ParquetValue {
                 Ok(ary.into_value_with(handle))
             }
             ParquetValue::Map(m) => {
+                #[cfg(ruby_lt_3_2)]
                 let hash = handle.hash_new_capa(m.len());
+
+                #[cfg(not(ruby_lt_3_2))]
+                let hash = handle.hash_new();
+
                 m.into_iter().try_for_each(|(k, v)| {
                     hash.aset(
                         k.try_into_value_with(handle)?,
@@ -228,6 +237,18 @@ impl ParquetValue {
                     let v = NumericConverter::<f64>::convert_with_string_fallback(ruby, value)?;
                     Ok(ParquetValue::Float64(v))
                 }
+                PrimitiveType::Decimal128(_precision, scale) => {
+                    if value.is_kind_of(ruby.class_string()) {
+                        convert_to_decimal128(value, *scale)
+                    } else if let Ok(s) = value.funcall::<_, _, RString>("to_s", ()) {
+                        convert_to_decimal128(s.as_value(), *scale)
+                    } else {
+                        Err(MagnusError::new(
+                            magnus::exception::type_error(),
+                            "Expected a string for a decimal type",
+                        ))
+                    }
+                }
                 PrimitiveType::String => {
                     let v = convert_to_string(value)?;
                     Ok(ParquetValue::String(v))
@@ -351,6 +372,184 @@ impl ParquetValue {
         }
     }
 }
+/// Unified helper to parse a decimal string and apply scaling
+fn parse_decimal_string(input_str: &str, input_scale: i8) -> Result<i128, MagnusError> {
+    let s = input_str.trim();
+    
+    // 1. Handle scientific notation case (e.g., "0.12345e3")
+    if let Some(e_pos) = s.to_lowercase().find('e') {
+        let base = &s[0..e_pos];
+        let exp = &s[e_pos + 1..];
+
+        // Parse the exponent with detailed error message
+        let exp_val = exp.parse::<i32>().map_err(|e| {
+            MagnusError::new(
+                magnus::exception::type_error(),
+                format!("Failed to parse exponent '{}' in decimal string '{}': {}", exp, s, e),
+            )
+        })?;
+
+        // Limit exponent to reasonable range to prevent overflow
+        if exp_val.abs() > 38 {
+            return Err(MagnusError::new(
+                magnus::exception::range_error(),
+                format!("Exponent {} is out of range for decimal value '{}'. Must be between -38 and 38.", exp_val, s),
+            ));
+        }
+
+        // Handle the base part which might contain a decimal point
+        let (base_val, base_scale) = if let Some(decimal_pos) = base.find('.') {
+            let mut base_without_point = base.to_string();
+            base_without_point.remove(decimal_pos);
+
+            let base_scale = base.len() - decimal_pos - 1;
+
+            let base_val = base_without_point.parse::<i128>().map_err(|e| {
+                MagnusError::new(
+                    magnus::exception::type_error(),
+                    format!("Failed to parse base '{}' in scientific notation '{}': {}", base, s, e),
+                )
+            })?;
+
+            (base_val, base_scale as i32)
+        } else {
+            // No decimal point in base
+            let base_val = base.parse::<i128>().map_err(|e| {
+                MagnusError::new(
+                    magnus::exception::type_error(),
+                    format!("Failed to parse base '{}' in scientific notation '{}': {}", base, s, e),
+                )
+            })?;
+
+            (base_val, 0)
+        };
+
+        // Calculate the effective scale: base_scale - exp_val
+        let effective_scale = base_scale - exp_val;
+
+        // Adjust the value based on the difference between effective scale and requested scale
+        match effective_scale.cmp(&(input_scale as i32)) {
+            std::cmp::Ordering::Less => {
+                // Need to multiply to increase scale
+                let scale_diff = (input_scale as i32 - effective_scale) as u32;
+                if scale_diff > 38 {
+                    return Err(MagnusError::new(
+                        magnus::exception::range_error(),
+                        format!("Scale adjustment too large ({}) for decimal value '{}'. Consider using a smaller scale.", scale_diff, s),
+                    ));
+                }
+                Ok(base_val * 10_i128.pow(scale_diff))
+            }
+            std::cmp::Ordering::Greater => {
+                // Need to divide to decrease scale
+                let scale_diff = (effective_scale - input_scale as i32) as u32;
+                if scale_diff > 38 {
+                    return Err(MagnusError::new(
+                        magnus::exception::range_error(),
+                        format!("Scale adjustment too large ({}) for decimal value '{}'. Consider using a larger scale.", scale_diff, s),
+                    ));
+                }
+                Ok(base_val / 10_i128.pow(scale_diff))
+            }
+            std::cmp::Ordering::Equal => Ok(base_val),
+        }
+    }
+    // 2. Handle decimal point in the string (e.g., "123.456")
+    else if let Some(decimal_pos) = s.find('.') {
+        let mut s_without_point = s.to_string();
+        s_without_point.remove(decimal_pos);
+
+        // Calculate the actual scale from the decimal position
+        let actual_scale = s.len() - decimal_pos - 1;
+
+        // Parse the string without decimal point as i128
+        let v = s_without_point.parse::<i128>().map_err(|e| {
+            MagnusError::new(
+                magnus::exception::type_error(),
+                format!("Failed to parse decimal string '{}' (without decimal point: '{}'): {}", s, s_without_point, e),
+            )
+        })?;
+
+        // Scale the value if needed based on the difference between
+        // the actual scale and the requested scale
+        match actual_scale.cmp(&(input_scale as usize)) {
+            std::cmp::Ordering::Less => {
+                // Need to multiply to increase scale
+                let scale_diff = (input_scale - actual_scale as i8) as u32;
+                if scale_diff > 38 {
+                    return Err(MagnusError::new(
+                        magnus::exception::range_error(),
+                        format!("Scale adjustment too large ({}) for decimal value '{}'. Consider using a smaller scale.", scale_diff, s),
+                    ));
+                }
+                Ok(v * 10_i128.pow(scale_diff))
+            }
+            std::cmp::Ordering::Greater => {
+                // Need to divide to decrease scale
+                let scale_diff = (actual_scale as i8 - input_scale) as u32;
+                if scale_diff > 38 {
+                    return Err(MagnusError::new(
+                        magnus::exception::range_error(),
+                        format!("Scale adjustment too large ({}) for decimal value '{}'. Consider using a larger scale.", scale_diff, s),
+                    ));
+                }
+                Ok(v / 10_i128.pow(scale_diff))
+            }
+            std::cmp::Ordering::Equal => Ok(v),
+        }
+    } 
+    // 3. Plain integer value (e.g., "12345")
+    else {
+        // No decimal point, parse as i128 and scale appropriately
+        let v = s.parse::<i128>().map_err(|e| {
+            MagnusError::new(
+                magnus::exception::type_error(),
+                format!("Failed to parse integer string '{}' as decimal: {}", s, e),
+            )
+        })?;
+
+        // Apply scale - make sure it's reasonable
+        if input_scale > 38 {
+            return Err(MagnusError::new(
+                magnus::exception::range_error(),
+                format!("Scale {} is too large for decimal value '{}'. Must be ≤ 38.", input_scale, s),
+            ));
+        } else if input_scale < -38 {
+            return Err(MagnusError::new(
+                magnus::exception::range_error(),
+                format!("Scale {} is too small for decimal value '{}'. Must be ≥ -38.", input_scale, s),
+            ));
+        }
+
+        // Apply positive scale (multiply)
+        if input_scale >= 0 {
+            Ok(v * 10_i128.pow(input_scale as u32))
+        } else {
+            // Apply negative scale (divide)
+            Ok(v / 10_i128.pow((-input_scale) as u32))
+        }
+    }
+}
+
+fn convert_to_decimal128(value: Value, scale: i8) -> Result<ParquetValue, MagnusError> {
+    // Get the decimal string based on the type of value
+    let s = if unsafe { value.classname() } == "BigDecimal" {
+        value
+            .funcall::<_, _, RString>("to_s", ("F",))?
+            .to_string()?
+    } else {
+        value.to_r_string()?.to_string()?
+    };
+    
+    // Use our unified parser to convert the string to a decimal value with scaling
+    match parse_decimal_string(&s, scale) {
+        Ok(decimal_value) => Ok(ParquetValue::Decimal128(decimal_value)),
+        Err(e) => Err(MagnusError::new(
+            magnus::exception::type_error(),
+            format!("Failed to convert '{}' to decimal with scale {}: {}", s, scale, e),
+        ))
+    }
+}
 
 #[derive(Debug)]
 pub struct ParquetValueVec(Vec<ParquetValue>);
@@ -672,12 +871,10 @@ impl<'a> TryFrom<ArrayWrapper<'a>> for ParquetValueVec {
                 let x = downcast_array::<NullArray>(column.array);
                 Ok(ParquetValueVec(vec![ParquetValue::Null; x.len()]))
             }
-            _ => {
-                return Err(MagnusError::new(
-                    magnus::exception::type_error(),
-                    format!("Unsupported data type: {:?}", column.array.data_type()),
-                ))?;
-            }
+            _ => Err(MagnusError::new(
+                magnus::exception::type_error(),
+                format!("Unsupported data type: {:?}", column.array.data_type()),
+            ))?,
         }
     }
 }

data/ext/parquet/src/types/record_types.rs

@@ -1,7 +1,12 @@
+use std::sync::OnceLock;
+
 use itertools::Itertools;
+use parquet::data_type::AsBytes;
 
 use super::*;
 
+static LOADED_BIGDECIMAL: OnceLock<bool> = OnceLock::new();
+
 #[derive(Debug)]
 pub enum RowRecord<S: BuildHasher + Default> {
     Vec(Vec<ParquetField>),
@@ -29,8 +34,12 @@ impl<S: BuildHasher + Default> TryIntoValue for RowRecord<S> {
                 Ok(handle.into_value(ary))
             }
             RowRecord::Map(map) => {
+                #[cfg(ruby_lt_3_2)]
                 let hash = handle.hash_new_capa(map.len());
 
+                #[cfg(not(ruby_lt_3_2))]
+                let hash = handle.hash_new();
+
                 let mut values: [Value; 128] = [handle.qnil().as_value(); 128];
                 let mut i = 0;
 
@@ -78,8 +87,12 @@ impl<S: BuildHasher + Default> TryIntoValue for ColumnRecord<S> {
                 Ok(ary.into_value_with(handle))
             }
             ColumnRecord::Map(map) => {
+                #[cfg(ruby_lt_3_2)]
                 let hash = handle.hash_new_capa(map.len());
 
+                #[cfg(not(ruby_lt_3_2))]
+                let hash = handle.hash_new();
+
                 let mut values: [Value; 128] = [handle.qnil().as_value(); 128];
                 let mut i = 0;
 
@@ -137,8 +150,8 @@ impl TryIntoValue for ParquetField {
             Field::Str(s) => {
                 if self.1 {
                     Ok(simdutf8::basic::from_utf8(s.as_bytes())
-                        .map_err(|e| ParquetGemError::Utf8Error(e))
-                        .and_then(|s| Ok(s.into_value_with(handle)))?)
+                        .map_err(ParquetGemError::Utf8Error)
+                        .map(|s| s.into_value_with(handle))?)
                 } else {
                     let s = String::from_utf8_lossy(s.as_bytes());
                     Ok(s.into_value_with(handle))
@@ -175,9 +188,13 @@ impl TryIntoValue for ParquetField {
                 Ok(ary.into_value_with(handle))
             }
             Field::MapInternal(map) => {
-                let entries = map.entries();
-                let hash = handle.hash_new_capa(entries.len());
-                entries.iter().try_for_each(|(k, v)| {
+                #[cfg(ruby_lt_3_2)]
+                let hash = handle.hash_new_capa(map.len());
+
+                #[cfg(not(ruby_lt_3_2))]
+                let hash = handle.hash_new();
+
+                map.entries().iter().try_for_each(|(k, v)| {
                     hash.aset(
                         ParquetField(k.clone(), self.1).try_into_value_with(handle)?,
                         ParquetField(v.clone(), self.1).try_into_value_with(handle)?,
@@ -197,12 +214,18 @@ impl TryIntoValue for ParquetField {
                         format!("{}e-{}", unscaled, scale)
                     }
                     Decimal::Bytes { value, scale, .. } => {
-                        // Convert bytes to string representation of unscaled value
-                        let unscaled = String::from_utf8_lossy(value.data());
+                        // value is a byte array containing the bytes for an i128 value in big endian order
+                        let casted = value.as_bytes()[..16].try_into()?;
+                        let unscaled = i128::from_be_bytes(casted);
                         format!("{}e-{}", unscaled, scale)
                     }
                 };
-                Ok(handle.eval(&format!("BigDecimal(\"{value}\")"))?)
+
+                // Load the bigdecimal gem if it's not already loaded
+                LOADED_BIGDECIMAL.get_or_init(|| handle.require("bigdecimal").unwrap_or_default());
+
+                let kernel = handle.module_kernel();
+                Ok(kernel.funcall::<_, _, Value>("BigDecimal", (value,))?)
             }
             Field::Group(row) => {
                 let hash = handle.hash_new();

data/ext/parquet/src/types/schema_converter.rs

@@ -1,5 +1,5 @@
 use magnus::value::ReprValue; // Add ReprValue trait to scope
-use magnus::{Error as MagnusError, RArray, Ruby, TryConvert, Value};
+use magnus::{Error as MagnusError, IntoValue, RArray, Ruby, TryConvert, Value};
 
 use crate::types::{ParquetSchemaType as PST, PrimitiveType, SchemaField, SchemaNode};
 use crate::utils::parse_string_or_symbol;
@@ -22,7 +22,7 @@ fn convert_schema_field_to_node(field: &SchemaField) -> SchemaNode {
                     let item_field = SchemaField {
                         name: "item".to_string(),
                         type_: list_field.item_type.clone(),
-                        format: list_field.format.clone().map(String::from),
+                        format: list_field.format.map(String::from),
                         nullable: list_field.nullable,
                     };
                     convert_schema_field_to_node(&item_field)
@@ -33,7 +33,7 @@ fn convert_schema_field_to_node(field: &SchemaField) -> SchemaNode {
                     let item_field = SchemaField {
                         name: "item".to_string(),
                         type_: list_field.item_type.clone(),
-                        format: list_field.format.clone().map(String::from),
+                        format: list_field.format.map(String::from),
                         nullable: list_field.nullable,
                     };
                     convert_schema_field_to_node(&item_field)
@@ -50,13 +50,13 @@ fn convert_schema_field_to_node(field: &SchemaField) -> SchemaNode {
             let key_field = SchemaField {
                 name: "key".to_string(),
                 type_: map_field.key_type.clone(),
-                format: map_field.key_format.clone().map(String::from),
+                format: map_field.key_format.map(String::from),
                 nullable: false, // Map keys can never be null in Parquet
             };
             let value_field = SchemaField {
                 name: "value".to_string(),
                 type_: map_field.value_type.clone(),
-                format: map_field.value_format.clone().map(String::from),
+                format: map_field.value_format.map(String::from),
                 nullable: map_field.value_nullable,
             };
 
@@ -121,9 +121,7 @@ pub fn parse_legacy_schema(
                         ruby.exception_type_error(),
                         "Schema must be an array of field definitions or nil",
                     )
-                })?
-                .len()
-                == 0)
+                })?.is_empty())
     {
         // If schema is nil or an empty array, we'll handle this in the caller
         return Ok(Vec::new());
@@ -155,7 +153,7 @@ pub fn parse_legacy_schema(
             }
 
             let (name, type_value) = &entries[0];
-            let name_option = parse_string_or_symbol(ruby, name.clone())?;
+            let name_option = parse_string_or_symbol(ruby, *name)?;
             let name = name_option.ok_or_else(|| {
                 MagnusError::new(ruby.exception_runtime_error(), "Field name cannot be nil")
             })?;
@@ -166,6 +164,9 @@ pub fn parse_legacy_schema(
                 let mut format_str = None;
                 let mut nullable = true; // Default to true if not specified
 
+                let mut precision: Option<Value> = None;
+                let mut scale: Option<Value> = None;
+
                 for (key, value) in type_hash {
                     let key_option = parse_string_or_symbol(ruby, key)?;
                     let key = key_option.ok_or_else(|| {
@@ -181,6 +182,12 @@ pub fn parse_legacy_schema(
                             // Extract nullable if present - convert to boolean
                             nullable = bool::try_convert(value).unwrap_or(true);
                         }
+                        "precision" => {
+                            precision = Some(value);
+                        }
+                        "scale" => {
+                            scale = Some(value);
+                        }
                         _ => {
                             return Err(MagnusError::new(
                                 ruby.exception_type_error(),
@@ -197,9 +204,109 @@ pub fn parse_legacy_schema(
                     )
                 })?;
 
-                (PST::try_convert(type_str)?, format_str, nullable)
+                // Handle decimal type with precision and scale
+                let mut type_result = PST::try_convert(type_str)?;
+                
+                // If it's a decimal type and we have precision and scale, override the type
+                if let PST::Primitive(PrimitiveType::Decimal128(_, _)) = type_result {
+                    let precision_value = precision.unwrap_or_else(|| {
+                        let val: u8 = 18;
+                        val.into_value_with(ruby)
+                    });
+                    let scale_value = scale.unwrap_or_else(|| {
+                        let val: i8 = 2;
+                        val.into_value_with(ruby)
+                    });
+                    
+                    let precision_u8 = u8::try_convert(precision_value).map_err(|_| {
+                        MagnusError::new(
+                            ruby.exception_type_error(),
+                            "Invalid precision value for decimal type, expected a positive integer".to_string(),
+                        )
+                    })?;
+                    
+                    // Validate precision is in a valid range
+                    if precision_u8 < 1 {
+                        return Err(MagnusError::new(
+                            ruby.exception_arg_error(),
+                            format!(
+                                "Precision for decimal type must be at least 1, got {}", 
+                                precision_u8
+                            ),
+                        ));
+                    }
+                    
+                    if precision_u8 > 38 {
+                        return Err(MagnusError::new(
+                            ruby.exception_arg_error(),
+                            format!(
+                                "Precision for decimal type cannot exceed 38, got {}", 
+                                precision_u8
+                            ),
+                        ));
+                    }
+                    
+                    let scale_i8 = i8::try_convert(scale_value).map_err(|_| {
+                        MagnusError::new(
+                            ruby.exception_type_error(),
+                            "Invalid scale value for decimal type, expected an integer".to_string(),
+                        )
+                    })?;
+                    
+                    // Validate scale is in a valid range relative to precision
+                    if scale_i8 < 0 {
+                        return Err(MagnusError::new(
+                            ruby.exception_arg_error(),
+                            format!(
+                                "Scale for decimal type cannot be negative, got {}", 
+                                scale_i8
+                            ),
+                        ));
+                    }
+                    
+                    if scale_i8 as u8 > precision_u8 {
+                        return Err(MagnusError::new(
+                            ruby.exception_arg_error(),
+                            format!(
+                                "Scale ({}) cannot be larger than precision ({}) for decimal type", 
+                                scale_i8, precision_u8
+                            ),
+                        ));
+                    }
+                    
+                    type_result = PST::Primitive(PrimitiveType::Decimal128(precision_u8, scale_i8));
+                } else if let Some(type_name) = parse_string_or_symbol(ruby, type_str)? {
+                    if type_name == "decimal" {
+                        let precision_value = precision.unwrap_or_else(|| {
+                            let val: u8 = 18;
+                            val.into_value_with(ruby)
+                        });
+                        let scale_value = scale.unwrap_or_else(|| {
+                            let val: i8 = 2;
+                            val.into_value_with(ruby)
+                        });
+                        
+                        let precision_u8 = u8::try_convert(precision_value).map_err(|_| {
+                            MagnusError::new(
+                                ruby.exception_type_error(),
+                                "Invalid precision value for decimal type, expected a positive integer".to_string(),
+                            )
+                        })?;
+                        
+                        let scale_i8 = i8::try_convert(scale_value).map_err(|_| {
+                            MagnusError::new(
+                                ruby.exception_type_error(),
+                                "Invalid scale value for decimal type, expected an integer".to_string(),
+                            )
+                        })?;
+                        
+                        type_result = PST::Primitive(PrimitiveType::Decimal128(precision_u8, scale_i8));
+                    }
+                }
+
+                (type_result, format_str, nullable)
             } else {
-                (PST::try_convert(type_value.clone())?, None, true)
+                (PST::try_convert(*type_value)?, None, true)
             };
 
             schema.push(SchemaField {