parquet 0.5.9 → 0.5.10

data/ext/parquet/src/types/schema_node.rs

@@ -185,17 +185,18 @@ pub fn parse_schema_node(ruby: &Ruby, node_value: Value) -> Result<SchemaNode, M
             // 2. When precision only - use scale 0
             // 3. When scale only - use max precision (38)
             let (precision, scale) = match (precision_val, scale_val) {
-                (None, None) => (38, 0),  // Maximum accuracy, scale 0
+                (None, None) => (38, 0), // Maximum accuracy, scale 0
                 (Some(p), None) => {
                     // Precision provided, scale defaults to 0
                     let prec = u8::try_convert(p).map_err(|_| {
                         MagnusError::new(
                             ruby.exception_type_error(),
-                            "Invalid precision value for decimal type, expected a positive integer".to_string(),
+                            "Invalid precision value for decimal type, expected a positive integer"
+                                .to_string(),
                         )
                     })?;
                     (prec, 0)
-                },
+                }
                 (None, Some(s)) => {
                     // Scale provided, precision set to maximum (38)
                     let scl = i8::try_convert(s).map_err(|_| {
@@ -205,13 +206,14 @@ pub fn parse_schema_node(ruby: &Ruby, node_value: Value) -> Result<SchemaNode, M
                         )
                     })?;
                     (38, scl)
-                },
+                }
                 (Some(p), Some(s)) => {
                     // Both provided
                     let prec = u8::try_convert(p).map_err(|_| {
                         MagnusError::new(
                             ruby.exception_type_error(),
-                            "Invalid precision value for decimal type, expected a positive integer".to_string(),
+                            "Invalid precision value for decimal type, expected a positive integer"
+                                .to_string(),
                         )
                     })?;
                     let scl = i8::try_convert(s).map_err(|_| {
@@ -294,6 +296,7 @@ fn parse_primitive_type(s: &str) -> Option<PrimitiveType> {
         "timestamp_millis" | "timestamp_ms" => Some(PrimitiveType::TimestampMillis),
         "timestamp_micros" | "timestamp_us" => Some(PrimitiveType::TimestampMicros),
         "decimal" => Some(PrimitiveType::Decimal128(38, 0)), // Maximum precision, scale 0
+        "decimal256" => Some(PrimitiveType::Decimal256(38, 0)), // Maximum precision, scale 0
         _ => None,
     }
 }
@@ -321,6 +324,9 @@ pub fn schema_node_to_arrow_field(node: &SchemaNode) -> ArrowField {
                 PrimitiveType::Decimal128(precision, scale) => {
                     ArrowDataType::Decimal128(*precision, *scale)
                 }
+                PrimitiveType::Decimal256(precision, scale) => {
+                    ArrowDataType::Decimal256(*precision, *scale)
+                }
                 PrimitiveType::Boolean => ArrowDataType::Boolean,
                 PrimitiveType::String => ArrowDataType::Utf8,
                 PrimitiveType::Binary => ArrowDataType::Binary,

data/ext/parquet/src/types/type_conversion.rs

@@ -243,6 +243,7 @@ pub fn parquet_schema_type_to_arrow_data_type(
             PrimitiveType::Float32 => DataType::Float32,
             PrimitiveType::Float64 => DataType::Float64,
             PrimitiveType::Decimal128(precision, scale) => DataType::Decimal128(*precision, *scale),
+            PrimitiveType::Decimal256(precision, scale) => DataType::Decimal256(*precision, *scale),
             PrimitiveType::String => DataType::Utf8,
             PrimitiveType::Binary => DataType::Binary,
             PrimitiveType::Boolean => DataType::Boolean,
@@ -381,6 +382,22 @@ fn create_arrow_builder_for_type(
 
             Ok(Box::new(builder_with_precision))
         }
+        ParquetSchemaType::Primitive(PrimitiveType::Decimal256(precision, scale)) => {
+            // Create a Decimal128Builder since we're truncating Decimal256 to Decimal128
+            let builder = Decimal256Builder::with_capacity(cap);
+
+            // Set precision and scale for the decimal and return the new builder
+            let builder_with_precision = builder
+                .with_precision_and_scale(*precision, *scale)
+                .map_err(|e| {
+                    MagnusError::new(
+                        magnus::exception::runtime_error(),
+                        format!("Failed to set precision and scale: {}", e),
+                    )
+                })?;
+
+            Ok(Box::new(builder_with_precision))
+        }
         ParquetSchemaType::Primitive(PrimitiveType::String) => {
             Ok(Box::new(StringBuilder::with_capacity(cap, cap * 32)))
         }
@@ -891,6 +908,187 @@ fn fill_builder(
             }
             Ok(())
         }
+        ParquetSchemaType::Primitive(PrimitiveType::Decimal256(_precision, scale)) => {
+            let typed_builder = builder
+                .as_any_mut()
+                .downcast_mut::<Decimal256Builder>()
+                .expect("Builder mismatch: expected Decimal256Builder for Decimal256");
+
+            for val in values {
+                match val {
+                    ParquetValue::Decimal256(d, _scale) => typed_builder.append_value(*d),
+                    ParquetValue::Decimal128(d, _scale) => {
+                        // Convert i128 to i256
+                        typed_builder.append_value(arrow_buffer::i256::from_i128(*d))
+                    }
+                    ParquetValue::Float64(f) => {
+                        // Scale the float to the desired precision and scale
+                        // For large values, use BigInt to avoid overflow
+                        let scaled = *f * 10_f64.powi(*scale as i32);
+                        if scaled >= i128::MIN as f64 && scaled <= i128::MAX as f64 {
+                            let scaled_value = scaled as i128;
+                            typed_builder.append_value(arrow_buffer::i256::from_i128(scaled_value))
+                        } else {
+                            // Use BigInt for values that don't fit in i128
+                            use num::{BigInt, FromPrimitive};
+                            let bigint = BigInt::from_f64(scaled).ok_or_else(|| {
+                                MagnusError::new(
+                                    magnus::exception::type_error(),
+                                    format!("Failed to convert float {} to BigInt", f),
+                                )
+                            })?;
+                            let bytes = bigint.to_signed_bytes_le();
+                            if bytes.len() <= 32 {
+                                let mut buf = if bigint.sign() == num::bigint::Sign::Minus {
+                                    [0xff; 32]
+                                } else {
+                                    [0; 32]
+                                };
+                                buf[..bytes.len()].copy_from_slice(&bytes);
+                                typed_builder.append_value(arrow_buffer::i256::from_le_bytes(buf))
+                            } else {
+                                return Err(MagnusError::new(
+                                    magnus::exception::type_error(),
+                                    format!(
+                                        "Float value {} scaled to {} is too large for Decimal256",
+                                        f, scaled
+                                    ),
+                                ));
+                            }
+                        }
+                    }
+                    ParquetValue::Float32(flo) => {
+                        // Scale the float to the desired precision and scale
+                        let scaled = (*flo as f64) * 10_f64.powi(*scale as i32);
+                        if scaled >= i128::MIN as f64 && scaled <= i128::MAX as f64 {
+                            let scaled_value = scaled as i128;
+                            typed_builder.append_value(arrow_buffer::i256::from_i128(scaled_value))
+                        } else {
+                            // Use BigInt for values that don't fit in i128
+                            use num::{BigInt, FromPrimitive};
+                            let bigint = BigInt::from_f64(scaled).ok_or_else(|| {
+                                MagnusError::new(
+                                    magnus::exception::type_error(),
+                                    format!("Failed to convert float {} to BigInt", flo),
+                                )
+                            })?;
+                            let bytes = bigint.to_signed_bytes_le();
+                            if bytes.len() <= 32 {
+                                let mut buf = if bigint.sign() == num::bigint::Sign::Minus {
+                                    [0xff; 32]
+                                } else {
+                                    [0; 32]
+                                };
+                                buf[..bytes.len()].copy_from_slice(&bytes);
+                                typed_builder.append_value(arrow_buffer::i256::from_le_bytes(buf))
+                            } else {
+                                return Err(MagnusError::new(
+                                    magnus::exception::type_error(),
+                                    format!(
+                                        "Float value {} scaled is too large for Decimal256",
+                                        flo
+                                    ),
+                                ));
+                            }
+                        }
+                    }
+                    ParquetValue::Int64(i) => {
+                        // Scale the integer to the desired scale
+                        let base = arrow_buffer::i256::from_i128(*i as i128);
+                        if *scale <= 38 {
+                            // Can use i128 multiplication for scale <= 38
+                            let scale_factor =
+                                arrow_buffer::i256::from_i128(10_i128.pow(*scale as u32));
+                            match base.checked_mul(scale_factor) {
+                                Some(scaled) => typed_builder.append_value(scaled),
+                                None => {
+                                    return Err(MagnusError::new(
+                                        magnus::exception::type_error(),
+                                        format!(
+                                            "Integer {} scaled by {} overflows Decimal256",
+                                            i, scale
+                                        ),
+                                    ));
+                                }
+                            }
+                        } else {
+                            // For very large scales, use BigInt
+                            use num::BigInt;
+                            let bigint = BigInt::from(*i) * BigInt::from(10).pow(*scale as u32);
+                            let bytes = bigint.to_signed_bytes_le();
+                            if bytes.len() <= 32 {
+                                let mut buf = if bigint.sign() == num::bigint::Sign::Minus {
+                                    [0xff; 32]
+                                } else {
+                                    [0; 32]
+                                };
+                                buf[..bytes.len()].copy_from_slice(&bytes);
+                                typed_builder.append_value(arrow_buffer::i256::from_le_bytes(buf))
+                            } else {
+                                return Err(MagnusError::new(
+                                    magnus::exception::type_error(),
+                                    format!(
+                                        "Integer {} scaled by {} is too large for Decimal256",
+                                        i, scale
+                                    ),
+                                ));
+                            }
+                        }
+                    }
+                    ParquetValue::Int32(i) => {
+                        // Scale the integer to the desired scale
+                        let base = arrow_buffer::i256::from_i128(*i as i128);
+                        if *scale <= 38 {
+                            // Can use i128 multiplication for scale <= 38
+                            let scale_factor =
+                                arrow_buffer::i256::from_i128(10_i128.pow(*scale as u32));
+                            match base.checked_mul(scale_factor) {
+                                Some(scaled) => typed_builder.append_value(scaled),
+                                None => {
+                                    return Err(MagnusError::new(
+                                        magnus::exception::type_error(),
+                                        format!(
+                                            "Integer {} scaled by {} overflows Decimal256",
+                                            i, scale
+                                        ),
+                                    ));
+                                }
+                            }
+                        } else {
+                            // For very large scales, use BigInt
+                            use num::BigInt;
+                            let bigint = BigInt::from(*i) * BigInt::from(10).pow(*scale as u32);
+                            let bytes = bigint.to_signed_bytes_le();
+                            if bytes.len() <= 32 {
+                                let mut buf = if bigint.sign() == num::bigint::Sign::Minus {
+                                    [0xff; 32]
+                                } else {
+                                    [0; 32]
+                                };
+                                buf[..bytes.len()].copy_from_slice(&bytes);
+                                typed_builder.append_value(arrow_buffer::i256::from_le_bytes(buf))
+                            } else {
+                                return Err(MagnusError::new(
+                                    magnus::exception::type_error(),
+                                    format!(
+                                        "Integer {} scaled by {} is too large for Decimal256",
+                                        i, scale
+                                    ),
+                                ));
+                            }
+                        }
+                    }
+                    ParquetValue::Null => typed_builder.append_null(),
+                    other => {
+                        return Err(MagnusError::new(
+                            magnus::exception::type_error(),
+                            format!("Expected numeric value for Decimal256, got {:?}", other),
+                        ))
+                    }
+                }
+            }
+            Ok(())
+        }
         ParquetSchemaType::Primitive(PrimitiveType::Boolean) => {
             let typed_builder = builder
                 .as_any_mut()
@@ -1172,6 +1370,15 @@ fn fill_builder(
                                             )
                                         })?
                                         .append_value(*x),
+                                    ParquetValue::Decimal256(x, _scale) => typed_builder
+                                        .field_builder::<Decimal256Builder>(i)
+                                        .ok_or_else(|| {
+                                            MagnusError::new(
+                                                magnus::exception::type_error(),
+                                                "Failed to coerce into Decimal256Builder",
+                                            )
+                                        })?
+                                        .append_value(*x),
                                     ParquetValue::Date32(x) => typed_builder
                                         .field_builder::<Date32Builder>(i)
                                         .ok_or_else(|| {
@@ -1377,6 +1584,15 @@ fn fill_builder(
                                                 )
                                             })?
                                             .append_null(),
+                                        ParquetSchemaType::Primitive(PrimitiveType::Decimal256(_, _)) => typed_builder
+                                            .field_builder::<Decimal256Builder>(i)
+                                            .ok_or_else(|| {
+                                                MagnusError::new(
+                                                    magnus::exception::type_error(),
+                                                    "Failed to coerce into Decimal256Builder for Decimal256",
+                                                )
+                                            })?
+                                            .append_null(),
                                         ParquetSchemaType::Primitive(PrimitiveType::String) => typed_builder
                                             .field_builder::<StringBuilder>(i)
                                             .ok_or_else(|| {

data/ext/parquet/src/types/writer_types.rs

@@ -145,6 +145,53 @@ impl FromStr for ParquetSchemaType<'_> {
             }
         }
 
+        // Check if it's a decimal256 type with precision and scale
+        if let Some(decimal_params) = s.strip_prefix("decimal256(").and_then(|s| s.strip_suffix(")")) {
+            let parts: Vec<&str> = decimal_params.split(',').collect();
+
+            // Handle both single parameter (precision only) and two parameters (precision and scale)
+            if parts.len() == 1 {
+                // Only precision provided, scale defaults to 0
+                let precision = parts[0].trim().parse::<u8>().map_err(|_| {
+                    MagnusError::new(
+                        magnus::exception::runtime_error(),
+                        format!("Invalid precision value in decimal256 type: {}", parts[0]),
+                    )
+                })?;
+
+                return Ok(ParquetSchemaType::Primitive(PrimitiveType::Decimal256(
+                    precision, 0,
+                )));
+            } else if parts.len() == 2 {
+                // Both precision and scale provided
+                let precision = parts[0].trim().parse::<u8>().map_err(|_| {
+                    MagnusError::new(
+                        magnus::exception::runtime_error(),
+                        format!("Invalid precision value in decimal256 type: {}", parts[0]),
+                    )
+                })?;
+
+                let scale = parts[1].trim().parse::<i8>().map_err(|_| {
+                    MagnusError::new(
+                        magnus::exception::runtime_error(),
+                        format!("Invalid scale value in decimal256 type: {}", parts[1]),
+                    )
+                })?;
+
+                return Ok(ParquetSchemaType::Primitive(PrimitiveType::Decimal256(
+                    precision, scale,
+                )));
+            } else {
+                return Err(MagnusError::new(
+                    magnus::exception::runtime_error(),
+                    format!(
+                        "Invalid decimal256 format. Expected 'decimal256(precision)' or 'decimal256(precision,scale)', got '{}'",
+                        s
+                    ),
+                ));
+            }
+        }
+
         // Handle primitive types
         match s {
             "int8" => Ok(ParquetSchemaType::Primitive(PrimitiveType::Int8)),
@@ -166,6 +213,9 @@ impl FromStr for ParquetSchemaType<'_> {
             "decimal" => Ok(ParquetSchemaType::Primitive(PrimitiveType::Decimal128(
                 38, 0,
             ))),
+            "decimal256" => Ok(ParquetSchemaType::Primitive(PrimitiveType::Decimal256(
+                38, 0,
+            ))),
             "list" => Ok(ParquetSchemaType::List(Box::new(ListField {
                 item_type: ParquetSchemaType::Primitive(PrimitiveType::String),
                 format: None,

data/ext/parquet/src/writer/mod.rs

@@ -197,6 +197,9 @@ fn arrow_data_type_to_parquet_schema_type(dt: &DataType) -> Result<ParquetSchema
         DataType::Decimal128(precision, scale) => Ok(PST::Primitive(PrimitiveType::Decimal128(
             *precision, *scale,
         ))),
+        DataType::Decimal256(precision, scale) => Ok(PST::Primitive(PrimitiveType::Decimal256(
+            *precision, *scale,
+        ))),
         DataType::Date32 => Ok(PST::Primitive(PrimitiveType::Date32)),
         DataType::Date64 => {
             // Our code typically uses Date32 or Timestamp for 64. But Arrow has Date64

data/ext/parquet/src/writer/write_columns.rs

@@ -170,6 +170,9 @@ fn write_columns_impl(ruby: Rc<Ruby>, args: &[Value]) -> Result<(), ParquetGemEr
                                     PrimitiveType::TimestampMicros => {
                                         PST::Primitive(PrimitiveType::TimestampMicros)
                                     }
+                                    PrimitiveType::Decimal256(precision, scale) => {
+                                        PST::Primitive(PrimitiveType::Decimal256(precision, scale))
+                                    }
                                 },
                                 SchemaNode::List { .. }
                                 | SchemaNode::Map { .. }

data/ext/parquet/src/writer/write_rows.rs

@@ -258,6 +258,7 @@ pub fn estimate_value_size(
         | PST::Primitive(PrimitiveType::Float64) => Ok(8),
         PST::Primitive(PrimitiveType::Boolean) => Ok(1),
         PST::Primitive(PrimitiveType::Decimal128(_, _)) => Ok(16),
+        PST::Primitive(PrimitiveType::Decimal256(_, _)) => Ok(32),
         PST::Primitive(PrimitiveType::Date32)
         | PST::Primitive(PrimitiveType::TimestampMillis)
         | PST::Primitive(PrimitiveType::TimestampMicros) => Ok(8),

data/lib/parquet/version.rb

@@ -1,3 +1,3 @@
 module Parquet
-  VERSION = "0.5.9"
+  VERSION = "0.5.10"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parquet
 version: !ruby/object:Gem::Version
-  version: 0.5.9
+  version: 0.5.10
 platform: ruby
 authors:
 - Nathan Jaremko
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-06-03 00:00:00.000000000 Z
+date: 2025-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -62,7 +62,9 @@ files:
 - ext/parquet/src/header_cache.rs
 - ext/parquet/src/lib.rs
 - ext/parquet/src/logger.rs
+- ext/parquet/src/reader/arrow_reader.rs
 - ext/parquet/src/reader/common.rs
+- ext/parquet/src/reader/format_detector.rs
 - ext/parquet/src/reader/mod.rs
 - ext/parquet/src/reader/parquet_column_reader.rs
 - ext/parquet/src/reader/parquet_row_reader.rs