parquet 0.5.9 → 0.5.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f1a1f7c250b960dbe334145a537e865889fbc759e7b8bfbafcbadc77689972cd
-  data.tar.gz: 22116ec0b9fe89f0ad08a3674267bd00b141170b889091b476f3aab0d6be88a8
+  metadata.gz: 82528b663c4a577262db90b6d17ba473a81d0ea725ceba486b63a3619040fa73
+  data.tar.gz: 2e44daa9b4e36ef1503589daaa0815cbc3acee10c565d9942f6c0b6d35ced5f0
 SHA512:
-  metadata.gz: ef8485d03247dd0d31993a774117669c1aaef5b875e7cb5c6f4c53e237a72fb81113ea35251426a21ea1ba24f8ae568bd2c3a158e6a45ce2416a308251d0f467
-  data.tar.gz: 672f38dfbf703dae996283fba8d137529e3089f569797df87feaac32fb0f956ea7c4d7ae57032715d1a21bd5bfa4dd728c2a3fe80174fb5fc0abdef51c73110a
+  metadata.gz: 418951253384f5492385fcb30fa5b0113b85d9bc51346b6abad16105c124d8869266943c1a29bc0879cfee4270b94d32fb99004e233c6ebde4a70e1d329435af
+  data.tar.gz: bc0db4ebb36add314253b5b9b946cc2c84f315d51ba7fefbead6c7de3b65a3f7752fa4e4cf0be19704405b390ae0106d8383e30791e7fac4a86a75141c214de1

data/Cargo.lock

@@ -126,6 +126,7 @@ dependencies = [
  "arrow-data",
  "arrow-schema",
  "flatbuffers",
+ "lz4_flex",
 ]
 
 [[package]]
@@ -842,6 +843,8 @@ version = "0.1.0"
 dependencies = [
  "ahash",
  "arrow-array",
+ "arrow-buffer",
+ "arrow-ipc",
  "arrow-schema",
  "bytes",
  "either",

data/ext/parquet/Cargo.toml

@@ -12,6 +12,8 @@ rb-sys-env = "^0.2"
 [dependencies]
 ahash = "0.8"
 arrow-array = { git = "https://github.com/njaremko/arrow-rs", branch = "nathan/fix-time" }
+arrow-buffer = { git = "https://github.com/njaremko/arrow-rs", branch = "nathan/fix-time" }
+arrow-ipc = { git = "https://github.com/njaremko/arrow-rs", branch = "nathan/fix-time", features = ["lz4"] }
 arrow-schema = { git = "https://github.com/njaremko/arrow-rs", branch = "nathan/fix-time" }
 bytes = "^1.9"
 either = "1.9"

data/ext/parquet/build.rs

@@ -1,5 +1,5 @@
 pub fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let _rb_env = rb_sys_env::activate()?;
+    rb_sys_env::activate()?;
 
     Ok(())
 }

data/ext/parquet/src/lib.rs

@@ -19,6 +19,9 @@ use writer::write_rows;
 /// Initializes the Ruby extension and defines methods.
 #[magnus::init]
 fn init(ruby: &Ruby) -> Result<(), Error> {
+    // Require 'time' for Time.parse method
+    ruby.require("time")?;
+    
     let module = ruby.define_module("Parquet")?;
     module.define_module_function("metadata", magnus::method!(reader::parse_metadata, -1))?;
     module.define_module_function("each_row", magnus::method!(parse_parquet_rows, -1))?;

data/ext/parquet/src/types/core_types.rs

@@ -108,6 +108,7 @@ pub enum PrimitiveType {
     Float32,
     Float64,
     Decimal128(u8, i8),
+    Decimal256(u8, i8),
     Boolean,
     String,
     Binary,

data/ext/parquet/src/types/mod.rs

@@ -23,10 +23,11 @@ pub use writer_types::*;
 // Common imports used across the module
 use arrow_array::cast::downcast_array;
 use arrow_array::{
-    Array, BinaryArray, BooleanArray, Date32Array, Date64Array, Float16Array, Float32Array,
-    Float64Array, Int16Array, Int32Array, Int64Array, Int8Array, ListArray, NullArray, StringArray,
-    StructArray, TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,
-    TimestampSecondArray, UInt16Array, UInt32Array, UInt64Array, UInt8Array,
+    Array, BinaryArray, BooleanArray, Date32Array, Date64Array, Decimal128Array, Decimal256Array,
+    Float16Array, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array, Int8Array,
+    ListArray, NullArray, StringArray, StructArray, TimestampMicrosecondArray,
+    TimestampMillisecondArray, TimestampNanosecondArray, TimestampSecondArray, UInt16Array,
+    UInt32Array, UInt64Array, UInt8Array,
 };
 use arrow_schema::{DataType, TimeUnit};
 use magnus::{value::ReprValue, Error as MagnusError, IntoValue, Ruby, Value};
@@ -62,6 +63,8 @@ pub enum ParquetGemError {
     InvalidDecimal(String),
     #[error("Failed to parse UUID: {0}")]
     UuidError(#[from] uuid::Error),
+    #[error("Decimals larger than 128 bits are not supported")]
+    DecimalWouldBeTruncated,
 }
 
 #[derive(Debug)]

data/ext/parquet/src/types/parquet_value.rs

@@ -1,7 +1,7 @@
 use crate::{impl_date_conversion, impl_timestamp_array_conversion, impl_timestamp_conversion};
 
+use super::record_types::{format_decimal_with_i8_scale, format_i256_decimal_with_scale};
 use super::*;
-use super::record_types::format_decimal_with_i8_scale;
 use arrow_array::MapArray;
 use magnus::{RArray, RString};
 
@@ -24,6 +24,7 @@ pub enum ParquetValue {
     Date32(i32),
     Date64(i64),
     Decimal128(i128, i8),
+    Decimal256(arrow_buffer::i256, i8),
     TimestampSecond(i64, Option<Arc<str>>),
     TimestampMillis(i64, Option<Arc<str>>),
     TimestampMicros(i64, Option<Arc<str>>),
@@ -94,6 +95,15 @@ impl PartialEq for ParquetValue {
                     a_val == b_val
                 }
             }
+            (ParquetValue::Decimal256(a, scale_a), ParquetValue::Decimal256(b, scale_b)) => {
+                if scale_a == scale_b {
+                    // Same scale, compare directly
+                    a == b
+                } else {
+                    // TODO: Implement decimal256 comparison
+                    todo!("decimal256 comparison");
+                }
+            }
             (ParquetValue::TimestampSecond(a, _), ParquetValue::TimestampSecond(b, _)) => a == b,
             (ParquetValue::TimestampMillis(a, _), ParquetValue::TimestampMillis(b, _)) => a == b,
             (ParquetValue::TimestampMicros(a, _), ParquetValue::TimestampMicros(b, _)) => a == b,
@@ -130,6 +140,10 @@ impl std::hash::Hash for ParquetValue {
                 d.hash(state);
                 scale.hash(state);
             }
+            ParquetValue::Decimal256(d, scale) => {
+                d.hash(state);
+                scale.hash(state);
+            }
             ParquetValue::TimestampSecond(ts, tz) => {
                 ts.hash(state);
                 tz.hash(state);
@@ -185,6 +199,17 @@ impl TryIntoValue for ParquetValue {
                 let kernel = handle.module_kernel();
                 Ok(kernel.funcall::<_, _, Value>("BigDecimal", (value,))?)
             }
+            ParquetValue::Decimal256(d, scale) => {
+                // Load the bigdecimal gem if it's not already loaded
+                LOADED_BIGDECIMAL.get_or_init(|| handle.require("bigdecimal").unwrap_or_default());
+
+                // Format with proper scaling based on the sign of scale
+                // Use specialized function to preserve full precision
+                let value = format_i256_decimal_with_scale(d, scale)?;
+
+                let kernel = handle.module_kernel();
+                Ok(kernel.funcall::<_, _, Value>("BigDecimal", (value,))?)
+            }
             ParquetValue::Date32(d) => impl_date_conversion!(d, handle),
             ParquetValue::Date64(d) => impl_date_conversion!(d, handle),
             timestamp @ ParquetValue::TimestampSecond(_, _) => {
@@ -292,9 +317,21 @@ impl ParquetValue {
                 }
                 PrimitiveType::Decimal128(_precision, scale) => {
                     if value.is_kind_of(ruby.class_string()) {
-                        convert_to_decimal128(value, *scale)
+                        convert_to_decimal(value, *scale)
                     } else if let Ok(s) = value.funcall::<_, _, RString>("to_s", ()) {
-                        convert_to_decimal128(s.as_value(), *scale)
+                        convert_to_decimal(s.as_value(), *scale)
+                    } else {
+                        Err(MagnusError::new(
+                            magnus::exception::type_error(),
+                            "Expected a string for a decimal type",
+                        ))
+                    }
+                }
+                PrimitiveType::Decimal256(_precision, scale) => {
+                    if value.is_kind_of(ruby.class_string()) {
+                        convert_to_decimal(value, *scale)
+                    } else if let Ok(s) = value.funcall::<_, _, RString>("to_s", ()) {
+                        convert_to_decimal(s.as_value(), *scale)
                     } else {
                         Err(MagnusError::new(
                             magnus::exception::type_error(),
@@ -425,8 +462,14 @@ impl ParquetValue {
         }
     }
 }
+
+enum ParsedDecimal {
+    Int128(i128),
+    Int256(arrow_buffer::i256),
+}
+
 /// Unified helper to parse a decimal string and apply scaling
-fn parse_decimal_string(input_str: &str, input_scale: i8) -> Result<i128, MagnusError> {
+fn parse_decimal_string(input_str: &str, input_scale: i8) -> Result<ParsedDecimal, MagnusError> {
     let s = input_str.trim();
 
     // 1. Handle scientific notation case (e.g., "0.12345e3")
@@ -445,12 +488,9 @@ fn parse_decimal_string(input_str: &str, input_scale: i8) -> Result<i128, Magnus
             )
         })?;
 
-        // Limit exponent to reasonable range to prevent overflow
+        // For very large exponents, we'll need to use BigInt
         if exp_val.abs() > 38 {
-            return Err(MagnusError::new(
-                magnus::exception::range_error(),
-                format!("Exponent {} is out of range for decimal value '{}'. Must be between -38 and 38.", exp_val, s),
-            ));
+            return parse_large_decimal_with_bigint(s, input_scale);
         }
 
         // Handle the base part which might contain a decimal point
@@ -460,30 +500,23 @@ fn parse_decimal_string(input_str: &str, input_scale: i8) -> Result<i128, Magnus
 
             let base_scale = base.len() - decimal_pos - 1;
 
-            let base_val = base_without_point.parse::<i128>().map_err(|e| {
-                MagnusError::new(
-                    magnus::exception::type_error(),
-                    format!(
-                        "Failed to parse base '{}' in scientific notation '{}': {}",
-                        base, s, e
-                    ),
-                )
-            })?;
-
-            (base_val, base_scale as i32)
+            // Try to parse as i128 first
+            match base_without_point.parse::<i128>() {
+                Ok(v) => (v, base_scale as i32),
+                Err(_) => {
+                    // Value too large for i128, use BigInt
+                    return parse_large_decimal_with_bigint(s, input_scale);
+                }
+            }
         } else {
             // No decimal point in base
-            let base_val = base.parse::<i128>().map_err(|e| {
-                MagnusError::new(
-                    magnus::exception::type_error(),
-                    format!(
-                        "Failed to parse base '{}' in scientific notation '{}': {}",
-                        base, s, e
-                    ),
-                )
-            })?;
-
-            (base_val, 0)
+            match base.parse::<i128>() {
+                Ok(v) => (v, 0),
+                Err(_) => {
+                    // Value too large for i128, use BigInt
+                    return parse_large_decimal_with_bigint(s, input_scale);
+                }
+            }
         };
 
         // Calculate the effective scale: base_scale - exp_val
@@ -495,12 +528,14 @@ fn parse_decimal_string(input_str: &str, input_scale: i8) -> Result<i128, Magnus
                 // Need to multiply to increase scale
                 let scale_diff = (input_scale as i32 - effective_scale) as u32;
                 if scale_diff > 38 {
-                    return Err(MagnusError::new(
-                        magnus::exception::range_error(),
-                        format!("Scale adjustment too large ({}) for decimal value '{}'. Consider using a smaller scale.", scale_diff, s),
-                    ));
+                    return parse_large_decimal_with_bigint(s, input_scale);
+                }
+
+                // Check for overflow
+                match base_val.checked_mul(10_i128.pow(scale_diff)) {
+                    Some(v) => Ok(ParsedDecimal::Int128(v)),
+                    None => parse_large_decimal_with_bigint(s, input_scale),
                 }
-                Ok(base_val * 10_i128.pow(scale_diff))
             }
             std::cmp::Ordering::Greater => {
                 // Need to divide to decrease scale
@@ -511,9 +546,9 @@ fn parse_decimal_string(input_str: &str, input_scale: i8) -> Result<i128, Magnus
                         format!("Scale adjustment too large ({}) for decimal value '{}'. Consider using a larger scale.", scale_diff, s),
                     ));
                 }
-                Ok(base_val / 10_i128.pow(scale_diff))
+                Ok(ParsedDecimal::Int128(base_val / 10_i128.pow(scale_diff)))
             }
-            std::cmp::Ordering::Equal => Ok(base_val),
+            std::cmp::Ordering::Equal => Ok(ParsedDecimal::Int128(base_val)),
         }
     }
     // 2. Handle decimal point in the string (e.g., "123.456")
@@ -524,16 +559,14 @@ fn parse_decimal_string(input_str: &str, input_scale: i8) -> Result<i128, Magnus
         // Calculate the actual scale from the decimal position
         let actual_scale = s.len() - decimal_pos - 1;
 
-        // Parse the string without decimal point as i128
-        let v = s_without_point.parse::<i128>().map_err(|e| {
-            MagnusError::new(
-                magnus::exception::type_error(),
-                format!(
-                    "Failed to parse decimal string '{}' (without decimal point: '{}'): {}",
-                    s, s_without_point, e
-                ),
-            )
-        })?;
+        // Try to parse as i128 first
+        let v = match s_without_point.parse::<i128>() {
+            Ok(v) => v,
+            Err(_) => {
+                // Value too large for i128, use BigInt
+                return parse_large_decimal_with_bigint(s, input_scale);
+            }
+        };
 
         // Scale the value if needed based on the difference between
         // the actual scale and the requested scale
@@ -542,12 +575,14 @@ fn parse_decimal_string(input_str: &str, input_scale: i8) -> Result<i128, Magnus
                 // Need to multiply to increase scale
                 let scale_diff = (input_scale - actual_scale as i8) as u32;
                 if scale_diff > 38 {
-                    return Err(MagnusError::new(
-                        magnus::exception::range_error(),
-                        format!("Scale adjustment too large ({}) for decimal value '{}'. Consider using a smaller scale.", scale_diff, s),
-                    ));
+                    return parse_large_decimal_with_bigint(s, input_scale);
+                }
+
+                // Check for overflow
+                match v.checked_mul(10_i128.pow(scale_diff)) {
+                    Some(v) => Ok(ParsedDecimal::Int128(v)),
+                    None => parse_large_decimal_with_bigint(s, input_scale),
                 }
-                Ok(v * 10_i128.pow(scale_diff))
             }
             std::cmp::Ordering::Greater => {
                 // Need to divide to decrease scale
@@ -558,30 +593,25 @@ fn parse_decimal_string(input_str: &str, input_scale: i8) -> Result<i128, Magnus
                         format!("Scale adjustment too large ({}) for decimal value '{}'. Consider using a larger scale.", scale_diff, s),
                     ));
                 }
-                Ok(v / 10_i128.pow(scale_diff))
+                Ok(ParsedDecimal::Int128(v / 10_i128.pow(scale_diff)))
             }
-            std::cmp::Ordering::Equal => Ok(v),
+            std::cmp::Ordering::Equal => Ok(ParsedDecimal::Int128(v)),
         }
     }
     // 3. Plain integer value (e.g., "12345")
     else {
-        // No decimal point, parse as i128 and scale appropriately
-        let v = s.parse::<i128>().map_err(|e| {
-            MagnusError::new(
-                magnus::exception::type_error(),
-                format!("Failed to parse integer string '{}' as decimal: {}", s, e),
-            )
-        })?;
+        // No decimal point, try to parse as i128 first
+        let v = match s.parse::<i128>() {
+            Ok(v) => v,
+            Err(_) => {
+                // Value too large for i128, use BigInt
+                return parse_large_decimal_with_bigint(s, input_scale);
+            }
+        };
 
         // Apply scale - make sure it's reasonable
         if input_scale > 38 {
-            return Err(MagnusError::new(
-                magnus::exception::range_error(),
-                format!(
-                    "Scale {} is too large for decimal value '{}'. Must be ≤ 38.",
-                    input_scale, s
-                ),
-            ));
+            return parse_large_decimal_with_bigint(s, input_scale);
         } else if input_scale < -38 {
             return Err(MagnusError::new(
                 magnus::exception::range_error(),
@@ -594,15 +624,153 @@ fn parse_decimal_string(input_str: &str, input_scale: i8) -> Result<i128, Magnus
 
         // Apply positive scale (multiply)
         if input_scale >= 0 {
-            Ok(v * 10_i128.pow(input_scale as u32))
+            match v.checked_mul(10_i128.pow(input_scale as u32)) {
+                Some(v) => Ok(ParsedDecimal::Int128(v)),
+                None => parse_large_decimal_with_bigint(s, input_scale),
+            }
         } else {
             // Apply negative scale (divide)
-            Ok(v / 10_i128.pow((-input_scale) as u32))
+            Ok(ParsedDecimal::Int128(
+                v / 10_i128.pow((-input_scale) as u32),
+            ))
+        }
+    }
+}
+
+/// Parse large decimal values using BigInt when they would overflow i128
+fn parse_large_decimal_with_bigint(s: &str, input_scale: i8) -> Result<ParsedDecimal, MagnusError> {
+    use num::BigInt;
+    use std::str::FromStr;
+
+    // Parse the input string as a BigInt
+    let bigint = if let Some(e_pos) = s.to_lowercase().find('e') {
+        // Handle scientific notation
+        let base = &s[0..e_pos];
+        let exp = &s[e_pos + 1..];
+
+        let exp_val = exp.parse::<i32>().map_err(|e| {
+            MagnusError::new(
+                magnus::exception::type_error(),
+                format!("Failed to parse exponent '{}': {}", exp, e),
+            )
+        })?;
+
+        // Parse base as BigInt
+        let base_bigint = if let Some(decimal_pos) = base.find('.') {
+            let mut base_without_point = base.to_string();
+            base_without_point.remove(decimal_pos);
+            let base_scale = base.len() - decimal_pos - 1;
+
+            let bigint = BigInt::from_str(&base_without_point).map_err(|e| {
+                MagnusError::new(
+                    magnus::exception::type_error(),
+                    format!("Failed to parse decimal base '{}': {}", base, e),
+                )
+            })?;
+
+            // Adjust for the decimal point
+            let effective_exp = exp_val - base_scale as i32;
+
+            if effective_exp > 0 {
+                bigint * BigInt::from(10).pow(effective_exp as u32)
+            } else if effective_exp < 0 {
+                bigint / BigInt::from(10).pow((-effective_exp) as u32)
+            } else {
+                bigint
+            }
+        } else {
+            let bigint = BigInt::from_str(base).map_err(|e| {
+                MagnusError::new(
+                    magnus::exception::type_error(),
+                    format!("Failed to parse decimal base '{}': {}", base, e),
+                )
+            })?;
+
+            if exp_val > 0 {
+                bigint * BigInt::from(10).pow(exp_val as u32)
+            } else if exp_val < 0 {
+                bigint / BigInt::from(10).pow((-exp_val) as u32)
+            } else {
+                bigint
+            }
+        };
+
+        base_bigint
+    } else if let Some(decimal_pos) = s.find('.') {
+        // Handle decimal point
+        let mut s_without_point = s.to_string();
+        s_without_point.remove(decimal_pos);
+
+        let actual_scale = s.len() - decimal_pos - 1;
+        let bigint = BigInt::from_str(&s_without_point).map_err(|e| {
+            MagnusError::new(
+                magnus::exception::type_error(),
+                format!("Failed to parse decimal string '{}': {}", s, e),
+            )
+        })?;
+
+        // Adjust for scale difference
+        let scale_diff = actual_scale as i8 - input_scale;
+
+        if scale_diff > 0 {
+            bigint / BigInt::from(10).pow(scale_diff as u32)
+        } else if scale_diff < 0 {
+            bigint * BigInt::from(10).pow((-scale_diff) as u32)
+        } else {
+            bigint
+        }
+    } else {
+        // Plain integer
+        let bigint = BigInt::from_str(s).map_err(|e| {
+            MagnusError::new(
+                magnus::exception::type_error(),
+                format!("Failed to parse integer string '{}': {}", s, e),
+            )
+        })?;
+
+        if input_scale > 0 {
+            bigint * BigInt::from(10).pow(input_scale as u32)
+        } else if input_scale < 0 {
+            bigint / BigInt::from(10).pow((-input_scale) as u32)
+        } else {
+            bigint
         }
+    };
+
+    // Convert BigInt to bytes and then to i256
+    let bytes = bigint.to_signed_bytes_le();
+
+    if bytes.len() <= 16 {
+        // Fits in i128
+        let mut buf = if bigint.sign() == num::bigint::Sign::Minus {
+            [0xff; 16]
+        } else {
+            [0; 16]
+        };
+        buf[..bytes.len()].copy_from_slice(&bytes);
+
+        Ok(ParsedDecimal::Int128(i128::from_le_bytes(buf)))
+    } else if bytes.len() <= 32 {
+        // Fits in i256
+        let mut buf = if bigint.sign() == num::bigint::Sign::Minus {
+            [0xff; 32]
+        } else {
+            [0; 32]
+        };
+        buf[..bytes.len()].copy_from_slice(&bytes);
+
+        Ok(ParsedDecimal::Int256(arrow_buffer::i256::from_le_bytes(
+            buf,
+        )))
+    } else {
+        Err(MagnusError::new(
+            magnus::exception::range_error(),
+            format!("Decimal value '{}' is too large to fit in 256 bits", s),
+        ))
     }
 }
 
-fn convert_to_decimal128(value: Value, scale: i8) -> Result<ParquetValue, MagnusError> {
+fn convert_to_decimal(value: Value, scale: i8) -> Result<ParquetValue, MagnusError> {
     // Get the decimal string based on the type of value
     let s = if unsafe { value.classname() } == "BigDecimal" {
         value
@@ -614,7 +782,10 @@ fn convert_to_decimal128(value: Value, scale: i8) -> Result<ParquetValue, Magnus
 
     // Use our unified parser to convert the string to a decimal value with scaling
     match parse_decimal_string(&s, scale) {
-        Ok(decimal_value) => Ok(ParquetValue::Decimal128(decimal_value, scale)),
+        Ok(decimal_value) => match decimal_value {
+            ParsedDecimal::Int128(v) => Ok(ParquetValue::Decimal128(v, scale)),
+            ParsedDecimal::Int256(v) => Ok(ParquetValue::Decimal256(v, scale)),
+        },
         Err(e) => Err(MagnusError::new(
             magnus::exception::type_error(),
             format!(
@@ -731,6 +902,52 @@ impl<'a> TryFrom<ArrayWrapper<'a>> for ParquetValueVec {
             }
             DataType::Date32 => impl_numeric_array_conversion!(column.array, Date32Array, Date32),
             DataType::Date64 => impl_numeric_array_conversion!(column.array, Date64Array, Date64),
+            DataType::Decimal128(_precision, scale) => {
+                let array = downcast_array::<Decimal128Array>(column.array);
+                Ok(ParquetValueVec(if array.is_nullable() {
+                    array
+                        .values()
+                        .iter()
+                        .enumerate()
+                        .map(|(i, x)| {
+                            if array.is_null(i) {
+                                ParquetValue::Null
+                            } else {
+                                ParquetValue::Decimal128(*x, *scale)
+                            }
+                        })
+                        .collect()
+                } else {
+                    array
+                        .values()
+                        .iter()
+                        .map(|x| ParquetValue::Decimal128(*x, *scale))
+                        .collect()
+                }))
+            }
+            DataType::Decimal256(_precision, scale) => {
+                let array = downcast_array::<Decimal256Array>(column.array);
+                Ok(ParquetValueVec(if array.is_nullable() {
+                    array
+                        .values()
+                        .iter()
+                        .enumerate()
+                        .map(|(i, x)| {
+                            if array.is_null(i) {
+                                ParquetValue::Null
+                            } else {
+                                ParquetValue::Decimal256(*x, *scale)
+                            }
+                        })
+                        .collect()
+                } else {
+                    array
+                        .values()
+                        .iter()
+                        .map(|x| ParquetValue::Decimal256(*x, *scale))
+                        .collect()
+                }))
+            }
             DataType::Timestamp(TimeUnit::Second, tz) => {
                 impl_timestamp_array_conversion!(
                     column.array,

data/ext/parquet/src/types/record_types.rs

@@ -22,6 +22,19 @@ pub fn format_decimal_with_i8_scale<T: std::fmt::Display>(value: T, scale: i8) -
     }
 }
 
+/// Format i256 decimal value with appropriate scale for BigDecimal conversion
+/// Uses bytes conversion to preserve full precision
+pub fn format_i256_decimal_with_scale(
+    value: arrow_buffer::i256,
+    scale: i8,
+) -> Result<String, ParquetGemError> {
+    // Convert i256 to big-endian bytes
+    let bytes = value.to_be_bytes();
+
+    // Use the existing bytes_to_decimal function which handles full precision
+    bytes_to_decimal(&bytes, scale as i32)
+}
+
 /// Format decimal value with appropriate scale for BigDecimal conversion
 /// Handles positive and negative scales correctly for i32 scale
 pub fn format_decimal_with_i32_scale<T: std::fmt::Display>(value: T, scale: i32) -> String {
@@ -35,7 +48,7 @@ pub fn format_decimal_with_i32_scale<T: std::fmt::Display>(value: T, scale: i32)
 }
 
 /// Convert arbitrary-length big-endian byte array to decimal string
-/// Supports byte arrays from 1 to 16 bytes in length
+/// Supports byte arrays from 1 to 32 bytes in length
 fn bytes_to_decimal(bytes: &[u8], scale: i32) -> Result<String, ParquetGemError> {
     match bytes.len() {
         0 => Err(ParquetGemError::InvalidDecimal(
@@ -50,34 +63,34 @@ fn bytes_to_decimal(bytes: &[u8], scale: i32) -> Result<String, ParquetGemError>
             // For 2 bytes, use i16
             let mut value: i16 = 0;
             let is_negative = bytes[0] & 0x80 != 0;
-            
+
             for &byte in bytes {
                 value = (value << 8) | (byte as i16);
             }
-            
+
             // Sign extend if negative
             if is_negative {
                 let shift = 16 - (bytes.len() * 8);
                 value = (value << shift) >> shift;
             }
-            
+
             Ok(format_decimal_with_i32_scale(value, scale))
         }
         3..=4 => {
             // For 3-4 bytes, use i32
             let mut value: i32 = 0;
             let is_negative = bytes[0] & 0x80 != 0;
-            
+
             for &byte in bytes {
                 value = (value << 8) | (byte as i32);
             }
-            
+
             // Sign extend if negative
             if is_negative {
                 let shift = 32 - (bytes.len() * 8);
                 value = (value << shift) >> shift;
             }
-            
+
             Ok(format_decimal_with_i32_scale(value, scale))
         }
         5..=8 => {
@@ -114,8 +127,79 @@ fn bytes_to_decimal(bytes: &[u8], scale: i32) -> Result<String, ParquetGemError>
 
             Ok(format_decimal_with_i32_scale(value, scale))
         }
+        17..=32 => {
+            // For 17-32 bytes, we need arbitrary precision handling
+            // Check if the number is negative (MSB of first byte)
+            let is_negative = bytes[0] & 0x80 != 0;
+
+            if is_negative {
+                // For negative numbers, we need to compute two's complement
+                // First, invert all bits
+                let mut inverted = Vec::with_capacity(bytes.len());
+                for &byte in bytes {
+                    inverted.push(!byte);
+                }
+
+                // Then add 1
+                let mut carry = 1u8;
+                for i in (0..inverted.len()).rev() {
+                    let (sum, new_carry) = inverted[i].overflowing_add(carry);
+                    inverted[i] = sum;
+                    carry = if new_carry { 1 } else { 0 };
+                }
+
+                // Convert to decimal string
+                let mut result = String::new();
+                let mut remainder = inverted;
+
+                // Repeatedly divide by 10 to get decimal digits
+                while !remainder.iter().all(|&b| b == 0) {
+                    let mut carry = 0u16;
+                    for i in 0..remainder.len() {
+                        let temp = (carry << 8) | (remainder[i] as u16);
+                        remainder[i] = (temp / 10) as u8;
+                        carry = temp % 10;
+                    }
+                    result.push_str(&carry.to_string());
+                }
+
+                // The digits are in reverse order
+                if result.is_empty() {
+                    result = "0".to_string();
+                } else {
+                    result = result.chars().rev().collect();
+                }
+
+                // Add negative sign and format with scale
+                Ok(format_decimal_with_i32_scale(format!("-{}", result), scale))
+            } else {
+                // For positive numbers, direct conversion
+                let mut result = String::new();
+                let mut remainder = bytes.to_vec();
+
+                // Repeatedly divide by 10 to get decimal digits
+                while !remainder.iter().all(|&b| b == 0) {
+                    let mut carry = 0u16;
+                    for i in 0..remainder.len() {
+                        let temp = (carry << 8) | (remainder[i] as u16);
+                        remainder[i] = (temp / 10) as u8;
+                        carry = temp % 10;
+                    }
+                    result.push_str(&carry.to_string());
+                }
+
+                // The digits are in reverse order
+                if result.is_empty() {
+                    result = "0".to_string();
+                } else {
+                    result = result.chars().rev().collect();
+                }
+
+                Ok(format_decimal_with_i32_scale(result, scale))
+            }
+        }
         _ => Err(ParquetGemError::InvalidDecimal(format!(
-            "Unsupported decimal byte array size: {}",
+            "Unsupported decimal byte array size: {} (maximum 32 bytes)",
             bytes.len()
         ))),
     }

data/ext/parquet/src/types/schema_node.rs

@@ -185,17 +185,18 @@ pub fn parse_schema_node(ruby: &Ruby, node_value: Value) -> Result<SchemaNode, M
             // 2. When precision only - use scale 0
             // 3. When scale only - use max precision (38)
             let (precision, scale) = match (precision_val, scale_val) {
-                (None, None) => (38, 0),  // Maximum accuracy, scale 0
+                (None, None) => (38, 0), // Maximum accuracy, scale 0
                 (Some(p), None) => {
                     // Precision provided, scale defaults to 0
                     let prec = u8::try_convert(p).map_err(|_| {
                         MagnusError::new(
                             ruby.exception_type_error(),
-                            "Invalid precision value for decimal type, expected a positive integer".to_string(),
+                            "Invalid precision value for decimal type, expected a positive integer"
+                                .to_string(),
                         )
                     })?;
                     (prec, 0)
-                },
+                }
                 (None, Some(s)) => {
                     // Scale provided, precision set to maximum (38)
                     let scl = i8::try_convert(s).map_err(|_| {
@@ -205,13 +206,14 @@ pub fn parse_schema_node(ruby: &Ruby, node_value: Value) -> Result<SchemaNode, M
                         )
                     })?;
                     (38, scl)
-                },
+                }
                 (Some(p), Some(s)) => {
                     // Both provided
                     let prec = u8::try_convert(p).map_err(|_| {
                         MagnusError::new(
                             ruby.exception_type_error(),
-                            "Invalid precision value for decimal type, expected a positive integer".to_string(),
+                            "Invalid precision value for decimal type, expected a positive integer"
+                                .to_string(),
                         )
                     })?;
                     let scl = i8::try_convert(s).map_err(|_| {
@@ -294,6 +296,7 @@ fn parse_primitive_type(s: &str) -> Option<PrimitiveType> {
         "timestamp_millis" | "timestamp_ms" => Some(PrimitiveType::TimestampMillis),
         "timestamp_micros" | "timestamp_us" => Some(PrimitiveType::TimestampMicros),
         "decimal" => Some(PrimitiveType::Decimal128(38, 0)), // Maximum precision, scale 0
+        "decimal256" => Some(PrimitiveType::Decimal256(38, 0)), // Maximum precision, scale 0
         _ => None,
     }
 }
@@ -321,6 +324,9 @@ pub fn schema_node_to_arrow_field(node: &SchemaNode) -> ArrowField {
                 PrimitiveType::Decimal128(precision, scale) => {
                     ArrowDataType::Decimal128(*precision, *scale)
                 }
+                PrimitiveType::Decimal256(precision, scale) => {
+                    ArrowDataType::Decimal256(*precision, *scale)
+                }
                 PrimitiveType::Boolean => ArrowDataType::Boolean,
                 PrimitiveType::String => ArrowDataType::Utf8,
                 PrimitiveType::Binary => ArrowDataType::Binary,

data/ext/parquet/src/types/type_conversion.rs

@@ -243,6 +243,7 @@ pub fn parquet_schema_type_to_arrow_data_type(
             PrimitiveType::Float32 => DataType::Float32,
             PrimitiveType::Float64 => DataType::Float64,
             PrimitiveType::Decimal128(precision, scale) => DataType::Decimal128(*precision, *scale),
+            PrimitiveType::Decimal256(precision, scale) => DataType::Decimal256(*precision, *scale),
             PrimitiveType::String => DataType::Utf8,
             PrimitiveType::Binary => DataType::Binary,
             PrimitiveType::Boolean => DataType::Boolean,
@@ -381,6 +382,22 @@ fn create_arrow_builder_for_type(
 
             Ok(Box::new(builder_with_precision))
         }
+        ParquetSchemaType::Primitive(PrimitiveType::Decimal256(precision, scale)) => {
+            // Create a Decimal128Builder since we're truncating Decimal256 to Decimal128
+            let builder = Decimal256Builder::with_capacity(cap);
+
+            // Set precision and scale for the decimal and return the new builder
+            let builder_with_precision = builder
+                .with_precision_and_scale(*precision, *scale)
+                .map_err(|e| {
+                    MagnusError::new(
+                        magnus::exception::runtime_error(),
+                        format!("Failed to set precision and scale: {}", e),
+                    )
+                })?;
+
+            Ok(Box::new(builder_with_precision))
+        }
         ParquetSchemaType::Primitive(PrimitiveType::String) => {
             Ok(Box::new(StringBuilder::with_capacity(cap, cap * 32)))
         }
@@ -891,6 +908,187 @@ fn fill_builder(
             }
             Ok(())
         }
+        ParquetSchemaType::Primitive(PrimitiveType::Decimal256(_precision, scale)) => {
+            let typed_builder = builder
+                .as_any_mut()
+                .downcast_mut::<Decimal256Builder>()
+                .expect("Builder mismatch: expected Decimal256Builder for Decimal256");
+
+            for val in values {
+                match val {
+                    ParquetValue::Decimal256(d, _scale) => typed_builder.append_value(*d),
+                    ParquetValue::Decimal128(d, _scale) => {
+                        // Convert i128 to i256
+                        typed_builder.append_value(arrow_buffer::i256::from_i128(*d))
+                    }
+                    ParquetValue::Float64(f) => {
+                        // Scale the float to the desired precision and scale
+                        // For large values, use BigInt to avoid overflow
+                        let scaled = *f * 10_f64.powi(*scale as i32);
+                        if scaled >= i128::MIN as f64 && scaled <= i128::MAX as f64 {
+                            let scaled_value = scaled as i128;
+                            typed_builder.append_value(arrow_buffer::i256::from_i128(scaled_value))
+                        } else {
+                            // Use BigInt for values that don't fit in i128
+                            use num::{BigInt, FromPrimitive};
+                            let bigint = BigInt::from_f64(scaled).ok_or_else(|| {
+                                MagnusError::new(
+                                    magnus::exception::type_error(),
+                                    format!("Failed to convert float {} to BigInt", f),
+                                )
+                            })?;
+                            let bytes = bigint.to_signed_bytes_le();
+                            if bytes.len() <= 32 {
+                                let mut buf = if bigint.sign() == num::bigint::Sign::Minus {
+                                    [0xff; 32]
+                                } else {
+                                    [0; 32]
+                                };
+                                buf[..bytes.len()].copy_from_slice(&bytes);
+                                typed_builder.append_value(arrow_buffer::i256::from_le_bytes(buf))
+                            } else {
+                                return Err(MagnusError::new(
+                                    magnus::exception::type_error(),
+                                    format!(
+                                        "Float value {} scaled to {} is too large for Decimal256",
+                                        f, scaled
+                                    ),
+                                ));
+                            }
+                        }
+                    }
+                    ParquetValue::Float32(flo) => {
+                        // Scale the float to the desired precision and scale
+                        let scaled = (*flo as f64) * 10_f64.powi(*scale as i32);
+                        if scaled >= i128::MIN as f64 && scaled <= i128::MAX as f64 {
+                            let scaled_value = scaled as i128;
+                            typed_builder.append_value(arrow_buffer::i256::from_i128(scaled_value))
+                        } else {
+                            // Use BigInt for values that don't fit in i128
+                            use num::{BigInt, FromPrimitive};
+                            let bigint = BigInt::from_f64(scaled).ok_or_else(|| {
+                                MagnusError::new(
+                                    magnus::exception::type_error(),
+                                    format!("Failed to convert float {} to BigInt", flo),
+                                )
+                            })?;
+                            let bytes = bigint.to_signed_bytes_le();
+                            if bytes.len() <= 32 {
+                                let mut buf = if bigint.sign() == num::bigint::Sign::Minus {
+                                    [0xff; 32]
+                                } else {
+                                    [0; 32]
+                                };
+                                buf[..bytes.len()].copy_from_slice(&bytes);
+                                typed_builder.append_value(arrow_buffer::i256::from_le_bytes(buf))
+                            } else {
+                                return Err(MagnusError::new(
+                                    magnus::exception::type_error(),
+                                    format!(
+                                        "Float value {} scaled is too large for Decimal256",
+                                        flo
+                                    ),
+                                ));
+                            }
+                        }
+                    }
+                    ParquetValue::Int64(i) => {
+                        // Scale the integer to the desired scale
+                        let base = arrow_buffer::i256::from_i128(*i as i128);
+                        if *scale <= 38 {
+                            // Can use i128 multiplication for scale <= 38
+                            let scale_factor =
+                                arrow_buffer::i256::from_i128(10_i128.pow(*scale as u32));
+                            match base.checked_mul(scale_factor) {
+                                Some(scaled) => typed_builder.append_value(scaled),
+                                None => {
+                                    return Err(MagnusError::new(
+                                        magnus::exception::type_error(),
+                                        format!(
+                                            "Integer {} scaled by {} overflows Decimal256",
+                                            i, scale
+                                        ),
+                                    ));
+                                }
+                            }
+                        } else {
+                            // For very large scales, use BigInt
+                            use num::BigInt;
+                            let bigint = BigInt::from(*i) * BigInt::from(10).pow(*scale as u32);
+                            let bytes = bigint.to_signed_bytes_le();
+                            if bytes.len() <= 32 {
+                                let mut buf = if bigint.sign() == num::bigint::Sign::Minus {
+                                    [0xff; 32]
+                                } else {
+                                    [0; 32]
+                                };
+                                buf[..bytes.len()].copy_from_slice(&bytes);
+                                typed_builder.append_value(arrow_buffer::i256::from_le_bytes(buf))
+                            } else {
+                                return Err(MagnusError::new(
+                                    magnus::exception::type_error(),
+                                    format!(
+                                        "Integer {} scaled by {} is too large for Decimal256",
+                                        i, scale
+                                    ),
+                                ));
+                            }
+                        }
+                    }
+                    ParquetValue::Int32(i) => {
+                        // Scale the integer to the desired scale
+                        let base = arrow_buffer::i256::from_i128(*i as i128);
+                        if *scale <= 38 {
+                            // Can use i128 multiplication for scale <= 38
+                            let scale_factor =
+                                arrow_buffer::i256::from_i128(10_i128.pow(*scale as u32));
+                            match base.checked_mul(scale_factor) {
+                                Some(scaled) => typed_builder.append_value(scaled),
+                                None => {
+                                    return Err(MagnusError::new(
+                                        magnus::exception::type_error(),
+                                        format!(
+                                            "Integer {} scaled by {} overflows Decimal256",
+                                            i, scale
+                                        ),
+                                    ));
+                                }
+                            }
+                        } else {
+                            // For very large scales, use BigInt
+                            use num::BigInt;
+                            let bigint = BigInt::from(*i) * BigInt::from(10).pow(*scale as u32);
+                            let bytes = bigint.to_signed_bytes_le();
+                            if bytes.len() <= 32 {
+                                let mut buf = if bigint.sign() == num::bigint::Sign::Minus {
+                                    [0xff; 32]
+                                } else {
+                                    [0; 32]
+                                };
+                                buf[..bytes.len()].copy_from_slice(&bytes);
+                                typed_builder.append_value(arrow_buffer::i256::from_le_bytes(buf))
+                            } else {
+                                return Err(MagnusError::new(
+                                    magnus::exception::type_error(),
+                                    format!(
+                                        "Integer {} scaled by {} is too large for Decimal256",
+                                        i, scale
+                                    ),
+                                ));
+                            }
+                        }
+                    }
+                    ParquetValue::Null => typed_builder.append_null(),
+                    other => {
+                        return Err(MagnusError::new(
+                            magnus::exception::type_error(),
+                            format!("Expected numeric value for Decimal256, got {:?}", other),
+                        ))
+                    }
+                }
+            }
+            Ok(())
+        }
         ParquetSchemaType::Primitive(PrimitiveType::Boolean) => {
             let typed_builder = builder
                 .as_any_mut()
@@ -1172,6 +1370,15 @@ fn fill_builder(
                                             )
                                         })?
                                         .append_value(*x),
+                                    ParquetValue::Decimal256(x, _scale) => typed_builder
+                                        .field_builder::<Decimal256Builder>(i)
+                                        .ok_or_else(|| {
+                                            MagnusError::new(
+                                                magnus::exception::type_error(),
+                                                "Failed to coerce into Decimal256Builder",
+                                            )
+                                        })?
+                                        .append_value(*x),
                                     ParquetValue::Date32(x) => typed_builder
                                         .field_builder::<Date32Builder>(i)
                                         .ok_or_else(|| {
@@ -1377,6 +1584,15 @@ fn fill_builder(
                                                 )
                                             })?
                                             .append_null(),
+                                        ParquetSchemaType::Primitive(PrimitiveType::Decimal256(_, _)) => typed_builder
+                                            .field_builder::<Decimal256Builder>(i)
+                                            .ok_or_else(|| {
+                                                MagnusError::new(
+                                                    magnus::exception::type_error(),
+                                                    "Failed to coerce into Decimal256Builder for Decimal256",
+                                                )
+                                            })?
+                                            .append_null(),
                                         ParquetSchemaType::Primitive(PrimitiveType::String) => typed_builder
                                             .field_builder::<StringBuilder>(i)
                                             .ok_or_else(|| {

data/ext/parquet/src/types/writer_types.rs

@@ -145,6 +145,53 @@ impl FromStr for ParquetSchemaType<'_> {
             }
         }
 
+        // Check if it's a decimal256 type with precision and scale
+        if let Some(decimal_params) = s.strip_prefix("decimal256(").and_then(|s| s.strip_suffix(")")) {
+            let parts: Vec<&str> = decimal_params.split(',').collect();
+
+            // Handle both single parameter (precision only) and two parameters (precision and scale)
+            if parts.len() == 1 {
+                // Only precision provided, scale defaults to 0
+                let precision = parts[0].trim().parse::<u8>().map_err(|_| {
+                    MagnusError::new(
+                        magnus::exception::runtime_error(),
+                        format!("Invalid precision value in decimal256 type: {}", parts[0]),
+                    )
+                })?;
+
+                return Ok(ParquetSchemaType::Primitive(PrimitiveType::Decimal256(
+                    precision, 0,
+                )));
+            } else if parts.len() == 2 {
+                // Both precision and scale provided
+                let precision = parts[0].trim().parse::<u8>().map_err(|_| {
+                    MagnusError::new(
+                        magnus::exception::runtime_error(),
+                        format!("Invalid precision value in decimal256 type: {}", parts[0]),
+                    )
+                })?;
+
+                let scale = parts[1].trim().parse::<i8>().map_err(|_| {
+                    MagnusError::new(
+                        magnus::exception::runtime_error(),
+                        format!("Invalid scale value in decimal256 type: {}", parts[1]),
+                    )
+                })?;
+
+                return Ok(ParquetSchemaType::Primitive(PrimitiveType::Decimal256(
+                    precision, scale,
+                )));
+            } else {
+                return Err(MagnusError::new(
+                    magnus::exception::runtime_error(),
+                    format!(
+                        "Invalid decimal256 format. Expected 'decimal256(precision)' or 'decimal256(precision,scale)', got '{}'",
+                        s
+                    ),
+                ));
+            }
+        }
+
         // Handle primitive types
         match s {
             "int8" => Ok(ParquetSchemaType::Primitive(PrimitiveType::Int8)),
@@ -166,6 +213,9 @@ impl FromStr for ParquetSchemaType<'_> {
             "decimal" => Ok(ParquetSchemaType::Primitive(PrimitiveType::Decimal128(
                 38, 0,
             ))),
+            "decimal256" => Ok(ParquetSchemaType::Primitive(PrimitiveType::Decimal256(
+                38, 0,
+            ))),
             "list" => Ok(ParquetSchemaType::List(Box::new(ListField {
                 item_type: ParquetSchemaType::Primitive(PrimitiveType::String),
                 format: None,

data/ext/parquet/src/writer/mod.rs

@@ -197,6 +197,9 @@ fn arrow_data_type_to_parquet_schema_type(dt: &DataType) -> Result<ParquetSchema
         DataType::Decimal128(precision, scale) => Ok(PST::Primitive(PrimitiveType::Decimal128(
             *precision, *scale,
         ))),
+        DataType::Decimal256(precision, scale) => Ok(PST::Primitive(PrimitiveType::Decimal256(
+            *precision, *scale,
+        ))),
         DataType::Date32 => Ok(PST::Primitive(PrimitiveType::Date32)),
         DataType::Date64 => {
             // Our code typically uses Date32 or Timestamp for 64. But Arrow has Date64

data/ext/parquet/src/writer/write_columns.rs

@@ -170,6 +170,9 @@ fn write_columns_impl(ruby: Rc<Ruby>, args: &[Value]) -> Result<(), ParquetGemEr
                                     PrimitiveType::TimestampMicros => {
                                         PST::Primitive(PrimitiveType::TimestampMicros)
                                     }
+                                    PrimitiveType::Decimal256(precision, scale) => {
+                                        PST::Primitive(PrimitiveType::Decimal256(precision, scale))
+                                    }
                                 },
                                 SchemaNode::List { .. }
                                 | SchemaNode::Map { .. }

data/ext/parquet/src/writer/write_rows.rs

@@ -258,6 +258,7 @@ pub fn estimate_value_size(
         | PST::Primitive(PrimitiveType::Float64) => Ok(8),
         PST::Primitive(PrimitiveType::Boolean) => Ok(1),
         PST::Primitive(PrimitiveType::Decimal128(_, _)) => Ok(16),
+        PST::Primitive(PrimitiveType::Decimal256(_, _)) => Ok(32),
         PST::Primitive(PrimitiveType::Date32)
         | PST::Primitive(PrimitiveType::TimestampMillis)
         | PST::Primitive(PrimitiveType::TimestampMicros) => Ok(8),

data/lib/parquet/version.rb

@@ -1,3 +1,3 @@
 module Parquet
-  VERSION = "0.5.9"
+  VERSION = "0.5.11"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parquet
 version: !ruby/object:Gem::Version
-  version: 0.5.9
+  version: 0.5.11
 platform: ruby
 authors:
 - Nathan Jaremko
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-06-03 00:00:00.000000000 Z
+date: 2025-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys