parquet 0.5.5 → 0.5.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e2295ee94fe35758ae8e5137070e2206ec1e104aad6b9a0806aa508ad4799247
-  data.tar.gz: 340f86257082bdba22d6ced530ecd1d201c7b4e6d9116eebac41541ba2aaa257
+  metadata.gz: e8a79e74af0419282904a0041c09509520f64ce1e504e133237f4b87697dce14
+  data.tar.gz: 63391ffff73907caccc142f37550e85c12826f302f00ac726f826af391f8d8cd
 SHA512:
-  metadata.gz: f333ae2914cdd00468c390e8b3d876aec4e522a546d43ab29db5d777792105a38d2a40c49db0f0afe1e800bf32e54bb4c479441f8f9876937ba59917b444d15a
-  data.tar.gz: da2832c3514729cc0e99e16f70a10bbfc4e9093dc734de55715305121649ebc371dff93a7bb462b97fde27c79ad65cec12c5fa90a47f70bc64153a7fd2ce1a5c
+  metadata.gz: cddb7c6711e7e49ea785f6c0ab5ae3c40181756ad0e3fc23f298c291b725b178fdfbe5a8430fd9be10591b09e1b963255cb50637743054fe2173c9798e1e8bcc
+  data.tar.gz: 927a112ff1994800b3ed989f5000ed2a43438cebff886a545d0dd22018731b042b9052ead5b14983faf50fffc593cdd7512dd764bfed4de8ffa7781e6f2fda1a

data/Cargo.lock

@@ -63,9 +63,8 @@ dependencies = [
 
 [[package]]
 name = "arrow-array"
-version = "54.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57a4a6d2896083cfbdf84a71a863b22460d0708f8206a8373c52e326cc72ea1a"
+version = "55.1.0"
+source = "git+https://github.com/apache/arrow-rs?branch=main#e9df239980baa6d0f7eb4384eb01078bdd9b1701"
 dependencies = [
  "ahash",
  "arrow-buffer",
@@ -79,9 +78,8 @@ dependencies = [
 
 [[package]]
 name = "arrow-buffer"
-version = "54.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cef870583ce5e4f3b123c181706f2002fb134960f9a911900f64ba4830c7a43a"
+version = "55.1.0"
+source = "git+https://github.com/apache/arrow-rs?branch=main#e9df239980baa6d0f7eb4384eb01078bdd9b1701"
 dependencies = [
  "bytes",
  "half",
@@ -90,9 +88,8 @@ dependencies = [
 
 [[package]]
 name = "arrow-cast"
-version = "54.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ac7eba5a987f8b4a7d9629206ba48e19a1991762795bbe5d08497b7736017ee"
+version = "55.1.0"
+source = "git+https://github.com/apache/arrow-rs?branch=main#e9df239980baa6d0f7eb4384eb01078bdd9b1701"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -110,9 +107,8 @@ dependencies = [
 
 [[package]]
 name = "arrow-data"
-version = "54.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b095e8a4f3c309544935d53e04c3bfe4eea4e71c3de6fe0416d1f08bb4441a83"
+version = "55.1.0"
+source = "git+https://github.com/apache/arrow-rs?branch=main#e9df239980baa6d0f7eb4384eb01078bdd9b1701"
 dependencies = [
  "arrow-buffer",
  "arrow-schema",
@@ -122,9 +118,8 @@ dependencies = [
 
 [[package]]
 name = "arrow-ipc"
-version = "54.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "65c63da4afedde2b25ef69825cd4663ca76f78f79ffe2d057695742099130ff6"
+version = "55.1.0"
+source = "git+https://github.com/apache/arrow-rs?branch=main#e9df239980baa6d0f7eb4384eb01078bdd9b1701"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -135,15 +130,13 @@ dependencies = [
 
 [[package]]
 name = "arrow-schema"
-version = "54.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f40f6be8f78af1ab610db7d9b236e21d587b7168e368a36275d2e5670096735"
+version = "55.1.0"
+source = "git+https://github.com/apache/arrow-rs?branch=main#e9df239980baa6d0f7eb4384eb01078bdd9b1701"
 
 [[package]]
 name = "arrow-select"
-version = "54.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac265273864a820c4a179fc67182ccc41ea9151b97024e1be956f0f2369c2539"
+version = "55.1.0"
+source = "git+https://github.com/apache/arrow-rs?branch=main#e9df239980baa6d0f7eb4384eb01078bdd9b1701"
 dependencies = [
  "ahash",
  "arrow-array",
@@ -180,7 +173,7 @@ version = "0.69.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
 dependencies = [
- "bitflags 2.8.0",
+ "bitflags",
  "cexpr",
  "clang-sys",
  "itertools 0.12.1",
@@ -194,12 +187,6 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "bitflags"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
-
 [[package]]
 name = "bitflags"
 version = "2.8.0"
@@ -208,9 +195,9 @@ checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36"
 
 [[package]]
 name = "brotli"
-version = "7.0.0"
+version = "8.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd"
+checksum = "9991eea70ea4f293524138648e41ee89b0b2b12ddef3b255effa43c8056e0e0d"
 dependencies = [
  "alloc-no-stdlib",
  "alloc-stdlib",
@@ -219,9 +206,9 @@ dependencies = [
 
 [[package]]
 name = "brotli-decompressor"
-version = "4.0.2"
+version = "5.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "74fa05ad7d803d413eb8380983b092cbbaf9a85f151b871360e7b00cd7060b37"
+checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03"
 dependencies = [
  "alloc-no-stdlib",
  "alloc-stdlib",
@@ -359,11 +346,11 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
 
 [[package]]
 name = "flatbuffers"
-version = "24.12.23"
+version = "25.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096"
+checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1"
 dependencies = [
- "bitflags 1.3.2",
+ "bitflags",
  "rustc_version",
 ]
 
@@ -374,6 +361,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc"
 dependencies = [
  "crc32fast",
+ "libz-rs-sys",
  "miniz_oxide",
 ]
 
@@ -652,6 +640,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "libz-rs-sys"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "902bc563b5d65ad9bba616b490842ef0651066a1a1dc3ce1087113ffcb873c8d"
+dependencies = [
+ "zlib-rs",
+]
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.4.15"
@@ -670,7 +667,7 @@ version = "0.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5"
 dependencies = [
- "twox-hash",
+ "twox-hash 1.6.3",
 ]
 
 [[package]]
@@ -840,20 +837,20 @@ dependencies = [
  "magnus",
  "mimalloc",
  "num",
- "parquet 54.2.0",
+ "parquet 55.1.0",
  "rand",
  "rb-sys",
  "rb-sys-env 0.2.2",
  "simdutf8",
  "tempfile",
  "thiserror",
+ "uuid",
 ]
 
 [[package]]
 name = "parquet"
-version = "54.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "761c44d824fe83106e0600d2510c07bf4159a4985bf0569b513ea4288dc1b4fb"
+version = "55.1.0"
+source = "git+https://github.com/apache/arrow-rs?branch=main#e9df239980baa6d0f7eb4384eb01078bdd9b1701"
 dependencies = [
  "ahash",
  "arrow-array",
@@ -879,9 +876,8 @@ dependencies = [
  "simdutf8",
  "snap",
  "thrift",
- "twox-hash",
+ "twox-hash 2.1.0",
  "zstd",
- "zstd-sys",
 ]
 
 [[package]]
@@ -1055,7 +1051,7 @@ version = "0.38.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
 dependencies = [
- "bitflags 2.8.0",
+ "bitflags",
  "errno",
  "libc",
  "linux-raw-sys",
@@ -1223,12 +1219,24 @@ dependencies = [
  "static_assertions",
 ]
 
+[[package]]
+name = "twox-hash"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7b17f197b3050ba473acf9181f7b1d3b66d1cf7356c6cc57886662276e65908"
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "00e2473a93778eb0bad35909dff6a10d28e63f792f16ed15e404fca9d5eeedbe"
 
+[[package]]
+name = "uuid"
+version = "1.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9"
+
 [[package]]
 name = "version_check"
 version = "0.9.5"
@@ -1402,7 +1410,7 @@ version = "0.33.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c"
 dependencies = [
- "bitflags 2.8.0",
+ "bitflags",
 ]
 
 [[package]]
@@ -1446,6 +1454,12 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "zlib-rs"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b20717f0917c908dc63de2e44e97f1e6b126ca58d0e391cee86d504eb8fbd05"
+
 [[package]]
 name = "zstd"
 version = "0.13.3"

data/ext/parquet/Cargo.toml

@@ -11,20 +11,21 @@ rb-sys-env = "^0.2"
 
 [dependencies]
 ahash = "0.8"
-arrow-array = "54.0.0"
-arrow-schema = "54.0.0"
+arrow-array = { git = "https://github.com/apache/arrow-rs", branch = "main" }
+arrow-schema = { git = "https://github.com/apache/arrow-rs", branch = "main" }
 bytes = "^1.9"
 either = "1.9"
 itertools = "^0.14"
 jiff = "0.2"
 magnus = { version = "0.7", features = ["rb-sys"] }
-parquet = { version = "^54.0", features = ["json"] }
+parquet = { git = "https://github.com/apache/arrow-rs", branch = "main", features = ["json"] }
 rand = "0.9"
 rb-sys = "^0.9"
 simdutf8 = "0.1.5"
 tempfile = "^3.15"
 thiserror = "2.0"
 num = "0.4.3"
+uuid = "1.16.0"
 
 [target.'cfg(target_os = "linux")'.dependencies]
 jemallocator = { version = "0.5", features = ["disable_initial_exec_tls"] }

data/ext/parquet/src/reader/unified/mod.rs

@@ -2,8 +2,8 @@ use crate::header_cache::StringCache;
 use crate::logger::RubyLogger;
 use crate::types::TryIntoValue;
 use crate::{
-    create_column_enumerator, create_row_enumerator, ParquetField, ParquetGemError,
-    ParserResultType, ColumnEnumeratorArgs, RowEnumeratorArgs, RowRecord, ColumnRecord, ParquetValueVec,
+    create_column_enumerator, create_row_enumerator, ColumnEnumeratorArgs, ColumnRecord,
+    ParquetField, ParquetGemError, ParquetValueVec, ParserResultType, RowEnumeratorArgs, RowRecord,
 };
 use ahash::RandomState;
 use either::Either;
@@ -13,10 +13,10 @@ use std::collections::HashMap;
 use std::rc::Rc;
 use std::sync::OnceLock;
 
-use crate::types::ArrayWrapper;
 use super::common::{
     create_batch_reader, handle_block_or_enum, handle_empty_file, open_parquet_source,
 };
+use crate::types::ArrayWrapper;
 
 /// A unified parser configuration that can be used for both row and column parsing
 pub enum ParserType {
@@ -53,11 +53,11 @@ pub fn parse_parquet_unified(
     } = args;
 
     // Initialize the logger if provided
-    let ruby_logger = RubyLogger::new(&ruby, logger.clone())?;
-    
+    let ruby_logger = RubyLogger::new(&ruby, logger)?;
+
     // Clone values for the closure to avoid move issues
     let columns_clone = columns.clone();
-    
+
     // Determine if we're handling rows or columns for enumerator creation
     match &parser_type {
         ParserType::Row { strict } => {
@@ -75,13 +75,13 @@ pub fn parse_parquet_unified(
             })? {
                 return Ok(enum_value);
             }
-        },
+        }
         ParserType::Column { batch_size, strict } => {
             // For column-based parsing, log the batch size if present
             if let Some(ref bs) = batch_size {
                 ruby_logger.debug(|| format!("Using batch size: {}", bs))?;
             }
-            
+
             // Handle block or create column enumerator
             if let Some(enum_value) = handle_block_or_enum(&ruby, ruby.block_given(), || {
                 create_column_enumerator(ColumnEnumeratorArgs {
@@ -102,19 +102,34 @@ pub fn parse_parquet_unified(
 
     // Open the Parquet source
     let source = open_parquet_source(ruby.clone(), to_read)?;
-    
+
     // Based on the parser type, handle the data differently
     match parser_type {
         ParserType::Row { strict } => {
             // Handle row-based parsing
-            process_row_data(ruby.clone(), source, &columns, result_type, strict, &ruby_logger)?;
-        },
+            process_row_data(
+                ruby.clone(),
+                source,
+                &columns,
+                result_type,
+                strict,
+                &ruby_logger,
+            )?;
+        }
         ParserType::Column { batch_size, strict } => {
             // Handle column-based parsing
-            process_column_data(ruby.clone(), source, &columns, result_type, batch_size, strict, &ruby_logger)?;
+            process_column_data(
+                ruby.clone(),
+                source,
+                &columns,
+                result_type,
+                batch_size,
+                strict,
+                &ruby_logger,
+            )?;
         }
     }
-    
+
     Ok(ruby.qnil().into_value_with(&ruby))
 }
 
@@ -129,7 +144,7 @@ fn process_row_data(
 ) -> Result<(), ParquetGemError> {
     use parquet::file::reader::{FileReader, SerializedFileReader};
     use parquet::record::reader::RowIter as ParquetRowIter;
-    
+
     // Create the row-based reader
     let reader: Box<dyn FileReader> = match source {
         Either::Left(file) => {
@@ -174,8 +189,19 @@ fn process_row_data(
 
                     let mut map =
                         HashMap::with_capacity_and_hasher(headers.len(), RandomState::default());
-                    for (i, (_, v)) in row.get_column_iter().enumerate() {
-                        map.insert(headers[i], ParquetField(v.clone(), strict));
+                    for (i, ((_, v), t)) in
+                        row.get_column_iter().zip(schema.get_fields()).enumerate()
+                    {
+                        let type_info = t.get_basic_info();
+                        map.insert(
+                            headers[i],
+                            ParquetField {
+                                field: v.clone(),
+                                converted_type: type_info.converted_type(),
+                                logical_type: type_info.logical_type().clone(),
+                                strict,
+                            },
+                        );
                     }
                     map
                 })
@@ -193,8 +219,14 @@ fn process_row_data(
                 row.map(|row| {
                     let column_count = row.get_column_iter().count();
                     let mut vec = Vec::with_capacity(column_count);
-                    for (_, v) in row.get_column_iter() {
-                        vec.push(ParquetField(v.clone(), strict));
+                    for ((_, v), t) in row.get_column_iter().zip(schema.get_fields()) {
+                        let type_info = t.get_basic_info();
+                        vec.push(ParquetField {
+                            field: v.clone(),
+                            converted_type: type_info.converted_type(),
+                            logical_type: type_info.logical_type().clone(),
+                            strict,
+                        });
                     }
                     vec
                 })
@@ -309,7 +341,10 @@ fn process_column_data(
 }
 
 /// Helper function to create a projection schema
-fn create_projection_schema(schema: &parquet::schema::types::Type, columns: &[String]) -> parquet::schema::types::Type {
+fn create_projection_schema(
+    schema: &parquet::schema::types::Type,
+    columns: &[String],
+) -> parquet::schema::types::Type {
     if let parquet::schema::types::Type::GroupType { fields, .. } = schema {
         let projected_fields: Vec<std::sync::Arc<parquet::schema::types::Type>> = fields
             .iter()
@@ -325,4 +360,4 @@ fn create_projection_schema(schema: &parquet::schema::types::Type, columns: &[St
         // Return original schema if not a group type
         schema.clone()
     }
-}
+}

data/ext/parquet/src/types/mod.rs

@@ -61,6 +61,8 @@ pub enum ParquetGemError {
     Jiff(#[from] jiff::Error),
     #[error("Failed to cast slice to array: {0}")]
     InvalidDecimal(#[from] TryFromSliceError),
+    #[error("Failed to parse UUID: {0}")]
+    UuidError(#[from] uuid::Error),
 }
 
 #[derive(Debug)]

data/ext/parquet/src/types/record_types.rs

@@ -1,7 +1,10 @@
 use std::sync::OnceLock;
 
 use itertools::Itertools;
-use parquet::data_type::AsBytes;
+use parquet::{
+    basic::{ConvertedType, LogicalType},
+    data_type::AsBytes,
+};
 
 use super::*;
 
@@ -44,7 +47,13 @@ pub enum ColumnRecord<S: BuildHasher + Default> {
 }
 
 #[derive(Debug)]
-pub struct ParquetField(pub Field, pub bool);
+pub struct ParquetField {
+    pub field: Field,
+    #[allow(dead_code)]
+    pub converted_type: ConvertedType,
+    pub logical_type: Option<LogicalType>,
+    pub strict: bool,
+}
 
 impl<S: BuildHasher + Default> TryIntoValue for RowRecord<S> {
     fn try_into_value_with(self, handle: &Ruby) -> Result<Value, ParquetGemError> {
@@ -158,7 +167,7 @@ pub trait TryIntoValue {
 
 impl TryIntoValue for ParquetField {
     fn try_into_value_with(self, handle: &Ruby) -> Result<Value, ParquetGemError> {
-        match self.0 {
+        match self.field {
             Field::Null => Ok(handle.qnil().as_value()),
             Field::Bool(b) => Ok(b.into_value_with(handle)),
             Field::Short(s) => Ok(s.into_value_with(handle)),
@@ -172,7 +181,7 @@ impl TryIntoValue for ParquetField {
             Field::Float(f) => Ok(f.into_value_with(handle)),
             Field::Double(d) => Ok(d.into_value_with(handle)),
             Field::Str(s) => {
-                if self.1 {
+                if self.strict {
                     Ok(simdutf8::basic::from_utf8(s.as_bytes())
                         .map_err(ParquetGemError::Utf8Error)
                         .map(|s| s.into_value_with(handle))?)
@@ -182,7 +191,15 @@ impl TryIntoValue for ParquetField {
                 }
             }
             Field::Byte(b) => Ok(b.into_value_with(handle)),
-            Field::Bytes(b) => Ok(handle.str_from_slice(b.data()).as_value()),
+            Field::Bytes(b) => {
+                if matches!(self.logical_type, Some(parquet::basic::LogicalType::Uuid)) {
+                    let bytes = b.as_bytes();
+                    let uuid = uuid::Uuid::from_slice(bytes)?;
+                    Ok(uuid.to_string().into_value_with(handle))
+                } else {
+                    Ok(handle.str_from_slice(b.data()).as_value())
+                }
+            }
             Field::Date(d) => {
                 let ts = jiff::Timestamp::from_second((d as i64) * 86400)?;
                 let formatted = ts.strftime("%Y-%m-%d").to_string();
@@ -206,7 +223,15 @@ impl TryIntoValue for ParquetField {
                 let elements = list.elements();
                 let ary = handle.ary_new_capa(elements.len());
                 elements.iter().try_for_each(|e| {
-                    ary.push(ParquetField(e.clone(), self.1).try_into_value_with(handle)?)?;
+                    ary.push(
+                        ParquetField {
+                            field: e.clone(),
+                            logical_type: e.to_logical_type(),
+                            converted_type: e.to_converted_type(),
+                            strict: self.strict,
+                        }
+                        .try_into_value_with(handle)?,
+                    )?;
                     Ok::<_, ParquetGemError>(())
                 })?;
                 Ok(ary.into_value_with(handle))
@@ -220,8 +245,20 @@ impl TryIntoValue for ParquetField {
 
                 map.entries().iter().try_for_each(|(k, v)| {
                     hash.aset(
-                        ParquetField(k.clone(), self.1).try_into_value_with(handle)?,
-                        ParquetField(v.clone(), self.1).try_into_value_with(handle)?,
+                        ParquetField {
+                            field: k.clone(),
+                            converted_type: k.to_converted_type(),
+                            logical_type: k.to_logical_type(),
+                            strict: self.strict,
+                        }
+                        .try_into_value_with(handle)?,
+                        ParquetField {
+                            field: v.clone(),
+                            converted_type: v.to_converted_type(),
+                            logical_type: v.to_logical_type(),
+                            strict: self.strict,
+                        }
+                        .try_into_value_with(handle)?,
                     )?;
                     Ok::<_, ParquetGemError>(())
                 })?;
@@ -238,10 +275,32 @@ impl TryIntoValue for ParquetField {
                         format_decimal_with_i32_scale(unscaled, scale)
                     }
                     Decimal::Bytes { value, scale, .. } => {
-                        // value is a byte array containing the bytes for an i128 value in big endian order
-                        let casted = value.as_bytes()[..16].try_into()?;
-                        let unscaled = i128::from_be_bytes(casted);
-                        format_decimal_with_i32_scale(unscaled, scale)
+                        match value.len() {
+                            4 => {
+                                // value is a byte array containing the bytes for an i32 value in big endian order
+                                let casted = value.as_bytes()[..4].try_into()?;
+                                let unscaled = i32::from_be_bytes(casted);
+                                format_decimal_with_i32_scale(unscaled, scale)
+                            }
+                            8 => {
+                                // value is a byte array containing the bytes for an i64 value in big endian order
+                                let casted = value.as_bytes()[..8].try_into()?;
+                                let unscaled = i64::from_be_bytes(casted);
+                                format_decimal_with_i32_scale(unscaled, scale)
+                            }
+                            16 => {
+                                // value is a byte array containing the bytes for an i128 value in big endian order
+                                let casted = value.as_bytes()[..16].try_into()?;
+                                let unscaled = i128::from_be_bytes(casted);
+                                format_decimal_with_i32_scale(unscaled, scale)
+                            }
+                            _ => {
+                                unimplemented!(
+                                    "Unsupported decimal byte array size: {}",
+                                    value.len()
+                                );
+                            }
+                        }
                     }
                 };
 
@@ -256,7 +315,13 @@ impl TryIntoValue for ParquetField {
                 row.get_column_iter().try_for_each(|(k, v)| {
                     hash.aset(
                         k.clone().into_value_with(handle),
-                        ParquetField(v.clone(), self.1).try_into_value_with(handle)?,
+                        ParquetField {
+                            field: v.clone(),
+                            converted_type: v.to_converted_type(),
+                            logical_type: v.to_logical_type(),
+                            strict: self.strict,
+                        }
+                        .try_into_value_with(handle)?,
                     )?;
                     Ok::<_, ParquetGemError>(())
                 })?;
@@ -265,3 +330,111 @@ impl TryIntoValue for ParquetField {
         }
     }
 }
+
+trait ToTypeInfo {
+    fn to_converted_type(&self) -> ConvertedType;
+    fn to_logical_type(&self) -> Option<LogicalType>;
+}
+
+impl ToTypeInfo for &parquet::record::Field {
+    fn to_converted_type(&self) -> ConvertedType {
+        match self {
+            Field::Null => ConvertedType::NONE,
+            Field::Bool(_) => ConvertedType::INT_8,
+            Field::Byte(_) => ConvertedType::INT_8,
+            Field::Short(_) => ConvertedType::INT_16,
+            Field::Int(_) => ConvertedType::INT_32,
+            Field::Long(_) => ConvertedType::INT_64,
+            Field::UByte(_) => ConvertedType::UINT_8,
+            Field::UShort(_) => ConvertedType::UINT_16,
+            Field::UInt(_) => ConvertedType::UINT_32,
+            Field::ULong(_) => ConvertedType::UINT_64,
+            Field::Float16(_) => ConvertedType::NONE,
+            Field::Float(_) => ConvertedType::NONE,
+            Field::Double(_) => ConvertedType::NONE,
+            Field::Decimal(_) => ConvertedType::DECIMAL,
+            Field::Str(_) => ConvertedType::UTF8,
+            Field::Bytes(_) => ConvertedType::LIST,
+            Field::Date(_) => ConvertedType::DATE,
+            Field::TimestampMillis(_) => ConvertedType::TIMESTAMP_MILLIS,
+            Field::TimestampMicros(_) => ConvertedType::TIMESTAMP_MICROS,
+            Field::Group(_) => ConvertedType::NONE,
+            Field::ListInternal(_) => ConvertedType::LIST,
+            Field::MapInternal(_) => ConvertedType::MAP,
+        }
+    }
+    fn to_logical_type(&self) -> Option<LogicalType> {
+        Some(match self {
+            Field::Null => LogicalType::Unknown,
+            Field::Bool(_) => LogicalType::Integer {
+                bit_width: 1,
+                is_signed: false,
+            },
+            Field::Byte(_) => LogicalType::Integer {
+                bit_width: 8,
+                is_signed: false,
+            },
+            Field::Short(_) => LogicalType::Integer {
+                bit_width: 16,
+                is_signed: true,
+            },
+            Field::Int(_) => LogicalType::Integer {
+                bit_width: 32,
+                is_signed: true,
+            },
+            Field::Long(_) => LogicalType::Integer {
+                bit_width: 64,
+                is_signed: true,
+            },
+            Field::UByte(_) => LogicalType::Integer {
+                bit_width: 8,
+                is_signed: false,
+            },
+            Field::UShort(_) => LogicalType::Integer {
+                bit_width: 16,
+                is_signed: false,
+            },
+            Field::UInt(_) => LogicalType::Integer {
+                bit_width: 32,
+                is_signed: false,
+            },
+            Field::ULong(_) => LogicalType::Integer {
+                bit_width: 64,
+                is_signed: false,
+            },
+            Field::Float16(_) => LogicalType::Float16,
+            Field::Float(_) => LogicalType::Decimal {
+                scale: 7,
+                precision: 7,
+            },
+            Field::Double(_) => LogicalType::Decimal {
+                scale: 15,
+                precision: 15,
+            },
+            Field::Decimal(decimal) => LogicalType::Decimal {
+                scale: decimal.scale(),
+                precision: decimal.precision(),
+            },
+            Field::Str(_) => LogicalType::String,
+            Field::Bytes(b) => {
+                if b.data().len() == 16 && uuid::Uuid::from_slice(b.as_bytes()).is_ok() {
+                    LogicalType::Uuid
+                } else {
+                    LogicalType::Unknown
+                }
+            }
+            Field::Date(_) => LogicalType::Date,
+            Field::TimestampMillis(_) => LogicalType::Timestamp {
+                is_adjusted_to_u_t_c: true,
+                unit: parquet::basic::TimeUnit::MILLIS(parquet::format::MilliSeconds {}),
+            },
+            Field::TimestampMicros(_) => LogicalType::Timestamp {
+                is_adjusted_to_u_t_c: true,
+                unit: parquet::basic::TimeUnit::MICROS(parquet::format::MicroSeconds {}),
+            },
+            Field::Group(_) => LogicalType::Unknown,
+            Field::ListInternal(_) => LogicalType::List,
+            Field::MapInternal(_) => LogicalType::Map,
+        })
+    }
+}

data/ext/parquet/src/types/schema_converter.rs

@@ -121,7 +121,8 @@ pub fn parse_legacy_schema(
                         ruby.exception_type_error(),
                         "Schema must be an array of field definitions or nil",
                     )
-                })?.is_empty())
+                })?
+                .is_empty())
     {
         // If schema is nil or an empty array, we'll handle this in the caller
         return Ok(Vec::new());
@@ -206,101 +207,39 @@ pub fn parse_legacy_schema(
 
                 // Handle decimal type with precision and scale
                 let mut type_result = PST::try_convert(type_str)?;
-                
+
                 // If it's a decimal type and we have precision and scale, override the type
                 if let PST::Primitive(PrimitiveType::Decimal128(_, _)) = type_result {
-                    let precision_value = precision.unwrap_or_else(|| {
-                        let val: u8 = 18;
-                        val.into_value_with(ruby)
-                    });
-                    let scale_value = scale.unwrap_or_else(|| {
-                        let val: i8 = 2;
-                        val.into_value_with(ruby)
-                    });
-                    
-                    let precision_u8 = u8::try_convert(precision_value).map_err(|_| {
-                        MagnusError::new(
-                            ruby.exception_type_error(),
-                            "Invalid precision value for decimal type, expected a positive integer".to_string(),
-                        )
-                    })?;
-                    
-                    // Validate precision is in a valid range
-                    if precision_u8 < 1 {
-                        return Err(MagnusError::new(
-                            ruby.exception_arg_error(),
-                            format!(
-                                "Precision for decimal type must be at least 1, got {}", 
-                                precision_u8
-                            ),
-                        ));
-                    }
-                    
-                    if precision_u8 > 38 {
-                        return Err(MagnusError::new(
-                            ruby.exception_arg_error(),
-                            format!(
-                                "Precision for decimal type cannot exceed 38, got {}", 
-                                precision_u8
-                            ),
-                        ));
-                    }
-                    
-                    let scale_i8 = i8::try_convert(scale_value).map_err(|_| {
-                        MagnusError::new(
-                            ruby.exception_type_error(),
-                            "Invalid scale value for decimal type, expected an integer".to_string(),
-                        )
-                    })?;
-                    
-                    // Validate scale is in a valid range relative to precision
-                    if scale_i8 < 0 {
-                        return Err(MagnusError::new(
-                            ruby.exception_arg_error(),
-                            format!(
-                                "Scale for decimal type cannot be negative, got {}", 
-                                scale_i8
-                            ),
-                        ));
-                    }
-                    
-                    if scale_i8 as u8 > precision_u8 {
-                        return Err(MagnusError::new(
-                            ruby.exception_arg_error(),
-                            format!(
-                                "Scale ({}) cannot be larger than precision ({}) for decimal type", 
-                                scale_i8, precision_u8
-                            ),
-                        ));
-                    }
-                    
-                    type_result = PST::Primitive(PrimitiveType::Decimal128(precision_u8, scale_i8));
+                    // Do nothing
                 } else if let Some(type_name) = parse_string_or_symbol(ruby, type_str)? {
                     if type_name == "decimal" {
                         let precision_value = precision.unwrap_or_else(|| {
-                            let val: u8 = 18;
+                            let val: u8 = 38;
                             val.into_value_with(ruby)
                         });
+
                         let scale_value = scale.unwrap_or_else(|| {
-                            let val: i8 = 2;
+                            let val: i8 = 0;
                             val.into_value_with(ruby)
                         });
-                        
+
                         let precision_u8 = u8::try_convert(precision_value).map_err(|_| {
                             MagnusError::new(
                                 ruby.exception_type_error(),
                                 "Invalid precision value for decimal type, expected a positive integer".to_string(),
                             )
                         })?;
-                        
+
                         let scale_i8 = i8::try_convert(scale_value).map_err(|_| {
                             MagnusError::new(
                                 ruby.exception_type_error(),
-                                "Invalid scale value for decimal type, expected an integer".to_string(),
+                                "Invalid scale value for decimal type, expected an integer"
+                                    .to_string(),
                             )
                         })?;
-                        
-                        type_result = PST::Primitive(PrimitiveType::Decimal128(precision_u8, scale_i8));
+
+                        type_result =
+                            PST::Primitive(PrimitiveType::Decimal128(precision_u8, scale_i8));
                     }
                 }
 

data/lib/parquet/version.rb

@@ -1,3 +1,3 @@
 module Parquet
-  VERSION = "0.5.5"
+  VERSION = "0.5.7"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parquet
 version: !ruby/object:Gem::Version
-  version: 0.5.5
+  version: 0.5.7
 platform: ruby
 authors:
 - Nathan Jaremko
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-04-01 00:00:00.000000000 Z
+date: 2025-05-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys