parquet 0.5.4 → 0.5.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 936feb49be7a1bbbb36236551480ae0522d6b52443e76b4ebb7502abdb9d2903
-  data.tar.gz: bcc56665ec0cd132e22c262373e7b1294e085be364c93efbd214e434ada7dcb6
+  metadata.gz: dc1d1eda7d71aa6336fbf6cc94789517439df3fab1852ec7d2e9d265e0c016c4
+  data.tar.gz: 6fff5321a31d3fe19a59a4f47add56222dbeb274bef7a068163b48757d65252d
 SHA512:
-  metadata.gz: 7856d7f36820a8384faf564f166d39e0daca1c9d15457b6f6aae8ff56f4176a8b1302bfbc2cc5edcfedfcb0805cbe71029f5712e716a29dc4942a1e6453a3e5e
-  data.tar.gz: '08d1f4cfe357b22bad4c4fab4ddd4fa93069b13c65559d668fb704e2f7d8884fc8f081270e4dc43a5db60aab7147be36bfe7d26945f93c9ad6e9badbd0ad957e'
+  metadata.gz: ddd50f82df2b42cf844e379a7f07c0214e9aef925e7c43ec566b6b9f27be311676b6f887c163aa5d41d4523cd1d506266b15623205453bc8e08467c88e7c2b63
+  data.tar.gz: afb235ad09338d8c4cd59588dded3d312890c5d5d879b77040fcbf960be69653981fe5176cc591969a80ba54214d4c6a63cff96c36ceda7b9e00c75ba8e9e913

data/Cargo.lock

@@ -63,9 +63,8 @@ dependencies = [
 
 [[package]]
 name = "arrow-array"
-version = "54.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57a4a6d2896083cfbdf84a71a863b22460d0708f8206a8373c52e326cc72ea1a"
+version = "55.1.0"
+source = "git+https://github.com/njaremko/arrow-rs?branch=nathan%2Ffix-reading-int32-timestamp-records#f791b78a67cb5d9a0b4ec0fcab80780dcb61c346"
 dependencies = [
  "ahash",
  "arrow-buffer",
@@ -79,9 +78,8 @@ dependencies = [
 
 [[package]]
 name = "arrow-buffer"
-version = "54.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cef870583ce5e4f3b123c181706f2002fb134960f9a911900f64ba4830c7a43a"
+version = "55.1.0"
+source = "git+https://github.com/njaremko/arrow-rs?branch=nathan%2Ffix-reading-int32-timestamp-records#f791b78a67cb5d9a0b4ec0fcab80780dcb61c346"
 dependencies = [
  "bytes",
  "half",
@@ -90,9 +88,8 @@ dependencies = [
 
 [[package]]
 name = "arrow-cast"
-version = "54.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ac7eba5a987f8b4a7d9629206ba48e19a1991762795bbe5d08497b7736017ee"
+version = "55.1.0"
+source = "git+https://github.com/njaremko/arrow-rs?branch=nathan%2Ffix-reading-int32-timestamp-records#f791b78a67cb5d9a0b4ec0fcab80780dcb61c346"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -110,9 +107,8 @@ dependencies = [
 
 [[package]]
 name = "arrow-data"
-version = "54.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b095e8a4f3c309544935d53e04c3bfe4eea4e71c3de6fe0416d1f08bb4441a83"
+version = "55.1.0"
+source = "git+https://github.com/njaremko/arrow-rs?branch=nathan%2Ffix-reading-int32-timestamp-records#f791b78a67cb5d9a0b4ec0fcab80780dcb61c346"
 dependencies = [
  "arrow-buffer",
  "arrow-schema",
@@ -122,9 +118,8 @@ dependencies = [
 
 [[package]]
 name = "arrow-ipc"
-version = "54.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "65c63da4afedde2b25ef69825cd4663ca76f78f79ffe2d057695742099130ff6"
+version = "55.1.0"
+source = "git+https://github.com/njaremko/arrow-rs?branch=nathan%2Ffix-reading-int32-timestamp-records#f791b78a67cb5d9a0b4ec0fcab80780dcb61c346"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -135,15 +130,13 @@ dependencies = [
 
 [[package]]
 name = "arrow-schema"
-version = "54.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f40f6be8f78af1ab610db7d9b236e21d587b7168e368a36275d2e5670096735"
+version = "55.1.0"
+source = "git+https://github.com/njaremko/arrow-rs?branch=nathan%2Ffix-reading-int32-timestamp-records#f791b78a67cb5d9a0b4ec0fcab80780dcb61c346"
 
 [[package]]
 name = "arrow-select"
-version = "54.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac265273864a820c4a179fc67182ccc41ea9151b97024e1be956f0f2369c2539"
+version = "55.1.0"
+source = "git+https://github.com/njaremko/arrow-rs?branch=nathan%2Ffix-reading-int32-timestamp-records#f791b78a67cb5d9a0b4ec0fcab80780dcb61c346"
 dependencies = [
  "ahash",
  "arrow-array",
@@ -180,7 +173,7 @@ version = "0.69.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
 dependencies = [
- "bitflags 2.8.0",
+ "bitflags",
  "cexpr",
  "clang-sys",
  "itertools 0.12.1",
@@ -194,12 +187,6 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "bitflags"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
-
 [[package]]
 name = "bitflags"
 version = "2.8.0"
@@ -208,9 +195,9 @@ checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36"
 
 [[package]]
 name = "brotli"
-version = "7.0.0"
+version = "8.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd"
+checksum = "9991eea70ea4f293524138648e41ee89b0b2b12ddef3b255effa43c8056e0e0d"
 dependencies = [
  "alloc-no-stdlib",
  "alloc-stdlib",
@@ -219,9 +206,9 @@ dependencies = [
 
 [[package]]
 name = "brotli-decompressor"
-version = "4.0.2"
+version = "5.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "74fa05ad7d803d413eb8380983b092cbbaf9a85f151b871360e7b00cd7060b37"
+checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03"
 dependencies = [
  "alloc-no-stdlib",
  "alloc-stdlib",
@@ -359,11 +346,11 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
 
 [[package]]
 name = "flatbuffers"
-version = "24.12.23"
+version = "25.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096"
+checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1"
 dependencies = [
- "bitflags 1.3.2",
+ "bitflags",
  "rustc_version",
 ]
 
@@ -374,6 +361,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc"
 dependencies = [
  "crc32fast",
+ "libz-rs-sys",
  "miniz_oxide",
 ]
 
@@ -652,6 +640,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "libz-rs-sys"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "902bc563b5d65ad9bba616b490842ef0651066a1a1dc3ce1087113ffcb873c8d"
+dependencies = [
+ "zlib-rs",
+]
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.4.15"
@@ -670,7 +667,7 @@ version = "0.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5"
 dependencies = [
- "twox-hash",
+ "twox-hash 1.6.3",
 ]
 
 [[package]]
@@ -840,7 +837,7 @@ dependencies = [
  "magnus",
  "mimalloc",
  "num",
- "parquet 54.2.0",
+ "parquet 55.1.0",
  "rand",
  "rb-sys",
  "rb-sys-env 0.2.2",
@@ -851,9 +848,8 @@ dependencies = [
 
 [[package]]
 name = "parquet"
-version = "54.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "761c44d824fe83106e0600d2510c07bf4159a4985bf0569b513ea4288dc1b4fb"
+version = "55.1.0"
+source = "git+https://github.com/njaremko/arrow-rs?branch=nathan%2Ffix-reading-int32-timestamp-records#f791b78a67cb5d9a0b4ec0fcab80780dcb61c346"
 dependencies = [
  "ahash",
  "arrow-array",
@@ -879,9 +875,8 @@ dependencies = [
  "simdutf8",
  "snap",
  "thrift",
- "twox-hash",
+ "twox-hash 2.1.0",
  "zstd",
- "zstd-sys",
 ]
 
 [[package]]
@@ -1055,7 +1050,7 @@ version = "0.38.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
 dependencies = [
- "bitflags 2.8.0",
+ "bitflags",
  "errno",
  "libc",
  "linux-raw-sys",
@@ -1223,6 +1218,12 @@ dependencies = [
  "static_assertions",
 ]
 
+[[package]]
+name = "twox-hash"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7b17f197b3050ba473acf9181f7b1d3b66d1cf7356c6cc57886662276e65908"
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.17"
@@ -1402,7 +1403,7 @@ version = "0.33.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c"
 dependencies = [
- "bitflags 2.8.0",
+ "bitflags",
 ]
 
 [[package]]
@@ -1446,6 +1447,12 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "zlib-rs"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b20717f0917c908dc63de2e44e97f1e6b126ca58d0e391cee86d504eb8fbd05"
+
 [[package]]
 name = "zstd"
 version = "0.13.3"

data/ext/parquet/Cargo.toml

@@ -11,14 +11,14 @@ rb-sys-env = "^0.2"
 
 [dependencies]
 ahash = "0.8"
-arrow-array = "54.0.0"
-arrow-schema = "54.0.0"
+arrow-array = { git = "https://github.com/njaremko/arrow-rs", branch = "nathan/fix-reading-int32-timestamp-records" }
+arrow-schema = { git = "https://github.com/njaremko/arrow-rs", branch = "nathan/fix-reading-int32-timestamp-records" }
 bytes = "^1.9"
 either = "1.9"
 itertools = "^0.14"
 jiff = "0.2"
 magnus = { version = "0.7", features = ["rb-sys"] }
-parquet = { version = "^54.0", features = ["json"] }
+parquet = { git = "https://github.com/njaremko/arrow-rs", branch = "nathan/fix-reading-int32-timestamp-records", features = ["json"] }
 rand = "0.9"
 rb-sys = "^0.9"
 simdutf8 = "0.1.5"

data/ext/parquet/src/reader/mod.rs

@@ -1,6 +1,7 @@
 mod common;
 mod parquet_column_reader;
 mod parquet_row_reader;
+mod unified;
 use std::{fs::File, rc::Rc};
 
 use magnus::{value::ReprValue, Error as MagnusError, Ruby, Value};
@@ -207,4 +208,4 @@ pub fn parse_metadata(_rb_self: Value, args: &[Value]) -> Result<Value, MagnusEr
     let metadata = reader.finish().map_err(ParquetGemError::Parquet)?;
 
     Ok(RubyParquetMetaData(metadata).try_into_value_with(&ruby)?)
-}
+}

data/ext/parquet/src/reader/parquet_column_reader.rs

@@ -1,21 +1,9 @@
-use crate::header_cache::StringCache;
-use crate::logger::RubyLogger;
-use crate::types::{ArrayWrapper, ParquetGemError, TryIntoValue};
-use crate::{
-    create_column_enumerator, utils::*, ColumnEnumeratorArgs, ColumnRecord, ParquetValueVec,
-    ParserResultType,
-};
-use ahash::RandomState;
-use either::Either;
-use magnus::IntoValue;
+use crate::reader::unified::{parse_parquet_unified, ParserType, UnifiedParserArgs};
+use crate::utils::*;
+use crate::ParquetGemError;
+
 use magnus::{Error as MagnusError, Ruby, Value};
-use std::collections::HashMap;
 use std::rc::Rc;
-use std::sync::OnceLock;
-
-use super::common::{
-    create_batch_reader, handle_block_or_enum, handle_empty_file, open_parquet_source,
-};
 
 #[inline]
 pub fn parse_parquet_columns(rb_self: Value, args: &[Value]) -> Result<Value, MagnusError> {
@@ -41,116 +29,16 @@ fn parse_parquet_columns_impl(
         logger,
     } = parse_parquet_columns_args(&ruby, args)?;
 
-    // Initialize the logger if provided
-    let ruby_logger = RubyLogger::new(&ruby, logger)?;
-    if let Some(ref bs) = batch_size {
-        ruby_logger.debug(|| format!("Using batch size: {}", bs))?;
-    }
-
-    // Clone values for the closure to avoid move issues
-    let columns_clone = columns.clone();
-
-    // Handle block or create enumerator
-    if let Some(enum_value) = handle_block_or_enum(&ruby, ruby.block_given(), || {
-        create_column_enumerator(ColumnEnumeratorArgs {
-            rb_self,
+    // Use the unified parsing implementation
+    parse_parquet_unified(
+        ruby,
+        rb_self,
+        UnifiedParserArgs {
             to_read,
             result_type,
-            columns: columns_clone,
-            batch_size,
-            strict,
-            logger: logger.as_ref().map(|_| to_read),
-        })
-        .map(|yield_enum| yield_enum.into_value_with(&ruby))
-    })? {
-        return Ok(enum_value);
-    }
-
-    let source = open_parquet_source(ruby.clone(), to_read)?;
-
-    // Use the common function to create the batch reader
-
-    let (batch_reader, schema, num_rows) = match source {
-        Either::Left(file) => create_batch_reader(file, &columns, batch_size)?,
-        Either::Right(readable) => create_batch_reader(readable, &columns, batch_size)?,
-    };
-
-    match result_type {
-        ParserResultType::Hash => {
-            // For hash return type, we need to return a hash with column names pointing at empty arrays
-            if handle_empty_file(&ruby, &schema, num_rows)? {
-                return Ok(ruby.qnil().into_value_with(&ruby));
-            }
-
-            let headers = OnceLock::new();
-            let headers_clone = headers.clone();
-            let iter = batch_reader.map(move |batch| {
-                batch.map_err(ParquetGemError::Arrow).and_then(|batch| {
-                    let local_headers = headers_clone
-                        .get_or_init(|| {
-                            let schema = batch.schema();
-                            let fields = schema.fields();
-                            let mut header_string = Vec::with_capacity(fields.len());
-                            for field in fields {
-                                header_string.push(field.name().to_owned());
-                            }
-                            StringCache::intern_many(&header_string)
-                        })
-                        .as_ref()
-                        .map_err(|e| ParquetGemError::HeaderIntern(e.clone()))?;
-
-                    let mut map = HashMap::with_capacity_and_hasher(
-                        local_headers.len(),
-                        RandomState::default(),
-                    );
-
-                    batch
-                        .columns()
-                        .iter()
-                        .enumerate()
-                        .try_for_each(|(i, column)| {
-                            let header = local_headers[i];
-                            let values = ParquetValueVec::try_from(ArrayWrapper {
-                                array: column,
-                                strict,
-                            })?;
-                            map.insert(header, values.into_inner());
-                            Ok::<_, ParquetGemError>(())
-                        })?;
-
-                    Ok(ColumnRecord::Map::<RandomState>(map))
-                })
-            });
-
-            for result in iter {
-                let record = result?;
-                let _: Value = ruby.yield_value(record.try_into_value_with(&ruby)?)?;
-            }
-        }
-        ParserResultType::Array => {
-            let iter = batch_reader.map(|batch| {
-                batch.map_err(ParquetGemError::Arrow).and_then(|batch| {
-                    let vec = batch
-                        .columns()
-                        .iter()
-                        .map(|column| {
-                            let values = ParquetValueVec::try_from(ArrayWrapper {
-                                array: column,
-                                strict,
-                            })?;
-                            Ok::<_, ParquetGemError>(values.into_inner())
-                        })
-                        .collect::<Result<Vec<_>, _>>()?;
-                    Ok(ColumnRecord::Vec::<RandomState>(vec))
-                })
-            });
-
-            for result in iter {
-                let record = result?;
-                let _: Value = ruby.yield_value(record.try_into_value_with(&ruby)?)?;
-            }
-        }
-    }
-
-    Ok(ruby.qnil().into_value_with(&ruby))
-}
+            columns,
+            parser_type: ParserType::Column { batch_size, strict },
+            logger,
+        },
+    )
+}

data/ext/parquet/src/reader/parquet_row_reader.rs

@@ -1,22 +1,9 @@
-use crate::header_cache::StringCache;
-use crate::logger::RubyLogger;
-use crate::types::TryIntoValue;
-use crate::{
-    create_row_enumerator, utils::*, ParquetField, ParquetGemError, ParserResultType,
-    RowEnumeratorArgs, RowRecord,
-};
-use ahash::RandomState;
-use either::Either;
-use magnus::IntoValue;
+use crate::reader::unified::{parse_parquet_unified, ParserType, UnifiedParserArgs};
+use crate::utils::*;
+use crate::ParquetGemError;
+
 use magnus::{Error as MagnusError, Ruby, Value};
-use parquet::file::reader::{FileReader, SerializedFileReader};
-use parquet::record::reader::RowIter as ParquetRowIter;
-use parquet::schema::types::{Type as SchemaType, TypePtr};
-use std::collections::HashMap;
 use std::rc::Rc;
-use std::sync::OnceLock;
-
-use super::common::{handle_block_or_enum, open_parquet_source};
 
 #[inline]
 pub fn parse_parquet_rows(rb_self: Value, args: &[Value]) -> Result<Value, MagnusError> {
@@ -41,123 +28,16 @@ fn parse_parquet_rows_impl(
         logger,
     } = parse_parquet_rows_args(&ruby, args)?;
 
-    // Initialize the logger if provided
-    let ruby_logger = RubyLogger::new(&ruby, logger)?;
-
-    // Clone values for the closure to avoid move issues
-    let columns_clone = columns.clone();
-
-    // Handle block or create enumerator
-    if let Some(enum_value) = handle_block_or_enum(&ruby, ruby.block_given(), || {
-        create_row_enumerator(RowEnumeratorArgs {
-            rb_self,
+    // Use the unified parsing implementation
+    parse_parquet_unified(
+        ruby,
+        rb_self,
+        UnifiedParserArgs {
             to_read,
             result_type,
-            columns: columns_clone,
-            strict,
+            columns,
+            parser_type: ParserType::Row { strict },
             logger,
-        })
-        .map(|yield_enum| yield_enum.into_value_with(&ruby))
-    })? {
-        return Ok(enum_value);
-    }
-
-    let source = open_parquet_source(ruby.clone(), to_read)?;
-    let reader: Box<dyn FileReader> = match source {
-        Either::Left(file) => {
-            Box::new(SerializedFileReader::new(file).map_err(ParquetGemError::from)?)
-        }
-        Either::Right(readable) => {
-            Box::new(SerializedFileReader::new(readable).map_err(ParquetGemError::from)?)
-        }
-    };
-
-    let schema = reader.metadata().file_metadata().schema().clone();
-    ruby_logger.debug(|| format!("Schema loaded: {:?}", schema))?;
-
-    let mut iter = ParquetRowIter::from_file_into(reader);
-    if let Some(cols) = columns {
-        ruby_logger.debug(|| format!("Projecting columns: {:?}", cols))?;
-        let projection = create_projection_schema(&schema, &cols);
-        iter = iter.project(Some(projection.to_owned())).map_err(|e| {
-            MagnusError::new(
-                ruby.exception_runtime_error(),
-                format!("Failed to create projection: {}", e),
-            )
-        })?;
-    }
-
-    match result_type {
-        ParserResultType::Hash => {
-            let headers = OnceLock::new();
-            let headers_clone = headers.clone();
-            let iter = iter.map(move |row| {
-                row.map(|row| {
-                    let headers = headers_clone.get_or_init(|| {
-                        let column_count = row.get_column_iter().count();
-
-                        let mut header_string = Vec::with_capacity(column_count);
-                        for (k, _) in row.get_column_iter() {
-                            header_string.push(k.to_owned());
-                        }
-
-                        StringCache::intern_many(&header_string).expect("Failed to intern headers")
-                    });
-
-                    let mut map =
-                        HashMap::with_capacity_and_hasher(headers.len(), RandomState::default());
-                    for (i, (_, v)) in row.get_column_iter().enumerate() {
-                        map.insert(headers[i], ParquetField(v.clone(), strict));
-                    }
-                    map
-                })
-                .map(RowRecord::Map::<RandomState>)
-                .map_err(ParquetGemError::from)
-            });
-
-            for result in iter {
-                let record = result?;
-                let _: Value = ruby.yield_value(record.try_into_value_with(&ruby)?)?;
-            }
-        }
-        ParserResultType::Array => {
-            let iter = iter.map(|row| {
-                row.map(|row| {
-                    let column_count = row.get_column_iter().count();
-                    let mut vec = Vec::with_capacity(column_count);
-                    for (_, v) in row.get_column_iter() {
-                        vec.push(ParquetField(v.clone(), strict));
-                    }
-                    vec
-                })
-                .map(RowRecord::Vec::<RandomState>)
-                .map_err(ParquetGemError::from)
-            });
-
-            for result in iter {
-                let record = result?;
-                let _: Value = ruby.yield_value(record.try_into_value_with(&ruby)?)?;
-            }
-        }
-    }
-
-    Ok(ruby.qnil().into_value_with(&ruby))
-}
-
-fn create_projection_schema(schema: &SchemaType, columns: &[String]) -> SchemaType {
-    if let SchemaType::GroupType { fields, .. } = schema {
-        let projected_fields: Vec<TypePtr> = fields
-            .iter()
-            .filter(|field| columns.contains(&field.name().to_string()))
-            .cloned()
-            .collect();
-
-        SchemaType::GroupType {
-            basic_info: schema.get_basic_info().clone(),
-            fields: projected_fields,
-        }
-    } else {
-        // Return original schema if not a group type
-        schema.clone()
-    }
-}
+        },
+    )
+}