parquet 0.5.3 → 0.5.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e1ae8e2c64920df8527a16d7348fc37c5ae2cf5c783b648bed93e31cab25bd72
-  data.tar.gz: 2d7b45349d33679f96559683e31d7c9dd5718fb78611aad057bba92d7324c2d3
+  metadata.gz: e2295ee94fe35758ae8e5137070e2206ec1e104aad6b9a0806aa508ad4799247
+  data.tar.gz: 340f86257082bdba22d6ced530ecd1d201c7b4e6d9116eebac41541ba2aaa257
 SHA512:
-  metadata.gz: 1f56d8e538bdb095e43472940a8c3a57b6b54d74ab87d9c1519878d759962e6d844f9c992927dc22d22ebefee4bd64a858b2ed89ccc3c694d183bcb9fd154497
-  data.tar.gz: 5f5c8914d81ef297bebb021ba40e70725208e61c2bd1565f7d134341ac3c31489b501766266f7390ffde82a44e5821321b55f827467ac95c760cd08588788e9d
+  metadata.gz: f333ae2914cdd00468c390e8b3d876aec4e522a546d43ab29db5d777792105a38d2a40c49db0f0afe1e800bf32e54bb4c479441f8f9876937ba59917b444d15a
+  data.tar.gz: da2832c3514729cc0e99e16f70a10bbfc4e9093dc734de55715305121649ebc371dff93a7bb462b97fde27c79ad65cec12c5fa90a47f70bc64153a7fd2ce1a5c

data/README.md

@@ -8,6 +8,78 @@ This project is a Ruby library wrapping the [parquet-rs](https://github.com/apac
 
 This library provides high-level bindings to parquet-rs with two primary APIs for reading Parquet files: row-wise and column-wise iteration. The column-wise API generally offers better performance, especially when working with subset of columns.
 
+### Metadata
+
+The `metadata` method provides detailed information about a Parquet file's structure and contents:
+
+```ruby
+require "parquet"
+
+# Get metadata from a file path
+metadata = Parquet.metadata("data.parquet")
+
+# Or from an IO object
+File.open("data.parquet", "rb") do |file|
+  metadata = Parquet.metadata(file)
+end
+
+# Example metadata output:
+# {
+#   "num_rows" => 3,
+#   "created_by" => "parquet-rs version 54.2.0",
+#   "key_value_metadata" => [
+#     {
+#       "key" => "ARROW:schema",
+#       "value" => "base64_encoded_schema"
+#     }
+#   ],
+#   "schema" => {
+#     "name" => "arrow_schema",
+#     "fields" => [
+#       {
+#         "name" => "id",
+#         "type" => "primitive",
+#         "physical_type" => "INT64",
+#         "repetition" => "OPTIONAL",
+#         "converted_type" => "NONE"
+#       },
+#       # ... other fields
+#     ]
+#   },
+#   "row_groups" => [
+#     {
+#       "num_columns" => 5,
+#       "num_rows" => 3,
+#       "total_byte_size" => 379,
+#       "columns" => [
+#         {
+#           "column_path" => "id",
+#           "num_values" => 3,
+#           "compression" => "UNCOMPRESSED",
+#           "total_compressed_size" => 91,
+#           "encodings" => ["PLAIN", "RLE", "RLE_DICTIONARY"],
+#           "statistics" => {
+#             "min_is_exact" => true,
+#             "max_is_exact" => true
+#           }
+#         },
+#         # ... other columns
+#       ]
+#     }
+#   ]
+# }
+```
+
+The metadata includes:
+- Total number of rows
+- File creation information
+- Key-value metadata (including Arrow schema)
+- Detailed schema information for each column
+- Row group information including:
+  - Number of columns and rows
+  - Total byte size
+  - Column-level details (compression, encodings, statistics)
+
 ### Row-wise Iteration
 
 The `each_row` method provides sequential access to individual rows:
@@ -236,17 +308,169 @@ schema = Parquet::Schema.define do
     field :description, :string
   end
 
-  # Nested lists
+  # Nested lists (list of lists of strings)
   field :nested_lists, :list, item: :list do
-    field :item, :string  # For nested lists, inner item must be named 'item'
+    field :item, :string  # REQUIRED: Inner item field MUST be named 'item' for nested lists
   end
 
   # Map of lists
   field :map_of_lists, :map, key: :string, value: :list do
-    field :item, :int32  # For list items in maps, item must be named 'item'
+    field :item, :int32  # REQUIRED: List items in maps MUST be named 'item'
   end
 end
 
+### Nested Lists
+
+When working with nested lists (a list of lists), there are specific requirements:
+
+1. Using the Schema DSL:
+```ruby
+# A list of lists of strings
+field :nested_lists, :list, item: :list do
+  field :item, :string  # For nested lists, inner item MUST be named 'item'
+end
+```
+
+2. Using hash-based schema format:
+```ruby
+# A list of lists of integers
+{ "nested_numbers" => "list<list<int32>>" }
+```
+
+The data for nested lists is structured as an array of arrays:
+```ruby
+# Data for the nested_lists field
+[["a", "b"], ["c", "d", "e"], []]  # Last one is an empty inner list
+```
+
+### Decimal Data Type
+
+Parquet supports decimal numbers with configurable precision and scale, which is essential for financial applications where exact decimal representation is critical. The library seamlessly converts between Ruby's `BigDecimal` and Parquet's decimal type.
+
+#### Decimal Precision and Scale
+
+When working with decimal fields, you need to understand two key parameters:
+
+- **Precision**: The total number of significant digits (both before and after the decimal point)
+- **Scale**: The number of digits after the decimal point
+
+The rules for defining decimals are:
+
+```ruby
+# No precision/scale specified - uses maximum precision (38) with scale 0
+field :amount1, :decimal  # Equivalent to INTEGER with 38 digits
+
+# Only precision specified - scale defaults to 0
+field :amount2, :decimal, precision: 10  # 10 digits, no decimal places
+
+# Only scale specified - uses maximum precision (38)
+field :amount3, :decimal, scale: 2  # 38 digits with 2 decimal places
+
+# Both precision and scale specified
+field :amount4, :decimal, precision: 10, scale: 2  # 10 digits with 2 decimal places
+```
+
+#### Financial Data Example
+
+Here's a practical example for a financial application:
+
+```ruby
+require "parquet"
+require "bigdecimal"
+
+# Schema for financial transactions
+schema = Parquet::Schema.define do
+  field :transaction_id, :string, nullable: false
+  field :timestamp, :timestamp_millis, nullable: false
+  field :amount, :decimal, precision: 12, scale: 2  # Supports up to 10^10 with 2 decimal places
+  field :balance, :decimal, precision: 16, scale: 2  # Larger precision for running balances
+  field :currency, :string
+  field :exchange_rate, :decimal, precision: 10, scale: 6  # 6 decimal places for forex rates
+  field :fee, :decimal, precision: 8, scale: 2, nullable: true  # Optional fee
+  field :category, :string
+end
+
+# Sample financial data
+transactions = [
+  [
+    "T-12345",
+    Time.now,
+    BigDecimal("1256.99"),       # amount (directly using BigDecimal)
+    BigDecimal("10250.25"),      # balance
+    "USD",
+    BigDecimal("1.0"),           # exchange_rate
+    BigDecimal("2.50"),          # fee
+    "Groceries"
+  ],
+  [
+    "T-12346",
+    Time.now - 86400,            # yesterday
+    BigDecimal("-89.50"),        # negative amount for withdrawal
+    BigDecimal("10160.75"),      # updated balance
+    "USD",
+    BigDecimal("1.0"),           # exchange_rate
+    nil,                         # no fee
+    "Transportation"
+  ],
+  [
+    "T-12347",
+    Time.now - 172800,           # two days ago
+    BigDecimal("250.00"),        # amount
+    BigDecimal("10410.75"),      # balance
+    "EUR",                       # different currency
+    BigDecimal("1.05463"),       # exchange_rate
+    BigDecimal("1.75"),          # fee
+    "Entertainment"
+  ]
+]
+
+# Write financial data to Parquet file
+Parquet.write_rows(transactions.each, schema: schema, write_to: "financial_data.parquet")
+
+# Read back transactions
+Parquet.each_row("financial_data.parquet") do |transaction|
+  # Access decimal fields as BigDecimal objects
+  puts "Transaction: #{transaction['transaction_id']}"
+  puts "  Amount: #{transaction['currency']} #{transaction['amount']}"
+  puts "  Balance: $#{transaction['balance']}"
+  puts "  Fee: #{transaction['fee'] || 'No fee'}"
+
+  # You can perform precise decimal calculations
+  if transaction['currency'] != 'USD'
+    usd_amount = transaction['amount'] * transaction['exchange_rate']
+    puts "  USD Equivalent: $#{usd_amount.round(2)}"
+  end
+end
+```
+
+#### Decimal Type Storage Considerations
+
+Parquet optimizes storage based on the precision:
+- For precision ≤ 9: Uses 4-byte INT32
+- For precision ≤ 18: Uses 8-byte INT64
+- For precision ≤ 38: Uses 16-byte BYTE_ARRAY
+
+Choose appropriate precision and scale for your data to optimize storage while ensuring adequate range:
+
+```ruby
+# Banking examples
+field :account_balance, :decimal, precision: 16, scale: 2   # Up to 14 digits before decimal point
+field :interest_rate, :decimal, precision: 8, scale: 6      # Rate with 6 decimal places (e.g., 0.015625)
+
+# E-commerce examples
+field :product_price, :decimal, precision: 10, scale: 2     # Product price
+field :shipping_weight, :decimal, precision: 6, scale: 3    # Weight in kg with 3 decimal places
+
+# Analytics examples
+field :conversion_rate, :decimal, precision: 5, scale: 4    # Rate like 0.0123
+field :daily_revenue, :decimal, precision: 14, scale: 2     # Daily revenue with 2 decimal places
+```
+
+### Sample Data with Nested Structures
+
+Here's an example showing how to use the schema defined earlier with sample data:
+
+```ruby
 # Sample data with nested structures
 data = [
   [
@@ -271,7 +495,7 @@ data = [
       "feature1" => { "count" => 5, "description" => "Main feature" },
       "feature2" => { "count" => 3, "description" => "Secondary feature" }
     },
-    [["a", "b"], ["c", "d", "e"]],  # nested_lists
+    [["a", "b"], ["c", "d", "e"]],  # nested_lists (a list of lists of strings)
     {                                # map_of_lists
       "group1" => [1, 2, 3],
       "group2" => [4, 5, 6]

data/ext/parquet/src/reader/mod.rs

@@ -1,6 +1,7 @@
 mod common;
 mod parquet_column_reader;
 mod parquet_row_reader;
+mod unified;
 use std::{fs::File, rc::Rc};
 
 use magnus::{value::ReprValue, Error as MagnusError, Ruby, Value};
@@ -207,4 +208,4 @@ pub fn parse_metadata(_rb_self: Value, args: &[Value]) -> Result<Value, MagnusEr
     let metadata = reader.finish().map_err(ParquetGemError::Parquet)?;
 
     Ok(RubyParquetMetaData(metadata).try_into_value_with(&ruby)?)
-}
+}

data/ext/parquet/src/reader/parquet_column_reader.rs

@@ -1,21 +1,9 @@
-use crate::header_cache::StringCache;
-use crate::logger::RubyLogger;
-use crate::types::{ArrayWrapper, ParquetGemError, TryIntoValue};
-use crate::{
-    create_column_enumerator, utils::*, ColumnEnumeratorArgs, ColumnRecord, ParquetValueVec,
-    ParserResultType,
-};
-use ahash::RandomState;
-use either::Either;
-use magnus::IntoValue;
+use crate::reader::unified::{parse_parquet_unified, ParserType, UnifiedParserArgs};
+use crate::utils::*;
+use crate::ParquetGemError;
+
 use magnus::{Error as MagnusError, Ruby, Value};
-use std::collections::HashMap;
 use std::rc::Rc;
-use std::sync::OnceLock;
-
-use super::common::{
-    create_batch_reader, handle_block_or_enum, handle_empty_file, open_parquet_source,
-};
 
 #[inline]
 pub fn parse_parquet_columns(rb_self: Value, args: &[Value]) -> Result<Value, MagnusError> {
@@ -41,116 +29,16 @@ fn parse_parquet_columns_impl(
         logger,
     } = parse_parquet_columns_args(&ruby, args)?;
 
-    // Initialize the logger if provided
-    let ruby_logger = RubyLogger::new(&ruby, logger)?;
-    if let Some(ref bs) = batch_size {
-        ruby_logger.debug(|| format!("Using batch size: {}", bs))?;
-    }
-
-    // Clone values for the closure to avoid move issues
-    let columns_clone = columns.clone();
-
-    // Handle block or create enumerator
-    if let Some(enum_value) = handle_block_or_enum(&ruby, ruby.block_given(), || {
-        create_column_enumerator(ColumnEnumeratorArgs {
-            rb_self,
+    // Use the unified parsing implementation
+    parse_parquet_unified(
+        ruby,
+        rb_self,
+        UnifiedParserArgs {
             to_read,
             result_type,
-            columns: columns_clone,
-            batch_size,
-            strict,
-            logger: logger.as_ref().map(|_| to_read),
-        })
-        .map(|yield_enum| yield_enum.into_value_with(&ruby))
-    })? {
-        return Ok(enum_value);
-    }
-
-    let source = open_parquet_source(ruby.clone(), to_read)?;
-
-    // Use the common function to create the batch reader
-
-    let (batch_reader, schema, num_rows) = match source {
-        Either::Left(file) => create_batch_reader(file, &columns, batch_size)?,
-        Either::Right(readable) => create_batch_reader(readable, &columns, batch_size)?,
-    };
-
-    match result_type {
-        ParserResultType::Hash => {
-            // For hash return type, we need to return a hash with column names pointing at empty arrays
-            if handle_empty_file(&ruby, &schema, num_rows)? {
-                return Ok(ruby.qnil().into_value_with(&ruby));
-            }
-
-            let headers = OnceLock::new();
-            let headers_clone = headers.clone();
-            let iter = batch_reader.map(move |batch| {
-                batch.map_err(ParquetGemError::Arrow).and_then(|batch| {
-                    let local_headers = headers_clone
-                        .get_or_init(|| {
-                            let schema = batch.schema();
-                            let fields = schema.fields();
-                            let mut header_string = Vec::with_capacity(fields.len());
-                            for field in fields {
-                                header_string.push(field.name().to_owned());
-                            }
-                            StringCache::intern_many(&header_string)
-                        })
-                        .as_ref()
-                        .map_err(|e| ParquetGemError::HeaderIntern(e.clone()))?;
-
-                    let mut map = HashMap::with_capacity_and_hasher(
-                        local_headers.len(),
-                        RandomState::default(),
-                    );
-
-                    batch
-                        .columns()
-                        .iter()
-                        .enumerate()
-                        .try_for_each(|(i, column)| {
-                            let header = local_headers[i];
-                            let values = ParquetValueVec::try_from(ArrayWrapper {
-                                array: column,
-                                strict,
-                            })?;
-                            map.insert(header, values.into_inner());
-                            Ok::<_, ParquetGemError>(())
-                        })?;
-
-                    Ok(ColumnRecord::Map::<RandomState>(map))
-                })
-            });
-
-            for result in iter {
-                let record = result?;
-                let _: Value = ruby.yield_value(record.try_into_value_with(&ruby)?)?;
-            }
-        }
-        ParserResultType::Array => {
-            let iter = batch_reader.map(|batch| {
-                batch.map_err(ParquetGemError::Arrow).and_then(|batch| {
-                    let vec = batch
-                        .columns()
-                        .iter()
-                        .map(|column| {
-                            let values = ParquetValueVec::try_from(ArrayWrapper {
-                                array: column,
-                                strict,
-                            })?;
-                            Ok::<_, ParquetGemError>(values.into_inner())
-                        })
-                        .collect::<Result<Vec<_>, _>>()?;
-                    Ok(ColumnRecord::Vec::<RandomState>(vec))
-                })
-            });
-
-            for result in iter {
-                let record = result?;
-                let _: Value = ruby.yield_value(record.try_into_value_with(&ruby)?)?;
-            }
-        }
-    }
-
-    Ok(ruby.qnil().into_value_with(&ruby))
-}
+            columns,
+            parser_type: ParserType::Column { batch_size, strict },
+            logger,
+        },
+    )
+}

data/ext/parquet/src/reader/parquet_row_reader.rs

@@ -1,22 +1,9 @@
-use crate::header_cache::StringCache;
-use crate::logger::RubyLogger;
-use crate::types::TryIntoValue;
-use crate::{
-    create_row_enumerator, utils::*, ParquetField, ParquetGemError, ParserResultType,
-    RowEnumeratorArgs, RowRecord,
-};
-use ahash::RandomState;
-use either::Either;
-use magnus::IntoValue;
+use crate::reader::unified::{parse_parquet_unified, ParserType, UnifiedParserArgs};
+use crate::utils::*;
+use crate::ParquetGemError;
+
 use magnus::{Error as MagnusError, Ruby, Value};
-use parquet::file::reader::{FileReader, SerializedFileReader};
-use parquet::record::reader::RowIter as ParquetRowIter;
-use parquet::schema::types::{Type as SchemaType, TypePtr};
-use std::collections::HashMap;
 use std::rc::Rc;
-use std::sync::OnceLock;
-
-use super::common::{handle_block_or_enum, open_parquet_source};
 
 #[inline]
 pub fn parse_parquet_rows(rb_self: Value, args: &[Value]) -> Result<Value, MagnusError> {
@@ -41,123 +28,16 @@ fn parse_parquet_rows_impl(
         logger,
     } = parse_parquet_rows_args(&ruby, args)?;
 
-    // Initialize the logger if provided
-    let ruby_logger = RubyLogger::new(&ruby, logger)?;
-
-    // Clone values for the closure to avoid move issues
-    let columns_clone = columns.clone();
-
-    // Handle block or create enumerator
-    if let Some(enum_value) = handle_block_or_enum(&ruby, ruby.block_given(), || {
-        create_row_enumerator(RowEnumeratorArgs {
-            rb_self,
+    // Use the unified parsing implementation
+    parse_parquet_unified(
+        ruby,
+        rb_self,
+        UnifiedParserArgs {
             to_read,
             result_type,
-            columns: columns_clone,
-            strict,
+            columns,
+            parser_type: ParserType::Row { strict },
             logger,
-        })
-        .map(|yield_enum| yield_enum.into_value_with(&ruby))
-    })? {
-        return Ok(enum_value);
-    }
-
-    let source = open_parquet_source(ruby.clone(), to_read)?;
-    let reader: Box<dyn FileReader> = match source {
-        Either::Left(file) => {
-            Box::new(SerializedFileReader::new(file).map_err(ParquetGemError::from)?)
-        }
-        Either::Right(readable) => {
-            Box::new(SerializedFileReader::new(readable).map_err(ParquetGemError::from)?)
-        }
-    };
-
-    let schema = reader.metadata().file_metadata().schema().clone();
-    ruby_logger.debug(|| format!("Schema loaded: {:?}", schema))?;
-
-    let mut iter = ParquetRowIter::from_file_into(reader);
-    if let Some(cols) = columns {
-        ruby_logger.debug(|| format!("Projecting columns: {:?}", cols))?;
-        let projection = create_projection_schema(&schema, &cols);
-        iter = iter.project(Some(projection.to_owned())).map_err(|e| {
-            MagnusError::new(
-                ruby.exception_runtime_error(),
-                format!("Failed to create projection: {}", e),
-            )
-        })?;
-    }
-
-    match result_type {
-        ParserResultType::Hash => {
-            let headers = OnceLock::new();
-            let headers_clone = headers.clone();
-            let iter = iter.map(move |row| {
-                row.map(|row| {
-                    let headers = headers_clone.get_or_init(|| {
-                        let column_count = row.get_column_iter().count();
-
-                        let mut header_string = Vec::with_capacity(column_count);
-                        for (k, _) in row.get_column_iter() {
-                            header_string.push(k.to_owned());
-                        }
-
-                        StringCache::intern_many(&header_string).expect("Failed to intern headers")
-                    });
-
-                    let mut map =
-                        HashMap::with_capacity_and_hasher(headers.len(), RandomState::default());
-                    for (i, (_, v)) in row.get_column_iter().enumerate() {
-                        map.insert(headers[i], ParquetField(v.clone(), strict));
-                    }
-                    map
-                })
-                .map(RowRecord::Map::<RandomState>)
-                .map_err(ParquetGemError::from)
-            });
-
-            for result in iter {
-                let record = result?;
-                let _: Value = ruby.yield_value(record.try_into_value_with(&ruby)?)?;
-            }
-        }
-        ParserResultType::Array => {
-            let iter = iter.map(|row| {
-                row.map(|row| {
-                    let column_count = row.get_column_iter().count();
-                    let mut vec = Vec::with_capacity(column_count);
-                    for (_, v) in row.get_column_iter() {
-                        vec.push(ParquetField(v.clone(), strict));
-                    }
-                    vec
-                })
-                .map(RowRecord::Vec::<RandomState>)
-                .map_err(ParquetGemError::from)
-            });
-
-            for result in iter {
-                let record = result?;
-                let _: Value = ruby.yield_value(record.try_into_value_with(&ruby)?)?;
-            }
-        }
-    }
-
-    Ok(ruby.qnil().into_value_with(&ruby))
-}
-
-fn create_projection_schema(schema: &SchemaType, columns: &[String]) -> SchemaType {
-    if let SchemaType::GroupType { fields, .. } = schema {
-        let projected_fields: Vec<TypePtr> = fields
-            .iter()
-            .filter(|field| columns.contains(&field.name().to_string()))
-            .cloned()
-            .collect();
-
-        SchemaType::GroupType {
-            basic_info: schema.get_basic_info().clone(),
-            fields: projected_fields,
-        }
-    } else {
-        // Return original schema if not a group type
-        schema.clone()
-    }
-}
+        },
+    )
+}