parquet 0.5.1 → 0.5.3

data/ext/parquet/src/writer/write_columns.rs

@@ -11,12 +11,12 @@ use crate::{
 use crate::{types::PrimitiveType, SchemaNode};
 use arrow_array::{Array, RecordBatch};
 use magnus::{value::ReprValue, Error as MagnusError, RArray, Ruby, Value};
-use std::sync::Arc;
+use std::{rc::Rc, sync::Arc};
 
 #[inline]
 pub fn write_columns(args: &[Value]) -> Result<(), MagnusError> {
     let ruby = unsafe { Ruby::get_unchecked() };
-    write_columns_impl(Arc::new(ruby), args).map_err(|e| {
+    write_columns_impl(Rc::new(ruby), args).map_err(|e| {
         let z: MagnusError = e.into();
         z
     })?;
@@ -24,7 +24,7 @@ pub fn write_columns(args: &[Value]) -> Result<(), MagnusError> {
 }
 
 #[inline]
-fn write_columns_impl(ruby: Arc<Ruby>, args: &[Value]) -> Result<(), ParquetGemError> {
+fn write_columns_impl(ruby: Rc<Ruby>, args: &[Value]) -> Result<(), ParquetGemError> {
     let ParquetWriteArgs {
         read_from,
         write_to,
@@ -94,7 +94,7 @@ fn write_columns_impl(ruby: Arc<Ruby>, args: &[Value]) -> Result<(), ParquetGemE
                     };
 
                     if batch_array.len() != schema_len {
-                        return Err(MagnusError::new(
+                        Err(MagnusError::new(
                             magnus::exception::type_error(),
                             format!(
                                 "Batch column count ({}) does not match schema length ({}). Schema expects columns: {:?}",
@@ -118,7 +118,7 @@ fn write_columns_impl(ruby: Arc<Ruby>, args: &[Value]) -> Result<(), ParquetGemE
                                 ))?,
                             };
                         if top_fields.len() != fields.len() {
-                            return Err(MagnusError::new(
+                            Err(MagnusError::new(
                                 magnus::exception::runtime_error(),
                                 "Mismatch top-level DSL fields vs Arrow fields",
                             ))?;
@@ -140,31 +140,34 @@ fn write_columns_impl(ruby: Arc<Ruby>, args: &[Value]) -> Result<(), ParquetGemE
                                     parquet_type,
                                     // Format is handled internally now
                                     ..
-                                } => match parquet_type {
-                                    &PrimitiveType::Int8 => PST::Primitive(PrimitiveType::Int8),
-                                    &PrimitiveType::Int16 => PST::Primitive(PrimitiveType::Int16),
-                                    &PrimitiveType::Int32 => PST::Primitive(PrimitiveType::Int32),
-                                    &PrimitiveType::Int64 => PST::Primitive(PrimitiveType::Int64),
-                                    &PrimitiveType::UInt8 => PST::Primitive(PrimitiveType::UInt8),
-                                    &PrimitiveType::UInt16 => PST::Primitive(PrimitiveType::UInt16),
-                                    &PrimitiveType::UInt32 => PST::Primitive(PrimitiveType::UInt32),
-                                    &PrimitiveType::UInt64 => PST::Primitive(PrimitiveType::UInt64),
-                                    &PrimitiveType::Float32 => {
+                                } => match *parquet_type {
+                                    PrimitiveType::Int8 => PST::Primitive(PrimitiveType::Int8),
+                                    PrimitiveType::Int16 => PST::Primitive(PrimitiveType::Int16),
+                                    PrimitiveType::Int32 => PST::Primitive(PrimitiveType::Int32),
+                                    PrimitiveType::Int64 => PST::Primitive(PrimitiveType::Int64),
+                                    PrimitiveType::UInt8 => PST::Primitive(PrimitiveType::UInt8),
+                                    PrimitiveType::UInt16 => PST::Primitive(PrimitiveType::UInt16),
+                                    PrimitiveType::UInt32 => PST::Primitive(PrimitiveType::UInt32),
+                                    PrimitiveType::UInt64 => PST::Primitive(PrimitiveType::UInt64),
+                                    PrimitiveType::Float32 => {
                                         PST::Primitive(PrimitiveType::Float32)
                                     }
-                                    &PrimitiveType::Float64 => {
+                                    PrimitiveType::Float64 => {
                                         PST::Primitive(PrimitiveType::Float64)
                                     }
-                                    &PrimitiveType::String => PST::Primitive(PrimitiveType::String),
-                                    &PrimitiveType::Binary => PST::Primitive(PrimitiveType::Binary),
-                                    &PrimitiveType::Boolean => {
+                                    PrimitiveType::Decimal128(precision, scale) => {
+                                        PST::Primitive(PrimitiveType::Decimal128(precision, scale))
+                                    }
+                                    PrimitiveType::String => PST::Primitive(PrimitiveType::String),
+                                    PrimitiveType::Binary => PST::Primitive(PrimitiveType::Binary),
+                                    PrimitiveType::Boolean => {
                                         PST::Primitive(PrimitiveType::Boolean)
                                     }
-                                    &PrimitiveType::Date32 => PST::Primitive(PrimitiveType::Date32),
-                                    &PrimitiveType::TimestampMillis => {
+                                    PrimitiveType::Date32 => PST::Primitive(PrimitiveType::Date32),
+                                    PrimitiveType::TimestampMillis => {
                                         PST::Primitive(PrimitiveType::TimestampMillis)
                                     }
-                                    &PrimitiveType::TimestampMicros => {
+                                    PrimitiveType::TimestampMicros => {
                                         PST::Primitive(PrimitiveType::TimestampMicros)
                                     }
                                 },
@@ -205,12 +208,12 @@ fn write_columns_impl(ruby: Arc<Ruby>, args: &[Value]) -> Result<(), ParquetGemE
                     if e.is_kind_of(ruby.exception_stop_iteration()) {
                         break;
                     }
-                    return Err(e)?;
+                    Err(e)?;
                 }
             }
         }
     } else {
-        return Err(MagnusError::new(
+        Err(MagnusError::new(
             magnus::exception::type_error(),
             "read_from must be an Enumerator".to_string(),
         ))?;

data/ext/parquet/src/writer/write_rows.rs

@@ -1,7 +1,7 @@
 use super::{
     build_column_collectors_from_dsl, copy_temp_file_to_io_like, create_writer,
     parse_parquet_write_args, DEFAULT_MEMORY_THRESHOLD, INITIAL_BATCH_SIZE, MIN_BATCH_SIZE,
-    MIN_SAMPLES_FOR_ESTIMATE, SAMPLE_SIZE,
+    SAMPLE_SIZE,
 };
 use crate::{
     logger::RubyLogger,
@@ -16,12 +16,14 @@ use magnus::{
     value::ReprValue, Error as MagnusError, RArray, RHash, Ruby, Symbol, TryConvert, Value,
 };
 use rand::Rng;
-use std::sync::Arc;
+use std::{rc::Rc, sync::Arc};
+
+const MIN_SAMPLES_FOR_ESTIMATE: usize = 10;
 
 #[inline]
 pub fn write_rows(args: &[Value]) -> Result<(), MagnusError> {
     let ruby = unsafe { Ruby::get_unchecked() };
-    write_rows_impl(Arc::new(ruby), args).map_err(|e| {
+    write_rows_impl(Rc::new(ruby), args).map_err(|e| {
         let z: MagnusError = e.into();
         z
     })?;
@@ -29,7 +31,7 @@ pub fn write_rows(args: &[Value]) -> Result<(), MagnusError> {
 }
 
 #[inline]
-fn write_rows_impl(ruby: Arc<Ruby>, args: &[Value]) -> Result<(), ParquetGemError> {
+fn write_rows_impl(ruby: Rc<Ruby>, args: &[Value]) -> Result<(), ParquetGemError> {
     let ParquetWriteArgs {
         read_from,
         write_to,
@@ -81,8 +83,8 @@ fn write_rows_impl(ruby: Arc<Ruby>, args: &[Value]) -> Result<(), ParquetGemErro
                         })?;
                         let row_size = estimate_single_row_size(&row_array, &column_collectors)?;
                         size_samples.push(row_size);
-                    } else if rng.random_range(0..=total_rows) < sample_size as usize {
-                        let idx = rng.random_range(0..sample_size as usize);
+                    } else if rng.random_range(0..=total_rows) < sample_size {
+                        let idx = rng.random_range(0..sample_size);
                         let row_array = RArray::from_value(row).ok_or_else(|| {
                             MagnusError::new(ruby.exception_type_error(), "Row must be an array")
                         })?;
@@ -113,12 +115,12 @@ fn write_rows_impl(ruby: Arc<Ruby>, args: &[Value]) -> Result<(), ParquetGemErro
                         }
                         break;
                     }
-                    return Err(e)?;
+                    Err(e)?;
                 }
             }
         }
     } else {
-        return Err(MagnusError::new(
+        Err(MagnusError::new(
             magnus::exception::type_error(),
             "read_from must be an Enumerator".to_string(),
         ))?;
@@ -255,6 +257,7 @@ pub fn estimate_value_size(
         | PST::Primitive(PrimitiveType::UInt64)
         | PST::Primitive(PrimitiveType::Float64) => Ok(8),
         PST::Primitive(PrimitiveType::Boolean) => Ok(1),
+        PST::Primitive(PrimitiveType::Decimal128(_, _)) => Ok(16),
         PST::Primitive(PrimitiveType::Date32)
         | PST::Primitive(PrimitiveType::TimestampMillis)
         | PST::Primitive(PrimitiveType::TimestampMicros) => Ok(8),
@@ -427,15 +430,13 @@ pub fn estimate_value_size(
                             if let Some(field_value) = hash.get(&*field.name) {
                                 total_fields_size +=
                                     estimate_value_size(field_value, &field.type_)?;
+                            } else if field.nullable {
+                                total_fields_size += 0;
                             } else {
-                                if field.nullable {
-                                    total_fields_size += 0;
-                                } else {
-                                    return Err(MagnusError::new(
-                                        magnus::exception::runtime_error(),
-                                        format!("Missing field: {} in hash {:?}", field.name, hash),
-                                    ));
-                                }
+                                return Err(MagnusError::new(
+                                    magnus::exception::runtime_error(),
+                                    format!("Missing field: {} in hash {:?}", field.name, hash),
+                                ));
                             }
                         }
                     }

data/lib/parquet/schema.rb

@@ -11,6 +11,9 @@ module Parquet
     #     field :id, :int64, nullable: false  # ID cannot be null
     #     field :name, :string  # Default nullable: true
     #
+    #     # Decimal field with precision and scale
+    #     field :price, :decimal, precision: 10, scale: 2
+    #
     #     # List with non-nullable items
     #     field :scores, :list, item: :float, item_nullable: false
     #
@@ -45,7 +48,7 @@ module Parquet
 
       # Define a field in the schema
       # @param name [String, Symbol] field name
-      # @param type [Symbol] data type (:int32, :int64, :string, :list, :map, :struct, etc)
+      # @param type [Symbol] data type (:int32, :int64, :string, :list, :map, :struct, :decimal, etc)
       # @param nullable [Boolean] whether the field can be null (default: true)
       # @param kwargs [Hash] additional options depending on type
       #
@@ -55,6 +58,7 @@ module Parquet
       #   - `key:, value:` if type == :map
       #   - `key_nullable:, value_nullable:` controls nullability of map keys/values (default: true)
       #   - `format:` if you want to store some format string
+      #   - `precision:, scale:` if type == :decimal (precision defaults to 18, scale to 2)
       #   - `nullable:` default to true if not specified
       def field(name, type, nullable: true, **kwargs, &block)
         field_hash = { name: name.to_s, type: type, nullable: !!nullable }
@@ -73,7 +77,15 @@ module Parquet
           raise ArgumentError, "list field `#{name}` requires `item:` type" unless item_type
           # Pass item_nullable if provided, otherwise use true as default
           item_nullable = kwargs[:item_nullable].nil? ? true : !!kwargs[:item_nullable]
-          field_hash[:item] = wrap_subtype(item_type, nullable: item_nullable, &block)
+          
+          # Pass precision and scale if type is decimal
+          if item_type == :decimal
+            precision = kwargs[:precision]
+            scale = kwargs[:scale]
+            field_hash[:item] = wrap_subtype(item_type, nullable: item_nullable, precision: precision, scale: scale, &block)
+          else
+            field_hash[:item] = wrap_subtype(item_type, nullable: item_nullable, &block)
+          end
         when :map
           # user must specify key:, value:
           key_type = kwargs[:key]
@@ -82,8 +94,41 @@ module Parquet
           # Pass key_nullable and value_nullable if provided, otherwise use true as default
           key_nullable = kwargs[:key_nullable].nil? ? true : !!kwargs[:key_nullable]
           value_nullable = kwargs[:value_nullable].nil? ? true : !!kwargs[:value_nullable]
+          
           field_hash[:key] = wrap_subtype(key_type, nullable: key_nullable)
-          field_hash[:value] = wrap_subtype(value_type, nullable: value_nullable, &block)
+          
+          # Pass precision and scale if value type is decimal
+          if value_type == :decimal
+            precision = kwargs[:precision]
+            scale = kwargs[:scale]
+            field_hash[:value] = wrap_subtype(value_type, nullable: value_nullable, precision: precision, scale: scale, &block)
+          else
+            field_hash[:value] = wrap_subtype(value_type, nullable: value_nullable, &block)
+          end
+        when :decimal
+          # Store precision and scale for decimal type according to rules:
+          # 1. When neither precision nor scale is provided, use maximum precision (38)
+          # 2. When only precision is provided, scale defaults to 0
+          # 3. When only scale is provided, use maximum precision (38)
+          # 4. When both are provided, use the provided values
+          
+          if kwargs[:precision].nil? && kwargs[:scale].nil?
+            # No precision or scale provided - use maximum precision
+            field_hash[:precision] = 38
+            field_hash[:scale] = 0
+          elsif kwargs[:precision] && kwargs[:scale].nil?
+            # Precision only - scale defaults to 0
+            field_hash[:precision] = kwargs[:precision]
+            field_hash[:scale] = 0
+          elsif kwargs[:precision].nil? && kwargs[:scale]
+            # Scale only - use maximum precision
+            field_hash[:precision] = 38
+            field_hash[:scale] = kwargs[:scale]
+          else
+            # Both provided
+            field_hash[:precision] = kwargs[:precision]
+            field_hash[:scale] = kwargs[:scale]
+          end
         else
           # primitive type: :int32, :int64, :string, etc.
           # do nothing else special
@@ -122,7 +167,7 @@ module Parquet
       # If user said: field "something", :list, item: :struct do ... end
       # we want to recursively parse that sub-struct from the block.
       # So wrap_subtype might be:
-      def wrap_subtype(t, nullable: true, &block)
+      def wrap_subtype(t, nullable: true, precision: nil, scale: nil, &block)
         if t == :struct
           sub_builder = SchemaBuilder.new
           sub_builder.instance_eval(&block) if block
@@ -144,6 +189,34 @@ module Parquet
           end
 
           { type: :list, nullable: nullable, name: "item", item: sub_builder.fields[0] }
+        elsif t == :decimal
+          # Handle decimal type with precision and scale
+          result = { type: t, nullable: nullable, name: "item" }
+          
+          # Follow the same rules as in field() method:
+          # 1. When neither precision nor scale is provided, use maximum precision (38)
+          # 2. When only precision is provided, scale defaults to 0
+          # 3. When only scale is provided, use maximum precision (38)
+          # 4. When both are provided, use the provided values
+          if precision.nil? && scale.nil?
+            # No precision or scale provided - use maximum precision
+            result[:precision] = 38
+            result[:scale] = 0
+          elsif precision && scale.nil?
+            # Precision only - scale defaults to 0
+            result[:precision] = precision
+            result[:scale] = 0
+          elsif precision.nil? && scale
+            # Scale only - use maximum precision
+            result[:precision] = 38
+            result[:scale] = scale
+          else
+            # Both provided
+            result[:precision] = precision
+            result[:scale] = scale
+          end
+          
+          result
         else
           # e.g. :int32 => { type: :int32, nullable: true }
           { type: t, nullable: nullable, name: "item" }

data/lib/parquet/version.rb

@@ -1,3 +1,3 @@
 module Parquet
-  VERSION = "0.5.1"
+  VERSION = "0.5.3"
 end

data/lib/parquet.rbi

@@ -1,6 +1,17 @@
 # typed: true
 
 module Parquet
+  # Returns metadata information about a Parquet file
+  #
+  # The returned hash contains information about:
+  # - Basic file metadata (num_rows, created_by)
+  # - Schema information (fields, types, etc.)
+  # - Row group details
+  # - Column chunk information (compression, encodings, statistics)
+  sig { params(path: String).returns(T::Hash[String, T.untyped]) }
+  def self.metadata(path)
+  end
+
   # Options:
   #   - `input`: String, File, or IO object containing parquet data
   #   - `result_type`: String specifying the output format

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parquet
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.3
 platform: ruby
 authors:
 - Nathan Jaremko
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-02-27 00:00:00.000000000 Z
+date: 2025-04-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -55,6 +55,7 @@ files:
 - README.md
 - Rakefile
 - ext/parquet/Cargo.toml
+- ext/parquet/build.rs
 - ext/parquet/extconf.rb
 - ext/parquet/src/allocator.rs
 - ext/parquet/src/enumerator.rs