parquet 0.5.12 → 0.6.0

data/Cargo.toml

@@ -1,3 +1,3 @@
 [workspace]
-members = ["./ext/parquet"]
+members = ["./ext/parquet", "./ext/parquet-core", "./ext/parquet-ruby-adapter"]
 resolver = "2"

data/Gemfile

@@ -11,6 +11,7 @@ group :development do
   # gem "benchmark-ips", "~> 2.12"
   # gem "polars-df"
   # gem "duckdb"
+  gem "benchmark-memory"
 end
 
 group :test do

data/README.md

@@ -2,11 +2,11 @@
 
 [![Gem Version](https://badge.fury.io/rb/parquet.svg)](https://badge.fury.io/rb/parquet)
 
-This project is a Ruby library wrapping the [parquet-rs](https://github.com/apache/parquet-rs) rust crate.
+This project is a Ruby library wrapping the [`parquet`](https://github.com/apache/arrow-rs/tree/main/parquet) rust crate.
 
 ## Usage
 
-This library provides high-level bindings to parquet-rs with two primary APIs for reading Parquet files: row-wise and column-wise iteration. The column-wise API generally offers better performance, especially when working with subset of columns.
+This library provides high-level bindings to `parquet` with two primary APIs for reading Parquet files: row-wise and column-wise iteration. The column-wise API generally offers better performance, especially when working with subset of columns.
 
 ### Metadata
 
@@ -264,9 +264,100 @@ The following data types are supported in the schema:
 - `binary`
 - `boolean`
 - `date32`
-- `timestamp_millis`, `timestamp_micros`
+- `timestamp_millis`, `timestamp_micros`, `timestamp_second`, `timestamp_nanos`
 - `time_millis`, `time_micros`
 
+### Timestamp Timezone Handling
+
+**CRITICAL PARQUET SPECIFICATION LIMITATION**: The Apache Parquet format specification only supports two types of timestamps:
+1. **UTC-normalized timestamps** (when ANY timezone is specified) - `isAdjustedToUTC = true`
+2. **Local/unzoned timestamps** (when NO timezone is specified) - `isAdjustedToUTC = false`
+
+This means that specific timezone offsets like "+09:00" or "America/New_York" CANNOT be preserved in Parquet files. This is not a limitation of this Ruby library, but of the Parquet format itself.
+
+**When Writing:**
+- If the schema specifies ANY timezone (whether it's "UTC", "+09:00", "America/New_York", etc.):
+  - Time values are converted to UTC before storing
+  - The file metadata sets `isAdjustedToUTC = true`
+  - The original timezone information is LOST
+- If the schema doesn't specify a timezone:
+  - Timestamps are stored as local/unzoned time (no conversion)
+  - The file metadata sets `isAdjustedToUTC = false`
+  - These represent "wall clock" times without timezone context
+
+**When Reading:**
+- If the Parquet file has `isAdjustedToUTC = true` (ANY timezone was specified during writing):
+  - Time objects are returned in UTC
+  - The original timezone (e.g., "+09:00") is NOT recoverable
+- If the file has `isAdjustedToUTC = false` (NO timezone was specified):
+  - Time objects are returned as local time in your system's timezone
+  - These are "wall clock" times without timezone information
+
+```ruby
+# Preferred approach: use has_timezone to be explicit about UTC vs local storage
+schema = Parquet::Schema.define do
+  field :timestamp_utc, :timestamp_millis, has_timezone: true     # Stored as UTC (default)
+  field :timestamp_local, :timestamp_millis, has_timezone: false  # Stored as local/unzoned
+  field :timestamp_default, :timestamp_millis                     # Default: UTC storage
+end
+
+# Legacy approach still supported (any timezone value means UTC storage)
+schema_legacy = Parquet::Schema.define do
+  field :timestamp_utc, :timestamp_millis, timezone: "UTC"        # Stored as UTC
+  field :timestamp_tokyo, :timestamp_millis, timezone: "+09:00"  # Also stored as UTC!
+  field :timestamp_local, :timestamp_millis                       # No timezone - local
+end
+
+# Time values will be converted based on schema
+rows = [
+  [
+    Time.new(2024, 1, 1, 12, 0, 0, "+03:00"),  # Converted to UTC if has_timezone: true
+    Time.new(2024, 1, 1, 12, 0, 0, "-05:00"),  # Kept as local if has_timezone: false
+    Time.new(2024, 1, 1, 12, 0, 0)              # Kept as local (default)
+  ]
+]
+
+Parquet.write_rows(rows.each, schema: schema, write_to: "timestamps.parquet")
+
+# Reading back - timezone presence determines UTC vs local
+Parquet.each_row("timestamps.parquet") do |row|
+  # row["timestamp_utc"]     => Time object in UTC
+  # row["timestamp_local"]   => Time object in local timezone
+  # row["timestamp_default"] => Time object in local timezone
+end
+
+# If you need to preserve specific timezone information, store it separately:
+schema_with_tz = Parquet::Schema.define do
+  field :timestamp, :timestamp_millis, has_timezone: true  # Store as UTC
+  field :original_timezone, :string                        # Store timezone as string
+end
+```
+
+## Architecture
+
+This library uses a modular, trait-based architecture that separates language-agnostic Parquet operations from Ruby-specific bindings:
+
+- **parquet-core**: Language-agnostic core functionality for Parquet file operations
+  - Pure Rust implementation without Ruby dependencies
+  - Traits for customizable I/O operations (`ChunkReader`) and value conversion (`ValueConverter`)
+  - Efficient Arrow-based reader and writer implementations
+  
+- **parquet-ruby-adapter**: Ruby-specific adapter layer
+  - Implements core traits for Ruby integration
+  - Handles Ruby value conversion through the `ValueConverter` trait
+  - Manages Ruby I/O objects through the `ChunkReader` trait
+
+- **parquet gem**: Ruby FFI bindings
+  - Provides high-level Ruby API
+  - Manages memory safety between Ruby and Rust
+  - Supports both file-based and IO-based operations
+
+This architecture enables:
+- Clear separation of concerns between core functionality and language bindings
+- Easy testing of core logic without Ruby dependencies
+- Potential reuse of core functionality for other language bindings
+- Type-safe interfaces through Rust's trait system
+
 ### Schema DSL for Complex Data Types
 
 In addition to the hash-based schema definition shown above, this library provides a more expressive DSL for defining complex schemas with nested structures:

data/ext/parquet/Cargo.toml

@@ -11,16 +11,17 @@ rb-sys-env = "^0.2"
 
 [dependencies]
 ahash = "0.8"
-arrow-array = { git = "https://github.com/njaremko/arrow-rs", branch = "nathan/fix-time" }
-arrow-buffer = { git = "https://github.com/njaremko/arrow-rs", branch = "nathan/fix-time" }
-arrow-ipc = { git = "https://github.com/njaremko/arrow-rs", branch = "nathan/fix-time", features = ["lz4"] }
-arrow-schema = { git = "https://github.com/njaremko/arrow-rs", branch = "nathan/fix-time" }
+arrow-array = { git = "https://github.com/njaremko/arrow-rs", branch = "nathan_06-24-remove_primitive_map_key_assertion_on_record_reader" }
+arrow-buffer = { git = "https://github.com/njaremko/arrow-rs", branch = "nathan_06-24-remove_primitive_map_key_assertion_on_record_reader" }
+arrow-ipc = { git = "https://github.com/njaremko/arrow-rs", branch = "nathan_06-24-remove_primitive_map_key_assertion_on_record_reader", features = ["lz4"] }
+arrow-schema = { git = "https://github.com/njaremko/arrow-rs", branch = "nathan_06-24-remove_primitive_map_key_assertion_on_record_reader" }
 bytes = "^1.9"
 either = "1.9"
 itertools = "^0.14"
 jiff = "0.2"
 magnus = { version = "0.7", features = ["rb-sys"] }
-parquet = { git = "https://github.com/njaremko/arrow-rs", branch = "nathan/fix-time", features = ["json"] }
+parquet = { git = "https://github.com/njaremko/arrow-rs", branch = "nathan_06-24-remove_primitive_map_key_assertion_on_record_reader", features = ["json"] }
+parquet-ruby-adapter = { path = "../parquet-ruby-adapter" }
 rand = "0.9"
 rb-sys = "^0.9"
 simdutf8 = "0.1.5"
@@ -28,6 +29,8 @@ tempfile = "^3.15"
 thiserror = "2.0"
 num = "0.4.3"
 uuid = "1.16.0"
+ordered-float = "5.0.0"
+
 
 [target.'cfg(target_os = "linux")'.dependencies]
 jemallocator = { version = "0.5", features = ["disable_initial_exec_tls"] }

data/ext/parquet/src/adapter_ffi.rs

@@ -0,0 +1,156 @@
+use magnus::scan_args::{get_kwargs, scan_args};
+use magnus::value::ReprValue;
+use magnus::{Error as MagnusError, Ruby, Value};
+use parquet_ruby_adapter::{
+    logger::RubyLogger, types::ParserResultType, utils::parse_parquet_write_args,
+};
+pub fn each_row(rb_self: Value, args: &[Value]) -> Result<Value, MagnusError> {
+    let ruby = Ruby::get().map_err(|_| {
+        MagnusError::new(
+            magnus::exception::runtime_error(),
+            "Failed to get Ruby runtime",
+        )
+    })?;
+
+    // Parse arguments
+    let parsed_args = scan_args::<(Value,), (), (), (), _, ()>(args)?;
+    let (to_read,) = parsed_args.required;
+
+    // Parse keyword arguments
+    let kwargs = get_kwargs::<
+        _,
+        (),
+        (
+            Option<Option<Value>>,       // result_type
+            Option<Option<Vec<String>>>, // columns
+            Option<Option<bool>>,        // strict
+            Option<Option<Value>>,       // logger
+        ),
+        (),
+    >(
+        parsed_args.keywords,
+        &[],
+        &["result_type", "columns", "strict", "logger"],
+    )?;
+
+    let result_type: ParserResultType = if let Some(rt_value) = kwargs.optional.0.flatten() {
+        rt_value
+            .to_r_string()?
+            .to_string()?
+            .parse()
+            .map_err(|e| MagnusError::new(ruby.exception_arg_error(), e))?
+    } else {
+        ParserResultType::Hash
+    };
+    let columns = kwargs.optional.1.flatten();
+    let strict = kwargs.optional.2.flatten().unwrap_or(true);
+    let logger = RubyLogger::new(kwargs.optional.3.flatten())?;
+
+    // Delegate to parquet_ruby_adapter
+    parquet_ruby_adapter::reader::each_row(
+        &ruby,
+        rb_self,
+        to_read,
+        result_type,
+        columns,
+        strict,
+        logger,
+    )
+}
+
+pub fn each_column(rb_self: Value, args: &[Value]) -> Result<Value, MagnusError> {
+    let ruby = Ruby::get().map_err(|_| {
+        MagnusError::new(
+            magnus::exception::runtime_error(),
+            "Failed to get Ruby runtime",
+        )
+    })?;
+
+    // Parse arguments
+    let parsed_args = scan_args::<(Value,), (), (), (), _, ()>(args)?;
+    let (to_read,) = parsed_args.required;
+
+    // Parse keyword arguments
+    let kwargs = get_kwargs::<
+        _,
+        (),
+        (
+            Option<Option<Value>>,       // result_type
+            Option<Option<Vec<String>>>, // columns
+            Option<Option<usize>>,       // batch_size
+            Option<Option<bool>>,        // strict
+            Option<Option<Value>>,       // logger
+        ),
+        (),
+    >(
+        parsed_args.keywords,
+        &[],
+        &["result_type", "columns", "batch_size", "strict", "logger"],
+    )?;
+
+    let result_type: ParserResultType = if let Some(rt_value) = kwargs.optional.0.flatten() {
+        rt_value
+            .to_r_string()?
+            .to_string()?
+            .parse()
+            .map_err(|e| MagnusError::new(ruby.exception_arg_error(), e))?
+    } else {
+        ParserResultType::Hash
+    };
+    let columns = kwargs.optional.1.flatten();
+    let batch_size = if let Some(bs) = kwargs.optional.2.flatten() {
+        if bs == 0 {
+            return Err(MagnusError::new(
+                ruby.exception_arg_error(),
+                "batch_size must be greater than 0",
+            ));
+        }
+        Some(bs)
+    } else {
+        None
+    };
+    let strict = kwargs.optional.3.flatten().unwrap_or(true);
+    let logger = RubyLogger::new(kwargs.optional.4.flatten())?;
+
+    // Delegate to parquet_ruby_adapter
+    parquet_ruby_adapter::reader::each_column(
+        &ruby,
+        rb_self,
+        to_read,
+        result_type,
+        columns,
+        batch_size,
+        strict,
+        logger,
+    )
+}
+
+pub fn write_rows(args: &[Value]) -> Result<Value, MagnusError> {
+    let ruby = Ruby::get().map_err(|_| {
+        MagnusError::new(
+            magnus::exception::runtime_error(),
+            "Failed to get Ruby runtime",
+        )
+    })?;
+
+    // Parse arguments using the new parser
+    let write_args = parse_parquet_write_args(&ruby, args)?;
+
+    // Delegate to parquet_ruby_adapter
+    parquet_ruby_adapter::writer::write_rows(&ruby, write_args)
+}
+
+pub fn write_columns(args: &[Value]) -> Result<Value, MagnusError> {
+    let ruby = Ruby::get().map_err(|_| {
+        MagnusError::new(
+            magnus::exception::runtime_error(),
+            "Failed to get Ruby runtime",
+        )
+    })?;
+
+    // Parse arguments using the new parser
+    let write_args = parse_parquet_write_args(&ruby, args)?;
+
+    // Delegate to parquet_ruby_adapter
+    parquet_ruby_adapter::writer::write_columns(&ruby, write_args)
+}

data/ext/parquet/src/lib.rs

@@ -1,32 +1,24 @@
+mod adapter_ffi;
 mod allocator;
-mod enumerator;
-pub mod header_cache;
-mod logger;
-mod reader;
-mod ruby_reader;
-mod types;
-mod utils;
-mod writer;
 
-use crate::enumerator::*;
-use crate::reader::*;
-use crate::types::*;
+use magnus::{function, method, Error, Ruby};
 
-use magnus::{Error, Ruby};
-use writer::write_columns;
-use writer::write_rows;
+use crate::adapter_ffi::{each_column, each_row, write_columns, write_rows};
+use parquet_ruby_adapter::metadata::parse_metadata;
 
 /// Initializes the Ruby extension and defines methods.
 #[magnus::init]
 fn init(ruby: &Ruby) -> Result<(), Error> {
-    // Require 'time' for Time.parse method
     ruby.require("time")?;
-    
+    ruby.require("bigdecimal")?;
+
     let module = ruby.define_module("Parquet")?;
-    module.define_module_function("metadata", magnus::method!(reader::parse_metadata, -1))?;
-    module.define_module_function("each_row", magnus::method!(parse_parquet_rows, -1))?;
-    module.define_module_function("each_column", magnus::method!(parse_parquet_columns, -1))?;
-    module.define_module_function("write_rows", magnus::function!(write_rows, -1))?;
-    module.define_module_function("write_columns", magnus::function!(write_columns, -1))?;
+
+    module.define_module_function("metadata", function!(parse_metadata, 1))?;
+    module.define_module_function("each_row", method!(each_row, -1))?;
+    module.define_module_function("each_column", method!(each_column, -1))?;
+    module.define_module_function("write_rows", function!(write_rows, -1))?;
+    module.define_module_function("write_columns", function!(write_columns, -1))?;
+
     Ok(())
 }

data/ext/parquet-core/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "parquet-core"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+arrow = { git = "https://github.com/njaremko/arrow-rs", branch = "nathan_06-24-remove_primitive_map_key_assertion_on_record_reader" }
+arrow-array = { git = "https://github.com/njaremko/arrow-rs", branch = "nathan_06-24-remove_primitive_map_key_assertion_on_record_reader" }
+arrow-buffer = { git = "https://github.com/njaremko/arrow-rs", branch = "nathan_06-24-remove_primitive_map_key_assertion_on_record_reader" }
+arrow-schema = { git = "https://github.com/njaremko/arrow-rs", branch = "nathan_06-24-remove_primitive_map_key_assertion_on_record_reader" }
+bytes = "1.5"
+indexmap = "2.2"
+jiff = "0.2"
+num = "0.4.3"
+ordered-float = "5.0.0"
+parquet = { git = "https://github.com/njaremko/arrow-rs", branch = "nathan_06-24-remove_primitive_map_key_assertion_on_record_reader", features = ["arrow", "zstd", "lz4", "snap"] }
+rand = "0.9.1"
+serde = { version = "1.0", features = ["derive"] }
+thiserror = "2.0"
+
+[dev-dependencies]
+uuid = { version = "1.0", features = ["v4"] }
+tempfile = "3.8"