parquet 0.5.13-aarch64-linux-musl → 0.6.0-aarch64-linux-musl

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1ebeb87a44c0fbe4110a9d3157a6d1fc5317ece0928bcf991e3a77bb9732bd61
-  data.tar.gz: dd31a679579d4034fe69e3164150116b456a63b78adb46e58d50a0685a1a66c3
+  metadata.gz: ef860225882188543417167eb81cf2d330d1a13e04c4d0bc139775437bacea2b
+  data.tar.gz: 7040158d3e2541e0d807b082d3fd3f84db7188bc5692c1271e27af5a2cf79c94
 SHA512:
-  metadata.gz: bce130c43936c9dd571d85f1450193d5b610488bda9541a04e71346f73cc9305c11bda2fad03203776d377ebff95e459dc805bdbe6d44ed9df91908b8ad47355
-  data.tar.gz: f70824805cfa8423d93115b826fc8c26aa4e5e895042c245f834ecf025e66c896e1b8824de3f7d5f38d07c38776ab974217b0b8b3775ce639caec3871f86fa9e
+  metadata.gz: 1c8f29b9753dcb9d1eb0ccb36974337108237c8a18b33e835476f87071de99777792c74da683adeed7e04818737c62c8df98aea1bea64b39355967bcc0cfb688
+  data.tar.gz: 2ffdcb881ec52f38b0ea104baae9292c1c93a03282dce46fd45e6d2075f8b257ed13e783f5b8b227629b753b0e3929504aff0b4369685f32452262078a692814

data/Gemfile

@@ -11,6 +11,7 @@ group :development do
   # gem "benchmark-ips", "~> 2.12"
   # gem "polars-df"
   # gem "duckdb"
+  gem "benchmark-memory"
 end
 
 group :test do

data/README.md

@@ -2,11 +2,11 @@
 
 [![Gem Version](https://badge.fury.io/rb/parquet.svg)](https://badge.fury.io/rb/parquet)
 
-This project is a Ruby library wrapping the [parquet-rs](https://github.com/apache/parquet-rs) rust crate.
+This project is a Ruby library wrapping the [`parquet`](https://github.com/apache/arrow-rs/tree/main/parquet) rust crate.
 
 ## Usage
 
-This library provides high-level bindings to parquet-rs with two primary APIs for reading Parquet files: row-wise and column-wise iteration. The column-wise API generally offers better performance, especially when working with subset of columns.
+This library provides high-level bindings to `parquet` with two primary APIs for reading Parquet files: row-wise and column-wise iteration. The column-wise API generally offers better performance, especially when working with subset of columns.
 
 ### Metadata
 
@@ -264,9 +264,100 @@ The following data types are supported in the schema:
 - `binary`
 - `boolean`
 - `date32`
-- `timestamp_millis`, `timestamp_micros`
+- `timestamp_millis`, `timestamp_micros`, `timestamp_second`, `timestamp_nanos`
 - `time_millis`, `time_micros`
 
+### Timestamp Timezone Handling
+
+**CRITICAL PARQUET SPECIFICATION LIMITATION**: The Apache Parquet format specification only supports two types of timestamps:
+1. **UTC-normalized timestamps** (when ANY timezone is specified) - `isAdjustedToUTC = true`
+2. **Local/unzoned timestamps** (when NO timezone is specified) - `isAdjustedToUTC = false`
+
+This means that specific timezone offsets like "+09:00" or "America/New_York" CANNOT be preserved in Parquet files. This is not a limitation of this Ruby library, but of the Parquet format itself.
+
+**When Writing:**
+- If the schema specifies ANY timezone (whether it's "UTC", "+09:00", "America/New_York", etc.):
+  - Time values are converted to UTC before storing
+  - The file metadata sets `isAdjustedToUTC = true`
+  - The original timezone information is LOST
+- If the schema doesn't specify a timezone:
+  - Timestamps are stored as local/unzoned time (no conversion)
+  - The file metadata sets `isAdjustedToUTC = false`
+  - These represent "wall clock" times without timezone context
+
+**When Reading:**
+- If the Parquet file has `isAdjustedToUTC = true` (ANY timezone was specified during writing):
+  - Time objects are returned in UTC
+  - The original timezone (e.g., "+09:00") is NOT recoverable
+- If the file has `isAdjustedToUTC = false` (NO timezone was specified):
+  - Time objects are returned as local time in your system's timezone
+  - These are "wall clock" times without timezone information
+
+```ruby
+# Preferred approach: use has_timezone to be explicit about UTC vs local storage
+schema = Parquet::Schema.define do
+  field :timestamp_utc, :timestamp_millis, has_timezone: true     # Stored as UTC (default)
+  field :timestamp_local, :timestamp_millis, has_timezone: false  # Stored as local/unzoned
+  field :timestamp_default, :timestamp_millis                     # Default: UTC storage
+end
+
+# Legacy approach still supported (any timezone value means UTC storage)
+schema_legacy = Parquet::Schema.define do
+  field :timestamp_utc, :timestamp_millis, timezone: "UTC"        # Stored as UTC
+  field :timestamp_tokyo, :timestamp_millis, timezone: "+09:00"  # Also stored as UTC!
+  field :timestamp_local, :timestamp_millis                       # No timezone - local
+end
+
+# Time values will be converted based on schema
+rows = [
+  [
+    Time.new(2024, 1, 1, 12, 0, 0, "+03:00"),  # Converted to UTC if has_timezone: true
+    Time.new(2024, 1, 1, 12, 0, 0, "-05:00"),  # Kept as local if has_timezone: false
+    Time.new(2024, 1, 1, 12, 0, 0)              # Kept as local (default)
+  ]
+]
+
+Parquet.write_rows(rows.each, schema: schema, write_to: "timestamps.parquet")
+
+# Reading back - timezone presence determines UTC vs local
+Parquet.each_row("timestamps.parquet") do |row|
+  # row["timestamp_utc"]     => Time object in UTC
+  # row["timestamp_local"]   => Time object in local timezone
+  # row["timestamp_default"] => Time object in local timezone
+end
+
+# If you need to preserve specific timezone information, store it separately:
+schema_with_tz = Parquet::Schema.define do
+  field :timestamp, :timestamp_millis, has_timezone: true  # Store as UTC
+  field :original_timezone, :string                        # Store timezone as string
+end
+```
+
+## Architecture
+
+This library uses a modular, trait-based architecture that separates language-agnostic Parquet operations from Ruby-specific bindings:
+
+- **parquet-core**: Language-agnostic core functionality for Parquet file operations
+  - Pure Rust implementation without Ruby dependencies
+  - Traits for customizable I/O operations (`ChunkReader`) and value conversion (`ValueConverter`)
+  - Efficient Arrow-based reader and writer implementations
+  
+- **parquet-ruby-adapter**: Ruby-specific adapter layer
+  - Implements core traits for Ruby integration
+  - Handles Ruby value conversion through the `ValueConverter` trait
+  - Manages Ruby I/O objects through the `ChunkReader` trait
+
+- **parquet gem**: Ruby FFI bindings
+  - Provides high-level Ruby API
+  - Manages memory safety between Ruby and Rust
+  - Supports both file-based and IO-based operations
+
+This architecture enables:
+- Clear separation of concerns between core functionality and language bindings
+- Easy testing of core logic without Ruby dependencies
+- Potential reuse of core functionality for other language bindings
+- Type-safe interfaces through Rust's trait system
+
 ### Schema DSL for Complex Data Types
 
 In addition to the hash-based schema definition shown above, this library provides a more expressive DSL for defining complex schemas with nested structures:

data/lib/parquet/schema.rb

@@ -59,12 +59,31 @@ module Parquet
       #   - `key_nullable:, value_nullable:` controls nullability of map keys/values (default: true)
       #   - `format:` if you want to store some format string
       #   - `precision:, scale:` if type == :decimal (precision defaults to 38, scale to 0)
+      #   - `has_timezone:` if type is timestamp - true means UTC storage (default), false means local/unzoned
+      #   - `timezone:` (DEPRECATED) if type is timestamp - any value means UTC storage
       #   - `nullable:` default to true if not specified
       def field(name, type, nullable: true, **kwargs, &block)
         field_hash = { name: name.to_s, type: type, nullable: !!nullable }
 
         # Possibly store a format if provided
         field_hash[:format] = kwargs[:format] if kwargs.key?(:format)
+        
+        # Handle timezone for timestamp types
+        if [:timestamp_second, :timestamp_millis, :timestamp_micros, :timestamp_nanos].include?(type)
+          # Support new has_timezone parameter (preferred)
+          if kwargs.key?(:has_timezone)
+            # If has_timezone is true, store "UTC" to indicate timezone presence
+            # If explicitly false, don't store timezone (indicates local/unzoned)
+            field_hash[:timezone] = "UTC" if kwargs[:has_timezone]
+          elsif kwargs.key?(:timezone)
+            # Legacy support: any timezone value means UTC storage
+            # Store "UTC" regardless of the actual value to make it clear
+            field_hash[:timezone] = "UTC"
+          else
+            # Default behavior when neither parameter is specified: UTC storage
+            field_hash[:timezone] = "UTC"
+          end
+        end
 
         case type
         when :struct

data/lib/parquet/version.rb

@@ -1,3 +1,3 @@
 module Parquet
-  VERSION = "0.5.13"
+  VERSION = "0.6.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parquet
 version: !ruby/object:Gem::Version
-  version: 0.5.13
+  version: 0.6.0
 platform: aarch64-linux-musl
 authors:
 - Nathan Jaremko
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2025-06-24 00:00:00.000000000 Z
+date: 2025-07-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake-compiler