parquet-tyfoom 0.8.0

data/Cargo.toml

@@ -0,0 +1,3 @@
+[workspace]
+members = ["./ext/parquet", "./ext/parquet-core", "./ext/parquet-ruby-adapter"]
+resolver = "2"

data/Gemfile

@@ -0,0 +1,21 @@
+source "https://rubygems.org"
+
+gem "rb_sys", "~> 0.9.56"
+gem "rake"
+gem "bigdecimal"
+
+# Use local version of parquet
+gemspec
+
+group :development do
+  # gem "benchmark-ips", "~> 2.12"
+  # gem "polars-df"
+  # gem "duckdb"
+  gem "benchmark-memory"
+end
+
+group :test do
+  gem "csv"
+  gem "logger"
+  gem "minitest", "~> 5.0"
+end

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Nathan Jaremko
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,428 @@
+# parquet-ruby
+
+[![Gem Version](https://badge.fury.io/rb/parquet.svg)](https://badge.fury.io/rb/parquet)
+
+Read and write [Apache Parquet](https://parquet.apache.org/) files from Ruby. This gem wraps the official Apache [`parquet`](https://github.com/apache/arrow-rs/tree/main/parquet) rust crate, providing:
+
+- **High performance** columnar data storage and retrieval
+- **Memory-efficient** streaming APIs for large datasets
+- **Full compatibility** with the Apache Parquet specification
+- **Simple, Ruby-native** APIs that feel natural
+
+## Why Use This Library?
+
+Apache Parquet is the de facto standard for analytical data storage, offering:
+- **Efficient compression** - typically 2-10x smaller than CSV
+- **Fast columnar access** - read only the columns you need
+- **Rich type system** - preserves data types, including nested structures
+- **Wide ecosystem support** - works with Spark, Pandas, DuckDB, and more
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'parquet'
+```
+
+Then execute:
+
+```bash
+$ bundle install
+```
+
+Or install it directly:
+
+```bash
+$ gem install parquet
+```
+
+## Quick Start
+
+### Reading Data
+
+```ruby
+require "parquet"
+
+# Read Parquet files row by row
+Parquet.each_row("data.parquet") do |row|
+  puts row  # => {"id" => 1, "name" => "Alice", "score" => 95.5}
+end
+
+# Or column by column for better performance
+Parquet.each_column("data.parquet", batch_size: 1000) do |batch|
+  puts batch  # => {"id" => [1, 2, ...], "name" => ["Alice", "Bob", ...]}
+end
+```
+
+### Writing Data
+
+```ruby
+# Define your schema
+schema = [
+  { "id" => "int64" },
+  { "name" => "string" },
+  { "score" => "double" }
+]
+
+# Write row by row
+rows = [
+  [1, "Alice", 95.5],
+  [2, "Bob", 82.3]
+]
+
+Parquet.write_rows(rows.each, schema: schema, write_to: "output.parquet")
+```
+
+## Reading Parquet Files
+
+The library provides two APIs for reading data, each optimized for different use cases:
+
+### Row-wise Reading (Sequential Access)
+
+Best for: Processing records one at a time, data transformations, ETL pipelines
+
+```ruby
+# Basic usage - returns hashes
+Parquet.each_row("data.parquet") do |row|
+  puts row  # => {"id" => 1, "name" => "Alice"}
+end
+
+# Memory-efficient array format
+Parquet.each_row("data.parquet", result_type: :array) do |row|
+  puts row  # => [1, "Alice"]
+end
+
+# Read specific columns only
+Parquet.each_row("data.parquet", columns: ["id", "name"]) do |row|
+  # Only requested columns are loaded from disk
+end
+
+# Works with IO objects
+File.open("data.parquet", "rb") do |file|
+  Parquet.each_row(file) do |row|
+    # Process row
+  end
+end
+```
+
+### Column-wise Reading (Analytical Access)
+
+Best for: Analytics, aggregations, when you need few columns from wide tables
+
+```ruby
+# Process data in column batches
+Parquet.each_column("data.parquet", batch_size: 1000) do |batch|
+  # batch is a hash of column_name => array_of_values
+  ids = batch["id"]      # => [1, 2, 3, ..., 1000]
+  names = batch["name"]  # => ["Alice", "Bob", ...]
+
+  # Perform columnar operations
+  avg_id = ids.sum.to_f / ids.length
+end
+
+# Array format for more control
+Parquet.each_column("data.parquet",
+                    result_type: :array,
+                    columns: ["id", "name"]) do |batch|
+  # batch is an array of arrays
+  # [[1, 2, ...], ["Alice", "Bob", ...]]
+end
+```
+
+### File Metadata
+
+Inspect file structure without reading data:
+
+```ruby
+metadata = Parquet.metadata("data.parquet")
+
+puts metadata["num_rows"]           # Total row count
+puts metadata["created_by"]         # Writer identification
+puts metadata["schema"]["fields"]   # Column definitions
+puts metadata["row_groups"].size    # Number of row groups
+```
+
+## Writing Parquet Files
+
+### Row-wise Writing
+
+Best for: Streaming data, converting from other formats, memory-constrained environments
+
+```ruby
+# Basic schema definition
+schema = [
+  { "id" => "int64" },
+  { "name" => "string" },
+  { "active" => "boolean" },
+  { "balance" => "double" }
+]
+
+# Stream data from any enumerable
+rows = CSV.foreach("input.csv").map do |row|
+  [row[0].to_i, row[1], row[2] == "true", row[3].to_f]
+end
+
+Parquet.write_rows(rows,
+  schema: schema,
+  write_to: "output.parquet",
+  batch_size: 5000  # Positive rows per batch (default: 1000)
+)
+```
+
+### Column-wise Writing
+
+Best for: Pre-columnar data, better compression, higher performance
+
+```ruby
+# Prepare columnar data
+ids = [1, 2, 3, 4, 5]
+names = ["Alice", "Bob", "Charlie", "Diana", "Eve"]
+scores = [95.5, 82.3, 88.7, 91.2, 79.8]
+
+# Create batches
+batches = [[
+  ids,    # First column
+  names,  # Second column
+  scores  # Third column
+]]
+
+schema = [
+  { "id" => "int64" },
+  { "name" => "string" },
+  { "score" => "double" }
+]
+
+Parquet.write_columns(batches.each,
+  schema: schema,
+  write_to: "output.parquet",
+  compression: "snappy"  # Options: none, snappy, gzip, lz4, zstd
+)
+```
+
+`write_columns` also accepts `logger:` with the same Ruby logger interface as
+row writes.
+
+## Data Types
+
+### Basic Types
+
+```ruby
+schema = [
+  # Integers
+  { "tiny" => "int8" },         # -128 to 127
+  { "small" => "int16" },       # -32,768 to 32,767
+  { "medium" => "int32" },      # ±2 billion
+  { "large" => "int64" },       # ±9 quintillion
+
+  # Unsigned integers
+  { "ubyte" => "uint8" },       # 0 to 255
+  { "ushort" => "uint16" },     # 0 to 65,535
+  { "uint" => "uint32" },       # 0 to 4 billion
+  { "ulong" => "uint64" },      # 0 to 18 quintillion
+
+  # Floating point
+  { "price" => "float" },       # 32-bit precision
+  { "amount" => "double" },     # 64-bit precision
+
+  # Other basics
+  { "name" => "string" },
+  { "data" => "binary" },
+  { "active" => "boolean" }
+]
+```
+
+### Date and Time Types
+
+```ruby
+schema = [
+  # Date (days since Unix epoch)
+  { "date" => "date32" },
+
+  # Timestamps (with different precisions)
+  { "created_sec" => "timestamp_second" },
+  { "created_ms" => "timestamp_millis" },    # Most common
+  { "created_us" => "timestamp_micros" },
+  { "created_ns" => "timestamp_nanos" },
+
+  # Time of day (without date)
+  { "time_ms" => "time_millis" },    # Milliseconds since midnight
+  { "time_us" => "time_micros" }     # Microseconds since midnight
+]
+```
+
+### Decimal Type (Financial Data)
+
+For exact decimal arithmetic (no floating-point errors):
+
+```ruby
+require "bigdecimal"
+
+schema = [
+  # Financial amounts with 2 decimal places
+  { "price" => "decimal", "precision" => 10, "scale" => 2 },  # Up to 99,999,999.99
+  { "balance" => "decimal", "precision" => 15, "scale" => 2 }, # Larger amounts
+
+  # High-precision calculations
+  { "rate" => "decimal", "precision" => 10, "scale" => 8 }     # 8 decimal places
+]
+
+# Use BigDecimal for exact values
+data = [[
+  BigDecimal("19.99"),
+  BigDecimal("1234567.89"),
+  BigDecimal("0.00000123")
+]]
+```
+
+## Complex Data Structures
+
+The library includes a powerful Schema DSL for defining nested data:
+
+### Using the Schema DSL
+
+```ruby
+schema = Parquet::Schema.define do
+  # Simple fields
+  field :id, :int64, nullable: false      # Required field
+  field :name, :string                    # Optional by default
+
+  # Nested structure
+  field :address, :struct do
+    field :street, :string
+    field :city, :string
+    field :location, :struct do
+      field :lat, :double
+      field :lng, :double
+    end
+  end
+
+  # Lists
+  field :tags, :list, item: :string
+  field :scores, :list, item: :int32
+
+  # Maps (dictionaries)
+  field :metadata, :map, key: :string, value: :string
+
+  # Complex combinations
+  field :contacts, :list, item: :struct do
+    field :name, :string
+    field :email, :string
+    field :primary, :boolean
+  end
+end
+```
+
+### Writing Complex Data
+
+```ruby
+data = [[
+  1,                              # id
+  "Alice Johnson",                # name
+  {                               # address
+    "street" => "123 Main St",
+    "city" => "Springfield",
+    "location" => {
+      "lat" => 40.7128,
+      "lng" => -74.0060
+    }
+  },
+  ["ruby", "parquet", "data"],    # tags
+  [85, 92, 88],                   # scores
+  { "dept" => "Engineering" },    # metadata
+  [                               # contacts
+    { "name" => "Bob", "email" => "bob@example.com", "primary" => true },
+    { "name" => "Carol", "email" => "carol@example.com", "primary" => false }
+  ]
+]]
+
+Parquet.write_rows(data.each, schema: schema, write_to: "complex.parquet")
+```
+
+## ⚠️ Important Limitations
+
+### Timezone Handling in Parquet
+
+The Parquet specification has a fundamental limitation with timezone storage:
+
+1. **UTC-normalized**: Any timestamp with timezone info (including "+09:00" or "America/New_York") is converted to UTC
+2. **Local/unzoned**: Timestamps without timezone info are stored as-is
+
+**The original timezone information is permanently lost.** This is not a limitation of this library but of the Parquet format itself.
+
+```ruby
+schema = Parquet::Schema.define do
+  # These BOTH store in UTC - timezone info is lost!
+  field :timestamp_utc, :timestamp_millis, timezone: "UTC"
+  field :timestamp_tokyo, :timestamp_millis, timezone: "+09:00"
+
+  # This stores as local time (no timezone)
+  field :timestamp_local, :timestamp_millis
+end
+
+# If you need timezone preservation, store it separately:
+schema = Parquet::Schema.define do
+  field :timestamp, :timestamp_millis, has_timezone: true  # UTC storage
+  field :original_tz, :string                              # "America/New_York"
+end
+```
+
+## Performance Tips
+
+1. **Use column-wise reading** when you need only a few columns from wide tables
+2. **Specify columns parameter** to avoid reading unnecessary data
+3. **Choose appropriate batch sizes**:
+   - Larger batches = better throughput but more memory
+   - Smaller batches = less memory but more overhead
+4. **Pre-sort data** by commonly filtered columns for better compression
+
+
+## Memory Management
+
+Writes are streamed: an Enumerator (or any Enumerable) passed to `write_rows`
+is consumed in bounded slices rather than materialized up front, and completed
+row groups are flushed to the destination while the input is still being
+enumerated. Peak memory is bounded by `batch_size` and `flush_threshold`, not
+by the total dataset size:
+
+```ruby
+Parquet.write_rows(huge_dataset.each,
+  schema: schema,
+  write_to: "output.parquet",
+  batch_size: 1000,              # Rows buffered per write batch (also the
+                                 # slice size pulled from an Enumerator)
+  flush_threshold: 32 * 1024**2  # Flush a row group to the destination once
+                                 # ~32MB of raw row data is staged (default 100MB)
+)
+```
+
+`flush_threshold` bounds both the raw bytes staged since the last flush and
+the writer's encoded in-progress buffer, so row groups reach the destination
+incrementally even when compression shrinks the encoded data dramatically.
+`write_columns` flushes the same way after each batch of columns.
+
+When `write_to:` is an IO object instead of a file path, output is staged in a
+temporary file on disk (memory stays bounded) and copied to the IO after the
+write completes, so the IO receives its bytes only at the end.
+
+Write batch and sample sizes are bounded before buffer allocation. Very large
+batch sizes are rejected, and wide schemas have a lower effective batch cap so
+the writer cannot reserve unbounded per-column value slots.
+
+## Architecture
+
+This gem uses a modular architecture:
+
+- **parquet-core**: Language-agnostic Rust core for Parquet operations
+- **parquet-ruby-adapter**: Ruby-specific FFI adapter layer
+- **parquet gem**: High-level Ruby API
+
+Take a look at [ARCH.md](./ARCH.md)
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/njaremko/parquet-ruby.
+
+## License
+
+The gem is available as open source under the terms of the MIT License.

data/Rakefile

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "rake/testtask"
+require "rb_sys/extensiontask"
+
+task default: :test
+
+GEMSPEC = Gem::Specification.load("parquet.gemspec")
+
+
+platforms = [
+  "x86_64-linux",
+  "x86_64-linux-musl",
+  "aarch64-linux",
+  "aarch64-linux-musl",
+  "x86_64-darwin",
+  "arm64-darwin"
+]
+
+RbSys::ExtensionTask.new("parquet", GEMSPEC) do |ext|
+  ext.lib_dir = "lib/parquet"
+  ext.ext_dir = "ext/parquet"
+  ext.cross_compile = true
+  ext.cross_platform = platforms
+  ext.cross_compiling do |spec|
+    spec.dependencies.reject! { |dep| dep.name == "rb_sys" }
+    spec.files.reject! { |file| File.fnmatch?("ext/*", file, File::FNM_EXTGLOB) }
+  end
+end
+
+Rake::TestTask.new do |t|
+  t.deps << :compile
+  t.test_files = FileList[File.expand_path("test/*_test.rb", __dir__)]
+  t.libs << "lib"
+  t.libs << "test"
+end
+
+task :release do
+  sh "bundle exec rake test"
+  sh "mkdir -p pkg"
+  sh "gem build parquet.gemspec -o pkg/parquet-#{Parquet::VERSION}.gem"
+  sh "gem push pkg/parquet-#{Parquet::VERSION}.gem"
+end

data/ext/parquet/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+name = "parquet"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+crate-type = ["cdylib"]
+
+[build-dependencies]
+rb-sys-env = "^0.2"
+
+[dependencies]
+ahash = "0.8"
+arrow-array = "58.3.0"
+arrow-buffer = "58.3.0"
+arrow-ipc = { version = "58.3.0", features = ["lz4"] }
+arrow-schema = "58.3.0"
+bytes = "^1.9"
+either = "1.9"
+itertools = "^0.14"
+jiff = "0.2"
+magnus = { version = "0.8", features = ["rb-sys"] }
+parquet = { version = "58.3.0", features = ["json"] }
+parquet-ruby-adapter = { path = "../parquet-ruby-adapter" }
+rand = "0.9"
+rb-sys = "^0.9"
+simdutf8 = "0.1.5"
+tempfile = "^3.15"
+thiserror = "2.0"
+num = "0.4.3"
+uuid = "1.16.0"
+ordered-float = "5.0.0"
+
+
+[target.'cfg(target_os = "linux")'.dependencies]
+jemallocator = { version = "0.5", features = ["disable_initial_exec_tls"] }
+
+[target.'cfg(not(any(target_os = "linux", target_os = "windows")))'.dependencies]
+mimalloc = { version = "0.1", default-features = false }

data/ext/parquet/build.rs

@@ -0,0 +1,5 @@
+pub fn main() -> Result<(), Box<dyn std::error::Error>> {
+    rb_sys_env::activate()?;
+
+    Ok(())
+}

data/ext/parquet/extconf.rb

@@ -0,0 +1,4 @@
+require "mkmf"
+require "rb_sys/mkmf"
+
+create_rust_makefile("parquet/parquet")