parquet-tyfoom 0.8.0-aarch64-linux

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2623381ad8d8b9ce01cca102fe15d33c9a72694fe34633f5e148f795cc1164e5
+  data.tar.gz: 8525cecb2735315e85cfe8c22b0406823b536d70c131e00aeaa5975c56b31881
+SHA512:
+  metadata.gz: 33d06cf0b70a54a24b65285dbe8827e6afd1ce43a0f520648a5d292a0c3f1234226ecf8b9f971514881f318f7f4becfc7f524542efe8b9b4a786845ad1babfe8
+  data.tar.gz: 0f89b7a93fb38173db4d468576f81f04cf9217ed380de551858eeb0f85b008b07b47344d439b2ee01139821d4b61fb1c41ee6e447eb0a61287143982d90e842e

data/Gemfile

@@ -0,0 +1,21 @@
+source "https://rubygems.org"
+
+gem "rb_sys", "~> 0.9.56"
+gem "rake"
+gem "bigdecimal"
+
+# Use local version of parquet
+gemspec
+
+group :development do
+  # gem "benchmark-ips", "~> 2.12"
+  # gem "polars-df"
+  # gem "duckdb"
+  gem "benchmark-memory"
+end
+
+group :test do
+  gem "csv"
+  gem "logger"
+  gem "minitest", "~> 5.0"
+end

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Nathan Jaremko
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,428 @@
+# parquet-ruby
+
+[![Gem Version](https://badge.fury.io/rb/parquet.svg)](https://badge.fury.io/rb/parquet)
+
+Read and write [Apache Parquet](https://parquet.apache.org/) files from Ruby. This gem wraps the official Apache [`parquet`](https://github.com/apache/arrow-rs/tree/main/parquet) rust crate, providing:
+
+- **High performance** columnar data storage and retrieval
+- **Memory-efficient** streaming APIs for large datasets
+- **Full compatibility** with the Apache Parquet specification
+- **Simple, Ruby-native** APIs that feel natural
+
+## Why Use This Library?
+
+Apache Parquet is the de facto standard for analytical data storage, offering:
+- **Efficient compression** - typically 2-10x smaller than CSV
+- **Fast columnar access** - read only the columns you need
+- **Rich type system** - preserves data types, including nested structures
+- **Wide ecosystem support** - works with Spark, Pandas, DuckDB, and more
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'parquet'
+```
+
+Then execute:
+
+```bash
+$ bundle install
+```
+
+Or install it directly:
+
+```bash
+$ gem install parquet
+```
+
+## Quick Start
+
+### Reading Data
+
+```ruby
+require "parquet"
+
+# Read Parquet files row by row
+Parquet.each_row("data.parquet") do |row|
+  puts row  # => {"id" => 1, "name" => "Alice", "score" => 95.5}
+end
+
+# Or column by column for better performance
+Parquet.each_column("data.parquet", batch_size: 1000) do |batch|
+  puts batch  # => {"id" => [1, 2, ...], "name" => ["Alice", "Bob", ...]}
+end
+```
+
+### Writing Data
+
+```ruby
+# Define your schema
+schema = [
+  { "id" => "int64" },
+  { "name" => "string" },
+  { "score" => "double" }
+]
+
+# Write row by row
+rows = [
+  [1, "Alice", 95.5],
+  [2, "Bob", 82.3]
+]
+
+Parquet.write_rows(rows.each, schema: schema, write_to: "output.parquet")
+```
+
+## Reading Parquet Files
+
+The library provides two APIs for reading data, each optimized for different use cases:
+
+### Row-wise Reading (Sequential Access)
+
+Best for: Processing records one at a time, data transformations, ETL pipelines
+
+```ruby
+# Basic usage - returns hashes
+Parquet.each_row("data.parquet") do |row|
+  puts row  # => {"id" => 1, "name" => "Alice"}
+end
+
+# Memory-efficient array format
+Parquet.each_row("data.parquet", result_type: :array) do |row|
+  puts row  # => [1, "Alice"]
+end
+
+# Read specific columns only
+Parquet.each_row("data.parquet", columns: ["id", "name"]) do |row|
+  # Only requested columns are loaded from disk
+end
+
+# Works with IO objects
+File.open("data.parquet", "rb") do |file|
+  Parquet.each_row(file) do |row|
+    # Process row
+  end
+end
+```
+
+### Column-wise Reading (Analytical Access)
+
+Best for: Analytics, aggregations, when you need few columns from wide tables
+
+```ruby
+# Process data in column batches
+Parquet.each_column("data.parquet", batch_size: 1000) do |batch|
+  # batch is a hash of column_name => array_of_values
+  ids = batch["id"]      # => [1, 2, 3, ..., 1000]
+  names = batch["name"]  # => ["Alice", "Bob", ...]
+
+  # Perform columnar operations
+  avg_id = ids.sum.to_f / ids.length
+end
+
+# Array format for more control
+Parquet.each_column("data.parquet",
+                    result_type: :array,
+                    columns: ["id", "name"]) do |batch|
+  # batch is an array of arrays
+  # [[1, 2, ...], ["Alice", "Bob", ...]]
+end
+```
+
+### File Metadata
+
+Inspect file structure without reading data:
+
+```ruby
+metadata = Parquet.metadata("data.parquet")
+
+puts metadata["num_rows"]           # Total row count
+puts metadata["created_by"]         # Writer identification
+puts metadata["schema"]["fields"]   # Column definitions
+puts metadata["row_groups"].size    # Number of row groups
+```
+
+## Writing Parquet Files
+
+### Row-wise Writing
+
+Best for: Streaming data, converting from other formats, memory-constrained environments
+
+```ruby
+# Basic schema definition
+schema = [
+  { "id" => "int64" },
+  { "name" => "string" },
+  { "active" => "boolean" },
+  { "balance" => "double" }
+]
+
+# Stream data from any enumerable
+rows = CSV.foreach("input.csv").map do |row|
+  [row[0].to_i, row[1], row[2] == "true", row[3].to_f]
+end
+
+Parquet.write_rows(rows,
+  schema: schema,
+  write_to: "output.parquet",
+  batch_size: 5000  # Positive rows per batch (default: 1000)
+)
+```
+
+### Column-wise Writing
+
+Best for: Pre-columnar data, better compression, higher performance
+
+```ruby
+# Prepare columnar data
+ids = [1, 2, 3, 4, 5]
+names = ["Alice", "Bob", "Charlie", "Diana", "Eve"]
+scores = [95.5, 82.3, 88.7, 91.2, 79.8]
+
+# Create batches
+batches = [[
+  ids,    # First column
+  names,  # Second column
+  scores  # Third column
+]]
+
+schema = [
+  { "id" => "int64" },
+  { "name" => "string" },
+  { "score" => "double" }
+]
+
+Parquet.write_columns(batches.each,
+  schema: schema,
+  write_to: "output.parquet",
+  compression: "snappy"  # Options: none, snappy, gzip, lz4, zstd
+)
+```
+
+`write_columns` also accepts `logger:` with the same Ruby logger interface as
+row writes.
+
+## Data Types
+
+### Basic Types
+
+```ruby
+schema = [
+  # Integers
+  { "tiny" => "int8" },         # -128 to 127
+  { "small" => "int16" },       # -32,768 to 32,767
+  { "medium" => "int32" },      # ±2 billion
+  { "large" => "int64" },       # ±9 quintillion
+
+  # Unsigned integers
+  { "ubyte" => "uint8" },       # 0 to 255
+  { "ushort" => "uint16" },     # 0 to 65,535
+  { "uint" => "uint32" },       # 0 to 4 billion
+  { "ulong" => "uint64" },      # 0 to 18 quintillion
+
+  # Floating point
+  { "price" => "float" },       # 32-bit precision
+  { "amount" => "double" },     # 64-bit precision
+
+  # Other basics
+  { "name" => "string" },
+  { "data" => "binary" },
+  { "active" => "boolean" }
+]
+```
+
+### Date and Time Types
+
+```ruby
+schema = [
+  # Date (days since Unix epoch)
+  { "date" => "date32" },
+
+  # Timestamps (with different precisions)
+  { "created_sec" => "timestamp_second" },
+  { "created_ms" => "timestamp_millis" },    # Most common
+  { "created_us" => "timestamp_micros" },
+  { "created_ns" => "timestamp_nanos" },
+
+  # Time of day (without date)
+  { "time_ms" => "time_millis" },    # Milliseconds since midnight
+  { "time_us" => "time_micros" }     # Microseconds since midnight
+]
+```
+
+### Decimal Type (Financial Data)
+
+For exact decimal arithmetic (no floating-point errors):
+
+```ruby
+require "bigdecimal"
+
+schema = [
+  # Financial amounts with 2 decimal places
+  { "price" => "decimal", "precision" => 10, "scale" => 2 },  # Up to 99,999,999.99
+  { "balance" => "decimal", "precision" => 15, "scale" => 2 }, # Larger amounts
+
+  # High-precision calculations
+  { "rate" => "decimal", "precision" => 10, "scale" => 8 }     # 8 decimal places
+]
+
+# Use BigDecimal for exact values
+data = [[
+  BigDecimal("19.99"),
+  BigDecimal("1234567.89"),
+  BigDecimal("0.00000123")
+]]
+```
+
+## Complex Data Structures
+
+The library includes a powerful Schema DSL for defining nested data:
+
+### Using the Schema DSL
+
+```ruby
+schema = Parquet::Schema.define do
+  # Simple fields
+  field :id, :int64, nullable: false      # Required field
+  field :name, :string                    # Optional by default
+
+  # Nested structure
+  field :address, :struct do
+    field :street, :string
+    field :city, :string
+    field :location, :struct do
+      field :lat, :double
+      field :lng, :double
+    end
+  end
+
+  # Lists
+  field :tags, :list, item: :string
+  field :scores, :list, item: :int32
+
+  # Maps (dictionaries)
+  field :metadata, :map, key: :string, value: :string
+
+  # Complex combinations
+  field :contacts, :list, item: :struct do
+    field :name, :string
+    field :email, :string
+    field :primary, :boolean
+  end
+end
+```
+
+### Writing Complex Data
+
+```ruby
+data = [[
+  1,                              # id
+  "Alice Johnson",                # name
+  {                               # address
+    "street" => "123 Main St",
+    "city" => "Springfield",
+    "location" => {
+      "lat" => 40.7128,
+      "lng" => -74.0060
+    }
+  },
+  ["ruby", "parquet", "data"],    # tags
+  [85, 92, 88],                   # scores
+  { "dept" => "Engineering" },    # metadata
+  [                               # contacts
+    { "name" => "Bob", "email" => "bob@example.com", "primary" => true },
+    { "name" => "Carol", "email" => "carol@example.com", "primary" => false }
+  ]
+]]
+
+Parquet.write_rows(data.each, schema: schema, write_to: "complex.parquet")
+```
+
+## ⚠️ Important Limitations
+
+### Timezone Handling in Parquet
+
+The Parquet specification has a fundamental limitation with timezone storage:
+
+1. **UTC-normalized**: Any timestamp with timezone info (including "+09:00" or "America/New_York") is converted to UTC
+2. **Local/unzoned**: Timestamps without timezone info are stored as-is
+
+**The original timezone information is permanently lost.** This is not a limitation of this library but of the Parquet format itself.
+
+```ruby
+schema = Parquet::Schema.define do
+  # These BOTH store in UTC - timezone info is lost!
+  field :timestamp_utc, :timestamp_millis, timezone: "UTC"
+  field :timestamp_tokyo, :timestamp_millis, timezone: "+09:00"
+
+  # This stores as local time (no timezone)
+  field :timestamp_local, :timestamp_millis
+end
+
+# If you need timezone preservation, store it separately:
+schema = Parquet::Schema.define do
+  field :timestamp, :timestamp_millis, has_timezone: true  # UTC storage
+  field :original_tz, :string                              # "America/New_York"
+end
+```
+
+## Performance Tips
+
+1. **Use column-wise reading** when you need only a few columns from wide tables
+2. **Specify columns parameter** to avoid reading unnecessary data
+3. **Choose appropriate batch sizes**:
+   - Larger batches = better throughput but more memory
+   - Smaller batches = less memory but more overhead
+4. **Pre-sort data** by commonly filtered columns for better compression
+
+
+## Memory Management
+
+Writes are streamed: an Enumerator (or any Enumerable) passed to `write_rows`
+is consumed in bounded slices rather than materialized up front, and completed
+row groups are flushed to the destination while the input is still being
+enumerated. Peak memory is bounded by `batch_size` and `flush_threshold`, not
+by the total dataset size:
+
+```ruby
+Parquet.write_rows(huge_dataset.each,
+  schema: schema,
+  write_to: "output.parquet",
+  batch_size: 1000,              # Rows buffered per write batch (also the
+                                 # slice size pulled from an Enumerator)
+  flush_threshold: 32 * 1024**2  # Flush a row group to the destination once
+                                 # ~32MB of raw row data is staged (default 100MB)
+)
+```
+
+`flush_threshold` bounds both the raw bytes staged since the last flush and
+the writer's encoded in-progress buffer, so row groups reach the destination
+incrementally even when compression shrinks the encoded data dramatically.
+`write_columns` flushes the same way after each batch of columns.
+
+When `write_to:` is an IO object instead of a file path, output is staged in a
+temporary file on disk (memory stays bounded) and copied to the IO after the
+write completes, so the IO receives its bytes only at the end.
+
+Write batch and sample sizes are bounded before buffer allocation. Very large
+batch sizes are rejected, and wide schemas have a lower effective batch cap so
+the writer cannot reserve unbounded per-column value slots.
+
+## Architecture
+
+This gem uses a modular architecture:
+
+- **parquet-core**: Language-agnostic Rust core for Parquet operations
+- **parquet-ruby-adapter**: Ruby-specific FFI adapter layer
+- **parquet gem**: High-level Ruby API
+
+Take a look at [ARCH.md](./ARCH.md)
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/njaremko/parquet-ruby.
+
+## License
+
+The gem is available as open source under the terms of the MIT License.

data/Rakefile

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "rake/testtask"
+require "rb_sys/extensiontask"
+
+task default: :test
+
+GEMSPEC = Gem::Specification.load("parquet.gemspec")
+
+
+platforms = [
+  "x86_64-linux",
+  "x86_64-linux-musl",
+  "aarch64-linux",
+  "aarch64-linux-musl",
+  "x86_64-darwin",
+  "arm64-darwin"
+]
+
+RbSys::ExtensionTask.new("parquet", GEMSPEC) do |ext|
+  ext.lib_dir = "lib/parquet"
+  ext.ext_dir = "ext/parquet"
+  ext.cross_compile = true
+  ext.cross_platform = platforms
+  ext.cross_compiling do |spec|
+    spec.dependencies.reject! { |dep| dep.name == "rb_sys" }
+    spec.files.reject! { |file| File.fnmatch?("ext/*", file, File::FNM_EXTGLOB) }
+  end
+end
+
+Rake::TestTask.new do |t|
+  t.deps << :compile
+  t.test_files = FileList[File.expand_path("test/*_test.rb", __dir__)]
+  t.libs << "lib"
+  t.libs << "test"
+end
+
+task :release do
+  sh "bundle exec rake test"
+  sh "mkdir -p pkg"
+  sh "gem build parquet.gemspec -o pkg/parquet-#{Parquet::VERSION}.gem"
+  sh "gem push pkg/parquet-#{Parquet::VERSION}.gem"
+end

data/lib/parquet/schema.rb

@@ -0,0 +1,262 @@
+# frozen_string_literal: true
+
+module Parquet
+  # Schema definition for Parquet files
+  class Schema
+    # Define a new schema using the DSL
+    # @return [Hash] schema definition hash
+    #
+    # @example Define a schema with nullable and non-nullable fields
+    #   Parquet::Schema.define do
+    #     field :id, :int64, nullable: false  # ID cannot be null
+    #     field :name, :string  # Default nullable: true
+    #
+    #     # Decimal field with precision and scale
+    #     field :price, :decimal, precision: 10, scale: 2
+    #
+    #     # List with non-nullable items
+    #     field :scores, :list, item: :float, item_nullable: false
+    #
+    #     # Map with nullable values
+    #     field :metadata, :map,
+    #           key: :string,
+    #           value: :string,
+    #           value_nullable: true
+    #
+    #     # Nested struct with non-nullable fields
+    #     field :address, :struct, nullable: true do
+    #       field :street, :string, nullable: false
+    #       field :city, :string, nullable: false
+    #       field :zip, :string, nullable: false
+    #     end
+    #   end
+    def self.define(&block)
+      builder = SchemaBuilder.new
+      builder.instance_eval(&block)
+
+      # Return a structured hash representing the schema
+      { type: :struct, fields: builder.fields }
+    end
+
+    # Internal builder class that provides the DSL methods
+    class SchemaBuilder
+      attr_reader :fields
+
+      def initialize
+        @fields = []
+      end
+
+      # Define a field in the schema
+      # @param name [String, Symbol] field name
+      # @param type [Symbol] data type (:int32, :int64, :string, :list, :map, :struct, :decimal, etc)
+      # @param nullable [Boolean] whether the field can be null (default: true)
+      # @param kwargs [Hash] additional options depending on type
+      #
+      # Additional keyword args:
+      #   - `item:` if type == :list
+      #   - `item_nullable:` controls nullability of list items (default: true)
+      #   - `key:, value:` if type == :map
+      #   - `key_nullable:, value_nullable:` controls nullability of map keys/values (default: true)
+      #   - `format:` if you want to store some format string
+      #   - `precision:, scale:` if type == :decimal (precision defaults to 38, scale to 0)
+      #   - `has_timezone:` if type is timestamp - true means UTC storage (default), false means local/unzoned
+      #   - `timezone:` (DEPRECATED) if type is timestamp - any value means UTC storage
+      #   - `nullable:` default to true if not specified
+      def field(name, type, nullable: true, **kwargs, &block)
+        field_hash = { name: name.to_s, type: type, nullable: !!nullable }
+
+        # Possibly store a format if provided
+        field_hash[:format] = kwargs[:format] if kwargs.key?(:format)
+        
+        # Handle timezone for timestamp types
+        if [:timestamp_second, :timestamp_millis, :timestamp_micros, :timestamp_nanos].include?(type)
+          # Support new has_timezone parameter (preferred)
+          if kwargs.key?(:has_timezone)
+            # If has_timezone is true, store "UTC" to indicate timezone presence
+            # If explicitly false, don't store timezone (indicates local/unzoned)
+            field_hash[:timezone] = "UTC" if kwargs[:has_timezone]
+          elsif kwargs.key?(:timezone)
+            # Legacy support: any timezone value means UTC storage
+            # Store "UTC" regardless of the actual value to make it clear
+            field_hash[:timezone] = "UTC"
+          else
+            # Default behavior when neither parameter is specified: UTC storage
+            field_hash[:timezone] = "UTC"
+          end
+        end
+
+        case type
+        when :struct
+          # We'll parse subfields from the block
+          sub_builder = SchemaBuilder.new
+          sub_builder.instance_eval(&block) if block
+          field_hash[:fields] = sub_builder.fields
+        when :list
+          item_type = kwargs[:item]
+          raise ArgumentError, "list field `#{name}` requires `item:` type" unless item_type
+          # Pass item_nullable if provided, otherwise use true as default
+          item_nullable = kwargs[:item_nullable].nil? ? true : !!kwargs[:item_nullable]
+
+          # Pass precision and scale if type is decimal
+          if item_type == :decimal
+            precision = kwargs[:precision]
+            scale = kwargs[:scale]
+            field_hash[:item] = wrap_subtype(
+              item_type,
+              nullable: item_nullable,
+              precision: precision,
+              scale: scale,
+              &block
+            )
+          else
+            field_hash[:item] = wrap_subtype(item_type, nullable: item_nullable, &block)
+          end
+        when :map
+          # user must specify key:, value:
+          key_type = kwargs[:key]
+          value_type = kwargs[:value]
+          raise ArgumentError, "map field `#{name}` requires `key:` and `value:`" if key_type.nil? || value_type.nil?
+          # Map keys are required by the Parquet spec. Reject an explicit nullable
+          # key at this boundary rather than letting it fail deep in the writer.
+          if kwargs[:key_nullable]
+            raise ArgumentError, "map field `#{name}` keys are always required; remove `key_nullable: true`"
+          end
+          key_nullable = false
+          value_nullable = kwargs[:value_nullable].nil? ? true : !!kwargs[:value_nullable]
+
+          field_hash[:key] = wrap_subtype(key_type, nullable: key_nullable)
+
+          # Pass precision and scale if value type is decimal
+          if value_type == :decimal
+            precision = kwargs[:precision]
+            scale = kwargs[:scale]
+            field_hash[:value] = wrap_subtype(
+              value_type,
+              nullable: value_nullable,
+              precision: precision,
+              scale: scale,
+              &block
+            )
+          else
+            field_hash[:value] = wrap_subtype(value_type, nullable: value_nullable, &block)
+          end
+        when :decimal
+          # Store precision and scale for decimal type according to rules:
+          # 1. When neither precision nor scale is provided, use maximum precision (38)
+          # 2. When only precision is provided, scale defaults to 0
+          # 3. When only scale is provided, use maximum precision (38)
+          # 4. When both are provided, use the provided values
+
+          if kwargs[:precision].nil? && kwargs[:scale].nil?
+            # No precision or scale provided - use maximum precision
+            field_hash[:precision] = 38
+            field_hash[:scale] = 0
+          elsif kwargs[:precision] && kwargs[:scale].nil?
+            # Precision only - scale defaults to 0
+            field_hash[:precision] = kwargs[:precision]
+            field_hash[:scale] = 0
+          elsif kwargs[:precision].nil? && kwargs[:scale]
+            # Scale only - use maximum precision
+            field_hash[:precision] = 38
+            field_hash[:scale] = kwargs[:scale]
+          else
+            # Both provided
+            field_hash[:precision] = kwargs[:precision]
+            field_hash[:scale] = kwargs[:scale]
+          end
+        else
+          # primitive type: :int32, :int64, :string, etc.
+          # do nothing else special
+        end
+
+        @fields << field_hash
+      end
+
+      def build_map(key_type, value_type, key_nullable: false, value_nullable: true, nullable: true, &block)
+        # Wrap the key type (maps typically use non-nullable keys)
+        key = wrap_subtype(key_type, nullable: key_nullable)
+
+        # Handle the case where value_type is a complex type (:struct or :list) and a block is provided
+        value =
+          if (value_type == :struct || value_type == :list) && block
+            wrap_subtype(value_type, nullable: value_nullable, &block)
+          else
+            wrap_subtype(value_type, nullable: value_nullable)
+          end
+
+        # Map is represented as a list of key/value pairs in Parquet
+        {
+          type: :map,
+          nullable: nullable,
+          item: {
+            type: :struct,
+            nullable: false,
+            name: "key_value",
+            fields: [key, value]
+          }
+        }
+      end
+
+      private
+
+      # If user said: field "something", :list, item: :struct do ... end
+      # we want to recursively parse that sub-struct from the block.
+      # So wrap_subtype might be:
+      def wrap_subtype(t, nullable: true, precision: nil, scale: nil, &block)
+        if t == :struct
+          sub_builder = SchemaBuilder.new
+          sub_builder.instance_eval(&block) if block
+
+          # Validate that the struct has at least one field
+          if sub_builder.fields.empty?
+            raise ArgumentError, "Cannot create a struct with zero fields. Parquet doesn't support empty structs."
+          end
+
+          { type: :struct, nullable: nullable, name: "item", fields: sub_builder.fields }
+        elsif t == :list && block
+          # Handle nested lists by processing the block to define the item type
+          sub_builder = SchemaBuilder.new
+          sub_builder.instance_eval(&block) if block
+
+          # We expect a single field named "item" that defines the inner list's item type
+          if sub_builder.fields.empty? || sub_builder.fields.length > 1 || sub_builder.fields[0][:name] != "item"
+            raise ArgumentError, "Nested list must define exactly one field named 'item' for the inner list's item type"
+          end
+
+          { type: :list, nullable: nullable, name: "item", item: sub_builder.fields[0] }
+        elsif t == :decimal
+          # Handle decimal type with precision and scale
+          result = { type: t, nullable: nullable, name: "item" }
+
+          # Follow the same rules as in field() method:
+          # 1. When neither precision nor scale is provided, use maximum precision (38)
+          # 2. When only precision is provided, scale defaults to 0
+          # 3. When only scale is provided, use maximum precision (38)
+          # 4. When both are provided, use the provided values
+          if precision.nil? && scale.nil?
+            # No precision or scale provided - use maximum precision
+            result[:precision] = 38
+            result[:scale] = 0
+          elsif precision && scale.nil?
+            # Precision only - scale defaults to 0
+            result[:precision] = precision
+            result[:scale] = 0
+          elsif precision.nil? && scale
+            # Scale only - use maximum precision
+            result[:precision] = 38
+            result[:scale] = scale
+          else
+            # Both provided
+            result[:precision] = precision
+            result[:scale] = scale
+          end
+
+          result
+        else
+          # e.g. :int32 => { type: :int32, nullable: true }
+          { type: t, nullable: nullable, name: "item" }
+        end
+      end
+    end
+  end
+end

data/lib/parquet/version.rb

@@ -0,0 +1,3 @@
+module Parquet
+  VERSION = "0.8.0"
+end

data/lib/parquet.rb

@@ -0,0 +1,11 @@
+require_relative "parquet/version"
+require_relative "parquet/schema"
+
+begin
+  require "parquet/#{RUBY_VERSION.to_f}/parquet"
+rescue LoadError
+  require "parquet/parquet"
+end
+
+module Parquet
+end

data/lib/parquet.rbi

@@ -0,0 +1,181 @@
+# typed: true
+
+module Parquet
+  # Returns metadata information about a Parquet file
+  #
+  # The returned hash contains information about:
+  # - Basic file metadata (num_rows, created_by)
+  # - Schema information (fields, types, etc.)
+  # - Row group details
+  # - Column chunk information (compression, encodings, statistics)
+  sig { params(path: String).returns(T::Hash[String, T.untyped]) }
+  def self.metadata(path)
+  end
+
+  # Options:
+  #   - `input`: String, File, or IO object containing parquet data
+  #   - `result_type`: String specifying the output format
+  #                    ("hash" or "array" or :hash or :array)
+  #   - `columns`: When present, only the specified columns will be included in the output.
+  #                This is useful for reducing how much data is read and improving performance.
+  #   - `string_storage`: How string *values* become Ruby strings (default `:copy`). Hash keys
+  #                       (struct field names and top-level column names) are always interned and
+  #                       reused regardless of this setting.
+  #                       - `:copy` allocates a fresh mutable String per value.
+  #                       - `:intern` deduplicates low-cardinality equal values into frozen interned
+  #                         Strings up to a bounded per-read cache, then falls back to frozen copies.
+  #                         A transient copy still happens per value, so it is not a per-value speedup.
+  #                       - `:shared` returns frozen, zero-copy strings backed by Rust memory for
+  #                         short, repeated, low-cardinality values. Each read returns at most the
+  #                         configured number of shared values and only values up to the configured
+  #                         byte size; values past those bounds become frozen copies. New process-wide
+  #                         leaks are also capped by the requested budget and hard process ceilings.
+  #                         All `:shared` results are frozen. Not recommended for high-cardinality or
+  #                         large-blob string columns.
+  #                       Pass a hash to set the `:shared` leak budget, e.g.
+  #                       `{ mode: :shared, max_entries: 16_384, max_value_bytes: 1024 }`.
+  sig do
+    params(
+      input: T.any(String, File, StringIO, IO),
+      result_type: T.nilable(T.any(String, Symbol)),
+      columns: T.nilable(T::Array[String]),
+      strict: T.nilable(T::Boolean),
+      string_storage: T.nilable(T.any(String, Symbol, T::Hash[Symbol, T.untyped]))
+    ).returns(T::Enumerator[T.any(T::Hash[String, T.untyped], T::Array[T.untyped])])
+  end
+  sig do
+    params(
+      input: T.any(String, File, StringIO, IO),
+      result_type: T.nilable(T.any(String, Symbol)),
+      columns: T.nilable(T::Array[String]),
+      strict: T.nilable(T::Boolean),
+      string_storage: T.nilable(T.any(String, Symbol, T::Hash[Symbol, T.untyped])),
+      blk: T.nilable(T.proc.params(row: T.any(T::Hash[String, T.untyped], T::Array[T.untyped])).void)
+    ).returns(NilClass)
+  end
+  def self.each_row(input, result_type: nil, columns: nil, strict: nil, string_storage: nil, &blk)
+  end
+
+  # Options:
+  #   - `input`: String, File, or IO object containing parquet data
+  #   - `result_type`: String specifying the output format
+  #                    ("hash" or "array" or :hash or :array)
+  #   - `columns`: When present, only the specified columns will be included in the output.
+  #   - `batch_size`: When present, specifies the number of rows per batch
+  #   - `string_storage`: How string values become Ruby strings (`:copy` (default), `:intern`,
+  #                       or `:shared`). See `each_row` for the semantics of each mode.
+  sig do
+    params(
+      input: T.any(String, File, StringIO, IO),
+      result_type: T.nilable(T.any(String, Symbol)),
+      columns: T.nilable(T::Array[String]),
+      batch_size: T.nilable(Integer),
+      strict: T.nilable(T::Boolean),
+      string_storage: T.nilable(T.any(String, Symbol, T::Hash[Symbol, T.untyped]))
+    ).returns(T::Enumerator[T.any(T::Hash[String, T.untyped], T::Array[T.untyped])])
+  end
+  sig do
+    params(
+      input: T.any(String, File, StringIO, IO),
+      result_type: T.nilable(T.any(String, Symbol)),
+      columns: T.nilable(T::Array[String]),
+      batch_size: T.nilable(Integer),
+      strict: T.nilable(T::Boolean),
+      string_storage: T.nilable(T.any(String, Symbol, T::Hash[Symbol, T.untyped])),
+      blk:
+        T.nilable(T.proc.params(batch: T.any(T::Hash[String, T::Array[T.untyped]], T::Array[T::Array[T.untyped]])).void)
+    ).returns(NilClass)
+  end
+  def self.each_column(input, result_type: nil, columns: nil, batch_size: nil, strict: nil, string_storage: nil, &blk)
+  end
+
+  # Options:
+  #   - `read_from`: An Enumerator yielding arrays of values representing each row
+  #   - `schema`: Array of hashes specifying column names and types. Supported types:
+  #     - `int8`, `int16`, `int32`, `int64`
+  #     - `uint8`, `uint16`, `uint32`, `uint64`
+  #     - `float`, `double`
+  #     - `string`
+  #     - `binary`
+  #     - `boolean`
+  #     - `date32`
+  #     - `timestamp_millis`, `timestamp_micros`
+  #   - `write_to`: String path or IO object to write the parquet file to
+  #   - `batch_size`: Optional positive batch size for writing (defaults to 1000, at most 1_000_000
+  #                   for one-column schemas; wide schemas may have a lower safety cap). Enumerator
+  #                   inputs are consumed in slices of this many rows, never materialized in full.
+  #   - `flush_threshold`: Optional threshold in bytes before a row group is flushed to the
+  #                        destination; bounds both the raw bytes staged since the last flush
+  #                        and the writer's encoded in-progress buffer (defaults to 100MB)
+  #   - `compression`: Optional compression type to use (defaults to "zstd")
+  #                   Supported values: "none", "uncompressed", "snappy", "gzip", "lz4", "zstd"
+  #   - `sample_size`: Optional positive number of rows to sample for size estimation
+  #                    (defaults to 100, at most 10_000)
+  #   - `string_cache`: Deduplicate repeated string values while writing. `false` (default)
+  #                     disables it, `true` enables it with a default capacity, and an Integer
+  #                     enables it with that many retained distinct strings (at most 65_536).
+  #                     Retention also skips values larger than 4KB and stops after 16MB of
+  #                     cached string content.
+  sig do
+    params(
+      read_from: T::Enumerator[T::Array[T.untyped]],
+      schema: T::Array[T::Hash[String, String]],
+      write_to: T.any(String, IO),
+      batch_size: T.nilable(Integer),
+      flush_threshold: T.nilable(Integer),
+      compression: T.nilable(String),
+      sample_size: T.nilable(Integer),
+      string_cache: T.nilable(T.any(T::Boolean, Integer))
+    ).void
+  end
+  def self.write_rows(
+    read_from,
+    schema:,
+    write_to:,
+    batch_size: nil,
+    flush_threshold: nil,
+    compression: nil,
+    sample_size: nil,
+    string_cache: nil
+  )
+  end
+
+  # Options:
+  #   - `read_from`: An Enumerator yielding arrays of column batches
+  #   - `schema`: Array of hashes specifying column names and types. Supported types:
+  #     - `int8`, `int16`, `int32`, `int64`
+  #     - `uint8`, `uint16`, `uint32`, `uint64`
+  #     - `float`, `double`
+  #     - `string`
+  #     - `binary`
+  #     - `boolean`
+  #     - `date32`
+  #     - `timestamp_millis`, `timestamp_micros`
+  #     - Looks like [{"column_name" => {"type" => "date32", "format" => "%Y-%m-%d"}}, {"column_name" => "int8"}]
+  #   - `write_to`: String path or IO object to write the parquet file to
+  #   - `flush_threshold`: Optional threshold in bytes before a row group is flushed to the
+  #                        destination; bounds both the raw bytes staged since the last flush
+  #                        and the writer's encoded in-progress buffer (defaults to 100MB)
+  #   - `compression`: Optional compression type to use (defaults to "zstd")
+  #                   Supported values: "none", "uncompressed", "snappy", "gzip", "lz4", "zstd"
+  #   - `logger`: Optional Ruby logger for column-write progress messages
+  sig do
+    params(
+      read_from: T::Enumerator[T::Array[T::Array[T.untyped]]],
+      schema: T::Array[T::Hash[String, String]],
+      write_to: T.any(String, IO),
+      flush_threshold: T.nilable(Integer),
+      compression: T.nilable(String),
+      logger: T.nilable(T.untyped)
+    ).void
+  end
+  def self.write_columns(
+    read_from,
+    schema:,
+    write_to:,
+    flush_threshold: nil,
+    compression: nil,
+    logger: nil
+  )
+  end
+end

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification
+name: parquet-tyfoom
+version: !ruby/object:Gem::Version
+  version: 0.8.0
+platform: aarch64-linux
+authors:
+- Nathan Jaremko
+- Cameron McCord
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2026-07-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bigdecimal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+description: |2
+      Tyfoom's fork of the `parquet` gem (github.com/njaremko/parquet-ruby), published while the
+      incremental streaming-write fix is pending upstream. It wraps the official Apache Rust
+      implementation and bounds write memory by streaming row groups to disk instead of buffering the
+      whole file. Drop-in compatible with the upstream gem: the library is still required as
+      `require "parquet"` and exposes the same `Parquet` API.
+email:
+- nathan@jaremko.ca
+- cameron.mccord@tyfoom.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/parquet.rb
+- lib/parquet.rbi
+- lib/parquet/3.2/parquet.so
+- lib/parquet/3.3/parquet.so
+- lib/parquet/3.4/parquet.so
+- lib/parquet/4.0/parquet.so
+- lib/parquet/schema.rb
+- lib/parquet/version.rb
+homepage: https://github.com/cameronmccord2/parquet-ruby
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/cameronmccord2/parquet-ruby
+  source_code_uri: https://github.com/cameronmccord2/parquet-ruby
+  readme_uri: https://github.com/cameronmccord2/parquet-ruby/blob/stream-writes-incrementally/README.md
+  changelog_uri: https://github.com/cameronmccord2/parquet-ruby/blob/stream-writes-incrementally/CHANGELOG.md
+  documentation_uri: https://www.rubydoc.info/gems/parquet-tyfoom
+  funding_uri: https://github.com/sponsors/njaremko
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.2'
+  - - "<"
+    - !ruby/object:Gem::Version
+      version: 4.1.dev
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.23
+signing_key: 
+specification_version: 4
+summary: Tyfoom fork of the parquet gem (Rust), with incremental streaming writes
+test_files: []