parquet 0.7.3-aarch64-linux → 0.9.0-aarch64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6f9f906c15b3bb0dd6ffebdf8805583e3d70bc4744126a6e5b5b8a9141f7aac3
-  data.tar.gz: 9e5dccc0659a2a0bebc5585cd99bf6ccf9a688d685ad122006a6c6bc426e2450
+  metadata.gz: 06deeee590e4572ff7d4b302597e26804ca0a7dc4a8eccfd7687f926276a5ef7
+  data.tar.gz: 5514690113ee28379aa505d49ee88cd0e806ad8f4fb7e442c3ffef6e0d36305c
 SHA512:
-  metadata.gz: 5b35a9d549a84fdfe670ede85172336ef3fecffb0be714962984830183cec4ace99abd2189647d3bc6f2fbf8488a401d2cb0470aa4e4f0edd633de551c1366db
-  data.tar.gz: 351a0d0964446a2a2016d7e088ae435e3f11e6cd1974ce171376fb3789649da58b8d7c281be7478ffb50084b2b07df1f11176c49d447a009391af7267c2840c0
+  metadata.gz: e68aced5c5b45ae7dbcd5f3b8215817c10a0cc72379044523895870e3e164a4212fd24d34b9eca9e7913d3f6ff8f43c040ac3d3094a0af955b4353a710db38e2
+  data.tar.gz: 44998090036e3de92c39f2418ccb68369564209e2a3d165405fa56e14d96b1b283aa3e3980cd5880c861e578b1fb90f0332412ee472669e09ad5f2256af11d6c

data/Gemfile

@@ -16,5 +16,6 @@ end
 
 group :test do
   gem "csv"
+  gem "logger"
   gem "minitest", "~> 5.0"
 end

data/README.md

@@ -145,6 +145,33 @@ puts metadata["row_groups"].size    # Number of row groups
 
 ## Writing Parquet Files
 
+`write_rows` and `write_columns` stream an enumerable to a path or writable IO
+and return `nil`. Each row is an array in schema order. For column writing, each
+yielded batch contains one array per field, all of the same length.
+
+Both methods require `schema:` and `write_to:`. `schema:` accepts the array form
+shown below, a `Parquet::Schema` DSL result, or a `fields` schema hash. Use `nil`
+or `[]` to infer string columns named `f0`, `f1`, ... from the first row or
+batch. Empty input requires an explicit schema.
+
+`compression:` accepts `"none"`, `"uncompressed"`, `"snappy"`, `"gzip"`,
+`"lz4"`, `"zstd"`, and `"brotli"`; `nil` defaults to Snappy.
+`flush_threshold:` defaults to 100 MiB. `logger:` accepts a Ruby logger with
+`debug`, `info`, `warn`, and `error` methods.
+
+`write_rows` also accepts `batch_size:`, `sample_size:`, and `string_cache:`.
+Without `batch_size:`, sizing starts at 1,000 rows and adapts to row size; the
+cap is 1,000,000 rows and lower for wide schemas. Sampling defaults to 100 rows
+and is capped at 10,000. `string_cache: true` uses a capacity of 100, or you can
+pass a capacity up to 65,536; `nil` and `false` disable it.
+
+Path output is staged and atomically published only after the complete file has
+been written. On Unix, replacing an existing path preserves its uid, gid, and
+mode and uses standard last-committer-wins rename semantics; creating a path is
+no-clobber. Extended attributes and ACLs are not preserved. IO output is first
+staged on disk, then copied to the IO; a failed copy may leave the IO partially
+written.
+
 ### Row-wise Writing
 
 Best for: Streaming data, converting from other formats, memory-constrained environments
@@ -159,17 +186,109 @@ schema = [
 ]
 
 # Stream data from any enumerable
-rows = CSV.foreach("input.csv").map do |row|
+rows = CSV.foreach("input.csv").lazy.map do |row|
   [row[0].to_i, row[1], row[2] == "true", row[3].to_f]
 end
 
 Parquet.write_rows(rows,
   schema: schema,
   write_to: "output.parquet",
-  batch_size: 5000  # Rows per batch (default: 1000)
+  batch_size: 5000
+)
+```
+
+### Repacking Existing Parquet Files
+
+Concatenate Parquet files and re-split them into differently sized files without
+translating rows through Ruby.
+
+```ruby
+Parquet.repack(
+  ["input-0.parquet", "input-1.parquet"],
+  output_dir: "repacked",
+  rows_per_file: 100_000
 )
+# => [{ "path" => "repacked/batch-0.parquet", "num_rows" => 100_000 },
+#     { "path" => "repacked/batch-1.parquet", "num_rows" => 42_137 }]
+```
+
+The outputs hold exactly the input rows, in input order. Every output but the
+last holds `rows_per_file` rows, and there is always at least one output even
+when the inputs are empty. Omit `rows_per_file:` to concatenate everything into
+a single file.
+
+Each output's Parquet schema is identical to the first input's, and that input's
+file-level key/value metadata (`ARROW:schema`, `pandas`, and so on) is carried
+over. Inputs must agree on leaf column shape — path, physical and logical type,
+nesting — but may differ in key/value metadata and Parquet field ids.
+
+#### Compression and copying
+
+With no `compression:`, each column keeps its own codec — Parquet records one
+per column, and a file may legitimately use several. Naming a codec applies it
+to every column instead:
+
+```ruby
+Parquet.repack("input.parquet", output_dir: "out", compression: "zstd")
 ```
 
+Keeping the inputs' codecs also lets repack copy whole row groups into the
+output byte-for-byte, skipping decompression and re-encoding entirely. A row
+group is copied when it fits the output's remaining row budget, is large enough
+to be worth copying, and the request did not ask for a different codec;
+otherwise its rows are decoded and re-encoded. Both routes produce the same
+rows, so which one runs is not something callers need to reason about — but the
+copy route is dramatically faster, so a plain concatenation is close to an
+I/O-bound copy.
+
+Small row groups are deliberately merged rather than copied: copying them
+one-for-one would make a compaction of many small files reproduce exactly the
+fragmentation it was meant to remove.
+
+#### Output directory ownership
+
+`repack` owns the `{output_file_prefix}-{n}.parquet` names in `output_dir`. If
+any already exist it raises `ArgumentError` rather than mixing two runs' files
+in one directory:
+
+```ruby
+Parquet.repack("input.parquet", output_dir: "out", rows_per_file: 1000, overwrite: true)
+```
+
+`overwrite: true` replaces that set and deletes members left over from a longer
+earlier run, so the returned list always equals what a reader finds in the
+directory. Files outside the set are never touched.
+
+#### Bounds
+
+`max_read_rows_per_chunk:` (default 8192, reduced for wide schemas) bounds rows
+buffered while reading; output row groups are bounded in rows by the same slot
+budget. Both are resource controls: varying them cannot change the returned
+list, the rows, the schema, or the codecs. They do shift compressed byte counts
+and page boundaries, which are representation rather than contract.
+
+Input metadata is read one file at a time, so peak memory is set by the widest
+single file and its row-group size, not by how many files you pass or how many
+rows they hold in total. Note that the row-group bound is in rows, not bytes: a
+schema with very large values still buffers one row group's worth of encoded
+data.
+
+Reading and writing run with the GVL released, so other Ruby threads keep
+running and `Interrupt` / `Timeout` are honoured. An interrupted call leaves no
+output behind.
+
+A repacked output is capped at 32,767 row groups. Merging small groups keeps
+that limit out of reach in practice; if a single output would exceed it,
+repack raises rather than writing an unreadable file, and `rows_per_file:` is
+the way out.
+
+#### Page indexes and other optional structures
+
+Every row group in a Parquet file must agree on whether it carries a page index,
+and a copied row group can only contribute the index its source had. So an
+output carries one exactly when every contributing input does. Bloom filters are
+not carried over on either route.
+
 ### Column-wise Writing
 
 Best for: Pre-columnar data, better compression, higher performance
@@ -196,7 +315,7 @@ schema = [
 Parquet.write_columns(batches.each,
   schema: schema,
   write_to: "output.parquet",
-  compression: "snappy"  # Options: none, snappy, gzip, lz4, zstd
+  compression: "snappy"
 )
 ```
 
@@ -340,7 +459,7 @@ Parquet.write_rows(data.each, schema: schema, write_to: "complex.parquet")
 
 ### Timezone Handling in Parquet
 
-**Critical**: The Parquet specification has a fundamental limitation with timezone storage:
+The Parquet specification has a fundamental limitation with timezone storage:
 
 1. **UTC-normalized**: Any timestamp with timezone info (including "+09:00" or "America/New_York") is converted to UTC
 2. **Local/unzoned**: Timestamps without timezone info are stored as-is
@@ -382,11 +501,20 @@ Control memory usage with flush thresholds:
 Parquet.write_rows(huge_dataset.each,
   schema: schema,
   write_to: "output.parquet",
-  batch_size: 1000,              # Rows before considering flush
-  flush_threshold: 32 * 1024**2  # Flush if batch exceeds 32MB
+  batch_size: 1_000,
+  flush_threshold: 32 * 1024**2 # 32 MiB
 )
 ```
 
+Writer-owned memory stays bounded as the file grows. `flush_threshold` controls
+the converted-value buffer; a single larger row may exceed it temporarily. The
+row-group target is at least 8 MiB, and each file may contain up to 32,768 row
+groups.
+
+Encoded data and completed row-group metadata are staged on disk, so large
+writes need temporary disk space. Ruby still owns the current row or batch, and
+an in-memory destination such as `StringIO` holds the output in memory.
+
 ## Architecture
 
 This gem uses a modular architecture:

data/lib/parquet/schema.rb

@@ -116,8 +116,12 @@ module Parquet
           key_type = kwargs[:key]
           value_type = kwargs[:value]
           raise ArgumentError, "map field `#{name}` requires `key:` and `value:`" if key_type.nil? || value_type.nil?
-          # Pass key_nullable and value_nullable if provided, otherwise use true as default
-          key_nullable = kwargs[:key_nullable].nil? ? true : !!kwargs[:key_nullable]
+          # Map keys are required by the Parquet spec. Reject an explicit nullable
+          # key at this boundary rather than letting it fail deep in the writer.
+          if kwargs[:key_nullable]
+            raise ArgumentError, "map field `#{name}` keys are always required; remove `key_nullable: true`"
+          end
+          key_nullable = false
           value_nullable = kwargs[:value_nullable].nil? ? true : !!kwargs[:value_nullable]
 
           field_hash[:key] = wrap_subtype(key_type, nullable: key_nullable)

data/lib/parquet/version.rb

@@ -1,3 +1,3 @@
 module Parquet
-  VERSION = "0.7.3"
+  VERSION = "0.9.0"
 end

data/lib/parquet.rbi

@@ -18,12 +18,29 @@ module Parquet
   #                    ("hash" or "array" or :hash or :array)
   #   - `columns`: When present, only the specified columns will be included in the output.
   #                This is useful for reducing how much data is read and improving performance.
+  #   - `string_storage`: How string *values* become Ruby strings (default `:copy`). Hash keys
+  #                       (struct field names and top-level column names) are always interned and
+  #                       reused regardless of this setting.
+  #                       - `:copy` allocates a fresh mutable String per value.
+  #                       - `:intern` deduplicates low-cardinality equal values into frozen interned
+  #                         Strings up to a bounded per-read cache, then falls back to frozen copies.
+  #                         A transient copy still happens per value, so it is not a per-value speedup.
+  #                       - `:shared` returns frozen, zero-copy strings backed by Rust memory for
+  #                         short, repeated, low-cardinality values. Each read returns at most the
+  #                         configured number of shared values and only values up to the configured
+  #                         byte size; values past those bounds become frozen copies. New process-wide
+  #                         leaks are also capped by the requested budget and hard process ceilings.
+  #                         All `:shared` results are frozen. Not recommended for high-cardinality or
+  #                         large-blob string columns.
+  #                       Pass a hash to set the `:shared` leak budget, e.g.
+  #                       `{ mode: :shared, max_entries: 16_384, max_value_bytes: 1024 }`.
   sig do
     params(
       input: T.any(String, File, StringIO, IO),
       result_type: T.nilable(T.any(String, Symbol)),
       columns: T.nilable(T::Array[String]),
-      strict: T.nilable(T::Boolean)
+      strict: T.nilable(T::Boolean),
+      string_storage: T.nilable(T.any(String, Symbol, T::Hash[Symbol, T.untyped]))
     ).returns(T::Enumerator[T.any(T::Hash[String, T.untyped], T::Array[T.untyped])])
   end
   sig do
@@ -32,10 +49,11 @@ module Parquet
       result_type: T.nilable(T.any(String, Symbol)),
       columns: T.nilable(T::Array[String]),
       strict: T.nilable(T::Boolean),
+      string_storage: T.nilable(T.any(String, Symbol, T::Hash[Symbol, T.untyped])),
       blk: T.nilable(T.proc.params(row: T.any(T::Hash[String, T.untyped], T::Array[T.untyped])).void)
     ).returns(NilClass)
   end
-  def self.each_row(input, result_type: nil, columns: nil, strict: nil, &blk)
+  def self.each_row(input, result_type: nil, columns: nil, strict: nil, string_storage: nil, &blk)
   end
 
   # Options:
@@ -44,13 +62,16 @@ module Parquet
   #                    ("hash" or "array" or :hash or :array)
   #   - `columns`: When present, only the specified columns will be included in the output.
   #   - `batch_size`: When present, specifies the number of rows per batch
+  #   - `string_storage`: How string values become Ruby strings (`:copy` (default), `:intern`,
+  #                       or `:shared`). See `each_row` for the semantics of each mode.
   sig do
     params(
       input: T.any(String, File, StringIO, IO),
       result_type: T.nilable(T.any(String, Symbol)),
       columns: T.nilable(T::Array[String]),
       batch_size: T.nilable(Integer),
-      strict: T.nilable(T::Boolean)
+      strict: T.nilable(T::Boolean),
+      string_storage: T.nilable(T.any(String, Symbol, T::Hash[Symbol, T.untyped]))
     ).returns(T::Enumerator[T.any(T::Hash[String, T.untyped], T::Array[T.untyped])])
   end
   sig do
@@ -60,15 +81,17 @@ module Parquet
       columns: T.nilable(T::Array[String]),
       batch_size: T.nilable(Integer),
       strict: T.nilable(T::Boolean),
+      string_storage: T.nilable(T.any(String, Symbol, T::Hash[Symbol, T.untyped])),
       blk:
         T.nilable(T.proc.params(batch: T.any(T::Hash[String, T::Array[T.untyped]], T::Array[T::Array[T.untyped]])).void)
     ).returns(NilClass)
   end
-  def self.each_column(input, result_type: nil, columns: nil, batch_size: nil, strict: nil, &blk)
+  def self.each_column(input, result_type: nil, columns: nil, batch_size: nil, strict: nil, string_storage: nil, &blk)
   end
 
   # Options:
-  #   - `read_from`: An Enumerator yielding arrays of values representing each row
+  #   - `read_from`: An Enumerable yielding arrays of values representing each row. The outer
+  #                  enumerable is pulled incrementally and is never materialized with `to_a`.
   #   - `schema`: Array of hashes specifying column names and types. Supported types:
   #     - `int8`, `int16`, `int32`, `int64`
   #     - `uint8`, `uint16`, `uint32`, `uint64`
@@ -79,20 +102,30 @@ module Parquet
   #     - `date32`
   #     - `timestamp_millis`, `timestamp_micros`
   #   - `write_to`: String path or IO object to write the parquet file to
-  #   - `batch_size`: Optional batch size for writing (defaults to 1000)
-  #   - `flush_threshold`: Optional memory threshold in bytes before flushing (defaults to 64MB)
+  #   - `batch_size`: Optional positive batch size for writing (defaults to 1000, at most 1_000_000
+  #                   for one-column schemas; wide schemas may have a lower safety cap)
+  #   - `flush_threshold`: Optional positive byte quantum for converted native values (defaults
+  #                        to 100MB). One larger row is written alone. Encoded row groups use this
+  #                        value with an 8MB minimum; completed footer metadata is disk-spooled.
   #   - `compression`: Optional compression type to use (defaults to "zstd")
   #                   Supported values: "none", "uncompressed", "snappy", "gzip", "lz4", "zstd"
-  #   - `sample_size`: Optional number of rows to sample for size estimation (defaults to 100)
+  #   - `sample_size`: Optional positive number of rows to sample for size estimation
+  #                    (defaults to 100, at most 10_000)
+  #   - `string_cache`: Deduplicate repeated string values while writing. `false` (default)
+  #                     disables it, `true` enables it with a default capacity, and an Integer
+  #                     enables it with that many retained distinct strings (at most 65_536).
+  #                     Retention also skips values larger than 4KB and stops after 16MB of
+  #                     cached string content.
   sig do
     params(
-      read_from: T::Enumerator[T::Array[T.untyped]],
+      read_from: T::Enumerable[T::Array[T.untyped]],
       schema: T::Array[T::Hash[String, String]],
       write_to: T.any(String, IO),
       batch_size: T.nilable(Integer),
       flush_threshold: T.nilable(Integer),
       compression: T.nilable(String),
-      sample_size: T.nilable(Integer)
+      sample_size: T.nilable(Integer),
+      string_cache: T.nilable(T.any(T::Boolean, Integer))
     ).void
   end
   def self.write_rows(
@@ -102,12 +135,14 @@ module Parquet
     batch_size: nil,
     flush_threshold: nil,
     compression: nil,
-    sample_size: nil
+    sample_size: nil,
+    string_cache: nil
   )
   end
 
   # Options:
-  #   - `read_from`: An Enumerator yielding arrays of column batches
+  #   - `read_from`: An Enumerable yielding arrays of column batches. Batches are validated and
+  #                  consumed one at a time; all columns in each batch must have equal length.
   #   - `schema`: Array of hashes specifying column names and types. Supported types:
   #     - `int8`, `int16`, `int32`, `int64`
   #     - `uint8`, `uint16`, `uint32`, `uint64`
@@ -119,18 +154,82 @@ module Parquet
   #     - `timestamp_millis`, `timestamp_micros`
   #     - Looks like [{"column_name" => {"type" => "date32", "format" => "%Y-%m-%d"}}, {"column_name" => "int8"}]
   #   - `write_to`: String path or IO object to write the parquet file to
-  #   - `flush_threshold`: Optional memory threshold in bytes before flushing (defaults to 64MB)
+  #   - `flush_threshold`: Optional positive byte quantum for converted native values (defaults
+  #                        to 100MB). One larger row is written alone. Encoded row groups use this
+  #                        value with an 8MB minimum; completed footer metadata is disk-spooled.
   #   - `compression`: Optional compression type to use (defaults to "zstd")
   #                   Supported values: "none", "uncompressed", "snappy", "gzip", "lz4", "zstd"
+  #   - `logger`: Optional Ruby logger for column-write progress messages
   sig do
     params(
-      read_from: T::Enumerator[T::Array[T::Array[T.untyped]]],
+      read_from: T::Enumerable[T::Array[T::Array[T.untyped]]],
       schema: T::Array[T::Hash[String, String]],
       write_to: T.any(String, IO),
       flush_threshold: T.nilable(Integer),
-      compression: T.nilable(String)
+      compression: T.nilable(String),
+      logger: T.nilable(T.untyped)
     ).void
   end
-  def self.write_columns(read_from, schema:, write_to:, flush_threshold: nil, compression: nil)
+  def self.write_columns(
+    read_from,
+    schema:,
+    write_to:,
+    flush_threshold: nil,
+    compression: nil,
+    logger: nil
+  )
+  end
+
+  # Concatenates Parquet files and re-splits them into a new set of files
+  # without translating rows through Ruby. Returns one hash per output file,
+  # `{"path" => String, "num_rows" => Integer}`, in output order.
+  #
+  # The outputs hold exactly the input rows, in input order. Every output but
+  # the last holds `rows_per_file` rows; there is always at least one output,
+  # even when the inputs are empty. Each output's Parquet schema is identical
+  # to the first input's, and that input's file-level key/value metadata (such
+  # as `ARROW:schema` or `pandas`) is carried over.
+  #
+  # Inputs must agree on leaf column shape — path, physical and logical type,
+  # nesting. They may differ in key/value metadata and Parquet field ids.
+  #
+  # Options:
+  #   - `read_from`: String path or array of paths to Parquet files with matching schemas
+  #   - `output_dir`: Directory where {output_file_prefix}-{n}.parquet files will be written
+  #   - `output_file_prefix`: Single filename component used for outputs, default "batch".
+  #     Path separators and `..` are rejected.
+  #   - `rows_per_file`: Optional maximum number of rows per output file. When nil, all input
+  #     rows are concatenated into one file.
+  #   - `max_read_rows_per_chunk`: Optional upper bound for rows read per chunk, default 8192
+  #     and reduced for wide schemas. It bounds memory only; it never changes the returned
+  #     list, the rows, the schema, or the codecs.
+  #   - `compression`: Optional codec for the outputs. When nil each column keeps its own
+  #     codec, which also lets whole row groups be copied without re-encoding.
+  #   - `overwrite`: When false (default), a non-empty `{output_file_prefix}-*.parquet` set in
+  #     `output_dir` raises ArgumentError. When true, that set is replaced and any files left
+  #     over from a longer earlier run are removed. Files outside the set are never touched.
+  #
+  # Raises ArgumentError for an invalid request or mismatched input schemas, and
+  # IOError when an input or output cannot be read or written.
+  sig do
+    params(
+      read_from: T.any(String, T::Array[String]),
+      output_dir: String,
+      output_file_prefix: T.nilable(String),
+      rows_per_file: T.nilable(Integer),
+      max_read_rows_per_chunk: T.nilable(Integer),
+      compression: T.nilable(String),
+      overwrite: T.nilable(T::Boolean)
+    ).returns(T::Array[T::Hash[String, T.any(String, Integer)]])
+  end
+  def self.repack(
+    read_from,
+    output_dir:,
+    output_file_prefix: nil,
+    rows_per_file: nil,
+    max_read_rows_per_chunk: nil,
+    compression: nil,
+    overwrite: nil
+  )
   end
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: parquet
 version: !ruby/object:Gem::Version
-  version: 0.7.3
+  version: 0.9.0
 platform: aarch64-linux
 authors:
 - Nathan Jaremko
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2025-07-05 00:00:00.000000000 Z
+date: 2026-08-30 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: bigdecimal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake-compiler
   requirement: !ruby/object:Gem::Requirement
@@ -42,6 +56,7 @@ files:
 - lib/parquet/3.2/parquet.so
 - lib/parquet/3.3/parquet.so
 - lib/parquet/3.4/parquet.so
+- lib/parquet/4.0/parquet.so
 - lib/parquet/schema.rb
 - lib/parquet/version.rb
 homepage: https://github.com/njaremko/parquet-ruby
@@ -65,7 +80,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '3.2'
   - - "<"
     - !ruby/object:Gem::Version
-      version: 3.5.dev
+      version: 4.1.dev
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="