parquet 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4b40af9f0e3f469b64b66a59e050c801ab97ee7aad4927179c9923deeab278d7
-  data.tar.gz: 18cf939257c201b9485ce009194098260cfffd5b5db8f3e92d59e243b3e54b58
+  metadata.gz: 9d6e1c27061809c378723ba1917a02d427034cbcca336cb13018b6782f02a39f
+  data.tar.gz: 8682402a2c0007384c851a7a3d5f68f2b758f11ca3b6c196fbae3153f3751848
 SHA512:
-  metadata.gz: 0aad32ede402031bb3a6cb9ad617a76c0fc16e03305ad8bf0eea60db31b256cd47a0875d36f690c6cd030735646b880c70490eee461b712e1ad9c20e9cf2f847
-  data.tar.gz: 83d099424fe612c95d6f16f544594804aed5e5f8e987aa2cac8eff686852fc67c90dec747b8ea8d6137e4833faa02651f1a4f9a0f964e3c1c1d1c08064d70b4d
+  metadata.gz: 19075351b7cdc6616c896979d9b4d871aa599f8c8c0d13e52e48aa75d3df3f94f6a1824ae27b81b9321329183a5ad4b2db38647711583b0e8bf885f3f2ab0a3f
+  data.tar.gz: e3240b3224aac0e536a8c88e65cf022ded919afb6b8cd193f0f1e064934e1449a13d3ca0c36a9c559dc48a780b76e390dba45b313b4deb3a19ec4f764b61dffa

data/Cargo.lock

@@ -1105,6 +1105,7 @@ dependencies = [
  "serde",
  "tempfile",
  "thiserror",
+ "thrift",
  "triomphe",
  "uuid",
 ]

data/README.md

@@ -145,6 +145,33 @@ puts metadata["row_groups"].size    # Number of row groups
 
 ## Writing Parquet Files
 
+`write_rows` and `write_columns` stream an enumerable to a path or writable IO
+and return `nil`. Each row is an array in schema order. For column writing, each
+yielded batch contains one array per field, all of the same length.
+
+Both methods require `schema:` and `write_to:`. `schema:` accepts the array form
+shown below, a `Parquet::Schema` DSL result, or a `fields` schema hash. Use `nil`
+or `[]` to infer string columns named `f0`, `f1`, ... from the first row or
+batch. Empty input requires an explicit schema.
+
+`compression:` accepts `"none"`, `"uncompressed"`, `"snappy"`, `"gzip"`,
+`"lz4"`, `"zstd"`, and `"brotli"`; `nil` defaults to Snappy.
+`flush_threshold:` defaults to 100 MiB. `logger:` accepts a Ruby logger with
+`debug`, `info`, `warn`, and `error` methods.
+
+`write_rows` also accepts `batch_size:`, `sample_size:`, and `string_cache:`.
+Without `batch_size:`, sizing starts at 1,000 rows and adapts to row size; the
+cap is 1,000,000 rows and lower for wide schemas. Sampling defaults to 100 rows
+and is capped at 10,000. `string_cache: true` uses a capacity of 100, or you can
+pass a capacity up to 65,536; `nil` and `false` disable it.
+
+Path output is staged and atomically published only after the complete file has
+been written. On Unix, replacing an existing path preserves its uid, gid, and
+mode and uses standard last-committer-wins rename semantics; creating a path is
+no-clobber. Extended attributes and ACLs are not preserved. IO output is first
+staged on disk, then copied to the IO; a failed copy may leave the IO partially
+written.
+
 ### Row-wise Writing
 
 Best for: Streaming data, converting from other formats, memory-constrained environments
@@ -159,17 +186,109 @@ schema = [
 ]
 
 # Stream data from any enumerable
-rows = CSV.foreach("input.csv").map do |row|
+rows = CSV.foreach("input.csv").lazy.map do |row|
   [row[0].to_i, row[1], row[2] == "true", row[3].to_f]
 end
 
 Parquet.write_rows(rows,
   schema: schema,
   write_to: "output.parquet",
-  batch_size: 5000  # Positive rows per batch (default: 1000)
+  batch_size: 5000
+)
+```
+
+### Repacking Existing Parquet Files
+
+Concatenate Parquet files and re-split them into differently sized files without
+translating rows through Ruby.
+
+```ruby
+Parquet.repack(
+  ["input-0.parquet", "input-1.parquet"],
+  output_dir: "repacked",
+  rows_per_file: 100_000
 )
+# => [{ "path" => "repacked/batch-0.parquet", "num_rows" => 100_000 },
+#     { "path" => "repacked/batch-1.parquet", "num_rows" => 42_137 }]
+```
+
+The outputs hold exactly the input rows, in input order. Every output but the
+last holds `rows_per_file` rows, and there is always at least one output even
+when the inputs are empty. Omit `rows_per_file:` to concatenate everything into
+a single file.
+
+Each output's Parquet schema is identical to the first input's, and that input's
+file-level key/value metadata (`ARROW:schema`, `pandas`, and so on) is carried
+over. Inputs must agree on leaf column shape — path, physical and logical type,
+nesting — but may differ in key/value metadata and Parquet field ids.
+
+#### Compression and copying
+
+With no `compression:`, each column keeps its own codec — Parquet records one
+per column, and a file may legitimately use several. Naming a codec applies it
+to every column instead:
+
+```ruby
+Parquet.repack("input.parquet", output_dir: "out", compression: "zstd")
 ```
 
+Keeping the inputs' codecs also lets repack copy whole row groups into the
+output byte-for-byte, skipping decompression and re-encoding entirely. A row
+group is copied when it fits the output's remaining row budget, is large enough
+to be worth copying, and the request did not ask for a different codec;
+otherwise its rows are decoded and re-encoded. Both routes produce the same
+rows, so which one runs is not something callers need to reason about — but the
+copy route is dramatically faster, so a plain concatenation is close to an
+I/O-bound copy.
+
+Small row groups are deliberately merged rather than copied: copying them
+one-for-one would make a compaction of many small files reproduce exactly the
+fragmentation it was meant to remove.
+
+#### Output directory ownership
+
+`repack` owns the `{output_file_prefix}-{n}.parquet` names in `output_dir`. If
+any already exist it raises `ArgumentError` rather than mixing two runs' files
+in one directory:
+
+```ruby
+Parquet.repack("input.parquet", output_dir: "out", rows_per_file: 1000, overwrite: true)
+```
+
+`overwrite: true` replaces that set and deletes members left over from a longer
+earlier run, so the returned list always equals what a reader finds in the
+directory. Files outside the set are never touched.
+
+#### Bounds
+
+`max_read_rows_per_chunk:` (default 8192, reduced for wide schemas) bounds rows
+buffered while reading; output row groups are bounded in rows by the same slot
+budget. Both are resource controls: varying them cannot change the returned
+list, the rows, the schema, or the codecs. They do shift compressed byte counts
+and page boundaries, which are representation rather than contract.
+
+Input metadata is read one file at a time, so peak memory is set by the widest
+single file and its row-group size, not by how many files you pass or how many
+rows they hold in total. Note that the row-group bound is in rows, not bytes: a
+schema with very large values still buffers one row group's worth of encoded
+data.
+
+Reading and writing run with the GVL released, so other Ruby threads keep
+running and `Interrupt` / `Timeout` are honoured. An interrupted call leaves no
+output behind.
+
+A repacked output is capped at 32,767 row groups. Merging small groups keeps
+that limit out of reach in practice; if a single output would exceed it,
+repack raises rather than writing an unreadable file, and `rows_per_file:` is
+the way out.
+
+#### Page indexes and other optional structures
+
+Every row group in a Parquet file must agree on whether it carries a page index,
+and a copied row group can only contribute the index its source had. So an
+output carries one exactly when every contributing input does. Bloom filters are
+not carried over on either route.
+
 ### Column-wise Writing
 
 Best for: Pre-columnar data, better compression, higher performance
@@ -196,13 +315,10 @@ schema = [
 Parquet.write_columns(batches.each,
   schema: schema,
   write_to: "output.parquet",
-  compression: "snappy"  # Options: none, snappy, gzip, lz4, zstd
+  compression: "snappy"
 )
 ```
 
-`write_columns` also accepts `logger:` with the same Ruby logger interface as
-row writes.
-
 ## Data Types
 
 ### Basic Types
@@ -385,14 +501,19 @@ Control memory usage with flush thresholds:
 Parquet.write_rows(huge_dataset.each,
   schema: schema,
   write_to: "output.parquet",
-  batch_size: 1000,              # Positive rows before considering flush
-  flush_threshold: 32 * 1024**2  # Flush if batch exceeds 32MB
+  batch_size: 1_000,
+  flush_threshold: 32 * 1024**2 # 32 MiB
 )
 ```
 
-Write batch and sample sizes are bounded before buffer allocation. Very large
-batch sizes are rejected, and wide schemas have a lower effective batch cap so
-the writer cannot reserve unbounded per-column value slots.
+Writer-owned memory stays bounded as the file grows. `flush_threshold` controls
+the converted-value buffer; a single larger row may exceed it temporarily. The
+row-group target is at least 8 MiB, and each file may contain up to 32,768 row
+groups.
+
+Encoded data and completed row-group metadata are staged on disk, so large
+writes need temporary disk space. Ruby still owns the current row or batch, and
+an in-memory destination such as `StringIO` holds the output in memory.
 
 ## Architecture
 

data/ext/parquet/src/adapter_ffi.rs

@@ -275,6 +275,12 @@ pub fn write_columns(args: &[Value]) -> Result<Value, MagnusError> {
     parquet_ruby_adapter::writer::write_columns(&ruby, write_args)
 }
 
+pub fn repack(args: &[Value]) -> Result<Value, MagnusError> {
+    let ruby = Ruby::get().expect("Ruby FFI entry point runs while the Ruby GVL is held");
+
+    parquet_ruby_adapter::repack::repack(&ruby, args)
+}
+
 fn reject_row_only_column_write_options(
     write_args: &parquet_ruby_adapter::types::ParquetWriteArgs,
 ) -> Result<(), MagnusError> {

data/ext/parquet/src/lib.rs

@@ -3,7 +3,7 @@ mod allocator;
 
 use magnus::{function, method, Error, Ruby};
 
-use crate::adapter_ffi::{each_column, each_row, write_columns, write_rows};
+use crate::adapter_ffi::{each_column, each_row, repack, write_columns, write_rows};
 use parquet_ruby_adapter::metadata::parse_metadata;
 
 /// Initializes the Ruby extension and defines methods.
@@ -19,6 +19,7 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     module.define_module_function("each_column", method!(each_column, -1))?;
     module.define_module_function("write_rows", function!(write_rows, -1))?;
     module.define_module_function("write_columns", function!(write_columns, -1))?;
+    module.define_module_function("repack", function!(repack, -1))?;
 
     Ok(())
 }

data/ext/parquet-core/Cargo.toml

@@ -16,9 +16,8 @@ ordered-float = "5.0.0"
 parquet = { version = "58.3.0", features = ["arrow", "zstd", "lz4", "snap"] }
 rand = "0.9.1"
 serde = { version = "1.0", features = ["derive"] }
+tempfile = "3.8"
 thiserror = "2.0"
+thrift = { version = "0.17", default-features = false }
 triomphe = "0.1.15"
 uuid = { version = "1.0", features = ["v4"] }
-
-[dev-dependencies]
-tempfile = "3.8"

data/ext/parquet-core/src/lib.rs

@@ -46,6 +46,7 @@ pub mod arrow_conversion;
 pub mod error;
 pub mod reader;
 pub mod schema;
+mod streaming_file_writer;
 pub mod traits;
 pub mod value;
 pub mod writer;
@@ -57,4 +58,6 @@ pub use error::{ErrorContext, ParquetError, Result};
 pub use reader::Reader;
 pub use schema::{PrimitiveType, Repetition, Schema, SchemaBuilder, SchemaNode};
 pub use value::ParquetValue;
-pub use writer::{Writer, WriterBuilder, MAX_BATCH_SIZE, MAX_SAMPLE_SIZE};
+pub use writer::{
+    max_batch_size_for_column_count, Writer, WriterBuilder, MAX_BATCH_SIZE, MAX_SAMPLE_SIZE,
+};