parquet 0.7.3 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fd1247418c45b1f64165c2fddbb835adb487a2d938e31b80eb015431caa167ed
-  data.tar.gz: 07ef3cb8663a956736200121b4c99046079a52b7cb2136f2755f6fdafba2102c
+  metadata.gz: 9d6e1c27061809c378723ba1917a02d427034cbcca336cb13018b6782f02a39f
+  data.tar.gz: 8682402a2c0007384c851a7a3d5f68f2b758f11ca3b6c196fbae3153f3751848
 SHA512:
-  metadata.gz: 4b5c65cbae6d67c499e778835ba68a4bc0dd223006028e1c5ad832beedbf0c296b569af6362d0554464b512f25f3c1874e3cd18d8ada83668c92b79de4926079
-  data.tar.gz: 4d1a8e2ee202b9fc810d25307bfe3315a9ba38fd50f9d9f7a796e69bcfe7cd95904820d509a86e5792974143eacc4cfd87a7ecf66cfe421db24fbf5b8f6afe82
+  metadata.gz: 19075351b7cdc6616c896979d9b4d871aa599f8c8c0d13e52e48aa75d3df3f94f6a1824ae27b81b9321329183a5ad4b2db38647711583b0e8bf885f3f2ab0a3f
+  data.tar.gz: e3240b3224aac0e536a8c88e65cf022ded919afb6b8cd193f0f1e064934e1449a13d3ca0c36a9c559dc48a780b76e390dba45b313b4deb3a19ec4f764b61dffa

data/Cargo.lock

@@ -63,8 +63,9 @@ dependencies = [
 
 [[package]]
 name = "arrow"
-version = "55.2.0"
-source = "git+https://github.com/njaremko/arrow-rs?branch=nathan_06-24-remove_primitive_map_key_assertion_on_record_reader#54858bf019ff3faeb8f5b562da8c01012162aef0"
+version = "58.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -83,21 +84,23 @@ dependencies = [
 
 [[package]]
 name = "arrow-arith"
-version = "55.2.0"
-source = "git+https://github.com/njaremko/arrow-rs?branch=nathan_06-24-remove_primitive_map_key_assertion_on_record_reader#54858bf019ff3faeb8f5b562da8c01012162aef0"
+version = "58.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
  "arrow-data",
  "arrow-schema",
  "chrono",
- "num",
+ "num-traits",
 ]
 
 [[package]]
 name = "arrow-array"
-version = "55.2.0"
-source = "git+https://github.com/njaremko/arrow-rs?branch=nathan_06-24-remove_primitive_map_key_assertion_on_record_reader#54858bf019ff3faeb8f5b562da8c01012162aef0"
+version = "58.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e"
 dependencies = [
  "ahash",
  "arrow-buffer",
@@ -105,28 +108,34 @@ dependencies = [
  "arrow-schema",
  "chrono",
  "half",
- "hashbrown",
- "num",
+ "hashbrown 0.17.1",
+ "num-complex",
+ "num-integer",
+ "num-traits",
 ]
 
 [[package]]
 name = "arrow-buffer"
-version = "55.2.0"
-source = "git+https://github.com/njaremko/arrow-rs?branch=nathan_06-24-remove_primitive_map_key_assertion_on_record_reader#54858bf019ff3faeb8f5b562da8c01012162aef0"
+version = "58.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0"
 dependencies = [
  "bytes",
  "half",
- "num",
+ "num-bigint",
+ "num-traits",
 ]
 
 [[package]]
 name = "arrow-cast"
-version = "55.2.0"
-source = "git+https://github.com/njaremko/arrow-rs?branch=nathan_06-24-remove_primitive_map_key_assertion_on_record_reader#54858bf019ff3faeb8f5b562da8c01012162aef0"
+version = "58.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
  "arrow-data",
+ "arrow-ord",
  "arrow-schema",
  "arrow-select",
  "atoi",
@@ -134,14 +143,15 @@ dependencies = [
  "chrono",
  "half",
  "lexical-core",
- "num",
+ "num-traits",
  "ryu",
 ]
 
 [[package]]
 name = "arrow-csv"
-version = "55.2.0"
-source = "git+https://github.com/njaremko/arrow-rs?branch=nathan_06-24-remove_primitive_map_key_assertion_on_record_reader#54858bf019ff3faeb8f5b562da8c01012162aef0"
+version = "58.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de"
 dependencies = [
  "arrow-array",
  "arrow-cast",
@@ -154,53 +164,62 @@ dependencies = [
 
 [[package]]
 name = "arrow-data"
-version = "55.2.0"
-source = "git+https://github.com/njaremko/arrow-rs?branch=nathan_06-24-remove_primitive_map_key_assertion_on_record_reader#54858bf019ff3faeb8f5b562da8c01012162aef0"
+version = "58.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0"
 dependencies = [
  "arrow-buffer",
  "arrow-schema",
  "half",
- "num",
+ "num-integer",
+ "num-traits",
 ]
 
 [[package]]
 name = "arrow-ipc"
-version = "55.2.0"
-source = "git+https://github.com/njaremko/arrow-rs?branch=nathan_06-24-remove_primitive_map_key_assertion_on_record_reader#54858bf019ff3faeb8f5b562da8c01012162aef0"
+version = "58.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
  "arrow-data",
  "arrow-schema",
+ "arrow-select",
  "flatbuffers",
  "lz4_flex",
 ]
 
 [[package]]
 name = "arrow-json"
-version = "55.2.0"
-source = "git+https://github.com/njaremko/arrow-rs?branch=nathan_06-24-remove_primitive_map_key_assertion_on_record_reader#54858bf019ff3faeb8f5b562da8c01012162aef0"
+version = "58.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
  "arrow-cast",
- "arrow-data",
+ "arrow-ord",
  "arrow-schema",
+ "arrow-select",
  "chrono",
  "half",
  "indexmap",
+ "itoa",
  "lexical-core",
  "memchr",
- "num",
- "serde",
+ "num-traits",
+ "ryu",
+ "serde_core",
  "serde_json",
  "simdutf8",
 ]
 
 [[package]]
 name = "arrow-ord"
-version = "55.2.0"
-source = "git+https://github.com/njaremko/arrow-rs?branch=nathan_06-24-remove_primitive_map_key_assertion_on_record_reader#54858bf019ff3faeb8f5b562da8c01012162aef0"
+version = "58.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -211,8 +230,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-row"
-version = "55.2.0"
-source = "git+https://github.com/njaremko/arrow-rs?branch=nathan_06-24-remove_primitive_map_key_assertion_on_record_reader#54858bf019ff3faeb8f5b562da8c01012162aef0"
+version = "58.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -223,30 +243,33 @@ dependencies = [
 
 [[package]]
 name = "arrow-schema"
-version = "55.2.0"
-source = "git+https://github.com/njaremko/arrow-rs?branch=nathan_06-24-remove_primitive_map_key_assertion_on_record_reader#54858bf019ff3faeb8f5b562da8c01012162aef0"
+version = "58.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed"
 dependencies = [
- "serde",
+ "serde_core",
  "serde_json",
 ]
 
 [[package]]
 name = "arrow-select"
-version = "55.2.0"
-source = "git+https://github.com/njaremko/arrow-rs?branch=nathan_06-24-remove_primitive_map_key_assertion_on_record_reader#54858bf019ff3faeb8f5b562da8c01012162aef0"
+version = "58.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222"
 dependencies = [
  "ahash",
  "arrow-array",
  "arrow-buffer",
  "arrow-data",
  "arrow-schema",
- "num",
+ "num-traits",
 ]
 
 [[package]]
 name = "arrow-string"
-version = "55.2.0"
-source = "git+https://github.com/njaremko/arrow-rs?branch=nathan_06-24-remove_primitive_map_key_assertion_on_record_reader#54858bf019ff3faeb8f5b562da8c01012162aef0"
+version = "58.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -254,7 +277,7 @@ dependencies = [
  "arrow-schema",
  "arrow-select",
  "memchr",
- "num",
+ "num-traits",
  "regex",
  "regex-syntax",
 ]
@@ -551,6 +574,12 @@ version = "0.15.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5"
 
+[[package]]
+name = "hashbrown"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+
 [[package]]
 name = "iana-time-zone"
 version = "0.1.63"
@@ -582,7 +611,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661"
 dependencies = [
  "equivalent",
- "hashbrown",
+ "hashbrown 0.15.4",
 ]
 
 [[package]]
@@ -827,30 +856,30 @@ checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
 
 [[package]]
 name = "lz4_flex"
-version = "0.11.5"
+version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a"
+checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e"
 dependencies = [
  "twox-hash",
 ]
 
 [[package]]
 name = "magnus"
-version = "0.7.1"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d87ae53030f3a22e83879e666cb94e58a7bdf31706878a0ba48752994146dab"
+checksum = "3b36a5b126bbe97eb0d02d07acfeb327036c6319fd816139a49824a83b7f9012"
 dependencies = [
  "magnus-macros",
  "rb-sys",
- "rb-sys-env 0.1.2",
+ "rb-sys-env",
  "seq-macro",
 ]
 
 [[package]]
 name = "magnus-macros"
-version = "0.6.0"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5968c820e2960565f647819f5928a42d6e874551cab9d88d75e3e0660d7f71e3"
+checksum = "47607461fd8e1513cb4f2076c197d8092d921a1ea75bd08af97398f593751892"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1013,11 +1042,11 @@ dependencies = [
  "mimalloc",
  "num",
  "ordered-float 5.0.0",
- "parquet 55.2.0",
+ "parquet 58.3.0",
  "parquet-ruby-adapter",
  "rand",
  "rb-sys",
- "rb-sys-env 0.2.2",
+ "rb-sys-env",
  "simdutf8",
  "tempfile",
  "thiserror",
@@ -1026,13 +1055,13 @@ dependencies = [
 
 [[package]]
 name = "parquet"
-version = "55.2.0"
-source = "git+https://github.com/njaremko/arrow-rs?branch=nathan_06-24-remove_primitive_map_key_assertion_on_record_reader#54858bf019ff3faeb8f5b562da8c01012162aef0"
+version = "58.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908"
 dependencies = [
  "ahash",
  "arrow-array",
  "arrow-buffer",
- "arrow-cast",
  "arrow-data",
  "arrow-ipc",
  "arrow-schema",
@@ -1043,10 +1072,11 @@ dependencies = [
  "chrono",
  "flate2",
  "half",
- "hashbrown",
+ "hashbrown 0.17.1",
  "lz4_flex",
- "num",
  "num-bigint",
+ "num-integer",
+ "num-traits",
  "paste",
  "seq-macro",
  "serde_json",
@@ -1070,11 +1100,13 @@ dependencies = [
  "jiff",
  "num",
  "ordered-float 5.0.0",
- "parquet 55.2.0",
+ "parquet 58.3.0",
  "rand",
  "serde",
  "tempfile",
  "thiserror",
+ "thrift",
+ "triomphe",
  "uuid",
 ]
 
@@ -1090,12 +1122,13 @@ dependencies = [
  "magnus",
  "num",
  "ordered-float 5.0.0",
- "parquet 55.2.0",
+ "parquet 58.3.0",
  "parquet-core",
  "rb-sys",
- "rb-sys-env 0.2.2",
+ "rb-sys-env",
  "tempfile",
  "thiserror",
+ "triomphe",
  "uuid",
 ]
 
@@ -1190,18 +1223,18 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.116"
+version = "0.9.124"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7059846f68396df83155779c75336ca24567741cb95256e6308c9fcc370e8dad"
+checksum = "c85c4188462601e2aa1469def389c17228566f82ea72f137ed096f21591bc489"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.116"
+version = "0.9.124"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac217510df41b9ffc041573e68d7a02aaff770c49943c7494441c4b224b0ecd0"
+checksum = "568068db4102230882e6d4ae8de6632e224ca75fe5970f6e026a04e91ed635d3"
 dependencies = [
  "bindgen",
  "lazy_static",
@@ -1212,12 +1245,6 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "rb-sys-env"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a35802679f07360454b418a5d1735c89716bde01d35b1560fc953c1415a0b3bb"
-
 [[package]]
 name = "rb-sys-env"
 version = "0.2.2"
@@ -1307,18 +1334,28 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
 
 [[package]]
 name = "serde"
-version = "1.0.219"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
+dependencies = [
+ "serde_core",
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.228"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6"
+checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.219"
+version = "1.0.228"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
+checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1361,6 +1398,12 @@ version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b"
 
+[[package]]
+name = "stable_deref_trait"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
+
 [[package]]
 name = "static_assertions"
 version = "1.1.0"
@@ -1431,6 +1474,16 @@ dependencies = [
  "crunchy",
 ]
 
+[[package]]
+name = "triomphe"
+version = "0.1.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd69c5aa8f924c7519d6372789a74eac5b94fb0f8fcf0d4a97eb0bfc3e785f39"
+dependencies = [
+ "serde",
+ "stable_deref_trait",
+]
+
 [[package]]
 name = "twox-hash"
 version = "2.1.1"

data/Gemfile

@@ -16,5 +16,6 @@ end
 
 group :test do
   gem "csv"
+  gem "logger"
   gem "minitest", "~> 5.0"
 end

data/README.md

@@ -145,6 +145,33 @@ puts metadata["row_groups"].size    # Number of row groups
 
 ## Writing Parquet Files
 
+`write_rows` and `write_columns` stream an enumerable to a path or writable IO
+and return `nil`. Each row is an array in schema order. For column writing, each
+yielded batch contains one array per field, all of the same length.
+
+Both methods require `schema:` and `write_to:`. `schema:` accepts the array form
+shown below, a `Parquet::Schema` DSL result, or a `fields` schema hash. Use `nil`
+or `[]` to infer string columns named `f0`, `f1`, ... from the first row or
+batch. Empty input requires an explicit schema.
+
+`compression:` accepts `"none"`, `"uncompressed"`, `"snappy"`, `"gzip"`,
+`"lz4"`, `"zstd"`, and `"brotli"`; `nil` defaults to Snappy.
+`flush_threshold:` defaults to 100 MiB. `logger:` accepts a Ruby logger with
+`debug`, `info`, `warn`, and `error` methods.
+
+`write_rows` also accepts `batch_size:`, `sample_size:`, and `string_cache:`.
+Without `batch_size:`, sizing starts at 1,000 rows and adapts to row size; the
+cap is 1,000,000 rows and lower for wide schemas. Sampling defaults to 100 rows
+and is capped at 10,000. `string_cache: true` uses a capacity of 100, or you can
+pass a capacity up to 65,536; `nil` and `false` disable it.
+
+Path output is staged and atomically published only after the complete file has
+been written. On Unix, replacing an existing path preserves its uid, gid, and
+mode and uses standard last-committer-wins rename semantics; creating a path is
+no-clobber. Extended attributes and ACLs are not preserved. IO output is first
+staged on disk, then copied to the IO; a failed copy may leave the IO partially
+written.
+
 ### Row-wise Writing
 
 Best for: Streaming data, converting from other formats, memory-constrained environments
@@ -159,17 +186,109 @@ schema = [
 ]
 
 # Stream data from any enumerable
-rows = CSV.foreach("input.csv").map do |row|
+rows = CSV.foreach("input.csv").lazy.map do |row|
   [row[0].to_i, row[1], row[2] == "true", row[3].to_f]
 end
 
 Parquet.write_rows(rows,
   schema: schema,
   write_to: "output.parquet",
-  batch_size: 5000  # Rows per batch (default: 1000)
+  batch_size: 5000
+)
+```
+
+### Repacking Existing Parquet Files
+
+Concatenate Parquet files and re-split them into differently sized files without
+translating rows through Ruby.
+
+```ruby
+Parquet.repack(
+  ["input-0.parquet", "input-1.parquet"],
+  output_dir: "repacked",
+  rows_per_file: 100_000
 )
+# => [{ "path" => "repacked/batch-0.parquet", "num_rows" => 100_000 },
+#     { "path" => "repacked/batch-1.parquet", "num_rows" => 42_137 }]
+```
+
+The outputs hold exactly the input rows, in input order. Every output but the
+last holds `rows_per_file` rows, and there is always at least one output even
+when the inputs are empty. Omit `rows_per_file:` to concatenate everything into
+a single file.
+
+Each output's Parquet schema is identical to the first input's, and that input's
+file-level key/value metadata (`ARROW:schema`, `pandas`, and so on) is carried
+over. Inputs must agree on leaf column shape — path, physical and logical type,
+nesting — but may differ in key/value metadata and Parquet field ids.
+
+#### Compression and copying
+
+With no `compression:`, each column keeps its own codec — Parquet records one
+per column, and a file may legitimately use several. Naming a codec applies it
+to every column instead:
+
+```ruby
+Parquet.repack("input.parquet", output_dir: "out", compression: "zstd")
 ```
 
+Keeping the inputs' codecs also lets repack copy whole row groups into the
+output byte-for-byte, skipping decompression and re-encoding entirely. A row
+group is copied when it fits the output's remaining row budget, is large enough
+to be worth copying, and the request did not ask for a different codec;
+otherwise its rows are decoded and re-encoded. Both routes produce the same
+rows, so which one runs is not something callers need to reason about — but the
+copy route is dramatically faster, so a plain concatenation is close to an
+I/O-bound copy.
+
+Small row groups are deliberately merged rather than copied: copying them
+one-for-one would make a compaction of many small files reproduce exactly the
+fragmentation it was meant to remove.
+
+#### Output directory ownership
+
+`repack` owns the `{output_file_prefix}-{n}.parquet` names in `output_dir`. If
+any already exist it raises `ArgumentError` rather than mixing two runs' files
+in one directory:
+
+```ruby
+Parquet.repack("input.parquet", output_dir: "out", rows_per_file: 1000, overwrite: true)
+```
+
+`overwrite: true` replaces that set and deletes members left over from a longer
+earlier run, so the returned list always equals what a reader finds in the
+directory. Files outside the set are never touched.
+
+#### Bounds
+
+`max_read_rows_per_chunk:` (default 8192, reduced for wide schemas) bounds rows
+buffered while reading; output row groups are bounded in rows by the same slot
+budget. Both are resource controls: varying them cannot change the returned
+list, the rows, the schema, or the codecs. They do shift compressed byte counts
+and page boundaries, which are representation rather than contract.
+
+Input metadata is read one file at a time, so peak memory is set by the widest
+single file and its row-group size, not by how many files you pass or how many
+rows they hold in total. Note that the row-group bound is in rows, not bytes: a
+schema with very large values still buffers one row group's worth of encoded
+data.
+
+Reading and writing run with the GVL released, so other Ruby threads keep
+running and `Interrupt` / `Timeout` are honoured. An interrupted call leaves no
+output behind.
+
+A repacked output is capped at 32,767 row groups. Merging small groups keeps
+that limit out of reach in practice; if a single output would exceed it,
+repack raises rather than writing an unreadable file, and `rows_per_file:` is
+the way out.
+
+#### Page indexes and other optional structures
+
+Every row group in a Parquet file must agree on whether it carries a page index,
+and a copied row group can only contribute the index its source had. So an
+output carries one exactly when every contributing input does. Bloom filters are
+not carried over on either route.
+
 ### Column-wise Writing
 
 Best for: Pre-columnar data, better compression, higher performance
@@ -196,7 +315,7 @@ schema = [
 Parquet.write_columns(batches.each,
   schema: schema,
   write_to: "output.parquet",
-  compression: "snappy"  # Options: none, snappy, gzip, lz4, zstd
+  compression: "snappy"
 )
 ```
 
@@ -340,7 +459,7 @@ Parquet.write_rows(data.each, schema: schema, write_to: "complex.parquet")
 
 ### Timezone Handling in Parquet
 
-**Critical**: The Parquet specification has a fundamental limitation with timezone storage:
+The Parquet specification has a fundamental limitation with timezone storage:
 
 1. **UTC-normalized**: Any timestamp with timezone info (including "+09:00" or "America/New_York") is converted to UTC
 2. **Local/unzoned**: Timestamps without timezone info are stored as-is
@@ -382,11 +501,20 @@ Control memory usage with flush thresholds:
 Parquet.write_rows(huge_dataset.each,
   schema: schema,
   write_to: "output.parquet",
-  batch_size: 1000,              # Rows before considering flush
-  flush_threshold: 32 * 1024**2  # Flush if batch exceeds 32MB
+  batch_size: 1_000,
+  flush_threshold: 32 * 1024**2 # 32 MiB
 )
 ```
 
+Writer-owned memory stays bounded as the file grows. `flush_threshold` controls
+the converted-value buffer; a single larger row may exceed it temporarily. The
+row-group target is at least 8 MiB, and each file may contain up to 32,768 row
+groups.
+
+Encoded data and completed row-group metadata are staged on disk, so large
+writes need temporary disk space. Ruby still owns the current row or batch, and
+an in-memory destination such as `StringIO` holds the output in memory.
+
 ## Architecture
 
 This gem uses a modular architecture: