herringbone 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 36e3b2444245fc90b36b577284aab2a81e4991ff04e30dd781f9ba94df43ae90
-  data.tar.gz: 9e2e4dbe2ab635a03aa743206cd5e6d74d6039cc5e790cefae8b04a7afd8304b
+  metadata.gz: e9b95c84373ccf0215789628229613c8899227be711dc204c31be6778352f2d6
+  data.tar.gz: 9314fc6d6d55660e210db0fe62369d9732e33dc3910e306f500b3ebc5c7090e9
 SHA512:
-  metadata.gz: 162e38811c7e6371ed61a777fddcfa8e1e811de5225842e13dad1cf67da20490c7b7eb2347eaeb6515322a6ace971d453fa4ce1556cc75f4e70acfdcda7a9179
-  data.tar.gz: 0e5798748be214eb32a7939799ecb7a885283099908c05003065a98569c0572dc17a25fd4e6d017c5635256cefdd3b0db5072e28c1536fe4e26867c1d019eca0
+  metadata.gz: 63292b6fde84386d41068903db5267fe7fa94b15c8c191d12c229f888cd94bfd1e6c78e3d01ed5f562643565e8126e9ee2f372e80ab96e08609293afeff1c3bf
+  data.tar.gz: e9946d5f2ac156222b3bca852f5fc26194ec304fa2b150fd9520311b7e3a323af3ba28e1662bc3b47e593508b22b216e148cf6af67d55318830272f1d87578e5

data/CHANGELOG.md

@@ -2,6 +2,46 @@
 
 ## Unreleased
 
+## 0.7.0
+
+- `Herringbone.redact` and `Herringbone::Redaction#apply` take a `Herringbone::Reader` as well as an
+  IO, using the Reader's decryption.
+- `Herringbone::Schema#union` (`+`) and `#intersect` (`&`) match fields by name at every level of
+  nesting and widen their types without loss (int32 + int64 -> int64, uint32 + int8 -> int64,
+  int32 + float -> double, millis + micros -> micros, string + binary -> binary).
+- `Herringbone::IncompatibleSchema` lists every field two schemas cannot agree on, at once.
+- `Herringbone::Schema#==` compares structure: field names, order, nullability, types, field ids
+  and nesting.
+- `Herringbone.combine(ios_or_readers, output_io)` concatenates Parquet files, copying their column
+  chunks byte for byte wherever the stored values stay the same. `schema: :union` fills fields an
+  input lacks with nulls and widens the others, `schema: :intersect` keeps the fields all inputs
+  have.
+- `Herringbone.combine` checks every input before writing and raises one
+  `Herringbone::IncompatibleSchema` naming every input and field that does not fit.
+- `Herringbone.combine` encrypts the output like the encrypted inputs unless `encryption:` says
+  otherwise.
+- `Herringbone::Combiner::Report#inputs` says which fields of each input were filled with nulls,
+  widened or dropped.
+- `herringbone combine INPUT... --output=FILE` does the same from the command line, with
+  `--schema=union|intersect`, and `--encrypt-key=KEY` or `--plaintext` to encrypt the output
+  otherwise.
+- `Herringbone.redact` raises `Herringbone::FormatError` for a corrupt page header in a row group it
+  copies, instead of retrying forever.
+- Writing allocates less: compressed pages are no longer copied before they reach the IO, and GZIP,
+  ZSTD and pure-Ruby Snappy compress a page's levels and values without joining them first.
+- Writing metadata whose integers don't fit their Thrift field raises `Herringbone::Thrift::Error`
+  instead of producing a file Herringbone itself cannot read back.
+- Column names and key-value metadata in other encodings are converted to UTF-8 on write, and
+  ones that aren't valid UTF-8 raise `Herringbone::Thrift::Error`.
+- **Breaking:** `Herringbone::Reader#io` returns a `Herringbone::RestrictedReadableIO` wrapping the
+  given IO, which answers only `#read`, `#seek`, `#pos`, `#size` and `#path`.
+
+## 0.6.1
+
+- `Herringbone::Reader` raises `Herringbone::FormatError` for forged metadata or page headers that
+  made it loop endlessly, overflow the stack or raise `ArgumentError`, `RangeError` or
+  `NoMemoryError`.
+
 ## 0.6.0
 
 - `require "herringbone"` loads the rest of the gem on first use: under 1 ms instead of about

data/MANUAL.md

@@ -33,6 +33,9 @@ The full reference for Herringbone. For a quick start, see the [README](README.m
   - [What gets erased, and what doesn't](#what-gets-erased-and-what-doesnt)
   - [Encrypted files](#encrypted-files)
   - [Recipes](#recipes)
+- [Combining files](#combining-files)
+  - [Comparing and uniting schemas](#comparing-and-uniting-schemas)
+  - [Combining encrypted files](#combining-encrypted-files)
 - [Type mapping](#type-mapping)
 - [Inspecting files](#inspecting-files)
 - [Command line](#command-line)
@@ -520,22 +523,22 @@ nothing touches are copied byte for byte.
 
 ```ruby
 # Forget me: remove the rows
-Herringbone.redact(input, output) do |r|
+Herringbone.redact(io, output_io) do |r|
   r.where(user_id: 42).delete
 end
 
 # Forget me, but keep the row for accounting: blank the personal columns
-Herringbone.redact(input, output) do |r|
+Herringbone.redact(io, output_io) do |r|
   r.where(user_id: 42).replace(email: nil, name: nil, address: nil)
 end
 
 # Forget me, but keep the row linkable: pseudonymize just that user's email
-Herringbone.redact(input, output) do |r|
+Herringbone.redact(io, output_io) do |r|
   r.where(user_id: 42).replace(:email) { |email| OpenSSL::HMAC.hexdigest("SHA256", KEY, email) }
 end
 
 # Pseudonymize a column across the whole file, mask another, drop a third
-Herringbone.redact(input, output) do |r|
+Herringbone.redact(io, output_io) do |r|
   r.replace(:email) { |email| OpenSSL::HMAC.hexdigest("SHA256", KEY, email.downcase) }
   r.replace(:phone) { |phone| phone && "***#{phone[-3..]}" }
   r.drop :ssn, :ip_address
@@ -575,8 +578,8 @@ forget = Herringbone::Redaction.new do |r|
   r.where(created_at: ..2.years.ago).delete      # retention works the same way
 end
 
-if forget.affects?(input)      # statistics and bloom filters first, then the where columns only
-  report = forget.apply(input, output)
+if forget.affects?(io)         # statistics and bloom filters first, then the where columns only
+  report = forget.apply(io, output_io)
   report.rows_deleted          # => 3
   report.row_groups            # => { copied: 61, rewritten: 1 }
 end
@@ -588,7 +591,16 @@ The `Redaction::Report` that `apply` returns has `rows_read`, `rows_deleted`, `r
 ### Input and output
 
 `apply` and `Herringbone.redact` take IOs like `Reader` and `Writer` do: the input must be
-seekable, the output only needs `#write`, and neither is closed. To redact in place, write to a
+seekable, the output only needs `#write`, and neither is closed. Herringbone doesn't open files by
+path, so open them yourself. The input can also be a `Herringbone::Reader` you already have, and
+then its IO and its decryption are used. Whatever `keys:` or `time_zone:` it was built with, the
+redaction reads values the same way (and blocks get rows with String keys):
+
+```ruby
+reader = Herringbone::Reader.new(io, keys: :symbol, decryption: {keys: kms_lookup})
+Herringbone.redact(reader, output_io) { |r| r.where(user_id: 42).delete }
+```
+ To redact in place, write to a
 temporary file and rename it over the original; on S3, read the object and write the new one
 through `upload_stream`.
 
@@ -619,10 +631,11 @@ there), and other files with the same person in them need their own pass.
 Pass the input's keys in `decryption:`. The output is then encrypted the same way: same algorithm,
 footer mode, AAD prefix, keys and key metadata, for the columns that are kept. `encryption:` (as
 for `Writer`) writes it with other settings, and `encryption: false` writes a plaintext file.
-`Redaction#affects?` takes `decryption:` too.
+`Redaction#affects?` takes `decryption:` too. A `Herringbone::Reader` brings its own keys, so
+`decryption:` is not given with one.
 
 ```ruby
-Herringbone.redact(input, output, decryption: {keys: kms_lookup}) do |r|
+Herringbone.redact(io, output_io, decryption: {keys: kms_lookup}) do |r|
   r.where(user_id: 42).delete
 end
 ```
@@ -639,7 +652,7 @@ rotate or destroy it to cut the link:
 ```ruby
 require "openssl"
 KEY = ENV.fetch("PSEUDONYM_KEY")
-Herringbone.redact(input, output) do |r|
+Herringbone.redact(io, output_io) do |r|
   r.replace(:email) { |email| email && OpenSSL::HMAC.hexdigest("SHA256", KEY, email.downcase) }
 end
 ```
@@ -647,7 +660,7 @@ end
 Masking keeps enough to be recognizable to the person but not to anyone else:
 
 ```ruby
-Herringbone.redact(input, output) do |r|
+Herringbone.redact(io, output_io) do |r|
   r.replace(:card_number) { |number| number && number[-4..].rjust(number.size, "*") }
   r.replace(:email) { |email| email&.sub(/\A(.).*@/, '\1***@') }
   r.replace(:birth_date) { |date| date && Date.new(date.year, 1, 1) }
@@ -660,7 +673,7 @@ original value to get the same fake for the same person in every file:
 ```ruby
 require "faker"
 require "zlib"
-Herringbone.redact(input, output) do |r|
+Herringbone.redact(io, output_io) do |r|
   r.replace(:name) do |name|
     next nil unless name
     Faker::Config.random = Random.new(Zlib.crc32(name))
@@ -672,6 +685,132 @@ Herringbone.redact(input, output) do |r|
 end
 ```
 
+## Combining files
+
+`Herringbone.combine` concatenates Parquet files into one: every row group of every input, in
+order. The inputs are an Array (or any Enumerable) of seekable IOs or `Herringbone::Reader`s, and
+the output any IO with `#write`, like for `Reader` and `Writer`. Herringbone never opens files by
+path, and closes nothing it was given.
+
+```ruby
+paths = %w[2026-01.parquet 2026-02.parquet 2026-03.parquet]
+ios = paths.map { |path| File.open(path, "rb") }
+File.open("2026-q1.parquet", "wb") { |output_io| Herringbone.combine(ios, output_io) }
+ios.each(&:close)
+```
+
+Every input is read twice: its footer first, while `combine` works out the output schema and
+checks every input against it, then its column chunks. Keep the IOs open until `combine` returns,
+and mind the limit on open files (256 by default on macOS) when combining very many of them.
+
+When the inputs have the same schema, nothing is decoded: the column chunks are copied byte for
+byte, with their statistics, page indexes and bloom filters, and only the offsets pointing at them
+are rebased. Row groups keep their boundaries, so a hundred small files make a file of a hundred
+small row groups; to merge row groups, read the files and write the rows with `Writer` instead.
+
+The footer's key/value metadata is the first input's (without `ARROW:schema` and `pandas` when the
+output schema is another than the first input's), unless `metadata:` is given. Writer options
+(`compression:`, `page_rows:`...) apply to the column chunks that have to be encoded again;
+`row_group_bytes:` and `row_group_rows:` are refused. Those chunks keep the codec of the original
+chunk, and get a bloom filter if they had one.
+
+### Inputs with different schemas
+
+`schema:` says what the output holds:
+
+| `schema:`      | Output schema                                                    | Inputs                                       |
+|----------------|------------------------------------------------------------------|----------------------------------------------|
+| not given      | the schema of the first input                                    | must all have it (`Schema#==`)               |
+| `:union`       | every field of every input (`Schema#union`)                      | fields an input lacks are written as nulls   |
+| `:intersect`   | the fields all inputs have (`Schema#intersect`)                  | the other fields are dropped                 |
+| a `Schema`     | as given                                                         | must each fit it: the same fields or fewer, the same types or narrower |
+
+```ruby
+Herringbone.combine([old, new], output_io, schema: :union)
+Herringbone.combine([old, new], output_io, schema: :intersect)
+```
+
+Fields are matched by name at every level: struct members, list elements, map keys and values. A
+field one input lacks, or has nullable, is nullable in a union. The types of fields both have are
+widened to one that holds the values of both without loss:
+
+| Types                                    | Widened to                                  |
+|------------------------------------------|---------------------------------------------|
+| two signed or two unsigned integers      | the wider one                               |
+| unsigned and signed integer              | signed, twice as wide: `uint32 + int8` -> `int64` |
+| two floats                               | the wider one                               |
+| integer and float                        | the narrowest float holding every integer exactly: `int16 + float` -> `float`, `int32 + float` -> `double` |
+| timestamps (or times) of different units | the finer unit: `millis + micros` -> `micros` |
+| string, enum, json                       | string                                      |
+| any of those and binary                  | binary                                      |
+
+Anything else does not fit: `uint64` with a signed integer, `int64` with any float, decimals of
+another precision or scale, UTC with local timestamps, dates with timestamps, a struct with a
+list, fields with different field ids, a bare `repeated` field only some inputs have (it cannot be
+null), and inputs with no fields in common.
+
+Column chunks are copied wherever the stored bytes stay the same, which includes widenings of the
+annotation alone (`int8` into `int32`, `uint16` into `int32`, `string` into `binary`) and list
+elements named another way (pyarrow's `list.item`). Columns whose physical type, time unit or
+nullability changes (`int32` into `int64`, `millis` into `micros`, required into nullable) are
+encoded again from their values, and so are struct members that lose their siblings to an
+intersection.
+
+`combine` returns a `Combiner::Report`: `rows`, `row_groups` (`{ copied:, rewritten: }`) and
+`inputs`, which says per input which fields were `filled` with nulls, `widened` or `dropped`:
+
+```ruby
+report = Herringbone.combine(ios, output_io, schema: :union)
+report.inputs.each { |input| puts "#{input.name}: nulls in #{input.filled.join(", ")}" if input.filled.any? }
+```
+
+### When inputs do not fit
+
+Every input is checked before anything is written, and `Herringbone::IncompatibleSchema` names
+every input and every field that does not fit at once. Inputs are named by their path when the
+IO has one:
+
+```
+2 of 4 inputs have another schema than input 0 (2026-01.parquet):
+
+  input 2 (2026-03.parquet)
+    address.zip  int32 in input 0, string here
+
+  input 3 (2026-04.parquet)
+    id     required in input 0, nullable here
+    email  missing here, input 0 has it
+    mail   only here, input 0 lacks it
+
+Combine files with the same schema, or pass schema: :union to fill the fields an input lacks
+with nulls and widen the others, or schema: :intersect to keep only the fields all inputs have.
+```
+
+`error.conflicts` has the same as `Conflict`s (`path`, `left`, `right`, `reason`, and `input`, the
+position of the input). `Schema#union` and `Schema#intersect` raise the same error class, with
+`input` nil.
+
+### Combining encrypted files
+
+Pass the keys of encrypted inputs in `decryption:`, as for `Reader` (`[key, older_key]` lets each
+file find its own), or pass Readers, which bring their own keys. An encrypted column chunk can't
+be copied, since its encryption is tied to the file and to its position in it, so it is decrypted
+and encoded again.
+
+The output is encrypted like the encrypted inputs: the same algorithm, footer mode, footer key and
+key metadata, and the same key for each column. Plaintext inputs in the same call get that
+encryption too, so nothing gets decrypted into a plaintext file by accident. The AAD prefix the
+inputs share is kept.
+
+```ruby
+Herringbone.combine(ios, output_io, decryption: key)                     # encrypted with key, like the inputs
+Herringbone.combine(ios, output_io, decryption: key, encryption: new_key) # re-keyed
+Herringbone.combine(ios, output_io, decryption: key, encryption: false)   # plaintext, on purpose
+```
+
+Inputs encrypted differently (another footer key, algorithm, footer mode or AAD prefix, a column
+encrypted with different keys) have no single "like the inputs", so `combine` raises `ArgumentError` listing
+how they differ, and `encryption:` decides.
+
 ## Type mapping
 
 | Parquet | Ruby |
@@ -725,10 +864,18 @@ bin/herringbone inspect FILE [--pages]                 # text report (--pages li
 bin/herringbone inspect FILE --format=json             # everything as JSON
 bin/herringbone inspect FILE --format=html             # the HTML page, opened in your browser
 bin/herringbone inspect FILE --format=html > out.html  # the HTML page, saved
+bin/herringbone combine A B C --output=all.parquet     # concatenate files with the same schema
+bin/herringbone combine A B --schema=union > all.parquet      # fill fields an input lacks with nulls
+bin/herringbone combine A B --schema=intersect > all.parquet  # keep the fields all inputs have
 ```
 
 Add `--verify-checksums` to any `inspect` form to check page CRCs.
 
+`combine` does what [`Herringbone.combine`](#combining-files) does and prints how many row groups
+it copied and encoded again. Without `--output` it writes to stdout when that is redirected.
+`--compression=CODEC` sets the codec of the chunks it encodes again. If it fails, the output file
+is removed rather than left without a footer, and an output that is also an input is refused.
+
 Encrypted files take keys on the command line: as they are (picked by their fingerprint id, like
 a keyring), or for the footer, a column path or the key id the file stores (which `inspect`
 shows):
@@ -743,6 +890,8 @@ bin/herringbone inspect FILE --aad-prefix=orders/part-0 --no-prompt
 Keys are hex, `base64:...` or `raw:...` (a 16, 24 or 32-character key that isn't valid hex is
 taken as typed too). A key the file needs but wasn't given is asked for on stdin, without echo in
 a terminal; an empty answer leaves that column unreadable. `--no-prompt` asks for nothing.
+`combine` reads encrypted inputs with the same flags, and encrypts the output like them;
+`--encrypt-key=KEY` encrypts it with one key instead, and `--plaintext` writes it decrypted.
 
 `--format` is `text` (the default), `json` or `html`; `--pages` only goes with `text`.
 In a terminal, `--format=html` writes the page to a temp file, prints its path and opens it with

data/README.md

@@ -13,7 +13,7 @@ A pure-Ruby reader and writer for [Apache Parquet](https://parquet.apache.org/)
 
 This README covers the common cases. The [MANUAL](MANUAL.md) has everything else: reader and
 writer options, schemas and column types, bloom filters, Numo arrays, encryption, the redaction
-rules, type mapping and the inspector.
+rules, combining files, type mapping and the inspector.
 
 ## Installation
 
@@ -125,20 +125,22 @@ footers and which tools read what.
 
 `Herringbone.redact` rewrites a file with rows removed or values replaced, for GDPR "forget me"
 requests and pseudonymization. The parts of the file nothing touches are copied byte for byte.
+It reads from a seekable IO or from a `Herringbone::Reader`, which brings its own decryption, and
+writes to any IO with `#write`.
 
 ```ruby
 # Forget me: remove the rows
-Herringbone.redact(input, output) do |r|
+Herringbone.redact(io, output_io) do |r|
   r.where(user_id: 42).delete
 end
 
 # Forget me, but keep the row for accounting: blank the personal columns
-Herringbone.redact(input, output) do |r|
+Herringbone.redact(io, output_io) do |r|
   r.where(user_id: 42).replace(email: nil, name: nil, address: nil)
 end
 
 # Pseudonymize a column across the whole file, mask another, drop a third
-Herringbone.redact(input, output) do |r|
+Herringbone.redact(io, output_io) do |r|
   r.replace(:email) { |email| OpenSSL::HMAC.hexdigest("SHA256", KEY, email.downcase) }
   r.replace(:phone) { |phone| phone && "***#{phone[-3..]}" }
   r.drop :ssn, :ip_address
@@ -147,6 +149,19 @@ end
 
 See [Redaction](MANUAL.md#redaction) for reusable redactions, reports and recipes.
 
+## Combining files
+
+`Herringbone.combine` concatenates files with the same schema by copying their row groups byte for
+byte, without decoding anything:
+
+```ruby
+Herringbone.combine([january, february, march], output_io)
+```
+
+Files with different columns combine with `schema: :union`, which fills the gaps with nulls and
+widens types (`int32` and `int64` to `int64`), or `schema: :intersect`, which keeps the fields all
+of them have. See [Combining files](MANUAL.md#combining-files).
+
 ## Looking inside a file
 
 ```

data/bin/herringbone

@@ -9,6 +9,12 @@
 #   herringbone inspect FILE --format=html      write a self-contained HTML page showing the file's layout
 #                                               and open it in the browser (printed when stdout is redirected)
 #   add --verify-checksums to any inspect form to read page bodies and check their CRCs
+#   herringbone combine INPUT... --output=FILE  concatenate files with the same schema, copying their
+#                                               row groups (to stdout when it is redirected and
+#                                               --output is not given); --schema=union fills fields an
+#                                               input lacks with nulls, --schema=intersect keeps only
+#                                               the fields all inputs have, --compression=CODEC sets
+#                                               the codec of the chunks that are encoded again
 # `inspect` reads only the footer, page headers and page indexes: no values are decompressed.
 #
 # Encrypted files: both commands take --key=KEY (a key matched by its fingerprint id, or used for
@@ -17,6 +23,8 @@
 # base64:..., or raw:... for the bytes as typed (a 16, 24 or 32-character String that is not
 # valid hex is taken as is too). Keys that are needed but not given are asked for on stdin
 # (without echo in a terminal; an empty answer skips the key), unless --no-prompt is given.
+# `combine` takes the same flags for its inputs, and encrypts the output like the encrypted inputs;
+# --encrypt-key=KEY encrypts it with that key instead, and --plaintext not at all.
 $LOAD_PATH.unshift File.expand_path("../lib", __dir__)
 require "herringbone"
 require "json"
@@ -110,13 +118,16 @@ def decryption_from(key_flags, prompt)
     end
   end
   decryption[:footer_key] ||= plain.first.bytes if plain.size == 1
-  decryption[:keys] = ->(metadata, owner) { by_metadata.fetch(metadata) { prompt ? ask_key(metadata, owner) : nil } }
+  asked = Hash.new { |h, (metadata, owner)| h[[metadata, owner]] = ask_key(metadata, owner) } # combine's inputs may share keys
+  decryption[:keys] = ->(metadata, owner) { by_metadata.fetch(metadata) { asked[[metadata, owner]] if prompt } }
   decryption
 end
 
 USAGE = "usage: herringbone cat FILE [N] [KEY OPTIONS]\n" \
   "       herringbone inspect FILE [--format=text|json|html] [--pages] [--verify-checksums] [KEY OPTIONS]\n" \
   "       (--pages only with --format=text)\n" \
+  "       herringbone combine INPUT... [--output=FILE] [--schema=union|intersect] [--compression=CODEC]\n" \
+  "       [--encrypt-key=KEY | --plaintext] [KEY OPTIONS]\n" \
   "key options, for encrypted files: --key=KEY --key=KEY_ID=KEY --footer-key=KEY --column-key=PATH=KEY\n" \
   "       --aad-prefix=PREFIX --no-prompt (KEY is hex, base64:... or raw:...)"
 
@@ -135,15 +146,54 @@ when "inspect"
   path && limit.nil? && format_flags.size <= 1 && %w[text json html].include?(format) &&
     (other_flags - %w[--pages --verify-checksums]).empty? &&
     (format == "text" || !other_flags.include?("--pages"))
+when "combine"
+  valued, other_flags = flags.partition { |f| f.start_with?("--output=", "--compression=", "--encrypt-key=", "--schema=") }
+  options = valued.to_h { |f| f.delete_prefix("--").split("=", 2) }
+  plaintext = other_flags.delete("--plaintext")
+  !args.empty? && options.size == valued.size && other_flags.empty? && !(plaintext && options["encrypt-key"]) &&
+    [nil, "union", "intersect"].include?(options["schema"]) &&
+    (options.key?("output") ? !options["output"].empty? : !$stdout.tty?)
 end
 unless valid
   warn USAGE
   exit 1
 end
 
+# Writes the inputs into one file, removing the output again when that fails
+# @return [Herringbone::Combiner::Report]
+def combine(paths, options, plaintext, decryption)
+  output_path = options["output"]
+  if output_path && paths.any? { |p| File.expand_path(p) == File.expand_path(output_path) || (File.exist?(output_path) && File.identical?(p, output_path)) }
+    raise ArgumentError, "#{output_path} is one of the inputs"
+  end
+  writer_options = {}
+  writer_options[:compression] = options["compression"].to_sym if options["compression"]
+  writer_options[:encryption] = Herringbone::Key.new(parse_key(options["encrypt-key"], "--encrypt-key")) if options["encrypt-key"]
+  writer_options[:encryption] = false if plaintext
+  inputs = paths.map { |p| File.open(p, "rb") }
+  output = output_path ? File.open(output_path, "wb") : $stdout.binmode
+  Herringbone.combine(inputs, output, schema: options["schema"]&.to_sym, decryption: decryption, **writer_options)
+rescue Exception # standard:disable Lint/RescueException
+  # A file without its footer is of no use to anyone
+  if output_path && output
+    output.close
+    File.delete(output_path)
+  end
+  raise
+ensure
+  output.close if output_path && output && !output.closed?
+  inputs&.each(&:close)
+end
+
 # Parquet, file and key errors are reported in one line, without a backtrace
 begin
   decryption = decryption_from(key_flags, prompt)
+  if command == "combine"
+    report = combine(args, options, plaintext, decryption)
+    copied, rewritten = report.row_groups.values_at(:copied, :rewritten)
+    warn "herringbone: #{report.rows} rows, #{copied} row groups copied, #{rewritten} encoded again"
+    exit
+  end
   File.open(path, "rb") do |io|
     if command == "cat"
       n = limit&.to_i
@@ -171,5 +221,10 @@ begin
 rescue Herringbone::Error, SystemCallError, ArgumentError => e
   warn "herringbone: #{e.message}"
   warn "herringbone: give keys with --footer-key=KEY, --column-key=PATH=KEY or --key=KEY_METADATA=KEY" if e.is_a?(Herringbone::DecryptionError)
+  warn "herringbone: give --encrypt-key=KEY to encrypt the output, or --plaintext" if e.message.include?("Pass encryption: for the output")
+  if e.is_a?(Herringbone::IncompatibleSchema) && command == "combine"
+    warn "herringbone: give --schema=union to fill the fields an input lacks with nulls, or --schema=intersect to keep " \
+      "only the fields all inputs have"
+  end
   exit 1
 end

data/lib/herringbone/active_record.rb

@@ -56,6 +56,8 @@ module Herringbone
     #   hstore    map<string, string>
     #   anything else (string, text, citext, inet, cidr, macaddr, ...) -> string
     #   Postgres arrays (column.array, or a sql_type ending in "[]") -> list of the element type
+    #
+    # @api private
     module ActiveRecordMapping
       module_function
 

data/lib/herringbone/bloom_filter.rb

@@ -12,6 +12,8 @@ module Herringbone
   # like the writer converts them (the column's encoder), so a Date, Time, BigDecimal or UUID
   # String hashes the same bytes as the stored value. Nulls are never in a bloom filter.
   # The writer builds them (bloom_filters: option) and reads with where: consult them.
+  #
+  # @api private
   class BloomFilter
     # The spec's eight salt constants, one per word of a block
     SALT = [0x47b6137b, 0x44974d91, 0x8824ad5b, 0xa2b7289d, 0x705495c7, 0x2df1424b, 0x9efc4947, 0x5c6bfb31].freeze

data/lib/herringbone/byte_values.rb

@@ -11,6 +11,8 @@ module Herringbone
   # It starts out in dictionary mode (when allowed) and switches to raw bytes once the dictionary
   # grows too large or too many values turn out to be distinct. Strings are only rebuilt when the
   # row group is flushed, one column at a time.
+  #
+  # @api private
   class ByteValues
     # Values in the dictionary may take this many bytes before switching to raw bytes
     MAX_DICTIONARY_BYTES = 1024 * 1024

data/lib/herringbone/codecs/lz4.rb

@@ -4,6 +4,8 @@ module Herringbone
   module Codecs
     # Pure-Ruby LZ4: raw block format (Parquet LZ4_RAW), Hadoop-framed blocks
     # (Parquet's deprecated LZ4) and a decoder for the LZ4 frame format.
+    #
+    # @api private
     module LZ4
       # Raised for corrupt, truncated or unsupported LZ4 input.
       class Error < StandardError; end
@@ -101,11 +103,13 @@ module Herringbone
 
       # Compress into a single raw LZ4 block.
       # @param input [String] bytes to compress
+      # @param skip [Integer] zero bytes to put before the block, for a header filled in later
       # @return [String] raw LZ4 block in ASCII-8BIT
-      def compress_block(input)
+      def compress_block(input, skip: 0)
         src = binary(input)
         n = src.bytesize
-        out = String.new(capacity: n + (n / 255) + 16, encoding: BINARY)
+        out = String.new(capacity: skip + n + (n / 255) + 16, encoding: BINARY)
+        out << ("\0" * skip) if skip > 0
         anchor = 0
 
         if n > MFLIMIT
@@ -171,8 +175,9 @@ module Herringbone
       # @return [String] Hadoop-framed LZ4 data in ASCII-8BIT
       def compress_hadoop(input)
         src = binary(input)
-        block = compress_block(src)
-        [src.bytesize, block.bytesize].pack("NN") << block
+        out = compress_block(src, skip: HADOOP_PREFIX)
+        out[0, HADOOP_PREFIX] = [src.bytesize, out.bytesize - HADOOP_PREFIX].pack("NN")
+        out
       end
 
       # -- internals --

data/lib/herringbone/codecs/lzo.rb

@@ -8,6 +8,8 @@ module Herringbone
     #
     # Written from the bitstream description in the Linux kernel's
     # Documentation/staging/lzo.rst, not from the (GPL) LZO sources.
+    #
+    # @api private
     module LZO
       # Raised for corrupt or truncated LZO input.
       class Error < StandardError; end

data/lib/herringbone/codecs/snappy.rb

@@ -14,6 +14,8 @@ module Herringbone
     #
     # 32-bit loads are done with four getbyte calls rather than unpack1(offset:) to stay
     # compatible with Ruby 3.0 - the speed difference on MRI is marginal.
+    #
+    # @api private
     module Snappy
       # Raised for corrupt or truncated Snappy input, and for input too large to compress.
       class Error < StandardError; end
@@ -227,32 +229,72 @@ module Herringbone
         out
       end
 
-      # @param input [String] bytes to compress
+      # Several Strings are compressed as their concatenation without joining them: fragments are
+      # compressed straight out of the part holding them, and only a fragment straddling two parts
+      # is copied out. The output is the same as for the joined input.
+      #
+      # @param input [String, Array<String>] bytes to compress, or the parts of them
       # @return [String] raw snappy block in ASCII-8BIT
       # @raise [Error] if the input is larger than MAX_UNCOMPRESSED
       def compress(input)
-        src = (input.encoding == Encoding::BINARY) ? input : input.b
-        n = src.bytesize
+        parts = input.is_a?(Array) ? input.map { |part| binary(part) } : [binary(input)]
+        n = parts.sum(&:bytesize)
         # Checked for libsnappy too: it would truncate the 32-bit length preamble without complaint
         raise Error, "input too large for snappy" if n > MAX_UNCOMPRESSED
         if (lib = native)
-          return lib.deflate(src)
+          return lib.deflate((parts.size == 1) ? parts.first : parts.join)
         end
 
         out = String.new(capacity: 32 + n + n / 6, encoding: Encoding::BINARY)
         write_varint(out, n)
         table = Array.new(1 << HASH_BITS, 0)
+        part_index = 0
+        offset = 0
         start = 0
         while start < n
           len = n - start
           len = BLOCK_SIZE if len > BLOCK_SIZE
           table.fill(0)
-          compress_block(src, start, len, out, table)
+          part_index, offset = next_part(parts, part_index, offset)
+          part = parts[part_index]
+          if offset + len <= part.bytesize
+            compress_block(part, offset, len, out, table)
+            offset += len
+          else
+            fragment = String.new(capacity: len, encoding: Encoding::BINARY)
+            while fragment.bytesize < len
+              part_index, offset = next_part(parts, part_index, offset)
+              take = [len - fragment.bytesize, parts[part_index].bytesize - offset].min
+              fragment << parts[part_index].byteslice(offset, take)
+              offset += take
+            end
+            compress_block(fragment, 0, len, out, table)
+          end
           start += len
         end
         out
       end
 
+      # @param str [String] bytes in any encoding
+      # @return [String] +str+, or a binary copy when it is in another encoding
+      def binary(str)
+        (str.encoding == Encoding::BINARY) ? str : str.b
+      end
+
+      # Skips past parts that are used up
+      #
+      # @param parts [Array<String>] the input parts
+      # @param index [Integer] index of the current part
+      # @param offset [Integer] offset in the current part
+      # @return [Array(Integer, Integer)] index of a part with bytes left at the offset, and that offset
+      def next_part(parts, index, offset)
+        while offset >= parts[index].bytesize
+          index += 1
+          offset = 0
+        end
+        [index, offset]
+      end
+
       # Reads the uncompressed-length preamble (a little-endian base-128 varint) at the start of +src+.
       # @param src [String] raw snappy block
       # @param n [Integer] byte size of +src+
@@ -490,7 +532,7 @@ module Herringbone
         end
       end
 
-      private_class_method :native, :native_library, :decompress_io_buffer, :decompress_string, :read_varint, :write_varint, :compress_block, :block_words, :match_length_words,
+      private_class_method :native, :native_library, :binary, :next_part, :decompress_io_buffer, :decompress_string, :read_varint, :write_varint, :compress_block, :block_words, :match_length_words,
         :match_length, :emit_literal, :emit_copy, :emit_copy_upto64
     end
   end