iostreams 2.0.0 → 2.1.0

data/lib/io_streams/pgp/reader.rb

@@ -3,6 +3,10 @@ require "open3"
 module IOStreams
   module Pgp
     class Reader < IOStreams::Reader
+      def self.option_names
+        %i[passphrase ignore_mdc_error verify_first]
+      end
+
       # Passphrase to use to open the private key to decrypt the received file
       class << self
         attr_writer :default_passphrase
@@ -16,6 +20,15 @@ module IOStreams
 
       # Read from a PGP / GPG file , decompressing the contents as it is read.
       #
+      # SECURITY WARNING:
+      #   By default the decrypted contents are passed to the block as gpg decrypts them,
+      #   before gpg has checked the file's integrity (MDC) and any signature, since those are
+      #   only known once the whole file has been read. When either check fails,
+      #   `IOStreams::Pgp::Failure` is raised after the block has processed the data.
+      #   Do not commit any side effects, such as database updates, until the block returns
+      #   without raising, for example by processing the file within a database transaction.
+      #   Otherwise supply `verify_first: true`.
+      #
       # file_name: [String]
       #   Name of file to read from
       #
@@ -30,7 +43,15 @@ module IOStreams
       #   Only enable this for files from a trusted source: without MDC the decrypted
       #   contents are not protected against tampering.
       #   Default: false
-      def self.file(file_name, passphrase: nil, ignore_mdc_error: false)
+      #
+      # verify_first: [true|false]
+      #   Decrypt the whole file into a temporary file, only readable by the current user,
+      #   and only pass its contents to the block once gpg has checked the file's integrity
+      #   and any signature.
+      #   Requires local disk space for the decrypted contents, which are deleted afterwards,
+      #   and an extra pass over the data.
+      #   Default: false
+      def self.file(file_name, passphrase: nil, ignore_mdc_error: false, verify_first: false, &)
         # Cannot use `passphrase: self.default_passphrase` since it is considered private
         passphrase ||= default_passphrase
 
@@ -43,11 +64,13 @@ module IOStreams
         args += ["--batch", "--no-tty", "--yes", "--decrypt"]
         # Only feed a passphrase when one is supplied; sign-only files need none.
         args += ["--passphrase-fd", "0"] if passphrase
-        args << file_name.to_s
+        args += ["--", file_name.to_s]
 
         command = IOStreams::Pgp.gpg_command(*args)
         IOStreams.logger&.debug { "IOStreams::Pgp::Reader.open: #{command.shelljoin}" }
 
+        return decrypt_then_read(command, file_name, passphrase, &) if verify_first
+
         # Read decrypted contents from stdout
         Open3.popen3(*command) do |stdin, stdout, stderr, waith_thr|
           stdin.puts(passphrase) if passphrase
@@ -65,6 +88,23 @@ module IOStreams
           result
         end
       end
+
+      # Decrypts the file into a temporary file, and only yields it once gpg has succeeded.
+      def self.decrypt_then_read(command, file_name, passphrase, &block)
+        Utils.private_temp_file("iostreams_pgp") do |temp_file_name|
+          Open3.popen3(*command) do |stdin, stdout, stderr, waith_thr|
+            stdin.puts(passphrase) if passphrase
+            stdin.close
+            ::File.open(temp_file_name, "wb") { |io| ::IO.copy_stream(stdout, io) }
+            unless waith_thr.value.success?
+              raise(Pgp::Failure, "GPG Failed to decrypt file: #{file_name}: #{stderr.read.chomp}")
+            end
+          end
+
+          ::File.open(temp_file_name, "rb", &block)
+        end
+      end
+      private_class_method :decrypt_then_read
     end
   end
 end

data/lib/io_streams/pgp/writer.rb

@@ -3,6 +3,14 @@ require "open3"
 module IOStreams
   module Pgp
     class Writer < IOStreams::Writer
+      # File descriptor in the gpg process that the signer passphrase is read from.
+      PASSPHRASE_FD = 3
+
+      def self.option_names
+        %i[encrypt recipient import_and_trust_key import_and_trust_level signer signer_passphrase
+           compress compress_level]
+      end
+
       class << self
         # Sign all encrypted files with this users key.
         # Default: Do not sign encrypted files.
@@ -122,13 +130,22 @@ module IOStreams
           recipients:        recipients
         )
         command = IOStreams::Pgp.gpg_command(*args)
+        IOStreams.logger&.debug { "IOStreams::Pgp::Writer.open: #{command.shelljoin}" }
 
-        # Do not log the command, it may contain the signer passphrase.
-        action = encrypt ? "encrypt" : "sign"
-        IOStreams.logger&.debug { "IOStreams::Pgp::Writer.open: #{action} -o #{file_name}" }
+        # Since stdin carries the data, supply the signer passphrase on file descriptor 3
+        # so that it is not visible in the process list.
+        spawn_options = {}
+        if signer_passphrase
+          passphrase_reader, passphrase_writer = IO.pipe
+          passphrase_writer.puts(signer_passphrase.to_s)
+          passphrase_writer.close
+          spawn_options[PASSPHRASE_FD] = passphrase_reader
+        end
 
         result = nil
-        Open3.popen2e(*command) do |stdin, out, waith_thr|
+        Open3.popen2e(*command, spawn_options) do |stdin, out, waith_thr|
+          # Only the gpg process needs the passphrase.
+          passphrase_reader&.close
           begin
             stdin.binmode
             result = yield(stdin)
@@ -144,6 +161,8 @@ module IOStreams
           end
         end
         result
+      ensure
+        passphrase_reader&.close
       end
 
       def self.build_args(file_name:, encrypt:, signer:, signer_passphrase:, compress:, compress_level:, recipients:)
@@ -153,7 +172,7 @@ module IOStreams
         if signer_passphrase
           args += ["--pinentry-mode", "loopback"] if IOStreams::Pgp.pgp_version.to_f >= 2.1
           args << "--no-symkey-cache" if IOStreams::Pgp.pgp_version.to_f >= 2.4
-          args += ["--passphrase", signer_passphrase.to_s]
+          args += ["--passphrase-fd", PASSPHRASE_FD.to_s]
         end
         args += ["-z", compress_level.to_s] if compress_level != 6
         args += ["--compress-algo", compress.to_s] unless compress == :none
@@ -168,7 +187,8 @@ module IOStreams
         recipients << audit_recipient if audit_recipient
 
         Array(import_and_trust_key).each do |key|
-          recipients << IOStreams::Pgp.import_and_trust(key: key, trust_level: import_and_trust_level)
+          # Encrypt to the imported key's fingerprint, since its email address could match another key in the keyring.
+          recipients << IOStreams::Pgp.import_and_trust_recipient(key: key, trust_level: import_and_trust_level)
         end
         recipients
       end

data/lib/io_streams/pgp.rb

@@ -160,13 +160,14 @@ module IOStreams
       end
     end
 
-    # Delete all private and public keys for a particular email.
+    # Delete all private and public keys for a particular email or key id.
     #
     # Returns false if no key was found.
     # Raises an exception if it fails to delete the key.
+    # Raises ArgumentError when neither :email nor :key_id is supplied.
     #
-    # email: [String] Optional email address for the key.
-    # key_id: [String] Optional id for the key.
+    # email: [String] Email address for the key.
+    # key_id: [String] Id for the key.
     #
     # public: [true|false]
     #   Whether to delete the public key
@@ -176,6 +177,8 @@ module IOStreams
     #   Whether to delete the private key
     #   Default: false
     def self.delete_keys(email: nil, key_id: nil, public: true, private: false)
+      raise(ArgumentError, "Either :email, or :key_id must be supplied") if email.nil? && key_id.nil?
+
       version_check
       # Version 2.1+ uses delete_public_or_private_keys
       # Version < 2.1 uses delete_public_or_private_keys_v1
@@ -207,7 +210,8 @@ module IOStreams
     def self.list_keys(email: nil, key_id: nil, private: false)
       version_check
       args = [private ? "--list-secret-keys" : "--list-keys"]
-      args << (email || key_id).to_s if email || key_id
+      # `--` stops gpg from treating the email or key id as an option.
+      args += ["--", (email || key_id).to_s] if email || key_id
       command = gpg_command(*args)
 
       out, err, status = Open3.capture3(*command, binmode: true)
@@ -255,14 +259,26 @@ module IOStreams
       parse_list_output(out)
     end
 
-    # Returns [String] containing all the public keys for the supplied email address.
+    # Returns [String] containing all the public keys for the supplied email address or key id.
+    #
+    # Raises ArgumentError when neither :email nor :key_id is supplied.
+    # Raises Pgp::Failure when `email: nil` is supplied without a :key_id, as before :key_id was added.
+    # In v3.0 this will raise ArgumentError instead.
     #
     # email: [String] Email address for requested key.
     #
+    # key_id: [String] Id for the requested key.
+    #
     # ascii: [true|false]
     #   Whether to export as ASCII text instead of binary format
     #   Default: true
-    def self.export(email:, ascii: true, private: false, passphrase: nil)
+    def self.export(email: (email_omitted = true) && nil, key_id: nil, ascii: true, private: false, passphrase: nil)
+      if email.nil? && key_id.nil?
+        raise(ArgumentError, "Either :email, or :key_id must be supplied") if email_omitted
+
+        raise(Pgp::Failure, "GPG Failed reading key: no email address or key id was supplied")
+      end
+
       version_check
 
       args = []
@@ -270,15 +286,16 @@ module IOStreams
       args << "--no-symkey-cache" if pgp_version.to_f >= 2.4
       args << "--armor" if ascii
       args += ["--no-tty", "--batch"]
-      args += passphrase ? ["--passphrase", passphrase] : ["--passphrase-fd", "0"]
-      args += private ? ["--export-secret-keys", email.to_s] : ["--export", email.to_s]
+      # Supply the passphrase on stdin so that it is not visible in the process list.
+      args += ["--passphrase-fd", "0"]
+      args << (private ? "--export-secret-keys" : "--export")
+      args += ["--", (email || key_id).to_s]
       command = gpg_command(*args)
 
-      out, err, status = Open3.capture3(*command, binmode: true)
-      # Do not log the command, it may contain the passphrase.
-      IOStreams.logger&.debug { "IOStreams::Pgp.export: #{email}\n#{err}" }
+      out, err, status = Open3.capture3(*command, binmode: true, stdin_data: passphrase.to_s)
+      IOStreams.logger&.debug { "IOStreams::Pgp.export: #{command.shelljoin}\n#{err}" }
 
-      raise(Pgp::Failure, "GPG Failed reading key: #{email}: #{err}") unless status.success? && out.length.positive?
+      raise(Pgp::Failure, "GPG Failed reading key: #{email || key_id}: #{err}") unless status.success? && out.length.positive?
 
       out
     end
@@ -424,18 +441,47 @@ module IOStreams
     # Notes:
     # - If the same email address has multiple keys then only the first is currently trusted.
     def self.import_and_trust(key:, trust_level: 5)
+      info = import_and_trust_key_info(key: key, trust_level: trust_level)
+      info[:email] || info[:key_id]
+    end
+
+    # Imports and trusts the supplied key, see #import_and_trust.
+    #
+    # Returns [String] the recipient to encrypt to: the key's fingerprint when gpg supplies it,
+    # otherwise the email address or key id returned by #import_and_trust.
+    #
+    # Encrypting to the fingerprint ensures that the imported key is used, since gpg looks up an
+    # email address in the keyring, where another key may have the same email address.
+    #
+    # Used internally by the PGP writer.
+    def self.import_and_trust_recipient(key:, trust_level: 5)
+      info   = import_and_trust_key_info(key: key, trust_level: trust_level)
+      key_id = info[:key_id].to_s
+      # gpg v2.1 and later supply the full fingerprint: 40 hex digits for v4 keys, 64 for v5 and v6 keys.
+      # Earlier versions only supply a short key id, which is not unique.
+      return key_id if key_id.match?(/\A(\h{40}|\h{64})\z/)
+
+      info[:email] || info[:key_id]
+    end
+
+    # Returns [Hash] the key info for the supplied key, after importing and trusting it.
+    def self.import_and_trust_key_info(key:, trust_level:)
       raise(ArgumentError, "Key cannot be empty") if key.nil? || (key == "")
 
-      key_info = key_info(key: key).last
+      infos = key_info(key: key)
+      info  = infos.last&.dup || {}
+      # When a key has several user ids, only the entry for its first user id includes the key id.
+      info[:key_id] ||= infos.reverse_each.find { |entry| entry[:key_id] }&.fetch(:key_id)
 
-      email = key_info.fetch(:email, nil)
-      key_id = key_info.fetch(:key_id, nil)
+      email  = info[:email]
+      key_id = info[:key_id]
       raise(ArgumentError, "Recipient email or key id cannot be extracted from supplied key") unless email || key_id
 
       import(key: key)
       set_trust(email: email, key_id: key_id, level: trust_level)
-      email || key_id
+      info
     end
+    private_class_method :import_and_trust_key_info
 
     # Set the trust level for an existing key.
     #
@@ -444,6 +490,10 @@ module IOStreams
     #
     # After importing keys, they are not trusted and the relevant trust level must be set.
     #
+    # key_id: [String]
+    #   The fingerprint of the key, as hexadecimal digits only.
+    #   Raises ArgumentError when it contains any other characters.
+    #
     # level: [Integer]
     #   The owner-trust level to assign to the key:
     #     1 : Undefined  (no opinion)
@@ -461,6 +511,12 @@ module IOStreams
     #   key at this level allows that attacker to impersonate other recipients.
     #   When the key cannot be fully verified, supply a lower `level`.
     def self.set_trust(email: nil, key_id: nil, level: 5)
+      # The key_id is written into gpg's ownertrust input, where any other character, such as a newline,
+      # could add trust lines for other keys.
+      if key_id && !key_id.to_s.match?(/\A\h+\z/)
+        raise(ArgumentError, "Invalid :key_id, it must only contain hexadecimal digits: #{key_id.inspect}")
+      end
+
       version_check
       fingerprint = key_id || fingerprint(email: email)
       return unless fingerprint
@@ -479,7 +535,7 @@ module IOStreams
     # Public callers should identify keys by `key_id` (see #list_keys / #key_info).
     def self.fingerprint(email:)
       version_check
-      command = gpg_command("--list-keys", "--fingerprint", "--with-colons", email.to_s)
+      command = gpg_command("--list-keys", "--fingerprint", "--with-colons", "--", email.to_s)
       Open3.popen2e(*command) do |_stdin, out, waith_thr|
         output = out.read.chomp
         if !waith_thr.value.success? && output !~ /(public key not found|No public key)/i
@@ -609,10 +665,11 @@ module IOStreams
           hash[:trust] = match[2].to_s.strip if match[1]
           results << hash
           hash = {}
-        elsif (match = line.match(/\s+([A-Z0-9]{16,40})/))
+        elsif (match = line.match(/\s+([A-Z0-9]{16,64})/))
           # v2.2/v2.4 key id on separate line:
           # 18A0FC1C09C0D8AE34CE659257DC4AE323C7368C
           # Or shorter format: 7932AB23D7238F6B
+          # Or a 64 digit fingerprint for v5 and v6 keys.
           hash[:key_id] ||= match[1]
         end
       end
@@ -636,7 +693,7 @@ module IOStreams
         key_id = key_info[:key_id]
         next unless key_id
 
-        command          = gpg_command("--batch", "--no-tty", "--yes", "--delete-#{keys}", key_id)
+        command          = gpg_command("--batch", "--no-tty", "--yes", "--delete-#{keys}", "--", key_id)
         out, err, status = Open3.capture3(*command, binmode: true)
         IOStreams.logger&.debug { "IOStreams::Pgp.delete_keys: #{command.shelljoin}\n#{err}#{out}" }
 
@@ -653,7 +710,7 @@ module IOStreams
 
       # List the fingerprints, then delete each one. Previously this shelled out
       # to a `for` loop, which allowed shell injection via :email / :key_id.
-      list_command        = gpg_command("--list-#{keys}", "--with-colons", "--fingerprint", (email || key_id).to_s)
+      list_command        = gpg_command("--list-#{keys}", "--with-colons", "--fingerprint", "--", (email || key_id).to_s)
       list_out, list_err, = Open3.capture3(*list_command, binmode: true)
       IOStreams.logger&.debug { "IOStreams::Pgp.delete_keys: #{list_command.shelljoin}\n#{list_err}: #{list_out}" }
 
@@ -663,7 +720,7 @@ module IOStreams
       return false if fingerprints.empty?
 
       fingerprints.each do |fingerprint|
-        command          = gpg_command("--batch", "--no-tty", "--yes", "--delete-#{keys}", fingerprint)
+        command          = gpg_command("--batch", "--no-tty", "--yes", "--delete-#{keys}", "--", fingerprint)
         out, err, status = Open3.capture3(*command, binmode: true)
         IOStreams.logger&.debug { "IOStreams::Pgp.delete_keys: #{command.shelljoin}\n#{err}: #{out}" }
 

data/lib/io_streams/reader.rb

@@ -1,9 +1,18 @@
 module IOStreams
   class Reader
+    # Returns [Array<Symbol>] the names of the options this reader accepts,
+    # or [nil] when the reader does not declare them.
+    #
+    # When declared, `IOStreams::Builder` rejects any other option before the reader is opened,
+    # naming the direction an option belongs to when it is only valid for the other direction.
+    def self.option_names
+      nil
+    end
+
     # When a Reader does not support streams, we copy the stream to a local temp file
     # and then pass that filename in for this reader.
     def self.stream(input_stream, **args, &block)
-      Utils.temp_file_name("iostreams_reader") do |file_name|
+      Utils.private_temp_file("iostreams_reader") do |file_name|
         ::File.open(file_name, "wb") { |target| ::IO.copy_stream(input_stream, target) }
         file(file_name, **args, &block)
       end

data/lib/io_streams/record/reader.rb

@@ -56,6 +56,10 @@ module IOStreams
       #       #as_hash will skip these additional columns entirely as if they were not in the file at all.
       #     false:
       #       Raises Tabular::InvalidHeader when a column is supplied that is not in the whitelist.
+      #
+      # Note:
+      # * `allowed_columns`, `required_columns` and `skip_unknown` only apply to every input, including JSON records,
+      #   supplied `columns` and `cleanse_header: false`, when `IOStreams.enforce_column_restrictions?` is true.
       def initialize(line_reader, cleanse_header: true, original_file_name: nil, **args)
         unless line_reader.respond_to?(:each)
           raise(ArgumentError, "Stream must be a IOStreams::Line::Reader or implement #each")
@@ -64,18 +68,84 @@ module IOStreams
         @tabular        = IOStreams::Tabular.new(file_name: original_file_name, **args)
         @line_reader    = line_reader
         @cleanse_header = cleanse_header
+        @warned         = false
+
+        # Supplied columns take the place of a header row, so apply the allowed and required columns to them.
+        restrict_columns if restricted? && !@tabular.header?
       end
 
       def each
         @line_reader.each do |line|
           if @tabular.header?
             @tabular.parse_header(line)
-            @tabular.cleanse_header! if @cleanse_header
+            if @cleanse_header
+              cleanse_columns
+            elsif restricted?
+              restrict_columns
+            end
           else
-            yield @tabular.record_parse(line)
+            yield restrict(@tabular.record_parse(line))
           end
         end
       end
+
+      private
+
+      def restricted?
+        @tabular.header.restricted?
+      end
+
+      def cleanse_columns
+        @tabular.header.cleanse!(rename: @cleanse_header)
+      end
+
+      # Apply the allowed and required columns to supplied columns, or to a header row read with
+      # `cleanse_header: false`. Unless `IOStreams.enforce_column_restrictions?`, only warn when they would
+      # change the columns.
+      def restrict_columns
+        return cleanse_columns if IOStreams.enforce_column_restrictions?
+        return if @warned
+
+        header  = @tabular.header
+        columns = header.columns
+        changed = changed_by_restriction? do
+          copy = IOStreams::Tabular::Header.new(
+            columns:          columns,
+            allowed_columns:  header.allowed_columns,
+            required_columns: header.required_columns,
+            skip_unknown:     header.skip_unknown
+          )
+          copy.cleanse!(rename: @cleanse_header)
+          copy.columns != columns
+        end
+        warn_restriction if changed
+      end
+
+      # Formats such as JSON have no header row, so apply the allowed and required columns to each record's keys.
+      # Unless `IOStreams.enforce_column_restrictions?`, only warn when they would change the record.
+      def restrict(record)
+        return record unless record.is_a?(Hash) && restricted? && @tabular.header.columns.nil?
+        return @tabular.header.restrict_hash(record, rename: @cleanse_header) if IOStreams.enforce_column_restrictions?
+        return record if @warned
+
+        warn_restriction if changed_by_restriction? { @tabular.header.restrict_hash(record, rename: @cleanse_header) != record }
+        record
+      end
+
+      def changed_by_restriction?
+        yield
+      rescue IOStreams::Errors::InvalidHeader
+        true
+      end
+
+      # Warn once per reader, since the same columns usually apply to every record.
+      def warn_restriction
+        @warned = true
+        IOStreams.logger&.warn(
+          "allowed_columns and required_columns are not applied to this input, but would change the records read. " \
+          "In v3.0 they will apply to every input. Set `IOStreams.enforce_column_restrictions = true` to apply them now."
+        )
+      end
     end
   end
 end

data/lib/io_streams/stream.rb

@@ -87,9 +87,11 @@ module IOStreams
     #   end
     #
     # Notes:
-    # - Embedded lines (within double quotes) will be skipped if
-    #   1. The file name contains .csv
-    #   2. Or the embedded_within argument is set
+    # - Newlines embedded within quoted fields are kept within the same line when
+    #   1. The resolved tabular format quotes its fields (e.g. CSV, whether detected from a
+    #      `.csv` file name or set explicitly via `.format(:csv)`)
+    #   2. Or the `embedded_within` argument is supplied (e.g. `embedded_within: '"'`)
+    # - Pass `embedded_within: nil` to disable quote-aware line joining for a quoted format.
     def each(mode = :line, **args, &block)
       raise(ArgumentError, "Invalid mode: #{mode.inspect}") if mode == :stream
 
@@ -331,8 +333,11 @@ module IOStreams
       builder.reader(io_stream, &)
     end
 
-    def line_reader(embedded_within: nil, **args)
-      embedded_within = '"' if embedded_within.nil? && builder.file_name&.include?(".csv")
+    def line_reader(embedded_within: :auto, **args)
+      # `:auto` defers the decision to the resolved tabular format (e.g. CSV quotes with `"`),
+      # while distinguishing "not supplied" from an explicit value such as `nil` (disable) or
+      # `'"'` (force). Centralizing this in the builder keeps all format-based decisions there.
+      embedded_within = builder.quote_character if embedded_within == :auto
 
       stream_reader do |io|
         yield IOStreams::Line::Reader.new(
@@ -344,7 +349,7 @@ module IOStreams
     end
 
     # Iterate over a file / stream returning each line as an array, one at a time.
-    def row_reader(delimiter: nil, embedded_within: nil, **args)
+    def row_reader(delimiter: nil, embedded_within: :auto, **args)
       line_reader(delimiter: delimiter, embedded_within: embedded_within) do |io|
         yield IOStreams::Row::Reader.new(
           io,
@@ -357,7 +362,7 @@ module IOStreams
     end
 
     # Iterate over a file / stream returning each line as a hash, one at a time.
-    def record_reader(delimiter: nil, embedded_within: nil, **args)
+    def record_reader(delimiter: nil, embedded_within: :auto, **args)
       line_reader(delimiter: delimiter, embedded_within: embedded_within) do |io|
         yield IOStreams::Record::Reader.new(
           io,

data/lib/io_streams/symmetric_encryption/reader.rb

@@ -1,6 +1,10 @@
 module IOStreams
   module SymmetricEncryption
     class Reader < IOStreams::Reader
+      def self.option_names
+        %i[buffer_size version]
+      end
+
       # read from a file/stream using Symmetric Encryption
       def self.stream(input_stream, **args, &)
         Utils.load_soft_dependency("symmetric-encryption", ".enc streaming") unless defined?(SymmetricEncryption)

data/lib/io_streams/symmetric_encryption/writer.rb

@@ -1,6 +1,10 @@
 module IOStreams
   module SymmetricEncryption
     class Writer < IOStreams::Writer
+      def self.option_names
+        %i[compress version cipher_name header random_key random_iv]
+      end
+
       # Write to stream using Symmetric Encryption
       # By default the output stream is compressed.
       # If the input_stream is already compressed consider setting compress: false.

data/lib/io_streams/tabular/header.rb

@@ -52,16 +52,22 @@ module IOStreams
       # - Spaces and '-' are converted to '_'.
       # - All characters except for letters, digits, and '_' are stripped.
       #
+      # Parameters:
+      #   rename [true|false]
+      #     Whether to cleanse the column names as described above.
+      #     When false, the column names are compared to `allowed_columns` and `required_columns` as-is.
+      #     Default: true
+      #
       # Notes:
       # * So that rejected columns can be identified in subsequent steps, they will be prefixed with `__rejected__`.
       #   For example, `Unknown Column` would be cleansed as `__rejected__Unknown Column`.
       # * Raises Tabular::InvalidHeader when there are no rejected columns left after cleansing.
-      def cleanse!
+      def cleanse!(rename: true)
         return [] if columns.nil? || columns.empty?
 
         ignored_columns = []
         self.columns    = columns.collect do |column|
-          cleansed = cleanse_column(column)
+          cleansed = rename ? cleanse_column(column) : column
           if allowed_columns.nil? || allowed_columns.include?(cleansed)
             cleansed
           else
@@ -112,6 +118,26 @@ module IOStreams
         end
       end
 
+      # Returns [true|false] whether `allowed_columns` or `required_columns` restrict the columns.
+      def restricted?
+        !allowed_columns.nil? || !required_columns.nil?
+      end
+
+      # Returns [Hash] the supplied hash after applying `allowed_columns`, `required_columns` and `skip_unknown`
+      # to its keys, as if its keys were the header row.
+      #
+      # Used for formats such as JSON where each record supplies its own keys instead of a header row.
+      def restrict_hash(hash, rename: true)
+        header = self.class.new(
+          columns:          hash.keys,
+          allowed_columns:  allowed_columns,
+          required_columns: required_columns,
+          skip_unknown:     skip_unknown
+        )
+        header.cleanse!(rename: rename)
+        header.to_hash(hash.values)
+      end
+
       def to_array(row, cleanse = true)
         if row.is_a?(Hash) && columns
           row = cleanse_hash(row) if cleanse
@@ -142,12 +168,13 @@ module IOStreams
       # For example, avoids issues with case etc.
       def cleanse_hash(hash)
         hash      = hash.transform_keys(&:to_s) unless hash.keys.all?(String)
-        unmatched = columns - hash.keys
+        allowed   = columns.reject { |column| column.start_with?(IGNORE_PREFIX) }
+        unmatched = allowed - hash.keys
         unless unmatched.empty?
           hash = hash.dup
           unmatched.each { |name| hash[cleanse_column(name)] = hash.delete(name) }
         end
-        hash.slice(*columns)
+        hash.slice(*allowed)
       end
 
       def cleanse_column(name)

data/lib/io_streams/tabular/parser/base.rb

@@ -2,6 +2,16 @@ module IOStreams
   class Tabular
     module Parser
       class Base
+        # Returns [String] the quote character within which field delimiters and embedded
+        # newlines may appear for this format, or [nil] when the format has no such quoting.
+        #
+        # Used by the line reader to avoid treating a newline as a line ending when it is
+        # embedded within a quoted field (e.g. CSV). Defined at the class level since it is a
+        # static property of the format, independent of any per-instance format options.
+        def self.quote_character
+          nil
+        end
+
         # Returns [true|false] whether a header row is required for this format.
         def requires_header?
           true

data/lib/io_streams/tabular/parser/csv.rb

@@ -3,6 +3,11 @@ module IOStreams
   class Tabular
     module Parser
       class Csv < Base
+        # CSV fields may contain embedded delimiters and newlines when wrapped in double quotes.
+        def self.quote_character
+          '"'
+        end
+
         # Returns [Array] the parsed CSV line
         def parse(row)
           return row if row.is_a?(::Array)

data/lib/io_streams/tabular/parser/fixed.rb

@@ -143,6 +143,7 @@ module IOStreams
 
         class Column
           TYPES = %i[string integer float].freeze
+          LINE_BREAK = /\r\n|\r|\n/
 
           attr_reader :key, :size, :type, :decimals
 
@@ -179,7 +180,8 @@ module IOStreams
             formatted =
               case type
               when :string
-                value = value.to_s
+                # Replace line breaks with a space so that a value cannot add records.
+                value = value.to_s.gsub(LINE_BREAK, " ")
                 return value if size == -1
 
                 format(truncate ? "%-#{size}.#{size}s" : "%-#{size}s", value)