iostreams 1.11.0 → 2.1.0

data/lib/io_streams/record/reader.rb

@@ -16,7 +16,7 @@ module IOStreams
 
       # When reading from a file also add the line reader stream
       def self.file(file_name, original_file_name: file_name, delimiter: $/, **args)
-        IOStreams::Line::Reader.file(file_name, original_file_name: original_file_name, delimiter: delimiter) do |io|
+        IOStreams::Line::Reader.file(file_name, delimiter: delimiter) do |io|
           yield new(io, original_file_name: original_file_name, **args)
         end
       end
@@ -35,11 +35,10 @@ module IOStreams
       #   format_options: [Hash]
       #     Any specialized format specific options. For example, `:fixed` format requires the file definition.
       #
-      #   columns [Array<String>]
+      #   columns [Array<String|Symbol>]
       #     The header columns when the file does not include a header row.
       #     Note:
-      #       It is recommended to keep all columns as strings to avoid any issues when persistence
-      #       with MongoDB when it converts symbol keys to strings.
+      #       Column names are converted to strings.
       #
       #   allowed_columns [Array<String>]
       #     List of columns to allow.
@@ -57,6 +56,10 @@ module IOStreams
       #       #as_hash will skip these additional columns entirely as if they were not in the file at all.
       #     false:
       #       Raises Tabular::InvalidHeader when a column is supplied that is not in the whitelist.
+      #
+      # Note:
+      # * `allowed_columns`, `required_columns` and `skip_unknown` only apply to every input, including JSON records,
+      #   supplied `columns` and `cleanse_header: false`, when `IOStreams.enforce_column_restrictions?` is true.
       def initialize(line_reader, cleanse_header: true, original_file_name: nil, **args)
         unless line_reader.respond_to?(:each)
           raise(ArgumentError, "Stream must be a IOStreams::Line::Reader or implement #each")
@@ -65,18 +68,84 @@ module IOStreams
         @tabular        = IOStreams::Tabular.new(file_name: original_file_name, **args)
         @line_reader    = line_reader
         @cleanse_header = cleanse_header
+        @warned         = false
+
+        # Supplied columns take the place of a header row, so apply the allowed and required columns to them.
+        restrict_columns if restricted? && !@tabular.header?
       end
 
       def each
         @line_reader.each do |line|
           if @tabular.header?
             @tabular.parse_header(line)
-            @tabular.cleanse_header! if @cleanse_header
+            if @cleanse_header
+              cleanse_columns
+            elsif restricted?
+              restrict_columns
+            end
           else
-            yield @tabular.record_parse(line)
+            yield restrict(@tabular.record_parse(line))
           end
         end
       end
+
+      private
+
+      def restricted?
+        @tabular.header.restricted?
+      end
+
+      def cleanse_columns
+        @tabular.header.cleanse!(rename: @cleanse_header)
+      end
+
+      # Apply the allowed and required columns to supplied columns, or to a header row read with
+      # `cleanse_header: false`. Unless `IOStreams.enforce_column_restrictions?`, only warn when they would
+      # change the columns.
+      def restrict_columns
+        return cleanse_columns if IOStreams.enforce_column_restrictions?
+        return if @warned
+
+        header  = @tabular.header
+        columns = header.columns
+        changed = changed_by_restriction? do
+          copy = IOStreams::Tabular::Header.new(
+            columns:          columns,
+            allowed_columns:  header.allowed_columns,
+            required_columns: header.required_columns,
+            skip_unknown:     header.skip_unknown
+          )
+          copy.cleanse!(rename: @cleanse_header)
+          copy.columns != columns
+        end
+        warn_restriction if changed
+      end
+
+      # Formats such as JSON have no header row, so apply the allowed and required columns to each record's keys.
+      # Unless `IOStreams.enforce_column_restrictions?`, only warn when they would change the record.
+      def restrict(record)
+        return record unless record.is_a?(Hash) && restricted? && @tabular.header.columns.nil?
+        return @tabular.header.restrict_hash(record, rename: @cleanse_header) if IOStreams.enforce_column_restrictions?
+        return record if @warned
+
+        warn_restriction if changed_by_restriction? { @tabular.header.restrict_hash(record, rename: @cleanse_header) != record }
+        record
+      end
+
+      def changed_by_restriction?
+        yield
+      rescue IOStreams::Errors::InvalidHeader
+        true
+      end
+
+      # Warn once per reader, since the same columns usually apply to every record.
+      def warn_restriction
+        @warned = true
+        IOStreams.logger&.warn(
+          "allowed_columns and required_columns are not applied to this input, but would change the records read. " \
+          "In v3.0 they will apply to every input. Set `IOStreams.enforce_column_restrictions = true` to apply them now."
+        )
+      end
     end
   end
 end

data/lib/io_streams/record/writer.rb

@@ -18,7 +18,7 @@ module IOStreams
 
       # When writing to a file also add the line writer stream
       def self.file(file_name, original_file_name: file_name, delimiter: $/, **args, &block)
-        IOStreams::Line::Writer.file(file_name, original_file_name: original_file_name, delimiter: delimiter) do |io|
+        IOStreams::Line::Writer.file(file_name, delimiter: delimiter) do |io|
           yield new(io, original_file_name: original_file_name, **args, &block)
         end
       end
@@ -37,11 +37,10 @@ module IOStreams
       #   format_options: [Hash]
       #     Any specialized format specific options. For example, `:fixed` format requires the file definition.
       #
-      #   columns [Array<String>]
+      #   columns [Array<String|Symbol>]
       #     The header columns when the file does not include a header row.
       #     Note:
-      #       It is recommended to keep all columns as strings to avoid any issues when persistence
-      #       with MongoDB when it converts symbol keys to strings.
+      #       Column names are converted to strings.
       #
       #   allowed_columns [Array<String>]
       #     List of columns to allow.

data/lib/io_streams/row/reader.rb

@@ -14,7 +14,7 @@ module IOStreams
 
       # When reading from a file also add the line reader stream
       def self.file(file_name, original_file_name: file_name, delimiter: $/, **args)
-        IOStreams::Line::Reader.file(file_name, original_file_name: original_file_name, delimiter: delimiter) do |io|
+        IOStreams::Line::Reader.file(file_name, delimiter: delimiter) do |io|
           yield new(io, original_file_name: original_file_name, **args)
         end
       end

data/lib/io_streams/row/writer.rb

@@ -21,7 +21,7 @@ module IOStreams
 
       # When writing to a file also add the line writer stream
       def self.file(file_name, original_file_name: file_name, delimiter: $/, **args, &block)
-        IOStreams::Line::Writer.file(file_name, original_file_name: original_file_name, delimiter: delimiter) do |io|
+        IOStreams::Line::Writer.file(file_name, delimiter: delimiter) do |io|
           yield new(io, original_file_name: original_file_name, **args, &block)
         end
       end

data/lib/io_streams/stream.rb

@@ -18,8 +18,8 @@ module IOStreams
     # Example:
     #
     # IOStreams.path("tempfile2527").stream(:zip).stream(:pgp, passphrase: "receiver_passphrase").read
-    def stream(stream, **options)
-      builder.stream(stream, **options)
+    def stream(stream, **)
+      builder.stream(stream, **)
       self
     end
 
@@ -33,15 +33,15 @@ module IOStreams
     # IOStreams.path("keep_safe.enc").option(:pgp, passphrase: "receiver_passphrase").read
     #
     # IOStreams.path(output_file_name).option(:pgp, passphrase: "receiver_passphrase").read
-    def option(stream, **options)
-      builder.option(stream, **options)
+    def option(stream, **)
+      builder.option(stream, **)
       self
     end
 
     # Adds the options for the specified stream as an option,
     # but if streams have already been added it is instead added as a stream.
-    def option_or_stream(stream, **options)
-      builder.option_or_stream(stream, **options)
+    def option_or_stream(stream, **)
+      builder.option_or_stream(stream, **)
       self
     end
 
@@ -87,27 +87,34 @@ module IOStreams
     #   end
     #
     # Notes:
-    # - Embedded lines (within double quotes) will be skipped if
-    #   1. The file name contains .csv
-    #   2. Or the embedded_within argument is set
+    # - Newlines embedded within quoted fields are kept within the same line when
+    #   1. The resolved tabular format quotes its fields (e.g. CSV, whether detected from a
+    #      `.csv` file name or set explicitly via `.format(:csv)`)
+    #   2. Or the `embedded_within` argument is supplied (e.g. `embedded_within: '"'`)
+    # - Pass `embedded_within: nil` to disable quote-aware line joining for a quoted format.
     def each(mode = :line, **args, &block)
       raise(ArgumentError, "Invalid mode: #{mode.inspect}") if mode == :stream
 
-      #    return enum_for __method__ unless block_given?
+      # Deliberately not returning an Enumerator when no block is given.
+      # The stream pipeline manages resources via block scope: every stream is opened with an
+      # `ensure` that closes the file handle, reaps the gpg subprocess, deletes temp files, etc.
+      # A Fiber-backed Enumerator (e.g. `to_enum(__method__, mode, **args)`) would leave that block
+      # suspended; if the caller abandons a partially-consumed enumerator, none of the cleanup runs
+      # until GC collects the Fiber, leaking file descriptors, gpg processes, and temp files.
       reader(mode, **args) { |stream| stream.each(&block) }
     end
 
     # Returns a Reader for reading a file / stream
-    def reader(mode = :stream, **args, &block)
+    def reader(mode = :stream, **args, &)
       case mode
       when :stream
-        stream_reader(&block)
+        stream_reader(&)
       when :line
-        line_reader(**args, &block)
+        line_reader(**args, &)
       when :array
-        row_reader(**args, &block)
+        row_reader(**args, &)
       when :hash
-        record_reader(**args, &block)
+        record_reader(**args, &)
       else
         raise(ArgumentError, "Invalid mode: #{mode.inspect}")
       end
@@ -124,16 +131,16 @@ module IOStreams
     end
 
     # Returns a Writer for writing to a file / stream
-    def writer(mode = :stream, **args, &block)
+    def writer(mode = :stream, **args, &)
       case mode
       when :stream
-        stream_writer(&block)
+        stream_writer(&)
       when :line
-        line_writer(**args, &block)
+        line_writer(**args, &)
       when :array
-        row_writer(**args, &block)
+        row_writer(**args, &)
       when :hash
-        record_writer(**args, &block)
+        record_writer(**args, &)
       else
         raise(ArgumentError, "Invalid mode: #{mode.inspect}")
       end
@@ -171,7 +178,7 @@ module IOStreams
     # IOStreams.path("target_file.json").copy_from("source_file_name.csv.gz", convert: false)
     #
     # # Advanced copy with custom stream conversions on source and target.
-    # source = IOStreams.path("source_file").stream(encoding: "BINARY")
+    # source = IOStreams.path("source_file").stream(:encode, encoding: "BINARY")
     # IOStreams.path("target_file.pgp").option(:pgp, passphrase: "hello").copy_from(source)
     def copy_from(source, convert: true, mode: nil, **args)
       if convert
@@ -322,23 +329,27 @@ module IOStreams
       @builder ||= IOStreams::Builder.new
     end
 
-    def stream_reader(&block)
-      builder.reader(io_stream, &block)
+    def stream_reader(&)
+      builder.reader(io_stream, &)
     end
 
-    def line_reader(embedded_within: nil, **args)
-      embedded_within = '"' if embedded_within.nil? && builder.file_name&.include?(".csv")
+    def line_reader(embedded_within: :auto, **args)
+      # `:auto` defers the decision to the resolved tabular format (e.g. CSV quotes with `"`),
+      # while distinguishing "not supplied" from an explicit value such as `nil` (disable) or
+      # `'"'` (force). Centralizing this in the builder keeps all format-based decisions there.
+      embedded_within = builder.quote_character if embedded_within == :auto
 
       stream_reader do |io|
-        yield IOStreams::Line::Reader.new(io,
-                                          original_file_name: builder.file_name,
-                                          embedded_within:    embedded_within,
-                                          **args)
+        yield IOStreams::Line::Reader.new(
+          io,
+          embedded_within: embedded_within,
+          **args
+        )
       end
     end
 
     # Iterate over a file / stream returning each line as an array, one at a time.
-    def row_reader(delimiter: nil, embedded_within: nil, **args)
+    def row_reader(delimiter: nil, embedded_within: :auto, **args)
       line_reader(delimiter: delimiter, embedded_within: embedded_within) do |io|
         yield IOStreams::Row::Reader.new(
           io,
@@ -351,7 +362,7 @@ module IOStreams
     end
 
     # Iterate over a file / stream returning each line as a hash, one at a time.
-    def record_reader(delimiter: nil, embedded_within: nil, **args)
+    def record_reader(delimiter: nil, embedded_within: :auto, **args)
       line_reader(delimiter: delimiter, embedded_within: embedded_within) do |io|
         yield IOStreams::Record::Reader.new(
           io,
@@ -363,20 +374,20 @@ module IOStreams
       end
     end
 
-    def stream_writer(&block)
-      builder.writer(io_stream, &block)
+    def stream_writer(&)
+      builder.writer(io_stream, &)
     end
 
     def line_writer(**args, &block)
-      return block.call(io_stream) if io_stream&.is_a?(IOStreams::Line::Writer)
+      return block.call(io_stream) if io_stream.is_a?(IOStreams::Line::Writer)
 
       writer do |io|
-        IOStreams::Line::Writer.stream(io, original_file_name: builder.file_name, **args, &block)
+        IOStreams::Line::Writer.stream(io, **args, &block)
       end
     end
 
     def row_writer(delimiter: $/, **args, &block)
-      return block.call(io_stream) if io_stream&.is_a?(IOStreams::Row::Writer)
+      return block.call(io_stream) if io_stream.is_a?(IOStreams::Row::Writer)
 
       line_writer(delimiter: delimiter) do |io|
         IOStreams::Row::Writer.stream(
@@ -391,7 +402,7 @@ module IOStreams
     end
 
     def record_writer(delimiter: $/, **args, &block)
-      return block.call(io_stream) if io_stream&.is_a?(IOStreams::Record::Writer)
+      return block.call(io_stream) if io_stream.is_a?(IOStreams::Record::Writer)
 
       line_writer(delimiter: delimiter) do |io|
         IOStreams::Record::Writer.stream(

data/lib/io_streams/symmetric_encryption/reader.rb

@@ -1,11 +1,15 @@
 module IOStreams
   module SymmetricEncryption
     class Reader < IOStreams::Reader
+      def self.option_names
+        %i[buffer_size version]
+      end
+
       # read from a file/stream using Symmetric Encryption
-      def self.stream(input_stream, **args, &block)
+      def self.stream(input_stream, **args, &)
         Utils.load_soft_dependency("symmetric-encryption", ".enc streaming") unless defined?(SymmetricEncryption)
 
-        ::SymmetricEncryption::Reader.open(input_stream, **args, &block)
+        ::SymmetricEncryption::Reader.open(input_stream, **args, &)
       end
     end
   end

data/lib/io_streams/symmetric_encryption/writer.rb

@@ -1,21 +1,25 @@
 module IOStreams
   module SymmetricEncryption
     class Writer < IOStreams::Writer
+      def self.option_names
+        %i[compress version cipher_name header random_key random_iv]
+      end
+
       # Write to stream using Symmetric Encryption
       # By default the output stream is compressed.
       # If the input_stream is already compressed consider setting compress: false.
-      def self.stream(input_stream, compress: true, **args, &block)
+      def self.stream(input_stream, compress: true, **args, &)
         Utils.load_soft_dependency("symmetric-encryption", ".enc streaming") unless defined?(SymmetricEncryption)
 
-        ::SymmetricEncryption::Writer.open(input_stream, compress: compress, **args, &block)
+        ::SymmetricEncryption::Writer.open(input_stream, compress: compress, **args, &)
       end
 
       # Write to stream using Symmetric Encryption
       # By default the output stream is compressed unless the file_name extension indicates the file is already compressed.
-      def self.file(file_name, compress: nil, **args, &block)
+      def self.file(file_name, compress: nil, **args, &)
         Utils.load_soft_dependency("symmetric-encryption", ".enc streaming") unless defined?(SymmetricEncryption)
 
-        ::SymmetricEncryption::Writer.open(file_name, compress: compress, **args, &block)
+        ::SymmetricEncryption::Writer.open(file_name, compress: compress, **args, &)
       end
     end
   end

data/lib/io_streams/tabular/header.rb

@@ -5,16 +5,16 @@ module IOStreams
       # Column names that begin with this prefix have been rejected and should be ignored.
       IGNORE_PREFIX = "__rejected__".freeze
 
-      attr_accessor :columns, :allowed_columns, :required_columns, :skip_unknown
+      attr_accessor :allowed_columns, :required_columns, :skip_unknown
+      attr_reader :columns
 
       # Header
       #
       # Parameters
-      #   columns [Array<String>]
+      #   columns [Array<String|Symbol>]
       #     Columns in this header.
       #     Note:
-      #       It is recommended to keep all columns as strings to avoid any issues when persistence
-      #       with MongoDB when it converts symbol keys to strings.
+      #       Column names are converted to strings.
       #
       #   allowed_columns [Array<String>]
       #     List of columns to allow.
@@ -33,12 +33,17 @@ module IOStreams
       #     false:
       #       Raises Tabular::InvalidHeader when a column is supplied that is not in the whitelist.
       def initialize(columns: nil, allowed_columns: nil, required_columns: nil, skip_unknown: true)
-        @columns          = columns
+        @columns          = stringify(columns)
         @required_columns = required_columns
         @allowed_columns  = allowed_columns
         @skip_unknown     = skip_unknown
       end
 
+      # Set the columns in this header, converting the column names to strings.
+      def columns=(columns)
+        @columns = stringify(columns)
+      end
+
       # Returns [Array<String>] list columns that were ignored during cleansing.
       #
       # Each column is cleansed as follows:
@@ -47,16 +52,22 @@ module IOStreams
       # - Spaces and '-' are converted to '_'.
       # - All characters except for letters, digits, and '_' are stripped.
       #
+      # Parameters:
+      #   rename [true|false]
+      #     Whether to cleanse the column names as described above.
+      #     When false, the column names are compared to `allowed_columns` and `required_columns` as-is.
+      #     Default: true
+      #
       # Notes:
       # * So that rejected columns can be identified in subsequent steps, they will be prefixed with `__rejected__`.
       #   For example, `Unknown Column` would be cleansed as `__rejected__Unknown Column`.
       # * Raises Tabular::InvalidHeader when there are no rejected columns left after cleansing.
-      def cleanse!
+      def cleanse!(rename: true)
         return [] if columns.nil? || columns.empty?
 
         ignored_columns = []
         self.columns    = columns.collect do |column|
-          cleansed = cleanse_column(column)
+          cleansed = rename ? cleanse_column(column) : column
           if allowed_columns.nil? || allowed_columns.include?(cleansed)
             cleansed
           else
@@ -107,6 +118,26 @@ module IOStreams
         end
       end
 
+      # Returns [true|false] whether `allowed_columns` or `required_columns` restrict the columns.
+      def restricted?
+        !allowed_columns.nil? || !required_columns.nil?
+      end
+
+      # Returns [Hash] the supplied hash after applying `allowed_columns`, `required_columns` and `skip_unknown`
+      # to its keys, as if its keys were the header row.
+      #
+      # Used for formats such as JSON where each record supplies its own keys instead of a header row.
+      def restrict_hash(hash, rename: true)
+        header = self.class.new(
+          columns:          hash.keys,
+          allowed_columns:  allowed_columns,
+          required_columns: required_columns,
+          skip_unknown:     skip_unknown
+        )
+        header.cleanse!(rename: rename)
+        header.to_hash(hash.values)
+      end
+
       def to_array(row, cleanse = true)
         if row.is_a?(Hash) && columns
           row = cleanse_hash(row) if cleanse
@@ -127,19 +158,23 @@ module IOStreams
 
       def array_to_hash(row)
         h = {}
-        columns.each_with_index { |col, i| h[col] = row[i] unless IOStreams::Utils.blank?(col) || col.start_with?(IGNORE_PREFIX) }
+        columns.each_with_index do |col, i|
+          h[col] = row[i] unless IOStreams::Utils.blank?(col) || col.start_with?(IGNORE_PREFIX)
+        end
         h
       end
 
       # Perform cleansing on returned Hash keys during the narrowing process.
       # For example, avoids issues with case etc.
       def cleanse_hash(hash)
-        unmatched = columns - hash.keys
+        hash      = hash.transform_keys(&:to_s) unless hash.keys.all?(String)
+        allowed   = columns.reject { |column| column.start_with?(IGNORE_PREFIX) }
+        unmatched = allowed - hash.keys
         unless unmatched.empty?
           hash = hash.dup
           unmatched.each { |name| hash[cleanse_column(name)] = hash.delete(name) }
         end
-        hash.slice(*columns)
+        hash.slice(*allowed)
       end
 
       def cleanse_column(name)
@@ -149,6 +184,10 @@ module IOStreams
         cleansed.gsub!(/\W+/, "")
         cleansed
       end
+
+      def stringify(columns)
+        columns&.collect { |column| column&.to_s }
+      end
     end
   end
 end

data/lib/io_streams/tabular/parser/array.rb

@@ -3,16 +3,6 @@ module IOStreams
   class Tabular
     module Parser
       class Array < Base
-        # Returns [Array<String>] the header row.
-        # Returns nil if the row is blank.
-        def parse_header(row)
-          unless row.is_a?(::Array)
-            raise(IOStreams::Errors::InvalidHeader, "Format is :array. Invalid input header: #{row.class.name}")
-          end
-
-          row
-        end
-
         # Returns Array
         def parse(row)
           raise(IOStreams::Errors::TypeMismatch, "Format is :array. Invalid input: #{row.class.name}") unless row.is_a?(::Array)

data/lib/io_streams/tabular/parser/base.rb

@@ -2,6 +2,16 @@ module IOStreams
   class Tabular
     module Parser
       class Base
+        # Returns [String] the quote character within which field delimiters and embedded
+        # newlines may appear for this format, or [nil] when the format has no such quoting.
+        #
+        # Used by the line reader to avoid treating a newline as a line ending when it is
+        # embedded within a quoted field (e.g. CSV). Defined at the class level since it is a
+        # static property of the format, independent of any per-instance format options.
+        def self.quote_character
+          nil
+        end
+
         # Returns [true|false] whether a header row is required for this format.
         def requires_header?
           true

data/lib/io_streams/tabular/parser/csv.rb

@@ -3,24 +3,9 @@ module IOStreams
   class Tabular
     module Parser
       class Csv < Base
-        attr_reader :csv_parser
-
-        unless RUBY_VERSION.to_f >= 2.6
-          def initialize
-            @csv_parser = Utility::CSVRow.new
-          end
-        end
-
-        # Returns [Array<String>] the header row.
-        # Returns nil if the row is blank.
-        def parse_header(row)
-          return row if row.is_a?(::Array)
-
-          unless row.is_a?(String)
-            raise(IOStreams::Errors::InvalidHeader, "Format is :csv. Invalid input header: #{row.class.name}")
-          end
-
-          parse_line(row)
+        # CSV fields may contain embedded delimiters and newlines when wrapped in double quotes.
+        def self.quote_character
+          '"'
         end
 
         # Returns [Array] the parsed CSV line
@@ -40,26 +25,14 @@ module IOStreams
 
         private
 
-        if RUBY_VERSION.to_f >= 2.6
-          # About 10 times slower than the approach used in Ruby 2.5 and earlier,
-          # but at least it works on Ruby 2.6 and above.
-          def parse_line(line)
-            return if IOStreams::Utils.blank?(line)
+        def parse_line(line)
+          return if IOStreams::Utils.blank?(line)
 
-            CSV.parse_line(line)
-          end
-
-          def render_array(array)
-            CSV.generate_line(array, encoding: "UTF-8", row_sep: "")
-          end
-        else
-          def parse_line(line)
-            csv_parser.parse(line)
-          end
+          CSV.parse_line(line)
+        end
 
-          def render_array(array)
-            csv_parser.to_csv(array)
-          end
+        def render_array(array)
+          CSV.generate_line(array, encoding: "UTF-8", row_sep: "")
         end
       end
     end