iostreams 1.10.3 → 2.0.0

data/lib/io_streams/stream.rb

@@ -18,8 +18,8 @@ module IOStreams
     # Example:
     #
     # IOStreams.path("tempfile2527").stream(:zip).stream(:pgp, passphrase: "receiver_passphrase").read
-    def stream(stream, **options)
-      builder.stream(stream, **options)
+    def stream(stream, **)
+      builder.stream(stream, **)
       self
     end
 
@@ -33,15 +33,15 @@ module IOStreams
     # IOStreams.path("keep_safe.enc").option(:pgp, passphrase: "receiver_passphrase").read
     #
     # IOStreams.path(output_file_name).option(:pgp, passphrase: "receiver_passphrase").read
-    def option(stream, **options)
-      builder.option(stream, **options)
+    def option(stream, **)
+      builder.option(stream, **)
       self
     end
 
     # Adds the options for the specified stream as an option,
     # but if streams have already been added it is instead added as a stream.
-    def option_or_stream(stream, **options)
-      builder.option_or_stream(stream, **options)
+    def option_or_stream(stream, **)
+      builder.option_or_stream(stream, **)
       self
     end
 
@@ -93,21 +93,26 @@ module IOStreams
     def each(mode = :line, **args, &block)
       raise(ArgumentError, "Invalid mode: #{mode.inspect}") if mode == :stream
 
-      #    return enum_for __method__ unless block_given?
+      # Deliberately not returning an Enumerator when no block is given.
+      # The stream pipeline manages resources via block scope: every stream is opened with an
+      # `ensure` that closes the file handle, reaps the gpg subprocess, deletes temp files, etc.
+      # A Fiber-backed Enumerator (e.g. `to_enum(__method__, mode, **args)`) would leave that block
+      # suspended; if the caller abandons a partially-consumed enumerator, none of the cleanup runs
+      # until GC collects the Fiber, leaking file descriptors, gpg processes, and temp files.
       reader(mode, **args) { |stream| stream.each(&block) }
     end
 
     # Returns a Reader for reading a file / stream
-    def reader(mode = :stream, **args, &block)
+    def reader(mode = :stream, **args, &)
       case mode
       when :stream
-        stream_reader(&block)
+        stream_reader(&)
       when :line
-        line_reader(**args, &block)
+        line_reader(**args, &)
       when :array
-        row_reader(**args, &block)
+        row_reader(**args, &)
       when :hash
-        record_reader(**args, &block)
+        record_reader(**args, &)
       else
         raise(ArgumentError, "Invalid mode: #{mode.inspect}")
       end
@@ -124,16 +129,16 @@ module IOStreams
     end
 
     # Returns a Writer for writing to a file / stream
-    def writer(mode = :stream, **args, &block)
+    def writer(mode = :stream, **args, &)
       case mode
       when :stream
-        stream_writer(&block)
+        stream_writer(&)
       when :line
-        line_writer(**args, &block)
+        line_writer(**args, &)
       when :array
-        row_writer(**args, &block)
+        row_writer(**args, &)
       when :hash
-        record_writer(**args, &block)
+        record_writer(**args, &)
       else
         raise(ArgumentError, "Invalid mode: #{mode.inspect}")
       end
@@ -171,7 +176,7 @@ module IOStreams
     # IOStreams.path("target_file.json").copy_from("source_file_name.csv.gz", convert: false)
     #
     # # Advanced copy with custom stream conversions on source and target.
-    # source = IOStreams.path("source_file").stream(encoding: "BINARY")
+    # source = IOStreams.path("source_file").stream(:encode, encoding: "BINARY")
     # IOStreams.path("target_file.pgp").option(:pgp, passphrase: "hello").copy_from(source)
     def copy_from(source, convert: true, mode: nil, **args)
       if convert
@@ -322,18 +327,19 @@ module IOStreams
       @builder ||= IOStreams::Builder.new
     end
 
-    def stream_reader(&block)
-      builder.reader(io_stream, &block)
+    def stream_reader(&)
+      builder.reader(io_stream, &)
     end
 
     def line_reader(embedded_within: nil, **args)
       embedded_within = '"' if embedded_within.nil? && builder.file_name&.include?(".csv")
 
       stream_reader do |io|
-        yield IOStreams::Line::Reader.new(io,
-                                          original_file_name: builder.file_name,
-                                          embedded_within:    embedded_within,
-                                          **args)
+        yield IOStreams::Line::Reader.new(
+          io,
+          embedded_within: embedded_within,
+          **args
+        )
       end
     end
 
@@ -363,20 +369,20 @@ module IOStreams
       end
     end
 
-    def stream_writer(&block)
-      builder.writer(io_stream, &block)
+    def stream_writer(&)
+      builder.writer(io_stream, &)
     end
 
     def line_writer(**args, &block)
-      return block.call(io_stream) if io_stream&.is_a?(IOStreams::Line::Writer)
+      return block.call(io_stream) if io_stream.is_a?(IOStreams::Line::Writer)
 
       writer do |io|
-        IOStreams::Line::Writer.stream(io, original_file_name: builder.file_name, **args, &block)
+        IOStreams::Line::Writer.stream(io, **args, &block)
       end
     end
 
     def row_writer(delimiter: $/, **args, &block)
-      return block.call(io_stream) if io_stream&.is_a?(IOStreams::Row::Writer)
+      return block.call(io_stream) if io_stream.is_a?(IOStreams::Row::Writer)
 
       line_writer(delimiter: delimiter) do |io|
         IOStreams::Row::Writer.stream(
@@ -391,7 +397,7 @@ module IOStreams
     end
 
     def record_writer(delimiter: $/, **args, &block)
-      return block.call(io_stream) if io_stream&.is_a?(IOStreams::Record::Writer)
+      return block.call(io_stream) if io_stream.is_a?(IOStreams::Record::Writer)
 
       line_writer(delimiter: delimiter) do |io|
         IOStreams::Record::Writer.stream(

data/lib/io_streams/symmetric_encryption/reader.rb

@@ -2,10 +2,10 @@ module IOStreams
   module SymmetricEncryption
     class Reader < IOStreams::Reader
       # read from a file/stream using Symmetric Encryption
-      def self.stream(input_stream, **args, &block)
+      def self.stream(input_stream, **args, &)
         Utils.load_soft_dependency("symmetric-encryption", ".enc streaming") unless defined?(SymmetricEncryption)
 
-        ::SymmetricEncryption::Reader.open(input_stream, **args, &block)
+        ::SymmetricEncryption::Reader.open(input_stream, **args, &)
       end
     end
   end

data/lib/io_streams/symmetric_encryption/writer.rb

@@ -4,18 +4,18 @@ module IOStreams
       # Write to stream using Symmetric Encryption
       # By default the output stream is compressed.
       # If the input_stream is already compressed consider setting compress: false.
-      def self.stream(input_stream, compress: true, **args, &block)
+      def self.stream(input_stream, compress: true, **args, &)
         Utils.load_soft_dependency("symmetric-encryption", ".enc streaming") unless defined?(SymmetricEncryption)
 
-        ::SymmetricEncryption::Writer.open(input_stream, compress: compress, **args, &block)
+        ::SymmetricEncryption::Writer.open(input_stream, compress: compress, **args, &)
       end
 
       # Write to stream using Symmetric Encryption
       # By default the output stream is compressed unless the file_name extension indicates the file is already compressed.
-      def self.file(file_name, compress: nil, **args, &block)
+      def self.file(file_name, compress: nil, **args, &)
         Utils.load_soft_dependency("symmetric-encryption", ".enc streaming") unless defined?(SymmetricEncryption)
 
-        ::SymmetricEncryption::Writer.open(file_name, compress: compress, **args, &block)
+        ::SymmetricEncryption::Writer.open(file_name, compress: compress, **args, &)
       end
     end
   end

data/lib/io_streams/tabular/header.rb

@@ -5,16 +5,16 @@ module IOStreams
       # Column names that begin with this prefix have been rejected and should be ignored.
       IGNORE_PREFIX = "__rejected__".freeze
 
-      attr_accessor :columns, :allowed_columns, :required_columns, :skip_unknown
+      attr_accessor :allowed_columns, :required_columns, :skip_unknown
+      attr_reader :columns
 
       # Header
       #
       # Parameters
-      #   columns [Array<String>]
+      #   columns [Array<String|Symbol>]
       #     Columns in this header.
       #     Note:
-      #       It is recommended to keep all columns as strings to avoid any issues when persistence
-      #       with MongoDB when it converts symbol keys to strings.
+      #       Column names are converted to strings.
       #
       #   allowed_columns [Array<String>]
       #     List of columns to allow.
@@ -33,12 +33,17 @@ module IOStreams
       #     false:
       #       Raises Tabular::InvalidHeader when a column is supplied that is not in the whitelist.
       def initialize(columns: nil, allowed_columns: nil, required_columns: nil, skip_unknown: true)
-        @columns          = columns
+        @columns          = stringify(columns)
         @required_columns = required_columns
         @allowed_columns  = allowed_columns
         @skip_unknown     = skip_unknown
       end
 
+      # Set the columns in this header, converting the column names to strings.
+      def columns=(columns)
+        @columns = stringify(columns)
+      end
+
       # Returns [Array<String>] list columns that were ignored during cleansing.
       #
       # Each column is cleansed as follows:
@@ -127,13 +132,16 @@ module IOStreams
 
       def array_to_hash(row)
         h = {}
-        columns.each_with_index { |col, i| h[col] = row[i] unless IOStreams::Utils.blank?(col) || col.start_with?(IGNORE_PREFIX) }
+        columns.each_with_index do |col, i|
+          h[col] = row[i] unless IOStreams::Utils.blank?(col) || col.start_with?(IGNORE_PREFIX)
+        end
         h
       end
 
       # Perform cleansing on returned Hash keys during the narrowing process.
       # For example, avoids issues with case etc.
       def cleanse_hash(hash)
+        hash      = hash.transform_keys(&:to_s) unless hash.keys.all?(String)
         unmatched = columns - hash.keys
         unless unmatched.empty?
           hash = hash.dup
@@ -149,6 +157,10 @@ module IOStreams
         cleansed.gsub!(/\W+/, "")
         cleansed
       end
+
+      def stringify(columns)
+        columns&.collect { |column| column&.to_s }
+      end
     end
   end
 end

data/lib/io_streams/tabular/parser/array.rb

@@ -3,16 +3,6 @@ module IOStreams
   class Tabular
     module Parser
       class Array < Base
-        # Returns [Array<String>] the header row.
-        # Returns nil if the row is blank.
-        def parse_header(row)
-          unless row.is_a?(::Array)
-            raise(IOStreams::Errors::InvalidHeader, "Format is :array. Invalid input header: #{row.class.name}")
-          end
-
-          row
-        end
-
         # Returns Array
         def parse(row)
           raise(IOStreams::Errors::TypeMismatch, "Format is :array. Invalid input: #{row.class.name}") unless row.is_a?(::Array)

data/lib/io_streams/tabular/parser/csv.rb

@@ -3,26 +3,6 @@ module IOStreams
   class Tabular
     module Parser
       class Csv < Base
-        attr_reader :csv_parser
-
-        unless RUBY_VERSION.to_f >= 2.6
-          def initialize
-            @csv_parser = Utility::CSVRow.new
-          end
-        end
-
-        # Returns [Array<String>] the header row.
-        # Returns nil if the row is blank.
-        def parse_header(row)
-          return row if row.is_a?(::Array)
-
-          unless row.is_a?(String)
-            raise(IOStreams::Errors::InvalidHeader, "Format is :csv. Invalid input header: #{row.class.name}")
-          end
-
-          parse_line(row)
-        end
-
         # Returns [Array] the parsed CSV line
         def parse(row)
           return row if row.is_a?(::Array)
@@ -40,26 +20,14 @@ module IOStreams
 
         private
 
-        if RUBY_VERSION.to_f >= 2.6
-          # About 10 times slower than the approach used in Ruby 2.5 and earlier,
-          # but at least it works on Ruby 2.6 and above.
-          def parse_line(line)
-            return if IOStreams::Utils.blank?(line)
+        def parse_line(line)
+          return if IOStreams::Utils.blank?(line)
 
-            CSV.parse_line(line)
-          end
-
-          def render_array(array)
-            CSV.generate_line(array, encoding: "UTF-8", row_sep: "")
-          end
-        else
-          def parse_line(line)
-            csv_parser.parse(line)
-          end
+          CSV.parse_line(line)
+        end
 
-          def render_array(array)
-            csv_parser.to_csv(array)
-          end
+        def render_array(array)
+          CSV.generate_line(array, encoding: "UTF-8", row_sep: "")
         end
       end
     end

data/lib/io_streams/tabular/parser/fixed.rb

@@ -71,7 +71,7 @@ module IOStreams
         def render(row, header)
           hash = header.to_hash(row)
 
-          result = ""
+          result = +""
           layout.columns.each do |column|
             result << column.render(hash[column.key], truncate)
           end
@@ -93,7 +93,7 @@ module IOStreams
           index = 0
           layout.columns.each do |column|
             if column.size == -1
-              hash[column.key] = column.parse(line[index..-1]) if column.key
+              hash[column.key] = column.parse(line[index..]) if column.key
               break
             end
 
@@ -148,7 +148,7 @@ module IOStreams
 
           def initialize(size:, key: nil, type: :string, decimals: 2)
             @key      = key
-            @size     = (size == :remainder || size == "remainder") ? -1 : size.to_i
+            @size     = [:remainder, "remainder"].include?(size) ? -1 : size.to_i
             @type     = type.to_sym
             @decimals = decimals
 
@@ -167,9 +167,9 @@ module IOStreams
             when :string
               stripped_value
             when :integer
-              stripped_value.length.zero? ? nil : value.to_i
+              stripped_value.empty? ? nil : value.to_i
             when :float
-              stripped_value.length.zero? ? nil : value.to_f
+              stripped_value.empty? ? nil : value.to_f
             else
               raise(Errors::InvalidLayout, "Unsupported type: #{type.inspect}")
             end

data/lib/io_streams/tabular/parser/psv.rb

@@ -3,18 +3,6 @@ module IOStreams
     module Parser
       # For parsing a single line of Pipe-separated values
       class Psv < Base
-        # Returns [Array<String>] the header row.
-        # Returns nil if the row is blank.
-        def parse_header(row)
-          return row if row.is_a?(::Array)
-
-          unless row.is_a?(String)
-            raise(IOStreams::Errors::InvalidHeader, "Format is :psv. Invalid input header: #{row.class.name}")
-          end
-
-          row.split("|")
-        end
-
         # Returns [Array] the parsed PSV line
         def parse(row)
           return row if row.is_a?(::Array)

data/lib/io_streams/tabular.rb

@@ -40,10 +40,6 @@ module IOStreams
       autoload :Psv, "io_streams/tabular/parser/psv"
     end
 
-    module Utility
-      autoload :CSVRow, "io_streams/tabular/utility/csv_row"
-    end
-
     attr_reader :format, :header, :parser
 
     # Parse a delimited data source.
@@ -59,11 +55,10 @@ module IOStreams
     #   format_options: [Hash]
     #     Any specialized format specific options. For example, `:fixed` format requires the file definition.
     #
-    #   columns [Array<String>]
+    #   columns [Array<String|Symbol>]
     #     The header columns when the file does not include a header row.
     #     Note:
-    #       It is recommended to keep all columns as strings to avoid any issues when persistence
-    #       with MongoDB when it converts symbol keys to strings.
+    #       Column names are converted to strings.
     #
     #   allowed_columns [Array<String>]
     #     List of columns to allow.
@@ -91,7 +86,7 @@ module IOStreams
       @header = Header.new(**args)
       @format = file_name && format.nil? ? self.class.format_from_file_name(file_name) : format
       @format ||= default_format
-      raise(UnknownFormat, "The format cannot be inferred from the file name: #{file_name}") unless @format
+      raise(Errors::UnknownFormat, "The format cannot be inferred from the file name: #{file_name}") unless @format
 
       klass   = self.class.parser_class(@format)
       @parser = format_options ? klass.new(**format_options) : klass.new
@@ -177,7 +172,7 @@ module IOStreams
     # Returns [Symbol] the format removed, or nil if the format was not registered
     #
     # Example:
-    #   register_extension(:xls)
+    #   deregister_format(:psv)
     def self.deregister_format(format)
       raise(ArgumentError, "Invalid format #{format.inspect}") unless format.to_s =~ /\A\w+\Z/
 
@@ -200,7 +195,7 @@ module IOStreams
 
     # Returns the parser class for the registered format.
     def self.parser_class(format)
-      @formats[format.nil? ? nil : format.to_sym] ||
+      @formats[format&.to_sym] ||
         raise(ArgumentError, "Unknown Tabular Format: #{format.inspect}")
     end
 

data/lib/io_streams/utils.rb

@@ -1,3 +1,4 @@
+require "cgi"
 require "uri"
 require "tmpdir"
 module IOStreams
@@ -13,13 +14,10 @@ module IOStreams
 
     # Helper method: Returns [true|false] if a value is blank?
     def self.blank?(value)
-      if value.nil?
-        true
-      elsif value.is_a?(String)
-        value !~ /\S/
-      else
-        value.respond_to?(:empty?) ? value.empty? : !value
-      end
+      return true if value.nil?
+      return value !~ /\S/ if value.is_a?(String)
+
+      value.respond_to?(:empty?) ? value.empty? : !value
     end
 
     # Yields the path to a temporary file_name.
@@ -30,7 +28,7 @@ module IOStreams
       ::Dir::Tmpname.create([basename, extension], IOStreams.temp_dir, max_try: MAX_TEMP_FILE_NAME_ATTEMPTS) do |tmpname|
         result = yield(tmpname)
       ensure
-        ::File.unlink(tmpname) if ::File.exist?(tmpname)
+        ::FileUtils.rm_f(tmpname)
       end
       result
     end

data/lib/io_streams/version.rb

@@ -1,3 +1,3 @@
 module IOStreams
-  VERSION = "1.10.3".freeze
+  VERSION = "2.0.0".freeze
 end

data/lib/io_streams/writer.rb

@@ -2,22 +2,22 @@ module IOStreams
   class Writer
     # When a Writer does not support streams, we copy the stream to a local temp file
     # and then pass that filename in for this reader.
-    def self.stream(output_stream, original_file_name: nil, **args, &block)
+    def self.stream(output_stream, **args, &block)
       Utils.temp_file_name("iostreams_writer") do |file_name|
-        count = file(file_name, original_file_name: original_file_name, **args, &block)
+        count = file(file_name, **args, &block)
         ::File.open(file_name, "rb") { |source| ::IO.copy_stream(source, output_stream) }
         count
       end
     end
 
     # When a Writer supports streams, also allow it to simply support a file
-    def self.file(file_name, original_file_name: file_name, **args, &block)
-      ::File.open(file_name, "wb") { |file| stream(file, original_file_name: original_file_name, **args, &block) }
+    def self.file(file_name, **args, &block)
+      ::File.open(file_name, "wb") { |file| stream(file, **args, &block) }
     end
 
     # For processing by either a file name or an open IO stream.
-    def self.open(file_name_or_io, **args, &block)
-      file_name_or_io.is_a?(String) ? file(file_name_or_io, **args, &block) : stream(file_name_or_io, **args, &block)
+    def self.open(file_name_or_io, **args, &)
+      file_name_or_io.is_a?(String) ? file(file_name_or_io, **args, &) : stream(file_name_or_io, **args, &)
     end
 
     attr_reader :output_stream

data/lib/io_streams/xlsx/reader.rb

@@ -4,7 +4,7 @@ module IOStreams
   module Xlsx
     class Reader < IOStreams::Reader
       # Convert a xlsx, or xlsm file into CSV format.
-      def self.file(file_name, original_file_name: file_name, &block)
+      def self.file(file_name, &block)
         # Stream into a temp file as csv
         Utils.temp_file_name("iostreams_csv") do |temp_file_name|
           ::File.open(temp_file_name, "wb") { |io| new(file_name).each { |lines| io << lines.to_csv } }

data/lib/io_streams/zip/writer.rb

@@ -1,30 +1,42 @@
 module IOStreams
   module Zip
     class Writer < IOStreams::Writer
+      # When writing to a file, default the entry name within the zip to the file name
+      # without the `.zip` extension, unless an entry name was explicitly supplied.
+      def self.file(file_name, zip_file_name: nil, entry_file_name: zip_file_name, &)
+        entry_file_name = file_name.to_s[0..-5] if entry_file_name.nil? && file_name.to_s =~ /\.zip\z/i
+
+        super(file_name, entry_file_name: entry_file_name, &)
+      end
+
       # Write a single file in Zip format to the supplied output stream
       #
       # Parameters
       #   output_stream [IO]
       #     Output stream to write to
       #
-      #   original_file_name [String]
-      #     Since this is a stream the original file name is used to create the entry_file_name if not supplied
-      #
       #   entry_file_name: [String]
       #     Name of the file entry within the Zip file.
+      #     Default: "file"
       #
       # The stream supplied to the block only responds to #write
-      def self.stream(output_stream, original_file_name: nil, zip_file_name: nil, entry_file_name: zip_file_name)
-        # Default the name of the file within the zip to the supplied file_name without the zip extension
-        if entry_file_name.nil? && original_file_name && (original_file_name =~ /\.(zip)\z/i)
-          entry_file_name = original_file_name.to_s[0..-5]
-        end
+      #
+      # Note:
+      #   This writer uses `zip_kit` rather than `rubyzip` on purpose. `rubyzip`'s
+      #   `Zip::OutputStream` requires a seekable output: it seeks back to rewrite each
+      #   entry's local header with the CRC and sizes once the entry is finished. That
+      #   means it cannot write directly to a non-seekable destination (S3, SFTP, HTTP,
+      #   a socket); the output would first have to be spooled to a temporary file and
+      #   then copied across. `zip_kit` streams to non-seekable outputs by emitting
+      #   data descriptors instead, so we can write straight to the output stream and
+      #   avoid the temp file round-trip.
+      def self.stream(output_stream, zip_file_name: nil, entry_file_name: zip_file_name)
         entry_file_name ||= "file"
 
-        Utils.load_soft_dependency("zip_tricks", "Zip") unless defined?(ZipTricks::Streamer)
+        Utils.load_soft_dependency("zip_kit", "Zip") unless defined?(ZipKit::Streamer)
 
         result = nil
-        ZipTricks::Streamer.open(output_stream) do |zip|
+        ZipKit::Streamer.open(output_stream) do |zip|
           zip.write_deflated_file(entry_file_name) { |io| result = yield(io) }
         end
         result

data/lib/iostreams.rb

@@ -63,5 +63,4 @@ module IOStreams
     autoload :Writer, "io_streams/zip/writer"
   end
 end
-require "io_streams/deprecated"
 require "io_streams/io_streams"