iostreams 1.11.0 → 2.1.0

data/lib/io_streams/tabular/parser/fixed.rb

@@ -71,7 +71,7 @@ module IOStreams
         def render(row, header)
           hash = header.to_hash(row)
 
-          result = ""
+          result = +""
           layout.columns.each do |column|
             result << column.render(hash[column.key], truncate)
           end
@@ -93,7 +93,7 @@ module IOStreams
           index = 0
           layout.columns.each do |column|
             if column.size == -1
-              hash[column.key] = column.parse(line[index..-1]) if column.key
+              hash[column.key] = column.parse(line[index..]) if column.key
               break
             end
 
@@ -143,12 +143,13 @@ module IOStreams
 
         class Column
           TYPES = %i[string integer float].freeze
+          LINE_BREAK = /\r\n|\r|\n/
 
           attr_reader :key, :size, :type, :decimals
 
           def initialize(size:, key: nil, type: :string, decimals: 2)
             @key      = key
-            @size     = (size == :remainder || size == "remainder") ? -1 : size.to_i
+            @size     = [:remainder, "remainder"].include?(size) ? -1 : size.to_i
             @type     = type.to_sym
             @decimals = decimals
 
@@ -167,9 +168,9 @@ module IOStreams
             when :string
               stripped_value
             when :integer
-              stripped_value.length.zero? ? nil : value.to_i
+              stripped_value.empty? ? nil : value.to_i
             when :float
-              stripped_value.length.zero? ? nil : value.to_f
+              stripped_value.empty? ? nil : value.to_f
             else
               raise(Errors::InvalidLayout, "Unsupported type: #{type.inspect}")
             end
@@ -179,7 +180,8 @@ module IOStreams
             formatted =
               case type
               when :string
-                value = value.to_s
+                # Replace line breaks with a space so that a value cannot add records.
+                value = value.to_s.gsub(LINE_BREAK, " ")
                 return value if size == -1
 
                 format(truncate ? "%-#{size}.#{size}s" : "%-#{size}s", value)

data/lib/io_streams/tabular/parser/psv.rb

@@ -3,18 +3,7 @@ module IOStreams
     module Parser
       # For parsing a single line of Pipe-separated values
       class Psv < Base
-        # Returns [Array<String>] the header row.
-        # Returns nil if the row is blank.
-        def parse_header(row)
-          return row if row.is_a?(::Array)
-
-          unless row.is_a?(String)
-            raise(IOStreams::Errors::InvalidHeader, "Format is :psv. Invalid input header: #{row.class.name}")
-          end
-
-          row.split("|")
-        end
-
+        LINE_BREAK = /\r\n|\r|\n/
         # Returns [Array] the parsed PSV line
         def parse(row)
           return row if row.is_a?(::Array)
@@ -24,11 +13,14 @@ module IOStreams
           row.split("|")
         end
 
-        # Return the supplied array as a single line JSON string.
+        # Return the supplied array as a single line PSV string.
+        #
+        # Since PSV has no escaping, any `|` within a value is replaced with `:`,
+        # and any line break with a space, so that a value cannot add columns or records.
         def render(row, header)
           array          = header.to_array(row)
           cleansed_array = array.collect do |i|
-            i.is_a?(String) ? i.tr("|", ":") : i
+            i.is_a?(String) ? i.tr("|", ":").gsub(LINE_BREAK, " ") : i
           end
           cleansed_array.join("|")
         end

data/lib/io_streams/tabular.rb

@@ -40,10 +40,6 @@ module IOStreams
       autoload :Psv, "io_streams/tabular/parser/psv"
     end
 
-    module Utility
-      autoload :CSVRow, "io_streams/tabular/utility/csv_row"
-    end
-
     attr_reader :format, :header, :parser
 
     # Parse a delimited data source.
@@ -59,11 +55,10 @@ module IOStreams
     #   format_options: [Hash]
     #     Any specialized format specific options. For example, `:fixed` format requires the file definition.
     #
-    #   columns [Array<String>]
+    #   columns [Array<String|Symbol>]
     #     The header columns when the file does not include a header row.
     #     Note:
-    #       It is recommended to keep all columns as strings to avoid any issues when persistence
-    #       with MongoDB when it converts symbol keys to strings.
+    #       Column names are converted to strings.
     #
     #   allowed_columns [Array<String>]
     #     List of columns to allow.
@@ -91,7 +86,7 @@ module IOStreams
       @header = Header.new(**args)
       @format = file_name && format.nil? ? self.class.format_from_file_name(file_name) : format
       @format ||= default_format
-      raise(UnknownFormat, "The format cannot be inferred from the file name: #{file_name}") unless @format
+      raise(Errors::UnknownFormat, "The format cannot be inferred from the file name: #{file_name}") unless @format
 
       klass   = self.class.parser_class(@format)
       @parser = format_options ? klass.new(**format_options) : klass.new
@@ -177,7 +172,7 @@ module IOStreams
     # Returns [Symbol] the format removed, or nil if the format was not registered
     #
     # Example:
-    #   register_extension(:xls)
+    #   deregister_format(:psv)
     def self.deregister_format(format)
       raise(ArgumentError, "Invalid format #{format.inspect}") unless format.to_s =~ /\A\w+\Z/
 
@@ -200,7 +195,7 @@ module IOStreams
 
     # Returns the parser class for the registered format.
     def self.parser_class(format)
-      @formats[format.nil? ? nil : format.to_sym] ||
+      @formats[format&.to_sym] ||
         raise(ArgumentError, "Unknown Tabular Format: #{format.inspect}")
     end
 

data/lib/io_streams/utils.rb

@@ -1,3 +1,4 @@
+require "cgi"
 require "uri"
 require "tmpdir"
 module IOStreams
@@ -11,11 +12,22 @@ module IOStreams
       raise(LoadError, "Please install the gem '#{gem_name}' to support #{stream_type}. #{e.message}")
     end
 
+    # Log a warning for options that a stream ignores, until they raise `ArgumentError` in v3.0.
+    def self.warn_unknown_options(unknown, stream, direction, valid)
+      return if unknown.empty?
+
+      names = unknown.keys.map(&:inspect).join(", ")
+      IOStreams.logger&.warn(
+        "Ignoring unknown #{unknown.size == 1 ? 'option' : 'options'} #{names} when #{direction} a #{stream.inspect} " \
+        "stream. Valid options: #{valid.map(&:inspect).join(', ')}. In v3.0 this will raise ArgumentError."
+      )
+    end
+
     # Helper method: Returns [true|false] if a value is blank?
     def self.blank?(value)
       return true if value.nil?
       return value !~ /\S/ if value.is_a?(String)
-        
+
       value.respond_to?(:empty?) ? value.empty? : !value
     end
 
@@ -27,11 +39,31 @@ module IOStreams
       ::Dir::Tmpname.create([basename, extension], IOStreams.temp_dir, max_try: MAX_TEMP_FILE_NAME_ATTEMPTS) do |tmpname|
         result = yield(tmpname)
       ensure
-        ::File.unlink(tmpname) if ::File.exist?(tmpname)
+        ::FileUtils.rm_f(tmpname)
       end
       result
     end
 
+    # Yields the name of a new, empty temporary file that only the current user can read or write.
+    #
+    # The file is created exclusively, so that an existing file, or a link planted in a shared
+    # temp directory, is never written to. Only a name collision when creating the file is retried,
+    # and the file is only deleted once it was created here, so that another process's file is never removed.
+    #
+    # Returns the value from the block.
+    def self.private_temp_file(basename, extension = "")
+      file_name = ::Dir::Tmpname.create([basename, extension], IOStreams.temp_dir,
+                                        max_try: MAX_TEMP_FILE_NAME_ATTEMPTS) do |tmpname|
+        ::File.open(tmpname, ::File::WRONLY | ::File::CREAT | ::File::EXCL, 0o600, &:close)
+      end
+
+      begin
+        yield(file_name)
+      ensure
+        ::FileUtils.rm_f(file_name)
+      end
+    end
+
     class URI
       attr_reader :scheme, :hostname, :path, :user, :password, :port, :query
 

data/lib/io_streams/version.rb

@@ -1,3 +1,3 @@
 module IOStreams
-  VERSION = "1.11.0".freeze
+  VERSION = "2.1.0".freeze
 end

data/lib/io_streams/writer.rb

@@ -1,23 +1,32 @@
 module IOStreams
   class Writer
+    # Returns [Array<Symbol>] the names of the options this writer accepts,
+    # or [nil] when the writer does not declare them.
+    #
+    # When declared, `IOStreams::Builder` rejects any other option before the writer is opened,
+    # naming the direction an option belongs to when it is only valid for the other direction.
+    def self.option_names
+      nil
+    end
+
     # When a Writer does not support streams, we copy the stream to a local temp file
     # and then pass that filename in for this reader.
-    def self.stream(output_stream, original_file_name: nil, **args, &block)
-      Utils.temp_file_name("iostreams_writer") do |file_name|
-        count = file(file_name, original_file_name: original_file_name, **args, &block)
+    def self.stream(output_stream, **args, &block)
+      Utils.private_temp_file("iostreams_writer") do |file_name|
+        count = file(file_name, **args, &block)
         ::File.open(file_name, "rb") { |source| ::IO.copy_stream(source, output_stream) }
         count
       end
     end
 
     # When a Writer supports streams, also allow it to simply support a file
-    def self.file(file_name, original_file_name: file_name, **args, &block)
-      ::File.open(file_name, "wb") { |file| stream(file, original_file_name: original_file_name, **args, &block) }
+    def self.file(file_name, **args, &block)
+      ::File.open(file_name, "wb") { |file| stream(file, **args, &block) }
     end
 
     # For processing by either a file name or an open IO stream.
-    def self.open(file_name_or_io, **args, &block)
-      file_name_or_io.is_a?(String) ? file(file_name_or_io, **args, &block) : stream(file_name_or_io, **args, &block)
+    def self.open(file_name_or_io, **args, &)
+      file_name_or_io.is_a?(String) ? file(file_name_or_io, **args, &) : stream(file_name_or_io, **args, &)
     end
 
     attr_reader :output_stream

data/lib/io_streams/xlsx/reader.rb

@@ -3,10 +3,14 @@ require "csv"
 module IOStreams
   module Xlsx
     class Reader < IOStreams::Reader
+      def self.option_names
+        []
+      end
+
       # Convert a xlsx, or xlsm file into CSV format.
-      def self.file(file_name, original_file_name: file_name, &block)
+      def self.file(file_name, &block)
         # Stream into a temp file as csv
-        Utils.temp_file_name("iostreams_csv") do |temp_file_name|
+        Utils.private_temp_file("iostreams_csv") do |temp_file_name|
           ::File.open(temp_file_name, "wb") { |io| new(file_name).each { |lines| io << lines.to_csv } }
           ::File.open(temp_file_name, "rb", &block)
         end

data/lib/io_streams/zip/reader.rb

@@ -1,6 +1,10 @@
 module IOStreams
   module Zip
     class Reader < IOStreams::Reader
+      def self.option_names
+        %i[entry_file_name]
+      end
+
       # Read from a zip file or stream, decompressing the contents as it is read
       # The input stream from the first file found in the zip file is passed
       # to the supplied block.

data/lib/io_streams/zip/writer.rb

@@ -1,30 +1,46 @@
 module IOStreams
   module Zip
     class Writer < IOStreams::Writer
+      def self.option_names
+        %i[zip_file_name entry_file_name]
+      end
+
+      # When writing to a file, default the entry name within the zip to the file name
+      # without the `.zip` extension, unless an entry name was explicitly supplied.
+      def self.file(file_name, zip_file_name: nil, entry_file_name: zip_file_name, &)
+        entry_file_name = file_name.to_s[0..-5] if entry_file_name.nil? && file_name.to_s =~ /\.zip\z/i
+
+        super(file_name, entry_file_name: entry_file_name, &)
+      end
+
       # Write a single file in Zip format to the supplied output stream
       #
       # Parameters
       #   output_stream [IO]
       #     Output stream to write to
       #
-      #   original_file_name [String]
-      #     Since this is a stream the original file name is used to create the entry_file_name if not supplied
-      #
       #   entry_file_name: [String]
       #     Name of the file entry within the Zip file.
+      #     Default: "file"
       #
       # The stream supplied to the block only responds to #write
-      def self.stream(output_stream, original_file_name: nil, zip_file_name: nil, entry_file_name: zip_file_name)
-        # Default the name of the file within the zip to the supplied file_name without the zip extension
-        if entry_file_name.nil? && original_file_name && (original_file_name =~ /\.(zip)\z/i)
-          entry_file_name = original_file_name.to_s[0..-5]
-        end
+      #
+      # Note:
+      #   This writer uses `zip_kit` rather than `rubyzip` on purpose. `rubyzip`'s
+      #   `Zip::OutputStream` requires a seekable output: it seeks back to rewrite each
+      #   entry's local header with the CRC and sizes once the entry is finished. That
+      #   means it cannot write directly to a non-seekable destination (S3, SFTP, HTTP,
+      #   a socket); the output would first have to be spooled to a temporary file and
+      #   then copied across. `zip_kit` streams to non-seekable outputs by emitting
+      #   data descriptors instead, so we can write straight to the output stream and
+      #   avoid the temp file round-trip.
+      def self.stream(output_stream, zip_file_name: nil, entry_file_name: zip_file_name)
         entry_file_name ||= "file"
 
-        Utils.load_soft_dependency("zip_tricks", "Zip") unless defined?(ZipTricks::Streamer)
+        Utils.load_soft_dependency("zip_kit", "Zip") unless defined?(ZipKit::Streamer)
 
         result = nil
-        ZipTricks::Streamer.open(output_stream) do |zip|
+        ZipKit::Streamer.open(output_stream) do |zip|
           zip.write_deflated_file(entry_file_name) { |io| result = yield(io) }
         end
         result

data/lib/iostreams.rb

@@ -63,5 +63,4 @@ module IOStreams
     autoload :Writer, "io_streams/zip/writer"
   end
 end
-require "io_streams/deprecated"
 require "io_streams/io_streams"

metadata

@@ -1,17 +1,32 @@
 --- !ruby/object:Gem::Specification
 name: iostreams
 version: !ruby/object:Gem::Version
-  version: 1.11.0
+  version: 2.1.0
 platform: ruby
 authors:
 - Reid Morrison
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-09-30 00:00:00.000000000 Z
-dependencies: []
-description:
-email:
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: IOStreams makes file formats, compression (gzip, zip, bzip2), encryption
+  (PGP, symmetric), and storage location (local file, S3, SFTP, HTTP) transparent
+  to your application code. Files of any size are read and written one block at a
+  time, without loading the entire file into memory.
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -19,10 +34,20 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- docs/CLAUDE.md
+- docs/config.md
+- docs/copy_files.md
+- docs/extensions.md
+- docs/formats.md
+- docs/index.md
+- docs/path.md
+- docs/pgp.md
+- docs/streams.md
+- docs/tutorial.md
+- docs/upgrading.md
 - lib/io_streams/builder.rb
 - lib/io_streams/bzip2/reader.rb
 - lib/io_streams/bzip2/writer.rb
-- lib/io_streams/deprecated.rb
 - lib/io_streams/encode/reader.rb
 - lib/io_streams/encode/writer.rb
 - lib/io_streams/errors.rb
@@ -37,6 +62,7 @@ files:
 - lib/io_streams/paths/matcher.rb
 - lib/io_streams/paths/s3.rb
 - lib/io_streams/paths/sftp.rb
+- lib/io_streams/paths/sftp/net_ssh.rb
 - lib/io_streams/pgp.rb
 - lib/io_streams/pgp/reader.rb
 - lib/io_streams/pgp/writer.rb
@@ -57,7 +83,6 @@ files:
 - lib/io_streams/tabular/parser/hash.rb
 - lib/io_streams/tabular/parser/json.rb
 - lib/io_streams/tabular/parser/psv.rb
-- lib/io_streams/tabular/utility/csv_row.rb
 - lib/io_streams/utils.rb
 - lib/io_streams/version.rb
 - lib/io_streams/writer.rb
@@ -65,59 +90,16 @@ files:
 - lib/io_streams/zip/reader.rb
 - lib/io_streams/zip/writer.rb
 - lib/iostreams.rb
-- test/builder_test.rb
-- test/bzip2_reader_test.rb
-- test/bzip2_writer_test.rb
-- test/deprecated_test.rb
-- test/encode_reader_test.rb
-- test/encode_writer_test.rb
-- test/files/embedded_lines_test.csv
-- test/files/multiple_files.zip
-- test/files/spreadsheet.xlsx
-- test/files/test.csv
-- test/files/test.json
-- test/files/test.psv
-- test/files/text file.txt
-- test/files/text.txt
-- test/files/text.txt.bz2
-- test/files/text.txt.gz
-- test/files/text.txt.gz.zip
-- test/files/text.zip
-- test/files/text.zip.gz
-- test/files/unclosed_quote_large_test.csv
-- test/files/unclosed_quote_test.csv
-- test/files/unclosed_quote_test2.csv
-- test/gzip_reader_test.rb
-- test/gzip_writer_test.rb
-- test/io_streams_test.rb
-- test/line_reader_test.rb
-- test/line_writer_test.rb
-- test/minimal_file_reader.rb
-- test/path_test.rb
-- test/paths/file_test.rb
-- test/paths/http_test.rb
-- test/paths/matcher_test.rb
-- test/paths/s3_test.rb
-- test/paths/sftp_test.rb
-- test/pgp_reader_test.rb
-- test/pgp_test.rb
-- test/pgp_writer_test.rb
-- test/record_reader_test.rb
-- test/record_writer_test.rb
-- test/row_reader_test.rb
-- test/row_writer_test.rb
-- test/stream_test.rb
-- test/tabular_test.rb
-- test/test_helper.rb
-- test/utils_test.rb
-- test/xlsx_reader_test.rb
-- test/zip_reader_test.rb
-- test/zip_writer_test.rb
-homepage: https://iostreams.rocketjob.io
+homepage: https://iostreams.reidmorrison.com
 licenses:
 - Apache-2.0
-metadata: {}
-post_install_message:
+metadata:
+  bug_tracker_uri: https://github.com/reidmorrison/iostreams/issues
+  changelog_uri: https://github.com/reidmorrison/iostreams/blob/main/CHANGELOG.md
+  documentation_uri: https://iostreams.reidmorrison.com
+  homepage_uri: https://iostreams.reidmorrison.com
+  source_code_uri: https://github.com/reidmorrison/iostreams/tree/v2.1.0
+  rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:
 - lib
@@ -125,63 +107,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.5'
+      version: '3.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.19
-signing_key:
+rubygems_version: 3.6.9
 specification_version: 4
-summary: Input and Output streaming for Ruby.
-test_files:
-- test/builder_test.rb
-- test/bzip2_reader_test.rb
-- test/bzip2_writer_test.rb
-- test/deprecated_test.rb
-- test/encode_reader_test.rb
-- test/encode_writer_test.rb
-- test/files/embedded_lines_test.csv
-- test/files/multiple_files.zip
-- test/files/spreadsheet.xlsx
-- test/files/test.csv
-- test/files/test.json
-- test/files/test.psv
-- test/files/text file.txt
-- test/files/text.txt
-- test/files/text.txt.bz2
-- test/files/text.txt.gz
-- test/files/text.txt.gz.zip
-- test/files/text.zip
-- test/files/text.zip.gz
-- test/files/unclosed_quote_large_test.csv
-- test/files/unclosed_quote_test.csv
-- test/files/unclosed_quote_test2.csv
-- test/gzip_reader_test.rb
-- test/gzip_writer_test.rb
-- test/io_streams_test.rb
-- test/line_reader_test.rb
-- test/line_writer_test.rb
-- test/minimal_file_reader.rb
-- test/path_test.rb
-- test/paths/file_test.rb
-- test/paths/http_test.rb
-- test/paths/matcher_test.rb
-- test/paths/s3_test.rb
-- test/paths/sftp_test.rb
-- test/pgp_reader_test.rb
-- test/pgp_test.rb
-- test/pgp_writer_test.rb
-- test/record_reader_test.rb
-- test/record_writer_test.rb
-- test/row_reader_test.rb
-- test/row_writer_test.rb
-- test/stream_test.rb
-- test/tabular_test.rb
-- test/test_helper.rb
-- test/utils_test.rb
-- test/xlsx_reader_test.rb
-- test/zip_reader_test.rb
-- test/zip_writer_test.rb
+summary: 'Streaming I/O for Ruby: compression, encryption, file format, and storage
+  location transparent to your code.'
+test_files: []