iostreams 1.11.0 → 2.1.0

data/lib/io_streams/builder.rb

@@ -50,13 +50,13 @@ module IOStreams
       self
     end
 
-    def option_or_stream(stream, **options)
+    def option_or_stream(stream, **)
       if streams
-        stream(stream, **options)
+        stream(stream, **)
       elsif file_name
-        option(stream, **options)
+        option(stream, **)
       else
-        stream(stream, **options)
+        stream(stream, **)
       end
     end
 
@@ -67,12 +67,12 @@ module IOStreams
       options[stream] if options
     end
 
-    def reader(io_stream, &block)
-      execute(:reader, pipeline, io_stream, &block)
+    def reader(io_stream, &)
+      execute(:reader, pipeline, io_stream, &)
     end
 
-    def writer(io_stream, &block)
-      execute(:writer, pipeline, io_stream, &block)
+    def writer(io_stream, &)
+      execute(:writer, pipeline, io_stream, &)
     end
 
     # Returns [Hash<Symbol:Hash>] the pipeline of streams
@@ -105,6 +105,18 @@ module IOStreams
       @format = format
     end
 
+    # Returns [String] the quote character within which field delimiters and newlines may be
+    # embedded for the current tabular format, or [nil] when the format has no such quoting,
+    # or when the format cannot be determined.
+    #
+    # Used by the line reader to avoid treating a newline as a line ending when it is embedded
+    # within a quoted field (e.g. CSV). Delegates to the format's parser, so the per-format
+    # knowledge lives with the parser. Driven entirely by `format`, so an explicitly set format
+    # (e.g. `.format(:psv)`) overrides any extension auto-detected from the `file_name`.
+    def quote_character
+      format && IOStreams::Tabular.parser_class(format).quote_character
+    end
+
     private
 
     def build_pipeline
@@ -120,7 +132,7 @@ module IOStreams
     end
 
     def class_for_stream(type, stream)
-      ext = IOStreams.extensions[stream.nil? ? nil : stream.to_sym] ||
+      ext = IOStreams.extensions[stream&.to_sym] ||
             raise(ArgumentError, "Unknown Stream type: #{stream.inspect}")
       ext.send("#{type}_class") || raise(ArgumentError, "No #{type} registered for Stream type: #{stream.inspect}")
     end
@@ -146,14 +158,62 @@ module IOStreams
         block.call(io_stream)
       elsif pipeline.size == 1
         stream, opts = pipeline.first
-        class_for_stream(type, stream).open(io_stream, **opts, &block)
+        open_stream(type, stream, io_stream, opts, &block)
       else
         # Daisy chain multiple streams together
         last = pipeline.keys.inject(block) do |inner, stream_sym|
-          ->(io) { class_for_stream(type, stream_sym).open(io, **pipeline[stream_sym], &inner) }
+          ->(io) { open_stream(type, stream_sym, io, pipeline[stream_sym], &inner) }
         end
         last.call(io_stream)
       end
     end
+
+    def open_stream(type, stream, io_stream, opts, &)
+      klass = class_for_stream(type, stream)
+      validate_options(type, stream, klass, opts)
+      klass.open(io_stream, **opts, &)
+    end
+
+    # Options are strict: an option the stream does not accept raises instead of being ignored.
+    # One option hash is shared by the reader and the writer for a stream, so when the option
+    # is only valid in the other direction the message says so.
+    #
+    # Streams registered via `IOStreams.register_extension` need not inherit from `IOStreams::Reader`
+    # or `IOStreams::Writer`, so a class that does not declare `option_names` is not validated here.
+    def validate_options(type, stream, klass, opts)
+      accepted = option_names(klass)
+      return if accepted.nil?
+
+      unknown = opts.keys - accepted
+      return if unknown.empty?
+
+      other_type  = type == :reader ? :writer : :reader
+      other_names = option_names(IOStreams.extensions[stream].send("#{other_type}_class")) || []
+      other_only  = unknown & other_names
+      invalid     = unknown - other_names
+      direction   = type == :reader ? "reading" : "writing"
+
+      messages = []
+      if other_only.any?
+        messages << "#{list(other_only)} only #{other_only.size == 1 ? 'applies' : 'apply'} when " \
+                    "#{type == :reader ? 'writing' : 'reading'} a #{stream.inspect} stream and cannot be used when " \
+                    "#{direction}. Configure a separate path or stream without #{other_only.size == 1 ? 'it' : 'them'} " \
+                    "for #{direction}."
+      end
+      if invalid.any?
+        valid = accepted.empty? ? "none" : list(accepted)
+        messages << "Unknown #{invalid.size == 1 ? 'option' : 'options'} #{list(invalid)} when #{direction} " \
+                    "a #{stream.inspect} stream. Valid options: #{valid}."
+      end
+      raise(ArgumentError, messages.join(" "))
+    end
+
+    def option_names(klass)
+      klass.option_names if klass.respond_to?(:option_names)
+    end
+
+    def list(names)
+      names.map(&:inspect).join(", ")
+    end
   end
 end

data/lib/io_streams/bzip2/reader.rb

@@ -1,12 +1,35 @@
 module IOStreams
   module Bzip2
     class Reader < IOStreams::Reader
+      OPTION_NAMES = %i[autoclose first_only small].freeze
+
+      # Not declared until v3.0, so that an unknown option logs a warning instead of raising `ArgumentError`.
+      def self.option_names
+        nil
+      end
+
       # Read from a Bzip2 stream, decompressing the contents as it is read
-      def self.stream(input_stream, **args)
+      #
+      # Any other option is ignored and logs a warning. It will raise `ArgumentError` in v3.0.
+      #
+      # Parameters are passed through to `Bzip2::FFI::Reader`:
+      #   autoclose: [true|false]
+      #     Close the input stream when the reader is closed.
+      #     Default: false
+      #
+      #   first_only: [true|false]
+      #     Only decompress the first of any consecutive bzip2 structures in the input.
+      #     Default: false
+      #
+      #   small: [true|false]
+      #     Use an alternative decompression algorithm that uses less memory but is slower.
+      #     Default: false
+      def self.stream(input_stream, autoclose: false, first_only: false, small: false, **unknown)
+        Utils.warn_unknown_options(unknown, :bz2, "reading", OPTION_NAMES)
         Utils.load_soft_dependency("bzip2-ffi", "Bzip2", "bzip2/ffi") unless defined?(::Bzip2::FFI)
 
         begin
-          io = ::Bzip2::FFI::Reader.new(input_stream, args)
+          io = ::Bzip2::FFI::Reader.new(input_stream, autoclose: autoclose, first_only: first_only, small: small)
           yield io
         ensure
           io&.close

data/lib/io_streams/bzip2/writer.rb

@@ -1,12 +1,36 @@
 module IOStreams
   module Bzip2
     class Writer < IOStreams::Writer
+      OPTION_NAMES = %i[autoclose block_size work_factor].freeze
+
+      # Not declared until v3.0, so that an unknown option logs a warning instead of raising `ArgumentError`.
+      def self.option_names
+        nil
+      end
+
       # Write to a stream, compressing with Bzip2
-      def self.stream(input_stream, original_file_name: nil, **args)
+      #
+      # Any other option is ignored and logs a warning. It will raise `ArgumentError` in v3.0.
+      #
+      # Parameters are passed through to `Bzip2::FFI::Writer`:
+      #   autoclose: [true|false]
+      #     Close the output stream when the writer is closed.
+      #     Default: false
+      #
+      #   block_size: [Integer]
+      #     Compression block size, from 1 (100k) to 9 (900k).
+      #     Default: 9
+      #
+      #   work_factor: [Integer]
+      #     How much effort to spend on highly repetitive input before falling back
+      #     to a slower algorithm, from 0 to 250. 0 uses the libbz2 default.
+      #     Default: 0
+      def self.stream(input_stream, autoclose: false, block_size: nil, work_factor: nil, **unknown)
+        Utils.warn_unknown_options(unknown, :bz2, "writing", OPTION_NAMES)
         Utils.load_soft_dependency("bzip2-ffi", "Bzip2", "bzip2/ffi") unless defined?(::Bzip2::FFI)
 
         begin
-          io = ::Bzip2::FFI::Writer.new(input_stream, args)
+          io = ::Bzip2::FFI::Writer.new(input_stream, autoclose: autoclose, block_size: block_size, work_factor: work_factor)
           yield io
         ensure
           io&.close

data/lib/io_streams/encode/reader.rb

@@ -1,9 +1,13 @@
 module IOStreams
   module Encode
     class Reader < IOStreams::Reader
+      def self.option_names
+        %i[encoding cleaner replace]
+      end
+
       attr_reader :encoding, :cleaner
 
-      NOT_PRINTABLE = Regexp.compile(/[^[:print:]|\r|\n]/).freeze
+      NOT_PRINTABLE = /[^[:print:]|\r\n]/
       # Builtin strip options to apply after encoding the read data.
       CLEANSE_RULES = {
         # Strips all non printable characters
@@ -13,7 +17,7 @@ module IOStreams
       }.freeze
 
       # Read a line at a time from a file or stream
-      def self.stream(input_stream, original_file_name: nil, **args)
+      def self.stream(input_stream, **args)
         yield new(input_stream, **args)
       end
 

data/lib/io_streams/encode/writer.rb

@@ -1,10 +1,14 @@
 module IOStreams
   module Encode
     class Writer < IOStreams::Writer
+      def self.option_names
+        %i[encoding cleaner replace]
+      end
+
       attr_reader :encoding, :cleaner
 
       # Write a line at a time to a file or stream
-      def self.stream(input_stream, original_file_name: nil, **args)
+      def self.stream(input_stream, **args)
         yield new(input_stream, **args)
       end
 
@@ -46,7 +50,7 @@ module IOStreams
       # Write a line to the output stream
       #
       # Example:
-      #   IOStreams.writer('a.txt', encoding: 'UTF-8') do |stream|
+      #   IOStreams.path('a.txt').option(:encode, encoding: 'UTF-8').writer do |stream|
       #     stream << 'first line' << 'second line'
       #   end
       def <<(record)
@@ -54,13 +58,13 @@ module IOStreams
         self
       end
 
-      # Write a line to the output stream followed by the delimiter.
+      # Encode data and write it to the output stream.
       # Returns [Integer] the number of bytes written.
       #
       # Example:
-      #   IOStreams.writer('a.txt', encoding: 'UTF-8') do |stream|
+      #   IOStreams.path('a.txt').option(:encode, encoding: 'UTF-8').writer do |stream|
       #     count = stream.write('first line')
-      #     puts "Wrote #{count} bytes to the output file, including the delimiter"
+      #     puts "Wrote #{count} bytes to the output file"
       #   end
       def write(data)
         return 0 if data.nil?

data/lib/io_streams/errors.rb

@@ -18,6 +18,10 @@ module IOStreams
     class CommunicationsFailure < Error
     end
 
+    # When a path is not within any of the allowed paths, see `IOStreams.add_allowed_path`.
+    class AccessDenied < Error
+    end
+
     # When the specified delimiter is not found in the supplied stream / file
     class DelimiterNotFound < Error
     end

data/lib/io_streams/gzip/reader.rb

@@ -1,8 +1,12 @@
 module IOStreams
   module Gzip
     class Reader < IOStreams::Reader
+      def self.option_names
+        []
+      end
+
       # Read from a gzip stream, decompressing the contents as it is read
-      def self.stream(input_stream, original_file_name: nil)
+      def self.stream(input_stream)
         io = ::Zlib::GzipReader.new(input_stream)
         yield io
       ensure

data/lib/io_streams/gzip/writer.rb

@@ -1,9 +1,18 @@
 module IOStreams
   module Gzip
     class Writer < IOStreams::Writer
+      def self.option_names
+        %i[level]
+      end
+
       # Write to a stream, compressing with GZip
-      def self.stream(input_stream, original_file_name: nil, &block)
-        io = ::Zlib::GzipWriter.new(input_stream)
+      #
+      # Parameters
+      #   level: [Integer]
+      #     Compression level, from 0 (no compression) to 9 (best compression).
+      #     Default: Zlib::DEFAULT_COMPRESSION
+      def self.stream(input_stream, level: nil, &block)
+        io = ::Zlib::GzipWriter.new(input_stream, level)
         block.call(io)
       ensure
         io&.close

data/lib/io_streams/io_streams.rb

@@ -23,7 +23,7 @@ module IOStreams
   #    # => "/usr/local/sample"
   #
   #    IOStreams.path("s3://mybucket/path/file.xls")
-  #    # => #<IOStreams::S3::Path:0x00007fec66e3a288, @path="s3://mybucket/path/file.xls">
+  #    # => #<IOStreams::Paths::S3:0x00007fec66e3a288 @path="s3://mybucket/path/file.xls">
   #
   #    IOStreams.path("s3://mybucket/path/file.xls").to_s
   #    # => "s3://mybucket/path/file.xls"
@@ -36,10 +36,9 @@ module IOStreams
   #
   # For Files
   # IOStreams.path('blah.zip').option(:encode, encoding: 'BINARY').each(:line) { |line| puts line }
-  # IOStreams.path('blah.zip').option(:encode, encoding: 'UTF-8').each(:line).first
-  # IOStreams.path('blah.zip').option(:encode, encoding: 'UTF-8').each(:hash).last
-  # IOStreams.path('blah.zip').option(:encode, encoding: 'UTF-8').each(:hash).size
-  # IOStreams.path('blah.zip').option(:encode, encoding: 'UTF-8').reader.size
+  # IOStreams.path('blah.zip').option(:encode, encoding: 'UTF-8').each(:line) { |line| puts line }
+  # IOStreams.path('blah.zip').option(:encode, encoding: 'UTF-8').each(:hash) { |hash| p hash }
+  # IOStreams.path('blah.zip').option(:encode, encoding: 'UTF-8').read
   # IOStreams.path('blah.csv.zip').each(:line) { |line| puts line }
   # IOStreams.path('blah.zip').option(:pgp, passphrase: 'receiver_passphrase').read
   # IOStreams.path('blah.zip').stream(:zip).stream(:pgp, passphrase: 'receiver_passphrase').read
@@ -75,14 +74,25 @@ module IOStreams
 
   # Join the supplied path elements to a root path.
   #
+  # Roots allow paths to reference a particular root directory, so that all path names
+  # are appended to that root. Use `IOStreams.join` instead of `IOStreams.path` so that
+  # the exact same code can run in production and development, yet use completely
+  # different data sources in each. For example, in production the root can point to
+  # an S3 bucket, while in development it points to the local file system.
+  #
+  # Roots are configured via an initializer at startup. Multiple roots can be setup,
+  # for example one for input files, another for output files, another for reports, etc.
+  # The `:default` root is used whenever a root is not supplied when calling `IOStreams.join`.
+  #
   # Example:
   #    IOStreams.add_root(:default, "tmp/export")
+  #    IOStreams.add_root(:ftp, "tmp/ftp")
   #
   #    IOStreams.join('file.xls')
-  #    # => #<IOStreams::Paths::File:0x00007fec70391bd8 @path="tmp/export/sample">
+  #    # => #<IOStreams::Paths::File:0x00007fec70391bd8 @path="tmp/export/file.xls">
   #
   #    IOStreams.join('file.xls').to_s
-  #    # => "tmp/export/sample"
+  #    # => "tmp/export/file.xls"
   #
   #    IOStreams.join('sample', 'file.xls', root: :ftp)
   #    # => #<IOStreams::Paths::File:0x00007fec6ee329b8 @path="tmp/ftp/sample/file.xls">
@@ -108,9 +118,13 @@ module IOStreams
   #     Optional extension to add to the tempfile.
   #
   # Example:
-  #   IOStreams.temp_file
+  #   IOStreams.temp_file("export", ".csv") { |path| path.write("Hello World") }
+  #
+  # Note: The temp file is accessible even when it is not within the allowed paths, see `IOStreams.add_allowed_path`.
   def self.temp_file(basename, extension = "")
-    Utils.temp_file_name(basename, extension) { |file_name| yield(Paths::File.new(file_name).stream(:none)) }
+    Utils.temp_file_name(basename, extension) do |file_name|
+      yield(Paths::File.new(file_name).send(:permit!).stream(:none))
+    end
   end
 
   # Returns [IOStreams::Paths::File] current or named users home path
@@ -193,9 +207,9 @@ module IOStreams
   # "\a"        "a"              true     # escaped ordinary remains ordinary
   # "[\?]"      "?"              true     # can escape inside bracket expression
   #
-  # "*"         ".profile"       false    # wildcard doesn't match leading
-  # "*"         ".profile"       true     # period by default.
-  # ".*"        ".profile"       true                                   {hidden: true}
+  # "*"         ".profile"       false    # wildcard doesn't match leading period by default
+  # "*"         ".profile"       true     # unless hidden is enabled    {hidden: true}
+  # ".*"        ".profile"       true     # leading period is explicit
   #
   # "**/*.rb"   "main.rb"        false
   # "**/*.rb"   "./main.rb"      false
@@ -231,10 +245,81 @@ module IOStreams
     @root_paths.dup
   end
 
+  # Restrict IOStreams to only access paths within the supplied path.
+  #
+  # Once any allowed path has been added, reading, writing, listing, deleting or otherwise accessing
+  # a path that is not within one of the allowed paths raises `IOStreams::Errors::AccessDenied`.
+  # This prevents an untrusted file name, for example one supplied by a user, from accessing anything
+  # else that the process can access.
+  #
+  # Parameters: Same as `IOStreams.path`
+  #
+  # Returns [String] the normalized path that was added, against which paths are compared.
+  #
+  # Example:
+  #    IOStreams.add_allowed_path("/var/data/uploads")
+  #    IOStreams.add_allowed_path("s3://my-bucket/exports")
+  #
+  #    IOStreams.path("/var/data/uploads/file.csv").read
+  #    IOStreams.path("/etc/passwd").read
+  #    # => IOStreams::Errors::AccessDenied
+  #
+  # Notes:
+  # * By default no allowed paths are added, and every path is accessible.
+  # * Add allowed paths in an initializer at startup, where they cannot be changed by untrusted input.
+  # * Paths are normalized before they are compared, so `..` cannot be used to leave an allowed path:
+  #   * Local file names are resolved to their real path, following symbolic links.
+  #     A relative path is resolved against the current working directory when it is added.
+  #   * For S3 the bucket must match, and keys containing `.` or `..` segments are denied.
+  #   * For SFTP and HTTP the host and port must match, and `.` and `..` segments are resolved.
+  # * `#each_child` skips children that are not within the allowed paths, for example a symbolic link
+  #   to a file elsewhere.
+  # * Temp files created by `IOStreams.temp_file` are always accessible.
+  # * Paths from a scheme registered with `IOStreams.register_scheme` are denied, unless its path class
+  #   implements the private method `#allowed_location`.
+  # * A local file could be replaced with a symbolic link after it is checked but before it is opened.
+  #   Allowed paths do not prevent this, so do not allow paths where untrusted users can create files.
+  def self.add_allowed_path(*elements, **args)
+    location = allowed_location(path(*elements, **args))
+    @allowed_paths_mutex.synchronize { @allowed_paths = (@allowed_paths + [location]).uniq.freeze }
+    location
+  end
+
+  # Removes a path previously added with `IOStreams.add_allowed_path`.
+  #
+  # Returns [String] the normalized path that was removed.
+  def self.delete_allowed_path(*elements, **args)
+    location = allowed_location(path(*elements, **args))
+    @allowed_paths_mutex.synchronize { @allowed_paths = (@allowed_paths - [location]).freeze }
+    location
+  end
+
+  # Returns [Array<String>] the normalized allowed paths, see `IOStreams.add_allowed_path`.
+  def self.allowed_paths
+    @allowed_paths
+  end
+
+  # Returns [true|false] whether the supplied path can be accessed, see `IOStreams.add_allowed_path`.
+  #
+  # Always true when no allowed paths have been added.
+  #
+  # Parameters: Same as `IOStreams.path`
+  def self.allowed_path?(*elements, **args)
+    path(*elements, **args).send(:allowed?)
+  end
+
+  def self.allowed_location(path)
+    path.send(:allowed_location)
+  rescue Errors::AccessDenied => e
+    raise(ArgumentError, e.message)
+  end
+
+  private_class_method :allowed_location
+
   # Set the temporary path to use when creating local temp files.
   def self.temp_dir=(temp_dir)
     temp_dir = File.expand_path(temp_dir)
-    FileUtils.mkdir_p(temp_dir) unless ::File.exist?(temp_dir)
+    FileUtils.mkdir_p(temp_dir)
 
     @temp_dir = temp_dir
   end
@@ -249,6 +334,54 @@ module IOStreams
 
   @temp_dir = nil
 
+  # Apply `allowed_columns`, `required_columns` and `skip_unknown` to every input when reading records.
+  #
+  # When false, they only apply to a header row read from the file, and only when `cleanse_header` is true.
+  # They are ignored for JSON and `:hash` input, when `columns:` is supplied, and with `cleanse_header: false`,
+  # and a warning is logged when applying them would change the result.
+  #
+  # When true, they also apply to the supplied `columns:`, to a header row read with `cleanse_header: false`,
+  # and to the keys of each JSON or `:hash` record. JSON keys are cleansed like a header row, unless
+  # `cleanse_header` is false, unknown keys are skipped or raise `IOStreams::Errors::InvalidHeader`,
+  # and a record missing a required column raises `IOStreams::Errors::InvalidHeader`.
+  #
+  # Since the format is usually inferred from the file name, set this to true when the allowed columns
+  # restrict what an uploaded file can set, so that renaming the file to `.json` cannot bypass them.
+  #
+  # Default: false. It will default to true in v3.0.
+  #
+  # Example:
+  #   IOStreams.enforce_column_restrictions = true
+  def self.enforce_column_restrictions=(enforce)
+    raise(ArgumentError, "enforce_column_restrictions must be true or false") unless [true, false].include?(enforce)
+
+    @enforce_column_restrictions = enforce
+  end
+
+  # Returns [true|false] whether column restrictions apply to every input, see `IOStreams.enforce_column_restrictions=`.
+  def self.enforce_column_restrictions?
+    @enforce_column_restrictions
+  end
+
+  @enforce_column_restrictions = false
+
+  # Returns [Logger] the logger used by IOStreams for debug logging.
+  #
+  # When SemanticLogger is loaded a SemanticLogger instance is used by default,
+  # otherwise no logging is performed unless a logger is assigned via #logger=.
+  def self.logger
+    @logger
+  end
+
+  # Replace the logger used by IOStreams.
+  #
+  # Set to nil to disable logging.
+  def self.logger=(logger)
+    @logger = logger
+  end
+
+  @logger = (SemanticLogger[IOStreams] if defined?(SemanticLogger::Logger))
+
   # Register a file extension and the reader and writer streaming classes
   #
   # Example:
@@ -257,7 +390,7 @@ module IOStreams
   def self.register_extension(extension, reader_class, writer_class)
     raise(ArgumentError, "Invalid extension #{extension.inspect}") unless extension.nil? || extension.to_s =~ /\A\w+\Z/
 
-    @extensions[extension.nil? ? nil : extension.to_sym] = Extension.new(reader_class, writer_class)
+    @extensions[extension&.to_sym] = Extension.new(reader_class, writer_class)
   end
 
   # De-Register a file extension
@@ -265,7 +398,7 @@ module IOStreams
   # Returns [Symbol] the extension removed, or nil if the extension was not registered
   #
   # Example:
-  #   register_extension(:xls)
+  #   deregister_extension(:xls)
   def self.deregister_extension(extension)
     raise(ArgumentError, "Invalid extension #{extension.inspect}") unless extension.to_s =~ /\A\w+\Z/
 
@@ -277,15 +410,14 @@ module IOStreams
     @extensions.dup
   end
 
-  # Register a file extension and the reader and writer streaming classes
+  # Register a URI scheme and the path class that handles it
   #
   # Example:
-  #   # MyXls::Reader and MyXls::Writer must implement .open
-  #   register_scheme(:xls, MyXls::Reader, MyXls::Writer)
+  #   register_scheme(:gcs, MyGoogleCloudStoragePath)
   def self.register_scheme(scheme, klass)
     raise(ArgumentError, "Invalid scheme #{scheme.inspect}") unless scheme.nil? || scheme.to_s =~ /\A\w+\Z/
 
-    @schemes[scheme.nil? ? nil : scheme.to_sym] = klass
+    @schemes[scheme&.to_sym] = klass
   end
 
   def self.schemes
@@ -293,7 +425,7 @@ module IOStreams
   end
 
   def self.scheme(scheme_name)
-    @schemes[scheme_name.nil? ? nil : scheme_name.to_sym] || raise(ArgumentError, "Unknown Scheme type: #{scheme_name.inspect}")
+    @schemes[scheme_name&.to_sym] || raise(ArgumentError, "Unknown Scheme type: #{scheme_name.inspect}")
   end
 
   Extension = Struct.new(:reader_class, :writer_class)
@@ -301,6 +433,10 @@ module IOStreams
   # Hold root paths
   @root_paths = {}
 
+  # Hold allowed paths. Replaced rather than modified, so that it can be read without a lock.
+  @allowed_paths       = [].freeze
+  @allowed_paths_mutex = Mutex.new
+
   # A registry to hold formats for processing files during upload or download
   @extensions = {}
   @schemes    = {}

data/lib/io_streams/line/reader.rb

@@ -6,7 +6,7 @@ module IOStreams
       # Prevent denial of service when a delimiter is not found before this number * `buffer_size` characters are read.
       MAX_BLOCKS_MULTIPLIER = 100
 
-      LINEFEED_REGEXP = Regexp.compile(/\r\n|\n|\r/).freeze
+      LINEFEED_REGEXP = /\r\n|\n|\r/
 
       # Read a line at a time from a stream
       def self.stream(input_stream, **args)
@@ -43,8 +43,9 @@ module IOStreams
       #     For CSV files set `embedded_within: '"'`
       #
       # Note:
-      # * When using a line reader and the file_name ends with ".csv" then embedded_within is automatically set to `"`
-      def initialize(input_stream, delimiter: nil, buffer_size: 65_536, embedded_within: nil, original_file_name: nil)
+      # * When reached via `IOStreams::Stream`, `embedded_within` defaults to the quote character of the
+      #   resolved tabular format (e.g. `"` for CSV). See `IOStreams::Builder#quote_character`.
+      def initialize(input_stream, delimiter: nil, buffer_size: 65_536, embedded_within: nil)
         super(input_stream)
 
         @embedded_within = embedded_within
@@ -93,7 +94,9 @@ module IOStreams
         line = _readline
         if line && @embedded_within
           initial_line_number = @line_number
-          while line.count(@embedded_within).odd?
+          # Count the delimiters incrementally, since recounting the whole line each time is quadratic.
+          embedded_count      = line.count(@embedded_within)
+          while embedded_count.odd?
             if eof? || line.length > @buffer_size * 10
               raise(Errors::MalformedDataError.new(
                       "Unbalanced delimited field, delimiter: #{@embedded_within}",
@@ -101,6 +104,7 @@ module IOStreams
                     ))
             end
             line << @delimiter
+            embedded_count += @delimiter.count(@embedded_within)
             next_line = _readline
             if next_line.nil?
               raise(Errors::MalformedDataError.new(
@@ -109,6 +113,7 @@ module IOStreams
                     ))
             end
             line << next_line
+            embedded_count += next_line.count(@embedded_within)
           end
         end
         line

data/lib/io_streams/line/writer.rb

@@ -24,7 +24,7 @@ module IOStreams
       #     Add the specified delimiter after every record when writing it
       #     to the output stream
       #     Default: OS Specific. Linux: "\n"
-      def initialize(output_stream, delimiter: $/, original_file_name: nil)
+      def initialize(output_stream, delimiter: $/)
         super(output_stream)
         @delimiter = delimiter
       end