iostreams 1.11.0 → 2.0.0

data/lib/io_streams/deprecated.rb

@@ -1,216 +0,0 @@
-module IOStreams
-  UTF8_ENCODING   = Encoding.find("UTF-8").freeze
-  BINARY_ENCODING = Encoding.find("BINARY").freeze
-
-  # Deprecated IOStreams from v0.x. Do not use, will be removed soon.
-  module Deprecated
-    def self.included(base)
-      base.extend ClassMethods
-    end
-
-    module ClassMethods
-      # DEPRECATED. Use `#path` or `#io`
-      # Examples:
-      #   IOStreams.path("data.zip").reader { |f| f.read(100) }
-      #
-      #   IOStreams.path(file_name).option(:encode, encoding: "BINARY").reader { |f| f.read(100) }
-      #
-      #   io_stream = StringIO.new("Hello World")
-      #   IOStreams.stream(io_stream).reader { |f| f.read(100) }
-      def reader(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, &block)
-        path = build_path(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
-        path.reader(&block)
-      end
-
-      # DEPRECATED
-      def each_line(file_name_or_io, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
-        path = build_path(file_name_or_io, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
-        path.each(:line, **args, &block)
-      end
-
-      # DEPRECATED
-      def each_row(file_name_or_io, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
-        path = build_path(file_name_or_io, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
-        path.each(:array, **args, &block)
-      end
-
-      # DEPRECATED
-      def each_record(file_name_or_io, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
-        path = build_path(file_name_or_io, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
-        path.each(:hash, **args, &block)
-      end
-
-      # DEPRECATED. Use `#path` or `#io`
-      # Examples:
-      #   IOStreams.path("data.zip").writer { |f| f.write("Hello World") }
-      #
-      #   IOStreams.path(file_name).option(:encode, encoding: "BINARY").writer { |f| f.write("Hello World") }
-      #
-      #   io_stream = StringIO.new("Hello World")
-      #   IOStreams.stream(io_stream).writer { |f| f.write("Hello World") }
-      def writer(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, &block)
-        path = build_path(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
-        path.writer(&block)
-      end
-
-      # DEPRECATED
-      def line_writer(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
-        path = build_path(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
-        path.writer(:line, **args, &block)
-      end
-
-      # DEPRECATED
-      def row_writer(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
-        path = build_path(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
-        path.writer(:array, **args, &block)
-      end
-
-      # DEPRECATED
-      def record_writer(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
-        path = build_path(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
-        path.writer(:hash, **args, &block)
-      end
-
-      # Copies the source file/stream to the target file/stream.
-      # Returns [Integer] the number of bytes copied
-      #
-      # Example: Copy between 2 files
-      #   IOStreams.copy('a.csv', 'b.csv')
-      #
-      # Example: Read content from a Xlsx file and write it out in CSV form.
-      #   IOStreams.copy('a.xlsx', 'b.csv')
-      #
-      # Example:
-      #   # Read content from a JSON file and write it out in CSV form.
-      #   #
-      #   # The output header for the CSV file is extracted from the first row in the JSON file.
-      #   # If the first JSON row does not contain all the column names then they will be ignored
-      #   # for the rest of the file.
-      #   IOStreams.copy('a.json', 'b.csv')
-      #
-      # Example:
-      #   # Read a PSV file and write out a CSV file from it.
-      #   IOStreams.copy('a.psv', 'b.csv')
-      #
-      # Example:
-      #   # Copy between 2 files, encrypting the target file with Symmetric Encryption
-      #   # Since the target file_name already includes `.enc` in the filename, it is automatically
-      #   # encrypted.
-      #   IOStreams.copy('a.csv', 'b.csv.enc')
-      #
-      # Example:
-      #   # Copy between 2 files, encrypting the target file with Symmetric Encryption
-      #   # Since the target file_name does not include `.enc` in the filename, to encrypt it
-      #   # the encryption stream is added.
-      #   IOStreams.copy('a.csv', 'b', target_options: [:enc])
-      #
-      # Example:
-      #   # Copy between 2 files, encrypting the target file with Symmetric Encryption
-      #   # Since the target file_name does not include `.enc` in the filename, to encrypt it
-      #   # the encryption stream is added, along with the optional compress option.
-      #   IOStreams.copy('a.csv', 'b', target_options: [enc: { compress: true }])
-      #
-      # Example:
-      #   # Create a pgp encrypted file.
-      #   # For PGP Encryption the recipients email address is required.
-      #   IOStreams.copy('a.xlsx', 'b.csv.pgp', target_options: [:csv, pgp: { recipient_email: 'user@nospam.org' }])
-      #
-      # Example: Copy between 2 existing streams
-      #   IOStreams.reader('a.csv') do |source_stream|
-      #     IOStreams.writer('b.csv.enc') do |target_stream|
-      #       IOStreams.copy(source_stream, target_stream)
-      #     end
-      #   end
-      #
-      # Example:
-      #   # Copy between 2 csv files, reducing the number of columns present and encrypting the
-      #   # target file with Symmetric Encryption
-      #   output_headers = %w[name address]
-      #   IOStreams.copy(
-      #     'a.csv',
-      #     'b.csv.enc',
-      #     target_options: [csv:{headers: output_headers}, enc: {compress: true}]
-      #   )
-      #
-      # Example:
-      #   # Copy a locally encrypted file to AWS S3.
-      #   # Decrypts the file, then compresses it with gzip as it is being streamed into S3.
-      #   # Useful for when the entire bucket is encrypted on S3.
-      #   IOStreams.copy('a.csv.enc', 's3://my_bucket/b.csv.gz')
-      def copy(source_file_name_or_io, target_file_name_or_io, buffer_size: nil, source_options: {}, target_options: {})
-        # TODO: prevent stream conversions when reader and writer streams are the same!
-        reader(source_file_name_or_io, **source_options) do |source_stream|
-          writer(target_file_name_or_io, **target_options) do |target_stream|
-            IO.copy_stream(source_stream, target_stream)
-          end
-        end
-      end
-
-      # DEPRECATED
-      def reader_stream?(file_name_or_io)
-        file_name_or_io.respond_to?(:read)
-      end
-
-      # DEPRECATED
-      def writer_stream?(file_name_or_io)
-        file_name_or_io.respond_to?(:write)
-      end
-
-      # DEPRECATED. Use Path#compressed?
-      def compressed?(file_name)
-        Path.new(file_name).compressed?
-      end
-
-      # DEPRECATED. Use Path#encrypted?
-      def encrypted?(file_name)
-        Path.new(file_name).encrypted?
-      end
-
-      # DEPRECATED
-      def line_reader(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
-        path = build_path(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
-        path.reader(:line, **args, &block)
-      end
-
-      # DEPRECATED
-      def row_reader(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
-        path = build_path(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
-        path.reader(:line, **args, &block)
-      end
-
-      # DEPRECATED
-      def record_reader(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
-        path = build_path(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
-        path.reader(:hash, **args, &block)
-      end
-
-      private
-
-      def build_path(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil)
-        path = new(file_name_or_io)
-        path.file_name(file_name) if file_name
-
-        apply_old_style_streams(path, streams) if streams
-
-        if encoding || encode_cleaner || encode_replace
-          path.option_or_stream(:encode, encoding: encoding, cleaner: encode_cleaner, replace: encode_replace)
-        end
-
-        path
-      end
-
-      # Applies old form streams to the path
-      def apply_old_style_streams(path, streams)
-        if streams.is_a?(Symbol)
-          path.stream(streams)
-        elsif streams.is_a?(Array)
-          streams.each { |stream| apply_old_style_streams(path, stream) }
-        elsif streams.is_a?(Hash)
-          streams.each_pair { |stream, options| path.stream(stream, **options) }
-        else
-          raise ArgumentError, "Invalid old style stream supplied: #{params.inspect}"
-        end
-      end
-    end
-  end
-end

data/lib/io_streams/tabular/utility/csv_row.rb

@@ -1,105 +0,0 @@
-require "csv"
-module IOStreams
-  class Tabular
-    module Utility
-      # For parsing a single line of CSV at a time
-      # 2 to 3 times better performance than CSV.parse_line and considerably less
-      # garbage collection required.
-      #
-      # Note: Only used prior to Ruby 2.6
-      class CSVRow < ::CSV
-        UTF8_ENCODING = Encoding.find("UTF-8").freeze
-
-        def initialize(encoding = UTF8_ENCODING)
-          @io = StringIO.new("".force_encoding(encoding))
-          super(@io, row_sep: "")
-        end
-
-        # Parse a single line of CSV data
-        # Parameters
-        #   line [String]
-        #     A single line of CSV data without any line terminators
-        def parse(line)
-          return if IOStreams::Utils.blank?(line)
-          return if @skip_lines&.match(line)
-
-          in_extended_col = false
-          csv             = []
-          parts           = line.split(@col_sep, -1)
-          csv << nil if parts.empty?
-
-          # This loop is the hot path of csv parsing. Some things may be non-dry
-          # for a reason. Make sure to benchmark when refactoring.
-          parts.each do |part|
-            if in_extended_col
-              # If we are continuing a previous column
-              if part[-1] == @quote_char && part.count(@quote_char).odd?
-                # extended column ends
-                csv.last << part[0..-2]
-                raise MalformedCSVError, "Missing or stray quote in line #{lineno + 1}" if csv.last =~ @parsers[:stray_quote]
-
-                csv.last.gsub!(@quote_char * 2, @quote_char)
-                in_extended_col = false
-              else
-                csv.last << part
-                csv.last << @col_sep
-              end
-            elsif part[0] == @quote_char
-              # If we are starting a new quoted column
-              if part[-1] != @quote_char || part.count(@quote_char).odd?
-                # start an extended column
-                csv << part[1..-1]
-                csv.last << @col_sep
-                in_extended_col = true
-              else
-                # regular quoted column
-                csv << part[1..-2]
-                raise MalformedCSVError, "Missing or stray quote in line #{lineno + 1}" if csv.last =~ @parsers[:stray_quote]
-
-                csv.last.gsub!(@quote_char * 2, @quote_char)
-              end
-            elsif part =~ @parsers[:quote_or_nl]
-              # Unquoted field with bad characters.
-              if part =~ @parsers[:nl_or_lf]
-                raise MalformedCSVError, "Unquoted fields do not allow \\r or \\n (line #{lineno + 1})."
-              else
-                raise MalformedCSVError, "Illegal quoting in line #{lineno + 1}."
-              end
-            else
-              # Regular ole unquoted field.
-              csv << (part.empty? ? nil : part)
-            end
-          end
-
-          # Replace tacked on @col_sep with @row_sep if we are still in an extended
-          # column.
-          csv[-1][-1] = @row_sep if in_extended_col
-
-          raise MalformedCSVError, "Unclosed quoted field on line #{lineno + 1}." if in_extended_col
-
-          @lineno += 1
-
-          # save fields unconverted fields, if needed...
-          unconverted = csv.dup if @unconverted_fields
-
-          # convert fields, if needed...
-          csv         = convert_fields(csv) unless @use_headers || @converters.empty?
-          # parse out header rows and handle CSV::Row conversions...
-          csv         = parse_headers(csv) if @use_headers
-
-          # inject unconverted fields and accessor, if requested...
-          add_unconverted_fields(csv, unconverted) if @unconverted_fields && (!csv.respond_to? :unconverted_fields)
-
-          csv
-        end
-
-        # Return the supplied array as a single line CSV string.
-        def render(row)
-          row.map(&@quote).join(@col_sep) + @row_sep # quote and separate
-        end
-
-        alias to_csv render
-      end
-    end
-  end
-end

data/test/builder_test.rb

@@ -1,311 +0,0 @@
-require_relative "test_helper"
-
-class BuilderTest < Minitest::Test
-  describe IOStreams::Builder do
-    let(:file_name) { "my/path/abc.bcd.xlsx.zip.gz.pgp" }
-    let(:streams) { IOStreams::Builder.new(file_name) }
-
-    describe "#option" do
-      it "adds one option" do
-        streams.option(:pgp, passphrase: "unlock-me")
-        assert_equal({pgp: {passphrase: "unlock-me"}}, streams.options)
-      end
-
-      it "adds options in order" do
-        streams.option(:pgp, passphrase: "unlock-me")
-        streams.option(:enc, compress: false)
-        assert_equal({pgp: {passphrase: "unlock-me"}, enc: {compress: false}}, streams.options)
-      end
-
-      it "will not add an option if a stream was already set" do
-        streams.stream(:pgp, passphrase: "unlock-me")
-        assert_raises ArgumentError do
-          streams.option(:pgp, passphrase: "unlock-me")
-        end
-      end
-
-      it "will not add an invalid option" do
-        assert_raises ArgumentError do
-          streams.option(:blah, value: 23)
-        end
-      end
-
-      describe "with no file_name" do
-        let(:file_name) { nil }
-
-        it "prevents options being set" do
-          assert_raises ArgumentError do
-            streams.option(:pgp, passphrase: "unlock-me")
-          end
-        end
-      end
-    end
-
-    describe "#format" do
-      it "detects the format from the file name" do
-        streams = IOStreams::Builder.new("abc.json")
-        assert_equal :json, streams.format
-      end
-
-      it "is nil if the file name has no meaningful format" do
-        assert_nil streams.format
-      end
-
-      it "returns set format with no file_name" do
-        streams        = IOStreams::Builder.new
-        streams.format = :csv
-        assert_equal :csv, streams.format
-      end
-
-      it "returns set format with file_name" do
-        streams        = IOStreams::Builder.new("abc.json")
-        streams.format = :csv
-        assert_equal :csv, streams.format
-      end
-
-      it "validates bad format" do
-        assert_raises ArgumentError do
-          streams.format = :blah
-        end
-      end
-    end
-
-    describe "#stream" do
-      it "adds one stream" do
-        streams.stream(:pgp, passphrase: "unlock-me")
-        assert_equal({pgp: {passphrase: "unlock-me"}}, streams.streams)
-      end
-
-      it "adds streams in order" do
-        streams.stream(:pgp, passphrase: "unlock-me")
-        streams.stream(:enc, compress: false)
-        assert_equal({pgp: {passphrase: "unlock-me"}, enc: {compress: false}}, streams.streams)
-      end
-
-      it "will not add a stream if an option was already set" do
-        streams.option(:pgp, passphrase: "unlock-me")
-        assert_raises ArgumentError do
-          streams.stream(:pgp, passphrase: "unlock-me")
-        end
-      end
-
-      it "will not add an invalid stream" do
-        assert_raises ArgumentError do
-          streams.stream(:blah, value: 23)
-        end
-      end
-    end
-
-    describe "#reader" do
-      let :gzip_string do
-        io_string = StringIO.new("".b)
-        IOStreams::Gzip::Writer.stream(io_string) do |io|
-          io.write("Hello World")
-        end
-        io_string.string
-      end
-
-      it "directly calls block for an empty stream" do
-        string_io = StringIO.new
-        value     = nil
-        streams.stream(:none)
-        streams.reader(string_io) do |io|
-          assert_equal io, string_io
-          value = 32
-        end
-        assert_equal 32, value
-      end
-
-      it "returns the reader" do
-        string_io = StringIO.new(gzip_string)
-        streams.stream(:gz)
-        streams.reader(string_io) do |io|
-          assert io.is_a?(::Zlib::GzipReader), io
-        end
-      end
-
-      it "returns the last reader" do
-        string_io = StringIO.new(gzip_string)
-        streams.stream(:encode)
-        streams.stream(:gz)
-        streams.reader(string_io) do |io|
-          assert io.is_a?(IOStreams::Encode::Reader), io
-        end
-      end
-    end
-
-    describe "#writer" do
-      it "directly calls block for an empty stream" do
-        string_io = StringIO.new
-        value     = nil
-        streams.stream(:none)
-        streams.writer(string_io) do |io|
-          assert_equal io, string_io
-          value = 32
-        end
-        assert_equal 32, value
-      end
-
-      it "returns the reader" do
-        string_io = StringIO.new
-        streams.stream(:zip)
-        streams.writer(string_io) do |io|
-          assert io.is_a?(ZipTricks::Streamer::Writable), io
-        end
-      end
-
-      it "returns the last reader" do
-        string_io = StringIO.new
-        streams.stream(:encode)
-        streams.stream(:zip)
-        streams.writer(string_io) do |io|
-          assert io.is_a?(IOStreams::Encode::Writer), io
-        end
-      end
-    end
-
-    # Internal methods
-
-    describe "#class_for_stream" do
-      it "xlsx" do
-        assert_equal IOStreams::Xlsx::Reader, streams.send(:class_for_stream, :reader, :xlsx)
-      end
-
-      it "gzip" do
-        assert_equal IOStreams::Gzip::Writer, streams.send(:class_for_stream, :writer, :gzip)
-      end
-
-      it "unknown" do
-        assert_raises ArgumentError do
-          streams.send(:class_for_stream, :reader, :unknown)
-        end
-      end
-    end
-
-    describe "#parse_extensions" do
-      it "single stream" do
-        streams = IOStreams::Builder.new("my/path/abc.xlsx")
-        assert_equal %i[xlsx], streams.send(:parse_extensions)
-      end
-
-      it "empty" do
-        streams = IOStreams::Builder.new("my/path/abc.csv")
-        assert_equal [], streams.send(:parse_extensions)
-      end
-
-      it "handles multiple extensions" do
-        assert_equal %i[xlsx zip gz pgp], streams.send(:parse_extensions)
-      end
-
-      describe "case-insensitive" do
-        let(:file_name) { "a.XlsX.GzIp" }
-
-        it "is case-insensitive" do
-          assert_equal %i[xlsx gzip], streams.send(:parse_extensions)
-        end
-      end
-    end
-
-    describe "#pipeline" do
-      it "with stream and file name" do
-        expected = {enc: {compress: false}}
-        streams.stream(:enc, compress: false)
-        assert_equal expected, streams.pipeline
-      end
-
-      it "no file name, streams, or options" do
-        expected = {}
-        streams  = IOStreams::Builder.new
-        assert_equal expected, streams.pipeline
-      end
-
-      it "file name without options" do
-        expected = {xlsx: {}, zip: {}, gz: {}, pgp: {}}
-        assert_equal expected, streams.pipeline
-      end
-
-      it "file name with encode option" do
-        expected = {encode: {encoding: "BINARY"}, xlsx: {}, zip: {}, gz: {}, pgp: {}}
-        streams.option(:encode, encoding: "BINARY")
-        assert_equal expected, streams.pipeline
-      end
-
-      it "file name with option" do
-        expected = {xlsx: {}, zip: {}, gz: {}, pgp: {passphrase: "unlock-me"}}
-        streams.option(:pgp, passphrase: "unlock-me")
-        assert_equal expected, streams.pipeline
-      end
-    end
-
-    describe "#remove_from_pipeline" do
-      let(:file_name) { "my/path/abc.bz2.pgp" }
-      it "removes a named stream from the pipeline" do
-        assert_equal({bz2: {}, pgp: {}}, streams.pipeline)
-        streams.remove_from_pipeline(:bz2)
-        assert_equal({pgp: {}}, streams.pipeline)
-      end
-      it "removes a named stream from the pipeline with options" do
-        streams.option(:pgp, passphrase: "unlock-me")
-        assert_equal({bz2: {}, pgp: {passphrase: "unlock-me"}}, streams.pipeline)
-        streams.remove_from_pipeline(:bz2)
-        assert_equal({pgp: {passphrase: "unlock-me"}}, streams.pipeline)
-      end
-    end
-
-    describe "#execute" do
-      it "directly calls block for an empty stream" do
-        string_io = StringIO.new
-        value     = nil
-        streams.send(:execute, :writer, {}, string_io) do |io|
-          assert_equal io, string_io
-          value = 32
-        end
-        assert_equal 32, value
-      end
-
-      it "calls last block in one element stream" do
-        pipeline  = {simple: {arg: "first"}}
-        string_io = StringIO.new
-        streams.send(:execute, :writer, pipeline, string_io) { |io| io.write("last") }
-        assert_equal "first>last", string_io.string
-      end
-
-      it "chains blocks in 2 element stream" do
-        pipeline  = {simple: {arg: "first"}, simple2: {arg: "second"}}
-        string_io = StringIO.new
-        streams.send(:execute, :writer, pipeline, string_io) { |io| io.write("last") }
-        assert_equal "second>first>last", string_io.string
-      end
-
-      it "chains blocks in 3 element stream" do
-        pipeline  = {simple: {arg: "first"}, simple2: {arg: "second"}, simple3: {arg: "third"}}
-        string_io = StringIO.new
-        streams.send(:execute, :writer, pipeline, string_io) { |io| io.write("last") }
-        assert_equal "third>second>first>last", string_io.string
-      end
-    end
-
-    class SimpleStream
-      def self.stream(io, **args)
-        yield new(io, **args)
-      end
-
-      def self.open(file_name_or_io, **args, &block)
-        file_name_or_io.is_a?(String) ? file(file_name_or_io, **args, &block) : stream(file_name_or_io, **args, &block)
-      end
-
-      def initialize(io, arg:)
-        @io  = io
-        @arg = arg
-      end
-
-      def write(data)
-        @io.write("#{@arg}>#{data}")
-      end
-    end
-
-    IOStreams.register_extension(:simple, nil, SimpleStream)
-    IOStreams.register_extension(:simple2, nil, SimpleStream)
-    IOStreams.register_extension(:simple3, nil, SimpleStream)
-  end
-end

data/test/bzip2_reader_test.rb

@@ -1,27 +0,0 @@
-require_relative "test_helper"
-
-class Bzip2ReaderTest < Minitest::Test
-  describe IOStreams::Bzip2::Reader do
-    let :file_name do
-      File.join(File.dirname(__FILE__), "files", "text.txt.bz2")
-    end
-
-    let :decompressed do
-      File.read(File.join(File.dirname(__FILE__), "files", "text.txt"))
-    end
-
-    describe ".file" do
-      it "file" do
-        result = IOStreams::Bzip2::Reader.file(file_name, &:read)
-        assert_equal decompressed, result
-      end
-
-      it "stream" do
-        result = File.open(file_name) do |file|
-          IOStreams::Bzip2::Reader.stream(file, &:read)
-        end
-        assert_equal decompressed, result
-      end
-    end
-  end
-end