iostreams 1.7.0 → 1.10.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 849ceda63eb30f95762a7c985cd215d424e62afd68ab20776e8d16c188dd6aed
-  data.tar.gz: 8e26af86c40bb673ce36855a7fb30d1c4b401edc3eac0b27a71b9760cfe865dd
+  metadata.gz: c93dac4c226c66c4e36554311858ff328299fc4202c257cdb0e7f2c8e82e323b
+  data.tar.gz: fa96f9d6007769b812ab5506e1ca5ca20866ed9d6fc6bc98d75b50a0d50a23b2
 SHA512:
-  metadata.gz: 99318c4c64e0133df57b84429b1c2f9caa064abb1405ace5d55208e41b6bf8bb8fa83a75db8ae46d53753f10d566bab53971d95871ed4011bab4571d31bebe8a
-  data.tar.gz: bfba3a033c753e3fe05f798177b8f3c7ee8f566eabaf9223984fa902b288cb3515154f621a4e97f75b6c5bc31da88f284390c2d82c5015d7682ecf08c2a671d3
+  metadata.gz: 2538af2be40ad81287b4c3501f0ff9672ea82289adcb2de1e976d868fd9aa655f01d496b6299ba851193e1ed832750e6bcb21f6ed88179aab303f348e4bef60f
+  data.tar.gz: e7d6c8f09d0377f0bdfb83fdfc61559afd1581d1e8425cc9a3a76754be2e92003f202246c0a43b83b12107774965388aa71f2c2950835797363ce057ca5fe24b

data/lib/io_streams/builder.rb

@@ -79,15 +79,16 @@ module IOStreams
     # with their options that will be applied when the reader or writer is invoked.
     def pipeline
       return streams.dup.freeze if streams
-      return {}.freeze unless file_name
 
-      built_streams          = {}
-      # Encode stream is always first
-      built_streams[:encode] = options[:encode] if options&.key?(:encode)
+      build_pipeline.freeze
+    end
 
-      opts = options || {}
-      parse_extensions.each { |stream| built_streams[stream] = opts[stream] || {} }
-      built_streams.freeze
+    # Removes the named stream from the current pipeline.
+    # If the stream pipeline has not yet been built it will be built from the file_name if present.
+    # Note: Any options must be set _before_ calling this method.
+    def remove_from_pipeline(stream_name)
+      @streams ||= build_pipeline
+      @streams.delete(stream_name.to_sym)
     end
 
     # Returns the tabular format if set, otherwise tries to autodetect the format if the file_name has been set
@@ -106,6 +107,18 @@ module IOStreams
 
     private
 
+    def build_pipeline
+      return {} unless file_name
+
+      built_streams          = {}
+      # Encode stream is always first
+      built_streams[:encode] = options[:encode] if options&.key?(:encode)
+
+      opts = options || {}
+      parse_extensions.each { |stream| built_streams[stream] = opts[stream] || {} }
+      built_streams
+    end
+
     def class_for_stream(type, stream)
       ext = IOStreams.extensions[stream.nil? ? nil : stream.to_sym] ||
             raise(ArgumentError, "Unknown Stream type: #{stream.inspect}")

data/lib/io_streams/line/reader.rb

@@ -75,6 +75,8 @@ module IOStreams
       # Note:
       # * The line delimiter is _not_ returned.
       def each
+        return to_enum(__method__) unless block_given?
+
         line_count = 0
         until eof?
           line = readline
@@ -146,8 +148,8 @@ module IOStreams
         data
       end
 
-      # Returns [Integer] the number of characters read into the internal buffer
-      # Returns 0 on EOF
+      # Returns whether more data is available to read
+      # Returns false on EOF
       def read_block
         return false if @eof
 
@@ -157,7 +159,8 @@ module IOStreams
               @input_stream.read(@buffer_size, @read_cache_buffer)
             rescue ArgumentError
               # Handle arity of -1 when just 0..1
-              @read_cache_buffer = nil
+              @read_cache_buffer     = nil
+              @use_read_cache_buffer = false
               @input_stream.read(@buffer_size)
             end
           else

data/lib/io_streams/paths/file.rb

@@ -89,6 +89,11 @@ module IOStreams
       # "**.rb"     "lib/song.rb"    true
       # "*"         "dave/.profile"  true
       def each_child(pattern = "*", case_sensitive: false, directories: false, hidden: false)
+        unless block_given?
+          return to_enum(__method__, pattern,
+                         case_sensitive: case_sensitive, directories: directories, hidden: hidden)
+        end
+
         flags = 0
         flags |= ::File::FNM_CASEFOLD unless case_sensitive
         flags |= ::File::FNM_DOTMATCH if hidden

data/lib/io_streams/paths/s3.rb

@@ -284,6 +284,11 @@ module IOStreams
       # Notes:
       # - Currently all S3 lookups are recursive as of the pattern regardless of whether the pattern includes `**`.
       def each_child(pattern = "*", case_sensitive: false, directories: false, hidden: false)
+        unless block_given?
+          return to_enum(__method__, pattern,
+                         case_sensitive: case_sensitive, directories: directories, hidden: hidden)
+        end
+
         matcher = Matcher.new(self, pattern, case_sensitive: case_sensitive, hidden: hidden)
 
         # When the pattern includes an exact file name without any pattern characters

data/lib/io_streams/paths/sftp.rb

@@ -142,6 +142,11 @@ module IOStreams
       # sftp://sftp.example.org/a/b/c/test.txt {:type=>1, :size=>37, :owner=>"test_owner", :group=>"test_group",
       #   :permissions=>420, :atime=>1572378136, :mtime=>1572378136, :link_count=>1, :extended=>{}}
       def each_child(pattern = "*", case_sensitive: true, directories: false, hidden: false)
+        unless block_given?
+          return to_enum(__method__, pattern,
+                         case_sensitive: case_sensitive, directories: directories, hidden: hidden)
+        end
+
         Utils.load_soft_dependency("net-sftp", "SFTP glob capability", "net/sftp") unless defined?(Net::SFTP)
 
         flags = ::File::FNM_EXTGLOB

data/lib/io_streams/record/reader.rb

@@ -68,6 +68,8 @@ module IOStreams
       end
 
       def each
+        return to_enum(__method__) unless block_given?
+
         @line_reader.each do |line|
           if @tabular.header?
             @tabular.parse_header(line)

data/lib/io_streams/row/reader.rb

@@ -40,6 +40,8 @@ module IOStreams
       end
 
       def each
+        return to_enum(__method__) unless block_given?
+
         @line_reader.each do |line|
           if @tabular.header?
             columns = @tabular.parse_header(line)

data/lib/io_streams/stream.rb

@@ -56,6 +56,14 @@ module IOStreams
       builder.pipeline
     end
 
+    # Removes the named stream from the current pipeline.
+    # If the stream pipeline has not yet been built it will be built from the file_name if present.
+    # Note: Any options must be set _before_ calling this method.
+    def remove_from_pipeline(stream_name)
+      builder.remove_from_pipeline(stream_name)
+      self
+    end
+
     # Iterate over a file / stream returning one line at a time.
     #
     # Example: Read a line at a time

data/lib/io_streams/tabular/header.rb

@@ -2,6 +2,9 @@ module IOStreams
   class Tabular
     # Process files / streams that start with a header.
     class Header
+      # Column names that begin with this prefix have been rejected and should be ignored.
+      IGNORE_PREFIX = "__rejected__".freeze
+
       attr_accessor :columns, :allowed_columns, :required_columns, :skip_unknown
 
       # Header
@@ -17,8 +20,8 @@ module IOStreams
       #     List of columns to allow.
       #     Default: nil ( Allow all columns )
       #     Note:
-      #       When supplied any columns that are rejected will be returned in the cleansed columns
-      #       as nil so that they can be ignored during processing.
+      #     * So that rejected columns can be identified in subsequent steps, they will be prefixed with `__rejected__`.
+      #       For example, `Unknown Column` would be cleansed as `__rejected__Unknown Column`.
       #
       #   required_columns [Array<String>]
       #     List of columns that must be present, otherwise an Exception is raised.
@@ -44,8 +47,10 @@ module IOStreams
       # - Spaces and '-' are converted to '_'.
       # - All characters except for letters, digits, and '_' are stripped.
       #
-      # Notes
-      # * Raises Tabular::InvalidHeader when there are no non-nil columns left after cleansing.
+      # Notes:
+      # * So that rejected columns can be identified in subsequent steps, they will be prefixed with `__rejected__`.
+      #   For example, `Unknown Column` would be cleansed as `__rejected__Unknown Column`.
+      # * Raises Tabular::InvalidHeader when there are no rejected columns left after cleansing.
       def cleanse!
         return [] if columns.nil? || columns.empty?
 
@@ -56,7 +61,7 @@ module IOStreams
             cleansed
           else
             ignored_columns << column
-            nil
+            "#{IGNORE_PREFIX}#{column}"
           end
         end
 
@@ -122,7 +127,7 @@ module IOStreams
 
       def array_to_hash(row)
         h = {}
-        columns.each_with_index { |col, i| h[col] = row[i] unless IOStreams::Utils.blank?(col) }
+        columns.each_with_index { |col, i| h[col] = row[i] unless IOStreams::Utils.blank?(col) || col.start_with?(IGNORE_PREFIX) }
         h
       end
 
@@ -134,12 +139,7 @@ module IOStreams
           hash = hash.dup
           unmatched.each { |name| hash[cleanse_column(name)] = hash.delete(name) }
         end
-        # Hash#slice as of Ruby 2.5
-        if hash.respond_to?(:slice)
-          hash.slice(*columns)
-        else
-          columns.each_with_object({}) { |column, new_hash| new_hash[column] = hash[column] }
-        end
+        hash.slice(*columns)
       end
 
       def cleanse_column(name)

data/lib/io_streams/version.rb

@@ -1,3 +1,3 @@
 module IOStreams
-  VERSION = "1.7.0".freeze
+  VERSION = "1.10.1".freeze
 end

data/test/builder_test.rb

@@ -237,6 +237,21 @@ class BuilderTest < Minitest::Test
       end
     end
 
+    describe "#remove_from_pipeline" do
+      let(:file_name) { "my/path/abc.bz2.pgp" }
+      it "removes a named stream from the pipeline" do
+        assert_equal({bz2: {}, pgp: {}}, streams.pipeline)
+        streams.remove_from_pipeline(:bz2)
+        assert_equal({pgp: {}}, streams.pipeline)
+      end
+      it "removes a named stream from the pipeline with options" do
+        streams.option(:pgp, passphrase: "unlock-me")
+        assert_equal({bz2: {}, pgp: {passphrase: "unlock-me"}}, streams.pipeline)
+        streams.remove_from_pipeline(:bz2)
+        assert_equal({pgp: {passphrase: "unlock-me"}}, streams.pipeline)
+      end
+    end
+
     describe "#execute" do
       it "directly calls block for an empty stream" do
         string_io = StringIO.new

data/test/line_reader_test.rb

@@ -98,6 +98,13 @@ class LineReaderTest < Minitest::Test
         assert_equal data.size, count
       end
 
+      it "with no block returns enumerator" do
+        lines = IOStreams::Line::Reader.file(file_name) do |io|
+          io.each.first(100)
+        end
+        assert_equal data, lines
+      end
+
       it "each_line stream" do
         lines = []
         count = File.open(file_name) do |file|

data/test/paths/file_test.rb

@@ -5,7 +5,7 @@ module Paths
     describe IOStreams::Paths::File do
       let(:root) { IOStreams::Paths::File.new("/tmp/iostreams").delete_all }
       let(:directory) { root.join("/some_test_dir") }
-      let(:data) { "Hello World" }
+      let(:data) { "Hello World\nHow are you doing?\nOn this fine day" }
       let(:file_path) do
         path = root.join("some_test_dir/test_file.txt")
         path.writer { |io| io << data }
@@ -17,6 +17,20 @@ module Paths
         path
       end
 
+      describe "#each" do
+        it "reads lines" do
+          records = []
+          count   = file_path.each { |line| records << line }
+          assert_equal count, data.lines.size
+          assert_equal data.lines.collect(&:strip), records
+        end
+
+        it "reads lines without block" do
+          records = file_path.each.first(100)
+          assert_equal data.lines.collect(&:strip), records
+        end
+      end
+
       describe "#each_child" do
         it "iterates an empty path" do
           none = nil
@@ -48,6 +62,12 @@ module Paths
           actual   = root.children("**/Test*.TXT", case_sensitive: true).collect(&:to_s)
           refute_equal expected, actual.sort
         end
+
+        it "with no block returns enumerator" do
+          expected = [file_path.to_s, file_path2.to_s]
+          actual   = root.each_child("**/*").first(100).collect(&:to_s)
+          assert_equal expected.sort, actual.sort
+        end
       end
 
       describe "#mkpath" do

data/test/record_reader_test.rb

@@ -46,6 +46,13 @@ class RecordReaderTest < Minitest::Test
         end
         assert_equal expected, rows
       end
+
+      it "with no block returns enumerator" do
+        records = IOStreams::Record::Reader.file(file_name, cleanse_header: false) do |io|
+          io.each.first(100)
+        end
+        assert_equal expected, records
+      end
     end
 
     describe "#collect" do

data/test/row_reader_test.rb

@@ -10,7 +10,7 @@ class RowReaderTest < Minitest::Test
       CSV.read(file_name)
     end
 
-    describe ".open" do
+    describe "#each" do
       it "file" do
         rows  = []
         count = IOStreams::Row::Reader.file(file_name) do |io|
@@ -20,6 +20,13 @@ class RowReaderTest < Minitest::Test
         assert_equal expected.size, count
       end
 
+      it "with no block returns enumerator" do
+        rows = IOStreams::Row::Reader.file(file_name) do |io|
+          io.each.first(100)
+        end
+        assert_equal expected, rows
+      end
+
       it "stream" do
         rows  = []
         count = IOStreams::Line::Reader.file(file_name) do |file|

data/test/tabular_test.rb

@@ -58,12 +58,12 @@ class TabularTest < Minitest::Test
           assert_equal header, tabular.header.columns
         end
 
-        it "white listed snake cased alphanumeric columns" do
+        it "allowed list snake cased alphanumeric columns" do
           tabular = IOStreams::Tabular.new(
-            columns:         ["Ard Vark", "password", "robot version", "$$$"],
+            columns:         ["Ard Vark", "Password", "robot version", "$$$"],
             allowed_columns: %w[ard_vark robot_version]
           )
-          expected_header = ["ard_vark", nil, "robot_version", nil]
+          expected_header = ["ard_vark", "__rejected__Password", "robot_version", "__rejected__$$$"]
           cleansed_header = tabular.cleanse_header!
           assert_equal(expected_header, cleansed_header)
         end
@@ -82,13 +82,13 @@ class TabularTest < Minitest::Test
           assert_equal @allowed_columns, tabular.header.allowed_columns
         end
 
-        it "nils columns not in the whitelist" do
+        it "nils columns not in the allowed list" do
           tabular = IOStreams::Tabular.new(columns: ["   first ", "Unknown Column", "thirD   "], allowed_columns: @allowed_columns)
           header  = tabular.cleanse_header!
-          assert_equal ["first", nil, "third"], header
+          assert_equal ["first", "__rejected__Unknown Column", "third"], header
         end
 
-        it "raises exception for columns not in the whitelist" do
+        it "raises exception for columns not in the allowed list" do
           tabular = IOStreams::Tabular.new(columns: ["   first ", "Unknown Column", "thirD   "], allowed_columns: @allowed_columns, skip_unknown: false)
           exc     = assert_raises IOStreams::Errors::InvalidHeader do
             tabular.cleanse_header!
@@ -218,7 +218,7 @@ class TabularTest < Minitest::Test
         end
       end
 
-      it "skips columns not in the whitelist" do
+      it "skips columns not in the allowed list" do
         tabular.header.allowed_columns = %w[first second third fourth fifth]
         tabular.cleanse_header!
         assert hash = tabular.record_parse("1,2,3")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: iostreams
 version: !ruby/object:Gem::Version
-  version: 1.7.0
+  version: 1.10.1
 platform: ruby
 authors:
 - Reid Morrison
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-23 00:00:00.000000000 Z
+date: 2021-08-30 00:00:00.000000000 Z
 dependencies: []
 description:
 email:
@@ -132,7 +132,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.15
+rubygems_version: 3.2.22
 signing_key:
 specification_version: 4
 summary: Input and Output streaming for Ruby.