iop 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2631f9835b20e87513aed68b469ca251815ccd9517c1b6879555ab851bdae02a
-  data.tar.gz: b1a7d1794c962323692117181c1d5485343ed38d3fec65738af5c21076801fd4
+  metadata.gz: e00c4ba2a50b1aa72e48cfb31653ee740059b737d9ca6139fc4ba3bf4f1ebbbd
+  data.tar.gz: 643526a878056882bdaf5c840e36cfa2068912850692a53878a70c3e612727a3
 SHA512:
-  metadata.gz: d1cbb150db80d8de4aae4caa144618c143c84572949488508ee149b7fdcb8040385c966022a242fea08caf7533a96035fa0ee7e466cc8a583faec70b3e3c48b4
-  data.tar.gz: 42f468cd335d5087d4471b0f17fd190d0b7346611fe6f00da8a3a565c85042ce8a41f0a351cbe8bd40ef316311eaeae5456ec8fa79af71db3461621313435f6c
+  metadata.gz: 336b3d9df782302ea3d4391683ccddcf662da77b59a324971158b7f1404eaf08547f552394e2a055f842c02176ace16b37f62a37950cbd6fcb4eca8f0fa11b25
+  data.tar.gz: 752c73de778d5ef811a2874ff80b755b8172c78f07f5be7ad5ab4df1a97a6ab94643625b08ea7fcec375144083f7823d540b786efdfb5ed1c7f27e8b95f8bcec

data/.yardopts

@@ -0,0 +1,7 @@
+--no-private
+--protected
+--readme README.md
+-m markdown
+-
+lib/**/*.rb
+*.md

data/CHANGES.md

@@ -1,3 +1,10 @@
+# 0.2.0
+
+SFTP file reading/writing.
+
+Documentation updates.
+
+
 # 0.1.0
 
 Initial release.

data/README.md

@@ -13,9 +13,9 @@ Consider the example:
 (FileReader.new('input.dat') | GzipCompressor.new | DigestComputer.new(MD5.new) | FileWriter.new('output.dat.gz')).process!
 ```
 
-The above snippet reads input file and compresses it into the GZip-compatible output file simultaneously computing the MD5 hash of compressed data being written.
+The above snippet reads input file and compresses it into the Gzip-compatible output file simultaneously computing the MD5 hash of compressed data being written.
 
-The next snippet presents the incremental pipeline construction capability - a feature not easily implementable with the standard Ruby I/O blocks nesting.
+The next snippet presents the incremental pipeline construction capability - a feature not easily implementable with the standard Ruby I/O block nesting.
 
 ```ruby
 # Incremental pipeline construction example
@@ -25,7 +25,7 @@ pipe |= FileWriter.new('output')
 pipe.process!
 ```
 
-Here the GZip compression is made optional and is thrown in depending on external condition.
+Here the Gzip compression is made optional and is thrown in depending on external condition.
 
 ## Features
 
@@ -35,7 +35,7 @@ The following capabilities are currently implemented:
 - IO or local file reading/writing
 - FTP file reading/writing
 - Digest computing
-- GZip/Zlib (de)compression
+- Gzip/Zlib (de)compression
 - Zstd (de)compression
 - Symmetric cipher (de,en)cryption
 - Random data generation
@@ -61,7 +61,7 @@ include IOP
 
 ```ruby
 pipe  = StringSplitter.new('Greetings from IOP', 10)
-pipe |= GzipCompressor.new | (digest = DigestComputer.new(MD5.new))
+pipe |= GzipCompressor.new | (d = DigestComputer.new(MD5.new))
 pipe |= FileWriter.new('output.gz')
 ```
 
@@ -78,7 +78,7 @@ The IOP instances do normally perform self-cleanup operations, such as closing f
 - The variable-bound instances can be then examined.
 
 ```ruby
-puts digest.hexdigest
+puts d.digest.hexdigest
 ```
 
 For further information refer to IOP documentation.

data/lib/iop.rb

@@ -74,7 +74,7 @@
 module IOP
 
 
-  VERSION = '0.1.0'
+  VERSION = '0.2.0'
 
 
   # Default read block size in bytes for adapters which don't have this parameter externally imposed.
@@ -191,9 +191,71 @@ module IOP
   end
 
 
+  #
+  # Abstract reader class for seekable streams which can read with blocks of specified size.
+  #
+  # Sequentially reads specified number of bytes starting at specified byte offset.
+  #
+  # @since 0.2
+  #
+  class RandomAccessReader
+
+    include Feed
+
+    # Sets up the reader parameters.
+    #
+    # @param size [Integer] total number of bytes to read; +nil+ value instructs to read until end-of-data is reached
+    #
+    # @param offset [Integer] offset in bytes from the stream start to seek to; +nil+ value means no seeking is performed
+    #
+    # @param block_size [Integer] size of blocks to read data with
+    def initialize(size: nil, offset: nil, block_size: DEFAULT_BLOCK_SIZE)
+      @block_size = size.nil? ? block_size : IOP.min(size, block_size)
+      @left = @size = size
+      @offset = offset
+    end
+
+    def process!
+      seek! unless @offset.nil?
+      buffer = IOP.allocate_string(@block_size)
+      loop do
+        read_size = @size.nil? ? @block_size : IOP.min(@left, @block_size)
+        break if read_size.zero?
+        if (data = read!(read_size, buffer)).nil?
+          if @size.nil?
+            break
+          else
+            raise EOFError, INSUFFICIENT_DATA
+          end
+        else
+          unless @left.nil?
+            @left -= data.size
+            raise IOError, EXTRA_DATA if @left < 0
+          end
+        end
+        process(data) unless data.size.zero?
+      end
+      process
+    end
+
+    protected
+
+    # @abstract
+    def seek!() end
+    remove_method :seek!
+
+    # @abstract
+    def read!(read_size, buffer) nil end
+    remove_method :read!
+
+  end
+
+
   #
   # @private
   #
+  # Feed implementation for sequential streams which can not seek or request for the block size to read.
+  #
   # @note a class including this module must implement the {#next_data} method.
   #
   # @since 0.1
@@ -234,14 +296,13 @@ module IOP
       process
     end
 
-    # @abstract
-    #
+    protected
+
     # Returns the data portion of non-zero size or +nil+ on EOF.
     #
+    # @abstract
     # @return [String] data chunk recently read or +nil+
-    def next_data
-      raise
-    end
+    def next_data; nil end
     remove_method :next_data
 
   end

data/lib/iop/digest.rb

@@ -7,7 +7,7 @@ module IOP
 
   #
   # Filter class to compute digest of the data being passed through.
-  # It can be used with digest computing classes from +digest+ and +openssl+ standard Ruby modules.
+  # It can be used with digest computing classes from the standard Ruby +Digest::+ and +OpenSSL::Digest::+ modules.
   #
   # ### Use case: generate 1024 bytes of random data and compute and print MD5 hash sum of it.
   #

data/lib/iop/file.rb

@@ -26,48 +26,23 @@ module IOP
   #
   # @since 0.1
   #
-  class IOReader
-
-    include Feed
+  class IOReader < RandomAccessReader
 
     # Creates class instance.
     #
     # @param io [IO] +IO+ instance to read data from
     #
-    # @param size [Integer] total number of bytes to read; +nil+ value instructs to read until end-of-data is reached
-    #
-    # @param offset [Integer] offset in bytes from the stream start to seek to; +nil+ value means no seeking is performed
-    #
-    # @param block_size [Integer] size of blocks to read data with
-    def initialize(io, size: nil, offset: nil, block_size: DEFAULT_BLOCK_SIZE)
-      @block_size = size.nil? ? block_size : IOP.min(size, block_size)
-      @left = @size = size
-      @offset = offset
+    # @param options [Hash] keyword arguments passed to {RandomAccessReader} constructor
+    def initialize(io, **options)
+      super(**options)
       @io = io
     end
 
-    def process!
-      @io.seek(@offset) unless @offset.nil?
-      data = IOP.allocate_string(@block_size)
-      loop do
-        read_size = @size.nil? ? @block_size : IOP.min(@left, @block_size)
-        break if read_size.zero?
-        if @io.read(read_size, data).nil?
-          if @size.nil?
-            break
-          else
-            raise EOFError, INSUFFICIENT_DATA
-          end
-        else
-          unless @left.nil?
-            @left -= data.size
-            raise IOError, EXTRA_DATA if @left < 0
-          end
-        end
-        process(data) unless data.size.zero?
-      end
-      process
-    end
+    private
+
+    def seek!() @io.seek(@offset) end
+
+    def read!(read_size, data) @io.read(read_size, data) end
 
   end
 
@@ -94,7 +69,7 @@ module IOP
     #
     # @param mode [String] open mode for the file; refer to {File} for details
     #
-    # @param options [Hash] extra keyword parameters passed to {IOReader} constructor
+    # @param options [Hash] keyword parameters passed to {IOReader} constructor
     def initialize(file, mode: 'rb', **options)
       super(nil, **options)
       @file = file

data/lib/iop/net/ftp.rb

@@ -12,7 +12,7 @@ module IOP
 
 
   # @private
-  module FTPFile
+  module FTPCommons
 
     private
 
@@ -45,6 +45,7 @@ module IOP
 
   end
 
+
   #
   # Feed class to read file from FTP server.
   #
@@ -52,6 +53,7 @@ module IOP
   #
   # ### Use case: retrieve file from FTP server and store it locally.
   #
+  #     require 'iop/file'
   #     require 'iop/net/ftp'
   #     ( IOP::FTPFileReader.new('ftp.gnu.org', '/pub/README') | IOP::FileWriter.new('README') ).process!
   #
@@ -60,7 +62,7 @@ module IOP
   class FTPFileReader
 
     include Feed
-    include FTPFile
+    include FTPCommons
 
     # Creates class instance.
     #
@@ -133,7 +135,6 @@ module IOP
   end
 
 
-
   #
   # Sink class to write file to FTP server.
   #
@@ -158,7 +159,7 @@ module IOP
   class FTPFileWriter
 
     include Sink
-    include FTPFile
+    include FTPCommons
 
     # Creates class instance.
     #

data/lib/iop/net/sftp.rb

@@ -0,0 +1,176 @@
+require 'iop'
+require 'net/sftp'
+
+
+module IOP
+
+
+  # @private
+  module SFTPCommons
+
+    private
+
+    def setup
+      if @sftp.is_a?(String)
+        @sftp = Net::SFTP.start(@sftp, @options.delete(:username), **@options)
+        @managed = true
+      end
+    end
+
+    def cleanup
+      if @managed
+        ssh = @sftp.session
+        @sftp.close_channel
+        ssh.close
+      end
+    end
+
+  end
+
+
+  #
+  # Feed class to read file from SFTP server.
+  #
+  # This class an adapter for +Net::SFTP::Session+ class.
+  #
+  # ### Use case: retrieve current user's _~/.profile_ file from SFTP server running on local machine and and compute its MD5 hash sum.
+  #
+  #     require 'iop/digest'
+  #     require 'iop/net/sftp'
+  #     ( IOP::SFTPFileReader.new('localhost', '.profile') | (d = IOP::DigestComputer.new(Digest::MD5.new)) ).process!
+  #     puts d.digest.hexdigest
+  #
+  # @note this class depends on external +net-sftp+ gem.
+  # @since 0.2
+  #
+  class SFTPFileReader < RandomAccessReader
+
+    include Feed
+    include SFTPCommons
+
+    # Creates class instance.
+    #
+    # @param sftp [String, Net::SFTP::Session] SFTP server to connect to
+    #
+    # @param file [String] file name to process
+    #
+    # @param size [Integer] total number of bytes to read; +nil+ value instructs to read until end-of-data is reached
+    #
+    # @param offset [Integer] offset in bytes from the stream start to seek to; +nil+ value means no seeking is performed
+    #
+    # @param block_size [Integer] size of blocks to process data with
+    #
+    # @param options [Hash] extra keyword parameters passed to +Net::SFTP::Session+ constructor, such as username, password etc.
+    #
+    # _sftp_ can be either a +String+ of +Net::SFTP::Session+ instance.
+    # If it is a string a corresponding +Net::SFTP::Session+ instance will be created with _options_ passed to its constructor.
+    #
+    # If _sftp_ is a string, a created SFTP session is managed, e.g. it is closed after the process is complete,
+    # otherwise supplied object is left as is and no closing is performed.
+    # This allows to reuse SFTP session for a sequence of operations.
+    #
+    # Refer to +Net::SFTP+ documentation for available options.
+    def initialize(sftp, file, size: nil, offset: nil, block_size: DEFAULT_BLOCK_SIZE, **options)
+      super(size: size, offset: offset, block_size: block_size)
+      @options = options
+      @file = file
+      @sftp = sftp
+    end
+
+    def process!
+      setup
+      begin
+        @io = @sftp.open!(@file, 'r')
+        begin
+          super
+        ensure
+          @sftp.close(@io)
+        end
+      ensure
+        cleanup
+      end
+    end
+
+    private
+
+    def read!(read_size, buffer)
+      data = @sftp.read!(@io, @offset ||= 0, read_size)
+      @offset += data.size unless data.nil?
+      data
+    end
+
+  end
+
+
+  #
+  # Sink class to write file to SFTP server.
+  #
+  # This class an adapter for +Net::SFTP::Session+ class.
+  #
+  # ### Use case: store a number of files filled with random data to remote SFTP server reusing session.
+  #
+  #     require 'iop/net/sftp'
+  #     require 'iop/securerandom'
+  #     sftp = Net::SFTP.start('sftp.server', username: 'user',  password: '123')
+  #     begin
+  #       (1..3).each do |i|
+  #         ( IOP::SecureRandomGenerator.new(1024) | IOP::SFTPFileWriter.new(sftp, "random#{i}.dat") ).process!
+  #       end
+  #     ensure
+  #       sftp.session.close
+  #     end
+  #
+  # @note this class depends on external +net-sftp+ gem.
+  # @since 0.2
+  #
+  class SFTPFileWriter
+
+    include Sink
+    include SFTPCommons
+
+    # Creates class instance.
+    #
+    # @param sftp [String, Net::SFTP::Session] SFTP server to connect to
+    #
+    # @param file [String] file name to process
+    #
+    # @param options [Hash] extra keyword parameters passed to +Net::SFTP::Session+ constructor
+    #
+    # _sftp_ can be either a +String+ of +Net::SFTP::Session+ instance.
+    # If it is a string a corresponding +Net::SFTP::Session+ instance will be created with _options_ passed to its constructor.
+    #
+    # If _sftp_ is a string, a created SFTP session is managed, e.g. it is closed after the process is complete,
+    # otherwise supplied object is left as is and no closing is performed.
+    # This allows to reuse SFTP session for a sequence of operations.
+    def initialize(sftp, file, **options)
+      @options = options
+      @file = file
+      @sftp = sftp
+    end
+
+    def process!
+      setup
+      begin
+        @io = @sftp.open!(@file, 'w')
+        @offset = 0
+        begin
+          super
+        ensure
+          @sftp.close(@io)
+        end
+      ensure
+        cleanup
+      end
+    end
+
+    def process(data = nil)
+      unless data.nil?
+        @sftp.write!(@io, @offset, data)
+        @offset += data.size
+      end
+    end
+
+  end
+
+
+end

data/lib/iop/securerandom.rb

@@ -8,7 +8,7 @@ module IOP
   #
   # Feed class to generate and send a random sequence of bytes of specified size.
   #
-  # This is the adapter for standard {SecureRandom} generator module.
+  # This is the adapter for the standard Ruby +SecureRandom+ module.
   #
   # ### Use case: generate 1024 bytes of random data and compute MD5 hash sum of it.
   #

metadata

@@ -1,38 +1,41 @@
 --- !ruby/object:Gem::Specification
 name: iop
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Oleg A. Khlybov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-18 00:00:00.000000000 Z
+date: 2019-04-22 00:00:00.000000000 Z
 dependencies: []
 description: |2
-      Allows to construct data and file processing pipelines in a manner of UNIX shell pipes.
+      I/O pipeline construction framework.
+      Allows to construct data processing pipelines in a manner of UNIX shell pipes.
       Implemented features:
-      - String splitting/merging
-      - IO or local file reading/writing
-      - FTP file reading/writing
-      - Digest computing
-      - GZip/Zlib (de)compression
-      - Zstd (de)compression
-      - Symmetric cipher (de,en)cryption
-      - Random data generation
+        string splitting/merging,
+        IO or local file reading/writing,
+        FTP/SFTP file reading/writing,
+        digest computing,
+        Gzip/Zlib (de)compression,
+        Zstd (de)compression,
+        symmetric cipher (de,en)cryption,
+        random data generation.
 email:
 - fougas@mail.ru
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".yardopts"
 - CHANGES.md
 - README.md
 - lib/iop.rb
 - lib/iop/digest.rb
 - lib/iop/file.rb
 - lib/iop/net/ftp.rb
+- lib/iop/net/sftp.rb
 - lib/iop/openssl.rb
 - lib/iop/securerandom.rb
 - lib/iop/string.rb