iop 0.1.0

data/lib/iop/securerandom.rb

@@ -0,0 +1,49 @@
+require 'iop'
+require 'securerandom'
+
+
+module IOP
+
+
+  #
+  # Feed class to generate and send a random sequence of bytes of specified size.
+  #
+  # This is the adapter for standard {SecureRandom} generator module.
+  #
+  # ### Use case: generate 1024 bytes of random data and compute MD5 hash sum of it.
+  #
+  #     require 'iop/digest'
+  #     require 'iop/securerandom'
+  #     ( IOP::SecureRandomGenerator.new(1024) | IOP::DigestComputer.new(Digest::MD5.new) ).process!
+  #
+  # @since 0.1
+  #
+  class SecureRandomGenerator
+
+    include Feed
+
+    # Creates class instance.
+    #
+    # @param size [Integer] total random data size
+    #
+    # @param block_size [Integer] size of block the data in split into
+    def initialize(size, block_size: DEFAULT_BLOCK_SIZE)
+      @size = size
+      @block_size = block_size
+    end
+
+    def process!
+      written = 0
+      (0..@size/@block_size - 1).each do
+        process(SecureRandom.bytes(@block_size))
+        written += @block_size
+      end
+      left = @size - written
+      process(SecureRandom.bytes(left)) unless left.zero?
+      process
+    end
+
+  end
+
+
+end

data/lib/iop/string.rb

@@ -0,0 +1,89 @@
+require 'iop'
+
+
+module IOP
+
+
+  #
+  # Feed class to send arbitrary string in blocks of specified size.
+  #
+  # ### Use case: split the string into 3-byte blocks and reconstruct it.
+  #
+  #     require 'iop/string'
+  #     ( IOP::StringSplitter.new('Hello IOP', 3) | IOP::StringMerger.new ).process!
+  #
+  # @since 0.1
+  #
+  class StringSplitter
+
+    include Feed
+
+    # Creates class instance.
+    #
+    # @param string [String] string to be sent in blocks
+    #
+    # @param block_size [Integer] size of block the string is split into
+    def initialize(string, block_size: DEFAULT_BLOCK_SIZE)
+      @string = string
+      @block_size = block_size
+    end
+
+    def process!
+      offset = 0
+      (0..@string.size / @block_size - 1).each do
+        process(@string[offset, @block_size])
+        offset += @block_size
+      end
+      process(offset.zero? ? @string : @string[offset..-1]) unless offset == @string.size
+      process
+    end
+
+  end
+
+
+  #
+  # Sink class to receive data blocks and merge them into a single string.
+  #
+  # ### Use case: read current source file into a string.
+  #
+  #     require 'iop/file'
+  #     require 'iop/string'
+  #     ( IOP::FileReader.new($0) | (s = IOP::StringMerger.new) ).process!
+  #     puts s.to_s
+  #
+  # The actual string assembly is performed by the {#to_s} method.
+  #
+  # @note instance of this class can be used to collect data from multiple processing runs.
+  #
+  # @since 0.1
+  #
+  class StringMerger
+
+    include Sink
+
+    # Creates class instance.
+    def initialize
+      @size = 0
+      @data = []
+    end
+
+    def process(data = nil)
+      unless data.nil?
+        @data << data.dup # CHECKME is duplication really needed when the upstream continuously resending its internal data buffer with new contents
+        @size += data.size
+      end
+    end
+
+    # Returns concatenation of all received data blocks into a single string.
+    #
+    # @return [String]
+    def to_s
+      string = IOP.allocate_string(@size)
+      @data.each {|x| string << x}
+      string
+    end
+
+  end
+
+
+end

data/lib/iop/zlib.rb

@@ -0,0 +1,179 @@
+require 'iop'
+require 'zlib'
+
+
+module IOP
+
+
+  #
+  # Filter class to perform data compression with Zlib algorithm.
+  #
+  # This class is an adapter for the standard Ruby +Zlib::Deflate+ class.
+  #
+  # Note that this class does not produce valid _.gz_ files - use {GzipCompressor} for this purpose.
+  #
+  # ### Use case: compress a string.
+  #
+  #     require 'iop/zlib'
+  #     require 'iop/string'
+  #     ( IOP::StringSplitter.new('Hello IOP') | IOP::ZlibCompressor.new | (s = IOP::StringMerger.new) ).process!
+  #     puts s.to_s
+  #
+  # @since 0.1
+  #
+  class ZlibCompressor
+
+    include Feed
+    include Sink
+
+    # Creates class instance.
+    #
+    # @param args [Array] arguments passed to +Zlib::Deflate+ constructor
+    def initialize(*args)
+      @args = args
+    end
+
+    def process(data = nil)
+      if data.nil?
+        super(@deflate.finish)
+        super
+      else
+        super(@deflate.deflate(data))
+      end
+    end
+
+    def process!
+      @deflate = Zlib::Deflate.new(*@args)
+      begin
+        super
+      ensure
+        @deflate.close
+      end
+    end
+  end
+
+
+  #
+  # Filter class to perform data decompression with Zlib algorithm.
+  #
+  # This class is an adapter for the standard Ruby +Zlib::Inflate+ class.
+  #
+  # Note that this class can not decompress _.gz_ files - use {GzipDecompressor} for this purpose.
+  #
+  # ### Use case: decompress a Zlib-compressed part of a file skipping a header and compute MD5 hash sum of the uncompressed data.
+  #
+  #     require 'iop/zlib'
+  #     require 'iop/file'
+  #     require 'iop/digest'
+  #     ( IOP::FileReader.new('input.dat', offset: 16) | IOP::ZlibDecompressor.new | (d = IOP::DigestComputer.new(Digest::MD5.new)) ).process!
+  #     puts d.digest.hexdigest
+  #
+  # @since 0.1
+  #
+  class ZlibDecompressor
+
+    include Feed
+    include Sink
+
+    # Creates class instance.
+    #
+    # @param args [Array] arguments passed to +Zlib::Inflate+ constructor
+    def initialize(*args)
+      @args = args
+    end
+
+    def process(data = nil)
+      if data.nil?
+        super(@inflate.finish)
+        super
+      else
+        super(@inflate.inflate(data))
+      end
+    end
+
+    def process!
+      @inflate = Zlib::Inflate.new(*@args)
+      begin
+        super
+      ensure
+        @inflate.close
+      end
+    end
+  end
+
+
+  #
+  # Filter class to perform Gzip data compression.
+  #
+  # This class is an adapter for the standard Ruby +Zlib::GzipWriter+ class.
+  #
+  # This class produces valid _.gz_ files.
+  #
+  # ### Use case: compress a string and store it to .gz file.
+  #
+  #     require 'iop/zlib'
+  #     require 'iop/file'
+  #     require 'iop/string'
+  #     ( IOP::StringSplitter.new('Hello IOP') | IOP::GzipCompressor.new | IOP::FileWriter.new('hello.gz') ).process!
+  #
+  # @since 0.1
+  #
+  class GzipCompressor
+
+    include Feed
+    include Sink
+
+    # Creates class instance.
+    #
+    # @param args [Array] arguments passed to +Zlib::GzipWriter+ constructor
+    def initialize(*args)
+      @args = args
+    end
+
+    def process(data = nil)
+      if data.nil?
+        @compressor.finish
+        super
+      else
+        @compressor.write(data)
+      end
+    end
+
+    def write(data)
+      downstream&.process(data)
+    end
+
+    def process!
+      @compressor = Zlib::GzipWriter.new(self, *@args)
+      super
+    ensure
+      @compressor.close unless @compressor.nil?
+    end
+  end
+
+
+  #
+  # Filter class to perform Gzip data compression.
+  #
+  # This class is an adapter for the standard Ruby +Zlib::GzipWriter+ class.
+  #
+  # This class can decompress _.gz_ files.
+  #
+  # ### Use case: decompress a .gz file and compute MD5 hash sum of uncompressed data.
+  #
+  #     require 'iop/zlib'
+  #     require 'iop/file'
+  #     require 'iop/digest'
+  #     ( IOP::FileReader.new('hello.gz') | IOP::GzipDecompressor.new | (d = IOP::DigestComputer.new(Digest::MD5.new)) ).process!
+  #     puts d.digest.hexdigest
+  #
+  # @since 0.1
+  #
+  class GzipDecompressor < ZlibDecompressor
+    def initialize
+      super(16)
+    end
+  end
+
+
+end

data/lib/iop/zstdlib.rb

@@ -0,0 +1,104 @@
+require 'iop'
+require 'zstdlib'
+
+
+module IOP
+
+
+  #
+  # Filter class to perform data compression with Zstandard algorithm.
+  #
+  # This class produces valid _.zst_ files.
+  #
+  # ### Use case: compress a string and store it to .zst file.
+  #
+  #     require 'iop/file'
+  #     require 'iop/string'
+  #     require 'iop/zstdlib'
+  #     ( IOP::StringSplitter.new('Hello IOP') | IOP::ZstdCompressor.new(Zstdlib::BEST_COMPRESSION) | IOP::FileWriter.new('hello.zst') ).process!
+  #
+  # @note this class depends on external +zstdlib+ gem.
+  # @since 0.1
+  #
+  class ZstdCompressor
+
+    include Feed
+    include Sink
+
+    # Creates class instance.
+    #
+    # @param args [Array] arguments passed to +Zstdlib::Deflate+ constructor
+    def initialize(*args)
+      @args = args
+    end
+
+    def process(data = nil)
+      if data.nil?
+        super(@deflate.finish)
+        super
+      else
+        super(@deflate.deflate(data))
+      end
+    end
+
+    def process!
+      @deflate = Zstdlib::Deflate.new(*@args)
+      begin
+        super
+      ensure
+        @deflate.close
+      end
+    end
+  end
+
+
+  #
+  # Filter class to perform Gzip data compression.
+  #
+  # This class is an adapter for the standard Ruby +Zlib::GzipWriter+ class.
+  #
+  # This class can decompress _.zst_ files.
+  #
+  # ### Use case: decompress a .zst file and compute MD5 hash sum of uncompressed data.
+  #
+  #     require 'iop/file'
+  #     require 'iop/digest'
+  #     require 'iop/zstdlib'
+  #     ( IOP::FileReader.new('hello.zst') | IOP::ZstdDecompressor.new | (d = IOP::DigestComputer.new(Digest::MD5.new)) ).process!
+  #     puts d.digest.hexdigest
+  #
+  # @note this class depends on external +zstdlib+ gem.
+  # @since 0.1
+  #
+  class ZstdDecompressor
+
+    include Feed
+    include Sink
+
+    # Creates class instance.
+    #
+    # @param args [Array] arguments passed to +Zstdlib::Inflate+ constructor
+    def initialize(*args)
+      @args = args
+    end
+
+    def process(data = nil)
+      if data.nil?
+        super(@inflate.finish)
+        super
+      else
+        super(@inflate.inflate(data))
+      end
+    end
+
+    def process!
+      @inflate = Zstdlib::Inflate.new(*@args)
+      begin
+        super
+      ensure
+        @inflate.close
+      end
+    end
+  end
+
+  end

data/lib/iop.rb

@@ -0,0 +1,250 @@
+#
+# IOP is intended for constructing the data processing pipelines in a manner of UNIX command-line pipes.
+#
+# There are three principle types of the pipe nodes which can be composed:
+#
+# * Feed node.
+#
+# This is the start point of the pipe. It has no upstream node and may have downstream node.
+# Its purpose its to generate blocks of data and send them downstream in sequence.
+# A typical feed class is implemented by including the {Feed} module and defining the +#process!+ method
+# which calls {Feed#process} method to send the data.
+# An example of the feed node is a file reader ({FileReader}) which reads file and sends its contents in blocks.
+#
+# * Sink node.
+#
+# This is the end point of the pipe. It has upstream node and no downstream node.
+# Its purpose is to consume the received data.
+# A typical sink class is implemented by including the {Sink} module and defining the +#process+ method.
+# An example of the sink node is a file writer ({FileWriter}) which receives the data in blocks and writes it into file.
+#
+# * Filter node.
+#
+# A filter is a pass-through node which sits between feed and sink and therefore has both upstream and downstream nodes.
+# The simplest way to create a filter class is to include both {Feed} and {Sink} which manifest
+# both mandatory +#process!+ and +#process+ methods. Such filter is a no-op that is it does nothing apart passing
+# the received data downstream.
+# An example of the filter node is the digest computer ({DigestComputer}) which computes hash sum of the data it passes through.
+# In order to perform intended processing of the data a filter class overrides the {Feed#process} method.
+#
+# The basic control flow for an {IOP}-aware pipe is as follows:
+#
+# 1. The pipe is constructed from one or more {IOP}-aware class instances. The two or more objects are linked together
+# with the | operator implemented as the {Feed#|} method by default.
+#
+# 2. The actual processing is then triggered by the {Sink#process!} method of the very last object in the pipe.
+# By default, this method calls the same method of the upstream node thus forming the stack of nested calls
+# for all objects in the pipe.
+#
+# 3. Upon reaching the very first object in the pipe (which by definition has no upstream node),
+# the feed, starts sending blocks of data downstream with the {Feed#process} method. All objects' method implementations
+# (except for the one of the last object in the pipe) are expected to push either this or transformed data further downstream.
+#
+# 4. After all data has been processed the finalizing call +#process(nil)+ signifies the end-of-data after which
+# no data should be sent.
+#
+# In case the {Sink#process!} method is overridden in concrete class it is normally organized as follows:
+#
+#     def process!
+#       # ...initialization code...
+#       super
+#     ensure
+#       # ...finalization code...
+#     end
+#
+# to perform specific setup/cleanup actions, including exception handling and to pass the control flow upstream
+# with +super+ call.
+#
+# Note that when an exception is caught and processed in overridden +#process!+ method it must be re-raised in order
+# for other upstream objects to have a chance to react to it as well.
+#
+# In case the {Feed#process} is overridden in concrete class it is organized as follows:
+#
+#     def process(data = nil)
+#       # ... do something with data, convert data to new_data...
+#       super(new_data)
+#     end
+#
+# The data being sent is expected to be a +String+ of arbitrary size. It is however advisable to detect and omit
+# zero-sized strings.
+#
+# Note that the data passed to this method may be a reusable buffer of some other upstream object therefore a duplication
+# (or cloning) should be performed if the data is stored between the method invocations.
+#
+module IOP
+
+
+  VERSION = '0.1.0'
+
+
+  # Default read block size in bytes for adapters which don't have this parameter externally imposed.
+  DEFAULT_BLOCK_SIZE = 1024**2
+
+
+  if RUBY_VERSION >= '2.4'
+    # @private
+    def self.allocate_string(size)
+      String.new(capacity: size)
+    end
+  else
+    # @private
+    def self.allocate_string(size)
+      String.new
+    end
+  end
+
+
+  # @private
+  INSUFFICIENT_DATA = 'premature end-of-data encountered'.freeze
+
+
+  # @private
+  EXTRA_DATA = 'superfluous data received'.freeze
+
+
+  # @private
+  # Finds minimum of the values
+  def self.min(a, b)
+    a < b ? a : b
+  end
+
+
+  #
+  # Module to be included into classes which generate and send the data downstream.
+  #
+  # @since 0.1
+  #
+  module Feed
+
+    #
+    # Commences the data processing operation.
+    #
+    # @abstract
+    #
+    # @note this method should be implemented in concrete classes including this module.
+    #
+    # Refer to {Sink#process!} for details.
+    #
+    def process!
+      raise
+    end
+    remove_method :process!
+
+    #
+    # Sends the data block downstream.
+    #
+    # @note by convention, the very last call to this method should pass +nil+ to indicate the end-of-data and no data should be sent afterwards.
+    #
+    # This implementation simply passes through the received data block downstream if there exists an attached downstream
+    # object otherwise the data is simply thrown away.
+    #
+    # The overriding method in concrete class which includes {Feed} would normally want to call this one as +super+ after
+    # performing specific actions.
+    #
+    def process(data = nil)
+      downstream&.process(data) # Ruby 2.3+
+    end
+
+    # Returns the downstream object or +nil+ if +self+ is the last object in processing pipe.
+    attr_reader :downstream
+
+    #
+    # Links +self+ and +downstream+ together forming a processing pipe.
+    # The subsequent objects may be linked in turn.
+    # @return downstream object
+    #
+    def |(downstream)
+      downstream.upstream = self
+      @downstream = downstream
+    end
+
+  end
+
+
+  #
+  # Module to be included into classes which receive and process the upstream data.
+  #
+  # @since 0.1
+  #
+  module Sink
+
+    # Commences the data processing operation.
+    #
+    # This implementation calls {#process!} method of the upstream object.
+    def process!
+      upstream.process!
+    end
+
+    # @abstract
+    #
+    # @note this method should be implemented in concrete classes including this module.
+    #
+    # Refer to {Feed#process} for more information.
+    def process(data = nil)
+      raise
+    end
+    remove_method :process
+
+    # Returns the upstream object or +nil+ if +self+ is the first object in processing pipe.
+    attr_accessor :upstream
+
+  end
+
+
+  #
+  # @private
+  #
+  # @note a class including this module must implement the {#next_data} method.
+  #
+  # @since 0.1
+  #
+  module BufferingFeed
+
+    include Feed
+
+    def read!(size)
+      @left = @size = size
+      self
+    end
+
+    def process!
+      unless @buffer.nil?
+        if @buffer.size > @size
+          @left = 0
+          process(@buffer[0, @size])
+          @buffer = @buffer[@size..-1]
+        else
+          @left -= @buffer.size
+          process(@buffer)
+          @buffer = nil
+        end
+      end
+      until @left.zero?
+        raise EOFError, INSUFFICIENT_DATA if (data = next_data).nil?
+        if @left < data.size
+          process(data[0, @left])
+          @buffer = data[@left..-1]
+          @left = 0
+        else
+          process(data)
+          @left -= data.size
+        end
+      end
+      @left = @size = nil
+      process
+    end
+
+    # @abstract
+    #
+    # Returns the data portion of non-zero size or +nil+ on EOF.
+    #
+    # @return [String] data chunk recently read or +nil+
+    def next_data
+      raise
+    end
+    remove_method :next_data
+
+  end
+
+
+end

data/test/digest_test.rb

@@ -0,0 +1,18 @@
+require 'test/unit'
+require 'iop/digest'
+require 'iop/file'
+require 'openssl'
+
+class DigestTest < Test::Unit::TestCase
+
+  include IOP
+
+  def test_digest
+    ( FileReader.new(__FILE__) | DigestComputer.new(Digest::MD5.new) ).process!
+  end
+
+  def test_openssl_digest
+    ( FileReader.new(__FILE__) | DigestComputer.new(OpenSSL::Digest::MD5.new) ).process!
+  end
+
+end

data/test/io_test.rb

@@ -0,0 +1,23 @@
+require 'test/unit'
+require 'iop/file'
+require 'iop/string'
+require 'stringio'
+
+class IOTest < Test::Unit::TestCase
+
+  include IOP
+
+  def test_iosegmentreader_small
+    s = '0123456789'
+    (1..s.size-1).each do |b|
+      (1..11).each do |i|
+        m = StringMerger.new
+        r = IOSegmentReader.new(StringIO.open(s), block_size: i)
+        (r.read!(b) | m).process!
+        (r.read!(s.size-b) | m).process!
+        assert_equal s, m.to_s
+      end
+    end
+  end
+
+end