adsp 1.0.0

data/lib/adsp/stream/delegates.rb

@@ -0,0 +1,39 @@
+# Abstract data stream processor.
+# Copyright (c) 2021 AUTHORS, MIT License.
+
+require "forwardable"
+
+module ADSP
+  module Stream
+    # ADSP::Stream::Delegates class.
+    module Delegates
+      # List of methods to be forwarded for native stream.
+      DELEGATES = %i[
+        autoclose=
+        autoclose?
+        binmode
+        binmode?
+        close_on_exec=
+        close_on_exec?
+        fcntl
+        fdatasync
+        fileno
+        fsync
+        ioctl
+        isatty
+        pid
+        sync
+        sync=
+        to_i
+        tty?
+      ]
+      .freeze
+
+      # Defines delegates for native stream after module included.
+      def self.included(klass)
+        klass.extend ::Forwardable
+        klass.def_delegators :@io, *DELEGATES
+      end
+    end
+  end
+end

data/lib/adsp/stream/raw/abstract.rb

@@ -0,0 +1,69 @@
+# Abstract data stream processor.
+# Copyright (c) 2021 AUTHORS, MIT License.
+
+require_relative "../../error"
+require_relative "../../validation"
+
+module ADSP
+  module Stream
+    module Raw
+      # ADSP::Stream::Raw::Abstract class.
+      class Abstract
+        # Initializes raw stream using +native_stream+.
+        def initialize(native_stream)
+          @native_stream = native_stream
+          @is_closed     = false
+        end
+
+        # -- write --
+
+        # Flushes raw stream and writes next result using +writer+ proc.
+        def flush(&writer)
+          do_not_use_after_close
+
+          Validation.validate_proc writer
+
+          write_result(&writer)
+
+          nil
+        end
+
+        # Writes next result using +writer+ proc and frees destination buffer.
+        protected def more_destination(&writer)
+          result_bytesize = write_result(&writer)
+          raise NotEnoughDestinationError, "not enough destination" if result_bytesize.zero?
+        end
+
+        # Writes next result using block.
+        protected def write_result(&_writer)
+          result = @native_stream.read_result
+          yield result
+
+          result.bytesize
+        end
+
+        # -- close --
+
+        # Raises error when raw stream is closed.
+        protected def do_not_use_after_close
+          raise UsedAfterCloseError, "used after close" if closed?
+        end
+
+        # Writes next result using +writer+ proc and closes raw stream.
+        def close(&writer)
+          write_result(&writer)
+
+          @native_stream.close
+          @is_closed = true
+
+          nil
+        end
+
+        # Returns whether raw stream is closed.
+        def closed?
+          @is_closed
+        end
+      end
+    end
+  end
+end

data/lib/adsp/stream/raw/compressor.rb

@@ -0,0 +1,110 @@
+# Abstract data stream processor.
+# Copyright (c) 2021 AUTHORS, MIT License.
+
+require_relative "abstract"
+require_relative "native_compressor"
+require_relative "../../error"
+require_relative "../../option"
+require_relative "../../validation"
+
+module ADSP
+  module Stream
+    module Raw
+      # ADSP::Stream::Raw::Compressor class.
+      class Compressor < Abstract
+        # Current native compressor class.
+        NativeCompressor = Raw::NativeCompressor
+
+        # Current option class.
+        Option = ADSP::Option
+
+        # Current buffer length names.
+        # It is a part of compressor options.
+        BUFFER_LENGTH_NAMES = %i[destination_buffer_length].freeze
+
+        # Initializes compressor.
+        # Option: +:destination_buffer_length+ destination buffer length.
+        def initialize(options = {})
+          options       = self.class::Option.get_compressor_options options, BUFFER_LENGTH_NAMES
+          native_stream = self.class::NativeCompressor.new options
+
+          super native_stream
+        end
+
+        # Writes +source+ string, writes result using +writer+ proc.
+        # Returns amount of bytes written from +source+.
+        def write(source, &writer)
+          do_not_use_after_close
+
+          Validation.validate_string source
+          Validation.validate_proc writer
+
+          total_bytes_written = 0
+
+          loop do
+            bytes_written, need_more_destination  = @native_stream.write source
+            total_bytes_written                  += bytes_written
+
+            if need_more_destination
+              source = source.byteslice bytes_written, source.bytesize - bytes_written
+              more_destination(&writer)
+              next
+            end
+
+            unless bytes_written == source.bytesize
+              # Compressor write should eat all provided "source" without remainder.
+              # :nocov:
+              raise UnexpectedError, "unexpected error"
+              # :nocov:
+            end
+
+            break
+          end
+
+          total_bytes_written
+        end
+
+        # Flushes compressor, writes result using +writer+ proc and closes compressor.
+        def flush(&writer)
+          do_not_use_after_close
+
+          Validation.validate_proc writer
+
+          loop do
+            need_more_destination = @native_stream.flush
+
+            if need_more_destination
+              more_destination(&writer)
+              next
+            end
+
+            break
+          end
+
+          super
+        end
+
+        # Finishes compressor, writes result using +writer+ proc and closes compressor.
+        # Raises +UsedAfterCloseError+ when used after close.
+        def close(&writer)
+          return nil if closed?
+
+          Validation.validate_proc writer
+
+          loop do
+            need_more_destination = @native_stream.finish
+
+            if need_more_destination
+              more_destination(&writer)
+              next
+            end
+
+            break
+          end
+
+          super
+        end
+      end
+    end
+  end
+end

data/lib/adsp/stream/raw/decompressor.rb

@@ -0,0 +1,80 @@
+# Abstract data stream processor.
+# Copyright (c) 2021 AUTHORS, MIT License.
+
+require_relative "abstract"
+require_relative "native_decompressor"
+require_relative "../../option"
+require_relative "../../validation"
+
+module ADSP
+  module Stream
+    module Raw
+      # ADSP::Stream::Raw::Decompressor class.
+      class Decompressor < Abstract
+        # Current native decompressor class.
+        NativeDecompressor = Raw::NativeDecompressor
+
+        # Current option class.
+        Option = ADSP::Option
+
+        # Current buffer length names.
+        # It is a part of decompressor options.
+        BUFFER_LENGTH_NAMES = %i[destination_buffer_length].freeze
+
+        # Initializes decompressor.
+        # Option: +:destination_buffer_length+ destination buffer length.
+        def initialize(options = {})
+          options       = self.class::Option.get_decompressor_options options, BUFFER_LENGTH_NAMES
+          native_stream = self.class::NativeDecompressor.new options
+
+          super native_stream
+        end
+
+        # Reads +source+ string, writes result using +writer+ proc.
+        # Returns amount of bytes read from +source+.
+        def read(source, &writer)
+          do_not_use_after_close
+
+          Validation.validate_string source
+          Validation.validate_proc writer
+
+          total_bytes_read = 0
+
+          loop do
+            bytes_read, need_more_destination  = @native_stream.read source
+            total_bytes_read                  += bytes_read
+
+            if need_more_destination
+              source = source.byteslice bytes_read, source.bytesize - bytes_read
+              more_destination(&writer)
+              next
+            end
+
+            break
+          end
+
+          total_bytes_read
+        end
+
+        # Flushes decompressor, writes result using +writer+ proc and closes decompressor.
+        def flush(&writer)
+          do_not_use_after_close
+
+          Validation.validate_proc writer
+
+          super
+        end
+
+        # Writes result using +writer+ proc and closes decompressor.
+        # Raises +UsedAfterCloseError+ when used after close.
+        def close(&writer)
+          return nil if closed?
+
+          Validation.validate_proc writer
+
+          super
+        end
+      end
+    end
+  end
+end

data/lib/adsp/stream/raw/native_compressor.rb

@@ -0,0 +1,58 @@
+# Abstract data stream processor.
+# Copyright (c) 2021 AUTHORS, MIT License.
+
+require_relative "../../error"
+
+module ADSP
+  module Stream
+    module Raw
+      # ADSP::Stream::Raw::NativeCompressor class.
+      class NativeCompressor
+        # :nocov:
+
+        # Initializes native compressor.
+        # Option: +:destination_buffer_length+ destination buffer length.
+        def initialize(options)
+          raise NotImplementedError
+        end
+
+        # Writes part of +source+ string.
+        # Returns array of 2 values:
+        # 1. number of bytes written from +source+.
+        # 2. boolean that can be named as "need more destination".
+        # User needs to call +read_result+ if "need more destination" is true.
+        def write(source)
+          raise NotImplementedError
+        end
+
+        # Provides next part of unread result.
+        # Returns empty string if there is no unread result.
+        def read_result
+          raise NotImplementedError
+        end
+
+        # Flushes internal buffers and prepares result for +read_result+.
+        # Returns boolean that can be named as "need more destination".
+        # User needs to call +read_result+ if "need more destination" is true.
+        def flush
+          raise NotImplementedError
+        end
+
+        # Finishes compressor and prepares result for +read_result+.
+        # Returns boolean that can be named as "need more destination".
+        # User needs to call +read_result+ if "need more destination" is true.
+        def finish
+          raise NotImplementedError
+        end
+
+        # Closes compressor and cleans up internal resources.
+        # Raises +UsedAfterCloseError+ when used after close.
+        def close
+          raise NotImplementedError
+        end
+
+        # :nocov:
+      end
+    end
+  end
+end

data/lib/adsp/stream/raw/native_decompressor.rb

@@ -0,0 +1,44 @@
+# Abstract data stream processor.
+# Copyright (c) 2021 AUTHORS, MIT License.
+
+require_relative "../../error"
+
+module ADSP
+  module Stream
+    module Raw
+      # ADSP::Stream::Raw::NativeDecompressor class.
+      class NativeDecompressor
+        # :nocov:
+
+        # Initializes native decompressor.
+        # Option: +:destination_buffer_length+ destination buffer length.
+        def initialize(options)
+          raise NotImplementedError
+        end
+
+        # Reads part of +source+ string.
+        # Returns array of 2 values:
+        # 1. number of bytes read from +source+.
+        # 2. boolean that can be named as "need more destination".
+        # User needs to call +read_result+ if "need more destination" is true.
+        def read(source)
+          raise NotImplementedError
+        end
+
+        # Provides next part of unread result.
+        # Returns empty string if there is no unread result.
+        def read_result
+          raise NotImplementedError
+        end
+
+        # Closes decompressor and cleans up internal resources.
+        # Raises +UsedAfterCloseError+ when used after close.
+        def close
+          raise NotImplementedError
+        end
+
+        # :nocov:
+      end
+    end
+  end
+end

data/lib/adsp/stream/reader.rb

@@ -0,0 +1,234 @@
+# Abstract data stream processor.
+# Copyright (c) 2021 AUTHORS, MIT License.
+
+require_relative "abstract"
+require_relative "raw/decompressor"
+require_relative "reader_helpers"
+require_relative "../validation"
+
+module ADSP
+  module Stream
+    # ADSP::Stream::Reader class.
+    class Reader < Abstract
+      include ReaderHelpers
+
+      # Default source buffer length.
+      DEFAULT_SOURCE_BUFFER_LENGTH = 1 << 18 # 256 KB
+
+      # Current raw stream class.
+      RawDecompressor = Raw::Decompressor
+
+      # Current line for source data.
+      attr_accessor :lineno
+
+      # Initializes stream using +source_io+ native stream and +options+.
+      # Option: +:external_encoding+ encoding name for destination data.
+      # Option: +:internal_encoding+ encoding name for source data.
+      # Option: +:transcode_options+ transcode options for data.
+      def initialize(source_io, options = {}, *args)
+        @options = options
+
+        super source_io, *args
+
+        initialize_source_buffer_length
+        reset_io_remainder
+        reset_need_to_flush
+
+        @lineno = 0
+      end
+
+      # Creates raw stream.
+      protected def create_raw_stream
+        self.class::RawDecompressor.new @options
+      end
+
+      # Initializes source buffer length.
+      protected def initialize_source_buffer_length
+        source_buffer_length = @options[:source_buffer_length]
+        Validation.validate_not_negative_integer source_buffer_length unless source_buffer_length.nil?
+
+        if source_buffer_length.nil? || source_buffer_length.zero?
+          source_buffer_length = self.class::DEFAULT_SOURCE_BUFFER_LENGTH
+        end
+
+        @source_buffer_length = source_buffer_length
+      end
+
+      # Resets io remainder.
+      protected def reset_io_remainder
+        @io_remainder = ::String.new :encoding => ::Encoding::BINARY
+      end
+
+      # Resets need to flush flag.
+      protected def reset_need_to_flush
+        @need_to_flush = false
+      end
+
+      # -- synchronous --
+
+      # Reads +bytes_to_read+ bytes from stream.
+      # If +out_buffer+ is defined than it will be used as output destination.
+      def read(bytes_to_read = nil, out_buffer = nil)
+        Validation.validate_not_negative_integer bytes_to_read unless bytes_to_read.nil?
+        Validation.validate_string out_buffer unless out_buffer.nil?
+
+        raise ValidateError, "io should be responsible to read and eof" unless
+          @io.respond_to?(:read) && @io.respond_to?(:eof?)
+
+        unless bytes_to_read.nil?
+          return ::String.new :encoding => ::Encoding::BINARY if bytes_to_read.zero?
+          return nil if eof?
+
+          append_io_data @io.read(@source_buffer_length) while @buffer.bytesize < bytes_to_read && !@io.eof?
+          flush_io_data if @buffer.bytesize < bytes_to_read
+
+          return read_bytes_from_buffer bytes_to_read, out_buffer
+        end
+
+        append_io_data @io.read(@source_buffer_length) until @io.eof?
+        flush_io_data
+
+        read_buffer out_buffer
+      end
+
+      # Resets stream.
+      def rewind
+        raw_wrapper :close
+
+        reset_io_remainder
+        reset_need_to_flush
+
+        super
+      end
+
+      # Closes stream.
+      def close
+        raw_wrapper :close
+
+        super
+      end
+
+      # Returns whether we are at the end of stream.
+      def eof?
+        raise ValidateError, "io should be responsible to eof" unless @io.respond_to? :eof?
+
+        empty? && @io.eof?
+      end
+
+      # -- asynchronous --
+
+      # Reads +bytes_to_read+ bytes from stream.
+      # If +out_buffer+ is defined than it will be used as output destination.
+      # Raises +::EOFError+ when no data available.
+      def readpartial(bytes_to_read, out_buffer = nil)
+        raise ValidateError, "io should be responsible to readpartial" unless @io.respond_to? :readpartial
+
+        read_more_nonblock(bytes_to_read, out_buffer) { @io.readpartial @source_buffer_length }
+      end
+
+      # Reads +bytes_to_read+ bytes from stream.
+      # If +out_buffer+ is defined than it will be used as output destination.
+      # +options+ will be passed to native stream.
+      def read_nonblock(bytes_to_read, out_buffer = nil, *options)
+        raise ValidateError, "io should be responsible to read nonblock" unless @io.respond_to? :read_nonblock
+
+        read_more_nonblock(bytes_to_read, out_buffer) { @io.read_nonblock(@source_buffer_length, *options) }
+      end
+
+      # Reads +bytes_to_read+ bytes from stream.
+      # If +out_buffer+ is defined than it will be used as output destination.
+      protected def read_more_nonblock(bytes_to_read, out_buffer, &_block)
+        Validation.validate_not_negative_integer bytes_to_read
+        Validation.validate_string out_buffer unless out_buffer.nil?
+
+        return ::String.new :encoding => ::Encoding::BINARY if bytes_to_read.zero?
+
+        io_provided_eof_error = false
+
+        if @buffer.bytesize < bytes_to_read
+          begin
+            append_io_data yield
+          rescue ::EOFError
+            io_provided_eof_error = true
+          end
+        end
+
+        flush_io_data if @buffer.bytesize < bytes_to_read
+        raise ::EOFError if empty? && io_provided_eof_error
+
+        read_bytes_from_buffer bytes_to_read, out_buffer
+      end
+
+      # -- common --
+
+      # Appends +io_data+ from native stream to internal storage.
+      protected def append_io_data(io_data)
+        io_portion    = @io_remainder + io_data
+        bytes_read    = raw_wrapper :read, io_portion
+        @io_remainder = io_portion.byteslice bytes_read, io_portion.bytesize - bytes_read
+
+        # Even empty io data may require flush.
+        @need_to_flush = true
+      end
+
+      # Triggers flush method for native stream.
+      protected def flush_io_data
+        raw_wrapper :flush
+
+        @need_to_flush = false
+      end
+
+      # Returns whether stream is empty.
+      protected def empty?
+        !@need_to_flush && @buffer.bytesize.zero?
+      end
+
+      # Reads +bytes_to_read+ bytes from buffer.
+      # If +out_buffer+ is defined than it will be used as output destination.
+      protected def read_bytes_from_buffer(bytes_to_read, out_buffer)
+        bytes_read = [@buffer.bytesize, bytes_to_read].min
+
+        # Result uses buffer binary encoding.
+        result   = @buffer.byteslice 0, bytes_read
+        @buffer  = @buffer.byteslice bytes_read, @buffer.bytesize - bytes_read
+        @pos    += bytes_read
+
+        result = out_buffer.replace result unless out_buffer.nil?
+        result
+      end
+
+      # Reads data from buffer.
+      # If +out_buffer+ is defined than it will be used as output destination.
+      protected def read_buffer(out_buffer)
+        result = @buffer
+        reset_buffer
+        @pos += result.bytesize
+
+        result.force_encoding @external_encoding unless @external_encoding.nil?
+        result = transcode_to_internal result
+
+        result = out_buffer.replace result unless out_buffer.nil?
+        result
+      end
+
+      # Transcodes +data+ to internal encoding.
+      protected def transcode_to_internal(data)
+        data = data.encode @internal_encoding, **@transcode_options unless @internal_encoding.nil?
+        data
+      end
+
+      # Transcodes +data+ to external encoding.
+      # We should be able to return data back to buffer.
+      # We won't use any transcode options because transcoded data should be backward compatible.
+      protected def transcode_to_external(data)
+        data = data.encode @external_encoding unless @external_encoding.nil?
+        data
+      end
+
+      # Wraps +method_name+ for raw stream.
+      protected def raw_wrapper(method_name, *args)
+        @raw_stream.send(method_name, *args) { |portion| @buffer << portion }
+      end
+    end
+  end
+end