ruby-zstds 1.0.0

data/lib/zstds/stream/abstract.rb

@@ -0,0 +1,153 @@
+# Ruby bindings for zstd library.
+# Copyright (c) 2019 AUTHORS, MIT License.
+
+require_relative "delegates"
+require_relative "stat"
+require_relative "../error"
+require_relative "../validation"
+
+module ZSTDS
+  module Stream
+    class Abstract
+      # Native stream is not seekable by design.
+      # Related methods like "seek" and "pos=" can't be implemented.
+
+      # It is not possible to maintain correspondance between bytes consumed from source and bytes written to destination by design.
+      # We will consume all source bytes and maintain buffer with remaining destination data.
+
+      include Delegates
+
+      attr_reader :io
+      attr_reader :stat
+      attr_reader :external_encoding
+      attr_reader :internal_encoding
+      attr_reader :transcode_options
+      attr_reader :pos
+      alias tell pos
+
+      def initialize(io, external_encoding: nil, internal_encoding: nil, transcode_options: {})
+        @raw_stream = create_raw_stream
+
+        Validation.validate_io io
+        @io = io
+
+        @stat = Stat.new @io.stat
+
+        set_encoding external_encoding, internal_encoding, transcode_options
+        reset_buffer
+        reset_io_advise
+
+        @pos = 0
+      end
+
+      # -- buffer --
+
+      protected def reset_buffer
+        @buffer = ::String.new :encoding => ::Encoding::BINARY
+      end
+
+      # -- advise --
+
+      protected def reset_io_advise
+        # Both compressor and decompressor need sequential io access.
+        @io.advise :sequential
+      rescue ::Errno::ESPIPE # rubocop:disable Lint/HandleExceptions
+        # ok
+      end
+
+      def advise
+        # Noop
+        nil
+      end
+
+      # -- encoding --
+
+      def set_encoding(*args)
+        external_encoding, internal_encoding, transcode_options = process_set_encoding_arguments(*args)
+
+        set_target_encoding :@external_encoding, external_encoding
+        set_target_encoding :@internal_encoding, internal_encoding
+        @transcode_options = transcode_options
+
+        self
+      end
+
+      protected def process_set_encoding_arguments(*args)
+        external_encoding = args[0]
+
+        unless external_encoding.nil? || external_encoding.is_a?(::Encoding)
+          Validation.validate_string external_encoding
+
+          # First argument can be "external_encoding:internal_encoding".
+          match = %r{(.+?):(.+)}.match external_encoding
+
+          unless match.nil?
+            external_encoding = match[0]
+            internal_encoding = match[1]
+
+            transcode_options = args[1]
+            Validation.validate_hash transcode_options unless transcode_options.nil?
+
+            return [external_encoding, internal_encoding, transcode_options]
+          end
+        end
+
+        internal_encoding = args[1]
+        Validation.validate_string internal_encoding \
+          unless internal_encoding.nil? || internal_encoding.is_a?(::Encoding)
+
+        transcode_options = args[2]
+        Validation.validate_hash transcode_options unless transcode_options.nil?
+
+        [external_encoding, internal_encoding, transcode_options]
+      end
+
+      protected def set_target_encoding(name, value)
+        unless value.nil? || value.is_a?(::Encoding)
+          begin
+            value = ::Encoding.find value
+          rescue ::ArgumentError
+            raise ValidateError, "invalid #{name} encoding"
+          end
+        end
+
+        instance_variable_set name, value
+      end
+
+      protected def target_encoding
+        return @internal_encoding unless @internal_encoding.nil?
+        return @external_encoding unless @external_encoding.nil?
+
+        ::Encoding::BINARY
+      end
+
+      # -- etc --
+
+      def rewind
+        @raw_stream = create_raw_stream
+
+        @io.rewind
+        reset_buffer
+        reset_io_advise
+
+        @pos = 0
+
+        0
+      end
+
+      def close
+        @io.close
+
+        nil
+      end
+
+      def closed?
+        @raw_stream.closed? && @io.closed?
+      end
+
+      def to_io
+        self
+      end
+    end
+  end
+end

data/lib/zstds/stream/delegates.rb

@@ -0,0 +1,36 @@
+# Ruby bindings for zstd library.
+# Copyright (c) 2019 AUTHORS, MIT License.
+
+require "forwardable"
+
+module ZSTDS
+  module Stream
+    module Delegates
+      DELEGATES = %i[
+        autoclose=
+        autoclose?
+        binmode
+        binmode?
+        close_on_exec=
+        close_on_exec?
+        fcntl
+        fdatasync
+        fileno
+        fsync
+        ioctl
+        isatty
+        pid
+        sync
+        sync=
+        to_i
+        tty?
+      ]
+      .freeze
+
+      def self.included(klass)
+        klass.extend ::Forwardable
+        klass.def_delegators :@io, *DELEGATES
+      end
+    end
+  end
+end

data/lib/zstds/stream/raw/abstract.rb

@@ -0,0 +1,55 @@
+# Ruby bindings for zstd library.
+# Copyright (c) 2019 AUTHORS, MIT License.
+
+require "zstds_ext"
+
+require_relative "../../error"
+require_relative "../../validation"
+
+module ZSTDS
+  module Stream
+    module Raw
+      class Abstract
+        def initialize(native_stream)
+          @native_stream = native_stream
+          @is_closed     = false
+        end
+
+        # -- write --
+
+        def flush(&writer)
+          write_result(&writer)
+        end
+
+        protected def flush_destination_buffer(&writer)
+          result_bytesize = write_result(&writer)
+          raise NotEnoughDestinationError, "not enough destination" if result_bytesize == 0
+        end
+
+        protected def write_result(&_writer)
+          result = @native_stream.read_result
+          yield result
+
+          result.bytesize
+        end
+
+        # -- close --
+
+        protected def do_not_use_after_close
+          raise UsedAfterCloseError, "used after close" if closed?
+        end
+
+        def close(&writer)
+          write_result(&writer)
+
+          @native_stream.close
+          @is_closed = true
+        end
+
+        def closed?
+          @is_closed
+        end
+      end
+    end
+  end
+end

data/lib/zstds/stream/raw/compressor.rb

@@ -0,0 +1,101 @@
+# Ruby bindings for zstd library.
+# Copyright (c) 2019 AUTHORS, MIT License.
+
+require "zstds_ext"
+
+require_relative "abstract"
+require_relative "../../error"
+require_relative "../../option"
+require_relative "../../validation"
+
+module ZSTDS
+  module Stream
+    module Raw
+      class Compressor < Abstract
+        BUFFER_LENGTH_NAMES = %i[destination_buffer_length].freeze
+
+        def initialize(options = {})
+          options = Option.get_compressor_options options, BUFFER_LENGTH_NAMES
+
+          pledged_size = options[:pledged_size]
+          Validation.validate_not_negative_integer pledged_size unless pledged_size.nil?
+
+          native_stream = NativeCompressor.new options
+
+          super native_stream
+        end
+
+        def write(source, &writer)
+          do_not_use_after_close
+
+          Validation.validate_string source
+          Validation.validate_proc writer
+
+          total_bytes_written = 0
+
+          loop do
+            bytes_written, need_more_destination  = @native_stream.write source
+            total_bytes_written                  += bytes_written
+
+            if need_more_destination
+              source = source.byteslice bytes_written, source.bytesize - bytes_written
+              flush_destination_buffer(&writer)
+              next
+            end
+
+            unless bytes_written == source.bytesize
+              # Compressor write should eat all provided "source" without remainder.
+              raise UnexpectedError, "unexpected error"
+            end
+
+            break
+          end
+
+          total_bytes_written
+        end
+
+        def flush(&writer)
+          do_not_use_after_close
+
+          Validation.validate_proc writer
+
+          loop do
+            need_more_destination = @native_stream.flush
+
+            if need_more_destination
+              flush_destination_buffer(&writer)
+              next
+            end
+
+            break
+          end
+
+          super
+
+          nil
+        end
+
+        def close(&writer)
+          return nil if closed?
+
+          Validation.validate_proc writer
+
+          loop do
+            need_more_destination = @native_stream.finish
+
+            if need_more_destination
+              flush_destination_buffer(&writer)
+              next
+            end
+
+            break
+          end
+
+          super
+
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/zstds/stream/raw/decompressor.rb

@@ -0,0 +1,70 @@
+# Ruby bindings for zstd library.
+# Copyright (c) 2019 AUTHORS, MIT License.
+
+require "zstds_ext"
+
+require_relative "abstract"
+require_relative "../../option"
+require_relative "../../validation"
+
+module ZSTDS
+  module Stream
+    module Raw
+      class Decompressor < Abstract
+        BUFFER_LENGTH_NAMES = %i[destination_buffer_length].freeze
+
+        def initialize(options = {})
+          options       = Option.get_decompressor_options options, BUFFER_LENGTH_NAMES
+          native_stream = NativeDecompressor.new options
+
+          super native_stream
+        end
+
+        def read(source, &writer)
+          do_not_use_after_close
+
+          Validation.validate_string source
+          Validation.validate_proc writer
+
+          total_bytes_read = 0
+
+          loop do
+            bytes_read, need_more_destination  = @native_stream.read source
+            total_bytes_read                  += bytes_read
+
+            if need_more_destination
+              source = source.byteslice bytes_read, source.bytesize - bytes_read
+              flush_destination_buffer(&writer)
+              next
+            end
+
+            break
+          end
+
+          # Please remember that "total_bytes_read" can not be equal to "source.bytesize".
+          total_bytes_read
+        end
+
+        def flush(&writer)
+          do_not_use_after_close
+
+          Validation.validate_proc writer
+
+          super
+
+          nil
+        end
+
+        def close(&writer)
+          return nil if closed?
+
+          Validation.validate_proc writer
+
+          super
+
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/zstds/stream/reader.rb

@@ -0,0 +1,166 @@
+# Ruby bindings for zstd library.
+# Copyright (c) 2019 AUTHORS, MIT License.
+
+require_relative "abstract"
+require_relative "raw/decompressor"
+require_relative "reader_helpers"
+require_relative "../validation"
+
+module ZSTDS
+  module Stream
+    class Reader < Abstract
+      include ReaderHelpers
+
+      attr_accessor :lineno
+
+      def initialize(source_io, options = {}, *args)
+        @options = options
+
+        super source_io, *args
+
+        initialize_source_buffer_length
+        reset_io_remainder
+
+        @lineno = 0
+      end
+
+      protected def initialize_source_buffer_length
+        source_buffer_length = @options[:source_buffer_length]
+        Validation.validate_not_negative_integer source_buffer_length unless source_buffer_length.nil?
+
+        source_buffer_length = Buffer::DEFAULT_SOURCE_BUFFER_LENGTH_FOR_DECOMPRESSOR \
+          if source_buffer_length == 0 || source_buffer_length.nil?
+
+        @source_buffer_length = source_buffer_length
+      end
+
+      protected def create_raw_stream
+        Raw::Decompressor.new @options
+      end
+
+      protected def reset_io_remainder
+        @io_remainder = ::String.new :encoding => ::Encoding::BINARY
+      end
+
+      # -- synchronous --
+
+      def read(bytes_to_read = nil, out_buffer = nil)
+        Validation.validate_not_negative_integer bytes_to_read unless bytes_to_read.nil?
+        Validation.validate_string out_buffer unless out_buffer.nil?
+
+        return ::String.new :encoding => ::Encoding::BINARY if bytes_to_read == 0
+
+        unless bytes_to_read.nil?
+          return nil if eof?
+
+          read_more_from_buffer until @buffer.bytesize >= bytes_to_read || @io.eof?
+
+          return read_bytes_from_buffer bytes_to_read, out_buffer
+        end
+
+        read_more_from_buffer until @io.eof?
+
+        result = @buffer
+        reset_buffer
+        @pos += result.bytesize
+
+        result.force_encoding @external_encoding unless @external_encoding.nil?
+        result = transcode_to_internal result
+        result = out_buffer.replace result unless out_buffer.nil?
+
+        result
+      end
+
+      protected def read_more_from_buffer
+        io_data = @io.read @source_buffer_length
+        append_io_data_to_buffer io_data
+      end
+
+      def readpartial(bytes_to_read = nil, out_buffer = nil)
+        raise ::EOFError if eof?
+
+        readpartial_from_buffer until @buffer.bytesize >= bytes_to_read || @io.eof?
+
+        read_bytes_from_buffer bytes_to_read, out_buffer
+      end
+
+      protected def readpartial_from_buffer
+        io_data = @io.readpartial @source_buffer_length
+        append_io_data_to_buffer io_data
+      end
+
+      def rewind
+        raw_wrapper :close
+
+        reset_io_remainder
+
+        super
+      end
+
+      def close
+        raw_wrapper :close
+
+        super
+      end
+
+      # -- asynchronous --
+
+      def read_nonblock(bytes_to_read, out_buffer = nil, *options)
+        raise ::EOFError if eof?
+
+        read_more_from_buffer_nonblock(*options) until @buffer.bytesize >= bytes_to_read || @io.eof?
+
+        read_bytes_from_buffer bytes_to_read, out_buffer
+      end
+
+      protected def read_more_from_buffer_nonblock(*options)
+        io_data = @io.read_nonblock @source_buffer_length, *options
+        append_io_data_to_buffer io_data
+      end
+
+      # -- common --
+
+      def eof?
+        @io.eof? && @buffer.bytesize == 0
+      end
+
+      protected def read_bytes_from_buffer(bytes_to_read, out_buffer)
+        bytes_read = [@buffer.bytesize, bytes_to_read].min
+
+        # Result uses buffer binary encoding.
+        result   = @buffer.byteslice 0, bytes_read
+        @buffer  = @buffer.byteslice bytes_read, @buffer.bytesize - bytes_read
+        @pos    += bytes_read
+
+        result = out_buffer.replace result unless out_buffer.nil?
+        result
+      end
+
+      protected def append_io_data_to_buffer(io_data)
+        io_portion    = @io_remainder + io_data
+        bytes_read    = raw_wrapper :read, io_portion
+        @io_remainder = io_portion.byteslice bytes_read, io_portion.bytesize - bytes_read
+
+        # We should just ignore case when "io.eof?" appears but "io_remainder" is not empty.
+        # Ancient compress implementations can write bytes from not initialized buffer parts to output.
+        raw_wrapper :flush if @io.eof?
+      end
+
+      protected def transcode_to_internal(data)
+        data = data.encode @internal_encoding, @transcode_options unless @internal_encoding.nil?
+        data
+      end
+
+      # We should be able to return data back to buffer.
+      # We won't use any transcode options because transcoded data should be backward compatible.
+      protected def transcode_to_external(data)
+        data = data.encode @external_encoding unless @external_encoding.nil?
+        data
+      end
+
+      protected def raw_wrapper(method_name, *args)
+        @raw_stream.send(method_name, *args) { |portion| @buffer << portion }
+      end
+    end
+  end
+end