ruby-zstds 1.0.3 → 1.1.1

data/lib/zstds/file.rb

@@ -19,9 +19,11 @@ module ZSTDS
 
       options[:pledged_size] = ::File.size source
 
-      open_files(source, destination) do |source_io, destination_io|
+      open_files source, destination do |source_io, destination_io|
         ZSTDS._native_compress_io source_io, destination_io, options
       end
+
+      nil
     end
 
     def self.decompress(source, destination, options = {})
@@ -30,9 +32,11 @@ module ZSTDS
 
       options = Option.get_decompressor_options options, BUFFER_LENGTH_NAMES
 
-      open_files(source, destination) do |source_io, destination_io|
+      open_files source, destination do |source_io, destination_io|
         ZSTDS._native_decompress_io source_io, destination_io, options
       end
+
+      nil
     end
 
     private_class_method def self.open_files(source, destination, &_block)

data/lib/zstds/option.rb

@@ -12,6 +12,7 @@ module ZSTDS
     DEFAULT_BUFFER_LENGTH = 0
 
     COMPRESSOR_DEFAULTS = {
+      :gvl                           => false,
       :compression_level             => nil,
       :window_log                    => nil,
       :hash_log                      => nil,
@@ -36,6 +37,7 @@ module ZSTDS
     .freeze
 
     DECOMPRESSOR_DEFAULTS = {
+      :gvl            => false,
       :window_log_max => nil,
       :dictionary     => nil
     }
@@ -44,11 +46,16 @@ module ZSTDS
     def self.get_compressor_options(options, buffer_length_names)
       Validation.validate_hash options
 
-      buffer_length_defaults = buffer_length_names.each_with_object({}) { |name, defaults| defaults[name] = DEFAULT_BUFFER_LENGTH }
-      options                = COMPRESSOR_DEFAULTS.merge(buffer_length_defaults).merge options
+      buffer_length_defaults = buffer_length_names.each_with_object({}) do |name, defaults|
+        defaults[name] = DEFAULT_BUFFER_LENGTH
+      end
+
+      options = COMPRESSOR_DEFAULTS.merge(buffer_length_defaults).merge options
 
       buffer_length_names.each { |name| Validation.validate_not_negative_integer options[name] }
 
+      Validation.validate_bool options[:gvl]
+
       compression_level = options[:compression_level]
       unless compression_level.nil?
         Validation.validate_integer compression_level
@@ -166,9 +173,8 @@ module ZSTDS
       end
 
       dictionary = options[:dictionary]
-      unless dictionary.nil?
-        raise ValidateError, "invalid dictionary" unless dictionary.is_a? Dictionary
-      end
+      raise ValidateError, "invalid dictionary" unless
+        dictionary.nil? || dictionary.is_a?(Dictionary)
 
       options
     end
@@ -176,11 +182,16 @@ module ZSTDS
     def self.get_decompressor_options(options, buffer_length_names)
       Validation.validate_hash options
 
-      buffer_length_defaults = buffer_length_names.each_with_object({}) { |name, defaults| defaults[name] = DEFAULT_BUFFER_LENGTH }
-      options                = DECOMPRESSOR_DEFAULTS.merge(buffer_length_defaults).merge options
+      buffer_length_defaults = buffer_length_names.each_with_object({}) do |name, defaults|
+        defaults[name] = DEFAULT_BUFFER_LENGTH
+      end
+
+      options = DECOMPRESSOR_DEFAULTS.merge(buffer_length_defaults).merge options
 
       buffer_length_names.each { |name| Validation.validate_not_negative_integer options[name] }
 
+      Validation.validate_bool options[:gvl]
+
       window_log_max = options[:window_log_max]
       unless window_log_max.nil?
         Validation.validate_not_negative_integer window_log_max

data/lib/zstds/stream/abstract.rb

@@ -12,17 +12,13 @@ module ZSTDS
       # Native stream is not seekable by design.
       # Related methods like "seek" and "pos=" can't be implemented.
 
-      # It is not possible to maintain correspondance between bytes consumed from source and bytes written to destination by design.
+      # It is not possible to maintain correspondance between bytes
+      # consumed from source and bytes written to destination by design.
       # We will consume all source bytes and maintain buffer with remaining destination data.
 
       include Delegates
 
-      attr_reader :io
-      attr_reader :stat
-      attr_reader :external_encoding
-      attr_reader :internal_encoding
-      attr_reader :transcode_options
-      attr_reader :pos
+      attr_reader :io, :stat, :external_encoding, :internal_encoding, :transcode_options, :pos
 
       alias tell pos
 
@@ -94,8 +90,9 @@ module ZSTDS
         end
 
         internal_encoding = args[1]
-        Validation.validate_string internal_encoding \
-          unless internal_encoding.nil? || internal_encoding.is_a?(::Encoding)
+        unless internal_encoding.nil? || internal_encoding.is_a?(::Encoding)
+          Validation.validate_string internal_encoding
+        end
 
         transcode_options = args[2]
         Validation.validate_hash transcode_options unless transcode_options.nil?

data/lib/zstds/stream/raw/abstract.rb

@@ -19,11 +19,13 @@ module ZSTDS
 
         def flush(&writer)
           write_result(&writer)
+
+          nil
         end
 
-        protected def flush_destination_buffer(&writer)
+        protected def more_destination(&writer)
           result_bytesize = write_result(&writer)
-          raise NotEnoughDestinationError, "not enough destination" if result_bytesize == 0
+          raise NotEnoughDestinationError, "not enough destination" if result_bytesize.zero?
         end
 
         protected def write_result(&_writer)
@@ -44,6 +46,8 @@ module ZSTDS
 
           @native_stream.close
           @is_closed = true
+
+          nil
         end
 
         def closed?

data/lib/zstds/stream/raw/compressor.rb

@@ -39,7 +39,7 @@ module ZSTDS
 
             if need_more_destination
               source = source.byteslice bytes_written, source.bytesize - bytes_written
-              flush_destination_buffer(&writer)
+              more_destination(&writer)
               next
             end
 
@@ -65,7 +65,7 @@ module ZSTDS
             need_more_destination = @native_stream.flush
 
             if need_more_destination
-              flush_destination_buffer(&writer)
+              more_destination(&writer)
               next
             end
 
@@ -73,8 +73,6 @@ module ZSTDS
           end
 
           super
-
-          nil
         end
 
         def close(&writer)
@@ -86,7 +84,7 @@ module ZSTDS
             need_more_destination = @native_stream.finish
 
             if need_more_destination
-              flush_destination_buffer(&writer)
+              more_destination(&writer)
               next
             end
 
@@ -94,8 +92,6 @@ module ZSTDS
           end
 
           super
-
-          nil
         end
       end
     end

data/lib/zstds/stream/raw/decompressor.rb

@@ -34,7 +34,7 @@ module ZSTDS
 
             if need_more_destination
               source = source.byteslice bytes_read, source.bytesize - bytes_read
-              flush_destination_buffer(&writer)
+              more_destination(&writer)
               next
             end
 
@@ -51,8 +51,6 @@ module ZSTDS
           Validation.validate_proc writer
 
           super
-
-          nil
         end
 
         def close(&writer)
@@ -61,8 +59,6 @@ module ZSTDS
           Validation.validate_proc writer
 
           super
-
-          nil
         end
       end
     end

data/lib/zstds/stream/reader.rb

@@ -20,79 +20,61 @@ module ZSTDS
 
         initialize_source_buffer_length
         reset_io_remainder
+        reset_need_to_flush
 
         @lineno = 0
       end
 
+      protected def create_raw_stream
+        Raw::Decompressor.new @options
+      end
+
       protected def initialize_source_buffer_length
         source_buffer_length = @options[:source_buffer_length]
         Validation.validate_not_negative_integer source_buffer_length unless source_buffer_length.nil?
 
-        source_buffer_length = Buffer::DEFAULT_SOURCE_BUFFER_LENGTH_FOR_DECOMPRESSOR \
-          if source_buffer_length == 0 || source_buffer_length.nil?
+        if source_buffer_length.nil? || source_buffer_length.zero?
+          source_buffer_length = Buffer::DEFAULT_SOURCE_BUFFER_LENGTH_FOR_DECOMPRESSOR
+        end
 
         @source_buffer_length = source_buffer_length
       end
 
-      protected def create_raw_stream
-        Raw::Decompressor.new @options
-      end
-
       protected def reset_io_remainder
         @io_remainder = ::String.new :encoding => ::Encoding::BINARY
       end
 
+      protected def reset_need_to_flush
+        @need_to_flush = false
+      end
+
       # -- synchronous --
 
       def read(bytes_to_read = nil, out_buffer = nil)
         Validation.validate_not_negative_integer bytes_to_read unless bytes_to_read.nil?
         Validation.validate_string out_buffer unless out_buffer.nil?
 
-        return ::String.new :encoding => ::Encoding::BINARY if bytes_to_read == 0
-
         unless bytes_to_read.nil?
+          return ::String.new :encoding => ::Encoding::BINARY if bytes_to_read.zero?
           return nil if eof?
 
-          read_more_to_buffer until @buffer.bytesize >= bytes_to_read || @io.eof?
+          append_io_data @io.read(@source_buffer_length) while @buffer.bytesize < bytes_to_read && !@io.eof?
+          flush_io_data if @buffer.bytesize < bytes_to_read
 
           return read_bytes_from_buffer bytes_to_read, out_buffer
         end
 
-        read_more_to_buffer until @io.eof?
-
-        result = @buffer
-        reset_buffer
-        @pos += result.bytesize
-
-        result.force_encoding @external_encoding unless @external_encoding.nil?
-        result = transcode_to_internal result
-        result = out_buffer.replace result unless out_buffer.nil?
-
-        result
-      end
-
-      protected def read_more_to_buffer
-        io_data = @io.read @source_buffer_length
-        append_io_data_to_buffer io_data
-      end
-
-      def readpartial(bytes_to_read = nil, out_buffer = nil)
-        raise ::EOFError if eof?
-
-        readpartial_to_buffer until @buffer.bytesize >= bytes_to_read || @io.eof?
-
-        read_bytes_from_buffer bytes_to_read, out_buffer
-      end
+        append_io_data @io.read(@source_buffer_length) until @io.eof?
+        flush_io_data
 
-      protected def readpartial_to_buffer
-        io_data = @io.readpartial @source_buffer_length
-        append_io_data_to_buffer io_data
+        read_buffer out_buffer
       end
 
       def rewind
         raw_wrapper :close
 
         reset_io_remainder
+        reset_need_to_flush
 
         super
       end
@@ -103,25 +85,61 @@ module ZSTDS
         super
       end
 
+      def eof?
+        empty? && @io.eof?
+      end
+
       # -- asynchronous --
 
+      def readpartial(bytes_to_read, out_buffer = nil)
+        read_more_nonblock(bytes_to_read, out_buffer) { @io.readpartial @source_buffer_length }
+      end
+
       def read_nonblock(bytes_to_read, out_buffer = nil, *options)
-        raise ::EOFError if eof?
+        read_more_nonblock(bytes_to_read, out_buffer) { @io.read_nonblock(@source_buffer_length, *options) }
+      end
+
+      protected def read_more_nonblock(bytes_to_read, out_buffer, &_block)
+        Validation.validate_not_negative_integer bytes_to_read
+        Validation.validate_string out_buffer unless out_buffer.nil?
+
+        return ::String.new :encoding => ::Encoding::BINARY if bytes_to_read.zero?
+
+        io_provided_eof_error = false
 
-        read_more_to_buffer_nonblock(*options) until @buffer.bytesize >= bytes_to_read || @io.eof?
+        if @buffer.bytesize < bytes_to_read
+          begin
+            append_io_data yield
+          rescue ::EOFError
+            io_provided_eof_error = true
+          end
+        end
+
+        flush_io_data if @buffer.bytesize < bytes_to_read
+        raise ::EOFError if empty? && io_provided_eof_error
 
         read_bytes_from_buffer bytes_to_read, out_buffer
       end
 
-      protected def read_more_to_buffer_nonblock(*options)
-        io_data = @io.read_nonblock @source_buffer_length, *options
-        append_io_data_to_buffer io_data
+      # -- common --
+
+      protected def append_io_data(io_data)
+        io_portion    = @io_remainder + io_data
+        bytes_read    = raw_wrapper :read, io_portion
+        @io_remainder = io_portion.byteslice bytes_read, io_portion.bytesize - bytes_read
+
+        # Even empty io data may require flush.
+        @need_to_flush = true
       end
 
-      # -- common --
+      protected def flush_io_data
+        raw_wrapper :flush
 
-      def eof?
-        @io.eof? && @buffer.bytesize == 0
+        @need_to_flush = false
+      end
+
+      protected def empty?
+        !@need_to_flush && @buffer.bytesize.zero?
       end
 
       protected def read_bytes_from_buffer(bytes_to_read, out_buffer)
@@ -136,14 +154,16 @@ module ZSTDS
         result
       end
 
-      protected def append_io_data_to_buffer(io_data)
-        io_portion    = @io_remainder + io_data
-        bytes_read    = raw_wrapper :read, io_portion
-        @io_remainder = io_portion.byteslice bytes_read, io_portion.bytesize - bytes_read
+      protected def read_buffer(out_buffer)
+        result = @buffer
+        reset_buffer
+        @pos += result.bytesize
 
-        # We should just ignore case when "io.eof?" appears but "io_remainder" is not empty.
-        # Ancient compress implementations can write bytes from not initialized buffer parts to output.
-        raw_wrapper :flush if @io.eof?
+        result.force_encoding @external_encoding unless @external_encoding.nil?
+        result = transcode_to_internal result
+
+        result = out_buffer.replace result unless out_buffer.nil?
+        result
       end
 
       protected def transcode_to_internal(data)

data/lib/zstds/stream/reader_helpers.rb

@@ -143,7 +143,7 @@ module ZSTDS
       end
 
       protected def each_string(each_proc, &block)
-        return enum_for __method__ unless block.is_a? ::Proc
+        return enum_for __method__, each_proc unless block.is_a? ::Proc
 
         loop do
           string = each_proc.call

data/lib/zstds/stream/writer.rb

@@ -64,7 +64,7 @@ module ZSTDS
       end
 
       protected def write_remaining_buffer
-        return nil if @buffer.bytesize == 0
+        return nil if @buffer.bytesize.zero?
 
         @io.write @buffer
 
@@ -77,6 +77,11 @@ module ZSTDS
 
       # -- asynchronous --
 
+      # IO write nonblock can raise wait writable error.
+      # After resolving this error user may provide same content again.
+      # It is not possible to revert accepted content after error.
+      # So we have to accept content after processing IO write nonblock.
+      # It means that first write nonblock won't call IO write nonblock.
       def write_nonblock(object, *options)
         return 0 unless write_remaining_buffer_nonblock(*options)
 
@@ -120,14 +125,14 @@ module ZSTDS
       end
 
       protected def write_remaining_buffer_nonblock(*options)
-        return true if @buffer.bytesize == 0
+        return true if @buffer.bytesize.zero?
 
         bytes_written = @io.write_nonblock @buffer, *options
-        return false if bytes_written == 0
+        return false if bytes_written.zero?
 
         @buffer = @buffer.byteslice bytes_written, @buffer.bytesize - bytes_written
 
-        @buffer.bytesize == 0
+        @buffer.bytesize.zero?
       end
 
       protected def raw_nonblock_wrapper(method_name, *args)