ruby-zstds 1.0.1 → 1.0.6

data/ext/zstds_ext/main.c

@@ -3,7 +3,6 @@
 
 #include "ruby.h"
 #include "zstds_ext/buffer.h"
-#include "zstds_ext/common.h"
 #include "zstds_ext/dictionary.h"
 #include "zstds_ext/io.h"
 #include "zstds_ext/option.h"

data/ext/zstds_ext/option.c

@@ -3,13 +3,9 @@
 
 #include "zstds_ext/option.h"
 
-#include <stdbool.h>
-#include <stdint.h>
-#include <stdlib.h>
 #include <zstd.h>
 
 #include "ruby.h"
-#include "zstds_ext/common.h"
 #include "zstds_ext/dictionary.h"
 #include "zstds_ext/error.h"
 

data/ext/zstds_ext/option.h

@@ -5,8 +5,6 @@
 #define ZSTDS_EXT_OPTIONS_H
 
 #include <stdbool.h>
-#include <stdint.h>
-#include <stdlib.h>
 #include <zstd.h>
 
 #include "ruby.h"
@@ -22,9 +20,9 @@ enum {
   ZSTDS_EXT_OPTION_TYPE_STRATEGY
 };
 
-typedef uint_fast8_t       zstds_ext_option_type_t;
-typedef int                zstds_ext_option_value_t;
-typedef unsigned long long zstds_ext_ull_option_value_t;
+typedef zstds_ext_byte_fast_t zstds_ext_option_type_t;
+typedef int                   zstds_ext_option_value_t;
+typedef unsigned long long    zstds_ext_ull_option_value_t;
 
 typedef struct {
   bool                     has_value;

data/ext/zstds_ext/stream/compressor.c

@@ -3,12 +3,9 @@
 
 #include "zstds_ext/stream/compressor.h"
 
-#include <stdint.h>
-#include <stdlib.h>
 #include <zstd.h>
 
 #include "ruby.h"
-#include "zstds_ext/common.h"
 #include "zstds_ext/error.h"
 #include "zstds_ext/option.h"
 
@@ -19,7 +16,7 @@ static void free_compressor(zstds_ext_compressor_t* compressor_ptr)
     ZSTD_freeCCtx(ctx);
   }
 
-  uint8_t* destination_buffer = compressor_ptr->destination_buffer;
+  zstds_ext_byte_t* destination_buffer = compressor_ptr->destination_buffer;
   if (destination_buffer != NULL) {
     free(destination_buffer);
   }
@@ -68,7 +65,7 @@ VALUE zstds_ext_initialize_compressor(VALUE self, VALUE options)
     destination_buffer_length = ZSTD_CStreamOutSize();
   }
 
-  uint8_t* destination_buffer = malloc(destination_buffer_length);
+  zstds_ext_byte_t* destination_buffer = malloc(destination_buffer_length);
   if (destination_buffer == NULL) {
     ZSTD_freeCCtx(ctx);
     zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
@@ -185,9 +182,9 @@ VALUE zstds_ext_compressor_read_result(VALUE self)
   GET_COMPRESSOR(self);
   DO_NOT_USE_AFTER_CLOSE(compressor_ptr);
 
-  uint8_t* destination_buffer                  = compressor_ptr->destination_buffer;
-  size_t   destination_buffer_length           = compressor_ptr->destination_buffer_length;
-  size_t   remaining_destination_buffer_length = compressor_ptr->remaining_destination_buffer_length;
+  zstds_ext_byte_t* destination_buffer                  = compressor_ptr->destination_buffer;
+  size_t            destination_buffer_length           = compressor_ptr->destination_buffer_length;
+  size_t            remaining_destination_buffer_length = compressor_ptr->remaining_destination_buffer_length;
 
   const char* result        = (const char*)destination_buffer;
   size_t      result_length = destination_buffer_length - remaining_destination_buffer_length;
@@ -212,7 +209,7 @@ VALUE zstds_ext_compressor_close(VALUE self)
     compressor_ptr->ctx = NULL;
   }
 
-  uint8_t* destination_buffer = compressor_ptr->destination_buffer;
+  zstds_ext_byte_t* destination_buffer = compressor_ptr->destination_buffer;
   if (destination_buffer != NULL) {
     free(destination_buffer);
 

data/ext/zstds_ext/stream/compressor.h

@@ -4,18 +4,17 @@
 #if !defined(ZSTDS_EXT_STREAM_COMPRESSOR_H)
 #define ZSTDS_EXT_STREAM_COMPRESSOR_H
 
-#include <stdint.h>
-#include <stdlib.h>
 #include <zstd.h>
 
 #include "ruby.h"
+#include "zstds_ext/common.h"
 
 typedef struct {
-  ZSTD_CCtx* ctx;
-  uint8_t*   destination_buffer;
-  size_t     destination_buffer_length;
-  uint8_t*   remaining_destination_buffer;
-  size_t     remaining_destination_buffer_length;
+  ZSTD_CCtx*        ctx;
+  zstds_ext_byte_t* destination_buffer;
+  size_t            destination_buffer_length;
+  zstds_ext_byte_t* remaining_destination_buffer;
+  size_t            remaining_destination_buffer_length;
 } zstds_ext_compressor_t;
 
 VALUE zstds_ext_allocate_compressor(VALUE klass);

data/ext/zstds_ext/stream/decompressor.c

@@ -3,12 +3,9 @@
 
 #include "zstds_ext/stream/decompressor.h"
 
-#include <stdint.h>
-#include <stdlib.h>
 #include <zstd.h>
 
 #include "ruby.h"
-#include "zstds_ext/common.h"
 #include "zstds_ext/error.h"
 #include "zstds_ext/option.h"
 
@@ -19,7 +16,7 @@ static void free_decompressor(zstds_ext_decompressor_t* decompressor_ptr)
     ZSTD_freeDCtx(ctx);
   }
 
-  uint8_t* destination_buffer = decompressor_ptr->destination_buffer;
+  zstds_ext_byte_t* destination_buffer = decompressor_ptr->destination_buffer;
   if (destination_buffer != NULL) {
     free(destination_buffer);
   }
@@ -68,7 +65,7 @@ VALUE zstds_ext_initialize_decompressor(VALUE self, VALUE options)
     destination_buffer_length = ZSTD_DStreamOutSize();
   }
 
-  uint8_t* destination_buffer = malloc(destination_buffer_length);
+  zstds_ext_byte_t* destination_buffer = malloc(destination_buffer_length);
   if (destination_buffer == NULL) {
     ZSTD_freeDCtx(ctx);
     zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
@@ -129,9 +126,9 @@ VALUE zstds_ext_decompressor_read_result(VALUE self)
   GET_DECOMPRESSOR(self);
   DO_NOT_USE_AFTER_CLOSE(decompressor_ptr);
 
-  uint8_t* destination_buffer                  = decompressor_ptr->destination_buffer;
-  size_t   destination_buffer_length           = decompressor_ptr->destination_buffer_length;
-  size_t   remaining_destination_buffer_length = decompressor_ptr->remaining_destination_buffer_length;
+  zstds_ext_byte_t* destination_buffer                  = decompressor_ptr->destination_buffer;
+  size_t            destination_buffer_length           = decompressor_ptr->destination_buffer_length;
+  size_t            remaining_destination_buffer_length = decompressor_ptr->remaining_destination_buffer_length;
 
   const char* result        = (const char*)destination_buffer;
   size_t      result_length = destination_buffer_length - remaining_destination_buffer_length;
@@ -156,7 +153,7 @@ VALUE zstds_ext_decompressor_close(VALUE self)
     decompressor_ptr->ctx = NULL;
   }
 
-  uint8_t* destination_buffer = decompressor_ptr->destination_buffer;
+  zstds_ext_byte_t* destination_buffer = decompressor_ptr->destination_buffer;
   if (destination_buffer != NULL) {
     free(destination_buffer);
 

data/ext/zstds_ext/stream/decompressor.h

@@ -4,18 +4,17 @@
 #if !defined(ZSTDS_EXT_STREAM_DECOMPRESSOR_H)
 #define ZSTDS_EXT_STREAM_DECOMPRESSOR_H
 
-#include <stdint.h>
-#include <stdlib.h>
 #include <zstd.h>
 
 #include "ruby.h"
+#include "zstds_ext/common.h"
 
 typedef struct {
-  ZSTD_DCtx* ctx;
-  uint8_t*   destination_buffer;
-  size_t     destination_buffer_length;
-  uint8_t*   remaining_destination_buffer;
-  size_t     remaining_destination_buffer_length;
+  ZSTD_DCtx*        ctx;
+  zstds_ext_byte_t* destination_buffer;
+  size_t            destination_buffer_length;
+  zstds_ext_byte_t* remaining_destination_buffer;
+  size_t            remaining_destination_buffer_length;
 } zstds_ext_decompressor_t;
 
 VALUE zstds_ext_allocate_decompressor(VALUE klass);

data/ext/zstds_ext/string.c

@@ -3,13 +3,10 @@
 
 #include "zstds_ext/string.h"
 
-#include <stdint.h>
-#include <stdlib.h>
 #include <zstd.h>
 
 #include "ruby.h"
 #include "zstds_ext/buffer.h"
-#include "zstds_ext/common.h"
 #include "zstds_ext/error.h"
 #include "zstds_ext/macro.h"
 #include "zstds_ext/option.h"
@@ -65,7 +62,7 @@ static inline zstds_ext_result_t compress(
   size_t         remaining_destination_buffer_length = destination_buffer_length;
 
   while (true) {
-    out_buffer.dst  = (uint8_t*)RSTRING_PTR(destination_value) + destination_length;
+    out_buffer.dst  = (zstds_ext_byte_t*)RSTRING_PTR(destination_value) + destination_length;
     out_buffer.size = remaining_destination_buffer_length;
     out_buffer.pos  = 0;
 
@@ -166,7 +163,7 @@ static inline zstds_ext_result_t decompress(
   size_t         remaining_destination_buffer_length = destination_buffer_length;
 
   while (true) {
-    out_buffer.dst  = (uint8_t*)RSTRING_PTR(destination_value) + destination_length;
+    out_buffer.dst  = (zstds_ext_byte_t*)RSTRING_PTR(destination_value) + destination_length;
     out_buffer.size = remaining_destination_buffer_length;
     out_buffer.pos  = 0;
 

data/lib/zstds/file.rb

@@ -22,6 +22,8 @@ module ZSTDS
       open_files(source, destination) do |source_io, destination_io|
         ZSTDS._native_compress_io source_io, destination_io, options
       end
+
+      nil
     end
 
     def self.decompress(source, destination, options = {})
@@ -33,6 +35,8 @@ module ZSTDS
       open_files(source, destination) do |source_io, destination_io|
         ZSTDS._native_decompress_io source_io, destination_io, options
       end
+
+      nil
     end
 
     private_class_method def self.open_files(source, destination, &_block)

data/lib/zstds/option.rb

@@ -166,9 +166,8 @@ module ZSTDS
       end
 
       dictionary = options[:dictionary]
-      unless dictionary.nil?
-        raise ValidateError, "invalid dictionary" unless dictionary.is_a? Dictionary
-      end
+      raise ValidateError, "invalid dictionary" unless
+        dictionary.nil? || dictionary.is_a?(Dictionary)
 
       options
     end

data/lib/zstds/stream/abstract.rb

@@ -17,23 +17,19 @@ module ZSTDS
 
       include Delegates
 
-      attr_reader :io
-      attr_reader :stat
-      attr_reader :external_encoding
-      attr_reader :internal_encoding
-      attr_reader :transcode_options
-      attr_reader :pos
+      attr_reader :io, :stat, :external_encoding, :internal_encoding, :transcode_options, :pos
+
       alias tell pos
 
-      def initialize(io, external_encoding: nil, internal_encoding: nil, transcode_options: {})
+      def initialize(io, options = {})
         @raw_stream = create_raw_stream
 
         Validation.validate_io io
         @io = io
 
-        @stat = Stat.new @io.stat
+        @stat = Stat.new @io.stat if @io.respond_to? :stat
 
-        set_encoding external_encoding, internal_encoding, transcode_options
+        set_encoding options[:external_encoding], options[:internal_encoding], options[:transcode_options]
         reset_buffer
         reset_io_advise
 
@@ -50,8 +46,8 @@ module ZSTDS
 
       protected def reset_io_advise
         # Both compressor and decompressor need sequential io access.
-        @io.advise :sequential
-      rescue ::Errno::ESPIPE # rubocop:disable Lint/HandleExceptions
+        @io.advise :sequential if @io.respond_to? :advise
+      rescue ::Errno::ESPIPE
         # ok
       end
 
@@ -126,7 +122,8 @@ module ZSTDS
       def rewind
         @raw_stream = create_raw_stream
 
-        @io.rewind
+        @io.rewind if @io.respond_to? :rewind
+
         reset_buffer
         reset_io_advise
 

data/lib/zstds/stream/raw/abstract.rb

@@ -19,11 +19,13 @@ module ZSTDS
 
         def flush(&writer)
           write_result(&writer)
+
+          nil
         end
 
-        protected def flush_destination_buffer(&writer)
+        protected def more_destination(&writer)
           result_bytesize = write_result(&writer)
-          raise NotEnoughDestinationError, "not enough destination" if result_bytesize == 0
+          raise NotEnoughDestinationError, "not enough destination" if result_bytesize.zero?
         end
 
         protected def write_result(&_writer)
@@ -44,6 +46,8 @@ module ZSTDS
 
           @native_stream.close
           @is_closed = true
+
+          nil
         end
 
         def closed?

data/lib/zstds/stream/raw/compressor.rb

@@ -39,13 +39,15 @@ module ZSTDS
 
             if need_more_destination
               source = source.byteslice bytes_written, source.bytesize - bytes_written
-              flush_destination_buffer(&writer)
+              more_destination(&writer)
               next
             end
 
             unless bytes_written == source.bytesize
+              # :nocov:
               # Compressor write should eat all provided "source" without remainder.
               raise UnexpectedError, "unexpected error"
+              # :nocov:
             end
 
             break
@@ -63,7 +65,7 @@ module ZSTDS
             need_more_destination = @native_stream.flush
 
             if need_more_destination
-              flush_destination_buffer(&writer)
+              more_destination(&writer)
               next
             end
 
@@ -71,8 +73,6 @@ module ZSTDS
           end
 
           super
-
-          nil
         end
 
         def close(&writer)
@@ -84,7 +84,7 @@ module ZSTDS
             need_more_destination = @native_stream.finish
 
             if need_more_destination
-              flush_destination_buffer(&writer)
+              more_destination(&writer)
               next
             end
 
@@ -92,8 +92,6 @@ module ZSTDS
           end
 
           super
-
-          nil
         end
       end
     end

data/lib/zstds/stream/raw/decompressor.rb

@@ -34,7 +34,7 @@ module ZSTDS
 
             if need_more_destination
               source = source.byteslice bytes_read, source.bytesize - bytes_read
-              flush_destination_buffer(&writer)
+              more_destination(&writer)
               next
             end
 
@@ -51,8 +51,6 @@ module ZSTDS
           Validation.validate_proc writer
 
           super
-
-          nil
         end
 
         def close(&writer)
@@ -61,8 +59,6 @@ module ZSTDS
           Validation.validate_proc writer
 
           super
-
-          nil
         end
       end
     end

data/lib/zstds/stream/reader.rb

@@ -20,79 +20,60 @@ module ZSTDS
 
         initialize_source_buffer_length
         reset_io_remainder
+        reset_need_to_flush
 
         @lineno = 0
       end
 
+      protected def create_raw_stream
+        Raw::Decompressor.new @options
+      end
+
       protected def initialize_source_buffer_length
         source_buffer_length = @options[:source_buffer_length]
         Validation.validate_not_negative_integer source_buffer_length unless source_buffer_length.nil?
 
         source_buffer_length = Buffer::DEFAULT_SOURCE_BUFFER_LENGTH_FOR_DECOMPRESSOR \
-          if source_buffer_length == 0 || source_buffer_length.nil?
+          if source_buffer_length.nil? || source_buffer_length.zero?
 
         @source_buffer_length = source_buffer_length
       end
 
-      protected def create_raw_stream
-        Raw::Decompressor.new @options
-      end
-
       protected def reset_io_remainder
         @io_remainder = ::String.new :encoding => ::Encoding::BINARY
       end
 
+      protected def reset_need_to_flush
+        @need_to_flush = false
+      end
+
       # -- synchronous --
 
       def read(bytes_to_read = nil, out_buffer = nil)
         Validation.validate_not_negative_integer bytes_to_read unless bytes_to_read.nil?
         Validation.validate_string out_buffer unless out_buffer.nil?
 
-        return ::String.new :encoding => ::Encoding::BINARY if bytes_to_read == 0
-
         unless bytes_to_read.nil?
+          return ::String.new :encoding => ::Encoding::BINARY if bytes_to_read.zero?
           return nil if eof?
 
-          read_more_from_buffer until @buffer.bytesize >= bytes_to_read || @io.eof?
+          append_io_data @io.read(@source_buffer_length) while @buffer.bytesize < bytes_to_read && !@io.eof?
+          flush_io_data if @buffer.bytesize < bytes_to_read
 
           return read_bytes_from_buffer bytes_to_read, out_buffer
         end
 
-        read_more_from_buffer until @io.eof?
-
-        result = @buffer
-        reset_buffer
-        @pos += result.bytesize
+        append_io_data @io.read(@source_buffer_length) until @io.eof?
+        flush_io_data
 
-        result.force_encoding @external_encoding unless @external_encoding.nil?
-        result = transcode_to_internal result
-        result = out_buffer.replace result unless out_buffer.nil?
-
-        result
-      end
-
-      protected def read_more_from_buffer
-        io_data = @io.read @source_buffer_length
-        append_io_data_to_buffer io_data
-      end
-
-      def readpartial(bytes_to_read = nil, out_buffer = nil)
-        raise ::EOFError if eof?
-
-        readpartial_from_buffer until @buffer.bytesize >= bytes_to_read || @io.eof?
-
-        read_bytes_from_buffer bytes_to_read, out_buffer
-      end
-
-      protected def readpartial_from_buffer
-        io_data = @io.readpartial @source_buffer_length
-        append_io_data_to_buffer io_data
+        read_buffer out_buffer
       end
 
       def rewind
         raw_wrapper :close
 
         reset_io_remainder
+        reset_need_to_flush
 
         super
       end
@@ -103,25 +84,61 @@ module ZSTDS
         super
       end
 
+      def eof?
+        empty? && @io.eof?
+      end
+
       # -- asynchronous --
 
+      def readpartial(bytes_to_read, out_buffer = nil)
+        read_more_nonblock(bytes_to_read, out_buffer) { @io.readpartial @source_buffer_length }
+      end
+
       def read_nonblock(bytes_to_read, out_buffer = nil, *options)
-        raise ::EOFError if eof?
+        read_more_nonblock(bytes_to_read, out_buffer) { @io.read_nonblock(@source_buffer_length, *options) }
+      end
+
+      protected def read_more_nonblock(bytes_to_read, out_buffer, &_block)
+        Validation.validate_not_negative_integer bytes_to_read
+        Validation.validate_string out_buffer unless out_buffer.nil?
+
+        return ::String.new :encoding => ::Encoding::BINARY if bytes_to_read.zero?
+
+        io_provided_eof_error = false
 
-        read_more_from_buffer_nonblock(*options) until @buffer.bytesize >= bytes_to_read || @io.eof?
+        if @buffer.bytesize < bytes_to_read
+          begin
+            append_io_data yield
+          rescue ::EOFError
+            io_provided_eof_error = true
+          end
+        end
+
+        flush_io_data if @buffer.bytesize < bytes_to_read
+        raise ::EOFError if empty? && io_provided_eof_error
 
         read_bytes_from_buffer bytes_to_read, out_buffer
       end
 
-      protected def read_more_from_buffer_nonblock(*options)
-        io_data = @io.read_nonblock @source_buffer_length, *options
-        append_io_data_to_buffer io_data
+      # -- common --
+
+      protected def append_io_data(io_data)
+        io_portion    = @io_remainder + io_data
+        bytes_read    = raw_wrapper :read, io_portion
+        @io_remainder = io_portion.byteslice bytes_read, io_portion.bytesize - bytes_read
+
+        # Even empty io data may require flush.
+        @need_to_flush = true
       end
 
-      # -- common --
+      protected def flush_io_data
+        raw_wrapper :flush
 
-      def eof?
-        @io.eof? && @buffer.bytesize == 0
+        @need_to_flush = false
+      end
+
+      protected def empty?
+        !@need_to_flush && @buffer.bytesize.zero?
       end
 
       protected def read_bytes_from_buffer(bytes_to_read, out_buffer)
@@ -136,18 +153,20 @@ module ZSTDS
         result
       end
 
-      protected def append_io_data_to_buffer(io_data)
-        io_portion    = @io_remainder + io_data
-        bytes_read    = raw_wrapper :read, io_portion
-        @io_remainder = io_portion.byteslice bytes_read, io_portion.bytesize - bytes_read
+      protected def read_buffer(out_buffer)
+        result = @buffer
+        reset_buffer
+        @pos += result.bytesize
 
-        # We should just ignore case when "io.eof?" appears but "io_remainder" is not empty.
-        # Ancient compress implementations can write bytes from not initialized buffer parts to output.
-        raw_wrapper :flush if @io.eof?
+        result.force_encoding @external_encoding unless @external_encoding.nil?
+        result = transcode_to_internal result
+
+        result = out_buffer.replace result unless out_buffer.nil?
+        result
       end
 
       protected def transcode_to_internal(data)
-        data = data.encode @internal_encoding, @transcode_options unless @internal_encoding.nil?
+        data = data.encode @internal_encoding, **@transcode_options unless @internal_encoding.nil?
         data
       end