ruby-zstds 1.2.0 → 1.3.1

data/lib/zstds/dictionary.rb

@@ -7,15 +7,36 @@ require_relative "error"
 require_relative "validation"
 
 module ZSTDS
+  # ZSTDS::Dictionary class.
   class Dictionary
+    # Current train defaults.
     TRAIN_DEFAULTS = {
       :gvl      => false,
       :capacity => 0
     }
     .freeze
 
+    # Current finalize defaults.
+    FINALIZE_DEFAULTS = {
+      :gvl                => false,
+      :max_size           => 0,
+      :dictionary_options => {}
+    }
+    .freeze
+
+    # Current finalize dictionary defaults.
+    FINALIZE_DICTIONARY_DEFAULTS = {
+      :compression_level  => 0,
+      :notification_level => 0,
+      :dictionary_id      => 0
+    }
+    .freeze
+
+    # Reads current +buffer+ binary data.
     attr_reader :buffer
 
+    # Initializes compressor.
+    # Uses +buffer+ binary data.
     def initialize(buffer)
       Validation.validate_string buffer
       raise ValidateError, "dictionary buffer should not be empty" if buffer.empty?
@@ -23,13 +44,14 @@ module ZSTDS
       @buffer = buffer
     end
 
+    # Trains dictionary.
+    # Uses +samples+ list of binary datas.
+    # Uses +options+ options hash.
+    # Option +gvl+ is global interpreter lock enabled.
+    # Option +capacity+ capacity of dictionary buffer.
+    # Returns dictionary based on new buffer.
     def self.train(samples, options = {})
-      Validation.validate_array samples
-
-      samples.each do |sample|
-        Validation.validate_string sample
-        raise ValidateError, "dictionary sample should not be empty" if sample.empty?
-      end
+      validate_samples samples
 
       Validation.validate_hash options
 
@@ -42,10 +64,58 @@ module ZSTDS
       new buffer
     end
 
+    # Finalizes dictionary.
+    # Uses +content+ binary data.
+    # Uses +samples+ list of binary datas.
+    # Uses +options+ options hash.
+    # Option +gvl+ is global interpreter lock enabled.
+    # Option +max_size+ max size of dictionary buffer.
+    # Option +dictionary_options+ standard dictionary options hash.
+    # Returns dictionary based on new buffer.
+    def self.finalize(content, samples, options = {})
+      Validation.validate_string content
+      raise ValidateError, "content should not be empty" if content.empty?
+
+      validate_samples samples
+
+      Validation.validate_hash options
+
+      options = FINALIZE_DEFAULTS.merge options
+
+      Validation.validate_bool                 options[:gvl]
+      Validation.validate_not_negative_integer options[:max_size]
+      Validation.validate_hash                 options[:dictionary_options]
+
+      dictionary_options = FINALIZE_DICTIONARY_DEFAULTS.merge options[:dictionary_options]
+
+      compression_level = dictionary_options[:compression_level]
+      Validation.validate_integer compression_level
+      raise ValidateError, "invalid compression level" if
+        compression_level < Option::MIN_COMPRESSION_LEVEL || compression_level > Option::MAX_COMPRESSION_LEVEL
+
+      Validation.validate_not_negative_integer dictionary_options[:notification_level]
+      Validation.validate_not_negative_integer dictionary_options[:dictionary_id]
+
+      buffer = finalize_buffer content, samples, options
+      new buffer
+    end
+
+    # Raises error when +samples+ are not list of not empty strings.
+    def self.validate_samples(samples)
+      Validation.validate_array samples
+
+      samples.each do |sample|
+        Validation.validate_string sample
+        raise ValidateError, "dictionary sample should not be empty" if sample.empty?
+      end
+    end
+
+    # Returns current dictionary id.
     def id
       self.class.get_buffer_id @buffer
     end
 
+    # Returns current dictionary header size.
     def header_size
       self.class.get_header_size @buffer
     end

data/lib/zstds/error.rb

@@ -1,16 +1,15 @@
 # Ruby bindings for zstd library.
 # Copyright (c) 2019 AUTHORS, MIT License.
 
+require "adsp"
+
 module ZSTDS
   class BaseError < ::StandardError; end
 
   class AllocateError < BaseError; end
-  class ValidateError < BaseError; end
 
-  class UsedAfterCloseError              < BaseError; end
   class NotEnoughSourceBufferError       < BaseError; end
   class NotEnoughDestinationBufferError  < BaseError; end
-  class NotEnoughDestinationError        < BaseError; end
   class DecompressorCorruptedSourceError < BaseError; end
   class CorruptedDictionaryError         < BaseError; end
 
@@ -18,5 +17,11 @@ module ZSTDS
   class ReadIOError   < BaseError; end
   class WriteIOError  < BaseError; end
 
-  class UnexpectedError < BaseError; end
+  ValidateError = ADSP::ValidateError
+
+  NotEnoughDestinationError = ADSP::NotEnoughDestinationError
+  UsedAfterCloseError       = ADSP::UsedAfterCloseError
+
+  NotImplementedError = ADSP::NotImplementedError
+  UnexpectedError     = ADSP::UnexpectedError
 end

data/lib/zstds/file.rb

@@ -1,50 +1,40 @@
 # Ruby bindings for zstd library.
 # Copyright (c) 2019 AUTHORS, MIT License.
 
+require "adsp/file"
 require "zstds_ext"
 
-require_relative "error"
 require_relative "option"
 require_relative "validation"
 
 module ZSTDS
-  module File
-    BUFFER_LENGTH_NAMES = %i[source_buffer_length destination_buffer_length].freeze
-
+  # ZSTDS::File class.
+  class File < ADSP::File
+    # Current option class.
+    Option = ZSTDS::Option
+
+    # Compresses data from +source+ file path to +destination+ file path.
+    # Option: +:source_buffer_length+ source buffer length.
+    # Option: +:destination_buffer_length+ destination buffer length.
+    # Option: +:pledged_size+ source bytesize.
     def self.compress(source, destination, options = {})
       Validation.validate_string source
-      Validation.validate_string destination
 
       options = Option.get_compressor_options options, BUFFER_LENGTH_NAMES
 
       options[:pledged_size] = ::File.size source
 
-      open_files source, destination do |source_io, destination_io|
-        ZSTDS._native_compress_io source_io, destination_io, options
-      end
-
-      nil
+      super source, destination, options
     end
 
-    def self.decompress(source, destination, options = {})
-      Validation.validate_string source
-      Validation.validate_string destination
-
-      options = Option.get_decompressor_options options, BUFFER_LENGTH_NAMES
-
-      open_files source, destination do |source_io, destination_io|
-        ZSTDS._native_decompress_io source_io, destination_io, options
-      end
-
-      nil
+    # Bypass native compress.
+    def self.native_compress_io(*args)
+      ZSTDS._native_compress_io(*args)
     end
 
-    private_class_method def self.open_files(source, destination, &_block)
-      ::File.open source, "rb" do |source_io|
-        ::File.open destination, "wb" do |destination_io|
-          yield source_io, destination_io
-        end
-      end
+    # Bypass native decompress.
+    def self.native_decompress_io(*args)
+      ZSTDS._native_decompress_io(*args)
     end
   end
 end

data/lib/zstds/option.rb

@@ -8,41 +8,95 @@ require_relative "error"
 require_relative "validation"
 
 module ZSTDS
+  # ZSTDS::Option module.
   module Option
+    # Current default buffer length.
     DEFAULT_BUFFER_LENGTH = 0
 
+    # Current compressor defaults.
     COMPRESSOR_DEFAULTS = {
+      # Enables global VM lock where possible.
       :gvl                           => false,
+      # Compression level.
       :compression_level             => nil,
+      # Maximum back-reference distance (power of 2).
       :window_log                    => nil,
+      # Size of the initial probe table (power of 2).
       :hash_log                      => nil,
+      # Size of the multi-probe search table (power of 2).
       :chain_log                     => nil,
+      # Number of search attempts (power of 2).
       :search_log                    => nil,
+      # Minimum size of searched matches.
       :min_match                     => nil,
+      # Distance between match sampling (for :fast strategy),
+      #   length of match considered "good enough" for (for other strategies).
       :target_length                 => nil,
+      # Choses strategy.
       :strategy                      => nil,
+      # Enables long distance matching.
       :enable_long_distance_matching => nil,
+      # Size of the table for long distance matching (power of 2).
       :ldm_hash_log                  => nil,
+      # Minimum match size for long distance matcher.
       :ldm_min_match                 => nil,
+      # Log size of each bucket in the LDM hash table for collision resolution.
       :ldm_bucket_size_log           => nil,
+      # Frequency of inserting/looking up entries into the LDM hash table.
       :ldm_hash_rate_log             => nil,
+      # Enables writing of content size into frame header (if known).
       :content_size_flag             => nil,
+      # Enables writing of 32-bits checksum of content at end of frame.
       :checksum_flag                 => nil,
+      # Enables writing of dictionary id into frame header.
       :dict_id_flag                  => nil,
+      # Number of threads spawned in parallel.
       :nb_workers                    => nil,
+      # Size of job (nb_workers >= 1).
       :job_size                      => nil,
+      # Overlap size, as a fraction of window size.
       :overlap_log                   => nil,
+      # Chose dictionary.
       :dictionary                    => nil
     }
     .freeze
 
+    # Current decompressor defaults.
     DECOMPRESSOR_DEFAULTS = {
+      # Enables global VM lock where possible.
       :gvl            => false,
+      # Size limit (power of 2).
       :window_log_max => nil,
+      # Chose dictionary.
       :dictionary     => nil
     }
     .freeze
 
+    # Processes compressor +options+ and +buffer_length_names+.
+    # Option: +:source_buffer_length+ source buffer length.
+    # Option: +:destination_buffer_length+ destination buffer length.
+    # Option: +:gvl+ enables global VM lock where possible.
+    # Option: +:compression_level+ compression level.
+    # Option: +:window_log+ maximum back-reference distance (power of 2).
+    # Option: +:hash_log+ size of the initial probe table (power of 2).
+    # Option: +:chain_log+ size of the multi-probe search table (power of 2).
+    # Option: +:search_log+ number of search attempts (power of 2).
+    # Option: +:min_match+ minimum size of searched matches.
+    # Option: +:target_length+ distance between match sampling (for :fast strategy),
+    #   length of match considered "good enough" for (for other strategies).
+    # Option: +:strategy+ choses strategy.
+    # Option: +:ldm_hash_log+ size of the table for long distance matching (power of 2).
+    # Option: +:ldm_min_match+ minimum match size for long distance matcher.
+    # Option: +:ldm_bucket_size_log+ log size of each bucket in the LDM hash table for collision resolution.
+    # Option: +:ldm_hash_rate_log+ frequency of inserting/looking up entries into the LDM hash table.
+    # Option: +:content_size_flag+ enables writing of content size into frame header (if known).
+    # Option: +:checksum_flag+ enables writing of 32-bits checksum of content at end of frame.
+    # Option: +:dict_id_flag+ enables writing of dictionary id into frame header.
+    # Option: +:nb_workers+ number of threads spawned in parallel.
+    # Option: +:job_size+ size of job (nb_workers >= 1).
+    # Option: +:overlap_log+ overlap size, as a fraction of window size.
+    # Option: +:dictionary+ chose dictionary.
+    # Returns processed compressor options.
     def self.get_compressor_options(options, buffer_length_names)
       Validation.validate_hash options
 
@@ -179,6 +233,12 @@ module ZSTDS
       options
     end
 
+    # Processes decompressor +options+ and +buffer_length_names+.
+    # Option: +:source_buffer_length+ source buffer length.
+    # Option: +:destination_buffer_length+ destination buffer length.
+    # Option: +:gvl+ enables global VM lock where possible.
+    # Option: +:window_log_max+ size limit (power of 2).
+    # Returns processed decompressor options.
     def self.get_decompressor_options(options, buffer_length_names)
       Validation.validate_hash options
 

data/lib/zstds/stream/raw/compressor.rb

@@ -1,97 +1,33 @@
 # Ruby bindings for zstd library.
 # Copyright (c) 2019 AUTHORS, MIT License.
 
+require "adsp/stream/raw/compressor"
 require "zstds_ext"
 
-require_relative "abstract"
-require_relative "../../error"
 require_relative "../../option"
 require_relative "../../validation"
 
 module ZSTDS
   module Stream
     module Raw
-      class Compressor < Abstract
-        BUFFER_LENGTH_NAMES = %i[destination_buffer_length].freeze
+      # ZSTDS::Stream::Raw::Compressor class.
+      class Compressor < ADSP::Stream::Raw::Compressor
+        # Current native compressor class.
+        NativeCompressor = Stream::NativeCompressor
 
+        # Current option class.
+        Option = ZSTDS::Option
+
+        # Initializes compressor.
+        # Option: +:destination_buffer_length+ destination buffer length.
+        # Option: +:pledged_size+ source bytesize.
         def initialize(options = {})
           options = Option.get_compressor_options options, BUFFER_LENGTH_NAMES
 
           pledged_size = options[:pledged_size]
           Validation.validate_not_negative_integer pledged_size unless pledged_size.nil?
 
-          native_stream = NativeCompressor.new options
-
-          super native_stream
-        end
-
-        def write(source, &writer)
-          do_not_use_after_close
-
-          Validation.validate_string source
-          Validation.validate_proc writer
-
-          total_bytes_written = 0
-
-          loop do
-            bytes_written, need_more_destination  = @native_stream.write source
-            total_bytes_written                  += bytes_written
-
-            if need_more_destination
-              source = source.byteslice bytes_written, source.bytesize - bytes_written
-              more_destination(&writer)
-              next
-            end
-
-            unless bytes_written == source.bytesize
-              # :nocov:
-              # Compressor write should eat all provided "source" without remainder.
-              raise UnexpectedError, "unexpected error"
-              # :nocov:
-            end
-
-            break
-          end
-
-          total_bytes_written
-        end
-
-        def flush(&writer)
-          do_not_use_after_close
-
-          Validation.validate_proc writer
-
-          loop do
-            need_more_destination = @native_stream.flush
-
-            if need_more_destination
-              more_destination(&writer)
-              next
-            end
-
-            break
-          end
-
-          super
-        end
-
-        def close(&writer)
-          return nil if closed?
-
-          Validation.validate_proc writer
-
-          loop do
-            need_more_destination = @native_stream.finish
-
-            if need_more_destination
-              more_destination(&writer)
-              next
-            end
-
-            break
-          end
-
-          super
+          super options
         end
       end
     end

data/lib/zstds/stream/raw/decompressor.rb

@@ -1,65 +1,21 @@
 # Ruby bindings for zstd library.
 # Copyright (c) 2019 AUTHORS, MIT License.
 
+require "adsp/stream/raw/decompressor"
 require "zstds_ext"
 
-require_relative "abstract"
 require_relative "../../option"
-require_relative "../../validation"
 
 module ZSTDS
   module Stream
     module Raw
-      class Decompressor < Abstract
-        BUFFER_LENGTH_NAMES = %i[destination_buffer_length].freeze
+      # ZSTDS::Stream::Raw::Decompressor class.
+      class Decompressor < ADSP::Stream::Raw::Decompressor
+        # Current native decompressor class.
+        NativeDecompressor = Stream::NativeDecompressor
 
-        def initialize(options = {})
-          options       = Option.get_decompressor_options options, BUFFER_LENGTH_NAMES
-          native_stream = NativeDecompressor.new options
-
-          super native_stream
-        end
-
-        def read(source, &writer)
-          do_not_use_after_close
-
-          Validation.validate_string source
-          Validation.validate_proc writer
-
-          total_bytes_read = 0
-
-          loop do
-            bytes_read, need_more_destination  = @native_stream.read source
-            total_bytes_read                  += bytes_read
-
-            if need_more_destination
-              source = source.byteslice bytes_read, source.bytesize - bytes_read
-              more_destination(&writer)
-              next
-            end
-
-            break
-          end
-
-          # Please remember that "total_bytes_read" can not be equal to "source.bytesize".
-          total_bytes_read
-        end
-
-        def flush(&writer)
-          do_not_use_after_close
-
-          Validation.validate_proc writer
-
-          super
-        end
-
-        def close(&writer)
-          return nil if closed?
-
-          Validation.validate_proc writer
-
-          super
-        end
+        # Current option class.
+        Option = ZSTDS::Option
       end
     end
   end

data/lib/zstds/stream/reader.rb

@@ -1,186 +1,16 @@
 # Ruby bindings for zstd library.
 # Copyright (c) 2019 AUTHORS, MIT License.
 
-require_relative "abstract"
+require "adsp/stream/reader"
+
 require_relative "raw/decompressor"
-require_relative "reader_helpers"
-require_relative "../validation"
 
 module ZSTDS
   module Stream
-    class Reader < Abstract
-      include ReaderHelpers
-
-      attr_accessor :lineno
-
-      def initialize(source_io, options = {}, *args)
-        @options = options
-
-        super source_io, *args
-
-        initialize_source_buffer_length
-        reset_io_remainder
-        reset_need_to_flush
-
-        @lineno = 0
-      end
-
-      protected def create_raw_stream
-        Raw::Decompressor.new @options
-      end
-
-      protected def initialize_source_buffer_length
-        source_buffer_length = @options[:source_buffer_length]
-        Validation.validate_not_negative_integer source_buffer_length unless source_buffer_length.nil?
-
-        if source_buffer_length.nil? || source_buffer_length.zero?
-          source_buffer_length = Buffer::DEFAULT_SOURCE_BUFFER_LENGTH_FOR_DECOMPRESSOR
-        end
-
-        @source_buffer_length = source_buffer_length
-      end
-
-      protected def reset_io_remainder
-        @io_remainder = ::String.new :encoding => ::Encoding::BINARY
-      end
-
-      protected def reset_need_to_flush
-        @need_to_flush = false
-      end
-
-      # -- synchronous --
-
-      def read(bytes_to_read = nil, out_buffer = nil)
-        Validation.validate_not_negative_integer bytes_to_read unless bytes_to_read.nil?
-        Validation.validate_string out_buffer unless out_buffer.nil?
-
-        unless bytes_to_read.nil?
-          return ::String.new :encoding => ::Encoding::BINARY if bytes_to_read.zero?
-          return nil if eof?
-
-          append_io_data @io.read(@source_buffer_length) while @buffer.bytesize < bytes_to_read && !@io.eof?
-          flush_io_data if @buffer.bytesize < bytes_to_read
-
-          return read_bytes_from_buffer bytes_to_read, out_buffer
-        end
-
-        append_io_data @io.read(@source_buffer_length) until @io.eof?
-        flush_io_data
-
-        read_buffer out_buffer
-      end
-
-      def rewind
-        raw_wrapper :close
-
-        reset_io_remainder
-        reset_need_to_flush
-
-        super
-      end
-
-      def close
-        raw_wrapper :close
-
-        super
-      end
-
-      def eof?
-        empty? && @io.eof?
-      end
-
-      # -- asynchronous --
-
-      def readpartial(bytes_to_read, out_buffer = nil)
-        read_more_nonblock(bytes_to_read, out_buffer) { @io.readpartial @source_buffer_length }
-      end
-
-      def read_nonblock(bytes_to_read, out_buffer = nil, *options)
-        read_more_nonblock(bytes_to_read, out_buffer) { @io.read_nonblock(@source_buffer_length, *options) }
-      end
-
-      protected def read_more_nonblock(bytes_to_read, out_buffer, &_block)
-        Validation.validate_not_negative_integer bytes_to_read
-        Validation.validate_string out_buffer unless out_buffer.nil?
-
-        return ::String.new :encoding => ::Encoding::BINARY if bytes_to_read.zero?
-
-        io_provided_eof_error = false
-
-        if @buffer.bytesize < bytes_to_read
-          begin
-            append_io_data yield
-          rescue ::EOFError
-            io_provided_eof_error = true
-          end
-        end
-
-        flush_io_data if @buffer.bytesize < bytes_to_read
-        raise ::EOFError if empty? && io_provided_eof_error
-
-        read_bytes_from_buffer bytes_to_read, out_buffer
-      end
-
-      # -- common --
-
-      protected def append_io_data(io_data)
-        io_portion    = @io_remainder + io_data
-        bytes_read    = raw_wrapper :read, io_portion
-        @io_remainder = io_portion.byteslice bytes_read, io_portion.bytesize - bytes_read
-
-        # Even empty io data may require flush.
-        @need_to_flush = true
-      end
-
-      protected def flush_io_data
-        raw_wrapper :flush
-
-        @need_to_flush = false
-      end
-
-      protected def empty?
-        !@need_to_flush && @buffer.bytesize.zero?
-      end
-
-      protected def read_bytes_from_buffer(bytes_to_read, out_buffer)
-        bytes_read = [@buffer.bytesize, bytes_to_read].min
-
-        # Result uses buffer binary encoding.
-        result   = @buffer.byteslice 0, bytes_read
-        @buffer  = @buffer.byteslice bytes_read, @buffer.bytesize - bytes_read
-        @pos    += bytes_read
-
-        result = out_buffer.replace result unless out_buffer.nil?
-        result
-      end
-
-      protected def read_buffer(out_buffer)
-        result = @buffer
-        reset_buffer
-        @pos += result.bytesize
-
-        result.force_encoding @external_encoding unless @external_encoding.nil?
-        result = transcode_to_internal result
-
-        result = out_buffer.replace result unless out_buffer.nil?
-        result
-      end
-
-      protected def transcode_to_internal(data)
-        data = data.encode @internal_encoding, **@transcode_options unless @internal_encoding.nil?
-        data
-      end
-
-      # We should be able to return data back to buffer.
-      # We won't use any transcode options because transcoded data should be backward compatible.
-      protected def transcode_to_external(data)
-        data = data.encode @external_encoding unless @external_encoding.nil?
-        data
-      end
-
-      protected def raw_wrapper(method_name, *args)
-        @raw_stream.send(method_name, *args) { |portion| @buffer << portion }
-      end
+    # ZSTDS::Stream::Reader class.
+    class Reader < ADSP::Stream::Reader
+      # Current raw stream class.
+      RawDecompressor = Raw::Decompressor
     end
   end
 end