ruby-zstds 1.0.0

data/ext/zstds_ext/stream/decompressor.h

@@ -0,0 +1,29 @@
+// Ruby bindings for zstd library.
+// Copyright (c) 2019 AUTHORS, MIT License.
+
+#if !defined(ZSTDS_EXT_STREAM_DECOMPRESSOR_H)
+#define ZSTDS_EXT_STREAM_DECOMPRESSOR_H
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <zstd.h>
+
+#include "ruby.h"
+
+typedef struct {
+  ZSTD_DCtx* ctx;
+  uint8_t*   destination_buffer;
+  size_t     destination_buffer_length;
+  uint8_t*   remaining_destination_buffer;
+  size_t     remaining_destination_buffer_length;
+} zstds_ext_decompressor_t;
+
+VALUE zstds_ext_allocate_decompressor(VALUE klass);
+VALUE zstds_ext_initialize_decompressor(VALUE self, VALUE options);
+VALUE zstds_ext_decompress(VALUE self, VALUE source);
+VALUE zstds_ext_decompressor_read_result(VALUE self);
+VALUE zstds_ext_decompressor_close(VALUE self);
+
+void zstds_ext_decompressor_exports(VALUE root_module);
+
+#endif // ZSTDS_EXT_STREAM_DECOMPRESSOR_H

data/ext/zstds_ext/string.c

@@ -0,0 +1,254 @@
+// Ruby bindings for zstd library.
+// Copyright (c) 2019 AUTHORS, MIT License.
+
+#include "zstds_ext/string.h"
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <zstd.h>
+
+#include "ruby.h"
+#include "zstds_ext/buffer.h"
+#include "zstds_ext/common.h"
+#include "zstds_ext/error.h"
+#include "zstds_ext/macro.h"
+#include "zstds_ext/option.h"
+
+// -- buffer --
+
+static inline zstds_ext_result_t increase_destination_buffer(
+  VALUE destination_value, size_t destination_length,
+  size_t* remaining_destination_buffer_length_ptr, size_t destination_buffer_length)
+{
+  if (*remaining_destination_buffer_length_ptr == destination_buffer_length) {
+    // We want to write more data at once, than buffer has.
+    return ZSTDS_EXT_ERROR_NOT_ENOUGH_DESTINATION_BUFFER;
+  }
+
+  int exception;
+
+  ZSTDS_EXT_RESIZE_STRING_BUFFER(destination_value, destination_length + destination_buffer_length, exception);
+  if (exception != 0) {
+    return ZSTDS_EXT_ERROR_ALLOCATE_FAILED;
+  }
+
+  *remaining_destination_buffer_length_ptr = destination_buffer_length;
+
+  return 0;
+}
+
+// -- utils --
+
+#define GET_SOURCE_DATA(source_value)                    \
+  Check_Type(source_value, T_STRING);                    \
+                                                         \
+  const char* source        = RSTRING_PTR(source_value); \
+  size_t      source_length = RSTRING_LEN(source_value);
+
+// -- compress --
+
+static inline zstds_ext_result_t compress(
+  ZSTD_CCtx*  ctx,
+  const char* source, size_t source_length,
+  VALUE destination_value, size_t destination_buffer_length)
+{
+  zstds_result_t     result;
+  zstds_ext_result_t ext_result;
+
+  ZSTD_inBuffer in_buffer;
+  in_buffer.src  = source;
+  in_buffer.size = source_length;
+  in_buffer.pos  = 0;
+
+  ZSTD_outBuffer out_buffer;
+  size_t         destination_length                  = 0;
+  size_t         remaining_destination_buffer_length = destination_buffer_length;
+
+  while (true) {
+    out_buffer.dst  = (uint8_t*)RSTRING_PTR(destination_value) + destination_length;
+    out_buffer.size = remaining_destination_buffer_length;
+    out_buffer.pos  = 0;
+
+    result = ZSTD_compressStream2(ctx, &out_buffer, &in_buffer, ZSTD_e_end);
+    if (ZSTD_isError(result)) {
+      return zstds_ext_get_error(ZSTD_getErrorCode(result));
+    }
+
+    destination_length += out_buffer.pos;
+    remaining_destination_buffer_length -= out_buffer.pos;
+
+    if (result != 0) {
+      ext_result = increase_destination_buffer(
+        destination_value, destination_length,
+        &remaining_destination_buffer_length, destination_buffer_length);
+
+      if (ext_result != 0) {
+        return ext_result;
+      }
+
+      continue;
+    }
+
+    break;
+  }
+
+  int exception;
+
+  ZSTDS_EXT_RESIZE_STRING_BUFFER(destination_value, destination_length, exception);
+  if (exception != 0) {
+    return ZSTDS_EXT_ERROR_ALLOCATE_FAILED;
+  }
+
+  return 0;
+}
+
+VALUE zstds_ext_compress_string(VALUE ZSTDS_EXT_UNUSED(self), VALUE source_value, VALUE options)
+{
+  GET_SOURCE_DATA(source_value);
+  Check_Type(options, T_HASH);
+  ZSTDS_EXT_GET_COMPRESSOR_OPTIONS(options);
+  ZSTDS_EXT_GET_BUFFER_LENGTH_OPTION(options, destination_buffer_length);
+
+  ZSTD_CCtx* ctx = ZSTD_createCCtx();
+  if (ctx == NULL) {
+    zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
+  }
+
+  zstds_ext_result_t ext_result = zstds_ext_set_compressor_options(ctx, &compressor_options);
+  if (ext_result != 0) {
+    ZSTD_freeCCtx(ctx);
+    zstds_ext_raise_error(ext_result);
+  }
+
+  if (destination_buffer_length == 0) {
+    destination_buffer_length = ZSTD_CStreamOutSize();
+  }
+
+  int exception;
+
+  ZSTDS_EXT_CREATE_STRING_BUFFER(destination_value, destination_buffer_length, exception);
+  if (exception != 0) {
+    ZSTD_freeCCtx(ctx);
+    zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
+  }
+
+  ext_result = compress(
+    ctx,
+    source, source_length,
+    destination_value, destination_buffer_length);
+
+  ZSTD_freeCCtx(ctx);
+
+  if (ext_result != 0) {
+    zstds_ext_raise_error(ext_result);
+  }
+
+  return destination_value;
+}
+
+// -- decompress --
+
+static inline zstds_ext_result_t decompress(
+  ZSTD_DCtx*  ctx,
+  const char* source, size_t source_length,
+  VALUE destination_value, size_t destination_buffer_length)
+{
+  zstds_result_t     result;
+  zstds_ext_result_t ext_result;
+
+  ZSTD_inBuffer in_buffer;
+  in_buffer.src  = source;
+  in_buffer.size = source_length;
+  in_buffer.pos  = 0;
+
+  ZSTD_outBuffer out_buffer;
+  size_t         destination_length                  = 0;
+  size_t         remaining_destination_buffer_length = destination_buffer_length;
+
+  while (true) {
+    out_buffer.dst  = (uint8_t*)RSTRING_PTR(destination_value) + destination_length;
+    out_buffer.size = remaining_destination_buffer_length;
+    out_buffer.pos  = 0;
+
+    result = ZSTD_decompressStream(ctx, &out_buffer, &in_buffer);
+    if (ZSTD_isError(result)) {
+      return zstds_ext_get_error(ZSTD_getErrorCode(result));
+    }
+
+    destination_length += out_buffer.pos;
+    remaining_destination_buffer_length -= out_buffer.pos;
+
+    if (remaining_destination_buffer_length == 0) {
+      ext_result = increase_destination_buffer(
+        destination_value, destination_length,
+        &remaining_destination_buffer_length, destination_buffer_length);
+
+      if (ext_result != 0) {
+        return ext_result;
+      }
+
+      continue;
+    }
+
+    break;
+  }
+
+  int exception;
+
+  ZSTDS_EXT_RESIZE_STRING_BUFFER(destination_value, destination_length, exception);
+  if (exception != 0) {
+    zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
+  }
+
+  return 0;
+}
+
+VALUE zstds_ext_decompress_string(VALUE ZSTDS_EXT_UNUSED(self), VALUE source_value, VALUE options)
+{
+  GET_SOURCE_DATA(source_value);
+  Check_Type(options, T_HASH);
+  ZSTDS_EXT_GET_DECOMPRESSOR_OPTIONS(options);
+  ZSTDS_EXT_GET_BUFFER_LENGTH_OPTION(options, destination_buffer_length);
+
+  ZSTD_DCtx* ctx = ZSTD_createDCtx();
+  if (ctx == NULL) {
+    zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
+  }
+
+  zstds_ext_result_t ext_result = zstds_ext_set_decompressor_options(ctx, &decompressor_options);
+  if (ext_result != 0) {
+    ZSTD_freeDCtx(ctx);
+    zstds_ext_raise_error(ext_result);
+  }
+
+  if (destination_buffer_length == 0) {
+    destination_buffer_length = ZSTD_DStreamOutSize();
+  }
+
+  int exception;
+
+  ZSTDS_EXT_CREATE_STRING_BUFFER(destination_value, destination_buffer_length, exception);
+  if (exception != 0) {
+    ZSTD_freeDCtx(ctx);
+    zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
+  }
+
+  ext_result = decompress(
+    ctx,
+    source, source_length,
+    destination_value, destination_buffer_length);
+
+  ZSTD_freeDCtx(ctx);
+
+  if (ext_result != 0) {
+    zstds_ext_raise_error(ext_result);
+  }
+
+  return destination_value;
+}
+
+void zstds_ext_string_exports(VALUE root_module)
+{
+  rb_define_module_function(root_module, "_native_compress_string", RUBY_METHOD_FUNC(zstds_ext_compress_string), 2);
+  rb_define_module_function(root_module, "_native_decompress_string", RUBY_METHOD_FUNC(zstds_ext_decompress_string), 2);
+}

data/ext/zstds_ext/string.h

@@ -0,0 +1,14 @@
+// Ruby bindings for zstd library.
+// Copyright (c) 2019 AUTHORS, MIT License.
+
+#if !defined(ZSTDS_EXT_STRING_H)
+#define ZSTDS_EXT_STRING_H
+
+#include "ruby.h"
+
+VALUE zstds_ext_compress_string(VALUE self, VALUE source, VALUE options);
+VALUE zstds_ext_decompress_string(VALUE self, VALUE source, VALUE options);
+
+void zstds_ext_string_exports(VALUE root_module);
+
+#endif // ZSTDS_EXT_STRING_H

data/lib/zstds.rb

@@ -0,0 +1,9 @@
+# Ruby bindings for zstd library.
+# Copyright (c) 2019 AUTHORS, MIT License.
+
+require_relative "zstds/stream/reader"
+require_relative "zstds/stream/writer"
+require_relative "zstds/dictionary"
+require_relative "zstds/file"
+require_relative "zstds/string"
+require_relative "zstds/version"

data/lib/zstds/dictionary.rb

@@ -0,0 +1,47 @@
+# Ruby bindings for zstd library.
+# Copyright (c) 2019 AUTHORS, MIT License.
+
+require "zstds_ext"
+
+require_relative "error"
+require_relative "validation"
+
+module ZSTDS
+  class Dictionary
+    TRAIN_DEFAULTS = {
+      :capacity => 0
+    }
+    .freeze
+
+    attr_reader :buffer
+
+    def initialize(buffer)
+      Validation.validate_string buffer
+      raise ValidateError, "dictionary buffer should not be empty" if buffer.empty?
+
+      @buffer = buffer
+    end
+
+    def id
+      self.class.get_buffer_id @buffer
+    end
+
+    def self.train(samples, options = {})
+      Validation.validate_array samples
+
+      samples.each do |sample|
+        Validation.validate_string sample
+        raise ValidateError, "dictionary sample should not be empty" if sample.empty?
+      end
+
+      Validation.validate_hash options
+
+      options = TRAIN_DEFAULTS.merge options
+
+      Validation.validate_not_negative_integer options[:capacity]
+
+      buffer = train_buffer samples, options
+      new buffer
+    end
+  end
+end

data/lib/zstds/error.rb

@@ -0,0 +1,22 @@
+# Ruby bindings for zstd library.
+# Copyright (c) 2019 AUTHORS, MIT License.
+
+module ZSTDS
+  class BaseError < ::StandardError; end
+
+  class AllocateError < BaseError; end
+  class ValidateError < BaseError; end
+
+  class UsedAfterCloseError              < BaseError; end
+  class NotEnoughSourceBufferError       < BaseError; end
+  class NotEnoughDestinationBufferError  < BaseError; end
+  class NotEnoughDestinationError        < BaseError; end
+  class DecompressorCorruptedSourceError < BaseError; end
+  class CorruptedDictionaryError         < BaseError; end
+
+  class AccessIOError < BaseError; end
+  class ReadIOError   < BaseError; end
+  class WriteIOError  < BaseError; end
+
+  class UnexpectedError < BaseError; end
+end

data/lib/zstds/file.rb

@@ -0,0 +1,46 @@
+# Ruby bindings for zstd library.
+# Copyright (c) 2019 AUTHORS, MIT License.
+
+require "zstds_ext"
+
+require_relative "error"
+require_relative "option"
+require_relative "validation"
+
+module ZSTDS
+  module File
+    BUFFER_LENGTH_NAMES = %i[source_buffer_length destination_buffer_length].freeze
+
+    def self.compress(source, destination, options = {})
+      Validation.validate_string source
+      Validation.validate_string destination
+
+      options = Option.get_compressor_options options, BUFFER_LENGTH_NAMES
+
+      options[:pledged_size] = ::File.size source
+
+      open_files(source, destination) do |source_io, destination_io|
+        ZSTDS._native_compress_io source_io, destination_io, options
+      end
+    end
+
+    def self.decompress(source, destination, options = {})
+      Validation.validate_string source
+      Validation.validate_string destination
+
+      options = Option.get_decompressor_options options, BUFFER_LENGTH_NAMES
+
+      open_files(source, destination) do |source_io, destination_io|
+        ZSTDS._native_decompress_io source_io, destination_io, options
+      end
+    end
+
+    private_class_method def self.open_files(source, destination, &_block)
+      ::File.open source, "rb" do |source_io|
+        ::File.open destination, "wb" do |destination_io|
+          yield source_io, destination_io
+        end
+      end
+    end
+  end
+end

data/lib/zstds/option.rb

@@ -0,0 +1,194 @@
+# Ruby bindings for zstd library.
+# Copyright (c) 2019 AUTHORS, MIT License.
+
+require "zstds_ext"
+
+require_relative "dictionary"
+require_relative "error"
+require_relative "validation"
+
+module ZSTDS
+  module Option
+    DEFAULT_BUFFER_LENGTH = 0
+
+    COMPRESSOR_DEFAULTS = {
+      :compression_level             => nil,
+      :window_log                    => nil,
+      :hash_log                      => nil,
+      :chain_log                     => nil,
+      :search_log                    => nil,
+      :min_match                     => nil,
+      :target_length                 => nil,
+      :strategy                      => nil,
+      :enable_long_distance_matching => nil,
+      :ldm_hash_log                  => nil,
+      :ldm_min_match                 => nil,
+      :ldm_bucket_size_log           => nil,
+      :ldm_hash_rate_log             => nil,
+      :content_size_flag             => nil,
+      :checksum_flag                 => nil,
+      :dict_id_flag                  => nil,
+      :nb_workers                    => nil,
+      :job_size                      => nil,
+      :overlap_log                   => nil,
+      :dictionary                    => nil
+    }
+    .freeze
+
+    DECOMPRESSOR_DEFAULTS = {
+      :window_log_max => nil,
+      :dictionary     => nil
+    }
+    .freeze
+
+    def self.get_compressor_options(options, buffer_length_names)
+      Validation.validate_hash options
+
+      buffer_length_defaults = buffer_length_names.each_with_object({}) { |name, defaults| defaults[name] = DEFAULT_BUFFER_LENGTH }
+      options                = COMPRESSOR_DEFAULTS.merge(buffer_length_defaults).merge options
+
+      buffer_length_names.each { |name| Validation.validate_not_negative_integer options[name] }
+
+      compression_level = options[:compression_level]
+      unless compression_level.nil?
+        Validation.validate_integer compression_level
+        raise ValidateError, "invalid compression level" if
+          compression_level < MIN_COMPRESSION_LEVEL || compression_level > MAX_COMPRESSION_LEVEL
+      end
+
+      window_log = options[:window_log]
+      unless window_log.nil?
+        Validation.validate_not_negative_integer window_log
+        raise ValidateError, "invalid window log" if
+          window_log < MIN_WINDOW_LOG || window_log > MAX_WINDOW_LOG
+      end
+
+      hash_log = options[:hash_log]
+      unless hash_log.nil?
+        Validation.validate_not_negative_integer hash_log
+        raise ValidateError, "invalid hash log" if
+          hash_log < MIN_HASH_LOG || hash_log > MAX_HASH_LOG
+      end
+
+      chain_log = options[:chain_log]
+      unless chain_log.nil?
+        Validation.validate_not_negative_integer chain_log
+        raise ValidateError, "invalid chain log" if
+          chain_log < MIN_CHAIN_LOG || chain_log > MAX_CHAIN_LOG
+      end
+
+      search_log = options[:search_log]
+      unless search_log.nil?
+        Validation.validate_not_negative_integer search_log
+        raise ValidateError, "invalid search log" if
+          search_log < MIN_SEARCH_LOG || search_log > MAX_SEARCH_LOG
+      end
+
+      min_match = options[:min_match]
+      unless min_match.nil?
+        Validation.validate_not_negative_integer min_match
+        raise ValidateError, "invalid min match" if
+          min_match < MIN_MIN_MATCH || min_match > MAX_MIN_MATCH
+      end
+
+      target_length = options[:target_length]
+      unless target_length.nil?
+        Validation.validate_not_negative_integer target_length
+        raise ValidateError, "invalid target length" if
+          target_length < MIN_TARGET_LENGTH || target_length > MAX_TARGET_LENGTH
+      end
+
+      strategy = options[:strategy]
+      unless strategy.nil?
+        Validation.validate_symbol strategy
+        raise ValidateError, "invalid strategy" unless STRATEGIES.include? strategy
+      end
+
+      enable_long_distance_matching = options[:enable_long_distance_matching]
+      Validation.validate_bool enable_long_distance_matching unless enable_long_distance_matching.nil?
+
+      ldm_hash_log = options[:ldm_hash_log]
+      unless ldm_hash_log.nil?
+        Validation.validate_not_negative_integer ldm_hash_log
+        raise ValidateError, "invalid ldm hash log" if
+          ldm_hash_log < MIN_LDM_HASH_LOG || ldm_hash_log > MAX_LDM_HASH_LOG
+      end
+
+      ldm_min_match = options[:ldm_min_match]
+      unless ldm_min_match.nil?
+        Validation.validate_not_negative_integer ldm_min_match
+        raise ValidateError, "invalid ldm min match" if
+          ldm_min_match < MIN_LDM_MIN_MATCH || ldm_min_match > MAX_LDM_MIN_MATCH
+      end
+
+      ldm_bucket_size_log = options[:ldm_bucket_size_log]
+      unless ldm_bucket_size_log.nil?
+        Validation.validate_not_negative_integer ldm_bucket_size_log
+        raise ValidateError, "invalid ldm bucket size log" if
+          ldm_bucket_size_log < MIN_LDM_BUCKET_SIZE_LOG || ldm_bucket_size_log > MAX_LDM_BUCKET_SIZE_LOG
+      end
+
+      ldm_hash_rate_log = options[:ldm_hash_rate_log]
+      unless ldm_hash_rate_log.nil?
+        Validation.validate_not_negative_integer ldm_hash_rate_log
+        raise ValidateError, "invalid ldm hash rate log" if
+          ldm_hash_rate_log < MIN_LDM_HASH_RATE_LOG || ldm_hash_rate_log > MAX_LDM_HASH_RATE_LOG
+      end
+
+      content_size_flag = options[:content_size_flag]
+      Validation.validate_bool content_size_flag unless content_size_flag.nil?
+
+      checksum_flag = options[:checksum_flag]
+      Validation.validate_bool checksum_flag unless checksum_flag.nil?
+
+      dict_id_flag = options[:dict_id_flag]
+      Validation.validate_bool dict_id_flag unless dict_id_flag.nil?
+
+      nb_workers = options[:nb_workers]
+      unless nb_workers.nil?
+        Validation.validate_not_negative_integer nb_workers
+        raise ValidateError, "invalid nb workers" if
+          nb_workers < MIN_NB_WORKERS || nb_workers > MAX_NB_WORKERS
+      end
+
+      job_size = options[:job_size]
+      unless job_size.nil?
+        Validation.validate_not_negative_integer job_size
+        raise ValidateError, "invalid job size" if
+          job_size < MIN_JOB_SIZE || job_size > MAX_JOB_SIZE
+      end
+
+      overlap_log = options[:overlap_log]
+      unless overlap_log.nil?
+        Validation.validate_not_negative_integer overlap_log
+        raise ValidateError, "invalid overlap log" if
+          overlap_log < MIN_OVERLAP_LOG || overlap_log > MAX_OVERLAP_LOG
+      end
+
+      dictionary = options[:dictionary]
+      unless dictionary.nil?
+        raise ValidateError, "invalid dictionary" unless dictionary.is_a? Dictionary
+      end
+
+      options
+    end
+
+    def self.get_decompressor_options(options, buffer_length_names)
+      Validation.validate_hash options
+
+      buffer_length_defaults = buffer_length_names.each_with_object({}) { |name, defaults| defaults[name] = DEFAULT_BUFFER_LENGTH }
+      options                = DECOMPRESSOR_DEFAULTS.merge(buffer_length_defaults).merge options
+
+      buffer_length_names.each { |name| Validation.validate_not_negative_integer options[name] }
+
+      window_log_max = options[:window_log_max]
+      unless window_log_max.nil?
+        Validation.validate_not_negative_integer window_log_max
+        raise ValidateError, "invalid window log max" if
+          window_log_max < MIN_WINDOW_LOG_MAX || window_log_max > MAX_WINDOW_LOG_MAX
+      end
+
+      options
+    end
+  end
+end