ruby-zstds 1.1.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 882ef3555df6569fdb51a8689d2e7b8e710255f98a08a631e023ae0160d3ad86
-  data.tar.gz: 233e455b9819cab316812fd7c977735972ad5ff2eeee1a816c329e7a5b7661f1
+  metadata.gz: ec16246ba1102583f6f309e555e4ac4dbbf49135f5adee4e0332b86be7388de4
+  data.tar.gz: 7d8d40b406c9da3ed50c5929f565c51fc8cf46558bee937fdeaa0d942d0fe84f
 SHA512:
-  metadata.gz: 69ce6c1b92687ca88e87619a3314acf3b8c0584c56a1694730e5e5bf7139d5db3c7c280f01df79e1fb010a3c61c0c46af76a82bb1f4815b481421fb7b885f3d4
-  data.tar.gz: b77df0f6f9b98523ca3e611bbab7d5655a94c7e637fc61511096da614cb95672fab113894de375f6f406015aebc6eded0d0dabb5025644891775bec8a04cadda
+  metadata.gz: fd92bd835bf7eb14db13ecc1761a89467e3f38bbcb9d373573fa7c3a8839f4050a478091ada05fec97bb8e2824e9913d23ef9593f3388e317c5b9df456e7e7cd
+  data.tar.gz: 00d8bd37067fd788a7033abd8cf4c6d590b4f672f8986e7da7b80e47ed9ffa4a91886db7442a2a118a6ba45c8f9b2b232665455a5fc85c6cf4c8fcbe119c4e9c

data/AUTHORS

@@ -1 +1,3 @@
 Andrew Aladjev
+Ivan Takarlikov
+Jenner La Fave

data/README.md

@@ -8,7 +8,17 @@ See [zstd library](https://github.com/facebook/zstd).
 
 ## Installation
 
-Please install zstd library first, use latest 1.4.0+ version.
+Operating systems: GNU/Linux, FreeBSD, OSX, Windows (MinGW).
+
+Dependencies: [zstd](https://github.com/facebook/zstd) 1.4.0+ version.
+
+| Popular OS | Dependencies              |
+|------------|---------------------------|
+| Ubuntu     | `libzstd-dev`             |
+| CentOS     | `libzstd-devel`           |
+| ArchLinux  | `zstd`                    |
+| OSX        | `zstd`                    |
+| Windows    | `mingw-w64-x86_64-zstd`   |
 
 ```sh
 gem install ruby-zstds
@@ -23,6 +33,22 @@ gem install pkg/ruby-zstds-*.gem
 
 You can also use [overlay](https://github.com/andrew-aladev/overlay) for gentoo.
 
+### Installation in macOS on Apple Silicon
+
+On M1 Macs, Homebrew installs to /opt/homebrew, so you'll need to specify its
+include and lib paths when building the native extension for zstd.
+
+```sh
+brew install zstd
+gem install ruby-zstds -- --with-opt-include=/opt/homebrew/include --with-opt-lib=/opt/homebrew/lib
+```
+
+You can also configure Bundler to use those options when installing:
+
+```sh
+bundle config set build.ruby-zstds "--with-opt-include=/opt/homebrew/include --with-opt-lib=/opt/homebrew/lib"
+```
+
 ## Usage
 
 There are simple APIs: `String` and `File`. Also you can use generic streaming API: `Stream::Writer` and `Stream::Reader`.
@@ -449,10 +475,6 @@ You should lock all shared data between threads.
 For example: you should not use same compressor/decompressor inside multiple threads.
 Please verify that you are using each processor inside single thread at the same time.
 
-## Operating systems
-
-GNU/Linux, FreeBSD, OSX, Windows (MinGW).
-
 ## CI
 
 Please visit [scripts/test-images](scripts/test-images).

data/ext/extconf.rb

@@ -9,26 +9,97 @@ have_func "rb_thread_call_without_gvl", "ruby/thread.h"
 # https://bugs.gentoo.org/713940
 $LDFLAGS << " -pthread" # rubocop:disable Style/GlobalVars
 
-def require_header(name, types = [])
+def require_header(name, constants: [], macroses: [], types: [])
   abort "Can't find #{name} header" unless find_header name
 
+  constants.each do |constant|
+    abort "Can't find #{constant} constant in #{name} header" unless have_const constant, name
+  end
+
+  macroses.each do |macro|
+    abort "Can't find #{macro} macro in #{name} header" unless have_macro macro, name
+  end
+
   types.each do |type|
     abort "Can't find #{type} type in #{name} header" unless find_type type, nil, name
   end
 end
 
-require_header "zstd_errors.h", %w[
-  ZSTD_ErrorCode
-]
-require_header "zstd.h", [
-  "ZSTD_CCtx *",
-  "ZSTD_DCtx *",
-  "ZSTD_strategy",
-  "ZSTD_bounds",
-  "ZSTD_inBuffer",
-  "ZSTD_outBuffer"
-]
 require_header "zdict.h"
+require_header(
+  "zstd.h",
+  :constants => %w[
+    ZSTD_btlazy2
+    ZSTD_btopt
+    ZSTD_btultra
+    ZSTD_btultra2
+    ZSTD_c_chainLog
+    ZSTD_c_checksumFlag
+    ZSTD_c_compressionLevel
+    ZSTD_c_contentSizeFlag
+    ZSTD_c_dictIDFlag
+    ZSTD_c_enableLongDistanceMatching
+    ZSTD_c_hashLog
+    ZSTD_c_jobSize
+    ZSTD_c_ldmBucketSizeLog
+    ZSTD_c_ldmHashLog
+    ZSTD_c_ldmHashRateLog
+    ZSTD_c_ldmMinMatch
+    ZSTD_c_minMatch
+    ZSTD_c_nbWorkers
+    ZSTD_c_overlapLog
+    ZSTD_c_searchLog
+    ZSTD_c_strategy
+    ZSTD_c_targetLength
+    ZSTD_c_windowLog
+    ZSTD_dfast
+    ZSTD_d_windowLogMax
+    ZSTD_e_continue
+    ZSTD_e_end
+    ZSTD_e_flush
+    ZSTD_fast
+    ZSTD_greedy
+    ZSTD_lazy
+    ZSTD_lazy2
+  ],
+  :macroses  => %w[ZSTD_VERSION_STRING],
+  :types     => [
+    "ZSTD_bounds",
+    "ZSTD_CCtx *",
+    "ZSTD_DCtx *",
+    "ZSTD_inBuffer",
+    "ZSTD_outBuffer",
+    "ZSTD_strategy"
+  ]
+)
+
+require_header(
+  "zstd_errors.h",
+  :constants => %w[
+    ZSTD_error_checksum_wrong
+    ZSTD_error_corruption_detected
+    ZSTD_error_dictionaryCreation_failed
+    ZSTD_error_dictionary_corrupted
+    ZSTD_error_dictionary_wrong
+    ZSTD_error_dstBuffer_null
+    ZSTD_error_dstSize_tooSmall
+    ZSTD_error_frameParameter_unsupported
+    ZSTD_error_frameParameter_windowTooLarge
+    ZSTD_error_init_missing
+    ZSTD_error_maxSymbolValue_tooLarge
+    ZSTD_error_maxSymbolValue_tooSmall
+    ZSTD_error_memory_allocation
+    ZSTD_error_parameter_outOfBound
+    ZSTD_error_parameter_unsupported
+    ZSTD_error_prefix_unknown
+    ZSTD_error_srcSize_wrong
+    ZSTD_error_stage_wrong
+    ZSTD_error_tableLog_tooLarge
+    ZSTD_error_version_unsupported
+    ZSTD_error_workSpace_tooSmall
+  ],
+  :types     => %w[ZSTD_ErrorCode]
+)
 
 def require_library(name, functions)
   functions.each do |function|
@@ -36,29 +107,44 @@ def require_library(name, functions)
   end
 end
 
+# rubocop:disable Style/GlobalVars
+if find_library "zstd", "ZDICT_getDictHeaderSize"
+  $defs.push "-DHAVE_ZDICT_HEADER_SIZE"
+end
+
+zdict_has_params   = find_type "ZDICT_params_t", nil, "zdict.h"
+zdict_has_finalize = find_library "zstd", "ZDICT_finalizeDictionary"
+
+if zdict_has_params && zdict_has_finalize
+  $defs.push "-DHAVE_ZDICT_FINALIZE"
+end
+# rubocop:enable Style/GlobalVars
+
 require_library(
   "zstd",
   %w[
-    ZSTD_isError
-    ZSTD_getErrorCode
-    ZSTD_createCCtx
-    ZSTD_createDCtx
-    ZSTD_freeCCtx
-    ZSTD_freeDCtx
+    ZDICT_getDictID
+    ZDICT_isError
+    ZDICT_trainFromBuffer
+    ZSTD_CCtx_loadDictionary
     ZSTD_CCtx_setParameter
-    ZSTD_DCtx_setParameter
     ZSTD_CCtx_setPledgedSrcSize
-    ZSTD_cParam_getBounds
-    ZSTD_dParam_getBounds
     ZSTD_CStreamInSize
     ZSTD_CStreamOutSize
+    ZSTD_compressStream2
+    ZSTD_cParam_getBounds
+    ZSTD_createCCtx
+    ZSTD_createDCtx
+    ZSTD_DCtx_setParameter
+    ZSTD_DCtx_loadDictionary
     ZSTD_DStreamInSize
     ZSTD_DStreamOutSize
-    ZSTD_compressStream2
     ZSTD_decompressStream
-    ZDICT_isError
-    ZDICT_getDictID
-    ZDICT_trainFromBuffer
+    ZSTD_dParam_getBounds
+    ZSTD_freeCCtx
+    ZSTD_freeDCtx
+    ZSTD_getErrorCode
+    ZSTD_isError
   ]
 )
 

data/ext/zstds_ext/buffer.c

@@ -5,8 +5,6 @@
 
 #include <zstd.h>
 
-#include "ruby.h"
-
 VALUE zstds_ext_create_string_buffer(VALUE length)
 {
   return rb_str_new(NULL, NUM2SIZET(length));

data/ext/zstds_ext/dictionary.c

@@ -6,14 +6,12 @@
 #include <string.h>
 #include <zdict.h>
 
-#include "ruby.h"
 #include "zstds_ext/buffer.h"
 #include "zstds_ext/error.h"
 #include "zstds_ext/gvl.h"
-#include "zstds_ext/macro.h"
 #include "zstds_ext/option.h"
 
-// -- initialization --
+// -- common --
 
 typedef struct
 {
@@ -21,43 +19,64 @@ typedef struct
   size_t      size;
 } sample_t;
 
-typedef struct
+static inline void check_raw_samples(VALUE raw_samples)
 {
-  const sample_t*    samples;
-  size_t             length;
-  char*              buffer;
-  size_t             capacity;
-  zstds_result_t     result;
-  zstds_ext_result_t ext_result;
-} train_args_t;
+  Check_Type(raw_samples, T_ARRAY);
 
-static inline void* train_wrapper(void* data)
+  size_t samples_length = RARRAY_LEN(raw_samples);
+
+  for (size_t index = 0; index < samples_length; index++) {
+    Check_Type(rb_ary_entry(raw_samples, index), T_STRING);
+  }
+}
+
+static inline sample_t* prepare_samples(VALUE raw_samples, size_t* samples_length_ptr)
 {
-  train_args_t*   args    = data;
-  const sample_t* samples = args->samples;
-  size_t          length  = args->length;
-  size_t          size    = 0;
+  size_t    samples_length = RARRAY_LEN(raw_samples);
+  sample_t* samples        = malloc(sizeof(sample_t) * samples_length);
+  if (samples == NULL) {
+    zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
+  }
 
-  for (size_t index = 0; index < length; index++) {
+  for (size_t index = 0; index < samples_length; index++) {
+    VALUE     raw_sample = rb_ary_entry(raw_samples, index);
+    sample_t* sample     = &samples[index];
+
+    sample->data = RSTRING_PTR(raw_sample);
+    sample->size = RSTRING_LEN(raw_sample);
+  }
+
+  *samples_length_ptr = samples_length;
+
+  return samples;
+}
+
+static inline zstds_ext_result_t prepare_samples_group(
+  const sample_t*    samples,
+  size_t             samples_length,
+  zstds_ext_byte_t** group_ptr,
+  size_t**           sizes_ptr)
+{
+  size_t size = 0;
+
+  for (size_t index = 0; index < samples_length; index++) {
     size += samples[index].size;
   }
 
   zstds_ext_byte_t* group = malloc(size);
   if (group == NULL) {
-    args->ext_result = ZSTDS_EXT_ERROR_ALLOCATE_FAILED;
-    return NULL;
+    return ZSTDS_EXT_ERROR_ALLOCATE_FAILED;
   }
 
-  size_t* sizes = malloc(length * sizeof(size_t));
+  size_t* sizes = malloc(samples_length * sizeof(size_t));
   if (sizes == NULL) {
     free(group);
-    args->ext_result = ZSTDS_EXT_ERROR_ALLOCATE_FAILED;
-    return NULL;
+    return ZSTDS_EXT_ERROR_ALLOCATE_FAILED;
   }
 
   size_t offset = 0;
 
-  for (size_t index = 0; index < length; index++) {
+  for (size_t index = 0; index < samples_length; index++) {
     const sample_t* sample_ptr  = &samples[index];
     size_t          sample_size = sample_ptr->size;
 
@@ -67,7 +86,38 @@ static inline void* train_wrapper(void* data)
     sizes[index] = sample_size;
   }
 
-  args->result = ZDICT_trainFromBuffer((void*) args->buffer, args->capacity, group, sizes, (unsigned int) length);
+  *group_ptr = group;
+  *sizes_ptr = sizes;
+
+  return 0;
+}
+
+// -- training --
+
+typedef struct
+{
+  const sample_t*    samples;
+  size_t             samples_length;
+  char*              buffer;
+  size_t             capacity;
+  zstds_result_t     result;
+  zstds_ext_result_t ext_result;
+} train_args_t;
+
+static inline void* train_wrapper(void* data)
+{
+  train_args_t* args = data;
+
+  zstds_ext_byte_t*  group;
+  size_t*            sizes;
+  zstds_ext_result_t result = prepare_samples_group(args->samples, args->samples_length, &group, &sizes);
+  if (result != 0) {
+    args->ext_result = result;
+    return NULL;
+  }
+
+  args->result =
+    ZDICT_trainFromBuffer((void*) args->buffer, args->capacity, group, sizes, (unsigned int) args->samples_length);
 
   free(group);
   free(sizes);
@@ -84,17 +134,13 @@ static inline void* train_wrapper(void* data)
 
 VALUE zstds_ext_train_dictionary_buffer(VALUE ZSTDS_EXT_UNUSED(self), VALUE raw_samples, VALUE options)
 {
-  Check_Type(raw_samples, T_ARRAY);
-
-  size_t length = RARRAY_LEN(raw_samples);
-
-  for (size_t index = 0; index < length; index++) {
-    Check_Type(rb_ary_entry(raw_samples, index), T_STRING);
-  }
-
+  check_raw_samples(raw_samples);
   Check_Type(options, T_HASH);
-  ZSTDS_EXT_GET_BOOL_OPTION(options, gvl);
   ZSTDS_EXT_GET_SIZE_OPTION(options, capacity);
+  ZSTDS_EXT_GET_BOOL_OPTION(options, gvl);
+
+  size_t    samples_length;
+  sample_t* samples = prepare_samples(raw_samples, &samples_length);
 
   if (capacity == 0) {
     capacity = ZSTDS_EXT_DEFAULT_DICTIONARY_CAPACITY;
@@ -107,27 +153,147 @@ VALUE zstds_ext_train_dictionary_buffer(VALUE ZSTDS_EXT_UNUSED(self), VALUE raw_
     zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
   }
 
-  sample_t* samples = malloc(sizeof(sample_t) * length);
-  if (samples == NULL) {
+  train_args_t args = {
+    .samples        = samples,
+    .samples_length = samples_length,
+    .buffer         = RSTRING_PTR(buffer),
+    .capacity       = capacity,
+  };
+
+  ZSTDS_EXT_GVL_WRAP(gvl, train_wrapper, &args);
+  free(samples);
+
+  if (args.ext_result != 0) {
+    zstds_ext_raise_error(args.ext_result);
+  }
+
+  ZSTDS_EXT_RESIZE_STRING_BUFFER(buffer, args.result, exception);
+  if (exception != 0) {
     zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
   }
 
-  for (size_t index = 0; index < length; index++) {
-    VALUE     raw_sample = rb_ary_entry(raw_samples, index);
-    sample_t* sample     = &samples[index];
+  return buffer;
+}
 
-    sample->data = RSTRING_PTR(raw_sample);
-    sample->size = RSTRING_LEN(raw_sample);
+// -- finalizing --
+
+#if defined(HAVE_ZDICT_FINALIZE)
+typedef struct
+{
+  const sample_t*                samples;
+  size_t                         samples_length;
+  char*                          buffer;
+  size_t                         max_size;
+  char*                          content;
+  size_t                         content_length;
+  zstds_ext_dictionary_options_t dictionary_options;
+  zstds_result_t                 result;
+  zstds_ext_result_t             ext_result;
+} finalize_args_t;
+
+static inline void* finalize_wrapper(void* data)
+{
+  finalize_args_t* args = data;
+
+  zstds_ext_byte_t*  group;
+  size_t*            sizes;
+  zstds_ext_result_t result = prepare_samples_group(args->samples, args->samples_length, &group, &sizes);
+  if (result != 0) {
+    args->ext_result = result;
+    return NULL;
   }
 
-  train_args_t args = {
-    .samples  = samples,
-    .length   = length,
-    .buffer   = RSTRING_PTR(buffer),
-    .capacity = capacity,
+  int                compressionLevel;
+  zstds_ext_option_t compression_level = args->dictionary_options.compression_level;
+  if (compression_level.has_value) {
+    compressionLevel = compression_level.value;
+  } else {
+    compressionLevel = 0;
+  }
+
+  unsigned int       notificationLevel;
+  zstds_ext_option_t notification_level = args->dictionary_options.notification_level;
+  if (notification_level.has_value) {
+    notificationLevel = notification_level.value;
+  } else {
+    notificationLevel = 0;
+  }
+
+  unsigned int       dictID;
+  zstds_ext_option_t dictionary_id = args->dictionary_options.dictionary_id;
+  if (dictionary_id.has_value) {
+    dictID = dictionary_id.value;
+  } else {
+    dictID = 0;
+  }
+
+  ZDICT_params_t dictionary_params = {
+    .compressionLevel  = compressionLevel,
+    .notificationLevel = notificationLevel,
+    .dictID            = dictID,
   };
 
-  ZSTDS_EXT_GVL_WRAP(gvl, train_wrapper, &args);
+  args->result = ZDICT_finalizeDictionary(
+    (void*) args->buffer,
+    args->max_size,
+    (void*) args->content,
+    args->content_length,
+    group,
+    sizes,
+    (unsigned int) args->samples_length,
+    dictionary_params);
+
+  free(group);
+  free(sizes);
+
+  if (ZDICT_isError(args->result)) {
+    args->ext_result = zstds_ext_get_error(ZSTD_getErrorCode(args->result));
+    return NULL;
+  }
+
+  args->ext_result = 0;
+
+  return NULL;
+}
+
+VALUE zstds_ext_finalize_dictionary_buffer(
+  VALUE ZSTDS_EXT_UNUSED(self),
+  VALUE content,
+  VALUE raw_samples,
+  VALUE options)
+{
+  Check_Type(content, T_STRING);
+  check_raw_samples(raw_samples);
+  Check_Type(options, T_HASH);
+  ZSTDS_EXT_GET_SIZE_OPTION(options, max_size);
+  ZSTDS_EXT_GET_BOOL_OPTION(options, gvl);
+  ZSTDS_EXT_GET_DICTIONARY_OPTIONS(options);
+
+  size_t    samples_length;
+  sample_t* samples = prepare_samples(raw_samples, &samples_length);
+
+  if (max_size == 0) {
+    max_size = ZSTDS_EXT_DEFAULT_DICTIONARY_MAX_SIZE;
+  }
+
+  int exception;
+
+  ZSTDS_EXT_CREATE_STRING_BUFFER(buffer, max_size, exception);
+  if (exception != 0) {
+    zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
+  }
+
+  finalize_args_t args = {
+    .samples            = samples,
+    .samples_length     = samples_length,
+    .buffer             = RSTRING_PTR(buffer),
+    .max_size           = max_size,
+    .content            = RSTRING_PTR(content),
+    .content_length     = RSTRING_LEN(content),
+    .dictionary_options = dictionary_options,
+  };
+
+  ZSTDS_EXT_GVL_WRAP(gvl, finalize_wrapper, &args);
   free(samples);
 
   if (args.ext_result != 0) {
@@ -142,6 +308,17 @@ VALUE zstds_ext_train_dictionary_buffer(VALUE ZSTDS_EXT_UNUSED(self), VALUE raw_
   return buffer;
 }
 
+#else
+ZSTDS_EXT_NORETURN VALUE zstds_ext_finalize_dictionary_buffer(
+  VALUE ZSTDS_EXT_UNUSED(self),
+  VALUE ZSTDS_EXT_UNUSED(content),
+  VALUE ZSTDS_EXT_UNUSED(raw_samples),
+  VALUE ZSTDS_EXT_UNUSED(options))
+{
+  zstds_ext_raise_error(ZSTDS_EXT_ERROR_NOT_IMPLEMENTED);
+}
+#endif // HAVE_ZDICT_FINALIZE
+
 // -- other --
 
 VALUE zstds_ext_get_dictionary_buffer_id(VALUE ZSTDS_EXT_UNUSED(self), VALUE buffer)
@@ -154,12 +331,33 @@ VALUE zstds_ext_get_dictionary_buffer_id(VALUE ZSTDS_EXT_UNUSED(self), VALUE buf
   return UINT2NUM(id);
 }
 
+#if defined(HAVE_ZDICT_HEADER_SIZE)
+VALUE zstds_ext_get_dictionary_header_size(VALUE ZSTDS_EXT_UNUSED(self), VALUE buffer)
+{
+  zstds_result_t result = ZDICT_getDictHeaderSize(RSTRING_PTR(buffer), RSTRING_LEN(buffer));
+  if (ZDICT_isError(result)) {
+    zstds_ext_raise_error(zstds_ext_get_error(ZSTD_getErrorCode(result)));
+  }
+
+  return SIZET2NUM(result);
+}
+
+#else
+ZSTDS_EXT_NORETURN VALUE
+  zstds_ext_get_dictionary_header_size(VALUE ZSTDS_EXT_UNUSED(self), VALUE ZSTDS_EXT_UNUSED(buffer))
+{
+  zstds_ext_raise_error(ZSTDS_EXT_ERROR_NOT_IMPLEMENTED);
+};
+#endif // HAVE_ZDICT_HEADER_SIZE
+
 // -- exports --
 
 void zstds_ext_dictionary_exports(VALUE root_module)
 {
   VALUE dictionary = rb_define_class_under(root_module, "Dictionary", rb_cObject);
 
+  rb_define_singleton_method(dictionary, "finalize_buffer", zstds_ext_finalize_dictionary_buffer, 3);
   rb_define_singleton_method(dictionary, "get_buffer_id", zstds_ext_get_dictionary_buffer_id, 1);
+  rb_define_singleton_method(dictionary, "get_header_size", zstds_ext_get_dictionary_header_size, 1);
   rb_define_singleton_method(dictionary, "train_buffer", zstds_ext_train_dictionary_buffer, 2);
 }

data/ext/zstds_ext/dictionary.h

@@ -6,11 +6,33 @@
 
 #include "ruby.h"
 
+#include "zstds_ext/macro.h"
+
 #define ZSTDS_EXT_DEFAULT_DICTIONARY_CAPACITY (1 << 17); // 128 KB
+#define ZSTDS_EXT_DEFAULT_DICTIONARY_MAX_SIZE ZSTDS_EXT_DEFAULT_DICTIONARY_CAPACITY
+
+// -- training --
 
 VALUE zstds_ext_train_dictionary_buffer(VALUE self, VALUE samples, VALUE options);
+
+// -- finalizing --
+
+#if defined(HAVE_ZDICT_FINALIZE)
+VALUE zstds_ext_finalize_dictionary_buffer(VALUE self, VALUE content, VALUE samples, VALUE options);
+#else
+ZSTDS_EXT_NORETURN VALUE zstds_ext_finalize_dictionary_buffer(VALUE self, VALUE content, VALUE samples, VALUE options);
+#endif // HAVE_ZDICT_FINALIZE
+
+// -- other --
+
 VALUE zstds_ext_get_dictionary_buffer_id(VALUE self, VALUE buffer);
 
+#if defined(HAVE_ZDICT_HEADER_SIZE)
+VALUE zstds_ext_get_dictionary_header_size(VALUE self, VALUE buffer);
+#else
+ZSTDS_EXT_NORETURN VALUE zstds_ext_get_dictionary_header_size(VALUE self, VALUE buffer);
+#endif // HAVE_ZDICT_HEADER_SIZE
+
 void zstds_ext_dictionary_exports(VALUE root_module);
 
 #endif // ZSTDS_EXT_DICTIONARY_H

data/ext/zstds_ext/error.c

@@ -3,10 +3,6 @@
 
 #include "zstds_ext/error.h"
 
-#include <zstd_errors.h>
-
-#include "ruby.h"
-
 zstds_ext_result_t zstds_ext_get_error(ZSTD_ErrorCode error_code)
 {
   switch (error_code) {
@@ -73,6 +69,9 @@ void zstds_ext_raise_error(zstds_ext_result_t ext_result)
     case ZSTDS_EXT_ERROR_WRITE_IO:
       raise_error("WriteIOError", "failed to write IO");
 
+    case ZSTDS_EXT_ERROR_NOT_IMPLEMENTED:
+      raise_error("NotImplementedError", "not implemented error");
+
     default:
       // ZSTDS_EXT_ERROR_UNEXPECTED
       raise_error("UnexpectedError", "unexpected error");

data/ext/zstds_ext/error.h

@@ -26,6 +26,7 @@ enum
   ZSTDS_EXT_ERROR_READ_IO,
   ZSTDS_EXT_ERROR_WRITE_IO,
 
+  ZSTDS_EXT_ERROR_NOT_IMPLEMENTED,
   ZSTDS_EXT_ERROR_UNEXPECTED
 };
 

data/ext/zstds_ext/gvl.h

@@ -4,7 +4,7 @@
 #if !defined(ZSTDS_EXT_GVL_H)
 #define ZSTDS_EXT_GVL_H
 
-#ifdef HAVE_RB_THREAD_CALL_WITHOUT_GVL
+#if defined(HAVE_RB_THREAD_CALL_WITHOUT_GVL)
 
 #include "ruby/thread.h"
 
@@ -19,6 +19,6 @@
 
 #define ZSTDS_EXT_GVL_WRAP(_gvl, function, data) function((void*) data);
 
-#endif
+#endif // HAVE_RB_THREAD_CALL_WITHOUT_GVL
 
 #endif // ZSTDS_EXT_GVL_H

data/ext/zstds_ext/io.c

@@ -7,7 +7,6 @@
 #include <string.h>
 #include <zstd.h>
 
-#include "ruby.h"
 #include "ruby/io.h"
 #include "zstds_ext/error.h"
 #include "zstds_ext/gvl.h"

data/ext/zstds_ext/macro.h

@@ -8,6 +8,12 @@
 #define ZSTDS_EXT_UNUSED(x) x __attribute__((__unused__))
 #else
 #define ZSTDS_EXT_UNUSED(x) x
-#endif
+#endif // __GNUC__
+
+#if defined(__GNUC__)
+#define ZSTDS_EXT_NORETURN __attribute__((__noreturn__))
+#else
+#define ZSTDS_EXT_NORETURN
+#endif // __GNUC__
 
 #endif // ZSTDS_EXT_MACRO_H

data/ext/zstds_ext/main.c

@@ -1,7 +1,6 @@
 // Ruby bindings for zstd library.
 // Copyright (c) 2019 AUTHORS, MIT License.
 
-#include "ruby.h"
 #include "zstds_ext/buffer.h"
 #include "zstds_ext/dictionary.h"
 #include "zstds_ext/io.h"
@@ -21,4 +20,7 @@ void Init_zstds_ext()
   zstds_ext_compressor_exports(root_module);
   zstds_ext_decompressor_exports(root_module);
   zstds_ext_string_exports(root_module);
+
+  VALUE version = rb_str_new2(ZSTD_VERSION_STRING);
+  rb_define_const(root_module, "LIBRARY_VERSION", rb_obj_freeze(version));
 }

data/ext/zstds_ext/option.c

@@ -3,9 +3,6 @@
 
 #include "zstds_ext/option.h"
 
-#include <zstd.h>
-
-#include "ruby.h"
 #include "zstds_ext/dictionary.h"
 #include "zstds_ext/error.h"
 

data/ext/zstds_ext/option.h

@@ -68,6 +68,13 @@ typedef struct
   VALUE              dictionary;
 } zstds_ext_decompressor_options_t;
 
+typedef struct
+{
+  zstds_ext_option_t compression_level;
+  zstds_ext_option_t notification_level;
+  zstds_ext_option_t dictionary_id;
+} zstds_ext_dictionary_options_t;
+
 void zstds_ext_resolve_option(
   VALUE                   options,
   zstds_ext_option_t*     option,
@@ -117,6 +124,13 @@ void zstds_ext_resolve_dictionary_option(VALUE options, VALUE* option, const cha
   ZSTDS_EXT_RESOLVE_OPTION(options, decompressor_options, ZSTDS_EXT_OPTION_TYPE_UINT, window_log_max); \
   ZSTDS_EXT_RESOLVE_DICTIONARY_OPTION(options, decompressor_options, dictionary);
 
+#define ZSTDS_EXT_GET_DICTIONARY_OPTIONS(options)                                                        \
+  zstds_ext_dictionary_options_t dictionary_options;                                                     \
+                                                                                                         \
+  ZSTDS_EXT_RESOLVE_OPTION(options, dictionary_options, ZSTDS_EXT_OPTION_TYPE_INT, compression_level);   \
+  ZSTDS_EXT_RESOLVE_OPTION(options, dictionary_options, ZSTDS_EXT_OPTION_TYPE_UINT, notification_level); \
+  ZSTDS_EXT_RESOLVE_OPTION(options, dictionary_options, ZSTDS_EXT_OPTION_TYPE_UINT, dictionary_id);
+
 bool   zstds_ext_get_bool_option_value(VALUE options, const char* name);
 size_t zstds_ext_get_size_option_value(VALUE options, const char* name);
 

data/ext/zstds_ext/stream/compressor.c

@@ -3,9 +3,6 @@
 
 #include "zstds_ext/stream/compressor.h"
 
-#include <zstd.h>
-
-#include "ruby.h"
 #include "zstds_ext/error.h"
 #include "zstds_ext/gvl.h"
 #include "zstds_ext/option.h"

data/ext/zstds_ext/stream/decompressor.c

@@ -3,9 +3,6 @@
 
 #include "zstds_ext/stream/decompressor.h"
 
-#include <zstd.h>
-
-#include "ruby.h"
 #include "zstds_ext/error.h"
 #include "zstds_ext/gvl.h"
 #include "zstds_ext/option.h"

data/ext/zstds_ext/string.c

@@ -5,7 +5,6 @@
 
 #include <zstd.h>
 
-#include "ruby.h"
 #include "zstds_ext/buffer.h"
 #include "zstds_ext/error.h"
 #include "zstds_ext/gvl.h"

data/lib/zstds/dictionary.rb

@@ -14,6 +14,20 @@ module ZSTDS
     }
     .freeze
 
+    FINALIZE_DEFAULTS = {
+      :gvl                => false,
+      :max_size           => 0,
+      :dictionary_options => {}
+    }
+    .freeze
+
+    FINALIZE_DICTIONARY_DEFAULTS = {
+      :compression_level  => 0,
+      :notification_level => 0,
+      :dictionary_id      => 0
+    }
+    .freeze
+
     attr_reader :buffer
 
     def initialize(buffer)
@@ -23,17 +37,8 @@ module ZSTDS
       @buffer = buffer
     end
 
-    def id
-      self.class.get_buffer_id @buffer
-    end
-
     def self.train(samples, options = {})
-      Validation.validate_array samples
-
-      samples.each do |sample|
-        Validation.validate_string sample
-        raise ValidateError, "dictionary sample should not be empty" if sample.empty?
-      end
+      validate_samples samples
 
       Validation.validate_hash options
 
@@ -45,5 +50,50 @@ module ZSTDS
       buffer = train_buffer samples, options
       new buffer
     end
+
+    def self.finalize(content, samples, options = {})
+      Validation.validate_string content
+      raise ValidateError, "content should not be empty" if content.empty?
+
+      validate_samples samples
+
+      Validation.validate_hash options
+
+      options = FINALIZE_DEFAULTS.merge options
+
+      Validation.validate_bool                 options[:gvl]
+      Validation.validate_not_negative_integer options[:max_size]
+      Validation.validate_hash                 options[:dictionary_options]
+
+      dictionary_options = FINALIZE_DICTIONARY_DEFAULTS.merge options[:dictionary_options]
+
+      compression_level = dictionary_options[:compression_level]
+      Validation.validate_integer compression_level
+      raise ValidateError, "invalid compression level" if
+        compression_level < Option::MIN_COMPRESSION_LEVEL || compression_level > Option::MAX_COMPRESSION_LEVEL
+
+      Validation.validate_not_negative_integer dictionary_options[:notification_level]
+      Validation.validate_not_negative_integer dictionary_options[:dictionary_id]
+
+      buffer = finalize_buffer content, samples, options
+      new buffer
+    end
+
+    def self.validate_samples(samples)
+      Validation.validate_array samples
+
+      samples.each do |sample|
+        Validation.validate_string sample
+        raise ValidateError, "dictionary sample should not be empty" if sample.empty?
+      end
+    end
+
+    def id
+      self.class.get_buffer_id @buffer
+    end
+
+    def header_size
+      self.class.get_header_size @buffer
+    end
   end
 end

data/lib/zstds/error.rb

@@ -18,5 +18,6 @@ module ZSTDS
   class ReadIOError   < BaseError; end
   class WriteIOError  < BaseError; end
 
-  class UnexpectedError < BaseError; end
+  class NotImplementedError < BaseError; end
+  class UnexpectedError     < BaseError; end
 end

data/lib/zstds/stream/abstract.rb

@@ -24,9 +24,7 @@ module ZSTDS
 
       def initialize(io, options = {})
         @raw_stream = create_raw_stream
-
-        Validation.validate_io io
-        @io = io
+        @io         = io
 
         @stat = Stat.new @io.stat if @io.respond_to? :stat
 
@@ -135,13 +133,19 @@ module ZSTDS
       end
 
       def close
-        @io.close
+        @io.close if @io.respond_to? :close
 
         nil
       end
 
       def closed?
-        @raw_stream.closed? && @io.closed?
+        return false unless @raw_stream.closed?
+
+        if @io.respond_to? :closed
+          @io.closed?
+        else
+          true
+        end
       end
 
       def to_io

data/lib/zstds/stream/reader.rb

@@ -54,6 +54,9 @@ module ZSTDS
         Validation.validate_not_negative_integer bytes_to_read unless bytes_to_read.nil?
         Validation.validate_string out_buffer unless out_buffer.nil?
 
+        raise ValidateError, "io should be responsible to read and eof" unless
+          @io.respond_to?(:read) && @io.respond_to?(:eof?)
+
         unless bytes_to_read.nil?
           return ::String.new :encoding => ::Encoding::BINARY if bytes_to_read.zero?
           return nil if eof?
@@ -86,16 +89,22 @@ module ZSTDS
       end
 
       def eof?
+        raise ValidateError, "io should be responsible to eof" unless @io.respond_to? :eof?
+
         empty? && @io.eof?
       end
 
       # -- asynchronous --
 
       def readpartial(bytes_to_read, out_buffer = nil)
+        raise ValidateError, "io should be responsible to readpartial" unless @io.respond_to? :readpartial
+
         read_more_nonblock(bytes_to_read, out_buffer) { @io.readpartial @source_buffer_length }
       end
 
       def read_nonblock(bytes_to_read, out_buffer = nil, *options)
+        raise ValidateError, "io should be responsible to read nonblock" unless @io.respond_to? :read_nonblock
+
         read_more_nonblock(bytes_to_read, out_buffer) { @io.read_nonblock(@source_buffer_length, *options) }
       end
 

data/lib/zstds/stream/writer.rb

@@ -23,6 +23,8 @@ module ZSTDS
       # -- synchronous --
 
       def write(*objects)
+        validate_write
+
         write_remaining_buffer
 
         bytes_written = 0
@@ -38,20 +40,26 @@ module ZSTDS
       end
 
       def flush
+        validate_write
+
         finish :flush
 
-        @io.flush
+        @io.flush if @io.respond_to? :flush
 
         self
       end
 
       def rewind
+        validate_write
+
         finish :close
 
         super
       end
 
       def close
+        validate_write
+
         finish :close
 
         super
@@ -75,6 +83,10 @@ module ZSTDS
         @raw_stream.send(method_name, *args) { |portion| @io.write portion }
       end
 
+      def validate_write
+        raise ValidateError, "io should be responsible to write" unless @io.respond_to? :write
+      end
+
       # -- asynchronous --
 
       # IO write nonblock can raise wait writable error.
@@ -83,6 +95,8 @@ module ZSTDS
       # So we have to accept content after processing IO write nonblock.
       # It means that first write nonblock won't call IO write nonblock.
       def write_nonblock(object, *options)
+        validate_write_nonblock
+
         return 0 unless write_remaining_buffer_nonblock(*options)
 
         source         = transcode object.to_s
@@ -93,14 +107,18 @@ module ZSTDS
       end
 
       def flush_nonblock(*options)
+        validate_write_nonblock
+
         return false unless finish_nonblock :flush, *options
 
-        @io.flush
+        @io.flush if @io.respond_to? :flush
 
         true
       end
 
       def rewind_nonblock(*options)
+        validate_write_nonblock
+
         return false unless finish_nonblock :close, *options
 
         method(:rewind).super_method.call
@@ -109,6 +127,8 @@ module ZSTDS
       end
 
       def close_nonblock(*options)
+        validate_write_nonblock
+
         return false unless finish_nonblock :close, *options
 
         method(:close).super_method.call
@@ -139,6 +159,10 @@ module ZSTDS
         @raw_stream.send(method_name, *args) { |portion| @buffer << portion }
       end
 
+      def validate_write_nonblock
+        raise ValidateError, "io should be responsible to write nonblock" unless @io.respond_to? :write_nonblock
+      end
+
       # -- common --
 
       protected def transcode(data)

data/lib/zstds/validation.rb

@@ -5,33 +5,34 @@ require_relative "error"
 
 module ZSTDS
   module Validation
-    IO_METHODS = %i[
-      read
-      write
-      readpartial
-      read_nonblock
-      write_nonblock
-      eof?
-      flush
-      close
-      closed?
-    ]
-    .freeze
+    def self.validate_array(value)
+      raise ValidateError, "invalid array" unless value.is_a? ::Array
+    end
 
     def self.validate_bool(value)
       raise ValidateError, "invalid bool" unless value.is_a?(::TrueClass) || value.is_a?(::FalseClass)
     end
 
+    def self.validate_hash(value)
+      raise ValidateError, "invalid hash" unless value.is_a? ::Hash
+    end
+
     def self.validate_integer(value)
       raise ValidateError, "invalid integer" unless value.is_a? ::Integer
     end
 
+    def self.validate_not_negative_integer(value)
+      raise ValidateError, "invalid not negative integer" unless value.is_a?(::Integer) && value >= 0
+    end
+
     def self.validate_positive_integer(value)
       raise ValidateError, "invalid positive integer" unless value.is_a?(::Integer) && value.positive?
     end
 
-    def self.validate_not_negative_integer(value)
-      raise ValidateError, "invalid not negative integer" unless value.is_a?(::Integer) && value >= 0
+    def self.validate_proc(value)
+      unless value.is_a?(::Proc) || value.is_a?(::Method) || value.is_a?(::UnboundMethod)
+        raise ValidateError, "invalid proc"
+      end
     end
 
     def self.validate_string(value)
@@ -41,23 +42,5 @@ module ZSTDS
     def self.validate_symbol(value)
       raise ValidateError, "invalid symbol" unless value.is_a? ::Symbol
     end
-
-    def self.validate_io(value)
-      raise ValidateError, "invalid io" unless IO_METHODS.all? { |method| value.respond_to? method }
-    end
-
-    def self.validate_array(value)
-      raise ValidateError, "invalid array" unless value.is_a? ::Array
-    end
-
-    def self.validate_hash(value)
-      raise ValidateError, "invalid hash" unless value.is_a? ::Hash
-    end
-
-    def self.validate_proc(value)
-      unless value.is_a?(::Proc) || value.is_a?(::Method) || value.is_a?(::UnboundMethod)
-        raise ValidateError, "invalid proc"
-      end
-    end
   end
 end

data/lib/zstds/version.rb

@@ -2,5 +2,5 @@
 # Copyright (c) 2019 AUTHORS, MIT License.
 
 module ZSTDS
-  VERSION = "1.1.1".freeze
+  VERSION = "1.3.0".freeze
 end

metadata

@@ -1,14 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: ruby-zstds
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.3.0
 platform: ruby
 authors:
 - Andrew Aladjev
+- Ivan Takarlikov
+- Jenner La Fave
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-04 00:00:00.000000000 Z
+date: 2022-03-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: codecov
@@ -58,28 +60,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.14'
+        version: '5.15'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.14'
+        version: '5.15'
 - !ruby/object:Gem::Dependency
   name: ocg
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
@@ -128,56 +130,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '1.26'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '1.26'
 - !ruby/object:Gem::Dependency
   name: rubocop-minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.12'
+        version: '0.17'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.12'
+        version: '0.17'
 - !ruby/object:Gem::Dependency
   name: rubocop-performance
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '1.13'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '1.13'
 - !ruby/object:Gem::Dependency
   name: rubocop-rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.6'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -244,7 +246,8 @@ files:
 homepage: https://github.com/andrew-aladev/ruby-zstds
 licenses:
 - MIT
-metadata: {}
+metadata:
+  rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -260,7 +263,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.15
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: Ruby bindings for zstd library.