ruby-zstds 1.0.6 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 359ef05325e4af260831df1c3d71b70d64d1b4f235631ad3cd0bf10be63be308
-  data.tar.gz: 39b138930c23a914317db548081db091a607b00867300788d3de3c22e7e56616
+  metadata.gz: 80ccd3c8a7210dcf3fabdfadcca8e37ce82d5fe78c7f761f5ae7c355ff746854
+  data.tar.gz: a4ed4e5dd4c89b7f95438c5e3033d09eb25721ab32fda8e1eac9351774ab3215
 SHA512:
-  metadata.gz: 0c76ce3fe240fb963dab49e55ff167a2cfea690e1bb48b107d164821d5bba1b5090eb81b73ec221a86aa4c94c05f9082d540d6c0423ae99326face5deee199d8
-  data.tar.gz: 00db3a70938e3ca436d253f9c2e715941837c252569bb6d72d1ddb94b1c8c9e85aeeb1bd67e9c85a0ab965c5393f785f273e67a0a3869a1ae770a915ae23ca9e
+  metadata.gz: 967b5d866cd79b139bdbca97e1fdacf9063e3bc425f441be239980dd839d123173fcad15c0830576a32fe7c0bb5166f4fd5159717a9ac7f038fe57a435859f70
+  data.tar.gz: 78c54fc20f8facaa556eb68438407cfa7ff96d33425a9dce73584fc26435177dda8478c729a412f00d9b28a9d860b3dec6a059a728926a587f3a83dae9fe1836

data/README.md

@@ -1,14 +1,14 @@
 # Ruby bindings for zstd library
 
-| Travis | AppVeyor | Circle | Codecov |
-| :---:  | :---:    | :---:  | :---:   |
-| [![Travis test status](https://travis-ci.com/andrew-aladev/ruby-zstds.svg?branch=master)](https://travis-ci.com/andrew-aladev/ruby-zstds) | [![AppVeyor test status](https://ci.appveyor.com/api/projects/status/github/andrew-aladev/ruby-zstds?branch=master&svg=true)](https://ci.appveyor.com/project/andrew-aladev/ruby-zstds/branch/master) | [![Circle test status](https://circleci.com/gh/andrew-aladev/ruby-zstds/tree/master.svg?style=shield)](https://circleci.com/gh/andrew-aladev/ruby-zstds/tree/master) | [![Codecov](https://codecov.io/gh/andrew-aladev/ruby-zstds/branch/master/graph/badge.svg)](https://codecov.io/gh/andrew-aladev/ruby-zstds) |
+| AppVeyor | Circle | Github actions | Codecov | Gem  |
+| :------: | :----: | :------------: | :-----: | :--: |
+| [![AppVeyor test status](https://ci.appveyor.com/api/projects/status/github/andrew-aladev/ruby-zstds?branch=master&svg=true)](https://ci.appveyor.com/project/andrew-aladev/ruby-zstds/branch/master) | [![Circle test status](https://circleci.com/gh/andrew-aladev/ruby-zstds/tree/master.svg?style=shield)](https://circleci.com/gh/andrew-aladev/ruby-zstds/tree/master) | [![Github Actions test status](https://github.com/andrew-aladev/ruby-zstds/workflows/test/badge.svg?branch=master)](https://github.com/andrew-aladev/ruby-zstds/actions) | [![Codecov](https://codecov.io/gh/andrew-aladev/ruby-zstds/branch/master/graph/badge.svg)](https://codecov.io/gh/andrew-aladev/ruby-zstds) | [![Gem](https://img.shields.io/gem/v/ruby-zstds.svg)](https://rubygems.org/gems/ruby-zstds) |
 
 See [zstd library](https://github.com/facebook/zstd).
 
 ## Installation
 
-Please install zstd library first, use latest 1.4.3+ version.
+Please install zstd library first, use latest 1.4.0+ version.
 
 ```sh
 gem install ruby-zstds
@@ -70,16 +70,16 @@ require "zstds"
 samples = (Array.new(8) { ::SecureRandom.random_bytes(1 << 8) } + ["sample string"]).shuffle
 
 dictionary = ZSTDS::Dictionary.train samples
-File.write "dictionary.bin", dictionary.buffer
+File.write "dictionary.bin", dictionary.buffer, :mode => "wb"
 
-dictionary_buffer = File.read "dictionary.bin"
+dictionary_buffer = File.read "dictionary.bin", :mode => "rb"
 dictionary        = ZSTDS::Dictionary.new dictionary_buffer
 
 data = ZSTDS::String.compress "sample string", :dictionary => dictionary
 puts ZSTDS::String.decompress(data, :dictionary => dictionary)
 ```
 
-You can create and read `tar.zst` archives with `minitar` for example.
+You can create and read `tar.zst` archives with [minitar](https://github.com/halostatue/minitar).
 
 ```ruby
 require "zstds"
@@ -101,12 +101,37 @@ ZSTDS::Stream::Reader.open "file.tar.zst" do |reader|
 end
 ```
 
+You can also use `Content-Encoding: zstd` with [sinatra](http://sinatrarb.com):
+
+```ruby
+require "zstds"
+require "sinatra"
+
+get "/" do
+  headers["Content-Encoding"] = "zstd"
+  ZSTDS::String.compress "sample string"
+end
+```
+
+All functionality (including streaming) can be used inside multiple threads with [parallel](https://github.com/grosser/parallel).
+This code will provide heavy load for your CPU.
+
+```ruby
+require "zstds"
+require "parallel"
+
+Parallel.each large_datas do |large_data|
+  ZSTDS::String.compress large_data
+end
+```
+
 ## Options
 
 | Option                          | Values         | Default    | Description |
 |---------------------------------|----------------|------------|-------------|
 | `source_buffer_length`          | 0 - inf        | 0 (auto)   | internal buffer length for source data |
 | `destination_buffer_length`     | 0 - inf        | 0 (auto)   | internal buffer length for description data |
+| `gvl`                           | true/false     | false      | enables global VM lock where possible |
 | `compression_level`             | -131072 - 22   | 0 (auto)   | compression level |
 | `window_log`                    | 10 - 31        | 0 (auto)   | maximum back-reference distance (power of 2) |
 | `hash_log`                      | 6 - 30         | 0 (auto)   | size of the initial probe table (power of 2) |
@@ -134,6 +159,10 @@ There are internal buffers for compressed and decompressed data.
 For example you want to use 1 KB as `source_buffer_length` for compressor - please use 256 B as `destination_buffer_length`.
 You want to use 256 B as `source_buffer_length` for decompressor - please use 1 KB as `destination_buffer_length`.
 
+`gvl` is disabled by default, this mode allows running multiple compressors/decompressors in different threads simultaneously.
+Please consider enabling `gvl` if you don't want to launch processors in separate threads.
+If `gvl` is enabled ruby won't waste time on acquiring/releasing VM lock.
+
 `String` and `File` will set `:pledged_size` automaticaly.
 
 You can also read zstd docs for more info about options.
@@ -161,6 +190,7 @@ Possible compressor options:
 ```
 :source_buffer_length
 :destination_buffer_length
+:gvl
 :compression_level
 :window_log
 :hash_log
@@ -188,6 +218,7 @@ Possible decompressor options:
 ```
 :source_buffer_length
 :destination_buffer_length
+:gvl
 :window_log_max
 :dictionary
 ```
@@ -201,18 +232,6 @@ data = ZSTDS::String.compress "sample string", :compression_level => 5
 puts ZSTDS::String.decompress(data, :window_log_max => 11)
 ```
 
-HTTP encoding (`Content-Encoding: zstd`) using default options:
-
-```ruby
-require "zstds"
-require "sinatra"
-
-get "/" do
-  headers["Content-Encoding"] = "zstd"
-  ZSTDS::String.compress "sample string"
-end
-```
-
 ## String
 
 String maintains destination buffer only, so it accepts `destination_buffer_length` option only.
@@ -237,7 +256,7 @@ File maintains both source and destination buffers, it accepts both `source_buff
 
 ## Stream::Writer
 
-Its behaviour is similar to builtin [`Zlib::GzipWriter`](https://ruby-doc.org/stdlib-2.7.0/libdoc/zlib/rdoc/Zlib/GzipWriter.html).
+Its behaviour is similar to builtin [`Zlib::GzipWriter`](https://ruby-doc.org/stdlib/libdoc/zlib/rdoc/Zlib/GzipWriter.html).
 
 Writer maintains destination buffer only, so it accepts `destination_buffer_length` option only.
 
@@ -275,7 +294,7 @@ Set another encodings, `nil` is just for compatibility with `IO`.
 #tell
 ```
 
-See [`IO`](https://ruby-doc.org/core-2.7.0/IO.html) docs.
+See [`IO`](https://ruby-doc.org/core/IO.html) docs.
 
 ```
 #write(*objects)
@@ -285,7 +304,7 @@ See [`IO`](https://ruby-doc.org/core-2.7.0/IO.html) docs.
 #closed?
 ```
 
-See [`Zlib::GzipWriter`](https://ruby-doc.org/stdlib-2.7.0/libdoc/zlib/rdoc/Zlib/GzipWriter.html) docs.
+See [`Zlib::GzipWriter`](https://ruby-doc.org/stdlib/libdoc/zlib/rdoc/Zlib/GzipWriter.html) docs.
 
 ```
 #write_nonblock(object, *options)
@@ -299,6 +318,9 @@ Special asynchronous methods missing in `Zlib::GzipWriter`.
 So it is possible to have asynchronous variants for these synchronous methods.
 Behaviour is the same as `IO#write_nonblock` method.
 
+All nonblock operations for file will raise `EBADF` error on Windows.
+Setting file into nonblocking mode is [not available on Windows](https://github.com/ruby/ruby/blob/master/win32/win32.c#L4388).
+
 ```
 #<<(object)
 #print(*objects)
@@ -307,11 +329,11 @@ Behaviour is the same as `IO#write_nonblock` method.
 #puts(*objects)
 ```
 
-Typical helpers, see [`Zlib::GzipWriter`](https://ruby-doc.org/stdlib-2.7.0/libdoc/zlib/rdoc/Zlib/GzipWriter.html) docs.
+Typical helpers, see [`Zlib::GzipWriter`](https://ruby-doc.org/stdlib/libdoc/zlib/rdoc/Zlib/GzipWriter.html) docs.
 
 ## Stream::Reader
 
-Its behaviour is similar to builtin [`Zlib::GzipReader`](https://ruby-doc.org/stdlib-2.7.0/libdoc/zlib/rdoc/Zlib/GzipReader.html).
+Its behaviour is similar to builtin [`Zlib::GzipReader`](https://ruby-doc.org/stdlib/libdoc/zlib/rdoc/Zlib/GzipReader.html).
 
 Reader maintains both source and destination buffers, it accepts both `source_buffer_length` and `destination_buffer_length` options.
 
@@ -346,7 +368,7 @@ Set another encodings.
 #tell
 ```
 
-See [`IO`](https://ruby-doc.org/core-2.7.0/IO.html) docs.
+See [`IO`](https://ruby-doc.org/core/IO.html) docs.
 
 ```
 #read(bytes_to_read = nil, out_buffer = nil)
@@ -356,14 +378,14 @@ See [`IO`](https://ruby-doc.org/core-2.7.0/IO.html) docs.
 #closed?
 ```
 
-See [`Zlib::GzipReader`](https://ruby-doc.org/stdlib-2.7.0/libdoc/zlib/rdoc/Zlib/GzipReader.html) docs.
+See [`Zlib::GzipReader`](https://ruby-doc.org/stdlib/libdoc/zlib/rdoc/Zlib/GzipReader.html) docs.
 
 ```
 #readpartial(bytes_to_read = nil, out_buffer = nil)
 #read_nonblock(bytes_to_read, out_buffer = nil, *options)
 ```
 
-See [`IO`](https://ruby-doc.org/core-2.7.0/IO.html) docs.
+See [`IO`](https://ruby-doc.org/core/IO.html) docs.
 
 ```
 #getbyte
@@ -386,7 +408,7 @@ See [`IO`](https://ruby-doc.org/core-2.7.0/IO.html) docs.
 #ungetline(line)
 ```
 
-Typical helpers, see [`Zlib::GzipReader`](https://ruby-doc.org/stdlib-2.7.0/libdoc/zlib/rdoc/Zlib/GzipReader.html) docs.
+Typical helpers, see [`Zlib::GzipReader`](https://ruby-doc.org/stdlib/libdoc/zlib/rdoc/Zlib/GzipReader.html) docs.
 
 ## Dictionary
 
@@ -418,12 +440,24 @@ Please use regular constructor to create dictionary from buffer.
 
 Read dictionary id from buffer.
 
+## Thread safety
+
+`:gvl` option is disabled by default, you can use bindings effectively in multiple threads.
+Please be careful: bindings are not thread safe.
+You should lock all shared data between threads.
+
+For example: you should not use same compressor/decompressor inside multiple threads.
+Please verify that you are using each processor inside single thread at the same time.
+
+## Operating systems
+
+GNU/Linux, FreeBSD, OSX, Windows (MinGW).
+
 ## CI
 
-See universal test script [scripts/ci_test.sh](scripts/ci_test.sh) for CI.
 Please visit [scripts/test-images](scripts/test-images).
-You can run this test script using many native and cross images.
+See universal test script [scripts/ci_test.sh](scripts/ci_test.sh) for CI.
 
 ## License
 
-MIT license, see LICENSE and AUTHORS.
+MIT license, see [LICENSE](LICENSE) and [AUTHORS](AUTHORS).

data/ext/extconf.rb

@@ -3,57 +3,142 @@
 
 require "mkmf"
 
+have_func "rb_thread_call_without_gvl", "ruby/thread.h"
+
+# Old zstd versions has bug: underlinking against pthreads.
+# https://bugs.gentoo.org/713940
 $LDFLAGS << " -pthread" # rubocop:disable Style/GlobalVars
 
-def require_header(name, types = [])
+def require_header(name, constants: [], macroses: [], types: [])
   abort "Can't find #{name} header" unless find_header name
 
+  constants.each do |constant|
+    abort "Can't find #{constant} constant in #{name} header" unless have_const constant, name
+  end
+
+  macroses.each do |macro|
+    abort "Can't find #{macro} macro in #{name} header" unless have_macro macro, name
+  end
+
   types.each do |type|
     abort "Can't find #{type} type in #{name} header" unless find_type type, nil, name
   end
 end
 
-require_header "zstd_errors.h", %w[
-  ZSTD_ErrorCode
-]
-require_header "zstd.h", [
-  "ZSTD_CCtx *",
-  "ZSTD_DCtx *",
-  "ZSTD_strategy",
-  "ZSTD_bounds",
-  "ZSTD_inBuffer",
-  "ZSTD_outBuffer"
-]
 require_header "zdict.h"
 
+require_header(
+  "zstd.h",
+  :constants => %w[
+    ZSTD_btlazy2
+    ZSTD_btopt
+    ZSTD_btultra
+    ZSTD_btultra2
+    ZSTD_c_chainLog
+    ZSTD_c_checksumFlag
+    ZSTD_c_compressionLevel
+    ZSTD_c_contentSizeFlag
+    ZSTD_c_dictIDFlag
+    ZSTD_c_enableLongDistanceMatching
+    ZSTD_c_hashLog
+    ZSTD_c_jobSize
+    ZSTD_c_ldmBucketSizeLog
+    ZSTD_c_ldmHashLog
+    ZSTD_c_ldmHashRateLog
+    ZSTD_c_ldmMinMatch
+    ZSTD_c_minMatch
+    ZSTD_c_nbWorkers
+    ZSTD_c_overlapLog
+    ZSTD_c_searchLog
+    ZSTD_c_strategy
+    ZSTD_c_targetLength
+    ZSTD_c_windowLog
+    ZSTD_dfast
+    ZSTD_d_windowLogMax
+    ZSTD_e_continue
+    ZSTD_e_end
+    ZSTD_e_flush
+    ZSTD_fast
+    ZSTD_greedy
+    ZSTD_lazy
+    ZSTD_lazy2
+  ],
+  :macroses  => %w[ZSTD_VERSION_STRING],
+  :types     => [
+    "ZSTD_bounds",
+    "ZSTD_CCtx *",
+    "ZSTD_DCtx *",
+    "ZSTD_inBuffer",
+    "ZSTD_outBuffer",
+    "ZSTD_strategy"
+  ]
+)
+
+require_header(
+  "zstd_errors.h",
+  :constants => %w[
+    ZSTD_error_checksum_wrong
+    ZSTD_error_corruption_detected
+    ZSTD_error_dictionaryCreation_failed
+    ZSTD_error_dictionary_corrupted
+    ZSTD_error_dictionary_wrong
+    ZSTD_error_dstBuffer_null
+    ZSTD_error_dstSize_tooSmall
+    ZSTD_error_frameParameter_unsupported
+    ZSTD_error_frameParameter_windowTooLarge
+    ZSTD_error_init_missing
+    ZSTD_error_maxSymbolValue_tooLarge
+    ZSTD_error_maxSymbolValue_tooSmall
+    ZSTD_error_memory_allocation
+    ZSTD_error_parameter_outOfBound
+    ZSTD_error_parameter_unsupported
+    ZSTD_error_prefix_unknown
+    ZSTD_error_srcSize_wrong
+    ZSTD_error_stage_wrong
+    ZSTD_error_tableLog_tooLarge
+    ZSTD_error_version_unsupported
+    ZSTD_error_workSpace_tooSmall
+  ],
+  :types     => %w[ZSTD_ErrorCode]
+)
+
 def require_library(name, functions)
   functions.each do |function|
     abort "Can't find #{function} function in #{name} library" unless find_library name, function
   end
 end
 
+# rubocop:disable Style/GlobalVars
+if find_library "zstd", "ZDICT_getDictHeaderSize"
+  $defs.push "-DHAVE_ZDICT_HEADER_SIZE"
+end
+# rubocop:enable Style/GlobalVars
+
 require_library(
   "zstd",
   %w[
-    ZSTD_isError
-    ZSTD_getErrorCode
-    ZSTD_createCCtx
-    ZSTD_createDCtx
-    ZSTD_freeCCtx
-    ZSTD_freeDCtx
+    ZDICT_getDictID
+    ZDICT_isError
+    ZDICT_trainFromBuffer
+    ZSTD_CCtx_loadDictionary
     ZSTD_CCtx_setParameter
-    ZSTD_DCtx_setParameter
     ZSTD_CCtx_setPledgedSrcSize
-    ZSTD_cParam_getBounds
-    ZSTD_dParam_getBounds
     ZSTD_CStreamInSize
     ZSTD_CStreamOutSize
+    ZSTD_compressStream2
+    ZSTD_cParam_getBounds
+    ZSTD_createCCtx
+    ZSTD_createDCtx
+    ZSTD_DCtx_setParameter
+    ZSTD_DCtx_loadDictionary
     ZSTD_DStreamInSize
     ZSTD_DStreamOutSize
-    ZSTD_compressStream2
     ZSTD_decompressStream
-    ZDICT_getDictID
-    ZDICT_trainFromBuffer
+    ZSTD_dParam_getBounds
+    ZSTD_freeCCtx
+    ZSTD_freeDCtx
+    ZSTD_getErrorCode
+    ZSTD_isError
   ]
 )
 
@@ -82,7 +167,7 @@ $libs = $libs.split(%r{\s})
   .uniq
   .join " "
 
-if ENV["CI"] || ENV["COVERAGE"]
+if ENV["CI"]
   $CFLAGS << " --coverage"
   $LDFLAGS << " --coverage"
 end

data/ext/zstds_ext/buffer.c

@@ -5,17 +5,16 @@
 
 #include <zstd.h>
 
-#include "ruby.h"
-
 VALUE zstds_ext_create_string_buffer(VALUE length)
 {
   return rb_str_new(NULL, NUM2SIZET(length));
 }
 
-VALUE zstds_ext_resize_string_buffer(VALUE args)
+VALUE zstds_ext_resize_string_buffer(VALUE buffer_args)
 {
-  VALUE buffer = rb_ary_entry(args, 0);
-  VALUE length = rb_ary_entry(args, 1);
+  VALUE buffer = rb_ary_entry(buffer_args, 0);
+  VALUE length = rb_ary_entry(buffer_args, 1);
+
   return rb_str_resize(buffer, NUM2SIZET(length));
 }
 

data/ext/zstds_ext/buffer.h

@@ -11,12 +11,12 @@ VALUE zstds_ext_create_string_buffer(VALUE length);
 #define ZSTDS_EXT_CREATE_STRING_BUFFER(buffer, length, exception) \
   VALUE buffer = rb_protect(zstds_ext_create_string_buffer, SIZET2NUM(length), &exception);
 
-VALUE zstds_ext_resize_string_buffer(VALUE args);
+VALUE zstds_ext_resize_string_buffer(VALUE buffer_args);
 
-#define ZSTDS_EXT_RESIZE_STRING_BUFFER(buffer, length, exception)            \
-  VALUE args = rb_ary_new_from_args(2, buffer, SIZET2NUM(length));           \
-  buffer     = rb_protect(zstds_ext_resize_string_buffer, args, &exception); \
-  RB_GC_GUARD(args);
+#define ZSTDS_EXT_RESIZE_STRING_BUFFER(buffer, length, exception)                          \
+  VALUE buffer_args = rb_ary_new_from_args(2, buffer, SIZET2NUM(length));                  \
+  buffer            = rb_protect(zstds_ext_resize_string_buffer, buffer_args, &exception); \
+  RB_GC_GUARD(buffer_args);
 
 void zstds_ext_buffer_exports(VALUE root_module);
 

data/ext/zstds_ext/dictionary.c

@@ -6,39 +6,93 @@
 #include <string.h>
 #include <zdict.h>
 
-#include "ruby.h"
 #include "zstds_ext/buffer.h"
 #include "zstds_ext/error.h"
-#include "zstds_ext/macro.h"
+#include "zstds_ext/gvl.h"
 #include "zstds_ext/option.h"
 
-VALUE zstds_ext_get_dictionary_buffer_id(VALUE ZSTDS_EXT_UNUSED(self), VALUE buffer)
+// -- initialization --
+
+typedef struct
 {
-  unsigned int id = ZDICT_getDictID(RSTRING_PTR(buffer), RSTRING_LEN(buffer));
-  if (id == 0) {
-    zstds_ext_raise_error(ZSTDS_EXT_ERROR_VALIDATE_FAILED);
+  const char* data;
+  size_t      size;
+} sample_t;
+
+typedef struct
+{
+  const sample_t*    samples;
+  size_t             length;
+  char*              buffer;
+  size_t             capacity;
+  zstds_result_t     result;
+  zstds_ext_result_t ext_result;
+} train_args_t;
+
+static inline void* train_wrapper(void* data)
+{
+  train_args_t*   args    = data;
+  const sample_t* samples = args->samples;
+  size_t          length  = args->length;
+  size_t          size    = 0;
+
+  for (size_t index = 0; index < length; index++) {
+    size += samples[index].size;
   }
 
-  return UINT2NUM(id);
+  zstds_ext_byte_t* group = malloc(size);
+  if (group == NULL) {
+    args->ext_result = ZSTDS_EXT_ERROR_ALLOCATE_FAILED;
+    return NULL;
+  }
+
+  size_t* sizes = malloc(length * sizeof(size_t));
+  if (sizes == NULL) {
+    free(group);
+    args->ext_result = ZSTDS_EXT_ERROR_ALLOCATE_FAILED;
+    return NULL;
+  }
+
+  size_t offset = 0;
+
+  for (size_t index = 0; index < length; index++) {
+    const sample_t* sample_ptr  = &samples[index];
+    size_t          sample_size = sample_ptr->size;
+
+    memmove(group + offset, sample_ptr->data, sample_size);
+    offset += sample_size;
+
+    sizes[index] = sample_size;
+  }
+
+  args->result = ZDICT_trainFromBuffer((void*) args->buffer, args->capacity, group, sizes, (unsigned int) length);
+
+  free(group);
+  free(sizes);
+
+  if (ZDICT_isError(args->result)) {
+    args->ext_result = zstds_ext_get_error(ZSTD_getErrorCode(args->result));
+    return NULL;
+  }
+
+  args->ext_result = 0;
+
+  return NULL;
 }
 
-VALUE zstds_ext_train_dictionary_buffer(VALUE ZSTDS_EXT_UNUSED(self), VALUE samples, VALUE options)
+VALUE zstds_ext_train_dictionary_buffer(VALUE ZSTDS_EXT_UNUSED(self), VALUE raw_samples, VALUE options)
 {
-  Check_Type(samples, T_ARRAY);
-
-  size_t       sample_index;
-  unsigned int samples_length = (unsigned int)RARRAY_LEN(samples);
-  size_t       samples_size   = 0;
+  Check_Type(raw_samples, T_ARRAY);
 
-  for (sample_index = 0; sample_index < samples_length; sample_index++) {
-    VALUE sample = rb_ary_entry(samples, sample_index);
-    Check_Type(sample, T_STRING);
+  size_t length = RARRAY_LEN(raw_samples);
 
-    samples_size += RSTRING_LEN(sample);
+  for (size_t index = 0; index < length; index++) {
+    Check_Type(rb_ary_entry(raw_samples, index), T_STRING);
   }
 
   Check_Type(options, T_HASH);
-  ZSTDS_EXT_GET_BUFFER_LENGTH_OPTION(options, capacity);
+  ZSTDS_EXT_GET_BOOL_OPTION(options, gvl);
+  ZSTDS_EXT_GET_SIZE_OPTION(options, capacity);
 
   if (capacity == 0) {
     capacity = ZSTDS_EXT_DEFAULT_DICTIONARY_CAPACITY;
@@ -51,53 +105,78 @@ VALUE zstds_ext_train_dictionary_buffer(VALUE ZSTDS_EXT_UNUSED(self), VALUE samp
     zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
   }
 
-  zstds_ext_byte_t* samples_buffer = malloc(samples_size);
-  if (samples_buffer == NULL) {
+  sample_t* samples = malloc(sizeof(sample_t) * length);
+  if (samples == NULL) {
     zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
   }
 
-  size_t* samples_sizes = malloc(samples_length * sizeof(size_t));
-  if (samples_sizes == NULL) {
-    free(samples_buffer);
-    zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
+  for (size_t index = 0; index < length; index++) {
+    VALUE     raw_sample = rb_ary_entry(raw_samples, index);
+    sample_t* sample     = &samples[index];
+
+    sample->data = RSTRING_PTR(raw_sample);
+    sample->size = RSTRING_LEN(raw_sample);
   }
 
-  size_t sample_offset = 0;
+  train_args_t args = {
+    .samples  = samples,
+    .length   = length,
+    .buffer   = RSTRING_PTR(buffer),
+    .capacity = capacity,
+  };
 
-  for (sample_index = 0; sample_index < samples_length; sample_index++) {
-    VALUE       sample      = rb_ary_entry(samples, sample_index);
-    const char* sample_data = RSTRING_PTR(sample);
-    size_t      sample_size = RSTRING_LEN(sample);
+  ZSTDS_EXT_GVL_WRAP(gvl, train_wrapper, &args);
+  free(samples);
 
-    memmove(samples_buffer + sample_offset, sample_data, sample_size);
-    sample_offset += sample_size;
+  if (args.ext_result != 0) {
+    zstds_ext_raise_error(args.ext_result);
+  }
 
-    samples_sizes[sample_index] = sample_size;
+  ZSTDS_EXT_RESIZE_STRING_BUFFER(buffer, args.result, exception);
+  if (exception != 0) {
+    zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
   }
 
-  zstds_result_t result = ZDICT_trainFromBuffer(
-    RSTRING_PTR(buffer), capacity,
-    samples_buffer, samples_sizes, samples_length);
+  return buffer;
+}
 
-  free(samples_buffer);
-  free(samples_sizes);
+// -- other --
 
-  if (ZSTD_isError(result)) {
-    zstds_ext_raise_error(zstds_ext_get_error(ZSTD_getErrorCode(result)));
+VALUE zstds_ext_get_dictionary_buffer_id(VALUE ZSTDS_EXT_UNUSED(self), VALUE buffer)
+{
+  unsigned int id = ZDICT_getDictID(RSTRING_PTR(buffer), RSTRING_LEN(buffer));
+  if (id == 0) {
+    zstds_ext_raise_error(ZSTDS_EXT_ERROR_VALIDATE_FAILED);
   }
 
-  ZSTDS_EXT_RESIZE_STRING_BUFFER(buffer, result, exception);
-  if (exception != 0) {
-    zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
+  return UINT2NUM(id);
+}
+
+#if defined(HAVE_ZDICT_HEADER_SIZE)
+VALUE zstds_ext_get_dictionary_header_size(VALUE self, VALUE buffer)
+{
+  zstds_result_t result = ZDICT_getDictHeaderSize(RSTRING_PTR(buffer), RSTRING_LEN(buffer));
+  if (ZDICT_isError(result)) {
+    zstds_ext_raise_error(zstds_ext_get_error(ZSTD_getErrorCode(result)));
   }
 
-  return buffer;
+  return SIZET2NUM(result);
 }
 
+#else
+ZSTDS_EXT_NORETURN VALUE zstds_ext_get_dictionary_header_size(VALUE self, VALUE buffer)
+{
+  zstds_ext_raise_error(ZSTDS_EXT_ERROR_NOT_IMPLEMENTED);
+};
+#endif
+
+// -- exports --
+
 void zstds_ext_dictionary_exports(VALUE root_module)
 {
   VALUE dictionary = rb_define_class_under(root_module, "Dictionary", rb_cObject);
 
   rb_define_singleton_method(dictionary, "get_buffer_id", zstds_ext_get_dictionary_buffer_id, 1);
+  rb_define_singleton_method(dictionary, "get_header_size", zstds_ext_get_dictionary_header_size, 1);
   rb_define_singleton_method(dictionary, "train_buffer", zstds_ext_train_dictionary_buffer, 2);
 }