ruby-zstds 1.0.3 → 1.1.1

data/ext/zstds_ext/stream/compressor.h

@@ -4,17 +4,20 @@
 #if !defined(ZSTDS_EXT_STREAM_COMPRESSOR_H)
 #define ZSTDS_EXT_STREAM_COMPRESSOR_H
 
+#include <stdbool.h>
 #include <zstd.h>
 
 #include "ruby.h"
 #include "zstds_ext/common.h"
 
-typedef struct {
+typedef struct
+{
   ZSTD_CCtx*        ctx;
   zstds_ext_byte_t* destination_buffer;
   size_t            destination_buffer_length;
   zstds_ext_byte_t* remaining_destination_buffer;
   size_t            remaining_destination_buffer_length;
+  bool              gvl;
 } zstds_ext_compressor_t;
 
 VALUE zstds_ext_allocate_compressor(VALUE klass);

data/ext/zstds_ext/stream/decompressor.c

@@ -7,8 +7,11 @@
 
 #include "ruby.h"
 #include "zstds_ext/error.h"
+#include "zstds_ext/gvl.h"
 #include "zstds_ext/option.h"
 
+// -- initialization --
+
 static void free_decompressor(zstds_ext_decompressor_t* decompressor_ptr)
 {
   ZSTD_DCtx* ctx = decompressor_ptr->ctx;
@@ -27,7 +30,6 @@ static void free_decompressor(zstds_ext_decompressor_t* decompressor_ptr)
 VALUE zstds_ext_allocate_decompressor(VALUE klass)
 {
   zstds_ext_decompressor_t* decompressor_ptr;
-
   VALUE self = Data_Make_Struct(klass, zstds_ext_decompressor_t, NULL, free_decompressor, decompressor_ptr);
 
   decompressor_ptr->ctx                                 = NULL;
@@ -47,8 +49,9 @@ VALUE zstds_ext_initialize_decompressor(VALUE self, VALUE options)
 {
   GET_DECOMPRESSOR(self);
   Check_Type(options, T_HASH);
+  ZSTDS_EXT_GET_SIZE_OPTION(options, destination_buffer_length);
+  ZSTDS_EXT_GET_BOOL_OPTION(options, gvl);
   ZSTDS_EXT_GET_DECOMPRESSOR_OPTIONS(options);
-  ZSTDS_EXT_GET_BUFFER_LENGTH_OPTION(options, destination_buffer_length);
 
   ZSTD_DCtx* ctx = ZSTD_createDCtx();
   if (ctx == NULL) {
@@ -76,40 +79,56 @@ VALUE zstds_ext_initialize_decompressor(VALUE self, VALUE options)
   decompressor_ptr->destination_buffer_length           = destination_buffer_length;
   decompressor_ptr->remaining_destination_buffer        = destination_buffer;
   decompressor_ptr->remaining_destination_buffer_length = destination_buffer_length;
+  decompressor_ptr->gvl                                 = gvl;
 
   return Qnil;
 }
 
+// -- decompress --
+
 #define DO_NOT_USE_AFTER_CLOSE(decompressor_ptr)                                       \
   if (decompressor_ptr->ctx == NULL || decompressor_ptr->destination_buffer == NULL) { \
     zstds_ext_raise_error(ZSTDS_EXT_ERROR_USED_AFTER_CLOSE);                           \
   }
 
-#define GET_SOURCE_DATA(source_value)                    \
-  Check_Type(source_value, T_STRING);                    \
-                                                         \
-  const char* source        = RSTRING_PTR(source_value); \
-  size_t      source_length = RSTRING_LEN(source_value);
+typedef struct
+{
+  zstds_ext_decompressor_t* decompressor_ptr;
+  ZSTD_inBuffer*            in_buffer_ptr;
+  ZSTD_outBuffer*           out_buffer_ptr;
+  zstds_result_t            result;
+} decompress_args_t;
+
+static inline void* decompress_wrapper(void* data)
+{
+  decompress_args_t* args = data;
+
+  args->result = ZSTD_decompressStream(args->decompressor_ptr->ctx, args->out_buffer_ptr, args->in_buffer_ptr);
+
+  return NULL;
+}
 
 VALUE zstds_ext_decompress(VALUE self, VALUE source_value)
 {
   GET_DECOMPRESSOR(self);
   DO_NOT_USE_AFTER_CLOSE(decompressor_ptr);
-  GET_SOURCE_DATA(source_value);
+  Check_Type(source_value, T_STRING);
+
+  const char* source        = RSTRING_PTR(source_value);
+  size_t      source_length = RSTRING_LEN(source_value);
 
-  ZSTD_inBuffer in_buffer;
-  in_buffer.src  = source;
-  in_buffer.size = source_length;
-  in_buffer.pos  = 0;
+  ZSTD_inBuffer  in_buffer  = {.src = source, .size = source_length, .pos = 0};
+  ZSTD_outBuffer out_buffer = {
+    .dst  = decompressor_ptr->remaining_destination_buffer,
+    .size = decompressor_ptr->remaining_destination_buffer_length,
+    .pos  = 0};
 
-  ZSTD_outBuffer out_buffer;
-  out_buffer.dst  = decompressor_ptr->remaining_destination_buffer;
-  out_buffer.size = decompressor_ptr->remaining_destination_buffer_length;
-  out_buffer.pos  = 0;
+  decompress_args_t args = {
+    .decompressor_ptr = decompressor_ptr, .in_buffer_ptr = &in_buffer, .out_buffer_ptr = &out_buffer};
 
-  zstds_result_t result = ZSTD_decompressStream(decompressor_ptr->ctx, &out_buffer, &in_buffer);
-  if (ZSTD_isError(result)) {
-    zstds_ext_raise_error(zstds_ext_get_error(ZSTD_getErrorCode(result)));
+  ZSTDS_EXT_GVL_WRAP(decompressor_ptr->gvl, decompress_wrapper, &args);
+  if (ZSTD_isError(args.result)) {
+    zstds_ext_raise_error(zstds_ext_get_error(ZSTD_getErrorCode(args.result)));
   }
 
   decompressor_ptr->remaining_destination_buffer += out_buffer.pos;
@@ -121,6 +140,8 @@ VALUE zstds_ext_decompress(VALUE self, VALUE source_value)
   return rb_ary_new_from_args(2, bytes_read, needs_more_destination);
 }
 
+// -- other --
+
 VALUE zstds_ext_decompressor_read_result(VALUE self)
 {
   GET_DECOMPRESSOR(self);
@@ -130,10 +151,9 @@ VALUE zstds_ext_decompressor_read_result(VALUE self)
   size_t            destination_buffer_length           = decompressor_ptr->destination_buffer_length;
   size_t            remaining_destination_buffer_length = decompressor_ptr->remaining_destination_buffer_length;
 
-  const char* result        = (const char*)destination_buffer;
+  const char* result        = (const char*) destination_buffer;
   size_t      result_length = destination_buffer_length - remaining_destination_buffer_length;
-
-  VALUE result_value = rb_str_new(result, result_length);
+  VALUE       result_value  = rb_str_new(result, result_length);
 
   decompressor_ptr->remaining_destination_buffer        = destination_buffer;
   decompressor_ptr->remaining_destination_buffer_length = destination_buffer_length;
@@ -141,6 +161,8 @@ VALUE zstds_ext_decompressor_read_result(VALUE self)
   return result_value;
 }
 
+// -- cleanup --
+
 VALUE zstds_ext_decompressor_close(VALUE self)
 {
   GET_DECOMPRESSOR(self);
@@ -166,6 +188,8 @@ VALUE zstds_ext_decompressor_close(VALUE self)
   return Qnil;
 }
 
+// -- exports --
+
 void zstds_ext_decompressor_exports(VALUE root_module)
 {
   VALUE module = rb_define_module_under(root_module, "Stream");

data/ext/zstds_ext/stream/decompressor.h

@@ -4,17 +4,20 @@
 #if !defined(ZSTDS_EXT_STREAM_DECOMPRESSOR_H)
 #define ZSTDS_EXT_STREAM_DECOMPRESSOR_H
 
+#include <stdbool.h>
 #include <zstd.h>
 
 #include "ruby.h"
 #include "zstds_ext/common.h"
 
-typedef struct {
+typedef struct
+{
   ZSTD_DCtx*        ctx;
   zstds_ext_byte_t* destination_buffer;
   size_t            destination_buffer_length;
   zstds_ext_byte_t* remaining_destination_buffer;
   size_t            remaining_destination_buffer_length;
+  bool              gvl;
 } zstds_ext_decompressor_t;
 
 VALUE zstds_ext_allocate_decompressor(VALUE klass);

data/ext/zstds_ext/string.c

@@ -8,14 +8,17 @@
 #include "ruby.h"
 #include "zstds_ext/buffer.h"
 #include "zstds_ext/error.h"
+#include "zstds_ext/gvl.h"
 #include "zstds_ext/macro.h"
 #include "zstds_ext/option.h"
 
 // -- buffer --
 
 static inline zstds_ext_result_t increase_destination_buffer(
-  VALUE destination_value, size_t destination_length,
-  size_t* remaining_destination_buffer_length_ptr, size_t destination_buffer_length)
+  VALUE   destination_value,
+  size_t  destination_length,
+  size_t* remaining_destination_buffer_length_ptr,
+  size_t  destination_buffer_length)
 {
   if (*remaining_destination_buffer_length_ptr == destination_buffer_length) {
     // We want to write more data at once, than buffer has.
@@ -34,50 +37,58 @@ static inline zstds_ext_result_t increase_destination_buffer(
   return 0;
 }
 
-// -- utils --
+// -- compress --
 
-#define GET_SOURCE_DATA(source_value)                    \
-  Check_Type(source_value, T_STRING);                    \
-                                                         \
-  const char* source        = RSTRING_PTR(source_value); \
-  size_t      source_length = RSTRING_LEN(source_value);
+typedef struct
+{
+  ZSTD_CCtx*      ctx;
+  ZSTD_inBuffer*  in_buffer_ptr;
+  ZSTD_outBuffer* out_buffer_ptr;
+  zstds_result_t  result;
+} compress_args_t;
 
-// -- compress --
+static inline void* compress_wrapper(void* data)
+{
+  compress_args_t* args = data;
+
+  args->result = ZSTD_compressStream2(args->ctx, args->out_buffer_ptr, args->in_buffer_ptr, ZSTD_e_end);
+
+  return NULL;
+}
 
 static inline zstds_ext_result_t compress(
   ZSTD_CCtx*  ctx,
-  const char* source, size_t source_length,
-  VALUE destination_value, size_t destination_buffer_length)
+  const char* source,
+  size_t      source_length,
+  VALUE       destination_value,
+  size_t      destination_buffer_length,
+  bool        gvl)
 {
-  zstds_result_t     result;
   zstds_ext_result_t ext_result;
-
-  ZSTD_inBuffer in_buffer;
-  in_buffer.src  = source;
-  in_buffer.size = source_length;
-  in_buffer.pos  = 0;
-
-  ZSTD_outBuffer out_buffer;
-  size_t         destination_length                  = 0;
-  size_t         remaining_destination_buffer_length = destination_buffer_length;
+  size_t             destination_length                  = 0;
+  size_t             remaining_destination_buffer_length = destination_buffer_length;
+  ZSTD_inBuffer      in_buffer                           = {.src = source, .size = source_length, .pos = 0};
+  compress_args_t    args                                = {.ctx = ctx, .in_buffer_ptr = &in_buffer};
 
   while (true) {
-    out_buffer.dst  = (zstds_ext_byte_t*)RSTRING_PTR(destination_value) + destination_length;
-    out_buffer.size = remaining_destination_buffer_length;
-    out_buffer.pos  = 0;
+    ZSTD_outBuffer out_buffer = {
+      .dst  = (zstds_ext_byte_t*) RSTRING_PTR(destination_value) + destination_length,
+      .size = remaining_destination_buffer_length,
+      .pos  = 0};
 
-    result = ZSTD_compressStream2(ctx, &out_buffer, &in_buffer, ZSTD_e_end);
-    if (ZSTD_isError(result)) {
-      return zstds_ext_get_error(ZSTD_getErrorCode(result));
+    args.out_buffer_ptr = &out_buffer;
+
+    ZSTDS_EXT_GVL_WRAP(gvl, compress_wrapper, &args);
+    if (ZSTD_isError(args.result)) {
+      return zstds_ext_get_error(ZSTD_getErrorCode(args.result));
     }
 
     destination_length += out_buffer.pos;
     remaining_destination_buffer_length -= out_buffer.pos;
 
-    if (result != 0) {
+    if (args.result != 0) {
       ext_result = increase_destination_buffer(
-        destination_value, destination_length,
-        &remaining_destination_buffer_length, destination_buffer_length);
+        destination_value, destination_length, &remaining_destination_buffer_length, destination_buffer_length);
 
       if (ext_result != 0) {
         return ext_result;
@@ -101,10 +112,11 @@ static inline zstds_ext_result_t compress(
 
 VALUE zstds_ext_compress_string(VALUE ZSTDS_EXT_UNUSED(self), VALUE source_value, VALUE options)
 {
-  GET_SOURCE_DATA(source_value);
+  Check_Type(source_value, T_STRING);
   Check_Type(options, T_HASH);
+  ZSTDS_EXT_GET_SIZE_OPTION(options, destination_buffer_length);
+  ZSTDS_EXT_GET_BOOL_OPTION(options, gvl);
   ZSTDS_EXT_GET_COMPRESSOR_OPTIONS(options);
-  ZSTDS_EXT_GET_BUFFER_LENGTH_OPTION(options, destination_buffer_length);
 
   ZSTD_CCtx* ctx = ZSTD_createCCtx();
   if (ctx == NULL) {
@@ -129,10 +141,10 @@ VALUE zstds_ext_compress_string(VALUE ZSTDS_EXT_UNUSED(self), VALUE source_value
     zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
   }
 
-  ext_result = compress(
-    ctx,
-    source, source_length,
-    destination_value, destination_buffer_length);
+  const char* source        = RSTRING_PTR(source_value);
+  size_t      source_length = RSTRING_LEN(source_value);
+
+  ext_result = compress(ctx, source, source_length, destination_value, destination_buffer_length, gvl);
 
   ZSTD_freeCCtx(ctx);
 
@@ -145,31 +157,48 @@ VALUE zstds_ext_compress_string(VALUE ZSTDS_EXT_UNUSED(self), VALUE source_value
 
 // -- decompress --
 
+typedef struct
+{
+  ZSTD_DCtx*      ctx;
+  ZSTD_inBuffer*  in_buffer_ptr;
+  ZSTD_outBuffer* out_buffer_ptr;
+  zstds_result_t  result;
+} decompress_args_t;
+
+static inline void* decompress_wrapper(void* data)
+{
+  decompress_args_t* args = data;
+
+  args->result = ZSTD_decompressStream(args->ctx, args->out_buffer_ptr, args->in_buffer_ptr);
+
+  return NULL;
+}
+
 static inline zstds_ext_result_t decompress(
   ZSTD_DCtx*  ctx,
-  const char* source, size_t source_length,
-  VALUE destination_value, size_t destination_buffer_length)
+  const char* source,
+  size_t      source_length,
+  VALUE       destination_value,
+  size_t      destination_buffer_length,
+  bool        gvl)
 {
-  zstds_result_t     result;
   zstds_ext_result_t ext_result;
-
-  ZSTD_inBuffer in_buffer;
-  in_buffer.src  = source;
-  in_buffer.size = source_length;
-  in_buffer.pos  = 0;
-
-  ZSTD_outBuffer out_buffer;
-  size_t         destination_length                  = 0;
-  size_t         remaining_destination_buffer_length = destination_buffer_length;
+  size_t             destination_length                  = 0;
+  size_t             remaining_destination_buffer_length = destination_buffer_length;
+  ZSTD_inBuffer      in_buffer                           = {.src = source, .size = source_length, .pos = 0};
+  decompress_args_t  args                                = {.ctx = ctx, .in_buffer_ptr = &in_buffer};
 
   while (true) {
-    out_buffer.dst  = (zstds_ext_byte_t*)RSTRING_PTR(destination_value) + destination_length;
-    out_buffer.size = remaining_destination_buffer_length;
-    out_buffer.pos  = 0;
+    ZSTD_outBuffer out_buffer = {
+      .dst  = (zstds_ext_byte_t*) RSTRING_PTR(destination_value) + destination_length,
+      .size = remaining_destination_buffer_length,
+      .pos  = 0};
+
+    args.out_buffer_ptr = &out_buffer;
 
-    result = ZSTD_decompressStream(ctx, &out_buffer, &in_buffer);
-    if (ZSTD_isError(result)) {
-      return zstds_ext_get_error(ZSTD_getErrorCode(result));
+    ZSTDS_EXT_GVL_WRAP(gvl, decompress_wrapper, &args);
+    if (ZSTD_isError(args.result)) {
+      return zstds_ext_get_error(ZSTD_getErrorCode(args.result));
     }
 
     destination_length += out_buffer.pos;
@@ -177,8 +206,7 @@ static inline zstds_ext_result_t decompress(
 
     if (remaining_destination_buffer_length == 0) {
       ext_result = increase_destination_buffer(
-        destination_value, destination_length,
-        &remaining_destination_buffer_length, destination_buffer_length);
+        destination_value, destination_length, &remaining_destination_buffer_length, destination_buffer_length);
 
       if (ext_result != 0) {
         return ext_result;
@@ -202,10 +230,11 @@ static inline zstds_ext_result_t decompress(
 
 VALUE zstds_ext_decompress_string(VALUE ZSTDS_EXT_UNUSED(self), VALUE source_value, VALUE options)
 {
-  GET_SOURCE_DATA(source_value);
+  Check_Type(source_value, T_STRING);
   Check_Type(options, T_HASH);
+  ZSTDS_EXT_GET_SIZE_OPTION(options, destination_buffer_length);
+  ZSTDS_EXT_GET_BOOL_OPTION(options, gvl);
   ZSTDS_EXT_GET_DECOMPRESSOR_OPTIONS(options);
-  ZSTDS_EXT_GET_BUFFER_LENGTH_OPTION(options, destination_buffer_length);
 
   ZSTD_DCtx* ctx = ZSTD_createDCtx();
   if (ctx == NULL) {
@@ -230,10 +259,10 @@ VALUE zstds_ext_decompress_string(VALUE ZSTDS_EXT_UNUSED(self), VALUE source_val
     zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
   }
 
-  ext_result = decompress(
-    ctx,
-    source, source_length,
-    destination_value, destination_buffer_length);
+  const char* source        = RSTRING_PTR(source_value);
+  size_t      source_length = RSTRING_LEN(source_value);
+
+  ext_result = decompress(ctx, source, source_length, destination_value, destination_buffer_length, gvl);
 
   ZSTD_freeDCtx(ctx);
 
@@ -244,6 +273,8 @@ VALUE zstds_ext_decompress_string(VALUE ZSTDS_EXT_UNUSED(self), VALUE source_val
   return destination_value;
 }
 
+// -- exports --
+
 void zstds_ext_string_exports(VALUE root_module)
 {
   rb_define_module_function(root_module, "_native_compress_string", RUBY_METHOD_FUNC(zstds_ext_compress_string), 2);

data/lib/zstds/dictionary.rb

@@ -9,6 +9,7 @@ require_relative "validation"
 module ZSTDS
   class Dictionary
     TRAIN_DEFAULTS = {
+      :gvl      => false,
       :capacity => 0
     }
     .freeze
@@ -38,6 +39,7 @@ module ZSTDS
 
       options = TRAIN_DEFAULTS.merge options
 
+      Validation.validate_bool                 options[:gvl]
       Validation.validate_not_negative_integer options[:capacity]
 
       buffer = train_buffer samples, options