zstd-ruby 1.5.6.1 → 1.5.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 40905f48d12335ff309bf984092915cf10b77425f3a913fa7ee721732fd200ec
-  data.tar.gz: 50af86c41600c37b2395aef57f967e76bac3d296271b257d106b64456f6ab4cf
+  metadata.gz: 729474c9a3f3196fe69a06dfdd280c5879bc368b1884b8d508b75164aa7e1dcd
+  data.tar.gz: 1cdf1b0554dd89aa1f7b6450be8eb183773a0f3b895024715ae7b3243cec598b
 SHA512:
-  metadata.gz: 1c01f7837f96849a1cd6bc3068b1b3b3d777e955849c26bc902529007184541a1b78838b33a2c27d4d50a11da4ca0e0f6b42c2c9318f9a1e23d81d4806d7f970
-  data.tar.gz: 843369ab0c7dcc5010b6d027a73f364e5765dd63898683227a839ff7139fbba6ece018d3c77885b7f21a6aec4846a52f86abe90ee47bb18d675985df4111e873
+  metadata.gz: d69bc35190b5dbc8ce36f9e5acf6ef26a2c85a899ddd409f8eee4a83b005166b20d22e2659cae1b101fb1b97f6a7bf12cf4066e3da94b2a1991d41bf255cd1ce
+  data.tar.gz: 2ee85e56a8ae268da1e4b69bf35a016a5d9bc677f240435725e082b5c5279e60d7f8be32dbe57283e618b7cf59b3937b57aa859f6cbb0aafa64458a903747fa2

data/README.md

@@ -34,20 +34,22 @@ Or install it yourself as:
 require 'zstd-ruby'
 ```
 
-### Simple Compression
+### Compression
+
+#### Simple Compression
 
 ```ruby
 compressed_data = Zstd.compress(data)
-compressed_data = Zstd.compress(data, complession_level) # default compression_level is 0
+compressed_data = Zstd.compress(data, level: complession_level) # default compression_level is 3
 ```
 
-### Compression using Dictionary
+#### Compression with Dictionary
 ```ruby
 # dictionary is supposed to have been created using `zstd --train`
-compressed_using_dict = Zstd.compress_using_dict("", IO.read('dictionary_file'))
+compressed_using_dict = Zstd.compress("", dict: File.read('dictionary_file'))
 ```
 
-### Streaming Compression
+#### Streaming Compression
 ```ruby
 stream = Zstd::StreamingCompress.new
 stream << "abc" << "def"
@@ -66,19 +68,39 @@ res << stream.compress("def")
 res << stream.finish
 ```
 
-### Simple Decompression
+#### Streaming Compression with Dictionary
+```ruby
+stream = Zstd::StreamingCompress.new(dict: File.read('dictionary_file'))
+stream << "abc" << "def"
+res = stream.flush
+stream << "ghi"
+res << stream.finish
+```
+
+#### Streaming Compression with level and Dictionary
+```ruby
+stream = Zstd::StreamingCompress.new(level: 5, dict: File.read('dictionary_file'))
+stream << "abc" << "def"
+res = stream.flush
+stream << "ghi"
+res << stream.finish
+```
+
+### Decompression
+
+#### Simple Decompression
 
 ```ruby
 data = Zstd.decompress(compressed_data)
 ```
 
-### Decomporession using Dictionary
+#### Decompression with Dictionary
 ```ruby
 # dictionary is supposed to have been created using `zstd --train`
-Zstd.decompress_using_dict(compressed_using_dict, IO.read('dictionary_file'))
+Zstd.decompress(compressed_using_dict, dict: File.read('dictionary_file'))
 ```
 
-### Streaming Decompression
+#### Streaming Decompression
 ```ruby
 cstr = "" # Compressed data
 stream = Zstd::StreamingDecompress.new
@@ -87,6 +109,15 @@ result << stream.decompress(cstr[0, 10])
 result << stream.decompress(cstr[10..-1])
 ```
 
+#### Streaming Decompression with dictionary
+```ruby
+cstr = "" # Compressed data
+stream = Zstd::StreamingDecompress.new(dict: File.read('dictionary_file'))
+result = ''
+result << stream.decompress(cstr[0, 10])
+result << stream.decompress(cstr[10..-1])
+```
+
 ### Skippable frame
 
 ```ruby

data/ext/zstdruby/common.h

@@ -1,7 +1,11 @@
 #ifndef ZSTD_RUBY_H
 #define ZSTD_RUBY_H 1
 
-#include "ruby.h"
+#include <ruby.h>
+#ifdef HAVE_RUBY_THREAD_H
+#include <ruby/thread.h>
+#endif
+#include <stdbool.h>
 #include "./libzstd/zstd.h"
 
 static int convert_compression_level(VALUE compression_level_value)
@@ -12,4 +16,109 @@ static int convert_compression_level(VALUE compression_level_value)
   return NUM2INT(compression_level_value);
 }
 
+static void set_compress_params(ZSTD_CCtx* const ctx, VALUE level_from_args, VALUE kwargs)
+{
+  ID kwargs_keys[2];
+  kwargs_keys[0] = rb_intern("level");
+  kwargs_keys[1] = rb_intern("dict");
+  VALUE kwargs_values[2];
+  rb_get_kwargs(kwargs, kwargs_keys, 0, 2, kwargs_values);
+
+  int compression_level = ZSTD_CLEVEL_DEFAULT;
+  if (kwargs_values[0] != Qundef && kwargs_values[0] != Qnil) {
+    compression_level = convert_compression_level(kwargs_values[0]);
+  } else if (!NIL_P(level_from_args)) {
+    rb_warn("`level` in args is deprecated; use keyword args `level:` instead.");
+    compression_level = convert_compression_level(level_from_args);
+  }
+  ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, compression_level);
+
+  if (kwargs_values[1] != Qundef && kwargs_values[1] != Qnil) {
+    char* dict_buffer = RSTRING_PTR(kwargs_values[1]);
+    size_t dict_size = RSTRING_LEN(kwargs_values[1]);
+    size_t load_dict_ret = ZSTD_CCtx_loadDictionary(ctx, dict_buffer, dict_size);
+    if (ZSTD_isError(load_dict_ret)) {
+      ZSTD_freeCCtx(ctx);
+      rb_raise(rb_eRuntimeError, "%s", "ZSTD_CCtx_loadDictionary failed");
+    }
+  }
+}
+
+struct compress_params {
+  ZSTD_CCtx* ctx;
+  ZSTD_outBuffer* output;
+  ZSTD_inBuffer* input;
+  ZSTD_EndDirective endOp;
+  size_t ret;
+};
+
+static void* compress_wrapper(void* args)
+{
+    struct compress_params* params = args;
+    params->ret = ZSTD_compressStream2(params->ctx, params->output, params->input, params->endOp);
+    return NULL;
+}
+
+static size_t zstd_compress(ZSTD_CCtx* const ctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input, ZSTD_EndDirective endOp, bool gvl)
+{
+#ifdef HAVE_RUBY_THREAD_H
+    if (gvl) {
+      return ZSTD_compressStream2(ctx, output, input, endOp);
+    } else {
+      struct compress_params params = { ctx, output, input, endOp };
+      rb_thread_call_without_gvl(compress_wrapper, &params, NULL, NULL);
+      return params.ret;
+    }
+#else
+    return ZSTD_compressStream2(ctx, output, input, endOp);
+#endif
+}
+
+static void set_decompress_params(ZSTD_DCtx* const dctx, VALUE kwargs)
+{
+  ID kwargs_keys[1];
+  kwargs_keys[0] = rb_intern("dict");
+  VALUE kwargs_values[1];
+  rb_get_kwargs(kwargs, kwargs_keys, 0, 1, kwargs_values);
+
+  if (kwargs_values[0] != Qundef && kwargs_values[0] != Qnil) {
+    char* dict_buffer = RSTRING_PTR(kwargs_values[0]);
+    size_t dict_size = RSTRING_LEN(kwargs_values[0]);
+    size_t load_dict_ret = ZSTD_DCtx_loadDictionary(dctx, dict_buffer, dict_size);
+    if (ZSTD_isError(load_dict_ret)) {
+      ZSTD_freeDCtx(dctx);
+      rb_raise(rb_eRuntimeError, "%s", "ZSTD_CCtx_loadDictionary failed");
+    }
+  }
+}
+
+struct decompress_params {
+  ZSTD_DCtx* dctx;
+  ZSTD_outBuffer* output;
+  ZSTD_inBuffer* input;
+  size_t ret;
+};
+
+static void* decompress_wrapper(void* args)
+{
+    struct decompress_params* params = args;
+    params->ret = ZSTD_decompressStream(params->dctx, params->output, params->input);
+    return NULL;
+}
+
+static size_t zstd_decompress(ZSTD_DCtx* const dctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input, bool gvl)
+{
+#ifdef HAVE_RUBY_THREAD_H
+    if (gvl) {
+      return ZSTD_decompressStream(dctx, output, input);
+    } else {
+      struct decompress_params params = { dctx, output, input };
+      rb_thread_call_without_gvl(decompress_wrapper, &params, NULL, NULL);
+      return params.ret;
+    }
+#else
+    return ZSTD_decompressStream(dctx, output, input);
+#endif
+}
+
 #endif /* ZSTD_RUBY_H */

data/ext/zstdruby/extconf.rb

@@ -2,7 +2,7 @@ require "mkmf"
 
 have_func('rb_gc_mark_movable')
 
-$CFLAGS = '-I. -O3 -std=c99 -DZSTD_STATIC_LINKING_ONLY'
+$CFLAGS = '-I. -O3 -std=c99 -DZSTD_STATIC_LINKING_ONLY -DZSTD_MULTITHREAD -pthread -DDEBUGLEVEL=0'
 $CPPFLAGS += " -fdeclspec" if CONFIG['CXX'] =~ /clang/
 
 Dir.chdir File.expand_path('..', __FILE__) do

data/ext/zstdruby/main.c

@@ -1,4 +1,5 @@
-#include <common.h>
+#include "common.h"
+
 VALUE rb_mZstd;
 void zstd_ruby_init(void);
 void zstd_ruby_skippable_frame_init(void);

data/ext/zstdruby/skippable_frame.c

@@ -1,4 +1,4 @@
-#include <common.h>
+#include "common.h"
 
 extern VALUE rb_mZstd;
 

data/ext/zstdruby/streaming_compress.c

@@ -1,5 +1,4 @@
-#include <common.h>
-#include <streaming_compress.h>
+#include "common.h"
 
 struct streaming_compress_t {
   ZSTD_CCtx* ctx;
@@ -71,9 +70,9 @@ rb_streaming_compress_allocate(VALUE klass)
 static VALUE
 rb_streaming_compress_initialize(int argc, VALUE *argv, VALUE obj)
 {
+  VALUE kwargs;
   VALUE compression_level_value;
-  rb_scan_args(argc, argv, "01", &compression_level_value);
-  int compression_level = convert_compression_level(compression_level_value);
+  rb_scan_args(argc, argv, "01:", &compression_level_value, &kwargs);
 
   struct streaming_compress_t* sc;
   TypedData_Get_Struct(obj, struct streaming_compress_t, &streaming_compress_type, sc);
@@ -83,7 +82,8 @@ rb_streaming_compress_initialize(int argc, VALUE *argv, VALUE obj)
   if (ctx == NULL) {
     rb_raise(rb_eRuntimeError, "%s", "ZSTD_createCCtx error");
   }
-  ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, compression_level);
+  set_compress_params(ctx, compression_level_value, kwargs);
+
   sc->ctx = ctx;
   sc->buf = rb_str_new(NULL, buffOutSize);
   sc->buf_size = buffOutSize;
@@ -106,7 +106,7 @@ no_compress(struct streaming_compress_t* sc, ZSTD_EndDirective endOp)
   do {
     ZSTD_outBuffer output = { (void*)output_data, sc->buf_size, 0 };
 
-    size_t const ret = ZSTD_compressStream2(sc->ctx, &output, &input, endOp);
+    size_t const ret = zstd_compress(sc->ctx, &output, &input, endOp, false);
     if (ZSTD_isError(ret)) {
       rb_raise(rb_eRuntimeError, "flush error error code: %s", ZSTD_getErrorName(ret));
     }
@@ -125,11 +125,12 @@ rb_streaming_compress_compress(VALUE obj, VALUE src)
 
   struct streaming_compress_t* sc;
   TypedData_Get_Struct(obj, struct streaming_compress_t, &streaming_compress_type, sc);
+
   const char* output_data = RSTRING_PTR(sc->buf);
   VALUE result = rb_str_new(0, 0);
   while (input.pos < input.size) {
     ZSTD_outBuffer output = { (void*)output_data, sc->buf_size, 0 };
-    size_t const ret = ZSTD_compressStream2(sc->ctx, &output, &input, ZSTD_e_continue);
+    size_t const ret = zstd_compress(sc->ctx, &output, &input, ZSTD_e_continue, false);
     if (ZSTD_isError(ret)) {
       rb_raise(rb_eRuntimeError, "compress error error code: %s", ZSTD_getErrorName(ret));
     }
@@ -139,27 +140,54 @@ rb_streaming_compress_compress(VALUE obj, VALUE src)
 }
 
 static VALUE
-rb_streaming_compress_addstr(VALUE obj, VALUE src)
+rb_streaming_compress_write(int argc, VALUE *argv, VALUE obj)
 {
-  StringValue(src);
-  const char* input_data = RSTRING_PTR(src);
-  size_t input_size = RSTRING_LEN(src);
-  ZSTD_inBuffer input = { input_data, input_size, 0 };
-
+  size_t total = 0;
+  VALUE result = rb_str_new(0, 0);
   struct streaming_compress_t* sc;
   TypedData_Get_Struct(obj, struct streaming_compress_t, &streaming_compress_type, sc);
   const char* output_data = RSTRING_PTR(sc->buf);
 
-  while (input.pos < input.size) {
-    ZSTD_outBuffer output = { (void*)output_data, sc->buf_size, 0 };
-    size_t const result = ZSTD_compressStream2(sc->ctx, &output, &input, ZSTD_e_continue);
-    if (ZSTD_isError(result)) {
-      rb_raise(rb_eRuntimeError, "compress error error code: %s", ZSTD_getErrorName(result));
+  while (argc-- > 0) {
+    VALUE str = *argv++;
+    StringValue(str);
+    const char* input_data = RSTRING_PTR(str);
+    size_t input_size = RSTRING_LEN(str);
+    ZSTD_inBuffer input = { input_data, input_size, 0 };
+
+    while (input.pos < input.size) {
+      ZSTD_outBuffer output = { (void*)output_data, sc->buf_size, 0 };
+      size_t const ret = zstd_compress(sc->ctx, &output, &input, ZSTD_e_continue, false);
+      if (ZSTD_isError(ret)) {
+        rb_raise(rb_eRuntimeError, "compress error error code: %s", ZSTD_getErrorName(ret));
+      }
+      total += RSTRING_LEN(str);
     }
   }
-  return obj;
+  return SIZET2NUM(total);
 }
 
+/*
+ * Document-method: <<
+ * Same as IO.
+ */
+#define rb_streaming_compress_addstr  rb_io_addstr
+/*
+ * Document-method: printf
+ * Same as IO.
+ */
+#define rb_streaming_compress_printf  rb_io_printf
+/*
+ * Document-method: print
+ * Same as IO.
+ */
+#define rb_streaming_compress_print  rb_io_print
+/*
+ * Document-method: puts
+ * Same as IO.
+ */
+#define rb_streaming_compress_puts  rb_io_puts
+
 static VALUE
 rb_streaming_compress_flush(VALUE obj)
 {
@@ -186,7 +214,12 @@ zstd_ruby_streaming_compress_init(void)
   rb_define_alloc_func(cStreamingCompress, rb_streaming_compress_allocate);
   rb_define_method(cStreamingCompress, "initialize", rb_streaming_compress_initialize, -1);
   rb_define_method(cStreamingCompress, "compress", rb_streaming_compress_compress, 1);
+  rb_define_method(cStreamingCompress, "write", rb_streaming_compress_write, -1);
   rb_define_method(cStreamingCompress, "<<", rb_streaming_compress_addstr, 1);
+  rb_define_method(cStreamingCompress, "printf", rb_streaming_compress_printf, -1);
+  rb_define_method(cStreamingCompress, "print", rb_streaming_compress_print, -1);
+  rb_define_method(cStreamingCompress, "puts", rb_streaming_compress_puts, -1);
+
   rb_define_method(cStreamingCompress, "flush", rb_streaming_compress_flush, 0);
   rb_define_method(cStreamingCompress, "finish", rb_streaming_compress_finish, 0);
 
@@ -194,4 +227,3 @@ zstd_ruby_streaming_compress_init(void)
   rb_define_const(cStreamingCompress, "FLUSH", INT2FIX(ZSTD_e_flush));
   rb_define_const(cStreamingCompress, "END", INT2FIX(ZSTD_e_end));
 }
-

data/ext/zstdruby/streaming_decompress.c

@@ -1,7 +1,7 @@
-#include <common.h>
+#include "common.h"
 
 struct streaming_decompress_t {
-  ZSTD_DCtx* ctx;
+  ZSTD_DCtx* dctx;
   VALUE buf;
   size_t buf_size;
 };
@@ -21,9 +21,9 @@ static void
 streaming_decompress_free(void *p)
 {
   struct streaming_decompress_t *sd = p;
-  ZSTD_DCtx* ctx = sd->ctx;
-  if (ctx != NULL) {
-    ZSTD_freeDCtx(ctx);
+  ZSTD_DCtx* dctx = sd->dctx;
+  if (dctx != NULL) {
+    ZSTD_freeDCtx(dctx);
   }
   xfree(sd);
 }
@@ -61,24 +61,29 @@ rb_streaming_decompress_allocate(VALUE klass)
 {
   struct streaming_decompress_t* sd;
   VALUE obj = TypedData_Make_Struct(klass, struct streaming_decompress_t, &streaming_decompress_type, sd);
-  sd->ctx = NULL;
+  sd->dctx = NULL;
   sd->buf = Qnil;
   sd->buf_size = 0;
   return obj;
 }
 
 static VALUE
-rb_streaming_decompress_initialize(VALUE obj)
+rb_streaming_decompress_initialize(int argc, VALUE *argv, VALUE obj)
 {
+  VALUE kwargs;
+  rb_scan_args(argc, argv, "00:", &kwargs);
+
   struct streaming_decompress_t* sd;
   TypedData_Get_Struct(obj, struct streaming_decompress_t, &streaming_decompress_type, sd);
   size_t const buffOutSize = ZSTD_DStreamOutSize();
 
-  ZSTD_DCtx* ctx = ZSTD_createDCtx();
-  if (ctx == NULL) {
+  ZSTD_DCtx* dctx = ZSTD_createDCtx();
+  if (dctx == NULL) {
     rb_raise(rb_eRuntimeError, "%s", "ZSTD_createDCtx error");
   }
-  sd->ctx = ctx;
+  set_decompress_params(dctx, kwargs);
+
+  sd->dctx = dctx;
   sd->buf = rb_str_new(NULL, buffOutSize);
   sd->buf_size = buffOutSize;
 
@@ -99,45 +104,21 @@ rb_streaming_decompress_decompress(VALUE obj, VALUE src)
   VALUE result = rb_str_new(0, 0);
   while (input.pos < input.size) {
     ZSTD_outBuffer output = { (void*)output_data, sd->buf_size, 0 };
-    size_t const ret = ZSTD_decompressStream(sd->ctx, &output, &input);
+    size_t const ret = zstd_decompress(sd->dctx, &output, &input, false);
     if (ZSTD_isError(ret)) {
-      rb_raise(rb_eRuntimeError, "compress error error code: %s", ZSTD_getErrorName(ret));
+      rb_raise(rb_eRuntimeError, "decompress error error code: %s", ZSTD_getErrorName(ret));
     }
     rb_str_cat(result, output.dst, output.pos);
   }
   return result;
 }
 
-static VALUE
-rb_streaming_decompress_addstr(VALUE obj, VALUE src)
-{
-  StringValue(src);
-  const char* input_data = RSTRING_PTR(src);
-  size_t input_size = RSTRING_LEN(src);
-  ZSTD_inBuffer input = { input_data, input_size, 0 };
-
-  struct streaming_decompress_t* sd;
-  TypedData_Get_Struct(obj, struct streaming_decompress_t, &streaming_decompress_type, sd);
-  const char* output_data = RSTRING_PTR(sd->buf);
-
-  while (input.pos < input.size) {
-    ZSTD_outBuffer output = { (void*)output_data, sd->buf_size, 0 };
-    size_t const result = ZSTD_decompressStream(sd->ctx, &output, &input);
-    if (ZSTD_isError(result)) {
-      rb_raise(rb_eRuntimeError, "compress error error code: %s", ZSTD_getErrorName(result));
-    }
-  }
-  return obj;
-}
-
 extern VALUE rb_mZstd, cStreamingDecompress;
 void
 zstd_ruby_streaming_decompress_init(void)
 {
   VALUE cStreamingDecompress = rb_define_class_under(rb_mZstd, "StreamingDecompress", rb_cObject);
   rb_define_alloc_func(cStreamingDecompress, rb_streaming_decompress_allocate);
-  rb_define_method(cStreamingDecompress, "initialize", rb_streaming_decompress_initialize, 0);
+  rb_define_method(cStreamingDecompress, "initialize", rb_streaming_decompress_initialize, -1);
   rb_define_method(cStreamingDecompress, "decompress", rb_streaming_decompress_decompress, 1);
-  rb_define_method(cStreamingDecompress, "<<", rb_streaming_decompress_addstr, 1);
 }
-

data/ext/zstdruby/zstdruby.c

@@ -12,28 +12,40 @@ static VALUE rb_compress(int argc, VALUE *argv, VALUE self)
 {
   VALUE input_value;
   VALUE compression_level_value;
-  rb_scan_args(argc, argv, "11", &input_value, &compression_level_value);
-  int compression_level = convert_compression_level(compression_level_value);
+  VALUE kwargs;
+  rb_scan_args(argc, argv, "11:", &input_value, &compression_level_value, &kwargs);
+
+  ZSTD_CCtx* const ctx = ZSTD_createCCtx();
+  if (ctx == NULL) {
+    rb_raise(rb_eRuntimeError, "%s", "ZSTD_createCCtx error");
+  }
+
+  set_compress_params(ctx, compression_level_value, kwargs);
 
   StringValue(input_value);
   char* input_data = RSTRING_PTR(input_value);
   size_t input_size = RSTRING_LEN(input_value);
+  ZSTD_inBuffer input = { input_data, input_size, 0 };
+  // ZSTD_compressBound causes SEGV under multi-thread
   size_t max_compressed_size = ZSTD_compressBound(input_size);
+  VALUE buf = rb_str_new(NULL, max_compressed_size);
+  char* output_data = RSTRING_PTR(buf);
+  ZSTD_outBuffer output = { (void*)output_data, max_compressed_size, 0 };
 
-  VALUE output = rb_str_new(NULL, max_compressed_size);
-  char* output_data = RSTRING_PTR(output);
-  size_t compressed_size = ZSTD_compress((void*)output_data, max_compressed_size,
-                                         (void*)input_data, input_size, compression_level);
-  if (ZSTD_isError(compressed_size)) {
-    rb_raise(rb_eRuntimeError, "%s: %s", "compress failed", ZSTD_getErrorName(compressed_size));
+  size_t const ret = zstd_compress(ctx, &output, &input, ZSTD_e_end, true);
+  if (ZSTD_isError(ret)) {
+    ZSTD_freeCCtx(ctx);
+    rb_raise(rb_eRuntimeError, "%s: %s", "compress failed", ZSTD_getErrorName(ret));
   }
-
-  rb_str_resize(output, compressed_size);
-  return output;
+  VALUE result = rb_str_new(0, 0);
+  rb_str_cat(result, output.dst, output.pos);
+  ZSTD_freeCCtx(ctx);
+  return result;
 }
 
 static VALUE rb_compress_using_dict(int argc, VALUE *argv, VALUE self)
 {
+  rb_warn("Zstd.compress_using_dict is deprecated; use Zstd.compress with `dict:` instead.");
   VALUE input_value;
   VALUE dict;
   VALUE compression_level_value;
@@ -76,61 +88,56 @@ static VALUE rb_compress_using_dict(int argc, VALUE *argv, VALUE self)
 }
 
 
-static VALUE decompress_buffered(const char* input_data, size_t input_size)
+static VALUE decompress_buffered(ZSTD_DCtx* dctx, const char* input_data, size_t input_size)
 {
-  const size_t outputBufferSize = 4096;
-
-  ZSTD_DStream* const dstream = ZSTD_createDStream();
-  if (dstream == NULL) {
-    rb_raise(rb_eRuntimeError, "%s", "ZSTD_createDStream failed");
-  }
-
-  size_t initResult = ZSTD_initDStream(dstream);
-  if (ZSTD_isError(initResult)) {
-    ZSTD_freeDStream(dstream);
-    rb_raise(rb_eRuntimeError, "%s: %s", "ZSTD_initDStream failed", ZSTD_getErrorName(initResult));
-  }
-
   VALUE output_string = rb_str_new(NULL, 0);
   ZSTD_outBuffer output = { NULL, 0, 0 };
 
   ZSTD_inBuffer input = { input_data, input_size, 0 };
   while (input.pos < input.size) {
-    output.size += outputBufferSize;
+    output.size += ZSTD_DStreamOutSize();
     rb_str_resize(output_string, output.size);
     output.dst = RSTRING_PTR(output_string);
 
-    size_t readHint = ZSTD_decompressStream(dstream, &output, &input);
-    if (ZSTD_isError(readHint)) {
-      ZSTD_freeDStream(dstream);
-      rb_raise(rb_eRuntimeError, "%s: %s", "ZSTD_decompressStream failed", ZSTD_getErrorName(readHint));
+    size_t ret = zstd_decompress(dctx, &output, &input, true);
+    if (ZSTD_isError(ret)) {
+      ZSTD_freeDCtx(dctx);
+      rb_raise(rb_eRuntimeError, "%s: %s", "ZSTD_decompressStream failed", ZSTD_getErrorName(ret));
     }
   }
-
-  ZSTD_freeDStream(dstream);
   rb_str_resize(output_string, output.pos);
+  ZSTD_freeDCtx(dctx);
   return output_string;
 }
 
-static VALUE rb_decompress(VALUE self, VALUE input_value)
+static VALUE rb_decompress(int argc, VALUE *argv, VALUE self)
 {
+  VALUE input_value;
+  VALUE kwargs;
+  rb_scan_args(argc, argv, "10:", &input_value, &kwargs);
   StringValue(input_value);
   char* input_data = RSTRING_PTR(input_value);
   size_t input_size = RSTRING_LEN(input_value);
+  ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+  if (dctx == NULL) {
+    rb_raise(rb_eRuntimeError, "%s", "ZSTD_createDCtx failed");
+  }
+  set_decompress_params(dctx, kwargs);
 
   unsigned long long const uncompressed_size = ZSTD_getFrameContentSize(input_data, input_size);
   if (uncompressed_size == ZSTD_CONTENTSIZE_ERROR) {
     rb_raise(rb_eRuntimeError, "%s: %s", "not compressed by zstd", ZSTD_getErrorName(uncompressed_size));
   }
+  // ZSTD_decompressStream may be called multiple times when ZSTD_CONTENTSIZE_UNKNOWN, causing slowness.
+  // Therefore, we will not standardize on ZSTD_decompressStream
   if (uncompressed_size == ZSTD_CONTENTSIZE_UNKNOWN) {
-    return decompress_buffered(input_data, input_size);
+    return decompress_buffered(dctx, input_data, input_size);
   }
 
   VALUE output = rb_str_new(NULL, uncompressed_size);
   char* output_data = RSTRING_PTR(output);
-  size_t const decompress_size = ZSTD_decompress((void*)output_data, uncompressed_size,
-                                           (void*)input_data, input_size);
 
+  size_t const decompress_size =  ZSTD_decompressDCtx(dctx, output_data, uncompressed_size, input_data, input_size);
   if (ZSTD_isError(decompress_size)) {
     rb_raise(rb_eRuntimeError, "%s: %s", "decompress error", ZSTD_getErrorName(decompress_size));
   }
@@ -140,6 +147,7 @@ static VALUE rb_decompress(VALUE self, VALUE input_value)
 
 static VALUE rb_decompress_using_dict(int argc, VALUE *argv, VALUE self)
 {
+  rb_warn("Zstd.decompress_using_dict is deprecated; use Zstd.decompress with `dict:` instead.");
   VALUE input_value;
   VALUE dict;
   rb_scan_args(argc, argv, "20", &input_value, &dict);
@@ -147,15 +155,6 @@ static VALUE rb_decompress_using_dict(int argc, VALUE *argv, VALUE self)
   StringValue(input_value);
   char* input_data = RSTRING_PTR(input_value);
   size_t input_size = RSTRING_LEN(input_value);
-  unsigned long long const uncompressed_size = ZSTD_getFrameContentSize(input_data, input_size);
-  if (uncompressed_size == ZSTD_CONTENTSIZE_ERROR) {
-    rb_raise(rb_eRuntimeError, "%s: %s", "not compressed by zstd", ZSTD_getErrorName(uncompressed_size));
-  }
-  if (uncompressed_size == ZSTD_CONTENTSIZE_UNKNOWN) {
-    return decompress_buffered(input_data, input_size);
-  }
-  VALUE output = rb_str_new(NULL, uncompressed_size);
-  char* output_data = RSTRING_PTR(output);
 
   char* dict_buffer = RSTRING_PTR(dict);
   size_t dict_size = RSTRING_LEN(dict);
@@ -163,12 +162,11 @@ static VALUE rb_decompress_using_dict(int argc, VALUE *argv, VALUE self)
   if (ddict == NULL) {
     rb_raise(rb_eRuntimeError, "%s", "ZSTD_createDDict failed");
   }
-
   unsigned const expected_dict_id = ZSTD_getDictID_fromDDict(ddict);
   unsigned const actual_dict_id = ZSTD_getDictID_fromFrame(input_data, input_size);
   if (expected_dict_id != actual_dict_id) {
     ZSTD_freeDDict(ddict);
-    rb_raise(rb_eRuntimeError, "%s: %s", "DictID mismatch", ZSTD_getErrorName(uncompressed_size));
+    rb_raise(rb_eRuntimeError, "DictID mismatch");
   }
 
   ZSTD_DCtx* const ctx = ZSTD_createDCtx();
@@ -176,6 +174,19 @@ static VALUE rb_decompress_using_dict(int argc, VALUE *argv, VALUE self)
     ZSTD_freeDDict(ddict);
     rb_raise(rb_eRuntimeError, "%s", "ZSTD_createDCtx failed");
   }
+
+  unsigned long long const uncompressed_size = ZSTD_getFrameContentSize(input_data, input_size);
+  if (uncompressed_size == ZSTD_CONTENTSIZE_ERROR) {
+    ZSTD_freeDDict(ddict);
+    ZSTD_freeDCtx(ctx);
+    rb_raise(rb_eRuntimeError, "%s: %s", "not compressed by zstd", ZSTD_getErrorName(uncompressed_size));
+  }
+  if (uncompressed_size == ZSTD_CONTENTSIZE_UNKNOWN) {
+    return decompress_buffered(ctx, input_data, input_size);
+  }
+
+  VALUE output = rb_str_new(NULL, uncompressed_size);
+  char* output_data = RSTRING_PTR(output);
   size_t const decompress_size = ZSTD_decompress_usingDDict(ctx, output_data, uncompressed_size, input_data, input_size, ddict);
   if (ZSTD_isError(decompress_size)) {
     ZSTD_freeDDict(ddict);
@@ -193,6 +204,6 @@ zstd_ruby_init(void)
   rb_define_module_function(rb_mZstd, "zstd_version", zstdVersion, 0);
   rb_define_module_function(rb_mZstd, "compress", rb_compress, -1);
   rb_define_module_function(rb_mZstd, "compress_using_dict", rb_compress_using_dict, -1);
-  rb_define_module_function(rb_mZstd, "decompress", rb_decompress, 1);
+  rb_define_module_function(rb_mZstd, "decompress", rb_decompress, -1);
   rb_define_module_function(rb_mZstd, "decompress_using_dict", rb_decompress_using_dict, -1);
 }

data/lib/zstd-ruby/stream_reader.rb

@@ -0,0 +1,22 @@
+module Zstd
+  # @todo Exprimental
+  class StreamReader
+    def initialize(io)
+      @io = io
+      @stream = Zstd::StreamingDecompress.new
+    end
+
+    def read(length)
+      if @io.eof?
+        raise StandardError, "EOF"
+      end
+      data = @io.read(length)
+      @stream.decompress(data)
+    end
+
+    def close
+      @io.write(@stream.finish)
+      @io.close
+    end
+  end
+end

data/lib/zstd-ruby/stream_writer.rb

@@ -0,0 +1,23 @@
+module Zstd
+  # @todo Exprimental
+  class StreamWriter
+    def initialize(io, level: nil)
+      @io = io
+      @stream = Zstd::StreamingCompress.new(level)
+    end
+
+    def write(*data)
+      @stream.write(*data)
+      @io.write(@stream.flush)
+    end
+
+    def finish
+      @io.write(@stream.finish)
+    end
+
+    def close
+      @io.write(@stream.finish)
+      @io.close
+    end
+  end
+end

data/lib/zstd-ruby/version.rb

@@ -1,3 +1,3 @@
 module Zstd
-  VERSION = "1.5.6.1"
+  VERSION = "1.5.6.3"
 end

data/zstd-ruby.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
   #end
 
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|
-    f.match(%r{^(test|spec|features|benchmarks|zstd|.github)/})
+    f.match(%r{^(test|spec|features|benchmarks|zstd|.github|examples)/})
   end
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
@@ -35,4 +35,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rake-compiler", '~> 1'
   spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "pry"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zstd-ruby
 version: !ruby/object:Gem::Version
-  version: 1.5.6.1
+  version: 1.5.6.3
 platform: ruby
 authors:
 - SpringMT
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-04-01 00:00:00.000000000 Z
+date: 2024-04-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Ruby binding for zstd(Zstandard - Fast real-time compression algorithm).
   See https://github.com/facebook/zstd
 email:
@@ -159,10 +173,11 @@ files:
 - ext/zstdruby/main.c
 - ext/zstdruby/skippable_frame.c
 - ext/zstdruby/streaming_compress.c
-- ext/zstdruby/streaming_compress.h
 - ext/zstdruby/streaming_decompress.c
 - ext/zstdruby/zstdruby.c
 - lib/zstd-ruby.rb
+- lib/zstd-ruby/stream_reader.rb
+- lib/zstd-ruby/stream_writer.rb
 - lib/zstd-ruby/version.rb
 - renovate.json
 - zstd-ruby.gemspec

data/ext/zstdruby/streaming_compress.h

@@ -1,5 +0,0 @@
-#if !defined(STREAMING_COMPRESS_H)
-#define STREAMING_COMPRESS_H
-
-
-#endif // STREAMING_COMPRESS_H