zstd-ruby 1.5.2.2 → 1.5.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '048d980344614b4a0087d7eb25cc01f9937fdf93439bdf4c1c830ca0d6fedeff'
-  data.tar.gz: 432606f67285abd98fb76650445a4950e28501552a55661ed41b6f87000a4aa1
+  metadata.gz: cb64917059ca60a73732ee8f460f31ab3fbd36bce1eb20b844f8e36de8c9ec75
+  data.tar.gz: 01a248c444714b1404b934745f47c73aac8a2acb1bd1b0cb36993b1b61593d33
 SHA512:
-  metadata.gz: 5f563ede0aa63ff7496f9206db6440f418ccf4b3fb7a9e1fb236bc9a7bfd78a6f4a4d0f4ce0eec497e786a44b7a5f3d74667d45a46952de8b3ba05df4e5c6a06
-  data.tar.gz: ea58de90bbd0f397245fdb8830b92eae55787999a1e739540492129ba3461119fd6980b39e53fb0083c2edfc57ad6cb3bd33f183bb917120326af3a342e03fcb
+  metadata.gz: 629f566ee687ad0e3cf0afd7329f32d67bc10bb58a316541fec249913cb5b7a81323138dd1eef10aed986fbdf57838214fa65218cb3c5ae1fb20b6ffb880aa99
+  data.tar.gz: f6ca1102e9fd7c1147b0a2187575eca9fe6539375e4eff0b3a90635fb973b739402ed0951cd8b8f68d55d33a1996593c5da4e6078b39b8a5d2be77d311004110

data/README.md

@@ -41,6 +41,12 @@ compressed_data = Zstd.compress(data)
 compressed_data = Zstd.compress(data, complession_level) # default compression_level is 0
 ```
 
+### Compression using Dictionary
+```ruby
+# dictionary is supposed to have been created using `zstd --train`
+compressed_using_dict = Zstd.compress_using_dict("", IO.read('dictionary_file'))
+```
+
 ### Streaming Compression
 ```
 stream = Zstd::StreamingCompress.new
@@ -66,6 +72,12 @@ res << stream.finish
 data = Zstd.decompress(compressed_data)
 ```
 
+### Decomporession using Dictionary
+```ruby
+# dictionary is supposed to have been created using `zstd --train`
+Zstd.decompress_using_dict(compressed_using_dict, IO.read('dictionary_file'))
+```
+
 ### Streaming Decompression
 ```
 cstr = "" # Compressed data

data/ext/zstdruby/common.h

@@ -4,5 +4,12 @@
 #include "ruby.h"
 #include "./libzstd/zstd.h"
 
+static int convert_compression_level(VALUE compression_level_value)
+{
+  if (NIL_P(compression_level_value)) {
+    return ZSTD_CLEVEL_DEFAULT;
+  }
+  return NUM2INT(compression_level_value);
+}
 
 #endif /* ZSTD_RUBY_H */

data/ext/zstdruby/streaming_compress.c

@@ -53,13 +53,7 @@ rb_streaming_compress_initialize(int argc, VALUE *argv, VALUE obj)
 {
   VALUE compression_level_value;
   rb_scan_args(argc, argv, "01", &compression_level_value);
-
-  int compression_level;
-  if (NIL_P(compression_level_value)) {
-    compression_level = ZSTD_CLEVEL_DEFAULT;
-  } else {
-    compression_level = NUM2INT(compression_level_value);
-  }
+  int compression_level = convert_compression_level(compression_level_value);
 
   struct streaming_compress_t* sc;
   TypedData_Get_Struct(obj, struct streaming_compress_t, &streaming_compress_type, sc);

data/ext/zstdruby/zstdruby.c

@@ -8,41 +8,74 @@ static VALUE zstdVersion(VALUE self)
   return INT2NUM(version);
 }
 
-static VALUE compress(int argc, VALUE *argv, VALUE self)
+static VALUE rb_compress(int argc, VALUE *argv, VALUE self)
 {
   VALUE input_value;
   VALUE compression_level_value;
   rb_scan_args(argc, argv, "11", &input_value, &compression_level_value);
+  int compression_level = convert_compression_level(compression_level_value);
 
   StringValue(input_value);
-  const char* input_data = RSTRING_PTR(input_value);
+  char* input_data = RSTRING_PTR(input_value);
   size_t input_size = RSTRING_LEN(input_value);
+  size_t max_compressed_size = ZSTD_compressBound(input_size);
 
-  int compression_level;
-  if (NIL_P(compression_level_value)) {
-    compression_level = 0; // The default. See ZSTD_CLEVEL_DEFAULT in zstd_compress.c
-  } else {
-    compression_level = NUM2INT(compression_level_value);
+  VALUE output = rb_str_new(NULL, max_compressed_size);
+  char* output_data = RSTRING_PTR(output);
+  size_t compressed_size = ZSTD_compress((void*)output_data, max_compressed_size,
+                                         (void*)input_data, input_size, compression_level);
+  if (ZSTD_isError(compressed_size)) {
+    rb_raise(rb_eRuntimeError, "%s: %s", "compress failed", ZSTD_getErrorName(compressed_size));
   }
 
-  // do compress
+  rb_str_resize(output, compressed_size);
+  return output;
+}
+
+static VALUE rb_compress_using_dict(int argc, VALUE *argv, VALUE self)
+{
+  VALUE input_value;
+  VALUE dict;
+  VALUE compression_level_value;
+  rb_scan_args(argc, argv, "21", &input_value, &dict, &compression_level_value);
+  int compression_level = convert_compression_level(compression_level_value);
+
+  StringValue(input_value);
+  char* input_data = RSTRING_PTR(input_value);
+  size_t input_size = RSTRING_LEN(input_value);
   size_t max_compressed_size = ZSTD_compressBound(input_size);
 
+  char* dict_buffer = RSTRING_PTR(dict);
+  size_t dict_size = RSTRING_LEN(dict);
+
+  ZSTD_CDict* const cdict = ZSTD_createCDict(dict_buffer, dict_size, compression_level);
+  if (cdict == NULL) {
+    rb_raise(rb_eRuntimeError, "%s", "ZSTD_createCDict failed");
+  }
+  ZSTD_CCtx* const ctx = ZSTD_createCCtx();
+  if (ctx == NULL) {
+    ZSTD_freeCDict(cdict);
+    rb_raise(rb_eRuntimeError, "%s", "ZSTD_createCCtx failed");
+  }
+
   VALUE output = rb_str_new(NULL, max_compressed_size);
   char* output_data = RSTRING_PTR(output);
-
-  size_t compressed_size = ZSTD_compress((void*)output_data, max_compressed_size,
-                                         (const void*)input_data, input_size, compression_level);
+  size_t const compressed_size = ZSTD_compress_usingCDict(ctx, (void*)output_data, max_compressed_size,
+                                             (void*)input_data, input_size, cdict);
 
   if (ZSTD_isError(compressed_size)) {
+    ZSTD_freeCDict(cdict);
+    ZSTD_freeCCtx(ctx);
     rb_raise(rb_eRuntimeError, "%s: %s", "compress failed", ZSTD_getErrorName(compressed_size));
-  } else {
-    rb_str_resize(output, compressed_size);
   }
 
+  rb_str_resize(output, compressed_size);
+  ZSTD_freeCDict(cdict);
+  ZSTD_freeCCtx(ctx);
   return output;
 }
 
+
 static VALUE decompress_buffered(const char* input_data, size_t input_size)
 {
   const size_t outputBufferSize = 4096;
@@ -58,7 +91,6 @@ static VALUE decompress_buffered(const char* input_data, size_t input_size)
     rb_raise(rb_eRuntimeError, "%s: %s", "ZSTD_initDStream failed", ZSTD_getErrorName(initResult));
   }
 
-
   VALUE output_string = rb_str_new(NULL, 0);
   ZSTD_outBuffer output = { NULL, 0, 0 };
 
@@ -80,23 +112,24 @@ static VALUE decompress_buffered(const char* input_data, size_t input_size)
   return output_string;
 }
 
-static VALUE decompress(VALUE self, VALUE input)
+static VALUE rb_decompress(VALUE self, VALUE input_value)
 {
-  StringValue(input);
-  const char* input_data = RSTRING_PTR(input);
-  size_t input_size = RSTRING_LEN(input);
-
-  uint64_t uncompressed_size = ZSTD_getDecompressedSize(input_data, input_size);
+  StringValue(input_value);
+  char* input_data = RSTRING_PTR(input_value);
+  size_t input_size = RSTRING_LEN(input_value);
 
-  if (uncompressed_size == 0) {
+  unsigned long long const uncompressed_size = ZSTD_getFrameContentSize(input_data, input_size);
+  if (uncompressed_size == ZSTD_CONTENTSIZE_ERROR) {
+    rb_raise(rb_eRuntimeError, "%s: %s", "not compressed by zstd", ZSTD_getErrorName(uncompressed_size));
+  }
+  if (uncompressed_size == ZSTD_CONTENTSIZE_UNKNOWN) {
     return decompress_buffered(input_data, input_size);
   }
 
   VALUE output = rb_str_new(NULL, uncompressed_size);
   char* output_data = RSTRING_PTR(output);
-
-  size_t decompress_size = ZSTD_decompress((void*)output_data, uncompressed_size,
-                                           (const void*)input_data, input_size);
+  size_t const decompress_size = ZSTD_decompress((void*)output_data, uncompressed_size,
+                                           (void*)input_data, input_size);
 
   if (ZSTD_isError(decompress_size)) {
     rb_raise(rb_eRuntimeError, "%s: %s", "decompress error", ZSTD_getErrorName(decompress_size));
@@ -105,10 +138,61 @@ static VALUE decompress(VALUE self, VALUE input)
   return output;
 }
 
+static VALUE rb_decompress_using_dict(int argc, VALUE *argv, VALUE self)
+{
+  VALUE input_value;
+  VALUE dict;
+  rb_scan_args(argc, argv, "20", &input_value, &dict);
+
+  StringValue(input_value);
+  char* input_data = RSTRING_PTR(input_value);
+  size_t input_size = RSTRING_LEN(input_value);
+  unsigned long long const uncompressed_size = ZSTD_getFrameContentSize(input_data, input_size);
+  if (uncompressed_size == ZSTD_CONTENTSIZE_ERROR) {
+    rb_raise(rb_eRuntimeError, "%s: %s", "not compressed by zstd", ZSTD_getErrorName(uncompressed_size));
+  }
+  if (uncompressed_size == ZSTD_CONTENTSIZE_UNKNOWN) {
+    return decompress_buffered(input_data, input_size);
+  }
+  VALUE output = rb_str_new(NULL, uncompressed_size);
+  char* output_data = RSTRING_PTR(output);
+
+  char* dict_buffer = RSTRING_PTR(dict);
+  size_t dict_size = RSTRING_LEN(dict);
+  ZSTD_DDict* const ddict = ZSTD_createDDict(dict_buffer, dict_size);
+  if (ddict == NULL) {
+    rb_raise(rb_eRuntimeError, "%s", "ZSTD_createDDict failed");
+  }
+
+  unsigned const expected_dict_id = ZSTD_getDictID_fromDDict(ddict);
+  unsigned const actual_dict_id = ZSTD_getDictID_fromFrame(input_data, input_size);
+  if (expected_dict_id != actual_dict_id) {
+    ZSTD_freeDDict(ddict);
+    rb_raise(rb_eRuntimeError, "%s: %s", "DictID mismatch", ZSTD_getErrorName(uncompressed_size));
+  }
+
+  ZSTD_DCtx* const ctx = ZSTD_createDCtx();
+  if (ctx == NULL) {
+    ZSTD_freeDDict(ddict);
+    rb_raise(rb_eRuntimeError, "%s", "ZSTD_createDCtx failed");
+  }
+  size_t const decompress_size = ZSTD_decompress_usingDDict(ctx, output_data, uncompressed_size, input_data, input_size, ddict);
+  if (ZSTD_isError(decompress_size)) {
+    ZSTD_freeDDict(ddict);
+    ZSTD_freeDCtx(ctx);
+    rb_raise(rb_eRuntimeError, "%s: %s", "decompress error", ZSTD_getErrorName(decompress_size));
+  }
+  ZSTD_freeDDict(ddict);
+  ZSTD_freeDCtx(ctx);
+  return output;
+}
+
 void
 zstd_ruby_init(void)
 {
   rb_define_module_function(rb_mZstd, "zstd_version", zstdVersion, 0);
-  rb_define_module_function(rb_mZstd, "compress", compress, -1);
-  rb_define_module_function(rb_mZstd, "decompress", decompress, 1);
+  rb_define_module_function(rb_mZstd, "compress", rb_compress, -1);
+  rb_define_module_function(rb_mZstd, "compress_using_dict", rb_compress_using_dict, -1);
+  rb_define_module_function(rb_mZstd, "decompress", rb_decompress, 1);
+  rb_define_module_function(rb_mZstd, "decompress_using_dict", rb_decompress_using_dict, -1);
 }

data/lib/zstd-ruby/version.rb

@@ -1,3 +1,3 @@
 module Zstd
-  VERSION = "1.5.2.2"
+  VERSION = "1.5.2.3"
 end

data/lib/zstd-ruby.rb

@@ -2,5 +2,4 @@ require "zstd-ruby/version"
 require "zstd-ruby/zstdruby"
 
 module Zstd
-  # Your code goes here...
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: zstd-ruby
 version: !ruby/object:Gem::Version
-  version: 1.5.2.2
+  version: 1.5.2.3
 platform: ruby
 authors:
 - SpringMT