zstd-ruby 1.5.1.0 → 1.5.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9c5390f204466dadf3276d3d7841d5dcc0c2be4f9cdafd11be705b6304b8cb27
-  data.tar.gz: 3538385c093224f15c6199ae51cdedb944a1b6e0ff4e9c3287ea907911fb07b9
+  metadata.gz: 70096397fcb0ba88b1332602dc743a979e6417bc07aa25270b2fa3e637922291
+  data.tar.gz: e3e2f2ddf2d2d01514e7128b01792e10d9b48cbc08f61a25e0cd9ddf3e91430e
 SHA512:
-  metadata.gz: 8fc0f3ca375ee6b9fd6564eb0807c8d9e5c0f39e19ba9a9692d07c28e55e2fc70a625d3dc682ae402eb7bce76caf114f0c5b103a047f46c88b033bc2bdccc52f
-  data.tar.gz: 2bdb03fade1bca81eedcac96a63b5cbaea4774373f06f483ce77978a3795241e523905b6845c5fac0c70465df3575691e70441cca7a68b7f32b81202c82e69c0
+  metadata.gz: fd02829550fef9d56add09f13c1bee7a5171830e6f7f08e728ef4140e922657b1c70df79e4dd4b56b1bf77cddedbcea4830405561d6cb027bbbce928ff5d7152
+  data.tar.gz: aa56ea5cbc0b9d6e6268691ad656d71f38afde239d9833f07c4ae02a6ab1e1ee2b81e92e5489db3e993774938e58d8fb884c8d882ba4126de15588164ceeff25

data/.gitignore

@@ -17,3 +17,5 @@ vendor/
 
 # rspec failure tracking
 .rspec_status
+
+.ruby-version

data/README.md

@@ -10,7 +10,7 @@ See https://github.com/facebook/zstd
 Fork from https://github.com/jarredholman/ruby-zstd.
 
 ## Zstd version
-v1.5.1 (https://github.com/facebook/zstd/tree/v1.5.1)
+v1.5.2 (https://github.com/facebook/zstd/tree/v1.5.2)
 
 ## Installation
 
@@ -34,20 +34,73 @@ Or install it yourself as:
 require 'zstd-ruby'
 ```
 
-### compression
+### Simple Compression
 
 ```ruby
 compressed_data = Zstd.compress(data)
 compressed_data = Zstd.compress(data, complession_level) # default compression_level is 0
 ```
 
+### Streaming Compression
+```
+stream = Zstd::StreamingCompress.new
+stream << "abc" << "def"
+res = stream.flush
+stream << "ghi"
+res << stream.finish
+```
+
+or
+
+```
+stream = Zstd::StreamingCompress.new
+res = stream.compress("abc")
+res << stream.flush
+res << stream.compress("def")
+res << stream.finish
+```
 
-### decompression
+### Simple Decompression
 
 ```ruby
 data = Zstd.decompress(compressed_data)
 ```
 
+### Streaming Decompression
+```
+cstr = "" # Compressed data
+stream = Zstd::StreamingDecompress.new
+result = ''
+result << stream.decompress(cstr[0, 10])
+result << stream.decompress(cstr[10..-1])
+```
+
+
+## JRuby
+This gem does not support JRuby.
+
+Please consider using https://github.com/luben/zstd-jni.
+
+Sample code is below.
+
+```
+require 'java'
+require_relative './zstd-jni-1.5.2-3.jar'
+
+str = "testtest"
+compressed = com.github.luben.zstd.Zstd.compress(str.to_java_bytes)
+puts com.github.luben.zstd.Zstd.decompress(compressed, str.length)
+```
+
+```
+% ls
+test.rb              zstd-jni-1.5.2-3.jar
+% ruby -v
+jruby 9.3.2.0 (2.6.8) 2021-12-01 0b8223f905 OpenJDK 64-Bit Server VM 11.0.12+0 on 11.0.12+0 +jit [darwin-x86_64]
+% ruby test.rb
+testtest
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/Rakefile

@@ -17,6 +17,12 @@ task :default => [:clobber, :compile, :spec]
 
 desc 'Sync zstd libs dirs to ext/zstdruby/libzstd'
 task :zstd_update do
-  FileUtils.rm_r('ext/zstdruby/libzstd')
-  FileUtils.cp_r('zstd/lib', 'ext/zstdruby/libzstd')
+  FileUtils.rm_r("ext/zstdruby/libzstd")
+  FileUtils.mkdir_p("ext/zstdruby/libzstd")
+  ["common", "compress", "decompress", "dictBuilder"].each do |dir|
+    FileUtils.cp_r("zstd/lib/#{dir}", "ext/zstdruby/libzstd/#{dir}")
+  end
+  FileUtils.cp_r('zstd/lib/zdict.h', 'ext/zstdruby/libzstd')
+  FileUtils.cp_r('zstd/lib/zstd.h', 'ext/zstdruby/libzstd')
+  FileUtils.cp_r('zstd/lib/zstd_errors.h', 'ext/zstdruby/libzstd')
 end

data/ext/zstdruby/{zstdruby.h → common.h}

@@ -2,5 +2,7 @@
 #define ZSTD_RUBY_H 1
 
 #include "ruby.h"
+#include "./libzstd/zstd.h"
+
 
 #endif /* ZSTD_RUBY_H */

data/ext/zstdruby/extconf.rb

@@ -4,7 +4,7 @@ $CFLAGS = '-I. -O3 -std=c99'
 $CPPFLAGS += " -fdeclspec" if CONFIG['CXX'] =~ /clang/
 
 Dir.chdir File.expand_path('..', __FILE__) do
-    $srcs = Dir['**/*.c']
+  $srcs = Dir['**/*.c', '**/*.S']
 
     Dir.glob('libzstd/*') do |path|
         if Dir.exist?(path)

data/ext/zstdruby/libzstd/common/pool.c

@@ -86,7 +86,7 @@ static void* POOL_thread(void* opaque) {
         {   POOL_job const job = ctx->queue[ctx->queueHead];
             ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize;
             ctx->numThreadsBusy++;
-            ctx->queueEmpty = ctx->queueHead == ctx->queueTail;
+            ctx->queueEmpty = (ctx->queueHead == ctx->queueTail);
             /* Unlock the mutex, signal a pusher, and run the job */
             ZSTD_pthread_cond_signal(&ctx->queuePushCond);
             ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
@@ -105,6 +105,7 @@ static void* POOL_thread(void* opaque) {
     assert(0);  /* Unreachable */
 }
 
+/* ZSTD_createThreadPool() : public access point */
 POOL_ctx* ZSTD_createThreadPool(size_t numThreads) {
     return POOL_create (numThreads, 0);
 }
@@ -114,7 +115,8 @@ POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
 }
 
 POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
-                               ZSTD_customMem customMem) {
+                               ZSTD_customMem customMem)
+{
     POOL_ctx* ctx;
     /* Check parameters */
     if (!numThreads) { return NULL; }
@@ -192,7 +194,7 @@ void ZSTD_freeThreadPool (ZSTD_threadPool* pool) {
   POOL_free (pool);
 }
 
-size_t POOL_sizeof(POOL_ctx *ctx) {
+size_t POOL_sizeof(const POOL_ctx* ctx) {
     if (ctx==NULL) return 0;  /* supports sizeof NULL */
     return sizeof(*ctx)
         + ctx->queueSize * sizeof(POOL_job)
@@ -257,7 +259,8 @@ static int isQueueFull(POOL_ctx const* ctx) {
 }
 
 
-static void POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque)
+static void
+POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque)
 {
     POOL_job const job = {function, opaque};
     assert(ctx != NULL);
@@ -313,7 +316,9 @@ POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
     return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem);
 }
 
-POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) {
+POOL_ctx*
+POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem)
+{
     (void)numThreads;
     (void)queueSize;
     (void)customMem;
@@ -341,7 +346,7 @@ int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) {
     return 1;
 }
 
-size_t POOL_sizeof(POOL_ctx* ctx) {
+size_t POOL_sizeof(const POOL_ctx* ctx) {
     if (ctx==NULL) return 0;  /* supports sizeof NULL */
     assert(ctx == &g_poolCtx);
     return sizeof(*ctx);

data/ext/zstdruby/libzstd/common/pool.h

@@ -53,7 +53,7 @@ int POOL_resize(POOL_ctx* ctx, size_t numThreads);
  * @return threadpool memory usage
  *  note : compatible with NULL (returns 0 in this case)
  */
-size_t POOL_sizeof(POOL_ctx* ctx);
+size_t POOL_sizeof(const POOL_ctx* ctx);
 
 /*! POOL_function :
  *  The function type that can be added to a thread pool.
@@ -70,7 +70,7 @@ void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque);
 
 
 /*! POOL_tryAdd() :
- *  Add the job `function(opaque)` to thread pool _if_ a worker is available.
+ *  Add the job `function(opaque)` to thread pool _if_ a queue slot is available.
  *  Returns immediately even if not (does not block).
  * @return : 1 if successful, 0 if not.
  */

data/ext/zstdruby/libzstd/common/portability_macros.h

@@ -65,6 +65,12 @@
 # endif
 #endif
 
+/* Mark the internal assembly functions as hidden  */
+#ifdef __ELF__
+# define ZSTD_HIDE_ASM_FUNCTION(func) .hidden func
+#else
+# define ZSTD_HIDE_ASM_FUNCTION(func)
+#endif
 
 /* Enable runtime BMI2 dispatch based on the CPU.
  * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default.

data/ext/zstdruby/libzstd/common/zstd_internal.h

@@ -67,7 +67,6 @@ extern "C" {
 #define ZSTD_OPT_NUM    (1<<12)
 
 #define ZSTD_REP_NUM      3                 /* number of repcodes */
-#define ZSTD_REP_MOVE     (ZSTD_REP_NUM-1)
 static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
 
 #define KB *(1 <<10)
@@ -285,9 +284,9 @@ typedef enum {
 *  Private declarations
 *********************************************/
 typedef struct seqDef_s {
-    U32 offset;         /* offset == rawOffset + ZSTD_REP_NUM, or equivalently, offCode + 1 */
+    U32 offBase;   /* offBase == Offset + ZSTD_REP_NUM, or repcode 1,2,3 */
     U16 litLength;
-    U16 matchLength;
+    U16 mlBase;    /* mlBase == matchLength - MINMATCH */
 } seqDef;
 
 /* Controls whether seqStore has a single "long" litLength or matchLength. See seqStore_t. */
@@ -329,7 +328,7 @@ MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore
 {
     ZSTD_sequenceLength seqLen;
     seqLen.litLength = seq->litLength;
-    seqLen.matchLength = seq->matchLength + MINMATCH;
+    seqLen.matchLength = seq->mlBase + MINMATCH;
     if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) {
         if (seqStore->longLengthType == ZSTD_llt_literalLength) {
             seqLen.litLength += 0xFFFF;