zstd-ruby 1.5.1.0 → 1.5.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/README.md +56 -3
  4. data/Rakefile +8 -2
  5. data/ext/zstdruby/{zstdruby.h → common.h} +2 -0
  6. data/ext/zstdruby/extconf.rb +1 -1
  7. data/ext/zstdruby/libzstd/common/pool.c +11 -6
  8. data/ext/zstdruby/libzstd/common/pool.h +2 -2
  9. data/ext/zstdruby/libzstd/common/portability_macros.h +6 -0
  10. data/ext/zstdruby/libzstd/common/zstd_internal.h +3 -4
  11. data/ext/zstdruby/libzstd/compress/zstd_compress.c +114 -96
  12. data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +72 -39
  13. data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.c +10 -10
  14. data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.c +1 -1
  15. data/ext/zstdruby/libzstd/compress/zstd_cwksp.h +38 -24
  16. data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +10 -10
  17. data/ext/zstdruby/libzstd/compress/zstd_fast.c +11 -11
  18. data/ext/zstdruby/libzstd/compress/zstd_lazy.c +66 -62
  19. data/ext/zstdruby/libzstd/compress/zstd_ldm.c +5 -3
  20. data/ext/zstdruby/libzstd/compress/zstd_opt.c +66 -43
  21. data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +17 -9
  22. data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +4 -1
  23. data/ext/zstdruby/libzstd/decompress/huf_decompress.c +2 -2
  24. data/ext/zstdruby/libzstd/decompress/huf_decompress_amd64.S +17 -3
  25. data/ext/zstdruby/libzstd/dictBuilder/zdict.c +2 -2
  26. data/ext/zstdruby/libzstd/zstd.h +1 -1
  27. data/ext/zstdruby/main.c +14 -0
  28. data/ext/zstdruby/streaming_compress.c +185 -0
  29. data/ext/zstdruby/streaming_compress.h +5 -0
  30. data/ext/zstdruby/streaming_decompress.c +125 -0
  31. data/ext/zstdruby/zstdruby.c +4 -6
  32. data/lib/zstd-ruby/version.rb +1 -1
  33. data/zstd-ruby.gemspec +1 -1
  34. metadata +11 -40
  35. data/.github/dependabot.yml +0 -8
  36. data/.github/workflows/ruby.yml +0 -35
  37. data/ext/zstdruby/libzstd/.gitignore +0 -3
  38. data/ext/zstdruby/libzstd/BUCK +0 -232
  39. data/ext/zstdruby/libzstd/Makefile +0 -357
  40. data/ext/zstdruby/libzstd/README.md +0 -217
  41. data/ext/zstdruby/libzstd/deprecated/zbuff.h +0 -214
  42. data/ext/zstdruby/libzstd/deprecated/zbuff_common.c +0 -26
  43. data/ext/zstdruby/libzstd/deprecated/zbuff_compress.c +0 -167
  44. data/ext/zstdruby/libzstd/deprecated/zbuff_decompress.c +0 -75
  45. data/ext/zstdruby/libzstd/dll/example/Makefile +0 -48
  46. data/ext/zstdruby/libzstd/dll/example/README.md +0 -63
  47. data/ext/zstdruby/libzstd/dll/example/build_package.bat +0 -20
  48. data/ext/zstdruby/libzstd/dll/example/fullbench-dll.sln +0 -25
  49. data/ext/zstdruby/libzstd/dll/example/fullbench-dll.vcxproj +0 -181
  50. data/ext/zstdruby/libzstd/legacy/zstd_legacy.h +0 -415
  51. data/ext/zstdruby/libzstd/legacy/zstd_v01.c +0 -2158
  52. data/ext/zstdruby/libzstd/legacy/zstd_v01.h +0 -94
  53. data/ext/zstdruby/libzstd/legacy/zstd_v02.c +0 -3518
  54. data/ext/zstdruby/libzstd/legacy/zstd_v02.h +0 -93
  55. data/ext/zstdruby/libzstd/legacy/zstd_v03.c +0 -3160
  56. data/ext/zstdruby/libzstd/legacy/zstd_v03.h +0 -93
  57. data/ext/zstdruby/libzstd/legacy/zstd_v04.c +0 -3647
  58. data/ext/zstdruby/libzstd/legacy/zstd_v04.h +0 -142
  59. data/ext/zstdruby/libzstd/legacy/zstd_v05.c +0 -4050
  60. data/ext/zstdruby/libzstd/legacy/zstd_v05.h +0 -162
  61. data/ext/zstdruby/libzstd/legacy/zstd_v06.c +0 -4154
  62. data/ext/zstdruby/libzstd/legacy/zstd_v06.h +0 -172
  63. data/ext/zstdruby/libzstd/legacy/zstd_v07.c +0 -4541
  64. data/ext/zstdruby/libzstd/legacy/zstd_v07.h +0 -187
  65. data/ext/zstdruby/libzstd/libzstd.mk +0 -185
  66. data/ext/zstdruby/libzstd/libzstd.pc.in +0 -16
  67. data/ext/zstdruby/libzstd/modulemap/module.modulemap +0 -4
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 9c5390f204466dadf3276d3d7841d5dcc0c2be4f9cdafd11be705b6304b8cb27
4
- data.tar.gz: 3538385c093224f15c6199ae51cdedb944a1b6e0ff4e9c3287ea907911fb07b9
3
+ metadata.gz: 70096397fcb0ba88b1332602dc743a979e6417bc07aa25270b2fa3e637922291
4
+ data.tar.gz: e3e2f2ddf2d2d01514e7128b01792e10d9b48cbc08f61a25e0cd9ddf3e91430e
5
5
  SHA512:
6
- metadata.gz: 8fc0f3ca375ee6b9fd6564eb0807c8d9e5c0f39e19ba9a9692d07c28e55e2fc70a625d3dc682ae402eb7bce76caf114f0c5b103a047f46c88b033bc2bdccc52f
7
- data.tar.gz: 2bdb03fade1bca81eedcac96a63b5cbaea4774373f06f483ce77978a3795241e523905b6845c5fac0c70465df3575691e70441cca7a68b7f32b81202c82e69c0
6
+ metadata.gz: fd02829550fef9d56add09f13c1bee7a5171830e6f7f08e728ef4140e922657b1c70df79e4dd4b56b1bf77cddedbcea4830405561d6cb027bbbce928ff5d7152
7
+ data.tar.gz: aa56ea5cbc0b9d6e6268691ad656d71f38afde239d9833f07c4ae02a6ab1e1ee2b81e92e5489db3e993774938e58d8fb884c8d882ba4126de15588164ceeff25
data/.gitignore CHANGED
@@ -17,3 +17,5 @@ vendor/
17
17
 
18
18
  # rspec failure tracking
19
19
  .rspec_status
20
+
21
+ .ruby-version
data/README.md CHANGED
@@ -10,7 +10,7 @@ See https://github.com/facebook/zstd
10
10
  Fork from https://github.com/jarredholman/ruby-zstd.
11
11
 
12
12
  ## Zstd version
13
- v1.5.1 (https://github.com/facebook/zstd/tree/v1.5.1)
13
+ v1.5.2 (https://github.com/facebook/zstd/tree/v1.5.2)
14
14
 
15
15
  ## Installation
16
16
 
@@ -34,20 +34,73 @@ Or install it yourself as:
34
34
  require 'zstd-ruby'
35
35
  ```
36
36
 
37
- ### compression
37
+ ### Simple Compression
38
38
 
39
39
  ```ruby
40
40
  compressed_data = Zstd.compress(data)
41
41
  compressed_data = Zstd.compress(data, complession_level) # default compression_level is 0
42
42
  ```
43
43
 
44
+ ### Streaming Compression
45
+ ```
46
+ stream = Zstd::StreamingCompress.new
47
+ stream << "abc" << "def"
48
+ res = stream.flush
49
+ stream << "ghi"
50
+ res << stream.finish
51
+ ```
52
+
53
+ or
54
+
55
+ ```
56
+ stream = Zstd::StreamingCompress.new
57
+ res = stream.compress("abc")
58
+ res << stream.flush
59
+ res << stream.compress("def")
60
+ res << stream.finish
61
+ ```
44
62
 
45
- ### decompression
63
+ ### Simple Decompression
46
64
 
47
65
  ```ruby
48
66
  data = Zstd.decompress(compressed_data)
49
67
  ```
50
68
 
69
+ ### Streaming Decompression
70
+ ```
71
+ cstr = "" # Compressed data
72
+ stream = Zstd::StreamingDecompress.new
73
+ result = ''
74
+ result << stream.decompress(cstr[0, 10])
75
+ result << stream.decompress(cstr[10..-1])
76
+ ```
77
+
78
+
79
+ ## JRuby
80
+ This gem does not support JRuby.
81
+
82
+ Please consider using https://github.com/luben/zstd-jni.
83
+
84
+ Sample code is below.
85
+
86
+ ```
87
+ require 'java'
88
+ require_relative './zstd-jni-1.5.2-3.jar'
89
+
90
+ str = "testtest"
91
+ compressed = com.github.luben.zstd.Zstd.compress(str.to_java_bytes)
92
+ puts com.github.luben.zstd.Zstd.decompress(compressed, str.length)
93
+ ```
94
+
95
+ ```
96
+ % ls
97
+ test.rb zstd-jni-1.5.2-3.jar
98
+ % ruby -v
99
+ jruby 9.3.2.0 (2.6.8) 2021-12-01 0b8223f905 OpenJDK 64-Bit Server VM 11.0.12+0 on 11.0.12+0 +jit [darwin-x86_64]
100
+ % ruby test.rb
101
+ testtest
102
+ ```
103
+
51
104
  ## Development
52
105
 
53
106
  After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
data/Rakefile CHANGED
@@ -17,6 +17,12 @@ task :default => [:clobber, :compile, :spec]
17
17
 
18
18
  desc 'Sync zstd libs dirs to ext/zstdruby/libzstd'
19
19
  task :zstd_update do
20
- FileUtils.rm_r('ext/zstdruby/libzstd')
21
- FileUtils.cp_r('zstd/lib', 'ext/zstdruby/libzstd')
20
+ FileUtils.rm_r("ext/zstdruby/libzstd")
21
+ FileUtils.mkdir_p("ext/zstdruby/libzstd")
22
+ ["common", "compress", "decompress", "dictBuilder"].each do |dir|
23
+ FileUtils.cp_r("zstd/lib/#{dir}", "ext/zstdruby/libzstd/#{dir}")
24
+ end
25
+ FileUtils.cp_r('zstd/lib/zdict.h', 'ext/zstdruby/libzstd')
26
+ FileUtils.cp_r('zstd/lib/zstd.h', 'ext/zstdruby/libzstd')
27
+ FileUtils.cp_r('zstd/lib/zstd_errors.h', 'ext/zstdruby/libzstd')
22
28
  end
@@ -2,5 +2,7 @@
2
2
  #define ZSTD_RUBY_H 1
3
3
 
4
4
  #include "ruby.h"
5
+ #include "./libzstd/zstd.h"
6
+
5
7
 
6
8
  #endif /* ZSTD_RUBY_H */
@@ -4,7 +4,7 @@ $CFLAGS = '-I. -O3 -std=c99'
4
4
  $CPPFLAGS += " -fdeclspec" if CONFIG['CXX'] =~ /clang/
5
5
 
6
6
  Dir.chdir File.expand_path('..', __FILE__) do
7
- $srcs = Dir['**/*.c']
7
+ $srcs = Dir['**/*.c', '**/*.S']
8
8
 
9
9
  Dir.glob('libzstd/*') do |path|
10
10
  if Dir.exist?(path)
@@ -86,7 +86,7 @@ static void* POOL_thread(void* opaque) {
86
86
  { POOL_job const job = ctx->queue[ctx->queueHead];
87
87
  ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize;
88
88
  ctx->numThreadsBusy++;
89
- ctx->queueEmpty = ctx->queueHead == ctx->queueTail;
89
+ ctx->queueEmpty = (ctx->queueHead == ctx->queueTail);
90
90
  /* Unlock the mutex, signal a pusher, and run the job */
91
91
  ZSTD_pthread_cond_signal(&ctx->queuePushCond);
92
92
  ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
@@ -105,6 +105,7 @@ static void* POOL_thread(void* opaque) {
105
105
  assert(0); /* Unreachable */
106
106
  }
107
107
 
108
+ /* ZSTD_createThreadPool() : public access point */
108
109
  POOL_ctx* ZSTD_createThreadPool(size_t numThreads) {
109
110
  return POOL_create (numThreads, 0);
110
111
  }
@@ -114,7 +115,8 @@ POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
114
115
  }
115
116
 
116
117
  POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
117
- ZSTD_customMem customMem) {
118
+ ZSTD_customMem customMem)
119
+ {
118
120
  POOL_ctx* ctx;
119
121
  /* Check parameters */
120
122
  if (!numThreads) { return NULL; }
@@ -192,7 +194,7 @@ void ZSTD_freeThreadPool (ZSTD_threadPool* pool) {
192
194
  POOL_free (pool);
193
195
  }
194
196
 
195
- size_t POOL_sizeof(POOL_ctx *ctx) {
197
+ size_t POOL_sizeof(const POOL_ctx* ctx) {
196
198
  if (ctx==NULL) return 0; /* supports sizeof NULL */
197
199
  return sizeof(*ctx)
198
200
  + ctx->queueSize * sizeof(POOL_job)
@@ -257,7 +259,8 @@ static int isQueueFull(POOL_ctx const* ctx) {
257
259
  }
258
260
 
259
261
 
260
- static void POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque)
262
+ static void
263
+ POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque)
261
264
  {
262
265
  POOL_job const job = {function, opaque};
263
266
  assert(ctx != NULL);
@@ -313,7 +316,9 @@ POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
313
316
  return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem);
314
317
  }
315
318
 
316
- POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) {
319
+ POOL_ctx*
320
+ POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem)
321
+ {
317
322
  (void)numThreads;
318
323
  (void)queueSize;
319
324
  (void)customMem;
@@ -341,7 +346,7 @@ int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) {
341
346
  return 1;
342
347
  }
343
348
 
344
- size_t POOL_sizeof(POOL_ctx* ctx) {
349
+ size_t POOL_sizeof(const POOL_ctx* ctx) {
345
350
  if (ctx==NULL) return 0; /* supports sizeof NULL */
346
351
  assert(ctx == &g_poolCtx);
347
352
  return sizeof(*ctx);
@@ -53,7 +53,7 @@ int POOL_resize(POOL_ctx* ctx, size_t numThreads);
53
53
  * @return threadpool memory usage
54
54
  * note : compatible with NULL (returns 0 in this case)
55
55
  */
56
- size_t POOL_sizeof(POOL_ctx* ctx);
56
+ size_t POOL_sizeof(const POOL_ctx* ctx);
57
57
 
58
58
  /*! POOL_function :
59
59
  * The function type that can be added to a thread pool.
@@ -70,7 +70,7 @@ void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque);
70
70
 
71
71
 
72
72
  /*! POOL_tryAdd() :
73
- * Add the job `function(opaque)` to thread pool _if_ a worker is available.
73
+ * Add the job `function(opaque)` to thread pool _if_ a queue slot is available.
74
74
  * Returns immediately even if not (does not block).
75
75
  * @return : 1 if successful, 0 if not.
76
76
  */
@@ -65,6 +65,12 @@
65
65
  # endif
66
66
  #endif
67
67
 
68
+ /* Mark the internal assembly functions as hidden */
69
+ #ifdef __ELF__
70
+ # define ZSTD_HIDE_ASM_FUNCTION(func) .hidden func
71
+ #else
72
+ # define ZSTD_HIDE_ASM_FUNCTION(func)
73
+ #endif
68
74
 
69
75
  /* Enable runtime BMI2 dispatch based on the CPU.
70
76
  * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default.
@@ -67,7 +67,6 @@ extern "C" {
67
67
  #define ZSTD_OPT_NUM (1<<12)
68
68
 
69
69
  #define ZSTD_REP_NUM 3 /* number of repcodes */
70
- #define ZSTD_REP_MOVE (ZSTD_REP_NUM-1)
71
70
  static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
72
71
 
73
72
  #define KB *(1 <<10)
@@ -285,9 +284,9 @@ typedef enum {
285
284
  * Private declarations
286
285
  *********************************************/
287
286
  typedef struct seqDef_s {
288
- U32 offset; /* offset == rawOffset + ZSTD_REP_NUM, or equivalently, offCode + 1 */
287
+ U32 offBase; /* offBase == Offset + ZSTD_REP_NUM, or repcode 1,2,3 */
289
288
  U16 litLength;
290
- U16 matchLength;
289
+ U16 mlBase; /* mlBase == matchLength - MINMATCH */
291
290
  } seqDef;
292
291
 
293
292
  /* Controls whether seqStore has a single "long" litLength or matchLength. See seqStore_t. */
@@ -329,7 +328,7 @@ MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore
329
328
  {
330
329
  ZSTD_sequenceLength seqLen;
331
330
  seqLen.litLength = seq->litLength;
332
- seqLen.matchLength = seq->matchLength + MINMATCH;
331
+ seqLen.matchLength = seq->mlBase + MINMATCH;
333
332
  if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) {
334
333
  if (seqStore->longLengthType == ZSTD_llt_literalLength) {
335
334
  seqLen.litLength += 0xFFFF;