zstd-ruby 1.5.1.0 → 1.5.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/README.md +56 -3
- data/Rakefile +8 -2
- data/ext/zstdruby/{zstdruby.h → common.h} +2 -0
- data/ext/zstdruby/extconf.rb +1 -1
- data/ext/zstdruby/libzstd/common/pool.c +11 -6
- data/ext/zstdruby/libzstd/common/pool.h +2 -2
- data/ext/zstdruby/libzstd/common/portability_macros.h +6 -0
- data/ext/zstdruby/libzstd/common/zstd_internal.h +3 -4
- data/ext/zstdruby/libzstd/compress/zstd_compress.c +114 -96
- data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +72 -39
- data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.c +10 -10
- data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.c +1 -1
- data/ext/zstdruby/libzstd/compress/zstd_cwksp.h +38 -24
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +10 -10
- data/ext/zstdruby/libzstd/compress/zstd_fast.c +11 -11
- data/ext/zstdruby/libzstd/compress/zstd_lazy.c +66 -62
- data/ext/zstdruby/libzstd/compress/zstd_ldm.c +5 -3
- data/ext/zstdruby/libzstd/compress/zstd_opt.c +66 -43
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +17 -9
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +4 -1
- data/ext/zstdruby/libzstd/decompress/huf_decompress.c +2 -2
- data/ext/zstdruby/libzstd/decompress/huf_decompress_amd64.S +17 -3
- data/ext/zstdruby/libzstd/dictBuilder/zdict.c +2 -2
- data/ext/zstdruby/libzstd/zstd.h +1 -1
- data/ext/zstdruby/main.c +14 -0
- data/ext/zstdruby/streaming_compress.c +185 -0
- data/ext/zstdruby/streaming_compress.h +5 -0
- data/ext/zstdruby/streaming_decompress.c +125 -0
- data/ext/zstdruby/zstdruby.c +4 -6
- data/lib/zstd-ruby/version.rb +1 -1
- data/zstd-ruby.gemspec +1 -1
- metadata +11 -40
- data/.github/dependabot.yml +0 -8
- data/.github/workflows/ruby.yml +0 -35
- data/ext/zstdruby/libzstd/.gitignore +0 -3
- data/ext/zstdruby/libzstd/BUCK +0 -232
- data/ext/zstdruby/libzstd/Makefile +0 -357
- data/ext/zstdruby/libzstd/README.md +0 -217
- data/ext/zstdruby/libzstd/deprecated/zbuff.h +0 -214
- data/ext/zstdruby/libzstd/deprecated/zbuff_common.c +0 -26
- data/ext/zstdruby/libzstd/deprecated/zbuff_compress.c +0 -167
- data/ext/zstdruby/libzstd/deprecated/zbuff_decompress.c +0 -75
- data/ext/zstdruby/libzstd/dll/example/Makefile +0 -48
- data/ext/zstdruby/libzstd/dll/example/README.md +0 -63
- data/ext/zstdruby/libzstd/dll/example/build_package.bat +0 -20
- data/ext/zstdruby/libzstd/dll/example/fullbench-dll.sln +0 -25
- data/ext/zstdruby/libzstd/dll/example/fullbench-dll.vcxproj +0 -181
- data/ext/zstdruby/libzstd/legacy/zstd_legacy.h +0 -415
- data/ext/zstdruby/libzstd/legacy/zstd_v01.c +0 -2158
- data/ext/zstdruby/libzstd/legacy/zstd_v01.h +0 -94
- data/ext/zstdruby/libzstd/legacy/zstd_v02.c +0 -3518
- data/ext/zstdruby/libzstd/legacy/zstd_v02.h +0 -93
- data/ext/zstdruby/libzstd/legacy/zstd_v03.c +0 -3160
- data/ext/zstdruby/libzstd/legacy/zstd_v03.h +0 -93
- data/ext/zstdruby/libzstd/legacy/zstd_v04.c +0 -3647
- data/ext/zstdruby/libzstd/legacy/zstd_v04.h +0 -142
- data/ext/zstdruby/libzstd/legacy/zstd_v05.c +0 -4050
- data/ext/zstdruby/libzstd/legacy/zstd_v05.h +0 -162
- data/ext/zstdruby/libzstd/legacy/zstd_v06.c +0 -4154
- data/ext/zstdruby/libzstd/legacy/zstd_v06.h +0 -172
- data/ext/zstdruby/libzstd/legacy/zstd_v07.c +0 -4541
- data/ext/zstdruby/libzstd/legacy/zstd_v07.h +0 -187
- data/ext/zstdruby/libzstd/libzstd.mk +0 -185
- data/ext/zstdruby/libzstd/libzstd.pc.in +0 -16
- data/ext/zstdruby/libzstd/modulemap/module.modulemap +0 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 70096397fcb0ba88b1332602dc743a979e6417bc07aa25270b2fa3e637922291
|
4
|
+
data.tar.gz: e3e2f2ddf2d2d01514e7128b01792e10d9b48cbc08f61a25e0cd9ddf3e91430e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fd02829550fef9d56add09f13c1bee7a5171830e6f7f08e728ef4140e922657b1c70df79e4dd4b56b1bf77cddedbcea4830405561d6cb027bbbce928ff5d7152
|
7
|
+
data.tar.gz: aa56ea5cbc0b9d6e6268691ad656d71f38afde239d9833f07c4ae02a6ab1e1ee2b81e92e5489db3e993774938e58d8fb884c8d882ba4126de15588164ceeff25
|
data/.gitignore
CHANGED
data/README.md
CHANGED
@@ -10,7 +10,7 @@ See https://github.com/facebook/zstd
|
|
10
10
|
Fork from https://github.com/jarredholman/ruby-zstd.
|
11
11
|
|
12
12
|
## Zstd version
|
13
|
-
v1.5.
|
13
|
+
v1.5.2 (https://github.com/facebook/zstd/tree/v1.5.2)
|
14
14
|
|
15
15
|
## Installation
|
16
16
|
|
@@ -34,20 +34,73 @@ Or install it yourself as:
|
|
34
34
|
require 'zstd-ruby'
|
35
35
|
```
|
36
36
|
|
37
|
-
###
|
37
|
+
### Simple Compression
|
38
38
|
|
39
39
|
```ruby
|
40
40
|
compressed_data = Zstd.compress(data)
|
41
41
|
compressed_data = Zstd.compress(data, complession_level) # default compression_level is 0
|
42
42
|
```
|
43
43
|
|
44
|
+
### Streaming Compression
|
45
|
+
```
|
46
|
+
stream = Zstd::StreamingCompress.new
|
47
|
+
stream << "abc" << "def"
|
48
|
+
res = stream.flush
|
49
|
+
stream << "ghi"
|
50
|
+
res << stream.finish
|
51
|
+
```
|
52
|
+
|
53
|
+
or
|
54
|
+
|
55
|
+
```
|
56
|
+
stream = Zstd::StreamingCompress.new
|
57
|
+
res = stream.compress("abc")
|
58
|
+
res << stream.flush
|
59
|
+
res << stream.compress("def")
|
60
|
+
res << stream.finish
|
61
|
+
```
|
44
62
|
|
45
|
-
###
|
63
|
+
### Simple Decompression
|
46
64
|
|
47
65
|
```ruby
|
48
66
|
data = Zstd.decompress(compressed_data)
|
49
67
|
```
|
50
68
|
|
69
|
+
### Streaming Decompression
|
70
|
+
```
|
71
|
+
cstr = "" # Compressed data
|
72
|
+
stream = Zstd::StreamingDecompress.new
|
73
|
+
result = ''
|
74
|
+
result << stream.decompress(cstr[0, 10])
|
75
|
+
result << stream.decompress(cstr[10..-1])
|
76
|
+
```
|
77
|
+
|
78
|
+
|
79
|
+
## JRuby
|
80
|
+
This gem does not support JRuby.
|
81
|
+
|
82
|
+
Please consider using https://github.com/luben/zstd-jni.
|
83
|
+
|
84
|
+
Sample code is below.
|
85
|
+
|
86
|
+
```
|
87
|
+
require 'java'
|
88
|
+
require_relative './zstd-jni-1.5.2-3.jar'
|
89
|
+
|
90
|
+
str = "testtest"
|
91
|
+
compressed = com.github.luben.zstd.Zstd.compress(str.to_java_bytes)
|
92
|
+
puts com.github.luben.zstd.Zstd.decompress(compressed, str.length)
|
93
|
+
```
|
94
|
+
|
95
|
+
```
|
96
|
+
% ls
|
97
|
+
test.rb zstd-jni-1.5.2-3.jar
|
98
|
+
% ruby -v
|
99
|
+
jruby 9.3.2.0 (2.6.8) 2021-12-01 0b8223f905 OpenJDK 64-Bit Server VM 11.0.12+0 on 11.0.12+0 +jit [darwin-x86_64]
|
100
|
+
% ruby test.rb
|
101
|
+
testtest
|
102
|
+
```
|
103
|
+
|
51
104
|
## Development
|
52
105
|
|
53
106
|
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
data/Rakefile
CHANGED
@@ -17,6 +17,12 @@ task :default => [:clobber, :compile, :spec]
|
|
17
17
|
|
18
18
|
desc 'Sync zstd libs dirs to ext/zstdruby/libzstd'
|
19
19
|
task :zstd_update do
|
20
|
-
FileUtils.rm_r(
|
21
|
-
FileUtils.
|
20
|
+
FileUtils.rm_r("ext/zstdruby/libzstd")
|
21
|
+
FileUtils.mkdir_p("ext/zstdruby/libzstd")
|
22
|
+
["common", "compress", "decompress", "dictBuilder"].each do |dir|
|
23
|
+
FileUtils.cp_r("zstd/lib/#{dir}", "ext/zstdruby/libzstd/#{dir}")
|
24
|
+
end
|
25
|
+
FileUtils.cp_r('zstd/lib/zdict.h', 'ext/zstdruby/libzstd')
|
26
|
+
FileUtils.cp_r('zstd/lib/zstd.h', 'ext/zstdruby/libzstd')
|
27
|
+
FileUtils.cp_r('zstd/lib/zstd_errors.h', 'ext/zstdruby/libzstd')
|
22
28
|
end
|
data/ext/zstdruby/extconf.rb
CHANGED
@@ -86,7 +86,7 @@ static void* POOL_thread(void* opaque) {
|
|
86
86
|
{ POOL_job const job = ctx->queue[ctx->queueHead];
|
87
87
|
ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize;
|
88
88
|
ctx->numThreadsBusy++;
|
89
|
-
ctx->queueEmpty = ctx->queueHead == ctx->queueTail;
|
89
|
+
ctx->queueEmpty = (ctx->queueHead == ctx->queueTail);
|
90
90
|
/* Unlock the mutex, signal a pusher, and run the job */
|
91
91
|
ZSTD_pthread_cond_signal(&ctx->queuePushCond);
|
92
92
|
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
@@ -105,6 +105,7 @@ static void* POOL_thread(void* opaque) {
|
|
105
105
|
assert(0); /* Unreachable */
|
106
106
|
}
|
107
107
|
|
108
|
+
/* ZSTD_createThreadPool() : public access point */
|
108
109
|
POOL_ctx* ZSTD_createThreadPool(size_t numThreads) {
|
109
110
|
return POOL_create (numThreads, 0);
|
110
111
|
}
|
@@ -114,7 +115,8 @@ POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
|
|
114
115
|
}
|
115
116
|
|
116
117
|
POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
|
117
|
-
ZSTD_customMem customMem)
|
118
|
+
ZSTD_customMem customMem)
|
119
|
+
{
|
118
120
|
POOL_ctx* ctx;
|
119
121
|
/* Check parameters */
|
120
122
|
if (!numThreads) { return NULL; }
|
@@ -192,7 +194,7 @@ void ZSTD_freeThreadPool (ZSTD_threadPool* pool) {
|
|
192
194
|
POOL_free (pool);
|
193
195
|
}
|
194
196
|
|
195
|
-
size_t POOL_sizeof(POOL_ctx
|
197
|
+
size_t POOL_sizeof(const POOL_ctx* ctx) {
|
196
198
|
if (ctx==NULL) return 0; /* supports sizeof NULL */
|
197
199
|
return sizeof(*ctx)
|
198
200
|
+ ctx->queueSize * sizeof(POOL_job)
|
@@ -257,7 +259,8 @@ static int isQueueFull(POOL_ctx const* ctx) {
|
|
257
259
|
}
|
258
260
|
|
259
261
|
|
260
|
-
static void
|
262
|
+
static void
|
263
|
+
POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque)
|
261
264
|
{
|
262
265
|
POOL_job const job = {function, opaque};
|
263
266
|
assert(ctx != NULL);
|
@@ -313,7 +316,9 @@ POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
|
|
313
316
|
return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem);
|
314
317
|
}
|
315
318
|
|
316
|
-
POOL_ctx*
|
319
|
+
POOL_ctx*
|
320
|
+
POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem)
|
321
|
+
{
|
317
322
|
(void)numThreads;
|
318
323
|
(void)queueSize;
|
319
324
|
(void)customMem;
|
@@ -341,7 +346,7 @@ int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) {
|
|
341
346
|
return 1;
|
342
347
|
}
|
343
348
|
|
344
|
-
size_t POOL_sizeof(POOL_ctx* ctx) {
|
349
|
+
size_t POOL_sizeof(const POOL_ctx* ctx) {
|
345
350
|
if (ctx==NULL) return 0; /* supports sizeof NULL */
|
346
351
|
assert(ctx == &g_poolCtx);
|
347
352
|
return sizeof(*ctx);
|
@@ -53,7 +53,7 @@ int POOL_resize(POOL_ctx* ctx, size_t numThreads);
|
|
53
53
|
* @return threadpool memory usage
|
54
54
|
* note : compatible with NULL (returns 0 in this case)
|
55
55
|
*/
|
56
|
-
size_t POOL_sizeof(POOL_ctx* ctx);
|
56
|
+
size_t POOL_sizeof(const POOL_ctx* ctx);
|
57
57
|
|
58
58
|
/*! POOL_function :
|
59
59
|
* The function type that can be added to a thread pool.
|
@@ -70,7 +70,7 @@ void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque);
|
|
70
70
|
|
71
71
|
|
72
72
|
/*! POOL_tryAdd() :
|
73
|
-
* Add the job `function(opaque)` to thread pool _if_ a
|
73
|
+
* Add the job `function(opaque)` to thread pool _if_ a queue slot is available.
|
74
74
|
* Returns immediately even if not (does not block).
|
75
75
|
* @return : 1 if successful, 0 if not.
|
76
76
|
*/
|
@@ -65,6 +65,12 @@
|
|
65
65
|
# endif
|
66
66
|
#endif
|
67
67
|
|
68
|
+
/* Mark the internal assembly functions as hidden */
|
69
|
+
#ifdef __ELF__
|
70
|
+
# define ZSTD_HIDE_ASM_FUNCTION(func) .hidden func
|
71
|
+
#else
|
72
|
+
# define ZSTD_HIDE_ASM_FUNCTION(func)
|
73
|
+
#endif
|
68
74
|
|
69
75
|
/* Enable runtime BMI2 dispatch based on the CPU.
|
70
76
|
* Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default.
|
@@ -67,7 +67,6 @@ extern "C" {
|
|
67
67
|
#define ZSTD_OPT_NUM (1<<12)
|
68
68
|
|
69
69
|
#define ZSTD_REP_NUM 3 /* number of repcodes */
|
70
|
-
#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1)
|
71
70
|
static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
|
72
71
|
|
73
72
|
#define KB *(1 <<10)
|
@@ -285,9 +284,9 @@ typedef enum {
|
|
285
284
|
* Private declarations
|
286
285
|
*********************************************/
|
287
286
|
typedef struct seqDef_s {
|
288
|
-
U32
|
287
|
+
U32 offBase; /* offBase == Offset + ZSTD_REP_NUM, or repcode 1,2,3 */
|
289
288
|
U16 litLength;
|
290
|
-
U16 matchLength
|
289
|
+
U16 mlBase; /* mlBase == matchLength - MINMATCH */
|
291
290
|
} seqDef;
|
292
291
|
|
293
292
|
/* Controls whether seqStore has a single "long" litLength or matchLength. See seqStore_t. */
|
@@ -329,7 +328,7 @@ MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore
|
|
329
328
|
{
|
330
329
|
ZSTD_sequenceLength seqLen;
|
331
330
|
seqLen.litLength = seq->litLength;
|
332
|
-
seqLen.matchLength = seq->
|
331
|
+
seqLen.matchLength = seq->mlBase + MINMATCH;
|
333
332
|
if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) {
|
334
333
|
if (seqStore->longLengthType == ZSTD_llt_literalLength) {
|
335
334
|
seqLen.litLength += 0xFFFF;
|