zstd-ruby 1.5.5.0 → 1.5.6.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/README.md +81 -11
- data/ext/zstdruby/common.h +172 -1
- data/ext/zstdruby/extconf.rb +3 -1
- data/ext/zstdruby/libzstd/common/allocations.h +1 -1
- data/ext/zstdruby/libzstd/common/bitstream.h +49 -29
- data/ext/zstdruby/libzstd/common/compiler.h +114 -22
- data/ext/zstdruby/libzstd/common/cpu.h +36 -0
- data/ext/zstdruby/libzstd/common/debug.c +6 -0
- data/ext/zstdruby/libzstd/common/debug.h +20 -11
- data/ext/zstdruby/libzstd/common/error_private.h +45 -36
- data/ext/zstdruby/libzstd/common/fse.h +3 -2
- data/ext/zstdruby/libzstd/common/fse_decompress.c +19 -17
- data/ext/zstdruby/libzstd/common/huf.h +14 -1
- data/ext/zstdruby/libzstd/common/mem.h +0 -9
- data/ext/zstdruby/libzstd/common/pool.c +1 -1
- data/ext/zstdruby/libzstd/common/pool.h +1 -1
- data/ext/zstdruby/libzstd/common/portability_macros.h +2 -0
- data/ext/zstdruby/libzstd/common/threading.c +8 -2
- data/ext/zstdruby/libzstd/common/xxhash.c +5 -11
- data/ext/zstdruby/libzstd/common/xxhash.h +2341 -1007
- data/ext/zstdruby/libzstd/common/zstd_internal.h +5 -5
- data/ext/zstdruby/libzstd/compress/fse_compress.c +8 -7
- data/ext/zstdruby/libzstd/compress/huf_compress.c +54 -25
- data/ext/zstdruby/libzstd/compress/zstd_compress.c +282 -161
- data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +29 -27
- data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.c +224 -113
- data/ext/zstdruby/libzstd/compress/zstd_cwksp.h +19 -13
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +17 -5
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +11 -0
- data/ext/zstdruby/libzstd/compress/zstd_fast.c +14 -6
- data/ext/zstdruby/libzstd/compress/zstd_lazy.c +129 -87
- data/ext/zstdruby/libzstd/compress/zstd_lazy.h +103 -28
- data/ext/zstdruby/libzstd/compress/zstd_ldm.c +8 -2
- data/ext/zstdruby/libzstd/compress/zstd_opt.c +216 -112
- data/ext/zstdruby/libzstd/compress/zstd_opt.h +31 -7
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +94 -79
- data/ext/zstdruby/libzstd/decompress/huf_decompress.c +188 -126
- data/ext/zstdruby/libzstd/decompress/huf_decompress_amd64.S +38 -19
- data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +84 -32
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.c +231 -208
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.h +1 -1
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_internal.h +2 -0
- data/ext/zstdruby/libzstd/dictBuilder/cover.c +16 -12
- data/ext/zstdruby/libzstd/dictBuilder/cover.h +2 -8
- data/ext/zstdruby/libzstd/dictBuilder/fastcover.c +2 -2
- data/ext/zstdruby/libzstd/dictBuilder/zdict.c +12 -6
- data/ext/zstdruby/libzstd/zstd.h +129 -60
- data/ext/zstdruby/main.c +2 -1
- data/ext/zstdruby/skippable_frame.c +1 -1
- data/ext/zstdruby/streaming_compress.c +75 -23
- data/ext/zstdruby/streaming_decompress.c +41 -40
- data/ext/zstdruby/zstdruby.c +60 -52
- data/lib/zstd-ruby/stream_reader.rb +22 -0
- data/lib/zstd-ruby/stream_writer.rb +23 -0
- data/lib/zstd-ruby/version.rb +1 -1
- data/lib/zstd-ruby.rb +2 -0
- data/renovate.json +6 -0
- data/zstd-ruby.gemspec +2 -1
- metadata +20 -4
- data/ext/zstdruby/streaming_compress.h +0 -5
@@ -1,5 +1,4 @@
|
|
1
|
-
#include
|
2
|
-
#include <streaming_compress.h>
|
1
|
+
#include "common.h"
|
3
2
|
|
4
3
|
struct streaming_compress_t {
|
5
4
|
ZSTD_CCtx* ctx;
|
@@ -11,7 +10,11 @@ static void
|
|
11
10
|
streaming_compress_mark(void *p)
|
12
11
|
{
|
13
12
|
struct streaming_compress_t *sc = p;
|
13
|
+
#ifdef HAVE_RB_GC_MARK_MOVABLE
|
14
|
+
rb_gc_mark_movable(sc->buf);
|
15
|
+
#else
|
14
16
|
rb_gc_mark(sc->buf);
|
17
|
+
#endif
|
15
18
|
}
|
16
19
|
|
17
20
|
static void
|
@@ -31,10 +34,26 @@ streaming_compress_memsize(const void *p)
|
|
31
34
|
return sizeof(struct streaming_compress_t);
|
32
35
|
}
|
33
36
|
|
37
|
+
#ifdef HAVE_RB_GC_MARK_MOVABLE
|
38
|
+
static void
|
39
|
+
streaming_compress_compact(void *p)
|
40
|
+
{
|
41
|
+
struct streaming_compress_t *sc = p;
|
42
|
+
sc->buf = rb_gc_location(sc->buf);
|
43
|
+
}
|
44
|
+
#endif
|
45
|
+
|
34
46
|
static const rb_data_type_t streaming_compress_type = {
|
35
|
-
|
36
|
-
|
37
|
-
|
47
|
+
"streaming_compress",
|
48
|
+
{
|
49
|
+
streaming_compress_mark,
|
50
|
+
streaming_compress_free,
|
51
|
+
streaming_compress_memsize,
|
52
|
+
#ifdef HAVE_RB_GC_MARK_MOVABLE
|
53
|
+
streaming_compress_compact,
|
54
|
+
#endif
|
55
|
+
},
|
56
|
+
0, 0, RUBY_TYPED_FREE_IMMEDIATELY
|
38
57
|
};
|
39
58
|
|
40
59
|
static VALUE
|
@@ -51,9 +70,9 @@ rb_streaming_compress_allocate(VALUE klass)
|
|
51
70
|
static VALUE
|
52
71
|
rb_streaming_compress_initialize(int argc, VALUE *argv, VALUE obj)
|
53
72
|
{
|
73
|
+
VALUE kwargs;
|
54
74
|
VALUE compression_level_value;
|
55
|
-
rb_scan_args(argc, argv, "01", &compression_level_value);
|
56
|
-
int compression_level = convert_compression_level(compression_level_value);
|
75
|
+
rb_scan_args(argc, argv, "01:", &compression_level_value, &kwargs);
|
57
76
|
|
58
77
|
struct streaming_compress_t* sc;
|
59
78
|
TypedData_Get_Struct(obj, struct streaming_compress_t, &streaming_compress_type, sc);
|
@@ -63,7 +82,8 @@ rb_streaming_compress_initialize(int argc, VALUE *argv, VALUE obj)
|
|
63
82
|
if (ctx == NULL) {
|
64
83
|
rb_raise(rb_eRuntimeError, "%s", "ZSTD_createCCtx error");
|
65
84
|
}
|
66
|
-
|
85
|
+
set_compress_params(ctx, compression_level_value, kwargs);
|
86
|
+
|
67
87
|
sc->ctx = ctx;
|
68
88
|
sc->buf = rb_str_new(NULL, buffOutSize);
|
69
89
|
sc->buf_size = buffOutSize;
|
@@ -86,7 +106,7 @@ no_compress(struct streaming_compress_t* sc, ZSTD_EndDirective endOp)
|
|
86
106
|
do {
|
87
107
|
ZSTD_outBuffer output = { (void*)output_data, sc->buf_size, 0 };
|
88
108
|
|
89
|
-
size_t const ret =
|
109
|
+
size_t const ret = zstd_stream_compress(sc->ctx, &output, &input, endOp, false);
|
90
110
|
if (ZSTD_isError(ret)) {
|
91
111
|
rb_raise(rb_eRuntimeError, "flush error error code: %s", ZSTD_getErrorName(ret));
|
92
112
|
}
|
@@ -105,11 +125,12 @@ rb_streaming_compress_compress(VALUE obj, VALUE src)
|
|
105
125
|
|
106
126
|
struct streaming_compress_t* sc;
|
107
127
|
TypedData_Get_Struct(obj, struct streaming_compress_t, &streaming_compress_type, sc);
|
128
|
+
|
108
129
|
const char* output_data = RSTRING_PTR(sc->buf);
|
109
130
|
VALUE result = rb_str_new(0, 0);
|
110
131
|
while (input.pos < input.size) {
|
111
132
|
ZSTD_outBuffer output = { (void*)output_data, sc->buf_size, 0 };
|
112
|
-
size_t const ret =
|
133
|
+
size_t const ret = zstd_stream_compress(sc->ctx, &output, &input, ZSTD_e_continue, false);
|
113
134
|
if (ZSTD_isError(ret)) {
|
114
135
|
rb_raise(rb_eRuntimeError, "compress error error code: %s", ZSTD_getErrorName(ret));
|
115
136
|
}
|
@@ -119,27 +140,54 @@ rb_streaming_compress_compress(VALUE obj, VALUE src)
|
|
119
140
|
}
|
120
141
|
|
121
142
|
static VALUE
|
122
|
-
|
143
|
+
rb_streaming_compress_write(int argc, VALUE *argv, VALUE obj)
|
123
144
|
{
|
124
|
-
|
125
|
-
|
126
|
-
size_t input_size = RSTRING_LEN(src);
|
127
|
-
ZSTD_inBuffer input = { input_data, input_size, 0 };
|
128
|
-
|
145
|
+
size_t total = 0;
|
146
|
+
VALUE result = rb_str_new(0, 0);
|
129
147
|
struct streaming_compress_t* sc;
|
130
148
|
TypedData_Get_Struct(obj, struct streaming_compress_t, &streaming_compress_type, sc);
|
131
149
|
const char* output_data = RSTRING_PTR(sc->buf);
|
132
150
|
|
133
|
-
while (
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
151
|
+
while (argc-- > 0) {
|
152
|
+
VALUE str = *argv++;
|
153
|
+
StringValue(str);
|
154
|
+
const char* input_data = RSTRING_PTR(str);
|
155
|
+
size_t input_size = RSTRING_LEN(str);
|
156
|
+
ZSTD_inBuffer input = { input_data, input_size, 0 };
|
157
|
+
|
158
|
+
while (input.pos < input.size) {
|
159
|
+
ZSTD_outBuffer output = { (void*)output_data, sc->buf_size, 0 };
|
160
|
+
size_t const ret = zstd_stream_compress(sc->ctx, &output, &input, ZSTD_e_continue, false);
|
161
|
+
if (ZSTD_isError(ret)) {
|
162
|
+
rb_raise(rb_eRuntimeError, "compress error error code: %s", ZSTD_getErrorName(ret));
|
163
|
+
}
|
164
|
+
total += RSTRING_LEN(str);
|
138
165
|
}
|
139
166
|
}
|
140
|
-
return
|
167
|
+
return SIZET2NUM(total);
|
141
168
|
}
|
142
169
|
|
170
|
+
/*
|
171
|
+
* Document-method: <<
|
172
|
+
* Same as IO.
|
173
|
+
*/
|
174
|
+
#define rb_streaming_compress_addstr rb_io_addstr
|
175
|
+
/*
|
176
|
+
* Document-method: printf
|
177
|
+
* Same as IO.
|
178
|
+
*/
|
179
|
+
#define rb_streaming_compress_printf rb_io_printf
|
180
|
+
/*
|
181
|
+
* Document-method: print
|
182
|
+
* Same as IO.
|
183
|
+
*/
|
184
|
+
#define rb_streaming_compress_print rb_io_print
|
185
|
+
/*
|
186
|
+
* Document-method: puts
|
187
|
+
* Same as IO.
|
188
|
+
*/
|
189
|
+
#define rb_streaming_compress_puts rb_io_puts
|
190
|
+
|
143
191
|
static VALUE
|
144
192
|
rb_streaming_compress_flush(VALUE obj)
|
145
193
|
{
|
@@ -166,7 +214,12 @@ zstd_ruby_streaming_compress_init(void)
|
|
166
214
|
rb_define_alloc_func(cStreamingCompress, rb_streaming_compress_allocate);
|
167
215
|
rb_define_method(cStreamingCompress, "initialize", rb_streaming_compress_initialize, -1);
|
168
216
|
rb_define_method(cStreamingCompress, "compress", rb_streaming_compress_compress, 1);
|
217
|
+
rb_define_method(cStreamingCompress, "write", rb_streaming_compress_write, -1);
|
169
218
|
rb_define_method(cStreamingCompress, "<<", rb_streaming_compress_addstr, 1);
|
219
|
+
rb_define_method(cStreamingCompress, "printf", rb_streaming_compress_printf, -1);
|
220
|
+
rb_define_method(cStreamingCompress, "print", rb_streaming_compress_print, -1);
|
221
|
+
rb_define_method(cStreamingCompress, "puts", rb_streaming_compress_puts, -1);
|
222
|
+
|
170
223
|
rb_define_method(cStreamingCompress, "flush", rb_streaming_compress_flush, 0);
|
171
224
|
rb_define_method(cStreamingCompress, "finish", rb_streaming_compress_finish, 0);
|
172
225
|
|
@@ -174,4 +227,3 @@ zstd_ruby_streaming_compress_init(void)
|
|
174
227
|
rb_define_const(cStreamingCompress, "FLUSH", INT2FIX(ZSTD_e_flush));
|
175
228
|
rb_define_const(cStreamingCompress, "END", INT2FIX(ZSTD_e_end));
|
176
229
|
}
|
177
|
-
|
@@ -1,7 +1,7 @@
|
|
1
|
-
#include
|
1
|
+
#include "common.h"
|
2
2
|
|
3
3
|
struct streaming_decompress_t {
|
4
|
-
ZSTD_DCtx*
|
4
|
+
ZSTD_DCtx* dctx;
|
5
5
|
VALUE buf;
|
6
6
|
size_t buf_size;
|
7
7
|
};
|
@@ -10,16 +10,20 @@ static void
|
|
10
10
|
streaming_decompress_mark(void *p)
|
11
11
|
{
|
12
12
|
struct streaming_decompress_t *sd = p;
|
13
|
+
#ifdef HAVE_RB_GC_MARK_MOVABLE
|
14
|
+
rb_gc_mark_movable(sd->buf);
|
15
|
+
#else
|
13
16
|
rb_gc_mark(sd->buf);
|
17
|
+
#endif
|
14
18
|
}
|
15
19
|
|
16
20
|
static void
|
17
21
|
streaming_decompress_free(void *p)
|
18
22
|
{
|
19
23
|
struct streaming_decompress_t *sd = p;
|
20
|
-
ZSTD_DCtx*
|
21
|
-
if (
|
22
|
-
ZSTD_freeDCtx(
|
24
|
+
ZSTD_DCtx* dctx = sd->dctx;
|
25
|
+
if (dctx != NULL) {
|
26
|
+
ZSTD_freeDCtx(dctx);
|
23
27
|
}
|
24
28
|
xfree(sd);
|
25
29
|
}
|
@@ -30,10 +34,26 @@ streaming_decompress_memsize(const void *p)
|
|
30
34
|
return sizeof(struct streaming_decompress_t);
|
31
35
|
}
|
32
36
|
|
37
|
+
#ifdef HAVE_RB_GC_MARK_MOVABLE
|
38
|
+
static void
|
39
|
+
streaming_decompress_compact(void *p)
|
40
|
+
{
|
41
|
+
struct streaming_decompress_t *sd = p;
|
42
|
+
sd->buf = rb_gc_location(sd->buf);
|
43
|
+
}
|
44
|
+
#endif
|
45
|
+
|
33
46
|
static const rb_data_type_t streaming_decompress_type = {
|
34
|
-
|
35
|
-
|
36
|
-
|
47
|
+
"streaming_decompress",
|
48
|
+
{
|
49
|
+
streaming_decompress_mark,
|
50
|
+
streaming_decompress_free,
|
51
|
+
streaming_decompress_memsize,
|
52
|
+
#ifdef HAVE_RB_GC_MARK_MOVABLE
|
53
|
+
streaming_decompress_compact,
|
54
|
+
#endif
|
55
|
+
},
|
56
|
+
0, 0, RUBY_TYPED_FREE_IMMEDIATELY
|
37
57
|
};
|
38
58
|
|
39
59
|
static VALUE
|
@@ -41,24 +61,29 @@ rb_streaming_decompress_allocate(VALUE klass)
|
|
41
61
|
{
|
42
62
|
struct streaming_decompress_t* sd;
|
43
63
|
VALUE obj = TypedData_Make_Struct(klass, struct streaming_decompress_t, &streaming_decompress_type, sd);
|
44
|
-
sd->
|
64
|
+
sd->dctx = NULL;
|
45
65
|
sd->buf = Qnil;
|
46
66
|
sd->buf_size = 0;
|
47
67
|
return obj;
|
48
68
|
}
|
49
69
|
|
50
70
|
static VALUE
|
51
|
-
rb_streaming_decompress_initialize(VALUE obj)
|
71
|
+
rb_streaming_decompress_initialize(int argc, VALUE *argv, VALUE obj)
|
52
72
|
{
|
73
|
+
VALUE kwargs;
|
74
|
+
rb_scan_args(argc, argv, "00:", &kwargs);
|
75
|
+
|
53
76
|
struct streaming_decompress_t* sd;
|
54
77
|
TypedData_Get_Struct(obj, struct streaming_decompress_t, &streaming_decompress_type, sd);
|
55
78
|
size_t const buffOutSize = ZSTD_DStreamOutSize();
|
56
79
|
|
57
|
-
ZSTD_DCtx*
|
58
|
-
if (
|
80
|
+
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
81
|
+
if (dctx == NULL) {
|
59
82
|
rb_raise(rb_eRuntimeError, "%s", "ZSTD_createDCtx error");
|
60
83
|
}
|
61
|
-
|
84
|
+
set_decompress_params(dctx, kwargs);
|
85
|
+
|
86
|
+
sd->dctx = dctx;
|
62
87
|
sd->buf = rb_str_new(NULL, buffOutSize);
|
63
88
|
sd->buf_size = buffOutSize;
|
64
89
|
|
@@ -79,45 +104,21 @@ rb_streaming_decompress_decompress(VALUE obj, VALUE src)
|
|
79
104
|
VALUE result = rb_str_new(0, 0);
|
80
105
|
while (input.pos < input.size) {
|
81
106
|
ZSTD_outBuffer output = { (void*)output_data, sd->buf_size, 0 };
|
82
|
-
size_t const ret =
|
107
|
+
size_t const ret = zstd_stream_decompress(sd->dctx, &output, &input, false);
|
83
108
|
if (ZSTD_isError(ret)) {
|
84
|
-
rb_raise(rb_eRuntimeError, "
|
109
|
+
rb_raise(rb_eRuntimeError, "decompress error error code: %s", ZSTD_getErrorName(ret));
|
85
110
|
}
|
86
111
|
rb_str_cat(result, output.dst, output.pos);
|
87
112
|
}
|
88
113
|
return result;
|
89
114
|
}
|
90
115
|
|
91
|
-
static VALUE
|
92
|
-
rb_streaming_decompress_addstr(VALUE obj, VALUE src)
|
93
|
-
{
|
94
|
-
StringValue(src);
|
95
|
-
const char* input_data = RSTRING_PTR(src);
|
96
|
-
size_t input_size = RSTRING_LEN(src);
|
97
|
-
ZSTD_inBuffer input = { input_data, input_size, 0 };
|
98
|
-
|
99
|
-
struct streaming_decompress_t* sd;
|
100
|
-
TypedData_Get_Struct(obj, struct streaming_decompress_t, &streaming_decompress_type, sd);
|
101
|
-
const char* output_data = RSTRING_PTR(sd->buf);
|
102
|
-
|
103
|
-
while (input.pos < input.size) {
|
104
|
-
ZSTD_outBuffer output = { (void*)output_data, sd->buf_size, 0 };
|
105
|
-
size_t const result = ZSTD_decompressStream(sd->ctx, &output, &input);
|
106
|
-
if (ZSTD_isError(result)) {
|
107
|
-
rb_raise(rb_eRuntimeError, "compress error error code: %s", ZSTD_getErrorName(result));
|
108
|
-
}
|
109
|
-
}
|
110
|
-
return obj;
|
111
|
-
}
|
112
|
-
|
113
116
|
extern VALUE rb_mZstd, cStreamingDecompress;
|
114
117
|
void
|
115
118
|
zstd_ruby_streaming_decompress_init(void)
|
116
119
|
{
|
117
120
|
VALUE cStreamingDecompress = rb_define_class_under(rb_mZstd, "StreamingDecompress", rb_cObject);
|
118
121
|
rb_define_alloc_func(cStreamingDecompress, rb_streaming_decompress_allocate);
|
119
|
-
rb_define_method(cStreamingDecompress, "initialize", rb_streaming_decompress_initialize,
|
122
|
+
rb_define_method(cStreamingDecompress, "initialize", rb_streaming_decompress_initialize, -1);
|
120
123
|
rb_define_method(cStreamingDecompress, "decompress", rb_streaming_decompress_decompress, 1);
|
121
|
-
rb_define_method(cStreamingDecompress, "<<", rb_streaming_decompress_addstr, 1);
|
122
124
|
}
|
123
|
-
|
data/ext/zstdruby/zstdruby.c
CHANGED
@@ -12,28 +12,37 @@ static VALUE rb_compress(int argc, VALUE *argv, VALUE self)
|
|
12
12
|
{
|
13
13
|
VALUE input_value;
|
14
14
|
VALUE compression_level_value;
|
15
|
-
|
16
|
-
|
15
|
+
VALUE kwargs;
|
16
|
+
rb_scan_args(argc, argv, "11:", &input_value, &compression_level_value, &kwargs);
|
17
|
+
|
18
|
+
ZSTD_CCtx* const ctx = ZSTD_createCCtx();
|
19
|
+
if (ctx == NULL) {
|
20
|
+
rb_raise(rb_eRuntimeError, "%s", "ZSTD_createCCtx error");
|
21
|
+
}
|
22
|
+
|
23
|
+
set_compress_params(ctx, compression_level_value, kwargs);
|
17
24
|
|
18
25
|
StringValue(input_value);
|
19
26
|
char* input_data = RSTRING_PTR(input_value);
|
20
27
|
size_t input_size = RSTRING_LEN(input_value);
|
21
|
-
size_t max_compressed_size = ZSTD_compressBound(input_size);
|
22
28
|
|
29
|
+
size_t max_compressed_size = ZSTD_compressBound(input_size);
|
23
30
|
VALUE output = rb_str_new(NULL, max_compressed_size);
|
24
31
|
char* output_data = RSTRING_PTR(output);
|
25
|
-
|
26
|
-
|
27
|
-
if (ZSTD_isError(
|
28
|
-
rb_raise(rb_eRuntimeError, "
|
32
|
+
|
33
|
+
size_t const ret = zstd_compress(ctx, output_data, max_compressed_size, input_data, input_size, false);
|
34
|
+
if (ZSTD_isError(ret)) {
|
35
|
+
rb_raise(rb_eRuntimeError, "compress error error code: %s", ZSTD_getErrorName(ret));
|
29
36
|
}
|
37
|
+
rb_str_resize(output, ret);
|
30
38
|
|
31
|
-
|
39
|
+
ZSTD_freeCCtx(ctx);
|
32
40
|
return output;
|
33
41
|
}
|
34
42
|
|
35
43
|
static VALUE rb_compress_using_dict(int argc, VALUE *argv, VALUE self)
|
36
44
|
{
|
45
|
+
rb_warn("Zstd.compress_using_dict is deprecated; use Zstd.compress with `dict:` instead.");
|
37
46
|
VALUE input_value;
|
38
47
|
VALUE dict;
|
39
48
|
VALUE compression_level_value;
|
@@ -76,70 +85,66 @@ static VALUE rb_compress_using_dict(int argc, VALUE *argv, VALUE self)
|
|
76
85
|
}
|
77
86
|
|
78
87
|
|
79
|
-
static VALUE decompress_buffered(const char* input_data, size_t input_size)
|
88
|
+
static VALUE decompress_buffered(ZSTD_DCtx* dctx, const char* input_data, size_t input_size)
|
80
89
|
{
|
81
|
-
const size_t outputBufferSize = 4096;
|
82
|
-
|
83
|
-
ZSTD_DStream* const dstream = ZSTD_createDStream();
|
84
|
-
if (dstream == NULL) {
|
85
|
-
rb_raise(rb_eRuntimeError, "%s", "ZSTD_createDStream failed");
|
86
|
-
}
|
87
|
-
|
88
|
-
size_t initResult = ZSTD_initDStream(dstream);
|
89
|
-
if (ZSTD_isError(initResult)) {
|
90
|
-
ZSTD_freeDStream(dstream);
|
91
|
-
rb_raise(rb_eRuntimeError, "%s: %s", "ZSTD_initDStream failed", ZSTD_getErrorName(initResult));
|
92
|
-
}
|
93
|
-
|
94
|
-
VALUE output_string = rb_str_new(NULL, 0);
|
95
|
-
ZSTD_outBuffer output = { NULL, 0, 0 };
|
96
|
-
|
97
90
|
ZSTD_inBuffer input = { input_data, input_size, 0 };
|
91
|
+
VALUE result = rb_str_new(0, 0);
|
92
|
+
|
98
93
|
while (input.pos < input.size) {
|
99
|
-
output
|
100
|
-
|
94
|
+
ZSTD_outBuffer output = { NULL, 0, 0 };
|
95
|
+
output.size += ZSTD_DStreamOutSize();
|
96
|
+
VALUE output_string = rb_str_new(NULL, output.size);
|
101
97
|
output.dst = RSTRING_PTR(output_string);
|
102
98
|
|
103
|
-
size_t
|
104
|
-
if (ZSTD_isError(
|
105
|
-
|
106
|
-
rb_raise(rb_eRuntimeError, "%s: %s", "ZSTD_decompressStream failed", ZSTD_getErrorName(
|
99
|
+
size_t ret = zstd_stream_decompress(dctx, &output, &input, false);
|
100
|
+
if (ZSTD_isError(ret)) {
|
101
|
+
ZSTD_freeDCtx(dctx);
|
102
|
+
rb_raise(rb_eRuntimeError, "%s: %s", "ZSTD_decompressStream failed", ZSTD_getErrorName(ret));
|
107
103
|
}
|
104
|
+
rb_str_cat(result, output.dst, output.pos);
|
108
105
|
}
|
109
|
-
|
110
|
-
|
111
|
-
rb_str_resize(output_string, output.pos);
|
112
|
-
return output_string;
|
106
|
+
ZSTD_freeDCtx(dctx);
|
107
|
+
return result;
|
113
108
|
}
|
114
109
|
|
115
|
-
static VALUE rb_decompress(VALUE
|
110
|
+
static VALUE rb_decompress(int argc, VALUE *argv, VALUE self)
|
116
111
|
{
|
112
|
+
VALUE input_value;
|
113
|
+
VALUE kwargs;
|
114
|
+
rb_scan_args(argc, argv, "10:", &input_value, &kwargs);
|
117
115
|
StringValue(input_value);
|
118
116
|
char* input_data = RSTRING_PTR(input_value);
|
119
117
|
size_t input_size = RSTRING_LEN(input_value);
|
118
|
+
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
119
|
+
if (dctx == NULL) {
|
120
|
+
rb_raise(rb_eRuntimeError, "%s", "ZSTD_createDCtx failed");
|
121
|
+
}
|
122
|
+
set_decompress_params(dctx, kwargs);
|
120
123
|
|
121
124
|
unsigned long long const uncompressed_size = ZSTD_getFrameContentSize(input_data, input_size);
|
122
125
|
if (uncompressed_size == ZSTD_CONTENTSIZE_ERROR) {
|
123
126
|
rb_raise(rb_eRuntimeError, "%s: %s", "not compressed by zstd", ZSTD_getErrorName(uncompressed_size));
|
124
127
|
}
|
128
|
+
// ZSTD_decompressStream may be called multiple times when ZSTD_CONTENTSIZE_UNKNOWN, causing slowness.
|
129
|
+
// Therefore, we will not standardize on ZSTD_decompressStream
|
125
130
|
if (uncompressed_size == ZSTD_CONTENTSIZE_UNKNOWN) {
|
126
|
-
return decompress_buffered(input_data, input_size);
|
131
|
+
return decompress_buffered(dctx, input_data, input_size);
|
127
132
|
}
|
128
133
|
|
129
134
|
VALUE output = rb_str_new(NULL, uncompressed_size);
|
130
135
|
char* output_data = RSTRING_PTR(output);
|
131
|
-
size_t const decompress_size = ZSTD_decompress((void*)output_data, uncompressed_size,
|
132
|
-
(void*)input_data, input_size);
|
133
136
|
|
137
|
+
size_t const decompress_size = zstd_decompress(dctx, output_data, uncompressed_size, input_data, input_size, false);
|
134
138
|
if (ZSTD_isError(decompress_size)) {
|
135
139
|
rb_raise(rb_eRuntimeError, "%s: %s", "decompress error", ZSTD_getErrorName(decompress_size));
|
136
140
|
}
|
137
|
-
|
141
|
+
ZSTD_freeDCtx(dctx);
|
138
142
|
return output;
|
139
143
|
}
|
140
144
|
|
141
145
|
static VALUE rb_decompress_using_dict(int argc, VALUE *argv, VALUE self)
|
142
146
|
{
|
147
|
+
rb_warn("Zstd.decompress_using_dict is deprecated; use Zstd.decompress with `dict:` instead.");
|
143
148
|
VALUE input_value;
|
144
149
|
VALUE dict;
|
145
150
|
rb_scan_args(argc, argv, "20", &input_value, &dict);
|
@@ -147,15 +152,6 @@ static VALUE rb_decompress_using_dict(int argc, VALUE *argv, VALUE self)
|
|
147
152
|
StringValue(input_value);
|
148
153
|
char* input_data = RSTRING_PTR(input_value);
|
149
154
|
size_t input_size = RSTRING_LEN(input_value);
|
150
|
-
unsigned long long const uncompressed_size = ZSTD_getFrameContentSize(input_data, input_size);
|
151
|
-
if (uncompressed_size == ZSTD_CONTENTSIZE_ERROR) {
|
152
|
-
rb_raise(rb_eRuntimeError, "%s: %s", "not compressed by zstd", ZSTD_getErrorName(uncompressed_size));
|
153
|
-
}
|
154
|
-
if (uncompressed_size == ZSTD_CONTENTSIZE_UNKNOWN) {
|
155
|
-
return decompress_buffered(input_data, input_size);
|
156
|
-
}
|
157
|
-
VALUE output = rb_str_new(NULL, uncompressed_size);
|
158
|
-
char* output_data = RSTRING_PTR(output);
|
159
155
|
|
160
156
|
char* dict_buffer = RSTRING_PTR(dict);
|
161
157
|
size_t dict_size = RSTRING_LEN(dict);
|
@@ -163,12 +159,11 @@ static VALUE rb_decompress_using_dict(int argc, VALUE *argv, VALUE self)
|
|
163
159
|
if (ddict == NULL) {
|
164
160
|
rb_raise(rb_eRuntimeError, "%s", "ZSTD_createDDict failed");
|
165
161
|
}
|
166
|
-
|
167
162
|
unsigned const expected_dict_id = ZSTD_getDictID_fromDDict(ddict);
|
168
163
|
unsigned const actual_dict_id = ZSTD_getDictID_fromFrame(input_data, input_size);
|
169
164
|
if (expected_dict_id != actual_dict_id) {
|
170
165
|
ZSTD_freeDDict(ddict);
|
171
|
-
rb_raise(rb_eRuntimeError, "
|
166
|
+
rb_raise(rb_eRuntimeError, "DictID mismatch");
|
172
167
|
}
|
173
168
|
|
174
169
|
ZSTD_DCtx* const ctx = ZSTD_createDCtx();
|
@@ -176,6 +171,19 @@ static VALUE rb_decompress_using_dict(int argc, VALUE *argv, VALUE self)
|
|
176
171
|
ZSTD_freeDDict(ddict);
|
177
172
|
rb_raise(rb_eRuntimeError, "%s", "ZSTD_createDCtx failed");
|
178
173
|
}
|
174
|
+
|
175
|
+
unsigned long long const uncompressed_size = ZSTD_getFrameContentSize(input_data, input_size);
|
176
|
+
if (uncompressed_size == ZSTD_CONTENTSIZE_ERROR) {
|
177
|
+
ZSTD_freeDDict(ddict);
|
178
|
+
ZSTD_freeDCtx(ctx);
|
179
|
+
rb_raise(rb_eRuntimeError, "%s: %s", "not compressed by zstd", ZSTD_getErrorName(uncompressed_size));
|
180
|
+
}
|
181
|
+
if (uncompressed_size == ZSTD_CONTENTSIZE_UNKNOWN) {
|
182
|
+
return decompress_buffered(ctx, input_data, input_size);
|
183
|
+
}
|
184
|
+
|
185
|
+
VALUE output = rb_str_new(NULL, uncompressed_size);
|
186
|
+
char* output_data = RSTRING_PTR(output);
|
179
187
|
size_t const decompress_size = ZSTD_decompress_usingDDict(ctx, output_data, uncompressed_size, input_data, input_size, ddict);
|
180
188
|
if (ZSTD_isError(decompress_size)) {
|
181
189
|
ZSTD_freeDDict(ddict);
|
@@ -193,6 +201,6 @@ zstd_ruby_init(void)
|
|
193
201
|
rb_define_module_function(rb_mZstd, "zstd_version", zstdVersion, 0);
|
194
202
|
rb_define_module_function(rb_mZstd, "compress", rb_compress, -1);
|
195
203
|
rb_define_module_function(rb_mZstd, "compress_using_dict", rb_compress_using_dict, -1);
|
196
|
-
rb_define_module_function(rb_mZstd, "decompress", rb_decompress, 1);
|
204
|
+
rb_define_module_function(rb_mZstd, "decompress", rb_decompress, -1);
|
197
205
|
rb_define_module_function(rb_mZstd, "decompress_using_dict", rb_decompress_using_dict, -1);
|
198
206
|
}
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module Zstd
|
2
|
+
# @todo Exprimental
|
3
|
+
class StreamReader
|
4
|
+
def initialize(io)
|
5
|
+
@io = io
|
6
|
+
@stream = Zstd::StreamingDecompress.new
|
7
|
+
end
|
8
|
+
|
9
|
+
def read(length)
|
10
|
+
if @io.eof?
|
11
|
+
raise StandardError, "EOF"
|
12
|
+
end
|
13
|
+
data = @io.read(length)
|
14
|
+
@stream.decompress(data)
|
15
|
+
end
|
16
|
+
|
17
|
+
def close
|
18
|
+
@io.write(@stream.finish)
|
19
|
+
@io.close
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module Zstd
|
2
|
+
# @todo Exprimental
|
3
|
+
class StreamWriter
|
4
|
+
def initialize(io, level: nil)
|
5
|
+
@io = io
|
6
|
+
@stream = Zstd::StreamingCompress.new(level)
|
7
|
+
end
|
8
|
+
|
9
|
+
def write(*data)
|
10
|
+
@stream.write(*data)
|
11
|
+
@io.write(@stream.flush)
|
12
|
+
end
|
13
|
+
|
14
|
+
def finish
|
15
|
+
@io.write(@stream.finish)
|
16
|
+
end
|
17
|
+
|
18
|
+
def close
|
19
|
+
@io.write(@stream.finish)
|
20
|
+
@io.close
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
data/lib/zstd-ruby/version.rb
CHANGED
data/lib/zstd-ruby.rb
CHANGED
data/renovate.json
ADDED
data/zstd-ruby.gemspec
CHANGED
@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
|
|
24
24
|
#end
|
25
25
|
|
26
26
|
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
27
|
-
f.match(%r{^(test|spec|features|benchmarks|zstd|.github)/})
|
27
|
+
f.match(%r{^(test|spec|features|benchmarks|zstd|.github|examples)/})
|
28
28
|
end
|
29
29
|
spec.bindir = "exe"
|
30
30
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
@@ -35,4 +35,5 @@ Gem::Specification.new do |spec|
|
|
35
35
|
spec.add_development_dependency "rake", "~> 13.0"
|
36
36
|
spec.add_development_dependency "rake-compiler", '~> 1'
|
37
37
|
spec.add_development_dependency "rspec", "~> 3.0"
|
38
|
+
spec.add_development_dependency "pry"
|
38
39
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zstd-ruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.5.
|
4
|
+
version: 1.5.6.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- SpringMT
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-05-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -66,6 +66,20 @@ dependencies:
|
|
66
66
|
- - "~>"
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '3.0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: pry
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
69
83
|
description: Ruby binding for zstd(Zstandard - Fast real-time compression algorithm).
|
70
84
|
See https://github.com/facebook/zstd
|
71
85
|
email:
|
@@ -159,11 +173,13 @@ files:
|
|
159
173
|
- ext/zstdruby/main.c
|
160
174
|
- ext/zstdruby/skippable_frame.c
|
161
175
|
- ext/zstdruby/streaming_compress.c
|
162
|
-
- ext/zstdruby/streaming_compress.h
|
163
176
|
- ext/zstdruby/streaming_decompress.c
|
164
177
|
- ext/zstdruby/zstdruby.c
|
165
178
|
- lib/zstd-ruby.rb
|
179
|
+
- lib/zstd-ruby/stream_reader.rb
|
180
|
+
- lib/zstd-ruby/stream_writer.rb
|
166
181
|
- lib/zstd-ruby/version.rb
|
182
|
+
- renovate.json
|
167
183
|
- zstd-ruby.gemspec
|
168
184
|
homepage: https://github.com/SpringMT/zstd-ruby
|
169
185
|
licenses:
|
@@ -184,7 +200,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
184
200
|
- !ruby/object:Gem::Version
|
185
201
|
version: '0'
|
186
202
|
requirements: []
|
187
|
-
rubygems_version: 3.3
|
203
|
+
rubygems_version: 3.5.3
|
188
204
|
signing_key:
|
189
205
|
specification_version: 4
|
190
206
|
summary: Ruby binding for zstd(Zstandard - Fast real-time compression algorithm)
|