zstd-ruby 1.5.5.0 → 1.5.6.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/README.md +81 -11
  4. data/ext/zstdruby/common.h +172 -1
  5. data/ext/zstdruby/extconf.rb +3 -1
  6. data/ext/zstdruby/libzstd/common/allocations.h +1 -1
  7. data/ext/zstdruby/libzstd/common/bitstream.h +49 -29
  8. data/ext/zstdruby/libzstd/common/compiler.h +114 -22
  9. data/ext/zstdruby/libzstd/common/cpu.h +36 -0
  10. data/ext/zstdruby/libzstd/common/debug.c +6 -0
  11. data/ext/zstdruby/libzstd/common/debug.h +20 -11
  12. data/ext/zstdruby/libzstd/common/error_private.h +45 -36
  13. data/ext/zstdruby/libzstd/common/fse.h +3 -2
  14. data/ext/zstdruby/libzstd/common/fse_decompress.c +19 -17
  15. data/ext/zstdruby/libzstd/common/huf.h +14 -1
  16. data/ext/zstdruby/libzstd/common/mem.h +0 -9
  17. data/ext/zstdruby/libzstd/common/pool.c +1 -1
  18. data/ext/zstdruby/libzstd/common/pool.h +1 -1
  19. data/ext/zstdruby/libzstd/common/portability_macros.h +2 -0
  20. data/ext/zstdruby/libzstd/common/threading.c +8 -2
  21. data/ext/zstdruby/libzstd/common/xxhash.c +5 -11
  22. data/ext/zstdruby/libzstd/common/xxhash.h +2341 -1007
  23. data/ext/zstdruby/libzstd/common/zstd_internal.h +5 -5
  24. data/ext/zstdruby/libzstd/compress/fse_compress.c +8 -7
  25. data/ext/zstdruby/libzstd/compress/huf_compress.c +54 -25
  26. data/ext/zstdruby/libzstd/compress/zstd_compress.c +282 -161
  27. data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +29 -27
  28. data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.c +224 -113
  29. data/ext/zstdruby/libzstd/compress/zstd_cwksp.h +19 -13
  30. data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +17 -5
  31. data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +11 -0
  32. data/ext/zstdruby/libzstd/compress/zstd_fast.c +14 -6
  33. data/ext/zstdruby/libzstd/compress/zstd_lazy.c +129 -87
  34. data/ext/zstdruby/libzstd/compress/zstd_lazy.h +103 -28
  35. data/ext/zstdruby/libzstd/compress/zstd_ldm.c +8 -2
  36. data/ext/zstdruby/libzstd/compress/zstd_opt.c +216 -112
  37. data/ext/zstdruby/libzstd/compress/zstd_opt.h +31 -7
  38. data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +94 -79
  39. data/ext/zstdruby/libzstd/decompress/huf_decompress.c +188 -126
  40. data/ext/zstdruby/libzstd/decompress/huf_decompress_amd64.S +38 -19
  41. data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +84 -32
  42. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.c +231 -208
  43. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.h +1 -1
  44. data/ext/zstdruby/libzstd/decompress/zstd_decompress_internal.h +2 -0
  45. data/ext/zstdruby/libzstd/dictBuilder/cover.c +16 -12
  46. data/ext/zstdruby/libzstd/dictBuilder/cover.h +2 -8
  47. data/ext/zstdruby/libzstd/dictBuilder/fastcover.c +2 -2
  48. data/ext/zstdruby/libzstd/dictBuilder/zdict.c +12 -6
  49. data/ext/zstdruby/libzstd/zstd.h +129 -60
  50. data/ext/zstdruby/main.c +2 -1
  51. data/ext/zstdruby/skippable_frame.c +1 -1
  52. data/ext/zstdruby/streaming_compress.c +75 -23
  53. data/ext/zstdruby/streaming_decompress.c +41 -40
  54. data/ext/zstdruby/zstdruby.c +60 -52
  55. data/lib/zstd-ruby/stream_reader.rb +22 -0
  56. data/lib/zstd-ruby/stream_writer.rb +23 -0
  57. data/lib/zstd-ruby/version.rb +1 -1
  58. data/lib/zstd-ruby.rb +2 -0
  59. data/renovate.json +6 -0
  60. data/zstd-ruby.gemspec +2 -1
  61. metadata +20 -4
  62. data/ext/zstdruby/streaming_compress.h +0 -5
@@ -1,5 +1,4 @@
1
- #include <common.h>
2
- #include <streaming_compress.h>
1
+ #include "common.h"
3
2
 
4
3
  struct streaming_compress_t {
5
4
  ZSTD_CCtx* ctx;
@@ -11,7 +10,11 @@ static void
11
10
  streaming_compress_mark(void *p)
12
11
  {
13
12
  struct streaming_compress_t *sc = p;
13
+ #ifdef HAVE_RB_GC_MARK_MOVABLE
14
+ rb_gc_mark_movable(sc->buf);
15
+ #else
14
16
  rb_gc_mark(sc->buf);
17
+ #endif
15
18
  }
16
19
 
17
20
  static void
@@ -31,10 +34,26 @@ streaming_compress_memsize(const void *p)
31
34
  return sizeof(struct streaming_compress_t);
32
35
  }
33
36
 
37
+ #ifdef HAVE_RB_GC_MARK_MOVABLE
38
+ static void
39
+ streaming_compress_compact(void *p)
40
+ {
41
+ struct streaming_compress_t *sc = p;
42
+ sc->buf = rb_gc_location(sc->buf);
43
+ }
44
+ #endif
45
+
34
46
  static const rb_data_type_t streaming_compress_type = {
35
- "streaming_compress",
36
- { streaming_compress_mark, streaming_compress_free, streaming_compress_memsize, },
37
- 0, 0, RUBY_TYPED_FREE_IMMEDIATELY
47
+ "streaming_compress",
48
+ {
49
+ streaming_compress_mark,
50
+ streaming_compress_free,
51
+ streaming_compress_memsize,
52
+ #ifdef HAVE_RB_GC_MARK_MOVABLE
53
+ streaming_compress_compact,
54
+ #endif
55
+ },
56
+ 0, 0, RUBY_TYPED_FREE_IMMEDIATELY
38
57
  };
39
58
 
40
59
  static VALUE
@@ -51,9 +70,9 @@ rb_streaming_compress_allocate(VALUE klass)
51
70
  static VALUE
52
71
  rb_streaming_compress_initialize(int argc, VALUE *argv, VALUE obj)
53
72
  {
73
+ VALUE kwargs;
54
74
  VALUE compression_level_value;
55
- rb_scan_args(argc, argv, "01", &compression_level_value);
56
- int compression_level = convert_compression_level(compression_level_value);
75
+ rb_scan_args(argc, argv, "01:", &compression_level_value, &kwargs);
57
76
 
58
77
  struct streaming_compress_t* sc;
59
78
  TypedData_Get_Struct(obj, struct streaming_compress_t, &streaming_compress_type, sc);
@@ -63,7 +82,8 @@ rb_streaming_compress_initialize(int argc, VALUE *argv, VALUE obj)
63
82
  if (ctx == NULL) {
64
83
  rb_raise(rb_eRuntimeError, "%s", "ZSTD_createCCtx error");
65
84
  }
66
- ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, compression_level);
85
+ set_compress_params(ctx, compression_level_value, kwargs);
86
+
67
87
  sc->ctx = ctx;
68
88
  sc->buf = rb_str_new(NULL, buffOutSize);
69
89
  sc->buf_size = buffOutSize;
@@ -86,7 +106,7 @@ no_compress(struct streaming_compress_t* sc, ZSTD_EndDirective endOp)
86
106
  do {
87
107
  ZSTD_outBuffer output = { (void*)output_data, sc->buf_size, 0 };
88
108
 
89
- size_t const ret = ZSTD_compressStream2(sc->ctx, &output, &input, endOp);
109
+ size_t const ret = zstd_stream_compress(sc->ctx, &output, &input, endOp, false);
90
110
  if (ZSTD_isError(ret)) {
91
111
  rb_raise(rb_eRuntimeError, "flush error error code: %s", ZSTD_getErrorName(ret));
92
112
  }
@@ -105,11 +125,12 @@ rb_streaming_compress_compress(VALUE obj, VALUE src)
105
125
 
106
126
  struct streaming_compress_t* sc;
107
127
  TypedData_Get_Struct(obj, struct streaming_compress_t, &streaming_compress_type, sc);
128
+
108
129
  const char* output_data = RSTRING_PTR(sc->buf);
109
130
  VALUE result = rb_str_new(0, 0);
110
131
  while (input.pos < input.size) {
111
132
  ZSTD_outBuffer output = { (void*)output_data, sc->buf_size, 0 };
112
- size_t const ret = ZSTD_compressStream2(sc->ctx, &output, &input, ZSTD_e_continue);
133
+ size_t const ret = zstd_stream_compress(sc->ctx, &output, &input, ZSTD_e_continue, false);
113
134
  if (ZSTD_isError(ret)) {
114
135
  rb_raise(rb_eRuntimeError, "compress error error code: %s", ZSTD_getErrorName(ret));
115
136
  }
@@ -119,27 +140,54 @@ rb_streaming_compress_compress(VALUE obj, VALUE src)
119
140
  }
120
141
 
121
142
  static VALUE
122
- rb_streaming_compress_addstr(VALUE obj, VALUE src)
143
+ rb_streaming_compress_write(int argc, VALUE *argv, VALUE obj)
123
144
  {
124
- StringValue(src);
125
- const char* input_data = RSTRING_PTR(src);
126
- size_t input_size = RSTRING_LEN(src);
127
- ZSTD_inBuffer input = { input_data, input_size, 0 };
128
-
145
+ size_t total = 0;
146
+ VALUE result = rb_str_new(0, 0);
129
147
  struct streaming_compress_t* sc;
130
148
  TypedData_Get_Struct(obj, struct streaming_compress_t, &streaming_compress_type, sc);
131
149
  const char* output_data = RSTRING_PTR(sc->buf);
132
150
 
133
- while (input.pos < input.size) {
134
- ZSTD_outBuffer output = { (void*)output_data, sc->buf_size, 0 };
135
- size_t const result = ZSTD_compressStream2(sc->ctx, &output, &input, ZSTD_e_continue);
136
- if (ZSTD_isError(result)) {
137
- rb_raise(rb_eRuntimeError, "compress error error code: %s", ZSTD_getErrorName(result));
151
+ while (argc-- > 0) {
152
+ VALUE str = *argv++;
153
+ StringValue(str);
154
+ const char* input_data = RSTRING_PTR(str);
155
+ size_t input_size = RSTRING_LEN(str);
156
+ ZSTD_inBuffer input = { input_data, input_size, 0 };
157
+
158
+ while (input.pos < input.size) {
159
+ ZSTD_outBuffer output = { (void*)output_data, sc->buf_size, 0 };
160
+ size_t const ret = zstd_stream_compress(sc->ctx, &output, &input, ZSTD_e_continue, false);
161
+ if (ZSTD_isError(ret)) {
162
+ rb_raise(rb_eRuntimeError, "compress error error code: %s", ZSTD_getErrorName(ret));
163
+ }
164
+ total += RSTRING_LEN(str);
138
165
  }
139
166
  }
140
- return obj;
167
+ return SIZET2NUM(total);
141
168
  }
142
169
 
170
+ /*
171
+ * Document-method: <<
172
+ * Same as IO.
173
+ */
174
+ #define rb_streaming_compress_addstr rb_io_addstr
175
+ /*
176
+ * Document-method: printf
177
+ * Same as IO.
178
+ */
179
+ #define rb_streaming_compress_printf rb_io_printf
180
+ /*
181
+ * Document-method: print
182
+ * Same as IO.
183
+ */
184
+ #define rb_streaming_compress_print rb_io_print
185
+ /*
186
+ * Document-method: puts
187
+ * Same as IO.
188
+ */
189
+ #define rb_streaming_compress_puts rb_io_puts
190
+
143
191
  static VALUE
144
192
  rb_streaming_compress_flush(VALUE obj)
145
193
  {
@@ -166,7 +214,12 @@ zstd_ruby_streaming_compress_init(void)
166
214
  rb_define_alloc_func(cStreamingCompress, rb_streaming_compress_allocate);
167
215
  rb_define_method(cStreamingCompress, "initialize", rb_streaming_compress_initialize, -1);
168
216
  rb_define_method(cStreamingCompress, "compress", rb_streaming_compress_compress, 1);
217
+ rb_define_method(cStreamingCompress, "write", rb_streaming_compress_write, -1);
169
218
  rb_define_method(cStreamingCompress, "<<", rb_streaming_compress_addstr, 1);
219
+ rb_define_method(cStreamingCompress, "printf", rb_streaming_compress_printf, -1);
220
+ rb_define_method(cStreamingCompress, "print", rb_streaming_compress_print, -1);
221
+ rb_define_method(cStreamingCompress, "puts", rb_streaming_compress_puts, -1);
222
+
170
223
  rb_define_method(cStreamingCompress, "flush", rb_streaming_compress_flush, 0);
171
224
  rb_define_method(cStreamingCompress, "finish", rb_streaming_compress_finish, 0);
172
225
 
@@ -174,4 +227,3 @@ zstd_ruby_streaming_compress_init(void)
174
227
  rb_define_const(cStreamingCompress, "FLUSH", INT2FIX(ZSTD_e_flush));
175
228
  rb_define_const(cStreamingCompress, "END", INT2FIX(ZSTD_e_end));
176
229
  }
177
-
@@ -1,7 +1,7 @@
1
- #include <common.h>
1
+ #include "common.h"
2
2
 
3
3
  struct streaming_decompress_t {
4
- ZSTD_DCtx* ctx;
4
+ ZSTD_DCtx* dctx;
5
5
  VALUE buf;
6
6
  size_t buf_size;
7
7
  };
@@ -10,16 +10,20 @@ static void
10
10
  streaming_decompress_mark(void *p)
11
11
  {
12
12
  struct streaming_decompress_t *sd = p;
13
+ #ifdef HAVE_RB_GC_MARK_MOVABLE
14
+ rb_gc_mark_movable(sd->buf);
15
+ #else
13
16
  rb_gc_mark(sd->buf);
17
+ #endif
14
18
  }
15
19
 
16
20
  static void
17
21
  streaming_decompress_free(void *p)
18
22
  {
19
23
  struct streaming_decompress_t *sd = p;
20
- ZSTD_DCtx* ctx = sd->ctx;
21
- if (ctx != NULL) {
22
- ZSTD_freeDCtx(ctx);
24
+ ZSTD_DCtx* dctx = sd->dctx;
25
+ if (dctx != NULL) {
26
+ ZSTD_freeDCtx(dctx);
23
27
  }
24
28
  xfree(sd);
25
29
  }
@@ -30,10 +34,26 @@ streaming_decompress_memsize(const void *p)
30
34
  return sizeof(struct streaming_decompress_t);
31
35
  }
32
36
 
37
+ #ifdef HAVE_RB_GC_MARK_MOVABLE
38
+ static void
39
+ streaming_decompress_compact(void *p)
40
+ {
41
+ struct streaming_decompress_t *sd = p;
42
+ sd->buf = rb_gc_location(sd->buf);
43
+ }
44
+ #endif
45
+
33
46
  static const rb_data_type_t streaming_decompress_type = {
34
- "streaming_decompress",
35
- { streaming_decompress_mark, streaming_decompress_free, streaming_decompress_memsize, },
36
- 0, 0, RUBY_TYPED_FREE_IMMEDIATELY
47
+ "streaming_decompress",
48
+ {
49
+ streaming_decompress_mark,
50
+ streaming_decompress_free,
51
+ streaming_decompress_memsize,
52
+ #ifdef HAVE_RB_GC_MARK_MOVABLE
53
+ streaming_decompress_compact,
54
+ #endif
55
+ },
56
+ 0, 0, RUBY_TYPED_FREE_IMMEDIATELY
37
57
  };
38
58
 
39
59
  static VALUE
@@ -41,24 +61,29 @@ rb_streaming_decompress_allocate(VALUE klass)
41
61
  {
42
62
  struct streaming_decompress_t* sd;
43
63
  VALUE obj = TypedData_Make_Struct(klass, struct streaming_decompress_t, &streaming_decompress_type, sd);
44
- sd->ctx = NULL;
64
+ sd->dctx = NULL;
45
65
  sd->buf = Qnil;
46
66
  sd->buf_size = 0;
47
67
  return obj;
48
68
  }
49
69
 
50
70
  static VALUE
51
- rb_streaming_decompress_initialize(VALUE obj)
71
+ rb_streaming_decompress_initialize(int argc, VALUE *argv, VALUE obj)
52
72
  {
73
+ VALUE kwargs;
74
+ rb_scan_args(argc, argv, "00:", &kwargs);
75
+
53
76
  struct streaming_decompress_t* sd;
54
77
  TypedData_Get_Struct(obj, struct streaming_decompress_t, &streaming_decompress_type, sd);
55
78
  size_t const buffOutSize = ZSTD_DStreamOutSize();
56
79
 
57
- ZSTD_DCtx* ctx = ZSTD_createDCtx();
58
- if (ctx == NULL) {
80
+ ZSTD_DCtx* dctx = ZSTD_createDCtx();
81
+ if (dctx == NULL) {
59
82
  rb_raise(rb_eRuntimeError, "%s", "ZSTD_createDCtx error");
60
83
  }
61
- sd->ctx = ctx;
84
+ set_decompress_params(dctx, kwargs);
85
+
86
+ sd->dctx = dctx;
62
87
  sd->buf = rb_str_new(NULL, buffOutSize);
63
88
  sd->buf_size = buffOutSize;
64
89
 
@@ -79,45 +104,21 @@ rb_streaming_decompress_decompress(VALUE obj, VALUE src)
79
104
  VALUE result = rb_str_new(0, 0);
80
105
  while (input.pos < input.size) {
81
106
  ZSTD_outBuffer output = { (void*)output_data, sd->buf_size, 0 };
82
- size_t const ret = ZSTD_decompressStream(sd->ctx, &output, &input);
107
+ size_t const ret = zstd_stream_decompress(sd->dctx, &output, &input, false);
83
108
  if (ZSTD_isError(ret)) {
84
- rb_raise(rb_eRuntimeError, "compress error error code: %s", ZSTD_getErrorName(ret));
109
+ rb_raise(rb_eRuntimeError, "decompress error error code: %s", ZSTD_getErrorName(ret));
85
110
  }
86
111
  rb_str_cat(result, output.dst, output.pos);
87
112
  }
88
113
  return result;
89
114
  }
90
115
 
91
- static VALUE
92
- rb_streaming_decompress_addstr(VALUE obj, VALUE src)
93
- {
94
- StringValue(src);
95
- const char* input_data = RSTRING_PTR(src);
96
- size_t input_size = RSTRING_LEN(src);
97
- ZSTD_inBuffer input = { input_data, input_size, 0 };
98
-
99
- struct streaming_decompress_t* sd;
100
- TypedData_Get_Struct(obj, struct streaming_decompress_t, &streaming_decompress_type, sd);
101
- const char* output_data = RSTRING_PTR(sd->buf);
102
-
103
- while (input.pos < input.size) {
104
- ZSTD_outBuffer output = { (void*)output_data, sd->buf_size, 0 };
105
- size_t const result = ZSTD_decompressStream(sd->ctx, &output, &input);
106
- if (ZSTD_isError(result)) {
107
- rb_raise(rb_eRuntimeError, "compress error error code: %s", ZSTD_getErrorName(result));
108
- }
109
- }
110
- return obj;
111
- }
112
-
113
116
  extern VALUE rb_mZstd, cStreamingDecompress;
114
117
  void
115
118
  zstd_ruby_streaming_decompress_init(void)
116
119
  {
117
120
  VALUE cStreamingDecompress = rb_define_class_under(rb_mZstd, "StreamingDecompress", rb_cObject);
118
121
  rb_define_alloc_func(cStreamingDecompress, rb_streaming_decompress_allocate);
119
- rb_define_method(cStreamingDecompress, "initialize", rb_streaming_decompress_initialize, 0);
122
+ rb_define_method(cStreamingDecompress, "initialize", rb_streaming_decompress_initialize, -1);
120
123
  rb_define_method(cStreamingDecompress, "decompress", rb_streaming_decompress_decompress, 1);
121
- rb_define_method(cStreamingDecompress, "<<", rb_streaming_decompress_addstr, 1);
122
124
  }
123
-
@@ -12,28 +12,37 @@ static VALUE rb_compress(int argc, VALUE *argv, VALUE self)
12
12
  {
13
13
  VALUE input_value;
14
14
  VALUE compression_level_value;
15
- rb_scan_args(argc, argv, "11", &input_value, &compression_level_value);
16
- int compression_level = convert_compression_level(compression_level_value);
15
+ VALUE kwargs;
16
+ rb_scan_args(argc, argv, "11:", &input_value, &compression_level_value, &kwargs);
17
+
18
+ ZSTD_CCtx* const ctx = ZSTD_createCCtx();
19
+ if (ctx == NULL) {
20
+ rb_raise(rb_eRuntimeError, "%s", "ZSTD_createCCtx error");
21
+ }
22
+
23
+ set_compress_params(ctx, compression_level_value, kwargs);
17
24
 
18
25
  StringValue(input_value);
19
26
  char* input_data = RSTRING_PTR(input_value);
20
27
  size_t input_size = RSTRING_LEN(input_value);
21
- size_t max_compressed_size = ZSTD_compressBound(input_size);
22
28
 
29
+ size_t max_compressed_size = ZSTD_compressBound(input_size);
23
30
  VALUE output = rb_str_new(NULL, max_compressed_size);
24
31
  char* output_data = RSTRING_PTR(output);
25
- size_t compressed_size = ZSTD_compress((void*)output_data, max_compressed_size,
26
- (void*)input_data, input_size, compression_level);
27
- if (ZSTD_isError(compressed_size)) {
28
- rb_raise(rb_eRuntimeError, "%s: %s", "compress failed", ZSTD_getErrorName(compressed_size));
32
+
33
+ size_t const ret = zstd_compress(ctx, output_data, max_compressed_size, input_data, input_size, false);
34
+ if (ZSTD_isError(ret)) {
35
+ rb_raise(rb_eRuntimeError, "compress error error code: %s", ZSTD_getErrorName(ret));
29
36
  }
37
+ rb_str_resize(output, ret);
30
38
 
31
- rb_str_resize(output, compressed_size);
39
+ ZSTD_freeCCtx(ctx);
32
40
  return output;
33
41
  }
34
42
 
35
43
  static VALUE rb_compress_using_dict(int argc, VALUE *argv, VALUE self)
36
44
  {
45
+ rb_warn("Zstd.compress_using_dict is deprecated; use Zstd.compress with `dict:` instead.");
37
46
  VALUE input_value;
38
47
  VALUE dict;
39
48
  VALUE compression_level_value;
@@ -76,70 +85,66 @@ static VALUE rb_compress_using_dict(int argc, VALUE *argv, VALUE self)
76
85
  }
77
86
 
78
87
 
79
- static VALUE decompress_buffered(const char* input_data, size_t input_size)
88
+ static VALUE decompress_buffered(ZSTD_DCtx* dctx, const char* input_data, size_t input_size)
80
89
  {
81
- const size_t outputBufferSize = 4096;
82
-
83
- ZSTD_DStream* const dstream = ZSTD_createDStream();
84
- if (dstream == NULL) {
85
- rb_raise(rb_eRuntimeError, "%s", "ZSTD_createDStream failed");
86
- }
87
-
88
- size_t initResult = ZSTD_initDStream(dstream);
89
- if (ZSTD_isError(initResult)) {
90
- ZSTD_freeDStream(dstream);
91
- rb_raise(rb_eRuntimeError, "%s: %s", "ZSTD_initDStream failed", ZSTD_getErrorName(initResult));
92
- }
93
-
94
- VALUE output_string = rb_str_new(NULL, 0);
95
- ZSTD_outBuffer output = { NULL, 0, 0 };
96
-
97
90
  ZSTD_inBuffer input = { input_data, input_size, 0 };
91
+ VALUE result = rb_str_new(0, 0);
92
+
98
93
  while (input.pos < input.size) {
99
- output.size += outputBufferSize;
100
- rb_str_resize(output_string, output.size);
94
+ ZSTD_outBuffer output = { NULL, 0, 0 };
95
+ output.size += ZSTD_DStreamOutSize();
96
+ VALUE output_string = rb_str_new(NULL, output.size);
101
97
  output.dst = RSTRING_PTR(output_string);
102
98
 
103
- size_t readHint = ZSTD_decompressStream(dstream, &output, &input);
104
- if (ZSTD_isError(readHint)) {
105
- ZSTD_freeDStream(dstream);
106
- rb_raise(rb_eRuntimeError, "%s: %s", "ZSTD_decompressStream failed", ZSTD_getErrorName(readHint));
99
+ size_t ret = zstd_stream_decompress(dctx, &output, &input, false);
100
+ if (ZSTD_isError(ret)) {
101
+ ZSTD_freeDCtx(dctx);
102
+ rb_raise(rb_eRuntimeError, "%s: %s", "ZSTD_decompressStream failed", ZSTD_getErrorName(ret));
107
103
  }
104
+ rb_str_cat(result, output.dst, output.pos);
108
105
  }
109
-
110
- ZSTD_freeDStream(dstream);
111
- rb_str_resize(output_string, output.pos);
112
- return output_string;
106
+ ZSTD_freeDCtx(dctx);
107
+ return result;
113
108
  }
114
109
 
115
- static VALUE rb_decompress(VALUE self, VALUE input_value)
110
+ static VALUE rb_decompress(int argc, VALUE *argv, VALUE self)
116
111
  {
112
+ VALUE input_value;
113
+ VALUE kwargs;
114
+ rb_scan_args(argc, argv, "10:", &input_value, &kwargs);
117
115
  StringValue(input_value);
118
116
  char* input_data = RSTRING_PTR(input_value);
119
117
  size_t input_size = RSTRING_LEN(input_value);
118
+ ZSTD_DCtx* const dctx = ZSTD_createDCtx();
119
+ if (dctx == NULL) {
120
+ rb_raise(rb_eRuntimeError, "%s", "ZSTD_createDCtx failed");
121
+ }
122
+ set_decompress_params(dctx, kwargs);
120
123
 
121
124
  unsigned long long const uncompressed_size = ZSTD_getFrameContentSize(input_data, input_size);
122
125
  if (uncompressed_size == ZSTD_CONTENTSIZE_ERROR) {
123
126
  rb_raise(rb_eRuntimeError, "%s: %s", "not compressed by zstd", ZSTD_getErrorName(uncompressed_size));
124
127
  }
128
+ // ZSTD_decompressStream may be called multiple times when ZSTD_CONTENTSIZE_UNKNOWN, causing slowness.
129
+ // Therefore, we will not standardize on ZSTD_decompressStream
125
130
  if (uncompressed_size == ZSTD_CONTENTSIZE_UNKNOWN) {
126
- return decompress_buffered(input_data, input_size);
131
+ return decompress_buffered(dctx, input_data, input_size);
127
132
  }
128
133
 
129
134
  VALUE output = rb_str_new(NULL, uncompressed_size);
130
135
  char* output_data = RSTRING_PTR(output);
131
- size_t const decompress_size = ZSTD_decompress((void*)output_data, uncompressed_size,
132
- (void*)input_data, input_size);
133
136
 
137
+ size_t const decompress_size = zstd_decompress(dctx, output_data, uncompressed_size, input_data, input_size, false);
134
138
  if (ZSTD_isError(decompress_size)) {
135
139
  rb_raise(rb_eRuntimeError, "%s: %s", "decompress error", ZSTD_getErrorName(decompress_size));
136
140
  }
137
-
141
+ ZSTD_freeDCtx(dctx);
138
142
  return output;
139
143
  }
140
144
 
141
145
  static VALUE rb_decompress_using_dict(int argc, VALUE *argv, VALUE self)
142
146
  {
147
+ rb_warn("Zstd.decompress_using_dict is deprecated; use Zstd.decompress with `dict:` instead.");
143
148
  VALUE input_value;
144
149
  VALUE dict;
145
150
  rb_scan_args(argc, argv, "20", &input_value, &dict);
@@ -147,15 +152,6 @@ static VALUE rb_decompress_using_dict(int argc, VALUE *argv, VALUE self)
147
152
  StringValue(input_value);
148
153
  char* input_data = RSTRING_PTR(input_value);
149
154
  size_t input_size = RSTRING_LEN(input_value);
150
- unsigned long long const uncompressed_size = ZSTD_getFrameContentSize(input_data, input_size);
151
- if (uncompressed_size == ZSTD_CONTENTSIZE_ERROR) {
152
- rb_raise(rb_eRuntimeError, "%s: %s", "not compressed by zstd", ZSTD_getErrorName(uncompressed_size));
153
- }
154
- if (uncompressed_size == ZSTD_CONTENTSIZE_UNKNOWN) {
155
- return decompress_buffered(input_data, input_size);
156
- }
157
- VALUE output = rb_str_new(NULL, uncompressed_size);
158
- char* output_data = RSTRING_PTR(output);
159
155
 
160
156
  char* dict_buffer = RSTRING_PTR(dict);
161
157
  size_t dict_size = RSTRING_LEN(dict);
@@ -163,12 +159,11 @@ static VALUE rb_decompress_using_dict(int argc, VALUE *argv, VALUE self)
163
159
  if (ddict == NULL) {
164
160
  rb_raise(rb_eRuntimeError, "%s", "ZSTD_createDDict failed");
165
161
  }
166
-
167
162
  unsigned const expected_dict_id = ZSTD_getDictID_fromDDict(ddict);
168
163
  unsigned const actual_dict_id = ZSTD_getDictID_fromFrame(input_data, input_size);
169
164
  if (expected_dict_id != actual_dict_id) {
170
165
  ZSTD_freeDDict(ddict);
171
- rb_raise(rb_eRuntimeError, "%s: %s", "DictID mismatch", ZSTD_getErrorName(uncompressed_size));
166
+ rb_raise(rb_eRuntimeError, "DictID mismatch");
172
167
  }
173
168
 
174
169
  ZSTD_DCtx* const ctx = ZSTD_createDCtx();
@@ -176,6 +171,19 @@ static VALUE rb_decompress_using_dict(int argc, VALUE *argv, VALUE self)
176
171
  ZSTD_freeDDict(ddict);
177
172
  rb_raise(rb_eRuntimeError, "%s", "ZSTD_createDCtx failed");
178
173
  }
174
+
175
+ unsigned long long const uncompressed_size = ZSTD_getFrameContentSize(input_data, input_size);
176
+ if (uncompressed_size == ZSTD_CONTENTSIZE_ERROR) {
177
+ ZSTD_freeDDict(ddict);
178
+ ZSTD_freeDCtx(ctx);
179
+ rb_raise(rb_eRuntimeError, "%s: %s", "not compressed by zstd", ZSTD_getErrorName(uncompressed_size));
180
+ }
181
+ if (uncompressed_size == ZSTD_CONTENTSIZE_UNKNOWN) {
182
+ return decompress_buffered(ctx, input_data, input_size);
183
+ }
184
+
185
+ VALUE output = rb_str_new(NULL, uncompressed_size);
186
+ char* output_data = RSTRING_PTR(output);
179
187
  size_t const decompress_size = ZSTD_decompress_usingDDict(ctx, output_data, uncompressed_size, input_data, input_size, ddict);
180
188
  if (ZSTD_isError(decompress_size)) {
181
189
  ZSTD_freeDDict(ddict);
@@ -193,6 +201,6 @@ zstd_ruby_init(void)
193
201
  rb_define_module_function(rb_mZstd, "zstd_version", zstdVersion, 0);
194
202
  rb_define_module_function(rb_mZstd, "compress", rb_compress, -1);
195
203
  rb_define_module_function(rb_mZstd, "compress_using_dict", rb_compress_using_dict, -1);
196
- rb_define_module_function(rb_mZstd, "decompress", rb_decompress, 1);
204
+ rb_define_module_function(rb_mZstd, "decompress", rb_decompress, -1);
197
205
  rb_define_module_function(rb_mZstd, "decompress_using_dict", rb_decompress_using_dict, -1);
198
206
  }
@@ -0,0 +1,22 @@
1
+ module Zstd
2
+ # @todo Exprimental
3
+ class StreamReader
4
+ def initialize(io)
5
+ @io = io
6
+ @stream = Zstd::StreamingDecompress.new
7
+ end
8
+
9
+ def read(length)
10
+ if @io.eof?
11
+ raise StandardError, "EOF"
12
+ end
13
+ data = @io.read(length)
14
+ @stream.decompress(data)
15
+ end
16
+
17
+ def close
18
+ @io.write(@stream.finish)
19
+ @io.close
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,23 @@
1
+ module Zstd
2
+ # @todo Exprimental
3
+ class StreamWriter
4
+ def initialize(io, level: nil)
5
+ @io = io
6
+ @stream = Zstd::StreamingCompress.new(level)
7
+ end
8
+
9
+ def write(*data)
10
+ @stream.write(*data)
11
+ @io.write(@stream.flush)
12
+ end
13
+
14
+ def finish
15
+ @io.write(@stream.finish)
16
+ end
17
+
18
+ def close
19
+ @io.write(@stream.finish)
20
+ @io.close
21
+ end
22
+ end
23
+ end
@@ -1,3 +1,3 @@
1
1
  module Zstd
2
- VERSION = "1.5.5.0"
2
+ VERSION = "1.5.6.6"
3
3
  end
data/lib/zstd-ruby.rb CHANGED
@@ -1,5 +1,7 @@
1
1
  require "zstd-ruby/version"
2
2
  require "zstd-ruby/zstdruby"
3
+ require "zstd-ruby/stream_writer"
4
+ require "zstd-ruby/stream_reader"
3
5
 
4
6
  module Zstd
5
7
  end
data/renovate.json ADDED
@@ -0,0 +1,6 @@
1
+ {
2
+ "$schema": "https://docs.renovatebot.com/renovate-schema.json",
3
+ "extends": [
4
+ "config:recommended"
5
+ ]
6
+ }
data/zstd-ruby.gemspec CHANGED
@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
24
24
  #end
25
25
 
26
26
  spec.files = `git ls-files -z`.split("\x0").reject do |f|
27
- f.match(%r{^(test|spec|features|benchmarks|zstd|.github)/})
27
+ f.match(%r{^(test|spec|features|benchmarks|zstd|.github|examples)/})
28
28
  end
29
29
  spec.bindir = "exe"
30
30
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
@@ -35,4 +35,5 @@ Gem::Specification.new do |spec|
35
35
  spec.add_development_dependency "rake", "~> 13.0"
36
36
  spec.add_development_dependency "rake-compiler", '~> 1'
37
37
  spec.add_development_dependency "rspec", "~> 3.0"
38
+ spec.add_development_dependency "pry"
38
39
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: zstd-ruby
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.5.5.0
4
+ version: 1.5.6.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - SpringMT
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2023-04-08 00:00:00.000000000 Z
11
+ date: 2024-05-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -66,6 +66,20 @@ dependencies:
66
66
  - - "~>"
67
67
  - !ruby/object:Gem::Version
68
68
  version: '3.0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: pry
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
69
83
  description: Ruby binding for zstd(Zstandard - Fast real-time compression algorithm).
70
84
  See https://github.com/facebook/zstd
71
85
  email:
@@ -159,11 +173,13 @@ files:
159
173
  - ext/zstdruby/main.c
160
174
  - ext/zstdruby/skippable_frame.c
161
175
  - ext/zstdruby/streaming_compress.c
162
- - ext/zstdruby/streaming_compress.h
163
176
  - ext/zstdruby/streaming_decompress.c
164
177
  - ext/zstdruby/zstdruby.c
165
178
  - lib/zstd-ruby.rb
179
+ - lib/zstd-ruby/stream_reader.rb
180
+ - lib/zstd-ruby/stream_writer.rb
166
181
  - lib/zstd-ruby/version.rb
182
+ - renovate.json
167
183
  - zstd-ruby.gemspec
168
184
  homepage: https://github.com/SpringMT/zstd-ruby
169
185
  licenses:
@@ -184,7 +200,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
184
200
  - !ruby/object:Gem::Version
185
201
  version: '0'
186
202
  requirements: []
187
- rubygems_version: 3.3.26
203
+ rubygems_version: 3.5.3
188
204
  signing_key:
189
205
  specification_version: 4
190
206
  summary: Ruby binding for zstd(Zstandard - Fast real-time compression algorithm)
@@ -1,5 +0,0 @@
1
- #if !defined(STREAMING_COMPRESS_H)
2
- #define STREAMING_COMPRESS_H
3
-
4
-
5
- #endif // STREAMING_COMPRESS_H