ruby-zstds 1.0.6 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,17 +4,20 @@
4
4
  #if !defined(ZSTDS_EXT_STREAM_COMPRESSOR_H)
5
5
  #define ZSTDS_EXT_STREAM_COMPRESSOR_H
6
6
 
7
+ #include <stdbool.h>
7
8
  #include <zstd.h>
8
9
 
9
10
  #include "ruby.h"
10
11
  #include "zstds_ext/common.h"
11
12
 
12
- typedef struct {
13
+ typedef struct
14
+ {
13
15
  ZSTD_CCtx* ctx;
14
16
  zstds_ext_byte_t* destination_buffer;
15
17
  size_t destination_buffer_length;
16
18
  zstds_ext_byte_t* remaining_destination_buffer;
17
19
  size_t remaining_destination_buffer_length;
20
+ bool gvl;
18
21
  } zstds_ext_compressor_t;
19
22
 
20
23
  VALUE zstds_ext_allocate_compressor(VALUE klass);
@@ -7,8 +7,11 @@
7
7
 
8
8
  #include "ruby.h"
9
9
  #include "zstds_ext/error.h"
10
+ #include "zstds_ext/gvl.h"
10
11
  #include "zstds_ext/option.h"
11
12
 
13
+ // -- initialization --
14
+
12
15
  static void free_decompressor(zstds_ext_decompressor_t* decompressor_ptr)
13
16
  {
14
17
  ZSTD_DCtx* ctx = decompressor_ptr->ctx;
@@ -27,7 +30,6 @@ static void free_decompressor(zstds_ext_decompressor_t* decompressor_ptr)
27
30
  VALUE zstds_ext_allocate_decompressor(VALUE klass)
28
31
  {
29
32
  zstds_ext_decompressor_t* decompressor_ptr;
30
-
31
33
  VALUE self = Data_Make_Struct(klass, zstds_ext_decompressor_t, NULL, free_decompressor, decompressor_ptr);
32
34
 
33
35
  decompressor_ptr->ctx = NULL;
@@ -47,8 +49,9 @@ VALUE zstds_ext_initialize_decompressor(VALUE self, VALUE options)
47
49
  {
48
50
  GET_DECOMPRESSOR(self);
49
51
  Check_Type(options, T_HASH);
52
+ ZSTDS_EXT_GET_SIZE_OPTION(options, destination_buffer_length);
53
+ ZSTDS_EXT_GET_BOOL_OPTION(options, gvl);
50
54
  ZSTDS_EXT_GET_DECOMPRESSOR_OPTIONS(options);
51
- ZSTDS_EXT_GET_BUFFER_LENGTH_OPTION(options, destination_buffer_length);
52
55
 
53
56
  ZSTD_DCtx* ctx = ZSTD_createDCtx();
54
57
  if (ctx == NULL) {
@@ -76,40 +79,56 @@ VALUE zstds_ext_initialize_decompressor(VALUE self, VALUE options)
76
79
  decompressor_ptr->destination_buffer_length = destination_buffer_length;
77
80
  decompressor_ptr->remaining_destination_buffer = destination_buffer;
78
81
  decompressor_ptr->remaining_destination_buffer_length = destination_buffer_length;
82
+ decompressor_ptr->gvl = gvl;
79
83
 
80
84
  return Qnil;
81
85
  }
82
86
 
87
+ // -- decompress --
88
+
83
89
  #define DO_NOT_USE_AFTER_CLOSE(decompressor_ptr) \
84
90
  if (decompressor_ptr->ctx == NULL || decompressor_ptr->destination_buffer == NULL) { \
85
91
  zstds_ext_raise_error(ZSTDS_EXT_ERROR_USED_AFTER_CLOSE); \
86
92
  }
87
93
 
88
- #define GET_SOURCE_DATA(source_value) \
89
- Check_Type(source_value, T_STRING); \
90
- \
91
- const char* source = RSTRING_PTR(source_value); \
92
- size_t source_length = RSTRING_LEN(source_value);
94
+ typedef struct
95
+ {
96
+ zstds_ext_decompressor_t* decompressor_ptr;
97
+ ZSTD_inBuffer* in_buffer_ptr;
98
+ ZSTD_outBuffer* out_buffer_ptr;
99
+ zstds_result_t result;
100
+ } decompress_args_t;
101
+
102
+ static inline void* decompress_wrapper(void* data)
103
+ {
104
+ decompress_args_t* args = data;
105
+
106
+ args->result = ZSTD_decompressStream(args->decompressor_ptr->ctx, args->out_buffer_ptr, args->in_buffer_ptr);
107
+
108
+ return NULL;
109
+ }
93
110
 
94
111
  VALUE zstds_ext_decompress(VALUE self, VALUE source_value)
95
112
  {
96
113
  GET_DECOMPRESSOR(self);
97
114
  DO_NOT_USE_AFTER_CLOSE(decompressor_ptr);
98
- GET_SOURCE_DATA(source_value);
115
+ Check_Type(source_value, T_STRING);
116
+
117
+ const char* source = RSTRING_PTR(source_value);
118
+ size_t source_length = RSTRING_LEN(source_value);
99
119
 
100
- ZSTD_inBuffer in_buffer;
101
- in_buffer.src = source;
102
- in_buffer.size = source_length;
103
- in_buffer.pos = 0;
120
+ ZSTD_inBuffer in_buffer = {.src = source, .size = source_length, .pos = 0};
121
+ ZSTD_outBuffer out_buffer = {
122
+ .dst = decompressor_ptr->remaining_destination_buffer,
123
+ .size = decompressor_ptr->remaining_destination_buffer_length,
124
+ .pos = 0};
104
125
 
105
- ZSTD_outBuffer out_buffer;
106
- out_buffer.dst = decompressor_ptr->remaining_destination_buffer;
107
- out_buffer.size = decompressor_ptr->remaining_destination_buffer_length;
108
- out_buffer.pos = 0;
126
+ decompress_args_t args = {
127
+ .decompressor_ptr = decompressor_ptr, .in_buffer_ptr = &in_buffer, .out_buffer_ptr = &out_buffer};
109
128
 
110
- zstds_result_t result = ZSTD_decompressStream(decompressor_ptr->ctx, &out_buffer, &in_buffer);
111
- if (ZSTD_isError(result)) {
112
- zstds_ext_raise_error(zstds_ext_get_error(ZSTD_getErrorCode(result)));
129
+ ZSTDS_EXT_GVL_WRAP(decompressor_ptr->gvl, decompress_wrapper, &args);
130
+ if (ZSTD_isError(args.result)) {
131
+ zstds_ext_raise_error(zstds_ext_get_error(ZSTD_getErrorCode(args.result)));
113
132
  }
114
133
 
115
134
  decompressor_ptr->remaining_destination_buffer += out_buffer.pos;
@@ -121,6 +140,8 @@ VALUE zstds_ext_decompress(VALUE self, VALUE source_value)
121
140
  return rb_ary_new_from_args(2, bytes_read, needs_more_destination);
122
141
  }
123
142
 
143
+ // -- other --
144
+
124
145
  VALUE zstds_ext_decompressor_read_result(VALUE self)
125
146
  {
126
147
  GET_DECOMPRESSOR(self);
@@ -130,10 +151,9 @@ VALUE zstds_ext_decompressor_read_result(VALUE self)
130
151
  size_t destination_buffer_length = decompressor_ptr->destination_buffer_length;
131
152
  size_t remaining_destination_buffer_length = decompressor_ptr->remaining_destination_buffer_length;
132
153
 
133
- const char* result = (const char*)destination_buffer;
154
+ const char* result = (const char*) destination_buffer;
134
155
  size_t result_length = destination_buffer_length - remaining_destination_buffer_length;
135
-
136
- VALUE result_value = rb_str_new(result, result_length);
156
+ VALUE result_value = rb_str_new(result, result_length);
137
157
 
138
158
  decompressor_ptr->remaining_destination_buffer = destination_buffer;
139
159
  decompressor_ptr->remaining_destination_buffer_length = destination_buffer_length;
@@ -141,6 +161,8 @@ VALUE zstds_ext_decompressor_read_result(VALUE self)
141
161
  return result_value;
142
162
  }
143
163
 
164
+ // -- cleanup --
165
+
144
166
  VALUE zstds_ext_decompressor_close(VALUE self)
145
167
  {
146
168
  GET_DECOMPRESSOR(self);
@@ -166,6 +188,8 @@ VALUE zstds_ext_decompressor_close(VALUE self)
166
188
  return Qnil;
167
189
  }
168
190
 
191
+ // -- exports --
192
+
169
193
  void zstds_ext_decompressor_exports(VALUE root_module)
170
194
  {
171
195
  VALUE module = rb_define_module_under(root_module, "Stream");
@@ -4,17 +4,20 @@
4
4
  #if !defined(ZSTDS_EXT_STREAM_DECOMPRESSOR_H)
5
5
  #define ZSTDS_EXT_STREAM_DECOMPRESSOR_H
6
6
 
7
+ #include <stdbool.h>
7
8
  #include <zstd.h>
8
9
 
9
10
  #include "ruby.h"
10
11
  #include "zstds_ext/common.h"
11
12
 
12
- typedef struct {
13
+ typedef struct
14
+ {
13
15
  ZSTD_DCtx* ctx;
14
16
  zstds_ext_byte_t* destination_buffer;
15
17
  size_t destination_buffer_length;
16
18
  zstds_ext_byte_t* remaining_destination_buffer;
17
19
  size_t remaining_destination_buffer_length;
20
+ bool gvl;
18
21
  } zstds_ext_decompressor_t;
19
22
 
20
23
  VALUE zstds_ext_allocate_decompressor(VALUE klass);
@@ -8,14 +8,17 @@
8
8
  #include "ruby.h"
9
9
  #include "zstds_ext/buffer.h"
10
10
  #include "zstds_ext/error.h"
11
+ #include "zstds_ext/gvl.h"
11
12
  #include "zstds_ext/macro.h"
12
13
  #include "zstds_ext/option.h"
13
14
 
14
15
  // -- buffer --
15
16
 
16
17
  static inline zstds_ext_result_t increase_destination_buffer(
17
- VALUE destination_value, size_t destination_length,
18
- size_t* remaining_destination_buffer_length_ptr, size_t destination_buffer_length)
18
+ VALUE destination_value,
19
+ size_t destination_length,
20
+ size_t* remaining_destination_buffer_length_ptr,
21
+ size_t destination_buffer_length)
19
22
  {
20
23
  if (*remaining_destination_buffer_length_ptr == destination_buffer_length) {
21
24
  // We want to write more data at once, than buffer has.
@@ -34,50 +37,58 @@ static inline zstds_ext_result_t increase_destination_buffer(
34
37
  return 0;
35
38
  }
36
39
 
37
- // -- utils --
40
+ // -- compress --
38
41
 
39
- #define GET_SOURCE_DATA(source_value) \
40
- Check_Type(source_value, T_STRING); \
41
- \
42
- const char* source = RSTRING_PTR(source_value); \
43
- size_t source_length = RSTRING_LEN(source_value);
42
+ typedef struct
43
+ {
44
+ ZSTD_CCtx* ctx;
45
+ ZSTD_inBuffer* in_buffer_ptr;
46
+ ZSTD_outBuffer* out_buffer_ptr;
47
+ zstds_result_t result;
48
+ } compress_args_t;
44
49
 
45
- // -- compress --
50
+ static inline void* compress_wrapper(void* data)
51
+ {
52
+ compress_args_t* args = data;
53
+
54
+ args->result = ZSTD_compressStream2(args->ctx, args->out_buffer_ptr, args->in_buffer_ptr, ZSTD_e_end);
55
+
56
+ return NULL;
57
+ }
46
58
 
47
59
  static inline zstds_ext_result_t compress(
48
60
  ZSTD_CCtx* ctx,
49
- const char* source, size_t source_length,
50
- VALUE destination_value, size_t destination_buffer_length)
61
+ const char* source,
62
+ size_t source_length,
63
+ VALUE destination_value,
64
+ size_t destination_buffer_length,
65
+ bool gvl)
51
66
  {
52
- zstds_result_t result;
53
67
  zstds_ext_result_t ext_result;
54
-
55
- ZSTD_inBuffer in_buffer;
56
- in_buffer.src = source;
57
- in_buffer.size = source_length;
58
- in_buffer.pos = 0;
59
-
60
- ZSTD_outBuffer out_buffer;
61
- size_t destination_length = 0;
62
- size_t remaining_destination_buffer_length = destination_buffer_length;
68
+ size_t destination_length = 0;
69
+ size_t remaining_destination_buffer_length = destination_buffer_length;
70
+ ZSTD_inBuffer in_buffer = {.src = source, .size = source_length, .pos = 0};
71
+ compress_args_t args = {.ctx = ctx, .in_buffer_ptr = &in_buffer};
63
72
 
64
73
  while (true) {
65
- out_buffer.dst = (zstds_ext_byte_t*)RSTRING_PTR(destination_value) + destination_length;
66
- out_buffer.size = remaining_destination_buffer_length;
67
- out_buffer.pos = 0;
74
+ ZSTD_outBuffer out_buffer = {
75
+ .dst = (zstds_ext_byte_t*) RSTRING_PTR(destination_value) + destination_length,
76
+ .size = remaining_destination_buffer_length,
77
+ .pos = 0};
68
78
 
69
- result = ZSTD_compressStream2(ctx, &out_buffer, &in_buffer, ZSTD_e_end);
70
- if (ZSTD_isError(result)) {
71
- return zstds_ext_get_error(ZSTD_getErrorCode(result));
79
+ args.out_buffer_ptr = &out_buffer;
80
+
81
+ ZSTDS_EXT_GVL_WRAP(gvl, compress_wrapper, &args);
82
+ if (ZSTD_isError(args.result)) {
83
+ return zstds_ext_get_error(ZSTD_getErrorCode(args.result));
72
84
  }
73
85
 
74
86
  destination_length += out_buffer.pos;
75
87
  remaining_destination_buffer_length -= out_buffer.pos;
76
88
 
77
- if (result != 0) {
89
+ if (args.result != 0) {
78
90
  ext_result = increase_destination_buffer(
79
- destination_value, destination_length,
80
- &remaining_destination_buffer_length, destination_buffer_length);
91
+ destination_value, destination_length, &remaining_destination_buffer_length, destination_buffer_length);
81
92
 
82
93
  if (ext_result != 0) {
83
94
  return ext_result;
@@ -101,10 +112,11 @@ static inline zstds_ext_result_t compress(
101
112
 
102
113
  VALUE zstds_ext_compress_string(VALUE ZSTDS_EXT_UNUSED(self), VALUE source_value, VALUE options)
103
114
  {
104
- GET_SOURCE_DATA(source_value);
115
+ Check_Type(source_value, T_STRING);
105
116
  Check_Type(options, T_HASH);
117
+ ZSTDS_EXT_GET_SIZE_OPTION(options, destination_buffer_length);
118
+ ZSTDS_EXT_GET_BOOL_OPTION(options, gvl);
106
119
  ZSTDS_EXT_GET_COMPRESSOR_OPTIONS(options);
107
- ZSTDS_EXT_GET_BUFFER_LENGTH_OPTION(options, destination_buffer_length);
108
120
 
109
121
  ZSTD_CCtx* ctx = ZSTD_createCCtx();
110
122
  if (ctx == NULL) {
@@ -129,10 +141,10 @@ VALUE zstds_ext_compress_string(VALUE ZSTDS_EXT_UNUSED(self), VALUE source_value
129
141
  zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
130
142
  }
131
143
 
132
- ext_result = compress(
133
- ctx,
134
- source, source_length,
135
- destination_value, destination_buffer_length);
144
+ const char* source = RSTRING_PTR(source_value);
145
+ size_t source_length = RSTRING_LEN(source_value);
146
+
147
+ ext_result = compress(ctx, source, source_length, destination_value, destination_buffer_length, gvl);
136
148
 
137
149
  ZSTD_freeCCtx(ctx);
138
150
 
@@ -145,31 +157,48 @@ VALUE zstds_ext_compress_string(VALUE ZSTDS_EXT_UNUSED(self), VALUE source_value
145
157
 
146
158
  // -- decompress --
147
159
 
160
+ typedef struct
161
+ {
162
+ ZSTD_DCtx* ctx;
163
+ ZSTD_inBuffer* in_buffer_ptr;
164
+ ZSTD_outBuffer* out_buffer_ptr;
165
+ zstds_result_t result;
166
+ } decompress_args_t;
167
+
168
+ static inline void* decompress_wrapper(void* data)
169
+ {
170
+ decompress_args_t* args = data;
171
+
172
+ args->result = ZSTD_decompressStream(args->ctx, args->out_buffer_ptr, args->in_buffer_ptr);
173
+
174
+ return NULL;
175
+ }
176
+
148
177
  static inline zstds_ext_result_t decompress(
149
178
  ZSTD_DCtx* ctx,
150
- const char* source, size_t source_length,
151
- VALUE destination_value, size_t destination_buffer_length)
179
+ const char* source,
180
+ size_t source_length,
181
+ VALUE destination_value,
182
+ size_t destination_buffer_length,
183
+ bool gvl)
152
184
  {
153
- zstds_result_t result;
154
185
  zstds_ext_result_t ext_result;
155
-
156
- ZSTD_inBuffer in_buffer;
157
- in_buffer.src = source;
158
- in_buffer.size = source_length;
159
- in_buffer.pos = 0;
160
-
161
- ZSTD_outBuffer out_buffer;
162
- size_t destination_length = 0;
163
- size_t remaining_destination_buffer_length = destination_buffer_length;
186
+ size_t destination_length = 0;
187
+ size_t remaining_destination_buffer_length = destination_buffer_length;
188
+ ZSTD_inBuffer in_buffer = {.src = source, .size = source_length, .pos = 0};
189
+ decompress_args_t args = {.ctx = ctx, .in_buffer_ptr = &in_buffer};
164
190
 
165
191
  while (true) {
166
- out_buffer.dst = (zstds_ext_byte_t*)RSTRING_PTR(destination_value) + destination_length;
167
- out_buffer.size = remaining_destination_buffer_length;
168
- out_buffer.pos = 0;
192
+ ZSTD_outBuffer out_buffer = {
193
+ .dst = (zstds_ext_byte_t*) RSTRING_PTR(destination_value) + destination_length,
194
+ .size = remaining_destination_buffer_length,
195
+ .pos = 0};
196
+
197
+ args.out_buffer_ptr = &out_buffer;
169
198
 
170
- result = ZSTD_decompressStream(ctx, &out_buffer, &in_buffer);
171
- if (ZSTD_isError(result)) {
172
- return zstds_ext_get_error(ZSTD_getErrorCode(result));
199
+ ZSTDS_EXT_GVL_WRAP(gvl, decompress_wrapper, &args);
200
+ if (ZSTD_isError(args.result)) {
201
+ return zstds_ext_get_error(ZSTD_getErrorCode(args.result));
173
202
  }
174
203
 
175
204
  destination_length += out_buffer.pos;
@@ -177,8 +206,7 @@ static inline zstds_ext_result_t decompress(
177
206
 
178
207
  if (remaining_destination_buffer_length == 0) {
179
208
  ext_result = increase_destination_buffer(
180
- destination_value, destination_length,
181
- &remaining_destination_buffer_length, destination_buffer_length);
209
+ destination_value, destination_length, &remaining_destination_buffer_length, destination_buffer_length);
182
210
 
183
211
  if (ext_result != 0) {
184
212
  return ext_result;
@@ -202,10 +230,11 @@ static inline zstds_ext_result_t decompress(
202
230
 
203
231
  VALUE zstds_ext_decompress_string(VALUE ZSTDS_EXT_UNUSED(self), VALUE source_value, VALUE options)
204
232
  {
205
- GET_SOURCE_DATA(source_value);
233
+ Check_Type(source_value, T_STRING);
206
234
  Check_Type(options, T_HASH);
235
+ ZSTDS_EXT_GET_SIZE_OPTION(options, destination_buffer_length);
236
+ ZSTDS_EXT_GET_BOOL_OPTION(options, gvl);
207
237
  ZSTDS_EXT_GET_DECOMPRESSOR_OPTIONS(options);
208
- ZSTDS_EXT_GET_BUFFER_LENGTH_OPTION(options, destination_buffer_length);
209
238
 
210
239
  ZSTD_DCtx* ctx = ZSTD_createDCtx();
211
240
  if (ctx == NULL) {
@@ -230,10 +259,10 @@ VALUE zstds_ext_decompress_string(VALUE ZSTDS_EXT_UNUSED(self), VALUE source_val
230
259
  zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
231
260
  }
232
261
 
233
- ext_result = decompress(
234
- ctx,
235
- source, source_length,
236
- destination_value, destination_buffer_length);
262
+ const char* source = RSTRING_PTR(source_value);
263
+ size_t source_length = RSTRING_LEN(source_value);
264
+
265
+ ext_result = decompress(ctx, source, source_length, destination_value, destination_buffer_length, gvl);
237
266
 
238
267
  ZSTD_freeDCtx(ctx);
239
268
 
@@ -244,6 +273,8 @@ VALUE zstds_ext_decompress_string(VALUE ZSTDS_EXT_UNUSED(self), VALUE source_val
244
273
  return destination_value;
245
274
  }
246
275
 
276
+ // -- exports --
277
+
247
278
  void zstds_ext_string_exports(VALUE root_module)
248
279
  {
249
280
  rb_define_module_function(root_module, "_native_compress_string", RUBY_METHOD_FUNC(zstds_ext_compress_string), 2);
@@ -9,6 +9,7 @@ require_relative "validation"
9
9
  module ZSTDS
10
10
  class Dictionary
11
11
  TRAIN_DEFAULTS = {
12
+ :gvl => false,
12
13
  :capacity => 0
13
14
  }
14
15
  .freeze
@@ -38,6 +39,7 @@ module ZSTDS
38
39
 
39
40
  options = TRAIN_DEFAULTS.merge options
40
41
 
42
+ Validation.validate_bool options[:gvl]
41
43
  Validation.validate_not_negative_integer options[:capacity]
42
44
 
43
45
  buffer = train_buffer samples, options
@@ -12,6 +12,7 @@ module ZSTDS
12
12
  DEFAULT_BUFFER_LENGTH = 0
13
13
 
14
14
  COMPRESSOR_DEFAULTS = {
15
+ :gvl => false,
15
16
  :compression_level => nil,
16
17
  :window_log => nil,
17
18
  :hash_log => nil,
@@ -36,6 +37,7 @@ module ZSTDS
36
37
  .freeze
37
38
 
38
39
  DECOMPRESSOR_DEFAULTS = {
40
+ :gvl => false,
39
41
  :window_log_max => nil,
40
42
  :dictionary => nil
41
43
  }
@@ -49,6 +51,8 @@ module ZSTDS
49
51
 
50
52
  buffer_length_names.each { |name| Validation.validate_not_negative_integer options[name] }
51
53
 
54
+ Validation.validate_bool options[:gvl]
55
+
52
56
  compression_level = options[:compression_level]
53
57
  unless compression_level.nil?
54
58
  Validation.validate_integer compression_level
@@ -180,6 +184,8 @@ module ZSTDS
180
184
 
181
185
  buffer_length_names.each { |name| Validation.validate_not_negative_integer options[name] }
182
186
 
187
+ Validation.validate_bool options[:gvl]
188
+
183
189
  window_log_max = options[:window_log_max]
184
190
  unless window_log_max.nil?
185
191
  Validation.validate_not_negative_integer window_log_max