ruby-zstds 1.2.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AUTHORS +2 -0
- data/README.md +27 -5
- data/ext/extconf.rb +7 -1
- data/ext/zstds_ext/dictionary.c +228 -47
- data/ext/zstds_ext/dictionary.h +15 -1
- data/ext/zstds_ext/gvl.h +1 -1
- data/ext/zstds_ext/macro.h +2 -2
- data/ext/zstds_ext/option.h +14 -0
- data/lib/zstds/dictionary.rb +52 -6
- data/lib/zstds/stream/abstract.rb +9 -5
- data/lib/zstds/stream/reader.rb +9 -0
- data/lib/zstds/stream/writer.rb +26 -2
- data/lib/zstds/validation.rb +15 -32
- data/lib/zstds/version.rb +1 -1
- metadata +7 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ec16246ba1102583f6f309e555e4ac4dbbf49135f5adee4e0332b86be7388de4
|
4
|
+
data.tar.gz: 7d8d40b406c9da3ed50c5929f565c51fc8cf46558bee937fdeaa0d942d0fe84f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fd92bd835bf7eb14db13ecc1761a89467e3f38bbcb9d373573fa7c3a8839f4050a478091ada05fec97bb8e2824e9913d23ef9593f3388e317c5b9df456e7e7cd
|
7
|
+
data.tar.gz: 00d8bd37067fd788a7033abd8cf4c6d590b4f672f8986e7da7b80e47ed9ffa4a91886db7442a2a118a6ba45c8f9b2b232665455a5fc85c6cf4c8fcbe119c4e9c
|
data/AUTHORS
CHANGED
data/README.md
CHANGED
@@ -8,7 +8,17 @@ See [zstd library](https://github.com/facebook/zstd).
|
|
8
8
|
|
9
9
|
## Installation
|
10
10
|
|
11
|
-
|
11
|
+
Operating systems: GNU/Linux, FreeBSD, OSX, Windows (MinGW).
|
12
|
+
|
13
|
+
Dependencies: [zstd](https://github.com/facebook/zstd) 1.4.0+ version.
|
14
|
+
|
15
|
+
| Popular OS | Dependencies |
|
16
|
+
|------------|---------------------------|
|
17
|
+
| Ubuntu | `libzstd-dev` |
|
18
|
+
| CentOS | `libzstd-devel` |
|
19
|
+
| ArchLinux | `zstd` |
|
20
|
+
| OSX | `zstd` |
|
21
|
+
| Windows | `mingw-w64-x86_64-zstd` |
|
12
22
|
|
13
23
|
```sh
|
14
24
|
gem install ruby-zstds
|
@@ -23,6 +33,22 @@ gem install pkg/ruby-zstds-*.gem
|
|
23
33
|
|
24
34
|
You can also use [overlay](https://github.com/andrew-aladev/overlay) for gentoo.
|
25
35
|
|
36
|
+
### Installation in macOS on Apple Silicon
|
37
|
+
|
38
|
+
On M1 Macs, Homebrew installs to /opt/homebrew, so you'll need to specify its
|
39
|
+
include and lib paths when building the native extension for zstd.
|
40
|
+
|
41
|
+
```sh
|
42
|
+
brew install zstd
|
43
|
+
gem install ruby-zstds -- --with-opt-include=/opt/homebrew/include --with-opt-lib=/opt/homebrew/lib
|
44
|
+
```
|
45
|
+
|
46
|
+
You can also configure Bundler to use those options when installing:
|
47
|
+
|
48
|
+
```sh
|
49
|
+
bundle config set build.ruby-zstds "--with-opt-include=/opt/homebrew/include --with-opt-lib=/opt/homebrew/lib"
|
50
|
+
```
|
51
|
+
|
26
52
|
## Usage
|
27
53
|
|
28
54
|
There are simple APIs: `String` and `File`. Also you can use generic streaming API: `Stream::Writer` and `Stream::Reader`.
|
@@ -449,10 +475,6 @@ You should lock all shared data between threads.
|
|
449
475
|
For example: you should not use same compressor/decompressor inside multiple threads.
|
450
476
|
Please verify that you are using each processor inside single thread at the same time.
|
451
477
|
|
452
|
-
## Operating systems
|
453
|
-
|
454
|
-
GNU/Linux, FreeBSD, OSX, Windows (MinGW).
|
455
|
-
|
456
478
|
## CI
|
457
479
|
|
458
480
|
Please visit [scripts/test-images](scripts/test-images).
|
data/ext/extconf.rb
CHANGED
@@ -26,7 +26,6 @@ def require_header(name, constants: [], macroses: [], types: [])
|
|
26
26
|
end
|
27
27
|
|
28
28
|
require_header "zdict.h"
|
29
|
-
|
30
29
|
require_header(
|
31
30
|
"zstd.h",
|
32
31
|
:constants => %w[
|
@@ -112,6 +111,13 @@ end
|
|
112
111
|
if find_library "zstd", "ZDICT_getDictHeaderSize"
|
113
112
|
$defs.push "-DHAVE_ZDICT_HEADER_SIZE"
|
114
113
|
end
|
114
|
+
|
115
|
+
zdict_has_params = find_type "ZDICT_params_t", nil, "zdict.h"
|
116
|
+
zdict_has_finalize = find_library "zstd", "ZDICT_finalizeDictionary"
|
117
|
+
|
118
|
+
if zdict_has_params && zdict_has_finalize
|
119
|
+
$defs.push "-DHAVE_ZDICT_FINALIZE"
|
120
|
+
end
|
115
121
|
# rubocop:enable Style/GlobalVars
|
116
122
|
|
117
123
|
require_library(
|
data/ext/zstds_ext/dictionary.c
CHANGED
@@ -11,7 +11,7 @@
|
|
11
11
|
#include "zstds_ext/gvl.h"
|
12
12
|
#include "zstds_ext/option.h"
|
13
13
|
|
14
|
-
// --
|
14
|
+
// -- common --
|
15
15
|
|
16
16
|
typedef struct
|
17
17
|
{
|
@@ -19,43 +19,64 @@ typedef struct
|
|
19
19
|
size_t size;
|
20
20
|
} sample_t;
|
21
21
|
|
22
|
-
|
22
|
+
static inline void check_raw_samples(VALUE raw_samples)
|
23
23
|
{
|
24
|
-
|
25
|
-
size_t length;
|
26
|
-
char* buffer;
|
27
|
-
size_t capacity;
|
28
|
-
zstds_result_t result;
|
29
|
-
zstds_ext_result_t ext_result;
|
30
|
-
} train_args_t;
|
24
|
+
Check_Type(raw_samples, T_ARRAY);
|
31
25
|
|
32
|
-
|
26
|
+
size_t samples_length = RARRAY_LEN(raw_samples);
|
27
|
+
|
28
|
+
for (size_t index = 0; index < samples_length; index++) {
|
29
|
+
Check_Type(rb_ary_entry(raw_samples, index), T_STRING);
|
30
|
+
}
|
31
|
+
}
|
32
|
+
|
33
|
+
static inline sample_t* prepare_samples(VALUE raw_samples, size_t* samples_length_ptr)
|
34
|
+
{
|
35
|
+
size_t samples_length = RARRAY_LEN(raw_samples);
|
36
|
+
sample_t* samples = malloc(sizeof(sample_t) * samples_length);
|
37
|
+
if (samples == NULL) {
|
38
|
+
zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
|
39
|
+
}
|
40
|
+
|
41
|
+
for (size_t index = 0; index < samples_length; index++) {
|
42
|
+
VALUE raw_sample = rb_ary_entry(raw_samples, index);
|
43
|
+
sample_t* sample = &samples[index];
|
44
|
+
|
45
|
+
sample->data = RSTRING_PTR(raw_sample);
|
46
|
+
sample->size = RSTRING_LEN(raw_sample);
|
47
|
+
}
|
48
|
+
|
49
|
+
*samples_length_ptr = samples_length;
|
50
|
+
|
51
|
+
return samples;
|
52
|
+
}
|
53
|
+
|
54
|
+
static inline zstds_ext_result_t prepare_samples_group(
|
55
|
+
const sample_t* samples,
|
56
|
+
size_t samples_length,
|
57
|
+
zstds_ext_byte_t** group_ptr,
|
58
|
+
size_t** sizes_ptr)
|
33
59
|
{
|
34
|
-
|
35
|
-
const sample_t* samples = args->samples;
|
36
|
-
size_t length = args->length;
|
37
|
-
size_t size = 0;
|
60
|
+
size_t size = 0;
|
38
61
|
|
39
|
-
for (size_t index = 0; index <
|
62
|
+
for (size_t index = 0; index < samples_length; index++) {
|
40
63
|
size += samples[index].size;
|
41
64
|
}
|
42
65
|
|
43
66
|
zstds_ext_byte_t* group = malloc(size);
|
44
67
|
if (group == NULL) {
|
45
|
-
|
46
|
-
return NULL;
|
68
|
+
return ZSTDS_EXT_ERROR_ALLOCATE_FAILED;
|
47
69
|
}
|
48
70
|
|
49
|
-
size_t* sizes = malloc(
|
71
|
+
size_t* sizes = malloc(samples_length * sizeof(size_t));
|
50
72
|
if (sizes == NULL) {
|
51
73
|
free(group);
|
52
|
-
|
53
|
-
return NULL;
|
74
|
+
return ZSTDS_EXT_ERROR_ALLOCATE_FAILED;
|
54
75
|
}
|
55
76
|
|
56
77
|
size_t offset = 0;
|
57
78
|
|
58
|
-
for (size_t index = 0; index <
|
79
|
+
for (size_t index = 0; index < samples_length; index++) {
|
59
80
|
const sample_t* sample_ptr = &samples[index];
|
60
81
|
size_t sample_size = sample_ptr->size;
|
61
82
|
|
@@ -65,7 +86,38 @@ static inline void* train_wrapper(void* data)
|
|
65
86
|
sizes[index] = sample_size;
|
66
87
|
}
|
67
88
|
|
68
|
-
|
89
|
+
*group_ptr = group;
|
90
|
+
*sizes_ptr = sizes;
|
91
|
+
|
92
|
+
return 0;
|
93
|
+
}
|
94
|
+
|
95
|
+
// -- training --
|
96
|
+
|
97
|
+
typedef struct
|
98
|
+
{
|
99
|
+
const sample_t* samples;
|
100
|
+
size_t samples_length;
|
101
|
+
char* buffer;
|
102
|
+
size_t capacity;
|
103
|
+
zstds_result_t result;
|
104
|
+
zstds_ext_result_t ext_result;
|
105
|
+
} train_args_t;
|
106
|
+
|
107
|
+
static inline void* train_wrapper(void* data)
|
108
|
+
{
|
109
|
+
train_args_t* args = data;
|
110
|
+
|
111
|
+
zstds_ext_byte_t* group;
|
112
|
+
size_t* sizes;
|
113
|
+
zstds_ext_result_t result = prepare_samples_group(args->samples, args->samples_length, &group, &sizes);
|
114
|
+
if (result != 0) {
|
115
|
+
args->ext_result = result;
|
116
|
+
return NULL;
|
117
|
+
}
|
118
|
+
|
119
|
+
args->result =
|
120
|
+
ZDICT_trainFromBuffer((void*) args->buffer, args->capacity, group, sizes, (unsigned int) args->samples_length);
|
69
121
|
|
70
122
|
free(group);
|
71
123
|
free(sizes);
|
@@ -82,17 +134,13 @@ static inline void* train_wrapper(void* data)
|
|
82
134
|
|
83
135
|
VALUE zstds_ext_train_dictionary_buffer(VALUE ZSTDS_EXT_UNUSED(self), VALUE raw_samples, VALUE options)
|
84
136
|
{
|
85
|
-
|
86
|
-
|
87
|
-
size_t length = RARRAY_LEN(raw_samples);
|
88
|
-
|
89
|
-
for (size_t index = 0; index < length; index++) {
|
90
|
-
Check_Type(rb_ary_entry(raw_samples, index), T_STRING);
|
91
|
-
}
|
92
|
-
|
137
|
+
check_raw_samples(raw_samples);
|
93
138
|
Check_Type(options, T_HASH);
|
94
|
-
ZSTDS_EXT_GET_BOOL_OPTION(options, gvl);
|
95
139
|
ZSTDS_EXT_GET_SIZE_OPTION(options, capacity);
|
140
|
+
ZSTDS_EXT_GET_BOOL_OPTION(options, gvl);
|
141
|
+
|
142
|
+
size_t samples_length;
|
143
|
+
sample_t* samples = prepare_samples(raw_samples, &samples_length);
|
96
144
|
|
97
145
|
if (capacity == 0) {
|
98
146
|
capacity = ZSTDS_EXT_DEFAULT_DICTIONARY_CAPACITY;
|
@@ -105,27 +153,147 @@ VALUE zstds_ext_train_dictionary_buffer(VALUE ZSTDS_EXT_UNUSED(self), VALUE raw_
|
|
105
153
|
zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
|
106
154
|
}
|
107
155
|
|
108
|
-
|
109
|
-
|
156
|
+
train_args_t args = {
|
157
|
+
.samples = samples,
|
158
|
+
.samples_length = samples_length,
|
159
|
+
.buffer = RSTRING_PTR(buffer),
|
160
|
+
.capacity = capacity,
|
161
|
+
};
|
162
|
+
|
163
|
+
ZSTDS_EXT_GVL_WRAP(gvl, train_wrapper, &args);
|
164
|
+
free(samples);
|
165
|
+
|
166
|
+
if (args.ext_result != 0) {
|
167
|
+
zstds_ext_raise_error(args.ext_result);
|
168
|
+
}
|
169
|
+
|
170
|
+
ZSTDS_EXT_RESIZE_STRING_BUFFER(buffer, args.result, exception);
|
171
|
+
if (exception != 0) {
|
110
172
|
zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
|
111
173
|
}
|
112
174
|
|
113
|
-
|
114
|
-
|
115
|
-
sample_t* sample = &samples[index];
|
175
|
+
return buffer;
|
176
|
+
}
|
116
177
|
|
117
|
-
|
118
|
-
|
178
|
+
// -- finalizing --
|
179
|
+
|
180
|
+
#if defined(HAVE_ZDICT_FINALIZE)
|
181
|
+
typedef struct
|
182
|
+
{
|
183
|
+
const sample_t* samples;
|
184
|
+
size_t samples_length;
|
185
|
+
char* buffer;
|
186
|
+
size_t max_size;
|
187
|
+
char* content;
|
188
|
+
size_t content_length;
|
189
|
+
zstds_ext_dictionary_options_t dictionary_options;
|
190
|
+
zstds_result_t result;
|
191
|
+
zstds_ext_result_t ext_result;
|
192
|
+
} finalize_args_t;
|
193
|
+
|
194
|
+
static inline void* finalize_wrapper(void* data)
|
195
|
+
{
|
196
|
+
finalize_args_t* args = data;
|
197
|
+
|
198
|
+
zstds_ext_byte_t* group;
|
199
|
+
size_t* sizes;
|
200
|
+
zstds_ext_result_t result = prepare_samples_group(args->samples, args->samples_length, &group, &sizes);
|
201
|
+
if (result != 0) {
|
202
|
+
args->ext_result = result;
|
203
|
+
return NULL;
|
119
204
|
}
|
120
205
|
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
206
|
+
int compressionLevel;
|
207
|
+
zstds_ext_option_t compression_level = args->dictionary_options.compression_level;
|
208
|
+
if (compression_level.has_value) {
|
209
|
+
compressionLevel = compression_level.value;
|
210
|
+
} else {
|
211
|
+
compressionLevel = 0;
|
212
|
+
}
|
213
|
+
|
214
|
+
unsigned int notificationLevel;
|
215
|
+
zstds_ext_option_t notification_level = args->dictionary_options.notification_level;
|
216
|
+
if (notification_level.has_value) {
|
217
|
+
notificationLevel = notification_level.value;
|
218
|
+
} else {
|
219
|
+
notificationLevel = 0;
|
220
|
+
}
|
221
|
+
|
222
|
+
unsigned int dictID;
|
223
|
+
zstds_ext_option_t dictionary_id = args->dictionary_options.dictionary_id;
|
224
|
+
if (dictionary_id.has_value) {
|
225
|
+
dictID = dictionary_id.value;
|
226
|
+
} else {
|
227
|
+
dictID = 0;
|
228
|
+
}
|
229
|
+
|
230
|
+
ZDICT_params_t dictionary_params = {
|
231
|
+
.compressionLevel = compressionLevel,
|
232
|
+
.notificationLevel = notificationLevel,
|
233
|
+
.dictID = dictID,
|
126
234
|
};
|
127
235
|
|
128
|
-
|
236
|
+
args->result = ZDICT_finalizeDictionary(
|
237
|
+
(void*) args->buffer,
|
238
|
+
args->max_size,
|
239
|
+
(void*) args->content,
|
240
|
+
args->content_length,
|
241
|
+
group,
|
242
|
+
sizes,
|
243
|
+
(unsigned int) args->samples_length,
|
244
|
+
dictionary_params);
|
245
|
+
|
246
|
+
free(group);
|
247
|
+
free(sizes);
|
248
|
+
|
249
|
+
if (ZDICT_isError(args->result)) {
|
250
|
+
args->ext_result = zstds_ext_get_error(ZSTD_getErrorCode(args->result));
|
251
|
+
return NULL;
|
252
|
+
}
|
253
|
+
|
254
|
+
args->ext_result = 0;
|
255
|
+
|
256
|
+
return NULL;
|
257
|
+
}
|
258
|
+
|
259
|
+
VALUE zstds_ext_finalize_dictionary_buffer(
|
260
|
+
VALUE ZSTDS_EXT_UNUSED(self),
|
261
|
+
VALUE content,
|
262
|
+
VALUE raw_samples,
|
263
|
+
VALUE options)
|
264
|
+
{
|
265
|
+
Check_Type(content, T_STRING);
|
266
|
+
check_raw_samples(raw_samples);
|
267
|
+
Check_Type(options, T_HASH);
|
268
|
+
ZSTDS_EXT_GET_SIZE_OPTION(options, max_size);
|
269
|
+
ZSTDS_EXT_GET_BOOL_OPTION(options, gvl);
|
270
|
+
ZSTDS_EXT_GET_DICTIONARY_OPTIONS(options);
|
271
|
+
|
272
|
+
size_t samples_length;
|
273
|
+
sample_t* samples = prepare_samples(raw_samples, &samples_length);
|
274
|
+
|
275
|
+
if (max_size == 0) {
|
276
|
+
max_size = ZSTDS_EXT_DEFAULT_DICTIONARY_MAX_SIZE;
|
277
|
+
}
|
278
|
+
|
279
|
+
int exception;
|
280
|
+
|
281
|
+
ZSTDS_EXT_CREATE_STRING_BUFFER(buffer, max_size, exception);
|
282
|
+
if (exception != 0) {
|
283
|
+
zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
|
284
|
+
}
|
285
|
+
|
286
|
+
finalize_args_t args = {
|
287
|
+
.samples = samples,
|
288
|
+
.samples_length = samples_length,
|
289
|
+
.buffer = RSTRING_PTR(buffer),
|
290
|
+
.max_size = max_size,
|
291
|
+
.content = RSTRING_PTR(content),
|
292
|
+
.content_length = RSTRING_LEN(content),
|
293
|
+
.dictionary_options = dictionary_options,
|
294
|
+
};
|
295
|
+
|
296
|
+
ZSTDS_EXT_GVL_WRAP(gvl, finalize_wrapper, &args);
|
129
297
|
free(samples);
|
130
298
|
|
131
299
|
if (args.ext_result != 0) {
|
@@ -140,6 +308,17 @@ VALUE zstds_ext_train_dictionary_buffer(VALUE ZSTDS_EXT_UNUSED(self), VALUE raw_
|
|
140
308
|
return buffer;
|
141
309
|
}
|
142
310
|
|
311
|
+
#else
|
312
|
+
ZSTDS_EXT_NORETURN VALUE zstds_ext_finalize_dictionary_buffer(
|
313
|
+
VALUE ZSTDS_EXT_UNUSED(self),
|
314
|
+
VALUE ZSTDS_EXT_UNUSED(content),
|
315
|
+
VALUE ZSTDS_EXT_UNUSED(raw_samples),
|
316
|
+
VALUE ZSTDS_EXT_UNUSED(options))
|
317
|
+
{
|
318
|
+
zstds_ext_raise_error(ZSTDS_EXT_ERROR_NOT_IMPLEMENTED);
|
319
|
+
}
|
320
|
+
#endif // HAVE_ZDICT_FINALIZE
|
321
|
+
|
143
322
|
// -- other --
|
144
323
|
|
145
324
|
VALUE zstds_ext_get_dictionary_buffer_id(VALUE ZSTDS_EXT_UNUSED(self), VALUE buffer)
|
@@ -153,7 +332,7 @@ VALUE zstds_ext_get_dictionary_buffer_id(VALUE ZSTDS_EXT_UNUSED(self), VALUE buf
|
|
153
332
|
}
|
154
333
|
|
155
334
|
#if defined(HAVE_ZDICT_HEADER_SIZE)
|
156
|
-
VALUE zstds_ext_get_dictionary_header_size(VALUE self, VALUE buffer)
|
335
|
+
VALUE zstds_ext_get_dictionary_header_size(VALUE ZSTDS_EXT_UNUSED(self), VALUE buffer)
|
157
336
|
{
|
158
337
|
zstds_result_t result = ZDICT_getDictHeaderSize(RSTRING_PTR(buffer), RSTRING_LEN(buffer));
|
159
338
|
if (ZDICT_isError(result)) {
|
@@ -164,11 +343,12 @@ VALUE zstds_ext_get_dictionary_header_size(VALUE self, VALUE buffer)
|
|
164
343
|
}
|
165
344
|
|
166
345
|
#else
|
167
|
-
ZSTDS_EXT_NORETURN VALUE
|
346
|
+
ZSTDS_EXT_NORETURN VALUE
|
347
|
+
zstds_ext_get_dictionary_header_size(VALUE ZSTDS_EXT_UNUSED(self), VALUE ZSTDS_EXT_UNUSED(buffer))
|
168
348
|
{
|
169
349
|
zstds_ext_raise_error(ZSTDS_EXT_ERROR_NOT_IMPLEMENTED);
|
170
350
|
};
|
171
|
-
#endif
|
351
|
+
#endif // HAVE_ZDICT_HEADER_SIZE
|
172
352
|
|
173
353
|
// -- exports --
|
174
354
|
|
@@ -176,6 +356,7 @@ void zstds_ext_dictionary_exports(VALUE root_module)
|
|
176
356
|
{
|
177
357
|
VALUE dictionary = rb_define_class_under(root_module, "Dictionary", rb_cObject);
|
178
358
|
|
359
|
+
rb_define_singleton_method(dictionary, "finalize_buffer", zstds_ext_finalize_dictionary_buffer, 3);
|
179
360
|
rb_define_singleton_method(dictionary, "get_buffer_id", zstds_ext_get_dictionary_buffer_id, 1);
|
180
361
|
rb_define_singleton_method(dictionary, "get_header_size", zstds_ext_get_dictionary_header_size, 1);
|
181
362
|
rb_define_singleton_method(dictionary, "train_buffer", zstds_ext_train_dictionary_buffer, 2);
|
data/ext/zstds_ext/dictionary.h
CHANGED
@@ -9,15 +9,29 @@
|
|
9
9
|
#include "zstds_ext/macro.h"
|
10
10
|
|
11
11
|
#define ZSTDS_EXT_DEFAULT_DICTIONARY_CAPACITY (1 << 17); // 128 KB
|
12
|
+
#define ZSTDS_EXT_DEFAULT_DICTIONARY_MAX_SIZE ZSTDS_EXT_DEFAULT_DICTIONARY_CAPACITY
|
13
|
+
|
14
|
+
// -- training --
|
12
15
|
|
13
16
|
VALUE zstds_ext_train_dictionary_buffer(VALUE self, VALUE samples, VALUE options);
|
17
|
+
|
18
|
+
// -- finalizing --
|
19
|
+
|
20
|
+
#if defined(HAVE_ZDICT_FINALIZE)
|
21
|
+
VALUE zstds_ext_finalize_dictionary_buffer(VALUE self, VALUE content, VALUE samples, VALUE options);
|
22
|
+
#else
|
23
|
+
ZSTDS_EXT_NORETURN VALUE zstds_ext_finalize_dictionary_buffer(VALUE self, VALUE content, VALUE samples, VALUE options);
|
24
|
+
#endif // HAVE_ZDICT_FINALIZE
|
25
|
+
|
26
|
+
// -- other --
|
27
|
+
|
14
28
|
VALUE zstds_ext_get_dictionary_buffer_id(VALUE self, VALUE buffer);
|
15
29
|
|
16
30
|
#if defined(HAVE_ZDICT_HEADER_SIZE)
|
17
31
|
VALUE zstds_ext_get_dictionary_header_size(VALUE self, VALUE buffer);
|
18
32
|
#else
|
19
33
|
ZSTDS_EXT_NORETURN VALUE zstds_ext_get_dictionary_header_size(VALUE self, VALUE buffer);
|
20
|
-
#endif
|
34
|
+
#endif // HAVE_ZDICT_HEADER_SIZE
|
21
35
|
|
22
36
|
void zstds_ext_dictionary_exports(VALUE root_module);
|
23
37
|
|
data/ext/zstds_ext/gvl.h
CHANGED
data/ext/zstds_ext/macro.h
CHANGED
@@ -8,12 +8,12 @@
|
|
8
8
|
#define ZSTDS_EXT_UNUSED(x) x __attribute__((__unused__))
|
9
9
|
#else
|
10
10
|
#define ZSTDS_EXT_UNUSED(x) x
|
11
|
-
#endif
|
11
|
+
#endif // __GNUC__
|
12
12
|
|
13
13
|
#if defined(__GNUC__)
|
14
14
|
#define ZSTDS_EXT_NORETURN __attribute__((__noreturn__))
|
15
15
|
#else
|
16
16
|
#define ZSTDS_EXT_NORETURN
|
17
|
-
#endif
|
17
|
+
#endif // __GNUC__
|
18
18
|
|
19
19
|
#endif // ZSTDS_EXT_MACRO_H
|
data/ext/zstds_ext/option.h
CHANGED
@@ -68,6 +68,13 @@ typedef struct
|
|
68
68
|
VALUE dictionary;
|
69
69
|
} zstds_ext_decompressor_options_t;
|
70
70
|
|
71
|
+
typedef struct
|
72
|
+
{
|
73
|
+
zstds_ext_option_t compression_level;
|
74
|
+
zstds_ext_option_t notification_level;
|
75
|
+
zstds_ext_option_t dictionary_id;
|
76
|
+
} zstds_ext_dictionary_options_t;
|
77
|
+
|
71
78
|
void zstds_ext_resolve_option(
|
72
79
|
VALUE options,
|
73
80
|
zstds_ext_option_t* option,
|
@@ -117,6 +124,13 @@ void zstds_ext_resolve_dictionary_option(VALUE options, VALUE* option, const cha
|
|
117
124
|
ZSTDS_EXT_RESOLVE_OPTION(options, decompressor_options, ZSTDS_EXT_OPTION_TYPE_UINT, window_log_max); \
|
118
125
|
ZSTDS_EXT_RESOLVE_DICTIONARY_OPTION(options, decompressor_options, dictionary);
|
119
126
|
|
127
|
+
#define ZSTDS_EXT_GET_DICTIONARY_OPTIONS(options) \
|
128
|
+
zstds_ext_dictionary_options_t dictionary_options; \
|
129
|
+
\
|
130
|
+
ZSTDS_EXT_RESOLVE_OPTION(options, dictionary_options, ZSTDS_EXT_OPTION_TYPE_INT, compression_level); \
|
131
|
+
ZSTDS_EXT_RESOLVE_OPTION(options, dictionary_options, ZSTDS_EXT_OPTION_TYPE_UINT, notification_level); \
|
132
|
+
ZSTDS_EXT_RESOLVE_OPTION(options, dictionary_options, ZSTDS_EXT_OPTION_TYPE_UINT, dictionary_id);
|
133
|
+
|
120
134
|
bool zstds_ext_get_bool_option_value(VALUE options, const char* name);
|
121
135
|
size_t zstds_ext_get_size_option_value(VALUE options, const char* name);
|
122
136
|
|
data/lib/zstds/dictionary.rb
CHANGED
@@ -14,6 +14,20 @@ module ZSTDS
|
|
14
14
|
}
|
15
15
|
.freeze
|
16
16
|
|
17
|
+
FINALIZE_DEFAULTS = {
|
18
|
+
:gvl => false,
|
19
|
+
:max_size => 0,
|
20
|
+
:dictionary_options => {}
|
21
|
+
}
|
22
|
+
.freeze
|
23
|
+
|
24
|
+
FINALIZE_DICTIONARY_DEFAULTS = {
|
25
|
+
:compression_level => 0,
|
26
|
+
:notification_level => 0,
|
27
|
+
:dictionary_id => 0
|
28
|
+
}
|
29
|
+
.freeze
|
30
|
+
|
17
31
|
attr_reader :buffer
|
18
32
|
|
19
33
|
def initialize(buffer)
|
@@ -24,12 +38,7 @@ module ZSTDS
|
|
24
38
|
end
|
25
39
|
|
26
40
|
def self.train(samples, options = {})
|
27
|
-
|
28
|
-
|
29
|
-
samples.each do |sample|
|
30
|
-
Validation.validate_string sample
|
31
|
-
raise ValidateError, "dictionary sample should not be empty" if sample.empty?
|
32
|
-
end
|
41
|
+
validate_samples samples
|
33
42
|
|
34
43
|
Validation.validate_hash options
|
35
44
|
|
@@ -42,6 +51,43 @@ module ZSTDS
|
|
42
51
|
new buffer
|
43
52
|
end
|
44
53
|
|
54
|
+
def self.finalize(content, samples, options = {})
|
55
|
+
Validation.validate_string content
|
56
|
+
raise ValidateError, "content should not be empty" if content.empty?
|
57
|
+
|
58
|
+
validate_samples samples
|
59
|
+
|
60
|
+
Validation.validate_hash options
|
61
|
+
|
62
|
+
options = FINALIZE_DEFAULTS.merge options
|
63
|
+
|
64
|
+
Validation.validate_bool options[:gvl]
|
65
|
+
Validation.validate_not_negative_integer options[:max_size]
|
66
|
+
Validation.validate_hash options[:dictionary_options]
|
67
|
+
|
68
|
+
dictionary_options = FINALIZE_DICTIONARY_DEFAULTS.merge options[:dictionary_options]
|
69
|
+
|
70
|
+
compression_level = dictionary_options[:compression_level]
|
71
|
+
Validation.validate_integer compression_level
|
72
|
+
raise ValidateError, "invalid compression level" if
|
73
|
+
compression_level < Option::MIN_COMPRESSION_LEVEL || compression_level > Option::MAX_COMPRESSION_LEVEL
|
74
|
+
|
75
|
+
Validation.validate_not_negative_integer dictionary_options[:notification_level]
|
76
|
+
Validation.validate_not_negative_integer dictionary_options[:dictionary_id]
|
77
|
+
|
78
|
+
buffer = finalize_buffer content, samples, options
|
79
|
+
new buffer
|
80
|
+
end
|
81
|
+
|
82
|
+
def self.validate_samples(samples)
|
83
|
+
Validation.validate_array samples
|
84
|
+
|
85
|
+
samples.each do |sample|
|
86
|
+
Validation.validate_string sample
|
87
|
+
raise ValidateError, "dictionary sample should not be empty" if sample.empty?
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
45
91
|
def id
|
46
92
|
self.class.get_buffer_id @buffer
|
47
93
|
end
|
@@ -24,9 +24,7 @@ module ZSTDS
|
|
24
24
|
|
25
25
|
def initialize(io, options = {})
|
26
26
|
@raw_stream = create_raw_stream
|
27
|
-
|
28
|
-
Validation.validate_io io
|
29
|
-
@io = io
|
27
|
+
@io = io
|
30
28
|
|
31
29
|
@stat = Stat.new @io.stat if @io.respond_to? :stat
|
32
30
|
|
@@ -135,13 +133,19 @@ module ZSTDS
|
|
135
133
|
end
|
136
134
|
|
137
135
|
def close
|
138
|
-
@io.close
|
136
|
+
@io.close if @io.respond_to? :close
|
139
137
|
|
140
138
|
nil
|
141
139
|
end
|
142
140
|
|
143
141
|
def closed?
|
144
|
-
|
142
|
+
return false unless @raw_stream.closed?
|
143
|
+
|
144
|
+
if @io.respond_to? :closed
|
145
|
+
@io.closed?
|
146
|
+
else
|
147
|
+
true
|
148
|
+
end
|
145
149
|
end
|
146
150
|
|
147
151
|
def to_io
|
data/lib/zstds/stream/reader.rb
CHANGED
@@ -54,6 +54,9 @@ module ZSTDS
|
|
54
54
|
Validation.validate_not_negative_integer bytes_to_read unless bytes_to_read.nil?
|
55
55
|
Validation.validate_string out_buffer unless out_buffer.nil?
|
56
56
|
|
57
|
+
raise ValidateError, "io should be responsible to read and eof" unless
|
58
|
+
@io.respond_to?(:read) && @io.respond_to?(:eof?)
|
59
|
+
|
57
60
|
unless bytes_to_read.nil?
|
58
61
|
return ::String.new :encoding => ::Encoding::BINARY if bytes_to_read.zero?
|
59
62
|
return nil if eof?
|
@@ -86,16 +89,22 @@ module ZSTDS
|
|
86
89
|
end
|
87
90
|
|
88
91
|
def eof?
|
92
|
+
raise ValidateError, "io should be responsible to eof" unless @io.respond_to? :eof?
|
93
|
+
|
89
94
|
empty? && @io.eof?
|
90
95
|
end
|
91
96
|
|
92
97
|
# -- asynchronous --
|
93
98
|
|
94
99
|
def readpartial(bytes_to_read, out_buffer = nil)
|
100
|
+
raise ValidateError, "io should be responsible to readpartial" unless @io.respond_to? :readpartial
|
101
|
+
|
95
102
|
read_more_nonblock(bytes_to_read, out_buffer) { @io.readpartial @source_buffer_length }
|
96
103
|
end
|
97
104
|
|
98
105
|
def read_nonblock(bytes_to_read, out_buffer = nil, *options)
|
106
|
+
raise ValidateError, "io should be responsible to read nonblock" unless @io.respond_to? :read_nonblock
|
107
|
+
|
99
108
|
read_more_nonblock(bytes_to_read, out_buffer) { @io.read_nonblock(@source_buffer_length, *options) }
|
100
109
|
end
|
101
110
|
|
data/lib/zstds/stream/writer.rb
CHANGED
@@ -23,6 +23,8 @@ module ZSTDS
|
|
23
23
|
# -- synchronous --
|
24
24
|
|
25
25
|
def write(*objects)
|
26
|
+
validate_write
|
27
|
+
|
26
28
|
write_remaining_buffer
|
27
29
|
|
28
30
|
bytes_written = 0
|
@@ -38,20 +40,26 @@ module ZSTDS
|
|
38
40
|
end
|
39
41
|
|
40
42
|
def flush
|
43
|
+
validate_write
|
44
|
+
|
41
45
|
finish :flush
|
42
46
|
|
43
|
-
@io.flush
|
47
|
+
@io.flush if @io.respond_to? :flush
|
44
48
|
|
45
49
|
self
|
46
50
|
end
|
47
51
|
|
48
52
|
def rewind
|
53
|
+
validate_write
|
54
|
+
|
49
55
|
finish :close
|
50
56
|
|
51
57
|
super
|
52
58
|
end
|
53
59
|
|
54
60
|
def close
|
61
|
+
validate_write
|
62
|
+
|
55
63
|
finish :close
|
56
64
|
|
57
65
|
super
|
@@ -75,6 +83,10 @@ module ZSTDS
|
|
75
83
|
@raw_stream.send(method_name, *args) { |portion| @io.write portion }
|
76
84
|
end
|
77
85
|
|
86
|
+
def validate_write
|
87
|
+
raise ValidateError, "io should be responsible to write" unless @io.respond_to? :write
|
88
|
+
end
|
89
|
+
|
78
90
|
# -- asynchronous --
|
79
91
|
|
80
92
|
# IO write nonblock can raise wait writable error.
|
@@ -83,6 +95,8 @@ module ZSTDS
|
|
83
95
|
# So we have to accept content after processing IO write nonblock.
|
84
96
|
# It means that first write nonblock won't call IO write nonblock.
|
85
97
|
def write_nonblock(object, *options)
|
98
|
+
validate_write_nonblock
|
99
|
+
|
86
100
|
return 0 unless write_remaining_buffer_nonblock(*options)
|
87
101
|
|
88
102
|
source = transcode object.to_s
|
@@ -93,14 +107,18 @@ module ZSTDS
|
|
93
107
|
end
|
94
108
|
|
95
109
|
def flush_nonblock(*options)
|
110
|
+
validate_write_nonblock
|
111
|
+
|
96
112
|
return false unless finish_nonblock :flush, *options
|
97
113
|
|
98
|
-
@io.flush
|
114
|
+
@io.flush if @io.respond_to? :flush
|
99
115
|
|
100
116
|
true
|
101
117
|
end
|
102
118
|
|
103
119
|
def rewind_nonblock(*options)
|
120
|
+
validate_write_nonblock
|
121
|
+
|
104
122
|
return false unless finish_nonblock :close, *options
|
105
123
|
|
106
124
|
method(:rewind).super_method.call
|
@@ -109,6 +127,8 @@ module ZSTDS
|
|
109
127
|
end
|
110
128
|
|
111
129
|
def close_nonblock(*options)
|
130
|
+
validate_write_nonblock
|
131
|
+
|
112
132
|
return false unless finish_nonblock :close, *options
|
113
133
|
|
114
134
|
method(:close).super_method.call
|
@@ -139,6 +159,10 @@ module ZSTDS
|
|
139
159
|
@raw_stream.send(method_name, *args) { |portion| @buffer << portion }
|
140
160
|
end
|
141
161
|
|
162
|
+
def validate_write_nonblock
|
163
|
+
raise ValidateError, "io should be responsible to write nonblock" unless @io.respond_to? :write_nonblock
|
164
|
+
end
|
165
|
+
|
142
166
|
# -- common --
|
143
167
|
|
144
168
|
protected def transcode(data)
|
data/lib/zstds/validation.rb
CHANGED
@@ -5,33 +5,34 @@ require_relative "error"
|
|
5
5
|
|
6
6
|
module ZSTDS
|
7
7
|
module Validation
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
readpartial
|
12
|
-
read_nonblock
|
13
|
-
write_nonblock
|
14
|
-
eof?
|
15
|
-
flush
|
16
|
-
close
|
17
|
-
closed?
|
18
|
-
]
|
19
|
-
.freeze
|
8
|
+
def self.validate_array(value)
|
9
|
+
raise ValidateError, "invalid array" unless value.is_a? ::Array
|
10
|
+
end
|
20
11
|
|
21
12
|
def self.validate_bool(value)
|
22
13
|
raise ValidateError, "invalid bool" unless value.is_a?(::TrueClass) || value.is_a?(::FalseClass)
|
23
14
|
end
|
24
15
|
|
16
|
+
def self.validate_hash(value)
|
17
|
+
raise ValidateError, "invalid hash" unless value.is_a? ::Hash
|
18
|
+
end
|
19
|
+
|
25
20
|
def self.validate_integer(value)
|
26
21
|
raise ValidateError, "invalid integer" unless value.is_a? ::Integer
|
27
22
|
end
|
28
23
|
|
24
|
+
def self.validate_not_negative_integer(value)
|
25
|
+
raise ValidateError, "invalid not negative integer" unless value.is_a?(::Integer) && value >= 0
|
26
|
+
end
|
27
|
+
|
29
28
|
def self.validate_positive_integer(value)
|
30
29
|
raise ValidateError, "invalid positive integer" unless value.is_a?(::Integer) && value.positive?
|
31
30
|
end
|
32
31
|
|
33
|
-
def self.
|
34
|
-
|
32
|
+
def self.validate_proc(value)
|
33
|
+
unless value.is_a?(::Proc) || value.is_a?(::Method) || value.is_a?(::UnboundMethod)
|
34
|
+
raise ValidateError, "invalid proc"
|
35
|
+
end
|
35
36
|
end
|
36
37
|
|
37
38
|
def self.validate_string(value)
|
@@ -41,23 +42,5 @@ module ZSTDS
|
|
41
42
|
def self.validate_symbol(value)
|
42
43
|
raise ValidateError, "invalid symbol" unless value.is_a? ::Symbol
|
43
44
|
end
|
44
|
-
|
45
|
-
def self.validate_io(value)
|
46
|
-
raise ValidateError, "invalid io" unless IO_METHODS.all? { |method| value.respond_to? method }
|
47
|
-
end
|
48
|
-
|
49
|
-
def self.validate_array(value)
|
50
|
-
raise ValidateError, "invalid array" unless value.is_a? ::Array
|
51
|
-
end
|
52
|
-
|
53
|
-
def self.validate_hash(value)
|
54
|
-
raise ValidateError, "invalid hash" unless value.is_a? ::Hash
|
55
|
-
end
|
56
|
-
|
57
|
-
def self.validate_proc(value)
|
58
|
-
unless value.is_a?(::Proc) || value.is_a?(::Method) || value.is_a?(::UnboundMethod)
|
59
|
-
raise ValidateError, "invalid proc"
|
60
|
-
end
|
61
|
-
end
|
62
45
|
end
|
63
46
|
end
|
data/lib/zstds/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,16 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ruby-zstds
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andrew Aladjev
|
8
|
+
- Ivan Takarlikov
|
9
|
+
- Jenner La Fave
|
8
10
|
autorequire:
|
9
11
|
bindir: bin
|
10
12
|
cert_chain: []
|
11
|
-
date: 2022-
|
13
|
+
date: 2022-03-18 00:00:00.000000000 Z
|
12
14
|
dependencies:
|
13
15
|
- !ruby/object:Gem::Dependency
|
14
16
|
name: codecov
|
@@ -128,14 +130,14 @@ dependencies:
|
|
128
130
|
requirements:
|
129
131
|
- - "~>"
|
130
132
|
- !ruby/object:Gem::Version
|
131
|
-
version: '1.
|
133
|
+
version: '1.26'
|
132
134
|
type: :development
|
133
135
|
prerelease: false
|
134
136
|
version_requirements: !ruby/object:Gem::Requirement
|
135
137
|
requirements:
|
136
138
|
- - "~>"
|
137
139
|
- !ruby/object:Gem::Version
|
138
|
-
version: '1.
|
140
|
+
version: '1.26'
|
139
141
|
- !ruby/object:Gem::Dependency
|
140
142
|
name: rubocop-minitest
|
141
143
|
requirement: !ruby/object:Gem::Requirement
|
@@ -261,7 +263,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
261
263
|
- !ruby/object:Gem::Version
|
262
264
|
version: '0'
|
263
265
|
requirements: []
|
264
|
-
rubygems_version: 3.3.
|
266
|
+
rubygems_version: 3.3.7
|
265
267
|
signing_key:
|
266
268
|
specification_version: 4
|
267
269
|
summary: Ruby bindings for zstd library.
|