ruby-zstds 1.2.1 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 80ccd3c8a7210dcf3fabdfadcca8e37ce82d5fe78c7f761f5ae7c355ff746854
4
- data.tar.gz: a4ed4e5dd4c89b7f95438c5e3033d09eb25721ab32fda8e1eac9351774ab3215
3
+ metadata.gz: ec16246ba1102583f6f309e555e4ac4dbbf49135f5adee4e0332b86be7388de4
4
+ data.tar.gz: 7d8d40b406c9da3ed50c5929f565c51fc8cf46558bee937fdeaa0d942d0fe84f
5
5
  SHA512:
6
- metadata.gz: 967b5d866cd79b139bdbca97e1fdacf9063e3bc425f441be239980dd839d123173fcad15c0830576a32fe7c0bb5166f4fd5159717a9ac7f038fe57a435859f70
7
- data.tar.gz: 78c54fc20f8facaa556eb68438407cfa7ff96d33425a9dce73584fc26435177dda8478c729a412f00d9b28a9d860b3dec6a059a728926a587f3a83dae9fe1836
6
+ metadata.gz: fd92bd835bf7eb14db13ecc1761a89467e3f38bbcb9d373573fa7c3a8839f4050a478091ada05fec97bb8e2824e9913d23ef9593f3388e317c5b9df456e7e7cd
7
+ data.tar.gz: 00d8bd37067fd788a7033abd8cf4c6d590b4f672f8986e7da7b80e47ed9ffa4a91886db7442a2a118a6ba45c8f9b2b232665455a5fc85c6cf4c8fcbe119c4e9c
data/AUTHORS CHANGED
@@ -1 +1,3 @@
1
1
  Andrew Aladjev
2
+ Ivan Takarlikov
3
+ Jenner La Fave
data/README.md CHANGED
@@ -8,7 +8,17 @@ See [zstd library](https://github.com/facebook/zstd).
8
8
 
9
9
  ## Installation
10
10
 
11
- Please install zstd library first, use latest 1.4.0+ version.
11
+ Operating systems: GNU/Linux, FreeBSD, OSX, Windows (MinGW).
12
+
13
+ Dependencies: [zstd](https://github.com/facebook/zstd) 1.4.0+ version.
14
+
15
+ | Popular OS | Dependencies |
16
+ |------------|---------------------------|
17
+ | Ubuntu | `libzstd-dev` |
18
+ | CentOS | `libzstd-devel` |
19
+ | ArchLinux | `zstd` |
20
+ | OSX | `zstd` |
21
+ | Windows | `mingw-w64-x86_64-zstd` |
12
22
 
13
23
  ```sh
14
24
  gem install ruby-zstds
@@ -23,6 +33,22 @@ gem install pkg/ruby-zstds-*.gem
23
33
 
24
34
  You can also use [overlay](https://github.com/andrew-aladev/overlay) for gentoo.
25
35
 
36
+ ### Installation in macOS on Apple Silicon
37
+
38
+ On M1 Macs, Homebrew installs to /opt/homebrew, so you'll need to specify its
39
+ include and lib paths when building the native extension for zstd.
40
+
41
+ ```sh
42
+ brew install zstd
43
+ gem install ruby-zstds -- --with-opt-include=/opt/homebrew/include --with-opt-lib=/opt/homebrew/lib
44
+ ```
45
+
46
+ You can also configure Bundler to use those options when installing:
47
+
48
+ ```sh
49
+ bundle config set build.ruby-zstds "--with-opt-include=/opt/homebrew/include --with-opt-lib=/opt/homebrew/lib"
50
+ ```
51
+
26
52
  ## Usage
27
53
 
28
54
  There are simple APIs: `String` and `File`. Also you can use generic streaming API: `Stream::Writer` and `Stream::Reader`.
@@ -449,10 +475,6 @@ You should lock all shared data between threads.
449
475
  For example: you should not use same compressor/decompressor inside multiple threads.
450
476
  Please verify that you are using each processor inside single thread at the same time.
451
477
 
452
- ## Operating systems
453
-
454
- GNU/Linux, FreeBSD, OSX, Windows (MinGW).
455
-
456
478
  ## CI
457
479
 
458
480
  Please visit [scripts/test-images](scripts/test-images).
data/ext/extconf.rb CHANGED
@@ -26,7 +26,6 @@ def require_header(name, constants: [], macroses: [], types: [])
26
26
  end
27
27
 
28
28
  require_header "zdict.h"
29
-
30
29
  require_header(
31
30
  "zstd.h",
32
31
  :constants => %w[
@@ -112,6 +111,13 @@ end
112
111
  if find_library "zstd", "ZDICT_getDictHeaderSize"
113
112
  $defs.push "-DHAVE_ZDICT_HEADER_SIZE"
114
113
  end
114
+
115
+ zdict_has_params = find_type "ZDICT_params_t", nil, "zdict.h"
116
+ zdict_has_finalize = find_library "zstd", "ZDICT_finalizeDictionary"
117
+
118
+ if zdict_has_params && zdict_has_finalize
119
+ $defs.push "-DHAVE_ZDICT_FINALIZE"
120
+ end
115
121
  # rubocop:enable Style/GlobalVars
116
122
 
117
123
  require_library(
@@ -11,7 +11,7 @@
11
11
  #include "zstds_ext/gvl.h"
12
12
  #include "zstds_ext/option.h"
13
13
 
14
- // -- initialization --
14
+ // -- common --
15
15
 
16
16
  typedef struct
17
17
  {
@@ -19,43 +19,64 @@ typedef struct
19
19
  size_t size;
20
20
  } sample_t;
21
21
 
22
- typedef struct
22
+ static inline void check_raw_samples(VALUE raw_samples)
23
23
  {
24
- const sample_t* samples;
25
- size_t length;
26
- char* buffer;
27
- size_t capacity;
28
- zstds_result_t result;
29
- zstds_ext_result_t ext_result;
30
- } train_args_t;
24
+ Check_Type(raw_samples, T_ARRAY);
31
25
 
32
- static inline void* train_wrapper(void* data)
26
+ size_t samples_length = RARRAY_LEN(raw_samples);
27
+
28
+ for (size_t index = 0; index < samples_length; index++) {
29
+ Check_Type(rb_ary_entry(raw_samples, index), T_STRING);
30
+ }
31
+ }
32
+
33
+ static inline sample_t* prepare_samples(VALUE raw_samples, size_t* samples_length_ptr)
34
+ {
35
+ size_t samples_length = RARRAY_LEN(raw_samples);
36
+ sample_t* samples = malloc(sizeof(sample_t) * samples_length);
37
+ if (samples == NULL) {
38
+ zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
39
+ }
40
+
41
+ for (size_t index = 0; index < samples_length; index++) {
42
+ VALUE raw_sample = rb_ary_entry(raw_samples, index);
43
+ sample_t* sample = &samples[index];
44
+
45
+ sample->data = RSTRING_PTR(raw_sample);
46
+ sample->size = RSTRING_LEN(raw_sample);
47
+ }
48
+
49
+ *samples_length_ptr = samples_length;
50
+
51
+ return samples;
52
+ }
53
+
54
+ static inline zstds_ext_result_t prepare_samples_group(
55
+ const sample_t* samples,
56
+ size_t samples_length,
57
+ zstds_ext_byte_t** group_ptr,
58
+ size_t** sizes_ptr)
33
59
  {
34
- train_args_t* args = data;
35
- const sample_t* samples = args->samples;
36
- size_t length = args->length;
37
- size_t size = 0;
60
+ size_t size = 0;
38
61
 
39
- for (size_t index = 0; index < length; index++) {
62
+ for (size_t index = 0; index < samples_length; index++) {
40
63
  size += samples[index].size;
41
64
  }
42
65
 
43
66
  zstds_ext_byte_t* group = malloc(size);
44
67
  if (group == NULL) {
45
- args->ext_result = ZSTDS_EXT_ERROR_ALLOCATE_FAILED;
46
- return NULL;
68
+ return ZSTDS_EXT_ERROR_ALLOCATE_FAILED;
47
69
  }
48
70
 
49
- size_t* sizes = malloc(length * sizeof(size_t));
71
+ size_t* sizes = malloc(samples_length * sizeof(size_t));
50
72
  if (sizes == NULL) {
51
73
  free(group);
52
- args->ext_result = ZSTDS_EXT_ERROR_ALLOCATE_FAILED;
53
- return NULL;
74
+ return ZSTDS_EXT_ERROR_ALLOCATE_FAILED;
54
75
  }
55
76
 
56
77
  size_t offset = 0;
57
78
 
58
- for (size_t index = 0; index < length; index++) {
79
+ for (size_t index = 0; index < samples_length; index++) {
59
80
  const sample_t* sample_ptr = &samples[index];
60
81
  size_t sample_size = sample_ptr->size;
61
82
 
@@ -65,7 +86,38 @@ static inline void* train_wrapper(void* data)
65
86
  sizes[index] = sample_size;
66
87
  }
67
88
 
68
- args->result = ZDICT_trainFromBuffer((void*) args->buffer, args->capacity, group, sizes, (unsigned int) length);
89
+ *group_ptr = group;
90
+ *sizes_ptr = sizes;
91
+
92
+ return 0;
93
+ }
94
+
95
+ // -- training --
96
+
97
+ typedef struct
98
+ {
99
+ const sample_t* samples;
100
+ size_t samples_length;
101
+ char* buffer;
102
+ size_t capacity;
103
+ zstds_result_t result;
104
+ zstds_ext_result_t ext_result;
105
+ } train_args_t;
106
+
107
+ static inline void* train_wrapper(void* data)
108
+ {
109
+ train_args_t* args = data;
110
+
111
+ zstds_ext_byte_t* group;
112
+ size_t* sizes;
113
+ zstds_ext_result_t result = prepare_samples_group(args->samples, args->samples_length, &group, &sizes);
114
+ if (result != 0) {
115
+ args->ext_result = result;
116
+ return NULL;
117
+ }
118
+
119
+ args->result =
120
+ ZDICT_trainFromBuffer((void*) args->buffer, args->capacity, group, sizes, (unsigned int) args->samples_length);
69
121
 
70
122
  free(group);
71
123
  free(sizes);
@@ -82,17 +134,13 @@ static inline void* train_wrapper(void* data)
82
134
 
83
135
  VALUE zstds_ext_train_dictionary_buffer(VALUE ZSTDS_EXT_UNUSED(self), VALUE raw_samples, VALUE options)
84
136
  {
85
- Check_Type(raw_samples, T_ARRAY);
86
-
87
- size_t length = RARRAY_LEN(raw_samples);
88
-
89
- for (size_t index = 0; index < length; index++) {
90
- Check_Type(rb_ary_entry(raw_samples, index), T_STRING);
91
- }
92
-
137
+ check_raw_samples(raw_samples);
93
138
  Check_Type(options, T_HASH);
94
- ZSTDS_EXT_GET_BOOL_OPTION(options, gvl);
95
139
  ZSTDS_EXT_GET_SIZE_OPTION(options, capacity);
140
+ ZSTDS_EXT_GET_BOOL_OPTION(options, gvl);
141
+
142
+ size_t samples_length;
143
+ sample_t* samples = prepare_samples(raw_samples, &samples_length);
96
144
 
97
145
  if (capacity == 0) {
98
146
  capacity = ZSTDS_EXT_DEFAULT_DICTIONARY_CAPACITY;
@@ -105,27 +153,147 @@ VALUE zstds_ext_train_dictionary_buffer(VALUE ZSTDS_EXT_UNUSED(self), VALUE raw_
105
153
  zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
106
154
  }
107
155
 
108
- sample_t* samples = malloc(sizeof(sample_t) * length);
109
- if (samples == NULL) {
156
+ train_args_t args = {
157
+ .samples = samples,
158
+ .samples_length = samples_length,
159
+ .buffer = RSTRING_PTR(buffer),
160
+ .capacity = capacity,
161
+ };
162
+
163
+ ZSTDS_EXT_GVL_WRAP(gvl, train_wrapper, &args);
164
+ free(samples);
165
+
166
+ if (args.ext_result != 0) {
167
+ zstds_ext_raise_error(args.ext_result);
168
+ }
169
+
170
+ ZSTDS_EXT_RESIZE_STRING_BUFFER(buffer, args.result, exception);
171
+ if (exception != 0) {
110
172
  zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
111
173
  }
112
174
 
113
- for (size_t index = 0; index < length; index++) {
114
- VALUE raw_sample = rb_ary_entry(raw_samples, index);
115
- sample_t* sample = &samples[index];
175
+ return buffer;
176
+ }
116
177
 
117
- sample->data = RSTRING_PTR(raw_sample);
118
- sample->size = RSTRING_LEN(raw_sample);
178
+ // -- finalizing --
179
+
180
+ #if defined(HAVE_ZDICT_FINALIZE)
181
+ typedef struct
182
+ {
183
+ const sample_t* samples;
184
+ size_t samples_length;
185
+ char* buffer;
186
+ size_t max_size;
187
+ char* content;
188
+ size_t content_length;
189
+ zstds_ext_dictionary_options_t dictionary_options;
190
+ zstds_result_t result;
191
+ zstds_ext_result_t ext_result;
192
+ } finalize_args_t;
193
+
194
+ static inline void* finalize_wrapper(void* data)
195
+ {
196
+ finalize_args_t* args = data;
197
+
198
+ zstds_ext_byte_t* group;
199
+ size_t* sizes;
200
+ zstds_ext_result_t result = prepare_samples_group(args->samples, args->samples_length, &group, &sizes);
201
+ if (result != 0) {
202
+ args->ext_result = result;
203
+ return NULL;
119
204
  }
120
205
 
121
- train_args_t args = {
122
- .samples = samples,
123
- .length = length,
124
- .buffer = RSTRING_PTR(buffer),
125
- .capacity = capacity,
206
+ int compressionLevel;
207
+ zstds_ext_option_t compression_level = args->dictionary_options.compression_level;
208
+ if (compression_level.has_value) {
209
+ compressionLevel = compression_level.value;
210
+ } else {
211
+ compressionLevel = 0;
212
+ }
213
+
214
+ unsigned int notificationLevel;
215
+ zstds_ext_option_t notification_level = args->dictionary_options.notification_level;
216
+ if (notification_level.has_value) {
217
+ notificationLevel = notification_level.value;
218
+ } else {
219
+ notificationLevel = 0;
220
+ }
221
+
222
+ unsigned int dictID;
223
+ zstds_ext_option_t dictionary_id = args->dictionary_options.dictionary_id;
224
+ if (dictionary_id.has_value) {
225
+ dictID = dictionary_id.value;
226
+ } else {
227
+ dictID = 0;
228
+ }
229
+
230
+ ZDICT_params_t dictionary_params = {
231
+ .compressionLevel = compressionLevel,
232
+ .notificationLevel = notificationLevel,
233
+ .dictID = dictID,
126
234
  };
127
235
 
128
- ZSTDS_EXT_GVL_WRAP(gvl, train_wrapper, &args);
236
+ args->result = ZDICT_finalizeDictionary(
237
+ (void*) args->buffer,
238
+ args->max_size,
239
+ (void*) args->content,
240
+ args->content_length,
241
+ group,
242
+ sizes,
243
+ (unsigned int) args->samples_length,
244
+ dictionary_params);
245
+
246
+ free(group);
247
+ free(sizes);
248
+
249
+ if (ZDICT_isError(args->result)) {
250
+ args->ext_result = zstds_ext_get_error(ZSTD_getErrorCode(args->result));
251
+ return NULL;
252
+ }
253
+
254
+ args->ext_result = 0;
255
+
256
+ return NULL;
257
+ }
258
+
259
+ VALUE zstds_ext_finalize_dictionary_buffer(
260
+ VALUE ZSTDS_EXT_UNUSED(self),
261
+ VALUE content,
262
+ VALUE raw_samples,
263
+ VALUE options)
264
+ {
265
+ Check_Type(content, T_STRING);
266
+ check_raw_samples(raw_samples);
267
+ Check_Type(options, T_HASH);
268
+ ZSTDS_EXT_GET_SIZE_OPTION(options, max_size);
269
+ ZSTDS_EXT_GET_BOOL_OPTION(options, gvl);
270
+ ZSTDS_EXT_GET_DICTIONARY_OPTIONS(options);
271
+
272
+ size_t samples_length;
273
+ sample_t* samples = prepare_samples(raw_samples, &samples_length);
274
+
275
+ if (max_size == 0) {
276
+ max_size = ZSTDS_EXT_DEFAULT_DICTIONARY_MAX_SIZE;
277
+ }
278
+
279
+ int exception;
280
+
281
+ ZSTDS_EXT_CREATE_STRING_BUFFER(buffer, max_size, exception);
282
+ if (exception != 0) {
283
+ zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
284
+ }
285
+
286
+ finalize_args_t args = {
287
+ .samples = samples,
288
+ .samples_length = samples_length,
289
+ .buffer = RSTRING_PTR(buffer),
290
+ .max_size = max_size,
291
+ .content = RSTRING_PTR(content),
292
+ .content_length = RSTRING_LEN(content),
293
+ .dictionary_options = dictionary_options,
294
+ };
295
+
296
+ ZSTDS_EXT_GVL_WRAP(gvl, finalize_wrapper, &args);
129
297
  free(samples);
130
298
 
131
299
  if (args.ext_result != 0) {
@@ -140,6 +308,17 @@ VALUE zstds_ext_train_dictionary_buffer(VALUE ZSTDS_EXT_UNUSED(self), VALUE raw_
140
308
  return buffer;
141
309
  }
142
310
 
311
+ #else
312
+ ZSTDS_EXT_NORETURN VALUE zstds_ext_finalize_dictionary_buffer(
313
+ VALUE ZSTDS_EXT_UNUSED(self),
314
+ VALUE ZSTDS_EXT_UNUSED(content),
315
+ VALUE ZSTDS_EXT_UNUSED(raw_samples),
316
+ VALUE ZSTDS_EXT_UNUSED(options))
317
+ {
318
+ zstds_ext_raise_error(ZSTDS_EXT_ERROR_NOT_IMPLEMENTED);
319
+ }
320
+ #endif // HAVE_ZDICT_FINALIZE
321
+
143
322
  // -- other --
144
323
 
145
324
  VALUE zstds_ext_get_dictionary_buffer_id(VALUE ZSTDS_EXT_UNUSED(self), VALUE buffer)
@@ -153,7 +332,7 @@ VALUE zstds_ext_get_dictionary_buffer_id(VALUE ZSTDS_EXT_UNUSED(self), VALUE buf
153
332
  }
154
333
 
155
334
  #if defined(HAVE_ZDICT_HEADER_SIZE)
156
- VALUE zstds_ext_get_dictionary_header_size(VALUE self, VALUE buffer)
335
+ VALUE zstds_ext_get_dictionary_header_size(VALUE ZSTDS_EXT_UNUSED(self), VALUE buffer)
157
336
  {
158
337
  zstds_result_t result = ZDICT_getDictHeaderSize(RSTRING_PTR(buffer), RSTRING_LEN(buffer));
159
338
  if (ZDICT_isError(result)) {
@@ -164,11 +343,12 @@ VALUE zstds_ext_get_dictionary_header_size(VALUE self, VALUE buffer)
164
343
  }
165
344
 
166
345
  #else
167
- ZSTDS_EXT_NORETURN VALUE zstds_ext_get_dictionary_header_size(VALUE self, VALUE buffer)
346
+ ZSTDS_EXT_NORETURN VALUE
347
+ zstds_ext_get_dictionary_header_size(VALUE ZSTDS_EXT_UNUSED(self), VALUE ZSTDS_EXT_UNUSED(buffer))
168
348
  {
169
349
  zstds_ext_raise_error(ZSTDS_EXT_ERROR_NOT_IMPLEMENTED);
170
350
  };
171
- #endif
351
+ #endif // HAVE_ZDICT_HEADER_SIZE
172
352
 
173
353
  // -- exports --
174
354
 
@@ -176,6 +356,7 @@ void zstds_ext_dictionary_exports(VALUE root_module)
176
356
  {
177
357
  VALUE dictionary = rb_define_class_under(root_module, "Dictionary", rb_cObject);
178
358
 
359
+ rb_define_singleton_method(dictionary, "finalize_buffer", zstds_ext_finalize_dictionary_buffer, 3);
179
360
  rb_define_singleton_method(dictionary, "get_buffer_id", zstds_ext_get_dictionary_buffer_id, 1);
180
361
  rb_define_singleton_method(dictionary, "get_header_size", zstds_ext_get_dictionary_header_size, 1);
181
362
  rb_define_singleton_method(dictionary, "train_buffer", zstds_ext_train_dictionary_buffer, 2);
@@ -9,15 +9,29 @@
9
9
  #include "zstds_ext/macro.h"
10
10
 
11
11
  #define ZSTDS_EXT_DEFAULT_DICTIONARY_CAPACITY (1 << 17); // 128 KB
12
+ #define ZSTDS_EXT_DEFAULT_DICTIONARY_MAX_SIZE ZSTDS_EXT_DEFAULT_DICTIONARY_CAPACITY
13
+
14
+ // -- training --
12
15
 
13
16
  VALUE zstds_ext_train_dictionary_buffer(VALUE self, VALUE samples, VALUE options);
17
+
18
+ // -- finalizing --
19
+
20
+ #if defined(HAVE_ZDICT_FINALIZE)
21
+ VALUE zstds_ext_finalize_dictionary_buffer(VALUE self, VALUE content, VALUE samples, VALUE options);
22
+ #else
23
+ ZSTDS_EXT_NORETURN VALUE zstds_ext_finalize_dictionary_buffer(VALUE self, VALUE content, VALUE samples, VALUE options);
24
+ #endif // HAVE_ZDICT_FINALIZE
25
+
26
+ // -- other --
27
+
14
28
  VALUE zstds_ext_get_dictionary_buffer_id(VALUE self, VALUE buffer);
15
29
 
16
30
  #if defined(HAVE_ZDICT_HEADER_SIZE)
17
31
  VALUE zstds_ext_get_dictionary_header_size(VALUE self, VALUE buffer);
18
32
  #else
19
33
  ZSTDS_EXT_NORETURN VALUE zstds_ext_get_dictionary_header_size(VALUE self, VALUE buffer);
20
- #endif
34
+ #endif // HAVE_ZDICT_HEADER_SIZE
21
35
 
22
36
  void zstds_ext_dictionary_exports(VALUE root_module);
23
37
 
data/ext/zstds_ext/gvl.h CHANGED
@@ -19,6 +19,6 @@
19
19
 
20
20
  #define ZSTDS_EXT_GVL_WRAP(_gvl, function, data) function((void*) data);
21
21
 
22
- #endif
22
+ #endif // HAVE_RB_THREAD_CALL_WITHOUT_GVL
23
23
 
24
24
  #endif // ZSTDS_EXT_GVL_H
@@ -8,12 +8,12 @@
8
8
  #define ZSTDS_EXT_UNUSED(x) x __attribute__((__unused__))
9
9
  #else
10
10
  #define ZSTDS_EXT_UNUSED(x) x
11
- #endif
11
+ #endif // __GNUC__
12
12
 
13
13
  #if defined(__GNUC__)
14
14
  #define ZSTDS_EXT_NORETURN __attribute__((__noreturn__))
15
15
  #else
16
16
  #define ZSTDS_EXT_NORETURN
17
- #endif
17
+ #endif // __GNUC__
18
18
 
19
19
  #endif // ZSTDS_EXT_MACRO_H
@@ -68,6 +68,13 @@ typedef struct
68
68
  VALUE dictionary;
69
69
  } zstds_ext_decompressor_options_t;
70
70
 
71
+ typedef struct
72
+ {
73
+ zstds_ext_option_t compression_level;
74
+ zstds_ext_option_t notification_level;
75
+ zstds_ext_option_t dictionary_id;
76
+ } zstds_ext_dictionary_options_t;
77
+
71
78
  void zstds_ext_resolve_option(
72
79
  VALUE options,
73
80
  zstds_ext_option_t* option,
@@ -117,6 +124,13 @@ void zstds_ext_resolve_dictionary_option(VALUE options, VALUE* option, const cha
117
124
  ZSTDS_EXT_RESOLVE_OPTION(options, decompressor_options, ZSTDS_EXT_OPTION_TYPE_UINT, window_log_max); \
118
125
  ZSTDS_EXT_RESOLVE_DICTIONARY_OPTION(options, decompressor_options, dictionary);
119
126
 
127
+ #define ZSTDS_EXT_GET_DICTIONARY_OPTIONS(options) \
128
+ zstds_ext_dictionary_options_t dictionary_options; \
129
+ \
130
+ ZSTDS_EXT_RESOLVE_OPTION(options, dictionary_options, ZSTDS_EXT_OPTION_TYPE_INT, compression_level); \
131
+ ZSTDS_EXT_RESOLVE_OPTION(options, dictionary_options, ZSTDS_EXT_OPTION_TYPE_UINT, notification_level); \
132
+ ZSTDS_EXT_RESOLVE_OPTION(options, dictionary_options, ZSTDS_EXT_OPTION_TYPE_UINT, dictionary_id);
133
+
120
134
  bool zstds_ext_get_bool_option_value(VALUE options, const char* name);
121
135
  size_t zstds_ext_get_size_option_value(VALUE options, const char* name);
122
136
 
@@ -14,6 +14,20 @@ module ZSTDS
14
14
  }
15
15
  .freeze
16
16
 
17
+ FINALIZE_DEFAULTS = {
18
+ :gvl => false,
19
+ :max_size => 0,
20
+ :dictionary_options => {}
21
+ }
22
+ .freeze
23
+
24
+ FINALIZE_DICTIONARY_DEFAULTS = {
25
+ :compression_level => 0,
26
+ :notification_level => 0,
27
+ :dictionary_id => 0
28
+ }
29
+ .freeze
30
+
17
31
  attr_reader :buffer
18
32
 
19
33
  def initialize(buffer)
@@ -24,12 +38,7 @@ module ZSTDS
24
38
  end
25
39
 
26
40
  def self.train(samples, options = {})
27
- Validation.validate_array samples
28
-
29
- samples.each do |sample|
30
- Validation.validate_string sample
31
- raise ValidateError, "dictionary sample should not be empty" if sample.empty?
32
- end
41
+ validate_samples samples
33
42
 
34
43
  Validation.validate_hash options
35
44
 
@@ -42,6 +51,43 @@ module ZSTDS
42
51
  new buffer
43
52
  end
44
53
 
54
+ def self.finalize(content, samples, options = {})
55
+ Validation.validate_string content
56
+ raise ValidateError, "content should not be empty" if content.empty?
57
+
58
+ validate_samples samples
59
+
60
+ Validation.validate_hash options
61
+
62
+ options = FINALIZE_DEFAULTS.merge options
63
+
64
+ Validation.validate_bool options[:gvl]
65
+ Validation.validate_not_negative_integer options[:max_size]
66
+ Validation.validate_hash options[:dictionary_options]
67
+
68
+ dictionary_options = FINALIZE_DICTIONARY_DEFAULTS.merge options[:dictionary_options]
69
+
70
+ compression_level = dictionary_options[:compression_level]
71
+ Validation.validate_integer compression_level
72
+ raise ValidateError, "invalid compression level" if
73
+ compression_level < Option::MIN_COMPRESSION_LEVEL || compression_level > Option::MAX_COMPRESSION_LEVEL
74
+
75
+ Validation.validate_not_negative_integer dictionary_options[:notification_level]
76
+ Validation.validate_not_negative_integer dictionary_options[:dictionary_id]
77
+
78
+ buffer = finalize_buffer content, samples, options
79
+ new buffer
80
+ end
81
+
82
+ def self.validate_samples(samples)
83
+ Validation.validate_array samples
84
+
85
+ samples.each do |sample|
86
+ Validation.validate_string sample
87
+ raise ValidateError, "dictionary sample should not be empty" if sample.empty?
88
+ end
89
+ end
90
+
45
91
  def id
46
92
  self.class.get_buffer_id @buffer
47
93
  end
@@ -24,9 +24,7 @@ module ZSTDS
24
24
 
25
25
  def initialize(io, options = {})
26
26
  @raw_stream = create_raw_stream
27
-
28
- Validation.validate_io io
29
- @io = io
27
+ @io = io
30
28
 
31
29
  @stat = Stat.new @io.stat if @io.respond_to? :stat
32
30
 
@@ -135,13 +133,19 @@ module ZSTDS
135
133
  end
136
134
 
137
135
  def close
138
- @io.close
136
+ @io.close if @io.respond_to? :close
139
137
 
140
138
  nil
141
139
  end
142
140
 
143
141
  def closed?
144
- @raw_stream.closed? && @io.closed?
142
+ return false unless @raw_stream.closed?
143
+
144
+ if @io.respond_to? :closed
145
+ @io.closed?
146
+ else
147
+ true
148
+ end
145
149
  end
146
150
 
147
151
  def to_io
@@ -54,6 +54,9 @@ module ZSTDS
54
54
  Validation.validate_not_negative_integer bytes_to_read unless bytes_to_read.nil?
55
55
  Validation.validate_string out_buffer unless out_buffer.nil?
56
56
 
57
+ raise ValidateError, "io should be responsible to read and eof" unless
58
+ @io.respond_to?(:read) && @io.respond_to?(:eof?)
59
+
57
60
  unless bytes_to_read.nil?
58
61
  return ::String.new :encoding => ::Encoding::BINARY if bytes_to_read.zero?
59
62
  return nil if eof?
@@ -86,16 +89,22 @@ module ZSTDS
86
89
  end
87
90
 
88
91
  def eof?
92
+ raise ValidateError, "io should be responsible to eof" unless @io.respond_to? :eof?
93
+
89
94
  empty? && @io.eof?
90
95
  end
91
96
 
92
97
  # -- asynchronous --
93
98
 
94
99
  def readpartial(bytes_to_read, out_buffer = nil)
100
+ raise ValidateError, "io should be responsible to readpartial" unless @io.respond_to? :readpartial
101
+
95
102
  read_more_nonblock(bytes_to_read, out_buffer) { @io.readpartial @source_buffer_length }
96
103
  end
97
104
 
98
105
  def read_nonblock(bytes_to_read, out_buffer = nil, *options)
106
+ raise ValidateError, "io should be responsible to read nonblock" unless @io.respond_to? :read_nonblock
107
+
99
108
  read_more_nonblock(bytes_to_read, out_buffer) { @io.read_nonblock(@source_buffer_length, *options) }
100
109
  end
101
110
 
@@ -23,6 +23,8 @@ module ZSTDS
23
23
  # -- synchronous --
24
24
 
25
25
  def write(*objects)
26
+ validate_write
27
+
26
28
  write_remaining_buffer
27
29
 
28
30
  bytes_written = 0
@@ -38,20 +40,26 @@ module ZSTDS
38
40
  end
39
41
 
40
42
  def flush
43
+ validate_write
44
+
41
45
  finish :flush
42
46
 
43
- @io.flush
47
+ @io.flush if @io.respond_to? :flush
44
48
 
45
49
  self
46
50
  end
47
51
 
48
52
  def rewind
53
+ validate_write
54
+
49
55
  finish :close
50
56
 
51
57
  super
52
58
  end
53
59
 
54
60
  def close
61
+ validate_write
62
+
55
63
  finish :close
56
64
 
57
65
  super
@@ -75,6 +83,10 @@ module ZSTDS
75
83
  @raw_stream.send(method_name, *args) { |portion| @io.write portion }
76
84
  end
77
85
 
86
+ def validate_write
87
+ raise ValidateError, "io should be responsible to write" unless @io.respond_to? :write
88
+ end
89
+
78
90
  # -- asynchronous --
79
91
 
80
92
  # IO write nonblock can raise wait writable error.
@@ -83,6 +95,8 @@ module ZSTDS
83
95
  # So we have to accept content after processing IO write nonblock.
84
96
  # It means that first write nonblock won't call IO write nonblock.
85
97
  def write_nonblock(object, *options)
98
+ validate_write_nonblock
99
+
86
100
  return 0 unless write_remaining_buffer_nonblock(*options)
87
101
 
88
102
  source = transcode object.to_s
@@ -93,14 +107,18 @@ module ZSTDS
93
107
  end
94
108
 
95
109
  def flush_nonblock(*options)
110
+ validate_write_nonblock
111
+
96
112
  return false unless finish_nonblock :flush, *options
97
113
 
98
- @io.flush
114
+ @io.flush if @io.respond_to? :flush
99
115
 
100
116
  true
101
117
  end
102
118
 
103
119
  def rewind_nonblock(*options)
120
+ validate_write_nonblock
121
+
104
122
  return false unless finish_nonblock :close, *options
105
123
 
106
124
  method(:rewind).super_method.call
@@ -109,6 +127,8 @@ module ZSTDS
109
127
  end
110
128
 
111
129
  def close_nonblock(*options)
130
+ validate_write_nonblock
131
+
112
132
  return false unless finish_nonblock :close, *options
113
133
 
114
134
  method(:close).super_method.call
@@ -139,6 +159,10 @@ module ZSTDS
139
159
  @raw_stream.send(method_name, *args) { |portion| @buffer << portion }
140
160
  end
141
161
 
162
+ def validate_write_nonblock
163
+ raise ValidateError, "io should be responsible to write nonblock" unless @io.respond_to? :write_nonblock
164
+ end
165
+
142
166
  # -- common --
143
167
 
144
168
  protected def transcode(data)
@@ -5,33 +5,34 @@ require_relative "error"
5
5
 
6
6
  module ZSTDS
7
7
  module Validation
8
- IO_METHODS = %i[
9
- read
10
- write
11
- readpartial
12
- read_nonblock
13
- write_nonblock
14
- eof?
15
- flush
16
- close
17
- closed?
18
- ]
19
- .freeze
8
+ def self.validate_array(value)
9
+ raise ValidateError, "invalid array" unless value.is_a? ::Array
10
+ end
20
11
 
21
12
  def self.validate_bool(value)
22
13
  raise ValidateError, "invalid bool" unless value.is_a?(::TrueClass) || value.is_a?(::FalseClass)
23
14
  end
24
15
 
16
+ def self.validate_hash(value)
17
+ raise ValidateError, "invalid hash" unless value.is_a? ::Hash
18
+ end
19
+
25
20
  def self.validate_integer(value)
26
21
  raise ValidateError, "invalid integer" unless value.is_a? ::Integer
27
22
  end
28
23
 
24
+ def self.validate_not_negative_integer(value)
25
+ raise ValidateError, "invalid not negative integer" unless value.is_a?(::Integer) && value >= 0
26
+ end
27
+
29
28
  def self.validate_positive_integer(value)
30
29
  raise ValidateError, "invalid positive integer" unless value.is_a?(::Integer) && value.positive?
31
30
  end
32
31
 
33
- def self.validate_not_negative_integer(value)
34
- raise ValidateError, "invalid not negative integer" unless value.is_a?(::Integer) && value >= 0
32
+ def self.validate_proc(value)
33
+ unless value.is_a?(::Proc) || value.is_a?(::Method) || value.is_a?(::UnboundMethod)
34
+ raise ValidateError, "invalid proc"
35
+ end
35
36
  end
36
37
 
37
38
  def self.validate_string(value)
@@ -41,23 +42,5 @@ module ZSTDS
41
42
  def self.validate_symbol(value)
42
43
  raise ValidateError, "invalid symbol" unless value.is_a? ::Symbol
43
44
  end
44
-
45
- def self.validate_io(value)
46
- raise ValidateError, "invalid io" unless IO_METHODS.all? { |method| value.respond_to? method }
47
- end
48
-
49
- def self.validate_array(value)
50
- raise ValidateError, "invalid array" unless value.is_a? ::Array
51
- end
52
-
53
- def self.validate_hash(value)
54
- raise ValidateError, "invalid hash" unless value.is_a? ::Hash
55
- end
56
-
57
- def self.validate_proc(value)
58
- unless value.is_a?(::Proc) || value.is_a?(::Method) || value.is_a?(::UnboundMethod)
59
- raise ValidateError, "invalid proc"
60
- end
61
- end
62
45
  end
63
46
  end
data/lib/zstds/version.rb CHANGED
@@ -2,5 +2,5 @@
2
2
  # Copyright (c) 2019 AUTHORS, MIT License.
3
3
 
4
4
  module ZSTDS
5
- VERSION = "1.2.1".freeze
5
+ VERSION = "1.3.0".freeze
6
6
  end
metadata CHANGED
@@ -1,14 +1,16 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ruby-zstds
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.1
4
+ version: 1.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Andrew Aladjev
8
+ - Ivan Takarlikov
9
+ - Jenner La Fave
8
10
  autorequire:
9
11
  bindir: bin
10
12
  cert_chain: []
11
- date: 2022-01-07 00:00:00.000000000 Z
13
+ date: 2022-03-18 00:00:00.000000000 Z
12
14
  dependencies:
13
15
  - !ruby/object:Gem::Dependency
14
16
  name: codecov
@@ -128,14 +130,14 @@ dependencies:
128
130
  requirements:
129
131
  - - "~>"
130
132
  - !ruby/object:Gem::Version
131
- version: '1.24'
133
+ version: '1.26'
132
134
  type: :development
133
135
  prerelease: false
134
136
  version_requirements: !ruby/object:Gem::Requirement
135
137
  requirements:
136
138
  - - "~>"
137
139
  - !ruby/object:Gem::Version
138
- version: '1.24'
140
+ version: '1.26'
139
141
  - !ruby/object:Gem::Dependency
140
142
  name: rubocop-minitest
141
143
  requirement: !ruby/object:Gem::Requirement
@@ -261,7 +263,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
261
263
  - !ruby/object:Gem::Version
262
264
  version: '0'
263
265
  requirements: []
264
- rubygems_version: 3.3.3
266
+ rubygems_version: 3.3.7
265
267
  signing_key:
266
268
  specification_version: 4
267
269
  summary: Ruby bindings for zstd library.