ruby-brs 1.1.5 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +32 -19
- data/ext/brs_ext/buffer.c +21 -7
- data/ext/brs_ext/buffer.h +7 -7
- data/ext/brs_ext/error.h +2 -1
- data/ext/brs_ext/gvl.h +24 -0
- data/ext/brs_ext/io.c +240 -122
- data/ext/brs_ext/option.c +37 -27
- data/ext/brs_ext/option.h +37 -29
- data/ext/brs_ext/stream/compressor.c +118 -52
- data/ext/brs_ext/stream/compressor.h +4 -1
- data/ext/brs_ext/stream/decompressor.c +54 -22
- data/ext/brs_ext/stream/decompressor.h +4 -1
- data/ext/brs_ext/string.c +118 -64
- data/ext/extconf.rb +2 -0
- data/lib/brs/option.rb +6 -0
- data/lib/brs/version.rb +1 -1
- metadata +23 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 239de67878a08e54f6b89af1381711ad94c70c1f29d99c8dca22ced06ebd850a
|
4
|
+
data.tar.gz: 2601159e2ede572b8bcdc8ce0ee7ca738f06f42cb0d6e59a159eb525e2f70420
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d12cfbf9280a63ad850ce4198e1bdaa044afc4502bccdeadeef8af746f1806182fb560bd580dd0bafafeaff93fd8e2e40435d7ac12079bad89a79b9cb22ef088
|
7
|
+
data.tar.gz: 13cb28b02d174fe3c448f291b85a80625cabb3f07ab32f5a5233a244425299ed04f248737be934d9f3171c6bc93c31062cbb1e7daf9f146e46a617163298a678
|
data/README.md
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
# Ruby bindings for brotli library
|
2
2
|
|
3
|
-
| Travis | AppVeyor | Circle | Codecov |
|
4
|
-
| :---: | :---: | :---: | :---: |
|
5
|
-
| [](https://travis-ci.com/andrew-aladev/ruby-brs) | [](https://ci.appveyor.com/project/andrew-aladev/ruby-brs/branch/master) | [](https://circleci.com/gh/andrew-aladev/ruby-brs/tree/master) | [](https://codecov.io/gh/andrew-aladev/ruby-brs) |
|
3
|
+
| Travis | AppVeyor | Circle | Codecov | Gem |
|
4
|
+
| :---: | :---: | :---: | :---: | :---: |
|
5
|
+
| [](https://travis-ci.com/andrew-aladev/ruby-brs) | [](https://ci.appveyor.com/project/andrew-aladev/ruby-brs/branch/master) | [](https://circleci.com/gh/andrew-aladev/ruby-brs/tree/master) | [](https://codecov.io/gh/andrew-aladev/ruby-brs) | [](https://rubygems.org/gems/ruby-brs) |
|
6
6
|
|
7
7
|
See [brotli library](https://github.com/google/brotli).
|
8
8
|
|
@@ -61,7 +61,7 @@ ensure
|
|
61
61
|
end
|
62
62
|
```
|
63
63
|
|
64
|
-
You can create and read `tar.br` archives with
|
64
|
+
You can create and read `tar.br` archives with [minitar](https://github.com/halostatue/minitar) for example.
|
65
65
|
|
66
66
|
```ruby
|
67
67
|
require "brs"
|
@@ -83,12 +83,25 @@ BRS::Stream::Reader.open "file.tar.br" do |reader|
|
|
83
83
|
end
|
84
84
|
```
|
85
85
|
|
86
|
+
You can also use `Content-Encoding: br` with [sinatra](http://sinatrarb.com):
|
87
|
+
|
88
|
+
```ruby
|
89
|
+
require "brs"
|
90
|
+
require "sinatra"
|
91
|
+
|
92
|
+
get "/" do
|
93
|
+
headers["Content-Encoding"] = "br"
|
94
|
+
BRS::String.compress "sample string"
|
95
|
+
end
|
96
|
+
```
|
97
|
+
|
86
98
|
## Options
|
87
99
|
|
88
100
|
| Option | Values | Default | Description |
|
89
101
|
|------------------------------------|------------|------------|-------------|
|
90
102
|
| `source_buffer_length` | 0 - inf | 0 (auto) | internal buffer length for source data |
|
91
103
|
| `destination_buffer_length` | 0 - inf | 0 (auto) | internal buffer length for description data |
|
104
|
+
| `gvl` | true/false | false | enables global VM lock where possible |
|
92
105
|
| `mode` | `MODES` | `:generic` | compressor mode |
|
93
106
|
| `quality` | 0 - 11 | 11 | compression level |
|
94
107
|
| `lgwin` | 10 - 24 | 22 | compressor window size |
|
@@ -102,6 +115,10 @@ There are internal buffers for compressed and decompressed data.
|
|
102
115
|
For example you want to use 1 KB as `source_buffer_length` for compressor - please use 256 B as `destination_buffer_length`.
|
103
116
|
You want to use 256 B as `source_buffer_length` for decompressor - please use 1 KB as `destination_buffer_length`.
|
104
117
|
|
118
|
+
`gvl` is disabled by default, this mode allows running multiple compressors/decompressors in different threads simultaneously.
|
119
|
+
Please consider enabling `gvl` if you don't want to launch processors in separate threads.
|
120
|
+
If `gvl` is enabled ruby won't waste time on acquiring/releasing VM lock.
|
121
|
+
|
105
122
|
`String` and `File` will set `:size_hint` automaticaly.
|
106
123
|
|
107
124
|
You can also read brotli docs for more info about options.
|
@@ -117,6 +134,7 @@ Possible compressor options:
|
|
117
134
|
```
|
118
135
|
:source_buffer_length
|
119
136
|
:destination_buffer_length
|
137
|
+
:gvl
|
120
138
|
:mode
|
121
139
|
:quality
|
122
140
|
:lgwin
|
@@ -130,6 +148,7 @@ Possible decompressor options:
|
|
130
148
|
```
|
131
149
|
:source_buffer_length
|
132
150
|
:destination_buffer_length
|
151
|
+
:gvl
|
133
152
|
:disable_ring_buffer_reallocation
|
134
153
|
:large_window
|
135
154
|
```
|
@@ -143,18 +162,6 @@ data = BRS::String.compress "sample string", :quality => 5
|
|
143
162
|
puts BRS::String.decompress(data, :disable_ring_buffer_reallocation => true)
|
144
163
|
```
|
145
164
|
|
146
|
-
HTTP encoding (`Content-Encoding: br`) using default options:
|
147
|
-
|
148
|
-
```ruby
|
149
|
-
require "brs"
|
150
|
-
require "sinatra"
|
151
|
-
|
152
|
-
get "/" do
|
153
|
-
headers["Content-Encoding"] = "br"
|
154
|
-
BRS::String.compress "sample string"
|
155
|
-
end
|
156
|
-
```
|
157
|
-
|
158
165
|
## String
|
159
166
|
|
160
167
|
String maintains destination buffer only, so it accepts `destination_buffer_length` option only.
|
@@ -330,12 +337,18 @@ See [`IO`](https://ruby-doc.org/core-2.7.0/IO.html) docs.
|
|
330
337
|
|
331
338
|
Typical helpers, see [`Zlib::GzipReader`](https://ruby-doc.org/stdlib-2.7.0/libdoc/zlib/rdoc/Zlib/GzipReader.html) docs.
|
332
339
|
|
340
|
+
## Thread safety
|
341
|
+
|
342
|
+
`:gvl` option is disabled by default, you can use bindings effectively in multiple threads.
|
343
|
+
Please be careful: bindings are not thread safe.
|
344
|
+
You should lock all shared data between threads.
|
345
|
+
|
333
346
|
## CI
|
334
347
|
|
335
|
-
See universal test script [scripts/ci_test.sh](scripts/ci_test.sh) for CI.
|
336
348
|
Please visit [scripts/test-images](scripts/test-images).
|
337
|
-
|
349
|
+
See universal test script [scripts/ci_test.sh](scripts/ci_test.sh) for CI.
|
350
|
+
You can run this script using many native and cross images.
|
338
351
|
|
339
352
|
## License
|
340
353
|
|
341
|
-
MIT license, see LICENSE and AUTHORS.
|
354
|
+
MIT license, see [LICENSE](LICENSE) and [AUTHORS](AUTHORS).
|
data/ext/brs_ext/buffer.c
CHANGED
@@ -10,10 +10,11 @@ VALUE brs_ext_create_string_buffer(VALUE length)
|
|
10
10
|
return rb_str_new(NULL, NUM2SIZET(length));
|
11
11
|
}
|
12
12
|
|
13
|
-
VALUE brs_ext_resize_string_buffer(VALUE
|
13
|
+
VALUE brs_ext_resize_string_buffer(VALUE buffer_args)
|
14
14
|
{
|
15
|
-
VALUE buffer = rb_ary_entry(
|
16
|
-
VALUE length = rb_ary_entry(
|
15
|
+
VALUE buffer = rb_ary_entry(buffer_args, 0);
|
16
|
+
VALUE length = rb_ary_entry(buffer_args, 1);
|
17
|
+
|
17
18
|
return rb_str_resize(buffer, NUM2SIZET(length));
|
18
19
|
}
|
19
20
|
|
@@ -21,8 +22,21 @@ void brs_ext_buffer_exports(VALUE root_module)
|
|
21
22
|
{
|
22
23
|
VALUE module = rb_define_module_under(root_module, "Buffer");
|
23
24
|
|
24
|
-
rb_define_const(
|
25
|
-
|
26
|
-
|
27
|
-
rb_define_const(
|
25
|
+
rb_define_const(
|
26
|
+
module, "DEFAULT_SOURCE_BUFFER_LENGTH_FOR_COMPRESSOR", SIZET2NUM(BRS_DEFAULT_SOURCE_BUFFER_LENGTH_FOR_COMPRESSOR));
|
27
|
+
|
28
|
+
rb_define_const(
|
29
|
+
module,
|
30
|
+
"DEFAULT_DESTINATION_BUFFER_LENGTH_FOR_COMPRESSOR",
|
31
|
+
SIZET2NUM(BRS_DEFAULT_DESTINATION_BUFFER_LENGTH_FOR_COMPRESSOR));
|
32
|
+
|
33
|
+
rb_define_const(
|
34
|
+
module,
|
35
|
+
"DEFAULT_SOURCE_BUFFER_LENGTH_FOR_DECOMPRESSOR",
|
36
|
+
SIZET2NUM(BRS_DEFAULT_SOURCE_BUFFER_LENGTH_FOR_DECOMPRESSOR));
|
37
|
+
|
38
|
+
rb_define_const(
|
39
|
+
module,
|
40
|
+
"DEFAULT_DESTINATION_BUFFER_LENGTH_FOR_DECOMPRESSOR",
|
41
|
+
SIZET2NUM(BRS_DEFAULT_DESTINATION_BUFFER_LENGTH_FOR_DECOMPRESSOR));
|
28
42
|
}
|
data/ext/brs_ext/buffer.h
CHANGED
@@ -6,10 +6,10 @@
|
|
6
6
|
|
7
7
|
#include "ruby.h"
|
8
8
|
|
9
|
-
#define BRS_DEFAULT_SOURCE_BUFFER_LENGTH_FOR_COMPRESSOR
|
9
|
+
#define BRS_DEFAULT_SOURCE_BUFFER_LENGTH_FOR_COMPRESSOR (1 << 18) // 256 KB
|
10
10
|
#define BRS_DEFAULT_DESTINATION_BUFFER_LENGTH_FOR_COMPRESSOR (1 << 16) // 64 KB
|
11
11
|
|
12
|
-
#define BRS_DEFAULT_SOURCE_BUFFER_LENGTH_FOR_DECOMPRESSOR
|
12
|
+
#define BRS_DEFAULT_SOURCE_BUFFER_LENGTH_FOR_DECOMPRESSOR (1 << 16) // 64 KB
|
13
13
|
#define BRS_DEFAULT_DESTINATION_BUFFER_LENGTH_FOR_DECOMPRESSOR (1 << 18) // 256 KB
|
14
14
|
|
15
15
|
VALUE brs_ext_create_string_buffer(VALUE length);
|
@@ -17,12 +17,12 @@ VALUE brs_ext_create_string_buffer(VALUE length);
|
|
17
17
|
#define BRS_EXT_CREATE_STRING_BUFFER(buffer, length, exception) \
|
18
18
|
VALUE buffer = rb_protect(brs_ext_create_string_buffer, SIZET2NUM(length), &exception);
|
19
19
|
|
20
|
-
VALUE brs_ext_resize_string_buffer(VALUE
|
20
|
+
VALUE brs_ext_resize_string_buffer(VALUE buffer_args);
|
21
21
|
|
22
|
-
#define BRS_EXT_RESIZE_STRING_BUFFER(buffer, length, exception)
|
23
|
-
VALUE
|
24
|
-
buffer
|
25
|
-
RB_GC_GUARD(
|
22
|
+
#define BRS_EXT_RESIZE_STRING_BUFFER(buffer, length, exception) \
|
23
|
+
VALUE buffer_args = rb_ary_new_from_args(2, buffer, SIZET2NUM(length)); \
|
24
|
+
buffer = rb_protect(brs_ext_resize_string_buffer, buffer_args, &exception); \
|
25
|
+
RB_GC_GUARD(buffer_args);
|
26
26
|
|
27
27
|
void brs_ext_buffer_exports(VALUE root_module);
|
28
28
|
|
data/ext/brs_ext/error.h
CHANGED
data/ext/brs_ext/gvl.h
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
// Ruby bindings for brotli library.
|
2
|
+
// Copyright (c) 2019 AUTHORS, MIT License.
|
3
|
+
|
4
|
+
#if !defined(BRS_EXT_GVL_H)
|
5
|
+
#define BRS_EXT_GVL_H
|
6
|
+
|
7
|
+
#ifdef HAVE_RB_THREAD_CALL_WITHOUT_GVL
|
8
|
+
|
9
|
+
#include "ruby/thread.h"
|
10
|
+
|
11
|
+
#define BRS_EXT_GVL_WRAP(gvl, function, data) \
|
12
|
+
if (gvl) { \
|
13
|
+
function((void*) data); \
|
14
|
+
} else { \
|
15
|
+
rb_thread_call_without_gvl(function, (void*) data, RUBY_UBF_IO, NULL); \
|
16
|
+
}
|
17
|
+
|
18
|
+
#else
|
19
|
+
|
20
|
+
#define BRS_EXT_GVL_WRAP(_gvl, function, data) function((void*) data);
|
21
|
+
|
22
|
+
#endif
|
23
|
+
|
24
|
+
#endif // BRS_EXT_GVL_H
|
data/ext/brs_ext/io.c
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
// Ruby bindings for brotli library.
|
2
2
|
// Copyright (c) 2019 AUTHORS, MIT License.
|
3
3
|
|
4
|
-
#include "
|
4
|
+
#include "brs_ext/io.h"
|
5
5
|
|
6
6
|
#include <brotli/decode.h>
|
7
7
|
#include <brotli/encode.h>
|
@@ -12,19 +12,22 @@
|
|
12
12
|
|
13
13
|
#include "brs_ext/buffer.h"
|
14
14
|
#include "brs_ext/error.h"
|
15
|
-
#include "brs_ext/
|
15
|
+
#include "brs_ext/gvl.h"
|
16
16
|
#include "brs_ext/macro.h"
|
17
17
|
#include "brs_ext/option.h"
|
18
18
|
#include "ruby.h"
|
19
|
+
#include "ruby/io.h"
|
19
20
|
|
20
21
|
// Additional possible results:
|
21
|
-
enum
|
22
|
+
enum
|
23
|
+
{
|
22
24
|
BRS_EXT_FILE_READ_FINISHED = 128
|
23
25
|
};
|
24
26
|
|
25
27
|
// -- file --
|
26
28
|
|
27
|
-
static inline brs_ext_result_t
|
29
|
+
static inline brs_ext_result_t
|
30
|
+
read_file(FILE* source_file, brs_ext_byte_t* source_buffer, size_t* source_length_ptr, size_t source_buffer_length)
|
28
31
|
{
|
29
32
|
size_t read_length = fread(source_buffer, 1, source_buffer_length, source_file);
|
30
33
|
if (read_length == 0 && feof(source_file)) {
|
@@ -40,7 +43,8 @@ static inline brs_ext_result_t read_file(FILE* source_file, brs_ext_byte_t* sour
|
|
40
43
|
return 0;
|
41
44
|
}
|
42
45
|
|
43
|
-
static inline brs_ext_result_t
|
46
|
+
static inline brs_ext_result_t
|
47
|
+
write_file(FILE* destination_file, brs_ext_byte_t* destination_buffer, size_t destination_length)
|
44
48
|
{
|
45
49
|
size_t written_length = fwrite(destination_buffer, 1, destination_length, destination_file);
|
46
50
|
if (written_length != destination_length) {
|
@@ -53,8 +57,10 @@ static inline brs_ext_result_t write_file(FILE* destination_file, brs_ext_byte_t
|
|
53
57
|
// -- buffer --
|
54
58
|
|
55
59
|
static inline brs_ext_result_t create_buffers(
|
56
|
-
brs_ext_byte_t** source_buffer_ptr,
|
57
|
-
|
60
|
+
brs_ext_byte_t** source_buffer_ptr,
|
61
|
+
size_t source_buffer_length,
|
62
|
+
brs_ext_byte_t** destination_buffer_ptr,
|
63
|
+
size_t destination_buffer_length)
|
58
64
|
{
|
59
65
|
brs_ext_byte_t* source_buffer = malloc(source_buffer_length);
|
60
66
|
if (source_buffer == NULL) {
|
@@ -81,8 +87,10 @@ static inline brs_ext_result_t create_buffers(
|
|
81
87
|
|
82
88
|
static inline brs_ext_result_t read_more_source(
|
83
89
|
FILE* source_file,
|
84
|
-
const brs_ext_byte_t** source_ptr,
|
85
|
-
|
90
|
+
const brs_ext_byte_t** source_ptr,
|
91
|
+
size_t* source_length_ptr,
|
92
|
+
brs_ext_byte_t* source_buffer,
|
93
|
+
size_t source_buffer_length)
|
86
94
|
{
|
87
95
|
const brs_ext_byte_t* source = *source_ptr;
|
88
96
|
size_t source_length = *source_length_ptr;
|
@@ -105,7 +113,9 @@ static inline brs_ext_result_t read_more_source(
|
|
105
113
|
brs_ext_byte_t* remaining_source_buffer = source_buffer + source_length;
|
106
114
|
size_t new_source_length;
|
107
115
|
|
108
|
-
brs_ext_result_t ext_result =
|
116
|
+
brs_ext_result_t ext_result =
|
117
|
+
read_file(source_file, remaining_source_buffer, &new_source_length, remaining_source_buffer_length);
|
118
|
+
|
109
119
|
if (ext_result != 0) {
|
110
120
|
return ext_result;
|
111
121
|
}
|
@@ -115,42 +125,37 @@ static inline brs_ext_result_t read_more_source(
|
|
115
125
|
return 0;
|
116
126
|
}
|
117
127
|
|
118
|
-
#define BUFFERED_READ_SOURCE(function, ...)
|
119
|
-
do {
|
120
|
-
bool is_function_called = false;
|
121
|
-
|
122
|
-
while (true) {
|
123
|
-
ext_result = read_more_source(
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
return ext_result;
|
137
|
-
}
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
ext_result = function(__VA_ARGS__); \
|
150
|
-
if (ext_result != 0) { \
|
151
|
-
return ext_result; \
|
152
|
-
} \
|
153
|
-
} \
|
128
|
+
#define BUFFERED_READ_SOURCE(function, ...) \
|
129
|
+
do { \
|
130
|
+
bool is_function_called = false; \
|
131
|
+
\
|
132
|
+
while (true) { \
|
133
|
+
ext_result = read_more_source(source_file, &source, &source_length, source_buffer, source_buffer_length); \
|
134
|
+
if (ext_result == BRS_EXT_FILE_READ_FINISHED) { \
|
135
|
+
if (source_length != 0) { \
|
136
|
+
/* Brotli won't provide any remainder by design. */ \
|
137
|
+
return BRS_EXT_ERROR_READ_IO; \
|
138
|
+
} \
|
139
|
+
break; \
|
140
|
+
} else if (ext_result != 0) { \
|
141
|
+
return ext_result; \
|
142
|
+
} \
|
143
|
+
\
|
144
|
+
ext_result = function(__VA_ARGS__); \
|
145
|
+
if (ext_result != 0) { \
|
146
|
+
return ext_result; \
|
147
|
+
} \
|
148
|
+
\
|
149
|
+
is_function_called = true; \
|
150
|
+
} \
|
151
|
+
\
|
152
|
+
if (!is_function_called) { \
|
153
|
+
/* Function should be called at least once. */ \
|
154
|
+
ext_result = function(__VA_ARGS__); \
|
155
|
+
if (ext_result != 0) { \
|
156
|
+
return ext_result; \
|
157
|
+
} \
|
158
|
+
} \
|
154
159
|
} while (false);
|
155
160
|
|
156
161
|
// Algorithm has written data into destination buffer.
|
@@ -159,7 +164,9 @@ static inline brs_ext_result_t read_more_source(
|
|
159
164
|
|
160
165
|
static inline brs_ext_result_t flush_destination_buffer(
|
161
166
|
FILE* destination_file,
|
162
|
-
brs_ext_byte_t* destination_buffer,
|
167
|
+
brs_ext_byte_t* destination_buffer,
|
168
|
+
size_t* destination_length_ptr,
|
169
|
+
size_t destination_buffer_length)
|
163
170
|
{
|
164
171
|
if (*destination_length_ptr == 0) {
|
165
172
|
// We want to write more data at once, than buffer has.
|
@@ -176,7 +183,8 @@ static inline brs_ext_result_t flush_destination_buffer(
|
|
176
183
|
return 0;
|
177
184
|
}
|
178
185
|
|
179
|
-
static inline brs_ext_result_t
|
186
|
+
static inline brs_ext_result_t
|
187
|
+
write_remaining_destination(FILE* destination_file, brs_ext_byte_t* destination_buffer, size_t destination_length)
|
180
188
|
{
|
181
189
|
if (destination_length == 0) {
|
182
190
|
return 0;
|
@@ -198,29 +206,57 @@ static inline brs_ext_result_t write_remaining_destination(FILE* destination_fil
|
|
198
206
|
brs_ext_raise_error(BRS_EXT_ERROR_ACCESS_IO); \
|
199
207
|
}
|
200
208
|
|
201
|
-
// -- compress --
|
209
|
+
// -- buffered compress --
|
210
|
+
|
211
|
+
typedef struct
|
212
|
+
{
|
213
|
+
BrotliEncoderState* state_ptr;
|
214
|
+
const brs_ext_byte_t** source_ptr;
|
215
|
+
size_t* source_length_ptr;
|
216
|
+
brs_ext_byte_t* remaining_destination_buffer;
|
217
|
+
size_t* remaining_destination_buffer_length_ptr;
|
218
|
+
BROTLI_BOOL result;
|
219
|
+
} compress_args_t;
|
220
|
+
|
221
|
+
static inline void* compress_wrapper(void* data)
|
222
|
+
{
|
223
|
+
compress_args_t* args = data;
|
224
|
+
|
225
|
+
args->result = BrotliEncoderCompressStream(
|
226
|
+
args->state_ptr,
|
227
|
+
BROTLI_OPERATION_PROCESS,
|
228
|
+
args->source_length_ptr,
|
229
|
+
args->source_ptr,
|
230
|
+
args->remaining_destination_buffer_length_ptr,
|
231
|
+
&args->remaining_destination_buffer,
|
232
|
+
NULL);
|
233
|
+
|
234
|
+
return NULL;
|
235
|
+
}
|
202
236
|
|
203
237
|
static inline brs_ext_result_t buffered_compress(
|
204
238
|
BrotliEncoderState* state_ptr,
|
205
|
-
const brs_ext_byte_t** source_ptr,
|
206
|
-
|
239
|
+
const brs_ext_byte_t** source_ptr,
|
240
|
+
size_t* source_length_ptr,
|
241
|
+
FILE* destination_file,
|
242
|
+
brs_ext_byte_t* destination_buffer,
|
243
|
+
size_t* destination_length_ptr,
|
244
|
+
size_t destination_buffer_length,
|
245
|
+
bool gvl)
|
207
246
|
{
|
208
|
-
BROTLI_BOOL result;
|
209
247
|
brs_ext_result_t ext_result;
|
248
|
+
compress_args_t args = {.state_ptr = state_ptr, .source_ptr = source_ptr, .source_length_ptr = source_length_ptr};
|
210
249
|
|
211
250
|
while (true) {
|
212
251
|
brs_ext_byte_t* remaining_destination_buffer = destination_buffer + *destination_length_ptr;
|
213
252
|
size_t remaining_destination_buffer_length = destination_buffer_length - *destination_length_ptr;
|
214
253
|
size_t prev_remaining_destination_buffer_length = remaining_destination_buffer_length;
|
215
254
|
|
216
|
-
|
217
|
-
|
218
|
-
BROTLI_OPERATION_PROCESS,
|
219
|
-
source_length_ptr, source_ptr,
|
220
|
-
&remaining_destination_buffer_length, &remaining_destination_buffer,
|
221
|
-
NULL);
|
255
|
+
args.remaining_destination_buffer = remaining_destination_buffer;
|
256
|
+
args.remaining_destination_buffer_length_ptr = &remaining_destination_buffer_length;
|
222
257
|
|
223
|
-
|
258
|
+
BRS_EXT_GVL_WRAP(gvl, compress_wrapper, &args);
|
259
|
+
if (!args.result) {
|
224
260
|
return BRS_EXT_ERROR_UNEXPECTED;
|
225
261
|
}
|
226
262
|
|
@@ -228,8 +264,7 @@ static inline brs_ext_result_t buffered_compress(
|
|
228
264
|
|
229
265
|
if (BrotliEncoderHasMoreOutput(state_ptr)) {
|
230
266
|
ext_result = flush_destination_buffer(
|
231
|
-
destination_file,
|
232
|
-
destination_buffer, destination_length_ptr, destination_buffer_length);
|
267
|
+
destination_file, destination_buffer, destination_length_ptr, destination_buffer_length);
|
233
268
|
|
234
269
|
if (ext_result != 0) {
|
235
270
|
return ext_result;
|
@@ -244,29 +279,57 @@ static inline brs_ext_result_t buffered_compress(
|
|
244
279
|
return 0;
|
245
280
|
}
|
246
281
|
|
282
|
+
// -- buffered compressor finish --
|
283
|
+
|
284
|
+
typedef struct
|
285
|
+
{
|
286
|
+
BrotliEncoderState* state_ptr;
|
287
|
+
const brs_ext_byte_t** source_ptr;
|
288
|
+
size_t* source_length_ptr;
|
289
|
+
brs_ext_byte_t* remaining_destination_buffer;
|
290
|
+
size_t* remaining_destination_buffer_length_ptr;
|
291
|
+
BROTLI_BOOL result;
|
292
|
+
} compressor_finish_args_t;
|
293
|
+
|
294
|
+
static inline void* compressor_finish_wrapper(void* data)
|
295
|
+
{
|
296
|
+
compressor_finish_args_t* args = data;
|
297
|
+
|
298
|
+
args->result = BrotliEncoderCompressStream(
|
299
|
+
args->state_ptr,
|
300
|
+
BROTLI_OPERATION_FINISH,
|
301
|
+
args->source_length_ptr,
|
302
|
+
args->source_ptr,
|
303
|
+
args->remaining_destination_buffer_length_ptr,
|
304
|
+
&args->remaining_destination_buffer,
|
305
|
+
NULL);
|
306
|
+
|
307
|
+
return NULL;
|
308
|
+
}
|
309
|
+
|
247
310
|
static inline brs_ext_result_t buffered_compressor_finish(
|
248
311
|
BrotliEncoderState* state_ptr,
|
249
|
-
FILE*
|
312
|
+
FILE* destination_file,
|
313
|
+
brs_ext_byte_t* destination_buffer,
|
314
|
+
size_t* destination_length_ptr,
|
315
|
+
size_t destination_buffer_length,
|
316
|
+
bool gvl)
|
250
317
|
{
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
size_t source_length = 0;
|
318
|
+
brs_ext_result_t ext_result;
|
319
|
+
const brs_ext_byte_t* source = NULL;
|
320
|
+
size_t source_length = 0;
|
321
|
+
compressor_finish_args_t args = {.state_ptr = state_ptr, .source_ptr = &source, .source_length_ptr = &source_length};
|
256
322
|
|
257
323
|
while (true) {
|
258
324
|
brs_ext_byte_t* remaining_destination_buffer = destination_buffer + *destination_length_ptr;
|
259
325
|
size_t remaining_destination_buffer_length = destination_buffer_length - *destination_length_ptr;
|
260
326
|
size_t prev_remaining_destination_buffer_length = remaining_destination_buffer_length;
|
261
327
|
|
262
|
-
|
263
|
-
|
264
|
-
BROTLI_OPERATION_FINISH,
|
265
|
-
&source_length, &source,
|
266
|
-
&remaining_destination_buffer_length, &remaining_destination_buffer,
|
267
|
-
NULL);
|
328
|
+
args.remaining_destination_buffer = remaining_destination_buffer;
|
329
|
+
args.remaining_destination_buffer_length_ptr = &remaining_destination_buffer_length;
|
268
330
|
|
269
|
-
|
331
|
+
BRS_EXT_GVL_WRAP(gvl, compressor_finish_wrapper, &args);
|
332
|
+
if (!args.result) {
|
270
333
|
return BRS_EXT_ERROR_UNEXPECTED;
|
271
334
|
}
|
272
335
|
|
@@ -274,8 +337,7 @@ static inline brs_ext_result_t buffered_compressor_finish(
|
|
274
337
|
|
275
338
|
if (BrotliEncoderHasMoreOutput(state_ptr) || !BrotliEncoderIsFinished(state_ptr)) {
|
276
339
|
ext_result = flush_destination_buffer(
|
277
|
-
destination_file,
|
278
|
-
destination_buffer, destination_length_ptr, destination_buffer_length);
|
340
|
+
destination_file, destination_buffer, destination_length_ptr, destination_buffer_length);
|
279
341
|
|
280
342
|
if (ext_result != 0) {
|
281
343
|
return ext_result;
|
@@ -290,13 +352,19 @@ static inline brs_ext_result_t buffered_compressor_finish(
|
|
290
352
|
return 0;
|
291
353
|
}
|
292
354
|
|
355
|
+
// -- compress --
|
356
|
+
|
293
357
|
static inline brs_ext_result_t compress(
|
294
358
|
BrotliEncoderState* state_ptr,
|
295
|
-
FILE*
|
296
|
-
|
359
|
+
FILE* source_file,
|
360
|
+
brs_ext_byte_t* source_buffer,
|
361
|
+
size_t source_buffer_length,
|
362
|
+
FILE* destination_file,
|
363
|
+
brs_ext_byte_t* destination_buffer,
|
364
|
+
size_t destination_buffer_length,
|
365
|
+
bool gvl)
|
297
366
|
{
|
298
|
-
brs_ext_result_t
|
299
|
-
|
367
|
+
brs_ext_result_t ext_result;
|
300
368
|
const brs_ext_byte_t* source = source_buffer;
|
301
369
|
size_t source_length = 0;
|
302
370
|
size_t destination_length = 0;
|
@@ -304,12 +372,16 @@ static inline brs_ext_result_t compress(
|
|
304
372
|
BUFFERED_READ_SOURCE(
|
305
373
|
buffered_compress,
|
306
374
|
state_ptr,
|
307
|
-
&source,
|
308
|
-
|
375
|
+
&source,
|
376
|
+
&source_length,
|
377
|
+
destination_file,
|
378
|
+
destination_buffer,
|
379
|
+
&destination_length,
|
380
|
+
destination_buffer_length,
|
381
|
+
gvl);
|
309
382
|
|
310
383
|
ext_result = buffered_compressor_finish(
|
311
|
-
state_ptr,
|
312
|
-
destination_file, destination_buffer, &destination_length, destination_buffer_length);
|
384
|
+
state_ptr, destination_file, destination_buffer, &destination_length, destination_buffer_length, gvl);
|
313
385
|
|
314
386
|
if (ext_result != 0) {
|
315
387
|
return ext_result;
|
@@ -323,9 +395,10 @@ VALUE brs_ext_compress_io(VALUE BRS_EXT_UNUSED(self), VALUE source, VALUE destin
|
|
323
395
|
GET_FILE(source);
|
324
396
|
GET_FILE(destination);
|
325
397
|
Check_Type(options, T_HASH);
|
398
|
+
BRS_EXT_GET_SIZE_OPTION(options, source_buffer_length);
|
399
|
+
BRS_EXT_GET_SIZE_OPTION(options, destination_buffer_length);
|
400
|
+
BRS_EXT_GET_BOOL_OPTION(options, gvl);
|
326
401
|
BRS_EXT_GET_COMPRESSOR_OPTIONS(options);
|
327
|
-
BRS_EXT_GET_BUFFER_LENGTH_OPTION(options, source_buffer_length);
|
328
|
-
BRS_EXT_GET_BUFFER_LENGTH_OPTION(options, destination_buffer_length);
|
329
402
|
|
330
403
|
BrotliEncoderState* state_ptr = BrotliEncoderCreateInstance(NULL, NULL, NULL);
|
331
404
|
if (state_ptr == NULL) {
|
@@ -348,10 +421,7 @@ VALUE brs_ext_compress_io(VALUE BRS_EXT_UNUSED(self), VALUE source, VALUE destin
|
|
348
421
|
brs_ext_byte_t* source_buffer;
|
349
422
|
brs_ext_byte_t* destination_buffer;
|
350
423
|
|
351
|
-
ext_result = create_buffers(
|
352
|
-
&source_buffer, source_buffer_length,
|
353
|
-
&destination_buffer, destination_buffer_length);
|
354
|
-
|
424
|
+
ext_result = create_buffers(&source_buffer, source_buffer_length, &destination_buffer, destination_buffer_length);
|
355
425
|
if (ext_result != 0) {
|
356
426
|
BrotliEncoderDestroyInstance(state_ptr);
|
357
427
|
brs_ext_raise_error(ext_result);
|
@@ -359,8 +429,13 @@ VALUE brs_ext_compress_io(VALUE BRS_EXT_UNUSED(self), VALUE source, VALUE destin
|
|
359
429
|
|
360
430
|
ext_result = compress(
|
361
431
|
state_ptr,
|
362
|
-
source_file,
|
363
|
-
|
432
|
+
source_file,
|
433
|
+
source_buffer,
|
434
|
+
source_buffer_length,
|
435
|
+
destination_file,
|
436
|
+
destination_buffer,
|
437
|
+
destination_buffer_length,
|
438
|
+
gvl);
|
364
439
|
|
365
440
|
free(source_buffer);
|
366
441
|
free(destination_buffer);
|
@@ -376,41 +451,68 @@ VALUE brs_ext_compress_io(VALUE BRS_EXT_UNUSED(self), VALUE source, VALUE destin
|
|
376
451
|
return Qnil;
|
377
452
|
}
|
378
453
|
|
379
|
-
// -- decompress --
|
454
|
+
// -- buffered decompress --
|
455
|
+
|
456
|
+
typedef struct
|
457
|
+
{
|
458
|
+
BrotliDecoderState* state_ptr;
|
459
|
+
const brs_ext_byte_t** source_ptr;
|
460
|
+
size_t* source_length_ptr;
|
461
|
+
brs_ext_byte_t* remaining_destination_buffer;
|
462
|
+
size_t* remaining_destination_buffer_length_ptr;
|
463
|
+
BrotliDecoderResult result;
|
464
|
+
} decompress_args_t;
|
465
|
+
|
466
|
+
static inline void* decompress_wrapper(void* data)
|
467
|
+
{
|
468
|
+
decompress_args_t* args = data;
|
469
|
+
|
470
|
+
args->result = BrotliDecoderDecompressStream(
|
471
|
+
args->state_ptr,
|
472
|
+
args->source_length_ptr,
|
473
|
+
args->source_ptr,
|
474
|
+
args->remaining_destination_buffer_length_ptr,
|
475
|
+
&args->remaining_destination_buffer,
|
476
|
+
NULL);
|
477
|
+
|
478
|
+
return NULL;
|
479
|
+
}
|
380
480
|
|
381
481
|
static inline brs_ext_result_t buffered_decompress(
|
382
482
|
BrotliDecoderState* state_ptr,
|
383
|
-
const brs_ext_byte_t** source_ptr,
|
384
|
-
|
483
|
+
const brs_ext_byte_t** source_ptr,
|
484
|
+
size_t* source_length_ptr,
|
485
|
+
FILE* destination_file,
|
486
|
+
brs_ext_byte_t* destination_buffer,
|
487
|
+
size_t* destination_length_ptr,
|
488
|
+
size_t destination_buffer_length,
|
489
|
+
bool gvl)
|
385
490
|
{
|
386
|
-
|
387
|
-
|
491
|
+
brs_ext_result_t ext_result;
|
492
|
+
decompress_args_t args = {.state_ptr = state_ptr, .source_ptr = source_ptr, .source_length_ptr = source_length_ptr};
|
388
493
|
|
389
494
|
while (true) {
|
390
495
|
brs_ext_byte_t* remaining_destination_buffer = destination_buffer + *destination_length_ptr;
|
391
496
|
size_t remaining_destination_buffer_length = destination_buffer_length - *destination_length_ptr;
|
392
497
|
size_t prev_remaining_destination_buffer_length = remaining_destination_buffer_length;
|
393
498
|
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
NULL);
|
499
|
+
args.remaining_destination_buffer = remaining_destination_buffer;
|
500
|
+
args.remaining_destination_buffer_length_ptr = &remaining_destination_buffer_length;
|
501
|
+
|
502
|
+
BRS_EXT_GVL_WRAP(gvl, decompress_wrapper, &args);
|
399
503
|
|
400
504
|
if (
|
401
|
-
result != BROTLI_DECODER_RESULT_SUCCESS &&
|
402
|
-
result !=
|
403
|
-
result != BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) {
|
505
|
+
args.result != BROTLI_DECODER_RESULT_SUCCESS && args.result != BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT &&
|
506
|
+
args.result != BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) {
|
404
507
|
BrotliDecoderErrorCode error_code = BrotliDecoderGetErrorCode(state_ptr);
|
405
508
|
return brs_ext_get_decompressor_error(error_code);
|
406
509
|
}
|
407
510
|
|
408
511
|
*destination_length_ptr += prev_remaining_destination_buffer_length - remaining_destination_buffer_length;
|
409
512
|
|
410
|
-
if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) {
|
513
|
+
if (args.result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) {
|
411
514
|
ext_result = flush_destination_buffer(
|
412
|
-
destination_file,
|
413
|
-
destination_buffer, destination_length_ptr, destination_buffer_length);
|
515
|
+
destination_file, destination_buffer, destination_length_ptr, destination_buffer_length);
|
414
516
|
|
415
517
|
if (ext_result != 0) {
|
416
518
|
return ext_result;
|
@@ -425,13 +527,19 @@ static inline brs_ext_result_t buffered_decompress(
|
|
425
527
|
return 0;
|
426
528
|
}
|
427
529
|
|
530
|
+
// -- decompress --
|
531
|
+
|
428
532
|
static inline brs_ext_result_t decompress(
|
429
533
|
BrotliDecoderState* state_ptr,
|
430
|
-
FILE*
|
431
|
-
|
534
|
+
FILE* source_file,
|
535
|
+
brs_ext_byte_t* source_buffer,
|
536
|
+
size_t source_buffer_length,
|
537
|
+
FILE* destination_file,
|
538
|
+
brs_ext_byte_t* destination_buffer,
|
539
|
+
size_t destination_buffer_length,
|
540
|
+
bool gvl)
|
432
541
|
{
|
433
|
-
brs_ext_result_t
|
434
|
-
|
542
|
+
brs_ext_result_t ext_result;
|
435
543
|
const brs_ext_byte_t* source = source_buffer;
|
436
544
|
size_t source_length = 0;
|
437
545
|
size_t destination_length = 0;
|
@@ -439,8 +547,13 @@ static inline brs_ext_result_t decompress(
|
|
439
547
|
BUFFERED_READ_SOURCE(
|
440
548
|
buffered_decompress,
|
441
549
|
state_ptr,
|
442
|
-
&source,
|
443
|
-
|
550
|
+
&source,
|
551
|
+
&source_length,
|
552
|
+
destination_file,
|
553
|
+
destination_buffer,
|
554
|
+
&destination_length,
|
555
|
+
destination_buffer_length,
|
556
|
+
gvl);
|
444
557
|
|
445
558
|
return write_remaining_destination(destination_file, destination_buffer, destination_length);
|
446
559
|
}
|
@@ -450,9 +563,10 @@ VALUE brs_ext_decompress_io(VALUE BRS_EXT_UNUSED(self), VALUE source, VALUE dest
|
|
450
563
|
GET_FILE(source);
|
451
564
|
GET_FILE(destination);
|
452
565
|
Check_Type(options, T_HASH);
|
566
|
+
BRS_EXT_GET_SIZE_OPTION(options, source_buffer_length);
|
567
|
+
BRS_EXT_GET_SIZE_OPTION(options, destination_buffer_length);
|
568
|
+
BRS_EXT_GET_BOOL_OPTION(options, gvl);
|
453
569
|
BRS_EXT_GET_DECOMPRESSOR_OPTIONS(options);
|
454
|
-
BRS_EXT_GET_BUFFER_LENGTH_OPTION(options, source_buffer_length);
|
455
|
-
BRS_EXT_GET_BUFFER_LENGTH_OPTION(options, destination_buffer_length);
|
456
570
|
|
457
571
|
BrotliDecoderState* state_ptr = BrotliDecoderCreateInstance(NULL, NULL, NULL);
|
458
572
|
if (state_ptr == NULL) {
|
@@ -475,10 +589,7 @@ VALUE brs_ext_decompress_io(VALUE BRS_EXT_UNUSED(self), VALUE source, VALUE dest
|
|
475
589
|
brs_ext_byte_t* source_buffer;
|
476
590
|
brs_ext_byte_t* destination_buffer;
|
477
591
|
|
478
|
-
ext_result = create_buffers(
|
479
|
-
&source_buffer, source_buffer_length,
|
480
|
-
&destination_buffer, destination_buffer_length);
|
481
|
-
|
592
|
+
ext_result = create_buffers(&source_buffer, source_buffer_length, &destination_buffer, destination_buffer_length);
|
482
593
|
if (ext_result != 0) {
|
483
594
|
BrotliDecoderDestroyInstance(state_ptr);
|
484
595
|
brs_ext_raise_error(ext_result);
|
@@ -486,8 +597,13 @@ VALUE brs_ext_decompress_io(VALUE BRS_EXT_UNUSED(self), VALUE source, VALUE dest
|
|
486
597
|
|
487
598
|
ext_result = decompress(
|
488
599
|
state_ptr,
|
489
|
-
source_file,
|
490
|
-
|
600
|
+
source_file,
|
601
|
+
source_buffer,
|
602
|
+
source_buffer_length,
|
603
|
+
destination_file,
|
604
|
+
destination_buffer,
|
605
|
+
destination_buffer_length,
|
606
|
+
gvl);
|
491
607
|
|
492
608
|
free(source_buffer);
|
493
609
|
free(destination_buffer);
|
@@ -503,6 +619,8 @@ VALUE brs_ext_decompress_io(VALUE BRS_EXT_UNUSED(self), VALUE source, VALUE dest
|
|
503
619
|
return Qnil;
|
504
620
|
}
|
505
621
|
|
622
|
+
// -- exports --
|
623
|
+
|
506
624
|
void brs_ext_io_exports(VALUE root_module)
|
507
625
|
{
|
508
626
|
rb_define_module_function(root_module, "_native_compress_io", RUBY_METHOD_FUNC(brs_ext_compress_io), 3);
|