burli 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +11 -0
- data/README.md +29 -0
- data/ext/burli_native/Cargo.toml +25 -0
- data/ext/burli_native/build.rs +24 -0
- data/ext/burli_native/extconf.rb +14 -0
- data/ext/burli_native/src/lib.rs +561 -0
- data/ext/burli_native/src/rb.rs +456 -0
- data/lib/burli/reader.rb +116 -0
- data/lib/burli/version.rb +6 -0
- data/lib/burli/writer.rb +71 -0
- data/lib/burli.rb +137 -0
- metadata +68 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: fd85643b4dc17f583b3c685edef250e86c7f6a567b6193c77bcc45207986543b
|
|
4
|
+
data.tar.gz: 83605da52ab070c33e4dcf6f54ee08ee8ae09bdf03ec9550bcb489aa85b95a6b
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 72b668abadb29e537c3aae01e786c153674293c3a65e67f38951c004455cfe0185f8d94cd7637a5d413279752b4351b5ce12694e9ef871934a3386f9378d0cad
|
|
7
|
+
data.tar.gz: 8605651ddfea605ba0843d19170e39d0d68417c978f0ba5a7ab7c8abf947aabb923ef2f6cb50c7a27a4d8250f729d8f13c8d94555bdb2a18db853627cab25536
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## [Unreleased]
|
|
4
|
+
|
|
5
|
+
## [0.1.0] - 2026-10-28
|
|
6
|
+
|
|
7
|
+
- Add the `burli` gem, with the API of the `brotli` gem: `Burli.deflate`,
|
|
8
|
+
`Burli.inflate`, `Burli::Compressor`, `Burli::Decompressor` with
|
|
9
|
+
`output_buffer_limit:`, `can_accept_more_data`, and `unused_data`,
|
|
10
|
+
`Burli::Writer`, `Burli::Reader`, and `Burli::Error`. Qualities 6 to 11 run
|
|
11
|
+
as 5. Raw dictionaries work for decompression.
|
data/README.md
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# burli
|
|
2
|
+
|
|
3
|
+
Brotli compression and decompression for Ruby, built on
|
|
4
|
+
[burli](https://github.com/paddor/burli), a Brotli codec in safe Rust. The
|
|
5
|
+
API matches the `brotli` gem:
|
|
6
|
+
|
|
7
|
+
```ruby
|
|
8
|
+
require "burli"
|
|
9
|
+
|
|
10
|
+
compressed = Burli.deflate(data)
|
|
11
|
+
Burli.inflate(compressed) == data # => true
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
Code written for the `brotli` gem can switch with `Brotli = Burli`.
|
|
15
|
+
`Burli::Compressor`, `Burli::Decompressor` (with `output_buffer_limit:`),
|
|
16
|
+
`Burli::Writer`, and `Burli::Reader` work as in `brotli`. Calls release the
|
|
17
|
+
GVL while they encode or decode.
|
|
18
|
+
|
|
19
|
+
Differences from `brotli`:
|
|
20
|
+
|
|
21
|
+
- Qualities 6 to 11 run as 5, and the default quality is 5.
|
|
22
|
+
- `dictionary:` works for decompression only. Compression with it raises
|
|
23
|
+
`ArgumentError`.
|
|
24
|
+
- `Burli.inflate` raises on data after the stream. `Burli::Decompressor`
|
|
25
|
+
keeps such data in `#unused_data`, as in `brotli`.
|
|
26
|
+
- Error messages differ, and `Burli.version` returns the gem version.
|
|
27
|
+
|
|
28
|
+
Gems come precompiled for Linux (glibc and musl), macOS, and Windows. Other
|
|
29
|
+
platforms build from source with Rust 1.89 or later.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
[package]
|
|
2
|
+
name = "burli_native"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
edition = "2024"
|
|
5
|
+
rust-version = "1.89"
|
|
6
|
+
description = "Ruby binding for burli"
|
|
7
|
+
repository = "https://github.com/paddor/burli"
|
|
8
|
+
license = "MIT"
|
|
9
|
+
publish = false
|
|
10
|
+
|
|
11
|
+
[lib]
|
|
12
|
+
name = "burli_native"
|
|
13
|
+
crate-type = ["cdylib"]
|
|
14
|
+
|
|
15
|
+
[dependencies]
|
|
16
|
+
burli = { version = "0.3.7", default-features = false, features = ["std"] }
|
|
17
|
+
rb-sys = "0.9.130"
|
|
18
|
+
|
|
19
|
+
[lints.clippy]
|
|
20
|
+
pedantic = { level = "warn", priority = -1 }
|
|
21
|
+
cast_possible_truncation = "allow"
|
|
22
|
+
cast_possible_wrap = "allow"
|
|
23
|
+
cast_sign_loss = "allow"
|
|
24
|
+
missing_errors_doc = "allow"
|
|
25
|
+
must_use_candidate = "allow"
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
use std::process::Command;
|
|
2
|
+
|
|
3
|
+
fn main() {
|
|
4
|
+
println!("cargo:rerun-if-env-changed=RUBY");
|
|
5
|
+
println!("cargo:rustc-check-cfg=cfg(ruby_engine, values(\"mri\", \"truffleruby\"))");
|
|
6
|
+
|
|
7
|
+
let ruby = std::env::var("RUBY").unwrap_or_else(|_| "ruby".to_string());
|
|
8
|
+
let output = Command::new(&ruby)
|
|
9
|
+
.arg("-rrbconfig")
|
|
10
|
+
.arg("-e")
|
|
11
|
+
.arg("print RbConfig::CONFIG.fetch('ruby_install_name')")
|
|
12
|
+
.output()
|
|
13
|
+
.unwrap_or_else(|err| panic!("failed to run {ruby}: {err}"));
|
|
14
|
+
|
|
15
|
+
assert!(
|
|
16
|
+
output.status.success(),
|
|
17
|
+
"failed to query Ruby engine with {ruby}"
|
|
18
|
+
);
|
|
19
|
+
|
|
20
|
+
match String::from_utf8_lossy(&output.stdout).trim() {
|
|
21
|
+
"truffleruby" => println!("cargo:rustc-cfg=ruby_engine=\"truffleruby\""),
|
|
22
|
+
_ => println!("cargo:rustc-cfg=ruby_engine=\"mri\""),
|
|
23
|
+
}
|
|
24
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "mkmf"
|
|
4
|
+
require "rb_sys/mkmf"
|
|
5
|
+
|
|
6
|
+
create_rust_makefile("burli/burli_native") do |r|
|
|
7
|
+
r.profile = ENV.fetch("RB_SYS_CARGO_PROFILE", :release).to_sym
|
|
8
|
+
# Gem installs build the crate outside the repo workspace and its profile.
|
|
9
|
+
r.env = {
|
|
10
|
+
"CARGO_PROFILE_RELEASE_LTO" => "true",
|
|
11
|
+
"CARGO_PROFILE_RELEASE_CODEGEN_UNITS" => "1",
|
|
12
|
+
"CARGO_PROFILE_RELEASE_STRIP" => "true"
|
|
13
|
+
}
|
|
14
|
+
end
|
|
@@ -0,0 +1,561 @@
|
|
|
1
|
+
//! Ruby binding for burli, shaped like the `brotli` gem. `lib/burli.rb`
|
|
2
|
+
//! checks options and calls the `_native_*` methods here.
|
|
3
|
+
//!
|
|
4
|
+
//! Qualities 6 to 11 run as 5.
|
|
5
|
+
|
|
6
|
+
mod rb;
|
|
7
|
+
|
|
8
|
+
use std::ffi::c_void;
|
|
9
|
+
use std::io::Write;
|
|
10
|
+
use std::panic::{AssertUnwindSafe, catch_unwind};
|
|
11
|
+
use std::sync::{Mutex, OnceLock, TryLockError};
|
|
12
|
+
|
|
13
|
+
use burli::decode::RawDictionary;
|
|
14
|
+
use burli::{BurliError, DecodeStatus, Mode, Options, PushDecoder, StreamEncoder};
|
|
15
|
+
use rb_sys::{VALUE, rb_data_type_struct__bindgen_ty_1, rb_data_type_t, size_t};
|
|
16
|
+
|
|
17
|
+
use crate::rb::{RbResult, RubyErr};
|
|
18
|
+
|
|
19
|
+
/// The highest quality the encoder implements. Higher ones run as this.
|
|
20
|
+
const MAX_QUALITY: i64 = 5;
|
|
21
|
+
/// The first output buffer of a decompress call. Later ones double it.
|
|
22
|
+
const FIRST_OUTPUT_ROOM: usize = 32 * 1024;
|
|
23
|
+
/// Work on fewer input and output bytes keeps the GVL, as in zrip. A
|
|
24
|
+
/// decompression without an output limit always releases it: its output
|
|
25
|
+
/// does not follow from its input.
|
|
26
|
+
const GVL_THRESHOLD: usize = 64 * 1024;
|
|
27
|
+
|
|
28
|
+
#[derive(Clone, Copy)]
|
|
29
|
+
struct GlobalValue(VALUE);
|
|
30
|
+
|
|
31
|
+
// SAFETY: the value is a class constant, which Ruby never frees or moves.
|
|
32
|
+
unsafe impl Send for GlobalValue {}
|
|
33
|
+
// SAFETY: as above.
|
|
34
|
+
unsafe impl Sync for GlobalValue {}
|
|
35
|
+
|
|
36
|
+
static ERROR: OnceLock<GlobalValue> = OnceLock::new();
|
|
37
|
+
|
|
38
|
+
/// A `Burli::Error`.
|
|
39
|
+
fn error(message: impl Into<String>) -> RubyErr {
|
|
40
|
+
RubyErr::new(ERROR.get().expect("Burli::Error is defined").0, message)
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
fn burli_error(err: &BurliError) -> RubyErr {
|
|
44
|
+
error(err.to_string())
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
fn encode_options(mode: VALUE, quality: VALUE, lgwin: VALUE, lgblock: VALUE) -> RbResult<Options> {
|
|
48
|
+
let mode = match rb::value_to_i64(mode)? {
|
|
49
|
+
1 => Mode::Text,
|
|
50
|
+
2 => Mode::Font,
|
|
51
|
+
_ => Mode::Generic,
|
|
52
|
+
};
|
|
53
|
+
let quality = rb::value_to_i64(quality)?.clamp(0, MAX_QUALITY) as u8;
|
|
54
|
+
let lgwin =
|
|
55
|
+
u8::try_from(rb::value_to_i64(lgwin)?).map_err(|_| RubyErr::arg("invalid lgwin"))?;
|
|
56
|
+
let lgblock =
|
|
57
|
+
u8::try_from(rb::value_to_i64(lgblock)?).map_err(|_| RubyErr::arg("invalid lgblock"))?;
|
|
58
|
+
let options = Options::default()
|
|
59
|
+
.with_quality(quality)
|
|
60
|
+
.and_then(|options| options.with_window_bits(lgwin))
|
|
61
|
+
.and_then(|options| options.with_block_bits((lgblock != 0).then_some(lgblock)))
|
|
62
|
+
.map_err(|err| RubyErr::arg(err.to_string()))?;
|
|
63
|
+
Ok(options.with_mode(mode))
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/// Runs `func` on the value behind `mutex`, without the GVL when
|
|
67
|
+
/// `release_gvl`. Waits for a held lock without the GVL too, so the thread
|
|
68
|
+
/// holding it can finish.
|
|
69
|
+
fn with_mutex<T, R, F>(
|
|
70
|
+
mutex: &Mutex<T>,
|
|
71
|
+
release_gvl: bool,
|
|
72
|
+
input: &mut rb::InputBytes,
|
|
73
|
+
func: F,
|
|
74
|
+
) -> RbResult<R>
|
|
75
|
+
where
|
|
76
|
+
F: FnOnce(&mut T, &[u8]) -> RbResult<R>,
|
|
77
|
+
{
|
|
78
|
+
let poisoned = || RubyErr::runtime("an earlier call panicked");
|
|
79
|
+
if !release_gvl {
|
|
80
|
+
match mutex.try_lock() {
|
|
81
|
+
Ok(mut guard) => return func(&mut guard, input.as_slice()),
|
|
82
|
+
Err(TryLockError::Poisoned(_)) => return Err(poisoned()),
|
|
83
|
+
Err(TryLockError::WouldBlock) => {}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
input.lock();
|
|
87
|
+
rb::maybe_without_gvl(true, || {
|
|
88
|
+
let mut guard = mutex.lock().map_err(|_| poisoned())?;
|
|
89
|
+
func(&mut guard, input.as_slice())
|
|
90
|
+
})?
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// ---------- module functions ----------
|
|
94
|
+
|
|
95
|
+
fn deflate_impl(
|
|
96
|
+
input: VALUE,
|
|
97
|
+
mode: VALUE,
|
|
98
|
+
quality: VALUE,
|
|
99
|
+
lgwin: VALUE,
|
|
100
|
+
lgblock: VALUE,
|
|
101
|
+
) -> RbResult<VALUE> {
|
|
102
|
+
let options = encode_options(mode, quality, lgwin, lgblock)?;
|
|
103
|
+
let mut input = rb::input_bytes(input)?;
|
|
104
|
+
let release_gvl = input.len() >= GVL_THRESHOLD;
|
|
105
|
+
if release_gvl {
|
|
106
|
+
input.lock();
|
|
107
|
+
}
|
|
108
|
+
let encoded = rb::maybe_without_gvl(release_gvl, || {
|
|
109
|
+
burli::compress_with_options(input.as_slice(), &options)
|
|
110
|
+
})?
|
|
111
|
+
.map_err(|err| burli_error(&err))?;
|
|
112
|
+
rb::new_binary_string(&encoded)
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
fn inflate_impl(input: VALUE, dictionary: VALUE) -> RbResult<VALUE> {
|
|
116
|
+
let dictionary = rb::value_to_option_bytes(dictionary)?.map(RawDictionary::from_vec);
|
|
117
|
+
let mut input = rb::input_bytes(input)?;
|
|
118
|
+
input.lock();
|
|
119
|
+
let decoded = rb::maybe_without_gvl(true, || match &dictionary {
|
|
120
|
+
Some(dictionary) => burli::decompress_with_raw_dictionary(input.as_slice(), dictionary),
|
|
121
|
+
None => burli::decompress(input.as_slice()),
|
|
122
|
+
})?
|
|
123
|
+
.map_err(|err| burli_error(&err))?;
|
|
124
|
+
rb::new_binary_string(&decoded)
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
unsafe extern "C" fn deflate(
|
|
128
|
+
_module: VALUE,
|
|
129
|
+
input: VALUE,
|
|
130
|
+
mode: VALUE,
|
|
131
|
+
quality: VALUE,
|
|
132
|
+
lgwin: VALUE,
|
|
133
|
+
lgblock: VALUE,
|
|
134
|
+
) -> VALUE {
|
|
135
|
+
rb::wrap(|| deflate_impl(input, mode, quality, lgwin, lgblock))
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
unsafe extern "C" fn inflate(_module: VALUE, input: VALUE, dictionary: VALUE) -> VALUE {
|
|
139
|
+
rb::wrap(|| inflate_impl(input, dictionary))
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// ---------- typed data ----------
|
|
143
|
+
|
|
144
|
+
struct NativeDataType(rb_data_type_t);
|
|
145
|
+
|
|
146
|
+
// SAFETY: the type is built once and never changed.
|
|
147
|
+
unsafe impl Send for NativeDataType {}
|
|
148
|
+
// SAFETY: as above.
|
|
149
|
+
unsafe impl Sync for NativeDataType {}
|
|
150
|
+
|
|
151
|
+
static COMPRESSOR_DATA_TYPE: OnceLock<NativeDataType> = OnceLock::new();
|
|
152
|
+
static DECOMPRESSOR_DATA_TYPE: OnceLock<NativeDataType> = OnceLock::new();
|
|
153
|
+
|
|
154
|
+
fn data_type(
|
|
155
|
+
cell: &'static OnceLock<NativeDataType>,
|
|
156
|
+
name: &'static std::ffi::CStr,
|
|
157
|
+
free: unsafe extern "C" fn(*mut c_void),
|
|
158
|
+
size: unsafe extern "C" fn(*const c_void) -> size_t,
|
|
159
|
+
) -> *const rb_data_type_t {
|
|
160
|
+
&raw const cell
|
|
161
|
+
.get_or_init(|| {
|
|
162
|
+
NativeDataType(rb_data_type_t {
|
|
163
|
+
wrap_struct_name: name.as_ptr(),
|
|
164
|
+
function: rb_data_type_struct__bindgen_ty_1 {
|
|
165
|
+
dmark: None,
|
|
166
|
+
dfree: Some(free),
|
|
167
|
+
dsize: Some(size),
|
|
168
|
+
dcompact: None,
|
|
169
|
+
reserved: [std::ptr::null_mut(); 1],
|
|
170
|
+
},
|
|
171
|
+
parent: std::ptr::null(),
|
|
172
|
+
data: std::ptr::null_mut(),
|
|
173
|
+
// RUBY_TYPED_FREE_IMMEDIATELY: freeing runs no Ruby code.
|
|
174
|
+
flags: 1,
|
|
175
|
+
})
|
|
176
|
+
})
|
|
177
|
+
.0
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
fn compressor_data_type() -> *const rb_data_type_t {
|
|
181
|
+
data_type(
|
|
182
|
+
&COMPRESSOR_DATA_TYPE,
|
|
183
|
+
c"burli_compressor",
|
|
184
|
+
free::<Compressor>,
|
|
185
|
+
native_size::<Compressor>,
|
|
186
|
+
)
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
fn decompressor_data_type() -> *const rb_data_type_t {
|
|
190
|
+
data_type(
|
|
191
|
+
&DECOMPRESSOR_DATA_TYPE,
|
|
192
|
+
c"burli_decompressor",
|
|
193
|
+
free::<Decompressor>,
|
|
194
|
+
native_size::<Decompressor>,
|
|
195
|
+
)
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
unsafe extern "C" fn free<T>(data: *mut c_void) {
|
|
199
|
+
if !data.is_null() {
|
|
200
|
+
// SAFETY: the object owned this box, and Ruby frees it once.
|
|
201
|
+
let _ = catch_unwind(AssertUnwindSafe(|| {
|
|
202
|
+
drop(unsafe { Box::from_raw(data.cast::<T>()) });
|
|
203
|
+
}));
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
unsafe extern "C" fn native_size<T>(_data: *const c_void) -> size_t {
|
|
208
|
+
std::mem::size_of::<T>() as size_t
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
// ---------- Compressor ----------
|
|
212
|
+
|
|
213
|
+
struct Compressor {
|
|
214
|
+
/// `None` after `finish`.
|
|
215
|
+
encoder: Mutex<Option<StreamEncoder<Vec<u8>>>>,
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
fn compressor_ref<'a>(value: VALUE) -> RbResult<&'a Compressor> {
|
|
219
|
+
// SAFETY: the data type wraps a `Compressor`, and Ruby keeps `value`
|
|
220
|
+
// alive while the method runs.
|
|
221
|
+
unsafe { rb::typed_data_ref(value, compressor_data_type()) }
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
fn compressor_new_impl(
|
|
225
|
+
class: VALUE,
|
|
226
|
+
mode: VALUE,
|
|
227
|
+
quality: VALUE,
|
|
228
|
+
lgwin: VALUE,
|
|
229
|
+
lgblock: VALUE,
|
|
230
|
+
) -> RbResult<VALUE> {
|
|
231
|
+
let options = encode_options(mode, quality, lgwin, lgblock)?;
|
|
232
|
+
let encoder =
|
|
233
|
+
StreamEncoder::with_options(Vec::new(), options).map_err(|err| burli_error(&err))?;
|
|
234
|
+
let compressor = Box::new(Compressor {
|
|
235
|
+
encoder: Mutex::new(Some(encoder)),
|
|
236
|
+
});
|
|
237
|
+
// SAFETY: the data type frees a `Box<Compressor>`.
|
|
238
|
+
unsafe { rb::wrap_typed_data(class, compressor, compressor_data_type()) }
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/// Writes `input` to the encoder and runs `step`; returns the output so far.
|
|
242
|
+
fn compressor_run(
|
|
243
|
+
rb_self: VALUE,
|
|
244
|
+
input: VALUE,
|
|
245
|
+
release_gvl: bool,
|
|
246
|
+
step: fn(&mut StreamEncoder<Vec<u8>>) -> std::io::Result<()>,
|
|
247
|
+
) -> RbResult<VALUE> {
|
|
248
|
+
let compressor = compressor_ref(rb_self)?;
|
|
249
|
+
let mut input = if input == rb::qnil() {
|
|
250
|
+
rb::InputBytes::Owned(Vec::new())
|
|
251
|
+
} else {
|
|
252
|
+
rb::input_bytes(input)?
|
|
253
|
+
};
|
|
254
|
+
let release_gvl = release_gvl || input.len() >= GVL_THRESHOLD;
|
|
255
|
+
let output = with_mutex(
|
|
256
|
+
&compressor.encoder,
|
|
257
|
+
release_gvl,
|
|
258
|
+
&mut input,
|
|
259
|
+
|encoder, input| {
|
|
260
|
+
let encoder = encoder
|
|
261
|
+
.as_mut()
|
|
262
|
+
.ok_or_else(|| error("Compressor is finished"))?;
|
|
263
|
+
encoder
|
|
264
|
+
.write_all(input)
|
|
265
|
+
.and_then(|()| step(encoder))
|
|
266
|
+
.map_err(|err| error(err.to_string()))?;
|
|
267
|
+
Ok(std::mem::take(encoder.get_mut()))
|
|
268
|
+
},
|
|
269
|
+
)?;
|
|
270
|
+
rb::new_binary_string(&output)
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
fn compressor_finish_impl(rb_self: VALUE) -> RbResult<VALUE> {
|
|
274
|
+
let compressor = compressor_ref(rb_self)?;
|
|
275
|
+
let mut input = rb::InputBytes::Owned(Vec::new());
|
|
276
|
+
let output = with_mutex(
|
|
277
|
+
&compressor.encoder,
|
|
278
|
+
true,
|
|
279
|
+
&mut input,
|
|
280
|
+
|encoder, _| match encoder.take() {
|
|
281
|
+
Some(encoder) => encoder.finish().map_err(|err| burli_error(&err)),
|
|
282
|
+
None => Ok(Vec::new()),
|
|
283
|
+
},
|
|
284
|
+
)?;
|
|
285
|
+
rb::new_binary_string(&output)
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
fn compressor_is_finished_impl(rb_self: VALUE) -> RbResult<VALUE> {
|
|
289
|
+
let compressor = compressor_ref(rb_self)?;
|
|
290
|
+
let mut input = rb::InputBytes::Owned(Vec::new());
|
|
291
|
+
let finished = with_mutex(&compressor.encoder, false, &mut input, |encoder, _| {
|
|
292
|
+
Ok(encoder.is_none())
|
|
293
|
+
})?;
|
|
294
|
+
Ok(rb::bool_value(finished))
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
unsafe extern "C" fn compressor_new(
|
|
298
|
+
class: VALUE,
|
|
299
|
+
mode: VALUE,
|
|
300
|
+
quality: VALUE,
|
|
301
|
+
lgwin: VALUE,
|
|
302
|
+
lgblock: VALUE,
|
|
303
|
+
) -> VALUE {
|
|
304
|
+
rb::wrap(|| compressor_new_impl(class, mode, quality, lgwin, lgblock))
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/// `process(input)`: compresses `input`, returning the output so far.
|
|
308
|
+
unsafe extern "C" fn compressor_process(rb_self: VALUE, input: VALUE) -> VALUE {
|
|
309
|
+
rb::wrap(|| compressor_run(rb_self, input, false, |_| Ok(())))
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
/// `flush`: returns output that decodes to all input so far.
|
|
313
|
+
unsafe extern "C" fn compressor_flush(rb_self: VALUE) -> VALUE {
|
|
314
|
+
rb::wrap(|| compressor_run(rb_self, rb::qnil(), true, Write::flush))
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/// `finish`: ends the stream, returning the rest of the output.
|
|
318
|
+
unsafe extern "C" fn compressor_finish(rb_self: VALUE) -> VALUE {
|
|
319
|
+
rb::wrap(|| compressor_finish_impl(rb_self))
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
unsafe extern "C" fn compressor_is_finished(rb_self: VALUE) -> VALUE {
|
|
323
|
+
rb::wrap(|| compressor_is_finished_impl(rb_self))
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
// ---------- Decompressor ----------
|
|
327
|
+
|
|
328
|
+
struct Decompressor {
|
|
329
|
+
state: Mutex<DecoderState>,
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
struct DecoderState {
|
|
333
|
+
decoder: PushDecoder,
|
|
334
|
+
/// Input the last call could not hand to the decoder before it reached
|
|
335
|
+
/// its output limit.
|
|
336
|
+
tail: Vec<u8>,
|
|
337
|
+
/// The decoder holds output the last call did not return.
|
|
338
|
+
pending: bool,
|
|
339
|
+
/// Input after the end of the stream.
|
|
340
|
+
unused: Vec<u8>,
|
|
341
|
+
/// A call failed; the decoder takes no more calls.
|
|
342
|
+
failed: bool,
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
impl DecoderState {
|
|
346
|
+
fn check(&self) -> RbResult<()> {
|
|
347
|
+
if self.failed {
|
|
348
|
+
return Err(error("Decompressor failed on earlier input"));
|
|
349
|
+
}
|
|
350
|
+
Ok(())
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
/// Decodes `data`, or held input when `data` is empty, into at most
|
|
354
|
+
/// `limit` bytes.
|
|
355
|
+
fn process(&mut self, data: &[u8], limit: usize) -> RbResult<Vec<u8>> {
|
|
356
|
+
self.check()?;
|
|
357
|
+
if self.decoder.is_finished() {
|
|
358
|
+
if data.is_empty() {
|
|
359
|
+
return Ok(Vec::new());
|
|
360
|
+
}
|
|
361
|
+
return Err(error("Decompressor is finished"));
|
|
362
|
+
}
|
|
363
|
+
if (self.pending || !self.tail.is_empty()) && !data.is_empty() {
|
|
364
|
+
return Err(error(
|
|
365
|
+
"Decompressor cannot accept more data until pending output is drained",
|
|
366
|
+
));
|
|
367
|
+
}
|
|
368
|
+
let tail = std::mem::take(&mut self.tail);
|
|
369
|
+
let input: &[u8] = if tail.is_empty() { data } else { &tail };
|
|
370
|
+
let (output, consumed, status) =
|
|
371
|
+
decode(&mut self.decoder, input, limit).map_err(|err| {
|
|
372
|
+
self.failed = true;
|
|
373
|
+
burli_error(&err)
|
|
374
|
+
})?;
|
|
375
|
+
match status {
|
|
376
|
+
DecodeStatus::Finished => self.unused = input[consumed..].to_vec(),
|
|
377
|
+
DecodeStatus::NeedsOutput => self.tail = input[consumed..].to_vec(),
|
|
378
|
+
DecodeStatus::NeedsInput => {}
|
|
379
|
+
}
|
|
380
|
+
self.pending = status == DecodeStatus::NeedsOutput;
|
|
381
|
+
Ok(output)
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/// Decodes `input` into a buffer that grows up to `limit` bytes. Returns
|
|
386
|
+
/// the output, the input bytes taken, and the decoder status.
|
|
387
|
+
fn decode(
|
|
388
|
+
decoder: &mut PushDecoder,
|
|
389
|
+
input: &[u8],
|
|
390
|
+
limit: usize,
|
|
391
|
+
) -> Result<(Vec<u8>, usize, DecodeStatus), BurliError> {
|
|
392
|
+
let mut output = Vec::new();
|
|
393
|
+
let mut consumed = 0;
|
|
394
|
+
loop {
|
|
395
|
+
let start = output.len();
|
|
396
|
+
let room = (limit - start).min(start.max(FIRST_OUTPUT_ROOM));
|
|
397
|
+
output.resize(start + room, 0);
|
|
398
|
+
let progress = decoder.decompress(&input[consumed..], &mut output[start..])?;
|
|
399
|
+
output.truncate(start + progress.written);
|
|
400
|
+
consumed += progress.consumed;
|
|
401
|
+
if progress.status != DecodeStatus::NeedsOutput || output.len() == limit {
|
|
402
|
+
return Ok((output, consumed, progress.status));
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
fn decompressor_ref<'a>(value: VALUE) -> RbResult<&'a Decompressor> {
|
|
408
|
+
// SAFETY: the data type wraps a `Decompressor`, and Ruby keeps `value`
|
|
409
|
+
// alive while the method runs.
|
|
410
|
+
unsafe { rb::typed_data_ref(value, decompressor_data_type()) }
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
fn decompressor_new_impl(class: VALUE, dictionary: VALUE) -> RbResult<VALUE> {
|
|
414
|
+
let decoder = match rb::value_to_option_bytes(dictionary)? {
|
|
415
|
+
Some(dictionary) => PushDecoder::with_raw_dictionary(RawDictionary::from_vec(dictionary)),
|
|
416
|
+
None => PushDecoder::new(),
|
|
417
|
+
};
|
|
418
|
+
let decompressor = Box::new(Decompressor {
|
|
419
|
+
state: Mutex::new(DecoderState {
|
|
420
|
+
decoder,
|
|
421
|
+
tail: Vec::new(),
|
|
422
|
+
pending: false,
|
|
423
|
+
unused: Vec::new(),
|
|
424
|
+
failed: false,
|
|
425
|
+
}),
|
|
426
|
+
});
|
|
427
|
+
// SAFETY: the data type frees a `Box<Decompressor>`.
|
|
428
|
+
unsafe { rb::wrap_typed_data(class, decompressor, decompressor_data_type()) }
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
fn decompressor_process_impl(rb_self: VALUE, input: VALUE, limit: VALUE) -> RbResult<VALUE> {
|
|
432
|
+
let decompressor = decompressor_ref(rb_self)?;
|
|
433
|
+
// Zero: no limit.
|
|
434
|
+
let limit = match usize::try_from(rb::value_to_i64(limit)?) {
|
|
435
|
+
Ok(0) => usize::MAX,
|
|
436
|
+
Ok(limit) => limit,
|
|
437
|
+
Err(_) => return Err(RubyErr::arg("output_buffer_limit must be positive")),
|
|
438
|
+
};
|
|
439
|
+
let mut input = rb::input_bytes(input)?;
|
|
440
|
+
let release_gvl = input.len().saturating_add(limit) >= GVL_THRESHOLD;
|
|
441
|
+
let output = with_mutex(
|
|
442
|
+
&decompressor.state,
|
|
443
|
+
release_gvl,
|
|
444
|
+
&mut input,
|
|
445
|
+
|state, input| state.process(input, limit),
|
|
446
|
+
)?;
|
|
447
|
+
rb::new_binary_string(&output)
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/// Runs `read` on the decoder state, with the GVL.
|
|
451
|
+
fn decompressor_read<R>(rb_self: VALUE, read: fn(&DecoderState) -> R) -> RbResult<R> {
|
|
452
|
+
let decompressor = decompressor_ref(rb_self)?;
|
|
453
|
+
let mut input = rb::InputBytes::Owned(Vec::new());
|
|
454
|
+
with_mutex(&decompressor.state, false, &mut input, |state, _| {
|
|
455
|
+
state.check()?;
|
|
456
|
+
Ok(read(state))
|
|
457
|
+
})
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
unsafe extern "C" fn decompressor_new(class: VALUE, dictionary: VALUE) -> VALUE {
|
|
461
|
+
rb::wrap(|| decompressor_new_impl(class, dictionary))
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
/// `_native_process(input, limit)`: `limit` is 0 for no limit.
|
|
465
|
+
unsafe extern "C" fn decompressor_process(rb_self: VALUE, input: VALUE, limit: VALUE) -> VALUE {
|
|
466
|
+
rb::wrap(|| decompressor_process_impl(rb_self, input, limit))
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
/// Whether the stream is decoded and all of its output returned.
|
|
470
|
+
unsafe extern "C" fn decompressor_is_finished(rb_self: VALUE) -> VALUE {
|
|
471
|
+
rb::wrap(|| decompressor_read(rb_self, |state| state.decoder.is_finished()).map(rb::bool_value))
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
/// Whether the decoder takes more input. False after the end of the stream,
|
|
475
|
+
/// and while it holds input or output from a call that reached its limit.
|
|
476
|
+
unsafe extern "C" fn decompressor_can_accept_more_data(rb_self: VALUE) -> VALUE {
|
|
477
|
+
rb::wrap(|| {
|
|
478
|
+
decompressor_read(rb_self, |state| {
|
|
479
|
+
!state.decoder.is_finished() && state.tail.is_empty() && !state.pending
|
|
480
|
+
})
|
|
481
|
+
.map(rb::bool_value)
|
|
482
|
+
})
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
/// Input after the end of the stream.
|
|
486
|
+
unsafe extern "C" fn decompressor_unused_data(rb_self: VALUE) -> VALUE {
|
|
487
|
+
rb::wrap(|| {
|
|
488
|
+
let unused = decompressor_read(rb_self, |state| state.unused.clone())?;
|
|
489
|
+
rb::new_binary_string(&unused)
|
|
490
|
+
})
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
// ---------- init ----------
|
|
494
|
+
|
|
495
|
+
/// # Safety
|
|
496
|
+
///
|
|
497
|
+
/// Ruby's extension loader calls this once, with the VM running.
|
|
498
|
+
#[unsafe(no_mangle)]
|
|
499
|
+
pub unsafe extern "C" fn Init_burli_native() {
|
|
500
|
+
rb::wrap(|| init().map(|()| rb::qnil()));
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
fn init() -> RbResult<()> {
|
|
504
|
+
// SAFETY: called while the extension loads. Objects keep their state
|
|
505
|
+
// behind their own lock, and no state is global.
|
|
506
|
+
#[cfg(ruby_engine = "mri")]
|
|
507
|
+
unsafe {
|
|
508
|
+
rb_sys::rb_ext_ractor_safe(true);
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
let module = rb::define_module(c"Burli")?;
|
|
512
|
+
// SAFETY: Ruby sets the class globals before it loads extensions.
|
|
513
|
+
let (object, standard_error) = unsafe { (rb_sys::rb_cObject, rb_sys::rb_eStandardError) };
|
|
514
|
+
let error_class = rb::define_class_under(module, c"Error", standard_error)?;
|
|
515
|
+
ERROR
|
|
516
|
+
.set(GlobalValue(error_class))
|
|
517
|
+
.map_err(|_| RubyErr::runtime("burli_native loaded twice"))?;
|
|
518
|
+
rb::define_singleton_method(module, c"_native_deflate", deflate as rb::Method5)?;
|
|
519
|
+
rb::define_singleton_method(module, c"_native_inflate", inflate as rb::Method2)?;
|
|
520
|
+
|
|
521
|
+
let compressor = rb::define_class_under(module, c"Compressor", object)?;
|
|
522
|
+
rb::undef_alloc_func(compressor)?;
|
|
523
|
+
rb::define_singleton_method(compressor, c"_native_new", compressor_new as rb::Method4)?;
|
|
524
|
+
rb::define_method(compressor, c"process", compressor_process as rb::Method1)?;
|
|
525
|
+
rb::define_method(compressor, c"flush", compressor_flush as rb::Method0)?;
|
|
526
|
+
rb::define_method(compressor, c"finish", compressor_finish as rb::Method0)?;
|
|
527
|
+
rb::define_method(
|
|
528
|
+
compressor,
|
|
529
|
+
c"finished?",
|
|
530
|
+
compressor_is_finished as rb::Method0,
|
|
531
|
+
)?;
|
|
532
|
+
|
|
533
|
+
let decompressor = rb::define_class_under(module, c"Decompressor", object)?;
|
|
534
|
+
rb::undef_alloc_func(decompressor)?;
|
|
535
|
+
rb::define_singleton_method(
|
|
536
|
+
decompressor,
|
|
537
|
+
c"_native_new",
|
|
538
|
+
decompressor_new as rb::Method1,
|
|
539
|
+
)?;
|
|
540
|
+
rb::define_method(
|
|
541
|
+
decompressor,
|
|
542
|
+
c"_native_process",
|
|
543
|
+
decompressor_process as rb::Method2,
|
|
544
|
+
)?;
|
|
545
|
+
rb::define_method(
|
|
546
|
+
decompressor,
|
|
547
|
+
c"finished?",
|
|
548
|
+
decompressor_is_finished as rb::Method0,
|
|
549
|
+
)?;
|
|
550
|
+
rb::define_method(
|
|
551
|
+
decompressor,
|
|
552
|
+
c"can_accept_more_data",
|
|
553
|
+
decompressor_can_accept_more_data as rb::Method0,
|
|
554
|
+
)?;
|
|
555
|
+
rb::define_method(
|
|
556
|
+
decompressor,
|
|
557
|
+
c"unused_data",
|
|
558
|
+
decompressor_unused_data as rb::Method0,
|
|
559
|
+
)?;
|
|
560
|
+
Ok(())
|
|
561
|
+
}
|