vibe_zstd 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.standard.yml +3 -0
- data/CHANGELOG.md +22 -0
- data/LICENSE.txt +21 -0
- data/README.md +978 -0
- data/Rakefile +20 -0
- data/benchmark/README.md +198 -0
- data/benchmark/compression_levels.rb +99 -0
- data/benchmark/context_reuse.rb +174 -0
- data/benchmark/decompression_speed_by_level.rb +65 -0
- data/benchmark/dictionary_training.rb +182 -0
- data/benchmark/dictionary_usage.rb +121 -0
- data/benchmark/for_readme.rb +157 -0
- data/benchmark/generate_fixture.rb +82 -0
- data/benchmark/helpers.rb +237 -0
- data/benchmark/multithreading.rb +105 -0
- data/benchmark/run_all.rb +150 -0
- data/benchmark/streaming.rb +154 -0
- data/ext/vibe_zstd/Makefile +270 -0
- data/ext/vibe_zstd/cctx.c +565 -0
- data/ext/vibe_zstd/dctx.c +493 -0
- data/ext/vibe_zstd/dict.c +587 -0
- data/ext/vibe_zstd/extconf.rb +52 -0
- data/ext/vibe_zstd/frames.c +132 -0
- data/ext/vibe_zstd/libzstd/LICENSE +30 -0
- data/ext/vibe_zstd/libzstd/common/allocations.h +55 -0
- data/ext/vibe_zstd/libzstd/common/bits.h +205 -0
- data/ext/vibe_zstd/libzstd/common/bitstream.h +454 -0
- data/ext/vibe_zstd/libzstd/common/compiler.h +464 -0
- data/ext/vibe_zstd/libzstd/common/cpu.h +249 -0
- data/ext/vibe_zstd/libzstd/common/debug.c +30 -0
- data/ext/vibe_zstd/libzstd/common/debug.h +107 -0
- data/ext/vibe_zstd/libzstd/common/entropy_common.c +340 -0
- data/ext/vibe_zstd/libzstd/common/error_private.c +64 -0
- data/ext/vibe_zstd/libzstd/common/error_private.h +158 -0
- data/ext/vibe_zstd/libzstd/common/fse.h +625 -0
- data/ext/vibe_zstd/libzstd/common/fse_decompress.c +315 -0
- data/ext/vibe_zstd/libzstd/common/huf.h +277 -0
- data/ext/vibe_zstd/libzstd/common/mem.h +422 -0
- data/ext/vibe_zstd/libzstd/common/pool.c +371 -0
- data/ext/vibe_zstd/libzstd/common/pool.h +81 -0
- data/ext/vibe_zstd/libzstd/common/portability_macros.h +171 -0
- data/ext/vibe_zstd/libzstd/common/threading.c +182 -0
- data/ext/vibe_zstd/libzstd/common/threading.h +142 -0
- data/ext/vibe_zstd/libzstd/common/xxhash.c +18 -0
- data/ext/vibe_zstd/libzstd/common/xxhash.h +7094 -0
- data/ext/vibe_zstd/libzstd/common/zstd_common.c +48 -0
- data/ext/vibe_zstd/libzstd/common/zstd_deps.h +123 -0
- data/ext/vibe_zstd/libzstd/common/zstd_internal.h +324 -0
- data/ext/vibe_zstd/libzstd/common/zstd_trace.h +156 -0
- data/ext/vibe_zstd/libzstd/compress/clevels.h +134 -0
- data/ext/vibe_zstd/libzstd/compress/fse_compress.c +625 -0
- data/ext/vibe_zstd/libzstd/compress/hist.c +191 -0
- data/ext/vibe_zstd/libzstd/compress/hist.h +82 -0
- data/ext/vibe_zstd/libzstd/compress/huf_compress.c +1464 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_compress.c +7843 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_compress_internal.h +1636 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_compress_literals.c +235 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_compress_literals.h +39 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_compress_sequences.c +442 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_compress_sequences.h +55 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_compress_superblock.c +688 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_compress_superblock.h +32 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_cwksp.h +765 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_double_fast.c +778 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_double_fast.h +42 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_fast.c +985 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_fast.h +30 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_lazy.c +2199 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_lazy.h +193 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_ldm.c +745 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_ldm.h +109 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_ldm_geartab.h +106 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_opt.c +1580 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_opt.h +72 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_preSplit.c +238 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_preSplit.h +33 -0
- data/ext/vibe_zstd/libzstd/compress/zstdmt_compress.c +1923 -0
- data/ext/vibe_zstd/libzstd/compress/zstdmt_compress.h +102 -0
- data/ext/vibe_zstd/libzstd/decompress/huf_decompress.c +1944 -0
- data/ext/vibe_zstd/libzstd/decompress/huf_decompress_amd64.S +602 -0
- data/ext/vibe_zstd/libzstd/decompress/zstd_ddict.c +244 -0
- data/ext/vibe_zstd/libzstd/decompress/zstd_ddict.h +44 -0
- data/ext/vibe_zstd/libzstd/decompress/zstd_decompress.c +2410 -0
- data/ext/vibe_zstd/libzstd/decompress/zstd_decompress_block.c +2209 -0
- data/ext/vibe_zstd/libzstd/decompress/zstd_decompress_block.h +73 -0
- data/ext/vibe_zstd/libzstd/decompress/zstd_decompress_internal.h +240 -0
- data/ext/vibe_zstd/libzstd/deprecated/zbuff.h +214 -0
- data/ext/vibe_zstd/libzstd/deprecated/zbuff_common.c +26 -0
- data/ext/vibe_zstd/libzstd/deprecated/zbuff_compress.c +167 -0
- data/ext/vibe_zstd/libzstd/deprecated/zbuff_decompress.c +77 -0
- data/ext/vibe_zstd/libzstd/dictBuilder/cover.c +1302 -0
- data/ext/vibe_zstd/libzstd/dictBuilder/cover.h +152 -0
- data/ext/vibe_zstd/libzstd/dictBuilder/divsufsort.c +1913 -0
- data/ext/vibe_zstd/libzstd/dictBuilder/divsufsort.h +57 -0
- data/ext/vibe_zstd/libzstd/dictBuilder/fastcover.c +766 -0
- data/ext/vibe_zstd/libzstd/dictBuilder/zdict.c +1133 -0
- data/ext/vibe_zstd/libzstd/zdict.h +481 -0
- data/ext/vibe_zstd/libzstd/zstd.h +3198 -0
- data/ext/vibe_zstd/libzstd/zstd_errors.h +107 -0
- data/ext/vibe_zstd/streaming.c +410 -0
- data/ext/vibe_zstd/vibe_zstd.c +293 -0
- data/ext/vibe_zstd/vibe_zstd.h +56 -0
- data/ext/vibe_zstd/vibe_zstd_internal.h +27 -0
- data/lib/vibe_zstd/constants.rb +67 -0
- data/lib/vibe_zstd/version.rb +5 -0
- data/lib/vibe_zstd.rb +255 -0
- data/sig/vibe_zstd.rbs +76 -0
- metadata +179 -0
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
#include "vibe_zstd_internal.h"
|
|
2
|
+
#include <ruby/thread.h>
|
|
3
|
+
#define ZDICT_STATIC_LINKING_ONLY
|
|
4
|
+
#include <zdict.h>
|
|
5
|
+
|
|
6
|
+
// Ruby module and class handles
|
|
7
|
+
VALUE rb_mVibeZstd;
|
|
8
|
+
VALUE rb_cVibeZstdCCtx;
|
|
9
|
+
VALUE rb_cVibeZstdDCtx;
|
|
10
|
+
VALUE rb_cVibeZstdCDict;
|
|
11
|
+
VALUE rb_cVibeZstdDDict;
|
|
12
|
+
VALUE rb_cVibeZstdCompressWriter;
|
|
13
|
+
VALUE rb_cVibeZstdDecompressReader;
|
|
14
|
+
|
|
15
|
+
// Forward declarations for free and mark functions
|
|
16
|
+
static void vibe_zstd_cctx_free(void* ptr);
|
|
17
|
+
static void vibe_zstd_dctx_free(void* ptr);
|
|
18
|
+
static void vibe_zstd_cdict_free(void* ptr);
|
|
19
|
+
static void vibe_zstd_ddict_free(void* ptr);
|
|
20
|
+
static void vibe_zstd_cstream_free(void* ptr);
|
|
21
|
+
static void vibe_zstd_cstream_mark(void* ptr);
|
|
22
|
+
static void vibe_zstd_dstream_free(void* ptr);
|
|
23
|
+
static void vibe_zstd_dstream_mark(void* ptr);
|
|
24
|
+
|
|
25
|
+
// TypedData type definitions (these are referenced by extern in the split files)
|
|
26
|
+
rb_data_type_t vibe_zstd_cctx_type = {
|
|
27
|
+
.wrap_struct_name = "vibe_zstd_cctx",
|
|
28
|
+
.function = {
|
|
29
|
+
.dmark = NULL,
|
|
30
|
+
.dfree = (RUBY_DATA_FUNC)vibe_zstd_cctx_free,
|
|
31
|
+
.dsize = NULL,
|
|
32
|
+
},
|
|
33
|
+
.data = NULL,
|
|
34
|
+
.flags = RUBY_TYPED_FREE_IMMEDIATELY,
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
rb_data_type_t vibe_zstd_dctx_type = {
|
|
38
|
+
.wrap_struct_name = "vibe_zstd_dctx",
|
|
39
|
+
.function = {
|
|
40
|
+
.dmark = NULL,
|
|
41
|
+
.dfree = (RUBY_DATA_FUNC)vibe_zstd_dctx_free,
|
|
42
|
+
.dsize = NULL,
|
|
43
|
+
},
|
|
44
|
+
.data = NULL,
|
|
45
|
+
.flags = RUBY_TYPED_FREE_IMMEDIATELY,
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
rb_data_type_t vibe_zstd_cdict_type = {
|
|
49
|
+
.wrap_struct_name = "vibe_zstd_cdict",
|
|
50
|
+
.function = {
|
|
51
|
+
.dmark = NULL,
|
|
52
|
+
.dfree = (RUBY_DATA_FUNC)vibe_zstd_cdict_free,
|
|
53
|
+
.dsize = NULL,
|
|
54
|
+
},
|
|
55
|
+
.data = NULL,
|
|
56
|
+
.flags = RUBY_TYPED_FREE_IMMEDIATELY,
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
rb_data_type_t vibe_zstd_ddict_type = {
|
|
60
|
+
.wrap_struct_name = "vibe_zstd_ddict",
|
|
61
|
+
.function = {
|
|
62
|
+
.dmark = NULL,
|
|
63
|
+
.dfree = (RUBY_DATA_FUNC)vibe_zstd_ddict_free,
|
|
64
|
+
.dsize = NULL,
|
|
65
|
+
},
|
|
66
|
+
.data = NULL,
|
|
67
|
+
.flags = RUBY_TYPED_FREE_IMMEDIATELY,
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
rb_data_type_t vibe_zstd_cstream_type = {
|
|
71
|
+
.wrap_struct_name = "vibe_zstd_cstream",
|
|
72
|
+
.function = {
|
|
73
|
+
.dmark = (RUBY_DATA_FUNC)vibe_zstd_cstream_mark,
|
|
74
|
+
.dfree = (RUBY_DATA_FUNC)vibe_zstd_cstream_free,
|
|
75
|
+
.dsize = NULL,
|
|
76
|
+
},
|
|
77
|
+
.data = NULL,
|
|
78
|
+
.flags = RUBY_TYPED_FREE_IMMEDIATELY,
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
rb_data_type_t vibe_zstd_dstream_type = {
|
|
82
|
+
.wrap_struct_name = "vibe_zstd_dstream",
|
|
83
|
+
.function = {
|
|
84
|
+
.dmark = (RUBY_DATA_FUNC)vibe_zstd_dstream_mark,
|
|
85
|
+
.dfree = (RUBY_DATA_FUNC)vibe_zstd_dstream_free,
|
|
86
|
+
.dsize = NULL,
|
|
87
|
+
},
|
|
88
|
+
.data = NULL,
|
|
89
|
+
.flags = RUBY_TYPED_FREE_IMMEDIATELY,
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
// Free functions
|
|
93
|
+
static void
|
|
94
|
+
vibe_zstd_cctx_free(void* ptr) {
|
|
95
|
+
vibe_zstd_cctx* cctx = ptr;
|
|
96
|
+
if (cctx->cctx) {
|
|
97
|
+
ZSTD_freeCCtx(cctx->cctx);
|
|
98
|
+
}
|
|
99
|
+
ruby_xfree(cctx);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
static void
|
|
103
|
+
vibe_zstd_dctx_free(void* ptr) {
|
|
104
|
+
vibe_zstd_dctx* dctx = ptr;
|
|
105
|
+
if (dctx->dctx) {
|
|
106
|
+
ZSTD_freeDCtx(dctx->dctx);
|
|
107
|
+
}
|
|
108
|
+
ruby_xfree(dctx);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
static void
|
|
112
|
+
vibe_zstd_cdict_free(void* ptr) {
|
|
113
|
+
vibe_zstd_cdict* cdict = ptr;
|
|
114
|
+
if (cdict->cdict) {
|
|
115
|
+
ZSTD_freeCDict(cdict->cdict);
|
|
116
|
+
}
|
|
117
|
+
ruby_xfree(cdict);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
static void
|
|
121
|
+
vibe_zstd_ddict_free(void* ptr) {
|
|
122
|
+
vibe_zstd_ddict* ddict = ptr;
|
|
123
|
+
if (ddict->ddict) {
|
|
124
|
+
ZSTD_freeDDict(ddict->ddict);
|
|
125
|
+
}
|
|
126
|
+
ruby_xfree(ddict);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
static void
|
|
130
|
+
vibe_zstd_cstream_mark(void* ptr) {
|
|
131
|
+
vibe_zstd_cstream* cstream = ptr;
|
|
132
|
+
rb_gc_mark(cstream->io);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
static void
|
|
136
|
+
vibe_zstd_cstream_free(void* ptr) {
|
|
137
|
+
vibe_zstd_cstream* cstream = ptr;
|
|
138
|
+
if (cstream->cstream) {
|
|
139
|
+
ZSTD_freeCStream(cstream->cstream);
|
|
140
|
+
}
|
|
141
|
+
ruby_xfree(cstream);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
static void
|
|
145
|
+
vibe_zstd_dstream_mark(void* ptr) {
|
|
146
|
+
vibe_zstd_dstream* dstream = ptr;
|
|
147
|
+
rb_gc_mark(dstream->io);
|
|
148
|
+
rb_gc_mark(dstream->input_data);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
static void
|
|
152
|
+
vibe_zstd_dstream_free(void* ptr) {
|
|
153
|
+
vibe_zstd_dstream* dstream = ptr;
|
|
154
|
+
if (dstream->dstream) {
|
|
155
|
+
ZSTD_freeDStream(dstream->dstream);
|
|
156
|
+
}
|
|
157
|
+
ruby_xfree(dstream);
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// Alloc functions
|
|
161
|
+
static VALUE
|
|
162
|
+
vibe_zstd_cctx_alloc(VALUE klass) {
|
|
163
|
+
vibe_zstd_cctx* cctx = ALLOC(vibe_zstd_cctx);
|
|
164
|
+
cctx->cctx = ZSTD_createCCtx();
|
|
165
|
+
if (!cctx->cctx) {
|
|
166
|
+
ruby_xfree(cctx);
|
|
167
|
+
rb_raise(rb_eRuntimeError, "Failed to create ZSTD_CCtx");
|
|
168
|
+
}
|
|
169
|
+
return TypedData_Wrap_Struct(klass, &vibe_zstd_cctx_type, cctx);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
static VALUE
|
|
173
|
+
vibe_zstd_dctx_alloc(VALUE klass) {
|
|
174
|
+
vibe_zstd_dctx* dctx = ALLOC(vibe_zstd_dctx);
|
|
175
|
+
dctx->dctx = ZSTD_createDCtx();
|
|
176
|
+
if (!dctx->dctx) {
|
|
177
|
+
ruby_xfree(dctx);
|
|
178
|
+
rb_raise(rb_eRuntimeError, "Failed to create ZSTD_DCtx");
|
|
179
|
+
}
|
|
180
|
+
dctx->initial_capacity = 0; // 0 = use class default
|
|
181
|
+
return TypedData_Wrap_Struct(klass, &vibe_zstd_dctx_type, dctx);
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
static VALUE
|
|
185
|
+
vibe_zstd_cdict_alloc(VALUE klass) {
|
|
186
|
+
vibe_zstd_cdict* cdict = ALLOC(vibe_zstd_cdict);
|
|
187
|
+
cdict->cdict = NULL; // Will be set in initialize
|
|
188
|
+
return TypedData_Wrap_Struct(klass, &vibe_zstd_cdict_type, cdict);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
static VALUE
|
|
192
|
+
vibe_zstd_ddict_alloc(VALUE klass) {
|
|
193
|
+
vibe_zstd_ddict* ddict = ALLOC(vibe_zstd_ddict);
|
|
194
|
+
ddict->ddict = NULL; // Will be set in initialize
|
|
195
|
+
return TypedData_Wrap_Struct(klass, &vibe_zstd_ddict_type, ddict);
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
static VALUE
|
|
199
|
+
vibe_zstd_cstream_alloc(VALUE klass) {
|
|
200
|
+
vibe_zstd_cstream* cstream = ALLOC(vibe_zstd_cstream);
|
|
201
|
+
cstream->cstream = NULL;
|
|
202
|
+
cstream->io = Qnil;
|
|
203
|
+
return TypedData_Wrap_Struct(klass, &vibe_zstd_cstream_type, cstream);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
static VALUE
|
|
207
|
+
vibe_zstd_dstream_alloc(VALUE klass) {
|
|
208
|
+
vibe_zstd_dstream* dstream = ALLOC(vibe_zstd_dstream);
|
|
209
|
+
dstream->dstream = NULL;
|
|
210
|
+
dstream->io = Qnil;
|
|
211
|
+
dstream->input_data = Qnil;
|
|
212
|
+
dstream->input.src = NULL;
|
|
213
|
+
dstream->input.size = 0;
|
|
214
|
+
dstream->input.pos = 0;
|
|
215
|
+
return TypedData_Wrap_Struct(klass, &vibe_zstd_dstream_type, dstream);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// Module-level version and compression level functions
|
|
219
|
+
static VALUE
|
|
220
|
+
vibe_zstd_version_number(VALUE self) {
|
|
221
|
+
(void)self;
|
|
222
|
+
return UINT2NUM(ZSTD_versionNumber());
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
static VALUE
|
|
226
|
+
vibe_zstd_version_string(VALUE self) {
|
|
227
|
+
(void)self;
|
|
228
|
+
return rb_str_new_cstr(ZSTD_versionString());
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
static VALUE
|
|
232
|
+
vibe_zstd_min_c_level(VALUE self) {
|
|
233
|
+
(void)self;
|
|
234
|
+
return INT2NUM(ZSTD_minCLevel());
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
static VALUE
|
|
238
|
+
vibe_zstd_max_c_level(VALUE self) {
|
|
239
|
+
(void)self;
|
|
240
|
+
return INT2NUM(ZSTD_maxCLevel());
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
static VALUE
|
|
244
|
+
vibe_zstd_default_c_level(VALUE self) {
|
|
245
|
+
(void)self;
|
|
246
|
+
return INT2NUM(ZSTD_defaultCLevel());
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
// Include the split implementation files
|
|
250
|
+
#include "cctx.c"
|
|
251
|
+
#include "dctx.c"
|
|
252
|
+
#include "dict.c"
|
|
253
|
+
#include "streaming.c"
|
|
254
|
+
#include "frames.c"
|
|
255
|
+
|
|
256
|
+
// Main initialization function
|
|
257
|
+
RUBY_FUNC_EXPORTED void
|
|
258
|
+
Init_vibe_zstd(void)
|
|
259
|
+
{
|
|
260
|
+
// Initialize parameter lookup tables
|
|
261
|
+
init_cctx_param_table();
|
|
262
|
+
init_dctx_param_table();
|
|
263
|
+
|
|
264
|
+
rb_mVibeZstd = rb_define_module("VibeZstd");
|
|
265
|
+
|
|
266
|
+
// Define classes
|
|
267
|
+
rb_cVibeZstdCCtx = rb_define_class_under(rb_mVibeZstd, "CCtx", rb_cObject);
|
|
268
|
+
rb_cVibeZstdDCtx = rb_define_class_under(rb_mVibeZstd, "DCtx", rb_cObject);
|
|
269
|
+
rb_cVibeZstdCDict = rb_define_class_under(rb_mVibeZstd, "CDict", rb_cObject);
|
|
270
|
+
rb_cVibeZstdDDict = rb_define_class_under(rb_mVibeZstd, "DDict", rb_cObject);
|
|
271
|
+
rb_cVibeZstdCompressWriter = rb_define_class_under(rb_mVibeZstd, "CompressWriter", rb_cObject);
|
|
272
|
+
rb_cVibeZstdDecompressReader = rb_define_class_under(rb_mVibeZstd, "DecompressReader", rb_cObject);
|
|
273
|
+
|
|
274
|
+
// Initialize each subsystem
|
|
275
|
+
vibe_zstd_cctx_init_class(rb_cVibeZstdCCtx);
|
|
276
|
+
vibe_zstd_dctx_init_class(rb_cVibeZstdDCtx);
|
|
277
|
+
vibe_zstd_dict_init_classes(rb_cVibeZstdCDict, rb_cVibeZstdDDict);
|
|
278
|
+
vibe_zstd_dict_init_module_methods(rb_mVibeZstd);
|
|
279
|
+
vibe_zstd_streaming_init_classes(rb_cVibeZstdCompressWriter, rb_cVibeZstdDecompressReader);
|
|
280
|
+
vibe_zstd_frames_init_module_methods(rb_mVibeZstd);
|
|
281
|
+
|
|
282
|
+
// Module-level version information
|
|
283
|
+
rb_define_module_function(rb_mVibeZstd, "version_number", vibe_zstd_version_number, 0);
|
|
284
|
+
rb_define_module_function(rb_mVibeZstd, "version_string", vibe_zstd_version_string, 0);
|
|
285
|
+
rb_define_module_function(rb_mVibeZstd, "min_compression_level", vibe_zstd_min_c_level, 0);
|
|
286
|
+
rb_define_module_function(rb_mVibeZstd, "max_compression_level", vibe_zstd_max_c_level, 0);
|
|
287
|
+
rb_define_module_function(rb_mVibeZstd, "default_compression_level", vibe_zstd_default_c_level, 0);
|
|
288
|
+
|
|
289
|
+
// Aliases
|
|
290
|
+
rb_define_module_function(rb_mVibeZstd, "min_level", vibe_zstd_min_c_level, 0);
|
|
291
|
+
rb_define_module_function(rb_mVibeZstd, "max_level", vibe_zstd_max_c_level, 0);
|
|
292
|
+
rb_define_module_function(rb_mVibeZstd, "default_level", vibe_zstd_default_c_level, 0);
|
|
293
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
#ifndef VIBE_ZSTD_H
|
|
2
|
+
#define VIBE_ZSTD_H 1
|
|
3
|
+
|
|
4
|
+
#include "ruby.h"
|
|
5
|
+
#define ZSTD_STATIC_LINKING_ONLY
|
|
6
|
+
#include <zstd.h>
|
|
7
|
+
|
|
8
|
+
// TypedData structs
|
|
9
|
+
typedef struct {
|
|
10
|
+
ZSTD_CCtx* cctx;
|
|
11
|
+
} vibe_zstd_cctx;
|
|
12
|
+
|
|
13
|
+
typedef struct {
|
|
14
|
+
ZSTD_DCtx* dctx;
|
|
15
|
+
size_t initial_capacity; // Initial capacity for unknown-size decompression (0 = use class default)
|
|
16
|
+
} vibe_zstd_dctx;
|
|
17
|
+
|
|
18
|
+
typedef struct {
|
|
19
|
+
ZSTD_CDict* cdict;
|
|
20
|
+
} vibe_zstd_cdict;
|
|
21
|
+
|
|
22
|
+
typedef struct {
|
|
23
|
+
ZSTD_DDict* ddict;
|
|
24
|
+
} vibe_zstd_ddict;
|
|
25
|
+
|
|
26
|
+
typedef struct {
|
|
27
|
+
ZSTD_CStream* cstream;
|
|
28
|
+
VALUE io;
|
|
29
|
+
} vibe_zstd_cstream;
|
|
30
|
+
|
|
31
|
+
typedef struct {
|
|
32
|
+
ZSTD_DStream* dstream;
|
|
33
|
+
VALUE io;
|
|
34
|
+
ZSTD_inBuffer input; // Zstandard manages the buffer state
|
|
35
|
+
VALUE input_data; // Ruby string holding input data
|
|
36
|
+
int eof; // Flag to track if we've reached end of stream
|
|
37
|
+
size_t initial_chunk_size; // Initial chunk size for unbounded reads (0 = use default)
|
|
38
|
+
} vibe_zstd_dstream;
|
|
39
|
+
|
|
40
|
+
// TypedData types
|
|
41
|
+
extern rb_data_type_t vibe_zstd_cctx_type;
|
|
42
|
+
extern rb_data_type_t vibe_zstd_dctx_type;
|
|
43
|
+
extern rb_data_type_t vibe_zstd_cdict_type;
|
|
44
|
+
extern rb_data_type_t vibe_zstd_ddict_type;
|
|
45
|
+
extern rb_data_type_t vibe_zstd_cstream_type;
|
|
46
|
+
extern rb_data_type_t vibe_zstd_dstream_type;
|
|
47
|
+
|
|
48
|
+
// Ruby classes and modules
|
|
49
|
+
extern VALUE rb_cVibeZstdCCtx;
|
|
50
|
+
extern VALUE rb_cVibeZstdDCtx;
|
|
51
|
+
extern VALUE rb_cVibeZstdCDict;
|
|
52
|
+
extern VALUE rb_cVibeZstdDDict;
|
|
53
|
+
extern VALUE rb_cVibeZstdCompressWriter;
|
|
54
|
+
extern VALUE rb_cVibeZstdDecompressReader;
|
|
55
|
+
|
|
56
|
+
#endif /* VIBE_ZSTD_H */
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
#ifndef VIBE_ZSTD_INTERNAL_H
|
|
2
|
+
#define VIBE_ZSTD_INTERNAL_H 1
|
|
3
|
+
|
|
4
|
+
#include "vibe_zstd.h"
|
|
5
|
+
#include <ruby/thread.h>
|
|
6
|
+
#define ZDICT_STATIC_LINKING_ONLY
|
|
7
|
+
#include <zdict.h>
|
|
8
|
+
|
|
9
|
+
// Function declarations for cross-file usage
|
|
10
|
+
|
|
11
|
+
// CCtx functions (cctx.c)
|
|
12
|
+
void vibe_zstd_cctx_init_class(VALUE rb_cVibeZstdCCtx);
|
|
13
|
+
|
|
14
|
+
// DCtx functions (dctx.c)
|
|
15
|
+
void vibe_zstd_dctx_init_class(VALUE rb_cVibeZstdDCtx);
|
|
16
|
+
|
|
17
|
+
// Dictionary functions (dict.c)
|
|
18
|
+
void vibe_zstd_dict_init_classes(VALUE rb_cVibeZstdCDict, VALUE rb_cVibeZstdDDict);
|
|
19
|
+
void vibe_zstd_dict_init_module_methods(VALUE rb_mVibeZstd);
|
|
20
|
+
|
|
21
|
+
// Streaming functions (streaming.c)
|
|
22
|
+
void vibe_zstd_streaming_init_classes(VALUE rb_cVibeZstdCompressWriter, VALUE rb_cVibeZstdDecompressReader);
|
|
23
|
+
|
|
24
|
+
// Frame utility functions (frames.c)
|
|
25
|
+
void vibe_zstd_frames_init_module_methods(VALUE rb_mVibeZstd);
|
|
26
|
+
|
|
27
|
+
#endif /* VIBE_ZSTD_INTERNAL_H */
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module VibeZstd
|
|
4
|
+
# Frame format constants for compression and decompression
|
|
5
|
+
module Format
|
|
6
|
+
# Standard zstd frame format with magic number (default)
|
|
7
|
+
STANDARD = 0
|
|
8
|
+
|
|
9
|
+
# Zstd frame format without initial 4-byte magic number
|
|
10
|
+
# Useful to save 4 bytes per frame, but decoder must be explicitly configured
|
|
11
|
+
MAGICLESS = 1
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# Compression strategy constants, listed from fastest to strongest
|
|
15
|
+
module Strategy
|
|
16
|
+
FAST = 1
|
|
17
|
+
DFAST = 2
|
|
18
|
+
GREEDY = 3
|
|
19
|
+
LAZY = 4
|
|
20
|
+
LAZY2 = 5
|
|
21
|
+
BTLAZY2 = 6
|
|
22
|
+
BTOPT = 7
|
|
23
|
+
BTULTRA = 8
|
|
24
|
+
BTULTRA2 = 9
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# Literal compression mode constants
|
|
28
|
+
module LiteralCompressionMode
|
|
29
|
+
# Automatically determine based on compression level
|
|
30
|
+
# Negative levels = uncompressed, positive levels = compressed
|
|
31
|
+
AUTO = 0
|
|
32
|
+
|
|
33
|
+
# Always attempt Huffman compression
|
|
34
|
+
# Uncompressed literals still emitted if not profitable
|
|
35
|
+
HUFFMAN = 1
|
|
36
|
+
|
|
37
|
+
# Always emit uncompressed literals
|
|
38
|
+
UNCOMPRESSED = 2
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Dictionary attachment preference constants
|
|
42
|
+
module DictAttachPref
|
|
43
|
+
# Automatically choose attachment method (default)
|
|
44
|
+
AUTO = 0
|
|
45
|
+
|
|
46
|
+
# Force dictionary to be attached by reference
|
|
47
|
+
FORCE_ATTACH = 1
|
|
48
|
+
|
|
49
|
+
# Force dictionary to be copied into working context
|
|
50
|
+
FORCE_COPY = 2
|
|
51
|
+
|
|
52
|
+
# Force dictionary to be reloaded
|
|
53
|
+
FORCE_LOAD = 3
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Context reset modes
|
|
57
|
+
module ResetDirective
|
|
58
|
+
# Reset session only, keep parameters
|
|
59
|
+
SESSION = 1
|
|
60
|
+
|
|
61
|
+
# Reset parameters to defaults, keep session state
|
|
62
|
+
PARAMETERS = 2
|
|
63
|
+
|
|
64
|
+
# Reset both session and parameters (full reset)
|
|
65
|
+
BOTH = 3
|
|
66
|
+
end
|
|
67
|
+
end
|
data/lib/vibe_zstd.rb
ADDED
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "vibe_zstd/version"
|
|
4
|
+
require_relative "vibe_zstd/vibe_zstd"
|
|
5
|
+
require_relative "vibe_zstd/constants"
|
|
6
|
+
|
|
7
|
+
module VibeZstd
|
|
8
|
+
class Error < StandardError; end
|
|
9
|
+
|
|
10
|
+
# Convenience method for one-off compression
|
|
11
|
+
# Supports all CCtx#compress options: level, dict, pledged_size
|
|
12
|
+
def self.compress(data, **options)
|
|
13
|
+
cctx = CCtx.new
|
|
14
|
+
cctx.compress(data, **options)
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# Convenience method for one-off decompression
|
|
18
|
+
# Supports all DCtx#decompress options: dict, initial_capacity
|
|
19
|
+
def self.decompress(data, **options)
|
|
20
|
+
dctx = DCtx.new
|
|
21
|
+
dctx.decompress(data, **options)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Get the decompressed content size from a compressed frame
|
|
25
|
+
# Returns nil if size is unknown or data is invalid
|
|
26
|
+
def self.frame_content_size(data)
|
|
27
|
+
DCtx.frame_content_size(data)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Iterate over all skippable frames in the data
|
|
31
|
+
# Yields [content, magic_variant, offset] for each skippable frame
|
|
32
|
+
def self.each_skippable_frame(data)
|
|
33
|
+
return enum_for(:each_skippable_frame, data) unless block_given?
|
|
34
|
+
|
|
35
|
+
offset = 0
|
|
36
|
+
while offset < data.bytesize
|
|
37
|
+
frame_data = data.byteslice(offset..-1)
|
|
38
|
+
frame_size = find_frame_compressed_size(frame_data)
|
|
39
|
+
|
|
40
|
+
# Defense: Prevent infinite loop on malformed data
|
|
41
|
+
# A valid frame must have non-zero size (at minimum: frame header)
|
|
42
|
+
if frame_size <= 0
|
|
43
|
+
raise Error, "Invalid frame: zero or negative size at offset #{offset}"
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
if skippable_frame?(frame_data)
|
|
47
|
+
content, magic_variant = read_skippable_frame(frame_data)
|
|
48
|
+
yield content, magic_variant, offset
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
offset += frame_size
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Convenient aliases for version/level methods
|
|
56
|
+
class << self
|
|
57
|
+
alias_method :min_level, :min_compression_level
|
|
58
|
+
alias_method :max_level, :max_compression_level
|
|
59
|
+
alias_method :default_level, :default_compression_level
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Add helper method to CDict for creating matching DDict
|
|
63
|
+
class CDict
|
|
64
|
+
# Get or create a matching DDict from this CDict's dictionary data
|
|
65
|
+
# The DDict is cached so it's only created once
|
|
66
|
+
#
|
|
67
|
+
# @return [DDict] Decompression dictionary matching this compression dictionary
|
|
68
|
+
def to_ddict
|
|
69
|
+
@ddict ||= DDict.new(@dict_data)
|
|
70
|
+
end
|
|
71
|
+
alias_method :ddict, :to_ddict
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# Thread-local context pooling for high-performance reuse
|
|
75
|
+
# Ideal for Rails/Puma applications where threads are reused across requests
|
|
76
|
+
#
|
|
77
|
+
# Example usage:
|
|
78
|
+
# # In Rails model with encrypted attributes
|
|
79
|
+
# class User < ApplicationRecord
|
|
80
|
+
# encrypts :email
|
|
81
|
+
# encrypts :preferences, dict: JSON_DICT
|
|
82
|
+
# end
|
|
83
|
+
#
|
|
84
|
+
# # Instead of:
|
|
85
|
+
# VibeZstd.decompress(data, dict: dict) # Creates new DCtx each time
|
|
86
|
+
#
|
|
87
|
+
# # Use:
|
|
88
|
+
# VibeZstd::ThreadLocal.decompress(data, dict: dict) # Reuses DCtx per thread
|
|
89
|
+
#
|
|
90
|
+
# Memory footprint: ~128KB per DCtx × unique dictionaries × threads
|
|
91
|
+
# Example: 3 dicts × 5 Puma threads = 1.9MB total
|
|
92
|
+
#
|
|
93
|
+
# Note: Only supports per-operation parameters (level, dict, pledged_size, initial_capacity)
|
|
94
|
+
# Does NOT support context-level settings (nb_workers, checksum_flag, etc.)
|
|
95
|
+
module ThreadLocal
|
|
96
|
+
# Compress data using thread-local context pool
|
|
97
|
+
# Contexts are keyed by dictionary ID for automatic isolation
|
|
98
|
+
#
|
|
99
|
+
# @param data [String] Data to compress
|
|
100
|
+
# @param level [Integer] Compression level (per-operation, can vary)
|
|
101
|
+
# @param dict [CDict] Compression dictionary (optional)
|
|
102
|
+
# @param pledged_size [Integer] Expected input size (optional)
|
|
103
|
+
# @return [String] Compressed data
|
|
104
|
+
def self.compress(data, level: nil, dict: nil, pledged_size: nil)
|
|
105
|
+
# Key by dictionary ID, or :default if no dict
|
|
106
|
+
key = dict ? dict.dict_id : :default
|
|
107
|
+
|
|
108
|
+
# Get or create thread-local context pool
|
|
109
|
+
Thread.current[:vibe_zstd_cctx_pool] ||= {}
|
|
110
|
+
cctx = Thread.current[:vibe_zstd_cctx_pool][key] ||= VibeZstd::CCtx.new
|
|
111
|
+
|
|
112
|
+
# Build options hash
|
|
113
|
+
options = {}
|
|
114
|
+
options[:level] = level if level
|
|
115
|
+
options[:dict] = dict if dict
|
|
116
|
+
options[:pledged_size] = pledged_size if pledged_size
|
|
117
|
+
|
|
118
|
+
cctx.compress(data, **options)
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# Decompress data using thread-local context pool
|
|
122
|
+
# Contexts are keyed by dictionary ID for automatic isolation
|
|
123
|
+
#
|
|
124
|
+
# @param data [String] Data to decompress
|
|
125
|
+
# @param dict [DDict] Decompression dictionary (optional)
|
|
126
|
+
# @param initial_capacity [Integer] Initial buffer size for unknown-size frames (optional)
|
|
127
|
+
# @return [String] Decompressed data
|
|
128
|
+
def self.decompress(data, dict: nil, initial_capacity: nil)
|
|
129
|
+
key = dict ? dict.dict_id : :default
|
|
130
|
+
|
|
131
|
+
# Get or create thread-local context pool
|
|
132
|
+
Thread.current[:vibe_zstd_dctx_pool] ||= {}
|
|
133
|
+
dctx = Thread.current[:vibe_zstd_dctx_pool][key] ||= VibeZstd::DCtx.new
|
|
134
|
+
|
|
135
|
+
# Build options hash
|
|
136
|
+
options = {}
|
|
137
|
+
options[:dict] = dict if dict
|
|
138
|
+
options[:initial_capacity] = initial_capacity if initial_capacity
|
|
139
|
+
|
|
140
|
+
# C code will validate dict matches frame requirements
|
|
141
|
+
dctx.decompress(data, **options)
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# Clear all thread-local context pools for the current thread
|
|
145
|
+
# Useful for testing or explicit memory management
|
|
146
|
+
def self.clear_thread_cache!
|
|
147
|
+
Thread.current[:vibe_zstd_cctx_pool] = {}
|
|
148
|
+
Thread.current[:vibe_zstd_dctx_pool] = {}
|
|
149
|
+
nil
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# Get statistics about the current thread's context pools
|
|
153
|
+
# @return [Hash] Pool statistics
|
|
154
|
+
def self.thread_cache_stats
|
|
155
|
+
{
|
|
156
|
+
compression_contexts: Thread.current[:vibe_zstd_cctx_pool]&.size || 0,
|
|
157
|
+
decompression_contexts: Thread.current[:vibe_zstd_dctx_pool]&.size || 0,
|
|
158
|
+
compression_keys: Thread.current[:vibe_zstd_cctx_pool]&.keys || [],
|
|
159
|
+
decompression_keys: Thread.current[:vibe_zstd_dctx_pool]&.keys || []
|
|
160
|
+
}
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
class CompressWriter
|
|
165
|
+
# Block-based resource management
|
|
166
|
+
# Automatically calls finish when block completes
|
|
167
|
+
def self.open(io, **options)
|
|
168
|
+
writer = new(io, **options)
|
|
169
|
+
return writer unless block_given?
|
|
170
|
+
|
|
171
|
+
begin
|
|
172
|
+
yield writer
|
|
173
|
+
ensure
|
|
174
|
+
writer.finish
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
class DecompressReader
|
|
180
|
+
include Enumerable
|
|
181
|
+
|
|
182
|
+
# Block-based resource management
|
|
183
|
+
# Automatically cleans up when block completes
|
|
184
|
+
def self.open(io, **options)
|
|
185
|
+
reader = new(io, **options)
|
|
186
|
+
return reader unless block_given?
|
|
187
|
+
|
|
188
|
+
yield reader
|
|
189
|
+
|
|
190
|
+
# Reader doesn't have finish, but this ensures cleanup
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
# Read all remaining data
|
|
194
|
+
def read_all
|
|
195
|
+
chunks = []
|
|
196
|
+
while (chunk = read)
|
|
197
|
+
chunks << chunk
|
|
198
|
+
end
|
|
199
|
+
chunks.join
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
# Alias for eof?
|
|
203
|
+
def eof
|
|
204
|
+
eof?
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
# Iterate over chunks (required for Enumerable)
|
|
208
|
+
def each(chunk_size = nil)
|
|
209
|
+
return enum_for(:each, chunk_size) unless block_given?
|
|
210
|
+
|
|
211
|
+
until eof?
|
|
212
|
+
chunk = read(chunk_size)
|
|
213
|
+
yield chunk if chunk
|
|
214
|
+
end
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
# Read a single line (up to newline or EOF)
|
|
218
|
+
def gets(sep = $/)
|
|
219
|
+
return nil if eof?
|
|
220
|
+
|
|
221
|
+
line = +""
|
|
222
|
+
until eof?
|
|
223
|
+
chunk = read(1)
|
|
224
|
+
break unless chunk
|
|
225
|
+
|
|
226
|
+
line << chunk
|
|
227
|
+
break if chunk.end_with?(sep)
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
line.empty? ? nil : line
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
# Iterate over lines
|
|
234
|
+
def each_line(sep = $/)
|
|
235
|
+
return enum_for(:each_line, sep) unless block_given?
|
|
236
|
+
|
|
237
|
+
while (line = gets(sep))
|
|
238
|
+
yield line
|
|
239
|
+
end
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
# Alias for gets
|
|
243
|
+
alias_method :readline, :gets
|
|
244
|
+
|
|
245
|
+
# Read exactly n bytes, or raise EOFError
|
|
246
|
+
def readpartial(maxlen)
|
|
247
|
+
raise EOFError, "end of file reached" if eof?
|
|
248
|
+
|
|
249
|
+
data = read(maxlen)
|
|
250
|
+
raise EOFError, "end of file reached" if data.nil?
|
|
251
|
+
|
|
252
|
+
data
|
|
253
|
+
end
|
|
254
|
+
end
|
|
255
|
+
end
|