vibe_zstd 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. checksums.yaml +7 -0
  2. data/.standard.yml +3 -0
  3. data/CHANGELOG.md +22 -0
  4. data/LICENSE.txt +21 -0
  5. data/README.md +978 -0
  6. data/Rakefile +20 -0
  7. data/benchmark/README.md +198 -0
  8. data/benchmark/compression_levels.rb +99 -0
  9. data/benchmark/context_reuse.rb +174 -0
  10. data/benchmark/decompression_speed_by_level.rb +65 -0
  11. data/benchmark/dictionary_training.rb +182 -0
  12. data/benchmark/dictionary_usage.rb +121 -0
  13. data/benchmark/for_readme.rb +157 -0
  14. data/benchmark/generate_fixture.rb +82 -0
  15. data/benchmark/helpers.rb +237 -0
  16. data/benchmark/multithreading.rb +105 -0
  17. data/benchmark/run_all.rb +150 -0
  18. data/benchmark/streaming.rb +154 -0
  19. data/ext/vibe_zstd/Makefile +270 -0
  20. data/ext/vibe_zstd/cctx.c +565 -0
  21. data/ext/vibe_zstd/dctx.c +493 -0
  22. data/ext/vibe_zstd/dict.c +587 -0
  23. data/ext/vibe_zstd/extconf.rb +52 -0
  24. data/ext/vibe_zstd/frames.c +132 -0
  25. data/ext/vibe_zstd/libzstd/LICENSE +30 -0
  26. data/ext/vibe_zstd/libzstd/common/allocations.h +55 -0
  27. data/ext/vibe_zstd/libzstd/common/bits.h +205 -0
  28. data/ext/vibe_zstd/libzstd/common/bitstream.h +454 -0
  29. data/ext/vibe_zstd/libzstd/common/compiler.h +464 -0
  30. data/ext/vibe_zstd/libzstd/common/cpu.h +249 -0
  31. data/ext/vibe_zstd/libzstd/common/debug.c +30 -0
  32. data/ext/vibe_zstd/libzstd/common/debug.h +107 -0
  33. data/ext/vibe_zstd/libzstd/common/entropy_common.c +340 -0
  34. data/ext/vibe_zstd/libzstd/common/error_private.c +64 -0
  35. data/ext/vibe_zstd/libzstd/common/error_private.h +158 -0
  36. data/ext/vibe_zstd/libzstd/common/fse.h +625 -0
  37. data/ext/vibe_zstd/libzstd/common/fse_decompress.c +315 -0
  38. data/ext/vibe_zstd/libzstd/common/huf.h +277 -0
  39. data/ext/vibe_zstd/libzstd/common/mem.h +422 -0
  40. data/ext/vibe_zstd/libzstd/common/pool.c +371 -0
  41. data/ext/vibe_zstd/libzstd/common/pool.h +81 -0
  42. data/ext/vibe_zstd/libzstd/common/portability_macros.h +171 -0
  43. data/ext/vibe_zstd/libzstd/common/threading.c +182 -0
  44. data/ext/vibe_zstd/libzstd/common/threading.h +142 -0
  45. data/ext/vibe_zstd/libzstd/common/xxhash.c +18 -0
  46. data/ext/vibe_zstd/libzstd/common/xxhash.h +7094 -0
  47. data/ext/vibe_zstd/libzstd/common/zstd_common.c +48 -0
  48. data/ext/vibe_zstd/libzstd/common/zstd_deps.h +123 -0
  49. data/ext/vibe_zstd/libzstd/common/zstd_internal.h +324 -0
  50. data/ext/vibe_zstd/libzstd/common/zstd_trace.h +156 -0
  51. data/ext/vibe_zstd/libzstd/compress/clevels.h +134 -0
  52. data/ext/vibe_zstd/libzstd/compress/fse_compress.c +625 -0
  53. data/ext/vibe_zstd/libzstd/compress/hist.c +191 -0
  54. data/ext/vibe_zstd/libzstd/compress/hist.h +82 -0
  55. data/ext/vibe_zstd/libzstd/compress/huf_compress.c +1464 -0
  56. data/ext/vibe_zstd/libzstd/compress/zstd_compress.c +7843 -0
  57. data/ext/vibe_zstd/libzstd/compress/zstd_compress_internal.h +1636 -0
  58. data/ext/vibe_zstd/libzstd/compress/zstd_compress_literals.c +235 -0
  59. data/ext/vibe_zstd/libzstd/compress/zstd_compress_literals.h +39 -0
  60. data/ext/vibe_zstd/libzstd/compress/zstd_compress_sequences.c +442 -0
  61. data/ext/vibe_zstd/libzstd/compress/zstd_compress_sequences.h +55 -0
  62. data/ext/vibe_zstd/libzstd/compress/zstd_compress_superblock.c +688 -0
  63. data/ext/vibe_zstd/libzstd/compress/zstd_compress_superblock.h +32 -0
  64. data/ext/vibe_zstd/libzstd/compress/zstd_cwksp.h +765 -0
  65. data/ext/vibe_zstd/libzstd/compress/zstd_double_fast.c +778 -0
  66. data/ext/vibe_zstd/libzstd/compress/zstd_double_fast.h +42 -0
  67. data/ext/vibe_zstd/libzstd/compress/zstd_fast.c +985 -0
  68. data/ext/vibe_zstd/libzstd/compress/zstd_fast.h +30 -0
  69. data/ext/vibe_zstd/libzstd/compress/zstd_lazy.c +2199 -0
  70. data/ext/vibe_zstd/libzstd/compress/zstd_lazy.h +193 -0
  71. data/ext/vibe_zstd/libzstd/compress/zstd_ldm.c +745 -0
  72. data/ext/vibe_zstd/libzstd/compress/zstd_ldm.h +109 -0
  73. data/ext/vibe_zstd/libzstd/compress/zstd_ldm_geartab.h +106 -0
  74. data/ext/vibe_zstd/libzstd/compress/zstd_opt.c +1580 -0
  75. data/ext/vibe_zstd/libzstd/compress/zstd_opt.h +72 -0
  76. data/ext/vibe_zstd/libzstd/compress/zstd_preSplit.c +238 -0
  77. data/ext/vibe_zstd/libzstd/compress/zstd_preSplit.h +33 -0
  78. data/ext/vibe_zstd/libzstd/compress/zstdmt_compress.c +1923 -0
  79. data/ext/vibe_zstd/libzstd/compress/zstdmt_compress.h +102 -0
  80. data/ext/vibe_zstd/libzstd/decompress/huf_decompress.c +1944 -0
  81. data/ext/vibe_zstd/libzstd/decompress/huf_decompress_amd64.S +602 -0
  82. data/ext/vibe_zstd/libzstd/decompress/zstd_ddict.c +244 -0
  83. data/ext/vibe_zstd/libzstd/decompress/zstd_ddict.h +44 -0
  84. data/ext/vibe_zstd/libzstd/decompress/zstd_decompress.c +2410 -0
  85. data/ext/vibe_zstd/libzstd/decompress/zstd_decompress_block.c +2209 -0
  86. data/ext/vibe_zstd/libzstd/decompress/zstd_decompress_block.h +73 -0
  87. data/ext/vibe_zstd/libzstd/decompress/zstd_decompress_internal.h +240 -0
  88. data/ext/vibe_zstd/libzstd/deprecated/zbuff.h +214 -0
  89. data/ext/vibe_zstd/libzstd/deprecated/zbuff_common.c +26 -0
  90. data/ext/vibe_zstd/libzstd/deprecated/zbuff_compress.c +167 -0
  91. data/ext/vibe_zstd/libzstd/deprecated/zbuff_decompress.c +77 -0
  92. data/ext/vibe_zstd/libzstd/dictBuilder/cover.c +1302 -0
  93. data/ext/vibe_zstd/libzstd/dictBuilder/cover.h +152 -0
  94. data/ext/vibe_zstd/libzstd/dictBuilder/divsufsort.c +1913 -0
  95. data/ext/vibe_zstd/libzstd/dictBuilder/divsufsort.h +57 -0
  96. data/ext/vibe_zstd/libzstd/dictBuilder/fastcover.c +766 -0
  97. data/ext/vibe_zstd/libzstd/dictBuilder/zdict.c +1133 -0
  98. data/ext/vibe_zstd/libzstd/zdict.h +481 -0
  99. data/ext/vibe_zstd/libzstd/zstd.h +3198 -0
  100. data/ext/vibe_zstd/libzstd/zstd_errors.h +107 -0
  101. data/ext/vibe_zstd/streaming.c +410 -0
  102. data/ext/vibe_zstd/vibe_zstd.c +293 -0
  103. data/ext/vibe_zstd/vibe_zstd.h +56 -0
  104. data/ext/vibe_zstd/vibe_zstd_internal.h +27 -0
  105. data/lib/vibe_zstd/constants.rb +67 -0
  106. data/lib/vibe_zstd/version.rb +5 -0
  107. data/lib/vibe_zstd.rb +255 -0
  108. data/sig/vibe_zstd.rbs +76 -0
  109. metadata +179 -0
@@ -0,0 +1,293 @@
1
+ #include "vibe_zstd_internal.h"
2
+ #include <ruby/thread.h>
3
+ #define ZDICT_STATIC_LINKING_ONLY
4
+ #include <zdict.h>
5
+
6
+ // Ruby module and class handles
7
+ VALUE rb_mVibeZstd;
8
+ VALUE rb_cVibeZstdCCtx;
9
+ VALUE rb_cVibeZstdDCtx;
10
+ VALUE rb_cVibeZstdCDict;
11
+ VALUE rb_cVibeZstdDDict;
12
+ VALUE rb_cVibeZstdCompressWriter;
13
+ VALUE rb_cVibeZstdDecompressReader;
14
+
15
+ // Forward declarations for free and mark functions
16
+ static void vibe_zstd_cctx_free(void* ptr);
17
+ static void vibe_zstd_dctx_free(void* ptr);
18
+ static void vibe_zstd_cdict_free(void* ptr);
19
+ static void vibe_zstd_ddict_free(void* ptr);
20
+ static void vibe_zstd_cstream_free(void* ptr);
21
+ static void vibe_zstd_cstream_mark(void* ptr);
22
+ static void vibe_zstd_dstream_free(void* ptr);
23
+ static void vibe_zstd_dstream_mark(void* ptr);
24
+
25
+ // TypedData type definitions (these are referenced by extern in the split files)
26
+ rb_data_type_t vibe_zstd_cctx_type = {
27
+ .wrap_struct_name = "vibe_zstd_cctx",
28
+ .function = {
29
+ .dmark = NULL,
30
+ .dfree = (RUBY_DATA_FUNC)vibe_zstd_cctx_free,
31
+ .dsize = NULL,
32
+ },
33
+ .data = NULL,
34
+ .flags = RUBY_TYPED_FREE_IMMEDIATELY,
35
+ };
36
+
37
+ rb_data_type_t vibe_zstd_dctx_type = {
38
+ .wrap_struct_name = "vibe_zstd_dctx",
39
+ .function = {
40
+ .dmark = NULL,
41
+ .dfree = (RUBY_DATA_FUNC)vibe_zstd_dctx_free,
42
+ .dsize = NULL,
43
+ },
44
+ .data = NULL,
45
+ .flags = RUBY_TYPED_FREE_IMMEDIATELY,
46
+ };
47
+
48
+ rb_data_type_t vibe_zstd_cdict_type = {
49
+ .wrap_struct_name = "vibe_zstd_cdict",
50
+ .function = {
51
+ .dmark = NULL,
52
+ .dfree = (RUBY_DATA_FUNC)vibe_zstd_cdict_free,
53
+ .dsize = NULL,
54
+ },
55
+ .data = NULL,
56
+ .flags = RUBY_TYPED_FREE_IMMEDIATELY,
57
+ };
58
+
59
+ rb_data_type_t vibe_zstd_ddict_type = {
60
+ .wrap_struct_name = "vibe_zstd_ddict",
61
+ .function = {
62
+ .dmark = NULL,
63
+ .dfree = (RUBY_DATA_FUNC)vibe_zstd_ddict_free,
64
+ .dsize = NULL,
65
+ },
66
+ .data = NULL,
67
+ .flags = RUBY_TYPED_FREE_IMMEDIATELY,
68
+ };
69
+
70
+ rb_data_type_t vibe_zstd_cstream_type = {
71
+ .wrap_struct_name = "vibe_zstd_cstream",
72
+ .function = {
73
+ .dmark = (RUBY_DATA_FUNC)vibe_zstd_cstream_mark,
74
+ .dfree = (RUBY_DATA_FUNC)vibe_zstd_cstream_free,
75
+ .dsize = NULL,
76
+ },
77
+ .data = NULL,
78
+ .flags = RUBY_TYPED_FREE_IMMEDIATELY,
79
+ };
80
+
81
+ rb_data_type_t vibe_zstd_dstream_type = {
82
+ .wrap_struct_name = "vibe_zstd_dstream",
83
+ .function = {
84
+ .dmark = (RUBY_DATA_FUNC)vibe_zstd_dstream_mark,
85
+ .dfree = (RUBY_DATA_FUNC)vibe_zstd_dstream_free,
86
+ .dsize = NULL,
87
+ },
88
+ .data = NULL,
89
+ .flags = RUBY_TYPED_FREE_IMMEDIATELY,
90
+ };
91
+
92
+ // Free functions
93
+ static void
94
+ vibe_zstd_cctx_free(void* ptr) {
95
+ vibe_zstd_cctx* cctx = ptr;
96
+ if (cctx->cctx) {
97
+ ZSTD_freeCCtx(cctx->cctx);
98
+ }
99
+ ruby_xfree(cctx);
100
+ }
101
+
102
+ static void
103
+ vibe_zstd_dctx_free(void* ptr) {
104
+ vibe_zstd_dctx* dctx = ptr;
105
+ if (dctx->dctx) {
106
+ ZSTD_freeDCtx(dctx->dctx);
107
+ }
108
+ ruby_xfree(dctx);
109
+ }
110
+
111
+ static void
112
+ vibe_zstd_cdict_free(void* ptr) {
113
+ vibe_zstd_cdict* cdict = ptr;
114
+ if (cdict->cdict) {
115
+ ZSTD_freeCDict(cdict->cdict);
116
+ }
117
+ ruby_xfree(cdict);
118
+ }
119
+
120
+ static void
121
+ vibe_zstd_ddict_free(void* ptr) {
122
+ vibe_zstd_ddict* ddict = ptr;
123
+ if (ddict->ddict) {
124
+ ZSTD_freeDDict(ddict->ddict);
125
+ }
126
+ ruby_xfree(ddict);
127
+ }
128
+
129
+ static void
130
+ vibe_zstd_cstream_mark(void* ptr) {
131
+ vibe_zstd_cstream* cstream = ptr;
132
+ rb_gc_mark(cstream->io);
133
+ }
134
+
135
+ static void
136
+ vibe_zstd_cstream_free(void* ptr) {
137
+ vibe_zstd_cstream* cstream = ptr;
138
+ if (cstream->cstream) {
139
+ ZSTD_freeCStream(cstream->cstream);
140
+ }
141
+ ruby_xfree(cstream);
142
+ }
143
+
144
+ static void
145
+ vibe_zstd_dstream_mark(void* ptr) {
146
+ vibe_zstd_dstream* dstream = ptr;
147
+ rb_gc_mark(dstream->io);
148
+ rb_gc_mark(dstream->input_data);
149
+ }
150
+
151
+ static void
152
+ vibe_zstd_dstream_free(void* ptr) {
153
+ vibe_zstd_dstream* dstream = ptr;
154
+ if (dstream->dstream) {
155
+ ZSTD_freeDStream(dstream->dstream);
156
+ }
157
+ ruby_xfree(dstream);
158
+ }
159
+
160
+ // Alloc functions
161
+ static VALUE
162
+ vibe_zstd_cctx_alloc(VALUE klass) {
163
+ vibe_zstd_cctx* cctx = ALLOC(vibe_zstd_cctx);
164
+ cctx->cctx = ZSTD_createCCtx();
165
+ if (!cctx->cctx) {
166
+ ruby_xfree(cctx);
167
+ rb_raise(rb_eRuntimeError, "Failed to create ZSTD_CCtx");
168
+ }
169
+ return TypedData_Wrap_Struct(klass, &vibe_zstd_cctx_type, cctx);
170
+ }
171
+
172
+ static VALUE
173
+ vibe_zstd_dctx_alloc(VALUE klass) {
174
+ vibe_zstd_dctx* dctx = ALLOC(vibe_zstd_dctx);
175
+ dctx->dctx = ZSTD_createDCtx();
176
+ if (!dctx->dctx) {
177
+ ruby_xfree(dctx);
178
+ rb_raise(rb_eRuntimeError, "Failed to create ZSTD_DCtx");
179
+ }
180
+ dctx->initial_capacity = 0; // 0 = use class default
181
+ return TypedData_Wrap_Struct(klass, &vibe_zstd_dctx_type, dctx);
182
+ }
183
+
184
+ static VALUE
185
+ vibe_zstd_cdict_alloc(VALUE klass) {
186
+ vibe_zstd_cdict* cdict = ALLOC(vibe_zstd_cdict);
187
+ cdict->cdict = NULL; // Will be set in initialize
188
+ return TypedData_Wrap_Struct(klass, &vibe_zstd_cdict_type, cdict);
189
+ }
190
+
191
+ static VALUE
192
+ vibe_zstd_ddict_alloc(VALUE klass) {
193
+ vibe_zstd_ddict* ddict = ALLOC(vibe_zstd_ddict);
194
+ ddict->ddict = NULL; // Will be set in initialize
195
+ return TypedData_Wrap_Struct(klass, &vibe_zstd_ddict_type, ddict);
196
+ }
197
+
198
+ static VALUE
199
+ vibe_zstd_cstream_alloc(VALUE klass) {
200
+ vibe_zstd_cstream* cstream = ALLOC(vibe_zstd_cstream);
201
+ cstream->cstream = NULL;
202
+ cstream->io = Qnil;
203
+ return TypedData_Wrap_Struct(klass, &vibe_zstd_cstream_type, cstream);
204
+ }
205
+
206
+ static VALUE
207
+ vibe_zstd_dstream_alloc(VALUE klass) {
208
+ vibe_zstd_dstream* dstream = ALLOC(vibe_zstd_dstream);
209
+ dstream->dstream = NULL;
210
+ dstream->io = Qnil;
211
+ dstream->input_data = Qnil;
212
+ dstream->input.src = NULL;
213
+ dstream->input.size = 0;
214
+ dstream->input.pos = 0;
215
+ return TypedData_Wrap_Struct(klass, &vibe_zstd_dstream_type, dstream);
216
+ }
217
+
218
+ // Module-level version and compression level functions
219
+ static VALUE
220
+ vibe_zstd_version_number(VALUE self) {
221
+ (void)self;
222
+ return UINT2NUM(ZSTD_versionNumber());
223
+ }
224
+
225
+ static VALUE
226
+ vibe_zstd_version_string(VALUE self) {
227
+ (void)self;
228
+ return rb_str_new_cstr(ZSTD_versionString());
229
+ }
230
+
231
+ static VALUE
232
+ vibe_zstd_min_c_level(VALUE self) {
233
+ (void)self;
234
+ return INT2NUM(ZSTD_minCLevel());
235
+ }
236
+
237
+ static VALUE
238
+ vibe_zstd_max_c_level(VALUE self) {
239
+ (void)self;
240
+ return INT2NUM(ZSTD_maxCLevel());
241
+ }
242
+
243
+ static VALUE
244
+ vibe_zstd_default_c_level(VALUE self) {
245
+ (void)self;
246
+ return INT2NUM(ZSTD_defaultCLevel());
247
+ }
248
+
249
+ // Include the split implementation files
250
+ #include "cctx.c"
251
+ #include "dctx.c"
252
+ #include "dict.c"
253
+ #include "streaming.c"
254
+ #include "frames.c"
255
+
256
+ // Main initialization function
257
+ RUBY_FUNC_EXPORTED void
258
+ Init_vibe_zstd(void)
259
+ {
260
+ // Initialize parameter lookup tables
261
+ init_cctx_param_table();
262
+ init_dctx_param_table();
263
+
264
+ rb_mVibeZstd = rb_define_module("VibeZstd");
265
+
266
+ // Define classes
267
+ rb_cVibeZstdCCtx = rb_define_class_under(rb_mVibeZstd, "CCtx", rb_cObject);
268
+ rb_cVibeZstdDCtx = rb_define_class_under(rb_mVibeZstd, "DCtx", rb_cObject);
269
+ rb_cVibeZstdCDict = rb_define_class_under(rb_mVibeZstd, "CDict", rb_cObject);
270
+ rb_cVibeZstdDDict = rb_define_class_under(rb_mVibeZstd, "DDict", rb_cObject);
271
+ rb_cVibeZstdCompressWriter = rb_define_class_under(rb_mVibeZstd, "CompressWriter", rb_cObject);
272
+ rb_cVibeZstdDecompressReader = rb_define_class_under(rb_mVibeZstd, "DecompressReader", rb_cObject);
273
+
274
+ // Initialize each subsystem
275
+ vibe_zstd_cctx_init_class(rb_cVibeZstdCCtx);
276
+ vibe_zstd_dctx_init_class(rb_cVibeZstdDCtx);
277
+ vibe_zstd_dict_init_classes(rb_cVibeZstdCDict, rb_cVibeZstdDDict);
278
+ vibe_zstd_dict_init_module_methods(rb_mVibeZstd);
279
+ vibe_zstd_streaming_init_classes(rb_cVibeZstdCompressWriter, rb_cVibeZstdDecompressReader);
280
+ vibe_zstd_frames_init_module_methods(rb_mVibeZstd);
281
+
282
+ // Module-level version information
283
+ rb_define_module_function(rb_mVibeZstd, "version_number", vibe_zstd_version_number, 0);
284
+ rb_define_module_function(rb_mVibeZstd, "version_string", vibe_zstd_version_string, 0);
285
+ rb_define_module_function(rb_mVibeZstd, "min_compression_level", vibe_zstd_min_c_level, 0);
286
+ rb_define_module_function(rb_mVibeZstd, "max_compression_level", vibe_zstd_max_c_level, 0);
287
+ rb_define_module_function(rb_mVibeZstd, "default_compression_level", vibe_zstd_default_c_level, 0);
288
+
289
+ // Aliases
290
+ rb_define_module_function(rb_mVibeZstd, "min_level", vibe_zstd_min_c_level, 0);
291
+ rb_define_module_function(rb_mVibeZstd, "max_level", vibe_zstd_max_c_level, 0);
292
+ rb_define_module_function(rb_mVibeZstd, "default_level", vibe_zstd_default_c_level, 0);
293
+ }
@@ -0,0 +1,56 @@
1
+ #ifndef VIBE_ZSTD_H
2
+ #define VIBE_ZSTD_H 1
3
+
4
+ #include "ruby.h"
5
+ #define ZSTD_STATIC_LINKING_ONLY
6
+ #include <zstd.h>
7
+
8
+ // TypedData structs
9
+ typedef struct {
10
+ ZSTD_CCtx* cctx;
11
+ } vibe_zstd_cctx;
12
+
13
+ typedef struct {
14
+ ZSTD_DCtx* dctx;
15
+ size_t initial_capacity; // Initial capacity for unknown-size decompression (0 = use class default)
16
+ } vibe_zstd_dctx;
17
+
18
+ typedef struct {
19
+ ZSTD_CDict* cdict;
20
+ } vibe_zstd_cdict;
21
+
22
+ typedef struct {
23
+ ZSTD_DDict* ddict;
24
+ } vibe_zstd_ddict;
25
+
26
+ typedef struct {
27
+ ZSTD_CStream* cstream;
28
+ VALUE io;
29
+ } vibe_zstd_cstream;
30
+
31
+ typedef struct {
32
+ ZSTD_DStream* dstream;
33
+ VALUE io;
34
+ ZSTD_inBuffer input; // Zstandard manages the buffer state
35
+ VALUE input_data; // Ruby string holding input data
36
+ int eof; // Flag to track if we've reached end of stream
37
+ size_t initial_chunk_size; // Initial chunk size for unbounded reads (0 = use default)
38
+ } vibe_zstd_dstream;
39
+
40
+ // TypedData types
41
+ extern rb_data_type_t vibe_zstd_cctx_type;
42
+ extern rb_data_type_t vibe_zstd_dctx_type;
43
+ extern rb_data_type_t vibe_zstd_cdict_type;
44
+ extern rb_data_type_t vibe_zstd_ddict_type;
45
+ extern rb_data_type_t vibe_zstd_cstream_type;
46
+ extern rb_data_type_t vibe_zstd_dstream_type;
47
+
48
+ // Ruby classes and modules
49
+ extern VALUE rb_cVibeZstdCCtx;
50
+ extern VALUE rb_cVibeZstdDCtx;
51
+ extern VALUE rb_cVibeZstdCDict;
52
+ extern VALUE rb_cVibeZstdDDict;
53
+ extern VALUE rb_cVibeZstdCompressWriter;
54
+ extern VALUE rb_cVibeZstdDecompressReader;
55
+
56
+ #endif /* VIBE_ZSTD_H */
@@ -0,0 +1,27 @@
1
+ #ifndef VIBE_ZSTD_INTERNAL_H
2
+ #define VIBE_ZSTD_INTERNAL_H 1
3
+
4
+ #include "vibe_zstd.h"
5
+ #include <ruby/thread.h>
6
+ #define ZDICT_STATIC_LINKING_ONLY
7
+ #include <zdict.h>
8
+
9
+ // Function declarations for cross-file usage
10
+
11
+ // CCtx functions (cctx.c)
12
+ void vibe_zstd_cctx_init_class(VALUE rb_cVibeZstdCCtx);
13
+
14
+ // DCtx functions (dctx.c)
15
+ void vibe_zstd_dctx_init_class(VALUE rb_cVibeZstdDCtx);
16
+
17
+ // Dictionary functions (dict.c)
18
+ void vibe_zstd_dict_init_classes(VALUE rb_cVibeZstdCDict, VALUE rb_cVibeZstdDDict);
19
+ void vibe_zstd_dict_init_module_methods(VALUE rb_mVibeZstd);
20
+
21
+ // Streaming functions (streaming.c)
22
+ void vibe_zstd_streaming_init_classes(VALUE rb_cVibeZstdCompressWriter, VALUE rb_cVibeZstdDecompressReader);
23
+
24
+ // Frame utility functions (frames.c)
25
+ void vibe_zstd_frames_init_module_methods(VALUE rb_mVibeZstd);
26
+
27
+ #endif /* VIBE_ZSTD_INTERNAL_H */
@@ -0,0 +1,67 @@
1
+ # frozen_string_literal: true
2
+
3
+ module VibeZstd
4
+ # Frame format constants for compression and decompression
5
+ module Format
6
+ # Standard zstd frame format with magic number (default)
7
+ STANDARD = 0
8
+
9
+ # Zstd frame format without initial 4-byte magic number
10
+ # Useful to save 4 bytes per frame, but decoder must be explicitly configured
11
+ MAGICLESS = 1
12
+ end
13
+
14
+ # Compression strategy constants, listed from fastest to strongest
15
+ module Strategy
16
+ FAST = 1
17
+ DFAST = 2
18
+ GREEDY = 3
19
+ LAZY = 4
20
+ LAZY2 = 5
21
+ BTLAZY2 = 6
22
+ BTOPT = 7
23
+ BTULTRA = 8
24
+ BTULTRA2 = 9
25
+ end
26
+
27
+ # Literal compression mode constants
28
+ module LiteralCompressionMode
29
+ # Automatically determine based on compression level
30
+ # Negative levels = uncompressed, positive levels = compressed
31
+ AUTO = 0
32
+
33
+ # Always attempt Huffman compression
34
+ # Uncompressed literals still emitted if not profitable
35
+ HUFFMAN = 1
36
+
37
+ # Always emit uncompressed literals
38
+ UNCOMPRESSED = 2
39
+ end
40
+
41
+ # Dictionary attachment preference constants
42
+ module DictAttachPref
43
+ # Automatically choose attachment method (default)
44
+ AUTO = 0
45
+
46
+ # Force dictionary to be attached by reference
47
+ FORCE_ATTACH = 1
48
+
49
+ # Force dictionary to be copied into working context
50
+ FORCE_COPY = 2
51
+
52
+ # Force dictionary to be reloaded
53
+ FORCE_LOAD = 3
54
+ end
55
+
56
+ # Context reset modes
57
+ module ResetDirective
58
+ # Reset session only, keep parameters
59
+ SESSION = 1
60
+
61
+ # Reset parameters to defaults, keep session state
62
+ PARAMETERS = 2
63
+
64
+ # Reset both session and parameters (full reset)
65
+ BOTH = 3
66
+ end
67
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module VibeZstd
4
+ VERSION = "1.0.0"
5
+ end
data/lib/vibe_zstd.rb ADDED
@@ -0,0 +1,255 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "vibe_zstd/version"
4
+ require_relative "vibe_zstd/vibe_zstd"
5
+ require_relative "vibe_zstd/constants"
6
+
7
+ module VibeZstd
8
+ class Error < StandardError; end
9
+
10
+ # Convenience method for one-off compression
11
+ # Supports all CCtx#compress options: level, dict, pledged_size
12
+ def self.compress(data, **options)
13
+ cctx = CCtx.new
14
+ cctx.compress(data, **options)
15
+ end
16
+
17
+ # Convenience method for one-off decompression
18
+ # Supports all DCtx#decompress options: dict, initial_capacity
19
+ def self.decompress(data, **options)
20
+ dctx = DCtx.new
21
+ dctx.decompress(data, **options)
22
+ end
23
+
24
+ # Get the decompressed content size from a compressed frame
25
+ # Returns nil if size is unknown or data is invalid
26
+ def self.frame_content_size(data)
27
+ DCtx.frame_content_size(data)
28
+ end
29
+
30
+ # Iterate over all skippable frames in the data
31
+ # Yields [content, magic_variant, offset] for each skippable frame
32
+ def self.each_skippable_frame(data)
33
+ return enum_for(:each_skippable_frame, data) unless block_given?
34
+
35
+ offset = 0
36
+ while offset < data.bytesize
37
+ frame_data = data.byteslice(offset..-1)
38
+ frame_size = find_frame_compressed_size(frame_data)
39
+
40
+ # Defense: Prevent infinite loop on malformed data
41
+ # A valid frame must have non-zero size (at minimum: frame header)
42
+ if frame_size <= 0
43
+ raise Error, "Invalid frame: zero or negative size at offset #{offset}"
44
+ end
45
+
46
+ if skippable_frame?(frame_data)
47
+ content, magic_variant = read_skippable_frame(frame_data)
48
+ yield content, magic_variant, offset
49
+ end
50
+
51
+ offset += frame_size
52
+ end
53
+ end
54
+
55
+ # Convenient aliases for version/level methods
56
+ class << self
57
+ alias_method :min_level, :min_compression_level
58
+ alias_method :max_level, :max_compression_level
59
+ alias_method :default_level, :default_compression_level
60
+ end
61
+
62
+ # Add helper method to CDict for creating matching DDict
63
+ class CDict
64
+ # Get or create a matching DDict from this CDict's dictionary data
65
+ # The DDict is cached so it's only created once
66
+ #
67
+ # @return [DDict] Decompression dictionary matching this compression dictionary
68
+ def to_ddict
69
+ @ddict ||= DDict.new(@dict_data)
70
+ end
71
+ alias_method :ddict, :to_ddict
72
+ end
73
+
74
+ # Thread-local context pooling for high-performance reuse
75
+ # Ideal for Rails/Puma applications where threads are reused across requests
76
+ #
77
+ # Example usage:
78
+ # # In Rails model with encrypted attributes
79
+ # class User < ApplicationRecord
80
+ # encrypts :email
81
+ # encrypts :preferences, dict: JSON_DICT
82
+ # end
83
+ #
84
+ # # Instead of:
85
+ # VibeZstd.decompress(data, dict: dict) # Creates new DCtx each time
86
+ #
87
+ # # Use:
88
+ # VibeZstd::ThreadLocal.decompress(data, dict: dict) # Reuses DCtx per thread
89
+ #
90
+ # Memory footprint: ~128KB per DCtx × unique dictionaries × threads
91
+ # Example: 3 dicts × 5 Puma threads = 1.9MB total
92
+ #
93
+ # Note: Only supports per-operation parameters (level, dict, pledged_size, initial_capacity)
94
+ # Does NOT support context-level settings (nb_workers, checksum_flag, etc.)
95
+ module ThreadLocal
96
+ # Compress data using thread-local context pool
97
+ # Contexts are keyed by dictionary ID for automatic isolation
98
+ #
99
+ # @param data [String] Data to compress
100
+ # @param level [Integer] Compression level (per-operation, can vary)
101
+ # @param dict [CDict] Compression dictionary (optional)
102
+ # @param pledged_size [Integer] Expected input size (optional)
103
+ # @return [String] Compressed data
104
+ def self.compress(data, level: nil, dict: nil, pledged_size: nil)
105
+ # Key by dictionary ID, or :default if no dict
106
+ key = dict ? dict.dict_id : :default
107
+
108
+ # Get or create thread-local context pool
109
+ Thread.current[:vibe_zstd_cctx_pool] ||= {}
110
+ cctx = Thread.current[:vibe_zstd_cctx_pool][key] ||= VibeZstd::CCtx.new
111
+
112
+ # Build options hash
113
+ options = {}
114
+ options[:level] = level if level
115
+ options[:dict] = dict if dict
116
+ options[:pledged_size] = pledged_size if pledged_size
117
+
118
+ cctx.compress(data, **options)
119
+ end
120
+
121
+ # Decompress data using thread-local context pool
122
+ # Contexts are keyed by dictionary ID for automatic isolation
123
+ #
124
+ # @param data [String] Data to decompress
125
+ # @param dict [DDict] Decompression dictionary (optional)
126
+ # @param initial_capacity [Integer] Initial buffer size for unknown-size frames (optional)
127
+ # @return [String] Decompressed data
128
+ def self.decompress(data, dict: nil, initial_capacity: nil)
129
+ key = dict ? dict.dict_id : :default
130
+
131
+ # Get or create thread-local context pool
132
+ Thread.current[:vibe_zstd_dctx_pool] ||= {}
133
+ dctx = Thread.current[:vibe_zstd_dctx_pool][key] ||= VibeZstd::DCtx.new
134
+
135
+ # Build options hash
136
+ options = {}
137
+ options[:dict] = dict if dict
138
+ options[:initial_capacity] = initial_capacity if initial_capacity
139
+
140
+ # C code will validate dict matches frame requirements
141
+ dctx.decompress(data, **options)
142
+ end
143
+
144
+ # Clear all thread-local context pools for the current thread
145
+ # Useful for testing or explicit memory management
146
+ def self.clear_thread_cache!
147
+ Thread.current[:vibe_zstd_cctx_pool] = {}
148
+ Thread.current[:vibe_zstd_dctx_pool] = {}
149
+ nil
150
+ end
151
+
152
+ # Get statistics about the current thread's context pools
153
+ # @return [Hash] Pool statistics
154
+ def self.thread_cache_stats
155
+ {
156
+ compression_contexts: Thread.current[:vibe_zstd_cctx_pool]&.size || 0,
157
+ decompression_contexts: Thread.current[:vibe_zstd_dctx_pool]&.size || 0,
158
+ compression_keys: Thread.current[:vibe_zstd_cctx_pool]&.keys || [],
159
+ decompression_keys: Thread.current[:vibe_zstd_dctx_pool]&.keys || []
160
+ }
161
+ end
162
+ end
163
+
164
+ class CompressWriter
165
+ # Block-based resource management
166
+ # Automatically calls finish when block completes
167
+ def self.open(io, **options)
168
+ writer = new(io, **options)
169
+ return writer unless block_given?
170
+
171
+ begin
172
+ yield writer
173
+ ensure
174
+ writer.finish
175
+ end
176
+ end
177
+ end
178
+
179
+ class DecompressReader
180
+ include Enumerable
181
+
182
+ # Block-based resource management
183
+ # Automatically cleans up when block completes
184
+ def self.open(io, **options)
185
+ reader = new(io, **options)
186
+ return reader unless block_given?
187
+
188
+ yield reader
189
+
190
+ # Reader doesn't have finish, but this ensures cleanup
191
+ end
192
+
193
+ # Read all remaining data
194
+ def read_all
195
+ chunks = []
196
+ while (chunk = read)
197
+ chunks << chunk
198
+ end
199
+ chunks.join
200
+ end
201
+
202
+ # Alias for eof?
203
+ def eof
204
+ eof?
205
+ end
206
+
207
+ # Iterate over chunks (required for Enumerable)
208
+ def each(chunk_size = nil)
209
+ return enum_for(:each, chunk_size) unless block_given?
210
+
211
+ until eof?
212
+ chunk = read(chunk_size)
213
+ yield chunk if chunk
214
+ end
215
+ end
216
+
217
+ # Read a single line (up to newline or EOF)
218
+ def gets(sep = $/)
219
+ return nil if eof?
220
+
221
+ line = +""
222
+ until eof?
223
+ chunk = read(1)
224
+ break unless chunk
225
+
226
+ line << chunk
227
+ break if chunk.end_with?(sep)
228
+ end
229
+
230
+ line.empty? ? nil : line
231
+ end
232
+
233
+ # Iterate over lines
234
+ def each_line(sep = $/)
235
+ return enum_for(:each_line, sep) unless block_given?
236
+
237
+ while (line = gets(sep))
238
+ yield line
239
+ end
240
+ end
241
+
242
+ # Alias for gets
243
+ alias_method :readline, :gets
244
+
245
+ # Read exactly n bytes, or raise EOFError
246
+ def readpartial(maxlen)
247
+ raise EOFError, "end of file reached" if eof?
248
+
249
+ data = read(maxlen)
250
+ raise EOFError, "end of file reached" if data.nil?
251
+
252
+ data
253
+ end
254
+ end
255
+ end