zstd-ruby 1.4.5.0 → 1.5.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (115) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/README.md +78 -5
  4. data/Rakefile +8 -2
  5. data/ext/zstdruby/common.h +15 -0
  6. data/ext/zstdruby/extconf.rb +3 -2
  7. data/ext/zstdruby/libzstd/common/allocations.h +55 -0
  8. data/ext/zstdruby/libzstd/common/bits.h +200 -0
  9. data/ext/zstdruby/libzstd/common/bitstream.h +45 -62
  10. data/ext/zstdruby/libzstd/common/compiler.h +205 -22
  11. data/ext/zstdruby/libzstd/common/cpu.h +1 -3
  12. data/ext/zstdruby/libzstd/common/debug.c +1 -1
  13. data/ext/zstdruby/libzstd/common/debug.h +12 -19
  14. data/ext/zstdruby/libzstd/common/entropy_common.c +172 -48
  15. data/ext/zstdruby/libzstd/common/error_private.c +10 -2
  16. data/ext/zstdruby/libzstd/common/error_private.h +82 -3
  17. data/ext/zstdruby/libzstd/common/fse.h +37 -86
  18. data/ext/zstdruby/libzstd/common/fse_decompress.c +117 -92
  19. data/ext/zstdruby/libzstd/common/huf.h +99 -166
  20. data/ext/zstdruby/libzstd/common/mem.h +124 -142
  21. data/ext/zstdruby/libzstd/common/pool.c +54 -27
  22. data/ext/zstdruby/libzstd/common/pool.h +10 -4
  23. data/ext/zstdruby/libzstd/common/portability_macros.h +156 -0
  24. data/ext/zstdruby/libzstd/common/threading.c +74 -19
  25. data/ext/zstdruby/libzstd/common/threading.h +5 -10
  26. data/ext/zstdruby/libzstd/common/xxhash.c +7 -847
  27. data/ext/zstdruby/libzstd/common/xxhash.h +5568 -167
  28. data/ext/zstdruby/libzstd/common/zstd_common.c +2 -37
  29. data/ext/zstdruby/libzstd/common/zstd_deps.h +111 -0
  30. data/ext/zstdruby/libzstd/common/zstd_internal.h +132 -187
  31. data/ext/zstdruby/libzstd/common/zstd_trace.h +163 -0
  32. data/ext/zstdruby/libzstd/compress/clevels.h +134 -0
  33. data/ext/zstdruby/libzstd/compress/fse_compress.c +83 -157
  34. data/ext/zstdruby/libzstd/compress/hist.c +27 -29
  35. data/ext/zstdruby/libzstd/compress/hist.h +2 -2
  36. data/ext/zstdruby/libzstd/compress/huf_compress.c +916 -279
  37. data/ext/zstdruby/libzstd/compress/zstd_compress.c +3773 -1019
  38. data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +610 -203
  39. data/ext/zstdruby/libzstd/compress/zstd_compress_literals.c +119 -42
  40. data/ext/zstdruby/libzstd/compress/zstd_compress_literals.h +16 -6
  41. data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.c +42 -19
  42. data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.h +1 -1
  43. data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.c +49 -317
  44. data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.h +1 -1
  45. data/ext/zstdruby/libzstd/compress/zstd_cwksp.h +320 -103
  46. data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +388 -151
  47. data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +3 -2
  48. data/ext/zstdruby/libzstd/compress/zstd_fast.c +729 -265
  49. data/ext/zstdruby/libzstd/compress/zstd_fast.h +3 -2
  50. data/ext/zstdruby/libzstd/compress/zstd_lazy.c +1270 -251
  51. data/ext/zstdruby/libzstd/compress/zstd_lazy.h +61 -1
  52. data/ext/zstdruby/libzstd/compress/zstd_ldm.c +324 -219
  53. data/ext/zstdruby/libzstd/compress/zstd_ldm.h +9 -2
  54. data/ext/zstdruby/libzstd/compress/zstd_ldm_geartab.h +106 -0
  55. data/ext/zstdruby/libzstd/compress/zstd_opt.c +481 -209
  56. data/ext/zstdruby/libzstd/compress/zstd_opt.h +1 -1
  57. data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +181 -457
  58. data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +34 -113
  59. data/ext/zstdruby/libzstd/decompress/huf_decompress.c +1199 -565
  60. data/ext/zstdruby/libzstd/decompress/huf_decompress_amd64.S +576 -0
  61. data/ext/zstdruby/libzstd/decompress/zstd_ddict.c +12 -12
  62. data/ext/zstdruby/libzstd/decompress/zstd_ddict.h +2 -2
  63. data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +627 -157
  64. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.c +1086 -326
  65. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.h +19 -5
  66. data/ext/zstdruby/libzstd/decompress/zstd_decompress_internal.h +62 -13
  67. data/ext/zstdruby/libzstd/dictBuilder/cover.c +73 -52
  68. data/ext/zstdruby/libzstd/dictBuilder/cover.h +7 -6
  69. data/ext/zstdruby/libzstd/dictBuilder/divsufsort.c +1 -1
  70. data/ext/zstdruby/libzstd/dictBuilder/fastcover.c +44 -35
  71. data/ext/zstdruby/libzstd/dictBuilder/zdict.c +103 -111
  72. data/ext/zstdruby/libzstd/{dictBuilder/zdict.h → zdict.h} +203 -34
  73. data/ext/zstdruby/libzstd/zstd.h +1217 -287
  74. data/ext/zstdruby/libzstd/{common/zstd_errors.h → zstd_errors.h} +28 -8
  75. data/ext/zstdruby/main.c +20 -0
  76. data/ext/zstdruby/skippable_frame.c +63 -0
  77. data/ext/zstdruby/streaming_compress.c +177 -0
  78. data/ext/zstdruby/streaming_compress.h +5 -0
  79. data/ext/zstdruby/streaming_decompress.c +123 -0
  80. data/ext/zstdruby/zstdruby.c +114 -32
  81. data/lib/zstd-ruby/version.rb +1 -1
  82. data/lib/zstd-ruby.rb +0 -1
  83. data/zstd-ruby.gemspec +1 -1
  84. metadata +19 -36
  85. data/.travis.yml +0 -14
  86. data/ext/zstdruby/libzstd/.gitignore +0 -3
  87. data/ext/zstdruby/libzstd/BUCK +0 -234
  88. data/ext/zstdruby/libzstd/Makefile +0 -354
  89. data/ext/zstdruby/libzstd/README.md +0 -179
  90. data/ext/zstdruby/libzstd/deprecated/zbuff.h +0 -214
  91. data/ext/zstdruby/libzstd/deprecated/zbuff_common.c +0 -26
  92. data/ext/zstdruby/libzstd/deprecated/zbuff_compress.c +0 -147
  93. data/ext/zstdruby/libzstd/deprecated/zbuff_decompress.c +0 -75
  94. data/ext/zstdruby/libzstd/dll/example/Makefile +0 -48
  95. data/ext/zstdruby/libzstd/dll/example/README.md +0 -69
  96. data/ext/zstdruby/libzstd/dll/example/build_package.bat +0 -20
  97. data/ext/zstdruby/libzstd/dll/example/fullbench-dll.sln +0 -25
  98. data/ext/zstdruby/libzstd/dll/example/fullbench-dll.vcxproj +0 -181
  99. data/ext/zstdruby/libzstd/legacy/zstd_legacy.h +0 -415
  100. data/ext/zstdruby/libzstd/legacy/zstd_v01.c +0 -2158
  101. data/ext/zstdruby/libzstd/legacy/zstd_v01.h +0 -94
  102. data/ext/zstdruby/libzstd/legacy/zstd_v02.c +0 -3518
  103. data/ext/zstdruby/libzstd/legacy/zstd_v02.h +0 -93
  104. data/ext/zstdruby/libzstd/legacy/zstd_v03.c +0 -3160
  105. data/ext/zstdruby/libzstd/legacy/zstd_v03.h +0 -93
  106. data/ext/zstdruby/libzstd/legacy/zstd_v04.c +0 -3647
  107. data/ext/zstdruby/libzstd/legacy/zstd_v04.h +0 -142
  108. data/ext/zstdruby/libzstd/legacy/zstd_v05.c +0 -4050
  109. data/ext/zstdruby/libzstd/legacy/zstd_v05.h +0 -162
  110. data/ext/zstdruby/libzstd/legacy/zstd_v06.c +0 -4154
  111. data/ext/zstdruby/libzstd/legacy/zstd_v06.h +0 -172
  112. data/ext/zstdruby/libzstd/legacy/zstd_v07.c +0 -4541
  113. data/ext/zstdruby/libzstd/legacy/zstd_v07.h +0 -187
  114. data/ext/zstdruby/libzstd/libzstd.pc.in +0 -15
  115. data/ext/zstdruby/zstdruby.h +0 -6
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -20,19 +20,31 @@ extern "C" {
20
20
 
21
21
 
22
22
  /* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */
23
- #ifndef ZSTDERRORLIB_VISIBILITY
24
- # if defined(__GNUC__) && (__GNUC__ >= 4)
25
- # define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default")))
23
+ #ifndef ZSTDERRORLIB_VISIBLE
24
+ /* Backwards compatibility with old macro name */
25
+ # ifdef ZSTDERRORLIB_VISIBILITY
26
+ # define ZSTDERRORLIB_VISIBLE ZSTDERRORLIB_VISIBILITY
27
+ # elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
28
+ # define ZSTDERRORLIB_VISIBLE __attribute__ ((visibility ("default")))
26
29
  # else
27
- # define ZSTDERRORLIB_VISIBILITY
30
+ # define ZSTDERRORLIB_VISIBLE
28
31
  # endif
29
32
  #endif
33
+
34
+ #ifndef ZSTDERRORLIB_HIDDEN
35
+ # if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
36
+ # define ZSTDERRORLIB_HIDDEN __attribute__ ((visibility ("hidden")))
37
+ # else
38
+ # define ZSTDERRORLIB_HIDDEN
39
+ # endif
40
+ #endif
41
+
30
42
  #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
31
- # define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY
43
+ # define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBLE
32
44
  #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
33
- # define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
45
+ # define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
34
46
  #else
35
- # define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY
47
+ # define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBLE
36
48
  #endif
37
49
 
38
50
  /*-*********************************************
@@ -58,14 +70,17 @@ typedef enum {
58
70
  ZSTD_error_frameParameter_windowTooLarge = 16,
59
71
  ZSTD_error_corruption_detected = 20,
60
72
  ZSTD_error_checksum_wrong = 22,
73
+ ZSTD_error_literals_headerWrong = 24,
61
74
  ZSTD_error_dictionary_corrupted = 30,
62
75
  ZSTD_error_dictionary_wrong = 32,
63
76
  ZSTD_error_dictionaryCreation_failed = 34,
64
77
  ZSTD_error_parameter_unsupported = 40,
78
+ ZSTD_error_parameter_combination_unsupported = 41,
65
79
  ZSTD_error_parameter_outOfBound = 42,
66
80
  ZSTD_error_tableLog_tooLarge = 44,
67
81
  ZSTD_error_maxSymbolValue_tooLarge = 46,
68
82
  ZSTD_error_maxSymbolValue_tooSmall = 48,
83
+ ZSTD_error_stabilityCondition_notRespected = 50,
69
84
  ZSTD_error_stage_wrong = 60,
70
85
  ZSTD_error_init_missing = 62,
71
86
  ZSTD_error_memory_allocation = 64,
@@ -73,10 +88,15 @@ typedef enum {
73
88
  ZSTD_error_dstSize_tooSmall = 70,
74
89
  ZSTD_error_srcSize_wrong = 72,
75
90
  ZSTD_error_dstBuffer_null = 74,
91
+ ZSTD_error_noForwardProgress_destFull = 80,
92
+ ZSTD_error_noForwardProgress_inputEmpty = 82,
76
93
  /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */
77
94
  ZSTD_error_frameIndex_tooLarge = 100,
78
95
  ZSTD_error_seekableIO = 102,
79
96
  ZSTD_error_dstBuffer_wrong = 104,
97
+ ZSTD_error_srcBuffer_wrong = 105,
98
+ ZSTD_error_sequenceProducer_failed = 106,
99
+ ZSTD_error_externalSequences_invalid = 107,
80
100
  ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */
81
101
  } ZSTD_ErrorCode;
82
102
 
@@ -0,0 +1,20 @@
1
+ #include <common.h>
2
+ VALUE rb_mZstd;
3
+ void zstd_ruby_init(void);
4
+ void zstd_ruby_skippable_frame_init(void);
5
+ void zstd_ruby_streaming_compress_init(void);
6
+ void zstd_ruby_streaming_decompress_init(void);
7
+
8
+ void
9
+ Init_zstdruby(void)
10
+ {
11
+ #ifdef HAVE_RB_EXT_RACTOR_SAFE
12
+ rb_ext_ractor_safe(true);
13
+ #endif
14
+
15
+ rb_mZstd = rb_define_module("Zstd");
16
+ zstd_ruby_init();
17
+ zstd_ruby_skippable_frame_init();
18
+ zstd_ruby_streaming_compress_init();
19
+ zstd_ruby_streaming_decompress_init();
20
+ }
@@ -0,0 +1,63 @@
1
+ #include <common.h>
2
+
3
+ extern VALUE rb_mZstd;
4
+
5
+ static VALUE rb_write_skippable_frame(int argc, VALUE *argv, VALUE self)
6
+ {
7
+ VALUE input_value;
8
+ VALUE skip_value;
9
+ VALUE kwargs;
10
+ rb_scan_args(argc, argv, "2:", &input_value, &skip_value, &kwargs);
11
+
12
+ ID kwargs_keys[1];
13
+ kwargs_keys[0] = rb_intern("magic_variant");
14
+ VALUE kwargs_values[1];
15
+ rb_get_kwargs(kwargs, kwargs_keys, 0, 1, kwargs_values);
16
+ unsigned magic_variant = (kwargs_values[0] != Qundef) ? (NUM2INT(kwargs_values[0])) : 0;
17
+
18
+ StringValue(input_value);
19
+ StringValue(skip_value);
20
+ char* input_data = RSTRING_PTR(input_value);
21
+ size_t input_size = RSTRING_LEN(input_value);
22
+ char* skip_data = RSTRING_PTR(skip_value);
23
+ size_t skip_size = RSTRING_LEN(skip_value);
24
+
25
+ size_t dst_size = input_size + ZSTD_SKIPPABLEHEADERSIZE + skip_size;
26
+ VALUE output = rb_str_new(input_data, dst_size);
27
+ char* output_data = RSTRING_PTR(output);
28
+ size_t output_size = ZSTD_writeSkippableFrame((void*)output_data, dst_size, (const void*)skip_data, skip_size, magic_variant);
29
+ if (ZSTD_isError(output_size)) {
30
+ rb_raise(rb_eRuntimeError, "%s: %s", "write skippable frame failed", ZSTD_getErrorName(output_size));
31
+ }
32
+
33
+ rb_str_resize(output, output_size);
34
+ return output;
35
+ }
36
+
37
+ static VALUE rb_read_skippable_frame(VALUE self, VALUE input_value)
38
+ {
39
+ char* input_data = RSTRING_PTR(input_value);
40
+ size_t input_size = RSTRING_LEN(input_value);
41
+
42
+ if (ZSTD_isSkippableFrame(input_data, input_size) == 0) {
43
+ return Qnil;
44
+ }
45
+ // ref https://github.com/facebook/zstd/blob/321490cd5b9863433b3d44816d04012874e5ecdb/tests/fuzzer.c#L2096
46
+ size_t const skipLen = 129 * 1024;
47
+ VALUE output = rb_str_new(NULL, skipLen);
48
+ char* output_data = RSTRING_PTR(output);
49
+ unsigned readMagic;
50
+ size_t output_size = ZSTD_readSkippableFrame((void*)output_data, skipLen, &readMagic, (const void*)input_data, input_size);
51
+ if (ZSTD_isError(output_size)) {
52
+ rb_raise(rb_eRuntimeError, "%s: %s", "read skippable frame failed", ZSTD_getErrorName(output_size));
53
+ }
54
+ rb_str_resize(output, output_size);
55
+ return output;
56
+ }
57
+
58
+ void
59
+ zstd_ruby_skippable_frame_init(void)
60
+ {
61
+ rb_define_module_function(rb_mZstd, "write_skippable_frame", rb_write_skippable_frame, -1);
62
+ rb_define_module_function(rb_mZstd, "read_skippable_frame", rb_read_skippable_frame, 1);
63
+ }
@@ -0,0 +1,177 @@
1
+ #include <common.h>
2
+ #include <streaming_compress.h>
3
+
4
+ struct streaming_compress_t {
5
+ ZSTD_CCtx* ctx;
6
+ VALUE buf;
7
+ size_t buf_size;
8
+ };
9
+
10
+ static void
11
+ streaming_compress_mark(void *p)
12
+ {
13
+ struct streaming_compress_t *sc = p;
14
+ rb_gc_mark(sc->buf);
15
+ }
16
+
17
+ static void
18
+ streaming_compress_free(void *p)
19
+ {
20
+ struct streaming_compress_t *sc = p;
21
+ ZSTD_CCtx* ctx = sc->ctx;
22
+ if (ctx != NULL) {
23
+ ZSTD_freeCCtx(ctx);
24
+ }
25
+ xfree(sc);
26
+ }
27
+
28
+ static size_t
29
+ streaming_compress_memsize(const void *p)
30
+ {
31
+ return sizeof(struct streaming_compress_t);
32
+ }
33
+
34
+ static const rb_data_type_t streaming_compress_type = {
35
+ "streaming_compress",
36
+ { streaming_compress_mark, streaming_compress_free, streaming_compress_memsize, },
37
+ 0, 0, RUBY_TYPED_FREE_IMMEDIATELY
38
+ };
39
+
40
+ static VALUE
41
+ rb_streaming_compress_allocate(VALUE klass)
42
+ {
43
+ struct streaming_compress_t* sc;
44
+ VALUE obj = TypedData_Make_Struct(klass, struct streaming_compress_t, &streaming_compress_type, sc);
45
+ sc->ctx = NULL;
46
+ sc->buf = Qnil;
47
+ sc->buf_size = 0;
48
+ return obj;
49
+ }
50
+
51
+ static VALUE
52
+ rb_streaming_compress_initialize(int argc, VALUE *argv, VALUE obj)
53
+ {
54
+ VALUE compression_level_value;
55
+ rb_scan_args(argc, argv, "01", &compression_level_value);
56
+ int compression_level = convert_compression_level(compression_level_value);
57
+
58
+ struct streaming_compress_t* sc;
59
+ TypedData_Get_Struct(obj, struct streaming_compress_t, &streaming_compress_type, sc);
60
+ size_t const buffOutSize = ZSTD_CStreamOutSize();
61
+
62
+ ZSTD_CCtx* ctx = ZSTD_createCCtx();
63
+ if (ctx == NULL) {
64
+ rb_raise(rb_eRuntimeError, "%s", "ZSTD_createCCtx error");
65
+ }
66
+ ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, compression_level);
67
+ sc->ctx = ctx;
68
+ sc->buf = rb_str_new(NULL, buffOutSize);
69
+ sc->buf_size = buffOutSize;
70
+
71
+ return obj;
72
+ }
73
+
74
+ #define FIXNUMARG(val, ifnil) \
75
+ (NIL_P((val)) ? (ifnil) \
76
+ : (FIX2INT((val))))
77
+ #define ARG_CONTINUE(val) FIXNUMARG((val), ZSTD_e_continue)
78
+
79
+ static VALUE
80
+ no_compress(struct streaming_compress_t* sc, ZSTD_EndDirective endOp)
81
+ {
82
+ ZSTD_inBuffer input = { NULL, 0, 0 };
83
+ const char* output_data = RSTRING_PTR(sc->buf);
84
+ VALUE result = rb_str_new(0, 0);
85
+ size_t ret;
86
+ do {
87
+ ZSTD_outBuffer output = { (void*)output_data, sc->buf_size, 0 };
88
+
89
+ size_t const ret = ZSTD_compressStream2(sc->ctx, &output, &input, endOp);
90
+ if (ZSTD_isError(ret)) {
91
+ rb_raise(rb_eRuntimeError, "flush error error code: %s", ZSTD_getErrorName(ret));
92
+ }
93
+ rb_str_cat(result, output.dst, output.pos);
94
+ } while (ret > 0);
95
+ return result;
96
+ }
97
+
98
+ static VALUE
99
+ rb_streaming_compress_compress(VALUE obj, VALUE src)
100
+ {
101
+ StringValue(src);
102
+ const char* input_data = RSTRING_PTR(src);
103
+ size_t input_size = RSTRING_LEN(src);
104
+ ZSTD_inBuffer input = { input_data, input_size, 0 };
105
+
106
+ struct streaming_compress_t* sc;
107
+ TypedData_Get_Struct(obj, struct streaming_compress_t, &streaming_compress_type, sc);
108
+ const char* output_data = RSTRING_PTR(sc->buf);
109
+ VALUE result = rb_str_new(0, 0);
110
+ while (input.pos < input.size) {
111
+ ZSTD_outBuffer output = { (void*)output_data, sc->buf_size, 0 };
112
+ size_t const ret = ZSTD_compressStream2(sc->ctx, &output, &input, ZSTD_e_continue);
113
+ if (ZSTD_isError(ret)) {
114
+ rb_raise(rb_eRuntimeError, "compress error error code: %s", ZSTD_getErrorName(ret));
115
+ }
116
+ rb_str_cat(result, output.dst, output.pos);
117
+ }
118
+ return result;
119
+ }
120
+
121
+ static VALUE
122
+ rb_streaming_compress_addstr(VALUE obj, VALUE src)
123
+ {
124
+ StringValue(src);
125
+ const char* input_data = RSTRING_PTR(src);
126
+ size_t input_size = RSTRING_LEN(src);
127
+ ZSTD_inBuffer input = { input_data, input_size, 0 };
128
+
129
+ struct streaming_compress_t* sc;
130
+ TypedData_Get_Struct(obj, struct streaming_compress_t, &streaming_compress_type, sc);
131
+ const char* output_data = RSTRING_PTR(sc->buf);
132
+
133
+ while (input.pos < input.size) {
134
+ ZSTD_outBuffer output = { (void*)output_data, sc->buf_size, 0 };
135
+ size_t const result = ZSTD_compressStream2(sc->ctx, &output, &input, ZSTD_e_continue);
136
+ if (ZSTD_isError(result)) {
137
+ rb_raise(rb_eRuntimeError, "compress error error code: %s", ZSTD_getErrorName(result));
138
+ }
139
+ }
140
+ return obj;
141
+ }
142
+
143
+ static VALUE
144
+ rb_streaming_compress_flush(VALUE obj)
145
+ {
146
+ struct streaming_compress_t* sc;
147
+ TypedData_Get_Struct(obj, struct streaming_compress_t, &streaming_compress_type, sc);
148
+ VALUE result = no_compress(sc, ZSTD_e_flush);
149
+ return result;
150
+ }
151
+
152
+ static VALUE
153
+ rb_streaming_compress_finish(VALUE obj)
154
+ {
155
+ struct streaming_compress_t* sc;
156
+ TypedData_Get_Struct(obj, struct streaming_compress_t, &streaming_compress_type, sc);
157
+ VALUE result = no_compress(sc, ZSTD_e_end);
158
+ return result;
159
+ }
160
+
161
+ extern VALUE rb_mZstd, cStreamingCompress;
162
+ void
163
+ zstd_ruby_streaming_compress_init(void)
164
+ {
165
+ VALUE cStreamingCompress = rb_define_class_under(rb_mZstd, "StreamingCompress", rb_cObject);
166
+ rb_define_alloc_func(cStreamingCompress, rb_streaming_compress_allocate);
167
+ rb_define_method(cStreamingCompress, "initialize", rb_streaming_compress_initialize, -1);
168
+ rb_define_method(cStreamingCompress, "compress", rb_streaming_compress_compress, 1);
169
+ rb_define_method(cStreamingCompress, "<<", rb_streaming_compress_addstr, 1);
170
+ rb_define_method(cStreamingCompress, "flush", rb_streaming_compress_flush, 0);
171
+ rb_define_method(cStreamingCompress, "finish", rb_streaming_compress_finish, 0);
172
+
173
+ rb_define_const(cStreamingCompress, "CONTINUE", INT2FIX(ZSTD_e_continue));
174
+ rb_define_const(cStreamingCompress, "FLUSH", INT2FIX(ZSTD_e_flush));
175
+ rb_define_const(cStreamingCompress, "END", INT2FIX(ZSTD_e_end));
176
+ }
177
+
@@ -0,0 +1,5 @@
1
+ #if !defined(STREAMING_COMPRESS_H)
2
+ #define STREAMING_COMPRESS_H
3
+
4
+
5
+ #endif // STREAMING_COMPRESS_H
@@ -0,0 +1,123 @@
1
+ #include <common.h>
2
+
3
+ struct streaming_decompress_t {
4
+ ZSTD_DCtx* ctx;
5
+ VALUE buf;
6
+ size_t buf_size;
7
+ };
8
+
9
+ static void
10
+ streaming_decompress_mark(void *p)
11
+ {
12
+ struct streaming_decompress_t *sd = p;
13
+ rb_gc_mark(sd->buf);
14
+ }
15
+
16
+ static void
17
+ streaming_decompress_free(void *p)
18
+ {
19
+ struct streaming_decompress_t *sd = p;
20
+ ZSTD_DCtx* ctx = sd->ctx;
21
+ if (ctx != NULL) {
22
+ ZSTD_freeDCtx(ctx);
23
+ }
24
+ xfree(sd);
25
+ }
26
+
27
+ static size_t
28
+ streaming_decompress_memsize(const void *p)
29
+ {
30
+ return sizeof(struct streaming_decompress_t);
31
+ }
32
+
33
+ static const rb_data_type_t streaming_decompress_type = {
34
+ "streaming_decompress",
35
+ { streaming_decompress_mark, streaming_decompress_free, streaming_decompress_memsize, },
36
+ 0, 0, RUBY_TYPED_FREE_IMMEDIATELY
37
+ };
38
+
39
+ static VALUE
40
+ rb_streaming_decompress_allocate(VALUE klass)
41
+ {
42
+ struct streaming_decompress_t* sd;
43
+ VALUE obj = TypedData_Make_Struct(klass, struct streaming_decompress_t, &streaming_decompress_type, sd);
44
+ sd->ctx = NULL;
45
+ sd->buf = Qnil;
46
+ sd->buf_size = 0;
47
+ return obj;
48
+ }
49
+
50
+ static VALUE
51
+ rb_streaming_decompress_initialize(VALUE obj)
52
+ {
53
+ struct streaming_decompress_t* sd;
54
+ TypedData_Get_Struct(obj, struct streaming_decompress_t, &streaming_decompress_type, sd);
55
+ size_t const buffOutSize = ZSTD_DStreamOutSize();
56
+
57
+ ZSTD_DCtx* ctx = ZSTD_createDCtx();
58
+ if (ctx == NULL) {
59
+ rb_raise(rb_eRuntimeError, "%s", "ZSTD_createDCtx error");
60
+ }
61
+ sd->ctx = ctx;
62
+ sd->buf = rb_str_new(NULL, buffOutSize);
63
+ sd->buf_size = buffOutSize;
64
+
65
+ return obj;
66
+ }
67
+
68
+ static VALUE
69
+ rb_streaming_decompress_decompress(VALUE obj, VALUE src)
70
+ {
71
+ StringValue(src);
72
+ const char* input_data = RSTRING_PTR(src);
73
+ size_t input_size = RSTRING_LEN(src);
74
+ ZSTD_inBuffer input = { input_data, input_size, 0 };
75
+
76
+ struct streaming_decompress_t* sd;
77
+ TypedData_Get_Struct(obj, struct streaming_decompress_t, &streaming_decompress_type, sd);
78
+ const char* output_data = RSTRING_PTR(sd->buf);
79
+ VALUE result = rb_str_new(0, 0);
80
+ while (input.pos < input.size) {
81
+ ZSTD_outBuffer output = { (void*)output_data, sd->buf_size, 0 };
82
+ size_t const ret = ZSTD_decompressStream(sd->ctx, &output, &input);
83
+ if (ZSTD_isError(ret)) {
84
+ rb_raise(rb_eRuntimeError, "compress error error code: %s", ZSTD_getErrorName(ret));
85
+ }
86
+ rb_str_cat(result, output.dst, output.pos);
87
+ }
88
+ return result;
89
+ }
90
+
91
+ static VALUE
92
+ rb_streaming_decompress_addstr(VALUE obj, VALUE src)
93
+ {
94
+ StringValue(src);
95
+ const char* input_data = RSTRING_PTR(src);
96
+ size_t input_size = RSTRING_LEN(src);
97
+ ZSTD_inBuffer input = { input_data, input_size, 0 };
98
+
99
+ struct streaming_decompress_t* sd;
100
+ TypedData_Get_Struct(obj, struct streaming_decompress_t, &streaming_decompress_type, sd);
101
+ const char* output_data = RSTRING_PTR(sd->buf);
102
+
103
+ while (input.pos < input.size) {
104
+ ZSTD_outBuffer output = { (void*)output_data, sd->buf_size, 0 };
105
+ size_t const result = ZSTD_decompressStream(sd->ctx, &output, &input);
106
+ if (ZSTD_isError(result)) {
107
+ rb_raise(rb_eRuntimeError, "compress error error code: %s", ZSTD_getErrorName(result));
108
+ }
109
+ }
110
+ return obj;
111
+ }
112
+
113
+ extern VALUE rb_mZstd, cStreamingDecompress;
114
+ void
115
+ zstd_ruby_streaming_decompress_init(void)
116
+ {
117
+ VALUE cStreamingDecompress = rb_define_class_under(rb_mZstd, "StreamingDecompress", rb_cObject);
118
+ rb_define_alloc_func(cStreamingDecompress, rb_streaming_decompress_allocate);
119
+ rb_define_method(cStreamingDecompress, "initialize", rb_streaming_decompress_initialize, 0);
120
+ rb_define_method(cStreamingDecompress, "decompress", rb_streaming_decompress_decompress, 1);
121
+ rb_define_method(cStreamingDecompress, "<<", rb_streaming_decompress_addstr, 1);
122
+ }
123
+
@@ -1,5 +1,6 @@
1
- #include "zstdruby.h"
2
- #include "./libzstd/zstd.h"
1
+ #include <common.h>
2
+
3
+ extern VALUE rb_mZstd;
3
4
 
4
5
  static VALUE zstdVersion(VALUE self)
5
6
  {
@@ -7,41 +8,74 @@ static VALUE zstdVersion(VALUE self)
7
8
  return INT2NUM(version);
8
9
  }
9
10
 
10
- static VALUE compress(int argc, VALUE *argv, VALUE self)
11
+ static VALUE rb_compress(int argc, VALUE *argv, VALUE self)
11
12
  {
12
13
  VALUE input_value;
13
14
  VALUE compression_level_value;
14
15
  rb_scan_args(argc, argv, "11", &input_value, &compression_level_value);
16
+ int compression_level = convert_compression_level(compression_level_value);
15
17
 
16
- Check_Type(input_value, RUBY_T_STRING);
17
- const char* input_data = RSTRING_PTR(input_value);
18
+ StringValue(input_value);
19
+ char* input_data = RSTRING_PTR(input_value);
18
20
  size_t input_size = RSTRING_LEN(input_value);
21
+ size_t max_compressed_size = ZSTD_compressBound(input_size);
19
22
 
20
- int compression_level;
21
- if (NIL_P(compression_level_value)) {
22
- compression_level = 0; // The default. See ZSTD_CLEVEL_DEFAULT in zstd_compress.c
23
- } else {
24
- compression_level = NUM2INT(compression_level_value);
23
+ VALUE output = rb_str_new(NULL, max_compressed_size);
24
+ char* output_data = RSTRING_PTR(output);
25
+ size_t compressed_size = ZSTD_compress((void*)output_data, max_compressed_size,
26
+ (void*)input_data, input_size, compression_level);
27
+ if (ZSTD_isError(compressed_size)) {
28
+ rb_raise(rb_eRuntimeError, "%s: %s", "compress failed", ZSTD_getErrorName(compressed_size));
25
29
  }
26
30
 
27
- // do compress
31
+ rb_str_resize(output, compressed_size);
32
+ return output;
33
+ }
34
+
35
+ static VALUE rb_compress_using_dict(int argc, VALUE *argv, VALUE self)
36
+ {
37
+ VALUE input_value;
38
+ VALUE dict;
39
+ VALUE compression_level_value;
40
+ rb_scan_args(argc, argv, "21", &input_value, &dict, &compression_level_value);
41
+ int compression_level = convert_compression_level(compression_level_value);
42
+
43
+ StringValue(input_value);
44
+ char* input_data = RSTRING_PTR(input_value);
45
+ size_t input_size = RSTRING_LEN(input_value);
28
46
  size_t max_compressed_size = ZSTD_compressBound(input_size);
29
47
 
48
+ char* dict_buffer = RSTRING_PTR(dict);
49
+ size_t dict_size = RSTRING_LEN(dict);
50
+
51
+ ZSTD_CDict* const cdict = ZSTD_createCDict(dict_buffer, dict_size, compression_level);
52
+ if (cdict == NULL) {
53
+ rb_raise(rb_eRuntimeError, "%s", "ZSTD_createCDict failed");
54
+ }
55
+ ZSTD_CCtx* const ctx = ZSTD_createCCtx();
56
+ if (ctx == NULL) {
57
+ ZSTD_freeCDict(cdict);
58
+ rb_raise(rb_eRuntimeError, "%s", "ZSTD_createCCtx failed");
59
+ }
60
+
30
61
  VALUE output = rb_str_new(NULL, max_compressed_size);
31
62
  char* output_data = RSTRING_PTR(output);
32
-
33
- size_t compressed_size = ZSTD_compress((void*)output_data, max_compressed_size,
34
- (const void*)input_data, input_size, compression_level);
63
+ size_t const compressed_size = ZSTD_compress_usingCDict(ctx, (void*)output_data, max_compressed_size,
64
+ (void*)input_data, input_size, cdict);
35
65
 
36
66
  if (ZSTD_isError(compressed_size)) {
67
+ ZSTD_freeCDict(cdict);
68
+ ZSTD_freeCCtx(ctx);
37
69
  rb_raise(rb_eRuntimeError, "%s: %s", "compress failed", ZSTD_getErrorName(compressed_size));
38
- } else {
39
- rb_str_resize(output, compressed_size);
40
70
  }
41
71
 
72
+ rb_str_resize(output, compressed_size);
73
+ ZSTD_freeCDict(cdict);
74
+ ZSTD_freeCCtx(ctx);
42
75
  return output;
43
76
  }
44
77
 
78
+
45
79
  static VALUE decompress_buffered(const char* input_data, size_t input_size)
46
80
  {
47
81
  const size_t outputBufferSize = 4096;
@@ -57,7 +91,6 @@ static VALUE decompress_buffered(const char* input_data, size_t input_size)
57
91
  rb_raise(rb_eRuntimeError, "%s: %s", "ZSTD_initDStream failed", ZSTD_getErrorName(initResult));
58
92
  }
59
93
 
60
-
61
94
  VALUE output_string = rb_str_new(NULL, 0);
62
95
  ZSTD_outBuffer output = { NULL, 0, 0 };
63
96
 
@@ -79,23 +112,24 @@ static VALUE decompress_buffered(const char* input_data, size_t input_size)
79
112
  return output_string;
80
113
  }
81
114
 
82
- static VALUE decompress(VALUE self, VALUE input)
115
+ static VALUE rb_decompress(VALUE self, VALUE input_value)
83
116
  {
84
- Check_Type(input, T_STRING);
85
- const char* input_data = RSTRING_PTR(input);
86
- size_t input_size = RSTRING_LEN(input);
87
-
88
- uint64_t uncompressed_size = ZSTD_getDecompressedSize(input_data, input_size);
117
+ StringValue(input_value);
118
+ char* input_data = RSTRING_PTR(input_value);
119
+ size_t input_size = RSTRING_LEN(input_value);
89
120
 
90
- if (uncompressed_size == 0) {
121
+ unsigned long long const uncompressed_size = ZSTD_getFrameContentSize(input_data, input_size);
122
+ if (uncompressed_size == ZSTD_CONTENTSIZE_ERROR) {
123
+ rb_raise(rb_eRuntimeError, "%s: %s", "not compressed by zstd", ZSTD_getErrorName(uncompressed_size));
124
+ }
125
+ if (uncompressed_size == ZSTD_CONTENTSIZE_UNKNOWN) {
91
126
  return decompress_buffered(input_data, input_size);
92
127
  }
93
128
 
94
129
  VALUE output = rb_str_new(NULL, uncompressed_size);
95
130
  char* output_data = RSTRING_PTR(output);
96
-
97
- size_t decompress_size = ZSTD_decompress((void*)output_data, uncompressed_size,
98
- (const void*)input_data, input_size);
131
+ size_t const decompress_size = ZSTD_decompress((void*)output_data, uncompressed_size,
132
+ (void*)input_data, input_size);
99
133
 
100
134
  if (ZSTD_isError(decompress_size)) {
101
135
  rb_raise(rb_eRuntimeError, "%s: %s", "decompress error", ZSTD_getErrorName(decompress_size));
@@ -104,13 +138,61 @@ static VALUE decompress(VALUE self, VALUE input)
104
138
  return output;
105
139
  }
106
140
 
107
- VALUE rb_mZstd;
141
+ static VALUE rb_decompress_using_dict(int argc, VALUE *argv, VALUE self)
142
+ {
143
+ VALUE input_value;
144
+ VALUE dict;
145
+ rb_scan_args(argc, argv, "20", &input_value, &dict);
146
+
147
+ StringValue(input_value);
148
+ char* input_data = RSTRING_PTR(input_value);
149
+ size_t input_size = RSTRING_LEN(input_value);
150
+ unsigned long long const uncompressed_size = ZSTD_getFrameContentSize(input_data, input_size);
151
+ if (uncompressed_size == ZSTD_CONTENTSIZE_ERROR) {
152
+ rb_raise(rb_eRuntimeError, "%s: %s", "not compressed by zstd", ZSTD_getErrorName(uncompressed_size));
153
+ }
154
+ if (uncompressed_size == ZSTD_CONTENTSIZE_UNKNOWN) {
155
+ return decompress_buffered(input_data, input_size);
156
+ }
157
+ VALUE output = rb_str_new(NULL, uncompressed_size);
158
+ char* output_data = RSTRING_PTR(output);
159
+
160
+ char* dict_buffer = RSTRING_PTR(dict);
161
+ size_t dict_size = RSTRING_LEN(dict);
162
+ ZSTD_DDict* const ddict = ZSTD_createDDict(dict_buffer, dict_size);
163
+ if (ddict == NULL) {
164
+ rb_raise(rb_eRuntimeError, "%s", "ZSTD_createDDict failed");
165
+ }
166
+
167
+ unsigned const expected_dict_id = ZSTD_getDictID_fromDDict(ddict);
168
+ unsigned const actual_dict_id = ZSTD_getDictID_fromFrame(input_data, input_size);
169
+ if (expected_dict_id != actual_dict_id) {
170
+ ZSTD_freeDDict(ddict);
171
+ rb_raise(rb_eRuntimeError, "%s: %s", "DictID mismatch", ZSTD_getErrorName(uncompressed_size));
172
+ }
173
+
174
+ ZSTD_DCtx* const ctx = ZSTD_createDCtx();
175
+ if (ctx == NULL) {
176
+ ZSTD_freeDDict(ddict);
177
+ rb_raise(rb_eRuntimeError, "%s", "ZSTD_createDCtx failed");
178
+ }
179
+ size_t const decompress_size = ZSTD_decompress_usingDDict(ctx, output_data, uncompressed_size, input_data, input_size, ddict);
180
+ if (ZSTD_isError(decompress_size)) {
181
+ ZSTD_freeDDict(ddict);
182
+ ZSTD_freeDCtx(ctx);
183
+ rb_raise(rb_eRuntimeError, "%s: %s", "decompress error", ZSTD_getErrorName(decompress_size));
184
+ }
185
+ ZSTD_freeDDict(ddict);
186
+ ZSTD_freeDCtx(ctx);
187
+ return output;
188
+ }
108
189
 
109
190
  void
110
- Init_zstdruby(void)
191
+ zstd_ruby_init(void)
111
192
  {
112
- rb_mZstd = rb_define_module("Zstd");
113
193
  rb_define_module_function(rb_mZstd, "zstd_version", zstdVersion, 0);
114
- rb_define_module_function(rb_mZstd, "compress", compress, -1);
115
- rb_define_module_function(rb_mZstd, "decompress", decompress, 1);
194
+ rb_define_module_function(rb_mZstd, "compress", rb_compress, -1);
195
+ rb_define_module_function(rb_mZstd, "compress_using_dict", rb_compress_using_dict, -1);
196
+ rb_define_module_function(rb_mZstd, "decompress", rb_decompress, 1);
197
+ rb_define_module_function(rb_mZstd, "decompress_using_dict", rb_decompress_using_dict, -1);
116
198
  }
@@ -1,3 +1,3 @@
1
1
  module Zstd
2
- VERSION = "1.4.5.0"
2
+ VERSION = "1.5.5.0"
3
3
  end
data/lib/zstd-ruby.rb CHANGED
@@ -2,5 +2,4 @@ require "zstd-ruby/version"
2
2
  require "zstd-ruby/zstdruby"
3
3
 
4
4
  module Zstd
5
- # Your code goes here...
6
5
  end