multi_compress 0.3.2 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (210) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +10 -3
  3. data/GET_STARTED.md +3 -3
  4. data/README.md +75 -66
  5. data/THIRD_PARTY_NOTICES.md +24 -0
  6. data/ext/multi_compress/brotli_dec_static_init.c +3 -0
  7. data/ext/multi_compress/brotli_enc_static_init.c +3 -0
  8. data/ext/multi_compress/extconf.rb +22 -1
  9. data/ext/multi_compress/vendor/.vendored +2 -2
  10. data/ext/multi_compress/vendor/brotli/LICENSE +19 -0
  11. data/ext/multi_compress/vendor/brotli/c/common/constants.c +7 -7
  12. data/ext/multi_compress/vendor/brotli/c/common/constants.h +2 -5
  13. data/ext/multi_compress/vendor/brotli/c/common/context.c +2 -2
  14. data/ext/multi_compress/vendor/brotli/c/common/context.h +1 -2
  15. data/ext/multi_compress/vendor/brotli/c/common/dictionary.c +4 -5856
  16. data/ext/multi_compress/vendor/brotli/c/common/dictionary.h +1 -2
  17. data/ext/multi_compress/vendor/brotli/c/common/dictionary_inc.h +5847 -0
  18. data/ext/multi_compress/vendor/brotli/c/common/platform.c +0 -4
  19. data/ext/multi_compress/vendor/brotli/c/common/platform.h +182 -43
  20. data/ext/multi_compress/vendor/brotli/c/common/shared_dictionary.c +3 -7
  21. data/ext/multi_compress/vendor/brotli/c/common/shared_dictionary_internal.h +1 -1
  22. data/ext/multi_compress/vendor/brotli/c/common/static_init.h +56 -0
  23. data/ext/multi_compress/vendor/brotli/c/common/transform.c +6 -4
  24. data/ext/multi_compress/vendor/brotli/c/common/transform.h +1 -2
  25. data/ext/multi_compress/vendor/brotli/c/common/version.h +3 -3
  26. data/ext/multi_compress/vendor/brotli/c/dec/bit_reader.c +2 -3
  27. data/ext/multi_compress/vendor/brotli/c/dec/bit_reader.h +0 -4
  28. data/ext/multi_compress/vendor/brotli/c/dec/decode.c +128 -39
  29. data/ext/multi_compress/vendor/brotli/c/dec/huffman.c +2 -5
  30. data/ext/multi_compress/vendor/brotli/c/dec/huffman.h +0 -2
  31. data/ext/multi_compress/vendor/brotli/c/dec/prefix.c +67 -0
  32. data/ext/multi_compress/vendor/brotli/c/dec/prefix.h +18 -708
  33. data/ext/multi_compress/vendor/brotli/c/dec/prefix_inc.h +707 -0
  34. data/ext/multi_compress/vendor/brotli/c/dec/state.c +18 -15
  35. data/ext/multi_compress/vendor/brotli/c/dec/state.h +2 -6
  36. data/ext/multi_compress/vendor/brotli/c/dec/static_init.c +53 -0
  37. data/ext/multi_compress/vendor/brotli/c/dec/static_init.h +30 -0
  38. data/ext/multi_compress/vendor/brotli/c/enc/backward_references.c +32 -8
  39. data/ext/multi_compress/vendor/brotli/c/enc/backward_references.h +1 -5
  40. data/ext/multi_compress/vendor/brotli/c/enc/backward_references_hq.c +15 -15
  41. data/ext/multi_compress/vendor/brotli/c/enc/backward_references_hq.h +1 -5
  42. data/ext/multi_compress/vendor/brotli/c/enc/bit_cost.c +28 -4
  43. data/ext/multi_compress/vendor/brotli/c/enc/bit_cost.h +8 -40
  44. data/ext/multi_compress/vendor/brotli/c/enc/bit_cost_inc.h +1 -1
  45. data/ext/multi_compress/vendor/brotli/c/enc/block_splitter.c +9 -12
  46. data/ext/multi_compress/vendor/brotli/c/enc/block_splitter.h +0 -3
  47. data/ext/multi_compress/vendor/brotli/c/enc/block_splitter_inc.h +14 -8
  48. data/ext/multi_compress/vendor/brotli/c/enc/brotli_bit_stream.c +10 -9
  49. data/ext/multi_compress/vendor/brotli/c/enc/brotli_bit_stream.h +0 -6
  50. data/ext/multi_compress/vendor/brotli/c/enc/cluster.c +0 -2
  51. data/ext/multi_compress/vendor/brotli/c/enc/cluster.h +0 -2
  52. data/ext/multi_compress/vendor/brotli/c/enc/command.c +1 -1
  53. data/ext/multi_compress/vendor/brotli/c/enc/command.h +8 -10
  54. data/ext/multi_compress/vendor/brotli/c/enc/compound_dictionary.c +3 -5
  55. data/ext/multi_compress/vendor/brotli/c/enc/compound_dictionary.h +1 -4
  56. data/ext/multi_compress/vendor/brotli/c/enc/compress_fragment.c +3 -13
  57. data/ext/multi_compress/vendor/brotli/c/enc/compress_fragment.h +0 -2
  58. data/ext/multi_compress/vendor/brotli/c/enc/compress_fragment_two_pass.c +5 -15
  59. data/ext/multi_compress/vendor/brotli/c/enc/compress_fragment_two_pass.h +0 -2
  60. data/ext/multi_compress/vendor/brotli/c/enc/dictionary_hash.c +127 -1830
  61. data/ext/multi_compress/vendor/brotli/c/enc/dictionary_hash.h +23 -3
  62. data/ext/multi_compress/vendor/brotli/c/enc/dictionary_hash_inc.h +1829 -0
  63. data/ext/multi_compress/vendor/brotli/c/enc/encode.c +77 -52
  64. data/ext/multi_compress/vendor/brotli/c/enc/encoder_dict.c +9 -7
  65. data/ext/multi_compress/vendor/brotli/c/enc/encoder_dict.h +2 -4
  66. data/ext/multi_compress/vendor/brotli/c/enc/entropy_encode.c +3 -6
  67. data/ext/multi_compress/vendor/brotli/c/enc/entropy_encode.h +2 -4
  68. data/ext/multi_compress/vendor/brotli/c/enc/entropy_encode_static.h +18 -12
  69. data/ext/multi_compress/vendor/brotli/c/enc/fast_log.c +1 -1
  70. data/ext/multi_compress/vendor/brotli/c/enc/fast_log.h +2 -3
  71. data/ext/multi_compress/vendor/brotli/c/enc/find_match_length.h +0 -2
  72. data/ext/multi_compress/vendor/brotli/c/enc/hash.h +38 -31
  73. data/ext/multi_compress/vendor/brotli/c/enc/hash_base.h +38 -0
  74. data/ext/multi_compress/vendor/brotli/c/enc/hash_forgetful_chain_inc.h +11 -1
  75. data/ext/multi_compress/vendor/brotli/c/enc/hash_longest_match64_inc.h +24 -7
  76. data/ext/multi_compress/vendor/brotli/c/enc/hash_longest_match64_simd_inc.h +304 -0
  77. data/ext/multi_compress/vendor/brotli/c/enc/hash_longest_match_inc.h +30 -11
  78. data/ext/multi_compress/vendor/brotli/c/enc/hash_longest_match_quickly_inc.h +4 -0
  79. data/ext/multi_compress/vendor/brotli/c/enc/hash_longest_match_simd_inc.h +278 -0
  80. data/ext/multi_compress/vendor/brotli/c/enc/histogram.c +1 -0
  81. data/ext/multi_compress/vendor/brotli/c/enc/histogram.h +0 -4
  82. data/ext/multi_compress/vendor/brotli/c/enc/literal_cost.c +4 -6
  83. data/ext/multi_compress/vendor/brotli/c/enc/literal_cost.h +0 -2
  84. data/ext/multi_compress/vendor/brotli/c/enc/matching_tag_mask.h +69 -0
  85. data/ext/multi_compress/vendor/brotli/c/enc/memory.c +0 -5
  86. data/ext/multi_compress/vendor/brotli/c/enc/memory.h +0 -4
  87. data/ext/multi_compress/vendor/brotli/c/enc/metablock.c +7 -9
  88. data/ext/multi_compress/vendor/brotli/c/enc/metablock.h +3 -3
  89. data/ext/multi_compress/vendor/brotli/c/enc/metablock_inc.h +4 -4
  90. data/ext/multi_compress/vendor/brotli/c/enc/params.h +0 -1
  91. data/ext/multi_compress/vendor/brotli/c/enc/prefix.h +0 -2
  92. data/ext/multi_compress/vendor/brotli/c/enc/quality.h +17 -10
  93. data/ext/multi_compress/vendor/brotli/c/enc/ringbuffer.h +1 -4
  94. data/ext/multi_compress/vendor/brotli/c/enc/state.h +2 -2
  95. data/ext/multi_compress/vendor/brotli/c/enc/static_dict.c +5 -11
  96. data/ext/multi_compress/vendor/brotli/c/enc/static_dict.h +1 -3
  97. data/ext/multi_compress/vendor/brotli/c/enc/static_dict_lut.c +224 -0
  98. data/ext/multi_compress/vendor/brotli/c/enc/static_dict_lut.h +20 -5837
  99. data/ext/multi_compress/vendor/brotli/c/enc/static_dict_lut_inc.h +5830 -0
  100. data/ext/multi_compress/vendor/brotli/c/enc/static_init.c +59 -0
  101. data/ext/multi_compress/vendor/brotli/c/enc/static_init.h +30 -0
  102. data/ext/multi_compress/vendor/brotli/c/enc/static_init_lazy.cc +26 -0
  103. data/ext/multi_compress/vendor/brotli/c/enc/utf8_util.c +1 -1
  104. data/ext/multi_compress/vendor/brotli/c/enc/utf8_util.h +0 -2
  105. data/ext/multi_compress/vendor/brotli/c/enc/write_bits.h +0 -2
  106. data/ext/multi_compress/vendor/brotli/c/include/brotli/decode.h +1 -1
  107. data/ext/multi_compress/vendor/brotli/c/include/brotli/encode.h +5 -1
  108. data/ext/multi_compress/vendor/brotli/c/include/brotli/port.h +4 -7
  109. data/ext/multi_compress/vendor/brotli/c/include/brotli/types.h +2 -2
  110. data/ext/multi_compress/vendor/lz4/LICENSE +12 -0
  111. data/ext/multi_compress/vendor/zstd/COPYING +339 -0
  112. data/ext/multi_compress/vendor/zstd/LICENSE +30 -0
  113. data/ext/multi_compress/vendor/zstd/lib/Makefile +67 -35
  114. data/ext/multi_compress/vendor/zstd/lib/README.md +33 -2
  115. data/ext/multi_compress/vendor/zstd/lib/common/allocations.h +55 -0
  116. data/ext/multi_compress/vendor/zstd/lib/common/bits.h +205 -0
  117. data/ext/multi_compress/vendor/zstd/lib/common/bitstream.h +84 -108
  118. data/ext/multi_compress/vendor/zstd/lib/common/compiler.h +170 -41
  119. data/ext/multi_compress/vendor/zstd/lib/common/cpu.h +37 -1
  120. data/ext/multi_compress/vendor/zstd/lib/common/debug.c +7 -1
  121. data/ext/multi_compress/vendor/zstd/lib/common/debug.h +21 -21
  122. data/ext/multi_compress/vendor/zstd/lib/common/entropy_common.c +12 -40
  123. data/ext/multi_compress/vendor/zstd/lib/common/error_private.c +10 -2
  124. data/ext/multi_compress/vendor/zstd/lib/common/error_private.h +46 -47
  125. data/ext/multi_compress/vendor/zstd/lib/common/fse.h +8 -100
  126. data/ext/multi_compress/vendor/zstd/lib/common/fse_decompress.c +28 -116
  127. data/ext/multi_compress/vendor/zstd/lib/common/huf.h +79 -166
  128. data/ext/multi_compress/vendor/zstd/lib/common/mem.h +46 -66
  129. data/ext/multi_compress/vendor/zstd/lib/common/pool.c +27 -11
  130. data/ext/multi_compress/vendor/zstd/lib/common/pool.h +8 -11
  131. data/ext/multi_compress/vendor/zstd/lib/common/portability_macros.h +45 -11
  132. data/ext/multi_compress/vendor/zstd/lib/common/threading.c +74 -14
  133. data/ext/multi_compress/vendor/zstd/lib/common/threading.h +5 -18
  134. data/ext/multi_compress/vendor/zstd/lib/common/xxhash.c +5 -11
  135. data/ext/multi_compress/vendor/zstd/lib/common/xxhash.h +2411 -1003
  136. data/ext/multi_compress/vendor/zstd/lib/common/zstd_common.c +1 -36
  137. data/ext/multi_compress/vendor/zstd/lib/common/zstd_deps.h +13 -1
  138. data/ext/multi_compress/vendor/zstd/lib/common/zstd_internal.h +13 -182
  139. data/ext/multi_compress/vendor/zstd/lib/common/zstd_trace.h +6 -13
  140. data/ext/multi_compress/vendor/zstd/lib/compress/clevels.h +1 -1
  141. data/ext/multi_compress/vendor/zstd/lib/compress/fse_compress.c +15 -131
  142. data/ext/multi_compress/vendor/zstd/lib/compress/hist.c +11 -1
  143. data/ext/multi_compress/vendor/zstd/lib/compress/hist.h +8 -1
  144. data/ext/multi_compress/vendor/zstd/lib/compress/huf_compress.c +283 -189
  145. data/ext/multi_compress/vendor/zstd/lib/compress/zstd_compress.c +2419 -903
  146. data/ext/multi_compress/vendor/zstd/lib/compress/zstd_compress_internal.h +423 -245
  147. data/ext/multi_compress/vendor/zstd/lib/compress/zstd_compress_literals.c +116 -40
  148. data/ext/multi_compress/vendor/zstd/lib/compress/zstd_compress_literals.h +16 -8
  149. data/ext/multi_compress/vendor/zstd/lib/compress/zstd_compress_sequences.c +10 -10
  150. data/ext/multi_compress/vendor/zstd/lib/compress/zstd_compress_sequences.h +8 -7
  151. data/ext/multi_compress/vendor/zstd/lib/compress/zstd_compress_superblock.c +254 -139
  152. data/ext/multi_compress/vendor/zstd/lib/compress/zstd_compress_superblock.h +1 -1
  153. data/ext/multi_compress/vendor/zstd/lib/compress/zstd_cwksp.h +184 -95
  154. data/ext/multi_compress/vendor/zstd/lib/compress/zstd_double_fast.c +163 -81
  155. data/ext/multi_compress/vendor/zstd/lib/compress/zstd_double_fast.h +18 -14
  156. data/ext/multi_compress/vendor/zstd/lib/compress/zstd_fast.c +507 -197
  157. data/ext/multi_compress/vendor/zstd/lib/compress/zstd_fast.h +7 -14
  158. data/ext/multi_compress/vendor/zstd/lib/compress/zstd_lazy.c +579 -484
  159. data/ext/multi_compress/vendor/zstd/lib/compress/zstd_lazy.h +133 -65
  160. data/ext/multi_compress/vendor/zstd/lib/compress/zstd_ldm.c +61 -40
  161. data/ext/multi_compress/vendor/zstd/lib/compress/zstd_ldm.h +7 -15
  162. data/ext/multi_compress/vendor/zstd/lib/compress/zstd_ldm_geartab.h +1 -1
  163. data/ext/multi_compress/vendor/zstd/lib/compress/zstd_opt.c +352 -218
  164. data/ext/multi_compress/vendor/zstd/lib/compress/zstd_opt.h +37 -21
  165. data/ext/multi_compress/vendor/zstd/lib/compress/zstd_preSplit.c +238 -0
  166. data/ext/multi_compress/vendor/zstd/lib/compress/zstd_preSplit.h +33 -0
  167. data/ext/multi_compress/vendor/zstd/lib/compress/zstdmt_compress.c +239 -175
  168. data/ext/multi_compress/vendor/zstd/lib/compress/zstdmt_compress.h +5 -16
  169. data/ext/multi_compress/vendor/zstd/lib/decompress/huf_decompress.c +543 -488
  170. data/ext/multi_compress/vendor/zstd/lib/decompress/huf_decompress_amd64.S +78 -61
  171. data/ext/multi_compress/vendor/zstd/lib/decompress/zstd_ddict.c +4 -4
  172. data/ext/multi_compress/vendor/zstd/lib/decompress/zstd_ddict.h +1 -1
  173. data/ext/multi_compress/vendor/zstd/lib/decompress/zstd_decompress.c +295 -115
  174. data/ext/multi_compress/vendor/zstd/lib/decompress/zstd_decompress_block.c +430 -293
  175. data/ext/multi_compress/vendor/zstd/lib/decompress/zstd_decompress_block.h +7 -2
  176. data/ext/multi_compress/vendor/zstd/lib/decompress/zstd_decompress_internal.h +11 -7
  177. data/ext/multi_compress/vendor/zstd/lib/deprecated/zbuff.h +1 -1
  178. data/ext/multi_compress/vendor/zstd/lib/deprecated/zbuff_common.c +1 -1
  179. data/ext/multi_compress/vendor/zstd/lib/deprecated/zbuff_compress.c +1 -1
  180. data/ext/multi_compress/vendor/zstd/lib/deprecated/zbuff_decompress.c +3 -1
  181. data/ext/multi_compress/vendor/zstd/lib/dictBuilder/cover.c +95 -46
  182. data/ext/multi_compress/vendor/zstd/lib/dictBuilder/cover.h +3 -9
  183. data/ext/multi_compress/vendor/zstd/lib/dictBuilder/divsufsort.h +0 -10
  184. data/ext/multi_compress/vendor/zstd/lib/dictBuilder/fastcover.c +4 -4
  185. data/ext/multi_compress/vendor/zstd/lib/dictBuilder/zdict.c +25 -97
  186. data/ext/multi_compress/vendor/zstd/lib/dll/example/Makefile +1 -1
  187. data/ext/multi_compress/vendor/zstd/lib/dll/example/README.md +1 -1
  188. data/ext/multi_compress/vendor/zstd/lib/legacy/zstd_legacy.h +38 -1
  189. data/ext/multi_compress/vendor/zstd/lib/legacy/zstd_v01.c +19 -50
  190. data/ext/multi_compress/vendor/zstd/lib/legacy/zstd_v01.h +1 -1
  191. data/ext/multi_compress/vendor/zstd/lib/legacy/zstd_v02.c +27 -80
  192. data/ext/multi_compress/vendor/zstd/lib/legacy/zstd_v02.h +1 -1
  193. data/ext/multi_compress/vendor/zstd/lib/legacy/zstd_v03.c +28 -83
  194. data/ext/multi_compress/vendor/zstd/lib/legacy/zstd_v03.h +1 -1
  195. data/ext/multi_compress/vendor/zstd/lib/legacy/zstd_v04.c +25 -74
  196. data/ext/multi_compress/vendor/zstd/lib/legacy/zstd_v04.h +1 -1
  197. data/ext/multi_compress/vendor/zstd/lib/legacy/zstd_v05.c +31 -76
  198. data/ext/multi_compress/vendor/zstd/lib/legacy/zstd_v05.h +1 -1
  199. data/ext/multi_compress/vendor/zstd/lib/legacy/zstd_v06.c +44 -88
  200. data/ext/multi_compress/vendor/zstd/lib/legacy/zstd_v06.h +1 -1
  201. data/ext/multi_compress/vendor/zstd/lib/legacy/zstd_v07.c +33 -84
  202. data/ext/multi_compress/vendor/zstd/lib/legacy/zstd_v07.h +1 -1
  203. data/ext/multi_compress/vendor/zstd/lib/libzstd.mk +65 -33
  204. data/ext/multi_compress/vendor/zstd/lib/libzstd.pc.in +5 -5
  205. data/ext/multi_compress/vendor/zstd/lib/module.modulemap +13 -3
  206. data/ext/multi_compress/vendor/zstd/lib/zdict.h +65 -36
  207. data/ext/multi_compress/vendor/zstd/lib/zstd.h +890 -267
  208. data/ext/multi_compress/vendor/zstd/lib/zstd_errors.h +28 -16
  209. data/lib/multi_compress/version.rb +1 -1
  210. metadata +29 -2
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -44,7 +44,6 @@
44
44
  #ifndef ZDICT_STATIC_LINKING_ONLY
45
45
  # define ZDICT_STATIC_LINKING_ONLY
46
46
  #endif
47
- #define HUF_STATIC_LINKING_ONLY
48
47
 
49
48
  #include "../common/mem.h" /* read */
50
49
  #include "../common/fse.h" /* FSE_normalizeCount, FSE_writeNCount */
@@ -54,6 +53,7 @@
54
53
  #include "../compress/zstd_compress_internal.h" /* ZSTD_loadCEntropy() */
55
54
  #include "../zdict.h"
56
55
  #include "divsufsort.h"
56
+ #include "../common/bits.h" /* ZSTD_NbCommonBytes */
57
57
 
58
58
 
59
59
  /*-*************************************
@@ -74,9 +74,9 @@ static const U32 g_selectivity_default = 9;
74
74
  * Console display
75
75
  ***************************************/
76
76
  #undef DISPLAY
77
- #define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush( stderr ); }
77
+ #define DISPLAY(...) do { fprintf(stderr, __VA_ARGS__); fflush( stderr ); } while (0)
78
78
  #undef DISPLAYLEVEL
79
- #define DISPLAYLEVEL(l, ...) if (notificationLevel>=l) { DISPLAY(__VA_ARGS__); } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
79
+ #define DISPLAYLEVEL(l, ...) do { if (notificationLevel>=l) { DISPLAY(__VA_ARGS__); } } while (0) /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
80
80
 
81
81
  static clock_t ZDICT_clockSpan(clock_t nPrevious) { return clock() - nPrevious; }
82
82
 
@@ -130,85 +130,6 @@ size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize)
130
130
  /*-********************************************************
131
131
  * Dictionary training functions
132
132
  **********************************************************/
133
- static unsigned ZDICT_NbCommonBytes (size_t val)
134
- {
135
- if (MEM_isLittleEndian()) {
136
- if (MEM_64bits()) {
137
- # if defined(_MSC_VER) && defined(_WIN64)
138
- if (val != 0) {
139
- unsigned long r;
140
- _BitScanForward64(&r, (U64)val);
141
- return (unsigned)(r >> 3);
142
- } else {
143
- /* Should not reach this code path */
144
- __assume(0);
145
- }
146
- # elif defined(__GNUC__) && (__GNUC__ >= 3)
147
- return (unsigned)(__builtin_ctzll((U64)val) >> 3);
148
- # else
149
- static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
150
- return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
151
- # endif
152
- } else { /* 32 bits */
153
- # if defined(_MSC_VER)
154
- if (val != 0) {
155
- unsigned long r;
156
- _BitScanForward(&r, (U32)val);
157
- return (unsigned)(r >> 3);
158
- } else {
159
- /* Should not reach this code path */
160
- __assume(0);
161
- }
162
- # elif defined(__GNUC__) && (__GNUC__ >= 3)
163
- return (unsigned)(__builtin_ctz((U32)val) >> 3);
164
- # else
165
- static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
166
- return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
167
- # endif
168
- }
169
- } else { /* Big Endian CPU */
170
- if (MEM_64bits()) {
171
- # if defined(_MSC_VER) && defined(_WIN64)
172
- if (val != 0) {
173
- unsigned long r;
174
- _BitScanReverse64(&r, val);
175
- return (unsigned)(r >> 3);
176
- } else {
177
- /* Should not reach this code path */
178
- __assume(0);
179
- }
180
- # elif defined(__GNUC__) && (__GNUC__ >= 3)
181
- return (unsigned)(__builtin_clzll(val) >> 3);
182
- # else
183
- unsigned r;
184
- const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */
185
- if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
186
- if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
187
- r += (!val);
188
- return r;
189
- # endif
190
- } else { /* 32 bits */
191
- # if defined(_MSC_VER)
192
- if (val != 0) {
193
- unsigned long r;
194
- _BitScanReverse(&r, (unsigned long)val);
195
- return (unsigned)(r >> 3);
196
- } else {
197
- /* Should not reach this code path */
198
- __assume(0);
199
- }
200
- # elif defined(__GNUC__) && (__GNUC__ >= 3)
201
- return (unsigned)(__builtin_clz((U32)val) >> 3);
202
- # else
203
- unsigned r;
204
- if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
205
- r += (!val);
206
- return r;
207
- # endif
208
- } }
209
- }
210
-
211
-
212
133
  /*! ZDICT_count() :
213
134
  Count the nb of common bytes between 2 pointers.
214
135
  Note : this function presumes end of buffer followed by noisy guard band.
@@ -223,7 +144,7 @@ static size_t ZDICT_count(const void* pIn, const void* pMatch)
223
144
  pMatch = (const char*)pMatch+sizeof(size_t);
224
145
  continue;
225
146
  }
226
- pIn = (const char*)pIn+ZDICT_NbCommonBytes(diff);
147
+ pIn = (const char*)pIn+ZSTD_NbCommonBytes(diff);
227
148
  return (size_t)((const char*)pIn - pStart);
228
149
  }
229
150
  }
@@ -451,7 +372,7 @@ static U32 ZDICT_tryMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, const
451
372
  elt = table[u];
452
373
  /* sort : improve rank */
453
374
  while ((u>1) && (table[u-1].savings < elt.savings))
454
- table[u] = table[u-1], u--;
375
+ table[u] = table[u-1], u--;
455
376
  table[u] = elt;
456
377
  return u;
457
378
  } }
@@ -556,10 +477,16 @@ static size_t ZDICT_trainBuffer_legacy(dictItem* dictList, U32 dictListSize,
556
477
  clock_t const refreshRate = CLOCKS_PER_SEC * 3 / 10;
557
478
 
558
479
  # undef DISPLAYUPDATE
559
- # define DISPLAYUPDATE(l, ...) if (notificationLevel>=l) { \
560
- if (ZDICT_clockSpan(displayClock) > refreshRate) \
561
- { displayClock = clock(); DISPLAY(__VA_ARGS__); \
562
- if (notificationLevel>=4) fflush(stderr); } }
480
+ # define DISPLAYUPDATE(l, ...) \
481
+ do { \
482
+ if (notificationLevel>=l) { \
483
+ if (ZDICT_clockSpan(displayClock) > refreshRate) { \
484
+ displayClock = clock(); \
485
+ DISPLAY(__VA_ARGS__); \
486
+ } \
487
+ if (notificationLevel>=4) fflush(stderr); \
488
+ } \
489
+ } while (0)
563
490
 
564
491
  /* init */
565
492
  DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */
@@ -602,7 +529,7 @@ static size_t ZDICT_trainBuffer_legacy(dictItem* dictList, U32 dictListSize,
602
529
  if (solution.length==0) { cursor++; continue; }
603
530
  ZDICT_insertDictItem(dictList, dictListSize, solution, buffer);
604
531
  cursor += solution.length;
605
- DISPLAYUPDATE(2, "\r%4.2f %% \r", (double)cursor / bufferSize * 100);
532
+ DISPLAYUPDATE(2, "\r%4.2f %% \r", (double)cursor / (double)bufferSize * 100.0);
606
533
  } }
607
534
 
608
535
  _cleanup:
@@ -645,15 +572,15 @@ static void ZDICT_countEStats(EStats_ress_t esr, const ZSTD_parameters* params,
645
572
  size_t cSize;
646
573
 
647
574
  if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */
648
- { size_t const errorCode = ZSTD_compressBegin_usingCDict(esr.zc, esr.dict);
575
+ { size_t const errorCode = ZSTD_compressBegin_usingCDict_deprecated(esr.zc, esr.dict);
649
576
  if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_compressBegin_usingCDict failed \n"); return; }
650
577
 
651
578
  }
652
- cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
579
+ cSize = ZSTD_compressBlock_deprecated(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
653
580
  if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (unsigned)srcSize); return; }
654
581
 
655
582
  if (cSize) { /* if == 0; block is not compressible */
656
- const seqStore_t* const seqStorePtr = ZSTD_getSeqStore(esr.zc);
583
+ const SeqStore_t* const seqStorePtr = ZSTD_getSeqStore(esr.zc);
657
584
 
658
585
  /* literals stats */
659
586
  { const BYTE* bytePtr;
@@ -681,7 +608,7 @@ static void ZDICT_countEStats(EStats_ress_t esr, const ZSTD_parameters* params,
681
608
  }
682
609
 
683
610
  if (nbSeq >= 2) { /* rep offsets */
684
- const seqDef* const seq = seqStorePtr->sequencesStart;
611
+ const SeqDef* const seq = seqStorePtr->sequencesStart;
685
612
  U32 offset1 = seq[0].offBase - ZSTD_REP_NUM;
686
613
  U32 offset2 = seq[1].offBase - ZSTD_REP_NUM;
687
614
  if (offset1 >= MAXREPOFFSET) offset1 = 0;
@@ -754,6 +681,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
754
681
  size_t const totalSrcSize = ZDICT_totalSampleSize(fileSizes, nbFiles);
755
682
  size_t const averageSampleSize = totalSrcSize / (nbFiles + !nbFiles);
756
683
  BYTE* dstPtr = (BYTE*)dstBuffer;
684
+ U32 wksp[HUF_CTABLE_WORKSPACE_SIZE_U32];
757
685
 
758
686
  /* init */
759
687
  DEBUGLOG(4, "ZDICT_analyzeEntropy");
@@ -794,7 +722,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
794
722
  } }
795
723
 
796
724
  /* analyze, build stats, starting with literals */
797
- { size_t maxNbBits = HUF_buildCTable (hufTable, countLit, 255, huffLog);
725
+ { size_t maxNbBits = HUF_buildCTable_wksp(hufTable, countLit, 255, huffLog, wksp, sizeof(wksp));
798
726
  if (HUF_isError(maxNbBits)) {
799
727
  eSize = maxNbBits;
800
728
  DISPLAYLEVEL(1, " HUF_buildCTable error \n");
@@ -803,7 +731,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
803
731
  if (maxNbBits==8) { /* not compressible : will fail on HUF_writeCTable() */
804
732
  DISPLAYLEVEL(2, "warning : pathological dataset : literals are not compressible : samples are noisy or too regular \n");
805
733
  ZDICT_flatLit(countLit); /* replace distribution by a fake "mostly flat but still compressible" distribution, that HUF_writeCTable() can encode */
806
- maxNbBits = HUF_buildCTable (hufTable, countLit, 255, huffLog);
734
+ maxNbBits = HUF_buildCTable_wksp(hufTable, countLit, 255, huffLog, wksp, sizeof(wksp));
807
735
  assert(maxNbBits==9);
808
736
  }
809
737
  huffLog = (U32)maxNbBits;
@@ -844,7 +772,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
844
772
  llLog = (U32)errorCode;
845
773
 
846
774
  /* write result to buffer */
847
- { size_t const hhSize = HUF_writeCTable(dstPtr, maxDstSize, hufTable, 255, huffLog);
775
+ { size_t const hhSize = HUF_writeCTable_wksp(dstPtr, maxDstSize, hufTable, 255, huffLog, wksp, sizeof(wksp));
848
776
  if (HUF_isError(hhSize)) {
849
777
  eSize = hhSize;
850
778
  DISPLAYLEVEL(1, "HUF_writeCTable error \n");
@@ -1,5 +1,5 @@
1
1
  # ################################################################
2
- # Copyright (c) Yann Collet, Facebook, Inc.
2
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
3
3
  # All rights reserved.
4
4
  #
5
5
  # This source code is licensed under both the BSD-style license (found in the
@@ -46,7 +46,7 @@ The compiled executable will require ZSTD DLL which is available at `dll\libzstd
46
46
  Open `example\fullbench-dll.sln` to compile `fullbench-dll` that uses a
47
47
  dynamic ZSTD library from the `dll` directory. The solution works with Visual C++
48
48
  2010 or newer. When one will open the solution with Visual C++ newer than 2010
49
- then the solution will upgraded to the current version.
49
+ then the solution will be upgraded to the current version.
50
50
 
51
51
  ## Using ZSTD DLL with Visual C++
52
52
 
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -124,6 +124,20 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
124
124
  const void* dict,size_t dictSize)
125
125
  {
126
126
  U32 const version = ZSTD_isLegacy(src, compressedSize);
127
+ char x;
128
+ /* Avoid passing NULL to legacy decoding. */
129
+ if (dst == NULL) {
130
+ assert(dstCapacity == 0);
131
+ dst = &x;
132
+ }
133
+ if (src == NULL) {
134
+ assert(compressedSize == 0);
135
+ src = &x;
136
+ }
137
+ if (dict == NULL) {
138
+ assert(dictSize == 0);
139
+ dict = &x;
140
+ }
127
141
  (void)dst; (void)dstCapacity; (void)dict; (void)dictSize; /* unused when ZSTD_LEGACY_SUPPORT >= 8 */
128
142
  switch(version)
129
143
  {
@@ -242,6 +256,13 @@ MEM_STATIC ZSTD_frameSizeInfo ZSTD_findFrameSizeInfoLegacy(const void *src, size
242
256
  frameSizeInfo.compressedSize = ERROR(srcSize_wrong);
243
257
  frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
244
258
  }
259
+ /* In all cases, decompressedBound == nbBlocks * ZSTD_BLOCKSIZE_MAX.
260
+ * So we can compute nbBlocks without having to change every function.
261
+ */
262
+ if (frameSizeInfo.decompressedBound != ZSTD_CONTENTSIZE_ERROR) {
263
+ assert((frameSizeInfo.decompressedBound & (ZSTD_BLOCKSIZE_MAX - 1)) == 0);
264
+ frameSizeInfo.nbBlocks = (size_t)(frameSizeInfo.decompressedBound / ZSTD_BLOCKSIZE_MAX);
265
+ }
245
266
  return frameSizeInfo;
246
267
  }
247
268
 
@@ -280,6 +301,12 @@ MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version)
280
301
  MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U32 newVersion,
281
302
  const void* dict, size_t dictSize)
282
303
  {
304
+ char x;
305
+ /* Avoid passing NULL to legacy decoding. */
306
+ if (dict == NULL) {
307
+ assert(dictSize == 0);
308
+ dict = &x;
309
+ }
283
310
  DEBUGLOG(5, "ZSTD_initLegacyStream for v0.%u", newVersion);
284
311
  if (prevVersion != newVersion) ZSTD_freeLegacyStreamContext(*legacyContext, prevVersion);
285
312
  switch(newVersion)
@@ -339,6 +366,16 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U
339
366
  MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version,
340
367
  ZSTD_outBuffer* output, ZSTD_inBuffer* input)
341
368
  {
369
+ static char x;
370
+ /* Avoid passing NULL to legacy decoding. */
371
+ if (output->dst == NULL) {
372
+ assert(output->size == 0);
373
+ output->dst = &x;
374
+ }
375
+ if (input->src == NULL) {
376
+ assert(input->size == 0);
377
+ input->src = &x;
378
+ }
342
379
  DEBUGLOG(5, "ZSTD_decompressLegacyStream for v0.%u", version);
343
380
  switch(version)
344
381
  {
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -14,6 +14,7 @@
14
14
  ******************************************/
15
15
  #include <stddef.h> /* size_t, ptrdiff_t */
16
16
  #include "zstd_v01.h"
17
+ #include "../common/compiler.h"
17
18
  #include "../common/error_private.h"
18
19
 
19
20
 
@@ -190,25 +191,6 @@ typedef signed long long S64;
190
191
  /****************************************************************
191
192
  * Memory I/O
192
193
  *****************************************************************/
193
- /* FSE_FORCE_MEMORY_ACCESS
194
- * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
195
- * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
196
- * The below switch allow to select different access method for improved performance.
197
- * Method 0 (default) : use `memcpy()`. Safe and portable.
198
- * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
199
- * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
200
- * Method 2 : direct access. This method is portable but violate C standard.
201
- * It can generate buggy code on targets generating assembly depending on alignment.
202
- * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
203
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
204
- * Prefer these methods in priority order (0 > 1 > 2)
205
- */
206
- #ifndef FSE_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
207
- # if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)
208
- # define FSE_FORCE_MEMORY_ACCESS 1
209
- # endif
210
- #endif
211
-
212
194
 
213
195
  static unsigned FSE_32bits(void)
214
196
  {
@@ -221,24 +203,6 @@ static unsigned FSE_isLittleEndian(void)
221
203
  return one.c[0];
222
204
  }
223
205
 
224
- #if defined(FSE_FORCE_MEMORY_ACCESS) && (FSE_FORCE_MEMORY_ACCESS==2)
225
-
226
- static U16 FSE_read16(const void* memPtr) { return *(const U16*) memPtr; }
227
- static U32 FSE_read32(const void* memPtr) { return *(const U32*) memPtr; }
228
- static U64 FSE_read64(const void* memPtr) { return *(const U64*) memPtr; }
229
-
230
- #elif defined(FSE_FORCE_MEMORY_ACCESS) && (FSE_FORCE_MEMORY_ACCESS==1)
231
-
232
- /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
233
- /* currently only defined for gcc and icc */
234
- typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign;
235
-
236
- static U16 FSE_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
237
- static U32 FSE_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
238
- static U64 FSE_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
239
-
240
- #else
241
-
242
206
  static U16 FSE_read16(const void* memPtr)
243
207
  {
244
208
  U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
@@ -254,8 +218,6 @@ static U64 FSE_read64(const void* memPtr)
254
218
  U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
255
219
  }
256
220
 
257
- #endif /* FSE_FORCE_MEMORY_ACCESS */
258
-
259
221
  static U16 FSE_readLE16(const void* memPtr)
260
222
  {
261
223
  if (FSE_isLittleEndian())
@@ -1190,7 +1152,7 @@ static size_t HUF_decompress (void* dst, size_t maxDstSize, const void* cSrc, si
1190
1152
  zstd - standard compression library
1191
1153
  Copyright (C) 2014-2015, Yann Collet.
1192
1154
 
1193
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
1155
+ BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
1194
1156
 
1195
1157
  Redistribution and use in source and binary forms, with or without
1196
1158
  modification, are permitted provided that the following conditions are
@@ -1421,7 +1383,7 @@ typedef struct {
1421
1383
  BYTE* matchLength;
1422
1384
  BYTE* dumpsStart;
1423
1385
  BYTE* dumps;
1424
- } seqStore_t;
1386
+ } SeqStore_t;
1425
1387
 
1426
1388
 
1427
1389
  typedef struct ZSTD_Cctx_s
@@ -1429,7 +1391,7 @@ typedef struct ZSTD_Cctx_s
1429
1391
  const BYTE* base;
1430
1392
  U32 current;
1431
1393
  U32 nextUpdate;
1432
- seqStore_t seqStore;
1394
+ SeqStore_t seqStore;
1433
1395
  #ifdef __AVX2__
1434
1396
  __m256i hashTable[HASH_TABLESIZE>>3];
1435
1397
  #else
@@ -1759,20 +1721,26 @@ static size_t ZSTD_execSequence(BYTE* op,
1759
1721
  static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
1760
1722
  static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* subtracted */
1761
1723
  const BYTE* const ostart = op;
1724
+ BYTE* const oLitEnd = op + sequence.litLength;
1762
1725
  const size_t litLength = sequence.litLength;
1763
1726
  BYTE* const endMatch = op + litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */
1764
1727
  const BYTE* const litEnd = *litPtr + litLength;
1765
1728
 
1766
- /* check */
1729
+ /* checks */
1730
+ size_t const seqLength = sequence.litLength + sequence.matchLength;
1731
+
1732
+ if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
1733
+ if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
1734
+ /* Now we know there are no overflow in literal nor match lengths, can use pointer checks */
1735
+ if (sequence.offset > (U32)(oLitEnd - base)) return ERROR(corruption_detected);
1736
+
1767
1737
  if (endMatch > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
1768
- if (litEnd > litLimit) return ERROR(corruption_detected);
1769
- if (sequence.matchLength > (size_t)(*litPtr-op)) return ERROR(dstSize_tooSmall); /* overwrite literal segment */
1738
+ if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
1739
+ if (sequence.matchLength > (size_t)(*litPtr-op)) return ERROR(dstSize_tooSmall); /* overwrite literal segment */
1770
1740
 
1771
1741
  /* copy Literals */
1772
- if (((size_t)(*litPtr - op) < 8) || ((size_t)(oend-litEnd) < 8) || (op+litLength > oend-8))
1773
- memmove(op, *litPtr, litLength); /* overwrite risk */
1774
- else
1775
- ZSTD_wildcopy(op, *litPtr, litLength);
1742
+ ZSTD_memmove(op, *litPtr, sequence.litLength); /* note : v0.1 seems to allow scenarios where output or input are close to end of buffer */
1743
+
1776
1744
  op += litLength;
1777
1745
  *litPtr = litEnd; /* update for next sequence */
1778
1746
 
@@ -2151,6 +2119,7 @@ size_t ZSTDv01_decompressContinue(ZSTDv01_Dctx* dctx, void* dst, size_t maxDstSi
2151
2119
  }
2152
2120
  ctx->phase = 1;
2153
2121
  ctx->expected = ZSTD_blockHeaderSize;
2122
+ if (ZSTDv01_isError(rSize)) return rSize;
2154
2123
  ctx->previousDstEnd = (void*)( ((char*)dst) + rSize);
2155
2124
  return rSize;
2156
2125
  }
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the