zstdlib 0.10.0-x64-mingw32 → 0.11.0-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (83) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGES.md +8 -0
  3. data/ext/zstdlib_c/extconf.rb +2 -2
  4. data/ext/zstdlib_c/ruby/zlib-3.2/zstdlib.c +5090 -0
  5. data/ext/zstdlib_c/zstd-1.5.5/lib/common/allocations.h +55 -0
  6. data/ext/zstdlib_c/zstd-1.5.5/lib/common/bits.h +200 -0
  7. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/bitstream.h +19 -60
  8. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/compiler.h +26 -3
  9. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/cpu.h +1 -1
  10. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/debug.c +1 -1
  11. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/debug.h +1 -1
  12. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/entropy_common.c +12 -40
  13. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/error_private.c +9 -2
  14. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/error_private.h +1 -1
  15. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/fse.h +5 -83
  16. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/fse_decompress.c +7 -99
  17. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/huf.h +65 -156
  18. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/mem.h +39 -46
  19. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/pool.c +26 -10
  20. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/pool.h +7 -1
  21. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/portability_macros.h +22 -3
  22. data/ext/zstdlib_c/zstd-1.5.5/lib/common/threading.c +176 -0
  23. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/threading.h +5 -10
  24. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/xxhash.c +2 -2
  25. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/xxhash.h +8 -8
  26. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/zstd_common.c +1 -36
  27. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/zstd_deps.h +1 -1
  28. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/zstd_internal.h +17 -118
  29. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/zstd_trace.h +3 -3
  30. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/clevels.h +1 -1
  31. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/fse_compress.c +7 -124
  32. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/hist.c +1 -1
  33. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/hist.h +1 -1
  34. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/huf_compress.c +234 -169
  35. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_compress.c +1243 -538
  36. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_compress_internal.h +225 -151
  37. data/ext/zstdlib_c/zstd-1.5.5/lib/compress/zstd_compress_literals.c +235 -0
  38. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_compress_literals.h +16 -8
  39. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_compress_sequences.c +3 -3
  40. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_compress_sequences.h +1 -1
  41. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_compress_superblock.c +25 -21
  42. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_compress_superblock.h +1 -1
  43. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_cwksp.h +128 -62
  44. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_double_fast.c +95 -33
  45. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_double_fast.h +3 -2
  46. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_fast.c +433 -148
  47. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_fast.h +3 -2
  48. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_lazy.c +398 -345
  49. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_lazy.h +4 -2
  50. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_ldm.c +5 -5
  51. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_ldm.h +1 -1
  52. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_ldm_geartab.h +1 -1
  53. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_opt.c +106 -80
  54. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_opt.h +1 -1
  55. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstdmt_compress.c +17 -9
  56. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstdmt_compress.h +1 -1
  57. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/decompress/huf_decompress.c +434 -441
  58. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/decompress/huf_decompress_amd64.S +30 -39
  59. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/decompress/zstd_ddict.c +4 -4
  60. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/decompress/zstd_ddict.h +1 -1
  61. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/decompress/zstd_decompress.c +205 -80
  62. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/decompress/zstd_decompress_block.c +201 -81
  63. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/decompress/zstd_decompress_block.h +6 -1
  64. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/decompress/zstd_decompress_internal.h +4 -2
  65. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/zdict.h +53 -31
  66. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/zstd.h +580 -135
  67. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/zstd_errors.h +27 -8
  68. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/zlibWrapper/gzclose.c +1 -1
  69. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/zlibWrapper/gzcompatibility.h +8 -8
  70. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/zlibWrapper/gzguts.h +10 -10
  71. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/zlibWrapper/gzlib.c +3 -3
  72. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/zlibWrapper/gzread.c +10 -10
  73. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/zlibWrapper/gzwrite.c +5 -5
  74. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/zlibWrapper/zstd_zlibwrapper.c +46 -44
  75. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/zlibWrapper/zstd_zlibwrapper.h +4 -1
  76. data/lib/2.4/zstdlib_c.so +0 -0
  77. data/lib/2.5/zstdlib_c.so +0 -0
  78. data/lib/2.6/zstdlib_c.so +0 -0
  79. data/lib/2.7/zstdlib_c.so +0 -0
  80. data/lib/3.0/zstdlib_c.so +0 -0
  81. metadata +80 -77
  82. data/ext/zstdlib_c/zstd-1.5.2/lib/common/threading.c +0 -122
  83. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_compress_literals.c +0 -159
@@ -0,0 +1,235 @@
1
+ /*
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
+ * All rights reserved.
4
+ *
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
9
+ */
10
+
11
+ /*-*************************************
12
+ * Dependencies
13
+ ***************************************/
14
+ #include "zstd_compress_literals.h"
15
+
16
+
17
+ /* **************************************************************
18
+ * Debug Traces
19
+ ****************************************************************/
20
+ #if DEBUGLEVEL >= 2
21
+
22
+ static size_t showHexa(const void* src, size_t srcSize)
23
+ {
24
+ const BYTE* const ip = (const BYTE*)src;
25
+ size_t u;
26
+ for (u=0; u<srcSize; u++) {
27
+ RAWLOG(5, " %02X", ip[u]); (void)ip;
28
+ }
29
+ RAWLOG(5, " \n");
30
+ return srcSize;
31
+ }
32
+
33
+ #endif
34
+
35
+
36
+ /* **************************************************************
37
+ * Literals compression - special cases
38
+ ****************************************************************/
39
+ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
40
+ {
41
+ BYTE* const ostart = (BYTE*)dst;
42
+ U32 const flSize = 1 + (srcSize>31) + (srcSize>4095);
43
+
44
+ DEBUGLOG(5, "ZSTD_noCompressLiterals: srcSize=%zu, dstCapacity=%zu", srcSize, dstCapacity);
45
+
46
+ RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall, "");
47
+
48
+ switch(flSize)
49
+ {
50
+ case 1: /* 2 - 1 - 5 */
51
+ ostart[0] = (BYTE)((U32)set_basic + (srcSize<<3));
52
+ break;
53
+ case 2: /* 2 - 2 - 12 */
54
+ MEM_writeLE16(ostart, (U16)((U32)set_basic + (1<<2) + (srcSize<<4)));
55
+ break;
56
+ case 3: /* 2 - 2 - 20 */
57
+ MEM_writeLE32(ostart, (U32)((U32)set_basic + (3<<2) + (srcSize<<4)));
58
+ break;
59
+ default: /* not necessary : flSize is {1,2,3} */
60
+ assert(0);
61
+ }
62
+
63
+ ZSTD_memcpy(ostart + flSize, src, srcSize);
64
+ DEBUGLOG(5, "Raw (uncompressed) literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize));
65
+ return srcSize + flSize;
66
+ }
67
+
68
+ static int allBytesIdentical(const void* src, size_t srcSize)
69
+ {
70
+ assert(srcSize >= 1);
71
+ assert(src != NULL);
72
+ { const BYTE b = ((const BYTE*)src)[0];
73
+ size_t p;
74
+ for (p=1; p<srcSize; p++) {
75
+ if (((const BYTE*)src)[p] != b) return 0;
76
+ }
77
+ return 1;
78
+ }
79
+ }
80
+
81
+ size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
82
+ {
83
+ BYTE* const ostart = (BYTE*)dst;
84
+ U32 const flSize = 1 + (srcSize>31) + (srcSize>4095);
85
+
86
+ assert(dstCapacity >= 4); (void)dstCapacity;
87
+ assert(allBytesIdentical(src, srcSize));
88
+
89
+ switch(flSize)
90
+ {
91
+ case 1: /* 2 - 1 - 5 */
92
+ ostart[0] = (BYTE)((U32)set_rle + (srcSize<<3));
93
+ break;
94
+ case 2: /* 2 - 2 - 12 */
95
+ MEM_writeLE16(ostart, (U16)((U32)set_rle + (1<<2) + (srcSize<<4)));
96
+ break;
97
+ case 3: /* 2 - 2 - 20 */
98
+ MEM_writeLE32(ostart, (U32)((U32)set_rle + (3<<2) + (srcSize<<4)));
99
+ break;
100
+ default: /* not necessary : flSize is {1,2,3} */
101
+ assert(0);
102
+ }
103
+
104
+ ostart[flSize] = *(const BYTE*)src;
105
+ DEBUGLOG(5, "RLE : Repeated Literal (%02X: %u times) -> %u bytes encoded", ((const BYTE*)src)[0], (U32)srcSize, (U32)flSize + 1);
106
+ return flSize+1;
107
+ }
108
+
109
+ /* ZSTD_minLiteralsToCompress() :
110
+ * returns minimal amount of literals
111
+ * for literal compression to even be attempted.
112
+ * Minimum is made tighter as compression strategy increases.
113
+ */
114
+ static size_t
115
+ ZSTD_minLiteralsToCompress(ZSTD_strategy strategy, HUF_repeat huf_repeat)
116
+ {
117
+ assert((int)strategy >= 0);
118
+ assert((int)strategy <= 9);
119
+ /* btultra2 : min 8 bytes;
120
+ * then 2x larger for each successive compression strategy
121
+ * max threshold 64 bytes */
122
+ { int const shift = MIN(9-(int)strategy, 3);
123
+ size_t const mintc = (huf_repeat == HUF_repeat_valid) ? 6 : (size_t)8 << shift;
124
+ DEBUGLOG(7, "minLiteralsToCompress = %zu", mintc);
125
+ return mintc;
126
+ }
127
+ }
128
+
129
+ size_t ZSTD_compressLiterals (
130
+ void* dst, size_t dstCapacity,
131
+ const void* src, size_t srcSize,
132
+ void* entropyWorkspace, size_t entropyWorkspaceSize,
133
+ const ZSTD_hufCTables_t* prevHuf,
134
+ ZSTD_hufCTables_t* nextHuf,
135
+ ZSTD_strategy strategy,
136
+ int disableLiteralCompression,
137
+ int suspectUncompressible,
138
+ int bmi2)
139
+ {
140
+ size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
141
+ BYTE* const ostart = (BYTE*)dst;
142
+ U32 singleStream = srcSize < 256;
143
+ symbolEncodingType_e hType = set_compressed;
144
+ size_t cLitSize;
145
+
146
+ DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i, srcSize=%u, dstCapacity=%zu)",
147
+ disableLiteralCompression, (U32)srcSize, dstCapacity);
148
+
149
+ DEBUGLOG(6, "Completed literals listing (%zu bytes)", showHexa(src, srcSize));
150
+
151
+ /* Prepare nextEntropy assuming reusing the existing table */
152
+ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
153
+
154
+ if (disableLiteralCompression)
155
+ return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
156
+
157
+ /* if too small, don't even attempt compression (speed opt) */
158
+ if (srcSize < ZSTD_minLiteralsToCompress(strategy, prevHuf->repeatMode))
159
+ return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
160
+
161
+ RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression");
162
+ { HUF_repeat repeat = prevHuf->repeatMode;
163
+ int const flags = 0
164
+ | (bmi2 ? HUF_flags_bmi2 : 0)
165
+ | (strategy < ZSTD_lazy && srcSize <= 1024 ? HUF_flags_preferRepeat : 0)
166
+ | (strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_flags_optimalDepth : 0)
167
+ | (suspectUncompressible ? HUF_flags_suspectUncompressible : 0);
168
+
169
+ typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int);
170
+ huf_compress_f huf_compress;
171
+ if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
172
+ huf_compress = singleStream ? HUF_compress1X_repeat : HUF_compress4X_repeat;
173
+ cLitSize = huf_compress(ostart+lhSize, dstCapacity-lhSize,
174
+ src, srcSize,
175
+ HUF_SYMBOLVALUE_MAX, LitHufLog,
176
+ entropyWorkspace, entropyWorkspaceSize,
177
+ (HUF_CElt*)nextHuf->CTable,
178
+ &repeat, flags);
179
+ DEBUGLOG(5, "%zu literals compressed into %zu bytes (before header)", srcSize, cLitSize);
180
+ if (repeat != HUF_repeat_none) {
181
+ /* reused the existing table */
182
+ DEBUGLOG(5, "reusing statistics from previous huffman block");
183
+ hType = set_repeat;
184
+ }
185
+ }
186
+
187
+ { size_t const minGain = ZSTD_minGain(srcSize, strategy);
188
+ if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) {
189
+ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
190
+ return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
191
+ } }
192
+ if (cLitSize==1) {
193
+ /* A return value of 1 signals that the alphabet consists of a single symbol.
194
+ * However, in some rare circumstances, it could be the compressed size (a single byte).
195
+ * For that outcome to have a chance to happen, it's necessary that `srcSize < 8`.
196
+ * (it's also necessary to not generate statistics).
197
+ * Therefore, in such a case, actively check that all bytes are identical. */
198
+ if ((srcSize >= 8) || allBytesIdentical(src, srcSize)) {
199
+ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
200
+ return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
201
+ } }
202
+
203
+ if (hType == set_compressed) {
204
+ /* using a newly constructed table */
205
+ nextHuf->repeatMode = HUF_repeat_check;
206
+ }
207
+
208
+ /* Build header */
209
+ switch(lhSize)
210
+ {
211
+ case 3: /* 2 - 2 - 10 - 10 */
212
+ if (!singleStream) assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
213
+ { U32 const lhc = hType + ((U32)(!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14);
214
+ MEM_writeLE24(ostart, lhc);
215
+ break;
216
+ }
217
+ case 4: /* 2 - 2 - 14 - 14 */
218
+ assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
219
+ { U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18);
220
+ MEM_writeLE32(ostart, lhc);
221
+ break;
222
+ }
223
+ case 5: /* 2 - 2 - 18 - 18 */
224
+ assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
225
+ { U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22);
226
+ MEM_writeLE32(ostart, lhc);
227
+ ostart[4] = (BYTE)(cLitSize >> 10);
228
+ break;
229
+ }
230
+ default: /* not possible : lhSize is {3,4,5} */
231
+ assert(0);
232
+ }
233
+ DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)srcSize, (U32)(lhSize+cLitSize));
234
+ return lhSize+cLitSize;
235
+ }
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -16,16 +16,24 @@
16
16
 
17
17
  size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize);
18
18
 
19
+ /* ZSTD_compressRleLiteralsBlock() :
20
+ * Conditions :
21
+ * - All bytes in @src are identical
22
+ * - dstCapacity >= 4 */
19
23
  size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize);
20
24
 
21
- /* If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
22
- size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
23
- ZSTD_hufCTables_t* nextHuf,
24
- ZSTD_strategy strategy, int disableLiteralCompression,
25
- void* dst, size_t dstCapacity,
25
+ /* ZSTD_compressLiterals():
26
+ * @entropyWorkspace: must be aligned on 4-bytes boundaries
27
+ * @entropyWorkspaceSize : must be >= HUF_WORKSPACE_SIZE
28
+ * @suspectUncompressible: sampling checks, to potentially skip huffman coding
29
+ */
30
+ size_t ZSTD_compressLiterals (void* dst, size_t dstCapacity,
26
31
  const void* src, size_t srcSize,
27
32
  void* entropyWorkspace, size_t entropyWorkspaceSize,
28
- const int bmi2,
29
- unsigned suspectUncompressible);
33
+ const ZSTD_hufCTables_t* prevHuf,
34
+ ZSTD_hufCTables_t* nextHuf,
35
+ ZSTD_strategy strategy, int disableLiteralCompression,
36
+ int suspectUncompressible,
37
+ int bmi2);
30
38
 
31
39
  #endif /* ZSTD_COMPRESS_LITERALS_H */
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -58,7 +58,7 @@ static unsigned ZSTD_useLowProbCount(size_t const nbSeq)
58
58
  {
59
59
  /* Heuristic: This should cover most blocks <= 16K and
60
60
  * start to fade out after 16K to about 32K depending on
61
- * comprssibility.
61
+ * compressibility.
62
62
  */
63
63
  return nbSeq >= 2048;
64
64
  }
@@ -166,7 +166,7 @@ ZSTD_selectEncodingType(
166
166
  if (mostFrequent == nbSeq) {
167
167
  *repeatMode = FSE_repeat_none;
168
168
  if (isDefaultAllowed && nbSeq <= 2) {
169
- /* Prefer set_basic over set_rle when there are 2 or less symbols,
169
+ /* Prefer set_basic over set_rle when there are 2 or fewer symbols,
170
170
  * since RLE uses 1 byte, but set_basic uses 5-6 bits per symbol.
171
171
  * If basic encoding isn't possible, always choose RLE.
172
172
  */
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -36,13 +36,14 @@
36
36
  * If it is set_compressed, first sub-block's literals section will be Treeless_Literals_Block
37
37
  * and the following sub-blocks' literals sections will be Treeless_Literals_Block.
38
38
  * @return : compressed size of literals section of a sub-block
39
- * Or 0 if it unable to compress.
39
+ * Or 0 if unable to compress.
40
40
  * Or error code */
41
- static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
42
- const ZSTD_hufCTablesMetadata_t* hufMetadata,
43
- const BYTE* literals, size_t litSize,
44
- void* dst, size_t dstSize,
45
- const int bmi2, int writeEntropy, int* entropyWritten)
41
+ static size_t
42
+ ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
43
+ const ZSTD_hufCTablesMetadata_t* hufMetadata,
44
+ const BYTE* literals, size_t litSize,
45
+ void* dst, size_t dstSize,
46
+ const int bmi2, int writeEntropy, int* entropyWritten)
46
47
  {
47
48
  size_t const header = writeEntropy ? 200 : 0;
48
49
  size_t const lhSize = 3 + (litSize >= (1 KB - header)) + (litSize >= (16 KB - header));
@@ -53,8 +54,6 @@ static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
53
54
  symbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat;
54
55
  size_t cLitSize = 0;
55
56
 
56
- (void)bmi2; /* TODO bmi2... */
57
-
58
57
  DEBUGLOG(5, "ZSTD_compressSubBlock_literal (litSize=%zu, lhSize=%zu, writeEntropy=%d)", litSize, lhSize, writeEntropy);
59
58
 
60
59
  *entropyWritten = 0;
@@ -76,9 +75,9 @@ static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
76
75
  DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize);
77
76
  }
78
77
 
79
- /* TODO bmi2 */
80
- { const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, oend-op, literals, litSize, hufTable)
81
- : HUF_compress4X_usingCTable(op, oend-op, literals, litSize, hufTable);
78
+ { int const flags = bmi2 ? HUF_flags_bmi2 : 0;
79
+ const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, oend-op, literals, litSize, hufTable, flags)
80
+ : HUF_compress4X_usingCTable(op, oend-op, literals, litSize, hufTable, flags);
82
81
  op += cSize;
83
82
  cLitSize += cSize;
84
83
  if (cSize == 0 || ERR_isError(cSize)) {
@@ -126,7 +125,11 @@ static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
126
125
  return op-ostart;
127
126
  }
128
127
 
129
- static size_t ZSTD_seqDecompressedSize(seqStore_t const* seqStore, const seqDef* sequences, size_t nbSeq, size_t litSize, int lastSequence) {
128
+ static size_t
129
+ ZSTD_seqDecompressedSize(seqStore_t const* seqStore,
130
+ const seqDef* sequences, size_t nbSeq,
131
+ size_t litSize, int lastSequence)
132
+ {
130
133
  const seqDef* const sstart = sequences;
131
134
  const seqDef* const send = sequences + nbSeq;
132
135
  const seqDef* sp = sstart;
@@ -156,13 +159,14 @@ static size_t ZSTD_seqDecompressedSize(seqStore_t const* seqStore, const seqDef*
156
159
  * @return : compressed size of sequences section of a sub-block
157
160
  * Or 0 if it is unable to compress
158
161
  * Or error code. */
159
- static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables,
160
- const ZSTD_fseCTablesMetadata_t* fseMetadata,
161
- const seqDef* sequences, size_t nbSeq,
162
- const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode,
163
- const ZSTD_CCtx_params* cctxParams,
164
- void* dst, size_t dstCapacity,
165
- const int bmi2, int writeEntropy, int* entropyWritten)
162
+ static size_t
163
+ ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables,
164
+ const ZSTD_fseCTablesMetadata_t* fseMetadata,
165
+ const seqDef* sequences, size_t nbSeq,
166
+ const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode,
167
+ const ZSTD_CCtx_params* cctxParams,
168
+ void* dst, size_t dstCapacity,
169
+ const int bmi2, int writeEntropy, int* entropyWritten)
166
170
  {
167
171
  const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN;
168
172
  BYTE* const ostart = (BYTE*)dst;
@@ -539,7 +543,7 @@ static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr,
539
543
  repcodes_t rep;
540
544
  ZSTD_memcpy(&rep, prevCBlock->rep, sizeof(rep));
541
545
  for (seq = sstart; seq < sp; ++seq) {
542
- ZSTD_updateRep(rep.rep, seq->offBase - 1, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0);
546
+ ZSTD_updateRep(rep.rep, seq->offBase, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0);
543
547
  }
544
548
  ZSTD_memcpy(nextCBlock->rep, &rep, sizeof(rep));
545
549
  }
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the