zstd-ruby 1.5.2.3 → 1.5.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +13 -5
- data/ext/zstdruby/extconf.rb +1 -1
- data/ext/zstdruby/libzstd/common/allocations.h +55 -0
- data/ext/zstdruby/libzstd/common/bits.h +200 -0
- data/ext/zstdruby/libzstd/common/bitstream.h +19 -60
- data/ext/zstdruby/libzstd/common/compiler.h +26 -3
- data/ext/zstdruby/libzstd/common/cpu.h +1 -1
- data/ext/zstdruby/libzstd/common/debug.c +1 -1
- data/ext/zstdruby/libzstd/common/debug.h +1 -1
- data/ext/zstdruby/libzstd/common/entropy_common.c +12 -40
- data/ext/zstdruby/libzstd/common/error_private.c +9 -2
- data/ext/zstdruby/libzstd/common/error_private.h +1 -1
- data/ext/zstdruby/libzstd/common/fse.h +5 -83
- data/ext/zstdruby/libzstd/common/fse_decompress.c +7 -99
- data/ext/zstdruby/libzstd/common/huf.h +65 -156
- data/ext/zstdruby/libzstd/common/mem.h +39 -46
- data/ext/zstdruby/libzstd/common/pool.c +26 -10
- data/ext/zstdruby/libzstd/common/pool.h +7 -1
- data/ext/zstdruby/libzstd/common/portability_macros.h +22 -3
- data/ext/zstdruby/libzstd/common/threading.c +68 -14
- data/ext/zstdruby/libzstd/common/threading.h +5 -10
- data/ext/zstdruby/libzstd/common/xxhash.c +2 -2
- data/ext/zstdruby/libzstd/common/xxhash.h +8 -8
- data/ext/zstdruby/libzstd/common/zstd_common.c +1 -36
- data/ext/zstdruby/libzstd/common/zstd_deps.h +1 -1
- data/ext/zstdruby/libzstd/common/zstd_internal.h +17 -118
- data/ext/zstdruby/libzstd/common/zstd_trace.h +3 -3
- data/ext/zstdruby/libzstd/compress/clevels.h +1 -1
- data/ext/zstdruby/libzstd/compress/fse_compress.c +7 -124
- data/ext/zstdruby/libzstd/compress/hist.c +1 -1
- data/ext/zstdruby/libzstd/compress/hist.h +1 -1
- data/ext/zstdruby/libzstd/compress/huf_compress.c +234 -169
- data/ext/zstdruby/libzstd/compress/zstd_compress.c +1243 -538
- data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +225 -151
- data/ext/zstdruby/libzstd/compress/zstd_compress_literals.c +115 -39
- data/ext/zstdruby/libzstd/compress/zstd_compress_literals.h +16 -8
- data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.c +3 -3
- data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.h +1 -1
- data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.c +25 -21
- data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.h +1 -1
- data/ext/zstdruby/libzstd/compress/zstd_cwksp.h +128 -62
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +95 -33
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +3 -2
- data/ext/zstdruby/libzstd/compress/zstd_fast.c +433 -148
- data/ext/zstdruby/libzstd/compress/zstd_fast.h +3 -2
- data/ext/zstdruby/libzstd/compress/zstd_lazy.c +398 -345
- data/ext/zstdruby/libzstd/compress/zstd_lazy.h +4 -2
- data/ext/zstdruby/libzstd/compress/zstd_ldm.c +5 -5
- data/ext/zstdruby/libzstd/compress/zstd_ldm.h +1 -1
- data/ext/zstdruby/libzstd/compress/zstd_ldm_geartab.h +1 -1
- data/ext/zstdruby/libzstd/compress/zstd_opt.c +106 -80
- data/ext/zstdruby/libzstd/compress/zstd_opt.h +1 -1
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +17 -9
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +1 -1
- data/ext/zstdruby/libzstd/decompress/huf_decompress.c +434 -441
- data/ext/zstdruby/libzstd/decompress/huf_decompress_amd64.S +30 -39
- data/ext/zstdruby/libzstd/decompress/zstd_ddict.c +4 -4
- data/ext/zstdruby/libzstd/decompress/zstd_ddict.h +1 -1
- data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +205 -80
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.c +201 -81
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.h +6 -1
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_internal.h +4 -2
- data/ext/zstdruby/libzstd/dictBuilder/cover.c +19 -15
- data/ext/zstdruby/libzstd/dictBuilder/cover.h +1 -1
- data/ext/zstdruby/libzstd/dictBuilder/fastcover.c +2 -2
- data/ext/zstdruby/libzstd/dictBuilder/zdict.c +11 -89
- data/ext/zstdruby/libzstd/zdict.h +53 -31
- data/ext/zstdruby/libzstd/zstd.h +580 -135
- data/ext/zstdruby/libzstd/zstd_errors.h +27 -8
- data/ext/zstdruby/main.c +6 -0
- data/ext/zstdruby/skippable_frame.c +63 -0
- data/lib/zstd-ruby/version.rb +1 -1
- metadata +9 -6
@@ -1,6 +1,6 @@
|
|
1
1
|
/* ******************************************************************
|
2
2
|
* Common functions of New Generation Entropy library
|
3
|
-
* Copyright (c)
|
3
|
+
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
4
4
|
*
|
5
5
|
* You can contact the author at :
|
6
6
|
* - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
@@ -19,8 +19,8 @@
|
|
19
19
|
#include "error_private.h" /* ERR_*, ERROR */
|
20
20
|
#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */
|
21
21
|
#include "fse.h"
|
22
|
-
#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */
|
23
22
|
#include "huf.h"
|
23
|
+
#include "bits.h" /* ZSDT_highbit32, ZSTD_countTrailingZeros32 */
|
24
24
|
|
25
25
|
|
26
26
|
/*=== Version ===*/
|
@@ -38,34 +38,6 @@ const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
|
38
38
|
/*-**************************************************************
|
39
39
|
* FSE NCount encoding-decoding
|
40
40
|
****************************************************************/
|
41
|
-
static U32 FSE_ctz(U32 val)
|
42
|
-
{
|
43
|
-
assert(val != 0);
|
44
|
-
{
|
45
|
-
# if defined(_MSC_VER) /* Visual */
|
46
|
-
if (val != 0) {
|
47
|
-
unsigned long r;
|
48
|
-
_BitScanForward(&r, val);
|
49
|
-
return (unsigned)r;
|
50
|
-
} else {
|
51
|
-
/* Should not reach this code path */
|
52
|
-
__assume(0);
|
53
|
-
}
|
54
|
-
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
|
55
|
-
return __builtin_ctz(val);
|
56
|
-
# elif defined(__ICCARM__) /* IAR Intrinsic */
|
57
|
-
return __CTZ(val);
|
58
|
-
# else /* Software version */
|
59
|
-
U32 count = 0;
|
60
|
-
while ((val & 1) == 0) {
|
61
|
-
val >>= 1;
|
62
|
-
++count;
|
63
|
-
}
|
64
|
-
return count;
|
65
|
-
# endif
|
66
|
-
}
|
67
|
-
}
|
68
|
-
|
69
41
|
FORCE_INLINE_TEMPLATE
|
70
42
|
size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
|
71
43
|
const void* headerBuffer, size_t hbSize)
|
@@ -113,7 +85,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne
|
|
113
85
|
* repeat.
|
114
86
|
* Avoid UB by setting the high bit to 1.
|
115
87
|
*/
|
116
|
-
int repeats =
|
88
|
+
int repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1;
|
117
89
|
while (repeats >= 12) {
|
118
90
|
charnum += 3 * 12;
|
119
91
|
if (LIKELY(ip <= iend-7)) {
|
@@ -124,7 +96,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne
|
|
124
96
|
ip = iend - 4;
|
125
97
|
}
|
126
98
|
bitStream = MEM_readLE32(ip) >> bitCount;
|
127
|
-
repeats =
|
99
|
+
repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1;
|
128
100
|
}
|
129
101
|
charnum += 3 * repeats;
|
130
102
|
bitStream >>= 2 * repeats;
|
@@ -189,7 +161,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne
|
|
189
161
|
* know that threshold > 1.
|
190
162
|
*/
|
191
163
|
if (remaining <= 1) break;
|
192
|
-
nbBits =
|
164
|
+
nbBits = ZSTD_highbit32(remaining) + 1;
|
193
165
|
threshold = 1 << (nbBits - 1);
|
194
166
|
}
|
195
167
|
if (charnum >= maxSV1) break;
|
@@ -264,7 +236,7 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
|
264
236
|
const void* src, size_t srcSize)
|
265
237
|
{
|
266
238
|
U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
|
267
|
-
return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /*
|
239
|
+
return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* flags */ 0);
|
268
240
|
}
|
269
241
|
|
270
242
|
FORCE_INLINE_TEMPLATE size_t
|
@@ -312,14 +284,14 @@ HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
|
312
284
|
if (weightTotal == 0) return ERROR(corruption_detected);
|
313
285
|
|
314
286
|
/* get last non-null symbol weight (implied, total must be 2^n) */
|
315
|
-
{ U32 const tableLog =
|
287
|
+
{ U32 const tableLog = ZSTD_highbit32(weightTotal) + 1;
|
316
288
|
if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected);
|
317
289
|
*tableLogPtr = tableLog;
|
318
290
|
/* determine last weight */
|
319
291
|
{ U32 const total = 1 << tableLog;
|
320
292
|
U32 const rest = total - weightTotal;
|
321
|
-
U32 const verif = 1 <<
|
322
|
-
U32 const lastWeight =
|
293
|
+
U32 const verif = 1 << ZSTD_highbit32(rest);
|
294
|
+
U32 const lastWeight = ZSTD_highbit32(rest) + 1;
|
323
295
|
if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */
|
324
296
|
huffWeight[oSize] = (BYTE)lastWeight;
|
325
297
|
rankStats[lastWeight]++;
|
@@ -356,13 +328,13 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
|
356
328
|
U32* nbSymbolsPtr, U32* tableLogPtr,
|
357
329
|
const void* src, size_t srcSize,
|
358
330
|
void* workSpace, size_t wkspSize,
|
359
|
-
int
|
331
|
+
int flags)
|
360
332
|
{
|
361
333
|
#if DYNAMIC_BMI2
|
362
|
-
if (
|
334
|
+
if (flags & HUF_flags_bmi2) {
|
363
335
|
return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
|
364
336
|
}
|
365
337
|
#endif
|
366
|
-
(void)
|
338
|
+
(void)flags;
|
367
339
|
return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
|
368
340
|
}
|
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c)
|
2
|
+
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -27,9 +27,11 @@ const char* ERR_getErrorString(ERR_enum code)
|
|
27
27
|
case PREFIX(version_unsupported): return "Version not supported";
|
28
28
|
case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
|
29
29
|
case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding";
|
30
|
-
case PREFIX(corruption_detected): return "
|
30
|
+
case PREFIX(corruption_detected): return "Data corruption detected";
|
31
31
|
case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
|
32
|
+
case PREFIX(literals_headerWrong): return "Header of Literals' block doesn't respect format specification";
|
32
33
|
case PREFIX(parameter_unsupported): return "Unsupported parameter";
|
34
|
+
case PREFIX(parameter_combination_unsupported): return "Unsupported combination of parameters";
|
33
35
|
case PREFIX(parameter_outOfBound): return "Parameter is out of bound";
|
34
36
|
case PREFIX(init_missing): return "Context should be init first";
|
35
37
|
case PREFIX(memory_allocation): return "Allocation error : not enough memory";
|
@@ -38,17 +40,22 @@ const char* ERR_getErrorString(ERR_enum code)
|
|
38
40
|
case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
|
39
41
|
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
|
40
42
|
case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
|
43
|
+
case PREFIX(stabilityCondition_notRespected): return "pledged buffer stability condition is not respected";
|
41
44
|
case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
|
42
45
|
case PREFIX(dictionary_wrong): return "Dictionary mismatch";
|
43
46
|
case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples";
|
44
47
|
case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
|
45
48
|
case PREFIX(srcSize_wrong): return "Src size is incorrect";
|
46
49
|
case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer";
|
50
|
+
case PREFIX(noForwardProgress_destFull): return "Operation made no progress over multiple calls, due to output buffer being full";
|
51
|
+
case PREFIX(noForwardProgress_inputEmpty): return "Operation made no progress over multiple calls, due to input being empty";
|
47
52
|
/* following error codes are not stable and may be removed or changed in a future version */
|
48
53
|
case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
|
49
54
|
case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
|
50
55
|
case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong";
|
51
56
|
case PREFIX(srcBuffer_wrong): return "Source buffer is wrong";
|
57
|
+
case PREFIX(sequenceProducer_failed): return "Block-level external sequence producer returned an error code";
|
58
|
+
case PREFIX(externalSequences_invalid): return "External sequences are not valid";
|
52
59
|
case PREFIX(maxCode):
|
53
60
|
default: return notErrorCode;
|
54
61
|
}
|
@@ -1,7 +1,7 @@
|
|
1
1
|
/* ******************************************************************
|
2
2
|
* FSE : Finite State Entropy codec
|
3
3
|
* Public Prototypes declaration
|
4
|
-
* Copyright (c)
|
4
|
+
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
5
5
|
*
|
6
6
|
* You can contact the author at :
|
7
7
|
* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
@@ -53,34 +53,6 @@ extern "C" {
|
|
53
53
|
FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */
|
54
54
|
|
55
55
|
|
56
|
-
/*-****************************************
|
57
|
-
* FSE simple functions
|
58
|
-
******************************************/
|
59
|
-
/*! FSE_compress() :
|
60
|
-
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
|
61
|
-
'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize).
|
62
|
-
@return : size of compressed data (<= dstCapacity).
|
63
|
-
Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
|
64
|
-
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead.
|
65
|
-
if FSE_isError(return), compression failed (more details using FSE_getErrorName())
|
66
|
-
*/
|
67
|
-
FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity,
|
68
|
-
const void* src, size_t srcSize);
|
69
|
-
|
70
|
-
/*! FSE_decompress():
|
71
|
-
Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',
|
72
|
-
into already allocated destination buffer 'dst', of size 'dstCapacity'.
|
73
|
-
@return : size of regenerated data (<= maxDstSize),
|
74
|
-
or an error code, which can be tested using FSE_isError() .
|
75
|
-
|
76
|
-
** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!!
|
77
|
-
Why ? : making this distinction requires a header.
|
78
|
-
Header management is intentionally delegated to the user layer, which can better manage special cases.
|
79
|
-
*/
|
80
|
-
FSE_PUBLIC_API size_t FSE_decompress(void* dst, size_t dstCapacity,
|
81
|
-
const void* cSrc, size_t cSrcSize);
|
82
|
-
|
83
|
-
|
84
56
|
/*-*****************************************
|
85
57
|
* Tool functions
|
86
58
|
******************************************/
|
@@ -91,20 +63,6 @@ FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return
|
|
91
63
|
FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */
|
92
64
|
|
93
65
|
|
94
|
-
/*-*****************************************
|
95
|
-
* FSE advanced functions
|
96
|
-
******************************************/
|
97
|
-
/*! FSE_compress2() :
|
98
|
-
Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog'
|
99
|
-
Both parameters can be defined as '0' to mean : use default value
|
100
|
-
@return : size of compressed data
|
101
|
-
Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!!
|
102
|
-
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
|
103
|
-
if FSE_isError(return), it's an error code.
|
104
|
-
*/
|
105
|
-
FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
106
|
-
|
107
|
-
|
108
66
|
/*-*****************************************
|
109
67
|
* FSE detailed API
|
110
68
|
******************************************/
|
@@ -164,8 +122,6 @@ FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize,
|
|
164
122
|
/*! Constructor and Destructor of FSE_CTable.
|
165
123
|
Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
|
166
124
|
typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */
|
167
|
-
FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog);
|
168
|
-
FSE_PUBLIC_API void FSE_freeCTable (FSE_CTable* ct);
|
169
125
|
|
170
126
|
/*! FSE_buildCTable():
|
171
127
|
Builds `ct`, which must be already allocated, using FSE_createCTable().
|
@@ -241,23 +197,7 @@ FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter,
|
|
241
197
|
unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
|
242
198
|
const void* rBuffer, size_t rBuffSize, int bmi2);
|
243
199
|
|
244
|
-
/*! Constructor and Destructor of FSE_DTable.
|
245
|
-
Note that its size depends on 'tableLog' */
|
246
200
|
typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
|
247
|
-
FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog);
|
248
|
-
FSE_PUBLIC_API void FSE_freeDTable(FSE_DTable* dt);
|
249
|
-
|
250
|
-
/*! FSE_buildDTable():
|
251
|
-
Builds 'dt', which must be already allocated, using FSE_createDTable().
|
252
|
-
return : 0, or an errorCode, which can be tested using FSE_isError() */
|
253
|
-
FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
|
254
|
-
|
255
|
-
/*! FSE_decompress_usingDTable():
|
256
|
-
Decompress compressed source `cSrc` of size `cSrcSize` using `dt`
|
257
|
-
into `dst` which must be already allocated.
|
258
|
-
@return : size of regenerated data (necessarily <= `dstCapacity`),
|
259
|
-
or an errorCode, which can be tested using FSE_isError() */
|
260
|
-
FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
|
261
201
|
|
262
202
|
/*!
|
263
203
|
Tutorial :
|
@@ -320,16 +260,6 @@ If there is an error, the function will return an error code, which can be teste
|
|
320
260
|
unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
|
321
261
|
/**< same as FSE_optimalTableLog(), which used `minus==2` */
|
322
262
|
|
323
|
-
/* FSE_compress_wksp() :
|
324
|
-
* Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
|
325
|
-
* FSE_COMPRESS_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable.
|
326
|
-
*/
|
327
|
-
#define FSE_COMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) )
|
328
|
-
size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
|
329
|
-
|
330
|
-
size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
|
331
|
-
/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */
|
332
|
-
|
333
263
|
size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
|
334
264
|
/**< build a fake FSE_CTable, designed to compress always the same symbolValue */
|
335
265
|
|
@@ -347,19 +277,11 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsi
|
|
347
277
|
FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
|
348
278
|
/**< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */
|
349
279
|
|
350
|
-
|
351
|
-
/**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */
|
352
|
-
|
353
|
-
size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
|
354
|
-
/**< build a fake FSE_DTable, designed to always generate the same symbolValue */
|
355
|
-
|
356
|
-
#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1)
|
280
|
+
#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + 1 + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1)
|
357
281
|
#define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned))
|
358
|
-
size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize);
|
359
|
-
/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)` */
|
360
|
-
|
361
282
|
size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2);
|
362
|
-
/**<
|
283
|
+
/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)`.
|
284
|
+
* Set bmi2 to 1 if your CPU supports BMI2 or 0 if it doesn't */
|
363
285
|
|
364
286
|
typedef enum {
|
365
287
|
FSE_repeat_none, /**< Cannot use the previous table */
|
@@ -555,7 +477,7 @@ MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePt
|
|
555
477
|
|
556
478
|
/* FSE_getMaxNbBits() :
|
557
479
|
* Approximate maximum cost of a symbol, in bits.
|
558
|
-
* Fractional get rounded up (i.e
|
480
|
+
* Fractional get rounded up (i.e. a symbol with a normalized frequency of 3 gives the same result as a frequency of 2)
|
559
481
|
* note 1 : assume symbolValue is valid (<= maxSymbolValue)
|
560
482
|
* note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
|
561
483
|
MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
/* ******************************************************************
|
2
2
|
* FSE : Finite State Entropy decoder
|
3
|
-
* Copyright (c)
|
3
|
+
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
4
4
|
*
|
5
5
|
* You can contact the author at :
|
6
6
|
* - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
@@ -24,6 +24,7 @@
|
|
24
24
|
#include "error_private.h"
|
25
25
|
#define ZSTD_DEPS_NEED_MALLOC
|
26
26
|
#include "zstd_deps.h"
|
27
|
+
#include "bits.h" /* ZSTD_highbit32 */
|
27
28
|
|
28
29
|
|
29
30
|
/* **************************************************************
|
@@ -55,19 +56,6 @@
|
|
55
56
|
#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)
|
56
57
|
#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)
|
57
58
|
|
58
|
-
|
59
|
-
/* Function templates */
|
60
|
-
FSE_DTable* FSE_createDTable (unsigned tableLog)
|
61
|
-
{
|
62
|
-
if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
|
63
|
-
return (FSE_DTable*)ZSTD_malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );
|
64
|
-
}
|
65
|
-
|
66
|
-
void FSE_freeDTable (FSE_DTable* dt)
|
67
|
-
{
|
68
|
-
ZSTD_free(dt);
|
69
|
-
}
|
70
|
-
|
71
59
|
static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
|
72
60
|
{
|
73
61
|
void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */
|
@@ -127,10 +115,10 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo
|
|
127
115
|
}
|
128
116
|
}
|
129
117
|
/* Now we spread those positions across the table.
|
130
|
-
* The benefit of doing it in two stages is that we avoid the
|
118
|
+
* The benefit of doing it in two stages is that we avoid the
|
131
119
|
* variable size inner loop, which caused lots of branch misses.
|
132
120
|
* Now we can run through all the positions without any branch misses.
|
133
|
-
* We unroll the loop twice, since that is what
|
121
|
+
* We unroll the loop twice, since that is what empirically worked best.
|
134
122
|
*/
|
135
123
|
{
|
136
124
|
size_t position = 0;
|
@@ -166,7 +154,7 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo
|
|
166
154
|
for (u=0; u<tableSize; u++) {
|
167
155
|
FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
|
168
156
|
U32 const nextState = symbolNext[symbol]++;
|
169
|
-
tableDecode[u].nbBits = (BYTE) (tableLog -
|
157
|
+
tableDecode[u].nbBits = (BYTE) (tableLog - ZSTD_highbit32(nextState) );
|
170
158
|
tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
|
171
159
|
} }
|
172
160
|
|
@@ -184,49 +172,6 @@ size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsi
|
|
184
172
|
/*-*******************************************************
|
185
173
|
* Decompression (Byte symbols)
|
186
174
|
*********************************************************/
|
187
|
-
size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
|
188
|
-
{
|
189
|
-
void* ptr = dt;
|
190
|
-
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
|
191
|
-
void* dPtr = dt + 1;
|
192
|
-
FSE_decode_t* const cell = (FSE_decode_t*)dPtr;
|
193
|
-
|
194
|
-
DTableH->tableLog = 0;
|
195
|
-
DTableH->fastMode = 0;
|
196
|
-
|
197
|
-
cell->newState = 0;
|
198
|
-
cell->symbol = symbolValue;
|
199
|
-
cell->nbBits = 0;
|
200
|
-
|
201
|
-
return 0;
|
202
|
-
}
|
203
|
-
|
204
|
-
|
205
|
-
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
|
206
|
-
{
|
207
|
-
void* ptr = dt;
|
208
|
-
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
|
209
|
-
void* dPtr = dt + 1;
|
210
|
-
FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr;
|
211
|
-
const unsigned tableSize = 1 << nbBits;
|
212
|
-
const unsigned tableMask = tableSize - 1;
|
213
|
-
const unsigned maxSV1 = tableMask+1;
|
214
|
-
unsigned s;
|
215
|
-
|
216
|
-
/* Sanity checks */
|
217
|
-
if (nbBits < 1) return ERROR(GENERIC); /* min size */
|
218
|
-
|
219
|
-
/* Build Decoding Table */
|
220
|
-
DTableH->tableLog = (U16)nbBits;
|
221
|
-
DTableH->fastMode = 1;
|
222
|
-
for (s=0; s<maxSV1; s++) {
|
223
|
-
dinfo[s].newState = 0;
|
224
|
-
dinfo[s].symbol = (BYTE)s;
|
225
|
-
dinfo[s].nbBits = (BYTE)nbBits;
|
226
|
-
}
|
227
|
-
|
228
|
-
return 0;
|
229
|
-
}
|
230
175
|
|
231
176
|
FORCE_INLINE_TEMPLATE size_t FSE_decompress_usingDTable_generic(
|
232
177
|
void* dst, size_t maxDstSize,
|
@@ -290,26 +235,6 @@ FORCE_INLINE_TEMPLATE size_t FSE_decompress_usingDTable_generic(
|
|
290
235
|
return op-ostart;
|
291
236
|
}
|
292
237
|
|
293
|
-
|
294
|
-
size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
|
295
|
-
const void* cSrc, size_t cSrcSize,
|
296
|
-
const FSE_DTable* dt)
|
297
|
-
{
|
298
|
-
const void* ptr = dt;
|
299
|
-
const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
|
300
|
-
const U32 fastMode = DTableH->fastMode;
|
301
|
-
|
302
|
-
/* select fast mode (static) */
|
303
|
-
if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
|
304
|
-
return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
|
305
|
-
}
|
306
|
-
|
307
|
-
|
308
|
-
size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize)
|
309
|
-
{
|
310
|
-
return FSE_decompress_wksp_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, /* bmi2 */ 0);
|
311
|
-
}
|
312
|
-
|
313
238
|
typedef struct {
|
314
239
|
short ncount[FSE_MAX_SYMBOL_VALUE + 1];
|
315
240
|
FSE_DTable dtable[1]; /* Dynamically sized */
|
@@ -342,7 +267,8 @@ FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body(
|
|
342
267
|
}
|
343
268
|
|
344
269
|
if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge);
|
345
|
-
|
270
|
+
assert(sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog) <= wkspSize);
|
271
|
+
workSpace = (BYTE*)workSpace + sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog);
|
346
272
|
wkspSize -= sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog);
|
347
273
|
|
348
274
|
CHECK_F( FSE_buildDTable_internal(wksp->dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) );
|
@@ -382,22 +308,4 @@ size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc,
|
|
382
308
|
return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize);
|
383
309
|
}
|
384
310
|
|
385
|
-
|
386
|
-
typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
|
387
|
-
|
388
|
-
#ifndef ZSTD_NO_UNUSED_FUNCTIONS
|
389
|
-
size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) {
|
390
|
-
U32 wksp[FSE_BUILD_DTABLE_WKSP_SIZE_U32(FSE_TABLELOG_ABSOLUTE_MAX, FSE_MAX_SYMBOL_VALUE)];
|
391
|
-
return FSE_buildDTable_wksp(dt, normalizedCounter, maxSymbolValue, tableLog, wksp, sizeof(wksp));
|
392
|
-
}
|
393
|
-
|
394
|
-
size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize)
|
395
|
-
{
|
396
|
-
/* Static analyzer seems unable to understand this table will be properly initialized later */
|
397
|
-
U32 wksp[FSE_DECOMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
|
398
|
-
return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, FSE_MAX_TABLELOG, wksp, sizeof(wksp));
|
399
|
-
}
|
400
|
-
#endif
|
401
|
-
|
402
|
-
|
403
311
|
#endif /* FSE_COMMONDEFS_ONLY */
|