extzstd 0.2 → 0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/HISTORY.ja.md +13 -0
- data/README.md +17 -14
- data/contrib/zstd/{NEWS → CHANGELOG} +115 -2
- data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
- data/contrib/zstd/Makefile +99 -53
- data/contrib/zstd/README.md +59 -39
- data/contrib/zstd/TESTING.md +1 -1
- data/contrib/zstd/appveyor.yml +17 -6
- data/contrib/zstd/lib/BUCK +29 -2
- data/contrib/zstd/lib/Makefile +118 -21
- data/contrib/zstd/lib/README.md +84 -44
- data/contrib/zstd/lib/common/bitstream.h +17 -33
- data/contrib/zstd/lib/common/compiler.h +62 -8
- data/contrib/zstd/lib/common/cpu.h +215 -0
- data/contrib/zstd/lib/common/debug.c +44 -0
- data/contrib/zstd/lib/common/debug.h +134 -0
- data/contrib/zstd/lib/common/entropy_common.c +16 -1
- data/contrib/zstd/lib/common/error_private.c +7 -0
- data/contrib/zstd/lib/common/fse.h +48 -44
- data/contrib/zstd/lib/common/fse_decompress.c +3 -3
- data/contrib/zstd/lib/common/huf.h +169 -113
- data/contrib/zstd/lib/common/mem.h +20 -2
- data/contrib/zstd/lib/common/pool.c +135 -49
- data/contrib/zstd/lib/common/pool.h +40 -21
- data/contrib/zstd/lib/common/threading.c +2 -2
- data/contrib/zstd/lib/common/threading.h +12 -12
- data/contrib/zstd/lib/common/xxhash.c +3 -2
- data/contrib/zstd/lib/common/zstd_common.c +3 -6
- data/contrib/zstd/lib/common/zstd_errors.h +17 -7
- data/contrib/zstd/lib/common/zstd_internal.h +76 -48
- data/contrib/zstd/lib/compress/fse_compress.c +89 -209
- data/contrib/zstd/lib/compress/hist.c +203 -0
- data/contrib/zstd/lib/compress/hist.h +95 -0
- data/contrib/zstd/lib/compress/huf_compress.c +188 -80
- data/contrib/zstd/lib/compress/zstd_compress.c +2500 -1203
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +463 -62
- data/contrib/zstd/lib/compress/zstd_double_fast.c +321 -131
- data/contrib/zstd/lib/compress/zstd_double_fast.h +13 -4
- data/contrib/zstd/lib/compress/zstd_fast.c +335 -108
- data/contrib/zstd/lib/compress/zstd_fast.h +12 -6
- data/contrib/zstd/lib/compress/zstd_lazy.c +654 -313
- data/contrib/zstd/lib/compress/zstd_lazy.h +44 -16
- data/contrib/zstd/lib/compress/zstd_ldm.c +310 -420
- data/contrib/zstd/lib/compress/zstd_ldm.h +63 -26
- data/contrib/zstd/lib/compress/zstd_opt.c +773 -325
- data/contrib/zstd/lib/compress/zstd_opt.h +31 -5
- data/contrib/zstd/lib/compress/zstdmt_compress.c +1468 -518
- data/contrib/zstd/lib/compress/zstdmt_compress.h +96 -45
- data/contrib/zstd/lib/decompress/huf_decompress.c +518 -282
- data/contrib/zstd/lib/decompress/zstd_ddict.c +240 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
- data/contrib/zstd/lib/decompress/zstd_decompress.c +613 -1513
- data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1311 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.h +59 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +175 -0
- data/contrib/zstd/lib/dictBuilder/cover.c +194 -113
- data/contrib/zstd/lib/dictBuilder/cover.h +112 -0
- data/contrib/zstd/lib/dictBuilder/divsufsort.c +3 -3
- data/contrib/zstd/lib/dictBuilder/fastcover.c +740 -0
- data/contrib/zstd/lib/dictBuilder/zdict.c +142 -106
- data/contrib/zstd/lib/dictBuilder/zdict.h +115 -49
- data/contrib/zstd/lib/legacy/zstd_legacy.h +44 -12
- data/contrib/zstd/lib/legacy/zstd_v01.c +41 -10
- data/contrib/zstd/lib/legacy/zstd_v01.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v02.c +37 -12
- data/contrib/zstd/lib/legacy/zstd_v02.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v03.c +38 -12
- data/contrib/zstd/lib/legacy/zstd_v03.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v04.c +55 -174
- data/contrib/zstd/lib/legacy/zstd_v04.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v05.c +59 -31
- data/contrib/zstd/lib/legacy/zstd_v05.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v06.c +48 -20
- data/contrib/zstd/lib/legacy/zstd_v06.h +10 -5
- data/contrib/zstd/lib/legacy/zstd_v07.c +62 -29
- data/contrib/zstd/lib/legacy/zstd_v07.h +10 -5
- data/contrib/zstd/lib/zstd.h +1346 -832
- data/ext/extzstd.c +27 -19
- data/ext/extzstd_stream.c +20 -4
- data/ext/zstd_compress.c +1 -0
- data/ext/zstd_decompress.c +4 -0
- data/ext/zstd_dictbuilder.c +4 -0
- data/ext/zstd_dictbuilder_fastcover.c +5 -0
- data/lib/extzstd.rb +52 -220
- data/lib/extzstd/version.rb +1 -1
- metadata +21 -7
- data/contrib/zstd/circle.yml +0 -63
@@ -0,0 +1,44 @@
|
|
1
|
+
/* ******************************************************************
|
2
|
+
debug
|
3
|
+
Part of FSE library
|
4
|
+
Copyright (C) 2013-present, Yann Collet.
|
5
|
+
|
6
|
+
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
7
|
+
|
8
|
+
Redistribution and use in source and binary forms, with or without
|
9
|
+
modification, are permitted provided that the following conditions are
|
10
|
+
met:
|
11
|
+
|
12
|
+
* Redistributions of source code must retain the above copyright
|
13
|
+
notice, this list of conditions and the following disclaimer.
|
14
|
+
* Redistributions in binary form must reproduce the above
|
15
|
+
copyright notice, this list of conditions and the following disclaimer
|
16
|
+
in the documentation and/or other materials provided with the
|
17
|
+
distribution.
|
18
|
+
|
19
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
20
|
+
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
21
|
+
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
22
|
+
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
23
|
+
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
24
|
+
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
25
|
+
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
26
|
+
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
27
|
+
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
28
|
+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
29
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
30
|
+
|
31
|
+
You can contact the author at :
|
32
|
+
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
33
|
+
****************************************************************** */
|
34
|
+
|
35
|
+
|
36
|
+
/*
|
37
|
+
* This module only hosts one global variable
|
38
|
+
* which can be used to dynamically influence the verbosity of traces,
|
39
|
+
* such as DEBUGLOG and RAWLOG
|
40
|
+
*/
|
41
|
+
|
42
|
+
#include "debug.h"
|
43
|
+
|
44
|
+
int g_debuglevel = DEBUGLEVEL;
|
@@ -0,0 +1,134 @@
|
|
1
|
+
/* ******************************************************************
|
2
|
+
debug
|
3
|
+
Part of FSE library
|
4
|
+
Copyright (C) 2013-present, Yann Collet.
|
5
|
+
|
6
|
+
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
7
|
+
|
8
|
+
Redistribution and use in source and binary forms, with or without
|
9
|
+
modification, are permitted provided that the following conditions are
|
10
|
+
met:
|
11
|
+
|
12
|
+
* Redistributions of source code must retain the above copyright
|
13
|
+
notice, this list of conditions and the following disclaimer.
|
14
|
+
* Redistributions in binary form must reproduce the above
|
15
|
+
copyright notice, this list of conditions and the following disclaimer
|
16
|
+
in the documentation and/or other materials provided with the
|
17
|
+
distribution.
|
18
|
+
|
19
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
20
|
+
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
21
|
+
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
22
|
+
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
23
|
+
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
24
|
+
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
25
|
+
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
26
|
+
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
27
|
+
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
28
|
+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
29
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
30
|
+
|
31
|
+
You can contact the author at :
|
32
|
+
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
33
|
+
****************************************************************** */
|
34
|
+
|
35
|
+
|
36
|
+
/*
|
37
|
+
* The purpose of this header is to enable debug functions.
|
38
|
+
* They regroup assert(), DEBUGLOG() and RAWLOG() for run-time,
|
39
|
+
* and DEBUG_STATIC_ASSERT() for compile-time.
|
40
|
+
*
|
41
|
+
* By default, DEBUGLEVEL==0, which means run-time debug is disabled.
|
42
|
+
*
|
43
|
+
* Level 1 enables assert() only.
|
44
|
+
* Starting level 2, traces can be generated and pushed to stderr.
|
45
|
+
* The higher the level, the more verbose the traces.
|
46
|
+
*
|
47
|
+
* It's possible to dynamically adjust level using variable g_debug_level,
|
48
|
+
* which is only declared if DEBUGLEVEL>=2,
|
49
|
+
* and is a global variable, not multi-thread protected (use with care)
|
50
|
+
*/
|
51
|
+
|
52
|
+
#ifndef DEBUG_H_12987983217
|
53
|
+
#define DEBUG_H_12987983217
|
54
|
+
|
55
|
+
#if defined (__cplusplus)
|
56
|
+
extern "C" {
|
57
|
+
#endif
|
58
|
+
|
59
|
+
|
60
|
+
/* static assert is triggered at compile time, leaving no runtime artefact.
|
61
|
+
* static assert only works with compile-time constants.
|
62
|
+
* Also, this variant can only be used inside a function. */
|
63
|
+
#define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1])
|
64
|
+
|
65
|
+
|
66
|
+
/* DEBUGLEVEL is expected to be defined externally,
|
67
|
+
* typically through compiler command line.
|
68
|
+
* Value must be a number. */
|
69
|
+
#ifndef DEBUGLEVEL
|
70
|
+
# define DEBUGLEVEL 0
|
71
|
+
#endif
|
72
|
+
|
73
|
+
|
74
|
+
/* DEBUGFILE can be defined externally,
|
75
|
+
* typically through compiler command line.
|
76
|
+
* note : currently useless.
|
77
|
+
* Value must be stderr or stdout */
|
78
|
+
#ifndef DEBUGFILE
|
79
|
+
# define DEBUGFILE stderr
|
80
|
+
#endif
|
81
|
+
|
82
|
+
|
83
|
+
/* recommended values for DEBUGLEVEL :
|
84
|
+
* 0 : release mode, no debug, all run-time checks disabled
|
85
|
+
* 1 : enables assert() only, no display
|
86
|
+
* 2 : reserved, for currently active debug path
|
87
|
+
* 3 : events once per object lifetime (CCtx, CDict, etc.)
|
88
|
+
* 4 : events once per frame
|
89
|
+
* 5 : events once per block
|
90
|
+
* 6 : events once per sequence (verbose)
|
91
|
+
* 7+: events at every position (*very* verbose)
|
92
|
+
*
|
93
|
+
* It's generally inconvenient to output traces > 5.
|
94
|
+
* In which case, it's possible to selectively trigger high verbosity levels
|
95
|
+
* by modifying g_debug_level.
|
96
|
+
*/
|
97
|
+
|
98
|
+
#if (DEBUGLEVEL>=1)
|
99
|
+
# include <assert.h>
|
100
|
+
#else
|
101
|
+
# ifndef assert /* assert may be already defined, due to prior #include <assert.h> */
|
102
|
+
# define assert(condition) ((void)0) /* disable assert (default) */
|
103
|
+
# endif
|
104
|
+
#endif
|
105
|
+
|
106
|
+
#if (DEBUGLEVEL>=2)
|
107
|
+
# include <stdio.h>
|
108
|
+
extern int g_debuglevel; /* the variable is only declared,
|
109
|
+
it actually lives in debug.c,
|
110
|
+
and is shared by the whole process.
|
111
|
+
It's not thread-safe.
|
112
|
+
It's useful when enabling very verbose levels
|
113
|
+
on selective conditions (such as position in src) */
|
114
|
+
|
115
|
+
# define RAWLOG(l, ...) { \
|
116
|
+
if (l<=g_debuglevel) { \
|
117
|
+
fprintf(stderr, __VA_ARGS__); \
|
118
|
+
} }
|
119
|
+
# define DEBUGLOG(l, ...) { \
|
120
|
+
if (l<=g_debuglevel) { \
|
121
|
+
fprintf(stderr, __FILE__ ": " __VA_ARGS__); \
|
122
|
+
fprintf(stderr, " \n"); \
|
123
|
+
} }
|
124
|
+
#else
|
125
|
+
# define RAWLOG(l, ...) {} /* disabled */
|
126
|
+
# define DEBUGLOG(l, ...) {} /* disabled */
|
127
|
+
#endif
|
128
|
+
|
129
|
+
|
130
|
+
#if defined (__cplusplus)
|
131
|
+
}
|
132
|
+
#endif
|
133
|
+
|
134
|
+
#endif /* DEBUG_H_12987983217 */
|
@@ -72,7 +72,21 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
|
|
72
72
|
unsigned charnum = 0;
|
73
73
|
int previous0 = 0;
|
74
74
|
|
75
|
-
if (hbSize < 4)
|
75
|
+
if (hbSize < 4) {
|
76
|
+
/* This function only works when hbSize >= 4 */
|
77
|
+
char buffer[4];
|
78
|
+
memset(buffer, 0, sizeof(buffer));
|
79
|
+
memcpy(buffer, headerBuffer, hbSize);
|
80
|
+
{ size_t const countSize = FSE_readNCount(normalizedCounter, maxSVPtr, tableLogPtr,
|
81
|
+
buffer, sizeof(buffer));
|
82
|
+
if (FSE_isError(countSize)) return countSize;
|
83
|
+
if (countSize > hbSize) return ERROR(corruption_detected);
|
84
|
+
return countSize;
|
85
|
+
} }
|
86
|
+
assert(hbSize >= 4);
|
87
|
+
|
88
|
+
/* init */
|
89
|
+
memset(normalizedCounter, 0, (*maxSVPtr+1) * sizeof(normalizedCounter[0])); /* all symbols not present in NCount have a frequency of 0 */
|
76
90
|
bitStream = MEM_readLE32(ip);
|
77
91
|
nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */
|
78
92
|
if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge);
|
@@ -105,6 +119,7 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
|
|
105
119
|
if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall);
|
106
120
|
while (charnum < n0) normalizedCounter[charnum++] = 0;
|
107
121
|
if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
|
122
|
+
assert((bitCount >> 3) <= 3); /* For first condition to work */
|
108
123
|
ip += bitCount>>3;
|
109
124
|
bitCount &= 7;
|
110
125
|
bitStream = MEM_readLE32(ip) >> bitCount;
|
@@ -14,6 +14,10 @@
|
|
14
14
|
|
15
15
|
const char* ERR_getErrorString(ERR_enum code)
|
16
16
|
{
|
17
|
+
#ifdef ZSTD_STRIP_ERROR_STRINGS
|
18
|
+
(void)code;
|
19
|
+
return "Error strings stripped";
|
20
|
+
#else
|
17
21
|
static const char* const notErrorCode = "Unspecified error code";
|
18
22
|
switch( code )
|
19
23
|
{
|
@@ -29,6 +33,7 @@ const char* ERR_getErrorString(ERR_enum code)
|
|
29
33
|
case PREFIX(parameter_outOfBound): return "Parameter is out of bound";
|
30
34
|
case PREFIX(init_missing): return "Context should be init first";
|
31
35
|
case PREFIX(memory_allocation): return "Allocation error : not enough memory";
|
36
|
+
case PREFIX(workSpace_tooSmall): return "workSpace buffer is not large enough";
|
32
37
|
case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
|
33
38
|
case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
|
34
39
|
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
|
@@ -38,10 +43,12 @@ const char* ERR_getErrorString(ERR_enum code)
|
|
38
43
|
case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples";
|
39
44
|
case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
|
40
45
|
case PREFIX(srcSize_wrong): return "Src size is incorrect";
|
46
|
+
case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer";
|
41
47
|
/* following error codes are not stable and may be removed or changed in a future version */
|
42
48
|
case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
|
43
49
|
case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
|
44
50
|
case PREFIX(maxCode):
|
45
51
|
default: return notErrorCode;
|
46
52
|
}
|
53
|
+
#endif
|
47
54
|
}
|
@@ -72,6 +72,7 @@ extern "C" {
|
|
72
72
|
#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE)
|
73
73
|
FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */
|
74
74
|
|
75
|
+
|
75
76
|
/*-****************************************
|
76
77
|
* FSE simple functions
|
77
78
|
******************************************/
|
@@ -129,7 +130,7 @@ FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src,
|
|
129
130
|
******************************************/
|
130
131
|
/*!
|
131
132
|
FSE_compress() does the following:
|
132
|
-
1. count symbol occurrence from source[] into table count[]
|
133
|
+
1. count symbol occurrence from source[] into table count[] (see hist.h)
|
133
134
|
2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
|
134
135
|
3. save normalized counters to memory buffer using writeNCount()
|
135
136
|
4. build encoding table 'CTable' from normalized counters
|
@@ -147,15 +148,6 @@ or to save and provide normalized distribution using external method.
|
|
147
148
|
|
148
149
|
/* *** COMPRESSION *** */
|
149
150
|
|
150
|
-
/*! FSE_count():
|
151
|
-
Provides the precise count of each byte within a table 'count'.
|
152
|
-
'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1).
|
153
|
-
*maxSymbolValuePtr will be updated if detected smaller than initial value.
|
154
|
-
@return : the count of the most frequent symbol (which is not identified).
|
155
|
-
if return == srcSize, there is only one symbol.
|
156
|
-
Can also return an error code, which can be tested with FSE_isError(). */
|
157
|
-
FSE_PUBLIC_API size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
|
158
|
-
|
159
151
|
/*! FSE_optimalTableLog():
|
160
152
|
dynamically downsize 'tableLog' when conditions are met.
|
161
153
|
It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
|
@@ -167,7 +159,8 @@ FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize
|
|
167
159
|
'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
|
168
160
|
@return : tableLog,
|
169
161
|
or an errorCode, which can be tested using FSE_isError() */
|
170
|
-
FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog,
|
162
|
+
FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog,
|
163
|
+
const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
|
171
164
|
|
172
165
|
/*! FSE_NCountWriteBound():
|
173
166
|
Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'.
|
@@ -178,8 +171,9 @@ FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tab
|
|
178
171
|
Compactly save 'normalizedCounter' into 'buffer'.
|
179
172
|
@return : size of the compressed table,
|
180
173
|
or an errorCode, which can be tested using FSE_isError(). */
|
181
|
-
FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize,
|
182
|
-
|
174
|
+
FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize,
|
175
|
+
const short* normalizedCounter,
|
176
|
+
unsigned maxSymbolValue, unsigned tableLog);
|
183
177
|
|
184
178
|
/*! Constructor and Destructor of FSE_CTable.
|
185
179
|
Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
|
@@ -250,7 +244,9 @@ If there is an error, the function will return an ErrorCode (which can be tested
|
|
250
244
|
@return : size read from 'rBuffer',
|
251
245
|
or an errorCode, which can be tested using FSE_isError().
|
252
246
|
maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
|
253
|
-
FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter,
|
247
|
+
FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter,
|
248
|
+
unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
|
249
|
+
const void* rBuffer, size_t rBuffSize);
|
254
250
|
|
255
251
|
/*! Constructor and Destructor of FSE_DTable.
|
256
252
|
Note that its size depends on 'tableLog' */
|
@@ -325,33 +321,8 @@ If there is an error, the function will return an error code, which can be teste
|
|
325
321
|
|
326
322
|
|
327
323
|
/* *****************************************
|
328
|
-
* FSE advanced API
|
329
|
-
|
330
|
-
/* FSE_count_wksp() :
|
331
|
-
* Same as FSE_count(), but using an externally provided scratch buffer.
|
332
|
-
* `workSpace` size must be table of >= `1024` unsigned
|
333
|
-
*/
|
334
|
-
size_t FSE_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
|
335
|
-
const void* source, size_t sourceSize, unsigned* workSpace);
|
336
|
-
|
337
|
-
/** FSE_countFast() :
|
338
|
-
* same as FSE_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr
|
339
|
-
*/
|
340
|
-
size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
|
341
|
-
|
342
|
-
/* FSE_countFast_wksp() :
|
343
|
-
* Same as FSE_countFast(), but using an externally provided scratch buffer.
|
344
|
-
* `workSpace` must be a table of minimum `1024` unsigned
|
345
|
-
*/
|
346
|
-
size_t FSE_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* workSpace);
|
347
|
-
|
348
|
-
/*! FSE_count_simple
|
349
|
-
* Same as FSE_countFast(), but does not use any additional memory (not even on stack).
|
350
|
-
* This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` (presuming it's also the size of `count`).
|
351
|
-
*/
|
352
|
-
size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
|
353
|
-
|
354
|
-
|
324
|
+
* FSE advanced API
|
325
|
+
***************************************** */
|
355
326
|
|
356
327
|
unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
|
357
328
|
/**< same as FSE_optimalTableLog(), which used `minus==2` */
|
@@ -387,7 +358,7 @@ size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size
|
|
387
358
|
typedef enum {
|
388
359
|
FSE_repeat_none, /**< Cannot use the previous table */
|
389
360
|
FSE_repeat_check, /**< Can use the previous table but it must be checked */
|
390
|
-
FSE_repeat_valid /**< Can use the previous table and it is
|
361
|
+
FSE_repeat_valid /**< Can use the previous table and it is assumed to be valid */
|
391
362
|
} FSE_repeat;
|
392
363
|
|
393
364
|
/* *****************************************
|
@@ -541,7 +512,7 @@ MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
|
|
541
512
|
const U32 tableLog = MEM_read16(ptr);
|
542
513
|
statePtr->value = (ptrdiff_t)1<<tableLog;
|
543
514
|
statePtr->stateTable = u16ptr+2;
|
544
|
-
statePtr->symbolTT =
|
515
|
+
statePtr->symbolTT = ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1);
|
545
516
|
statePtr->stateLog = tableLog;
|
546
517
|
}
|
547
518
|
|
@@ -560,7 +531,7 @@ MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U3
|
|
560
531
|
}
|
561
532
|
}
|
562
533
|
|
563
|
-
MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr,
|
534
|
+
MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, unsigned symbol)
|
564
535
|
{
|
565
536
|
FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
|
566
537
|
const U16* const stateTable = (const U16*)(statePtr->stateTable);
|
@@ -576,6 +547,39 @@ MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePt
|
|
576
547
|
}
|
577
548
|
|
578
549
|
|
550
|
+
/* FSE_getMaxNbBits() :
|
551
|
+
* Approximate maximum cost of a symbol, in bits.
|
552
|
+
* Fractional get rounded up (i.e : a symbol with a normalized frequency of 3 gives the same result as a frequency of 2)
|
553
|
+
* note 1 : assume symbolValue is valid (<= maxSymbolValue)
|
554
|
+
* note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
|
555
|
+
MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue)
|
556
|
+
{
|
557
|
+
const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr;
|
558
|
+
return (symbolTT[symbolValue].deltaNbBits + ((1<<16)-1)) >> 16;
|
559
|
+
}
|
560
|
+
|
561
|
+
/* FSE_bitCost() :
|
562
|
+
* Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits)
|
563
|
+
* note 1 : assume symbolValue is valid (<= maxSymbolValue)
|
564
|
+
* note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
|
565
|
+
MEM_STATIC U32 FSE_bitCost(const void* symbolTTPtr, U32 tableLog, U32 symbolValue, U32 accuracyLog)
|
566
|
+
{
|
567
|
+
const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr;
|
568
|
+
U32 const minNbBits = symbolTT[symbolValue].deltaNbBits >> 16;
|
569
|
+
U32 const threshold = (minNbBits+1) << 16;
|
570
|
+
assert(tableLog < 16);
|
571
|
+
assert(accuracyLog < 31-tableLog); /* ensure enough room for renormalization double shift */
|
572
|
+
{ U32 const tableSize = 1 << tableLog;
|
573
|
+
U32 const deltaFromThreshold = threshold - (symbolTT[symbolValue].deltaNbBits + tableSize);
|
574
|
+
U32 const normalizedDeltaFromThreshold = (deltaFromThreshold << accuracyLog) >> tableLog; /* linear interpolation (very approximate) */
|
575
|
+
U32 const bitMultiplier = 1 << accuracyLog;
|
576
|
+
assert(symbolTT[symbolValue].deltaNbBits + tableSize <= threshold);
|
577
|
+
assert(normalizedDeltaFromThreshold <= bitMultiplier);
|
578
|
+
return (minNbBits+1)*bitMultiplier - normalizedDeltaFromThreshold;
|
579
|
+
}
|
580
|
+
}
|
581
|
+
|
582
|
+
|
579
583
|
/* ====== Decompression ====== */
|
580
584
|
|
581
585
|
typedef struct {
|
@@ -49,7 +49,7 @@
|
|
49
49
|
* Error Management
|
50
50
|
****************************************************************/
|
51
51
|
#define FSE_isError ERR_isError
|
52
|
-
#define FSE_STATIC_ASSERT(c)
|
52
|
+
#define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */
|
53
53
|
|
54
54
|
/* check and forward error code */
|
55
55
|
#define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; }
|
@@ -139,8 +139,8 @@ size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned
|
|
139
139
|
{ U32 u;
|
140
140
|
for (u=0; u<tableSize; u++) {
|
141
141
|
FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
|
142
|
-
|
143
|
-
tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32
|
142
|
+
U32 const nextState = symbolNext[symbol]++;
|
143
|
+
tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32(nextState) );
|
144
144
|
tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
|
145
145
|
} }
|
146
146
|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
/* ******************************************************************
|
2
|
-
|
3
|
-
|
4
|
-
Copyright (C) 2013-
|
2
|
+
huff0 huffman codec,
|
3
|
+
part of Finite State Entropy library
|
4
|
+
Copyright (C) 2013-present, Yann Collet.
|
5
5
|
|
6
6
|
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
7
7
|
|
@@ -58,32 +58,32 @@ extern "C" {
|
|
58
58
|
#endif
|
59
59
|
|
60
60
|
|
61
|
-
/*
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
61
|
+
/* ========================== */
|
62
|
+
/* *** simple functions *** */
|
63
|
+
/* ========================== */
|
64
|
+
|
65
|
+
/** HUF_compress() :
|
66
|
+
* Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'.
|
67
|
+
* 'dst' buffer must be already allocated.
|
68
|
+
* Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).
|
69
|
+
* `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB.
|
70
|
+
* @return : size of compressed data (<= `dstCapacity`).
|
71
|
+
* Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
|
72
|
+
* if HUF_isError(return), compression failed (more details using HUF_getErrorName())
|
73
|
+
*/
|
73
74
|
HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity,
|
74
75
|
const void* src, size_t srcSize);
|
75
76
|
|
76
|
-
/**
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
*/
|
77
|
+
/** HUF_decompress() :
|
78
|
+
* Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
|
79
|
+
* into already allocated buffer 'dst', of minimum size 'dstSize'.
|
80
|
+
* `originalSize` : **must** be the ***exact*** size of original (uncompressed) data.
|
81
|
+
* Note : in contrast with FSE, HUF_decompress can regenerate
|
82
|
+
* RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
|
83
|
+
* because it knows size to regenerate (originalSize).
|
84
|
+
* @return : size of regenerated data (== originalSize),
|
85
|
+
* or an error code, which can be tested using HUF_isError()
|
86
|
+
*/
|
87
87
|
HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize,
|
88
88
|
const void* cSrc, size_t cSrcSize);
|
89
89
|
|
@@ -100,39 +100,32 @@ HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /**< provides error c
|
|
100
100
|
/* *** Advanced function *** */
|
101
101
|
|
102
102
|
/** HUF_compress2() :
|
103
|
-
* Same as HUF_compress(), but offers
|
104
|
-
*
|
105
|
-
|
103
|
+
* Same as HUF_compress(), but offers control over `maxSymbolValue` and `tableLog`.
|
104
|
+
* `maxSymbolValue` must be <= HUF_SYMBOLVALUE_MAX .
|
105
|
+
* `tableLog` must be `<= HUF_TABLELOG_MAX` . */
|
106
|
+
HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity,
|
107
|
+
const void* src, size_t srcSize,
|
108
|
+
unsigned maxSymbolValue, unsigned tableLog);
|
106
109
|
|
107
110
|
/** HUF_compress4X_wksp() :
|
108
111
|
* Same as HUF_compress2(), but uses externally allocated `workSpace`.
|
109
|
-
*
|
112
|
+
* `workspace` must have minimum alignment of 4, and be at least as large as HUF_WORKSPACE_SIZE */
|
110
113
|
#define HUF_WORKSPACE_SIZE (6 << 10)
|
111
114
|
#define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32))
|
112
|
-
HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity,
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
* HUF_readDTableX2_wksp() and HUF_readDTableX4_wksp().
|
117
|
-
*
|
118
|
-
* The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when
|
119
|
-
* HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15.
|
120
|
-
* Buffer overflow errors may potentially occur if code modifications result in
|
121
|
-
* a required workspace size greater than that specified in the following
|
122
|
-
* macro.
|
123
|
-
*/
|
124
|
-
#define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10)
|
125
|
-
#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
|
115
|
+
HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity,
|
116
|
+
const void* src, size_t srcSize,
|
117
|
+
unsigned maxSymbolValue, unsigned tableLog,
|
118
|
+
void* workSpace, size_t wkspSize);
|
126
119
|
|
127
120
|
#endif /* HUF_H_298734234 */
|
128
121
|
|
129
122
|
/* ******************************************************************
|
130
123
|
* WARNING !!
|
131
124
|
* The following section contains advanced and experimental definitions
|
132
|
-
* which shall never be used in the context of
|
125
|
+
* which shall never be used in the context of a dynamic library,
|
133
126
|
* because they are not guaranteed to remain stable in the future.
|
134
127
|
* Only consider them in association with static linking.
|
135
|
-
|
128
|
+
* *****************************************************************/
|
136
129
|
#if defined(HUF_STATIC_LINKING_ONLY) && !defined(HUF_H_HUF_STATIC_LINKING_ONLY)
|
137
130
|
#define HUF_H_HUF_STATIC_LINKING_ONLY
|
138
131
|
|
@@ -141,11 +134,11 @@ HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, const
|
|
141
134
|
|
142
135
|
|
143
136
|
/* *** Constants *** */
|
144
|
-
#define HUF_TABLELOG_MAX 12
|
145
|
-
#define HUF_TABLELOG_DEFAULT 11
|
137
|
+
#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
|
138
|
+
#define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */
|
146
139
|
#define HUF_SYMBOLVALUE_MAX 255
|
147
140
|
|
148
|
-
#define HUF_TABLELOG_ABSOLUTEMAX 15
|
141
|
+
#define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
|
149
142
|
#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
|
150
143
|
# error "HUF_TABLELOG_MAX is too large !"
|
151
144
|
#endif
|
@@ -170,130 +163,193 @@ HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, const
|
|
170
163
|
/* static allocation of HUF's DTable */
|
171
164
|
typedef U32 HUF_DTable;
|
172
165
|
#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog)))
|
173
|
-
#define
|
166
|
+
#define HUF_CREATE_STATIC_DTABLEX1(DTable, maxTableLog) \
|
174
167
|
HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) }
|
175
|
-
#define
|
168
|
+
#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
|
176
169
|
HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) }
|
177
170
|
|
178
171
|
|
179
172
|
/* ****************************************
|
180
173
|
* Advanced decompression functions
|
181
174
|
******************************************/
|
182
|
-
size_t
|
183
|
-
|
175
|
+
size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
176
|
+
#ifndef HUF_FORCE_DECOMPRESS_X1
|
177
|
+
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
178
|
+
#endif
|
184
179
|
|
185
180
|
size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */
|
186
181
|
size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
|
187
182
|
size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */
|
188
|
-
size_t
|
189
|
-
size_t
|
190
|
-
|
191
|
-
size_t
|
183
|
+
size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
184
|
+
size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
|
185
|
+
#ifndef HUF_FORCE_DECOMPRESS_X1
|
186
|
+
size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
187
|
+
size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
|
188
|
+
#endif
|
192
189
|
|
193
190
|
|
194
191
|
/* ****************************************
|
195
|
-
* HUF detailed API
|
196
|
-
|
197
|
-
|
198
|
-
HUF_compress() does the following:
|
199
|
-
1. count symbol occurrence from source[] into table count[] using FSE_count()
|
200
|
-
2. (optional) refine tableLog using HUF_optimalTableLog()
|
201
|
-
3. build Huffman table from count using HUF_buildCTable()
|
202
|
-
4. save Huffman table to memory buffer using HUF_writeCTable()
|
203
|
-
5. encode the data stream using HUF_compress4X_usingCTable()
|
204
|
-
|
205
|
-
The following API allows targeting specific sub-functions for advanced tasks.
|
206
|
-
For example, it's possible to compress several blocks using the same 'CTable',
|
207
|
-
or to save and regenerate 'CTable' using external methods.
|
208
|
-
*/
|
209
|
-
/* FSE_count() : find it within "fse.h" */
|
192
|
+
* HUF detailed API
|
193
|
+
* ****************************************/
|
194
|
+
|
195
|
+
/*! HUF_compress() does the following:
|
196
|
+
* 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h")
|
197
|
+
* 2. (optional) refine tableLog using HUF_optimalTableLog()
|
198
|
+
* 3. build Huffman table from count using HUF_buildCTable()
|
199
|
+
* 4. save Huffman table to memory buffer using HUF_writeCTable()
|
200
|
+
* 5. encode the data stream using HUF_compress4X_usingCTable()
|
201
|
+
*
|
202
|
+
* The following API allows targeting specific sub-functions for advanced tasks.
|
203
|
+
* For example, it's possible to compress several blocks using the same 'CTable',
|
204
|
+
* or to save and regenerate 'CTable' using external methods.
|
205
|
+
*/
|
210
206
|
unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
|
211
207
|
typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */
|
212
|
-
size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);
|
208
|
+
size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */
|
213
209
|
size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
|
214
210
|
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
|
215
211
|
|
216
212
|
typedef enum {
|
217
213
|
HUF_repeat_none, /**< Cannot use the previous table */
|
218
214
|
HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */
|
219
|
-
HUF_repeat_valid /**< Can use the previous table and it is
|
215
|
+
HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */
|
220
216
|
} HUF_repeat;
|
221
217
|
/** HUF_compress4X_repeat() :
|
222
|
-
*
|
223
|
-
*
|
224
|
-
*
|
225
|
-
*
|
226
|
-
size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
|
218
|
+
* Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
|
219
|
+
* If it uses hufTable it does not modify hufTable or repeat.
|
220
|
+
* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
|
221
|
+
* If preferRepeat then the old table will always be used if valid. */
|
222
|
+
size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
|
223
|
+
const void* src, size_t srcSize,
|
224
|
+
unsigned maxSymbolValue, unsigned tableLog,
|
225
|
+
void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
|
226
|
+
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2);
|
227
227
|
|
228
228
|
/** HUF_buildCTable_wksp() :
|
229
229
|
* Same as HUF_buildCTable(), but using externally allocated scratch buffer.
|
230
|
-
*
|
230
|
+
* `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE.
|
231
231
|
*/
|
232
|
-
|
232
|
+
#define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1)
|
233
|
+
#define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned))
|
234
|
+
size_t HUF_buildCTable_wksp (HUF_CElt* tree,
|
235
|
+
const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,
|
236
|
+
void* workSpace, size_t wkspSize);
|
233
237
|
|
234
238
|
/*! HUF_readStats() :
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
size_t HUF_readStats(BYTE* huffWeight, size_t hwSize,
|
240
|
-
U32* nbSymbolsPtr, U32* tableLogPtr,
|
239
|
+
* Read compact Huffman tree, saved by HUF_writeCTable().
|
240
|
+
* `huffWeight` is destination buffer.
|
241
|
+
* @return : size read from `src` , or an error Code .
|
242
|
+
* Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
|
243
|
+
size_t HUF_readStats(BYTE* huffWeight, size_t hwSize,
|
244
|
+
U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
|
241
245
|
const void* src, size_t srcSize);
|
242
246
|
|
243
247
|
/** HUF_readCTable() :
|
244
|
-
*
|
248
|
+
* Loading a CTable saved with HUF_writeCTable() */
|
245
249
|
size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
|
246
250
|
|
251
|
+
/** HUF_getNbBits() :
|
252
|
+
* Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX
|
253
|
+
* Note 1 : is not inlined, as HUF_CElt definition is private
|
254
|
+
* Note 2 : const void* used, so that it can provide a statically allocated table as argument (which uses type U32) */
|
255
|
+
U32 HUF_getNbBits(const void* symbolTable, U32 symbolValue);
|
247
256
|
|
248
257
|
/*
|
249
|
-
HUF_decompress() does the following:
|
250
|
-
1. select the decompression algorithm (
|
251
|
-
2. build Huffman table from save, using
|
252
|
-
3. decode 1 or 4 segments in parallel using
|
253
|
-
*/
|
258
|
+
* HUF_decompress() does the following:
|
259
|
+
* 1. select the decompression algorithm (X1, X2) based on pre-computed heuristics
|
260
|
+
* 2. build Huffman table from save, using HUF_readDTableX?()
|
261
|
+
* 3. decode 1 or 4 segments in parallel using HUF_decompress?X?_usingDTable()
|
262
|
+
*/
|
254
263
|
|
255
264
|
/** HUF_selectDecoder() :
|
256
|
-
*
|
257
|
-
*
|
258
|
-
*
|
259
|
-
*
|
265
|
+
* Tells which decoder is likely to decode faster,
|
266
|
+
* based on a set of pre-computed metrics.
|
267
|
+
* @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 .
|
268
|
+
* Assumption : 0 < dstSize <= 128 KB */
|
260
269
|
U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
|
261
270
|
|
271
|
+
/**
|
272
|
+
* The minimum workspace size for the `workSpace` used in
|
273
|
+
* HUF_readDTableX1_wksp() and HUF_readDTableX2_wksp().
|
274
|
+
*
|
275
|
+
* The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when
|
276
|
+
* HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15.
|
277
|
+
* Buffer overflow errors may potentially occur if code modifications result in
|
278
|
+
* a required workspace size greater than that specified in the following
|
279
|
+
* macro.
|
280
|
+
*/
|
281
|
+
#define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10)
|
282
|
+
#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
|
283
|
+
|
284
|
+
#ifndef HUF_FORCE_DECOMPRESS_X2
|
285
|
+
size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize);
|
286
|
+
size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
|
287
|
+
#endif
|
288
|
+
#ifndef HUF_FORCE_DECOMPRESS_X1
|
262
289
|
size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize);
|
263
290
|
size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
|
264
|
-
|
265
|
-
size_t HUF_readDTableX4_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
|
291
|
+
#endif
|
266
292
|
|
267
293
|
size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
294
|
+
#ifndef HUF_FORCE_DECOMPRESS_X2
|
295
|
+
size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
296
|
+
#endif
|
297
|
+
#ifndef HUF_FORCE_DECOMPRESS_X1
|
268
298
|
size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
269
|
-
|
299
|
+
#endif
|
270
300
|
|
271
301
|
|
302
|
+
/* ====================== */
|
272
303
|
/* single stream variants */
|
304
|
+
/* ====================== */
|
273
305
|
|
274
306
|
size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
275
307
|
size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
|
276
308
|
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
|
277
309
|
/** HUF_compress1X_repeat() :
|
278
|
-
*
|
279
|
-
*
|
280
|
-
*
|
281
|
-
*
|
282
|
-
size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
|
283
|
-
|
284
|
-
|
285
|
-
|
310
|
+
* Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
|
311
|
+
* If it uses hufTable it does not modify hufTable or repeat.
|
312
|
+
* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
|
313
|
+
* If preferRepeat then the old table will always be used if valid. */
|
314
|
+
size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
|
315
|
+
const void* src, size_t srcSize,
|
316
|
+
unsigned maxSymbolValue, unsigned tableLog,
|
317
|
+
void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
|
318
|
+
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2);
|
319
|
+
|
320
|
+
size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
|
321
|
+
#ifndef HUF_FORCE_DECOMPRESS_X1
|
322
|
+
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
|
323
|
+
#endif
|
286
324
|
|
287
325
|
size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
|
288
326
|
size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);
|
289
|
-
|
290
|
-
size_t
|
291
|
-
size_t
|
292
|
-
|
327
|
+
#ifndef HUF_FORCE_DECOMPRESS_X2
|
328
|
+
size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
329
|
+
size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
|
330
|
+
#endif
|
331
|
+
#ifndef HUF_FORCE_DECOMPRESS_X1
|
332
|
+
size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
333
|
+
size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
|
334
|
+
#endif
|
293
335
|
|
294
336
|
size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */
|
337
|
+
#ifndef HUF_FORCE_DECOMPRESS_X2
|
338
|
+
size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
339
|
+
#endif
|
340
|
+
#ifndef HUF_FORCE_DECOMPRESS_X1
|
295
341
|
size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
296
|
-
|
342
|
+
#endif
|
343
|
+
|
344
|
+
/* BMI2 variants.
|
345
|
+
* If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
|
346
|
+
*/
|
347
|
+
size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
|
348
|
+
#ifndef HUF_FORCE_DECOMPRESS_X2
|
349
|
+
size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
|
350
|
+
#endif
|
351
|
+
size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
|
352
|
+
size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
|
297
353
|
|
298
354
|
#endif /* HUF_STATIC_LINKING_ONLY */
|
299
355
|
|