extzstd 0.2 → 0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/HISTORY.ja.md +13 -0
- data/README.md +17 -14
- data/contrib/zstd/{NEWS → CHANGELOG} +115 -2
- data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
- data/contrib/zstd/Makefile +99 -53
- data/contrib/zstd/README.md +59 -39
- data/contrib/zstd/TESTING.md +1 -1
- data/contrib/zstd/appveyor.yml +17 -6
- data/contrib/zstd/lib/BUCK +29 -2
- data/contrib/zstd/lib/Makefile +118 -21
- data/contrib/zstd/lib/README.md +84 -44
- data/contrib/zstd/lib/common/bitstream.h +17 -33
- data/contrib/zstd/lib/common/compiler.h +62 -8
- data/contrib/zstd/lib/common/cpu.h +215 -0
- data/contrib/zstd/lib/common/debug.c +44 -0
- data/contrib/zstd/lib/common/debug.h +134 -0
- data/contrib/zstd/lib/common/entropy_common.c +16 -1
- data/contrib/zstd/lib/common/error_private.c +7 -0
- data/contrib/zstd/lib/common/fse.h +48 -44
- data/contrib/zstd/lib/common/fse_decompress.c +3 -3
- data/contrib/zstd/lib/common/huf.h +169 -113
- data/contrib/zstd/lib/common/mem.h +20 -2
- data/contrib/zstd/lib/common/pool.c +135 -49
- data/contrib/zstd/lib/common/pool.h +40 -21
- data/contrib/zstd/lib/common/threading.c +2 -2
- data/contrib/zstd/lib/common/threading.h +12 -12
- data/contrib/zstd/lib/common/xxhash.c +3 -2
- data/contrib/zstd/lib/common/zstd_common.c +3 -6
- data/contrib/zstd/lib/common/zstd_errors.h +17 -7
- data/contrib/zstd/lib/common/zstd_internal.h +76 -48
- data/contrib/zstd/lib/compress/fse_compress.c +89 -209
- data/contrib/zstd/lib/compress/hist.c +203 -0
- data/contrib/zstd/lib/compress/hist.h +95 -0
- data/contrib/zstd/lib/compress/huf_compress.c +188 -80
- data/contrib/zstd/lib/compress/zstd_compress.c +2500 -1203
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +463 -62
- data/contrib/zstd/lib/compress/zstd_double_fast.c +321 -131
- data/contrib/zstd/lib/compress/zstd_double_fast.h +13 -4
- data/contrib/zstd/lib/compress/zstd_fast.c +335 -108
- data/contrib/zstd/lib/compress/zstd_fast.h +12 -6
- data/contrib/zstd/lib/compress/zstd_lazy.c +654 -313
- data/contrib/zstd/lib/compress/zstd_lazy.h +44 -16
- data/contrib/zstd/lib/compress/zstd_ldm.c +310 -420
- data/contrib/zstd/lib/compress/zstd_ldm.h +63 -26
- data/contrib/zstd/lib/compress/zstd_opt.c +773 -325
- data/contrib/zstd/lib/compress/zstd_opt.h +31 -5
- data/contrib/zstd/lib/compress/zstdmt_compress.c +1468 -518
- data/contrib/zstd/lib/compress/zstdmt_compress.h +96 -45
- data/contrib/zstd/lib/decompress/huf_decompress.c +518 -282
- data/contrib/zstd/lib/decompress/zstd_ddict.c +240 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
- data/contrib/zstd/lib/decompress/zstd_decompress.c +613 -1513
- data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1311 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.h +59 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +175 -0
- data/contrib/zstd/lib/dictBuilder/cover.c +194 -113
- data/contrib/zstd/lib/dictBuilder/cover.h +112 -0
- data/contrib/zstd/lib/dictBuilder/divsufsort.c +3 -3
- data/contrib/zstd/lib/dictBuilder/fastcover.c +740 -0
- data/contrib/zstd/lib/dictBuilder/zdict.c +142 -106
- data/contrib/zstd/lib/dictBuilder/zdict.h +115 -49
- data/contrib/zstd/lib/legacy/zstd_legacy.h +44 -12
- data/contrib/zstd/lib/legacy/zstd_v01.c +41 -10
- data/contrib/zstd/lib/legacy/zstd_v01.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v02.c +37 -12
- data/contrib/zstd/lib/legacy/zstd_v02.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v03.c +38 -12
- data/contrib/zstd/lib/legacy/zstd_v03.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v04.c +55 -174
- data/contrib/zstd/lib/legacy/zstd_v04.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v05.c +59 -31
- data/contrib/zstd/lib/legacy/zstd_v05.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v06.c +48 -20
- data/contrib/zstd/lib/legacy/zstd_v06.h +10 -5
- data/contrib/zstd/lib/legacy/zstd_v07.c +62 -29
- data/contrib/zstd/lib/legacy/zstd_v07.h +10 -5
- data/contrib/zstd/lib/zstd.h +1346 -832
- data/ext/extzstd.c +27 -19
- data/ext/extzstd_stream.c +20 -4
- data/ext/zstd_compress.c +1 -0
- data/ext/zstd_decompress.c +4 -0
- data/ext/zstd_dictbuilder.c +4 -0
- data/ext/zstd_dictbuilder_fastcover.c +5 -0
- data/lib/extzstd.rb +52 -220
- data/lib/extzstd/version.rb +1 -1
- metadata +21 -7
- data/contrib/zstd/circle.yml +0 -63
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/* ******************************************************************
|
|
2
|
+
debug
|
|
3
|
+
Part of FSE library
|
|
4
|
+
Copyright (C) 2013-present, Yann Collet.
|
|
5
|
+
|
|
6
|
+
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
|
7
|
+
|
|
8
|
+
Redistribution and use in source and binary forms, with or without
|
|
9
|
+
modification, are permitted provided that the following conditions are
|
|
10
|
+
met:
|
|
11
|
+
|
|
12
|
+
* Redistributions of source code must retain the above copyright
|
|
13
|
+
notice, this list of conditions and the following disclaimer.
|
|
14
|
+
* Redistributions in binary form must reproduce the above
|
|
15
|
+
copyright notice, this list of conditions and the following disclaimer
|
|
16
|
+
in the documentation and/or other materials provided with the
|
|
17
|
+
distribution.
|
|
18
|
+
|
|
19
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
20
|
+
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
21
|
+
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
22
|
+
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
23
|
+
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
24
|
+
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
25
|
+
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
26
|
+
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
27
|
+
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
28
|
+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
29
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
30
|
+
|
|
31
|
+
You can contact the author at :
|
|
32
|
+
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
|
33
|
+
****************************************************************** */
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
/*
|
|
37
|
+
* This module only hosts one global variable
|
|
38
|
+
* which can be used to dynamically influence the verbosity of traces,
|
|
39
|
+
* such as DEBUGLOG and RAWLOG
|
|
40
|
+
*/
|
|
41
|
+
|
|
42
|
+
#include "debug.h"
|
|
43
|
+
|
|
44
|
+
int g_debuglevel = DEBUGLEVEL;
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/* ******************************************************************
|
|
2
|
+
debug
|
|
3
|
+
Part of FSE library
|
|
4
|
+
Copyright (C) 2013-present, Yann Collet.
|
|
5
|
+
|
|
6
|
+
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
|
7
|
+
|
|
8
|
+
Redistribution and use in source and binary forms, with or without
|
|
9
|
+
modification, are permitted provided that the following conditions are
|
|
10
|
+
met:
|
|
11
|
+
|
|
12
|
+
* Redistributions of source code must retain the above copyright
|
|
13
|
+
notice, this list of conditions and the following disclaimer.
|
|
14
|
+
* Redistributions in binary form must reproduce the above
|
|
15
|
+
copyright notice, this list of conditions and the following disclaimer
|
|
16
|
+
in the documentation and/or other materials provided with the
|
|
17
|
+
distribution.
|
|
18
|
+
|
|
19
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
20
|
+
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
21
|
+
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
22
|
+
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
23
|
+
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
24
|
+
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
25
|
+
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
26
|
+
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
27
|
+
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
28
|
+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
29
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
30
|
+
|
|
31
|
+
You can contact the author at :
|
|
32
|
+
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
|
33
|
+
****************************************************************** */
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
/*
|
|
37
|
+
* The purpose of this header is to enable debug functions.
|
|
38
|
+
* They regroup assert(), DEBUGLOG() and RAWLOG() for run-time,
|
|
39
|
+
* and DEBUG_STATIC_ASSERT() for compile-time.
|
|
40
|
+
*
|
|
41
|
+
* By default, DEBUGLEVEL==0, which means run-time debug is disabled.
|
|
42
|
+
*
|
|
43
|
+
* Level 1 enables assert() only.
|
|
44
|
+
* Starting level 2, traces can be generated and pushed to stderr.
|
|
45
|
+
* The higher the level, the more verbose the traces.
|
|
46
|
+
*
|
|
47
|
+
* It's possible to dynamically adjust level using variable g_debug_level,
|
|
48
|
+
* which is only declared if DEBUGLEVEL>=2,
|
|
49
|
+
* and is a global variable, not multi-thread protected (use with care)
|
|
50
|
+
*/
|
|
51
|
+
|
|
52
|
+
#ifndef DEBUG_H_12987983217
|
|
53
|
+
#define DEBUG_H_12987983217
|
|
54
|
+
|
|
55
|
+
#if defined (__cplusplus)
|
|
56
|
+
extern "C" {
|
|
57
|
+
#endif
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
/* static assert is triggered at compile time, leaving no runtime artefact.
|
|
61
|
+
* static assert only works with compile-time constants.
|
|
62
|
+
* Also, this variant can only be used inside a function. */
|
|
63
|
+
#define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1])
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
/* DEBUGLEVEL is expected to be defined externally,
|
|
67
|
+
* typically through compiler command line.
|
|
68
|
+
* Value must be a number. */
|
|
69
|
+
#ifndef DEBUGLEVEL
|
|
70
|
+
# define DEBUGLEVEL 0
|
|
71
|
+
#endif
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
/* DEBUGFILE can be defined externally,
|
|
75
|
+
* typically through compiler command line.
|
|
76
|
+
* note : currently useless.
|
|
77
|
+
* Value must be stderr or stdout */
|
|
78
|
+
#ifndef DEBUGFILE
|
|
79
|
+
# define DEBUGFILE stderr
|
|
80
|
+
#endif
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
/* recommended values for DEBUGLEVEL :
|
|
84
|
+
* 0 : release mode, no debug, all run-time checks disabled
|
|
85
|
+
* 1 : enables assert() only, no display
|
|
86
|
+
* 2 : reserved, for currently active debug path
|
|
87
|
+
* 3 : events once per object lifetime (CCtx, CDict, etc.)
|
|
88
|
+
* 4 : events once per frame
|
|
89
|
+
* 5 : events once per block
|
|
90
|
+
* 6 : events once per sequence (verbose)
|
|
91
|
+
* 7+: events at every position (*very* verbose)
|
|
92
|
+
*
|
|
93
|
+
* It's generally inconvenient to output traces > 5.
|
|
94
|
+
* In which case, it's possible to selectively trigger high verbosity levels
|
|
95
|
+
* by modifying g_debug_level.
|
|
96
|
+
*/
|
|
97
|
+
|
|
98
|
+
#if (DEBUGLEVEL>=1)
|
|
99
|
+
# include <assert.h>
|
|
100
|
+
#else
|
|
101
|
+
# ifndef assert /* assert may be already defined, due to prior #include <assert.h> */
|
|
102
|
+
# define assert(condition) ((void)0) /* disable assert (default) */
|
|
103
|
+
# endif
|
|
104
|
+
#endif
|
|
105
|
+
|
|
106
|
+
#if (DEBUGLEVEL>=2)
|
|
107
|
+
# include <stdio.h>
|
|
108
|
+
extern int g_debuglevel; /* the variable is only declared,
|
|
109
|
+
it actually lives in debug.c,
|
|
110
|
+
and is shared by the whole process.
|
|
111
|
+
It's not thread-safe.
|
|
112
|
+
It's useful when enabling very verbose levels
|
|
113
|
+
on selective conditions (such as position in src) */
|
|
114
|
+
|
|
115
|
+
# define RAWLOG(l, ...) { \
|
|
116
|
+
if (l<=g_debuglevel) { \
|
|
117
|
+
fprintf(stderr, __VA_ARGS__); \
|
|
118
|
+
} }
|
|
119
|
+
# define DEBUGLOG(l, ...) { \
|
|
120
|
+
if (l<=g_debuglevel) { \
|
|
121
|
+
fprintf(stderr, __FILE__ ": " __VA_ARGS__); \
|
|
122
|
+
fprintf(stderr, " \n"); \
|
|
123
|
+
} }
|
|
124
|
+
#else
|
|
125
|
+
# define RAWLOG(l, ...) {} /* disabled */
|
|
126
|
+
# define DEBUGLOG(l, ...) {} /* disabled */
|
|
127
|
+
#endif
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
#if defined (__cplusplus)
|
|
131
|
+
}
|
|
132
|
+
#endif
|
|
133
|
+
|
|
134
|
+
#endif /* DEBUG_H_12987983217 */
|
|
@@ -72,7 +72,21 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
|
|
|
72
72
|
unsigned charnum = 0;
|
|
73
73
|
int previous0 = 0;
|
|
74
74
|
|
|
75
|
-
if (hbSize < 4)
|
|
75
|
+
if (hbSize < 4) {
|
|
76
|
+
/* This function only works when hbSize >= 4 */
|
|
77
|
+
char buffer[4];
|
|
78
|
+
memset(buffer, 0, sizeof(buffer));
|
|
79
|
+
memcpy(buffer, headerBuffer, hbSize);
|
|
80
|
+
{ size_t const countSize = FSE_readNCount(normalizedCounter, maxSVPtr, tableLogPtr,
|
|
81
|
+
buffer, sizeof(buffer));
|
|
82
|
+
if (FSE_isError(countSize)) return countSize;
|
|
83
|
+
if (countSize > hbSize) return ERROR(corruption_detected);
|
|
84
|
+
return countSize;
|
|
85
|
+
} }
|
|
86
|
+
assert(hbSize >= 4);
|
|
87
|
+
|
|
88
|
+
/* init */
|
|
89
|
+
memset(normalizedCounter, 0, (*maxSVPtr+1) * sizeof(normalizedCounter[0])); /* all symbols not present in NCount have a frequency of 0 */
|
|
76
90
|
bitStream = MEM_readLE32(ip);
|
|
77
91
|
nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */
|
|
78
92
|
if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge);
|
|
@@ -105,6 +119,7 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
|
|
|
105
119
|
if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall);
|
|
106
120
|
while (charnum < n0) normalizedCounter[charnum++] = 0;
|
|
107
121
|
if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
|
|
122
|
+
assert((bitCount >> 3) <= 3); /* For first condition to work */
|
|
108
123
|
ip += bitCount>>3;
|
|
109
124
|
bitCount &= 7;
|
|
110
125
|
bitStream = MEM_readLE32(ip) >> bitCount;
|
|
@@ -14,6 +14,10 @@
|
|
|
14
14
|
|
|
15
15
|
const char* ERR_getErrorString(ERR_enum code)
|
|
16
16
|
{
|
|
17
|
+
#ifdef ZSTD_STRIP_ERROR_STRINGS
|
|
18
|
+
(void)code;
|
|
19
|
+
return "Error strings stripped";
|
|
20
|
+
#else
|
|
17
21
|
static const char* const notErrorCode = "Unspecified error code";
|
|
18
22
|
switch( code )
|
|
19
23
|
{
|
|
@@ -29,6 +33,7 @@ const char* ERR_getErrorString(ERR_enum code)
|
|
|
29
33
|
case PREFIX(parameter_outOfBound): return "Parameter is out of bound";
|
|
30
34
|
case PREFIX(init_missing): return "Context should be init first";
|
|
31
35
|
case PREFIX(memory_allocation): return "Allocation error : not enough memory";
|
|
36
|
+
case PREFIX(workSpace_tooSmall): return "workSpace buffer is not large enough";
|
|
32
37
|
case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
|
|
33
38
|
case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
|
|
34
39
|
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
|
|
@@ -38,10 +43,12 @@ const char* ERR_getErrorString(ERR_enum code)
|
|
|
38
43
|
case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples";
|
|
39
44
|
case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
|
|
40
45
|
case PREFIX(srcSize_wrong): return "Src size is incorrect";
|
|
46
|
+
case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer";
|
|
41
47
|
/* following error codes are not stable and may be removed or changed in a future version */
|
|
42
48
|
case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
|
|
43
49
|
case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
|
|
44
50
|
case PREFIX(maxCode):
|
|
45
51
|
default: return notErrorCode;
|
|
46
52
|
}
|
|
53
|
+
#endif
|
|
47
54
|
}
|
|
@@ -72,6 +72,7 @@ extern "C" {
|
|
|
72
72
|
#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE)
|
|
73
73
|
FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */
|
|
74
74
|
|
|
75
|
+
|
|
75
76
|
/*-****************************************
|
|
76
77
|
* FSE simple functions
|
|
77
78
|
******************************************/
|
|
@@ -129,7 +130,7 @@ FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src,
|
|
|
129
130
|
******************************************/
|
|
130
131
|
/*!
|
|
131
132
|
FSE_compress() does the following:
|
|
132
|
-
1. count symbol occurrence from source[] into table count[]
|
|
133
|
+
1. count symbol occurrence from source[] into table count[] (see hist.h)
|
|
133
134
|
2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
|
|
134
135
|
3. save normalized counters to memory buffer using writeNCount()
|
|
135
136
|
4. build encoding table 'CTable' from normalized counters
|
|
@@ -147,15 +148,6 @@ or to save and provide normalized distribution using external method.
|
|
|
147
148
|
|
|
148
149
|
/* *** COMPRESSION *** */
|
|
149
150
|
|
|
150
|
-
/*! FSE_count():
|
|
151
|
-
Provides the precise count of each byte within a table 'count'.
|
|
152
|
-
'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1).
|
|
153
|
-
*maxSymbolValuePtr will be updated if detected smaller than initial value.
|
|
154
|
-
@return : the count of the most frequent symbol (which is not identified).
|
|
155
|
-
if return == srcSize, there is only one symbol.
|
|
156
|
-
Can also return an error code, which can be tested with FSE_isError(). */
|
|
157
|
-
FSE_PUBLIC_API size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
|
|
158
|
-
|
|
159
151
|
/*! FSE_optimalTableLog():
|
|
160
152
|
dynamically downsize 'tableLog' when conditions are met.
|
|
161
153
|
It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
|
|
@@ -167,7 +159,8 @@ FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize
|
|
|
167
159
|
'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
|
|
168
160
|
@return : tableLog,
|
|
169
161
|
or an errorCode, which can be tested using FSE_isError() */
|
|
170
|
-
FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog,
|
|
162
|
+
FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog,
|
|
163
|
+
const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
|
|
171
164
|
|
|
172
165
|
/*! FSE_NCountWriteBound():
|
|
173
166
|
Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'.
|
|
@@ -178,8 +171,9 @@ FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tab
|
|
|
178
171
|
Compactly save 'normalizedCounter' into 'buffer'.
|
|
179
172
|
@return : size of the compressed table,
|
|
180
173
|
or an errorCode, which can be tested using FSE_isError(). */
|
|
181
|
-
FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize,
|
|
182
|
-
|
|
174
|
+
FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize,
|
|
175
|
+
const short* normalizedCounter,
|
|
176
|
+
unsigned maxSymbolValue, unsigned tableLog);
|
|
183
177
|
|
|
184
178
|
/*! Constructor and Destructor of FSE_CTable.
|
|
185
179
|
Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
|
|
@@ -250,7 +244,9 @@ If there is an error, the function will return an ErrorCode (which can be tested
|
|
|
250
244
|
@return : size read from 'rBuffer',
|
|
251
245
|
or an errorCode, which can be tested using FSE_isError().
|
|
252
246
|
maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
|
|
253
|
-
FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter,
|
|
247
|
+
FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter,
|
|
248
|
+
unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
|
|
249
|
+
const void* rBuffer, size_t rBuffSize);
|
|
254
250
|
|
|
255
251
|
/*! Constructor and Destructor of FSE_DTable.
|
|
256
252
|
Note that its size depends on 'tableLog' */
|
|
@@ -325,33 +321,8 @@ If there is an error, the function will return an error code, which can be teste
|
|
|
325
321
|
|
|
326
322
|
|
|
327
323
|
/* *****************************************
|
|
328
|
-
* FSE advanced API
|
|
329
|
-
|
|
330
|
-
/* FSE_count_wksp() :
|
|
331
|
-
* Same as FSE_count(), but using an externally provided scratch buffer.
|
|
332
|
-
* `workSpace` size must be table of >= `1024` unsigned
|
|
333
|
-
*/
|
|
334
|
-
size_t FSE_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
|
|
335
|
-
const void* source, size_t sourceSize, unsigned* workSpace);
|
|
336
|
-
|
|
337
|
-
/** FSE_countFast() :
|
|
338
|
-
* same as FSE_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr
|
|
339
|
-
*/
|
|
340
|
-
size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
|
|
341
|
-
|
|
342
|
-
/* FSE_countFast_wksp() :
|
|
343
|
-
* Same as FSE_countFast(), but using an externally provided scratch buffer.
|
|
344
|
-
* `workSpace` must be a table of minimum `1024` unsigned
|
|
345
|
-
*/
|
|
346
|
-
size_t FSE_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* workSpace);
|
|
347
|
-
|
|
348
|
-
/*! FSE_count_simple
|
|
349
|
-
* Same as FSE_countFast(), but does not use any additional memory (not even on stack).
|
|
350
|
-
* This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` (presuming it's also the size of `count`).
|
|
351
|
-
*/
|
|
352
|
-
size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
|
|
353
|
-
|
|
354
|
-
|
|
324
|
+
* FSE advanced API
|
|
325
|
+
***************************************** */
|
|
355
326
|
|
|
356
327
|
unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
|
|
357
328
|
/**< same as FSE_optimalTableLog(), which used `minus==2` */
|
|
@@ -387,7 +358,7 @@ size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size
|
|
|
387
358
|
typedef enum {
|
|
388
359
|
FSE_repeat_none, /**< Cannot use the previous table */
|
|
389
360
|
FSE_repeat_check, /**< Can use the previous table but it must be checked */
|
|
390
|
-
FSE_repeat_valid /**< Can use the previous table and it is
|
|
361
|
+
FSE_repeat_valid /**< Can use the previous table and it is assumed to be valid */
|
|
391
362
|
} FSE_repeat;
|
|
392
363
|
|
|
393
364
|
/* *****************************************
|
|
@@ -541,7 +512,7 @@ MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
|
|
|
541
512
|
const U32 tableLog = MEM_read16(ptr);
|
|
542
513
|
statePtr->value = (ptrdiff_t)1<<tableLog;
|
|
543
514
|
statePtr->stateTable = u16ptr+2;
|
|
544
|
-
statePtr->symbolTT =
|
|
515
|
+
statePtr->symbolTT = ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1);
|
|
545
516
|
statePtr->stateLog = tableLog;
|
|
546
517
|
}
|
|
547
518
|
|
|
@@ -560,7 +531,7 @@ MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U3
|
|
|
560
531
|
}
|
|
561
532
|
}
|
|
562
533
|
|
|
563
|
-
MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr,
|
|
534
|
+
MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, unsigned symbol)
|
|
564
535
|
{
|
|
565
536
|
FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
|
|
566
537
|
const U16* const stateTable = (const U16*)(statePtr->stateTable);
|
|
@@ -576,6 +547,39 @@ MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePt
|
|
|
576
547
|
}
|
|
577
548
|
|
|
578
549
|
|
|
550
|
+
/* FSE_getMaxNbBits() :
|
|
551
|
+
* Approximate maximum cost of a symbol, in bits.
|
|
552
|
+
* Fractional get rounded up (i.e : a symbol with a normalized frequency of 3 gives the same result as a frequency of 2)
|
|
553
|
+
* note 1 : assume symbolValue is valid (<= maxSymbolValue)
|
|
554
|
+
* note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
|
|
555
|
+
MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue)
|
|
556
|
+
{
|
|
557
|
+
const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr;
|
|
558
|
+
return (symbolTT[symbolValue].deltaNbBits + ((1<<16)-1)) >> 16;
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
/* FSE_bitCost() :
|
|
562
|
+
* Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits)
|
|
563
|
+
* note 1 : assume symbolValue is valid (<= maxSymbolValue)
|
|
564
|
+
* note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
|
|
565
|
+
MEM_STATIC U32 FSE_bitCost(const void* symbolTTPtr, U32 tableLog, U32 symbolValue, U32 accuracyLog)
|
|
566
|
+
{
|
|
567
|
+
const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr;
|
|
568
|
+
U32 const minNbBits = symbolTT[symbolValue].deltaNbBits >> 16;
|
|
569
|
+
U32 const threshold = (minNbBits+1) << 16;
|
|
570
|
+
assert(tableLog < 16);
|
|
571
|
+
assert(accuracyLog < 31-tableLog); /* ensure enough room for renormalization double shift */
|
|
572
|
+
{ U32 const tableSize = 1 << tableLog;
|
|
573
|
+
U32 const deltaFromThreshold = threshold - (symbolTT[symbolValue].deltaNbBits + tableSize);
|
|
574
|
+
U32 const normalizedDeltaFromThreshold = (deltaFromThreshold << accuracyLog) >> tableLog; /* linear interpolation (very approximate) */
|
|
575
|
+
U32 const bitMultiplier = 1 << accuracyLog;
|
|
576
|
+
assert(symbolTT[symbolValue].deltaNbBits + tableSize <= threshold);
|
|
577
|
+
assert(normalizedDeltaFromThreshold <= bitMultiplier);
|
|
578
|
+
return (minNbBits+1)*bitMultiplier - normalizedDeltaFromThreshold;
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
|
|
579
583
|
/* ====== Decompression ====== */
|
|
580
584
|
|
|
581
585
|
typedef struct {
|
|
@@ -49,7 +49,7 @@
|
|
|
49
49
|
* Error Management
|
|
50
50
|
****************************************************************/
|
|
51
51
|
#define FSE_isError ERR_isError
|
|
52
|
-
#define FSE_STATIC_ASSERT(c)
|
|
52
|
+
#define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */
|
|
53
53
|
|
|
54
54
|
/* check and forward error code */
|
|
55
55
|
#define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; }
|
|
@@ -139,8 +139,8 @@ size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned
|
|
|
139
139
|
{ U32 u;
|
|
140
140
|
for (u=0; u<tableSize; u++) {
|
|
141
141
|
FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
|
|
142
|
-
|
|
143
|
-
tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32
|
|
142
|
+
U32 const nextState = symbolNext[symbol]++;
|
|
143
|
+
tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32(nextState) );
|
|
144
144
|
tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
|
|
145
145
|
} }
|
|
146
146
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/* ******************************************************************
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
Copyright (C) 2013-
|
|
2
|
+
huff0 huffman codec,
|
|
3
|
+
part of Finite State Entropy library
|
|
4
|
+
Copyright (C) 2013-present, Yann Collet.
|
|
5
5
|
|
|
6
6
|
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
|
7
7
|
|
|
@@ -58,32 +58,32 @@ extern "C" {
|
|
|
58
58
|
#endif
|
|
59
59
|
|
|
60
60
|
|
|
61
|
-
/*
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
61
|
+
/* ========================== */
|
|
62
|
+
/* *** simple functions *** */
|
|
63
|
+
/* ========================== */
|
|
64
|
+
|
|
65
|
+
/** HUF_compress() :
|
|
66
|
+
* Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'.
|
|
67
|
+
* 'dst' buffer must be already allocated.
|
|
68
|
+
* Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).
|
|
69
|
+
* `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB.
|
|
70
|
+
* @return : size of compressed data (<= `dstCapacity`).
|
|
71
|
+
* Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
|
|
72
|
+
* if HUF_isError(return), compression failed (more details using HUF_getErrorName())
|
|
73
|
+
*/
|
|
73
74
|
HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity,
|
|
74
75
|
const void* src, size_t srcSize);
|
|
75
76
|
|
|
76
|
-
/**
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
*/
|
|
77
|
+
/** HUF_decompress() :
|
|
78
|
+
* Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
|
|
79
|
+
* into already allocated buffer 'dst', of minimum size 'dstSize'.
|
|
80
|
+
* `originalSize` : **must** be the ***exact*** size of original (uncompressed) data.
|
|
81
|
+
* Note : in contrast with FSE, HUF_decompress can regenerate
|
|
82
|
+
* RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
|
|
83
|
+
* because it knows size to regenerate (originalSize).
|
|
84
|
+
* @return : size of regenerated data (== originalSize),
|
|
85
|
+
* or an error code, which can be tested using HUF_isError()
|
|
86
|
+
*/
|
|
87
87
|
HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize,
|
|
88
88
|
const void* cSrc, size_t cSrcSize);
|
|
89
89
|
|
|
@@ -100,39 +100,32 @@ HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /**< provides error c
|
|
|
100
100
|
/* *** Advanced function *** */
|
|
101
101
|
|
|
102
102
|
/** HUF_compress2() :
|
|
103
|
-
* Same as HUF_compress(), but offers
|
|
104
|
-
*
|
|
105
|
-
|
|
103
|
+
* Same as HUF_compress(), but offers control over `maxSymbolValue` and `tableLog`.
|
|
104
|
+
* `maxSymbolValue` must be <= HUF_SYMBOLVALUE_MAX .
|
|
105
|
+
* `tableLog` must be `<= HUF_TABLELOG_MAX` . */
|
|
106
|
+
HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity,
|
|
107
|
+
const void* src, size_t srcSize,
|
|
108
|
+
unsigned maxSymbolValue, unsigned tableLog);
|
|
106
109
|
|
|
107
110
|
/** HUF_compress4X_wksp() :
|
|
108
111
|
* Same as HUF_compress2(), but uses externally allocated `workSpace`.
|
|
109
|
-
*
|
|
112
|
+
* `workspace` must have minimum alignment of 4, and be at least as large as HUF_WORKSPACE_SIZE */
|
|
110
113
|
#define HUF_WORKSPACE_SIZE (6 << 10)
|
|
111
114
|
#define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32))
|
|
112
|
-
HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity,
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
* HUF_readDTableX2_wksp() and HUF_readDTableX4_wksp().
|
|
117
|
-
*
|
|
118
|
-
* The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when
|
|
119
|
-
* HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15.
|
|
120
|
-
* Buffer overflow errors may potentially occur if code modifications result in
|
|
121
|
-
* a required workspace size greater than that specified in the following
|
|
122
|
-
* macro.
|
|
123
|
-
*/
|
|
124
|
-
#define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10)
|
|
125
|
-
#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
|
|
115
|
+
HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity,
|
|
116
|
+
const void* src, size_t srcSize,
|
|
117
|
+
unsigned maxSymbolValue, unsigned tableLog,
|
|
118
|
+
void* workSpace, size_t wkspSize);
|
|
126
119
|
|
|
127
120
|
#endif /* HUF_H_298734234 */
|
|
128
121
|
|
|
129
122
|
/* ******************************************************************
|
|
130
123
|
* WARNING !!
|
|
131
124
|
* The following section contains advanced and experimental definitions
|
|
132
|
-
* which shall never be used in the context of
|
|
125
|
+
* which shall never be used in the context of a dynamic library,
|
|
133
126
|
* because they are not guaranteed to remain stable in the future.
|
|
134
127
|
* Only consider them in association with static linking.
|
|
135
|
-
|
|
128
|
+
* *****************************************************************/
|
|
136
129
|
#if defined(HUF_STATIC_LINKING_ONLY) && !defined(HUF_H_HUF_STATIC_LINKING_ONLY)
|
|
137
130
|
#define HUF_H_HUF_STATIC_LINKING_ONLY
|
|
138
131
|
|
|
@@ -141,11 +134,11 @@ HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, const
|
|
|
141
134
|
|
|
142
135
|
|
|
143
136
|
/* *** Constants *** */
|
|
144
|
-
#define HUF_TABLELOG_MAX 12
|
|
145
|
-
#define HUF_TABLELOG_DEFAULT 11
|
|
137
|
+
#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
|
|
138
|
+
#define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */
|
|
146
139
|
#define HUF_SYMBOLVALUE_MAX 255
|
|
147
140
|
|
|
148
|
-
#define HUF_TABLELOG_ABSOLUTEMAX 15
|
|
141
|
+
#define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
|
|
149
142
|
#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
|
|
150
143
|
# error "HUF_TABLELOG_MAX is too large !"
|
|
151
144
|
#endif
|
|
@@ -170,130 +163,193 @@ HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, const
|
|
|
170
163
|
/* static allocation of HUF's DTable */
|
|
171
164
|
typedef U32 HUF_DTable;
|
|
172
165
|
#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog)))
|
|
173
|
-
#define
|
|
166
|
+
#define HUF_CREATE_STATIC_DTABLEX1(DTable, maxTableLog) \
|
|
174
167
|
HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) }
|
|
175
|
-
#define
|
|
168
|
+
#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
|
|
176
169
|
HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) }
|
|
177
170
|
|
|
178
171
|
|
|
179
172
|
/* ****************************************
|
|
180
173
|
* Advanced decompression functions
|
|
181
174
|
******************************************/
|
|
182
|
-
size_t
|
|
183
|
-
|
|
175
|
+
size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
|
176
|
+
#ifndef HUF_FORCE_DECOMPRESS_X1
|
|
177
|
+
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
|
178
|
+
#endif
|
|
184
179
|
|
|
185
180
|
size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */
|
|
186
181
|
size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
|
|
187
182
|
size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */
|
|
188
|
-
size_t
|
|
189
|
-
size_t
|
|
190
|
-
|
|
191
|
-
size_t
|
|
183
|
+
size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
|
184
|
+
size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
|
|
185
|
+
#ifndef HUF_FORCE_DECOMPRESS_X1
|
|
186
|
+
size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
|
187
|
+
size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
|
|
188
|
+
#endif
|
|
192
189
|
|
|
193
190
|
|
|
194
191
|
/* ****************************************
|
|
195
|
-
* HUF detailed API
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
HUF_compress() does the following:
|
|
199
|
-
1. count symbol occurrence from source[] into table count[] using FSE_count()
|
|
200
|
-
2. (optional) refine tableLog using HUF_optimalTableLog()
|
|
201
|
-
3. build Huffman table from count using HUF_buildCTable()
|
|
202
|
-
4. save Huffman table to memory buffer using HUF_writeCTable()
|
|
203
|
-
5. encode the data stream using HUF_compress4X_usingCTable()
|
|
204
|
-
|
|
205
|
-
The following API allows targeting specific sub-functions for advanced tasks.
|
|
206
|
-
For example, it's possible to compress several blocks using the same 'CTable',
|
|
207
|
-
or to save and regenerate 'CTable' using external methods.
|
|
208
|
-
*/
|
|
209
|
-
/* FSE_count() : find it within "fse.h" */
|
|
192
|
+
* HUF detailed API
|
|
193
|
+
* ****************************************/
|
|
194
|
+
|
|
195
|
+
/*! HUF_compress() does the following:
|
|
196
|
+
* 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h")
|
|
197
|
+
* 2. (optional) refine tableLog using HUF_optimalTableLog()
|
|
198
|
+
* 3. build Huffman table from count using HUF_buildCTable()
|
|
199
|
+
* 4. save Huffman table to memory buffer using HUF_writeCTable()
|
|
200
|
+
* 5. encode the data stream using HUF_compress4X_usingCTable()
|
|
201
|
+
*
|
|
202
|
+
* The following API allows targeting specific sub-functions for advanced tasks.
|
|
203
|
+
* For example, it's possible to compress several blocks using the same 'CTable',
|
|
204
|
+
* or to save and regenerate 'CTable' using external methods.
|
|
205
|
+
*/
|
|
210
206
|
unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
|
|
211
207
|
typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */
|
|
212
|
-
size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);
|
|
208
|
+
size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */
|
|
213
209
|
size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
|
|
214
210
|
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
|
|
215
211
|
|
|
216
212
|
typedef enum {
|
|
217
213
|
HUF_repeat_none, /**< Cannot use the previous table */
|
|
218
214
|
HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */
|
|
219
|
-
HUF_repeat_valid /**< Can use the previous table and it is
|
|
215
|
+
HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */
|
|
220
216
|
} HUF_repeat;
|
|
221
217
|
/** HUF_compress4X_repeat() :
|
|
222
|
-
*
|
|
223
|
-
*
|
|
224
|
-
*
|
|
225
|
-
*
|
|
226
|
-
size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
|
|
218
|
+
* Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
|
|
219
|
+
* If it uses hufTable it does not modify hufTable or repeat.
|
|
220
|
+
* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
|
|
221
|
+
* If preferRepeat then the old table will always be used if valid. */
|
|
222
|
+
size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
|
|
223
|
+
const void* src, size_t srcSize,
|
|
224
|
+
unsigned maxSymbolValue, unsigned tableLog,
|
|
225
|
+
void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
|
|
226
|
+
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2);
|
|
227
227
|
|
|
228
228
|
/** HUF_buildCTable_wksp() :
|
|
229
229
|
* Same as HUF_buildCTable(), but using externally allocated scratch buffer.
|
|
230
|
-
*
|
|
230
|
+
* `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE.
|
|
231
231
|
*/
|
|
232
|
-
|
|
232
|
+
#define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1)
|
|
233
|
+
#define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned))
|
|
234
|
+
size_t HUF_buildCTable_wksp (HUF_CElt* tree,
|
|
235
|
+
const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,
|
|
236
|
+
void* workSpace, size_t wkspSize);
|
|
233
237
|
|
|
234
238
|
/*! HUF_readStats() :
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
size_t HUF_readStats(BYTE* huffWeight, size_t hwSize,
|
|
240
|
-
U32* nbSymbolsPtr, U32* tableLogPtr,
|
|
239
|
+
* Read compact Huffman tree, saved by HUF_writeCTable().
|
|
240
|
+
* `huffWeight` is destination buffer.
|
|
241
|
+
* @return : size read from `src` , or an error Code .
|
|
242
|
+
* Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
|
|
243
|
+
size_t HUF_readStats(BYTE* huffWeight, size_t hwSize,
|
|
244
|
+
U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
|
|
241
245
|
const void* src, size_t srcSize);
|
|
242
246
|
|
|
243
247
|
/** HUF_readCTable() :
|
|
244
|
-
*
|
|
248
|
+
* Loading a CTable saved with HUF_writeCTable() */
|
|
245
249
|
size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
|
|
246
250
|
|
|
251
|
+
/** HUF_getNbBits() :
|
|
252
|
+
* Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX
|
|
253
|
+
* Note 1 : is not inlined, as HUF_CElt definition is private
|
|
254
|
+
* Note 2 : const void* used, so that it can provide a statically allocated table as argument (which uses type U32) */
|
|
255
|
+
U32 HUF_getNbBits(const void* symbolTable, U32 symbolValue);
|
|
247
256
|
|
|
248
257
|
/*
|
|
249
|
-
HUF_decompress() does the following:
|
|
250
|
-
1. select the decompression algorithm (
|
|
251
|
-
2. build Huffman table from save, using
|
|
252
|
-
3. decode 1 or 4 segments in parallel using
|
|
253
|
-
*/
|
|
258
|
+
* HUF_decompress() does the following:
|
|
259
|
+
* 1. select the decompression algorithm (X1, X2) based on pre-computed heuristics
|
|
260
|
+
* 2. build Huffman table from save, using HUF_readDTableX?()
|
|
261
|
+
* 3. decode 1 or 4 segments in parallel using HUF_decompress?X?_usingDTable()
|
|
262
|
+
*/
|
|
254
263
|
|
|
255
264
|
/** HUF_selectDecoder() :
|
|
256
|
-
*
|
|
257
|
-
*
|
|
258
|
-
*
|
|
259
|
-
*
|
|
265
|
+
* Tells which decoder is likely to decode faster,
|
|
266
|
+
* based on a set of pre-computed metrics.
|
|
267
|
+
* @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 .
|
|
268
|
+
* Assumption : 0 < dstSize <= 128 KB */
|
|
260
269
|
U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
|
|
261
270
|
|
|
271
|
+
/**
|
|
272
|
+
* The minimum workspace size for the `workSpace` used in
|
|
273
|
+
* HUF_readDTableX1_wksp() and HUF_readDTableX2_wksp().
|
|
274
|
+
*
|
|
275
|
+
* The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when
|
|
276
|
+
* HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15.
|
|
277
|
+
* Buffer overflow errors may potentially occur if code modifications result in
|
|
278
|
+
* a required workspace size greater than that specified in the following
|
|
279
|
+
* macro.
|
|
280
|
+
*/
|
|
281
|
+
#define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10)
|
|
282
|
+
#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
|
|
283
|
+
|
|
284
|
+
#ifndef HUF_FORCE_DECOMPRESS_X2
|
|
285
|
+
size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize);
|
|
286
|
+
size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
|
|
287
|
+
#endif
|
|
288
|
+
#ifndef HUF_FORCE_DECOMPRESS_X1
|
|
262
289
|
size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize);
|
|
263
290
|
size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
|
|
264
|
-
|
|
265
|
-
size_t HUF_readDTableX4_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
|
|
291
|
+
#endif
|
|
266
292
|
|
|
267
293
|
size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
|
294
|
+
#ifndef HUF_FORCE_DECOMPRESS_X2
|
|
295
|
+
size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
|
296
|
+
#endif
|
|
297
|
+
#ifndef HUF_FORCE_DECOMPRESS_X1
|
|
268
298
|
size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
|
269
|
-
|
|
299
|
+
#endif
|
|
270
300
|
|
|
271
301
|
|
|
302
|
+
/* ====================== */
|
|
272
303
|
/* single stream variants */
|
|
304
|
+
/* ====================== */
|
|
273
305
|
|
|
274
306
|
size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
|
275
307
|
size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
|
|
276
308
|
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
|
|
277
309
|
/** HUF_compress1X_repeat() :
|
|
278
|
-
*
|
|
279
|
-
*
|
|
280
|
-
*
|
|
281
|
-
*
|
|
282
|
-
size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
310
|
+
* Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
|
|
311
|
+
* If it uses hufTable it does not modify hufTable or repeat.
|
|
312
|
+
* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
|
|
313
|
+
* If preferRepeat then the old table will always be used if valid. */
|
|
314
|
+
size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
|
|
315
|
+
const void* src, size_t srcSize,
|
|
316
|
+
unsigned maxSymbolValue, unsigned tableLog,
|
|
317
|
+
void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
|
|
318
|
+
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2);
|
|
319
|
+
|
|
320
|
+
size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
|
|
321
|
+
#ifndef HUF_FORCE_DECOMPRESS_X1
|
|
322
|
+
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
|
|
323
|
+
#endif
|
|
286
324
|
|
|
287
325
|
size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
|
|
288
326
|
size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);
|
|
289
|
-
|
|
290
|
-
size_t
|
|
291
|
-
size_t
|
|
292
|
-
|
|
327
|
+
#ifndef HUF_FORCE_DECOMPRESS_X2
|
|
328
|
+
size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
|
329
|
+
size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
|
|
330
|
+
#endif
|
|
331
|
+
#ifndef HUF_FORCE_DECOMPRESS_X1
|
|
332
|
+
size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
|
333
|
+
size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
|
|
334
|
+
#endif
|
|
293
335
|
|
|
294
336
|
size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */
|
|
337
|
+
#ifndef HUF_FORCE_DECOMPRESS_X2
|
|
338
|
+
size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
|
339
|
+
#endif
|
|
340
|
+
#ifndef HUF_FORCE_DECOMPRESS_X1
|
|
295
341
|
size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
|
296
|
-
|
|
342
|
+
#endif
|
|
343
|
+
|
|
344
|
+
/* BMI2 variants.
|
|
345
|
+
* If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
|
|
346
|
+
*/
|
|
347
|
+
size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
|
|
348
|
+
#ifndef HUF_FORCE_DECOMPRESS_X2
|
|
349
|
+
size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
|
|
350
|
+
#endif
|
|
351
|
+
size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
|
|
352
|
+
size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
|
|
297
353
|
|
|
298
354
|
#endif /* HUF_STATIC_LINKING_ONLY */
|
|
299
355
|
|