zstdlib 0.3.0-x64-mingw32 → 0.8.0-x64-mingw32
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGES.md +30 -1
- data/README.md +2 -2
- data/Rakefile +1 -1
- data/ext/zstdlib/extconf.rb +3 -3
- data/ext/zstdlib/ruby/zlib-2.7/zstdlib.c +4895 -0
- data/ext/zstdlib/ruby/zlib-3.0/zstdlib.c +4994 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/bitstream.h +59 -51
- data/ext/zstdlib/zstd-1.5.0/lib/common/compiler.h +289 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/cpu.h +1 -3
- data/ext/zstdlib/zstd-1.5.0/lib/common/debug.c +24 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/debug.h +22 -49
- data/ext/zstdlib/zstd-1.5.0/lib/common/entropy_common.c +362 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/error_private.c +3 -1
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/error_private.h +8 -4
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/fse.h +50 -42
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/fse_decompress.c +149 -55
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/huf.h +43 -39
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/mem.h +69 -25
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/pool.c +30 -20
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/pool.h +3 -3
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/threading.c +51 -4
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/threading.h +36 -4
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/xxhash.c +40 -92
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/xxhash.h +12 -32
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/zstd_common.c +10 -10
- data/ext/zstdlib/zstd-1.5.0/lib/common/zstd_deps.h +111 -0
- data/ext/zstdlib/zstd-1.5.0/lib/common/zstd_internal.h +490 -0
- data/ext/zstdlib/zstd-1.5.0/lib/common/zstd_trace.h +154 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/fse_compress.c +47 -63
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/hist.c +41 -63
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/hist.h +13 -33
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/huf_compress.c +332 -193
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_compress.c +6393 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_compress_internal.h +522 -86
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_compress_literals.c +25 -16
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_compress_literals.h +2 -2
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_compress_sequences.c +50 -24
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_compress_sequences.h +11 -4
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_compress_superblock.c +572 -0
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_compress_superblock.h +32 -0
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_cwksp.h +662 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_double_fast.c +43 -41
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_double_fast.h +2 -2
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_fast.c +85 -80
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_fast.h +2 -2
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_lazy.c +2184 -0
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_lazy.h +125 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_ldm.c +333 -208
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_ldm.h +15 -3
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_ldm_geartab.h +103 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_opt.c +228 -129
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_opt.h +1 -1
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstdmt_compress.c +151 -440
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstdmt_compress.h +110 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/huf_decompress.c +395 -276
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_ddict.c +20 -16
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_ddict.h +3 -3
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_decompress.c +628 -231
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_decompress_block.c +606 -380
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_decompress_block.h +8 -5
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_decompress_internal.h +39 -9
- data/ext/zstdlib/zstd-1.5.0/lib/zdict.h +452 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/zstd.h +740 -153
- data/ext/zstdlib/{zstd-1.4.2/lib/common → zstd-1.5.0/lib}/zstd_errors.h +3 -1
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzclose.c +1 -1
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzcompatibility.h +1 -1
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzguts.h +0 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzlib.c +9 -9
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzread.c +16 -8
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzwrite.c +8 -8
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/zstd_zlibwrapper.c +131 -45
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/zstd_zlibwrapper.h +1 -1
- data/lib/2.2/zstdlib.so +0 -0
- data/lib/2.3/zstdlib.so +0 -0
- data/lib/2.4/zstdlib.so +0 -0
- data/lib/2.5/zstdlib.so +0 -0
- data/lib/2.6/zstdlib.so +0 -0
- data/lib/2.7/zstdlib.so +0 -0
- metadata +76 -67
- data/ext/zstdlib/zstd-1.4.2/lib/common/compiler.h +0 -147
- data/ext/zstdlib/zstd-1.4.2/lib/common/debug.c +0 -44
- data/ext/zstdlib/zstd-1.4.2/lib/common/entropy_common.c +0 -236
- data/ext/zstdlib/zstd-1.4.2/lib/common/zstd_internal.h +0 -371
- data/ext/zstdlib/zstd-1.4.2/lib/compress/zstd_compress.c +0 -3904
- data/ext/zstdlib/zstd-1.4.2/lib/compress/zstd_lazy.c +0 -1111
- data/ext/zstdlib/zstd-1.4.2/lib/compress/zstd_lazy.h +0 -67
- data/ext/zstdlib/zstd-1.4.2/lib/compress/zstdmt_compress.h +0 -192
@@ -1,40 +1,20 @@
|
|
1
1
|
/* ******************************************************************
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
* Redistributions in binary form must reproduce the above
|
15
|
-
copyright notice, this list of conditions and the following disclaimer
|
16
|
-
in the documentation and/or other materials provided with the
|
17
|
-
distribution.
|
18
|
-
|
19
|
-
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
20
|
-
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
21
|
-
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
22
|
-
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
23
|
-
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
24
|
-
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
25
|
-
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
26
|
-
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
27
|
-
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
28
|
-
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
29
|
-
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
30
|
-
|
31
|
-
You can contact the author at :
|
32
|
-
- FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
33
|
-
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
2
|
+
* hist : Histogram functions
|
3
|
+
* part of Finite State Entropy project
|
4
|
+
* Copyright (c) Yann Collet, Facebook, Inc.
|
5
|
+
*
|
6
|
+
* You can contact the author at :
|
7
|
+
* - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
8
|
+
* - Public forum : https://groups.google.com/forum/#!forum/lz4c
|
9
|
+
*
|
10
|
+
* This source code is licensed under both the BSD-style license (found in the
|
11
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
12
|
+
* in the COPYING file in the root directory of this source tree).
|
13
|
+
* You may select, at your option, one of the above-listed licenses.
|
34
14
|
****************************************************************** */
|
35
15
|
|
36
16
|
/* --- dependencies --- */
|
37
|
-
#include
|
17
|
+
#include "../common/zstd_deps.h" /* size_t */
|
38
18
|
|
39
19
|
|
40
20
|
/* --- simple histogram functions --- */
|
@@ -1,35 +1,15 @@
|
|
1
1
|
/* ******************************************************************
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
* Redistributions in binary form must reproduce the above
|
14
|
-
copyright notice, this list of conditions and the following disclaimer
|
15
|
-
in the documentation and/or other materials provided with the
|
16
|
-
distribution.
|
17
|
-
|
18
|
-
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
19
|
-
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
20
|
-
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
21
|
-
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
22
|
-
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
23
|
-
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
24
|
-
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
25
|
-
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
26
|
-
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
27
|
-
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
28
|
-
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
29
|
-
|
30
|
-
You can contact the author at :
|
31
|
-
- FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
32
|
-
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
2
|
+
* Huffman encoder, part of New Generation Entropy library
|
3
|
+
* Copyright (c) Yann Collet, Facebook, Inc.
|
4
|
+
*
|
5
|
+
* You can contact the author at :
|
6
|
+
* - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
7
|
+
* - Public forum : https://groups.google.com/forum/#!forum/lz4c
|
8
|
+
*
|
9
|
+
* This source code is licensed under both the BSD-style license (found in the
|
10
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
11
|
+
* in the COPYING file in the root directory of this source tree).
|
12
|
+
* You may select, at your option, one of the above-listed licenses.
|
33
13
|
****************************************************************** */
|
34
14
|
|
35
15
|
/* **************************************************************
|
@@ -43,16 +23,15 @@
|
|
43
23
|
/* **************************************************************
|
44
24
|
* Includes
|
45
25
|
****************************************************************/
|
46
|
-
#include
|
47
|
-
#include
|
48
|
-
#include "
|
49
|
-
#include "bitstream.h"
|
26
|
+
#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */
|
27
|
+
#include "../common/compiler.h"
|
28
|
+
#include "../common/bitstream.h"
|
50
29
|
#include "hist.h"
|
51
30
|
#define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */
|
52
|
-
#include "fse.h" /* header compression */
|
31
|
+
#include "../common/fse.h" /* header compression */
|
53
32
|
#define HUF_STATIC_LINKING_ONLY
|
54
|
-
#include "huf.h"
|
55
|
-
#include "error_private.h"
|
33
|
+
#include "../common/huf.h"
|
34
|
+
#include "../common/error_private.h"
|
56
35
|
|
57
36
|
|
58
37
|
/* **************************************************************
|
@@ -60,8 +39,6 @@
|
|
60
39
|
****************************************************************/
|
61
40
|
#define HUF_isError ERR_isError
|
62
41
|
#define HUF_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */
|
63
|
-
#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e
|
64
|
-
#define CHECK_F(f) { CHECK_V_F(_var_err__, f); }
|
65
42
|
|
66
43
|
|
67
44
|
/* **************************************************************
|
@@ -82,7 +59,15 @@ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
|
|
82
59
|
* Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX.
|
83
60
|
*/
|
84
61
|
#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6
|
85
|
-
|
62
|
+
|
63
|
+
typedef struct {
|
64
|
+
FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)];
|
65
|
+
U32 scratchBuffer[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(HUF_TABLELOG_MAX, MAX_FSE_TABLELOG_FOR_HUFF_HEADER)];
|
66
|
+
unsigned count[HUF_TABLELOG_MAX+1];
|
67
|
+
S16 norm[HUF_TABLELOG_MAX+1];
|
68
|
+
} HUF_CompressWeightsWksp;
|
69
|
+
|
70
|
+
static size_t HUF_compressWeights(void* dst, size_t dstSize, const void* weightTable, size_t wtSize, void* workspace, size_t workspaceSize)
|
86
71
|
{
|
87
72
|
BYTE* const ostart = (BYTE*) dst;
|
88
73
|
BYTE* op = ostart;
|
@@ -90,69 +75,65 @@ static size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weight
|
|
90
75
|
|
91
76
|
unsigned maxSymbolValue = HUF_TABLELOG_MAX;
|
92
77
|
U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER;
|
78
|
+
HUF_CompressWeightsWksp* wksp = (HUF_CompressWeightsWksp*)workspace;
|
93
79
|
|
94
|
-
|
95
|
-
BYTE scratchBuffer[1<<MAX_FSE_TABLELOG_FOR_HUFF_HEADER];
|
96
|
-
|
97
|
-
unsigned count[HUF_TABLELOG_MAX+1];
|
98
|
-
S16 norm[HUF_TABLELOG_MAX+1];
|
80
|
+
if (workspaceSize < sizeof(HUF_CompressWeightsWksp)) return ERROR(GENERIC);
|
99
81
|
|
100
82
|
/* init conditions */
|
101
83
|
if (wtSize <= 1) return 0; /* Not compressible */
|
102
84
|
|
103
85
|
/* Scan input and build symbol stats */
|
104
|
-
{ unsigned const maxCount = HIST_count_simple(count, &maxSymbolValue, weightTable, wtSize); /* never fails */
|
86
|
+
{ unsigned const maxCount = HIST_count_simple(wksp->count, &maxSymbolValue, weightTable, wtSize); /* never fails */
|
105
87
|
if (maxCount == wtSize) return 1; /* only a single symbol in src : rle */
|
106
88
|
if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */
|
107
89
|
}
|
108
90
|
|
109
91
|
tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue);
|
110
|
-
CHECK_F( FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue) );
|
92
|
+
CHECK_F( FSE_normalizeCount(wksp->norm, tableLog, wksp->count, wtSize, maxSymbolValue, /* useLowProbCount */ 0) );
|
111
93
|
|
112
94
|
/* Write table description header */
|
113
|
-
{ CHECK_V_F(hSize, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) );
|
95
|
+
{ CHECK_V_F(hSize, FSE_writeNCount(op, (size_t)(oend-op), wksp->norm, maxSymbolValue, tableLog) );
|
114
96
|
op += hSize;
|
115
97
|
}
|
116
98
|
|
117
99
|
/* Compress */
|
118
|
-
CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, sizeof(scratchBuffer)) );
|
119
|
-
{ CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable) );
|
100
|
+
CHECK_F( FSE_buildCTable_wksp(wksp->CTable, wksp->norm, maxSymbolValue, tableLog, wksp->scratchBuffer, sizeof(wksp->scratchBuffer)) );
|
101
|
+
{ CHECK_V_F(cSize, FSE_compress_usingCTable(op, (size_t)(oend - op), weightTable, wtSize, wksp->CTable) );
|
120
102
|
if (cSize == 0) return 0; /* not enough space for compressed data */
|
121
103
|
op += cSize;
|
122
104
|
}
|
123
105
|
|
124
|
-
return op-ostart;
|
106
|
+
return (size_t)(op-ostart);
|
125
107
|
}
|
126
108
|
|
127
109
|
|
128
|
-
struct
|
129
|
-
|
130
|
-
BYTE nbBits;
|
131
|
-
}; /* typedef'd to HUF_CElt within "huf.h" */
|
132
|
-
|
133
|
-
/*! HUF_writeCTable() :
|
134
|
-
`CTable` : Huffman tree to save, using huf representation.
|
135
|
-
@return : size of saved CTable */
|
136
|
-
size_t HUF_writeCTable (void* dst, size_t maxDstSize,
|
137
|
-
const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog)
|
138
|
-
{
|
110
|
+
typedef struct {
|
111
|
+
HUF_CompressWeightsWksp wksp;
|
139
112
|
BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */
|
140
113
|
BYTE huffWeight[HUF_SYMBOLVALUE_MAX];
|
114
|
+
} HUF_WriteCTableWksp;
|
115
|
+
|
116
|
+
size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize,
|
117
|
+
const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog,
|
118
|
+
void* workspace, size_t workspaceSize)
|
119
|
+
{
|
141
120
|
BYTE* op = (BYTE*)dst;
|
142
121
|
U32 n;
|
122
|
+
HUF_WriteCTableWksp* wksp = (HUF_WriteCTableWksp*)workspace;
|
143
123
|
|
144
|
-
|
124
|
+
/* check conditions */
|
125
|
+
if (workspaceSize < sizeof(HUF_WriteCTableWksp)) return ERROR(GENERIC);
|
145
126
|
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge);
|
146
127
|
|
147
128
|
/* convert to weight */
|
148
|
-
bitsToWeight[0] = 0;
|
129
|
+
wksp->bitsToWeight[0] = 0;
|
149
130
|
for (n=1; n<huffLog+1; n++)
|
150
|
-
bitsToWeight[n] = (BYTE)(huffLog + 1 - n);
|
131
|
+
wksp->bitsToWeight[n] = (BYTE)(huffLog + 1 - n);
|
151
132
|
for (n=0; n<maxSymbolValue; n++)
|
152
|
-
huffWeight[n] = bitsToWeight[CTable[n].nbBits];
|
133
|
+
wksp->huffWeight[n] = wksp->bitsToWeight[CTable[n].nbBits];
|
153
134
|
|
154
135
|
/* attempt weights compression by FSE */
|
155
|
-
{ CHECK_V_F(hSize, HUF_compressWeights(op+1, maxDstSize-1, huffWeight, maxSymbolValue) );
|
136
|
+
{ CHECK_V_F(hSize, HUF_compressWeights(op+1, maxDstSize-1, wksp->huffWeight, maxSymbolValue, &wksp->wksp, sizeof(wksp->wksp)) );
|
156
137
|
if ((hSize>1) & (hSize < maxSymbolValue/2)) { /* FSE compressed */
|
157
138
|
op[0] = (BYTE)hSize;
|
158
139
|
return hSize+1;
|
@@ -162,14 +143,24 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize,
|
|
162
143
|
if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */
|
163
144
|
if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */
|
164
145
|
op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1));
|
165
|
-
huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */
|
146
|
+
wksp->huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */
|
166
147
|
for (n=0; n<maxSymbolValue; n+=2)
|
167
|
-
op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]);
|
148
|
+
op[(n/2)+1] = (BYTE)((wksp->huffWeight[n] << 4) + wksp->huffWeight[n+1]);
|
168
149
|
return ((maxSymbolValue+1)/2) + 1;
|
169
150
|
}
|
170
151
|
|
152
|
+
/*! HUF_writeCTable() :
|
153
|
+
`CTable` : Huffman tree to save, using huf representation.
|
154
|
+
@return : size of saved CTable */
|
155
|
+
size_t HUF_writeCTable (void* dst, size_t maxDstSize,
|
156
|
+
const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog)
|
157
|
+
{
|
158
|
+
HUF_WriteCTableWksp wksp;
|
159
|
+
return HUF_writeCTable_wksp(dst, maxDstSize, CTable, maxSymbolValue, huffLog, &wksp, sizeof(wksp));
|
160
|
+
}
|
161
|
+
|
171
162
|
|
172
|
-
size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize)
|
163
|
+
size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* hasZeroWeights)
|
173
164
|
{
|
174
165
|
BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; /* init not required, even though some static analyzer may complain */
|
175
166
|
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
|
@@ -178,6 +169,7 @@ size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void
|
|
178
169
|
|
179
170
|
/* get symbol weights */
|
180
171
|
CHECK_V_F(readSize, HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize));
|
172
|
+
*hasZeroWeights = (rankVal[0] > 0);
|
181
173
|
|
182
174
|
/* check result */
|
183
175
|
if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
|
@@ -186,15 +178,15 @@ size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void
|
|
186
178
|
/* Prepare base value per rank */
|
187
179
|
{ U32 n, nextRankStart = 0;
|
188
180
|
for (n=1; n<=tableLog; n++) {
|
189
|
-
U32
|
181
|
+
U32 curr = nextRankStart;
|
190
182
|
nextRankStart += (rankVal[n] << (n-1));
|
191
|
-
rankVal[n] =
|
183
|
+
rankVal[n] = curr;
|
192
184
|
} }
|
193
185
|
|
194
186
|
/* fill nbBits */
|
195
187
|
{ U32 n; for (n=0; n<nbSymbols; n++) {
|
196
188
|
const U32 w = huffWeight[n];
|
197
|
-
CTable[n].nbBits = (BYTE)(tableLog + 1 - w);
|
189
|
+
CTable[n].nbBits = (BYTE)(tableLog + 1 - w) & -(w != 0);
|
198
190
|
} }
|
199
191
|
|
200
192
|
/* fill val */
|
@@ -232,106 +224,202 @@ typedef struct nodeElt_s {
|
|
232
224
|
BYTE nbBits;
|
233
225
|
} nodeElt;
|
234
226
|
|
227
|
+
/**
|
228
|
+
* HUF_setMaxHeight():
|
229
|
+
* Enforces maxNbBits on the Huffman tree described in huffNode.
|
230
|
+
*
|
231
|
+
* It sets all nodes with nbBits > maxNbBits to be maxNbBits. Then it adjusts
|
232
|
+
* the tree to so that it is a valid canonical Huffman tree.
|
233
|
+
*
|
234
|
+
* @pre The sum of the ranks of each symbol == 2^largestBits,
|
235
|
+
* where largestBits == huffNode[lastNonNull].nbBits.
|
236
|
+
* @post The sum of the ranks of each symbol == 2^largestBits,
|
237
|
+
* where largestBits is the return value <= maxNbBits.
|
238
|
+
*
|
239
|
+
* @param huffNode The Huffman tree modified in place to enforce maxNbBits.
|
240
|
+
* @param lastNonNull The symbol with the lowest count in the Huffman tree.
|
241
|
+
* @param maxNbBits The maximum allowed number of bits, which the Huffman tree
|
242
|
+
* may not respect. After this function the Huffman tree will
|
243
|
+
* respect maxNbBits.
|
244
|
+
* @return The maximum number of bits of the Huffman tree after adjustment,
|
245
|
+
* necessarily no more than maxNbBits.
|
246
|
+
*/
|
235
247
|
static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
|
236
248
|
{
|
237
249
|
const U32 largestBits = huffNode[lastNonNull].nbBits;
|
238
|
-
|
250
|
+
/* early exit : no elt > maxNbBits, so the tree is already valid. */
|
251
|
+
if (largestBits <= maxNbBits) return largestBits;
|
239
252
|
|
240
253
|
/* there are several too large elements (at least >= 2) */
|
241
254
|
{ int totalCost = 0;
|
242
255
|
const U32 baseCost = 1 << (largestBits - maxNbBits);
|
243
|
-
|
256
|
+
int n = (int)lastNonNull;
|
244
257
|
|
258
|
+
/* Adjust any ranks > maxNbBits to maxNbBits.
|
259
|
+
* Compute totalCost, which is how far the sum of the ranks is
|
260
|
+
* we are over 2^largestBits after adjust the offending ranks.
|
261
|
+
*/
|
245
262
|
while (huffNode[n].nbBits > maxNbBits) {
|
246
263
|
totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits));
|
247
264
|
huffNode[n].nbBits = (BYTE)maxNbBits;
|
248
|
-
n
|
249
|
-
}
|
250
|
-
|
265
|
+
n--;
|
266
|
+
}
|
267
|
+
/* n stops at huffNode[n].nbBits <= maxNbBits */
|
268
|
+
assert(huffNode[n].nbBits <= maxNbBits);
|
269
|
+
/* n end at index of smallest symbol using < maxNbBits */
|
270
|
+
while (huffNode[n].nbBits == maxNbBits) --n;
|
251
271
|
|
252
|
-
/* renorm totalCost
|
253
|
-
|
272
|
+
/* renorm totalCost from 2^largestBits to 2^maxNbBits
|
273
|
+
* note : totalCost is necessarily a multiple of baseCost */
|
274
|
+
assert((totalCost & (baseCost - 1)) == 0);
|
275
|
+
totalCost >>= (largestBits - maxNbBits);
|
276
|
+
assert(totalCost > 0);
|
254
277
|
|
255
278
|
/* repay normalized cost */
|
256
279
|
{ U32 const noSymbol = 0xF0F0F0F0;
|
257
280
|
U32 rankLast[HUF_TABLELOG_MAX+2];
|
258
|
-
int pos;
|
259
281
|
|
260
|
-
/* Get pos of last (smallest) symbol per rank */
|
261
|
-
|
282
|
+
/* Get pos of last (smallest = lowest cum. count) symbol per rank */
|
283
|
+
ZSTD_memset(rankLast, 0xF0, sizeof(rankLast));
|
262
284
|
{ U32 currentNbBits = maxNbBits;
|
285
|
+
int pos;
|
263
286
|
for (pos=n ; pos >= 0; pos--) {
|
264
287
|
if (huffNode[pos].nbBits >= currentNbBits) continue;
|
265
288
|
currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */
|
266
|
-
rankLast[maxNbBits-currentNbBits] = pos;
|
289
|
+
rankLast[maxNbBits-currentNbBits] = (U32)pos;
|
267
290
|
} }
|
268
291
|
|
269
292
|
while (totalCost > 0) {
|
270
|
-
|
293
|
+
/* Try to reduce the next power of 2 above totalCost because we
|
294
|
+
* gain back half the rank.
|
295
|
+
*/
|
296
|
+
U32 nBitsToDecrease = BIT_highbit32((U32)totalCost) + 1;
|
271
297
|
for ( ; nBitsToDecrease > 1; nBitsToDecrease--) {
|
272
|
-
U32 highPos = rankLast[nBitsToDecrease];
|
273
|
-
U32 lowPos = rankLast[nBitsToDecrease-1];
|
298
|
+
U32 const highPos = rankLast[nBitsToDecrease];
|
299
|
+
U32 const lowPos = rankLast[nBitsToDecrease-1];
|
274
300
|
if (highPos == noSymbol) continue;
|
301
|
+
/* Decrease highPos if no symbols of lowPos or if it is
|
302
|
+
* not cheaper to remove 2 lowPos than highPos.
|
303
|
+
*/
|
275
304
|
if (lowPos == noSymbol) break;
|
276
305
|
{ U32 const highTotal = huffNode[highPos].count;
|
277
306
|
U32 const lowTotal = 2 * huffNode[lowPos].count;
|
278
307
|
if (highTotal <= lowTotal) break;
|
279
308
|
} }
|
280
309
|
/* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */
|
310
|
+
assert(rankLast[nBitsToDecrease] != noSymbol || nBitsToDecrease == 1);
|
281
311
|
/* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */
|
282
312
|
while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol))
|
283
|
-
nBitsToDecrease
|
313
|
+
nBitsToDecrease++;
|
314
|
+
assert(rankLast[nBitsToDecrease] != noSymbol);
|
315
|
+
/* Increase the number of bits to gain back half the rank cost. */
|
284
316
|
totalCost -= 1 << (nBitsToDecrease-1);
|
317
|
+
huffNode[rankLast[nBitsToDecrease]].nbBits++;
|
318
|
+
|
319
|
+
/* Fix up the new rank.
|
320
|
+
* If the new rank was empty, this symbol is now its smallest.
|
321
|
+
* Otherwise, this symbol will be the largest in the new rank so no adjustment.
|
322
|
+
*/
|
285
323
|
if (rankLast[nBitsToDecrease-1] == noSymbol)
|
286
|
-
rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease];
|
287
|
-
|
324
|
+
rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease];
|
325
|
+
/* Fix up the old rank.
|
326
|
+
* If the symbol was at position 0, meaning it was the highest weight symbol in the tree,
|
327
|
+
* it must be the only symbol in its rank, so the old rank now has no symbols.
|
328
|
+
* Otherwise, since the Huffman nodes are sorted by count, the previous position is now
|
329
|
+
* the smallest node in the rank. If the previous position belongs to a different rank,
|
330
|
+
* then the rank is now empty.
|
331
|
+
*/
|
288
332
|
if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */
|
289
333
|
rankLast[nBitsToDecrease] = noSymbol;
|
290
334
|
else {
|
291
335
|
rankLast[nBitsToDecrease]--;
|
292
336
|
if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease)
|
293
337
|
rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */
|
294
|
-
|
295
|
-
|
338
|
+
}
|
339
|
+
} /* while (totalCost > 0) */
|
340
|
+
|
341
|
+
/* If we've removed too much weight, then we have to add it back.
|
342
|
+
* To avoid overshooting again, we only adjust the smallest rank.
|
343
|
+
* We take the largest nodes from the lowest rank 0 and move them
|
344
|
+
* to rank 1. There's guaranteed to be enough rank 0 symbols because
|
345
|
+
* TODO.
|
346
|
+
*/
|
296
347
|
while (totalCost < 0) { /* Sometimes, cost correction overshoot */
|
297
|
-
|
348
|
+
/* special case : no rank 1 symbol (using maxNbBits-1);
|
349
|
+
* let's create one from largest rank 0 (using maxNbBits).
|
350
|
+
*/
|
351
|
+
if (rankLast[1] == noSymbol) {
|
298
352
|
while (huffNode[n].nbBits == maxNbBits) n--;
|
299
353
|
huffNode[n+1].nbBits--;
|
300
|
-
|
354
|
+
assert(n >= 0);
|
355
|
+
rankLast[1] = (U32)(n+1);
|
301
356
|
totalCost++;
|
302
357
|
continue;
|
303
358
|
}
|
304
359
|
huffNode[ rankLast[1] + 1 ].nbBits--;
|
305
360
|
rankLast[1]++;
|
306
361
|
totalCost ++;
|
307
|
-
|
362
|
+
}
|
363
|
+
} /* repay normalized cost */
|
364
|
+
} /* there are several too large elements (at least >= 2) */
|
308
365
|
|
309
366
|
return maxNbBits;
|
310
367
|
}
|
311
368
|
|
312
|
-
|
313
369
|
typedef struct {
|
314
370
|
U32 base;
|
315
|
-
U32
|
371
|
+
U32 curr;
|
316
372
|
} rankPos;
|
317
373
|
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
U32 n;
|
374
|
+
typedef nodeElt huffNodeTable[HUF_CTABLE_WORKSPACE_SIZE_U32];
|
375
|
+
|
376
|
+
#define RANK_POSITION_TABLE_SIZE 32
|
322
377
|
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
378
|
+
typedef struct {
|
379
|
+
huffNodeTable huffNodeTbl;
|
380
|
+
rankPos rankPosition[RANK_POSITION_TABLE_SIZE];
|
381
|
+
} HUF_buildCTable_wksp_tables;
|
382
|
+
|
383
|
+
/**
|
384
|
+
* HUF_sort():
|
385
|
+
* Sorts the symbols [0, maxSymbolValue] by count[symbol] in decreasing order.
|
386
|
+
*
|
387
|
+
* @param[out] huffNode Sorted symbols by decreasing count. Only members `.count` and `.byte` are filled.
|
388
|
+
* Must have (maxSymbolValue + 1) entries.
|
389
|
+
* @param[in] count Histogram of the symbols.
|
390
|
+
* @param[in] maxSymbolValue Maximum symbol value.
|
391
|
+
* @param rankPosition This is a scratch workspace. Must have RANK_POSITION_TABLE_SIZE entries.
|
392
|
+
*/
|
393
|
+
static void HUF_sort(nodeElt* huffNode, const unsigned* count, U32 maxSymbolValue, rankPos* rankPosition)
|
394
|
+
{
|
395
|
+
int n;
|
396
|
+
int const maxSymbolValue1 = (int)maxSymbolValue + 1;
|
397
|
+
|
398
|
+
/* Compute base and set curr to base.
|
399
|
+
* For symbol s let lowerRank = BIT_highbit32(count[n]+1) and rank = lowerRank + 1.
|
400
|
+
* Then 2^lowerRank <= count[n]+1 <= 2^rank.
|
401
|
+
* We attribute each symbol to lowerRank's base value, because we want to know where
|
402
|
+
* each rank begins in the output, so for rank R we want to count ranks R+1 and above.
|
403
|
+
*/
|
404
|
+
ZSTD_memset(rankPosition, 0, sizeof(*rankPosition) * RANK_POSITION_TABLE_SIZE);
|
405
|
+
for (n = 0; n < maxSymbolValue1; ++n) {
|
406
|
+
U32 lowerRank = BIT_highbit32(count[n] + 1);
|
407
|
+
rankPosition[lowerRank].base++;
|
408
|
+
}
|
409
|
+
assert(rankPosition[RANK_POSITION_TABLE_SIZE - 1].base == 0);
|
410
|
+
for (n = RANK_POSITION_TABLE_SIZE - 1; n > 0; --n) {
|
411
|
+
rankPosition[n-1].base += rankPosition[n].base;
|
412
|
+
rankPosition[n-1].curr = rankPosition[n-1].base;
|
327
413
|
}
|
328
|
-
|
329
|
-
for (n=0; n<
|
330
|
-
for (n=0; n<=maxSymbolValue; n++) {
|
414
|
+
/* Sort */
|
415
|
+
for (n = 0; n < maxSymbolValue1; ++n) {
|
331
416
|
U32 const c = count[n];
|
332
417
|
U32 const r = BIT_highbit32(c+1) + 1;
|
333
|
-
U32 pos =
|
334
|
-
|
418
|
+
U32 pos = rankPosition[r].curr++;
|
419
|
+
/* Insert into the correct position in the rank.
|
420
|
+
* We have at most 256 symbols, so this insertion should be fine.
|
421
|
+
*/
|
422
|
+
while ((pos > rankPosition[r].base) && (c > huffNode[pos-1].count)) {
|
335
423
|
huffNode[pos] = huffNode[pos-1];
|
336
424
|
pos--;
|
337
425
|
}
|
@@ -343,45 +431,40 @@ static void HUF_sort(nodeElt* huffNode, const unsigned* count, U32 maxSymbolValu
|
|
343
431
|
|
344
432
|
/** HUF_buildCTable_wksp() :
|
345
433
|
* Same as HUF_buildCTable(), but using externally allocated scratch buffer.
|
346
|
-
* `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as
|
434
|
+
* `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as sizeof(HUF_buildCTable_wksp_tables).
|
347
435
|
*/
|
348
436
|
#define STARTNODE (HUF_SYMBOLVALUE_MAX+1)
|
349
|
-
|
350
|
-
|
437
|
+
|
438
|
+
/* HUF_buildTree():
|
439
|
+
* Takes the huffNode array sorted by HUF_sort() and builds an unlimited-depth Huffman tree.
|
440
|
+
*
|
441
|
+
* @param huffNode The array sorted by HUF_sort(). Builds the Huffman tree in this array.
|
442
|
+
* @param maxSymbolValue The maximum symbol value.
|
443
|
+
* @return The smallest node in the Huffman tree (by count).
|
444
|
+
*/
|
445
|
+
static int HUF_buildTree(nodeElt* huffNode, U32 maxSymbolValue)
|
351
446
|
{
|
352
|
-
nodeElt* const huffNode0 =
|
353
|
-
|
354
|
-
U32 n, nonNullRank;
|
447
|
+
nodeElt* const huffNode0 = huffNode - 1;
|
448
|
+
int nonNullRank;
|
355
449
|
int lowS, lowN;
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
/* safety checks */
|
360
|
-
if (((size_t)workSpace & 3) != 0) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */
|
361
|
-
if (wkspSize < sizeof(huffNodeTable)) return ERROR(workSpace_tooSmall);
|
362
|
-
if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT;
|
363
|
-
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge);
|
364
|
-
memset(huffNode0, 0, sizeof(huffNodeTable));
|
365
|
-
|
366
|
-
/* sort, decreasing order */
|
367
|
-
HUF_sort(huffNode, count, maxSymbolValue);
|
368
|
-
|
450
|
+
int nodeNb = STARTNODE;
|
451
|
+
int n, nodeRoot;
|
369
452
|
/* init for parents */
|
370
|
-
nonNullRank = maxSymbolValue;
|
453
|
+
nonNullRank = (int)maxSymbolValue;
|
371
454
|
while(huffNode[nonNullRank].count == 0) nonNullRank--;
|
372
455
|
lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb;
|
373
456
|
huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count;
|
374
|
-
huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb;
|
457
|
+
huffNode[lowS].parent = huffNode[lowS-1].parent = (U16)nodeNb;
|
375
458
|
nodeNb++; lowS-=2;
|
376
459
|
for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30);
|
377
460
|
huffNode0[0].count = (U32)(1U<<31); /* fake entry, strong barrier */
|
378
461
|
|
379
462
|
/* create parents */
|
380
463
|
while (nodeNb <= nodeRoot) {
|
381
|
-
|
382
|
-
|
464
|
+
int const n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
|
465
|
+
int const n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
|
383
466
|
huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count;
|
384
|
-
huffNode[n1].parent = huffNode[n2].parent = nodeNb;
|
467
|
+
huffNode[n1].parent = huffNode[n2].parent = (U16)nodeNb;
|
385
468
|
nodeNb++;
|
386
469
|
}
|
387
470
|
|
@@ -392,42 +475,73 @@ size_t HUF_buildCTable_wksp (HUF_CElt* tree, const unsigned* count, U32 maxSymbo
|
|
392
475
|
for (n=0; n<=nonNullRank; n++)
|
393
476
|
huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
|
394
477
|
|
395
|
-
|
396
|
-
maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits);
|
397
|
-
|
398
|
-
/* fill result into tree (val, nbBits) */
|
399
|
-
{ U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0};
|
400
|
-
U16 valPerRank[HUF_TABLELOG_MAX+1] = {0};
|
401
|
-
if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */
|
402
|
-
for (n=0; n<=nonNullRank; n++)
|
403
|
-
nbPerRank[huffNode[n].nbBits]++;
|
404
|
-
/* determine stating value per rank */
|
405
|
-
{ U16 min = 0;
|
406
|
-
for (n=maxNbBits; n>0; n--) {
|
407
|
-
valPerRank[n] = min; /* get starting value within each rank */
|
408
|
-
min += nbPerRank[n];
|
409
|
-
min >>= 1;
|
410
|
-
} }
|
411
|
-
for (n=0; n<=maxSymbolValue; n++)
|
412
|
-
tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */
|
413
|
-
for (n=0; n<=maxSymbolValue; n++)
|
414
|
-
tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */
|
415
|
-
}
|
416
|
-
|
417
|
-
return maxNbBits;
|
478
|
+
return nonNullRank;
|
418
479
|
}
|
419
480
|
|
420
|
-
/**
|
421
|
-
*
|
422
|
-
*
|
481
|
+
/**
|
482
|
+
* HUF_buildCTableFromTree():
|
483
|
+
* Build the CTable given the Huffman tree in huffNode.
|
484
|
+
*
|
485
|
+
* @param[out] CTable The output Huffman CTable.
|
486
|
+
* @param huffNode The Huffman tree.
|
487
|
+
* @param nonNullRank The last and smallest node in the Huffman tree.
|
488
|
+
* @param maxSymbolValue The maximum symbol value.
|
489
|
+
* @param maxNbBits The exact maximum number of bits used in the Huffman tree.
|
423
490
|
*/
|
424
|
-
|
491
|
+
static void HUF_buildCTableFromTree(HUF_CElt* CTable, nodeElt const* huffNode, int nonNullRank, U32 maxSymbolValue, U32 maxNbBits)
|
425
492
|
{
|
426
|
-
|
427
|
-
|
493
|
+
/* fill result into ctable (val, nbBits) */
|
494
|
+
int n;
|
495
|
+
U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0};
|
496
|
+
U16 valPerRank[HUF_TABLELOG_MAX+1] = {0};
|
497
|
+
int const alphabetSize = (int)(maxSymbolValue + 1);
|
498
|
+
for (n=0; n<=nonNullRank; n++)
|
499
|
+
nbPerRank[huffNode[n].nbBits]++;
|
500
|
+
/* determine starting value per rank */
|
501
|
+
{ U16 min = 0;
|
502
|
+
for (n=(int)maxNbBits; n>0; n--) {
|
503
|
+
valPerRank[n] = min; /* get starting value within each rank */
|
504
|
+
min += nbPerRank[n];
|
505
|
+
min >>= 1;
|
506
|
+
} }
|
507
|
+
for (n=0; n<alphabetSize; n++)
|
508
|
+
CTable[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */
|
509
|
+
for (n=0; n<alphabetSize; n++)
|
510
|
+
CTable[n].val = valPerRank[CTable[n].nbBits]++; /* assign value within rank, symbol order */
|
428
511
|
}
|
429
512
|
|
430
|
-
|
513
|
+
size_t HUF_buildCTable_wksp (HUF_CElt* tree, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize)
|
514
|
+
{
|
515
|
+
HUF_buildCTable_wksp_tables* const wksp_tables = (HUF_buildCTable_wksp_tables*)workSpace;
|
516
|
+
nodeElt* const huffNode0 = wksp_tables->huffNodeTbl;
|
517
|
+
nodeElt* const huffNode = huffNode0+1;
|
518
|
+
int nonNullRank;
|
519
|
+
|
520
|
+
/* safety checks */
|
521
|
+
if (((size_t)workSpace & 3) != 0) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */
|
522
|
+
if (wkspSize < sizeof(HUF_buildCTable_wksp_tables))
|
523
|
+
return ERROR(workSpace_tooSmall);
|
524
|
+
if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT;
|
525
|
+
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX)
|
526
|
+
return ERROR(maxSymbolValue_tooLarge);
|
527
|
+
ZSTD_memset(huffNode0, 0, sizeof(huffNodeTable));
|
528
|
+
|
529
|
+
/* sort, decreasing order */
|
530
|
+
HUF_sort(huffNode, count, maxSymbolValue, wksp_tables->rankPosition);
|
531
|
+
|
532
|
+
/* build tree */
|
533
|
+
nonNullRank = HUF_buildTree(huffNode, maxSymbolValue);
|
534
|
+
|
535
|
+
/* enforce maxTableLog */
|
536
|
+
maxNbBits = HUF_setMaxHeight(huffNode, (U32)nonNullRank, maxNbBits);
|
537
|
+
if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */
|
538
|
+
|
539
|
+
HUF_buildCTableFromTree(tree, huffNode, nonNullRank, maxSymbolValue, maxNbBits);
|
540
|
+
|
541
|
+
return maxNbBits;
|
542
|
+
}
|
543
|
+
|
544
|
+
size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue)
|
431
545
|
{
|
432
546
|
size_t nbBits = 0;
|
433
547
|
int s;
|
@@ -437,7 +551,7 @@ static size_t HUF_estimateCompressedSize(HUF_CElt* CTable, const unsigned* count
|
|
437
551
|
return nbBits >> 3;
|
438
552
|
}
|
439
553
|
|
440
|
-
|
554
|
+
int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) {
|
441
555
|
int bad = 0;
|
442
556
|
int s;
|
443
557
|
for (s = 0; s <= (int)maxSymbolValue; ++s) {
|
@@ -476,7 +590,7 @@ HUF_compress1X_usingCTable_internal_body(void* dst, size_t dstSize,
|
|
476
590
|
|
477
591
|
/* init */
|
478
592
|
if (dstSize < 8) return 0; /* not enough space to compress */
|
479
|
-
{ size_t const initErr = BIT_initCStream(&bitC, op, oend-op);
|
593
|
+
{ size_t const initErr = BIT_initCStream(&bitC, op, (size_t)(oend-op));
|
480
594
|
if (HUF_isError(initErr)) return 0; }
|
481
595
|
|
482
596
|
n = srcSize & ~3; /* join to mod 4 */
|
@@ -573,7 +687,8 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
|
|
573
687
|
if (srcSize < 12) return 0; /* no saving possible : too small input */
|
574
688
|
op += 6; /* jumpTable */
|
575
689
|
|
576
|
-
|
690
|
+
assert(op <= oend);
|
691
|
+
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) );
|
577
692
|
if (cSize==0) return 0;
|
578
693
|
assert(cSize <= 65535);
|
579
694
|
MEM_writeLE16(ostart, (U16)cSize);
|
@@ -581,7 +696,8 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
|
|
581
696
|
}
|
582
697
|
|
583
698
|
ip += segmentSize;
|
584
|
-
|
699
|
+
assert(op <= oend);
|
700
|
+
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) );
|
585
701
|
if (cSize==0) return 0;
|
586
702
|
assert(cSize <= 65535);
|
587
703
|
MEM_writeLE16(ostart+2, (U16)cSize);
|
@@ -589,7 +705,8 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
|
|
589
705
|
}
|
590
706
|
|
591
707
|
ip += segmentSize;
|
592
|
-
|
708
|
+
assert(op <= oend);
|
709
|
+
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) );
|
593
710
|
if (cSize==0) return 0;
|
594
711
|
assert(cSize <= 65535);
|
595
712
|
MEM_writeLE16(ostart+4, (U16)cSize);
|
@@ -597,12 +714,14 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
|
|
597
714
|
}
|
598
715
|
|
599
716
|
ip += segmentSize;
|
600
|
-
|
717
|
+
assert(op <= oend);
|
718
|
+
assert(ip <= iend);
|
719
|
+
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, (size_t)(iend-ip), CTable, bmi2) );
|
601
720
|
if (cSize==0) return 0;
|
602
721
|
op += cSize;
|
603
722
|
}
|
604
723
|
|
605
|
-
return op-ostart;
|
724
|
+
return (size_t)(op-ostart);
|
606
725
|
}
|
607
726
|
|
608
727
|
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
|
@@ -618,40 +737,47 @@ static size_t HUF_compressCTable_internal(
|
|
618
737
|
HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int bmi2)
|
619
738
|
{
|
620
739
|
size_t const cSize = (nbStreams==HUF_singleStream) ?
|
621
|
-
HUF_compress1X_usingCTable_internal(op, oend - op, src, srcSize, CTable, bmi2) :
|
622
|
-
HUF_compress4X_usingCTable_internal(op, oend - op, src, srcSize, CTable, bmi2);
|
740
|
+
HUF_compress1X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, bmi2) :
|
741
|
+
HUF_compress4X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, bmi2);
|
623
742
|
if (HUF_isError(cSize)) { return cSize; }
|
624
743
|
if (cSize==0) { return 0; } /* uncompressible */
|
625
744
|
op += cSize;
|
626
745
|
/* check compressibility */
|
746
|
+
assert(op >= ostart);
|
627
747
|
if ((size_t)(op-ostart) >= srcSize-1) { return 0; }
|
628
|
-
return op-ostart;
|
748
|
+
return (size_t)(op-ostart);
|
629
749
|
}
|
630
750
|
|
631
751
|
typedef struct {
|
632
752
|
unsigned count[HUF_SYMBOLVALUE_MAX + 1];
|
633
753
|
HUF_CElt CTable[HUF_SYMBOLVALUE_MAX + 1];
|
634
|
-
|
754
|
+
union {
|
755
|
+
HUF_buildCTable_wksp_tables buildCTable_wksp;
|
756
|
+
HUF_WriteCTableWksp writeCTable_wksp;
|
757
|
+
} wksps;
|
635
758
|
} HUF_compress_tables_t;
|
636
759
|
|
637
760
|
/* HUF_compress_internal() :
|
638
|
-
* `
|
761
|
+
* `workSpace_align4` must be aligned on 4-bytes boundaries,
|
762
|
+
* and occupies the same space as a table of HUF_WORKSPACE_SIZE_U32 unsigned */
|
639
763
|
static size_t
|
640
764
|
HUF_compress_internal (void* dst, size_t dstSize,
|
641
765
|
const void* src, size_t srcSize,
|
642
766
|
unsigned maxSymbolValue, unsigned huffLog,
|
643
767
|
HUF_nbStreams_e nbStreams,
|
644
|
-
void*
|
768
|
+
void* workSpace_align4, size_t wkspSize,
|
645
769
|
HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat,
|
646
770
|
const int bmi2)
|
647
771
|
{
|
648
|
-
HUF_compress_tables_t* const table = (HUF_compress_tables_t*)
|
772
|
+
HUF_compress_tables_t* const table = (HUF_compress_tables_t*)workSpace_align4;
|
649
773
|
BYTE* const ostart = (BYTE*)dst;
|
650
774
|
BYTE* const oend = ostart + dstSize;
|
651
775
|
BYTE* op = ostart;
|
652
776
|
|
777
|
+
HUF_STATIC_ASSERT(sizeof(*table) <= HUF_WORKSPACE_SIZE);
|
778
|
+
assert(((size_t)workSpace_align4 & 3) == 0); /* must be aligned on 4-bytes boundaries */
|
779
|
+
|
653
780
|
/* checks & inits */
|
654
|
-
if (((size_t)workSpace & 3) != 0) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */
|
655
781
|
if (wkspSize < HUF_WORKSPACE_SIZE) return ERROR(workSpace_tooSmall);
|
656
782
|
if (!srcSize) return 0; /* Uncompressed */
|
657
783
|
if (!dstSize) return 0; /* cannot fit anything within dst budget */
|
@@ -669,7 +795,7 @@ HUF_compress_internal (void* dst, size_t dstSize,
|
|
669
795
|
}
|
670
796
|
|
671
797
|
/* Scan input and build symbol stats */
|
672
|
-
{ CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize,
|
798
|
+
{ CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize, workSpace_align4, wkspSize) );
|
673
799
|
if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */
|
674
800
|
if (largest <= (srcSize >> 7)+4) return 0; /* heuristic : probably not compressible enough */
|
675
801
|
}
|
@@ -691,16 +817,17 @@ HUF_compress_internal (void* dst, size_t dstSize,
|
|
691
817
|
huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
|
692
818
|
{ size_t const maxBits = HUF_buildCTable_wksp(table->CTable, table->count,
|
693
819
|
maxSymbolValue, huffLog,
|
694
|
-
table->
|
820
|
+
&table->wksps.buildCTable_wksp, sizeof(table->wksps.buildCTable_wksp));
|
695
821
|
CHECK_F(maxBits);
|
696
822
|
huffLog = (U32)maxBits;
|
697
823
|
/* Zero unused symbols in CTable, so we can check it for validity */
|
698
|
-
|
824
|
+
ZSTD_memset(table->CTable + (maxSymbolValue + 1), 0,
|
699
825
|
sizeof(table->CTable) - ((maxSymbolValue + 1) * sizeof(HUF_CElt)));
|
700
826
|
}
|
701
827
|
|
702
828
|
/* Write table description header */
|
703
|
-
{ CHECK_V_F(hSize,
|
829
|
+
{ CHECK_V_F(hSize, HUF_writeCTable_wksp(op, dstSize, table->CTable, maxSymbolValue, huffLog,
|
830
|
+
&table->wksps.writeCTable_wksp, sizeof(table->wksps.writeCTable_wksp)) );
|
704
831
|
/* Check if using previous huffman table is beneficial */
|
705
832
|
if (repeat && *repeat != HUF_repeat_none) {
|
706
833
|
size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, table->count, maxSymbolValue);
|
@@ -716,7 +843,7 @@ HUF_compress_internal (void* dst, size_t dstSize,
|
|
716
843
|
op += hSize;
|
717
844
|
if (repeat) { *repeat = HUF_repeat_none; }
|
718
845
|
if (oldHufTable)
|
719
|
-
|
846
|
+
ZSTD_memcpy(oldHufTable, table->CTable, sizeof(table->CTable)); /* Save new table */
|
720
847
|
}
|
721
848
|
return HUF_compressCTable_internal(ostart, op, oend,
|
722
849
|
src, srcSize,
|
@@ -747,14 +874,6 @@ size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
|
|
747
874
|
repeat, preferRepeat, bmi2);
|
748
875
|
}
|
749
876
|
|
750
|
-
size_t HUF_compress1X (void* dst, size_t dstSize,
|
751
|
-
const void* src, size_t srcSize,
|
752
|
-
unsigned maxSymbolValue, unsigned huffLog)
|
753
|
-
{
|
754
|
-
unsigned workSpace[HUF_WORKSPACE_SIZE_U32];
|
755
|
-
return HUF_compress1X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace));
|
756
|
-
}
|
757
|
-
|
758
877
|
/* HUF_compress4X_repeat():
|
759
878
|
* compress input using 4 streams.
|
760
879
|
* provide workspace to generate compression tables */
|
@@ -784,6 +903,25 @@ size_t HUF_compress4X_repeat (void* dst, size_t dstSize,
|
|
784
903
|
hufTable, repeat, preferRepeat, bmi2);
|
785
904
|
}
|
786
905
|
|
906
|
+
#ifndef ZSTD_NO_UNUSED_FUNCTIONS
|
907
|
+
/** HUF_buildCTable() :
|
908
|
+
* @return : maxNbBits
|
909
|
+
* Note : count is used before tree is written, so they can safely overlap
|
910
|
+
*/
|
911
|
+
size_t HUF_buildCTable (HUF_CElt* tree, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits)
|
912
|
+
{
|
913
|
+
HUF_buildCTable_wksp_tables workspace;
|
914
|
+
return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, &workspace, sizeof(workspace));
|
915
|
+
}
|
916
|
+
|
917
|
+
size_t HUF_compress1X (void* dst, size_t dstSize,
|
918
|
+
const void* src, size_t srcSize,
|
919
|
+
unsigned maxSymbolValue, unsigned huffLog)
|
920
|
+
{
|
921
|
+
unsigned workSpace[HUF_WORKSPACE_SIZE_U32];
|
922
|
+
return HUF_compress1X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace));
|
923
|
+
}
|
924
|
+
|
787
925
|
size_t HUF_compress2 (void* dst, size_t dstSize,
|
788
926
|
const void* src, size_t srcSize,
|
789
927
|
unsigned maxSymbolValue, unsigned huffLog)
|
@@ -796,3 +934,4 @@ size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSi
|
|
796
934
|
{
|
797
935
|
return HUF_compress2(dst, maxDstSize, src, srcSize, 255, HUF_TABLELOG_DEFAULT);
|
798
936
|
}
|
937
|
+
#endif
|