extzstd 0.1 → 0.3.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/HISTORY.ja.md +39 -0
- data/README.md +38 -56
- data/contrib/zstd/CHANGELOG +613 -0
- data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
- data/contrib/zstd/CONTRIBUTING.md +406 -0
- data/contrib/zstd/COPYING +339 -0
- data/contrib/zstd/Makefile +420 -0
- data/contrib/zstd/README.md +179 -41
- data/contrib/zstd/TESTING.md +44 -0
- data/contrib/zstd/appveyor.yml +292 -0
- data/contrib/zstd/lib/BUCK +234 -0
- data/contrib/zstd/lib/Makefile +451 -0
- data/contrib/zstd/lib/README.md +207 -0
- data/contrib/zstd/{common → lib/common}/bitstream.h +187 -138
- data/contrib/zstd/lib/common/compiler.h +288 -0
- data/contrib/zstd/lib/common/cpu.h +213 -0
- data/contrib/zstd/lib/common/debug.c +24 -0
- data/contrib/zstd/lib/common/debug.h +107 -0
- data/contrib/zstd/lib/common/entropy_common.c +362 -0
- data/contrib/zstd/{common → lib/common}/error_private.c +25 -12
- data/contrib/zstd/{common → lib/common}/error_private.h +14 -10
- data/contrib/zstd/{common → lib/common}/fse.h +173 -92
- data/contrib/zstd/{common → lib/common}/fse_decompress.c +149 -85
- data/contrib/zstd/lib/common/huf.h +361 -0
- data/contrib/zstd/{common → lib/common}/mem.h +115 -59
- data/contrib/zstd/lib/common/pool.c +350 -0
- data/contrib/zstd/lib/common/pool.h +84 -0
- data/contrib/zstd/lib/common/threading.c +122 -0
- data/contrib/zstd/lib/common/threading.h +155 -0
- data/contrib/zstd/{common → lib/common}/xxhash.c +55 -96
- data/contrib/zstd/{common → lib/common}/xxhash.h +23 -47
- data/contrib/zstd/lib/common/zstd_common.c +83 -0
- data/contrib/zstd/lib/common/zstd_deps.h +111 -0
- data/contrib/zstd/lib/common/zstd_errors.h +95 -0
- data/contrib/zstd/lib/common/zstd_internal.h +478 -0
- data/contrib/zstd/{compress → lib/compress}/fse_compress.c +214 -319
- data/contrib/zstd/lib/compress/hist.c +181 -0
- data/contrib/zstd/lib/compress/hist.h +75 -0
- data/contrib/zstd/lib/compress/huf_compress.c +913 -0
- data/contrib/zstd/lib/compress/zstd_compress.c +5208 -0
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +1203 -0
- data/contrib/zstd/lib/compress/zstd_compress_literals.c +158 -0
- data/contrib/zstd/lib/compress/zstd_compress_literals.h +29 -0
- data/contrib/zstd/lib/compress/zstd_compress_sequences.c +433 -0
- data/contrib/zstd/lib/compress/zstd_compress_sequences.h +54 -0
- data/contrib/zstd/lib/compress/zstd_compress_superblock.c +849 -0
- data/contrib/zstd/lib/compress/zstd_compress_superblock.h +32 -0
- data/contrib/zstd/lib/compress/zstd_cwksp.h +561 -0
- data/contrib/zstd/lib/compress/zstd_double_fast.c +521 -0
- data/contrib/zstd/lib/compress/zstd_double_fast.h +38 -0
- data/contrib/zstd/lib/compress/zstd_fast.c +496 -0
- data/contrib/zstd/lib/compress/zstd_fast.h +37 -0
- data/contrib/zstd/lib/compress/zstd_lazy.c +1412 -0
- data/contrib/zstd/lib/compress/zstd_lazy.h +87 -0
- data/contrib/zstd/lib/compress/zstd_ldm.c +660 -0
- data/contrib/zstd/lib/compress/zstd_ldm.h +116 -0
- data/contrib/zstd/lib/compress/zstd_opt.c +1345 -0
- data/contrib/zstd/lib/compress/zstd_opt.h +56 -0
- data/contrib/zstd/lib/compress/zstdmt_compress.c +1811 -0
- data/contrib/zstd/lib/compress/zstdmt_compress.h +110 -0
- data/contrib/zstd/lib/decompress/huf_decompress.c +1350 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.c +244 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
- data/contrib/zstd/lib/decompress/zstd_decompress.c +1930 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1540 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.h +62 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +190 -0
- data/contrib/zstd/{common → lib/deprecated}/zbuff.h +68 -45
- data/contrib/zstd/lib/deprecated/zbuff_common.c +26 -0
- data/contrib/zstd/lib/deprecated/zbuff_compress.c +147 -0
- data/contrib/zstd/lib/deprecated/zbuff_decompress.c +75 -0
- data/contrib/zstd/lib/dictBuilder/cover.c +1245 -0
- data/contrib/zstd/lib/dictBuilder/cover.h +157 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.c +3 -3
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.h +0 -0
- data/contrib/zstd/lib/dictBuilder/fastcover.c +758 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/zdict.c +318 -194
- data/contrib/zstd/lib/dictBuilder/zdict.h +305 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_legacy.h +171 -15
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.c +191 -124
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.h +19 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.c +125 -125
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.h +19 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.c +125 -124
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.h +20 -6
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.c +151 -299
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.h +19 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.c +237 -243
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.h +19 -6
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.c +130 -143
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.h +18 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.c +158 -157
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.h +19 -5
- data/contrib/zstd/lib/libzstd.pc.in +15 -0
- data/contrib/zstd/lib/zstd.h +2391 -0
- data/ext/depend +2 -0
- data/ext/extconf.rb +15 -6
- data/ext/extzstd.c +76 -145
- data/ext/extzstd.h +80 -31
- data/ext/extzstd_stream.c +417 -142
- data/ext/libzstd_conf.h +8 -0
- data/ext/zstd_common.c +10 -7
- data/ext/zstd_compress.c +14 -5
- data/ext/zstd_decompress.c +5 -4
- data/ext/zstd_dictbuilder.c +9 -4
- data/ext/zstd_dictbuilder_fastcover.c +3 -0
- data/ext/zstd_legacy_v01.c +3 -1
- data/ext/zstd_legacy_v02.c +3 -1
- data/ext/zstd_legacy_v03.c +3 -1
- data/ext/zstd_legacy_v04.c +3 -1
- data/ext/zstd_legacy_v05.c +3 -1
- data/ext/zstd_legacy_v06.c +3 -1
- data/ext/zstd_legacy_v07.c +3 -1
- data/gemstub.rb +10 -24
- data/lib/extzstd.rb +64 -179
- data/lib/extzstd/version.rb +6 -1
- data/test/test_basic.rb +9 -6
- metadata +113 -57
- data/HISTORY.ja +0 -5
- data/contrib/zstd/common/entropy_common.c +0 -225
- data/contrib/zstd/common/huf.h +0 -228
- data/contrib/zstd/common/zstd_common.c +0 -83
- data/contrib/zstd/common/zstd_errors.h +0 -60
- data/contrib/zstd/common/zstd_internal.h +0 -267
- data/contrib/zstd/compress/huf_compress.c +0 -533
- data/contrib/zstd/compress/zbuff_compress.c +0 -319
- data/contrib/zstd/compress/zstd_compress.c +0 -3264
- data/contrib/zstd/compress/zstd_opt.h +0 -900
- data/contrib/zstd/decompress/huf_decompress.c +0 -883
- data/contrib/zstd/decompress/zbuff_decompress.c +0 -252
- data/contrib/zstd/decompress/zstd_decompress.c +0 -1842
- data/contrib/zstd/dictBuilder/zdict.h +0 -111
- data/contrib/zstd/zstd.h +0 -640
@@ -1,533 +0,0 @@
|
|
1
|
-
/* ******************************************************************
|
2
|
-
Huffman encoder, part of New Generation Entropy library
|
3
|
-
Copyright (C) 2013-2016, Yann Collet.
|
4
|
-
|
5
|
-
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
6
|
-
|
7
|
-
Redistribution and use in source and binary forms, with or without
|
8
|
-
modification, are permitted provided that the following conditions are
|
9
|
-
met:
|
10
|
-
|
11
|
-
* Redistributions of source code must retain the above copyright
|
12
|
-
notice, this list of conditions and the following disclaimer.
|
13
|
-
* Redistributions in binary form must reproduce the above
|
14
|
-
copyright notice, this list of conditions and the following disclaimer
|
15
|
-
in the documentation and/or other materials provided with the
|
16
|
-
distribution.
|
17
|
-
|
18
|
-
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
19
|
-
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
20
|
-
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
21
|
-
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
22
|
-
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
23
|
-
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
24
|
-
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
25
|
-
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
26
|
-
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
27
|
-
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
28
|
-
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
29
|
-
|
30
|
-
You can contact the author at :
|
31
|
-
- FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
32
|
-
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
33
|
-
****************************************************************** */
|
34
|
-
|
35
|
-
/* **************************************************************
|
36
|
-
* Compiler specifics
|
37
|
-
****************************************************************/
|
38
|
-
#ifdef _MSC_VER /* Visual Studio */
|
39
|
-
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
40
|
-
#endif
|
41
|
-
|
42
|
-
|
43
|
-
/* **************************************************************
|
44
|
-
* Includes
|
45
|
-
****************************************************************/
|
46
|
-
#include <string.h> /* memcpy, memset */
|
47
|
-
#include <stdio.h> /* printf (debug) */
|
48
|
-
#include "bitstream.h"
|
49
|
-
#define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */
|
50
|
-
#include "fse.h" /* header compression */
|
51
|
-
#define HUF_STATIC_LINKING_ONLY
|
52
|
-
#include "huf.h"
|
53
|
-
|
54
|
-
|
55
|
-
/* **************************************************************
|
56
|
-
* Error Management
|
57
|
-
****************************************************************/
|
58
|
-
#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
|
59
|
-
|
60
|
-
|
61
|
-
/* **************************************************************
|
62
|
-
* Utils
|
63
|
-
****************************************************************/
|
64
|
-
unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
|
65
|
-
{
|
66
|
-
return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
|
67
|
-
}
|
68
|
-
|
69
|
-
|
70
|
-
/* *******************************************************
|
71
|
-
* HUF : Huffman block compression
|
72
|
-
*********************************************************/
|
73
|
-
struct HUF_CElt_s {
|
74
|
-
U16 val;
|
75
|
-
BYTE nbBits;
|
76
|
-
}; /* typedef'd to HUF_CElt within "huf.h" */
|
77
|
-
|
78
|
-
typedef struct nodeElt_s {
|
79
|
-
U32 count;
|
80
|
-
U16 parent;
|
81
|
-
BYTE byte;
|
82
|
-
BYTE nbBits;
|
83
|
-
} nodeElt;
|
84
|
-
|
85
|
-
/*! HUF_writeCTable() :
|
86
|
-
`CTable` : huffman tree to save, using huf representation.
|
87
|
-
@return : size of saved CTable */
|
88
|
-
size_t HUF_writeCTable (void* dst, size_t maxDstSize,
|
89
|
-
const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
|
90
|
-
{
|
91
|
-
BYTE bitsToWeight[HUF_TABLELOG_MAX + 1];
|
92
|
-
BYTE huffWeight[HUF_SYMBOLVALUE_MAX];
|
93
|
-
BYTE* op = (BYTE*)dst;
|
94
|
-
U32 n;
|
95
|
-
|
96
|
-
/* check conditions */
|
97
|
-
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC);
|
98
|
-
|
99
|
-
/* convert to weight */
|
100
|
-
bitsToWeight[0] = 0;
|
101
|
-
for (n=1; n<huffLog+1; n++)
|
102
|
-
bitsToWeight[n] = (BYTE)(huffLog + 1 - n);
|
103
|
-
for (n=0; n<maxSymbolValue; n++)
|
104
|
-
huffWeight[n] = bitsToWeight[CTable[n].nbBits];
|
105
|
-
|
106
|
-
{ size_t const size = FSE_compress(op+1, maxDstSize-1, huffWeight, maxSymbolValue);
|
107
|
-
if (FSE_isError(size)) return size;
|
108
|
-
if ((size>1) & (size < maxSymbolValue/2)) { /* FSE compressed */
|
109
|
-
op[0] = (BYTE)size;
|
110
|
-
return size+1;
|
111
|
-
}
|
112
|
-
}
|
113
|
-
|
114
|
-
/* raw values */
|
115
|
-
if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen */
|
116
|
-
if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */
|
117
|
-
op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1));
|
118
|
-
huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */
|
119
|
-
for (n=0; n<maxSymbolValue; n+=2)
|
120
|
-
op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]);
|
121
|
-
return ((maxSymbolValue+1)/2) + 1;
|
122
|
-
|
123
|
-
}
|
124
|
-
|
125
|
-
|
126
|
-
size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, size_t srcSize)
|
127
|
-
{
|
128
|
-
BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
|
129
|
-
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
|
130
|
-
U32 tableLog = 0;
|
131
|
-
size_t readSize;
|
132
|
-
U32 nbSymbols = 0;
|
133
|
-
/*memset(huffWeight, 0, sizeof(huffWeight));*/ /* is not necessary, even though some analyzer complain ... */
|
134
|
-
|
135
|
-
/* get symbol weights */
|
136
|
-
readSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize);
|
137
|
-
if (HUF_isError(readSize)) return readSize;
|
138
|
-
|
139
|
-
/* check result */
|
140
|
-
if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
|
141
|
-
if (nbSymbols > maxSymbolValue+1) return ERROR(maxSymbolValue_tooSmall);
|
142
|
-
|
143
|
-
/* Prepare base value per rank */
|
144
|
-
{ U32 n, nextRankStart = 0;
|
145
|
-
for (n=1; n<=tableLog; n++) {
|
146
|
-
U32 current = nextRankStart;
|
147
|
-
nextRankStart += (rankVal[n] << (n-1));
|
148
|
-
rankVal[n] = current;
|
149
|
-
} }
|
150
|
-
|
151
|
-
/* fill nbBits */
|
152
|
-
{ U32 n; for (n=0; n<nbSymbols; n++) {
|
153
|
-
const U32 w = huffWeight[n];
|
154
|
-
CTable[n].nbBits = (BYTE)(tableLog + 1 - w);
|
155
|
-
} }
|
156
|
-
|
157
|
-
/* fill val */
|
158
|
-
{ U16 nbPerRank[HUF_TABLELOG_MAX+2] = {0}; /* support w=0=>n=tableLog+1 */
|
159
|
-
U16 valPerRank[HUF_TABLELOG_MAX+2] = {0};
|
160
|
-
{ U32 n; for (n=0; n<nbSymbols; n++) nbPerRank[CTable[n].nbBits]++; }
|
161
|
-
/* determine stating value per rank */
|
162
|
-
valPerRank[tableLog+1] = 0; /* for w==0 */
|
163
|
-
{ U16 min = 0;
|
164
|
-
U32 n; for (n=tableLog; n>0; n--) { /* start at n=tablelog <-> w=1 */
|
165
|
-
valPerRank[n] = min; /* get starting value within each rank */
|
166
|
-
min += nbPerRank[n];
|
167
|
-
min >>= 1;
|
168
|
-
} }
|
169
|
-
/* assign value within rank, symbol order */
|
170
|
-
{ U32 n; for (n=0; n<=maxSymbolValue; n++) CTable[n].val = valPerRank[CTable[n].nbBits]++; }
|
171
|
-
}
|
172
|
-
|
173
|
-
return readSize;
|
174
|
-
}
|
175
|
-
|
176
|
-
|
177
|
-
static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
|
178
|
-
{
|
179
|
-
const U32 largestBits = huffNode[lastNonNull].nbBits;
|
180
|
-
if (largestBits <= maxNbBits) return largestBits; /* early exit : no elt > maxNbBits */
|
181
|
-
|
182
|
-
/* there are several too large elements (at least >= 2) */
|
183
|
-
{ int totalCost = 0;
|
184
|
-
const U32 baseCost = 1 << (largestBits - maxNbBits);
|
185
|
-
U32 n = lastNonNull;
|
186
|
-
|
187
|
-
while (huffNode[n].nbBits > maxNbBits) {
|
188
|
-
totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits));
|
189
|
-
huffNode[n].nbBits = (BYTE)maxNbBits;
|
190
|
-
n --;
|
191
|
-
} /* n stops at huffNode[n].nbBits <= maxNbBits */
|
192
|
-
while (huffNode[n].nbBits == maxNbBits) n--; /* n end at index of smallest symbol using < maxNbBits */
|
193
|
-
|
194
|
-
/* renorm totalCost */
|
195
|
-
totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */
|
196
|
-
|
197
|
-
/* repay normalized cost */
|
198
|
-
{ U32 const noSymbol = 0xF0F0F0F0;
|
199
|
-
U32 rankLast[HUF_TABLELOG_MAX+2];
|
200
|
-
int pos;
|
201
|
-
|
202
|
-
/* Get pos of last (smallest) symbol per rank */
|
203
|
-
memset(rankLast, 0xF0, sizeof(rankLast));
|
204
|
-
{ U32 currentNbBits = maxNbBits;
|
205
|
-
for (pos=n ; pos >= 0; pos--) {
|
206
|
-
if (huffNode[pos].nbBits >= currentNbBits) continue;
|
207
|
-
currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */
|
208
|
-
rankLast[maxNbBits-currentNbBits] = pos;
|
209
|
-
} }
|
210
|
-
|
211
|
-
while (totalCost > 0) {
|
212
|
-
U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1;
|
213
|
-
for ( ; nBitsToDecrease > 1; nBitsToDecrease--) {
|
214
|
-
U32 highPos = rankLast[nBitsToDecrease];
|
215
|
-
U32 lowPos = rankLast[nBitsToDecrease-1];
|
216
|
-
if (highPos == noSymbol) continue;
|
217
|
-
if (lowPos == noSymbol) break;
|
218
|
-
{ U32 const highTotal = huffNode[highPos].count;
|
219
|
-
U32 const lowTotal = 2 * huffNode[lowPos].count;
|
220
|
-
if (highTotal <= lowTotal) break;
|
221
|
-
} }
|
222
|
-
/* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */
|
223
|
-
while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */
|
224
|
-
nBitsToDecrease ++;
|
225
|
-
totalCost -= 1 << (nBitsToDecrease-1);
|
226
|
-
if (rankLast[nBitsToDecrease-1] == noSymbol)
|
227
|
-
rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */
|
228
|
-
huffNode[rankLast[nBitsToDecrease]].nbBits ++;
|
229
|
-
if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */
|
230
|
-
rankLast[nBitsToDecrease] = noSymbol;
|
231
|
-
else {
|
232
|
-
rankLast[nBitsToDecrease]--;
|
233
|
-
if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease)
|
234
|
-
rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */
|
235
|
-
} } /* while (totalCost > 0) */
|
236
|
-
|
237
|
-
while (totalCost < 0) { /* Sometimes, cost correction overshoot */
|
238
|
-
if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */
|
239
|
-
while (huffNode[n].nbBits == maxNbBits) n--;
|
240
|
-
huffNode[n+1].nbBits--;
|
241
|
-
rankLast[1] = n+1;
|
242
|
-
totalCost++;
|
243
|
-
continue;
|
244
|
-
}
|
245
|
-
huffNode[ rankLast[1] + 1 ].nbBits--;
|
246
|
-
rankLast[1]++;
|
247
|
-
totalCost ++;
|
248
|
-
} } } /* there are several too large elements (at least >= 2) */
|
249
|
-
|
250
|
-
return maxNbBits;
|
251
|
-
}
|
252
|
-
|
253
|
-
|
254
|
-
typedef struct {
|
255
|
-
U32 base;
|
256
|
-
U32 current;
|
257
|
-
} rankPos;
|
258
|
-
|
259
|
-
static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue)
|
260
|
-
{
|
261
|
-
rankPos rank[32];
|
262
|
-
U32 n;
|
263
|
-
|
264
|
-
memset(rank, 0, sizeof(rank));
|
265
|
-
for (n=0; n<=maxSymbolValue; n++) {
|
266
|
-
U32 r = BIT_highbit32(count[n] + 1);
|
267
|
-
rank[r].base ++;
|
268
|
-
}
|
269
|
-
for (n=30; n>0; n--) rank[n-1].base += rank[n].base;
|
270
|
-
for (n=0; n<32; n++) rank[n].current = rank[n].base;
|
271
|
-
for (n=0; n<=maxSymbolValue; n++) {
|
272
|
-
U32 const c = count[n];
|
273
|
-
U32 const r = BIT_highbit32(c+1) + 1;
|
274
|
-
U32 pos = rank[r].current++;
|
275
|
-
while ((pos > rank[r].base) && (c > huffNode[pos-1].count)) huffNode[pos]=huffNode[pos-1], pos--;
|
276
|
-
huffNode[pos].count = c;
|
277
|
-
huffNode[pos].byte = (BYTE)n;
|
278
|
-
}
|
279
|
-
}
|
280
|
-
|
281
|
-
|
282
|
-
#define STARTNODE (HUF_SYMBOLVALUE_MAX+1)
|
283
|
-
size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits)
|
284
|
-
{
|
285
|
-
nodeElt huffNode0[2*HUF_SYMBOLVALUE_MAX+1 +1];
|
286
|
-
nodeElt* huffNode = huffNode0 + 1;
|
287
|
-
U32 n, nonNullRank;
|
288
|
-
int lowS, lowN;
|
289
|
-
U16 nodeNb = STARTNODE;
|
290
|
-
U32 nodeRoot;
|
291
|
-
|
292
|
-
/* safety checks */
|
293
|
-
if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT;
|
294
|
-
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC);
|
295
|
-
memset(huffNode0, 0, sizeof(huffNode0));
|
296
|
-
|
297
|
-
/* sort, decreasing order */
|
298
|
-
HUF_sort(huffNode, count, maxSymbolValue);
|
299
|
-
|
300
|
-
/* init for parents */
|
301
|
-
nonNullRank = maxSymbolValue;
|
302
|
-
while(huffNode[nonNullRank].count == 0) nonNullRank--;
|
303
|
-
lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb;
|
304
|
-
huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count;
|
305
|
-
huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb;
|
306
|
-
nodeNb++; lowS-=2;
|
307
|
-
for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30);
|
308
|
-
huffNode0[0].count = (U32)(1U<<31);
|
309
|
-
|
310
|
-
/* create parents */
|
311
|
-
while (nodeNb <= nodeRoot) {
|
312
|
-
U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
|
313
|
-
U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
|
314
|
-
huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count;
|
315
|
-
huffNode[n1].parent = huffNode[n2].parent = nodeNb;
|
316
|
-
nodeNb++;
|
317
|
-
}
|
318
|
-
|
319
|
-
/* distribute weights (unlimited tree height) */
|
320
|
-
huffNode[nodeRoot].nbBits = 0;
|
321
|
-
for (n=nodeRoot-1; n>=STARTNODE; n--)
|
322
|
-
huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
|
323
|
-
for (n=0; n<=nonNullRank; n++)
|
324
|
-
huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
|
325
|
-
|
326
|
-
/* enforce maxTableLog */
|
327
|
-
maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits);
|
328
|
-
|
329
|
-
/* fill result into tree (val, nbBits) */
|
330
|
-
{ U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0};
|
331
|
-
U16 valPerRank[HUF_TABLELOG_MAX+1] = {0};
|
332
|
-
if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */
|
333
|
-
for (n=0; n<=nonNullRank; n++)
|
334
|
-
nbPerRank[huffNode[n].nbBits]++;
|
335
|
-
/* determine stating value per rank */
|
336
|
-
{ U16 min = 0;
|
337
|
-
for (n=maxNbBits; n>0; n--) {
|
338
|
-
valPerRank[n] = min; /* get starting value within each rank */
|
339
|
-
min += nbPerRank[n];
|
340
|
-
min >>= 1;
|
341
|
-
} }
|
342
|
-
for (n=0; n<=maxSymbolValue; n++)
|
343
|
-
tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */
|
344
|
-
for (n=0; n<=maxSymbolValue; n++)
|
345
|
-
tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */
|
346
|
-
}
|
347
|
-
|
348
|
-
return maxNbBits;
|
349
|
-
}
|
350
|
-
|
351
|
-
static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable)
|
352
|
-
{
|
353
|
-
BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits);
|
354
|
-
}
|
355
|
-
|
356
|
-
size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
|
357
|
-
|
358
|
-
#define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
|
359
|
-
|
360
|
-
#define HUF_FLUSHBITS_1(stream) \
|
361
|
-
if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*2+7) HUF_FLUSHBITS(stream)
|
362
|
-
|
363
|
-
#define HUF_FLUSHBITS_2(stream) \
|
364
|
-
if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*4+7) HUF_FLUSHBITS(stream)
|
365
|
-
|
366
|
-
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
|
367
|
-
{
|
368
|
-
const BYTE* ip = (const BYTE*) src;
|
369
|
-
BYTE* const ostart = (BYTE*)dst;
|
370
|
-
BYTE* const oend = ostart + dstSize;
|
371
|
-
BYTE* op = ostart;
|
372
|
-
size_t n;
|
373
|
-
const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize));
|
374
|
-
BIT_CStream_t bitC;
|
375
|
-
|
376
|
-
/* init */
|
377
|
-
if (dstSize < 8) return 0; /* not enough space to compress */
|
378
|
-
{ size_t const errorCode = BIT_initCStream(&bitC, op, oend-op);
|
379
|
-
if (HUF_isError(errorCode)) return 0; }
|
380
|
-
|
381
|
-
n = srcSize & ~3; /* join to mod 4 */
|
382
|
-
switch (srcSize & 3)
|
383
|
-
{
|
384
|
-
case 3 : HUF_encodeSymbol(&bitC, ip[n+ 2], CTable);
|
385
|
-
HUF_FLUSHBITS_2(&bitC);
|
386
|
-
case 2 : HUF_encodeSymbol(&bitC, ip[n+ 1], CTable);
|
387
|
-
HUF_FLUSHBITS_1(&bitC);
|
388
|
-
case 1 : HUF_encodeSymbol(&bitC, ip[n+ 0], CTable);
|
389
|
-
HUF_FLUSHBITS(&bitC);
|
390
|
-
case 0 :
|
391
|
-
default: ;
|
392
|
-
}
|
393
|
-
|
394
|
-
for (; n>0; n-=4) { /* note : n&3==0 at this stage */
|
395
|
-
HUF_encodeSymbol(&bitC, ip[n- 1], CTable);
|
396
|
-
HUF_FLUSHBITS_1(&bitC);
|
397
|
-
HUF_encodeSymbol(&bitC, ip[n- 2], CTable);
|
398
|
-
HUF_FLUSHBITS_2(&bitC);
|
399
|
-
HUF_encodeSymbol(&bitC, ip[n- 3], CTable);
|
400
|
-
HUF_FLUSHBITS_1(&bitC);
|
401
|
-
HUF_encodeSymbol(&bitC, ip[n- 4], CTable);
|
402
|
-
HUF_FLUSHBITS(&bitC);
|
403
|
-
}
|
404
|
-
|
405
|
-
return BIT_closeCStream(&bitC);
|
406
|
-
}
|
407
|
-
|
408
|
-
|
409
|
-
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
|
410
|
-
{
|
411
|
-
size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */
|
412
|
-
const BYTE* ip = (const BYTE*) src;
|
413
|
-
const BYTE* const iend = ip + srcSize;
|
414
|
-
BYTE* const ostart = (BYTE*) dst;
|
415
|
-
BYTE* const oend = ostart + dstSize;
|
416
|
-
BYTE* op = ostart;
|
417
|
-
|
418
|
-
if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */
|
419
|
-
if (srcSize < 12) return 0; /* no saving possible : too small input */
|
420
|
-
op += 6; /* jumpTable */
|
421
|
-
|
422
|
-
{ size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
|
423
|
-
if (HUF_isError(cSize)) return cSize;
|
424
|
-
if (cSize==0) return 0;
|
425
|
-
MEM_writeLE16(ostart, (U16)cSize);
|
426
|
-
op += cSize;
|
427
|
-
}
|
428
|
-
|
429
|
-
ip += segmentSize;
|
430
|
-
{ size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
|
431
|
-
if (HUF_isError(cSize)) return cSize;
|
432
|
-
if (cSize==0) return 0;
|
433
|
-
MEM_writeLE16(ostart+2, (U16)cSize);
|
434
|
-
op += cSize;
|
435
|
-
}
|
436
|
-
|
437
|
-
ip += segmentSize;
|
438
|
-
{ size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
|
439
|
-
if (HUF_isError(cSize)) return cSize;
|
440
|
-
if (cSize==0) return 0;
|
441
|
-
MEM_writeLE16(ostart+4, (U16)cSize);
|
442
|
-
op += cSize;
|
443
|
-
}
|
444
|
-
|
445
|
-
ip += segmentSize;
|
446
|
-
{ size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable);
|
447
|
-
if (HUF_isError(cSize)) return cSize;
|
448
|
-
if (cSize==0) return 0;
|
449
|
-
op += cSize;
|
450
|
-
}
|
451
|
-
|
452
|
-
return op-ostart;
|
453
|
-
}
|
454
|
-
|
455
|
-
|
456
|
-
static size_t HUF_compress_internal (
|
457
|
-
void* dst, size_t dstSize,
|
458
|
-
const void* src, size_t srcSize,
|
459
|
-
unsigned maxSymbolValue, unsigned huffLog,
|
460
|
-
unsigned singleStream)
|
461
|
-
{
|
462
|
-
BYTE* const ostart = (BYTE*)dst;
|
463
|
-
BYTE* const oend = ostart + dstSize;
|
464
|
-
BYTE* op = ostart;
|
465
|
-
|
466
|
-
U32 count[HUF_SYMBOLVALUE_MAX+1];
|
467
|
-
HUF_CElt CTable[HUF_SYMBOLVALUE_MAX+1];
|
468
|
-
|
469
|
-
/* checks & inits */
|
470
|
-
if (!srcSize) return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */
|
471
|
-
if (!dstSize) return 0; /* cannot fit within dst budget */
|
472
|
-
if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */
|
473
|
-
if (huffLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
|
474
|
-
if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX;
|
475
|
-
if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT;
|
476
|
-
|
477
|
-
/* Scan input and build symbol stats */
|
478
|
-
{ size_t const largest = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize);
|
479
|
-
if (HUF_isError(largest)) return largest;
|
480
|
-
if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */
|
481
|
-
if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */
|
482
|
-
}
|
483
|
-
|
484
|
-
/* Build Huffman Tree */
|
485
|
-
huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
|
486
|
-
{ size_t const maxBits = HUF_buildCTable (CTable, count, maxSymbolValue, huffLog);
|
487
|
-
if (HUF_isError(maxBits)) return maxBits;
|
488
|
-
huffLog = (U32)maxBits;
|
489
|
-
}
|
490
|
-
|
491
|
-
/* Write table description header */
|
492
|
-
{ size_t const hSize = HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog);
|
493
|
-
if (HUF_isError(hSize)) return hSize;
|
494
|
-
if (hSize + 12 >= srcSize) return 0; /* not useful to try compression */
|
495
|
-
op += hSize;
|
496
|
-
}
|
497
|
-
|
498
|
-
/* Compress */
|
499
|
-
{ size_t const cSize = (singleStream) ?
|
500
|
-
HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) : /* single segment */
|
501
|
-
HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
|
502
|
-
if (HUF_isError(cSize)) return cSize;
|
503
|
-
if (cSize==0) return 0; /* uncompressible */
|
504
|
-
op += cSize;
|
505
|
-
}
|
506
|
-
|
507
|
-
/* check compressibility */
|
508
|
-
if ((size_t)(op-ostart) >= srcSize-1)
|
509
|
-
return 0;
|
510
|
-
|
511
|
-
return op-ostart;
|
512
|
-
}
|
513
|
-
|
514
|
-
|
515
|
-
size_t HUF_compress1X (void* dst, size_t dstSize,
|
516
|
-
const void* src, size_t srcSize,
|
517
|
-
unsigned maxSymbolValue, unsigned huffLog)
|
518
|
-
{
|
519
|
-
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1);
|
520
|
-
}
|
521
|
-
|
522
|
-
size_t HUF_compress2 (void* dst, size_t dstSize,
|
523
|
-
const void* src, size_t srcSize,
|
524
|
-
unsigned maxSymbolValue, unsigned huffLog)
|
525
|
-
{
|
526
|
-
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0);
|
527
|
-
}
|
528
|
-
|
529
|
-
|
530
|
-
size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
531
|
-
{
|
532
|
-
return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_TABLELOG_DEFAULT);
|
533
|
-
}
|