zstd 1.1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +15 -0
- data/.travis.yml +5 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +21 -0
- data/README.md +64 -0
- data/Rakefile +19 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/exe/zstd +3 -0
- data/ext/zstd/extconf.rb +20 -0
- data/ext/zstd/libzstd/.gitignore +2 -0
- data/ext/zstd/libzstd/LICENSE +1262 -0
- data/ext/zstd/libzstd/Makefile +133 -0
- data/ext/zstd/libzstd/PATENTS +1272 -0
- data/ext/zstd/libzstd/README.md +77 -0
- data/ext/zstd/libzstd/common/bitstream.h +414 -0
- data/ext/zstd/libzstd/common/entropy_common.c +227 -0
- data/ext/zstd/libzstd/common/error_private.c +43 -0
- data/ext/zstd/libzstd/common/error_private.h +76 -0
- data/ext/zstd/libzstd/common/fse.h +668 -0
- data/ext/zstd/libzstd/common/fse_decompress.c +329 -0
- data/ext/zstd/libzstd/common/huf.h +238 -0
- data/ext/zstd/libzstd/common/mem.h +372 -0
- data/ext/zstd/libzstd/common/xxhash.c +867 -0
- data/ext/zstd/libzstd/common/xxhash.h +309 -0
- data/ext/zstd/libzstd/common/zstd_common.c +77 -0
- data/ext/zstd/libzstd/common/zstd_errors.h +60 -0
- data/ext/zstd/libzstd/common/zstd_internal.h +270 -0
- data/ext/zstd/libzstd/compress/fse_compress.c +850 -0
- data/ext/zstd/libzstd/compress/huf_compress.c +609 -0
- data/ext/zstd/libzstd/compress/zstd_compress.c +3291 -0
- data/ext/zstd/libzstd/compress/zstd_opt.h +919 -0
- data/ext/zstd/libzstd/decompress/huf_decompress.c +885 -0
- data/ext/zstd/libzstd/decompress/zstd_decompress.c +2154 -0
- data/ext/zstd/libzstd/deprecated/zbuff.h +210 -0
- data/ext/zstd/libzstd/deprecated/zbuff_compress.c +145 -0
- data/ext/zstd/libzstd/deprecated/zbuff_decompress.c +74 -0
- data/ext/zstd/libzstd/dictBuilder/divsufsort.c +1913 -0
- data/ext/zstd/libzstd/dictBuilder/divsufsort.h +67 -0
- data/ext/zstd/libzstd/dictBuilder/zdict.c +1012 -0
- data/ext/zstd/libzstd/dictBuilder/zdict.h +111 -0
- data/ext/zstd/libzstd/dll/example/Makefile +47 -0
- data/ext/zstd/libzstd/dll/example/README.md +69 -0
- data/ext/zstd/libzstd/dll/example/build_package.bat +17 -0
- data/ext/zstd/libzstd/dll/example/fullbench-dll.sln +25 -0
- data/ext/zstd/libzstd/dll/example/fullbench-dll.vcxproj +179 -0
- data/ext/zstd/libzstd/dll/libzstd.def +86 -0
- data/ext/zstd/libzstd/legacy/zstd_legacy.h +259 -0
- data/ext/zstd/libzstd/legacy/zstd_v01.c +2095 -0
- data/ext/zstd/libzstd/legacy/zstd_v01.h +80 -0
- data/ext/zstd/libzstd/legacy/zstd_v02.c +3518 -0
- data/ext/zstd/libzstd/legacy/zstd_v02.h +79 -0
- data/ext/zstd/libzstd/legacy/zstd_v03.c +3159 -0
- data/ext/zstd/libzstd/legacy/zstd_v03.h +79 -0
- data/ext/zstd/libzstd/legacy/zstd_v04.c +3795 -0
- data/ext/zstd/libzstd/legacy/zstd_v04.h +128 -0
- data/ext/zstd/libzstd/legacy/zstd_v05.c +4056 -0
- data/ext/zstd/libzstd/legacy/zstd_v05.h +149 -0
- data/ext/zstd/libzstd/legacy/zstd_v06.c +4167 -0
- data/ext/zstd/libzstd/legacy/zstd_v06.h +159 -0
- data/ext/zstd/libzstd/legacy/zstd_v07.c +4540 -0
- data/ext/zstd/libzstd/legacy/zstd_v07.h +173 -0
- data/ext/zstd/libzstd/libzstd.pc.in +14 -0
- data/ext/zstd/libzstd/zstd.h +673 -0
- data/ext/zstd/zstd.c +185 -0
- data/ext/zstd/zstd.h +7 -0
- data/lib/zstd/version.rb +3 -0
- data/lib/zstd.rb +6 -0
- data/zstd.gemspec +38 -0
- metadata +172 -0
@@ -0,0 +1,329 @@
|
|
1
|
+
/* ******************************************************************
|
2
|
+
FSE : Finite State Entropy decoder
|
3
|
+
Copyright (C) 2013-2015, Yann Collet.
|
4
|
+
|
5
|
+
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
6
|
+
|
7
|
+
Redistribution and use in source and binary forms, with or without
|
8
|
+
modification, are permitted provided that the following conditions are
|
9
|
+
met:
|
10
|
+
|
11
|
+
* Redistributions of source code must retain the above copyright
|
12
|
+
notice, this list of conditions and the following disclaimer.
|
13
|
+
* Redistributions in binary form must reproduce the above
|
14
|
+
copyright notice, this list of conditions and the following disclaimer
|
15
|
+
in the documentation and/or other materials provided with the
|
16
|
+
distribution.
|
17
|
+
|
18
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
19
|
+
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
20
|
+
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
21
|
+
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
22
|
+
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
23
|
+
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
24
|
+
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
25
|
+
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
26
|
+
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
27
|
+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
28
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
29
|
+
|
30
|
+
You can contact the author at :
|
31
|
+
- FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
32
|
+
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
33
|
+
****************************************************************** */
|
34
|
+
|
35
|
+
|
36
|
+
/* **************************************************************
|
37
|
+
* Compiler specifics
|
38
|
+
****************************************************************/
|
39
|
+
#ifdef _MSC_VER /* Visual Studio */
|
40
|
+
# define FORCE_INLINE static __forceinline
|
41
|
+
# include <intrin.h> /* For Visual 2005 */
|
42
|
+
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
43
|
+
# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
|
44
|
+
#else
|
45
|
+
# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
|
46
|
+
# ifdef __GNUC__
|
47
|
+
# define FORCE_INLINE static inline __attribute__((always_inline))
|
48
|
+
# else
|
49
|
+
# define FORCE_INLINE static inline
|
50
|
+
# endif
|
51
|
+
# else
|
52
|
+
# define FORCE_INLINE static
|
53
|
+
# endif /* __STDC_VERSION__ */
|
54
|
+
#endif
|
55
|
+
|
56
|
+
|
57
|
+
/* **************************************************************
|
58
|
+
* Includes
|
59
|
+
****************************************************************/
|
60
|
+
#include <stdlib.h> /* malloc, free, qsort */
|
61
|
+
#include <string.h> /* memcpy, memset */
|
62
|
+
#include <stdio.h> /* printf (debug) */
|
63
|
+
#include "bitstream.h"
|
64
|
+
#define FSE_STATIC_LINKING_ONLY
|
65
|
+
#include "fse.h"
|
66
|
+
|
67
|
+
|
68
|
+
/* **************************************************************
|
69
|
+
* Error Management
|
70
|
+
****************************************************************/
|
71
|
+
#define FSE_isError ERR_isError
|
72
|
+
#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
|
73
|
+
|
74
|
+
/* check and forward error code */
|
75
|
+
#define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; }
|
76
|
+
|
77
|
+
|
78
|
+
/* **************************************************************
|
79
|
+
* Templates
|
80
|
+
****************************************************************/
|
81
|
+
/*
|
82
|
+
designed to be included
|
83
|
+
for type-specific functions (template emulation in C)
|
84
|
+
Objective is to write these functions only once, for improved maintenance
|
85
|
+
*/
|
86
|
+
|
87
|
+
/* safety checks */
|
88
|
+
#ifndef FSE_FUNCTION_EXTENSION
|
89
|
+
# error "FSE_FUNCTION_EXTENSION must be defined"
|
90
|
+
#endif
|
91
|
+
#ifndef FSE_FUNCTION_TYPE
|
92
|
+
# error "FSE_FUNCTION_TYPE must be defined"
|
93
|
+
#endif
|
94
|
+
|
95
|
+
/* Function names */
|
96
|
+
#define FSE_CAT(X,Y) X##Y
|
97
|
+
#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)
|
98
|
+
#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)
|
99
|
+
|
100
|
+
|
101
|
+
/* Function templates */
|
102
|
+
FSE_DTable* FSE_createDTable (unsigned tableLog)
|
103
|
+
{
|
104
|
+
if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
|
105
|
+
return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );
|
106
|
+
}
|
107
|
+
|
108
|
+
void FSE_freeDTable (FSE_DTable* dt)
|
109
|
+
{
|
110
|
+
free(dt);
|
111
|
+
}
|
112
|
+
|
113
|
+
size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
|
114
|
+
{
|
115
|
+
void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */
|
116
|
+
FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr);
|
117
|
+
U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1];
|
118
|
+
|
119
|
+
U32 const maxSV1 = maxSymbolValue + 1;
|
120
|
+
U32 const tableSize = 1 << tableLog;
|
121
|
+
U32 highThreshold = tableSize-1;
|
122
|
+
|
123
|
+
/* Sanity Checks */
|
124
|
+
if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge);
|
125
|
+
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
126
|
+
|
127
|
+
/* Init, lay down lowprob symbols */
|
128
|
+
{ FSE_DTableHeader DTableH;
|
129
|
+
DTableH.tableLog = (U16)tableLog;
|
130
|
+
DTableH.fastMode = 1;
|
131
|
+
{ S16 const largeLimit= (S16)(1 << (tableLog-1));
|
132
|
+
U32 s;
|
133
|
+
for (s=0; s<maxSV1; s++) {
|
134
|
+
if (normalizedCounter[s]==-1) {
|
135
|
+
tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s;
|
136
|
+
symbolNext[s] = 1;
|
137
|
+
} else {
|
138
|
+
if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
|
139
|
+
symbolNext[s] = normalizedCounter[s];
|
140
|
+
} } }
|
141
|
+
memcpy(dt, &DTableH, sizeof(DTableH));
|
142
|
+
}
|
143
|
+
|
144
|
+
/* Spread symbols */
|
145
|
+
{ U32 const tableMask = tableSize-1;
|
146
|
+
U32 const step = FSE_TABLESTEP(tableSize);
|
147
|
+
U32 s, position = 0;
|
148
|
+
for (s=0; s<maxSV1; s++) {
|
149
|
+
int i;
|
150
|
+
for (i=0; i<normalizedCounter[s]; i++) {
|
151
|
+
tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s;
|
152
|
+
position = (position + step) & tableMask;
|
153
|
+
while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */
|
154
|
+
} }
|
155
|
+
if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
|
156
|
+
}
|
157
|
+
|
158
|
+
/* Build Decoding table */
|
159
|
+
{ U32 u;
|
160
|
+
for (u=0; u<tableSize; u++) {
|
161
|
+
FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
|
162
|
+
U16 nextState = symbolNext[symbol]++;
|
163
|
+
tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32 ((U32)nextState) );
|
164
|
+
tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
|
165
|
+
} }
|
166
|
+
|
167
|
+
return 0;
|
168
|
+
}
|
169
|
+
|
170
|
+
|
171
|
+
#ifndef FSE_COMMONDEFS_ONLY
|
172
|
+
|
173
|
+
/*-*******************************************************
|
174
|
+
* Decompression (Byte symbols)
|
175
|
+
*********************************************************/
|
176
|
+
size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
|
177
|
+
{
|
178
|
+
void* ptr = dt;
|
179
|
+
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
|
180
|
+
void* dPtr = dt + 1;
|
181
|
+
FSE_decode_t* const cell = (FSE_decode_t*)dPtr;
|
182
|
+
|
183
|
+
DTableH->tableLog = 0;
|
184
|
+
DTableH->fastMode = 0;
|
185
|
+
|
186
|
+
cell->newState = 0;
|
187
|
+
cell->symbol = symbolValue;
|
188
|
+
cell->nbBits = 0;
|
189
|
+
|
190
|
+
return 0;
|
191
|
+
}
|
192
|
+
|
193
|
+
|
194
|
+
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
|
195
|
+
{
|
196
|
+
void* ptr = dt;
|
197
|
+
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
|
198
|
+
void* dPtr = dt + 1;
|
199
|
+
FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr;
|
200
|
+
const unsigned tableSize = 1 << nbBits;
|
201
|
+
const unsigned tableMask = tableSize - 1;
|
202
|
+
const unsigned maxSV1 = tableMask+1;
|
203
|
+
unsigned s;
|
204
|
+
|
205
|
+
/* Sanity checks */
|
206
|
+
if (nbBits < 1) return ERROR(GENERIC); /* min size */
|
207
|
+
|
208
|
+
/* Build Decoding Table */
|
209
|
+
DTableH->tableLog = (U16)nbBits;
|
210
|
+
DTableH->fastMode = 1;
|
211
|
+
for (s=0; s<maxSV1; s++) {
|
212
|
+
dinfo[s].newState = 0;
|
213
|
+
dinfo[s].symbol = (BYTE)s;
|
214
|
+
dinfo[s].nbBits = (BYTE)nbBits;
|
215
|
+
}
|
216
|
+
|
217
|
+
return 0;
|
218
|
+
}
|
219
|
+
|
220
|
+
FORCE_INLINE size_t FSE_decompress_usingDTable_generic(
|
221
|
+
void* dst, size_t maxDstSize,
|
222
|
+
const void* cSrc, size_t cSrcSize,
|
223
|
+
const FSE_DTable* dt, const unsigned fast)
|
224
|
+
{
|
225
|
+
BYTE* const ostart = (BYTE*) dst;
|
226
|
+
BYTE* op = ostart;
|
227
|
+
BYTE* const omax = op + maxDstSize;
|
228
|
+
BYTE* const olimit = omax-3;
|
229
|
+
|
230
|
+
BIT_DStream_t bitD;
|
231
|
+
FSE_DState_t state1;
|
232
|
+
FSE_DState_t state2;
|
233
|
+
|
234
|
+
/* Init */
|
235
|
+
CHECK_F(BIT_initDStream(&bitD, cSrc, cSrcSize));
|
236
|
+
|
237
|
+
FSE_initDState(&state1, &bitD, dt);
|
238
|
+
FSE_initDState(&state2, &bitD, dt);
|
239
|
+
|
240
|
+
#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD)
|
241
|
+
|
242
|
+
/* 4 symbols per loop */
|
243
|
+
for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) & (op<olimit) ; op+=4) {
|
244
|
+
op[0] = FSE_GETSYMBOL(&state1);
|
245
|
+
|
246
|
+
if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
|
247
|
+
BIT_reloadDStream(&bitD);
|
248
|
+
|
249
|
+
op[1] = FSE_GETSYMBOL(&state2);
|
250
|
+
|
251
|
+
if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
|
252
|
+
{ if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } }
|
253
|
+
|
254
|
+
op[2] = FSE_GETSYMBOL(&state1);
|
255
|
+
|
256
|
+
if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
|
257
|
+
BIT_reloadDStream(&bitD);
|
258
|
+
|
259
|
+
op[3] = FSE_GETSYMBOL(&state2);
|
260
|
+
}
|
261
|
+
|
262
|
+
/* tail */
|
263
|
+
/* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */
|
264
|
+
while (1) {
|
265
|
+
if (op>(omax-2)) return ERROR(dstSize_tooSmall);
|
266
|
+
*op++ = FSE_GETSYMBOL(&state1);
|
267
|
+
if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
|
268
|
+
*op++ = FSE_GETSYMBOL(&state2);
|
269
|
+
break;
|
270
|
+
}
|
271
|
+
|
272
|
+
if (op>(omax-2)) return ERROR(dstSize_tooSmall);
|
273
|
+
*op++ = FSE_GETSYMBOL(&state2);
|
274
|
+
if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
|
275
|
+
*op++ = FSE_GETSYMBOL(&state1);
|
276
|
+
break;
|
277
|
+
} }
|
278
|
+
|
279
|
+
return op-ostart;
|
280
|
+
}
|
281
|
+
|
282
|
+
|
283
|
+
size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
|
284
|
+
const void* cSrc, size_t cSrcSize,
|
285
|
+
const FSE_DTable* dt)
|
286
|
+
{
|
287
|
+
const void* ptr = dt;
|
288
|
+
const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
|
289
|
+
const U32 fastMode = DTableH->fastMode;
|
290
|
+
|
291
|
+
/* select fast mode (static) */
|
292
|
+
if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
|
293
|
+
return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
|
294
|
+
}
|
295
|
+
|
296
|
+
|
297
|
+
size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog)
|
298
|
+
{
|
299
|
+
const BYTE* const istart = (const BYTE*)cSrc;
|
300
|
+
const BYTE* ip = istart;
|
301
|
+
short counting[FSE_MAX_SYMBOL_VALUE+1];
|
302
|
+
unsigned tableLog;
|
303
|
+
unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
|
304
|
+
|
305
|
+
/* normal FSE decoding mode */
|
306
|
+
size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
|
307
|
+
if (FSE_isError(NCountLength)) return NCountLength;
|
308
|
+
//if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size; supposed to be already checked in NCountLength, only remaining case : NCountLength==cSrcSize */
|
309
|
+
if (tableLog > maxLog) return ERROR(tableLog_tooLarge);
|
310
|
+
ip += NCountLength;
|
311
|
+
cSrcSize -= NCountLength;
|
312
|
+
|
313
|
+
CHECK_F( FSE_buildDTable (workSpace, counting, maxSymbolValue, tableLog) );
|
314
|
+
|
315
|
+
return FSE_decompress_usingDTable (dst, dstCapacity, ip, cSrcSize, workSpace); /* always return, even if it is an error code */
|
316
|
+
}
|
317
|
+
|
318
|
+
|
319
|
+
typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
|
320
|
+
|
321
|
+
size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize)
|
322
|
+
{
|
323
|
+
DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */
|
324
|
+
return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, dt, FSE_MAX_TABLELOG);
|
325
|
+
}
|
326
|
+
|
327
|
+
|
328
|
+
|
329
|
+
#endif /* FSE_COMMONDEFS_ONLY */
|
@@ -0,0 +1,238 @@
|
|
1
|
+
/* ******************************************************************
|
2
|
+
Huffman coder, part of New Generation Entropy library
|
3
|
+
header file
|
4
|
+
Copyright (C) 2013-2016, Yann Collet.
|
5
|
+
|
6
|
+
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
7
|
+
|
8
|
+
Redistribution and use in source and binary forms, with or without
|
9
|
+
modification, are permitted provided that the following conditions are
|
10
|
+
met:
|
11
|
+
|
12
|
+
* Redistributions of source code must retain the above copyright
|
13
|
+
notice, this list of conditions and the following disclaimer.
|
14
|
+
* Redistributions in binary form must reproduce the above
|
15
|
+
copyright notice, this list of conditions and the following disclaimer
|
16
|
+
in the documentation and/or other materials provided with the
|
17
|
+
distribution.
|
18
|
+
|
19
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
20
|
+
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
21
|
+
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
22
|
+
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
23
|
+
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
24
|
+
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
25
|
+
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
26
|
+
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
27
|
+
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
28
|
+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
29
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
30
|
+
|
31
|
+
You can contact the author at :
|
32
|
+
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
33
|
+
****************************************************************** */
|
34
|
+
#ifndef HUF_H_298734234
|
35
|
+
#define HUF_H_298734234
|
36
|
+
|
37
|
+
#if defined (__cplusplus)
|
38
|
+
extern "C" {
|
39
|
+
#endif
|
40
|
+
|
41
|
+
|
42
|
+
/* *** Dependencies *** */
|
43
|
+
#include <stddef.h> /* size_t */
|
44
|
+
|
45
|
+
|
46
|
+
/* *** simple functions *** */
|
47
|
+
/**
|
48
|
+
HUF_compress() :
|
49
|
+
Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'.
|
50
|
+
'dst' buffer must be already allocated.
|
51
|
+
Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).
|
52
|
+
`srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB.
|
53
|
+
@return : size of compressed data (<= `dstCapacity`).
|
54
|
+
Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
|
55
|
+
if return == 1, srcData is a single repeated byte symbol (RLE compression).
|
56
|
+
if HUF_isError(return), compression failed (more details using HUF_getErrorName())
|
57
|
+
*/
|
58
|
+
size_t HUF_compress(void* dst, size_t dstCapacity,
|
59
|
+
const void* src, size_t srcSize);
|
60
|
+
|
61
|
+
/**
|
62
|
+
HUF_decompress() :
|
63
|
+
Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
|
64
|
+
into already allocated buffer 'dst', of minimum size 'dstSize'.
|
65
|
+
`originalSize` : **must** be the ***exact*** size of original (uncompressed) data.
|
66
|
+
Note : in contrast with FSE, HUF_decompress can regenerate
|
67
|
+
RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
|
68
|
+
because it knows size to regenerate.
|
69
|
+
@return : size of regenerated data (== originalSize),
|
70
|
+
or an error code, which can be tested using HUF_isError()
|
71
|
+
*/
|
72
|
+
size_t HUF_decompress(void* dst, size_t originalSize,
|
73
|
+
const void* cSrc, size_t cSrcSize);
|
74
|
+
|
75
|
+
|
76
|
+
/* *** Tool functions *** */
|
77
|
+
#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */
|
78
|
+
size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */
|
79
|
+
|
80
|
+
/* Error Management */
|
81
|
+
unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */
|
82
|
+
const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */
|
83
|
+
|
84
|
+
|
85
|
+
/* *** Advanced function *** */
|
86
|
+
|
87
|
+
/** HUF_compress2() :
|
88
|
+
* Same as HUF_compress(), but offers direct control over `maxSymbolValue` and `tableLog` .
|
89
|
+
* `tableLog` must be `<= HUF_TABLELOG_MAX` . */
|
90
|
+
size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
91
|
+
|
92
|
+
/** HUF_compress4X_wksp() :
|
93
|
+
* Same as HUF_compress2(), but uses externally allocated `workSpace`, which must be a table of >= 1024 unsigned */
|
94
|
+
size_t HUF_compress4X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least 1024 unsigned */
|
95
|
+
|
96
|
+
|
97
|
+
|
98
|
+
#ifdef HUF_STATIC_LINKING_ONLY
|
99
|
+
|
100
|
+
/* *** Dependencies *** */
|
101
|
+
#include "mem.h" /* U32 */
|
102
|
+
|
103
|
+
|
104
|
+
/* *** Constants *** */
|
105
|
+
#define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
|
106
|
+
#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
|
107
|
+
#define HUF_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */
|
108
|
+
#define HUF_SYMBOLVALUE_MAX 255
|
109
|
+
#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
|
110
|
+
# error "HUF_TABLELOG_MAX is too large !"
|
111
|
+
#endif
|
112
|
+
|
113
|
+
|
114
|
+
/* ****************************************
|
115
|
+
* Static allocation
|
116
|
+
******************************************/
|
117
|
+
/* HUF buffer bounds */
|
118
|
+
#define HUF_CTABLEBOUND 129
|
119
|
+
#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */
|
120
|
+
#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
|
121
|
+
|
122
|
+
/* static allocation of HUF's Compression Table */
|
123
|
+
#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
|
124
|
+
U32 name##hb[maxSymbolValue+1]; \
|
125
|
+
void* name##hv = &(name##hb); \
|
126
|
+
HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */
|
127
|
+
|
128
|
+
/* static allocation of HUF's DTable */
|
129
|
+
typedef U32 HUF_DTable;
|
130
|
+
#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog)))
|
131
|
+
#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
|
132
|
+
HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) }
|
133
|
+
#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \
|
134
|
+
HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) }
|
135
|
+
|
136
|
+
|
137
|
+
/* ****************************************
|
138
|
+
* Advanced decompression functions
|
139
|
+
******************************************/
|
140
|
+
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
141
|
+
size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
142
|
+
|
143
|
+
size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */
|
144
|
+
size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
|
145
|
+
size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
146
|
+
size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
147
|
+
|
148
|
+
|
149
|
+
/* ****************************************
|
150
|
+
* HUF detailed API
|
151
|
+
******************************************/
|
152
|
+
/*!
|
153
|
+
HUF_compress() does the following:
|
154
|
+
1. count symbol occurrence from source[] into table count[] using FSE_count()
|
155
|
+
2. (optional) refine tableLog using HUF_optimalTableLog()
|
156
|
+
3. build Huffman table from count using HUF_buildCTable()
|
157
|
+
4. save Huffman table to memory buffer using HUF_writeCTable()
|
158
|
+
5. encode the data stream using HUF_compress4X_usingCTable()
|
159
|
+
|
160
|
+
The following API allows targeting specific sub-functions for advanced tasks.
|
161
|
+
For example, it's possible to compress several blocks using the same 'CTable',
|
162
|
+
or to save and regenerate 'CTable' using external methods.
|
163
|
+
*/
|
164
|
+
/* FSE_count() : find it within "fse.h" */
|
165
|
+
unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
|
166
|
+
typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */
|
167
|
+
size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);
|
168
|
+
size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
|
169
|
+
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
|
170
|
+
|
171
|
+
|
172
|
+
/** HUF_buildCTable_wksp() :
|
173
|
+
* Same as HUF_buildCTable(), but using externally allocated scratch buffer.
|
174
|
+
* `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned.
|
175
|
+
*/
|
176
|
+
size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize);
|
177
|
+
|
178
|
+
/*! HUF_readStats() :
|
179
|
+
Read compact Huffman tree, saved by HUF_writeCTable().
|
180
|
+
`huffWeight` is destination buffer.
|
181
|
+
@return : size read from `src` , or an error Code .
|
182
|
+
Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
|
183
|
+
size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
184
|
+
U32* nbSymbolsPtr, U32* tableLogPtr,
|
185
|
+
const void* src, size_t srcSize);
|
186
|
+
|
187
|
+
/** HUF_readCTable() :
|
188
|
+
* Loading a CTable saved with HUF_writeCTable() */
|
189
|
+
size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize);
|
190
|
+
|
191
|
+
|
192
|
+
/*
|
193
|
+
HUF_decompress() does the following:
|
194
|
+
1. select the decompression algorithm (X2, X4) based on pre-computed heuristics
|
195
|
+
2. build Huffman table from save, using HUF_readDTableXn()
|
196
|
+
3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable
|
197
|
+
*/
|
198
|
+
|
199
|
+
/** HUF_selectDecoder() :
|
200
|
+
* Tells which decoder is likely to decode faster,
|
201
|
+
* based on a set of pre-determined metrics.
|
202
|
+
* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
|
203
|
+
* Assumption : 0 < cSrcSize < dstSize <= 128 KB */
|
204
|
+
U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
|
205
|
+
|
206
|
+
size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize);
|
207
|
+
size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize);
|
208
|
+
|
209
|
+
size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
210
|
+
size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
211
|
+
size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
212
|
+
|
213
|
+
|
214
|
+
/* single stream variants */
|
215
|
+
|
216
|
+
size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
217
|
+
size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least 1024 unsigned */
|
218
|
+
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
|
219
|
+
|
220
|
+
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
|
221
|
+
size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
|
222
|
+
|
223
|
+
size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
|
224
|
+
size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
225
|
+
size_t HUF_decompress1X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
226
|
+
|
227
|
+
size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */
|
228
|
+
size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
229
|
+
size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
230
|
+
|
231
|
+
#endif /* HUF_STATIC_LINKING_ONLY */
|
232
|
+
|
233
|
+
|
234
|
+
#if defined (__cplusplus)
|
235
|
+
}
|
236
|
+
#endif
|
237
|
+
|
238
|
+
#endif /* HUF_H_298734234 */
|