extzstd 0.0.2.CONCEPT → 0.0.3.CONCEPT
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +72 -22
- data/Rakefile +73 -19
- data/contrib/zstd/README.md +68 -0
- data/contrib/zstd/common/bitstream.h +414 -0
- data/contrib/zstd/common/entropy_common.c +231 -0
- data/contrib/zstd/common/error_private.h +125 -0
- data/contrib/zstd/common/error_public.h +77 -0
- data/contrib/zstd/common/fse.h +628 -0
- data/contrib/zstd/common/fse_decompress.c +331 -0
- data/contrib/zstd/common/huf.h +228 -0
- data/contrib/zstd/common/mem.h +377 -0
- data/contrib/zstd/common/xxhash.c +854 -0
- data/contrib/zstd/common/xxhash.h +273 -0
- data/contrib/zstd/common/zbuff.h +197 -0
- data/contrib/zstd/common/zstd.h +475 -0
- data/contrib/zstd/common/zstd_common.c +91 -0
- data/contrib/zstd/common/zstd_internal.h +238 -0
- data/contrib/zstd/compress/fse_compress.c +807 -0
- data/contrib/zstd/compress/huf_compress.c +577 -0
- data/contrib/zstd/compress/zbuff_compress.c +327 -0
- data/contrib/zstd/compress/zstd_compress.c +3074 -0
- data/contrib/zstd/compress/zstd_opt.h +1046 -0
- data/contrib/zstd/decompress/huf_decompress.c +894 -0
- data/contrib/zstd/decompress/zbuff_decompress.c +294 -0
- data/contrib/zstd/decompress/zstd_decompress.c +1362 -0
- data/contrib/zstd/dictBuilder/divsufsort.c +1913 -0
- data/contrib/zstd/dictBuilder/divsufsort.h +67 -0
- data/contrib/zstd/dictBuilder/zdict.c +1045 -0
- data/contrib/zstd/dictBuilder/zdict.h +113 -0
- data/contrib/zstd/legacy/zstd_legacy.h +140 -0
- data/contrib/zstd/legacy/zstd_v01.c +2178 -0
- data/contrib/zstd/{zstd.h → legacy/zstd_v01.h} +46 -39
- data/contrib/zstd/legacy/zstd_v02.c +3748 -0
- data/contrib/zstd/legacy/zstd_v02.h +99 -0
- data/contrib/zstd/legacy/zstd_v03.c +3389 -0
- data/contrib/zstd/legacy/zstd_v03.h +99 -0
- data/contrib/zstd/legacy/zstd_v04.c +4056 -0
- data/contrib/zstd/legacy/zstd_v04.h +148 -0
- data/contrib/zstd/legacy/zstd_v05.c +4325 -0
- data/contrib/zstd/legacy/zstd_v05.h +171 -0
- data/contrib/zstd/legacy/zstd_v06.c +4581 -0
- data/contrib/zstd/legacy/zstd_v06.h +185 -0
- data/ext/extconf.rb +10 -12
- data/ext/extzstd.c +497 -144
- data/ext/extzstd.h +127 -22
- data/ext/extzstd_buffered.c +265 -0
- data/ext/extzstd_nogvls.h +174 -0
- data/ext/zstd_amalgam.c +18 -0
- data/ext/zstd_legacy_v01.c +1 -0
- data/ext/zstd_legacy_v02.c +1 -0
- data/ext/zstd_legacy_v03.c +1 -0
- data/ext/zstd_legacy_v04.c +1 -0
- data/ext/zstd_legacy_v05.c +1 -0
- data/ext/zstd_legacy_v06.c +1 -0
- data/gemstub.rb +17 -1
- data/lib/extzstd.rb +197 -77
- data/lib/extzstd/version.rb +1 -1
- data/test/test_basic.rb +41 -0
- metadata +70 -20
- data/contrib/zstd/Makefile +0 -115
- data/contrib/zstd/fse.c +0 -2466
- data/contrib/zstd/fse.h +0 -320
- data/contrib/zstd/fse_static.h +0 -282
- data/contrib/zstd/libzstd.pc.in +0 -14
- data/contrib/zstd/zstd.c +0 -1768
- data/contrib/zstd/zstd_static.h +0 -89
- data/ext/extzstd-stream.c +0 -398
data/contrib/zstd/fse.h
DELETED
@@ -1,320 +0,0 @@
|
|
1
|
-
/* ******************************************************************
|
2
|
-
FSE : Finite State Entropy coder
|
3
|
-
header file
|
4
|
-
Copyright (C) 2013-2015, Yann Collet.
|
5
|
-
|
6
|
-
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
7
|
-
|
8
|
-
Redistribution and use in source and binary forms, with or without
|
9
|
-
modification, are permitted provided that the following conditions are
|
10
|
-
met:
|
11
|
-
|
12
|
-
* Redistributions of source code must retain the above copyright
|
13
|
-
notice, this list of conditions and the following disclaimer.
|
14
|
-
* Redistributions in binary form must reproduce the above
|
15
|
-
copyright notice, this list of conditions and the following disclaimer
|
16
|
-
in the documentation and/or other materials provided with the
|
17
|
-
distribution.
|
18
|
-
|
19
|
-
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
20
|
-
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
21
|
-
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
22
|
-
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
23
|
-
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
24
|
-
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
25
|
-
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
26
|
-
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
27
|
-
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
28
|
-
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
29
|
-
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
30
|
-
|
31
|
-
You can contact the author at :
|
32
|
-
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
33
|
-
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
34
|
-
****************************************************************** */
|
35
|
-
#pragma once
|
36
|
-
|
37
|
-
#if defined (__cplusplus)
|
38
|
-
extern "C" {
|
39
|
-
#endif
|
40
|
-
|
41
|
-
|
42
|
-
/******************************************
|
43
|
-
* Includes
|
44
|
-
******************************************/
|
45
|
-
#include <stddef.h> /* size_t, ptrdiff_t */
|
46
|
-
|
47
|
-
|
48
|
-
/******************************************
|
49
|
-
* FSE simple functions
|
50
|
-
******************************************/
|
51
|
-
size_t FSE_compress(void* dst, size_t maxDstSize,
|
52
|
-
const void* src, size_t srcSize);
|
53
|
-
size_t FSE_decompress(void* dst, size_t maxDstSize,
|
54
|
-
const void* cSrc, size_t cSrcSize);
|
55
|
-
/*
|
56
|
-
FSE_compress():
|
57
|
-
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
|
58
|
-
'dst' buffer must be already allocated. Compression runs faster is maxDstSize >= FSE_compressBound(srcSize)
|
59
|
-
return : size of compressed data (<= maxDstSize)
|
60
|
-
Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
|
61
|
-
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead.
|
62
|
-
if FSE_isError(return), compression failed (more details using FSE_getErrorName())
|
63
|
-
|
64
|
-
FSE_decompress():
|
65
|
-
Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',
|
66
|
-
into already allocated destination buffer 'dst', of size 'maxDstSize'.
|
67
|
-
return : size of regenerated data (<= maxDstSize)
|
68
|
-
or an error code, which can be tested using FSE_isError()
|
69
|
-
|
70
|
-
** Important ** : FSE_decompress() doesn't decompress non-compressible nor RLE data !!!
|
71
|
-
Why ? : making this distinction requires a header.
|
72
|
-
Header management is intentionally delegated to the user layer, which can better manage special cases.
|
73
|
-
*/
|
74
|
-
|
75
|
-
|
76
|
-
/******************************************
|
77
|
-
* Huff0 simple functions
|
78
|
-
******************************************/
|
79
|
-
size_t HUF_compress(void* dst, size_t maxDstSize,
|
80
|
-
const void* src, size_t srcSize);
|
81
|
-
size_t HUF_decompress(void* dst, size_t maxDstSize,
|
82
|
-
const void* cSrc, size_t cSrcSize);
|
83
|
-
/*
|
84
|
-
HUF_compress():
|
85
|
-
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
|
86
|
-
'dst' buffer must be already allocated. Compression runs faster is maxDstSize >= HUF_compressBound(srcSize)
|
87
|
-
return : size of compressed data (<= maxDstSize)
|
88
|
-
Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
|
89
|
-
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
|
90
|
-
if FSE_isError(return), compression failed (more details using FSE_getErrorName())
|
91
|
-
|
92
|
-
HUF_decompress():
|
93
|
-
Decompress Huff0 data from buffer 'cSrc', of size 'cSrcSize',
|
94
|
-
into already allocated destination buffer 'dst', of size 'maxDstSize'.
|
95
|
-
return : size of regenerated data (<= maxDstSize)
|
96
|
-
or an error code, which can be tested using FSE_isError()
|
97
|
-
|
98
|
-
** Important ** : HUF_decompress() doesn't decompress non-compressible nor RLE data !!!
|
99
|
-
*/
|
100
|
-
|
101
|
-
|
102
|
-
/******************************************
|
103
|
-
* Tool functions
|
104
|
-
******************************************/
|
105
|
-
size_t FSE_compressBound(size_t size); /* maximum compressed size */
|
106
|
-
|
107
|
-
/* Error Management */
|
108
|
-
unsigned FSE_isError(size_t code); /* tells if a return value is an error code */
|
109
|
-
const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */
|
110
|
-
|
111
|
-
|
112
|
-
/******************************************
|
113
|
-
* FSE advanced functions
|
114
|
-
******************************************/
|
115
|
-
/*
|
116
|
-
FSE_compress2():
|
117
|
-
Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog'
|
118
|
-
Both parameters can be defined as '0' to mean : use default value
|
119
|
-
return : size of compressed data
|
120
|
-
Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!!
|
121
|
-
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
|
122
|
-
if FSE_isError(return), it's an error code.
|
123
|
-
*/
|
124
|
-
size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
125
|
-
|
126
|
-
size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
127
|
-
|
128
|
-
|
129
|
-
/******************************************
|
130
|
-
* FSE detailed API
|
131
|
-
******************************************/
|
132
|
-
/*
|
133
|
-
FSE_compress() does the following:
|
134
|
-
1. count symbol occurrence from source[] into table count[]
|
135
|
-
2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
|
136
|
-
3. save normalized counters to memory buffer using writeNCount()
|
137
|
-
4. build encoding table 'CTable' from normalized counters
|
138
|
-
5. encode the data stream using encoding table 'CTable'
|
139
|
-
|
140
|
-
FSE_decompress() does the following:
|
141
|
-
1. read normalized counters with readNCount()
|
142
|
-
2. build decoding table 'DTable' from normalized counters
|
143
|
-
3. decode the data stream using decoding table 'DTable'
|
144
|
-
|
145
|
-
The following API allows targeting specific sub-functions for advanced tasks.
|
146
|
-
For example, it's possible to compress several blocks using the same 'CTable',
|
147
|
-
or to save and provide normalized distribution using external method.
|
148
|
-
*/
|
149
|
-
|
150
|
-
/* *** COMPRESSION *** */
|
151
|
-
|
152
|
-
/*
|
153
|
-
FSE_count():
|
154
|
-
Provides the precise count of each symbol within a table 'count'
|
155
|
-
'count' is a table of unsigned int, of minimum size (maxSymbolValuePtr[0]+1).
|
156
|
-
maxSymbolValuePtr[0] will be updated if detected smaller than initially expected
|
157
|
-
return : the count of the most frequent symbol (which is not identified)
|
158
|
-
if return == srcSize, there is only one symbol.
|
159
|
-
if FSE_isError(return), it's an error code. */
|
160
|
-
size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const unsigned char* src, size_t srcSize);
|
161
|
-
|
162
|
-
/*
|
163
|
-
FSE_optimalTableLog():
|
164
|
-
dynamically downsize 'tableLog' when conditions are met.
|
165
|
-
It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
|
166
|
-
return : recommended tableLog (necessarily <= initial 'tableLog') */
|
167
|
-
unsigned FSE_optimalTableLog(unsigned tableLog, size_t srcSize, unsigned maxSymbolValue);
|
168
|
-
|
169
|
-
/*
|
170
|
-
FSE_normalizeCount():
|
171
|
-
normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
|
172
|
-
'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
|
173
|
-
return : tableLog,
|
174
|
-
or an errorCode, which can be tested using FSE_isError() */
|
175
|
-
size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
|
176
|
-
|
177
|
-
/*
|
178
|
-
FSE_NCountWriteBound():
|
179
|
-
Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'
|
180
|
-
Typically useful for allocation purpose. */
|
181
|
-
size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);
|
182
|
-
|
183
|
-
/*
|
184
|
-
FSE_writeNCount():
|
185
|
-
Compactly save 'normalizedCounter' into 'buffer'.
|
186
|
-
return : size of the compressed table
|
187
|
-
or an errorCode, which can be tested using FSE_isError() */
|
188
|
-
size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
|
189
|
-
|
190
|
-
|
191
|
-
/*
|
192
|
-
Constructor and Destructor of type FSE_CTable
|
193
|
-
Note that its size depends on 'tableLog' and 'maxSymbolValue' */
|
194
|
-
typedef unsigned FSE_CTable; /* don't allocate that. It's just a way to be more restrictive than void* */
|
195
|
-
FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue);
|
196
|
-
void FSE_freeCTable (FSE_CTable* ct);
|
197
|
-
|
198
|
-
/*
|
199
|
-
FSE_buildCTable():
|
200
|
-
Builds 'ct', which must be already allocated, using FSE_createCTable()
|
201
|
-
return : 0
|
202
|
-
or an errorCode, which can be tested using FSE_isError() */
|
203
|
-
size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
|
204
|
-
|
205
|
-
/*
|
206
|
-
FSE_compress_usingCTable():
|
207
|
-
Compress 'src' using 'ct' into 'dst' which must be already allocated
|
208
|
-
return : size of compressed data (<= maxDstSize)
|
209
|
-
or 0 if compressed data could not fit into 'dst'
|
210
|
-
or an errorCode, which can be tested using FSE_isError() */
|
211
|
-
size_t FSE_compress_usingCTable (void* dst, size_t maxDstSize, const void* src, size_t srcSize, const FSE_CTable* ct);
|
212
|
-
|
213
|
-
/*
|
214
|
-
Tutorial :
|
215
|
-
----------
|
216
|
-
The first step is to count all symbols. FSE_count() does this job very fast.
|
217
|
-
Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells.
|
218
|
-
'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0]
|
219
|
-
maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value)
|
220
|
-
FSE_count() will return the number of occurrence of the most frequent symbol.
|
221
|
-
This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility.
|
222
|
-
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
|
223
|
-
|
224
|
-
The next step is to normalize the frequencies.
|
225
|
-
FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'.
|
226
|
-
It also guarantees a minimum of 1 to any Symbol with frequency >= 1.
|
227
|
-
You can use 'tableLog'==0 to mean "use default tableLog value".
|
228
|
-
If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(),
|
229
|
-
which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default").
|
230
|
-
|
231
|
-
The result of FSE_normalizeCount() will be saved into a table,
|
232
|
-
called 'normalizedCounter', which is a table of signed short.
|
233
|
-
'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells.
|
234
|
-
The return value is tableLog if everything proceeded as expected.
|
235
|
-
It is 0 if there is a single symbol within distribution.
|
236
|
-
If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()).
|
237
|
-
|
238
|
-
'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount().
|
239
|
-
'buffer' must be already allocated.
|
240
|
-
For guaranteed success, buffer size must be at least FSE_headerBound().
|
241
|
-
The result of the function is the number of bytes written into 'buffer'.
|
242
|
-
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small).
|
243
|
-
|
244
|
-
'normalizedCounter' can then be used to create the compression table 'CTable'.
|
245
|
-
The space required by 'CTable' must be already allocated, using FSE_createCTable().
|
246
|
-
You can then use FSE_buildCTable() to fill 'CTable'.
|
247
|
-
If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()).
|
248
|
-
|
249
|
-
'CTable' can then be used to compress 'src', with FSE_compress_usingCTable().
|
250
|
-
Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize'
|
251
|
-
The function returns the size of compressed data (without header), necessarily <= maxDstSize.
|
252
|
-
If it returns '0', compressed data could not fit into 'dst'.
|
253
|
-
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
|
254
|
-
*/
|
255
|
-
|
256
|
-
|
257
|
-
/* *** DECOMPRESSION *** */
|
258
|
-
|
259
|
-
/*
|
260
|
-
FSE_readNCount():
|
261
|
-
Read compactly saved 'normalizedCounter' from 'rBuffer'.
|
262
|
-
return : size read from 'rBuffer'
|
263
|
-
or an errorCode, which can be tested using FSE_isError()
|
264
|
-
maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
|
265
|
-
size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize);
|
266
|
-
|
267
|
-
/*
|
268
|
-
Constructor and Destructor of type FSE_DTable
|
269
|
-
Note that its size depends on 'tableLog' */
|
270
|
-
typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
|
271
|
-
FSE_DTable* FSE_createDTable(unsigned tableLog);
|
272
|
-
void FSE_freeDTable(FSE_DTable* dt);
|
273
|
-
|
274
|
-
/*
|
275
|
-
FSE_buildDTable():
|
276
|
-
Builds 'dt', which must be already allocated, using FSE_createDTable()
|
277
|
-
return : 0,
|
278
|
-
or an errorCode, which can be tested using FSE_isError() */
|
279
|
-
size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
|
280
|
-
|
281
|
-
/*
|
282
|
-
FSE_decompress_usingDTable():
|
283
|
-
Decompress compressed source 'cSrc' of size 'cSrcSize' using 'dt'
|
284
|
-
into 'dst' which must be already allocated.
|
285
|
-
return : size of regenerated data (necessarily <= maxDstSize)
|
286
|
-
or an errorCode, which can be tested using FSE_isError() */
|
287
|
-
size_t FSE_decompress_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
|
288
|
-
|
289
|
-
/*
|
290
|
-
Tutorial :
|
291
|
-
----------
|
292
|
-
(Note : these functions only decompress FSE-compressed blocks.
|
293
|
-
If block is uncompressed, use memcpy() instead
|
294
|
-
If block is a single repeated byte, use memset() instead )
|
295
|
-
|
296
|
-
The first step is to obtain the normalized frequencies of symbols.
|
297
|
-
This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount().
|
298
|
-
'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short.
|
299
|
-
In practice, that means it's necessary to know 'maxSymbolValue' beforehand,
|
300
|
-
or size the table to handle worst case situations (typically 256).
|
301
|
-
FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'.
|
302
|
-
The result of FSE_readNCount() is the number of bytes read from 'rBuffer'.
|
303
|
-
Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that.
|
304
|
-
If there is an error, the function will return an error code, which can be tested using FSE_isError().
|
305
|
-
|
306
|
-
The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'.
|
307
|
-
This is performed by the function FSE_buildDTable().
|
308
|
-
The space required by 'FSE_DTable' must be already allocated using FSE_createDTable().
|
309
|
-
If there is an error, the function will return an error code, which can be tested using FSE_isError().
|
310
|
-
|
311
|
-
'FSE_DTable' can then be used to decompress 'cSrc', with FSE_decompress_usingDTable().
|
312
|
-
'cSrcSize' must be strictly correct, otherwise decompression will fail.
|
313
|
-
FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=maxDstSize).
|
314
|
-
If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small)
|
315
|
-
*/
|
316
|
-
|
317
|
-
|
318
|
-
#if defined (__cplusplus)
|
319
|
-
}
|
320
|
-
#endif
|
data/contrib/zstd/fse_static.h
DELETED
@@ -1,282 +0,0 @@
|
|
1
|
-
/* ******************************************************************
|
2
|
-
FSE : Finite State Entropy coder
|
3
|
-
header file for static linking (only)
|
4
|
-
Copyright (C) 2013-2015, Yann Collet
|
5
|
-
|
6
|
-
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
7
|
-
|
8
|
-
Redistribution and use in source and binary forms, with or without
|
9
|
-
modification, are permitted provided that the following conditions are
|
10
|
-
met:
|
11
|
-
|
12
|
-
* Redistributions of source code must retain the above copyright
|
13
|
-
notice, this list of conditions and the following disclaimer.
|
14
|
-
* Redistributions in binary form must reproduce the above
|
15
|
-
copyright notice, this list of conditions and the following disclaimer
|
16
|
-
in the documentation and/or other materials provided with the
|
17
|
-
distribution.
|
18
|
-
|
19
|
-
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
20
|
-
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
21
|
-
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
22
|
-
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
23
|
-
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
24
|
-
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
25
|
-
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
26
|
-
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
27
|
-
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
28
|
-
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
29
|
-
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
30
|
-
|
31
|
-
You can contact the author at :
|
32
|
-
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
33
|
-
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
34
|
-
****************************************************************** */
|
35
|
-
#pragma once
|
36
|
-
|
37
|
-
#if defined (__cplusplus)
|
38
|
-
extern "C" {
|
39
|
-
#endif
|
40
|
-
|
41
|
-
|
42
|
-
/******************************************
|
43
|
-
* FSE API compatible with DLL
|
44
|
-
******************************************/
|
45
|
-
#include "fse.h"
|
46
|
-
|
47
|
-
|
48
|
-
/******************************************
|
49
|
-
* Static allocation
|
50
|
-
******************************************/
|
51
|
-
/* FSE buffer bounds */
|
52
|
-
#define FSE_NCOUNTBOUND 512
|
53
|
-
#define FSE_BLOCKBOUND(size) (size + (size>>7))
|
54
|
-
#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
|
55
|
-
|
56
|
-
/* You can statically allocate FSE CTable/DTable as a table of unsigned using below macro */
|
57
|
-
#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2))
|
58
|
-
#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))
|
59
|
-
|
60
|
-
/* Huff0 buffer bounds */
|
61
|
-
#define HUF_CTABLEBOUND 129
|
62
|
-
#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if pre-filtered with fast heuristic */
|
63
|
-
#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
|
64
|
-
|
65
|
-
/* You can statically allocate Huff0 DTable as a table of unsigned short using below macro */
|
66
|
-
#define HUF_DTABLE_SIZE_U16(maxTableLog) (1 + (1<<maxTableLog))
|
67
|
-
#define HUF_CREATE_STATIC_DTABLE(DTable, maxTableLog) \
|
68
|
-
unsigned short DTable[HUF_DTABLE_SIZE_U16(maxTableLog)] = { maxTableLog }
|
69
|
-
|
70
|
-
|
71
|
-
/******************************************
|
72
|
-
* Error Management
|
73
|
-
******************************************/
|
74
|
-
#define FSE_LIST_ERRORS(ITEM) \
|
75
|
-
ITEM(FSE_OK_NoError) ITEM(FSE_ERROR_GENERIC) \
|
76
|
-
ITEM(FSE_ERROR_tableLog_tooLarge) ITEM(FSE_ERROR_maxSymbolValue_tooLarge) ITEM(FSE_ERROR_maxSymbolValue_tooSmall) \
|
77
|
-
ITEM(FSE_ERROR_dstSize_tooSmall) ITEM(FSE_ERROR_srcSize_wrong)\
|
78
|
-
ITEM(FSE_ERROR_corruptionDetected) \
|
79
|
-
ITEM(FSE_ERROR_maxCode)
|
80
|
-
|
81
|
-
#define FSE_GENERATE_ENUM(ENUM) ENUM,
|
82
|
-
typedef enum { FSE_LIST_ERRORS(FSE_GENERATE_ENUM) } FSE_errorCodes; /* enum is exposed, to detect & handle specific errors; compare function result to -enum value */
|
83
|
-
|
84
|
-
|
85
|
-
/******************************************
|
86
|
-
* FSE advanced API
|
87
|
-
******************************************/
|
88
|
-
size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const unsigned char* src, size_t srcSize);
|
89
|
-
/* same as FSE_count(), but blindly trust that all values within src are <= maxSymbolValuePtr[0] */
|
90
|
-
|
91
|
-
size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
|
92
|
-
/* build a fake FSE_CTable, designed to not compress an input, where each symbol uses nbBits */
|
93
|
-
|
94
|
-
size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
|
95
|
-
/* build a fake FSE_CTable, designed to compress always the same symbolValue */
|
96
|
-
|
97
|
-
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits);
|
98
|
-
/* build a fake FSE_DTable, designed to read an uncompressed bitstream where each symbol uses nbBits */
|
99
|
-
|
100
|
-
size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
|
101
|
-
/* build a fake FSE_DTable, designed to always generate the same symbolValue */
|
102
|
-
|
103
|
-
|
104
|
-
/******************************************
|
105
|
-
* FSE symbol compression API
|
106
|
-
******************************************/
|
107
|
-
/*
|
108
|
-
This API consists of small unitary functions, which highly benefit from being inlined.
|
109
|
-
You will want to enable link-time-optimization to ensure these functions are properly inlined in your binary.
|
110
|
-
Visual seems to do it automatically.
|
111
|
-
For gcc or clang, you'll need to add -flto flag at compilation and linking stages.
|
112
|
-
If none of these solutions is applicable, include "fse.c" directly.
|
113
|
-
*/
|
114
|
-
|
115
|
-
typedef struct
|
116
|
-
{
|
117
|
-
size_t bitContainer;
|
118
|
-
int bitPos;
|
119
|
-
char* startPtr;
|
120
|
-
char* ptr;
|
121
|
-
char* endPtr;
|
122
|
-
} FSE_CStream_t;
|
123
|
-
|
124
|
-
typedef struct
|
125
|
-
{
|
126
|
-
ptrdiff_t value;
|
127
|
-
const void* stateTable;
|
128
|
-
const void* symbolTT;
|
129
|
-
unsigned stateLog;
|
130
|
-
} FSE_CState_t;
|
131
|
-
|
132
|
-
size_t FSE_initCStream(FSE_CStream_t* bitC, void* dstBuffer, size_t maxDstSize);
|
133
|
-
void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct);
|
134
|
-
|
135
|
-
void FSE_encodeSymbol(FSE_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol);
|
136
|
-
void FSE_addBits(FSE_CStream_t* bitC, size_t value, unsigned nbBits);
|
137
|
-
void FSE_flushBits(FSE_CStream_t* bitC);
|
138
|
-
|
139
|
-
void FSE_flushCState(FSE_CStream_t* bitC, const FSE_CState_t* CStatePtr);
|
140
|
-
size_t FSE_closeCStream(FSE_CStream_t* bitC);
|
141
|
-
|
142
|
-
/*
|
143
|
-
These functions are inner components of FSE_compress_usingCTable().
|
144
|
-
They allow the creation of custom streams, mixing multiple tables and bit sources.
|
145
|
-
|
146
|
-
A key property to keep in mind is that encoding and decoding are done **in reverse direction**.
|
147
|
-
So the first symbol you will encode is the last you will decode, like a LIFO stack.
|
148
|
-
|
149
|
-
You will need a few variables to track your CStream. They are :
|
150
|
-
|
151
|
-
FSE_CTable ct; // Provided by FSE_buildCTable()
|
152
|
-
FSE_CStream_t bitStream; // bitStream tracking structure
|
153
|
-
FSE_CState_t state; // State tracking structure (can have several)
|
154
|
-
|
155
|
-
|
156
|
-
The first thing to do is to init bitStream and state.
|
157
|
-
size_t errorCode = FSE_initCStream(&bitStream, dstBuffer, maxDstSize);
|
158
|
-
FSE_initCState(&state, ct);
|
159
|
-
|
160
|
-
Note that FSE_initCStream() can produce an error code, so its result should be tested, using FSE_isError();
|
161
|
-
You can then encode your input data, byte after byte.
|
162
|
-
FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time.
|
163
|
-
Remember decoding will be done in reverse direction.
|
164
|
-
FSE_encodeByte(&bitStream, &state, symbol);
|
165
|
-
|
166
|
-
At any time, you can also add any bit sequence.
|
167
|
-
Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders
|
168
|
-
FSE_addBits(&bitStream, bitField, nbBits);
|
169
|
-
|
170
|
-
The above methods don't commit data to memory, they just store it into local register, for speed.
|
171
|
-
Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
172
|
-
Writing data to memory is a manual operation, performed by the flushBits function.
|
173
|
-
FSE_flushBits(&bitStream);
|
174
|
-
|
175
|
-
Your last FSE encoding operation shall be to flush your last state value(s).
|
176
|
-
FSE_flushState(&bitStream, &state);
|
177
|
-
|
178
|
-
Finally, you must close the bitStream.
|
179
|
-
The function returns the size of CStream in bytes.
|
180
|
-
If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible)
|
181
|
-
If there is an error, it returns an errorCode (which can be tested using FSE_isError()).
|
182
|
-
size_t size = FSE_closeCStream(&bitStream);
|
183
|
-
*/
|
184
|
-
|
185
|
-
|
186
|
-
/******************************************
|
187
|
-
* FSE symbol decompression API
|
188
|
-
******************************************/
|
189
|
-
typedef struct
|
190
|
-
{
|
191
|
-
size_t bitContainer;
|
192
|
-
unsigned bitsConsumed;
|
193
|
-
const char* ptr;
|
194
|
-
const char* start;
|
195
|
-
} FSE_DStream_t;
|
196
|
-
|
197
|
-
typedef struct
|
198
|
-
{
|
199
|
-
size_t state;
|
200
|
-
const void* table; /* precise table may vary, depending on U16 */
|
201
|
-
} FSE_DState_t;
|
202
|
-
|
203
|
-
|
204
|
-
size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t srcSize);
|
205
|
-
void FSE_initDState(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD, const FSE_DTable* dt);
|
206
|
-
|
207
|
-
unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD);
|
208
|
-
size_t FSE_readBits(FSE_DStream_t* bitD, unsigned nbBits);
|
209
|
-
unsigned int FSE_reloadDStream(FSE_DStream_t* bitD);
|
210
|
-
|
211
|
-
unsigned FSE_endOfDStream(const FSE_DStream_t* bitD);
|
212
|
-
unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
|
213
|
-
|
214
|
-
typedef enum { FSE_DStream_unfinished = 0,
|
215
|
-
FSE_DStream_endOfBuffer = 1,
|
216
|
-
FSE_DStream_completed = 2,
|
217
|
-
FSE_DStream_tooFar = 3 } FSE_DStream_status; /* result of FSE_reloadDStream() */
|
218
|
-
/* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... ?! */
|
219
|
-
|
220
|
-
/*
|
221
|
-
Let's now decompose FSE_decompress_usingDTable() into its unitary components.
|
222
|
-
You will decode FSE-encoded symbols from the bitStream,
|
223
|
-
and also any other bitFields you put in, **in reverse order**.
|
224
|
-
|
225
|
-
You will need a few variables to track your bitStream. They are :
|
226
|
-
|
227
|
-
FSE_DStream_t DStream; // Stream context
|
228
|
-
FSE_DState_t DState; // State context. Multiple ones are possible
|
229
|
-
FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
|
230
|
-
|
231
|
-
The first thing to do is to init the bitStream.
|
232
|
-
errorCode = FSE_initDStream(&DStream, srcBuffer, srcSize);
|
233
|
-
|
234
|
-
You should then retrieve your initial state(s)
|
235
|
-
(in reverse flushing order if you have several ones) :
|
236
|
-
errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
|
237
|
-
|
238
|
-
You can then decode your data, symbol after symbol.
|
239
|
-
For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
|
240
|
-
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
|
241
|
-
unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
|
242
|
-
|
243
|
-
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
|
244
|
-
Note : maximum allowed nbBits is 25, for 32-bits compatibility
|
245
|
-
size_t bitField = FSE_readBits(&DStream, nbBits);
|
246
|
-
|
247
|
-
All above operations only read from local register (which size depends on size_t).
|
248
|
-
Refueling the register from memory is manually performed by the reload method.
|
249
|
-
endSignal = FSE_reloadDStream(&DStream);
|
250
|
-
|
251
|
-
FSE_reloadDStream() result tells if there is still some more data to read from DStream.
|
252
|
-
FSE_DStream_unfinished : there is still some data left into the DStream.
|
253
|
-
FSE_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
|
254
|
-
FSE_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
255
|
-
FSE_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
256
|
-
|
257
|
-
When reaching end of buffer (FSE_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
|
258
|
-
to properly detect the exact end of stream.
|
259
|
-
After each decoded symbol, check if DStream is fully consumed using this simple test :
|
260
|
-
FSE_reloadDStream(&DStream) >= FSE_DStream_completed
|
261
|
-
|
262
|
-
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
|
263
|
-
Checking if DStream has reached its end is performed by :
|
264
|
-
FSE_endOfDStream(&DStream);
|
265
|
-
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
|
266
|
-
FSE_endOfDState(&DState);
|
267
|
-
*/
|
268
|
-
|
269
|
-
|
270
|
-
/******************************************
|
271
|
-
* FSE unsafe symbol API
|
272
|
-
******************************************/
|
273
|
-
size_t FSE_readBitsFast(FSE_DStream_t* bitD, unsigned nbBits);
|
274
|
-
/* faster, but works only if nbBits >= 1 (otherwise, result will be corrupted) */
|
275
|
-
|
276
|
-
unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD);
|
277
|
-
/* faster, but works only if allways nbBits >= 1 (otherwise, result will be corrupted) */
|
278
|
-
|
279
|
-
|
280
|
-
#if defined (__cplusplus)
|
281
|
-
}
|
282
|
-
#endif
|