extzstd 0.0.2.CONCEPT → 0.0.3.CONCEPT

Sign up to get free protection for your applications and to get access to all the features.
Files changed (68) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +72 -22
  3. data/Rakefile +73 -19
  4. data/contrib/zstd/README.md +68 -0
  5. data/contrib/zstd/common/bitstream.h +414 -0
  6. data/contrib/zstd/common/entropy_common.c +231 -0
  7. data/contrib/zstd/common/error_private.h +125 -0
  8. data/contrib/zstd/common/error_public.h +77 -0
  9. data/contrib/zstd/common/fse.h +628 -0
  10. data/contrib/zstd/common/fse_decompress.c +331 -0
  11. data/contrib/zstd/common/huf.h +228 -0
  12. data/contrib/zstd/common/mem.h +377 -0
  13. data/contrib/zstd/common/xxhash.c +854 -0
  14. data/contrib/zstd/common/xxhash.h +273 -0
  15. data/contrib/zstd/common/zbuff.h +197 -0
  16. data/contrib/zstd/common/zstd.h +475 -0
  17. data/contrib/zstd/common/zstd_common.c +91 -0
  18. data/contrib/zstd/common/zstd_internal.h +238 -0
  19. data/contrib/zstd/compress/fse_compress.c +807 -0
  20. data/contrib/zstd/compress/huf_compress.c +577 -0
  21. data/contrib/zstd/compress/zbuff_compress.c +327 -0
  22. data/contrib/zstd/compress/zstd_compress.c +3074 -0
  23. data/contrib/zstd/compress/zstd_opt.h +1046 -0
  24. data/contrib/zstd/decompress/huf_decompress.c +894 -0
  25. data/contrib/zstd/decompress/zbuff_decompress.c +294 -0
  26. data/contrib/zstd/decompress/zstd_decompress.c +1362 -0
  27. data/contrib/zstd/dictBuilder/divsufsort.c +1913 -0
  28. data/contrib/zstd/dictBuilder/divsufsort.h +67 -0
  29. data/contrib/zstd/dictBuilder/zdict.c +1045 -0
  30. data/contrib/zstd/dictBuilder/zdict.h +113 -0
  31. data/contrib/zstd/legacy/zstd_legacy.h +140 -0
  32. data/contrib/zstd/legacy/zstd_v01.c +2178 -0
  33. data/contrib/zstd/{zstd.h → legacy/zstd_v01.h} +46 -39
  34. data/contrib/zstd/legacy/zstd_v02.c +3748 -0
  35. data/contrib/zstd/legacy/zstd_v02.h +99 -0
  36. data/contrib/zstd/legacy/zstd_v03.c +3389 -0
  37. data/contrib/zstd/legacy/zstd_v03.h +99 -0
  38. data/contrib/zstd/legacy/zstd_v04.c +4056 -0
  39. data/contrib/zstd/legacy/zstd_v04.h +148 -0
  40. data/contrib/zstd/legacy/zstd_v05.c +4325 -0
  41. data/contrib/zstd/legacy/zstd_v05.h +171 -0
  42. data/contrib/zstd/legacy/zstd_v06.c +4581 -0
  43. data/contrib/zstd/legacy/zstd_v06.h +185 -0
  44. data/ext/extconf.rb +10 -12
  45. data/ext/extzstd.c +497 -144
  46. data/ext/extzstd.h +127 -22
  47. data/ext/extzstd_buffered.c +265 -0
  48. data/ext/extzstd_nogvls.h +174 -0
  49. data/ext/zstd_amalgam.c +18 -0
  50. data/ext/zstd_legacy_v01.c +1 -0
  51. data/ext/zstd_legacy_v02.c +1 -0
  52. data/ext/zstd_legacy_v03.c +1 -0
  53. data/ext/zstd_legacy_v04.c +1 -0
  54. data/ext/zstd_legacy_v05.c +1 -0
  55. data/ext/zstd_legacy_v06.c +1 -0
  56. data/gemstub.rb +17 -1
  57. data/lib/extzstd.rb +197 -77
  58. data/lib/extzstd/version.rb +1 -1
  59. data/test/test_basic.rb +41 -0
  60. metadata +70 -20
  61. data/contrib/zstd/Makefile +0 -115
  62. data/contrib/zstd/fse.c +0 -2466
  63. data/contrib/zstd/fse.h +0 -320
  64. data/contrib/zstd/fse_static.h +0 -282
  65. data/contrib/zstd/libzstd.pc.in +0 -14
  66. data/contrib/zstd/zstd.c +0 -1768
  67. data/contrib/zstd/zstd_static.h +0 -89
  68. data/ext/extzstd-stream.c +0 -398
@@ -1,320 +0,0 @@
1
- /* ******************************************************************
2
- FSE : Finite State Entropy coder
3
- header file
4
- Copyright (C) 2013-2015, Yann Collet.
5
-
6
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
7
-
8
- Redistribution and use in source and binary forms, with or without
9
- modification, are permitted provided that the following conditions are
10
- met:
11
-
12
- * Redistributions of source code must retain the above copyright
13
- notice, this list of conditions and the following disclaimer.
14
- * Redistributions in binary form must reproduce the above
15
- copyright notice, this list of conditions and the following disclaimer
16
- in the documentation and/or other materials provided with the
17
- distribution.
18
-
19
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
-
31
- You can contact the author at :
32
- - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
33
- - Public forum : https://groups.google.com/forum/#!forum/lz4c
34
- ****************************************************************** */
35
- #pragma once
36
-
37
- #if defined (__cplusplus)
38
- extern "C" {
39
- #endif
40
-
41
-
42
- /******************************************
43
- * Includes
44
- ******************************************/
45
- #include <stddef.h> /* size_t, ptrdiff_t */
46
-
47
-
48
- /******************************************
49
- * FSE simple functions
50
- ******************************************/
51
- size_t FSE_compress(void* dst, size_t maxDstSize,
52
- const void* src, size_t srcSize);
53
- size_t FSE_decompress(void* dst, size_t maxDstSize,
54
- const void* cSrc, size_t cSrcSize);
55
- /*
56
- FSE_compress():
57
- Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
58
- 'dst' buffer must be already allocated. Compression runs faster is maxDstSize >= FSE_compressBound(srcSize)
59
- return : size of compressed data (<= maxDstSize)
60
- Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
61
- if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead.
62
- if FSE_isError(return), compression failed (more details using FSE_getErrorName())
63
-
64
- FSE_decompress():
65
- Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',
66
- into already allocated destination buffer 'dst', of size 'maxDstSize'.
67
- return : size of regenerated data (<= maxDstSize)
68
- or an error code, which can be tested using FSE_isError()
69
-
70
- ** Important ** : FSE_decompress() doesn't decompress non-compressible nor RLE data !!!
71
- Why ? : making this distinction requires a header.
72
- Header management is intentionally delegated to the user layer, which can better manage special cases.
73
- */
74
-
75
-
76
- /******************************************
77
- * Huff0 simple functions
78
- ******************************************/
79
- size_t HUF_compress(void* dst, size_t maxDstSize,
80
- const void* src, size_t srcSize);
81
- size_t HUF_decompress(void* dst, size_t maxDstSize,
82
- const void* cSrc, size_t cSrcSize);
83
- /*
84
- HUF_compress():
85
- Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
86
- 'dst' buffer must be already allocated. Compression runs faster is maxDstSize >= HUF_compressBound(srcSize)
87
- return : size of compressed data (<= maxDstSize)
88
- Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
89
- if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
90
- if FSE_isError(return), compression failed (more details using FSE_getErrorName())
91
-
92
- HUF_decompress():
93
- Decompress Huff0 data from buffer 'cSrc', of size 'cSrcSize',
94
- into already allocated destination buffer 'dst', of size 'maxDstSize'.
95
- return : size of regenerated data (<= maxDstSize)
96
- or an error code, which can be tested using FSE_isError()
97
-
98
- ** Important ** : HUF_decompress() doesn't decompress non-compressible nor RLE data !!!
99
- */
100
-
101
-
102
- /******************************************
103
- * Tool functions
104
- ******************************************/
105
- size_t FSE_compressBound(size_t size); /* maximum compressed size */
106
-
107
- /* Error Management */
108
- unsigned FSE_isError(size_t code); /* tells if a return value is an error code */
109
- const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */
110
-
111
-
112
- /******************************************
113
- * FSE advanced functions
114
- ******************************************/
115
- /*
116
- FSE_compress2():
117
- Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog'
118
- Both parameters can be defined as '0' to mean : use default value
119
- return : size of compressed data
120
- Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!!
121
- if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
122
- if FSE_isError(return), it's an error code.
123
- */
124
- size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
125
-
126
- size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
127
-
128
-
129
- /******************************************
130
- * FSE detailed API
131
- ******************************************/
132
- /*
133
- FSE_compress() does the following:
134
- 1. count symbol occurrence from source[] into table count[]
135
- 2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
136
- 3. save normalized counters to memory buffer using writeNCount()
137
- 4. build encoding table 'CTable' from normalized counters
138
- 5. encode the data stream using encoding table 'CTable'
139
-
140
- FSE_decompress() does the following:
141
- 1. read normalized counters with readNCount()
142
- 2. build decoding table 'DTable' from normalized counters
143
- 3. decode the data stream using decoding table 'DTable'
144
-
145
- The following API allows targeting specific sub-functions for advanced tasks.
146
- For example, it's possible to compress several blocks using the same 'CTable',
147
- or to save and provide normalized distribution using external method.
148
- */
149
-
150
- /* *** COMPRESSION *** */
151
-
152
- /*
153
- FSE_count():
154
- Provides the precise count of each symbol within a table 'count'
155
- 'count' is a table of unsigned int, of minimum size (maxSymbolValuePtr[0]+1).
156
- maxSymbolValuePtr[0] will be updated if detected smaller than initially expected
157
- return : the count of the most frequent symbol (which is not identified)
158
- if return == srcSize, there is only one symbol.
159
- if FSE_isError(return), it's an error code. */
160
- size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const unsigned char* src, size_t srcSize);
161
-
162
- /*
163
- FSE_optimalTableLog():
164
- dynamically downsize 'tableLog' when conditions are met.
165
- It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
166
- return : recommended tableLog (necessarily <= initial 'tableLog') */
167
- unsigned FSE_optimalTableLog(unsigned tableLog, size_t srcSize, unsigned maxSymbolValue);
168
-
169
- /*
170
- FSE_normalizeCount():
171
- normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
172
- 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
173
- return : tableLog,
174
- or an errorCode, which can be tested using FSE_isError() */
175
- size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
176
-
177
- /*
178
- FSE_NCountWriteBound():
179
- Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'
180
- Typically useful for allocation purpose. */
181
- size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);
182
-
183
- /*
184
- FSE_writeNCount():
185
- Compactly save 'normalizedCounter' into 'buffer'.
186
- return : size of the compressed table
187
- or an errorCode, which can be tested using FSE_isError() */
188
- size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
189
-
190
-
191
- /*
192
- Constructor and Destructor of type FSE_CTable
193
- Note that its size depends on 'tableLog' and 'maxSymbolValue' */
194
- typedef unsigned FSE_CTable; /* don't allocate that. It's just a way to be more restrictive than void* */
195
- FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue);
196
- void FSE_freeCTable (FSE_CTable* ct);
197
-
198
- /*
199
- FSE_buildCTable():
200
- Builds 'ct', which must be already allocated, using FSE_createCTable()
201
- return : 0
202
- or an errorCode, which can be tested using FSE_isError() */
203
- size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
204
-
205
- /*
206
- FSE_compress_usingCTable():
207
- Compress 'src' using 'ct' into 'dst' which must be already allocated
208
- return : size of compressed data (<= maxDstSize)
209
- or 0 if compressed data could not fit into 'dst'
210
- or an errorCode, which can be tested using FSE_isError() */
211
- size_t FSE_compress_usingCTable (void* dst, size_t maxDstSize, const void* src, size_t srcSize, const FSE_CTable* ct);
212
-
213
- /*
214
- Tutorial :
215
- ----------
216
- The first step is to count all symbols. FSE_count() does this job very fast.
217
- Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells.
218
- 'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0]
219
- maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value)
220
- FSE_count() will return the number of occurrence of the most frequent symbol.
221
- This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility.
222
- If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
223
-
224
- The next step is to normalize the frequencies.
225
- FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'.
226
- It also guarantees a minimum of 1 to any Symbol with frequency >= 1.
227
- You can use 'tableLog'==0 to mean "use default tableLog value".
228
- If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(),
229
- which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default").
230
-
231
- The result of FSE_normalizeCount() will be saved into a table,
232
- called 'normalizedCounter', which is a table of signed short.
233
- 'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells.
234
- The return value is tableLog if everything proceeded as expected.
235
- It is 0 if there is a single symbol within distribution.
236
- If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()).
237
-
238
- 'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount().
239
- 'buffer' must be already allocated.
240
- For guaranteed success, buffer size must be at least FSE_headerBound().
241
- The result of the function is the number of bytes written into 'buffer'.
242
- If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small).
243
-
244
- 'normalizedCounter' can then be used to create the compression table 'CTable'.
245
- The space required by 'CTable' must be already allocated, using FSE_createCTable().
246
- You can then use FSE_buildCTable() to fill 'CTable'.
247
- If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()).
248
-
249
- 'CTable' can then be used to compress 'src', with FSE_compress_usingCTable().
250
- Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize'
251
- The function returns the size of compressed data (without header), necessarily <= maxDstSize.
252
- If it returns '0', compressed data could not fit into 'dst'.
253
- If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
254
- */
255
-
256
-
257
- /* *** DECOMPRESSION *** */
258
-
259
- /*
260
- FSE_readNCount():
261
- Read compactly saved 'normalizedCounter' from 'rBuffer'.
262
- return : size read from 'rBuffer'
263
- or an errorCode, which can be tested using FSE_isError()
264
- maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
265
- size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize);
266
-
267
- /*
268
- Constructor and Destructor of type FSE_DTable
269
- Note that its size depends on 'tableLog' */
270
- typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
271
- FSE_DTable* FSE_createDTable(unsigned tableLog);
272
- void FSE_freeDTable(FSE_DTable* dt);
273
-
274
- /*
275
- FSE_buildDTable():
276
- Builds 'dt', which must be already allocated, using FSE_createDTable()
277
- return : 0,
278
- or an errorCode, which can be tested using FSE_isError() */
279
- size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
280
-
281
- /*
282
- FSE_decompress_usingDTable():
283
- Decompress compressed source 'cSrc' of size 'cSrcSize' using 'dt'
284
- into 'dst' which must be already allocated.
285
- return : size of regenerated data (necessarily <= maxDstSize)
286
- or an errorCode, which can be tested using FSE_isError() */
287
- size_t FSE_decompress_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
288
-
289
- /*
290
- Tutorial :
291
- ----------
292
- (Note : these functions only decompress FSE-compressed blocks.
293
- If block is uncompressed, use memcpy() instead
294
- If block is a single repeated byte, use memset() instead )
295
-
296
- The first step is to obtain the normalized frequencies of symbols.
297
- This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount().
298
- 'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short.
299
- In practice, that means it's necessary to know 'maxSymbolValue' beforehand,
300
- or size the table to handle worst case situations (typically 256).
301
- FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'.
302
- The result of FSE_readNCount() is the number of bytes read from 'rBuffer'.
303
- Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that.
304
- If there is an error, the function will return an error code, which can be tested using FSE_isError().
305
-
306
- The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'.
307
- This is performed by the function FSE_buildDTable().
308
- The space required by 'FSE_DTable' must be already allocated using FSE_createDTable().
309
- If there is an error, the function will return an error code, which can be tested using FSE_isError().
310
-
311
- 'FSE_DTable' can then be used to decompress 'cSrc', with FSE_decompress_usingDTable().
312
- 'cSrcSize' must be strictly correct, otherwise decompression will fail.
313
- FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=maxDstSize).
314
- If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small)
315
- */
316
-
317
-
318
- #if defined (__cplusplus)
319
- }
320
- #endif
@@ -1,282 +0,0 @@
1
- /* ******************************************************************
2
- FSE : Finite State Entropy coder
3
- header file for static linking (only)
4
- Copyright (C) 2013-2015, Yann Collet
5
-
6
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
7
-
8
- Redistribution and use in source and binary forms, with or without
9
- modification, are permitted provided that the following conditions are
10
- met:
11
-
12
- * Redistributions of source code must retain the above copyright
13
- notice, this list of conditions and the following disclaimer.
14
- * Redistributions in binary form must reproduce the above
15
- copyright notice, this list of conditions and the following disclaimer
16
- in the documentation and/or other materials provided with the
17
- distribution.
18
-
19
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
-
31
- You can contact the author at :
32
- - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
33
- - Public forum : https://groups.google.com/forum/#!forum/lz4c
34
- ****************************************************************** */
35
- #pragma once
36
-
37
- #if defined (__cplusplus)
38
- extern "C" {
39
- #endif
40
-
41
-
42
- /******************************************
43
- * FSE API compatible with DLL
44
- ******************************************/
45
- #include "fse.h"
46
-
47
-
48
- /******************************************
49
- * Static allocation
50
- ******************************************/
51
- /* FSE buffer bounds */
52
- #define FSE_NCOUNTBOUND 512
53
- #define FSE_BLOCKBOUND(size) (size + (size>>7))
54
- #define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
55
-
56
- /* You can statically allocate FSE CTable/DTable as a table of unsigned using below macro */
57
- #define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2))
58
- #define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))
59
-
60
- /* Huff0 buffer bounds */
61
- #define HUF_CTABLEBOUND 129
62
- #define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if pre-filtered with fast heuristic */
63
- #define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
64
-
65
- /* You can statically allocate Huff0 DTable as a table of unsigned short using below macro */
66
- #define HUF_DTABLE_SIZE_U16(maxTableLog) (1 + (1<<maxTableLog))
67
- #define HUF_CREATE_STATIC_DTABLE(DTable, maxTableLog) \
68
- unsigned short DTable[HUF_DTABLE_SIZE_U16(maxTableLog)] = { maxTableLog }
69
-
70
-
71
- /******************************************
72
- * Error Management
73
- ******************************************/
74
- #define FSE_LIST_ERRORS(ITEM) \
75
- ITEM(FSE_OK_NoError) ITEM(FSE_ERROR_GENERIC) \
76
- ITEM(FSE_ERROR_tableLog_tooLarge) ITEM(FSE_ERROR_maxSymbolValue_tooLarge) ITEM(FSE_ERROR_maxSymbolValue_tooSmall) \
77
- ITEM(FSE_ERROR_dstSize_tooSmall) ITEM(FSE_ERROR_srcSize_wrong)\
78
- ITEM(FSE_ERROR_corruptionDetected) \
79
- ITEM(FSE_ERROR_maxCode)
80
-
81
- #define FSE_GENERATE_ENUM(ENUM) ENUM,
82
- typedef enum { FSE_LIST_ERRORS(FSE_GENERATE_ENUM) } FSE_errorCodes; /* enum is exposed, to detect & handle specific errors; compare function result to -enum value */
83
-
84
-
85
- /******************************************
86
- * FSE advanced API
87
- ******************************************/
88
- size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const unsigned char* src, size_t srcSize);
89
- /* same as FSE_count(), but blindly trust that all values within src are <= maxSymbolValuePtr[0] */
90
-
91
- size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
92
- /* build a fake FSE_CTable, designed to not compress an input, where each symbol uses nbBits */
93
-
94
- size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
95
- /* build a fake FSE_CTable, designed to compress always the same symbolValue */
96
-
97
- size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits);
98
- /* build a fake FSE_DTable, designed to read an uncompressed bitstream where each symbol uses nbBits */
99
-
100
- size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
101
- /* build a fake FSE_DTable, designed to always generate the same symbolValue */
102
-
103
-
104
- /******************************************
105
- * FSE symbol compression API
106
- ******************************************/
107
- /*
108
- This API consists of small unitary functions, which highly benefit from being inlined.
109
- You will want to enable link-time-optimization to ensure these functions are properly inlined in your binary.
110
- Visual seems to do it automatically.
111
- For gcc or clang, you'll need to add -flto flag at compilation and linking stages.
112
- If none of these solutions is applicable, include "fse.c" directly.
113
- */
114
-
115
- typedef struct
116
- {
117
- size_t bitContainer;
118
- int bitPos;
119
- char* startPtr;
120
- char* ptr;
121
- char* endPtr;
122
- } FSE_CStream_t;
123
-
124
- typedef struct
125
- {
126
- ptrdiff_t value;
127
- const void* stateTable;
128
- const void* symbolTT;
129
- unsigned stateLog;
130
- } FSE_CState_t;
131
-
132
- size_t FSE_initCStream(FSE_CStream_t* bitC, void* dstBuffer, size_t maxDstSize);
133
- void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct);
134
-
135
- void FSE_encodeSymbol(FSE_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol);
136
- void FSE_addBits(FSE_CStream_t* bitC, size_t value, unsigned nbBits);
137
- void FSE_flushBits(FSE_CStream_t* bitC);
138
-
139
- void FSE_flushCState(FSE_CStream_t* bitC, const FSE_CState_t* CStatePtr);
140
- size_t FSE_closeCStream(FSE_CStream_t* bitC);
141
-
142
- /*
143
- These functions are inner components of FSE_compress_usingCTable().
144
- They allow the creation of custom streams, mixing multiple tables and bit sources.
145
-
146
- A key property to keep in mind is that encoding and decoding are done **in reverse direction**.
147
- So the first symbol you will encode is the last you will decode, like a LIFO stack.
148
-
149
- You will need a few variables to track your CStream. They are :
150
-
151
- FSE_CTable ct; // Provided by FSE_buildCTable()
152
- FSE_CStream_t bitStream; // bitStream tracking structure
153
- FSE_CState_t state; // State tracking structure (can have several)
154
-
155
-
156
- The first thing to do is to init bitStream and state.
157
- size_t errorCode = FSE_initCStream(&bitStream, dstBuffer, maxDstSize);
158
- FSE_initCState(&state, ct);
159
-
160
- Note that FSE_initCStream() can produce an error code, so its result should be tested, using FSE_isError();
161
- You can then encode your input data, byte after byte.
162
- FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time.
163
- Remember decoding will be done in reverse direction.
164
- FSE_encodeByte(&bitStream, &state, symbol);
165
-
166
- At any time, you can also add any bit sequence.
167
- Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders
168
- FSE_addBits(&bitStream, bitField, nbBits);
169
-
170
- The above methods don't commit data to memory, they just store it into local register, for speed.
171
- Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
172
- Writing data to memory is a manual operation, performed by the flushBits function.
173
- FSE_flushBits(&bitStream);
174
-
175
- Your last FSE encoding operation shall be to flush your last state value(s).
176
- FSE_flushState(&bitStream, &state);
177
-
178
- Finally, you must close the bitStream.
179
- The function returns the size of CStream in bytes.
180
- If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible)
181
- If there is an error, it returns an errorCode (which can be tested using FSE_isError()).
182
- size_t size = FSE_closeCStream(&bitStream);
183
- */
184
-
185
-
186
- /******************************************
187
- * FSE symbol decompression API
188
- ******************************************/
189
- typedef struct
190
- {
191
- size_t bitContainer;
192
- unsigned bitsConsumed;
193
- const char* ptr;
194
- const char* start;
195
- } FSE_DStream_t;
196
-
197
- typedef struct
198
- {
199
- size_t state;
200
- const void* table; /* precise table may vary, depending on U16 */
201
- } FSE_DState_t;
202
-
203
-
204
- size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t srcSize);
205
- void FSE_initDState(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD, const FSE_DTable* dt);
206
-
207
- unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD);
208
- size_t FSE_readBits(FSE_DStream_t* bitD, unsigned nbBits);
209
- unsigned int FSE_reloadDStream(FSE_DStream_t* bitD);
210
-
211
- unsigned FSE_endOfDStream(const FSE_DStream_t* bitD);
212
- unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
213
-
214
- typedef enum { FSE_DStream_unfinished = 0,
215
- FSE_DStream_endOfBuffer = 1,
216
- FSE_DStream_completed = 2,
217
- FSE_DStream_tooFar = 3 } FSE_DStream_status; /* result of FSE_reloadDStream() */
218
- /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... ?! */
219
-
220
- /*
221
- Let's now decompose FSE_decompress_usingDTable() into its unitary components.
222
- You will decode FSE-encoded symbols from the bitStream,
223
- and also any other bitFields you put in, **in reverse order**.
224
-
225
- You will need a few variables to track your bitStream. They are :
226
-
227
- FSE_DStream_t DStream; // Stream context
228
- FSE_DState_t DState; // State context. Multiple ones are possible
229
- FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
230
-
231
- The first thing to do is to init the bitStream.
232
- errorCode = FSE_initDStream(&DStream, srcBuffer, srcSize);
233
-
234
- You should then retrieve your initial state(s)
235
- (in reverse flushing order if you have several ones) :
236
- errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
237
-
238
- You can then decode your data, symbol after symbol.
239
- For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
240
- Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
241
- unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
242
-
243
- You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
244
- Note : maximum allowed nbBits is 25, for 32-bits compatibility
245
- size_t bitField = FSE_readBits(&DStream, nbBits);
246
-
247
- All above operations only read from local register (which size depends on size_t).
248
- Refueling the register from memory is manually performed by the reload method.
249
- endSignal = FSE_reloadDStream(&DStream);
250
-
251
- FSE_reloadDStream() result tells if there is still some more data to read from DStream.
252
- FSE_DStream_unfinished : there is still some data left into the DStream.
253
- FSE_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
254
- FSE_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
255
- FSE_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
256
-
257
- When reaching end of buffer (FSE_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
258
- to properly detect the exact end of stream.
259
- After each decoded symbol, check if DStream is fully consumed using this simple test :
260
- FSE_reloadDStream(&DStream) >= FSE_DStream_completed
261
-
262
- When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
263
- Checking if DStream has reached its end is performed by :
264
- FSE_endOfDStream(&DStream);
265
- Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
266
- FSE_endOfDState(&DState);
267
- */
268
-
269
-
270
- /******************************************
271
- * FSE unsafe symbol API
272
- ******************************************/
273
- size_t FSE_readBitsFast(FSE_DStream_t* bitD, unsigned nbBits);
274
- /* faster, but works only if nbBits >= 1 (otherwise, result will be corrupted) */
275
-
276
- unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD);
277
- /* faster, but works only if allways nbBits >= 1 (otherwise, result will be corrupted) */
278
-
279
-
280
- #if defined (__cplusplus)
281
- }
282
- #endif