extzstd 0.0.2.CONCEPT-x86-mingw32 → 0.0.3.CONCEPT-x86-mingw32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +72 -22
  3. data/Rakefile +73 -19
  4. data/contrib/zstd/README.md +68 -0
  5. data/contrib/zstd/common/bitstream.h +414 -0
  6. data/contrib/zstd/common/entropy_common.c +231 -0
  7. data/contrib/zstd/common/error_private.h +125 -0
  8. data/contrib/zstd/common/error_public.h +77 -0
  9. data/contrib/zstd/common/fse.h +628 -0
  10. data/contrib/zstd/common/fse_decompress.c +331 -0
  11. data/contrib/zstd/common/huf.h +228 -0
  12. data/contrib/zstd/common/mem.h +377 -0
  13. data/contrib/zstd/common/xxhash.c +854 -0
  14. data/contrib/zstd/common/xxhash.h +273 -0
  15. data/contrib/zstd/common/zbuff.h +197 -0
  16. data/contrib/zstd/common/zstd.h +475 -0
  17. data/contrib/zstd/common/zstd_common.c +91 -0
  18. data/contrib/zstd/common/zstd_internal.h +238 -0
  19. data/contrib/zstd/compress/fse_compress.c +807 -0
  20. data/contrib/zstd/compress/huf_compress.c +577 -0
  21. data/contrib/zstd/compress/zbuff_compress.c +327 -0
  22. data/contrib/zstd/compress/zstd_compress.c +3074 -0
  23. data/contrib/zstd/compress/zstd_opt.h +1046 -0
  24. data/contrib/zstd/decompress/huf_decompress.c +894 -0
  25. data/contrib/zstd/decompress/zbuff_decompress.c +294 -0
  26. data/contrib/zstd/decompress/zstd_decompress.c +1362 -0
  27. data/contrib/zstd/dictBuilder/divsufsort.c +1913 -0
  28. data/contrib/zstd/dictBuilder/divsufsort.h +67 -0
  29. data/contrib/zstd/dictBuilder/zdict.c +1045 -0
  30. data/contrib/zstd/dictBuilder/zdict.h +113 -0
  31. data/contrib/zstd/legacy/zstd_legacy.h +140 -0
  32. data/contrib/zstd/legacy/zstd_v01.c +2178 -0
  33. data/contrib/zstd/{zstd.h → legacy/zstd_v01.h} +46 -39
  34. data/contrib/zstd/legacy/zstd_v02.c +3748 -0
  35. data/contrib/zstd/legacy/zstd_v02.h +99 -0
  36. data/contrib/zstd/legacy/zstd_v03.c +3389 -0
  37. data/contrib/zstd/legacy/zstd_v03.h +99 -0
  38. data/contrib/zstd/legacy/zstd_v04.c +4056 -0
  39. data/contrib/zstd/legacy/zstd_v04.h +148 -0
  40. data/contrib/zstd/legacy/zstd_v05.c +4325 -0
  41. data/contrib/zstd/legacy/zstd_v05.h +171 -0
  42. data/contrib/zstd/legacy/zstd_v06.c +4581 -0
  43. data/contrib/zstd/legacy/zstd_v06.h +185 -0
  44. data/ext/extconf.rb +10 -12
  45. data/ext/extzstd.c +497 -144
  46. data/ext/extzstd.h +127 -22
  47. data/ext/extzstd_buffered.c +265 -0
  48. data/ext/extzstd_nogvls.h +174 -0
  49. data/ext/zstd_amalgam.c +18 -0
  50. data/ext/zstd_legacy_v01.c +1 -0
  51. data/ext/zstd_legacy_v02.c +1 -0
  52. data/ext/zstd_legacy_v03.c +1 -0
  53. data/ext/zstd_legacy_v04.c +1 -0
  54. data/ext/zstd_legacy_v05.c +1 -0
  55. data/ext/zstd_legacy_v06.c +1 -0
  56. data/gemstub.rb +17 -1
  57. data/lib/2.0/extzstd.so +0 -0
  58. data/lib/2.1/extzstd.so +0 -0
  59. data/lib/2.2/extzstd.so +0 -0
  60. data/lib/2.3/extzstd.so +0 -0
  61. data/lib/extzstd.rb +197 -77
  62. data/lib/extzstd/version.rb +1 -1
  63. data/test/test_basic.rb +41 -0
  64. metadata +71 -20
  65. data/contrib/zstd/Makefile +0 -115
  66. data/contrib/zstd/fse.c +0 -2466
  67. data/contrib/zstd/fse.h +0 -320
  68. data/contrib/zstd/fse_static.h +0 -282
  69. data/contrib/zstd/libzstd.pc.in +0 -14
  70. data/contrib/zstd/zstd.c +0 -1768
  71. data/contrib/zstd/zstd_static.h +0 -89
  72. data/ext/extzstd-stream.c +0 -398
@@ -0,0 +1,294 @@
1
+ /*
2
+ Buffered version of Zstd compression library
3
+ Copyright (C) 2015-2016, Yann Collet.
4
+
5
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
6
+
7
+ Redistribution and use in source and binary forms, with or without
8
+ modification, are permitted provided that the following conditions are
9
+ met:
10
+ * Redistributions of source code must retain the above copyright
11
+ notice, this list of conditions and the following disclaimer.
12
+ * Redistributions in binary form must reproduce the above
13
+ copyright notice, this list of conditions and the following disclaimer
14
+ in the documentation and/or other materials provided with the
15
+ distribution.
16
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
+
28
+ You can contact the author at :
29
+ - zstd homepage : http://www.zstd.net/
30
+ */
31
+
32
+
33
+ /* *************************************
34
+ * Dependencies
35
+ ***************************************/
36
+ #include <stdlib.h>
37
+ #include "error_private.h"
38
+ #include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize, ZSTD_BLOCKSIZE_MAX */
39
+ #define ZBUFF_STATIC_LINKING_ONLY
40
+ #include "zbuff.h"
41
+
42
+
43
+ /*-***************************************************************************
44
+ * Streaming decompression howto
45
+ *
46
+ * A ZBUFF_DCtx object is required to track streaming operations.
47
+ * Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources.
48
+ * Use ZBUFF_decompressInit() to start a new decompression operation,
49
+ * or ZBUFF_decompressInitDictionary() if decompression requires a dictionary.
50
+ * Note that ZBUFF_DCtx objects can be re-init multiple times.
51
+ *
52
+ * Use ZBUFF_decompressContinue() repetitively to consume your input.
53
+ * *srcSizePtr and *dstCapacityPtr can be any size.
54
+ * The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
55
+ * Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
56
+ * The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst.
57
+ * @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency),
58
+ * or 0 when a frame is completely decoded,
59
+ * or an error code, which can be tested using ZBUFF_isError().
60
+ *
61
+ * Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize()
62
+ * output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
63
+ * input : ZBUFF_recommendedDInSize == 128KB + 3;
64
+ * just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
65
+ * *******************************************************************************/
66
+
67
+ typedef enum { ZBUFFds_init, ZBUFFds_loadHeader,
68
+ ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage;
69
+
70
+ /* *** Resource management *** */
71
+ struct ZBUFF_DCtx_s {
72
+ ZSTD_DCtx* zd;
73
+ ZSTD_frameParams fParams;
74
+ ZBUFF_dStage stage;
75
+ char* inBuff;
76
+ size_t inBuffSize;
77
+ size_t inPos;
78
+ char* outBuff;
79
+ size_t outBuffSize;
80
+ size_t outStart;
81
+ size_t outEnd;
82
+ size_t blockSize;
83
+ BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
84
+ size_t lhSize;
85
+ ZSTD_customMem customMem;
86
+ }; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */
87
+
88
+
89
+ ZBUFF_DCtx* ZBUFF_createDCtx(void)
90
+ {
91
+ return ZBUFF_createDCtx_advanced(defaultCustomMem);
92
+ }
93
+
94
+ ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem)
95
+ {
96
+ ZBUFF_DCtx* zbd;
97
+
98
+ if (!customMem.customAlloc && !customMem.customFree)
99
+ customMem = defaultCustomMem;
100
+
101
+ if (!customMem.customAlloc || !customMem.customFree)
102
+ return NULL;
103
+
104
+ zbd = (ZBUFF_DCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_DCtx));
105
+ if (zbd==NULL) return NULL;
106
+ memset(zbd, 0, sizeof(ZBUFF_DCtx));
107
+ memcpy(&zbd->customMem, &customMem, sizeof(ZSTD_customMem));
108
+ zbd->zd = ZSTD_createDCtx_advanced(customMem);
109
+ if (zbd->zd == NULL) { ZBUFF_freeDCtx(zbd); return NULL; }
110
+ zbd->stage = ZBUFFds_init;
111
+ return zbd;
112
+ }
113
+
114
+ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd)
115
+ {
116
+ if (zbd==NULL) return 0; /* support free on null */
117
+ ZSTD_freeDCtx(zbd->zd);
118
+ if (zbd->inBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff);
119
+ if (zbd->outBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff);
120
+ zbd->customMem.customFree(zbd->customMem.opaque, zbd);
121
+ return 0;
122
+ }
123
+
124
+
125
+ /* *** Initialization *** */
126
+
127
+ size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbd, const void* dict, size_t dictSize)
128
+ {
129
+ zbd->stage = ZBUFFds_loadHeader;
130
+ zbd->lhSize = zbd->inPos = zbd->outStart = zbd->outEnd = 0;
131
+ return ZSTD_decompressBegin_usingDict(zbd->zd, dict, dictSize);
132
+ }
133
+
134
+ size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbd)
135
+ {
136
+ return ZBUFF_decompressInitDictionary(zbd, NULL, 0);
137
+ }
138
+
139
+
140
+ /* internal util function */
141
+ MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
142
+ {
143
+ size_t const length = MIN(dstCapacity, srcSize);
144
+ memcpy(dst, src, length);
145
+ return length;
146
+ }
147
+
148
+
149
+ /* *** Decompression *** */
150
+
151
+ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
152
+ void* dst, size_t* dstCapacityPtr,
153
+ const void* src, size_t* srcSizePtr)
154
+ {
155
+ const char* const istart = (const char*)src;
156
+ const char* const iend = istart + *srcSizePtr;
157
+ const char* ip = istart;
158
+ char* const ostart = (char*)dst;
159
+ char* const oend = ostart + *dstCapacityPtr;
160
+ char* op = ostart;
161
+ U32 notDone = 1;
162
+
163
+ while (notDone) {
164
+ switch(zbd->stage)
165
+ {
166
+ case ZBUFFds_init :
167
+ return ERROR(init_missing);
168
+
169
+ case ZBUFFds_loadHeader :
170
+ { size_t const hSize = ZSTD_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize);
171
+ if (hSize != 0) {
172
+ size_t const toLoad = hSize - zbd->lhSize; /* if hSize!=0, hSize > zbd->lhSize */
173
+ if (ZSTD_isError(hSize)) return hSize;
174
+ if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
175
+ memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
176
+ zbd->lhSize += iend-ip;
177
+ *dstCapacityPtr = 0;
178
+ return (hSize - zbd->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
179
+ }
180
+ memcpy(zbd->headerBuffer + zbd->lhSize, ip, toLoad); zbd->lhSize = hSize; ip += toLoad;
181
+ break;
182
+ } }
183
+
184
+ /* Consume header */
185
+ { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zbd->zd); /* == ZSTD_frameHeaderSize_min */
186
+ size_t const h1Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer, h1Size);
187
+ if (ZSTD_isError(h1Result)) return h1Result;
188
+ if (h1Size < zbd->lhSize) { /* long header */
189
+ size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zbd->zd);
190
+ size_t const h2Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer+h1Size, h2Size);
191
+ if (ZSTD_isError(h2Result)) return h2Result;
192
+ } }
193
+
194
+ zbd->fParams.windowSize = MAX(zbd->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
195
+
196
+ /* Frame header instruct buffer sizes */
197
+ { size_t const blockSize = MIN(zbd->fParams.windowSize, ZSTD_BLOCKSIZE_MAX);
198
+ zbd->blockSize = blockSize;
199
+ if (zbd->inBuffSize < blockSize) {
200
+ zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff);
201
+ zbd->inBuffSize = blockSize;
202
+ zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize);
203
+ if (zbd->inBuff == NULL) return ERROR(memory_allocation);
204
+ }
205
+ { size_t const neededOutSize = zbd->fParams.windowSize + blockSize;
206
+ if (zbd->outBuffSize < neededOutSize) {
207
+ zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff);
208
+ zbd->outBuffSize = neededOutSize;
209
+ zbd->outBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, neededOutSize);
210
+ if (zbd->outBuff == NULL) return ERROR(memory_allocation);
211
+ } } }
212
+ zbd->stage = ZBUFFds_read;
213
+
214
+ case ZBUFFds_read:
215
+ { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd);
216
+ if (neededInSize==0) { /* end of frame */
217
+ zbd->stage = ZBUFFds_init;
218
+ notDone = 0;
219
+ break;
220
+ }
221
+ if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
222
+ const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd);
223
+ size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
224
+ zbd->outBuff + zbd->outStart, (isSkipFrame ? 0 : zbd->outBuffSize - zbd->outStart),
225
+ ip, neededInSize);
226
+ if (ZSTD_isError(decodedSize)) return decodedSize;
227
+ ip += neededInSize;
228
+ if (!decodedSize && !isSkipFrame) break; /* this was just a header */
229
+ zbd->outEnd = zbd->outStart + decodedSize;
230
+ zbd->stage = ZBUFFds_flush;
231
+ break;
232
+ }
233
+ if (ip==iend) { notDone = 0; break; } /* no more input */
234
+ zbd->stage = ZBUFFds_load;
235
+ }
236
+
237
+ case ZBUFFds_load:
238
+ { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd);
239
+ size_t const toLoad = neededInSize - zbd->inPos; /* should always be <= remaining space within inBuff */
240
+ size_t loadedSize;
241
+ if (toLoad > zbd->inBuffSize - zbd->inPos) return ERROR(corruption_detected); /* should never happen */
242
+ loadedSize = ZBUFF_limitCopy(zbd->inBuff + zbd->inPos, toLoad, ip, iend-ip);
243
+ ip += loadedSize;
244
+ zbd->inPos += loadedSize;
245
+ if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */
246
+
247
+ /* decode loaded input */
248
+ { const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd);
249
+ size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
250
+ zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
251
+ zbd->inBuff, neededInSize);
252
+ if (ZSTD_isError(decodedSize)) return decodedSize;
253
+ zbd->inPos = 0; /* input is consumed */
254
+ if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */
255
+ zbd->outEnd = zbd->outStart + decodedSize;
256
+ zbd->stage = ZBUFFds_flush;
257
+ // break; /* ZBUFFds_flush follows */
258
+ } }
259
+
260
+ case ZBUFFds_flush:
261
+ { size_t const toFlushSize = zbd->outEnd - zbd->outStart;
262
+ size_t const flushedSize = ZBUFF_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize);
263
+ op += flushedSize;
264
+ zbd->outStart += flushedSize;
265
+ if (flushedSize == toFlushSize) {
266
+ zbd->stage = ZBUFFds_read;
267
+ if (zbd->outStart + zbd->blockSize > zbd->outBuffSize)
268
+ zbd->outStart = zbd->outEnd = 0;
269
+ break;
270
+ }
271
+ /* cannot flush everything */
272
+ notDone = 0;
273
+ break;
274
+ }
275
+ default: return ERROR(GENERIC); /* impossible */
276
+ } }
277
+
278
+ /* result */
279
+ *srcSizePtr = ip-istart;
280
+ *dstCapacityPtr = op-ostart;
281
+ { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbd->zd);
282
+ // if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize; /* get following block header too */
283
+ nextSrcSizeHint -= zbd->inPos; /* already loaded*/
284
+ return nextSrcSizeHint;
285
+ }
286
+ }
287
+
288
+
289
+
290
+ /* *************************************
291
+ * Tool functions
292
+ ***************************************/
293
+ size_t ZBUFF_recommendedDInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize /* block header size*/ ; }
294
+ size_t ZBUFF_recommendedDOutSize(void) { return ZSTD_BLOCKSIZE_MAX; }
@@ -0,0 +1,1362 @@
1
+ /*
2
+ zstd - standard compression library
3
+ Copyright (C) 2014-2016, Yann Collet.
4
+
5
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
6
+
7
+ Redistribution and use in source and binary forms, with or without
8
+ modification, are permitted provided that the following conditions are
9
+ met:
10
+ * Redistributions of source code must retain the above copyright
11
+ notice, this list of conditions and the following disclaimer.
12
+ * Redistributions in binary form must reproduce the above
13
+ copyright notice, this list of conditions and the following disclaimer
14
+ in the documentation and/or other materials provided with the
15
+ distribution.
16
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
+
28
+ You can contact the author at :
29
+ - zstd homepage : http://www.zstd.net
30
+ */
31
+
32
+ /* ***************************************************************
33
+ * Tuning parameters
34
+ *****************************************************************/
35
+ /*!
36
+ * HEAPMODE :
37
+ * Select how default decompression function ZSTD_decompress() will allocate memory,
38
+ * in memory stack (0), or in memory heap (1, requires malloc())
39
+ */
40
+ #ifndef ZSTD_HEAPMODE
41
+ # define ZSTD_HEAPMODE 1
42
+ #endif
43
+
44
+ /*!
45
+ * LEGACY_SUPPORT :
46
+ * if set to 1, ZSTD_decompress() can decode older formats (v0.1+)
47
+ */
48
+ #ifndef ZSTD_LEGACY_SUPPORT
49
+ # define ZSTD_LEGACY_SUPPORT 0
50
+ #endif
51
+
52
+
53
+ /*-*******************************************************
54
+ * Dependencies
55
+ *********************************************************/
56
+ #include <string.h> /* memcpy, memmove, memset */
57
+ #include <stdio.h> /* debug only : printf */
58
+ #include "mem.h" /* low level memory routines */
59
+ #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
60
+ #include "xxhash.h" /* XXH64_* */
61
+ #define FSE_STATIC_LINKING_ONLY
62
+ #include "fse.h"
63
+ #define HUF_STATIC_LINKING_ONLY
64
+ #include "huf.h"
65
+ #include "zstd_internal.h"
66
+
67
+ #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
68
+ # include "zstd_legacy.h"
69
+ #endif
70
+
71
+
72
+ /*-*******************************************************
73
+ * Compiler specifics
74
+ *********************************************************/
75
+ #ifdef _MSC_VER /* Visual Studio */
76
+ # define FORCE_INLINE static __forceinline
77
+ # include <intrin.h> /* For Visual 2005 */
78
+ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
79
+ # pragma warning(disable : 4324) /* disable: C4324: padded structure */
80
+ #else
81
+ # ifdef __GNUC__
82
+ # define FORCE_INLINE static inline __attribute__((always_inline))
83
+ # else
84
+ # define FORCE_INLINE static inline
85
+ # endif
86
+ #endif
87
+
88
+
89
+ /*-*************************************
90
+ * Macros
91
+ ***************************************/
92
+ #define ZSTD_isError ERR_isError /* for inlining */
93
+ #define FSE_isError ERR_isError
94
+ #define HUF_isError ERR_isError
95
+
96
+
97
+ /*_*******************************************************
98
+ * Memory operations
99
+ **********************************************************/
100
+ static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
101
+
102
+
103
+ /*-*************************************************************
104
+ * Context management
105
+ ***************************************************************/
106
+ typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
107
+ ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock,
108
+ ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage;
109
+
110
+ struct ZSTD_DCtx_s
111
+ {
112
+ FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
113
+ FSE_DTable OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
114
+ FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
115
+ HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */
116
+ const void* previousDstEnd;
117
+ const void* base;
118
+ const void* vBase;
119
+ const void* dictEnd;
120
+ size_t expected;
121
+ U32 rep[3];
122
+ ZSTD_frameParams fParams;
123
+ blockType_t bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
124
+ ZSTD_dStage stage;
125
+ U32 litEntropy;
126
+ U32 fseEntropy;
127
+ XXH64_state_t xxhState;
128
+ size_t headerSize;
129
+ U32 dictID;
130
+ const BYTE* litPtr;
131
+ ZSTD_customMem customMem;
132
+ size_t litBufSize;
133
+ size_t litSize;
134
+ BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH];
135
+ BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
136
+ }; /* typedef'd to ZSTD_DCtx within "zstd_static.h" */
137
+
138
+ size_t ZSTD_sizeofDCtx (const ZSTD_DCtx* dctx) { return sizeof(*dctx); }
139
+
140
+ size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); }
141
+
142
+ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
143
+ {
144
+ dctx->expected = ZSTD_frameHeaderSize_min;
145
+ dctx->stage = ZSTDds_getFrameHeaderSize;
146
+ dctx->previousDstEnd = NULL;
147
+ dctx->base = NULL;
148
+ dctx->vBase = NULL;
149
+ dctx->dictEnd = NULL;
150
+ dctx->hufTable[0] = (HUF_DTable)((HufLog)*0x1000001);
151
+ dctx->litEntropy = dctx->fseEntropy = 0;
152
+ dctx->dictID = 0;
153
+ { int i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->rep[i] = repStartValue[i]; }
154
+ return 0;
155
+ }
156
+
157
+ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
158
+ {
159
+ ZSTD_DCtx* dctx;
160
+
161
+ if (!customMem.customAlloc && !customMem.customFree)
162
+ customMem = defaultCustomMem;
163
+
164
+ if (!customMem.customAlloc || !customMem.customFree)
165
+ return NULL;
166
+
167
+ dctx = (ZSTD_DCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTD_DCtx));
168
+ if (!dctx) return NULL;
169
+ memcpy(&dctx->customMem, &customMem, sizeof(ZSTD_customMem));
170
+ ZSTD_decompressBegin(dctx);
171
+ return dctx;
172
+ }
173
+
174
+ ZSTD_DCtx* ZSTD_createDCtx(void)
175
+ {
176
+ return ZSTD_createDCtx_advanced(defaultCustomMem);
177
+ }
178
+
179
+ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
180
+ {
181
+ if (dctx==NULL) return 0; /* support free on NULL */
182
+ dctx->customMem.customFree(dctx->customMem.opaque, dctx);
183
+ return 0; /* reserved as a potential error code in the future */
184
+ }
185
+
186
+ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
187
+ {
188
+ memcpy(dstDCtx, srcDCtx,
189
+ sizeof(ZSTD_DCtx) - (ZSTD_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH + ZSTD_frameHeaderSize_max)); /* no need to copy workspace */
190
+ }
191
+
192
+
193
+ /*-*************************************************************
194
+ * Decompression section
195
+ ***************************************************************/
196
+
197
+ /* Frame format description
198
+ Frame Header - [ Block Header - Block ] - Frame End
199
+ 1) Frame Header
200
+ - 4 bytes - Magic Number : ZSTD_MAGICNUMBER (defined within zstd.h)
201
+ - 1 byte - Frame Descriptor
202
+ 2) Block Header
203
+ - 3 bytes, starting with a 2-bits descriptor
204
+ Uncompressed, Compressed, Frame End, unused
205
+ 3) Block
206
+ See Block Format Description
207
+ 4) Frame End
208
+ - 3 bytes, compatible with Block Header
209
+ */
210
+
211
+
212
+ /* Frame Header :
213
+
214
+ 1 byte - FrameHeaderDescription :
215
+ bit 0-1 : dictID (0, 1, 2 or 4 bytes)
216
+ bit 2 : checksumFlag
217
+ bit 3 : reserved (must be zero)
218
+ bit 4 : reserved (unused, can be any value)
219
+ bit 5 : Single Segment (if 1, WindowLog byte is not present)
220
+ bit 6-7 : FrameContentFieldSize (0, 2, 4, or 8)
221
+ if (SkippedWindowLog && !FrameContentFieldsize) FrameContentFieldsize=1;
222
+
223
+ Optional : WindowLog (0 or 1 byte)
224
+ bit 0-2 : octal Fractional (1/8th)
225
+ bit 3-7 : Power of 2, with 0 = 1 KB (up to 2 TB)
226
+
227
+ Optional : dictID (0, 1, 2 or 4 bytes)
228
+ Automatic adaptation
229
+ 0 : no dictID
230
+ 1 : 1 - 255
231
+ 2 : 256 - 65535
232
+ 4 : all other values
233
+
234
+ Optional : content size (0, 1, 2, 4 or 8 bytes)
235
+ 0 : unknown (fcfs==0 and swl==0)
236
+ 1 : 0-255 bytes (fcfs==0 and swl==1)
237
+ 2 : 256 - 65535+256 (fcfs==1)
238
+ 4 : 0 - 4GB-1 (fcfs==2)
239
+ 8 : 0 - 16EB-1 (fcfs==3)
240
+ */
241
+
242
+
243
+ /* Compressed Block, format description
244
+
245
+ Block = Literal Section - Sequences Section
246
+ Prerequisite : size of (compressed) block, maximum size of regenerated data
247
+
248
+ 1) Literal Section
249
+
250
+ 1.1) Header : 1-5 bytes
251
+ flags: 2 bits
252
+ 00 compressed by Huff0
253
+ 01 unused
254
+ 10 is Raw (uncompressed)
255
+ 11 is Rle
256
+ Note : using 01 => Huff0 with precomputed table ?
257
+ Note : delta map ? => compressed ?
258
+
259
+ 1.1.1) Huff0-compressed literal block : 3-5 bytes
260
+ srcSize < 1 KB => 3 bytes (2-2-10-10) => single stream
261
+ srcSize < 1 KB => 3 bytes (2-2-10-10)
262
+ srcSize < 16KB => 4 bytes (2-2-14-14)
263
+ else => 5 bytes (2-2-18-18)
264
+ big endian convention
265
+
266
+ 1.1.2) Raw (uncompressed) literal block header : 1-3 bytes
267
+ size : 5 bits: (IS_RAW<<6) + (0<<4) + size
268
+ 12 bits: (IS_RAW<<6) + (2<<4) + (size>>8)
269
+ size&255
270
+ 20 bits: (IS_RAW<<6) + (3<<4) + (size>>16)
271
+ size>>8&255
272
+ size&255
273
+
274
+ 1.1.3) Rle (repeated single byte) literal block header : 1-3 bytes
275
+ size : 5 bits: (IS_RLE<<6) + (0<<4) + size
276
+ 12 bits: (IS_RLE<<6) + (2<<4) + (size>>8)
277
+ size&255
278
+ 20 bits: (IS_RLE<<6) + (3<<4) + (size>>16)
279
+ size>>8&255
280
+ size&255
281
+
282
+ 1.1.4) Huff0-compressed literal block, using precomputed CTables : 3-5 bytes
283
+ srcSize < 1 KB => 3 bytes (2-2-10-10) => single stream
284
+ srcSize < 1 KB => 3 bytes (2-2-10-10)
285
+ srcSize < 16KB => 4 bytes (2-2-14-14)
286
+ else => 5 bytes (2-2-18-18)
287
+ big endian convention
288
+
289
+ 1- CTable available (stored into workspace ?)
290
+ 2- Small input (fast heuristic ? Full comparison ? depend on clevel ?)
291
+
292
+
293
+ 1.2) Literal block content
294
+
295
+ 1.2.1) Huff0 block, using sizes from header
296
+ See Huff0 format
297
+
298
+ 1.2.2) Huff0 block, using prepared table
299
+
300
+ 1.2.3) Raw content
301
+
302
+ 1.2.4) single byte
303
+
304
+
305
+ 2) Sequences section
306
+ TO DO
307
+ */
308
+
309
+ /** ZSTD_frameHeaderSize() :
310
+ * srcSize must be >= ZSTD_frameHeaderSize_min.
311
+ * @return : size of the Frame Header */
312
+ static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
313
+ {
314
+ if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);
315
+ { BYTE const fhd = ((const BYTE*)src)[4];
316
+ U32 const dictID= fhd & 3;
317
+ U32 const directMode = (fhd >> 5) & 1;
318
+ U32 const fcsId = fhd >> 6;
319
+ return ZSTD_frameHeaderSize_min + !directMode + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId]
320
+ + (directMode && !ZSTD_fcs_fieldSize[fcsId]);
321
+ }
322
+ }
323
+
324
+
325
+ /** ZSTD_getFrameParams() :
326
+ * decode Frame Header, or require larger `srcSize`.
327
+ * @return : 0, `fparamsPtr` is correctly filled,
328
+ * >0, `srcSize` is too small, result is expected `srcSize`,
329
+ * or an error code, which can be tested using ZSTD_isError() */
330
+ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize)
331
+ {
332
+ const BYTE* ip = (const BYTE*)src;
333
+
334
+ if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_min;
335
+ if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) {
336
+ if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
337
+ if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */
338
+ memset(fparamsPtr, 0, sizeof(*fparamsPtr));
339
+ fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4);
340
+ fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */
341
+ return 0;
342
+ }
343
+ return ERROR(prefix_unknown);
344
+ }
345
+
346
+ /* ensure there is enough `srcSize` to fully read/decode frame header */
347
+ { size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize);
348
+ if (srcSize < fhsize) return fhsize; }
349
+
350
+ { BYTE const fhdByte = ip[4];
351
+ size_t pos = 5;
352
+ U32 const dictIDSizeCode = fhdByte&3;
353
+ U32 const checksumFlag = (fhdByte>>2)&1;
354
+ U32 const directMode = (fhdByte>>5)&1;
355
+ U32 const fcsID = fhdByte>>6;
356
+ U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX;
357
+ U32 windowSize = 0;
358
+ U32 dictID = 0;
359
+ U64 frameContentSize = 0;
360
+ if ((fhdByte & 0x08) != 0) return ERROR(frameParameter_unsupported); /* reserved bits, which must be zero */
361
+ if (!directMode) {
362
+ BYTE const wlByte = ip[pos++];
363
+ U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
364
+ if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported);
365
+ windowSize = (1U << windowLog);
366
+ windowSize += (windowSize >> 3) * (wlByte&7);
367
+ }
368
+
369
+ switch(dictIDSizeCode)
370
+ {
371
+ default: /* impossible */
372
+ case 0 : break;
373
+ case 1 : dictID = ip[pos]; pos++; break;
374
+ case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break;
375
+ case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break;
376
+ }
377
+ switch(fcsID)
378
+ {
379
+ default: /* impossible */
380
+ case 0 : if (directMode) frameContentSize = ip[pos]; break;
381
+ case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break;
382
+ case 2 : frameContentSize = MEM_readLE32(ip+pos); break;
383
+ case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
384
+ }
385
+ if (!windowSize) windowSize = (U32)frameContentSize;
386
+ if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported);
387
+ fparamsPtr->frameContentSize = frameContentSize;
388
+ fparamsPtr->windowSize = windowSize;
389
+ fparamsPtr->dictID = dictID;
390
+ fparamsPtr->checksumFlag = checksumFlag;
391
+ }
392
+ return 0;
393
+ }
394
+
395
+
396
+ /** ZSTD_getDecompressedSize() :
397
+ * compatible with legacy mode
398
+ * @return : decompressed size if known, 0 otherwise
399
+ note : 0 can mean any of the following :
400
+ - decompressed size is not provided within frame header
401
+ - frame header unknown / not supported
402
+ - frame header not completely provided (`srcSize` too small) */
403
+ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize)
404
+ {
405
+ #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
406
+ if (ZSTD_isLegacy(src, srcSize)) return ZSTD_getDecompressedSize_legacy(src, srcSize);
407
+ #endif
408
+ { ZSTD_frameParams fparams;
409
+ size_t const frResult = ZSTD_getFrameParams(&fparams, src, srcSize);
410
+ if (frResult!=0) return 0;
411
+ return fparams.frameContentSize;
412
+ }
413
+ }
414
+
415
+
416
+ /** ZSTD_decodeFrameHeader() :
417
+ * `srcSize` must be the size provided by ZSTD_frameHeaderSize().
418
+ * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */
419
+ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t srcSize)
420
+ {
421
+ size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, srcSize);
422
+ if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong);
423
+ if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0);
424
+ return result;
425
+ }
426
+
427
+
428
+ typedef struct
429
+ {
430
+ blockType_t blockType;
431
+ U32 origSize;
432
+ } blockProperties_t;
433
+
434
+ /*! ZSTD_getcBlockSize() :
435
+ * Provides the size of compressed block from block header `src` */
436
+ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
437
+ {
438
+ const BYTE* const in = (const BYTE* const)src;
439
+ U32 cSize;
440
+
441
+ if (srcSize < ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
442
+
443
+ bpPtr->blockType = (blockType_t)((*in) >> 6);
444
+ cSize = in[2] + (in[1]<<8) + ((in[0] & 7)<<16);
445
+ bpPtr->origSize = (bpPtr->blockType == bt_rle) ? cSize : 0;
446
+
447
+ if (bpPtr->blockType == bt_end) return 0;
448
+ if (bpPtr->blockType == bt_rle) return 1;
449
+ return cSize;
450
+ }
451
+
452
+
453
+ static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
454
+ {
455
+ if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
456
+ memcpy(dst, src, srcSize);
457
+ return srcSize;
458
+ }
459
+
460
+
461
+ /*! ZSTD_decodeLiteralsBlock() :
462
+ @return : nb of bytes read from src (< srcSize ) */
463
+ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
464
+ const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
465
+ {
466
+ const BYTE* const istart = (const BYTE*) src;
467
+
468
+ if (srcSize < MIN_CBLOCK_SIZE) return ERROR(corruption_detected);
469
+
470
+ switch((litBlockType_t)(istart[0]>> 6))
471
+ {
472
+ case lbt_huffman:
473
+ { size_t litSize, litCSize, singleStream=0;
474
+ U32 lhSize = (istart[0] >> 4) & 3;
475
+ if (srcSize < 5) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for lhSize, + cSize (+nbSeq) */
476
+ switch(lhSize)
477
+ {
478
+ case 0: case 1: default: /* note : default is impossible, since lhSize into [0..3] */
479
+ /* 2 - 2 - 10 - 10 */
480
+ lhSize=3;
481
+ singleStream = istart[0] & 16;
482
+ litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2);
483
+ litCSize = ((istart[1] & 3) << 8) + istart[2];
484
+ break;
485
+ case 2:
486
+ /* 2 - 2 - 14 - 14 */
487
+ lhSize=4;
488
+ litSize = ((istart[0] & 15) << 10) + (istart[1] << 2) + (istart[2] >> 6);
489
+ litCSize = ((istart[2] & 63) << 8) + istart[3];
490
+ break;
491
+ case 3:
492
+ /* 2 - 2 - 18 - 18 */
493
+ lhSize=5;
494
+ litSize = ((istart[0] & 15) << 14) + (istart[1] << 6) + (istart[2] >> 2);
495
+ litCSize = ((istart[2] & 3) << 16) + (istart[3] << 8) + istart[4];
496
+ break;
497
+ }
498
+ if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected);
499
+ if (litCSize + lhSize > srcSize) return ERROR(corruption_detected);
500
+
501
+ if (HUF_isError(singleStream ?
502
+ HUF_decompress1X2_DCtx(dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) :
503
+ HUF_decompress4X_hufOnly (dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) ))
504
+ return ERROR(corruption_detected);
505
+
506
+ dctx->litPtr = dctx->litBuffer;
507
+ dctx->litBufSize = ZSTD_BLOCKSIZE_MAX+8;
508
+ dctx->litSize = litSize;
509
+ dctx->litEntropy = 1;
510
+ return litCSize + lhSize;
511
+ }
512
+ case lbt_repeat:
513
+ { size_t litSize, litCSize;
514
+ U32 lhSize = ((istart[0]) >> 4) & 3;
515
+ if (lhSize != 1) /* only case supported for now : small litSize, single stream */
516
+ return ERROR(corruption_detected);
517
+ if (dctx->litEntropy==0)
518
+ return ERROR(dictionary_corrupted);
519
+
520
+ /* 2 - 2 - 10 - 10 */
521
+ lhSize=3;
522
+ litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2);
523
+ litCSize = ((istart[1] & 3) << 8) + istart[2];
524
+ if (litCSize + lhSize > srcSize) return ERROR(corruption_detected);
525
+
526
+ { size_t const errorCode = HUF_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable);
527
+ if (HUF_isError(errorCode)) return ERROR(corruption_detected);
528
+ }
529
+ dctx->litPtr = dctx->litBuffer;
530
+ dctx->litBufSize = ZSTD_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH;
531
+ dctx->litSize = litSize;
532
+ return litCSize + lhSize;
533
+ }
534
+ case lbt_raw:
535
+ { size_t litSize;
536
+ U32 lhSize = ((istart[0]) >> 4) & 3;
537
+ switch(lhSize)
538
+ {
539
+ case 0: case 1: default: /* note : default is impossible, since lhSize into [0..3] */
540
+ lhSize=1;
541
+ litSize = istart[0] & 31;
542
+ break;
543
+ case 2:
544
+ litSize = ((istart[0] & 15) << 8) + istart[1];
545
+ break;
546
+ case 3:
547
+ litSize = ((istart[0] & 15) << 16) + (istart[1] << 8) + istart[2];
548
+ break;
549
+ }
550
+
551
+ if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */
552
+ if (litSize+lhSize > srcSize) return ERROR(corruption_detected);
553
+ memcpy(dctx->litBuffer, istart+lhSize, litSize);
554
+ dctx->litPtr = dctx->litBuffer;
555
+ dctx->litBufSize = ZSTD_BLOCKSIZE_MAX+8;
556
+ dctx->litSize = litSize;
557
+ return lhSize+litSize;
558
+ }
559
+ /* direct reference into compressed stream */
560
+ dctx->litPtr = istart+lhSize;
561
+ dctx->litBufSize = srcSize-lhSize;
562
+ dctx->litSize = litSize;
563
+ return lhSize+litSize;
564
+ }
565
+ case lbt_rle:
566
+ { size_t litSize;
567
+ U32 lhSize = ((istart[0]) >> 4) & 3;
568
+ switch(lhSize)
569
+ {
570
+ case 0: case 1: default: /* note : default is impossible, since lhSize into [0..3] */
571
+ lhSize = 1;
572
+ litSize = istart[0] & 31;
573
+ break;
574
+ case 2:
575
+ litSize = ((istart[0] & 15) << 8) + istart[1];
576
+ break;
577
+ case 3:
578
+ litSize = ((istart[0] & 15) << 16) + (istart[1] << 8) + istart[2];
579
+ if (srcSize<4) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */
580
+ break;
581
+ }
582
+ if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected);
583
+ memset(dctx->litBuffer, istart[lhSize], litSize);
584
+ dctx->litPtr = dctx->litBuffer;
585
+ dctx->litBufSize = ZSTD_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH;
586
+ dctx->litSize = litSize;
587
+ return lhSize+1;
588
+ }
589
+ default:
590
+ return ERROR(corruption_detected); /* impossible */
591
+ }
592
+ }
593
+
594
+
595
+ /*! ZSTD_buildSeqTable() :
596
+ @return : nb bytes read from src,
597
+ or an error code if it fails, testable with ZSTD_isError()
598
+ */
599
+ FORCE_INLINE size_t ZSTD_buildSeqTable(FSE_DTable* DTable, U32 type, U32 max, U32 maxLog,
600
+ const void* src, size_t srcSize,
601
+ const S16* defaultNorm, U32 defaultLog, U32 flagRepeatTable)
602
+ {
603
+ switch(type)
604
+ {
605
+ case FSE_ENCODING_RLE :
606
+ if (!srcSize) return ERROR(srcSize_wrong);
607
+ if ( (*(const BYTE*)src) > max) return ERROR(corruption_detected);
608
+ FSE_buildDTable_rle(DTable, *(const BYTE*)src); /* if *src > max, data is corrupted */
609
+ return 1;
610
+ case FSE_ENCODING_RAW :
611
+ FSE_buildDTable(DTable, defaultNorm, max, defaultLog);
612
+ return 0;
613
+ case FSE_ENCODING_STATIC:
614
+ if (!flagRepeatTable) return ERROR(corruption_detected);
615
+ return 0;
616
+ default : /* impossible */
617
+ case FSE_ENCODING_DYNAMIC :
618
+ { U32 tableLog;
619
+ S16 norm[MaxSeq+1];
620
+ size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize);
621
+ if (FSE_isError(headerSize)) return ERROR(corruption_detected);
622
+ if (tableLog > maxLog) return ERROR(corruption_detected);
623
+ FSE_buildDTable(DTable, norm, max, tableLog);
624
+ return headerSize;
625
+ } }
626
+ }
627
+
628
+
629
+ size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr,
630
+ FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, U32 flagRepeatTable,
631
+ const void* src, size_t srcSize)
632
+ {
633
+ const BYTE* const istart = (const BYTE* const)src;
634
+ const BYTE* const iend = istart + srcSize;
635
+ const BYTE* ip = istart;
636
+
637
+ /* check */
638
+ if (srcSize < MIN_SEQUENCES_SIZE) return ERROR(srcSize_wrong);
639
+
640
+ /* SeqHead */
641
+ { int nbSeq = *ip++;
642
+ if (!nbSeq) { *nbSeqPtr=0; return 1; }
643
+ if (nbSeq > 0x7F) {
644
+ if (nbSeq == 0xFF)
645
+ nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2;
646
+ else
647
+ nbSeq = ((nbSeq-0x80)<<8) + *ip++;
648
+ }
649
+ *nbSeqPtr = nbSeq;
650
+ }
651
+
652
+ /* FSE table descriptors */
653
+ { U32 const LLtype = *ip >> 6;
654
+ U32 const OFtype = (*ip >> 4) & 3;
655
+ U32 const MLtype = (*ip >> 2) & 3;
656
+ ip++;
657
+
658
+ /* check */
659
+ if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
660
+
661
+ /* Build DTables */
662
+ { size_t const llhSize = ZSTD_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
663
+ if (ZSTD_isError(llhSize)) return ERROR(corruption_detected);
664
+ ip += llhSize;
665
+ }
666
+ { size_t const ofhSize = ZSTD_buildSeqTable(DTableOffb, OFtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, flagRepeatTable);
667
+ if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected);
668
+ ip += ofhSize;
669
+ }
670
+ { size_t const mlhSize = ZSTD_buildSeqTable(DTableML, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, flagRepeatTable);
671
+ if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected);
672
+ ip += mlhSize;
673
+ } }
674
+
675
+ return ip-istart;
676
+ }
677
+
678
+
679
+ typedef struct {
680
+ size_t litLength;
681
+ size_t matchLength;
682
+ size_t offset;
683
+ } seq_t;
684
+
685
+ typedef struct {
686
+ BIT_DStream_t DStream;
687
+ FSE_DState_t stateLL;
688
+ FSE_DState_t stateOffb;
689
+ FSE_DState_t stateML;
690
+ size_t prevOffset[ZSTD_REP_INIT];
691
+ } seqState_t;
692
+
693
+
694
+ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
695
+ {
696
+ seq_t seq;
697
+
698
+ U32 const llCode = FSE_peekSymbol(&(seqState->stateLL));
699
+ U32 const mlCode = FSE_peekSymbol(&(seqState->stateML));
700
+ U32 const ofCode = FSE_peekSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */
701
+
702
+ U32 const llBits = LL_bits[llCode];
703
+ U32 const mlBits = ML_bits[mlCode];
704
+ U32 const ofBits = ofCode;
705
+ U32 const totalBits = llBits+mlBits+ofBits;
706
+
707
+ static const U32 LL_base[MaxLL+1] = {
708
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
709
+ 16, 18, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000,
710
+ 0x2000, 0x4000, 0x8000, 0x10000 };
711
+
712
+ static const U32 ML_base[MaxML+1] = {
713
+ 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
714
+ 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34,
715
+ 35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803,
716
+ 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 };
717
+
718
+ static const U32 OF_base[MaxOff+1] = {
719
+ 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D,
720
+ 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD,
721
+ 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD,
722
+ 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD };
723
+
724
+ /* sequence */
725
+ { size_t offset;
726
+ if (!ofCode)
727
+ offset = 0;
728
+ else {
729
+ offset = OF_base[ofCode] + BIT_readBits(&(seqState->DStream), ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */
730
+ if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream));
731
+ }
732
+
733
+ if (ofCode <= 1) {
734
+ if ((llCode == 0) & (offset <= 1)) offset = 1-offset;
735
+ if (offset) {
736
+ size_t const temp = seqState->prevOffset[offset];
737
+ if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1];
738
+ seqState->prevOffset[1] = seqState->prevOffset[0];
739
+ seqState->prevOffset[0] = offset = temp;
740
+ } else {
741
+ offset = seqState->prevOffset[0];
742
+ }
743
+ } else {
744
+ seqState->prevOffset[2] = seqState->prevOffset[1];
745
+ seqState->prevOffset[1] = seqState->prevOffset[0];
746
+ seqState->prevOffset[0] = offset;
747
+ }
748
+ seq.offset = offset;
749
+ }
750
+
751
+ seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */
752
+ if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&(seqState->DStream));
753
+
754
+ seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0); /* <= 16 bits */
755
+ if (MEM_32bits() ||
756
+ (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&(seqState->DStream));
757
+
758
+ /* ANS state update */
759
+ FSE_updateState(&(seqState->stateLL), &(seqState->DStream)); /* <= 9 bits */
760
+ FSE_updateState(&(seqState->stateML), &(seqState->DStream)); /* <= 9 bits */
761
+ if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); /* <= 18 bits */
762
+ FSE_updateState(&(seqState->stateOffb), &(seqState->DStream)); /* <= 8 bits */
763
+
764
+ return seq;
765
+ }
766
+
767
+
768
+ FORCE_INLINE
769
+ size_t ZSTD_execSequence(BYTE* op,
770
+ BYTE* const oend, seq_t sequence,
771
+ const BYTE** litPtr, const BYTE* const litLimit_w,
772
+ const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
773
+ {
774
+ BYTE* const oLitEnd = op + sequence.litLength;
775
+ size_t const sequenceLength = sequence.litLength + sequence.matchLength;
776
+ BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
777
+ BYTE* const oend_w = oend-WILDCOPY_OVERLENGTH;
778
+ const BYTE* const iLitEnd = *litPtr + sequence.litLength;
779
+ const BYTE* match = oLitEnd - sequence.offset;
780
+
781
+ /* check */
782
+ if ((oLitEnd>oend_w) | (oMatchEnd>oend)) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
783
+ if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */
784
+
785
+ /* copy Literals */
786
+ ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
787
+ op = oLitEnd;
788
+ *litPtr = iLitEnd; /* update for next sequence */
789
+
790
+ /* copy Match */
791
+ if (sequence.offset > (size_t)(oLitEnd - base)) {
792
+ /* offset beyond prefix */
793
+ if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected);
794
+ match = dictEnd - (base-match);
795
+ if (match + sequence.matchLength <= dictEnd) {
796
+ memmove(oLitEnd, match, sequence.matchLength);
797
+ return sequenceLength;
798
+ }
799
+ /* span extDict & currentPrefixSegment */
800
+ { size_t const length1 = dictEnd - match;
801
+ memmove(oLitEnd, match, length1);
802
+ op = oLitEnd + length1;
803
+ sequence.matchLength -= length1;
804
+ match = base;
805
+ } }
806
+
807
+ /* match within prefix */
808
+ if (sequence.offset < 8) {
809
+ /* close range match, overlap */
810
+ static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
811
+ static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */
812
+ int const sub2 = dec64table[sequence.offset];
813
+ op[0] = match[0];
814
+ op[1] = match[1];
815
+ op[2] = match[2];
816
+ op[3] = match[3];
817
+ match += dec32table[sequence.offset];
818
+ ZSTD_copy4(op+4, match);
819
+ match -= sub2;
820
+ } else {
821
+ ZSTD_copy8(op, match);
822
+ }
823
+ op += 8; match += 8;
824
+
825
+ if (oMatchEnd > oend-(16-MINMATCH)) {
826
+ if (op < oend_w) {
827
+ ZSTD_wildcopy(op, match, oend_w - op);
828
+ match += oend_w - op;
829
+ op = oend_w;
830
+ }
831
+ while (op < oMatchEnd) *op++ = *match++;
832
+ } else {
833
+ ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
834
+ }
835
+ return sequenceLength;
836
+ }
837
+
838
+
839
+ static size_t ZSTD_decompressSequences(
840
+ ZSTD_DCtx* dctx,
841
+ void* dst, size_t maxDstSize,
842
+ const void* seqStart, size_t seqSize)
843
+ {
844
+ const BYTE* ip = (const BYTE*)seqStart;
845
+ const BYTE* const iend = ip + seqSize;
846
+ BYTE* const ostart = (BYTE* const)dst;
847
+ BYTE* const oend = ostart + maxDstSize;
848
+ BYTE* op = ostart;
849
+ const BYTE* litPtr = dctx->litPtr;
850
+ const BYTE* const litLimit_w = litPtr + dctx->litBufSize - WILDCOPY_OVERLENGTH;
851
+ const BYTE* const litEnd = litPtr + dctx->litSize;
852
+ FSE_DTable* DTableLL = dctx->LLTable;
853
+ FSE_DTable* DTableML = dctx->MLTable;
854
+ FSE_DTable* DTableOffb = dctx->OffTable;
855
+ const BYTE* const base = (const BYTE*) (dctx->base);
856
+ const BYTE* const vBase = (const BYTE*) (dctx->vBase);
857
+ const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
858
+ int nbSeq;
859
+
860
+ /* Build Decoding Tables */
861
+ { size_t const seqHSize = ZSTD_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, dctx->fseEntropy, ip, seqSize);
862
+ if (ZSTD_isError(seqHSize)) return seqHSize;
863
+ ip += seqHSize;
864
+ }
865
+
866
+ /* Regen sequences */
867
+ if (nbSeq) {
868
+ seqState_t seqState;
869
+ dctx->fseEntropy = 1;
870
+ { U32 i; for (i=0; i<ZSTD_REP_INIT; i++) seqState.prevOffset[i] = dctx->rep[i]; }
871
+ { size_t const errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip);
872
+ if (ERR_isError(errorCode)) return ERROR(corruption_detected); }
873
+ FSE_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL);
874
+ FSE_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb);
875
+ FSE_initDState(&(seqState.stateML), &(seqState.DStream), DTableML);
876
+
877
+ for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) {
878
+ nbSeq--;
879
+ { seq_t const sequence = ZSTD_decodeSequence(&seqState);
880
+ size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_w, base, vBase, dictEnd);
881
+ if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
882
+ op += oneSeqSize;
883
+ } }
884
+
885
+ /* check if reached exact end */
886
+ if (nbSeq) return ERROR(corruption_detected);
887
+ /* save reps for next block */
888
+ { U32 i; for (i=0; i<ZSTD_REP_INIT; i++) dctx->rep[i] = (U32)(seqState.prevOffset[i]); }
889
+ }
890
+
891
+ /* last literal segment */
892
+ { size_t const lastLLSize = litEnd - litPtr;
893
+ //if (litPtr > litEnd) return ERROR(corruption_detected); /* too many literals already used */
894
+ if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall);
895
+ memcpy(op, litPtr, lastLLSize);
896
+ op += lastLLSize;
897
+ }
898
+
899
+ return op-ostart;
900
+ }
901
+
902
+
903
+ static void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst)
904
+ {
905
+ if (dst != dctx->previousDstEnd) { /* not contiguous */
906
+ dctx->dictEnd = dctx->previousDstEnd;
907
+ dctx->vBase = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base));
908
+ dctx->base = dst;
909
+ dctx->previousDstEnd = dst;
910
+ }
911
+ }
912
+
913
+
914
+ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
915
+ void* dst, size_t dstCapacity,
916
+ const void* src, size_t srcSize)
917
+ { /* blockType == blockCompressed */
918
+ const BYTE* ip = (const BYTE*)src;
919
+
920
+ if (srcSize >= ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong);
921
+
922
+ /* Decode literals sub-block */
923
+ { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize);
924
+ if (ZSTD_isError(litCSize)) return litCSize;
925
+ ip += litCSize;
926
+ srcSize -= litCSize;
927
+ }
928
+ return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize);
929
+ }
930
+
931
+
932
+ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
933
+ void* dst, size_t dstCapacity,
934
+ const void* src, size_t srcSize)
935
+ {
936
+ size_t dSize;
937
+ ZSTD_checkContinuity(dctx, dst);
938
+ dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
939
+ dctx->previousDstEnd = (char*)dst + dSize;
940
+ return dSize;
941
+ }
942
+
943
+
944
+ /** ZSTD_insertBlock() :
945
+ insert `src` block into `dctx` history. Useful to track uncompressed blocks. */
946
+ ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize)
947
+ {
948
+ ZSTD_checkContinuity(dctx, blockStart);
949
+ dctx->previousDstEnd = (const char*)blockStart + blockSize;
950
+ return blockSize;
951
+ }
952
+
953
+
954
+ size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length)
955
+ {
956
+ if (length > dstCapacity) return ERROR(dstSize_tooSmall);
957
+ memset(dst, byte, length);
958
+ return length;
959
+ }
960
+
961
+
962
+ /*! ZSTD_decompressFrame() :
963
+ * `dctx` must be properly initialized */
964
+ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
965
+ void* dst, size_t dstCapacity,
966
+ const void* src, size_t srcSize)
967
+ {
968
+ const BYTE* ip = (const BYTE*)src;
969
+ const BYTE* const iend = ip + srcSize;
970
+ BYTE* const ostart = (BYTE* const)dst;
971
+ BYTE* const oend = ostart + dstCapacity;
972
+ BYTE* op = ostart;
973
+ size_t remainingSize = srcSize;
974
+
975
+ /* check */
976
+ if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
977
+
978
+ /* Frame Header */
979
+ { size_t const frameHeaderSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min);
980
+ if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
981
+ if (srcSize < frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
982
+ if (ZSTD_decodeFrameHeader(dctx, src, frameHeaderSize)) return ERROR(corruption_detected);
983
+ ip += frameHeaderSize; remainingSize -= frameHeaderSize;
984
+ }
985
+
986
+ /* Loop on each block */
987
+ while (1) {
988
+ size_t decodedSize;
989
+ blockProperties_t blockProperties;
990
+ size_t const cBlockSize = ZSTD_getcBlockSize(ip, iend-ip, &blockProperties);
991
+ if (ZSTD_isError(cBlockSize)) return cBlockSize;
992
+
993
+ ip += ZSTD_blockHeaderSize;
994
+ remainingSize -= ZSTD_blockHeaderSize;
995
+ if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
996
+
997
+ switch(blockProperties.blockType)
998
+ {
999
+ case bt_compressed:
1000
+ decodedSize = ZSTD_decompressBlock_internal(dctx, op, oend-op, ip, cBlockSize);
1001
+ break;
1002
+ case bt_raw :
1003
+ decodedSize = ZSTD_copyRawBlock(op, oend-op, ip, cBlockSize);
1004
+ break;
1005
+ case bt_rle :
1006
+ decodedSize = ZSTD_generateNxBytes(op, oend-op, *ip, blockProperties.origSize);
1007
+ break;
1008
+ case bt_end :
1009
+ /* end of frame */
1010
+ if (remainingSize) return ERROR(srcSize_wrong);
1011
+ decodedSize = 0;
1012
+ break;
1013
+ default:
1014
+ return ERROR(GENERIC); /* impossible */
1015
+ }
1016
+ if (cBlockSize == 0) break; /* bt_end */
1017
+
1018
+ if (ZSTD_isError(decodedSize)) return decodedSize;
1019
+ if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, op, decodedSize);
1020
+ op += decodedSize;
1021
+ ip += cBlockSize;
1022
+ remainingSize -= cBlockSize;
1023
+ }
1024
+
1025
+ return op-ostart;
1026
+ }
1027
+
1028
+
1029
+ /*! ZSTD_decompress_usingPreparedDCtx() :
1030
+ * Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded.
1031
+ * It avoids reloading the dictionary each time.
1032
+ * `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict().
1033
+ * Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */
1034
+ size_t ZSTD_decompress_usingPreparedDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* refDCtx,
1035
+ void* dst, size_t dstCapacity,
1036
+ const void* src, size_t srcSize)
1037
+ {
1038
+ ZSTD_copyDCtx(dctx, refDCtx);
1039
+ ZSTD_checkContinuity(dctx, dst);
1040
+ return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
1041
+ }
1042
+
1043
+
1044
+ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
1045
+ void* dst, size_t dstCapacity,
1046
+ const void* src, size_t srcSize,
1047
+ const void* dict, size_t dictSize)
1048
+ {
1049
+ #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
1050
+ if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize);
1051
+ #endif
1052
+ ZSTD_decompressBegin_usingDict(dctx, dict, dictSize);
1053
+ ZSTD_checkContinuity(dctx, dst);
1054
+ return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
1055
+ }
1056
+
1057
+
1058
+ size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
1059
+ {
1060
+ return ZSTD_decompress_usingDict(dctx, dst, dstCapacity, src, srcSize, NULL, 0);
1061
+ }
1062
+
1063
+
1064
+ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
1065
+ {
1066
+ #if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE==1)
1067
+ size_t regenSize;
1068
+ ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1069
+ if (dctx==NULL) return ERROR(memory_allocation);
1070
+ regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize);
1071
+ ZSTD_freeDCtx(dctx);
1072
+ return regenSize;
1073
+ #else /* stack mode */
1074
+ ZSTD_DCtx dctx;
1075
+ return ZSTD_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize);
1076
+ #endif
1077
+ }
1078
+
1079
+
1080
+ /*_******************************
1081
+ * Streaming Decompression API
1082
+ ********************************/
1083
+ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx)
1084
+ {
1085
+ return dctx->expected;
1086
+ }
1087
+
1088
+ int ZSTD_isSkipFrame(ZSTD_DCtx* dctx)
1089
+ {
1090
+ return dctx->stage == ZSTDds_skipFrame;
1091
+ }
1092
+
1093
+ /** ZSTD_decompressContinue() :
1094
+ * @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity)
1095
+ * or an error code, which can be tested using ZSTD_isError() */
1096
+ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
1097
+ {
1098
+ /* Sanity check */
1099
+ if (srcSize != dctx->expected) return ERROR(srcSize_wrong);
1100
+ if (dstCapacity) ZSTD_checkContinuity(dctx, dst);
1101
+
1102
+ switch (dctx->stage)
1103
+ {
1104
+ case ZSTDds_getFrameHeaderSize :
1105
+ if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */
1106
+ if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
1107
+ memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
1108
+ dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_min; /* magic number + skippable frame length */
1109
+ dctx->stage = ZSTDds_decodeSkippableHeader;
1110
+ return 0;
1111
+ }
1112
+ dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min);
1113
+ if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
1114
+ memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
1115
+ if (dctx->headerSize > ZSTD_frameHeaderSize_min) {
1116
+ dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_min;
1117
+ dctx->stage = ZSTDds_decodeFrameHeader;
1118
+ return 0;
1119
+ }
1120
+ dctx->expected = 0; /* not necessary to copy more */
1121
+
1122
+ case ZSTDds_decodeFrameHeader:
1123
+ { size_t result;
1124
+ memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected);
1125
+ result = ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize);
1126
+ if (ZSTD_isError(result)) return result;
1127
+ dctx->expected = ZSTD_blockHeaderSize;
1128
+ dctx->stage = ZSTDds_decodeBlockHeader;
1129
+ return 0;
1130
+ }
1131
+ case ZSTDds_decodeBlockHeader:
1132
+ { blockProperties_t bp;
1133
+ size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
1134
+ if (ZSTD_isError(cBlockSize)) return cBlockSize;
1135
+ if (bp.blockType == bt_end) {
1136
+ if (dctx->fParams.checksumFlag) {
1137
+ U64 const h64 = XXH64_digest(&dctx->xxhState);
1138
+ U32 const h32 = (U32)(h64>>11) & ((1<<22)-1);
1139
+ const BYTE* const ip = (const BYTE*)src;
1140
+ U32 const check32 = ip[2] + (ip[1] << 8) + ((ip[0] & 0x3F) << 16);
1141
+ if (check32 != h32) return ERROR(checksum_wrong);
1142
+ }
1143
+ dctx->expected = 0;
1144
+ dctx->stage = ZSTDds_getFrameHeaderSize;
1145
+ } else {
1146
+ dctx->expected = cBlockSize;
1147
+ dctx->bType = bp.blockType;
1148
+ dctx->stage = ZSTDds_decompressBlock;
1149
+ }
1150
+ return 0;
1151
+ }
1152
+ case ZSTDds_decompressBlock:
1153
+ { size_t rSize;
1154
+ switch(dctx->bType)
1155
+ {
1156
+ case bt_compressed:
1157
+ rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
1158
+ break;
1159
+ case bt_raw :
1160
+ rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize);
1161
+ break;
1162
+ case bt_rle :
1163
+ return ERROR(GENERIC); /* not yet handled */
1164
+ break;
1165
+ case bt_end : /* should never happen (filtered at phase 1) */
1166
+ rSize = 0;
1167
+ break;
1168
+ default:
1169
+ return ERROR(GENERIC); /* impossible */
1170
+ }
1171
+ dctx->stage = ZSTDds_decodeBlockHeader;
1172
+ dctx->expected = ZSTD_blockHeaderSize;
1173
+ dctx->previousDstEnd = (char*)dst + rSize;
1174
+ if (ZSTD_isError(rSize)) return rSize;
1175
+ if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize);
1176
+ return rSize;
1177
+ }
1178
+ case ZSTDds_decodeSkippableHeader:
1179
+ { memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected);
1180
+ dctx->expected = MEM_readLE32(dctx->headerBuffer + 4);
1181
+ dctx->stage = ZSTDds_skipFrame;
1182
+ return 0;
1183
+ }
1184
+ case ZSTDds_skipFrame:
1185
+ { dctx->expected = 0;
1186
+ dctx->stage = ZSTDds_getFrameHeaderSize;
1187
+ return 0;
1188
+ }
1189
+ default:
1190
+ return ERROR(GENERIC); /* impossible */
1191
+ }
1192
+ }
1193
+
1194
+
1195
+ static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
1196
+ {
1197
+ dctx->dictEnd = dctx->previousDstEnd;
1198
+ dctx->vBase = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base));
1199
+ dctx->base = dict;
1200
+ dctx->previousDstEnd = (const char*)dict + dictSize;
1201
+ return 0;
1202
+ }
1203
+
1204
+ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t const dictSize)
1205
+ {
1206
+ const BYTE* dictPtr = (const BYTE*)dict;
1207
+ const BYTE* const dictEnd = dictPtr + dictSize;
1208
+
1209
+ { size_t const hSize = HUF_readDTableX4(dctx->hufTable, dict, dictSize);
1210
+ if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
1211
+ dictPtr += hSize;
1212
+ }
1213
+
1214
+ { short offcodeNCount[MaxOff+1];
1215
+ U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog;
1216
+ size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
1217
+ if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
1218
+ { size_t const errorCode = FSE_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog);
1219
+ if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
1220
+ dictPtr += offcodeHeaderSize;
1221
+ }
1222
+
1223
+ { short matchlengthNCount[MaxML+1];
1224
+ unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
1225
+ size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
1226
+ if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
1227
+ { size_t const errorCode = FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
1228
+ if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
1229
+ dictPtr += matchlengthHeaderSize;
1230
+ }
1231
+
1232
+ { short litlengthNCount[MaxLL+1];
1233
+ unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
1234
+ size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
1235
+ if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
1236
+ { size_t const errorCode = FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
1237
+ if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
1238
+ dictPtr += litlengthHeaderSize;
1239
+ }
1240
+
1241
+ if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
1242
+ dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
1243
+ dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
1244
+ dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
1245
+ dictPtr += 12;
1246
+
1247
+ dctx->litEntropy = dctx->fseEntropy = 1;
1248
+ return dictPtr - (const BYTE*)dict;
1249
+ }
1250
+
1251
+ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
1252
+ {
1253
+ if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize);
1254
+ { U32 const magic = MEM_readLE32(dict);
1255
+ if (magic != ZSTD_DICT_MAGIC) {
1256
+ return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */
1257
+ } }
1258
+ dctx->dictID = MEM_readLE32((const char*)dict + 4);
1259
+
1260
+ /* load entropy tables */
1261
+ dict = (const char*)dict + 8;
1262
+ dictSize -= 8;
1263
+ { size_t const eSize = ZSTD_loadEntropy(dctx, dict, dictSize);
1264
+ if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted);
1265
+ dict = (const char*)dict + eSize;
1266
+ dictSize -= eSize;
1267
+ }
1268
+
1269
+ /* reference dictionary content */
1270
+ return ZSTD_refDictContent(dctx, dict, dictSize);
1271
+ }
1272
+
1273
+
1274
+ size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
1275
+ {
1276
+ { size_t const errorCode = ZSTD_decompressBegin(dctx);
1277
+ if (ZSTD_isError(errorCode)) return errorCode; }
1278
+
1279
+ if (dict && dictSize) {
1280
+ size_t const errorCode = ZSTD_decompress_insertDictionary(dctx, dict, dictSize);
1281
+ if (ZSTD_isError(errorCode)) return ERROR(dictionary_corrupted);
1282
+ }
1283
+
1284
+ return 0;
1285
+ }
1286
+
1287
+
1288
+ struct ZSTD_DDict_s {
1289
+ void* dict;
1290
+ size_t dictSize;
1291
+ ZSTD_DCtx* refContext;
1292
+ }; /* typedef'd tp ZSTD_CDict within zstd.h */
1293
+
1294
+ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_customMem customMem)
1295
+ {
1296
+ if (!customMem.customAlloc && !customMem.customFree)
1297
+ customMem = defaultCustomMem;
1298
+
1299
+ if (!customMem.customAlloc || !customMem.customFree)
1300
+ return NULL;
1301
+
1302
+ { ZSTD_DDict* const ddict = (ZSTD_DDict*) customMem.customAlloc(customMem.opaque, sizeof(*ddict));
1303
+ void* const dictContent = customMem.customAlloc(customMem.opaque, dictSize);
1304
+ ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(customMem);
1305
+
1306
+ if (!dictContent || !ddict || !dctx) {
1307
+ customMem.customFree(customMem.opaque, dictContent);
1308
+ customMem.customFree(customMem.opaque, ddict);
1309
+ customMem.customFree(customMem.opaque, dctx);
1310
+ return NULL;
1311
+ }
1312
+
1313
+ memcpy(dictContent, dict, dictSize);
1314
+ { size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dictContent, dictSize);
1315
+ if (ZSTD_isError(errorCode)) {
1316
+ customMem.customFree(customMem.opaque, dictContent);
1317
+ customMem.customFree(customMem.opaque, ddict);
1318
+ customMem.customFree(customMem.opaque, dctx);
1319
+ return NULL;
1320
+ } }
1321
+
1322
+ ddict->dict = dictContent;
1323
+ ddict->dictSize = dictSize;
1324
+ ddict->refContext = dctx;
1325
+ return ddict;
1326
+ }
1327
+ }
1328
+
1329
+ /*! ZSTD_createDDict() :
1330
+ * Create a digested dictionary, ready to start decompression without startup delay.
1331
+ * `dict` can be released after `ZSTD_DDict` creation */
1332
+ ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
1333
+ {
1334
+ ZSTD_customMem const allocator = { NULL, NULL, NULL };
1335
+ return ZSTD_createDDict_advanced(dict, dictSize, allocator);
1336
+ }
1337
+
1338
+ size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
1339
+ {
1340
+ ZSTD_freeFunction const cFree = ddict->refContext->customMem.customFree;
1341
+ void* const opaque = ddict->refContext->customMem.opaque;
1342
+ ZSTD_freeDCtx(ddict->refContext);
1343
+ cFree(opaque, ddict->dict);
1344
+ cFree(opaque, ddict);
1345
+ return 0;
1346
+ }
1347
+
1348
+ /*! ZSTD_decompress_usingDDict() :
1349
+ * Decompression using a pre-digested Dictionary
1350
+ * Use dictionary without significant overhead. */
1351
+ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
1352
+ void* dst, size_t dstCapacity,
1353
+ const void* src, size_t srcSize,
1354
+ const ZSTD_DDict* ddict)
1355
+ {
1356
+ #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
1357
+ if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dict, ddict->dictSize);
1358
+ #endif
1359
+ return ZSTD_decompress_usingPreparedDCtx(dctx, ddict->refContext,
1360
+ dst, dstCapacity,
1361
+ src, srcSize);
1362
+ }