zstdlib 0.6.0-x86-mingw32 → 0.7.0-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGES.md +5 -0
- data/ext/zstdlib/extconf.rb +1 -1
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/bitstream.h +31 -37
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/compiler.h +19 -3
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/cpu.h +1 -1
- data/ext/zstdlib/zstd-1.4.5/lib/common/debug.c +24 -0
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/debug.h +11 -31
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/entropy_common.c +13 -33
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/error_private.c +2 -1
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/error_private.h +6 -2
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/fse.h +11 -31
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/fse_decompress.c +12 -37
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/huf.h +15 -33
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/mem.h +1 -1
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/pool.c +1 -1
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/pool.h +2 -2
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/threading.c +4 -3
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/threading.h +4 -3
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/xxhash.c +15 -33
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/xxhash.h +11 -31
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/zstd_common.c +1 -1
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/zstd_errors.h +2 -1
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/zstd_internal.h +112 -15
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/fse_compress.c +17 -40
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/hist.c +15 -35
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/hist.h +12 -32
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/huf_compress.c +92 -92
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_compress.c +450 -275
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_compress_internal.h +136 -14
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_compress_literals.c +10 -6
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_compress_literals.h +1 -1
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_compress_sequences.c +24 -20
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_compress_sequences.h +10 -3
- data/ext/zstdlib/zstd-1.4.5/lib/compress/zstd_compress_superblock.c +845 -0
- data/ext/zstdlib/zstd-1.4.5/lib/compress/zstd_compress_superblock.h +32 -0
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_cwksp.h +3 -13
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_double_fast.c +11 -8
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_double_fast.h +2 -2
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_fast.c +36 -24
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_fast.h +2 -2
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_lazy.c +34 -11
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_lazy.h +1 -1
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_ldm.c +27 -5
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_ldm.h +7 -2
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_opt.c +38 -84
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_opt.h +1 -1
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstdmt_compress.c +48 -21
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstdmt_compress.h +2 -2
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/huf_decompress.c +76 -62
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/zstd_ddict.c +12 -8
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/zstd_ddict.h +2 -2
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/zstd_decompress.c +264 -148
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/zstd_decompress_block.c +312 -203
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/zstd_decompress_block.h +3 -3
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/zstd_decompress_internal.h +18 -4
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/zstd.h +62 -21
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/gzclose.c +0 -0
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/gzcompatibility.h +1 -1
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/gzguts.h +0 -0
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/gzlib.c +0 -0
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/gzread.c +0 -0
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/gzwrite.c +0 -0
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/zstd_zlibwrapper.c +1 -1
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/zstd_zlibwrapper.h +1 -1
- data/lib/2.2/zstdlib.so +0 -0
- data/lib/2.3/zstdlib.so +0 -0
- data/lib/2.4/zstdlib.so +0 -0
- data/lib/2.5/zstdlib.so +0 -0
- data/lib/2.6/zstdlib.so +0 -0
- data/lib/2.7/zstdlib.so +0 -0
- metadata +64 -62
- data/ext/zstdlib/zstd-1.4.4/lib/common/debug.c +0 -44
@@ -1,47 +1,27 @@
|
|
1
1
|
/* ******************************************************************
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
notice, this list of conditions and the following disclaimer.
|
14
|
-
* Redistributions in binary form must reproduce the above
|
15
|
-
copyright notice, this list of conditions and the following disclaimer
|
16
|
-
in the documentation and/or other materials provided with the
|
17
|
-
distribution.
|
18
|
-
|
19
|
-
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
20
|
-
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
21
|
-
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
22
|
-
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
23
|
-
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
24
|
-
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
25
|
-
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
26
|
-
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
27
|
-
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
28
|
-
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
29
|
-
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
30
|
-
|
31
|
-
You can contact the author at :
|
32
|
-
- FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
2
|
+
* huff0 huffman decoder,
|
3
|
+
* part of Finite State Entropy library
|
4
|
+
* Copyright (c) 2013-2020, Yann Collet, Facebook, Inc.
|
5
|
+
*
|
6
|
+
* You can contact the author at :
|
7
|
+
* - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
8
|
+
*
|
9
|
+
* This source code is licensed under both the BSD-style license (found in the
|
10
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
11
|
+
* in the COPYING file in the root directory of this source tree).
|
12
|
+
* You may select, at your option, one of the above-listed licenses.
|
33
13
|
****************************************************************** */
|
34
14
|
|
35
15
|
/* **************************************************************
|
36
16
|
* Dependencies
|
37
17
|
****************************************************************/
|
38
18
|
#include <string.h> /* memcpy, memset */
|
39
|
-
#include "compiler.h"
|
40
|
-
#include "bitstream.h" /* BIT_* */
|
41
|
-
#include "fse.h" /* to compress headers */
|
19
|
+
#include "../common/compiler.h"
|
20
|
+
#include "../common/bitstream.h" /* BIT_* */
|
21
|
+
#include "../common/fse.h" /* to compress headers */
|
42
22
|
#define HUF_STATIC_LINKING_ONLY
|
43
|
-
#include "huf.h"
|
44
|
-
#include "error_private.h"
|
23
|
+
#include "../common/huf.h"
|
24
|
+
#include "../common/error_private.h"
|
45
25
|
|
46
26
|
/* **************************************************************
|
47
27
|
* Macros
|
@@ -61,9 +41,6 @@
|
|
61
41
|
* Error Management
|
62
42
|
****************************************************************/
|
63
43
|
#define HUF_isError ERR_isError
|
64
|
-
#ifndef CHECK_F
|
65
|
-
#define CHECK_F(f) { size_t const err_ = (f); if (HUF_isError(err_)) return err_; }
|
66
|
-
#endif
|
67
44
|
|
68
45
|
|
69
46
|
/* **************************************************************
|
@@ -181,17 +158,29 @@ size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize
|
|
181
158
|
|
182
159
|
/* fill DTable */
|
183
160
|
{ U32 n;
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
161
|
+
size_t const nEnd = nbSymbols;
|
162
|
+
for (n=0; n<nEnd; n++) {
|
163
|
+
size_t const w = huffWeight[n];
|
164
|
+
size_t const length = (1 << w) >> 1;
|
165
|
+
size_t const uStart = rankVal[w];
|
166
|
+
size_t const uEnd = uStart + length;
|
167
|
+
size_t u;
|
188
168
|
HUF_DEltX1 D;
|
189
|
-
D.byte = (BYTE)n;
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
169
|
+
D.byte = (BYTE)n;
|
170
|
+
D.nbBits = (BYTE)(tableLog + 1 - w);
|
171
|
+
rankVal[w] = (U32)uEnd;
|
172
|
+
if (length < 4) {
|
173
|
+
/* Use length in the loop bound so the compiler knows it is short. */
|
174
|
+
for (u = 0; u < length; ++u)
|
175
|
+
dt[uStart + u] = D;
|
176
|
+
} else {
|
177
|
+
/* Unroll the loop 4 times, we know it is a power of 2. */
|
178
|
+
for (u = uStart; u < uEnd; u += 4) {
|
179
|
+
dt[u + 0] = D;
|
180
|
+
dt[u + 1] = D;
|
181
|
+
dt[u + 2] = D;
|
182
|
+
dt[u + 3] = D;
|
183
|
+
} } } }
|
195
184
|
return iSize;
|
196
185
|
}
|
197
186
|
|
@@ -282,6 +271,7 @@ HUF_decompress4X1_usingDTable_internal_body(
|
|
282
271
|
{ const BYTE* const istart = (const BYTE*) cSrc;
|
283
272
|
BYTE* const ostart = (BYTE*) dst;
|
284
273
|
BYTE* const oend = ostart + dstSize;
|
274
|
+
BYTE* const olimit = oend - 3;
|
285
275
|
const void* const dtPtr = DTable + 1;
|
286
276
|
const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr;
|
287
277
|
|
@@ -306,9 +296,9 @@ HUF_decompress4X1_usingDTable_internal_body(
|
|
306
296
|
BYTE* op2 = opStart2;
|
307
297
|
BYTE* op3 = opStart3;
|
308
298
|
BYTE* op4 = opStart4;
|
309
|
-
U32 endSignal = BIT_DStream_unfinished;
|
310
299
|
DTableDesc const dtd = HUF_getDTableDesc(DTable);
|
311
300
|
U32 const dtLog = dtd.tableLog;
|
301
|
+
U32 endSignal = 1;
|
312
302
|
|
313
303
|
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
|
314
304
|
CHECK_F( BIT_initDStream(&bitD1, istart1, length1) );
|
@@ -317,8 +307,7 @@ HUF_decompress4X1_usingDTable_internal_body(
|
|
317
307
|
CHECK_F( BIT_initDStream(&bitD4, istart4, length4) );
|
318
308
|
|
319
309
|
/* up to 16 symbols per loop (4 symbols per stream) in 64-bit mode */
|
320
|
-
|
321
|
-
while ( (endSignal==BIT_DStream_unfinished) && (op4<(oend-3)) ) {
|
310
|
+
for ( ; (endSignal) & (op4 < olimit) ; ) {
|
322
311
|
HUF_DECODE_SYMBOLX1_2(op1, &bitD1);
|
323
312
|
HUF_DECODE_SYMBOLX1_2(op2, &bitD2);
|
324
313
|
HUF_DECODE_SYMBOLX1_2(op3, &bitD3);
|
@@ -335,10 +324,10 @@ HUF_decompress4X1_usingDTable_internal_body(
|
|
335
324
|
HUF_DECODE_SYMBOLX1_0(op2, &bitD2);
|
336
325
|
HUF_DECODE_SYMBOLX1_0(op3, &bitD3);
|
337
326
|
HUF_DECODE_SYMBOLX1_0(op4, &bitD4);
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
327
|
+
endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished;
|
328
|
+
endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished;
|
329
|
+
endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished;
|
330
|
+
endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished;
|
342
331
|
}
|
343
332
|
|
344
333
|
/* check corruption */
|
@@ -757,7 +746,6 @@ HUF_decompress1X2_usingDTable_internal_body(
|
|
757
746
|
return dstSize;
|
758
747
|
}
|
759
748
|
|
760
|
-
|
761
749
|
FORCE_INLINE_TEMPLATE size_t
|
762
750
|
HUF_decompress4X2_usingDTable_internal_body(
|
763
751
|
void* dst, size_t dstSize,
|
@@ -769,6 +757,7 @@ HUF_decompress4X2_usingDTable_internal_body(
|
|
769
757
|
{ const BYTE* const istart = (const BYTE*) cSrc;
|
770
758
|
BYTE* const ostart = (BYTE*) dst;
|
771
759
|
BYTE* const oend = ostart + dstSize;
|
760
|
+
BYTE* const olimit = oend - (sizeof(size_t)-1);
|
772
761
|
const void* const dtPtr = DTable+1;
|
773
762
|
const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr;
|
774
763
|
|
@@ -793,7 +782,7 @@ HUF_decompress4X2_usingDTable_internal_body(
|
|
793
782
|
BYTE* op2 = opStart2;
|
794
783
|
BYTE* op3 = opStart3;
|
795
784
|
BYTE* op4 = opStart4;
|
796
|
-
U32 endSignal;
|
785
|
+
U32 endSignal = 1;
|
797
786
|
DTableDesc const dtd = HUF_getDTableDesc(DTable);
|
798
787
|
U32 const dtLog = dtd.tableLog;
|
799
788
|
|
@@ -804,8 +793,29 @@ HUF_decompress4X2_usingDTable_internal_body(
|
|
804
793
|
CHECK_F( BIT_initDStream(&bitD4, istart4, length4) );
|
805
794
|
|
806
795
|
/* 16-32 symbols per loop (4-8 symbols per stream) */
|
807
|
-
|
808
|
-
|
796
|
+
for ( ; (endSignal) & (op4 < olimit); ) {
|
797
|
+
#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__))
|
798
|
+
HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
|
799
|
+
HUF_DECODE_SYMBOLX2_1(op1, &bitD1);
|
800
|
+
HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
|
801
|
+
HUF_DECODE_SYMBOLX2_0(op1, &bitD1);
|
802
|
+
HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
|
803
|
+
HUF_DECODE_SYMBOLX2_1(op2, &bitD2);
|
804
|
+
HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
|
805
|
+
HUF_DECODE_SYMBOLX2_0(op2, &bitD2);
|
806
|
+
endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished;
|
807
|
+
endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished;
|
808
|
+
HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
|
809
|
+
HUF_DECODE_SYMBOLX2_1(op3, &bitD3);
|
810
|
+
HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
|
811
|
+
HUF_DECODE_SYMBOLX2_0(op3, &bitD3);
|
812
|
+
HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
|
813
|
+
HUF_DECODE_SYMBOLX2_1(op4, &bitD4);
|
814
|
+
HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
|
815
|
+
HUF_DECODE_SYMBOLX2_0(op4, &bitD4);
|
816
|
+
endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished;
|
817
|
+
endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished;
|
818
|
+
#else
|
809
819
|
HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
|
810
820
|
HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
|
811
821
|
HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
|
@@ -822,8 +832,12 @@ HUF_decompress4X2_usingDTable_internal_body(
|
|
822
832
|
HUF_DECODE_SYMBOLX2_0(op2, &bitD2);
|
823
833
|
HUF_DECODE_SYMBOLX2_0(op3, &bitD3);
|
824
834
|
HUF_DECODE_SYMBOLX2_0(op4, &bitD4);
|
825
|
-
|
826
|
-
|
835
|
+
endSignal = (U32)LIKELY(
|
836
|
+
(BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished)
|
837
|
+
& (BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished)
|
838
|
+
& (BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished)
|
839
|
+
& (BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished));
|
840
|
+
#endif
|
827
841
|
}
|
828
842
|
|
829
843
|
/* check corruption */
|
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c) 2016-
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -15,17 +15,17 @@
|
|
15
15
|
* Dependencies
|
16
16
|
*********************************************************/
|
17
17
|
#include <string.h> /* memcpy, memmove, memset */
|
18
|
-
#include "cpu.h" /* bmi2 */
|
19
|
-
#include "mem.h" /* low level memory routines */
|
18
|
+
#include "../common/cpu.h" /* bmi2 */
|
19
|
+
#include "../common/mem.h" /* low level memory routines */
|
20
20
|
#define FSE_STATIC_LINKING_ONLY
|
21
|
-
#include "fse.h"
|
21
|
+
#include "../common/fse.h"
|
22
22
|
#define HUF_STATIC_LINKING_ONLY
|
23
|
-
#include "huf.h"
|
23
|
+
#include "../common/huf.h"
|
24
24
|
#include "zstd_decompress_internal.h"
|
25
25
|
#include "zstd_ddict.h"
|
26
26
|
|
27
27
|
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
|
28
|
-
# include "zstd_legacy.h"
|
28
|
+
# include "../legacy/zstd_legacy.h"
|
29
29
|
#endif
|
30
30
|
|
31
31
|
|
@@ -65,6 +65,10 @@ void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
|
|
65
65
|
dctx->virtualStart = ddict->dictContent;
|
66
66
|
dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize;
|
67
67
|
dctx->previousDstEnd = dctx->dictEnd;
|
68
|
+
#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
|
69
|
+
dctx->dictContentBeginForFuzzing = dctx->prefixStart;
|
70
|
+
dctx->dictContentEndForFuzzing = dctx->previousDstEnd;
|
71
|
+
#endif
|
68
72
|
if (ddict->entropyPresent) {
|
69
73
|
dctx->litEntropy = 1;
|
70
74
|
dctx->fseEntropy = 1;
|
@@ -107,7 +111,7 @@ ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict,
|
|
107
111
|
/* load entropy tables */
|
108
112
|
RETURN_ERROR_IF(ZSTD_isError(ZSTD_loadDEntropy(
|
109
113
|
&ddict->entropy, ddict->dictContent, ddict->dictSize)),
|
110
|
-
dictionary_corrupted);
|
114
|
+
dictionary_corrupted, "");
|
111
115
|
ddict->entropyPresent = 1;
|
112
116
|
return 0;
|
113
117
|
}
|
@@ -133,7 +137,7 @@ static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
|
|
133
137
|
ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
|
134
138
|
|
135
139
|
/* parse dictionary content */
|
136
|
-
FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) );
|
140
|
+
FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , "");
|
137
141
|
|
138
142
|
return 0;
|
139
143
|
}
|
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c) 2016-
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -16,7 +16,7 @@
|
|
16
16
|
* Dependencies
|
17
17
|
*********************************************************/
|
18
18
|
#include <stddef.h> /* size_t */
|
19
|
-
#include "zstd.h" /* ZSTD_DDict, and several public functions */
|
19
|
+
#include "../zstd.h" /* ZSTD_DDict, and several public functions */
|
20
20
|
|
21
21
|
|
22
22
|
/*-*******************************************************
|
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c) 2016-
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -56,19 +56,19 @@
|
|
56
56
|
* Dependencies
|
57
57
|
*********************************************************/
|
58
58
|
#include <string.h> /* memcpy, memmove, memset */
|
59
|
-
#include "cpu.h" /* bmi2 */
|
60
|
-
#include "mem.h" /* low level memory routines */
|
59
|
+
#include "../common/cpu.h" /* bmi2 */
|
60
|
+
#include "../common/mem.h" /* low level memory routines */
|
61
61
|
#define FSE_STATIC_LINKING_ONLY
|
62
|
-
#include "fse.h"
|
62
|
+
#include "../common/fse.h"
|
63
63
|
#define HUF_STATIC_LINKING_ONLY
|
64
|
-
#include "huf.h"
|
65
|
-
#include "zstd_internal.h" /* blockProperties_t */
|
64
|
+
#include "../common/huf.h"
|
65
|
+
#include "../common/zstd_internal.h" /* blockProperties_t */
|
66
66
|
#include "zstd_decompress_internal.h" /* ZSTD_DCtx */
|
67
67
|
#include "zstd_ddict.h" /* ZSTD_DDictDictContent */
|
68
68
|
#include "zstd_decompress_block.h" /* ZSTD_decompressBlock_internal */
|
69
69
|
|
70
70
|
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
|
71
|
-
# include "zstd_legacy.h"
|
71
|
+
# include "../legacy/zstd_legacy.h"
|
72
72
|
#endif
|
73
73
|
|
74
74
|
|
@@ -111,7 +111,12 @@ static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
|
|
111
111
|
dctx->legacyContext = NULL;
|
112
112
|
dctx->previousLegacyVersion = 0;
|
113
113
|
dctx->noForwardProgress = 0;
|
114
|
+
dctx->oversizedDuration = 0;
|
114
115
|
dctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid());
|
116
|
+
dctx->outBufferMode = ZSTD_obm_buffered;
|
117
|
+
#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
|
118
|
+
dctx->dictContentEndForFuzzing = NULL;
|
119
|
+
#endif
|
115
120
|
}
|
116
121
|
|
117
122
|
ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize)
|
@@ -208,7 +213,7 @@ unsigned ZSTD_isFrame(const void* buffer, size_t size)
|
|
208
213
|
static size_t ZSTD_frameHeaderSize_internal(const void* src, size_t srcSize, ZSTD_format_e format)
|
209
214
|
{
|
210
215
|
size_t const minInputSize = ZSTD_startingInputLength(format);
|
211
|
-
RETURN_ERROR_IF(srcSize < minInputSize, srcSize_wrong);
|
216
|
+
RETURN_ERROR_IF(srcSize < minInputSize, srcSize_wrong, "");
|
212
217
|
|
213
218
|
{ BYTE const fhd = ((const BYTE*)src)[minInputSize-1];
|
214
219
|
U32 const dictID= fhd & 3;
|
@@ -256,7 +261,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
|
|
256
261
|
zfhPtr->frameType = ZSTD_skippableFrame;
|
257
262
|
return 0;
|
258
263
|
}
|
259
|
-
RETURN_ERROR(prefix_unknown);
|
264
|
+
RETURN_ERROR(prefix_unknown, "");
|
260
265
|
}
|
261
266
|
|
262
267
|
/* ensure there is enough `srcSize` to fully read/decode frame header */
|
@@ -280,7 +285,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
|
|
280
285
|
if (!singleSegment) {
|
281
286
|
BYTE const wlByte = ip[pos++];
|
282
287
|
U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
|
283
|
-
RETURN_ERROR_IF(windowLog > ZSTD_WINDOWLOG_MAX, frameParameter_windowTooLarge);
|
288
|
+
RETURN_ERROR_IF(windowLog > ZSTD_WINDOWLOG_MAX, frameParameter_windowTooLarge, "");
|
284
289
|
windowSize = (1ULL << windowLog);
|
285
290
|
windowSize += (windowSize >> 3) * (wlByte&7);
|
286
291
|
}
|
@@ -352,14 +357,14 @@ static size_t readSkippableFrameSize(void const* src, size_t srcSize)
|
|
352
357
|
size_t const skippableHeaderSize = ZSTD_SKIPPABLEHEADERSIZE;
|
353
358
|
U32 sizeU32;
|
354
359
|
|
355
|
-
RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong);
|
360
|
+
RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, "");
|
356
361
|
|
357
362
|
sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE);
|
358
363
|
RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32,
|
359
|
-
frameParameter_unsupported);
|
364
|
+
frameParameter_unsupported, "");
|
360
365
|
{
|
361
366
|
size_t const skippableSize = skippableHeaderSize + sizeU32;
|
362
|
-
RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong);
|
367
|
+
RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong, "");
|
363
368
|
return skippableSize;
|
364
369
|
}
|
365
370
|
}
|
@@ -439,7 +444,7 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t he
|
|
439
444
|
* harder.
|
440
445
|
*/
|
441
446
|
RETURN_ERROR_IF(dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID),
|
442
|
-
dictionary_wrong);
|
447
|
+
dictionary_wrong, "");
|
443
448
|
#endif
|
444
449
|
if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0);
|
445
450
|
return 0;
|
@@ -559,17 +564,6 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
|
|
559
564
|
* Frame decoding
|
560
565
|
***************************************************************/
|
561
566
|
|
562
|
-
|
563
|
-
void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst)
|
564
|
-
{
|
565
|
-
if (dst != dctx->previousDstEnd) { /* not contiguous */
|
566
|
-
dctx->dictEnd = dctx->previousDstEnd;
|
567
|
-
dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart));
|
568
|
-
dctx->prefixStart = dst;
|
569
|
-
dctx->previousDstEnd = dst;
|
570
|
-
}
|
571
|
-
}
|
572
|
-
|
573
567
|
/** ZSTD_insertBlock() :
|
574
568
|
* insert `src` block into `dctx` history. Useful to track uncompressed blocks. */
|
575
569
|
size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize)
|
@@ -587,9 +581,9 @@ static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity,
|
|
587
581
|
DEBUGLOG(5, "ZSTD_copyRawBlock");
|
588
582
|
if (dst == NULL) {
|
589
583
|
if (srcSize == 0) return 0;
|
590
|
-
RETURN_ERROR(dstBuffer_null);
|
584
|
+
RETURN_ERROR(dstBuffer_null, "");
|
591
585
|
}
|
592
|
-
RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall);
|
586
|
+
RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, "");
|
593
587
|
memcpy(dst, src, srcSize);
|
594
588
|
return srcSize;
|
595
589
|
}
|
@@ -600,9 +594,9 @@ static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity,
|
|
600
594
|
{
|
601
595
|
if (dst == NULL) {
|
602
596
|
if (regenSize == 0) return 0;
|
603
|
-
RETURN_ERROR(dstBuffer_null);
|
597
|
+
RETURN_ERROR(dstBuffer_null, "");
|
604
598
|
}
|
605
|
-
RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall);
|
599
|
+
RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall, "");
|
606
600
|
memset(dst, b, regenSize);
|
607
601
|
return regenSize;
|
608
602
|
}
|
@@ -618,7 +612,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
|
618
612
|
{
|
619
613
|
const BYTE* ip = (const BYTE*)(*srcPtr);
|
620
614
|
BYTE* const ostart = (BYTE* const)dst;
|
621
|
-
BYTE* const oend = ostart + dstCapacity;
|
615
|
+
BYTE* const oend = dstCapacity != 0 ? ostart + dstCapacity : ostart;
|
622
616
|
BYTE* op = ostart;
|
623
617
|
size_t remainingSrcSize = *srcSizePtr;
|
624
618
|
|
@@ -627,15 +621,15 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
|
627
621
|
/* check */
|
628
622
|
RETURN_ERROR_IF(
|
629
623
|
remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN(dctx->format)+ZSTD_blockHeaderSize,
|
630
|
-
srcSize_wrong);
|
624
|
+
srcSize_wrong, "");
|
631
625
|
|
632
626
|
/* Frame Header */
|
633
627
|
{ size_t const frameHeaderSize = ZSTD_frameHeaderSize_internal(
|
634
628
|
ip, ZSTD_FRAMEHEADERSIZE_PREFIX(dctx->format), dctx->format);
|
635
629
|
if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
|
636
630
|
RETURN_ERROR_IF(remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize,
|
637
|
-
srcSize_wrong);
|
638
|
-
FORWARD_IF_ERROR( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) );
|
631
|
+
srcSize_wrong, "");
|
632
|
+
FORWARD_IF_ERROR( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) , "");
|
639
633
|
ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize;
|
640
634
|
}
|
641
635
|
|
@@ -648,7 +642,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
|
648
642
|
|
649
643
|
ip += ZSTD_blockHeaderSize;
|
650
644
|
remainingSrcSize -= ZSTD_blockHeaderSize;
|
651
|
-
RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong);
|
645
|
+
RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong, "");
|
652
646
|
|
653
647
|
switch(blockProperties.blockType)
|
654
648
|
{
|
@@ -663,13 +657,15 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
|
663
657
|
break;
|
664
658
|
case bt_reserved :
|
665
659
|
default:
|
666
|
-
RETURN_ERROR(corruption_detected);
|
660
|
+
RETURN_ERROR(corruption_detected, "invalid block type");
|
667
661
|
}
|
668
662
|
|
669
663
|
if (ZSTD_isError(decodedSize)) return decodedSize;
|
670
664
|
if (dctx->fParams.checksumFlag)
|
671
665
|
XXH64_update(&dctx->xxhState, op, decodedSize);
|
672
|
-
|
666
|
+
if (decodedSize != 0)
|
667
|
+
op += decodedSize;
|
668
|
+
assert(ip != NULL);
|
673
669
|
ip += cBlockSize;
|
674
670
|
remainingSrcSize -= cBlockSize;
|
675
671
|
if (blockProperties.lastBlock) break;
|
@@ -677,14 +673,14 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
|
677
673
|
|
678
674
|
if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) {
|
679
675
|
RETURN_ERROR_IF((U64)(op-ostart) != dctx->fParams.frameContentSize,
|
680
|
-
corruption_detected);
|
676
|
+
corruption_detected, "");
|
681
677
|
}
|
682
678
|
if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */
|
683
679
|
U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState);
|
684
680
|
U32 checkRead;
|
685
|
-
RETURN_ERROR_IF(remainingSrcSize<4, checksum_wrong);
|
681
|
+
RETURN_ERROR_IF(remainingSrcSize<4, checksum_wrong, "");
|
686
682
|
checkRead = MEM_readLE32(ip);
|
687
|
-
RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong);
|
683
|
+
RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong, "");
|
688
684
|
ip += 4;
|
689
685
|
remainingSrcSize -= 4;
|
690
686
|
}
|
@@ -741,7 +737,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
|
|
741
737
|
(unsigned)magicNumber, ZSTD_MAGICNUMBER);
|
742
738
|
if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
|
743
739
|
size_t const skippableSize = readSkippableFrameSize(src, srcSize);
|
744
|
-
FORWARD_IF_ERROR(skippableSize);
|
740
|
+
FORWARD_IF_ERROR(skippableSize, "readSkippableFrameSize failed");
|
745
741
|
assert(skippableSize <= srcSize);
|
746
742
|
|
747
743
|
src = (const BYTE *)src + skippableSize;
|
@@ -751,11 +747,11 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
|
|
751
747
|
|
752
748
|
if (ddict) {
|
753
749
|
/* we were called from ZSTD_decompress_usingDDict */
|
754
|
-
FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(dctx, ddict));
|
750
|
+
FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(dctx, ddict), "");
|
755
751
|
} else {
|
756
752
|
/* this will initialize correctly with no dict if dict == NULL, so
|
757
753
|
* use this in all cases but ddict */
|
758
|
-
FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize));
|
754
|
+
FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize), "");
|
759
755
|
}
|
760
756
|
ZSTD_checkContinuity(dctx, dst);
|
761
757
|
|
@@ -776,7 +772,8 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
|
|
776
772
|
"error.");
|
777
773
|
if (ZSTD_isError(res)) return res;
|
778
774
|
assert(res <= dstCapacity);
|
779
|
-
|
775
|
+
if (res != 0)
|
776
|
+
dst = (BYTE*)dst + res;
|
780
777
|
dstCapacity -= res;
|
781
778
|
}
|
782
779
|
moreThan1Frame = 1;
|
@@ -824,7 +821,7 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr
|
|
824
821
|
#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1)
|
825
822
|
size_t regenSize;
|
826
823
|
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
827
|
-
RETURN_ERROR_IF(dctx==NULL, memory_allocation);
|
824
|
+
RETURN_ERROR_IF(dctx==NULL, memory_allocation, "NULL pointer!");
|
828
825
|
regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize);
|
829
826
|
ZSTD_freeDCtx(dctx);
|
830
827
|
return regenSize;
|
@@ -842,6 +839,24 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr
|
|
842
839
|
****************************************/
|
843
840
|
size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; }
|
844
841
|
|
842
|
+
/**
|
843
|
+
* Similar to ZSTD_nextSrcSizeToDecompress(), but when when a block input can be streamed,
|
844
|
+
* we allow taking a partial block as the input. Currently only raw uncompressed blocks can
|
845
|
+
* be streamed.
|
846
|
+
*
|
847
|
+
* For blocks that can be streamed, this allows us to reduce the latency until we produce
|
848
|
+
* output, and avoid copying the input.
|
849
|
+
*
|
850
|
+
* @param inputSize - The total amount of input that the caller currently has.
|
851
|
+
*/
|
852
|
+
static size_t ZSTD_nextSrcSizeToDecompressWithInputSize(ZSTD_DCtx* dctx, size_t inputSize) {
|
853
|
+
if (!(dctx->stage == ZSTDds_decompressBlock || dctx->stage == ZSTDds_decompressLastBlock))
|
854
|
+
return dctx->expected;
|
855
|
+
if (dctx->bType != bt_raw)
|
856
|
+
return dctx->expected;
|
857
|
+
return MIN(MAX(inputSize, 1), dctx->expected);
|
858
|
+
}
|
859
|
+
|
845
860
|
ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) {
|
846
861
|
switch(dctx->stage)
|
847
862
|
{
|
@@ -874,7 +889,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
|
|
874
889
|
{
|
875
890
|
DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (unsigned)srcSize);
|
876
891
|
/* Sanity check */
|
877
|
-
RETURN_ERROR_IF(srcSize != dctx
|
892
|
+
RETURN_ERROR_IF(srcSize != ZSTD_nextSrcSizeToDecompressWithInputSize(dctx, srcSize), srcSize_wrong, "not allowed");
|
878
893
|
if (dstCapacity) ZSTD_checkContinuity(dctx, dst);
|
879
894
|
|
880
895
|
switch (dctx->stage)
|
@@ -899,7 +914,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
|
|
899
914
|
case ZSTDds_decodeFrameHeader:
|
900
915
|
assert(src != NULL);
|
901
916
|
memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize);
|
902
|
-
FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize));
|
917
|
+
FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize), "");
|
903
918
|
dctx->expected = ZSTD_blockHeaderSize;
|
904
919
|
dctx->stage = ZSTDds_decodeBlockHeader;
|
905
920
|
return 0;
|
@@ -941,29 +956,41 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
|
|
941
956
|
case bt_compressed:
|
942
957
|
DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed");
|
943
958
|
rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1);
|
959
|
+
dctx->expected = 0; /* Streaming not supported */
|
944
960
|
break;
|
945
961
|
case bt_raw :
|
962
|
+
assert(srcSize <= dctx->expected);
|
946
963
|
rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize);
|
964
|
+
FORWARD_IF_ERROR(rSize, "ZSTD_copyRawBlock failed");
|
965
|
+
assert(rSize == srcSize);
|
966
|
+
dctx->expected -= rSize;
|
947
967
|
break;
|
948
968
|
case bt_rle :
|
949
969
|
rSize = ZSTD_setRleBlock(dst, dstCapacity, *(const BYTE*)src, dctx->rleSize);
|
970
|
+
dctx->expected = 0; /* Streaming not supported */
|
950
971
|
break;
|
951
972
|
case bt_reserved : /* should never happen */
|
952
973
|
default:
|
953
|
-
RETURN_ERROR(corruption_detected);
|
974
|
+
RETURN_ERROR(corruption_detected, "invalid block type");
|
954
975
|
}
|
955
|
-
|
976
|
+
FORWARD_IF_ERROR(rSize, "");
|
956
977
|
RETURN_ERROR_IF(rSize > dctx->fParams.blockSizeMax, corruption_detected, "Decompressed Block Size Exceeds Maximum");
|
957
978
|
DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize);
|
958
979
|
dctx->decodedSize += rSize;
|
959
980
|
if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize);
|
981
|
+
dctx->previousDstEnd = (char*)dst + rSize;
|
982
|
+
|
983
|
+
/* Stay on the same stage until we are finished streaming the block. */
|
984
|
+
if (dctx->expected > 0) {
|
985
|
+
return rSize;
|
986
|
+
}
|
960
987
|
|
961
988
|
if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */
|
962
989
|
DEBUGLOG(4, "ZSTD_decompressContinue: decoded size from frame : %u", (unsigned)dctx->decodedSize);
|
963
990
|
RETURN_ERROR_IF(
|
964
991
|
dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
|
965
992
|
&& dctx->decodedSize != dctx->fParams.frameContentSize,
|
966
|
-
corruption_detected);
|
993
|
+
corruption_detected, "");
|
967
994
|
if (dctx->fParams.checksumFlag) { /* another round for frame checksum */
|
968
995
|
dctx->expected = 4;
|
969
996
|
dctx->stage = ZSTDds_checkChecksum;
|
@@ -974,7 +1001,6 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
|
|
974
1001
|
} else {
|
975
1002
|
dctx->stage = ZSTDds_decodeBlockHeader;
|
976
1003
|
dctx->expected = ZSTD_blockHeaderSize;
|
977
|
-
dctx->previousDstEnd = (char*)dst + rSize;
|
978
1004
|
}
|
979
1005
|
return rSize;
|
980
1006
|
}
|
@@ -984,7 +1010,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
|
|
984
1010
|
{ U32 const h32 = (U32)XXH64_digest(&dctx->xxhState);
|
985
1011
|
U32 const check32 = MEM_readLE32(src);
|
986
1012
|
DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32);
|
987
|
-
RETURN_ERROR_IF(check32 != h32, checksum_wrong);
|
1013
|
+
RETURN_ERROR_IF(check32 != h32, checksum_wrong, "");
|
988
1014
|
dctx->expected = 0;
|
989
1015
|
dctx->stage = ZSTDds_getFrameHeaderSize;
|
990
1016
|
return 0;
|
@@ -1005,7 +1031,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
|
|
1005
1031
|
|
1006
1032
|
default:
|
1007
1033
|
assert(0); /* impossible */
|
1008
|
-
RETURN_ERROR(GENERIC); /* some compiler require default to do something */
|
1034
|
+
RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */
|
1009
1035
|
}
|
1010
1036
|
}
|
1011
1037
|
|
@@ -1016,6 +1042,10 @@ static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dict
|
|
1016
1042
|
dctx->virtualStart = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart));
|
1017
1043
|
dctx->prefixStart = dict;
|
1018
1044
|
dctx->previousDstEnd = (const char*)dict + dictSize;
|
1045
|
+
#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
|
1046
|
+
dctx->dictContentBeginForFuzzing = dctx->prefixStart;
|
1047
|
+
dctx->dictContentEndForFuzzing = dctx->previousDstEnd;
|
1048
|
+
#endif
|
1019
1049
|
return 0;
|
1020
1050
|
}
|
1021
1051
|
|
@@ -1029,7 +1059,7 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
|
|
1029
1059
|
const BYTE* dictPtr = (const BYTE*)dict;
|
1030
1060
|
const BYTE* const dictEnd = dictPtr + dictSize;
|
1031
1061
|
|
1032
|
-
RETURN_ERROR_IF(dictSize <= 8, dictionary_corrupted);
|
1062
|
+
RETURN_ERROR_IF(dictSize <= 8, dictionary_corrupted, "dict is too small");
|
1033
1063
|
assert(MEM_readLE32(dict) == ZSTD_MAGIC_DICTIONARY); /* dict must be valid */
|
1034
1064
|
dictPtr += 8; /* skip header = magic + dictID */
|
1035
1065
|
|
@@ -1048,16 +1078,16 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
|
|
1048
1078
|
dictPtr, dictEnd - dictPtr,
|
1049
1079
|
workspace, workspaceSize);
|
1050
1080
|
#endif
|
1051
|
-
RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted);
|
1081
|
+
RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, "");
|
1052
1082
|
dictPtr += hSize;
|
1053
1083
|
}
|
1054
1084
|
|
1055
1085
|
{ short offcodeNCount[MaxOff+1];
|
1056
1086
|
unsigned offcodeMaxValue = MaxOff, offcodeLog;
|
1057
1087
|
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
|
1058
|
-
RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted);
|
1059
|
-
RETURN_ERROR_IF(offcodeMaxValue > MaxOff, dictionary_corrupted);
|
1060
|
-
RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted);
|
1088
|
+
RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, "");
|
1089
|
+
RETURN_ERROR_IF(offcodeMaxValue > MaxOff, dictionary_corrupted, "");
|
1090
|
+
RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, "");
|
1061
1091
|
ZSTD_buildFSETable( entropy->OFTable,
|
1062
1092
|
offcodeNCount, offcodeMaxValue,
|
1063
1093
|
OF_base, OF_bits,
|
@@ -1068,9 +1098,9 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
|
|
1068
1098
|
{ short matchlengthNCount[MaxML+1];
|
1069
1099
|
unsigned matchlengthMaxValue = MaxML, matchlengthLog;
|
1070
1100
|
size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
|
1071
|
-
RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted);
|
1072
|
-
RETURN_ERROR_IF(matchlengthMaxValue > MaxML, dictionary_corrupted);
|
1073
|
-
RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted);
|
1101
|
+
RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, "");
|
1102
|
+
RETURN_ERROR_IF(matchlengthMaxValue > MaxML, dictionary_corrupted, "");
|
1103
|
+
RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, "");
|
1074
1104
|
ZSTD_buildFSETable( entropy->MLTable,
|
1075
1105
|
matchlengthNCount, matchlengthMaxValue,
|
1076
1106
|
ML_base, ML_bits,
|
@@ -1081,9 +1111,9 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
|
|
1081
1111
|
{ short litlengthNCount[MaxLL+1];
|
1082
1112
|
unsigned litlengthMaxValue = MaxLL, litlengthLog;
|
1083
1113
|
size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
|
1084
|
-
RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted);
|
1085
|
-
RETURN_ERROR_IF(litlengthMaxValue > MaxLL, dictionary_corrupted);
|
1086
|
-
RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted);
|
1114
|
+
RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, "");
|
1115
|
+
RETURN_ERROR_IF(litlengthMaxValue > MaxLL, dictionary_corrupted, "");
|
1116
|
+
RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, "");
|
1087
1117
|
ZSTD_buildFSETable( entropy->LLTable,
|
1088
1118
|
litlengthNCount, litlengthMaxValue,
|
1089
1119
|
LL_base, LL_bits,
|
@@ -1091,13 +1121,13 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
|
|
1091
1121
|
dictPtr += litlengthHeaderSize;
|
1092
1122
|
}
|
1093
1123
|
|
1094
|
-
RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted);
|
1124
|
+
RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, "");
|
1095
1125
|
{ int i;
|
1096
1126
|
size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12));
|
1097
1127
|
for (i=0; i<3; i++) {
|
1098
1128
|
U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4;
|
1099
1129
|
RETURN_ERROR_IF(rep==0 || rep > dictContentSize,
|
1100
|
-
dictionary_corrupted);
|
1130
|
+
dictionary_corrupted, "");
|
1101
1131
|
entropy->rep[i] = rep;
|
1102
1132
|
} }
|
1103
1133
|
|
@@ -1115,7 +1145,7 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
|
|
1115
1145
|
|
1116
1146
|
/* load entropy tables */
|
1117
1147
|
{ size_t const eSize = ZSTD_loadDEntropy(&dctx->entropy, dict, dictSize);
|
1118
|
-
RETURN_ERROR_IF(ZSTD_isError(eSize), dictionary_corrupted);
|
1148
|
+
RETURN_ERROR_IF(ZSTD_isError(eSize), dictionary_corrupted, "");
|
1119
1149
|
dict = (const char*)dict + eSize;
|
1120
1150
|
dictSize -= eSize;
|
1121
1151
|
}
|
@@ -1138,6 +1168,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
|
|
1138
1168
|
dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
|
1139
1169
|
dctx->litEntropy = dctx->fseEntropy = 0;
|
1140
1170
|
dctx->dictID = 0;
|
1171
|
+
dctx->bType = bt_reserved;
|
1141
1172
|
ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue));
|
1142
1173
|
memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */
|
1143
1174
|
dctx->LLTptr = dctx->entropy.LLTable;
|
@@ -1149,11 +1180,11 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
|
|
1149
1180
|
|
1150
1181
|
size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
|
1151
1182
|
{
|
1152
|
-
FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) );
|
1183
|
+
FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , "");
|
1153
1184
|
if (dict && dictSize)
|
1154
1185
|
RETURN_ERROR_IF(
|
1155
1186
|
ZSTD_isError(ZSTD_decompress_insertDictionary(dctx, dict, dictSize)),
|
1156
|
-
dictionary_corrupted);
|
1187
|
+
dictionary_corrupted, "");
|
1157
1188
|
return 0;
|
1158
1189
|
}
|
1159
1190
|
|
@@ -1172,7 +1203,7 @@ size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
|
|
1172
1203
|
DEBUGLOG(4, "DDict is %s",
|
1173
1204
|
dctx->ddictIsCold ? "~cold~" : "hot!");
|
1174
1205
|
}
|
1175
|
-
FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) );
|
1206
|
+
FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , "");
|
1176
1207
|
if (ddict) { /* NULL ddict is equivalent to no dictionary */
|
1177
1208
|
ZSTD_copyDDictParameters(dctx, ddict);
|
1178
1209
|
}
|
@@ -1263,11 +1294,11 @@ size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx,
|
|
1263
1294
|
ZSTD_dictLoadMethod_e dictLoadMethod,
|
1264
1295
|
ZSTD_dictContentType_e dictContentType)
|
1265
1296
|
{
|
1266
|
-
RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
|
1297
|
+
RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
|
1267
1298
|
ZSTD_clearDict(dctx);
|
1268
1299
|
if (dict && dictSize != 0) {
|
1269
1300
|
dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem);
|
1270
|
-
RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation);
|
1301
|
+
RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation, "NULL pointer!");
|
1271
1302
|
dctx->ddict = dctx->ddictLocal;
|
1272
1303
|
dctx->dictUses = ZSTD_use_indefinitely;
|
1273
1304
|
}
|
@@ -1286,7 +1317,7 @@ size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSi
|
|
1286
1317
|
|
1287
1318
|
size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType)
|
1288
1319
|
{
|
1289
|
-
FORWARD_IF_ERROR(ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType));
|
1320
|
+
FORWARD_IF_ERROR(ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType), "");
|
1290
1321
|
dctx->dictUses = ZSTD_use_once;
|
1291
1322
|
return 0;
|
1292
1323
|
}
|
@@ -1303,8 +1334,8 @@ size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSiz
|
|
1303
1334
|
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)
|
1304
1335
|
{
|
1305
1336
|
DEBUGLOG(4, "ZSTD_initDStream_usingDict");
|
1306
|
-
FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) );
|
1307
|
-
FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) );
|
1337
|
+
FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) , "");
|
1338
|
+
FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) , "");
|
1308
1339
|
return ZSTD_startingInputLength(zds->format);
|
1309
1340
|
}
|
1310
1341
|
|
@@ -1320,8 +1351,8 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds)
|
|
1320
1351
|
* this function cannot fail */
|
1321
1352
|
size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict)
|
1322
1353
|
{
|
1323
|
-
FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) );
|
1324
|
-
FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) );
|
1354
|
+
FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , "");
|
1355
|
+
FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , "");
|
1325
1356
|
return ZSTD_startingInputLength(dctx->format);
|
1326
1357
|
}
|
1327
1358
|
|
@@ -1330,14 +1361,14 @@ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict)
|
|
1330
1361
|
* this function cannot fail */
|
1331
1362
|
size_t ZSTD_resetDStream(ZSTD_DStream* dctx)
|
1332
1363
|
{
|
1333
|
-
FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only));
|
1364
|
+
FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), "");
|
1334
1365
|
return ZSTD_startingInputLength(dctx->format);
|
1335
1366
|
}
|
1336
1367
|
|
1337
1368
|
|
1338
1369
|
size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
|
1339
1370
|
{
|
1340
|
-
RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
|
1371
|
+
RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
|
1341
1372
|
ZSTD_clearDict(dctx);
|
1342
1373
|
if (ddict) {
|
1343
1374
|
dctx->ddict = ddict;
|
@@ -1354,9 +1385,9 @@ size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize)
|
|
1354
1385
|
ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax);
|
1355
1386
|
size_t const min = (size_t)1 << bounds.lowerBound;
|
1356
1387
|
size_t const max = (size_t)1 << bounds.upperBound;
|
1357
|
-
RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
|
1358
|
-
RETURN_ERROR_IF(maxWindowSize < min, parameter_outOfBound);
|
1359
|
-
RETURN_ERROR_IF(maxWindowSize > max, parameter_outOfBound);
|
1388
|
+
RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
|
1389
|
+
RETURN_ERROR_IF(maxWindowSize < min, parameter_outOfBound, "");
|
1390
|
+
RETURN_ERROR_IF(maxWindowSize > max, parameter_outOfBound, "");
|
1360
1391
|
dctx->maxWindowSize = maxWindowSize;
|
1361
1392
|
return 0;
|
1362
1393
|
}
|
@@ -1379,6 +1410,10 @@ ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam)
|
|
1379
1410
|
bounds.upperBound = (int)ZSTD_f_zstd1_magicless;
|
1380
1411
|
ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless);
|
1381
1412
|
return bounds;
|
1413
|
+
case ZSTD_d_stableOutBuffer:
|
1414
|
+
bounds.lowerBound = (int)ZSTD_obm_buffered;
|
1415
|
+
bounds.upperBound = (int)ZSTD_obm_stable;
|
1416
|
+
return bounds;
|
1382
1417
|
default:;
|
1383
1418
|
}
|
1384
1419
|
bounds.error = ERROR(parameter_unsupported);
|
@@ -1398,12 +1433,12 @@ static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value)
|
|
1398
1433
|
}
|
1399
1434
|
|
1400
1435
|
#define CHECK_DBOUNDS(p,v) { \
|
1401
|
-
RETURN_ERROR_IF(!ZSTD_dParam_withinBounds(p, v), parameter_outOfBound); \
|
1436
|
+
RETURN_ERROR_IF(!ZSTD_dParam_withinBounds(p, v), parameter_outOfBound, ""); \
|
1402
1437
|
}
|
1403
1438
|
|
1404
1439
|
size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value)
|
1405
1440
|
{
|
1406
|
-
RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
|
1441
|
+
RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
|
1407
1442
|
switch(dParam) {
|
1408
1443
|
case ZSTD_d_windowLogMax:
|
1409
1444
|
if (value == 0) value = ZSTD_WINDOWLOG_LIMIT_DEFAULT;
|
@@ -1414,9 +1449,13 @@ size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value
|
|
1414
1449
|
CHECK_DBOUNDS(ZSTD_d_format, value);
|
1415
1450
|
dctx->format = (ZSTD_format_e)value;
|
1416
1451
|
return 0;
|
1452
|
+
case ZSTD_d_stableOutBuffer:
|
1453
|
+
CHECK_DBOUNDS(ZSTD_d_stableOutBuffer, value);
|
1454
|
+
dctx->outBufferMode = (ZSTD_outBufferMode_e)value;
|
1455
|
+
return 0;
|
1417
1456
|
default:;
|
1418
1457
|
}
|
1419
|
-
RETURN_ERROR(parameter_unsupported);
|
1458
|
+
RETURN_ERROR(parameter_unsupported, "");
|
1420
1459
|
}
|
1421
1460
|
|
1422
1461
|
size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset)
|
@@ -1428,7 +1467,7 @@ size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset)
|
|
1428
1467
|
}
|
1429
1468
|
if ( (reset == ZSTD_reset_parameters)
|
1430
1469
|
|| (reset == ZSTD_reset_session_and_parameters) ) {
|
1431
|
-
RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
|
1470
|
+
RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
|
1432
1471
|
ZSTD_clearDict(dctx);
|
1433
1472
|
dctx->format = ZSTD_f_zstd1;
|
1434
1473
|
dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
|
@@ -1449,7 +1488,7 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
|
|
1449
1488
|
unsigned long long const neededSize = MIN(frameContentSize, neededRBSize);
|
1450
1489
|
size_t const minRBSize = (size_t) neededSize;
|
1451
1490
|
RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize,
|
1452
|
-
frameParameter_windowTooLarge);
|
1491
|
+
frameParameter_windowTooLarge, "");
|
1453
1492
|
return minRBSize;
|
1454
1493
|
}
|
1455
1494
|
|
@@ -1467,30 +1506,94 @@ size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize)
|
|
1467
1506
|
ZSTD_frameHeader zfh;
|
1468
1507
|
size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize);
|
1469
1508
|
if (ZSTD_isError(err)) return err;
|
1470
|
-
RETURN_ERROR_IF(err>0, srcSize_wrong);
|
1509
|
+
RETURN_ERROR_IF(err>0, srcSize_wrong, "");
|
1471
1510
|
RETURN_ERROR_IF(zfh.windowSize > windowSizeMax,
|
1472
|
-
frameParameter_windowTooLarge);
|
1511
|
+
frameParameter_windowTooLarge, "");
|
1473
1512
|
return ZSTD_estimateDStreamSize((size_t)zfh.windowSize);
|
1474
1513
|
}
|
1475
1514
|
|
1476
1515
|
|
1477
1516
|
/* ***** Decompression ***** */
|
1478
1517
|
|
1479
|
-
|
1518
|
+
static int ZSTD_DCtx_isOverflow(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize)
|
1480
1519
|
{
|
1481
|
-
|
1482
|
-
|
1483
|
-
|
1520
|
+
return (zds->inBuffSize + zds->outBuffSize) >= (neededInBuffSize + neededOutBuffSize) * ZSTD_WORKSPACETOOLARGE_FACTOR;
|
1521
|
+
}
|
1522
|
+
|
1523
|
+
static void ZSTD_DCtx_updateOversizedDuration(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize)
|
1524
|
+
{
|
1525
|
+
if (ZSTD_DCtx_isOverflow(zds, neededInBuffSize, neededOutBuffSize))
|
1526
|
+
zds->oversizedDuration++;
|
1527
|
+
else
|
1528
|
+
zds->oversizedDuration = 0;
|
1529
|
+
}
|
1530
|
+
|
1531
|
+
static int ZSTD_DCtx_isOversizedTooLong(ZSTD_DStream* zds)
|
1532
|
+
{
|
1533
|
+
return zds->oversizedDuration >= ZSTD_WORKSPACETOOLARGE_MAXDURATION;
|
1534
|
+
}
|
1535
|
+
|
1536
|
+
/* Checks that the output buffer hasn't changed if ZSTD_obm_stable is used. */
|
1537
|
+
static size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const* output)
|
1538
|
+
{
|
1539
|
+
ZSTD_outBuffer const expect = zds->expectedOutBuffer;
|
1540
|
+
/* No requirement when ZSTD_obm_stable is not enabled. */
|
1541
|
+
if (zds->outBufferMode != ZSTD_obm_stable)
|
1542
|
+
return 0;
|
1543
|
+
/* Any buffer is allowed in zdss_init, this must be the same for every other call until
|
1544
|
+
* the context is reset.
|
1545
|
+
*/
|
1546
|
+
if (zds->streamStage == zdss_init)
|
1547
|
+
return 0;
|
1548
|
+
/* The buffer must match our expectation exactly. */
|
1549
|
+
if (expect.dst == output->dst && expect.pos == output->pos && expect.size == output->size)
|
1550
|
+
return 0;
|
1551
|
+
RETURN_ERROR(dstBuffer_wrong, "ZSTD_obm_stable enabled but output differs!");
|
1484
1552
|
}
|
1485
1553
|
|
1554
|
+
/* Calls ZSTD_decompressContinue() with the right parameters for ZSTD_decompressStream()
|
1555
|
+
* and updates the stage and the output buffer state. This call is extracted so it can be
|
1556
|
+
* used both when reading directly from the ZSTD_inBuffer, and in buffered input mode.
|
1557
|
+
* NOTE: You must break after calling this function since the streamStage is modified.
|
1558
|
+
*/
|
1559
|
+
static size_t ZSTD_decompressContinueStream(
|
1560
|
+
ZSTD_DStream* zds, char** op, char* oend,
|
1561
|
+
void const* src, size_t srcSize) {
|
1562
|
+
int const isSkipFrame = ZSTD_isSkipFrame(zds);
|
1563
|
+
if (zds->outBufferMode == ZSTD_obm_buffered) {
|
1564
|
+
size_t const dstSize = isSkipFrame ? 0 : zds->outBuffSize - zds->outStart;
|
1565
|
+
size_t const decodedSize = ZSTD_decompressContinue(zds,
|
1566
|
+
zds->outBuff + zds->outStart, dstSize, src, srcSize);
|
1567
|
+
FORWARD_IF_ERROR(decodedSize, "");
|
1568
|
+
if (!decodedSize && !isSkipFrame) {
|
1569
|
+
zds->streamStage = zdss_read;
|
1570
|
+
} else {
|
1571
|
+
zds->outEnd = zds->outStart + decodedSize;
|
1572
|
+
zds->streamStage = zdss_flush;
|
1573
|
+
}
|
1574
|
+
} else {
|
1575
|
+
/* Write directly into the output buffer */
|
1576
|
+
size_t const dstSize = isSkipFrame ? 0 : oend - *op;
|
1577
|
+
size_t const decodedSize = ZSTD_decompressContinue(zds, *op, dstSize, src, srcSize);
|
1578
|
+
FORWARD_IF_ERROR(decodedSize, "");
|
1579
|
+
*op += decodedSize;
|
1580
|
+
/* Flushing is not needed. */
|
1581
|
+
zds->streamStage = zdss_read;
|
1582
|
+
assert(*op <= oend);
|
1583
|
+
assert(zds->outBufferMode == ZSTD_obm_stable);
|
1584
|
+
}
|
1585
|
+
return 0;
|
1586
|
+
}
|
1486
1587
|
|
1487
1588
|
size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
|
1488
1589
|
{
|
1489
|
-
const char* const
|
1490
|
-
const char* const
|
1590
|
+
const char* const src = (const char*)input->src;
|
1591
|
+
const char* const istart = input->pos != 0 ? src + input->pos : src;
|
1592
|
+
const char* const iend = input->size != 0 ? src + input->size : src;
|
1491
1593
|
const char* ip = istart;
|
1492
|
-
char* const
|
1493
|
-
char* const
|
1594
|
+
char* const dst = (char*)output->dst;
|
1595
|
+
char* const ostart = output->pos != 0 ? dst + output->pos : dst;
|
1596
|
+
char* const oend = output->size != 0 ? dst + output->size : dst;
|
1494
1597
|
char* op = ostart;
|
1495
1598
|
U32 someMoreWork = 1;
|
1496
1599
|
|
@@ -1506,6 +1609,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|
1506
1609
|
"forbidden. out: pos: %u vs size: %u",
|
1507
1610
|
(U32)output->pos, (U32)output->size);
|
1508
1611
|
DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos));
|
1612
|
+
FORWARD_IF_ERROR(ZSTD_checkOutBuffer(zds, output), "");
|
1509
1613
|
|
1510
1614
|
while (someMoreWork) {
|
1511
1615
|
switch(zds->streamStage)
|
@@ -1516,6 +1620,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|
1516
1620
|
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
|
1517
1621
|
zds->legacyVersion = 0;
|
1518
1622
|
zds->hostageByte = 0;
|
1623
|
+
zds->expectedOutBuffer = *output;
|
1519
1624
|
/* fall-through */
|
1520
1625
|
|
1521
1626
|
case zdss_loadHeader :
|
@@ -1543,7 +1648,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|
1543
1648
|
"legacy support is incompatible with static dctx");
|
1544
1649
|
FORWARD_IF_ERROR(ZSTD_initLegacyStream(&zds->legacyContext,
|
1545
1650
|
zds->previousLegacyVersion, legacyVersion,
|
1546
|
-
dict, dictSize));
|
1651
|
+
dict, dictSize), "");
|
1547
1652
|
zds->legacyVersion = zds->previousLegacyVersion = legacyVersion;
|
1548
1653
|
{ size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, legacyVersion, output, input);
|
1549
1654
|
if (hint==0) zds->streamStage = zdss_init; /* or stay in stage zdss_loadHeader */
|
@@ -1570,7 +1675,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|
1570
1675
|
} }
|
1571
1676
|
|
1572
1677
|
/* check for single-pass mode opportunity */
|
1573
|
-
if (zds->fParams.frameContentSize
|
1678
|
+
if (zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
|
1679
|
+
&& zds->fParams.frameType != ZSTD_skippableFrame
|
1574
1680
|
&& (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) {
|
1575
1681
|
size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart);
|
1576
1682
|
if (cSize <= (size_t)(iend-istart)) {
|
@@ -1586,15 +1692,23 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|
1586
1692
|
break;
|
1587
1693
|
} }
|
1588
1694
|
|
1695
|
+
/* Check output buffer is large enough for ZSTD_odm_stable. */
|
1696
|
+
if (zds->outBufferMode == ZSTD_obm_stable
|
1697
|
+
&& zds->fParams.frameType != ZSTD_skippableFrame
|
1698
|
+
&& zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
|
1699
|
+
&& (U64)(size_t)(oend-op) < zds->fParams.frameContentSize) {
|
1700
|
+
RETURN_ERROR(dstSize_tooSmall, "ZSTD_obm_stable passed but ZSTD_outBuffer is too small");
|
1701
|
+
}
|
1702
|
+
|
1589
1703
|
/* Consume header (see ZSTDds_decodeFrameHeader) */
|
1590
1704
|
DEBUGLOG(4, "Consume header");
|
1591
|
-
FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)));
|
1705
|
+
FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)), "");
|
1592
1706
|
|
1593
1707
|
if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
|
1594
1708
|
zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE);
|
1595
1709
|
zds->stage = ZSTDds_skipFrame;
|
1596
1710
|
} else {
|
1597
|
-
FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize));
|
1711
|
+
FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize), "");
|
1598
1712
|
zds->expected = ZSTD_blockHeaderSize;
|
1599
1713
|
zds->stage = ZSTDds_decodeBlockHeader;
|
1600
1714
|
}
|
@@ -1605,40 +1719,48 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|
1605
1719
|
(U32)(zds->maxWindowSize >> 10) );
|
1606
1720
|
zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
|
1607
1721
|
RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize,
|
1608
|
-
frameParameter_windowTooLarge);
|
1722
|
+
frameParameter_windowTooLarge, "");
|
1609
1723
|
|
1610
1724
|
/* Adapt buffer sizes to frame header instructions */
|
1611
1725
|
{ size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */);
|
1612
|
-
size_t const neededOutBuffSize =
|
1613
|
-
|
1614
|
-
|
1615
|
-
|
1616
|
-
|
1617
|
-
|
1618
|
-
|
1619
|
-
|
1620
|
-
|
1621
|
-
|
1622
|
-
|
1623
|
-
|
1624
|
-
|
1625
|
-
|
1626
|
-
|
1627
|
-
zds->
|
1628
|
-
|
1629
|
-
|
1630
|
-
|
1631
|
-
|
1632
|
-
|
1633
|
-
|
1634
|
-
|
1635
|
-
|
1726
|
+
size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_obm_buffered
|
1727
|
+
? ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize)
|
1728
|
+
: 0;
|
1729
|
+
|
1730
|
+
ZSTD_DCtx_updateOversizedDuration(zds, neededInBuffSize, neededOutBuffSize);
|
1731
|
+
|
1732
|
+
{ int const tooSmall = (zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize);
|
1733
|
+
int const tooLarge = ZSTD_DCtx_isOversizedTooLong(zds);
|
1734
|
+
|
1735
|
+
if (tooSmall || tooLarge) {
|
1736
|
+
size_t const bufferSize = neededInBuffSize + neededOutBuffSize;
|
1737
|
+
DEBUGLOG(4, "inBuff : from %u to %u",
|
1738
|
+
(U32)zds->inBuffSize, (U32)neededInBuffSize);
|
1739
|
+
DEBUGLOG(4, "outBuff : from %u to %u",
|
1740
|
+
(U32)zds->outBuffSize, (U32)neededOutBuffSize);
|
1741
|
+
if (zds->staticSize) { /* static DCtx */
|
1742
|
+
DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize);
|
1743
|
+
assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */
|
1744
|
+
RETURN_ERROR_IF(
|
1745
|
+
bufferSize > zds->staticSize - sizeof(ZSTD_DCtx),
|
1746
|
+
memory_allocation, "");
|
1747
|
+
} else {
|
1748
|
+
ZSTD_free(zds->inBuff, zds->customMem);
|
1749
|
+
zds->inBuffSize = 0;
|
1750
|
+
zds->outBuffSize = 0;
|
1751
|
+
zds->inBuff = (char*)ZSTD_malloc(bufferSize, zds->customMem);
|
1752
|
+
RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation, "");
|
1753
|
+
}
|
1754
|
+
zds->inBuffSize = neededInBuffSize;
|
1755
|
+
zds->outBuff = zds->inBuff + zds->inBuffSize;
|
1756
|
+
zds->outBuffSize = neededOutBuffSize;
|
1757
|
+
} } }
|
1636
1758
|
zds->streamStage = zdss_read;
|
1637
1759
|
/* fall-through */
|
1638
1760
|
|
1639
1761
|
case zdss_read:
|
1640
1762
|
DEBUGLOG(5, "stage zdss_read");
|
1641
|
-
{ size_t const neededInSize =
|
1763
|
+
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompressWithInputSize(zds, iend - ip);
|
1642
1764
|
DEBUGLOG(5, "neededInSize = %u", (U32)neededInSize);
|
1643
1765
|
if (neededInSize==0) { /* end of frame */
|
1644
1766
|
zds->streamStage = zdss_init;
|
@@ -1646,15 +1768,9 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|
1646
1768
|
break;
|
1647
1769
|
}
|
1648
1770
|
if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
|
1649
|
-
|
1650
|
-
size_t const decodedSize = ZSTD_decompressContinue(zds,
|
1651
|
-
zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart),
|
1652
|
-
ip, neededInSize);
|
1653
|
-
if (ZSTD_isError(decodedSize)) return decodedSize;
|
1771
|
+
FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), "");
|
1654
1772
|
ip += neededInSize;
|
1655
|
-
|
1656
|
-
zds->outEnd = zds->outStart + decodedSize;
|
1657
|
-
zds->streamStage = zdss_flush;
|
1773
|
+
/* Function modifies the stage so we must break */
|
1658
1774
|
break;
|
1659
1775
|
} }
|
1660
1776
|
if (ip==iend) { someMoreWork = 0; break; } /* no more input */
|
@@ -1666,6 +1782,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|
1666
1782
|
size_t const toLoad = neededInSize - zds->inPos;
|
1667
1783
|
int const isSkipFrame = ZSTD_isSkipFrame(zds);
|
1668
1784
|
size_t loadedSize;
|
1785
|
+
/* At this point we shouldn't be decompressing a block that we can stream. */
|
1786
|
+
assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, iend - ip));
|
1669
1787
|
if (isSkipFrame) {
|
1670
1788
|
loadedSize = MIN(toLoad, (size_t)(iend-ip));
|
1671
1789
|
} else {
|
@@ -1679,17 +1797,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|
1679
1797
|
if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */
|
1680
1798
|
|
1681
1799
|
/* decode loaded input */
|
1682
|
-
|
1683
|
-
|
1684
|
-
|
1685
|
-
|
1686
|
-
|
1687
|
-
if (!decodedSize && !isSkipFrame) { zds->streamStage = zdss_read; break; } /* this was just a header */
|
1688
|
-
zds->outEnd = zds->outStart + decodedSize;
|
1689
|
-
} }
|
1690
|
-
zds->streamStage = zdss_flush;
|
1691
|
-
/* fall-through */
|
1692
|
-
|
1800
|
+
zds->inPos = 0; /* input is consumed */
|
1801
|
+
FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, zds->inBuff, neededInSize), "");
|
1802
|
+
/* Function modifies the stage so we must break */
|
1803
|
+
break;
|
1804
|
+
}
|
1693
1805
|
case zdss_flush:
|
1694
1806
|
{ size_t const toFlushSize = zds->outEnd - zds->outStart;
|
1695
1807
|
size_t const flushedSize = ZSTD_limitCopy(op, oend-op, zds->outBuff + zds->outStart, toFlushSize);
|
@@ -1712,17 +1824,21 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|
1712
1824
|
|
1713
1825
|
default:
|
1714
1826
|
assert(0); /* impossible */
|
1715
|
-
RETURN_ERROR(GENERIC); /* some compiler require default to do something */
|
1827
|
+
RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */
|
1716
1828
|
} }
|
1717
1829
|
|
1718
1830
|
/* result */
|
1719
1831
|
input->pos = (size_t)(ip - (const char*)(input->src));
|
1720
1832
|
output->pos = (size_t)(op - (char*)(output->dst));
|
1833
|
+
|
1834
|
+
/* Update the expected output buffer for ZSTD_obm_stable. */
|
1835
|
+
zds->expectedOutBuffer = *output;
|
1836
|
+
|
1721
1837
|
if ((ip==istart) && (op==ostart)) { /* no forward progress */
|
1722
1838
|
zds->noForwardProgress ++;
|
1723
1839
|
if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) {
|
1724
|
-
RETURN_ERROR_IF(op==oend, dstSize_tooSmall);
|
1725
|
-
RETURN_ERROR_IF(ip==iend, srcSize_wrong);
|
1840
|
+
RETURN_ERROR_IF(op==oend, dstSize_tooSmall, "");
|
1841
|
+
RETURN_ERROR_IF(ip==iend, srcSize_wrong, "");
|
1726
1842
|
assert(0);
|
1727
1843
|
}
|
1728
1844
|
} else {
|