extzstd 0.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/HISTORY.ja.md +39 -0
- data/README.md +38 -56
- data/contrib/zstd/CHANGELOG +613 -0
- data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
- data/contrib/zstd/CONTRIBUTING.md +406 -0
- data/contrib/zstd/COPYING +339 -0
- data/contrib/zstd/Makefile +420 -0
- data/contrib/zstd/README.md +179 -41
- data/contrib/zstd/TESTING.md +44 -0
- data/contrib/zstd/appveyor.yml +292 -0
- data/contrib/zstd/lib/BUCK +234 -0
- data/contrib/zstd/lib/Makefile +451 -0
- data/contrib/zstd/lib/README.md +207 -0
- data/contrib/zstd/{common → lib/common}/bitstream.h +187 -138
- data/contrib/zstd/lib/common/compiler.h +288 -0
- data/contrib/zstd/lib/common/cpu.h +213 -0
- data/contrib/zstd/lib/common/debug.c +24 -0
- data/contrib/zstd/lib/common/debug.h +107 -0
- data/contrib/zstd/lib/common/entropy_common.c +362 -0
- data/contrib/zstd/{common → lib/common}/error_private.c +25 -12
- data/contrib/zstd/{common → lib/common}/error_private.h +14 -10
- data/contrib/zstd/{common → lib/common}/fse.h +173 -92
- data/contrib/zstd/{common → lib/common}/fse_decompress.c +149 -85
- data/contrib/zstd/lib/common/huf.h +361 -0
- data/contrib/zstd/{common → lib/common}/mem.h +115 -59
- data/contrib/zstd/lib/common/pool.c +350 -0
- data/contrib/zstd/lib/common/pool.h +84 -0
- data/contrib/zstd/lib/common/threading.c +122 -0
- data/contrib/zstd/lib/common/threading.h +155 -0
- data/contrib/zstd/{common → lib/common}/xxhash.c +55 -96
- data/contrib/zstd/{common → lib/common}/xxhash.h +23 -47
- data/contrib/zstd/lib/common/zstd_common.c +83 -0
- data/contrib/zstd/lib/common/zstd_deps.h +111 -0
- data/contrib/zstd/lib/common/zstd_errors.h +95 -0
- data/contrib/zstd/lib/common/zstd_internal.h +478 -0
- data/contrib/zstd/{compress → lib/compress}/fse_compress.c +214 -319
- data/contrib/zstd/lib/compress/hist.c +181 -0
- data/contrib/zstd/lib/compress/hist.h +75 -0
- data/contrib/zstd/lib/compress/huf_compress.c +913 -0
- data/contrib/zstd/lib/compress/zstd_compress.c +5208 -0
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +1203 -0
- data/contrib/zstd/lib/compress/zstd_compress_literals.c +158 -0
- data/contrib/zstd/lib/compress/zstd_compress_literals.h +29 -0
- data/contrib/zstd/lib/compress/zstd_compress_sequences.c +433 -0
- data/contrib/zstd/lib/compress/zstd_compress_sequences.h +54 -0
- data/contrib/zstd/lib/compress/zstd_compress_superblock.c +849 -0
- data/contrib/zstd/lib/compress/zstd_compress_superblock.h +32 -0
- data/contrib/zstd/lib/compress/zstd_cwksp.h +561 -0
- data/contrib/zstd/lib/compress/zstd_double_fast.c +521 -0
- data/contrib/zstd/lib/compress/zstd_double_fast.h +38 -0
- data/contrib/zstd/lib/compress/zstd_fast.c +496 -0
- data/contrib/zstd/lib/compress/zstd_fast.h +37 -0
- data/contrib/zstd/lib/compress/zstd_lazy.c +1412 -0
- data/contrib/zstd/lib/compress/zstd_lazy.h +87 -0
- data/contrib/zstd/lib/compress/zstd_ldm.c +660 -0
- data/contrib/zstd/lib/compress/zstd_ldm.h +116 -0
- data/contrib/zstd/lib/compress/zstd_opt.c +1345 -0
- data/contrib/zstd/lib/compress/zstd_opt.h +56 -0
- data/contrib/zstd/lib/compress/zstdmt_compress.c +1811 -0
- data/contrib/zstd/lib/compress/zstdmt_compress.h +110 -0
- data/contrib/zstd/lib/decompress/huf_decompress.c +1350 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.c +244 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
- data/contrib/zstd/lib/decompress/zstd_decompress.c +1930 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1540 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.h +62 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +190 -0
- data/contrib/zstd/{common → lib/deprecated}/zbuff.h +68 -45
- data/contrib/zstd/lib/deprecated/zbuff_common.c +26 -0
- data/contrib/zstd/lib/deprecated/zbuff_compress.c +147 -0
- data/contrib/zstd/lib/deprecated/zbuff_decompress.c +75 -0
- data/contrib/zstd/lib/dictBuilder/cover.c +1245 -0
- data/contrib/zstd/lib/dictBuilder/cover.h +157 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.c +3 -3
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.h +0 -0
- data/contrib/zstd/lib/dictBuilder/fastcover.c +758 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/zdict.c +318 -194
- data/contrib/zstd/lib/dictBuilder/zdict.h +305 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_legacy.h +171 -15
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.c +191 -124
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.h +19 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.c +125 -125
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.h +19 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.c +125 -124
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.h +20 -6
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.c +151 -299
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.h +19 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.c +237 -243
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.h +19 -6
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.c +130 -143
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.h +18 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.c +158 -157
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.h +19 -5
- data/contrib/zstd/lib/libzstd.pc.in +15 -0
- data/contrib/zstd/lib/zstd.h +2391 -0
- data/ext/depend +2 -0
- data/ext/extconf.rb +15 -6
- data/ext/extzstd.c +76 -145
- data/ext/extzstd.h +80 -31
- data/ext/extzstd_stream.c +417 -142
- data/ext/libzstd_conf.h +8 -0
- data/ext/zstd_common.c +10 -7
- data/ext/zstd_compress.c +14 -5
- data/ext/zstd_decompress.c +5 -4
- data/ext/zstd_dictbuilder.c +9 -4
- data/ext/zstd_dictbuilder_fastcover.c +3 -0
- data/ext/zstd_legacy_v01.c +3 -1
- data/ext/zstd_legacy_v02.c +3 -1
- data/ext/zstd_legacy_v03.c +3 -1
- data/ext/zstd_legacy_v04.c +3 -1
- data/ext/zstd_legacy_v05.c +3 -1
- data/ext/zstd_legacy_v06.c +3 -1
- data/ext/zstd_legacy_v07.c +3 -1
- data/gemstub.rb +10 -24
- data/lib/extzstd.rb +64 -179
- data/lib/extzstd/version.rb +6 -1
- data/test/test_basic.rb +9 -6
- metadata +113 -57
- data/HISTORY.ja +0 -5
- data/contrib/zstd/common/entropy_common.c +0 -225
- data/contrib/zstd/common/huf.h +0 -228
- data/contrib/zstd/common/zstd_common.c +0 -83
- data/contrib/zstd/common/zstd_errors.h +0 -60
- data/contrib/zstd/common/zstd_internal.h +0 -267
- data/contrib/zstd/compress/huf_compress.c +0 -533
- data/contrib/zstd/compress/zbuff_compress.c +0 -319
- data/contrib/zstd/compress/zstd_compress.c +0 -3264
- data/contrib/zstd/compress/zstd_opt.h +0 -900
- data/contrib/zstd/decompress/huf_decompress.c +0 -883
- data/contrib/zstd/decompress/zbuff_decompress.c +0 -252
- data/contrib/zstd/decompress/zstd_decompress.c +0 -1842
- data/contrib/zstd/dictBuilder/zdict.h +0 -111
- data/contrib/zstd/zstd.h +0 -640
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
|
|
2
|
-
* Copyright (c) 2016-
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
|
-
* This source code is licensed under the BSD-style license found in the
|
|
6
|
-
* LICENSE file in the root directory of this source tree
|
|
7
|
-
*
|
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
|
8
9
|
*/
|
|
9
10
|
|
|
10
11
|
#ifndef ZSTD_V01_H_28739879432
|
|
@@ -34,6 +35,19 @@ ZSTDv01_decompress() : decompress ZSTD frames compliant with v0.1.x format
|
|
|
34
35
|
size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize,
|
|
35
36
|
const void* src, size_t compressedSize);
|
|
36
37
|
|
|
38
|
+
/**
|
|
39
|
+
ZSTDv01_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.1.x format
|
|
40
|
+
srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
|
41
|
+
cSize (output parameter) : the number of bytes that would be read to decompress this frame
|
|
42
|
+
or an error code if it fails (which can be tested using ZSTDv01_isError())
|
|
43
|
+
dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
|
|
44
|
+
or ZSTD_CONTENTSIZE_ERROR if an error occurs
|
|
45
|
+
|
|
46
|
+
note : assumes `cSize` and `dBound` are _not_ NULL.
|
|
47
|
+
*/
|
|
48
|
+
void ZSTDv01_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
|
|
49
|
+
size_t* cSize, unsigned long long* dBound);
|
|
50
|
+
|
|
37
51
|
/**
|
|
38
52
|
ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error
|
|
39
53
|
*/
|
|
@@ -1,16 +1,17 @@
|
|
|
1
|
-
|
|
2
|
-
* Copyright (c) 2016-
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
|
-
* This source code is licensed under the BSD-style license found in the
|
|
6
|
-
* LICENSE file in the root directory of this source tree
|
|
7
|
-
*
|
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
|
8
9
|
*/
|
|
9
10
|
|
|
10
11
|
|
|
11
12
|
#include <stddef.h> /* size_t, ptrdiff_t */
|
|
12
13
|
#include "zstd_v02.h"
|
|
13
|
-
#include "error_private.h"
|
|
14
|
+
#include "../common/error_private.h"
|
|
14
15
|
|
|
15
16
|
|
|
16
17
|
/******************************************
|
|
@@ -88,7 +89,11 @@ extern "C" {
|
|
|
88
89
|
* Basic Types
|
|
89
90
|
*****************************************************************/
|
|
90
91
|
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
|
91
|
-
#
|
|
92
|
+
# if defined(_AIX)
|
|
93
|
+
# include <inttypes.h>
|
|
94
|
+
# else
|
|
95
|
+
# include <stdint.h> /* intptr_t */
|
|
96
|
+
# endif
|
|
92
97
|
typedef uint8_t BYTE;
|
|
93
98
|
typedef uint16_t U16;
|
|
94
99
|
typedef int16_t S16;
|
|
@@ -188,7 +193,7 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
|
|
|
188
193
|
memcpy(memPtr, &value, sizeof(value));
|
|
189
194
|
}
|
|
190
195
|
|
|
191
|
-
#endif
|
|
196
|
+
#endif /* MEM_FORCE_MEMORY_ACCESS */
|
|
192
197
|
|
|
193
198
|
|
|
194
199
|
MEM_STATIC U16 MEM_readLE16(const void* memPtr)
|
|
@@ -216,6 +221,11 @@ MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
|
|
|
216
221
|
}
|
|
217
222
|
}
|
|
218
223
|
|
|
224
|
+
MEM_STATIC U32 MEM_readLE24(const void* memPtr)
|
|
225
|
+
{
|
|
226
|
+
return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
|
|
227
|
+
}
|
|
228
|
+
|
|
219
229
|
MEM_STATIC U32 MEM_readLE32(const void* memPtr)
|
|
220
230
|
{
|
|
221
231
|
if (MEM_isLittleEndian())
|
|
@@ -329,18 +339,6 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
|
|
|
329
339
|
MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
|
|
330
340
|
|
|
331
341
|
|
|
332
|
-
/*
|
|
333
|
-
* Start by invoking BIT_initDStream().
|
|
334
|
-
* A chunk of the bitStream is then stored into a local register.
|
|
335
|
-
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
|
336
|
-
* You can then retrieve bitFields stored into the local register, **in reverse order**.
|
|
337
|
-
* Local register is manually filled from memory by the BIT_reloadDStream() method.
|
|
338
|
-
* A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BIT_DStream_unfinished.
|
|
339
|
-
* Otherwise, it can be less than that, so proceed accordingly.
|
|
340
|
-
* Checking if DStream has reached its end can be performed with BIT_endOfDStream()
|
|
341
|
-
*/
|
|
342
|
-
|
|
343
|
-
|
|
344
342
|
/******************************************
|
|
345
343
|
* unsafe API
|
|
346
344
|
******************************************/
|
|
@@ -352,14 +350,14 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
|
|
|
352
350
|
/****************************************************************
|
|
353
351
|
* Helper functions
|
|
354
352
|
****************************************************************/
|
|
355
|
-
MEM_STATIC unsigned BIT_highbit32 (
|
|
353
|
+
MEM_STATIC unsigned BIT_highbit32 (U32 val)
|
|
356
354
|
{
|
|
357
355
|
# if defined(_MSC_VER) /* Visual */
|
|
358
356
|
unsigned long r=0;
|
|
359
357
|
_BitScanReverse ( &r, val );
|
|
360
358
|
return (unsigned) r;
|
|
361
359
|
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
|
|
362
|
-
return
|
|
360
|
+
return __builtin_clz (val) ^ 31;
|
|
363
361
|
# else /* Software version */
|
|
364
362
|
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
|
365
363
|
U32 v = val;
|
|
@@ -410,11 +408,17 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
|
|
410
408
|
switch(srcSize)
|
|
411
409
|
{
|
|
412
410
|
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
|
|
411
|
+
/* fallthrough */
|
|
413
412
|
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
|
|
413
|
+
/* fallthrough */
|
|
414
414
|
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
|
|
415
|
+
/* fallthrough */
|
|
415
416
|
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
|
|
417
|
+
/* fallthrough */
|
|
416
418
|
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
|
|
419
|
+
/* fallthrough */
|
|
417
420
|
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
|
|
421
|
+
/* fallthrough */
|
|
418
422
|
default:;
|
|
419
423
|
}
|
|
420
424
|
contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
|
|
@@ -426,13 +430,6 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
|
|
426
430
|
return srcSize;
|
|
427
431
|
}
|
|
428
432
|
|
|
429
|
-
/*!BIT_lookBits
|
|
430
|
-
* Provides next n bits from local register
|
|
431
|
-
* local register is not modified (bits are still present for next read/look)
|
|
432
|
-
* On 32-bits, maxNbBits==25
|
|
433
|
-
* On 64-bits, maxNbBits==57
|
|
434
|
-
* @return : value extracted
|
|
435
|
-
*/
|
|
436
433
|
MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits)
|
|
437
434
|
{
|
|
438
435
|
const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
|
@@ -452,11 +449,6 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
|
|
|
452
449
|
bitD->bitsConsumed += nbBits;
|
|
453
450
|
}
|
|
454
451
|
|
|
455
|
-
/*!BIT_readBits
|
|
456
|
-
* Read next n bits from local register.
|
|
457
|
-
* pay attention to not read more than nbBits contained into local register.
|
|
458
|
-
* @return : extracted value.
|
|
459
|
-
*/
|
|
460
452
|
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
|
|
461
453
|
{
|
|
462
454
|
size_t value = BIT_lookBits(bitD, nbBits);
|
|
@@ -475,8 +467,8 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
|
|
475
467
|
|
|
476
468
|
MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
|
|
477
469
|
{
|
|
478
|
-
|
|
479
|
-
|
|
470
|
+
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
|
|
471
|
+
return BIT_DStream_overflow;
|
|
480
472
|
|
|
481
473
|
if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer))
|
|
482
474
|
{
|
|
@@ -694,55 +686,6 @@ static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bi
|
|
|
694
686
|
|
|
695
687
|
static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
|
|
696
688
|
|
|
697
|
-
/*
|
|
698
|
-
Let's now decompose FSE_decompress_usingDTable() into its unitary components.
|
|
699
|
-
You will decode FSE-encoded symbols from the bitStream,
|
|
700
|
-
and also any other bitFields you put in, **in reverse order**.
|
|
701
|
-
|
|
702
|
-
You will need a few variables to track your bitStream. They are :
|
|
703
|
-
|
|
704
|
-
BIT_DStream_t DStream; // Stream context
|
|
705
|
-
FSE_DState_t DState; // State context. Multiple ones are possible
|
|
706
|
-
FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
|
|
707
|
-
|
|
708
|
-
The first thing to do is to init the bitStream.
|
|
709
|
-
errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
|
|
710
|
-
|
|
711
|
-
You should then retrieve your initial state(s)
|
|
712
|
-
(in reverse flushing order if you have several ones) :
|
|
713
|
-
errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
|
|
714
|
-
|
|
715
|
-
You can then decode your data, symbol after symbol.
|
|
716
|
-
For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
|
|
717
|
-
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
|
|
718
|
-
unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
|
|
719
|
-
|
|
720
|
-
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
|
|
721
|
-
Note : maximum allowed nbBits is 25, for 32-bits compatibility
|
|
722
|
-
size_t bitField = BIT_readBits(&DStream, nbBits);
|
|
723
|
-
|
|
724
|
-
All above operations only read from local register (which size depends on size_t).
|
|
725
|
-
Refueling the register from memory is manually performed by the reload method.
|
|
726
|
-
endSignal = FSE_reloadDStream(&DStream);
|
|
727
|
-
|
|
728
|
-
BIT_reloadDStream() result tells if there is still some more data to read from DStream.
|
|
729
|
-
BIT_DStream_unfinished : there is still some data left into the DStream.
|
|
730
|
-
BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
|
|
731
|
-
BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
|
732
|
-
BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
|
733
|
-
|
|
734
|
-
When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
|
|
735
|
-
to properly detect the exact end of stream.
|
|
736
|
-
After each decoded symbol, check if DStream is fully consumed using this simple test :
|
|
737
|
-
BIT_reloadDStream(&DStream) >= BIT_DStream_completed
|
|
738
|
-
|
|
739
|
-
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
|
|
740
|
-
Checking if DStream has reached its end is performed by :
|
|
741
|
-
BIT_endOfDStream(&DStream);
|
|
742
|
-
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
|
|
743
|
-
FSE_endOfDState(&DState);
|
|
744
|
-
*/
|
|
745
|
-
|
|
746
689
|
|
|
747
690
|
/******************************************
|
|
748
691
|
* FSE unsafe API
|
|
@@ -1334,8 +1277,8 @@ static size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsi
|
|
|
1334
1277
|
else
|
|
1335
1278
|
{
|
|
1336
1279
|
bitCount -= (int)(8 * (iend - 4 - ip));
|
|
1337
|
-
|
|
1338
|
-
|
|
1280
|
+
ip = iend - 4;
|
|
1281
|
+
}
|
|
1339
1282
|
bitStream = MEM_readLE32(ip) >> (bitCount & 31);
|
|
1340
1283
|
}
|
|
1341
1284
|
}
|
|
@@ -2040,7 +1983,7 @@ static size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
|
|
|
2040
1983
|
rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
|
|
2041
1984
|
}
|
|
2042
1985
|
|
|
2043
|
-
|
|
1986
|
+
/* Build rankVal */
|
|
2044
1987
|
{
|
|
2045
1988
|
const U32 minBits = tableLog+1 - maxW;
|
|
2046
1989
|
U32 nextRankVal = 0;
|
|
@@ -2374,7 +2317,7 @@ static size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
|
|
|
2374
2317
|
rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
|
|
2375
2318
|
}
|
|
2376
2319
|
|
|
2377
|
-
|
|
2320
|
+
/* Build rankVal */
|
|
2378
2321
|
{
|
|
2379
2322
|
const U32 minBits = tableLog+1 - maxW;
|
|
2380
2323
|
U32 nextRankVal = 0;
|
|
@@ -2794,6 +2737,8 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
|
|
|
2794
2737
|
#define LITERAL_NOENTROPY 63
|
|
2795
2738
|
#define COMMAND_NOENTROPY 7 /* to remove */
|
|
2796
2739
|
|
|
2740
|
+
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
|
|
2741
|
+
|
|
2797
2742
|
static const size_t ZSTD_blockHeaderSize = 3;
|
|
2798
2743
|
static const size_t ZSTD_frameHeaderSize = 4;
|
|
2799
2744
|
|
|
@@ -2808,7 +2753,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
|
|
|
2808
2753
|
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
|
|
2809
2754
|
|
|
2810
2755
|
/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
|
|
2811
|
-
static void ZSTD_wildcopy(void* dst, const void* src,
|
|
2756
|
+
static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
|
|
2812
2757
|
{
|
|
2813
2758
|
const BYTE* ip = (const BYTE*)src;
|
|
2814
2759
|
BYTE* op = (BYTE*)dst;
|
|
@@ -2868,7 +2813,6 @@ struct ZSTD_DCtx_s
|
|
|
2868
2813
|
blockType_t bType;
|
|
2869
2814
|
U32 phase;
|
|
2870
2815
|
const BYTE* litPtr;
|
|
2871
|
-
size_t litBufSize;
|
|
2872
2816
|
size_t litSize;
|
|
2873
2817
|
BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
|
|
2874
2818
|
}; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
|
|
@@ -2896,7 +2840,9 @@ static size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockPropertie
|
|
|
2896
2840
|
static size_t ZSTD_copyUncompressedBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
|
2897
2841
|
{
|
|
2898
2842
|
if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
|
|
2899
|
-
|
|
2843
|
+
if (srcSize > 0) {
|
|
2844
|
+
memcpy(dst, src, srcSize);
|
|
2845
|
+
}
|
|
2900
2846
|
return srcSize;
|
|
2901
2847
|
}
|
|
2902
2848
|
|
|
@@ -2940,8 +2886,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
|
2940
2886
|
size_t litSize = BLOCKSIZE;
|
|
2941
2887
|
const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
|
|
2942
2888
|
dctx->litPtr = dctx->litBuffer;
|
|
2943
|
-
dctx->litBufSize = BLOCKSIZE;
|
|
2944
2889
|
dctx->litSize = litSize;
|
|
2890
|
+
memset(dctx->litBuffer + dctx->litSize, 0, 8);
|
|
2945
2891
|
return readSize; /* works if it's an error too */
|
|
2946
2892
|
}
|
|
2947
2893
|
case IS_RAW:
|
|
@@ -2949,16 +2895,16 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
|
2949
2895
|
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
|
|
2950
2896
|
if (litSize > srcSize-11) /* risk of reading too far with wildcopy */
|
|
2951
2897
|
{
|
|
2952
|
-
|
|
2953
|
-
|
|
2954
|
-
|
|
2955
|
-
|
|
2956
|
-
|
|
2957
|
-
|
|
2958
|
-
|
|
2959
|
-
|
|
2898
|
+
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
|
|
2899
|
+
if (litSize > srcSize-3) return ERROR(corruption_detected);
|
|
2900
|
+
memcpy(dctx->litBuffer, istart, litSize);
|
|
2901
|
+
dctx->litPtr = dctx->litBuffer;
|
|
2902
|
+
dctx->litSize = litSize;
|
|
2903
|
+
memset(dctx->litBuffer + dctx->litSize, 0, 8);
|
|
2904
|
+
return litSize+3;
|
|
2905
|
+
}
|
|
2906
|
+
/* direct reference into compressed stream */
|
|
2960
2907
|
dctx->litPtr = istart+3;
|
|
2961
|
-
dctx->litBufSize = srcSize-3;
|
|
2962
2908
|
dctx->litSize = litSize;
|
|
2963
2909
|
return litSize+3;
|
|
2964
2910
|
}
|
|
@@ -2966,9 +2912,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
|
2966
2912
|
{
|
|
2967
2913
|
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
|
|
2968
2914
|
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
|
|
2969
|
-
memset(dctx->litBuffer, istart[3], litSize);
|
|
2915
|
+
memset(dctx->litBuffer, istart[3], litSize + 8);
|
|
2970
2916
|
dctx->litPtr = dctx->litBuffer;
|
|
2971
|
-
dctx->litBufSize = BLOCKSIZE;
|
|
2972
2917
|
dctx->litSize = litSize;
|
|
2973
2918
|
return 4;
|
|
2974
2919
|
}
|
|
@@ -3110,11 +3055,11 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
|
3110
3055
|
seqState->prevOffset = seq->offset;
|
|
3111
3056
|
if (litLength == MaxLL)
|
|
3112
3057
|
{
|
|
3113
|
-
U32 add = *dumps
|
|
3058
|
+
const U32 add = dumps<de ? *dumps++ : 0;
|
|
3114
3059
|
if (add < 255) litLength += add;
|
|
3115
|
-
else
|
|
3060
|
+
else if (dumps + 3 <= de)
|
|
3116
3061
|
{
|
|
3117
|
-
litLength =
|
|
3062
|
+
litLength = MEM_readLE24(dumps);
|
|
3118
3063
|
dumps += 3;
|
|
3119
3064
|
}
|
|
3120
3065
|
if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
|
|
@@ -3140,11 +3085,11 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
|
3140
3085
|
matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
|
|
3141
3086
|
if (matchLength == MaxML)
|
|
3142
3087
|
{
|
|
3143
|
-
U32 add = *dumps
|
|
3088
|
+
const U32 add = dumps<de ? *dumps++ : 0;
|
|
3144
3089
|
if (add < 255) matchLength += add;
|
|
3145
|
-
else
|
|
3090
|
+
else if (dumps + 3 <= de)
|
|
3146
3091
|
{
|
|
3147
|
-
matchLength =
|
|
3092
|
+
matchLength = MEM_readLE24(dumps);
|
|
3148
3093
|
dumps += 3;
|
|
3149
3094
|
}
|
|
3150
3095
|
if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
|
|
@@ -3165,7 +3110,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
|
3165
3110
|
BYTE* const base, BYTE* const oend)
|
|
3166
3111
|
{
|
|
3167
3112
|
static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
|
|
3168
|
-
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /*
|
|
3113
|
+
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* subtracted */
|
|
3169
3114
|
const BYTE* const ostart = op;
|
|
3170
3115
|
BYTE* const oLitEnd = op + sequence.litLength;
|
|
3171
3116
|
BYTE* const oMatchEnd = op + sequence.litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */
|
|
@@ -3175,7 +3120,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
|
3175
3120
|
/* checks */
|
|
3176
3121
|
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
|
|
3177
3122
|
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
|
|
3178
|
-
if (litEnd > litLimit
|
|
3123
|
+
if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
|
|
3179
3124
|
|
|
3180
3125
|
/* copy Literals */
|
|
3181
3126
|
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
|
@@ -3209,7 +3154,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
|
3209
3154
|
}
|
|
3210
3155
|
op += 8; match += 8;
|
|
3211
3156
|
|
|
3212
|
-
if (oMatchEnd > oend-
|
|
3157
|
+
if (oMatchEnd > oend-(16-MINMATCH))
|
|
3213
3158
|
{
|
|
3214
3159
|
if (op < oend_8)
|
|
3215
3160
|
{
|
|
@@ -3221,7 +3166,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
|
3221
3166
|
}
|
|
3222
3167
|
else
|
|
3223
3168
|
{
|
|
3224
|
-
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
|
|
3169
|
+
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
|
|
3225
3170
|
}
|
|
3226
3171
|
}
|
|
3227
3172
|
|
|
@@ -3241,7 +3186,6 @@ static size_t ZSTD_decompressSequences(
|
|
|
3241
3186
|
BYTE* const oend = ostart + maxDstSize;
|
|
3242
3187
|
size_t errorCode, dumpsLength;
|
|
3243
3188
|
const BYTE* litPtr = dctx->litPtr;
|
|
3244
|
-
const BYTE* const litMax = litPtr + dctx->litBufSize;
|
|
3245
3189
|
const BYTE* const litEnd = litPtr + dctx->litSize;
|
|
3246
3190
|
int nbSeq;
|
|
3247
3191
|
const BYTE* dumps;
|
|
@@ -3277,7 +3221,7 @@ static size_t ZSTD_decompressSequences(
|
|
|
3277
3221
|
size_t oneSeqSize;
|
|
3278
3222
|
nbSeq--;
|
|
3279
3223
|
ZSTD_decodeSequence(&sequence, &seqState);
|
|
3280
|
-
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr,
|
|
3224
|
+
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
|
|
3281
3225
|
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
|
|
3282
3226
|
op += oneSeqSize;
|
|
3283
3227
|
}
|
|
@@ -3291,8 +3235,10 @@ static size_t ZSTD_decompressSequences(
|
|
|
3291
3235
|
size_t lastLLSize = litEnd - litPtr;
|
|
3292
3236
|
if (litPtr > litEnd) return ERROR(corruption_detected);
|
|
3293
3237
|
if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
|
|
3294
|
-
if (
|
|
3295
|
-
|
|
3238
|
+
if (lastLLSize > 0) {
|
|
3239
|
+
if (op != litPtr) memmove(op, litPtr, lastLLSize);
|
|
3240
|
+
op += lastLLSize;
|
|
3241
|
+
}
|
|
3296
3242
|
}
|
|
3297
3243
|
}
|
|
3298
3244
|
|
|
@@ -3382,6 +3328,60 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
|
|
|
3382
3328
|
return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
|
3383
3329
|
}
|
|
3384
3330
|
|
|
3331
|
+
/* ZSTD_errorFrameSizeInfoLegacy() :
|
|
3332
|
+
assumes `cSize` and `dBound` are _not_ NULL */
|
|
3333
|
+
static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
|
|
3334
|
+
{
|
|
3335
|
+
*cSize = ret;
|
|
3336
|
+
*dBound = ZSTD_CONTENTSIZE_ERROR;
|
|
3337
|
+
}
|
|
3338
|
+
|
|
3339
|
+
void ZSTDv02_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
|
|
3340
|
+
{
|
|
3341
|
+
const BYTE* ip = (const BYTE*)src;
|
|
3342
|
+
size_t remainingSize = srcSize;
|
|
3343
|
+
size_t nbBlocks = 0;
|
|
3344
|
+
U32 magicNumber;
|
|
3345
|
+
blockProperties_t blockProperties;
|
|
3346
|
+
|
|
3347
|
+
/* Frame Header */
|
|
3348
|
+
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) {
|
|
3349
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
|
3350
|
+
return;
|
|
3351
|
+
}
|
|
3352
|
+
magicNumber = MEM_readLE32(src);
|
|
3353
|
+
if (magicNumber != ZSTD_magicNumber) {
|
|
3354
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
|
|
3355
|
+
return;
|
|
3356
|
+
}
|
|
3357
|
+
ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
|
|
3358
|
+
|
|
3359
|
+
/* Loop on each block */
|
|
3360
|
+
while (1)
|
|
3361
|
+
{
|
|
3362
|
+
size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
|
|
3363
|
+
if (ZSTD_isError(cBlockSize)) {
|
|
3364
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
|
|
3365
|
+
return;
|
|
3366
|
+
}
|
|
3367
|
+
|
|
3368
|
+
ip += ZSTD_blockHeaderSize;
|
|
3369
|
+
remainingSize -= ZSTD_blockHeaderSize;
|
|
3370
|
+
if (cBlockSize > remainingSize) {
|
|
3371
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
|
3372
|
+
return;
|
|
3373
|
+
}
|
|
3374
|
+
|
|
3375
|
+
if (cBlockSize == 0) break; /* bt_end */
|
|
3376
|
+
|
|
3377
|
+
ip += cBlockSize;
|
|
3378
|
+
remainingSize -= cBlockSize;
|
|
3379
|
+
nbBlocks++;
|
|
3380
|
+
}
|
|
3381
|
+
|
|
3382
|
+
*cSize = ip - (const BYTE*)src;
|
|
3383
|
+
*dBound = nbBlocks * BLOCKSIZE;
|
|
3384
|
+
}
|
|
3385
3385
|
|
|
3386
3386
|
/*******************************
|
|
3387
3387
|
* Streaming Decompression API
|
|
@@ -3487,36 +3487,36 @@ static size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSi
|
|
|
3487
3487
|
|
|
3488
3488
|
unsigned ZSTDv02_isError(size_t code)
|
|
3489
3489
|
{
|
|
3490
|
-
|
|
3490
|
+
return ZSTD_isError(code);
|
|
3491
3491
|
}
|
|
3492
3492
|
|
|
3493
3493
|
size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
|
|
3494
3494
|
const void* src, size_t compressedSize)
|
|
3495
3495
|
{
|
|
3496
|
-
|
|
3496
|
+
return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
|
|
3497
3497
|
}
|
|
3498
3498
|
|
|
3499
3499
|
ZSTDv02_Dctx* ZSTDv02_createDCtx(void)
|
|
3500
3500
|
{
|
|
3501
|
-
|
|
3501
|
+
return (ZSTDv02_Dctx*)ZSTD_createDCtx();
|
|
3502
3502
|
}
|
|
3503
3503
|
|
|
3504
3504
|
size_t ZSTDv02_freeDCtx(ZSTDv02_Dctx* dctx)
|
|
3505
3505
|
{
|
|
3506
|
-
|
|
3506
|
+
return ZSTD_freeDCtx((ZSTD_DCtx*)dctx);
|
|
3507
3507
|
}
|
|
3508
3508
|
|
|
3509
3509
|
size_t ZSTDv02_resetDCtx(ZSTDv02_Dctx* dctx)
|
|
3510
3510
|
{
|
|
3511
|
-
|
|
3511
|
+
return ZSTD_resetDCtx((ZSTD_DCtx*)dctx);
|
|
3512
3512
|
}
|
|
3513
3513
|
|
|
3514
3514
|
size_t ZSTDv02_nextSrcSizeToDecompress(ZSTDv02_Dctx* dctx)
|
|
3515
3515
|
{
|
|
3516
|
-
|
|
3516
|
+
return ZSTD_nextSrcSizeToDecompress((ZSTD_DCtx*)dctx);
|
|
3517
3517
|
}
|
|
3518
3518
|
|
|
3519
3519
|
size_t ZSTDv02_decompressContinue(ZSTDv02_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
|
3520
3520
|
{
|
|
3521
|
-
|
|
3521
|
+
return ZSTD_decompressContinue((ZSTD_DCtx*)dctx, dst, maxDstSize, src, srcSize);
|
|
3522
3522
|
}
|