extzstd 0.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/HISTORY.ja.md +39 -0
- data/README.md +38 -56
- data/contrib/zstd/CHANGELOG +613 -0
- data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
- data/contrib/zstd/CONTRIBUTING.md +406 -0
- data/contrib/zstd/COPYING +339 -0
- data/contrib/zstd/Makefile +420 -0
- data/contrib/zstd/README.md +179 -41
- data/contrib/zstd/TESTING.md +44 -0
- data/contrib/zstd/appveyor.yml +292 -0
- data/contrib/zstd/lib/BUCK +234 -0
- data/contrib/zstd/lib/Makefile +451 -0
- data/contrib/zstd/lib/README.md +207 -0
- data/contrib/zstd/{common → lib/common}/bitstream.h +187 -138
- data/contrib/zstd/lib/common/compiler.h +288 -0
- data/contrib/zstd/lib/common/cpu.h +213 -0
- data/contrib/zstd/lib/common/debug.c +24 -0
- data/contrib/zstd/lib/common/debug.h +107 -0
- data/contrib/zstd/lib/common/entropy_common.c +362 -0
- data/contrib/zstd/{common → lib/common}/error_private.c +25 -12
- data/contrib/zstd/{common → lib/common}/error_private.h +14 -10
- data/contrib/zstd/{common → lib/common}/fse.h +173 -92
- data/contrib/zstd/{common → lib/common}/fse_decompress.c +149 -85
- data/contrib/zstd/lib/common/huf.h +361 -0
- data/contrib/zstd/{common → lib/common}/mem.h +115 -59
- data/contrib/zstd/lib/common/pool.c +350 -0
- data/contrib/zstd/lib/common/pool.h +84 -0
- data/contrib/zstd/lib/common/threading.c +122 -0
- data/contrib/zstd/lib/common/threading.h +155 -0
- data/contrib/zstd/{common → lib/common}/xxhash.c +55 -96
- data/contrib/zstd/{common → lib/common}/xxhash.h +23 -47
- data/contrib/zstd/lib/common/zstd_common.c +83 -0
- data/contrib/zstd/lib/common/zstd_deps.h +111 -0
- data/contrib/zstd/lib/common/zstd_errors.h +95 -0
- data/contrib/zstd/lib/common/zstd_internal.h +478 -0
- data/contrib/zstd/{compress → lib/compress}/fse_compress.c +214 -319
- data/contrib/zstd/lib/compress/hist.c +181 -0
- data/contrib/zstd/lib/compress/hist.h +75 -0
- data/contrib/zstd/lib/compress/huf_compress.c +913 -0
- data/contrib/zstd/lib/compress/zstd_compress.c +5208 -0
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +1203 -0
- data/contrib/zstd/lib/compress/zstd_compress_literals.c +158 -0
- data/contrib/zstd/lib/compress/zstd_compress_literals.h +29 -0
- data/contrib/zstd/lib/compress/zstd_compress_sequences.c +433 -0
- data/contrib/zstd/lib/compress/zstd_compress_sequences.h +54 -0
- data/contrib/zstd/lib/compress/zstd_compress_superblock.c +849 -0
- data/contrib/zstd/lib/compress/zstd_compress_superblock.h +32 -0
- data/contrib/zstd/lib/compress/zstd_cwksp.h +561 -0
- data/contrib/zstd/lib/compress/zstd_double_fast.c +521 -0
- data/contrib/zstd/lib/compress/zstd_double_fast.h +38 -0
- data/contrib/zstd/lib/compress/zstd_fast.c +496 -0
- data/contrib/zstd/lib/compress/zstd_fast.h +37 -0
- data/contrib/zstd/lib/compress/zstd_lazy.c +1412 -0
- data/contrib/zstd/lib/compress/zstd_lazy.h +87 -0
- data/contrib/zstd/lib/compress/zstd_ldm.c +660 -0
- data/contrib/zstd/lib/compress/zstd_ldm.h +116 -0
- data/contrib/zstd/lib/compress/zstd_opt.c +1345 -0
- data/contrib/zstd/lib/compress/zstd_opt.h +56 -0
- data/contrib/zstd/lib/compress/zstdmt_compress.c +1811 -0
- data/contrib/zstd/lib/compress/zstdmt_compress.h +110 -0
- data/contrib/zstd/lib/decompress/huf_decompress.c +1350 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.c +244 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
- data/contrib/zstd/lib/decompress/zstd_decompress.c +1930 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1540 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.h +62 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +190 -0
- data/contrib/zstd/{common → lib/deprecated}/zbuff.h +68 -45
- data/contrib/zstd/lib/deprecated/zbuff_common.c +26 -0
- data/contrib/zstd/lib/deprecated/zbuff_compress.c +147 -0
- data/contrib/zstd/lib/deprecated/zbuff_decompress.c +75 -0
- data/contrib/zstd/lib/dictBuilder/cover.c +1245 -0
- data/contrib/zstd/lib/dictBuilder/cover.h +157 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.c +3 -3
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.h +0 -0
- data/contrib/zstd/lib/dictBuilder/fastcover.c +758 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/zdict.c +318 -194
- data/contrib/zstd/lib/dictBuilder/zdict.h +305 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_legacy.h +171 -15
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.c +191 -124
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.h +19 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.c +125 -125
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.h +19 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.c +125 -124
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.h +20 -6
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.c +151 -299
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.h +19 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.c +237 -243
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.h +19 -6
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.c +130 -143
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.h +18 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.c +158 -157
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.h +19 -5
- data/contrib/zstd/lib/libzstd.pc.in +15 -0
- data/contrib/zstd/lib/zstd.h +2391 -0
- data/ext/depend +2 -0
- data/ext/extconf.rb +15 -6
- data/ext/extzstd.c +76 -145
- data/ext/extzstd.h +80 -31
- data/ext/extzstd_stream.c +417 -142
- data/ext/libzstd_conf.h +8 -0
- data/ext/zstd_common.c +10 -7
- data/ext/zstd_compress.c +14 -5
- data/ext/zstd_decompress.c +5 -4
- data/ext/zstd_dictbuilder.c +9 -4
- data/ext/zstd_dictbuilder_fastcover.c +3 -0
- data/ext/zstd_legacy_v01.c +3 -1
- data/ext/zstd_legacy_v02.c +3 -1
- data/ext/zstd_legacy_v03.c +3 -1
- data/ext/zstd_legacy_v04.c +3 -1
- data/ext/zstd_legacy_v05.c +3 -1
- data/ext/zstd_legacy_v06.c +3 -1
- data/ext/zstd_legacy_v07.c +3 -1
- data/gemstub.rb +10 -24
- data/lib/extzstd.rb +64 -179
- data/lib/extzstd/version.rb +6 -1
- data/test/test_basic.rb +9 -6
- metadata +113 -57
- data/HISTORY.ja +0 -5
- data/contrib/zstd/common/entropy_common.c +0 -225
- data/contrib/zstd/common/huf.h +0 -228
- data/contrib/zstd/common/zstd_common.c +0 -83
- data/contrib/zstd/common/zstd_errors.h +0 -60
- data/contrib/zstd/common/zstd_internal.h +0 -267
- data/contrib/zstd/compress/huf_compress.c +0 -533
- data/contrib/zstd/compress/zbuff_compress.c +0 -319
- data/contrib/zstd/compress/zstd_compress.c +0 -3264
- data/contrib/zstd/compress/zstd_opt.h +0 -900
- data/contrib/zstd/decompress/huf_decompress.c +0 -883
- data/contrib/zstd/decompress/zbuff_decompress.c +0 -252
- data/contrib/zstd/decompress/zstd_decompress.c +0 -1842
- data/contrib/zstd/dictBuilder/zdict.h +0 -111
- data/contrib/zstd/zstd.h +0 -640
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
|
|
2
|
-
* Copyright (c) 2016-
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
|
-
* This source code is licensed under the BSD-style license found in the
|
|
6
|
-
* LICENSE file in the root directory of this source tree
|
|
7
|
-
*
|
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
|
8
9
|
*/
|
|
9
10
|
|
|
10
11
|
#ifndef ZSTD_V02_H_4174539423
|
|
@@ -34,6 +35,19 @@ ZSTDv02_decompress() : decompress ZSTD frames compliant with v0.2.x format
|
|
|
34
35
|
size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
|
|
35
36
|
const void* src, size_t compressedSize);
|
|
36
37
|
|
|
38
|
+
/**
|
|
39
|
+
ZSTDv02_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.2.x format
|
|
40
|
+
srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
|
41
|
+
cSize (output parameter) : the number of bytes that would be read to decompress this frame
|
|
42
|
+
or an error code if it fails (which can be tested using ZSTDv01_isError())
|
|
43
|
+
dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
|
|
44
|
+
or ZSTD_CONTENTSIZE_ERROR if an error occurs
|
|
45
|
+
|
|
46
|
+
note : assumes `cSize` and `dBound` are _not_ NULL.
|
|
47
|
+
*/
|
|
48
|
+
void ZSTDv02_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
|
|
49
|
+
size_t* cSize, unsigned long long* dBound);
|
|
50
|
+
|
|
37
51
|
/**
|
|
38
52
|
ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error
|
|
39
53
|
*/
|
|
@@ -1,16 +1,17 @@
|
|
|
1
|
-
|
|
2
|
-
* Copyright (c) 2016-
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
|
-
* This source code is licensed under the BSD-style license found in the
|
|
6
|
-
* LICENSE file in the root directory of this source tree
|
|
7
|
-
*
|
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
|
8
9
|
*/
|
|
9
10
|
|
|
10
11
|
|
|
11
12
|
#include <stddef.h> /* size_t, ptrdiff_t */
|
|
12
13
|
#include "zstd_v03.h"
|
|
13
|
-
#include "error_private.h"
|
|
14
|
+
#include "../common/error_private.h"
|
|
14
15
|
|
|
15
16
|
|
|
16
17
|
/******************************************
|
|
@@ -89,7 +90,11 @@ extern "C" {
|
|
|
89
90
|
* Basic Types
|
|
90
91
|
*****************************************************************/
|
|
91
92
|
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
|
92
|
-
#
|
|
93
|
+
# if defined(_AIX)
|
|
94
|
+
# include <inttypes.h>
|
|
95
|
+
# else
|
|
96
|
+
# include <stdint.h> /* intptr_t */
|
|
97
|
+
# endif
|
|
93
98
|
typedef uint8_t BYTE;
|
|
94
99
|
typedef uint16_t U16;
|
|
95
100
|
typedef int16_t S16;
|
|
@@ -190,7 +195,7 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
|
|
|
190
195
|
}
|
|
191
196
|
|
|
192
197
|
|
|
193
|
-
#endif
|
|
198
|
+
#endif /* MEM_FORCE_MEMORY_ACCESS */
|
|
194
199
|
|
|
195
200
|
|
|
196
201
|
MEM_STATIC U16 MEM_readLE16(const void* memPtr)
|
|
@@ -218,6 +223,11 @@ MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
|
|
|
218
223
|
}
|
|
219
224
|
}
|
|
220
225
|
|
|
226
|
+
MEM_STATIC U32 MEM_readLE24(const void* memPtr)
|
|
227
|
+
{
|
|
228
|
+
return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
|
|
229
|
+
}
|
|
230
|
+
|
|
221
231
|
MEM_STATIC U32 MEM_readLE32(const void* memPtr)
|
|
222
232
|
{
|
|
223
233
|
if (MEM_isLittleEndian())
|
|
@@ -331,17 +341,6 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
|
|
|
331
341
|
MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
|
|
332
342
|
|
|
333
343
|
|
|
334
|
-
/*
|
|
335
|
-
* Start by invoking BIT_initDStream().
|
|
336
|
-
* A chunk of the bitStream is then stored into a local register.
|
|
337
|
-
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
|
338
|
-
* You can then retrieve bitFields stored into the local register, **in reverse order**.
|
|
339
|
-
* Local register is manually filled from memory by the BIT_reloadDStream() method.
|
|
340
|
-
* A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BIT_DStream_unfinished.
|
|
341
|
-
* Otherwise, it can be less than that, so proceed accordingly.
|
|
342
|
-
* Checking if DStream has reached its end can be performed with BIT_endOfDStream()
|
|
343
|
-
*/
|
|
344
|
-
|
|
345
344
|
|
|
346
345
|
/******************************************
|
|
347
346
|
* unsafe API
|
|
@@ -354,14 +353,14 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
|
|
|
354
353
|
/****************************************************************
|
|
355
354
|
* Helper functions
|
|
356
355
|
****************************************************************/
|
|
357
|
-
MEM_STATIC unsigned BIT_highbit32 (
|
|
356
|
+
MEM_STATIC unsigned BIT_highbit32 (U32 val)
|
|
358
357
|
{
|
|
359
358
|
# if defined(_MSC_VER) /* Visual */
|
|
360
359
|
unsigned long r=0;
|
|
361
360
|
_BitScanReverse ( &r, val );
|
|
362
361
|
return (unsigned) r;
|
|
363
362
|
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
|
|
364
|
-
return
|
|
363
|
+
return __builtin_clz (val) ^ 31;
|
|
365
364
|
# else /* Software version */
|
|
366
365
|
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
|
367
366
|
U32 v = val;
|
|
@@ -412,11 +411,17 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
|
|
412
411
|
switch(srcSize)
|
|
413
412
|
{
|
|
414
413
|
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
|
|
414
|
+
/* fallthrough */
|
|
415
415
|
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
|
|
416
|
+
/* fallthrough */
|
|
416
417
|
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
|
|
418
|
+
/* fallthrough */
|
|
417
419
|
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
|
|
420
|
+
/* fallthrough */
|
|
418
421
|
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
|
|
422
|
+
/* fallthrough */
|
|
419
423
|
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
|
|
424
|
+
/* fallthrough */
|
|
420
425
|
default:;
|
|
421
426
|
}
|
|
422
427
|
contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
|
|
@@ -427,14 +432,6 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
|
|
427
432
|
|
|
428
433
|
return srcSize;
|
|
429
434
|
}
|
|
430
|
-
|
|
431
|
-
/*!BIT_lookBits
|
|
432
|
-
* Provides next n bits from local register
|
|
433
|
-
* local register is not modified (bits are still present for next read/look)
|
|
434
|
-
* On 32-bits, maxNbBits==25
|
|
435
|
-
* On 64-bits, maxNbBits==57
|
|
436
|
-
* @return : value extracted
|
|
437
|
-
*/
|
|
438
435
|
MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits)
|
|
439
436
|
{
|
|
440
437
|
const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
|
@@ -454,11 +451,6 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
|
|
|
454
451
|
bitD->bitsConsumed += nbBits;
|
|
455
452
|
}
|
|
456
453
|
|
|
457
|
-
/*!BIT_readBits
|
|
458
|
-
* Read next n bits from local register.
|
|
459
|
-
* pay attention to not read more than nbBits contained into local register.
|
|
460
|
-
* @return : extracted value.
|
|
461
|
-
*/
|
|
462
454
|
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
|
|
463
455
|
{
|
|
464
456
|
size_t value = BIT_lookBits(bitD, nbBits);
|
|
@@ -477,8 +469,8 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
|
|
477
469
|
|
|
478
470
|
MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
|
|
479
471
|
{
|
|
480
|
-
|
|
481
|
-
|
|
472
|
+
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
|
|
473
|
+
return BIT_DStream_overflow;
|
|
482
474
|
|
|
483
475
|
if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer))
|
|
484
476
|
{
|
|
@@ -696,55 +688,6 @@ static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bi
|
|
|
696
688
|
|
|
697
689
|
static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
|
|
698
690
|
|
|
699
|
-
/*
|
|
700
|
-
Let's now decompose FSE_decompress_usingDTable() into its unitary components.
|
|
701
|
-
You will decode FSE-encoded symbols from the bitStream,
|
|
702
|
-
and also any other bitFields you put in, **in reverse order**.
|
|
703
|
-
|
|
704
|
-
You will need a few variables to track your bitStream. They are :
|
|
705
|
-
|
|
706
|
-
BIT_DStream_t DStream; // Stream context
|
|
707
|
-
FSE_DState_t DState; // State context. Multiple ones are possible
|
|
708
|
-
FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
|
|
709
|
-
|
|
710
|
-
The first thing to do is to init the bitStream.
|
|
711
|
-
errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
|
|
712
|
-
|
|
713
|
-
You should then retrieve your initial state(s)
|
|
714
|
-
(in reverse flushing order if you have several ones) :
|
|
715
|
-
errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
|
|
716
|
-
|
|
717
|
-
You can then decode your data, symbol after symbol.
|
|
718
|
-
For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
|
|
719
|
-
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
|
|
720
|
-
unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
|
|
721
|
-
|
|
722
|
-
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
|
|
723
|
-
Note : maximum allowed nbBits is 25, for 32-bits compatibility
|
|
724
|
-
size_t bitField = BIT_readBits(&DStream, nbBits);
|
|
725
|
-
|
|
726
|
-
All above operations only read from local register (which size depends on size_t).
|
|
727
|
-
Refueling the register from memory is manually performed by the reload method.
|
|
728
|
-
endSignal = FSE_reloadDStream(&DStream);
|
|
729
|
-
|
|
730
|
-
BIT_reloadDStream() result tells if there is still some more data to read from DStream.
|
|
731
|
-
BIT_DStream_unfinished : there is still some data left into the DStream.
|
|
732
|
-
BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
|
|
733
|
-
BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
|
734
|
-
BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
|
735
|
-
|
|
736
|
-
When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
|
|
737
|
-
to properly detect the exact end of stream.
|
|
738
|
-
After each decoded symbol, check if DStream is fully consumed using this simple test :
|
|
739
|
-
BIT_reloadDStream(&DStream) >= BIT_DStream_completed
|
|
740
|
-
|
|
741
|
-
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
|
|
742
|
-
Checking if DStream has reached its end is performed by :
|
|
743
|
-
BIT_endOfDStream(&DStream);
|
|
744
|
-
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
|
|
745
|
-
FSE_endOfDState(&DState);
|
|
746
|
-
*/
|
|
747
|
-
|
|
748
691
|
|
|
749
692
|
/******************************************
|
|
750
693
|
* FSE unsafe API
|
|
@@ -1335,8 +1278,8 @@ static size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsi
|
|
|
1335
1278
|
else
|
|
1336
1279
|
{
|
|
1337
1280
|
bitCount -= (int)(8 * (iend - 4 - ip));
|
|
1338
|
-
|
|
1339
|
-
|
|
1281
|
+
ip = iend - 4;
|
|
1282
|
+
}
|
|
1340
1283
|
bitStream = MEM_readLE32(ip) >> (bitCount & 31);
|
|
1341
1284
|
}
|
|
1342
1285
|
}
|
|
@@ -2037,7 +1980,7 @@ static size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
|
|
|
2037
1980
|
rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
|
|
2038
1981
|
}
|
|
2039
1982
|
|
|
2040
|
-
|
|
1983
|
+
/* Build rankVal */
|
|
2041
1984
|
{
|
|
2042
1985
|
const U32 minBits = tableLog+1 - maxW;
|
|
2043
1986
|
U32 nextRankVal = 0;
|
|
@@ -2435,6 +2378,8 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
|
|
|
2435
2378
|
#define LITERAL_NOENTROPY 63
|
|
2436
2379
|
#define COMMAND_NOENTROPY 7 /* to remove */
|
|
2437
2380
|
|
|
2381
|
+
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
|
|
2382
|
+
|
|
2438
2383
|
static const size_t ZSTD_blockHeaderSize = 3;
|
|
2439
2384
|
static const size_t ZSTD_frameHeaderSize = 4;
|
|
2440
2385
|
|
|
@@ -2449,7 +2394,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
|
|
|
2449
2394
|
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
|
|
2450
2395
|
|
|
2451
2396
|
/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
|
|
2452
|
-
static void ZSTD_wildcopy(void* dst, const void* src,
|
|
2397
|
+
static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
|
|
2453
2398
|
{
|
|
2454
2399
|
const BYTE* ip = (const BYTE*)src;
|
|
2455
2400
|
BYTE* op = (BYTE*)dst;
|
|
@@ -2509,7 +2454,6 @@ struct ZSTD_DCtx_s
|
|
|
2509
2454
|
blockType_t bType;
|
|
2510
2455
|
U32 phase;
|
|
2511
2456
|
const BYTE* litPtr;
|
|
2512
|
-
size_t litBufSize;
|
|
2513
2457
|
size_t litSize;
|
|
2514
2458
|
BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
|
|
2515
2459
|
}; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
|
|
@@ -2537,7 +2481,9 @@ static size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockPropertie
|
|
|
2537
2481
|
static size_t ZSTD_copyUncompressedBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
|
2538
2482
|
{
|
|
2539
2483
|
if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
|
|
2540
|
-
|
|
2484
|
+
if (srcSize > 0) {
|
|
2485
|
+
memcpy(dst, src, srcSize);
|
|
2486
|
+
}
|
|
2541
2487
|
return srcSize;
|
|
2542
2488
|
}
|
|
2543
2489
|
|
|
@@ -2581,8 +2527,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
|
2581
2527
|
size_t litSize = BLOCKSIZE;
|
|
2582
2528
|
const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
|
|
2583
2529
|
dctx->litPtr = dctx->litBuffer;
|
|
2584
|
-
dctx->litBufSize = BLOCKSIZE;
|
|
2585
2530
|
dctx->litSize = litSize;
|
|
2531
|
+
memset(dctx->litBuffer + dctx->litSize, 0, 8);
|
|
2586
2532
|
return readSize; /* works if it's an error too */
|
|
2587
2533
|
}
|
|
2588
2534
|
case IS_RAW:
|
|
@@ -2590,16 +2536,16 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
|
2590
2536
|
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
|
|
2591
2537
|
if (litSize > srcSize-11) /* risk of reading too far with wildcopy */
|
|
2592
2538
|
{
|
|
2593
|
-
|
|
2594
|
-
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2539
|
+
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
|
|
2540
|
+
if (litSize > srcSize-3) return ERROR(corruption_detected);
|
|
2541
|
+
memcpy(dctx->litBuffer, istart, litSize);
|
|
2542
|
+
dctx->litPtr = dctx->litBuffer;
|
|
2543
|
+
dctx->litSize = litSize;
|
|
2544
|
+
memset(dctx->litBuffer + dctx->litSize, 0, 8);
|
|
2545
|
+
return litSize+3;
|
|
2546
|
+
}
|
|
2547
|
+
/* direct reference into compressed stream */
|
|
2601
2548
|
dctx->litPtr = istart+3;
|
|
2602
|
-
dctx->litBufSize = srcSize-3;
|
|
2603
2549
|
dctx->litSize = litSize;
|
|
2604
2550
|
return litSize+3;
|
|
2605
2551
|
}
|
|
@@ -2607,9 +2553,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
|
2607
2553
|
{
|
|
2608
2554
|
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
|
|
2609
2555
|
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
|
|
2610
|
-
memset(dctx->litBuffer, istart[3], litSize);
|
|
2556
|
+
memset(dctx->litBuffer, istart[3], litSize + 8);
|
|
2611
2557
|
dctx->litPtr = dctx->litBuffer;
|
|
2612
|
-
dctx->litBufSize = BLOCKSIZE;
|
|
2613
2558
|
dctx->litSize = litSize;
|
|
2614
2559
|
return 4;
|
|
2615
2560
|
}
|
|
@@ -2751,11 +2696,11 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
|
2751
2696
|
seqState->prevOffset = seq->offset;
|
|
2752
2697
|
if (litLength == MaxLL)
|
|
2753
2698
|
{
|
|
2754
|
-
U32 add = *dumps
|
|
2699
|
+
const U32 add = dumps<de ? *dumps++ : 0;
|
|
2755
2700
|
if (add < 255) litLength += add;
|
|
2756
|
-
else
|
|
2701
|
+
else if (dumps + 3 <= de)
|
|
2757
2702
|
{
|
|
2758
|
-
litLength =
|
|
2703
|
+
litLength = MEM_readLE24(dumps);
|
|
2759
2704
|
dumps += 3;
|
|
2760
2705
|
}
|
|
2761
2706
|
if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
|
|
@@ -2781,11 +2726,11 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
|
2781
2726
|
matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
|
|
2782
2727
|
if (matchLength == MaxML)
|
|
2783
2728
|
{
|
|
2784
|
-
U32 add = *dumps
|
|
2729
|
+
const U32 add = dumps<de ? *dumps++ : 0;
|
|
2785
2730
|
if (add < 255) matchLength += add;
|
|
2786
|
-
else
|
|
2731
|
+
else if (dumps + 3 <= de)
|
|
2787
2732
|
{
|
|
2788
|
-
matchLength =
|
|
2733
|
+
matchLength = MEM_readLE24(dumps);
|
|
2789
2734
|
dumps += 3;
|
|
2790
2735
|
}
|
|
2791
2736
|
if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
|
|
@@ -2806,7 +2751,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
|
2806
2751
|
BYTE* const base, BYTE* const oend)
|
|
2807
2752
|
{
|
|
2808
2753
|
static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
|
|
2809
|
-
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /*
|
|
2754
|
+
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* subtracted */
|
|
2810
2755
|
const BYTE* const ostart = op;
|
|
2811
2756
|
BYTE* const oLitEnd = op + sequence.litLength;
|
|
2812
2757
|
BYTE* const oMatchEnd = op + sequence.litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */
|
|
@@ -2816,7 +2761,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
|
2816
2761
|
/* checks */
|
|
2817
2762
|
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
|
|
2818
2763
|
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
|
|
2819
|
-
if (litEnd > litLimit
|
|
2764
|
+
if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
|
|
2820
2765
|
|
|
2821
2766
|
/* copy Literals */
|
|
2822
2767
|
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
|
@@ -2850,7 +2795,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
|
2850
2795
|
}
|
|
2851
2796
|
op += 8; match += 8;
|
|
2852
2797
|
|
|
2853
|
-
if (oMatchEnd > oend-
|
|
2798
|
+
if (oMatchEnd > oend-(16-MINMATCH))
|
|
2854
2799
|
{
|
|
2855
2800
|
if (op < oend_8)
|
|
2856
2801
|
{
|
|
@@ -2862,7 +2807,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
|
2862
2807
|
}
|
|
2863
2808
|
else
|
|
2864
2809
|
{
|
|
2865
|
-
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
|
|
2810
|
+
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
|
|
2866
2811
|
}
|
|
2867
2812
|
}
|
|
2868
2813
|
|
|
@@ -2882,7 +2827,6 @@ static size_t ZSTD_decompressSequences(
|
|
|
2882
2827
|
BYTE* const oend = ostart + maxDstSize;
|
|
2883
2828
|
size_t errorCode, dumpsLength;
|
|
2884
2829
|
const BYTE* litPtr = dctx->litPtr;
|
|
2885
|
-
const BYTE* const litMax = litPtr + dctx->litBufSize;
|
|
2886
2830
|
const BYTE* const litEnd = litPtr + dctx->litSize;
|
|
2887
2831
|
int nbSeq;
|
|
2888
2832
|
const BYTE* dumps;
|
|
@@ -2918,7 +2862,7 @@ static size_t ZSTD_decompressSequences(
|
|
|
2918
2862
|
size_t oneSeqSize;
|
|
2919
2863
|
nbSeq--;
|
|
2920
2864
|
ZSTD_decodeSequence(&sequence, &seqState);
|
|
2921
|
-
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr,
|
|
2865
|
+
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
|
|
2922
2866
|
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
|
|
2923
2867
|
op += oneSeqSize;
|
|
2924
2868
|
}
|
|
@@ -2932,8 +2876,10 @@ static size_t ZSTD_decompressSequences(
|
|
|
2932
2876
|
size_t lastLLSize = litEnd - litPtr;
|
|
2933
2877
|
if (litPtr > litEnd) return ERROR(corruption_detected);
|
|
2934
2878
|
if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
|
|
2935
|
-
if (
|
|
2936
|
-
|
|
2879
|
+
if (lastLLSize > 0) {
|
|
2880
|
+
if (op != litPtr) memmove(op, litPtr, lastLLSize);
|
|
2881
|
+
op += lastLLSize;
|
|
2882
|
+
}
|
|
2937
2883
|
}
|
|
2938
2884
|
}
|
|
2939
2885
|
|
|
@@ -3023,6 +2969,61 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
|
|
|
3023
2969
|
return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
|
3024
2970
|
}
|
|
3025
2971
|
|
|
2972
|
+
/* ZSTD_errorFrameSizeInfoLegacy() :
|
|
2973
|
+
assumes `cSize` and `dBound` are _not_ NULL */
|
|
2974
|
+
MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
|
|
2975
|
+
{
|
|
2976
|
+
*cSize = ret;
|
|
2977
|
+
*dBound = ZSTD_CONTENTSIZE_ERROR;
|
|
2978
|
+
}
|
|
2979
|
+
|
|
2980
|
+
void ZSTDv03_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
|
|
2981
|
+
{
|
|
2982
|
+
const BYTE* ip = (const BYTE*)src;
|
|
2983
|
+
size_t remainingSize = srcSize;
|
|
2984
|
+
size_t nbBlocks = 0;
|
|
2985
|
+
U32 magicNumber;
|
|
2986
|
+
blockProperties_t blockProperties;
|
|
2987
|
+
|
|
2988
|
+
/* Frame Header */
|
|
2989
|
+
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) {
|
|
2990
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
|
2991
|
+
return;
|
|
2992
|
+
}
|
|
2993
|
+
magicNumber = MEM_readLE32(src);
|
|
2994
|
+
if (magicNumber != ZSTD_magicNumber) {
|
|
2995
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
|
|
2996
|
+
return;
|
|
2997
|
+
}
|
|
2998
|
+
ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
|
|
2999
|
+
|
|
3000
|
+
/* Loop on each block */
|
|
3001
|
+
while (1)
|
|
3002
|
+
{
|
|
3003
|
+
size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
|
|
3004
|
+
if (ZSTD_isError(cBlockSize)) {
|
|
3005
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
|
|
3006
|
+
return;
|
|
3007
|
+
}
|
|
3008
|
+
|
|
3009
|
+
ip += ZSTD_blockHeaderSize;
|
|
3010
|
+
remainingSize -= ZSTD_blockHeaderSize;
|
|
3011
|
+
if (cBlockSize > remainingSize) {
|
|
3012
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
|
3013
|
+
return;
|
|
3014
|
+
}
|
|
3015
|
+
|
|
3016
|
+
if (cBlockSize == 0) break; /* bt_end */
|
|
3017
|
+
|
|
3018
|
+
ip += cBlockSize;
|
|
3019
|
+
remainingSize -= cBlockSize;
|
|
3020
|
+
nbBlocks++;
|
|
3021
|
+
}
|
|
3022
|
+
|
|
3023
|
+
*cSize = ip - (const BYTE*)src;
|
|
3024
|
+
*dBound = nbBlocks * BLOCKSIZE;
|
|
3025
|
+
}
|
|
3026
|
+
|
|
3026
3027
|
|
|
3027
3028
|
/*******************************
|
|
3028
3029
|
* Streaming Decompression API
|
|
@@ -3128,36 +3129,36 @@ static size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSi
|
|
|
3128
3129
|
|
|
3129
3130
|
unsigned ZSTDv03_isError(size_t code)
|
|
3130
3131
|
{
|
|
3131
|
-
|
|
3132
|
+
return ZSTD_isError(code);
|
|
3132
3133
|
}
|
|
3133
3134
|
|
|
3134
3135
|
size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
|
|
3135
3136
|
const void* src, size_t compressedSize)
|
|
3136
3137
|
{
|
|
3137
|
-
|
|
3138
|
+
return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
|
|
3138
3139
|
}
|
|
3139
3140
|
|
|
3140
3141
|
ZSTDv03_Dctx* ZSTDv03_createDCtx(void)
|
|
3141
3142
|
{
|
|
3142
|
-
|
|
3143
|
+
return (ZSTDv03_Dctx*)ZSTD_createDCtx();
|
|
3143
3144
|
}
|
|
3144
3145
|
|
|
3145
3146
|
size_t ZSTDv03_freeDCtx(ZSTDv03_Dctx* dctx)
|
|
3146
3147
|
{
|
|
3147
|
-
|
|
3148
|
+
return ZSTD_freeDCtx((ZSTD_DCtx*)dctx);
|
|
3148
3149
|
}
|
|
3149
3150
|
|
|
3150
3151
|
size_t ZSTDv03_resetDCtx(ZSTDv03_Dctx* dctx)
|
|
3151
3152
|
{
|
|
3152
|
-
|
|
3153
|
+
return ZSTD_resetDCtx((ZSTD_DCtx*)dctx);
|
|
3153
3154
|
}
|
|
3154
3155
|
|
|
3155
3156
|
size_t ZSTDv03_nextSrcSizeToDecompress(ZSTDv03_Dctx* dctx)
|
|
3156
3157
|
{
|
|
3157
|
-
|
|
3158
|
+
return ZSTD_nextSrcSizeToDecompress((ZSTD_DCtx*)dctx);
|
|
3158
3159
|
}
|
|
3159
3160
|
|
|
3160
3161
|
size_t ZSTDv03_decompressContinue(ZSTDv03_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
|
3161
3162
|
{
|
|
3162
|
-
|
|
3163
|
+
return ZSTD_decompressContinue((ZSTD_DCtx*)dctx, dst, maxDstSize, src, srcSize);
|
|
3163
3164
|
}
|