extzstd 0.1 → 0.3.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/HISTORY.ja.md +39 -0
- data/README.md +38 -56
- data/contrib/zstd/CHANGELOG +613 -0
- data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
- data/contrib/zstd/CONTRIBUTING.md +406 -0
- data/contrib/zstd/COPYING +339 -0
- data/contrib/zstd/Makefile +420 -0
- data/contrib/zstd/README.md +179 -41
- data/contrib/zstd/TESTING.md +44 -0
- data/contrib/zstd/appveyor.yml +292 -0
- data/contrib/zstd/lib/BUCK +234 -0
- data/contrib/zstd/lib/Makefile +451 -0
- data/contrib/zstd/lib/README.md +207 -0
- data/contrib/zstd/{common → lib/common}/bitstream.h +187 -138
- data/contrib/zstd/lib/common/compiler.h +288 -0
- data/contrib/zstd/lib/common/cpu.h +213 -0
- data/contrib/zstd/lib/common/debug.c +24 -0
- data/contrib/zstd/lib/common/debug.h +107 -0
- data/contrib/zstd/lib/common/entropy_common.c +362 -0
- data/contrib/zstd/{common → lib/common}/error_private.c +25 -12
- data/contrib/zstd/{common → lib/common}/error_private.h +14 -10
- data/contrib/zstd/{common → lib/common}/fse.h +173 -92
- data/contrib/zstd/{common → lib/common}/fse_decompress.c +149 -85
- data/contrib/zstd/lib/common/huf.h +361 -0
- data/contrib/zstd/{common → lib/common}/mem.h +115 -59
- data/contrib/zstd/lib/common/pool.c +350 -0
- data/contrib/zstd/lib/common/pool.h +84 -0
- data/contrib/zstd/lib/common/threading.c +122 -0
- data/contrib/zstd/lib/common/threading.h +155 -0
- data/contrib/zstd/{common → lib/common}/xxhash.c +55 -96
- data/contrib/zstd/{common → lib/common}/xxhash.h +23 -47
- data/contrib/zstd/lib/common/zstd_common.c +83 -0
- data/contrib/zstd/lib/common/zstd_deps.h +111 -0
- data/contrib/zstd/lib/common/zstd_errors.h +95 -0
- data/contrib/zstd/lib/common/zstd_internal.h +478 -0
- data/contrib/zstd/{compress → lib/compress}/fse_compress.c +214 -319
- data/contrib/zstd/lib/compress/hist.c +181 -0
- data/contrib/zstd/lib/compress/hist.h +75 -0
- data/contrib/zstd/lib/compress/huf_compress.c +913 -0
- data/contrib/zstd/lib/compress/zstd_compress.c +5208 -0
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +1203 -0
- data/contrib/zstd/lib/compress/zstd_compress_literals.c +158 -0
- data/contrib/zstd/lib/compress/zstd_compress_literals.h +29 -0
- data/contrib/zstd/lib/compress/zstd_compress_sequences.c +433 -0
- data/contrib/zstd/lib/compress/zstd_compress_sequences.h +54 -0
- data/contrib/zstd/lib/compress/zstd_compress_superblock.c +849 -0
- data/contrib/zstd/lib/compress/zstd_compress_superblock.h +32 -0
- data/contrib/zstd/lib/compress/zstd_cwksp.h +561 -0
- data/contrib/zstd/lib/compress/zstd_double_fast.c +521 -0
- data/contrib/zstd/lib/compress/zstd_double_fast.h +38 -0
- data/contrib/zstd/lib/compress/zstd_fast.c +496 -0
- data/contrib/zstd/lib/compress/zstd_fast.h +37 -0
- data/contrib/zstd/lib/compress/zstd_lazy.c +1412 -0
- data/contrib/zstd/lib/compress/zstd_lazy.h +87 -0
- data/contrib/zstd/lib/compress/zstd_ldm.c +660 -0
- data/contrib/zstd/lib/compress/zstd_ldm.h +116 -0
- data/contrib/zstd/lib/compress/zstd_opt.c +1345 -0
- data/contrib/zstd/lib/compress/zstd_opt.h +56 -0
- data/contrib/zstd/lib/compress/zstdmt_compress.c +1811 -0
- data/contrib/zstd/lib/compress/zstdmt_compress.h +110 -0
- data/contrib/zstd/lib/decompress/huf_decompress.c +1350 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.c +244 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
- data/contrib/zstd/lib/decompress/zstd_decompress.c +1930 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1540 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.h +62 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +190 -0
- data/contrib/zstd/{common → lib/deprecated}/zbuff.h +68 -45
- data/contrib/zstd/lib/deprecated/zbuff_common.c +26 -0
- data/contrib/zstd/lib/deprecated/zbuff_compress.c +147 -0
- data/contrib/zstd/lib/deprecated/zbuff_decompress.c +75 -0
- data/contrib/zstd/lib/dictBuilder/cover.c +1245 -0
- data/contrib/zstd/lib/dictBuilder/cover.h +157 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.c +3 -3
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.h +0 -0
- data/contrib/zstd/lib/dictBuilder/fastcover.c +758 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/zdict.c +318 -194
- data/contrib/zstd/lib/dictBuilder/zdict.h +305 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_legacy.h +171 -15
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.c +191 -124
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.h +19 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.c +125 -125
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.h +19 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.c +125 -124
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.h +20 -6
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.c +151 -299
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.h +19 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.c +237 -243
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.h +19 -6
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.c +130 -143
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.h +18 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.c +158 -157
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.h +19 -5
- data/contrib/zstd/lib/libzstd.pc.in +15 -0
- data/contrib/zstd/lib/zstd.h +2391 -0
- data/ext/depend +2 -0
- data/ext/extconf.rb +15 -6
- data/ext/extzstd.c +76 -145
- data/ext/extzstd.h +80 -31
- data/ext/extzstd_stream.c +417 -142
- data/ext/libzstd_conf.h +8 -0
- data/ext/zstd_common.c +10 -7
- data/ext/zstd_compress.c +14 -5
- data/ext/zstd_decompress.c +5 -4
- data/ext/zstd_dictbuilder.c +9 -4
- data/ext/zstd_dictbuilder_fastcover.c +3 -0
- data/ext/zstd_legacy_v01.c +3 -1
- data/ext/zstd_legacy_v02.c +3 -1
- data/ext/zstd_legacy_v03.c +3 -1
- data/ext/zstd_legacy_v04.c +3 -1
- data/ext/zstd_legacy_v05.c +3 -1
- data/ext/zstd_legacy_v06.c +3 -1
- data/ext/zstd_legacy_v07.c +3 -1
- data/gemstub.rb +10 -24
- data/lib/extzstd.rb +64 -179
- data/lib/extzstd/version.rb +6 -1
- data/test/test_basic.rb +9 -6
- metadata +113 -57
- data/HISTORY.ja +0 -5
- data/contrib/zstd/common/entropy_common.c +0 -225
- data/contrib/zstd/common/huf.h +0 -228
- data/contrib/zstd/common/zstd_common.c +0 -83
- data/contrib/zstd/common/zstd_errors.h +0 -60
- data/contrib/zstd/common/zstd_internal.h +0 -267
- data/contrib/zstd/compress/huf_compress.c +0 -533
- data/contrib/zstd/compress/zbuff_compress.c +0 -319
- data/contrib/zstd/compress/zstd_compress.c +0 -3264
- data/contrib/zstd/compress/zstd_opt.h +0 -900
- data/contrib/zstd/decompress/huf_decompress.c +0 -883
- data/contrib/zstd/decompress/zbuff_decompress.c +0 -252
- data/contrib/zstd/decompress/zstd_decompress.c +0 -1842
- data/contrib/zstd/dictBuilder/zdict.h +0 -111
- data/contrib/zstd/zstd.h +0 -640
@@ -1,10 +1,11 @@
|
|
1
|
-
|
2
|
-
* Copyright (c) 2016-
|
1
|
+
/*
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
|
-
* This source code is licensed under the BSD-style license found in the
|
6
|
-
* LICENSE file in the root directory of this source tree
|
7
|
-
*
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
8
9
|
*/
|
9
10
|
|
10
11
|
#ifndef ZSTD_V02_H_4174539423
|
@@ -34,6 +35,19 @@ ZSTDv02_decompress() : decompress ZSTD frames compliant with v0.2.x format
|
|
34
35
|
size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
|
35
36
|
const void* src, size_t compressedSize);
|
36
37
|
|
38
|
+
/**
|
39
|
+
ZSTDv02_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.2.x format
|
40
|
+
srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
41
|
+
cSize (output parameter) : the number of bytes that would be read to decompress this frame
|
42
|
+
or an error code if it fails (which can be tested using ZSTDv01_isError())
|
43
|
+
dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
|
44
|
+
or ZSTD_CONTENTSIZE_ERROR if an error occurs
|
45
|
+
|
46
|
+
note : assumes `cSize` and `dBound` are _not_ NULL.
|
47
|
+
*/
|
48
|
+
void ZSTDv02_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
|
49
|
+
size_t* cSize, unsigned long long* dBound);
|
50
|
+
|
37
51
|
/**
|
38
52
|
ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error
|
39
53
|
*/
|
@@ -1,16 +1,17 @@
|
|
1
|
-
|
2
|
-
* Copyright (c) 2016-
|
1
|
+
/*
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
|
-
* This source code is licensed under the BSD-style license found in the
|
6
|
-
* LICENSE file in the root directory of this source tree
|
7
|
-
*
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
8
9
|
*/
|
9
10
|
|
10
11
|
|
11
12
|
#include <stddef.h> /* size_t, ptrdiff_t */
|
12
13
|
#include "zstd_v03.h"
|
13
|
-
#include "error_private.h"
|
14
|
+
#include "../common/error_private.h"
|
14
15
|
|
15
16
|
|
16
17
|
/******************************************
|
@@ -89,7 +90,11 @@ extern "C" {
|
|
89
90
|
* Basic Types
|
90
91
|
*****************************************************************/
|
91
92
|
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
92
|
-
#
|
93
|
+
# if defined(_AIX)
|
94
|
+
# include <inttypes.h>
|
95
|
+
# else
|
96
|
+
# include <stdint.h> /* intptr_t */
|
97
|
+
# endif
|
93
98
|
typedef uint8_t BYTE;
|
94
99
|
typedef uint16_t U16;
|
95
100
|
typedef int16_t S16;
|
@@ -190,7 +195,7 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
|
|
190
195
|
}
|
191
196
|
|
192
197
|
|
193
|
-
#endif
|
198
|
+
#endif /* MEM_FORCE_MEMORY_ACCESS */
|
194
199
|
|
195
200
|
|
196
201
|
MEM_STATIC U16 MEM_readLE16(const void* memPtr)
|
@@ -218,6 +223,11 @@ MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
|
|
218
223
|
}
|
219
224
|
}
|
220
225
|
|
226
|
+
MEM_STATIC U32 MEM_readLE24(const void* memPtr)
|
227
|
+
{
|
228
|
+
return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
|
229
|
+
}
|
230
|
+
|
221
231
|
MEM_STATIC U32 MEM_readLE32(const void* memPtr)
|
222
232
|
{
|
223
233
|
if (MEM_isLittleEndian())
|
@@ -331,17 +341,6 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
|
|
331
341
|
MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
|
332
342
|
|
333
343
|
|
334
|
-
/*
|
335
|
-
* Start by invoking BIT_initDStream().
|
336
|
-
* A chunk of the bitStream is then stored into a local register.
|
337
|
-
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
338
|
-
* You can then retrieve bitFields stored into the local register, **in reverse order**.
|
339
|
-
* Local register is manually filled from memory by the BIT_reloadDStream() method.
|
340
|
-
* A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BIT_DStream_unfinished.
|
341
|
-
* Otherwise, it can be less than that, so proceed accordingly.
|
342
|
-
* Checking if DStream has reached its end can be performed with BIT_endOfDStream()
|
343
|
-
*/
|
344
|
-
|
345
344
|
|
346
345
|
/******************************************
|
347
346
|
* unsafe API
|
@@ -354,14 +353,14 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
|
|
354
353
|
/****************************************************************
|
355
354
|
* Helper functions
|
356
355
|
****************************************************************/
|
357
|
-
MEM_STATIC unsigned BIT_highbit32 (
|
356
|
+
MEM_STATIC unsigned BIT_highbit32 (U32 val)
|
358
357
|
{
|
359
358
|
# if defined(_MSC_VER) /* Visual */
|
360
359
|
unsigned long r=0;
|
361
360
|
_BitScanReverse ( &r, val );
|
362
361
|
return (unsigned) r;
|
363
362
|
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
|
364
|
-
return
|
363
|
+
return __builtin_clz (val) ^ 31;
|
365
364
|
# else /* Software version */
|
366
365
|
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
367
366
|
U32 v = val;
|
@@ -412,11 +411,17 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
|
412
411
|
switch(srcSize)
|
413
412
|
{
|
414
413
|
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
|
414
|
+
/* fallthrough */
|
415
415
|
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
|
416
|
+
/* fallthrough */
|
416
417
|
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
|
418
|
+
/* fallthrough */
|
417
419
|
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
|
420
|
+
/* fallthrough */
|
418
421
|
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
|
422
|
+
/* fallthrough */
|
419
423
|
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
|
424
|
+
/* fallthrough */
|
420
425
|
default:;
|
421
426
|
}
|
422
427
|
contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
|
@@ -427,14 +432,6 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
|
427
432
|
|
428
433
|
return srcSize;
|
429
434
|
}
|
430
|
-
|
431
|
-
/*!BIT_lookBits
|
432
|
-
* Provides next n bits from local register
|
433
|
-
* local register is not modified (bits are still present for next read/look)
|
434
|
-
* On 32-bits, maxNbBits==25
|
435
|
-
* On 64-bits, maxNbBits==57
|
436
|
-
* @return : value extracted
|
437
|
-
*/
|
438
435
|
MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits)
|
439
436
|
{
|
440
437
|
const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
@@ -454,11 +451,6 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
|
|
454
451
|
bitD->bitsConsumed += nbBits;
|
455
452
|
}
|
456
453
|
|
457
|
-
/*!BIT_readBits
|
458
|
-
* Read next n bits from local register.
|
459
|
-
* pay attention to not read more than nbBits contained into local register.
|
460
|
-
* @return : extracted value.
|
461
|
-
*/
|
462
454
|
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
|
463
455
|
{
|
464
456
|
size_t value = BIT_lookBits(bitD, nbBits);
|
@@ -477,8 +469,8 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
|
477
469
|
|
478
470
|
MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
|
479
471
|
{
|
480
|
-
|
481
|
-
|
472
|
+
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
|
473
|
+
return BIT_DStream_overflow;
|
482
474
|
|
483
475
|
if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer))
|
484
476
|
{
|
@@ -696,55 +688,6 @@ static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bi
|
|
696
688
|
|
697
689
|
static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
|
698
690
|
|
699
|
-
/*
|
700
|
-
Let's now decompose FSE_decompress_usingDTable() into its unitary components.
|
701
|
-
You will decode FSE-encoded symbols from the bitStream,
|
702
|
-
and also any other bitFields you put in, **in reverse order**.
|
703
|
-
|
704
|
-
You will need a few variables to track your bitStream. They are :
|
705
|
-
|
706
|
-
BIT_DStream_t DStream; // Stream context
|
707
|
-
FSE_DState_t DState; // State context. Multiple ones are possible
|
708
|
-
FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
|
709
|
-
|
710
|
-
The first thing to do is to init the bitStream.
|
711
|
-
errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
|
712
|
-
|
713
|
-
You should then retrieve your initial state(s)
|
714
|
-
(in reverse flushing order if you have several ones) :
|
715
|
-
errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
|
716
|
-
|
717
|
-
You can then decode your data, symbol after symbol.
|
718
|
-
For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
|
719
|
-
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
|
720
|
-
unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
|
721
|
-
|
722
|
-
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
|
723
|
-
Note : maximum allowed nbBits is 25, for 32-bits compatibility
|
724
|
-
size_t bitField = BIT_readBits(&DStream, nbBits);
|
725
|
-
|
726
|
-
All above operations only read from local register (which size depends on size_t).
|
727
|
-
Refueling the register from memory is manually performed by the reload method.
|
728
|
-
endSignal = FSE_reloadDStream(&DStream);
|
729
|
-
|
730
|
-
BIT_reloadDStream() result tells if there is still some more data to read from DStream.
|
731
|
-
BIT_DStream_unfinished : there is still some data left into the DStream.
|
732
|
-
BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
|
733
|
-
BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
734
|
-
BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
735
|
-
|
736
|
-
When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
|
737
|
-
to properly detect the exact end of stream.
|
738
|
-
After each decoded symbol, check if DStream is fully consumed using this simple test :
|
739
|
-
BIT_reloadDStream(&DStream) >= BIT_DStream_completed
|
740
|
-
|
741
|
-
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
|
742
|
-
Checking if DStream has reached its end is performed by :
|
743
|
-
BIT_endOfDStream(&DStream);
|
744
|
-
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
|
745
|
-
FSE_endOfDState(&DState);
|
746
|
-
*/
|
747
|
-
|
748
691
|
|
749
692
|
/******************************************
|
750
693
|
* FSE unsafe API
|
@@ -1335,8 +1278,8 @@ static size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsi
|
|
1335
1278
|
else
|
1336
1279
|
{
|
1337
1280
|
bitCount -= (int)(8 * (iend - 4 - ip));
|
1338
|
-
|
1339
|
-
|
1281
|
+
ip = iend - 4;
|
1282
|
+
}
|
1340
1283
|
bitStream = MEM_readLE32(ip) >> (bitCount & 31);
|
1341
1284
|
}
|
1342
1285
|
}
|
@@ -2037,7 +1980,7 @@ static size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
|
|
2037
1980
|
rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
|
2038
1981
|
}
|
2039
1982
|
|
2040
|
-
|
1983
|
+
/* Build rankVal */
|
2041
1984
|
{
|
2042
1985
|
const U32 minBits = tableLog+1 - maxW;
|
2043
1986
|
U32 nextRankVal = 0;
|
@@ -2435,6 +2378,8 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
|
|
2435
2378
|
#define LITERAL_NOENTROPY 63
|
2436
2379
|
#define COMMAND_NOENTROPY 7 /* to remove */
|
2437
2380
|
|
2381
|
+
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
|
2382
|
+
|
2438
2383
|
static const size_t ZSTD_blockHeaderSize = 3;
|
2439
2384
|
static const size_t ZSTD_frameHeaderSize = 4;
|
2440
2385
|
|
@@ -2449,7 +2394,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
|
|
2449
2394
|
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
|
2450
2395
|
|
2451
2396
|
/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
|
2452
|
-
static void ZSTD_wildcopy(void* dst, const void* src,
|
2397
|
+
static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
|
2453
2398
|
{
|
2454
2399
|
const BYTE* ip = (const BYTE*)src;
|
2455
2400
|
BYTE* op = (BYTE*)dst;
|
@@ -2509,7 +2454,6 @@ struct ZSTD_DCtx_s
|
|
2509
2454
|
blockType_t bType;
|
2510
2455
|
U32 phase;
|
2511
2456
|
const BYTE* litPtr;
|
2512
|
-
size_t litBufSize;
|
2513
2457
|
size_t litSize;
|
2514
2458
|
BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
|
2515
2459
|
}; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
|
@@ -2537,7 +2481,9 @@ static size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockPropertie
|
|
2537
2481
|
static size_t ZSTD_copyUncompressedBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
2538
2482
|
{
|
2539
2483
|
if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
|
2540
|
-
|
2484
|
+
if (srcSize > 0) {
|
2485
|
+
memcpy(dst, src, srcSize);
|
2486
|
+
}
|
2541
2487
|
return srcSize;
|
2542
2488
|
}
|
2543
2489
|
|
@@ -2581,8 +2527,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
2581
2527
|
size_t litSize = BLOCKSIZE;
|
2582
2528
|
const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
|
2583
2529
|
dctx->litPtr = dctx->litBuffer;
|
2584
|
-
dctx->litBufSize = BLOCKSIZE;
|
2585
2530
|
dctx->litSize = litSize;
|
2531
|
+
memset(dctx->litBuffer + dctx->litSize, 0, 8);
|
2586
2532
|
return readSize; /* works if it's an error too */
|
2587
2533
|
}
|
2588
2534
|
case IS_RAW:
|
@@ -2590,16 +2536,16 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
2590
2536
|
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
|
2591
2537
|
if (litSize > srcSize-11) /* risk of reading too far with wildcopy */
|
2592
2538
|
{
|
2593
|
-
|
2594
|
-
|
2595
|
-
|
2596
|
-
|
2597
|
-
|
2598
|
-
|
2599
|
-
|
2600
|
-
|
2539
|
+
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
|
2540
|
+
if (litSize > srcSize-3) return ERROR(corruption_detected);
|
2541
|
+
memcpy(dctx->litBuffer, istart, litSize);
|
2542
|
+
dctx->litPtr = dctx->litBuffer;
|
2543
|
+
dctx->litSize = litSize;
|
2544
|
+
memset(dctx->litBuffer + dctx->litSize, 0, 8);
|
2545
|
+
return litSize+3;
|
2546
|
+
}
|
2547
|
+
/* direct reference into compressed stream */
|
2601
2548
|
dctx->litPtr = istart+3;
|
2602
|
-
dctx->litBufSize = srcSize-3;
|
2603
2549
|
dctx->litSize = litSize;
|
2604
2550
|
return litSize+3;
|
2605
2551
|
}
|
@@ -2607,9 +2553,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
2607
2553
|
{
|
2608
2554
|
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
|
2609
2555
|
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
|
2610
|
-
memset(dctx->litBuffer, istart[3], litSize);
|
2556
|
+
memset(dctx->litBuffer, istart[3], litSize + 8);
|
2611
2557
|
dctx->litPtr = dctx->litBuffer;
|
2612
|
-
dctx->litBufSize = BLOCKSIZE;
|
2613
2558
|
dctx->litSize = litSize;
|
2614
2559
|
return 4;
|
2615
2560
|
}
|
@@ -2751,11 +2696,11 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
2751
2696
|
seqState->prevOffset = seq->offset;
|
2752
2697
|
if (litLength == MaxLL)
|
2753
2698
|
{
|
2754
|
-
U32 add = *dumps
|
2699
|
+
const U32 add = dumps<de ? *dumps++ : 0;
|
2755
2700
|
if (add < 255) litLength += add;
|
2756
|
-
else
|
2701
|
+
else if (dumps + 3 <= de)
|
2757
2702
|
{
|
2758
|
-
litLength =
|
2703
|
+
litLength = MEM_readLE24(dumps);
|
2759
2704
|
dumps += 3;
|
2760
2705
|
}
|
2761
2706
|
if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
|
@@ -2781,11 +2726,11 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
2781
2726
|
matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
|
2782
2727
|
if (matchLength == MaxML)
|
2783
2728
|
{
|
2784
|
-
U32 add = *dumps
|
2729
|
+
const U32 add = dumps<de ? *dumps++ : 0;
|
2785
2730
|
if (add < 255) matchLength += add;
|
2786
|
-
else
|
2731
|
+
else if (dumps + 3 <= de)
|
2787
2732
|
{
|
2788
|
-
matchLength =
|
2733
|
+
matchLength = MEM_readLE24(dumps);
|
2789
2734
|
dumps += 3;
|
2790
2735
|
}
|
2791
2736
|
if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
|
@@ -2806,7 +2751,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
2806
2751
|
BYTE* const base, BYTE* const oend)
|
2807
2752
|
{
|
2808
2753
|
static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
|
2809
|
-
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /*
|
2754
|
+
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* subtracted */
|
2810
2755
|
const BYTE* const ostart = op;
|
2811
2756
|
BYTE* const oLitEnd = op + sequence.litLength;
|
2812
2757
|
BYTE* const oMatchEnd = op + sequence.litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */
|
@@ -2816,7 +2761,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
2816
2761
|
/* checks */
|
2817
2762
|
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
|
2818
2763
|
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
|
2819
|
-
if (litEnd > litLimit
|
2764
|
+
if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
|
2820
2765
|
|
2821
2766
|
/* copy Literals */
|
2822
2767
|
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
@@ -2850,7 +2795,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
2850
2795
|
}
|
2851
2796
|
op += 8; match += 8;
|
2852
2797
|
|
2853
|
-
if (oMatchEnd > oend-
|
2798
|
+
if (oMatchEnd > oend-(16-MINMATCH))
|
2854
2799
|
{
|
2855
2800
|
if (op < oend_8)
|
2856
2801
|
{
|
@@ -2862,7 +2807,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
2862
2807
|
}
|
2863
2808
|
else
|
2864
2809
|
{
|
2865
|
-
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
|
2810
|
+
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
|
2866
2811
|
}
|
2867
2812
|
}
|
2868
2813
|
|
@@ -2882,7 +2827,6 @@ static size_t ZSTD_decompressSequences(
|
|
2882
2827
|
BYTE* const oend = ostart + maxDstSize;
|
2883
2828
|
size_t errorCode, dumpsLength;
|
2884
2829
|
const BYTE* litPtr = dctx->litPtr;
|
2885
|
-
const BYTE* const litMax = litPtr + dctx->litBufSize;
|
2886
2830
|
const BYTE* const litEnd = litPtr + dctx->litSize;
|
2887
2831
|
int nbSeq;
|
2888
2832
|
const BYTE* dumps;
|
@@ -2918,7 +2862,7 @@ static size_t ZSTD_decompressSequences(
|
|
2918
2862
|
size_t oneSeqSize;
|
2919
2863
|
nbSeq--;
|
2920
2864
|
ZSTD_decodeSequence(&sequence, &seqState);
|
2921
|
-
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr,
|
2865
|
+
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
|
2922
2866
|
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
|
2923
2867
|
op += oneSeqSize;
|
2924
2868
|
}
|
@@ -2932,8 +2876,10 @@ static size_t ZSTD_decompressSequences(
|
|
2932
2876
|
size_t lastLLSize = litEnd - litPtr;
|
2933
2877
|
if (litPtr > litEnd) return ERROR(corruption_detected);
|
2934
2878
|
if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
|
2935
|
-
if (
|
2936
|
-
|
2879
|
+
if (lastLLSize > 0) {
|
2880
|
+
if (op != litPtr) memmove(op, litPtr, lastLLSize);
|
2881
|
+
op += lastLLSize;
|
2882
|
+
}
|
2937
2883
|
}
|
2938
2884
|
}
|
2939
2885
|
|
@@ -3023,6 +2969,61 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
|
|
3023
2969
|
return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
3024
2970
|
}
|
3025
2971
|
|
2972
|
+
/* ZSTD_errorFrameSizeInfoLegacy() :
|
2973
|
+
assumes `cSize` and `dBound` are _not_ NULL */
|
2974
|
+
MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
|
2975
|
+
{
|
2976
|
+
*cSize = ret;
|
2977
|
+
*dBound = ZSTD_CONTENTSIZE_ERROR;
|
2978
|
+
}
|
2979
|
+
|
2980
|
+
void ZSTDv03_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
|
2981
|
+
{
|
2982
|
+
const BYTE* ip = (const BYTE*)src;
|
2983
|
+
size_t remainingSize = srcSize;
|
2984
|
+
size_t nbBlocks = 0;
|
2985
|
+
U32 magicNumber;
|
2986
|
+
blockProperties_t blockProperties;
|
2987
|
+
|
2988
|
+
/* Frame Header */
|
2989
|
+
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) {
|
2990
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
2991
|
+
return;
|
2992
|
+
}
|
2993
|
+
magicNumber = MEM_readLE32(src);
|
2994
|
+
if (magicNumber != ZSTD_magicNumber) {
|
2995
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
|
2996
|
+
return;
|
2997
|
+
}
|
2998
|
+
ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
|
2999
|
+
|
3000
|
+
/* Loop on each block */
|
3001
|
+
while (1)
|
3002
|
+
{
|
3003
|
+
size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
|
3004
|
+
if (ZSTD_isError(cBlockSize)) {
|
3005
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
|
3006
|
+
return;
|
3007
|
+
}
|
3008
|
+
|
3009
|
+
ip += ZSTD_blockHeaderSize;
|
3010
|
+
remainingSize -= ZSTD_blockHeaderSize;
|
3011
|
+
if (cBlockSize > remainingSize) {
|
3012
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
3013
|
+
return;
|
3014
|
+
}
|
3015
|
+
|
3016
|
+
if (cBlockSize == 0) break; /* bt_end */
|
3017
|
+
|
3018
|
+
ip += cBlockSize;
|
3019
|
+
remainingSize -= cBlockSize;
|
3020
|
+
nbBlocks++;
|
3021
|
+
}
|
3022
|
+
|
3023
|
+
*cSize = ip - (const BYTE*)src;
|
3024
|
+
*dBound = nbBlocks * BLOCKSIZE;
|
3025
|
+
}
|
3026
|
+
|
3026
3027
|
|
3027
3028
|
/*******************************
|
3028
3029
|
* Streaming Decompression API
|
@@ -3128,36 +3129,36 @@ static size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSi
|
|
3128
3129
|
|
3129
3130
|
unsigned ZSTDv03_isError(size_t code)
|
3130
3131
|
{
|
3131
|
-
|
3132
|
+
return ZSTD_isError(code);
|
3132
3133
|
}
|
3133
3134
|
|
3134
3135
|
size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
|
3135
3136
|
const void* src, size_t compressedSize)
|
3136
3137
|
{
|
3137
|
-
|
3138
|
+
return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
|
3138
3139
|
}
|
3139
3140
|
|
3140
3141
|
ZSTDv03_Dctx* ZSTDv03_createDCtx(void)
|
3141
3142
|
{
|
3142
|
-
|
3143
|
+
return (ZSTDv03_Dctx*)ZSTD_createDCtx();
|
3143
3144
|
}
|
3144
3145
|
|
3145
3146
|
size_t ZSTDv03_freeDCtx(ZSTDv03_Dctx* dctx)
|
3146
3147
|
{
|
3147
|
-
|
3148
|
+
return ZSTD_freeDCtx((ZSTD_DCtx*)dctx);
|
3148
3149
|
}
|
3149
3150
|
|
3150
3151
|
size_t ZSTDv03_resetDCtx(ZSTDv03_Dctx* dctx)
|
3151
3152
|
{
|
3152
|
-
|
3153
|
+
return ZSTD_resetDCtx((ZSTD_DCtx*)dctx);
|
3153
3154
|
}
|
3154
3155
|
|
3155
3156
|
size_t ZSTDv03_nextSrcSizeToDecompress(ZSTDv03_Dctx* dctx)
|
3156
3157
|
{
|
3157
|
-
|
3158
|
+
return ZSTD_nextSrcSizeToDecompress((ZSTD_DCtx*)dctx);
|
3158
3159
|
}
|
3159
3160
|
|
3160
3161
|
size_t ZSTDv03_decompressContinue(ZSTDv03_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
3161
3162
|
{
|
3162
|
-
|
3163
|
+
return ZSTD_decompressContinue((ZSTD_DCtx*)dctx, dst, maxDstSize, src, srcSize);
|
3163
3164
|
}
|