extzstd 0.1 → 0.3.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/HISTORY.ja.md +39 -0
- data/README.md +38 -56
- data/contrib/zstd/CHANGELOG +613 -0
- data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
- data/contrib/zstd/CONTRIBUTING.md +406 -0
- data/contrib/zstd/COPYING +339 -0
- data/contrib/zstd/Makefile +420 -0
- data/contrib/zstd/README.md +179 -41
- data/contrib/zstd/TESTING.md +44 -0
- data/contrib/zstd/appveyor.yml +292 -0
- data/contrib/zstd/lib/BUCK +234 -0
- data/contrib/zstd/lib/Makefile +451 -0
- data/contrib/zstd/lib/README.md +207 -0
- data/contrib/zstd/{common → lib/common}/bitstream.h +187 -138
- data/contrib/zstd/lib/common/compiler.h +288 -0
- data/contrib/zstd/lib/common/cpu.h +213 -0
- data/contrib/zstd/lib/common/debug.c +24 -0
- data/contrib/zstd/lib/common/debug.h +107 -0
- data/contrib/zstd/lib/common/entropy_common.c +362 -0
- data/contrib/zstd/{common → lib/common}/error_private.c +25 -12
- data/contrib/zstd/{common → lib/common}/error_private.h +14 -10
- data/contrib/zstd/{common → lib/common}/fse.h +173 -92
- data/contrib/zstd/{common → lib/common}/fse_decompress.c +149 -85
- data/contrib/zstd/lib/common/huf.h +361 -0
- data/contrib/zstd/{common → lib/common}/mem.h +115 -59
- data/contrib/zstd/lib/common/pool.c +350 -0
- data/contrib/zstd/lib/common/pool.h +84 -0
- data/contrib/zstd/lib/common/threading.c +122 -0
- data/contrib/zstd/lib/common/threading.h +155 -0
- data/contrib/zstd/{common → lib/common}/xxhash.c +55 -96
- data/contrib/zstd/{common → lib/common}/xxhash.h +23 -47
- data/contrib/zstd/lib/common/zstd_common.c +83 -0
- data/contrib/zstd/lib/common/zstd_deps.h +111 -0
- data/contrib/zstd/lib/common/zstd_errors.h +95 -0
- data/contrib/zstd/lib/common/zstd_internal.h +478 -0
- data/contrib/zstd/{compress → lib/compress}/fse_compress.c +214 -319
- data/contrib/zstd/lib/compress/hist.c +181 -0
- data/contrib/zstd/lib/compress/hist.h +75 -0
- data/contrib/zstd/lib/compress/huf_compress.c +913 -0
- data/contrib/zstd/lib/compress/zstd_compress.c +5208 -0
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +1203 -0
- data/contrib/zstd/lib/compress/zstd_compress_literals.c +158 -0
- data/contrib/zstd/lib/compress/zstd_compress_literals.h +29 -0
- data/contrib/zstd/lib/compress/zstd_compress_sequences.c +433 -0
- data/contrib/zstd/lib/compress/zstd_compress_sequences.h +54 -0
- data/contrib/zstd/lib/compress/zstd_compress_superblock.c +849 -0
- data/contrib/zstd/lib/compress/zstd_compress_superblock.h +32 -0
- data/contrib/zstd/lib/compress/zstd_cwksp.h +561 -0
- data/contrib/zstd/lib/compress/zstd_double_fast.c +521 -0
- data/contrib/zstd/lib/compress/zstd_double_fast.h +38 -0
- data/contrib/zstd/lib/compress/zstd_fast.c +496 -0
- data/contrib/zstd/lib/compress/zstd_fast.h +37 -0
- data/contrib/zstd/lib/compress/zstd_lazy.c +1412 -0
- data/contrib/zstd/lib/compress/zstd_lazy.h +87 -0
- data/contrib/zstd/lib/compress/zstd_ldm.c +660 -0
- data/contrib/zstd/lib/compress/zstd_ldm.h +116 -0
- data/contrib/zstd/lib/compress/zstd_opt.c +1345 -0
- data/contrib/zstd/lib/compress/zstd_opt.h +56 -0
- data/contrib/zstd/lib/compress/zstdmt_compress.c +1811 -0
- data/contrib/zstd/lib/compress/zstdmt_compress.h +110 -0
- data/contrib/zstd/lib/decompress/huf_decompress.c +1350 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.c +244 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
- data/contrib/zstd/lib/decompress/zstd_decompress.c +1930 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1540 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.h +62 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +190 -0
- data/contrib/zstd/{common → lib/deprecated}/zbuff.h +68 -45
- data/contrib/zstd/lib/deprecated/zbuff_common.c +26 -0
- data/contrib/zstd/lib/deprecated/zbuff_compress.c +147 -0
- data/contrib/zstd/lib/deprecated/zbuff_decompress.c +75 -0
- data/contrib/zstd/lib/dictBuilder/cover.c +1245 -0
- data/contrib/zstd/lib/dictBuilder/cover.h +157 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.c +3 -3
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.h +0 -0
- data/contrib/zstd/lib/dictBuilder/fastcover.c +758 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/zdict.c +318 -194
- data/contrib/zstd/lib/dictBuilder/zdict.h +305 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_legacy.h +171 -15
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.c +191 -124
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.h +19 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.c +125 -125
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.h +19 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.c +125 -124
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.h +20 -6
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.c +151 -299
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.h +19 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.c +237 -243
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.h +19 -6
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.c +130 -143
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.h +18 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.c +158 -157
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.h +19 -5
- data/contrib/zstd/lib/libzstd.pc.in +15 -0
- data/contrib/zstd/lib/zstd.h +2391 -0
- data/ext/depend +2 -0
- data/ext/extconf.rb +15 -6
- data/ext/extzstd.c +76 -145
- data/ext/extzstd.h +80 -31
- data/ext/extzstd_stream.c +417 -142
- data/ext/libzstd_conf.h +8 -0
- data/ext/zstd_common.c +10 -7
- data/ext/zstd_compress.c +14 -5
- data/ext/zstd_decompress.c +5 -4
- data/ext/zstd_dictbuilder.c +9 -4
- data/ext/zstd_dictbuilder_fastcover.c +3 -0
- data/ext/zstd_legacy_v01.c +3 -1
- data/ext/zstd_legacy_v02.c +3 -1
- data/ext/zstd_legacy_v03.c +3 -1
- data/ext/zstd_legacy_v04.c +3 -1
- data/ext/zstd_legacy_v05.c +3 -1
- data/ext/zstd_legacy_v06.c +3 -1
- data/ext/zstd_legacy_v07.c +3 -1
- data/gemstub.rb +10 -24
- data/lib/extzstd.rb +64 -179
- data/lib/extzstd/version.rb +6 -1
- data/test/test_basic.rb +9 -6
- metadata +113 -57
- data/HISTORY.ja +0 -5
- data/contrib/zstd/common/entropy_common.c +0 -225
- data/contrib/zstd/common/huf.h +0 -228
- data/contrib/zstd/common/zstd_common.c +0 -83
- data/contrib/zstd/common/zstd_errors.h +0 -60
- data/contrib/zstd/common/zstd_internal.h +0 -267
- data/contrib/zstd/compress/huf_compress.c +0 -533
- data/contrib/zstd/compress/zbuff_compress.c +0 -319
- data/contrib/zstd/compress/zstd_compress.c +0 -3264
- data/contrib/zstd/compress/zstd_opt.h +0 -900
- data/contrib/zstd/decompress/huf_decompress.c +0 -883
- data/contrib/zstd/decompress/zbuff_decompress.c +0 -252
- data/contrib/zstd/decompress/zstd_decompress.c +0 -1842
- data/contrib/zstd/dictBuilder/zdict.h +0 -111
- data/contrib/zstd/zstd.h +0 -640
@@ -1,10 +1,11 @@
|
|
1
|
-
|
2
|
-
* Copyright (c) 2016-
|
1
|
+
/*
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
|
-
* This source code is licensed under the BSD-style license found in the
|
6
|
-
* LICENSE file in the root directory of this source tree
|
7
|
-
*
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
8
9
|
*/
|
9
10
|
|
10
11
|
#ifndef ZSTD_V01_H_28739879432
|
@@ -34,6 +35,19 @@ ZSTDv01_decompress() : decompress ZSTD frames compliant with v0.1.x format
|
|
34
35
|
size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize,
|
35
36
|
const void* src, size_t compressedSize);
|
36
37
|
|
38
|
+
/**
|
39
|
+
ZSTDv01_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.1.x format
|
40
|
+
srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
41
|
+
cSize (output parameter) : the number of bytes that would be read to decompress this frame
|
42
|
+
or an error code if it fails (which can be tested using ZSTDv01_isError())
|
43
|
+
dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
|
44
|
+
or ZSTD_CONTENTSIZE_ERROR if an error occurs
|
45
|
+
|
46
|
+
note : assumes `cSize` and `dBound` are _not_ NULL.
|
47
|
+
*/
|
48
|
+
void ZSTDv01_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
|
49
|
+
size_t* cSize, unsigned long long* dBound);
|
50
|
+
|
37
51
|
/**
|
38
52
|
ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error
|
39
53
|
*/
|
@@ -1,16 +1,17 @@
|
|
1
|
-
|
2
|
-
* Copyright (c) 2016-
|
1
|
+
/*
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
|
-
* This source code is licensed under the BSD-style license found in the
|
6
|
-
* LICENSE file in the root directory of this source tree
|
7
|
-
*
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
8
9
|
*/
|
9
10
|
|
10
11
|
|
11
12
|
#include <stddef.h> /* size_t, ptrdiff_t */
|
12
13
|
#include "zstd_v02.h"
|
13
|
-
#include "error_private.h"
|
14
|
+
#include "../common/error_private.h"
|
14
15
|
|
15
16
|
|
16
17
|
/******************************************
|
@@ -88,7 +89,11 @@ extern "C" {
|
|
88
89
|
* Basic Types
|
89
90
|
*****************************************************************/
|
90
91
|
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
91
|
-
#
|
92
|
+
# if defined(_AIX)
|
93
|
+
# include <inttypes.h>
|
94
|
+
# else
|
95
|
+
# include <stdint.h> /* intptr_t */
|
96
|
+
# endif
|
92
97
|
typedef uint8_t BYTE;
|
93
98
|
typedef uint16_t U16;
|
94
99
|
typedef int16_t S16;
|
@@ -188,7 +193,7 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
|
|
188
193
|
memcpy(memPtr, &value, sizeof(value));
|
189
194
|
}
|
190
195
|
|
191
|
-
#endif
|
196
|
+
#endif /* MEM_FORCE_MEMORY_ACCESS */
|
192
197
|
|
193
198
|
|
194
199
|
MEM_STATIC U16 MEM_readLE16(const void* memPtr)
|
@@ -216,6 +221,11 @@ MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
|
|
216
221
|
}
|
217
222
|
}
|
218
223
|
|
224
|
+
MEM_STATIC U32 MEM_readLE24(const void* memPtr)
|
225
|
+
{
|
226
|
+
return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
|
227
|
+
}
|
228
|
+
|
219
229
|
MEM_STATIC U32 MEM_readLE32(const void* memPtr)
|
220
230
|
{
|
221
231
|
if (MEM_isLittleEndian())
|
@@ -329,18 +339,6 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
|
|
329
339
|
MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
|
330
340
|
|
331
341
|
|
332
|
-
/*
|
333
|
-
* Start by invoking BIT_initDStream().
|
334
|
-
* A chunk of the bitStream is then stored into a local register.
|
335
|
-
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
336
|
-
* You can then retrieve bitFields stored into the local register, **in reverse order**.
|
337
|
-
* Local register is manually filled from memory by the BIT_reloadDStream() method.
|
338
|
-
* A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BIT_DStream_unfinished.
|
339
|
-
* Otherwise, it can be less than that, so proceed accordingly.
|
340
|
-
* Checking if DStream has reached its end can be performed with BIT_endOfDStream()
|
341
|
-
*/
|
342
|
-
|
343
|
-
|
344
342
|
/******************************************
|
345
343
|
* unsafe API
|
346
344
|
******************************************/
|
@@ -352,14 +350,14 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
|
|
352
350
|
/****************************************************************
|
353
351
|
* Helper functions
|
354
352
|
****************************************************************/
|
355
|
-
MEM_STATIC unsigned BIT_highbit32 (
|
353
|
+
MEM_STATIC unsigned BIT_highbit32 (U32 val)
|
356
354
|
{
|
357
355
|
# if defined(_MSC_VER) /* Visual */
|
358
356
|
unsigned long r=0;
|
359
357
|
_BitScanReverse ( &r, val );
|
360
358
|
return (unsigned) r;
|
361
359
|
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
|
362
|
-
return
|
360
|
+
return __builtin_clz (val) ^ 31;
|
363
361
|
# else /* Software version */
|
364
362
|
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
365
363
|
U32 v = val;
|
@@ -410,11 +408,17 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
|
410
408
|
switch(srcSize)
|
411
409
|
{
|
412
410
|
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
|
411
|
+
/* fallthrough */
|
413
412
|
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
|
413
|
+
/* fallthrough */
|
414
414
|
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
|
415
|
+
/* fallthrough */
|
415
416
|
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
|
417
|
+
/* fallthrough */
|
416
418
|
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
|
419
|
+
/* fallthrough */
|
417
420
|
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
|
421
|
+
/* fallthrough */
|
418
422
|
default:;
|
419
423
|
}
|
420
424
|
contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
|
@@ -426,13 +430,6 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
|
426
430
|
return srcSize;
|
427
431
|
}
|
428
432
|
|
429
|
-
/*!BIT_lookBits
|
430
|
-
* Provides next n bits from local register
|
431
|
-
* local register is not modified (bits are still present for next read/look)
|
432
|
-
* On 32-bits, maxNbBits==25
|
433
|
-
* On 64-bits, maxNbBits==57
|
434
|
-
* @return : value extracted
|
435
|
-
*/
|
436
433
|
MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits)
|
437
434
|
{
|
438
435
|
const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
@@ -452,11 +449,6 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
|
|
452
449
|
bitD->bitsConsumed += nbBits;
|
453
450
|
}
|
454
451
|
|
455
|
-
/*!BIT_readBits
|
456
|
-
* Read next n bits from local register.
|
457
|
-
* pay attention to not read more than nbBits contained into local register.
|
458
|
-
* @return : extracted value.
|
459
|
-
*/
|
460
452
|
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
|
461
453
|
{
|
462
454
|
size_t value = BIT_lookBits(bitD, nbBits);
|
@@ -475,8 +467,8 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
|
475
467
|
|
476
468
|
MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
|
477
469
|
{
|
478
|
-
|
479
|
-
|
470
|
+
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
|
471
|
+
return BIT_DStream_overflow;
|
480
472
|
|
481
473
|
if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer))
|
482
474
|
{
|
@@ -694,55 +686,6 @@ static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bi
|
|
694
686
|
|
695
687
|
static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
|
696
688
|
|
697
|
-
/*
|
698
|
-
Let's now decompose FSE_decompress_usingDTable() into its unitary components.
|
699
|
-
You will decode FSE-encoded symbols from the bitStream,
|
700
|
-
and also any other bitFields you put in, **in reverse order**.
|
701
|
-
|
702
|
-
You will need a few variables to track your bitStream. They are :
|
703
|
-
|
704
|
-
BIT_DStream_t DStream; // Stream context
|
705
|
-
FSE_DState_t DState; // State context. Multiple ones are possible
|
706
|
-
FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
|
707
|
-
|
708
|
-
The first thing to do is to init the bitStream.
|
709
|
-
errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
|
710
|
-
|
711
|
-
You should then retrieve your initial state(s)
|
712
|
-
(in reverse flushing order if you have several ones) :
|
713
|
-
errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
|
714
|
-
|
715
|
-
You can then decode your data, symbol after symbol.
|
716
|
-
For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
|
717
|
-
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
|
718
|
-
unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
|
719
|
-
|
720
|
-
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
|
721
|
-
Note : maximum allowed nbBits is 25, for 32-bits compatibility
|
722
|
-
size_t bitField = BIT_readBits(&DStream, nbBits);
|
723
|
-
|
724
|
-
All above operations only read from local register (which size depends on size_t).
|
725
|
-
Refueling the register from memory is manually performed by the reload method.
|
726
|
-
endSignal = FSE_reloadDStream(&DStream);
|
727
|
-
|
728
|
-
BIT_reloadDStream() result tells if there is still some more data to read from DStream.
|
729
|
-
BIT_DStream_unfinished : there is still some data left into the DStream.
|
730
|
-
BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
|
731
|
-
BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
732
|
-
BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
733
|
-
|
734
|
-
When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
|
735
|
-
to properly detect the exact end of stream.
|
736
|
-
After each decoded symbol, check if DStream is fully consumed using this simple test :
|
737
|
-
BIT_reloadDStream(&DStream) >= BIT_DStream_completed
|
738
|
-
|
739
|
-
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
|
740
|
-
Checking if DStream has reached its end is performed by :
|
741
|
-
BIT_endOfDStream(&DStream);
|
742
|
-
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
|
743
|
-
FSE_endOfDState(&DState);
|
744
|
-
*/
|
745
|
-
|
746
689
|
|
747
690
|
/******************************************
|
748
691
|
* FSE unsafe API
|
@@ -1334,8 +1277,8 @@ static size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsi
|
|
1334
1277
|
else
|
1335
1278
|
{
|
1336
1279
|
bitCount -= (int)(8 * (iend - 4 - ip));
|
1337
|
-
|
1338
|
-
|
1280
|
+
ip = iend - 4;
|
1281
|
+
}
|
1339
1282
|
bitStream = MEM_readLE32(ip) >> (bitCount & 31);
|
1340
1283
|
}
|
1341
1284
|
}
|
@@ -2040,7 +1983,7 @@ static size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
|
|
2040
1983
|
rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
|
2041
1984
|
}
|
2042
1985
|
|
2043
|
-
|
1986
|
+
/* Build rankVal */
|
2044
1987
|
{
|
2045
1988
|
const U32 minBits = tableLog+1 - maxW;
|
2046
1989
|
U32 nextRankVal = 0;
|
@@ -2374,7 +2317,7 @@ static size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
|
|
2374
2317
|
rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
|
2375
2318
|
}
|
2376
2319
|
|
2377
|
-
|
2320
|
+
/* Build rankVal */
|
2378
2321
|
{
|
2379
2322
|
const U32 minBits = tableLog+1 - maxW;
|
2380
2323
|
U32 nextRankVal = 0;
|
@@ -2794,6 +2737,8 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
|
|
2794
2737
|
#define LITERAL_NOENTROPY 63
|
2795
2738
|
#define COMMAND_NOENTROPY 7 /* to remove */
|
2796
2739
|
|
2740
|
+
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
|
2741
|
+
|
2797
2742
|
static const size_t ZSTD_blockHeaderSize = 3;
|
2798
2743
|
static const size_t ZSTD_frameHeaderSize = 4;
|
2799
2744
|
|
@@ -2808,7 +2753,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
|
|
2808
2753
|
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
|
2809
2754
|
|
2810
2755
|
/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
|
2811
|
-
static void ZSTD_wildcopy(void* dst, const void* src,
|
2756
|
+
static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
|
2812
2757
|
{
|
2813
2758
|
const BYTE* ip = (const BYTE*)src;
|
2814
2759
|
BYTE* op = (BYTE*)dst;
|
@@ -2868,7 +2813,6 @@ struct ZSTD_DCtx_s
|
|
2868
2813
|
blockType_t bType;
|
2869
2814
|
U32 phase;
|
2870
2815
|
const BYTE* litPtr;
|
2871
|
-
size_t litBufSize;
|
2872
2816
|
size_t litSize;
|
2873
2817
|
BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
|
2874
2818
|
}; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
|
@@ -2896,7 +2840,9 @@ static size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockPropertie
|
|
2896
2840
|
static size_t ZSTD_copyUncompressedBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
2897
2841
|
{
|
2898
2842
|
if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
|
2899
|
-
|
2843
|
+
if (srcSize > 0) {
|
2844
|
+
memcpy(dst, src, srcSize);
|
2845
|
+
}
|
2900
2846
|
return srcSize;
|
2901
2847
|
}
|
2902
2848
|
|
@@ -2940,8 +2886,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
2940
2886
|
size_t litSize = BLOCKSIZE;
|
2941
2887
|
const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
|
2942
2888
|
dctx->litPtr = dctx->litBuffer;
|
2943
|
-
dctx->litBufSize = BLOCKSIZE;
|
2944
2889
|
dctx->litSize = litSize;
|
2890
|
+
memset(dctx->litBuffer + dctx->litSize, 0, 8);
|
2945
2891
|
return readSize; /* works if it's an error too */
|
2946
2892
|
}
|
2947
2893
|
case IS_RAW:
|
@@ -2949,16 +2895,16 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
2949
2895
|
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
|
2950
2896
|
if (litSize > srcSize-11) /* risk of reading too far with wildcopy */
|
2951
2897
|
{
|
2952
|
-
|
2953
|
-
|
2954
|
-
|
2955
|
-
|
2956
|
-
|
2957
|
-
|
2958
|
-
|
2959
|
-
|
2898
|
+
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
|
2899
|
+
if (litSize > srcSize-3) return ERROR(corruption_detected);
|
2900
|
+
memcpy(dctx->litBuffer, istart, litSize);
|
2901
|
+
dctx->litPtr = dctx->litBuffer;
|
2902
|
+
dctx->litSize = litSize;
|
2903
|
+
memset(dctx->litBuffer + dctx->litSize, 0, 8);
|
2904
|
+
return litSize+3;
|
2905
|
+
}
|
2906
|
+
/* direct reference into compressed stream */
|
2960
2907
|
dctx->litPtr = istart+3;
|
2961
|
-
dctx->litBufSize = srcSize-3;
|
2962
2908
|
dctx->litSize = litSize;
|
2963
2909
|
return litSize+3;
|
2964
2910
|
}
|
@@ -2966,9 +2912,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
2966
2912
|
{
|
2967
2913
|
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
|
2968
2914
|
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
|
2969
|
-
memset(dctx->litBuffer, istart[3], litSize);
|
2915
|
+
memset(dctx->litBuffer, istart[3], litSize + 8);
|
2970
2916
|
dctx->litPtr = dctx->litBuffer;
|
2971
|
-
dctx->litBufSize = BLOCKSIZE;
|
2972
2917
|
dctx->litSize = litSize;
|
2973
2918
|
return 4;
|
2974
2919
|
}
|
@@ -3110,11 +3055,11 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
3110
3055
|
seqState->prevOffset = seq->offset;
|
3111
3056
|
if (litLength == MaxLL)
|
3112
3057
|
{
|
3113
|
-
U32 add = *dumps
|
3058
|
+
const U32 add = dumps<de ? *dumps++ : 0;
|
3114
3059
|
if (add < 255) litLength += add;
|
3115
|
-
else
|
3060
|
+
else if (dumps + 3 <= de)
|
3116
3061
|
{
|
3117
|
-
litLength =
|
3062
|
+
litLength = MEM_readLE24(dumps);
|
3118
3063
|
dumps += 3;
|
3119
3064
|
}
|
3120
3065
|
if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
|
@@ -3140,11 +3085,11 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
3140
3085
|
matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
|
3141
3086
|
if (matchLength == MaxML)
|
3142
3087
|
{
|
3143
|
-
U32 add = *dumps
|
3088
|
+
const U32 add = dumps<de ? *dumps++ : 0;
|
3144
3089
|
if (add < 255) matchLength += add;
|
3145
|
-
else
|
3090
|
+
else if (dumps + 3 <= de)
|
3146
3091
|
{
|
3147
|
-
matchLength =
|
3092
|
+
matchLength = MEM_readLE24(dumps);
|
3148
3093
|
dumps += 3;
|
3149
3094
|
}
|
3150
3095
|
if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
|
@@ -3165,7 +3110,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
3165
3110
|
BYTE* const base, BYTE* const oend)
|
3166
3111
|
{
|
3167
3112
|
static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
|
3168
|
-
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /*
|
3113
|
+
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* subtracted */
|
3169
3114
|
const BYTE* const ostart = op;
|
3170
3115
|
BYTE* const oLitEnd = op + sequence.litLength;
|
3171
3116
|
BYTE* const oMatchEnd = op + sequence.litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */
|
@@ -3175,7 +3120,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
3175
3120
|
/* checks */
|
3176
3121
|
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
|
3177
3122
|
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
|
3178
|
-
if (litEnd > litLimit
|
3123
|
+
if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
|
3179
3124
|
|
3180
3125
|
/* copy Literals */
|
3181
3126
|
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
@@ -3209,7 +3154,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
3209
3154
|
}
|
3210
3155
|
op += 8; match += 8;
|
3211
3156
|
|
3212
|
-
if (oMatchEnd > oend-
|
3157
|
+
if (oMatchEnd > oend-(16-MINMATCH))
|
3213
3158
|
{
|
3214
3159
|
if (op < oend_8)
|
3215
3160
|
{
|
@@ -3221,7 +3166,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
3221
3166
|
}
|
3222
3167
|
else
|
3223
3168
|
{
|
3224
|
-
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
|
3169
|
+
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
|
3225
3170
|
}
|
3226
3171
|
}
|
3227
3172
|
|
@@ -3241,7 +3186,6 @@ static size_t ZSTD_decompressSequences(
|
|
3241
3186
|
BYTE* const oend = ostart + maxDstSize;
|
3242
3187
|
size_t errorCode, dumpsLength;
|
3243
3188
|
const BYTE* litPtr = dctx->litPtr;
|
3244
|
-
const BYTE* const litMax = litPtr + dctx->litBufSize;
|
3245
3189
|
const BYTE* const litEnd = litPtr + dctx->litSize;
|
3246
3190
|
int nbSeq;
|
3247
3191
|
const BYTE* dumps;
|
@@ -3277,7 +3221,7 @@ static size_t ZSTD_decompressSequences(
|
|
3277
3221
|
size_t oneSeqSize;
|
3278
3222
|
nbSeq--;
|
3279
3223
|
ZSTD_decodeSequence(&sequence, &seqState);
|
3280
|
-
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr,
|
3224
|
+
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
|
3281
3225
|
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
|
3282
3226
|
op += oneSeqSize;
|
3283
3227
|
}
|
@@ -3291,8 +3235,10 @@ static size_t ZSTD_decompressSequences(
|
|
3291
3235
|
size_t lastLLSize = litEnd - litPtr;
|
3292
3236
|
if (litPtr > litEnd) return ERROR(corruption_detected);
|
3293
3237
|
if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
|
3294
|
-
if (
|
3295
|
-
|
3238
|
+
if (lastLLSize > 0) {
|
3239
|
+
if (op != litPtr) memmove(op, litPtr, lastLLSize);
|
3240
|
+
op += lastLLSize;
|
3241
|
+
}
|
3296
3242
|
}
|
3297
3243
|
}
|
3298
3244
|
|
@@ -3382,6 +3328,60 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
|
|
3382
3328
|
return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
3383
3329
|
}
|
3384
3330
|
|
3331
|
+
/* ZSTD_errorFrameSizeInfoLegacy() :
|
3332
|
+
assumes `cSize` and `dBound` are _not_ NULL */
|
3333
|
+
static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
|
3334
|
+
{
|
3335
|
+
*cSize = ret;
|
3336
|
+
*dBound = ZSTD_CONTENTSIZE_ERROR;
|
3337
|
+
}
|
3338
|
+
|
3339
|
+
void ZSTDv02_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
|
3340
|
+
{
|
3341
|
+
const BYTE* ip = (const BYTE*)src;
|
3342
|
+
size_t remainingSize = srcSize;
|
3343
|
+
size_t nbBlocks = 0;
|
3344
|
+
U32 magicNumber;
|
3345
|
+
blockProperties_t blockProperties;
|
3346
|
+
|
3347
|
+
/* Frame Header */
|
3348
|
+
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) {
|
3349
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
3350
|
+
return;
|
3351
|
+
}
|
3352
|
+
magicNumber = MEM_readLE32(src);
|
3353
|
+
if (magicNumber != ZSTD_magicNumber) {
|
3354
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
|
3355
|
+
return;
|
3356
|
+
}
|
3357
|
+
ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
|
3358
|
+
|
3359
|
+
/* Loop on each block */
|
3360
|
+
while (1)
|
3361
|
+
{
|
3362
|
+
size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
|
3363
|
+
if (ZSTD_isError(cBlockSize)) {
|
3364
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
|
3365
|
+
return;
|
3366
|
+
}
|
3367
|
+
|
3368
|
+
ip += ZSTD_blockHeaderSize;
|
3369
|
+
remainingSize -= ZSTD_blockHeaderSize;
|
3370
|
+
if (cBlockSize > remainingSize) {
|
3371
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
3372
|
+
return;
|
3373
|
+
}
|
3374
|
+
|
3375
|
+
if (cBlockSize == 0) break; /* bt_end */
|
3376
|
+
|
3377
|
+
ip += cBlockSize;
|
3378
|
+
remainingSize -= cBlockSize;
|
3379
|
+
nbBlocks++;
|
3380
|
+
}
|
3381
|
+
|
3382
|
+
*cSize = ip - (const BYTE*)src;
|
3383
|
+
*dBound = nbBlocks * BLOCKSIZE;
|
3384
|
+
}
|
3385
3385
|
|
3386
3386
|
/*******************************
|
3387
3387
|
* Streaming Decompression API
|
@@ -3487,36 +3487,36 @@ static size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSi
|
|
3487
3487
|
|
3488
3488
|
unsigned ZSTDv02_isError(size_t code)
|
3489
3489
|
{
|
3490
|
-
|
3490
|
+
return ZSTD_isError(code);
|
3491
3491
|
}
|
3492
3492
|
|
3493
3493
|
size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
|
3494
3494
|
const void* src, size_t compressedSize)
|
3495
3495
|
{
|
3496
|
-
|
3496
|
+
return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
|
3497
3497
|
}
|
3498
3498
|
|
3499
3499
|
ZSTDv02_Dctx* ZSTDv02_createDCtx(void)
|
3500
3500
|
{
|
3501
|
-
|
3501
|
+
return (ZSTDv02_Dctx*)ZSTD_createDCtx();
|
3502
3502
|
}
|
3503
3503
|
|
3504
3504
|
size_t ZSTDv02_freeDCtx(ZSTDv02_Dctx* dctx)
|
3505
3505
|
{
|
3506
|
-
|
3506
|
+
return ZSTD_freeDCtx((ZSTD_DCtx*)dctx);
|
3507
3507
|
}
|
3508
3508
|
|
3509
3509
|
size_t ZSTDv02_resetDCtx(ZSTDv02_Dctx* dctx)
|
3510
3510
|
{
|
3511
|
-
|
3511
|
+
return ZSTD_resetDCtx((ZSTD_DCtx*)dctx);
|
3512
3512
|
}
|
3513
3513
|
|
3514
3514
|
size_t ZSTDv02_nextSrcSizeToDecompress(ZSTDv02_Dctx* dctx)
|
3515
3515
|
{
|
3516
|
-
|
3516
|
+
return ZSTD_nextSrcSizeToDecompress((ZSTD_DCtx*)dctx);
|
3517
3517
|
}
|
3518
3518
|
|
3519
3519
|
size_t ZSTDv02_decompressContinue(ZSTDv02_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
3520
3520
|
{
|
3521
|
-
|
3521
|
+
return ZSTD_decompressContinue((ZSTD_DCtx*)dctx, dst, maxDstSize, src, srcSize);
|
3522
3522
|
}
|