zstd-ruby 1.3.8.0 → 1.4.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +6 -5
- data/README.md +1 -1
- data/ext/zstdruby/libzstd/Makefile +133 -61
- data/ext/zstdruby/libzstd/README.md +51 -18
- data/ext/zstdruby/libzstd/common/bitstream.h +38 -39
- data/ext/zstdruby/libzstd/common/compiler.h +41 -6
- data/ext/zstdruby/libzstd/common/cpu.h +1 -1
- data/ext/zstdruby/libzstd/common/debug.c +11 -31
- data/ext/zstdruby/libzstd/common/debug.h +11 -31
- data/ext/zstdruby/libzstd/common/entropy_common.c +13 -33
- data/ext/zstdruby/libzstd/common/error_private.c +2 -1
- data/ext/zstdruby/libzstd/common/error_private.h +6 -2
- data/ext/zstdruby/libzstd/common/fse.h +13 -33
- data/ext/zstdruby/libzstd/common/fse_decompress.c +12 -35
- data/ext/zstdruby/libzstd/common/huf.h +15 -33
- data/ext/zstdruby/libzstd/common/mem.h +75 -2
- data/ext/zstdruby/libzstd/common/pool.c +8 -4
- data/ext/zstdruby/libzstd/common/pool.h +2 -2
- data/ext/zstdruby/libzstd/common/threading.c +52 -6
- data/ext/zstdruby/libzstd/common/threading.h +36 -4
- data/ext/zstdruby/libzstd/common/xxhash.c +25 -37
- data/ext/zstdruby/libzstd/common/xxhash.h +11 -31
- data/ext/zstdruby/libzstd/common/zstd_common.c +1 -1
- data/ext/zstdruby/libzstd/common/zstd_errors.h +2 -1
- data/ext/zstdruby/libzstd/common/zstd_internal.h +203 -22
- data/ext/zstdruby/libzstd/compress/fse_compress.c +19 -42
- data/ext/zstdruby/libzstd/compress/hist.c +15 -35
- data/ext/zstdruby/libzstd/compress/hist.h +12 -32
- data/ext/zstdruby/libzstd/compress/huf_compress.c +92 -92
- data/ext/zstdruby/libzstd/compress/zstd_compress.c +1460 -1472
- data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +330 -65
- data/ext/zstdruby/libzstd/compress/zstd_compress_literals.c +158 -0
- data/ext/zstdruby/libzstd/compress/zstd_compress_literals.h +29 -0
- data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.c +419 -0
- data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.h +54 -0
- data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.c +845 -0
- data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.h +32 -0
- data/ext/zstdruby/libzstd/compress/zstd_cwksp.h +525 -0
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +65 -43
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +2 -2
- data/ext/zstdruby/libzstd/compress/zstd_fast.c +264 -159
- data/ext/zstdruby/libzstd/compress/zstd_fast.h +2 -2
- data/ext/zstdruby/libzstd/compress/zstd_lazy.c +74 -42
- data/ext/zstdruby/libzstd/compress/zstd_lazy.h +2 -2
- data/ext/zstdruby/libzstd/compress/zstd_ldm.c +33 -11
- data/ext/zstdruby/libzstd/compress/zstd_ldm.h +7 -2
- data/ext/zstdruby/libzstd/compress/zstd_opt.c +108 -125
- data/ext/zstdruby/libzstd/compress/zstd_opt.h +1 -1
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +129 -93
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +46 -28
- data/ext/zstdruby/libzstd/decompress/huf_decompress.c +76 -60
- data/ext/zstdruby/libzstd/decompress/zstd_ddict.c +14 -10
- data/ext/zstdruby/libzstd/decompress/zstd_ddict.h +2 -2
- data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +471 -258
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.c +471 -346
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.h +3 -3
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_internal.h +25 -4
- data/ext/zstdruby/libzstd/deprecated/zbuff.h +9 -8
- data/ext/zstdruby/libzstd/deprecated/zbuff_common.c +2 -2
- data/ext/zstdruby/libzstd/deprecated/zbuff_compress.c +1 -1
- data/ext/zstdruby/libzstd/deprecated/zbuff_decompress.c +1 -1
- data/ext/zstdruby/libzstd/dictBuilder/cover.c +220 -65
- data/ext/zstdruby/libzstd/dictBuilder/cover.h +81 -7
- data/ext/zstdruby/libzstd/dictBuilder/fastcover.c +85 -56
- data/ext/zstdruby/libzstd/dictBuilder/zdict.c +43 -19
- data/ext/zstdruby/libzstd/dictBuilder/zdict.h +73 -35
- data/ext/zstdruby/libzstd/dll/example/Makefile +2 -1
- data/ext/zstdruby/libzstd/dll/example/build_package.bat +3 -2
- data/ext/zstdruby/libzstd/legacy/zstd_legacy.h +49 -15
- data/ext/zstdruby/libzstd/legacy/zstd_v01.c +142 -117
- data/ext/zstdruby/libzstd/legacy/zstd_v01.h +13 -8
- data/ext/zstdruby/libzstd/legacy/zstd_v02.c +54 -25
- data/ext/zstdruby/libzstd/legacy/zstd_v02.h +13 -8
- data/ext/zstdruby/libzstd/legacy/zstd_v03.c +55 -25
- data/ext/zstdruby/libzstd/legacy/zstd_v03.h +13 -8
- data/ext/zstdruby/libzstd/legacy/zstd_v04.c +62 -29
- data/ext/zstdruby/libzstd/legacy/zstd_v04.h +13 -8
- data/ext/zstdruby/libzstd/legacy/zstd_v05.c +145 -109
- data/ext/zstdruby/libzstd/legacy/zstd_v05.h +14 -9
- data/ext/zstdruby/libzstd/legacy/zstd_v06.c +56 -26
- data/ext/zstdruby/libzstd/legacy/zstd_v06.h +11 -6
- data/ext/zstdruby/libzstd/legacy/zstd_v07.c +65 -28
- data/ext/zstdruby/libzstd/legacy/zstd_v07.h +11 -6
- data/ext/zstdruby/libzstd/libzstd.pc.in +3 -2
- data/ext/zstdruby/libzstd/zstd.h +921 -597
- data/lib/zstd-ruby/version.rb +1 -1
- data/zstd-ruby.gemspec +2 -2
- metadata +19 -14
- data/ext/zstdruby/libzstd/dll/libzstd.def +0 -87
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c) 2016-
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -19,7 +19,7 @@ extern "C" {
|
|
19
19
|
* Dependencies
|
20
20
|
***************************************/
|
21
21
|
#include <stddef.h> /* size_t */
|
22
|
-
#include "mem.h" /* U64, U32 */
|
22
|
+
#include "../common/mem.h" /* U64, U32 */
|
23
23
|
|
24
24
|
|
25
25
|
/* *************************************
|
@@ -33,13 +33,18 @@ extern "C" {
|
|
33
33
|
size_t ZSTDv05_decompress( void* dst, size_t dstCapacity,
|
34
34
|
const void* src, size_t compressedSize);
|
35
35
|
|
36
|
-
/**
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
36
|
+
/**
|
37
|
+
ZSTDv05_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.5.x format
|
38
|
+
srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
39
|
+
cSize (output parameter) : the number of bytes that would be read to decompress this frame
|
40
|
+
or an error code if it fails (which can be tested using ZSTDv01_isError())
|
41
|
+
dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
|
42
|
+
or ZSTD_CONTENTSIZE_ERROR if an error occurs
|
43
|
+
|
44
|
+
note : assumes `cSize` and `dBound` are _not_ NULL.
|
45
|
+
*/
|
46
|
+
void ZSTDv05_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
|
47
|
+
size_t* cSize, unsigned long long* dBound);
|
43
48
|
|
44
49
|
/* *************************************
|
45
50
|
* Helper functions
|
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c) 2016-
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -14,7 +14,7 @@
|
|
14
14
|
#include <stddef.h> /* size_t, ptrdiff_t */
|
15
15
|
#include <string.h> /* memcpy */
|
16
16
|
#include <stdlib.h> /* malloc, free, qsort */
|
17
|
-
#include "error_private.h"
|
17
|
+
#include "../common/error_private.h"
|
18
18
|
|
19
19
|
|
20
20
|
|
@@ -506,6 +506,8 @@ typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
|
|
506
506
|
#define FSEv06_ENCODING_STATIC 2
|
507
507
|
#define FSEv06_ENCODING_DYNAMIC 3
|
508
508
|
|
509
|
+
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
|
510
|
+
|
509
511
|
static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
510
512
|
1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12,
|
511
513
|
13,14,15,16 };
|
@@ -858,7 +860,7 @@ MEM_STATIC unsigned BITv06_highbit32 ( U32 val)
|
|
858
860
|
_BitScanReverse ( &r, val );
|
859
861
|
return (unsigned) r;
|
860
862
|
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
|
861
|
-
return
|
863
|
+
return __builtin_clz (val) ^ 31;
|
862
864
|
# else /* Software version */
|
863
865
|
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
864
866
|
U32 v = val;
|
@@ -1860,7 +1862,7 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta
|
|
1860
1862
|
|
1861
1863
|
if (!srcSize) return ERROR(srcSize_wrong);
|
1862
1864
|
iSize = ip[0];
|
1863
|
-
|
1865
|
+
/* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */
|
1864
1866
|
|
1865
1867
|
if (iSize >= 128) { /* special header */
|
1866
1868
|
if (iSize >= (242)) { /* RLE */
|
@@ -2012,7 +2014,7 @@ size_t HUFv06_readDTableX2 (U16* DTable, const void* src, size_t srcSize)
|
|
2012
2014
|
HUFv06_DEltX2* const dt = (HUFv06_DEltX2*)dtPtr;
|
2013
2015
|
|
2014
2016
|
HUFv06_STATIC_ASSERT(sizeof(HUFv06_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */
|
2015
|
-
|
2017
|
+
/* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
|
2016
2018
|
|
2017
2019
|
iSize = HUFv06_readStats(huffWeight, HUFv06_MAX_SYMBOL_VALUE + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
|
2018
2020
|
if (HUFv06_isError(iSize)) return iSize;
|
@@ -2338,7 +2340,7 @@ size_t HUFv06_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
|
|
2338
2340
|
|
2339
2341
|
HUFv06_STATIC_ASSERT(sizeof(HUFv06_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */
|
2340
2342
|
if (memLog > HUFv06_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
2341
|
-
|
2343
|
+
/* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
|
2342
2344
|
|
2343
2345
|
iSize = HUFv06_readStats(weightList, HUFv06_MAX_SYMBOL_VALUE + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
|
2344
2346
|
if (HUFv06_isError(iSize)) return iSize;
|
@@ -2662,13 +2664,13 @@ size_t HUFv06_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
|
|
2662
2664
|
|
2663
2665
|
{ U32 algoNb = 0;
|
2664
2666
|
if (Dtime[1] < Dtime[0]) algoNb = 1;
|
2665
|
-
|
2667
|
+
/* if (Dtime[2] < Dtime[algoNb]) algoNb = 2; */ /* current speed of HUFv06_decompress4X6 is not good */
|
2666
2668
|
return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
|
2667
2669
|
}
|
2668
2670
|
|
2669
|
-
|
2670
|
-
|
2671
|
-
|
2671
|
+
/* return HUFv06_decompress4X2(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams single-symbol decoding */
|
2672
|
+
/* return HUFv06_decompress4X4(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams double-symbols decoding */
|
2673
|
+
/* return HUFv06_decompress4X6(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams quad-symbols decoding */
|
2672
2674
|
}
|
2673
2675
|
/*
|
2674
2676
|
Common functions of Zstd compression library
|
@@ -3240,14 +3242,12 @@ static size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
|
|
3240
3242
|
}
|
3241
3243
|
|
3242
3244
|
/* FSE table descriptors */
|
3245
|
+
if (ip + 4 > iend) return ERROR(srcSize_wrong); /* min : header byte + all 3 are "raw", hence no header, but at least xxLog bits per type */
|
3243
3246
|
{ U32 const LLtype = *ip >> 6;
|
3244
3247
|
U32 const Offtype = (*ip >> 4) & 3;
|
3245
3248
|
U32 const MLtype = (*ip >> 2) & 3;
|
3246
3249
|
ip++;
|
3247
3250
|
|
3248
|
-
/* check */
|
3249
|
-
if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
|
3250
|
-
|
3251
3251
|
/* Build DTables */
|
3252
3252
|
{ size_t const bhSize = ZSTDv06_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
|
3253
3253
|
if (ZSTDv06_isError(bhSize)) return ERROR(corruption_detected);
|
@@ -3406,7 +3406,7 @@ static size_t ZSTDv06_execSequence(BYTE* op,
|
|
3406
3406
|
if (sequence.offset < 8) {
|
3407
3407
|
/* close range match, overlap */
|
3408
3408
|
static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
|
3409
|
-
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /*
|
3409
|
+
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
|
3410
3410
|
int const sub2 = dec64table[sequence.offset];
|
3411
3411
|
op[0] = match[0];
|
3412
3412
|
op[1] = match[1];
|
@@ -3501,8 +3501,10 @@ static size_t ZSTDv06_decompressSequences(
|
|
3501
3501
|
{ size_t const lastLLSize = litEnd - litPtr;
|
3502
3502
|
if (litPtr > litEnd) return ERROR(corruption_detected); /* too many literals already used */
|
3503
3503
|
if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
|
3504
|
-
|
3505
|
-
|
3504
|
+
if (lastLLSize > 0) {
|
3505
|
+
memcpy(op, litPtr, lastLLSize);
|
3506
|
+
op += lastLLSize;
|
3507
|
+
}
|
3506
3508
|
}
|
3507
3509
|
|
3508
3510
|
return op-ostart;
|
@@ -3654,36 +3656,62 @@ size_t ZSTDv06_decompress(void* dst, size_t dstCapacity, const void* src, size_t
|
|
3654
3656
|
#endif
|
3655
3657
|
}
|
3656
3658
|
|
3657
|
-
|
3659
|
+
/* ZSTD_errorFrameSizeInfoLegacy() :
|
3660
|
+
assumes `cSize` and `dBound` are _not_ NULL */
|
3661
|
+
static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
|
3662
|
+
{
|
3663
|
+
*cSize = ret;
|
3664
|
+
*dBound = ZSTD_CONTENTSIZE_ERROR;
|
3665
|
+
}
|
3666
|
+
|
3667
|
+
void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
|
3658
3668
|
{
|
3659
3669
|
const BYTE* ip = (const BYTE*)src;
|
3660
3670
|
size_t remainingSize = srcSize;
|
3671
|
+
size_t nbBlocks = 0;
|
3661
3672
|
blockProperties_t blockProperties = { bt_compressed, 0 };
|
3662
3673
|
|
3663
3674
|
/* Frame Header */
|
3664
|
-
{ size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src,
|
3665
|
-
if (ZSTDv06_isError(frameHeaderSize))
|
3666
|
-
|
3667
|
-
|
3675
|
+
{ size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src, srcSize);
|
3676
|
+
if (ZSTDv06_isError(frameHeaderSize)) {
|
3677
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, frameHeaderSize);
|
3678
|
+
return;
|
3679
|
+
}
|
3680
|
+
if (MEM_readLE32(src) != ZSTDv06_MAGICNUMBER) {
|
3681
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
|
3682
|
+
return;
|
3683
|
+
}
|
3684
|
+
if (srcSize < frameHeaderSize+ZSTDv06_blockHeaderSize) {
|
3685
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
3686
|
+
return;
|
3687
|
+
}
|
3668
3688
|
ip += frameHeaderSize; remainingSize -= frameHeaderSize;
|
3669
3689
|
}
|
3670
3690
|
|
3671
3691
|
/* Loop on each block */
|
3672
3692
|
while (1) {
|
3673
3693
|
size_t const cBlockSize = ZSTDv06_getcBlockSize(ip, remainingSize, &blockProperties);
|
3674
|
-
if (ZSTDv06_isError(cBlockSize))
|
3694
|
+
if (ZSTDv06_isError(cBlockSize)) {
|
3695
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
|
3696
|
+
return;
|
3697
|
+
}
|
3675
3698
|
|
3676
3699
|
ip += ZSTDv06_blockHeaderSize;
|
3677
3700
|
remainingSize -= ZSTDv06_blockHeaderSize;
|
3678
|
-
if (cBlockSize > remainingSize)
|
3701
|
+
if (cBlockSize > remainingSize) {
|
3702
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
3703
|
+
return;
|
3704
|
+
}
|
3679
3705
|
|
3680
3706
|
if (cBlockSize == 0) break; /* bt_end */
|
3681
3707
|
|
3682
3708
|
ip += cBlockSize;
|
3683
3709
|
remainingSize -= cBlockSize;
|
3710
|
+
nbBlocks++;
|
3684
3711
|
}
|
3685
3712
|
|
3686
|
-
|
3713
|
+
*cSize = ip - (const BYTE*)src;
|
3714
|
+
*dBound = nbBlocks * ZSTDv06_BLOCKSIZE_MAX;
|
3687
3715
|
}
|
3688
3716
|
|
3689
3717
|
/*_******************************
|
@@ -3974,7 +4002,9 @@ size_t ZBUFFv06_decompressInit(ZBUFFv06_DCtx* zbd)
|
|
3974
4002
|
MEM_STATIC size_t ZBUFFv06_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
3975
4003
|
{
|
3976
4004
|
size_t length = MIN(dstCapacity, srcSize);
|
3977
|
-
|
4005
|
+
if (length > 0) {
|
4006
|
+
memcpy(dst, src, length);
|
4007
|
+
}
|
3978
4008
|
return length;
|
3979
4009
|
}
|
3980
4010
|
|
@@ -4083,7 +4113,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
|
|
4083
4113
|
if (!decodedSize) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */
|
4084
4114
|
zbd->outEnd = zbd->outStart + decodedSize;
|
4085
4115
|
zbd->stage = ZBUFFds_flush;
|
4086
|
-
|
4116
|
+
/* break; */ /* ZBUFFds_flush follows */
|
4087
4117
|
}
|
4088
4118
|
}
|
4089
4119
|
/* fall-through */
|
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c) 2016-
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -43,12 +43,17 @@ ZSTDLIBv06_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity,
|
|
43
43
|
const void* src, size_t compressedSize);
|
44
44
|
|
45
45
|
/**
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
46
|
+
ZSTDv06_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.6.x format
|
47
|
+
srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
48
|
+
cSize (output parameter) : the number of bytes that would be read to decompress this frame
|
49
|
+
or an error code if it fails (which can be tested using ZSTDv01_isError())
|
50
|
+
dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
|
51
|
+
or ZSTD_CONTENTSIZE_ERROR if an error occurs
|
52
|
+
|
53
|
+
note : assumes `cSize` and `dBound` are _not_ NULL.
|
50
54
|
*/
|
51
|
-
|
55
|
+
void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
|
56
|
+
size_t* cSize, unsigned long long* dBound);
|
52
57
|
|
53
58
|
/* *************************************
|
54
59
|
* Helper functions
|
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c) 2016-
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -17,14 +17,14 @@
|
|
17
17
|
#ifndef XXH_STATIC_LINKING_ONLY
|
18
18
|
# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
|
19
19
|
#endif
|
20
|
-
#include "xxhash.h" /* XXH64_* */
|
20
|
+
#include "../common/xxhash.h" /* XXH64_* */
|
21
21
|
#include "zstd_v07.h"
|
22
22
|
|
23
23
|
#define FSEv07_STATIC_LINKING_ONLY /* FSEv07_MIN_TABLELOG */
|
24
24
|
#define HUFv07_STATIC_LINKING_ONLY /* HUFv07_TABLELOG_ABSOLUTEMAX */
|
25
25
|
#define ZSTDv07_STATIC_LINKING_ONLY
|
26
26
|
|
27
|
-
#include "error_private.h"
|
27
|
+
#include "../common/error_private.h"
|
28
28
|
|
29
29
|
|
30
30
|
#ifdef ZSTDv07_STATIC_LINKING_ONLY
|
@@ -530,7 +530,7 @@ MEM_STATIC unsigned BITv07_highbit32 (U32 val)
|
|
530
530
|
_BitScanReverse ( &r, val );
|
531
531
|
return (unsigned) r;
|
532
532
|
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
|
533
|
-
return
|
533
|
+
return __builtin_clz (val) ^ 31;
|
534
534
|
# else /* Software version */
|
535
535
|
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
536
536
|
U32 v = val;
|
@@ -1314,7 +1314,7 @@ size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
|
1314
1314
|
|
1315
1315
|
if (!srcSize) return ERROR(srcSize_wrong);
|
1316
1316
|
iSize = ip[0];
|
1317
|
-
|
1317
|
+
/* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */
|
1318
1318
|
|
1319
1319
|
if (iSize >= 128) { /* special header */
|
1320
1320
|
if (iSize >= (242)) { /* RLE */
|
@@ -1784,7 +1784,7 @@ size_t HUFv07_readDTableX2 (HUFv07_DTable* DTable, const void* src, size_t srcSi
|
|
1784
1784
|
HUFv07_DEltX2* const dt = (HUFv07_DEltX2*)dtPtr;
|
1785
1785
|
|
1786
1786
|
HUFv07_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUFv07_DTable));
|
1787
|
-
|
1787
|
+
/* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
|
1788
1788
|
|
1789
1789
|
iSize = HUFv07_readStats(huffWeight, HUFv07_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
|
1790
1790
|
if (HUFv07_isError(iSize)) return iSize;
|
@@ -2148,7 +2148,7 @@ size_t HUFv07_readDTableX4 (HUFv07_DTable* DTable, const void* src, size_t srcSi
|
|
2148
2148
|
|
2149
2149
|
HUFv07_STATIC_ASSERT(sizeof(HUFv07_DEltX4) == sizeof(HUFv07_DTable)); /* if compilation fails here, assertion is false */
|
2150
2150
|
if (maxTableLog > HUFv07_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge);
|
2151
|
-
|
2151
|
+
/* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
|
2152
2152
|
|
2153
2153
|
iSize = HUFv07_readStats(weightList, HUFv07_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
|
2154
2154
|
if (HUFv07_isError(iSize)) return iSize;
|
@@ -2530,8 +2530,8 @@ size_t HUFv07_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
|
|
2530
2530
|
return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
|
2531
2531
|
}
|
2532
2532
|
|
2533
|
-
|
2534
|
-
|
2533
|
+
/* return HUFv07_decompress4X2(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams single-symbol decoding */
|
2534
|
+
/* return HUFv07_decompress4X4(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams double-symbols decoding */
|
2535
2535
|
}
|
2536
2536
|
|
2537
2537
|
size_t HUFv07_decompress4X_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
@@ -2740,6 +2740,8 @@ typedef enum { lbt_huffman, lbt_repeat, lbt_raw, lbt_rle } litBlockType_t;
|
|
2740
2740
|
#define FSEv07_ENCODING_STATIC 2
|
2741
2741
|
#define FSEv07_ENCODING_DYNAMIC 3
|
2742
2742
|
|
2743
|
+
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
|
2744
|
+
|
2743
2745
|
static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
2744
2746
|
1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12,
|
2745
2747
|
13,14,15,16 };
|
@@ -3270,7 +3272,9 @@ static size_t ZSTDv07_getcBlockSize(const void* src, size_t srcSize, blockProper
|
|
3270
3272
|
static size_t ZSTDv07_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
3271
3273
|
{
|
3272
3274
|
if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
|
3273
|
-
|
3275
|
+
if (srcSize > 0) {
|
3276
|
+
memcpy(dst, src, srcSize);
|
3277
|
+
}
|
3274
3278
|
return srcSize;
|
3275
3279
|
}
|
3276
3280
|
|
@@ -3468,14 +3472,12 @@ static size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr,
|
|
3468
3472
|
}
|
3469
3473
|
|
3470
3474
|
/* FSE table descriptors */
|
3475
|
+
if (ip + 4 > iend) return ERROR(srcSize_wrong); /* min : header byte + all 3 are "raw", hence no header, but at least xxLog bits per type */
|
3471
3476
|
{ U32 const LLtype = *ip >> 6;
|
3472
3477
|
U32 const OFtype = (*ip >> 4) & 3;
|
3473
3478
|
U32 const MLtype = (*ip >> 2) & 3;
|
3474
3479
|
ip++;
|
3475
3480
|
|
3476
|
-
/* check */
|
3477
|
-
if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
|
3478
|
-
|
3479
3481
|
/* Build DTables */
|
3480
3482
|
{ size_t const llhSize = ZSTDv07_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
|
3481
3483
|
if (ZSTDv07_isError(llhSize)) return ERROR(corruption_detected);
|
@@ -3631,7 +3633,7 @@ size_t ZSTDv07_execSequence(BYTE* op,
|
|
3631
3633
|
if (sequence.offset < 8) {
|
3632
3634
|
/* close range match, overlap */
|
3633
3635
|
static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
|
3634
|
-
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /*
|
3636
|
+
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
|
3635
3637
|
int const sub2 = dec64table[sequence.offset];
|
3636
3638
|
op[0] = match[0];
|
3637
3639
|
op[1] = match[1];
|
@@ -3712,10 +3714,12 @@ static size_t ZSTDv07_decompressSequences(
|
|
3712
3714
|
|
3713
3715
|
/* last literal segment */
|
3714
3716
|
{ size_t const lastLLSize = litEnd - litPtr;
|
3715
|
-
|
3717
|
+
/* if (litPtr > litEnd) return ERROR(corruption_detected); */ /* too many literals already used */
|
3716
3718
|
if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall);
|
3717
|
-
|
3718
|
-
|
3719
|
+
if (lastLLSize > 0) {
|
3720
|
+
memcpy(op, litPtr, lastLLSize);
|
3721
|
+
op += lastLLSize;
|
3722
|
+
}
|
3719
3723
|
}
|
3720
3724
|
|
3721
3725
|
return op-ostart;
|
@@ -3776,7 +3780,9 @@ ZSTDLIBv07_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockS
|
|
3776
3780
|
static size_t ZSTDv07_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length)
|
3777
3781
|
{
|
3778
3782
|
if (length > dstCapacity) return ERROR(dstSize_tooSmall);
|
3779
|
-
|
3783
|
+
if (length > 0) {
|
3784
|
+
memset(dst, byte, length);
|
3785
|
+
}
|
3780
3786
|
return length;
|
3781
3787
|
}
|
3782
3788
|
|
@@ -3895,19 +3901,40 @@ size_t ZSTDv07_decompress(void* dst, size_t dstCapacity, const void* src, size_t
|
|
3895
3901
|
#endif
|
3896
3902
|
}
|
3897
3903
|
|
3898
|
-
|
3904
|
+
/* ZSTD_errorFrameSizeInfoLegacy() :
|
3905
|
+
assumes `cSize` and `dBound` are _not_ NULL */
|
3906
|
+
static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
|
3907
|
+
{
|
3908
|
+
*cSize = ret;
|
3909
|
+
*dBound = ZSTD_CONTENTSIZE_ERROR;
|
3910
|
+
}
|
3911
|
+
|
3912
|
+
void ZSTDv07_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
|
3899
3913
|
{
|
3900
3914
|
const BYTE* ip = (const BYTE*)src;
|
3901
3915
|
size_t remainingSize = srcSize;
|
3916
|
+
size_t nbBlocks = 0;
|
3902
3917
|
|
3903
3918
|
/* check */
|
3904
|
-
if (srcSize < ZSTDv07_frameHeaderSize_min+ZSTDv07_blockHeaderSize)
|
3919
|
+
if (srcSize < ZSTDv07_frameHeaderSize_min+ZSTDv07_blockHeaderSize) {
|
3920
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
3921
|
+
return;
|
3922
|
+
}
|
3905
3923
|
|
3906
3924
|
/* Frame Header */
|
3907
|
-
{ size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src,
|
3908
|
-
if (ZSTDv07_isError(frameHeaderSize))
|
3909
|
-
|
3910
|
-
|
3925
|
+
{ size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src, srcSize);
|
3926
|
+
if (ZSTDv07_isError(frameHeaderSize)) {
|
3927
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, frameHeaderSize);
|
3928
|
+
return;
|
3929
|
+
}
|
3930
|
+
if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) {
|
3931
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
|
3932
|
+
return;
|
3933
|
+
}
|
3934
|
+
if (srcSize < frameHeaderSize+ZSTDv07_blockHeaderSize) {
|
3935
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
3936
|
+
return;
|
3937
|
+
}
|
3911
3938
|
ip += frameHeaderSize; remainingSize -= frameHeaderSize;
|
3912
3939
|
}
|
3913
3940
|
|
@@ -3915,20 +3942,28 @@ size_t ZSTDv07_findFrameCompressedSize(const void* src, size_t srcSize)
|
|
3915
3942
|
while (1) {
|
3916
3943
|
blockProperties_t blockProperties;
|
3917
3944
|
size_t const cBlockSize = ZSTDv07_getcBlockSize(ip, remainingSize, &blockProperties);
|
3918
|
-
if (ZSTDv07_isError(cBlockSize))
|
3945
|
+
if (ZSTDv07_isError(cBlockSize)) {
|
3946
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
|
3947
|
+
return;
|
3948
|
+
}
|
3919
3949
|
|
3920
3950
|
ip += ZSTDv07_blockHeaderSize;
|
3921
3951
|
remainingSize -= ZSTDv07_blockHeaderSize;
|
3922
3952
|
|
3923
3953
|
if (blockProperties.blockType == bt_end) break;
|
3924
3954
|
|
3925
|
-
if (cBlockSize > remainingSize)
|
3955
|
+
if (cBlockSize > remainingSize) {
|
3956
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
3957
|
+
return;
|
3958
|
+
}
|
3926
3959
|
|
3927
3960
|
ip += cBlockSize;
|
3928
3961
|
remainingSize -= cBlockSize;
|
3962
|
+
nbBlocks++;
|
3929
3963
|
}
|
3930
3964
|
|
3931
|
-
|
3965
|
+
*cSize = ip - (const BYTE*)src;
|
3966
|
+
*dBound = nbBlocks * ZSTDv07_BLOCKSIZE_ABSOLUTEMAX;
|
3932
3967
|
}
|
3933
3968
|
|
3934
3969
|
/*_******************************
|
@@ -4349,7 +4384,9 @@ size_t ZBUFFv07_decompressInit(ZBUFFv07_DCtx* zbd)
|
|
4349
4384
|
MEM_STATIC size_t ZBUFFv07_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
4350
4385
|
{
|
4351
4386
|
size_t const length = MIN(dstCapacity, srcSize);
|
4352
|
-
|
4387
|
+
if (length > 0) {
|
4388
|
+
memcpy(dst, src, length);
|
4389
|
+
}
|
4353
4390
|
return length;
|
4354
4391
|
}
|
4355
4392
|
|