zstd-ruby 1.3.8.0 → 1.4.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +6 -5
- data/README.md +1 -1
- data/ext/zstdruby/libzstd/Makefile +133 -61
- data/ext/zstdruby/libzstd/README.md +51 -18
- data/ext/zstdruby/libzstd/common/bitstream.h +38 -39
- data/ext/zstdruby/libzstd/common/compiler.h +41 -6
- data/ext/zstdruby/libzstd/common/cpu.h +1 -1
- data/ext/zstdruby/libzstd/common/debug.c +11 -31
- data/ext/zstdruby/libzstd/common/debug.h +11 -31
- data/ext/zstdruby/libzstd/common/entropy_common.c +13 -33
- data/ext/zstdruby/libzstd/common/error_private.c +2 -1
- data/ext/zstdruby/libzstd/common/error_private.h +6 -2
- data/ext/zstdruby/libzstd/common/fse.h +13 -33
- data/ext/zstdruby/libzstd/common/fse_decompress.c +12 -35
- data/ext/zstdruby/libzstd/common/huf.h +15 -33
- data/ext/zstdruby/libzstd/common/mem.h +75 -2
- data/ext/zstdruby/libzstd/common/pool.c +8 -4
- data/ext/zstdruby/libzstd/common/pool.h +2 -2
- data/ext/zstdruby/libzstd/common/threading.c +52 -6
- data/ext/zstdruby/libzstd/common/threading.h +36 -4
- data/ext/zstdruby/libzstd/common/xxhash.c +25 -37
- data/ext/zstdruby/libzstd/common/xxhash.h +11 -31
- data/ext/zstdruby/libzstd/common/zstd_common.c +1 -1
- data/ext/zstdruby/libzstd/common/zstd_errors.h +2 -1
- data/ext/zstdruby/libzstd/common/zstd_internal.h +203 -22
- data/ext/zstdruby/libzstd/compress/fse_compress.c +19 -42
- data/ext/zstdruby/libzstd/compress/hist.c +15 -35
- data/ext/zstdruby/libzstd/compress/hist.h +12 -32
- data/ext/zstdruby/libzstd/compress/huf_compress.c +92 -92
- data/ext/zstdruby/libzstd/compress/zstd_compress.c +1460 -1472
- data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +330 -65
- data/ext/zstdruby/libzstd/compress/zstd_compress_literals.c +158 -0
- data/ext/zstdruby/libzstd/compress/zstd_compress_literals.h +29 -0
- data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.c +419 -0
- data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.h +54 -0
- data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.c +845 -0
- data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.h +32 -0
- data/ext/zstdruby/libzstd/compress/zstd_cwksp.h +525 -0
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +65 -43
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +2 -2
- data/ext/zstdruby/libzstd/compress/zstd_fast.c +264 -159
- data/ext/zstdruby/libzstd/compress/zstd_fast.h +2 -2
- data/ext/zstdruby/libzstd/compress/zstd_lazy.c +74 -42
- data/ext/zstdruby/libzstd/compress/zstd_lazy.h +2 -2
- data/ext/zstdruby/libzstd/compress/zstd_ldm.c +33 -11
- data/ext/zstdruby/libzstd/compress/zstd_ldm.h +7 -2
- data/ext/zstdruby/libzstd/compress/zstd_opt.c +108 -125
- data/ext/zstdruby/libzstd/compress/zstd_opt.h +1 -1
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +129 -93
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +46 -28
- data/ext/zstdruby/libzstd/decompress/huf_decompress.c +76 -60
- data/ext/zstdruby/libzstd/decompress/zstd_ddict.c +14 -10
- data/ext/zstdruby/libzstd/decompress/zstd_ddict.h +2 -2
- data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +471 -258
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.c +471 -346
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.h +3 -3
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_internal.h +25 -4
- data/ext/zstdruby/libzstd/deprecated/zbuff.h +9 -8
- data/ext/zstdruby/libzstd/deprecated/zbuff_common.c +2 -2
- data/ext/zstdruby/libzstd/deprecated/zbuff_compress.c +1 -1
- data/ext/zstdruby/libzstd/deprecated/zbuff_decompress.c +1 -1
- data/ext/zstdruby/libzstd/dictBuilder/cover.c +220 -65
- data/ext/zstdruby/libzstd/dictBuilder/cover.h +81 -7
- data/ext/zstdruby/libzstd/dictBuilder/fastcover.c +85 -56
- data/ext/zstdruby/libzstd/dictBuilder/zdict.c +43 -19
- data/ext/zstdruby/libzstd/dictBuilder/zdict.h +73 -35
- data/ext/zstdruby/libzstd/dll/example/Makefile +2 -1
- data/ext/zstdruby/libzstd/dll/example/build_package.bat +3 -2
- data/ext/zstdruby/libzstd/legacy/zstd_legacy.h +49 -15
- data/ext/zstdruby/libzstd/legacy/zstd_v01.c +142 -117
- data/ext/zstdruby/libzstd/legacy/zstd_v01.h +13 -8
- data/ext/zstdruby/libzstd/legacy/zstd_v02.c +54 -25
- data/ext/zstdruby/libzstd/legacy/zstd_v02.h +13 -8
- data/ext/zstdruby/libzstd/legacy/zstd_v03.c +55 -25
- data/ext/zstdruby/libzstd/legacy/zstd_v03.h +13 -8
- data/ext/zstdruby/libzstd/legacy/zstd_v04.c +62 -29
- data/ext/zstdruby/libzstd/legacy/zstd_v04.h +13 -8
- data/ext/zstdruby/libzstd/legacy/zstd_v05.c +145 -109
- data/ext/zstdruby/libzstd/legacy/zstd_v05.h +14 -9
- data/ext/zstdruby/libzstd/legacy/zstd_v06.c +56 -26
- data/ext/zstdruby/libzstd/legacy/zstd_v06.h +11 -6
- data/ext/zstdruby/libzstd/legacy/zstd_v07.c +65 -28
- data/ext/zstdruby/libzstd/legacy/zstd_v07.h +11 -6
- data/ext/zstdruby/libzstd/libzstd.pc.in +3 -2
- data/ext/zstdruby/libzstd/zstd.h +921 -597
- data/lib/zstd-ruby/version.rb +1 -1
- data/zstd-ruby.gemspec +2 -2
- metadata +19 -14
- data/ext/zstdruby/libzstd/dll/libzstd.def +0 -87
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/*
|
|
2
|
-
* Copyright (c) 2016-
|
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
|
@@ -19,7 +19,7 @@ extern "C" {
|
|
|
19
19
|
* Dependencies
|
|
20
20
|
***************************************/
|
|
21
21
|
#include <stddef.h> /* size_t */
|
|
22
|
-
#include "mem.h" /* U64, U32 */
|
|
22
|
+
#include "../common/mem.h" /* U64, U32 */
|
|
23
23
|
|
|
24
24
|
|
|
25
25
|
/* *************************************
|
|
@@ -33,13 +33,18 @@ extern "C" {
|
|
|
33
33
|
size_t ZSTDv05_decompress( void* dst, size_t dstCapacity,
|
|
34
34
|
const void* src, size_t compressedSize);
|
|
35
35
|
|
|
36
|
-
/**
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
36
|
+
/**
|
|
37
|
+
ZSTDv05_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.5.x format
|
|
38
|
+
srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
|
39
|
+
cSize (output parameter) : the number of bytes that would be read to decompress this frame
|
|
40
|
+
or an error code if it fails (which can be tested using ZSTDv01_isError())
|
|
41
|
+
dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
|
|
42
|
+
or ZSTD_CONTENTSIZE_ERROR if an error occurs
|
|
43
|
+
|
|
44
|
+
note : assumes `cSize` and `dBound` are _not_ NULL.
|
|
45
|
+
*/
|
|
46
|
+
void ZSTDv05_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
|
|
47
|
+
size_t* cSize, unsigned long long* dBound);
|
|
43
48
|
|
|
44
49
|
/* *************************************
|
|
45
50
|
* Helper functions
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/*
|
|
2
|
-
* Copyright (c) 2016-
|
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
#include <stddef.h> /* size_t, ptrdiff_t */
|
|
15
15
|
#include <string.h> /* memcpy */
|
|
16
16
|
#include <stdlib.h> /* malloc, free, qsort */
|
|
17
|
-
#include "error_private.h"
|
|
17
|
+
#include "../common/error_private.h"
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
|
|
@@ -506,6 +506,8 @@ typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
|
|
|
506
506
|
#define FSEv06_ENCODING_STATIC 2
|
|
507
507
|
#define FSEv06_ENCODING_DYNAMIC 3
|
|
508
508
|
|
|
509
|
+
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
|
|
510
|
+
|
|
509
511
|
static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
510
512
|
1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12,
|
|
511
513
|
13,14,15,16 };
|
|
@@ -858,7 +860,7 @@ MEM_STATIC unsigned BITv06_highbit32 ( U32 val)
|
|
|
858
860
|
_BitScanReverse ( &r, val );
|
|
859
861
|
return (unsigned) r;
|
|
860
862
|
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
|
|
861
|
-
return
|
|
863
|
+
return __builtin_clz (val) ^ 31;
|
|
862
864
|
# else /* Software version */
|
|
863
865
|
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
|
864
866
|
U32 v = val;
|
|
@@ -1860,7 +1862,7 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta
|
|
|
1860
1862
|
|
|
1861
1863
|
if (!srcSize) return ERROR(srcSize_wrong);
|
|
1862
1864
|
iSize = ip[0];
|
|
1863
|
-
|
|
1865
|
+
/* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */
|
|
1864
1866
|
|
|
1865
1867
|
if (iSize >= 128) { /* special header */
|
|
1866
1868
|
if (iSize >= (242)) { /* RLE */
|
|
@@ -2012,7 +2014,7 @@ size_t HUFv06_readDTableX2 (U16* DTable, const void* src, size_t srcSize)
|
|
|
2012
2014
|
HUFv06_DEltX2* const dt = (HUFv06_DEltX2*)dtPtr;
|
|
2013
2015
|
|
|
2014
2016
|
HUFv06_STATIC_ASSERT(sizeof(HUFv06_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */
|
|
2015
|
-
|
|
2017
|
+
/* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
|
|
2016
2018
|
|
|
2017
2019
|
iSize = HUFv06_readStats(huffWeight, HUFv06_MAX_SYMBOL_VALUE + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
|
|
2018
2020
|
if (HUFv06_isError(iSize)) return iSize;
|
|
@@ -2338,7 +2340,7 @@ size_t HUFv06_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
|
|
|
2338
2340
|
|
|
2339
2341
|
HUFv06_STATIC_ASSERT(sizeof(HUFv06_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */
|
|
2340
2342
|
if (memLog > HUFv06_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
|
2341
|
-
|
|
2343
|
+
/* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
|
|
2342
2344
|
|
|
2343
2345
|
iSize = HUFv06_readStats(weightList, HUFv06_MAX_SYMBOL_VALUE + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
|
|
2344
2346
|
if (HUFv06_isError(iSize)) return iSize;
|
|
@@ -2662,13 +2664,13 @@ size_t HUFv06_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
|
|
|
2662
2664
|
|
|
2663
2665
|
{ U32 algoNb = 0;
|
|
2664
2666
|
if (Dtime[1] < Dtime[0]) algoNb = 1;
|
|
2665
|
-
|
|
2667
|
+
/* if (Dtime[2] < Dtime[algoNb]) algoNb = 2; */ /* current speed of HUFv06_decompress4X6 is not good */
|
|
2666
2668
|
return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
|
|
2667
2669
|
}
|
|
2668
2670
|
|
|
2669
|
-
|
|
2670
|
-
|
|
2671
|
-
|
|
2671
|
+
/* return HUFv06_decompress4X2(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams single-symbol decoding */
|
|
2672
|
+
/* return HUFv06_decompress4X4(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams double-symbols decoding */
|
|
2673
|
+
/* return HUFv06_decompress4X6(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams quad-symbols decoding */
|
|
2672
2674
|
}
|
|
2673
2675
|
/*
|
|
2674
2676
|
Common functions of Zstd compression library
|
|
@@ -3240,14 +3242,12 @@ static size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
|
|
|
3240
3242
|
}
|
|
3241
3243
|
|
|
3242
3244
|
/* FSE table descriptors */
|
|
3245
|
+
if (ip + 4 > iend) return ERROR(srcSize_wrong); /* min : header byte + all 3 are "raw", hence no header, but at least xxLog bits per type */
|
|
3243
3246
|
{ U32 const LLtype = *ip >> 6;
|
|
3244
3247
|
U32 const Offtype = (*ip >> 4) & 3;
|
|
3245
3248
|
U32 const MLtype = (*ip >> 2) & 3;
|
|
3246
3249
|
ip++;
|
|
3247
3250
|
|
|
3248
|
-
/* check */
|
|
3249
|
-
if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
|
|
3250
|
-
|
|
3251
3251
|
/* Build DTables */
|
|
3252
3252
|
{ size_t const bhSize = ZSTDv06_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
|
|
3253
3253
|
if (ZSTDv06_isError(bhSize)) return ERROR(corruption_detected);
|
|
@@ -3406,7 +3406,7 @@ static size_t ZSTDv06_execSequence(BYTE* op,
|
|
|
3406
3406
|
if (sequence.offset < 8) {
|
|
3407
3407
|
/* close range match, overlap */
|
|
3408
3408
|
static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
|
|
3409
|
-
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /*
|
|
3409
|
+
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
|
|
3410
3410
|
int const sub2 = dec64table[sequence.offset];
|
|
3411
3411
|
op[0] = match[0];
|
|
3412
3412
|
op[1] = match[1];
|
|
@@ -3501,8 +3501,10 @@ static size_t ZSTDv06_decompressSequences(
|
|
|
3501
3501
|
{ size_t const lastLLSize = litEnd - litPtr;
|
|
3502
3502
|
if (litPtr > litEnd) return ERROR(corruption_detected); /* too many literals already used */
|
|
3503
3503
|
if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
|
|
3504
|
-
|
|
3505
|
-
|
|
3504
|
+
if (lastLLSize > 0) {
|
|
3505
|
+
memcpy(op, litPtr, lastLLSize);
|
|
3506
|
+
op += lastLLSize;
|
|
3507
|
+
}
|
|
3506
3508
|
}
|
|
3507
3509
|
|
|
3508
3510
|
return op-ostart;
|
|
@@ -3654,36 +3656,62 @@ size_t ZSTDv06_decompress(void* dst, size_t dstCapacity, const void* src, size_t
|
|
|
3654
3656
|
#endif
|
|
3655
3657
|
}
|
|
3656
3658
|
|
|
3657
|
-
|
|
3659
|
+
/* ZSTD_errorFrameSizeInfoLegacy() :
|
|
3660
|
+
assumes `cSize` and `dBound` are _not_ NULL */
|
|
3661
|
+
static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
|
|
3662
|
+
{
|
|
3663
|
+
*cSize = ret;
|
|
3664
|
+
*dBound = ZSTD_CONTENTSIZE_ERROR;
|
|
3665
|
+
}
|
|
3666
|
+
|
|
3667
|
+
void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
|
|
3658
3668
|
{
|
|
3659
3669
|
const BYTE* ip = (const BYTE*)src;
|
|
3660
3670
|
size_t remainingSize = srcSize;
|
|
3671
|
+
size_t nbBlocks = 0;
|
|
3661
3672
|
blockProperties_t blockProperties = { bt_compressed, 0 };
|
|
3662
3673
|
|
|
3663
3674
|
/* Frame Header */
|
|
3664
|
-
{ size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src,
|
|
3665
|
-
if (ZSTDv06_isError(frameHeaderSize))
|
|
3666
|
-
|
|
3667
|
-
|
|
3675
|
+
{ size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src, srcSize);
|
|
3676
|
+
if (ZSTDv06_isError(frameHeaderSize)) {
|
|
3677
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, frameHeaderSize);
|
|
3678
|
+
return;
|
|
3679
|
+
}
|
|
3680
|
+
if (MEM_readLE32(src) != ZSTDv06_MAGICNUMBER) {
|
|
3681
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
|
|
3682
|
+
return;
|
|
3683
|
+
}
|
|
3684
|
+
if (srcSize < frameHeaderSize+ZSTDv06_blockHeaderSize) {
|
|
3685
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
|
3686
|
+
return;
|
|
3687
|
+
}
|
|
3668
3688
|
ip += frameHeaderSize; remainingSize -= frameHeaderSize;
|
|
3669
3689
|
}
|
|
3670
3690
|
|
|
3671
3691
|
/* Loop on each block */
|
|
3672
3692
|
while (1) {
|
|
3673
3693
|
size_t const cBlockSize = ZSTDv06_getcBlockSize(ip, remainingSize, &blockProperties);
|
|
3674
|
-
if (ZSTDv06_isError(cBlockSize))
|
|
3694
|
+
if (ZSTDv06_isError(cBlockSize)) {
|
|
3695
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
|
|
3696
|
+
return;
|
|
3697
|
+
}
|
|
3675
3698
|
|
|
3676
3699
|
ip += ZSTDv06_blockHeaderSize;
|
|
3677
3700
|
remainingSize -= ZSTDv06_blockHeaderSize;
|
|
3678
|
-
if (cBlockSize > remainingSize)
|
|
3701
|
+
if (cBlockSize > remainingSize) {
|
|
3702
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
|
3703
|
+
return;
|
|
3704
|
+
}
|
|
3679
3705
|
|
|
3680
3706
|
if (cBlockSize == 0) break; /* bt_end */
|
|
3681
3707
|
|
|
3682
3708
|
ip += cBlockSize;
|
|
3683
3709
|
remainingSize -= cBlockSize;
|
|
3710
|
+
nbBlocks++;
|
|
3684
3711
|
}
|
|
3685
3712
|
|
|
3686
|
-
|
|
3713
|
+
*cSize = ip - (const BYTE*)src;
|
|
3714
|
+
*dBound = nbBlocks * ZSTDv06_BLOCKSIZE_MAX;
|
|
3687
3715
|
}
|
|
3688
3716
|
|
|
3689
3717
|
/*_******************************
|
|
@@ -3974,7 +4002,9 @@ size_t ZBUFFv06_decompressInit(ZBUFFv06_DCtx* zbd)
|
|
|
3974
4002
|
MEM_STATIC size_t ZBUFFv06_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
|
3975
4003
|
{
|
|
3976
4004
|
size_t length = MIN(dstCapacity, srcSize);
|
|
3977
|
-
|
|
4005
|
+
if (length > 0) {
|
|
4006
|
+
memcpy(dst, src, length);
|
|
4007
|
+
}
|
|
3978
4008
|
return length;
|
|
3979
4009
|
}
|
|
3980
4010
|
|
|
@@ -4083,7 +4113,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
|
|
|
4083
4113
|
if (!decodedSize) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */
|
|
4084
4114
|
zbd->outEnd = zbd->outStart + decodedSize;
|
|
4085
4115
|
zbd->stage = ZBUFFds_flush;
|
|
4086
|
-
|
|
4116
|
+
/* break; */ /* ZBUFFds_flush follows */
|
|
4087
4117
|
}
|
|
4088
4118
|
}
|
|
4089
4119
|
/* fall-through */
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/*
|
|
2
|
-
* Copyright (c) 2016-
|
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
|
@@ -43,12 +43,17 @@ ZSTDLIBv06_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity,
|
|
|
43
43
|
const void* src, size_t compressedSize);
|
|
44
44
|
|
|
45
45
|
/**
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
46
|
+
ZSTDv06_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.6.x format
|
|
47
|
+
srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
|
48
|
+
cSize (output parameter) : the number of bytes that would be read to decompress this frame
|
|
49
|
+
or an error code if it fails (which can be tested using ZSTDv01_isError())
|
|
50
|
+
dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
|
|
51
|
+
or ZSTD_CONTENTSIZE_ERROR if an error occurs
|
|
52
|
+
|
|
53
|
+
note : assumes `cSize` and `dBound` are _not_ NULL.
|
|
50
54
|
*/
|
|
51
|
-
|
|
55
|
+
void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
|
|
56
|
+
size_t* cSize, unsigned long long* dBound);
|
|
52
57
|
|
|
53
58
|
/* *************************************
|
|
54
59
|
* Helper functions
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/*
|
|
2
|
-
* Copyright (c) 2016-
|
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
|
@@ -17,14 +17,14 @@
|
|
|
17
17
|
#ifndef XXH_STATIC_LINKING_ONLY
|
|
18
18
|
# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
|
|
19
19
|
#endif
|
|
20
|
-
#include "xxhash.h" /* XXH64_* */
|
|
20
|
+
#include "../common/xxhash.h" /* XXH64_* */
|
|
21
21
|
#include "zstd_v07.h"
|
|
22
22
|
|
|
23
23
|
#define FSEv07_STATIC_LINKING_ONLY /* FSEv07_MIN_TABLELOG */
|
|
24
24
|
#define HUFv07_STATIC_LINKING_ONLY /* HUFv07_TABLELOG_ABSOLUTEMAX */
|
|
25
25
|
#define ZSTDv07_STATIC_LINKING_ONLY
|
|
26
26
|
|
|
27
|
-
#include "error_private.h"
|
|
27
|
+
#include "../common/error_private.h"
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
#ifdef ZSTDv07_STATIC_LINKING_ONLY
|
|
@@ -530,7 +530,7 @@ MEM_STATIC unsigned BITv07_highbit32 (U32 val)
|
|
|
530
530
|
_BitScanReverse ( &r, val );
|
|
531
531
|
return (unsigned) r;
|
|
532
532
|
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
|
|
533
|
-
return
|
|
533
|
+
return __builtin_clz (val) ^ 31;
|
|
534
534
|
# else /* Software version */
|
|
535
535
|
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
|
536
536
|
U32 v = val;
|
|
@@ -1314,7 +1314,7 @@ size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
|
|
1314
1314
|
|
|
1315
1315
|
if (!srcSize) return ERROR(srcSize_wrong);
|
|
1316
1316
|
iSize = ip[0];
|
|
1317
|
-
|
|
1317
|
+
/* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */
|
|
1318
1318
|
|
|
1319
1319
|
if (iSize >= 128) { /* special header */
|
|
1320
1320
|
if (iSize >= (242)) { /* RLE */
|
|
@@ -1784,7 +1784,7 @@ size_t HUFv07_readDTableX2 (HUFv07_DTable* DTable, const void* src, size_t srcSi
|
|
|
1784
1784
|
HUFv07_DEltX2* const dt = (HUFv07_DEltX2*)dtPtr;
|
|
1785
1785
|
|
|
1786
1786
|
HUFv07_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUFv07_DTable));
|
|
1787
|
-
|
|
1787
|
+
/* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
|
|
1788
1788
|
|
|
1789
1789
|
iSize = HUFv07_readStats(huffWeight, HUFv07_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
|
|
1790
1790
|
if (HUFv07_isError(iSize)) return iSize;
|
|
@@ -2148,7 +2148,7 @@ size_t HUFv07_readDTableX4 (HUFv07_DTable* DTable, const void* src, size_t srcSi
|
|
|
2148
2148
|
|
|
2149
2149
|
HUFv07_STATIC_ASSERT(sizeof(HUFv07_DEltX4) == sizeof(HUFv07_DTable)); /* if compilation fails here, assertion is false */
|
|
2150
2150
|
if (maxTableLog > HUFv07_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge);
|
|
2151
|
-
|
|
2151
|
+
/* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
|
|
2152
2152
|
|
|
2153
2153
|
iSize = HUFv07_readStats(weightList, HUFv07_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
|
|
2154
2154
|
if (HUFv07_isError(iSize)) return iSize;
|
|
@@ -2530,8 +2530,8 @@ size_t HUFv07_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
|
|
|
2530
2530
|
return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
|
|
2531
2531
|
}
|
|
2532
2532
|
|
|
2533
|
-
|
|
2534
|
-
|
|
2533
|
+
/* return HUFv07_decompress4X2(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams single-symbol decoding */
|
|
2534
|
+
/* return HUFv07_decompress4X4(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams double-symbols decoding */
|
|
2535
2535
|
}
|
|
2536
2536
|
|
|
2537
2537
|
size_t HUFv07_decompress4X_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
|
@@ -2740,6 +2740,8 @@ typedef enum { lbt_huffman, lbt_repeat, lbt_raw, lbt_rle } litBlockType_t;
|
|
|
2740
2740
|
#define FSEv07_ENCODING_STATIC 2
|
|
2741
2741
|
#define FSEv07_ENCODING_DYNAMIC 3
|
|
2742
2742
|
|
|
2743
|
+
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
|
|
2744
|
+
|
|
2743
2745
|
static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
2744
2746
|
1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12,
|
|
2745
2747
|
13,14,15,16 };
|
|
@@ -3270,7 +3272,9 @@ static size_t ZSTDv07_getcBlockSize(const void* src, size_t srcSize, blockProper
|
|
|
3270
3272
|
static size_t ZSTDv07_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
|
3271
3273
|
{
|
|
3272
3274
|
if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
|
|
3273
|
-
|
|
3275
|
+
if (srcSize > 0) {
|
|
3276
|
+
memcpy(dst, src, srcSize);
|
|
3277
|
+
}
|
|
3274
3278
|
return srcSize;
|
|
3275
3279
|
}
|
|
3276
3280
|
|
|
@@ -3468,14 +3472,12 @@ static size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr,
|
|
|
3468
3472
|
}
|
|
3469
3473
|
|
|
3470
3474
|
/* FSE table descriptors */
|
|
3475
|
+
if (ip + 4 > iend) return ERROR(srcSize_wrong); /* min : header byte + all 3 are "raw", hence no header, but at least xxLog bits per type */
|
|
3471
3476
|
{ U32 const LLtype = *ip >> 6;
|
|
3472
3477
|
U32 const OFtype = (*ip >> 4) & 3;
|
|
3473
3478
|
U32 const MLtype = (*ip >> 2) & 3;
|
|
3474
3479
|
ip++;
|
|
3475
3480
|
|
|
3476
|
-
/* check */
|
|
3477
|
-
if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
|
|
3478
|
-
|
|
3479
3481
|
/* Build DTables */
|
|
3480
3482
|
{ size_t const llhSize = ZSTDv07_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
|
|
3481
3483
|
if (ZSTDv07_isError(llhSize)) return ERROR(corruption_detected);
|
|
@@ -3631,7 +3633,7 @@ size_t ZSTDv07_execSequence(BYTE* op,
|
|
|
3631
3633
|
if (sequence.offset < 8) {
|
|
3632
3634
|
/* close range match, overlap */
|
|
3633
3635
|
static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
|
|
3634
|
-
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /*
|
|
3636
|
+
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
|
|
3635
3637
|
int const sub2 = dec64table[sequence.offset];
|
|
3636
3638
|
op[0] = match[0];
|
|
3637
3639
|
op[1] = match[1];
|
|
@@ -3712,10 +3714,12 @@ static size_t ZSTDv07_decompressSequences(
|
|
|
3712
3714
|
|
|
3713
3715
|
/* last literal segment */
|
|
3714
3716
|
{ size_t const lastLLSize = litEnd - litPtr;
|
|
3715
|
-
|
|
3717
|
+
/* if (litPtr > litEnd) return ERROR(corruption_detected); */ /* too many literals already used */
|
|
3716
3718
|
if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall);
|
|
3717
|
-
|
|
3718
|
-
|
|
3719
|
+
if (lastLLSize > 0) {
|
|
3720
|
+
memcpy(op, litPtr, lastLLSize);
|
|
3721
|
+
op += lastLLSize;
|
|
3722
|
+
}
|
|
3719
3723
|
}
|
|
3720
3724
|
|
|
3721
3725
|
return op-ostart;
|
|
@@ -3776,7 +3780,9 @@ ZSTDLIBv07_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockS
|
|
|
3776
3780
|
static size_t ZSTDv07_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length)
|
|
3777
3781
|
{
|
|
3778
3782
|
if (length > dstCapacity) return ERROR(dstSize_tooSmall);
|
|
3779
|
-
|
|
3783
|
+
if (length > 0) {
|
|
3784
|
+
memset(dst, byte, length);
|
|
3785
|
+
}
|
|
3780
3786
|
return length;
|
|
3781
3787
|
}
|
|
3782
3788
|
|
|
@@ -3895,19 +3901,40 @@ size_t ZSTDv07_decompress(void* dst, size_t dstCapacity, const void* src, size_t
|
|
|
3895
3901
|
#endif
|
|
3896
3902
|
}
|
|
3897
3903
|
|
|
3898
|
-
|
|
3904
|
+
/* ZSTD_errorFrameSizeInfoLegacy() :
|
|
3905
|
+
assumes `cSize` and `dBound` are _not_ NULL */
|
|
3906
|
+
static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
|
|
3907
|
+
{
|
|
3908
|
+
*cSize = ret;
|
|
3909
|
+
*dBound = ZSTD_CONTENTSIZE_ERROR;
|
|
3910
|
+
}
|
|
3911
|
+
|
|
3912
|
+
void ZSTDv07_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
|
|
3899
3913
|
{
|
|
3900
3914
|
const BYTE* ip = (const BYTE*)src;
|
|
3901
3915
|
size_t remainingSize = srcSize;
|
|
3916
|
+
size_t nbBlocks = 0;
|
|
3902
3917
|
|
|
3903
3918
|
/* check */
|
|
3904
|
-
if (srcSize < ZSTDv07_frameHeaderSize_min+ZSTDv07_blockHeaderSize)
|
|
3919
|
+
if (srcSize < ZSTDv07_frameHeaderSize_min+ZSTDv07_blockHeaderSize) {
|
|
3920
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
|
3921
|
+
return;
|
|
3922
|
+
}
|
|
3905
3923
|
|
|
3906
3924
|
/* Frame Header */
|
|
3907
|
-
{ size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src,
|
|
3908
|
-
if (ZSTDv07_isError(frameHeaderSize))
|
|
3909
|
-
|
|
3910
|
-
|
|
3925
|
+
{ size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src, srcSize);
|
|
3926
|
+
if (ZSTDv07_isError(frameHeaderSize)) {
|
|
3927
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, frameHeaderSize);
|
|
3928
|
+
return;
|
|
3929
|
+
}
|
|
3930
|
+
if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) {
|
|
3931
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
|
|
3932
|
+
return;
|
|
3933
|
+
}
|
|
3934
|
+
if (srcSize < frameHeaderSize+ZSTDv07_blockHeaderSize) {
|
|
3935
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
|
3936
|
+
return;
|
|
3937
|
+
}
|
|
3911
3938
|
ip += frameHeaderSize; remainingSize -= frameHeaderSize;
|
|
3912
3939
|
}
|
|
3913
3940
|
|
|
@@ -3915,20 +3942,28 @@ size_t ZSTDv07_findFrameCompressedSize(const void* src, size_t srcSize)
|
|
|
3915
3942
|
while (1) {
|
|
3916
3943
|
blockProperties_t blockProperties;
|
|
3917
3944
|
size_t const cBlockSize = ZSTDv07_getcBlockSize(ip, remainingSize, &blockProperties);
|
|
3918
|
-
if (ZSTDv07_isError(cBlockSize))
|
|
3945
|
+
if (ZSTDv07_isError(cBlockSize)) {
|
|
3946
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
|
|
3947
|
+
return;
|
|
3948
|
+
}
|
|
3919
3949
|
|
|
3920
3950
|
ip += ZSTDv07_blockHeaderSize;
|
|
3921
3951
|
remainingSize -= ZSTDv07_blockHeaderSize;
|
|
3922
3952
|
|
|
3923
3953
|
if (blockProperties.blockType == bt_end) break;
|
|
3924
3954
|
|
|
3925
|
-
if (cBlockSize > remainingSize)
|
|
3955
|
+
if (cBlockSize > remainingSize) {
|
|
3956
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
|
3957
|
+
return;
|
|
3958
|
+
}
|
|
3926
3959
|
|
|
3927
3960
|
ip += cBlockSize;
|
|
3928
3961
|
remainingSize -= cBlockSize;
|
|
3962
|
+
nbBlocks++;
|
|
3929
3963
|
}
|
|
3930
3964
|
|
|
3931
|
-
|
|
3965
|
+
*cSize = ip - (const BYTE*)src;
|
|
3966
|
+
*dBound = nbBlocks * ZSTDv07_BLOCKSIZE_ABSOLUTEMAX;
|
|
3932
3967
|
}
|
|
3933
3968
|
|
|
3934
3969
|
/*_******************************
|
|
@@ -4349,7 +4384,9 @@ size_t ZBUFFv07_decompressInit(ZBUFFv07_DCtx* zbd)
|
|
|
4349
4384
|
MEM_STATIC size_t ZBUFFv07_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
|
4350
4385
|
{
|
|
4351
4386
|
size_t const length = MIN(dstCapacity, srcSize);
|
|
4352
|
-
|
|
4387
|
+
if (length > 0) {
|
|
4388
|
+
memcpy(dst, src, length);
|
|
4389
|
+
}
|
|
4353
4390
|
return length;
|
|
4354
4391
|
}
|
|
4355
4392
|
|