extzstd 0.2 → 0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/HISTORY.ja.md +13 -0
- data/README.md +17 -14
- data/contrib/zstd/{NEWS → CHANGELOG} +115 -2
- data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
- data/contrib/zstd/Makefile +99 -53
- data/contrib/zstd/README.md +59 -39
- data/contrib/zstd/TESTING.md +1 -1
- data/contrib/zstd/appveyor.yml +17 -6
- data/contrib/zstd/lib/BUCK +29 -2
- data/contrib/zstd/lib/Makefile +118 -21
- data/contrib/zstd/lib/README.md +84 -44
- data/contrib/zstd/lib/common/bitstream.h +17 -33
- data/contrib/zstd/lib/common/compiler.h +62 -8
- data/contrib/zstd/lib/common/cpu.h +215 -0
- data/contrib/zstd/lib/common/debug.c +44 -0
- data/contrib/zstd/lib/common/debug.h +134 -0
- data/contrib/zstd/lib/common/entropy_common.c +16 -1
- data/contrib/zstd/lib/common/error_private.c +7 -0
- data/contrib/zstd/lib/common/fse.h +48 -44
- data/contrib/zstd/lib/common/fse_decompress.c +3 -3
- data/contrib/zstd/lib/common/huf.h +169 -113
- data/contrib/zstd/lib/common/mem.h +20 -2
- data/contrib/zstd/lib/common/pool.c +135 -49
- data/contrib/zstd/lib/common/pool.h +40 -21
- data/contrib/zstd/lib/common/threading.c +2 -2
- data/contrib/zstd/lib/common/threading.h +12 -12
- data/contrib/zstd/lib/common/xxhash.c +3 -2
- data/contrib/zstd/lib/common/zstd_common.c +3 -6
- data/contrib/zstd/lib/common/zstd_errors.h +17 -7
- data/contrib/zstd/lib/common/zstd_internal.h +76 -48
- data/contrib/zstd/lib/compress/fse_compress.c +89 -209
- data/contrib/zstd/lib/compress/hist.c +203 -0
- data/contrib/zstd/lib/compress/hist.h +95 -0
- data/contrib/zstd/lib/compress/huf_compress.c +188 -80
- data/contrib/zstd/lib/compress/zstd_compress.c +2500 -1203
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +463 -62
- data/contrib/zstd/lib/compress/zstd_double_fast.c +321 -131
- data/contrib/zstd/lib/compress/zstd_double_fast.h +13 -4
- data/contrib/zstd/lib/compress/zstd_fast.c +335 -108
- data/contrib/zstd/lib/compress/zstd_fast.h +12 -6
- data/contrib/zstd/lib/compress/zstd_lazy.c +654 -313
- data/contrib/zstd/lib/compress/zstd_lazy.h +44 -16
- data/contrib/zstd/lib/compress/zstd_ldm.c +310 -420
- data/contrib/zstd/lib/compress/zstd_ldm.h +63 -26
- data/contrib/zstd/lib/compress/zstd_opt.c +773 -325
- data/contrib/zstd/lib/compress/zstd_opt.h +31 -5
- data/contrib/zstd/lib/compress/zstdmt_compress.c +1468 -518
- data/contrib/zstd/lib/compress/zstdmt_compress.h +96 -45
- data/contrib/zstd/lib/decompress/huf_decompress.c +518 -282
- data/contrib/zstd/lib/decompress/zstd_ddict.c +240 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
- data/contrib/zstd/lib/decompress/zstd_decompress.c +613 -1513
- data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1311 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.h +59 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +175 -0
- data/contrib/zstd/lib/dictBuilder/cover.c +194 -113
- data/contrib/zstd/lib/dictBuilder/cover.h +112 -0
- data/contrib/zstd/lib/dictBuilder/divsufsort.c +3 -3
- data/contrib/zstd/lib/dictBuilder/fastcover.c +740 -0
- data/contrib/zstd/lib/dictBuilder/zdict.c +142 -106
- data/contrib/zstd/lib/dictBuilder/zdict.h +115 -49
- data/contrib/zstd/lib/legacy/zstd_legacy.h +44 -12
- data/contrib/zstd/lib/legacy/zstd_v01.c +41 -10
- data/contrib/zstd/lib/legacy/zstd_v01.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v02.c +37 -12
- data/contrib/zstd/lib/legacy/zstd_v02.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v03.c +38 -12
- data/contrib/zstd/lib/legacy/zstd_v03.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v04.c +55 -174
- data/contrib/zstd/lib/legacy/zstd_v04.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v05.c +59 -31
- data/contrib/zstd/lib/legacy/zstd_v05.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v06.c +48 -20
- data/contrib/zstd/lib/legacy/zstd_v06.h +10 -5
- data/contrib/zstd/lib/legacy/zstd_v07.c +62 -29
- data/contrib/zstd/lib/legacy/zstd_v07.h +10 -5
- data/contrib/zstd/lib/zstd.h +1346 -832
- data/ext/extzstd.c +27 -19
- data/ext/extzstd_stream.c +20 -4
- data/ext/zstd_compress.c +1 -0
- data/ext/zstd_decompress.c +4 -0
- data/ext/zstd_dictbuilder.c +4 -0
- data/ext/zstd_dictbuilder_fastcover.c +5 -0
- data/lib/extzstd.rb +52 -220
- data/lib/extzstd/version.rb +1 -1
- metadata +21 -7
- data/contrib/zstd/circle.yml +0 -63
|
@@ -35,13 +35,18 @@ ZSTDv04_decompress() : decompress ZSTD frames compliant with v0.4.x format
|
|
|
35
35
|
size_t ZSTDv04_decompress( void* dst, size_t maxOriginalSize,
|
|
36
36
|
const void* src, size_t compressedSize);
|
|
37
37
|
|
|
38
|
-
/**
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
38
|
+
/**
|
|
39
|
+
ZSTDv04_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.4.x format
|
|
40
|
+
srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
|
41
|
+
cSize (output parameter) : the number of bytes that would be read to decompress this frame
|
|
42
|
+
or an error code if it fails (which can be tested using ZSTDv01_isError())
|
|
43
|
+
dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
|
|
44
|
+
or ZSTD_CONTENTSIZE_ERROR if an error occurs
|
|
45
|
+
|
|
46
|
+
note : assumes `cSize` and `dBound` are _not_ NULL.
|
|
47
|
+
*/
|
|
48
|
+
void ZSTDv04_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
|
|
49
|
+
size_t* cSize, unsigned long long* dBound);
|
|
45
50
|
|
|
46
51
|
/**
|
|
47
52
|
ZSTDv04_isError() : tells if the result of ZSTDv04_decompress() is an error
|
|
@@ -491,6 +491,8 @@ static const size_t ZSTDv05_frameHeaderSize_min = 5;
|
|
|
491
491
|
|
|
492
492
|
#define WILDCOPY_OVERLENGTH 8
|
|
493
493
|
|
|
494
|
+
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
|
|
495
|
+
|
|
494
496
|
typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
|
|
495
497
|
|
|
496
498
|
|
|
@@ -836,7 +838,7 @@ MEM_STATIC void BITv05_skipBits(BITv05_DStream_t* bitD, U32 nbBits)
|
|
|
836
838
|
bitD->bitsConsumed += nbBits;
|
|
837
839
|
}
|
|
838
840
|
|
|
839
|
-
MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD,
|
|
841
|
+
MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, unsigned nbBits)
|
|
840
842
|
{
|
|
841
843
|
size_t value = BITv05_lookBits(bitD, nbBits);
|
|
842
844
|
BITv05_skipBits(bitD, nbBits);
|
|
@@ -845,7 +847,7 @@ MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, U32 nbBits)
|
|
|
845
847
|
|
|
846
848
|
/*!BITv05_readBitsFast :
|
|
847
849
|
* unsafe version; only works only if nbBits >= 1 */
|
|
848
|
-
MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD,
|
|
850
|
+
MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, unsigned nbBits)
|
|
849
851
|
{
|
|
850
852
|
size_t value = BITv05_lookBitsFast(bitD, nbBits);
|
|
851
853
|
BITv05_skipBits(bitD, nbBits);
|
|
@@ -1162,7 +1164,7 @@ MEM_STATIC unsigned FSEv05_endOfDState(const FSEv05_DState_t* DStatePtr)
|
|
|
1162
1164
|
/* **************************************************************
|
|
1163
1165
|
* Complex types
|
|
1164
1166
|
****************************************************************/
|
|
1165
|
-
typedef
|
|
1167
|
+
typedef unsigned DTable_max_t[FSEv05_DTABLE_SIZE_U32(FSEv05_MAX_TABLELOG)];
|
|
1166
1168
|
|
|
1167
1169
|
|
|
1168
1170
|
/* **************************************************************
|
|
@@ -1224,6 +1226,7 @@ size_t FSEv05_buildDTable(FSEv05_DTable* dt, const short* normalizedCounter, uns
|
|
|
1224
1226
|
if (tableLog > FSEv05_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
|
1225
1227
|
|
|
1226
1228
|
/* Init, lay down lowprob symbols */
|
|
1229
|
+
memset(tableDecode, 0, sizeof(FSEv05_FUNCTION_TYPE) * (maxSymbolValue+1) ); /* useless init, but keep static analyzer happy, and we don't need to performance optimize legacy decoders */
|
|
1227
1230
|
DTableH.tableLog = (U16)tableLog;
|
|
1228
1231
|
for (s=0; s<=maxSymbolValue; s++) {
|
|
1229
1232
|
if (normalizedCounter[s]==-1) {
|
|
@@ -2190,7 +2193,7 @@ static void HUFv05_fillDTableX4(HUFv05_DEltX4* DTable, const U32 targetLog,
|
|
|
2190
2193
|
}
|
|
2191
2194
|
}
|
|
2192
2195
|
|
|
2193
|
-
size_t HUFv05_readDTableX4 (
|
|
2196
|
+
size_t HUFv05_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize)
|
|
2194
2197
|
{
|
|
2195
2198
|
BYTE weightList[HUFv05_MAX_SYMBOL_VALUE + 1];
|
|
2196
2199
|
sortedSymbol_t sortedSymbol[HUFv05_MAX_SYMBOL_VALUE + 1];
|
|
@@ -2204,7 +2207,7 @@ size_t HUFv05_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
|
|
|
2204
2207
|
void* dtPtr = DTable;
|
|
2205
2208
|
HUFv05_DEltX4* const dt = ((HUFv05_DEltX4*)dtPtr) + 1;
|
|
2206
2209
|
|
|
2207
|
-
HUFv05_STATIC_ASSERT(sizeof(HUFv05_DEltX4) == sizeof(
|
|
2210
|
+
HUFv05_STATIC_ASSERT(sizeof(HUFv05_DEltX4) == sizeof(unsigned)); /* if compilation fails here, assertion is false */
|
|
2208
2211
|
if (memLog > HUFv05_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
|
2209
2212
|
//memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
|
|
2210
2213
|
|
|
@@ -2331,7 +2334,7 @@ static inline size_t HUFv05_decodeStreamX4(BYTE* p, BITv05_DStream_t* bitDPtr, B
|
|
|
2331
2334
|
size_t HUFv05_decompress1X4_usingDTable(
|
|
2332
2335
|
void* dst, size_t dstSize,
|
|
2333
2336
|
const void* cSrc, size_t cSrcSize,
|
|
2334
|
-
const
|
|
2337
|
+
const unsigned* DTable)
|
|
2335
2338
|
{
|
|
2336
2339
|
const BYTE* const istart = (const BYTE*) cSrc;
|
|
2337
2340
|
BYTE* const ostart = (BYTE*) dst;
|
|
@@ -2374,7 +2377,7 @@ size_t HUFv05_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t
|
|
|
2374
2377
|
size_t HUFv05_decompress4X4_usingDTable(
|
|
2375
2378
|
void* dst, size_t dstSize,
|
|
2376
2379
|
const void* cSrc, size_t cSrcSize,
|
|
2377
|
-
const
|
|
2380
|
+
const unsigned* DTable)
|
|
2378
2381
|
{
|
|
2379
2382
|
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
|
|
2380
2383
|
|
|
@@ -2658,6 +2661,7 @@ struct ZSTDv05_DCtx_s
|
|
|
2658
2661
|
BYTE headerBuffer[ZSTDv05_frameHeaderSize_max];
|
|
2659
2662
|
}; /* typedef'd to ZSTDv05_DCtx within "zstd_static.h" */
|
|
2660
2663
|
|
|
2664
|
+
size_t ZSTDv05_sizeofDCtx (void); /* Hidden declaration */
|
|
2661
2665
|
size_t ZSTDv05_sizeofDCtx (void) { return sizeof(ZSTDv05_DCtx); }
|
|
2662
2666
|
|
|
2663
2667
|
size_t ZSTDv05_decompressBegin(ZSTDv05_DCtx* dctx)
|
|
@@ -2822,7 +2826,7 @@ static size_t ZSTDv05_decodeFrameHeader_Part2(ZSTDv05_DCtx* zc, const void* src,
|
|
|
2822
2826
|
}
|
|
2823
2827
|
|
|
2824
2828
|
|
|
2825
|
-
size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
|
2829
|
+
static size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
|
2826
2830
|
{
|
|
2827
2831
|
const BYTE* const in = (const BYTE* const)src;
|
|
2828
2832
|
BYTE headerFlags;
|
|
@@ -2845,6 +2849,7 @@ size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t*
|
|
|
2845
2849
|
|
|
2846
2850
|
static size_t ZSTDv05_copyRawBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
|
2847
2851
|
{
|
|
2852
|
+
if (dst==NULL) return ERROR(dstSize_tooSmall);
|
|
2848
2853
|
if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
|
|
2849
2854
|
memcpy(dst, src, srcSize);
|
|
2850
2855
|
return srcSize;
|
|
@@ -2853,8 +2858,8 @@ static size_t ZSTDv05_copyRawBlock(void* dst, size_t maxDstSize, const void* src
|
|
|
2853
2858
|
|
|
2854
2859
|
/*! ZSTDv05_decodeLiteralsBlock() :
|
|
2855
2860
|
@return : nb of bytes read from src (< srcSize ) */
|
|
2856
|
-
size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
|
|
2857
|
-
|
|
2861
|
+
static size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
|
|
2862
|
+
const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
|
|
2858
2863
|
{
|
|
2859
2864
|
const BYTE* const istart = (const BYTE*) src;
|
|
2860
2865
|
|
|
@@ -2988,7 +2993,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
|
|
|
2988
2993
|
}
|
|
2989
2994
|
|
|
2990
2995
|
|
|
2991
|
-
size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
|
|
2996
|
+
static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
|
|
2992
2997
|
FSEv05_DTable* DTableLL, FSEv05_DTable* DTableML, FSEv05_DTable* DTableOffb,
|
|
2993
2998
|
const void* src, size_t srcSize, U32 flagStaticTable)
|
|
2994
2999
|
{
|
|
@@ -2996,7 +3001,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
|
|
|
2996
3001
|
const BYTE* ip = istart;
|
|
2997
3002
|
const BYTE* const iend = istart + srcSize;
|
|
2998
3003
|
U32 LLtype, Offtype, MLtype;
|
|
2999
|
-
|
|
3004
|
+
unsigned LLlog, Offlog, MLlog;
|
|
3000
3005
|
size_t dumpsLength;
|
|
3001
3006
|
|
|
3002
3007
|
/* check */
|
|
@@ -3054,7 +3059,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
|
|
|
3054
3059
|
break;
|
|
3055
3060
|
case FSEv05_ENCODING_DYNAMIC :
|
|
3056
3061
|
default : /* impossible */
|
|
3057
|
-
{
|
|
3062
|
+
{ unsigned max = MaxLL;
|
|
3058
3063
|
headerSize = FSEv05_readNCount(norm, &max, &LLlog, ip, iend-ip);
|
|
3059
3064
|
if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
|
|
3060
3065
|
if (LLlog > LLFSEv05Log) return ERROR(corruption_detected);
|
|
@@ -3078,7 +3083,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
|
|
|
3078
3083
|
break;
|
|
3079
3084
|
case FSEv05_ENCODING_DYNAMIC :
|
|
3080
3085
|
default : /* impossible */
|
|
3081
|
-
{
|
|
3086
|
+
{ unsigned max = MaxOff;
|
|
3082
3087
|
headerSize = FSEv05_readNCount(norm, &max, &Offlog, ip, iend-ip);
|
|
3083
3088
|
if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
|
|
3084
3089
|
if (Offlog > OffFSEv05Log) return ERROR(corruption_detected);
|
|
@@ -3102,7 +3107,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
|
|
|
3102
3107
|
break;
|
|
3103
3108
|
case FSEv05_ENCODING_DYNAMIC :
|
|
3104
3109
|
default : /* impossible */
|
|
3105
|
-
{
|
|
3110
|
+
{ unsigned max = MaxML;
|
|
3106
3111
|
headerSize = FSEv05_readNCount(norm, &max, &MLlog, ip, iend-ip);
|
|
3107
3112
|
if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
|
|
3108
3113
|
if (MLlog > MLFSEv05Log) return ERROR(corruption_detected);
|
|
@@ -3214,7 +3219,7 @@ static size_t ZSTDv05_execSequence(BYTE* op,
|
|
|
3214
3219
|
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
|
|
3215
3220
|
{
|
|
3216
3221
|
static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
|
|
3217
|
-
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /*
|
|
3222
|
+
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
|
|
3218
3223
|
BYTE* const oLitEnd = op + sequence.litLength;
|
|
3219
3224
|
const size_t sequenceLength = sequence.litLength + sequence.matchLength;
|
|
3220
3225
|
BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
|
|
@@ -3297,14 +3302,14 @@ static size_t ZSTDv05_decompressSequences(
|
|
|
3297
3302
|
BYTE* const ostart = (BYTE* const)dst;
|
|
3298
3303
|
BYTE* op = ostart;
|
|
3299
3304
|
BYTE* const oend = ostart + maxDstSize;
|
|
3300
|
-
size_t errorCode, dumpsLength;
|
|
3305
|
+
size_t errorCode, dumpsLength=0;
|
|
3301
3306
|
const BYTE* litPtr = dctx->litPtr;
|
|
3302
3307
|
const BYTE* const litEnd = litPtr + dctx->litSize;
|
|
3303
|
-
int nbSeq;
|
|
3304
|
-
const BYTE* dumps;
|
|
3305
|
-
|
|
3306
|
-
|
|
3307
|
-
|
|
3308
|
+
int nbSeq=0;
|
|
3309
|
+
const BYTE* dumps = NULL;
|
|
3310
|
+
unsigned* DTableLL = dctx->LLTable;
|
|
3311
|
+
unsigned* DTableML = dctx->MLTable;
|
|
3312
|
+
unsigned* DTableOffb = dctx->OffTable;
|
|
3308
3313
|
const BYTE* const base = (const BYTE*) (dctx->base);
|
|
3309
3314
|
const BYTE* const vBase = (const BYTE*) (dctx->vBase);
|
|
3310
3315
|
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
|
|
@@ -3410,10 +3415,10 @@ static size_t ZSTDv05_decompress_continueDCtx(ZSTDv05_DCtx* dctx,
|
|
|
3410
3415
|
BYTE* const oend = ostart + maxDstSize;
|
|
3411
3416
|
size_t remainingSize = srcSize;
|
|
3412
3417
|
blockProperties_t blockProperties;
|
|
3418
|
+
memset(&blockProperties, 0, sizeof(blockProperties));
|
|
3413
3419
|
|
|
3414
3420
|
/* Frame Header */
|
|
3415
|
-
{
|
|
3416
|
-
size_t frameHeaderSize;
|
|
3421
|
+
{ size_t frameHeaderSize;
|
|
3417
3422
|
if (srcSize < ZSTDv05_frameHeaderSize_min+ZSTDv05_blockHeaderSize) return ERROR(srcSize_wrong);
|
|
3418
3423
|
frameHeaderSize = ZSTDv05_decodeFrameHeader_Part1(dctx, src, ZSTDv05_frameHeaderSize_min);
|
|
3419
3424
|
if (ZSTDv05_isError(frameHeaderSize)) return frameHeaderSize;
|
|
@@ -3505,34 +3510,57 @@ size_t ZSTDv05_decompress(void* dst, size_t maxDstSize, const void* src, size_t
|
|
|
3505
3510
|
#endif
|
|
3506
3511
|
}
|
|
3507
3512
|
|
|
3508
|
-
|
|
3513
|
+
/* ZSTD_errorFrameSizeInfoLegacy() :
|
|
3514
|
+
assumes `cSize` and `dBound` are _not_ NULL */
|
|
3515
|
+
static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
|
|
3516
|
+
{
|
|
3517
|
+
*cSize = ret;
|
|
3518
|
+
*dBound = ZSTD_CONTENTSIZE_ERROR;
|
|
3519
|
+
}
|
|
3520
|
+
|
|
3521
|
+
void ZSTDv05_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
|
|
3509
3522
|
{
|
|
3510
3523
|
const BYTE* ip = (const BYTE*)src;
|
|
3511
3524
|
size_t remainingSize = srcSize;
|
|
3525
|
+
size_t nbBlocks = 0;
|
|
3512
3526
|
blockProperties_t blockProperties;
|
|
3513
3527
|
|
|
3514
3528
|
/* Frame Header */
|
|
3515
|
-
if (srcSize < ZSTDv05_frameHeaderSize_min)
|
|
3516
|
-
|
|
3529
|
+
if (srcSize < ZSTDv05_frameHeaderSize_min) {
|
|
3530
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
|
3531
|
+
return;
|
|
3532
|
+
}
|
|
3533
|
+
if (MEM_readLE32(src) != ZSTDv05_MAGICNUMBER) {
|
|
3534
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
|
|
3535
|
+
return;
|
|
3536
|
+
}
|
|
3517
3537
|
ip += ZSTDv05_frameHeaderSize_min; remainingSize -= ZSTDv05_frameHeaderSize_min;
|
|
3518
3538
|
|
|
3519
3539
|
/* Loop on each block */
|
|
3520
3540
|
while (1)
|
|
3521
3541
|
{
|
|
3522
3542
|
size_t cBlockSize = ZSTDv05_getcBlockSize(ip, remainingSize, &blockProperties);
|
|
3523
|
-
if (ZSTDv05_isError(cBlockSize))
|
|
3543
|
+
if (ZSTDv05_isError(cBlockSize)) {
|
|
3544
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
|
|
3545
|
+
return;
|
|
3546
|
+
}
|
|
3524
3547
|
|
|
3525
3548
|
ip += ZSTDv05_blockHeaderSize;
|
|
3526
3549
|
remainingSize -= ZSTDv05_blockHeaderSize;
|
|
3527
|
-
if (cBlockSize > remainingSize)
|
|
3550
|
+
if (cBlockSize > remainingSize) {
|
|
3551
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
|
3552
|
+
return;
|
|
3553
|
+
}
|
|
3528
3554
|
|
|
3529
3555
|
if (cBlockSize == 0) break; /* bt_end */
|
|
3530
3556
|
|
|
3531
3557
|
ip += cBlockSize;
|
|
3532
3558
|
remainingSize -= cBlockSize;
|
|
3559
|
+
nbBlocks++;
|
|
3533
3560
|
}
|
|
3534
3561
|
|
|
3535
|
-
|
|
3562
|
+
*cSize = ip - (const BYTE*)src;
|
|
3563
|
+
*dBound = nbBlocks * BLOCKSIZE;
|
|
3536
3564
|
}
|
|
3537
3565
|
|
|
3538
3566
|
/* ******************************
|
|
@@ -3630,7 +3658,7 @@ static size_t ZSTDv05_loadEntropy(ZSTDv05_DCtx* dctx, const void* dict, size_t d
|
|
|
3630
3658
|
{
|
|
3631
3659
|
size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, errorCode, litlengthHeaderSize;
|
|
3632
3660
|
short offcodeNCount[MaxOff+1];
|
|
3633
|
-
|
|
3661
|
+
unsigned offcodeMaxValue=MaxOff, offcodeLog;
|
|
3634
3662
|
short matchlengthNCount[MaxML+1];
|
|
3635
3663
|
unsigned matchlengthMaxValue = MaxML, matchlengthLog;
|
|
3636
3664
|
short litlengthNCount[MaxLL+1];
|
|
@@ -33,13 +33,18 @@ extern "C" {
|
|
|
33
33
|
size_t ZSTDv05_decompress( void* dst, size_t dstCapacity,
|
|
34
34
|
const void* src, size_t compressedSize);
|
|
35
35
|
|
|
36
|
-
/**
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
36
|
+
/**
|
|
37
|
+
ZSTDv05_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.5.x format
|
|
38
|
+
srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
|
39
|
+
cSize (output parameter) : the number of bytes that would be read to decompress this frame
|
|
40
|
+
or an error code if it fails (which can be tested using ZSTDv01_isError())
|
|
41
|
+
dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
|
|
42
|
+
or ZSTD_CONTENTSIZE_ERROR if an error occurs
|
|
43
|
+
|
|
44
|
+
note : assumes `cSize` and `dBound` are _not_ NULL.
|
|
45
|
+
*/
|
|
46
|
+
void ZSTDv05_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
|
|
47
|
+
size_t* cSize, unsigned long long* dBound);
|
|
43
48
|
|
|
44
49
|
/* *************************************
|
|
45
50
|
* Helper functions
|
|
@@ -189,7 +189,7 @@ MEM_STATIC U32 MEM_swap32(U32 in)
|
|
|
189
189
|
{
|
|
190
190
|
#if defined(_MSC_VER) /* Visual Studio */
|
|
191
191
|
return _byteswap_ulong(in);
|
|
192
|
-
#elif defined (__GNUC__)
|
|
192
|
+
#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
|
|
193
193
|
return __builtin_bswap32(in);
|
|
194
194
|
#else
|
|
195
195
|
return ((in << 24) & 0xff000000 ) |
|
|
@@ -203,7 +203,7 @@ MEM_STATIC U64 MEM_swap64(U64 in)
|
|
|
203
203
|
{
|
|
204
204
|
#if defined(_MSC_VER) /* Visual Studio */
|
|
205
205
|
return _byteswap_uint64(in);
|
|
206
|
-
#elif defined (__GNUC__)
|
|
206
|
+
#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
|
|
207
207
|
return __builtin_bswap64(in);
|
|
208
208
|
#else
|
|
209
209
|
return ((in << 56) & 0xff00000000000000ULL) |
|
|
@@ -506,6 +506,8 @@ typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
|
|
|
506
506
|
#define FSEv06_ENCODING_STATIC 2
|
|
507
507
|
#define FSEv06_ENCODING_DYNAMIC 3
|
|
508
508
|
|
|
509
|
+
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
|
|
510
|
+
|
|
509
511
|
static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
510
512
|
1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12,
|
|
511
513
|
13,14,15,16 };
|
|
@@ -1250,9 +1252,7 @@ const char* FSEv06_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
|
|
1250
1252
|
/* **************************************************************
|
|
1251
1253
|
* HUF Error Management
|
|
1252
1254
|
****************************************************************/
|
|
1253
|
-
unsigned HUFv06_isError(size_t code) { return ERR_isError(code); }
|
|
1254
|
-
|
|
1255
|
-
const char* HUFv06_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
|
1255
|
+
static unsigned HUFv06_isError(size_t code) { return ERR_isError(code); }
|
|
1256
1256
|
|
|
1257
1257
|
|
|
1258
1258
|
/*-**************************************************************
|
|
@@ -2823,7 +2823,8 @@ struct ZSTDv06_DCtx_s
|
|
|
2823
2823
|
BYTE headerBuffer[ZSTDv06_FRAMEHEADERSIZE_MAX];
|
|
2824
2824
|
}; /* typedef'd to ZSTDv06_DCtx within "zstd_static.h" */
|
|
2825
2825
|
|
|
2826
|
-
size_t ZSTDv06_sizeofDCtx (void)
|
|
2826
|
+
size_t ZSTDv06_sizeofDCtx (void); /* Hidden declaration */
|
|
2827
|
+
size_t ZSTDv06_sizeofDCtx (void) { return sizeof(ZSTDv06_DCtx); }
|
|
2827
2828
|
|
|
2828
2829
|
size_t ZSTDv06_decompressBegin(ZSTDv06_DCtx* dctx)
|
|
2829
2830
|
{
|
|
@@ -3022,7 +3023,7 @@ typedef struct
|
|
|
3022
3023
|
|
|
3023
3024
|
/*! ZSTDv06_getcBlockSize() :
|
|
3024
3025
|
* Provides the size of compressed block from block header `src` */
|
|
3025
|
-
size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
|
3026
|
+
static size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
|
3026
3027
|
{
|
|
3027
3028
|
const BYTE* const in = (const BYTE* const)src;
|
|
3028
3029
|
U32 cSize;
|
|
@@ -3041,6 +3042,7 @@ size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t*
|
|
|
3041
3042
|
|
|
3042
3043
|
static size_t ZSTDv06_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
|
3043
3044
|
{
|
|
3045
|
+
if (dst==NULL) return ERROR(dstSize_tooSmall);
|
|
3044
3046
|
if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
|
|
3045
3047
|
memcpy(dst, src, srcSize);
|
|
3046
3048
|
return srcSize;
|
|
@@ -3049,7 +3051,7 @@ static size_t ZSTDv06_copyRawBlock(void* dst, size_t dstCapacity, const void* sr
|
|
|
3049
3051
|
|
|
3050
3052
|
/*! ZSTDv06_decodeLiteralsBlock() :
|
|
3051
3053
|
@return : nb of bytes read from src (< srcSize ) */
|
|
3052
|
-
size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
|
|
3054
|
+
static size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
|
|
3053
3055
|
const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
|
|
3054
3056
|
{
|
|
3055
3057
|
const BYTE* const istart = (const BYTE*) src;
|
|
@@ -3183,7 +3185,7 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
|
|
|
3183
3185
|
@return : nb bytes read from src,
|
|
3184
3186
|
or an error code if it fails, testable with ZSTDv06_isError()
|
|
3185
3187
|
*/
|
|
3186
|
-
size_t ZSTDv06_buildSeqTable(FSEv06_DTable* DTable, U32 type, U32 max, U32 maxLog,
|
|
3188
|
+
static size_t ZSTDv06_buildSeqTable(FSEv06_DTable* DTable, U32 type, U32 max, U32 maxLog,
|
|
3187
3189
|
const void* src, size_t srcSize,
|
|
3188
3190
|
const S16* defaultNorm, U32 defaultLog, U32 flagRepeatTable)
|
|
3189
3191
|
{
|
|
@@ -3213,7 +3215,7 @@ size_t ZSTDv06_buildSeqTable(FSEv06_DTable* DTable, U32 type, U32 max, U32 maxLo
|
|
|
3213
3215
|
}
|
|
3214
3216
|
|
|
3215
3217
|
|
|
3216
|
-
size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
|
|
3218
|
+
static size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
|
|
3217
3219
|
FSEv06_DTable* DTableLL, FSEv06_DTable* DTableML, FSEv06_DTable* DTableOffb, U32 flagRepeatTable,
|
|
3218
3220
|
const void* src, size_t srcSize)
|
|
3219
3221
|
{
|
|
@@ -3358,7 +3360,7 @@ static void ZSTDv06_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
|
3358
3360
|
}
|
|
3359
3361
|
|
|
3360
3362
|
|
|
3361
|
-
size_t ZSTDv06_execSequence(BYTE* op,
|
|
3363
|
+
static size_t ZSTDv06_execSequence(BYTE* op,
|
|
3362
3364
|
BYTE* const oend, seq_t sequence,
|
|
3363
3365
|
const BYTE** litPtr, const BYTE* const litLimit,
|
|
3364
3366
|
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
|
|
@@ -3406,7 +3408,7 @@ size_t ZSTDv06_execSequence(BYTE* op,
|
|
|
3406
3408
|
if (sequence.offset < 8) {
|
|
3407
3409
|
/* close range match, overlap */
|
|
3408
3410
|
static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
|
|
3409
|
-
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /*
|
|
3411
|
+
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
|
|
3410
3412
|
int const sub2 = dec64table[sequence.offset];
|
|
3411
3413
|
op[0] = match[0];
|
|
3412
3414
|
op[1] = match[1];
|
|
@@ -3654,36 +3656,62 @@ size_t ZSTDv06_decompress(void* dst, size_t dstCapacity, const void* src, size_t
|
|
|
3654
3656
|
#endif
|
|
3655
3657
|
}
|
|
3656
3658
|
|
|
3657
|
-
|
|
3659
|
+
/* ZSTD_errorFrameSizeInfoLegacy() :
|
|
3660
|
+
assumes `cSize` and `dBound` are _not_ NULL */
|
|
3661
|
+
static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
|
|
3662
|
+
{
|
|
3663
|
+
*cSize = ret;
|
|
3664
|
+
*dBound = ZSTD_CONTENTSIZE_ERROR;
|
|
3665
|
+
}
|
|
3666
|
+
|
|
3667
|
+
void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
|
|
3658
3668
|
{
|
|
3659
3669
|
const BYTE* ip = (const BYTE*)src;
|
|
3660
3670
|
size_t remainingSize = srcSize;
|
|
3671
|
+
size_t nbBlocks = 0;
|
|
3661
3672
|
blockProperties_t blockProperties = { bt_compressed, 0 };
|
|
3662
3673
|
|
|
3663
3674
|
/* Frame Header */
|
|
3664
3675
|
{ size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src, ZSTDv06_frameHeaderSize_min);
|
|
3665
|
-
if (ZSTDv06_isError(frameHeaderSize))
|
|
3666
|
-
|
|
3667
|
-
|
|
3676
|
+
if (ZSTDv06_isError(frameHeaderSize)) {
|
|
3677
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, frameHeaderSize);
|
|
3678
|
+
return;
|
|
3679
|
+
}
|
|
3680
|
+
if (MEM_readLE32(src) != ZSTDv06_MAGICNUMBER) {
|
|
3681
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
|
|
3682
|
+
return;
|
|
3683
|
+
}
|
|
3684
|
+
if (srcSize < frameHeaderSize+ZSTDv06_blockHeaderSize) {
|
|
3685
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
|
3686
|
+
return;
|
|
3687
|
+
}
|
|
3668
3688
|
ip += frameHeaderSize; remainingSize -= frameHeaderSize;
|
|
3669
3689
|
}
|
|
3670
3690
|
|
|
3671
3691
|
/* Loop on each block */
|
|
3672
3692
|
while (1) {
|
|
3673
3693
|
size_t const cBlockSize = ZSTDv06_getcBlockSize(ip, remainingSize, &blockProperties);
|
|
3674
|
-
if (ZSTDv06_isError(cBlockSize))
|
|
3694
|
+
if (ZSTDv06_isError(cBlockSize)) {
|
|
3695
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
|
|
3696
|
+
return;
|
|
3697
|
+
}
|
|
3675
3698
|
|
|
3676
3699
|
ip += ZSTDv06_blockHeaderSize;
|
|
3677
3700
|
remainingSize -= ZSTDv06_blockHeaderSize;
|
|
3678
|
-
if (cBlockSize > remainingSize)
|
|
3701
|
+
if (cBlockSize > remainingSize) {
|
|
3702
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
|
3703
|
+
return;
|
|
3704
|
+
}
|
|
3679
3705
|
|
|
3680
3706
|
if (cBlockSize == 0) break; /* bt_end */
|
|
3681
3707
|
|
|
3682
3708
|
ip += cBlockSize;
|
|
3683
3709
|
remainingSize -= cBlockSize;
|
|
3710
|
+
nbBlocks++;
|
|
3684
3711
|
}
|
|
3685
3712
|
|
|
3686
|
-
|
|
3713
|
+
*cSize = ip - (const BYTE*)src;
|
|
3714
|
+
*dBound = nbBlocks * ZSTDv06_BLOCKSIZE_MAX;
|
|
3687
3715
|
}
|
|
3688
3716
|
|
|
3689
3717
|
/*_******************************
|
|
@@ -4006,7 +4034,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
|
|
|
4006
4034
|
if (ZSTDv06_isError(hSize)) return hSize;
|
|
4007
4035
|
if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
|
|
4008
4036
|
memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
|
|
4009
|
-
zbd->lhSize += iend-ip;
|
|
4037
|
+
zbd->lhSize += iend-ip;
|
|
4010
4038
|
*dstCapacityPtr = 0;
|
|
4011
4039
|
return (hSize - zbd->lhSize) + ZSTDv06_blockHeaderSize; /* remaining header bytes + next block header */
|
|
4012
4040
|
}
|