extzstd 0.2 → 0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/HISTORY.ja.md +13 -0
- data/README.md +17 -14
- data/contrib/zstd/{NEWS → CHANGELOG} +115 -2
- data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
- data/contrib/zstd/Makefile +99 -53
- data/contrib/zstd/README.md +59 -39
- data/contrib/zstd/TESTING.md +1 -1
- data/contrib/zstd/appveyor.yml +17 -6
- data/contrib/zstd/lib/BUCK +29 -2
- data/contrib/zstd/lib/Makefile +118 -21
- data/contrib/zstd/lib/README.md +84 -44
- data/contrib/zstd/lib/common/bitstream.h +17 -33
- data/contrib/zstd/lib/common/compiler.h +62 -8
- data/contrib/zstd/lib/common/cpu.h +215 -0
- data/contrib/zstd/lib/common/debug.c +44 -0
- data/contrib/zstd/lib/common/debug.h +134 -0
- data/contrib/zstd/lib/common/entropy_common.c +16 -1
- data/contrib/zstd/lib/common/error_private.c +7 -0
- data/contrib/zstd/lib/common/fse.h +48 -44
- data/contrib/zstd/lib/common/fse_decompress.c +3 -3
- data/contrib/zstd/lib/common/huf.h +169 -113
- data/contrib/zstd/lib/common/mem.h +20 -2
- data/contrib/zstd/lib/common/pool.c +135 -49
- data/contrib/zstd/lib/common/pool.h +40 -21
- data/contrib/zstd/lib/common/threading.c +2 -2
- data/contrib/zstd/lib/common/threading.h +12 -12
- data/contrib/zstd/lib/common/xxhash.c +3 -2
- data/contrib/zstd/lib/common/zstd_common.c +3 -6
- data/contrib/zstd/lib/common/zstd_errors.h +17 -7
- data/contrib/zstd/lib/common/zstd_internal.h +76 -48
- data/contrib/zstd/lib/compress/fse_compress.c +89 -209
- data/contrib/zstd/lib/compress/hist.c +203 -0
- data/contrib/zstd/lib/compress/hist.h +95 -0
- data/contrib/zstd/lib/compress/huf_compress.c +188 -80
- data/contrib/zstd/lib/compress/zstd_compress.c +2500 -1203
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +463 -62
- data/contrib/zstd/lib/compress/zstd_double_fast.c +321 -131
- data/contrib/zstd/lib/compress/zstd_double_fast.h +13 -4
- data/contrib/zstd/lib/compress/zstd_fast.c +335 -108
- data/contrib/zstd/lib/compress/zstd_fast.h +12 -6
- data/contrib/zstd/lib/compress/zstd_lazy.c +654 -313
- data/contrib/zstd/lib/compress/zstd_lazy.h +44 -16
- data/contrib/zstd/lib/compress/zstd_ldm.c +310 -420
- data/contrib/zstd/lib/compress/zstd_ldm.h +63 -26
- data/contrib/zstd/lib/compress/zstd_opt.c +773 -325
- data/contrib/zstd/lib/compress/zstd_opt.h +31 -5
- data/contrib/zstd/lib/compress/zstdmt_compress.c +1468 -518
- data/contrib/zstd/lib/compress/zstdmt_compress.h +96 -45
- data/contrib/zstd/lib/decompress/huf_decompress.c +518 -282
- data/contrib/zstd/lib/decompress/zstd_ddict.c +240 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
- data/contrib/zstd/lib/decompress/zstd_decompress.c +613 -1513
- data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1311 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.h +59 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +175 -0
- data/contrib/zstd/lib/dictBuilder/cover.c +194 -113
- data/contrib/zstd/lib/dictBuilder/cover.h +112 -0
- data/contrib/zstd/lib/dictBuilder/divsufsort.c +3 -3
- data/contrib/zstd/lib/dictBuilder/fastcover.c +740 -0
- data/contrib/zstd/lib/dictBuilder/zdict.c +142 -106
- data/contrib/zstd/lib/dictBuilder/zdict.h +115 -49
- data/contrib/zstd/lib/legacy/zstd_legacy.h +44 -12
- data/contrib/zstd/lib/legacy/zstd_v01.c +41 -10
- data/contrib/zstd/lib/legacy/zstd_v01.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v02.c +37 -12
- data/contrib/zstd/lib/legacy/zstd_v02.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v03.c +38 -12
- data/contrib/zstd/lib/legacy/zstd_v03.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v04.c +55 -174
- data/contrib/zstd/lib/legacy/zstd_v04.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v05.c +59 -31
- data/contrib/zstd/lib/legacy/zstd_v05.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v06.c +48 -20
- data/contrib/zstd/lib/legacy/zstd_v06.h +10 -5
- data/contrib/zstd/lib/legacy/zstd_v07.c +62 -29
- data/contrib/zstd/lib/legacy/zstd_v07.h +10 -5
- data/contrib/zstd/lib/zstd.h +1346 -832
- data/ext/extzstd.c +27 -19
- data/ext/extzstd_stream.c +20 -4
- data/ext/zstd_compress.c +1 -0
- data/ext/zstd_decompress.c +4 -0
- data/ext/zstd_dictbuilder.c +4 -0
- data/ext/zstd_dictbuilder_fastcover.c +5 -0
- data/lib/extzstd.rb +52 -220
- data/lib/extzstd/version.rb +1 -1
- metadata +21 -7
- data/contrib/zstd/circle.yml +0 -63
@@ -35,13 +35,18 @@ ZSTDv04_decompress() : decompress ZSTD frames compliant with v0.4.x format
|
|
35
35
|
size_t ZSTDv04_decompress( void* dst, size_t maxOriginalSize,
|
36
36
|
const void* src, size_t compressedSize);
|
37
37
|
|
38
|
-
/**
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
38
|
+
/**
|
39
|
+
ZSTDv04_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.4.x format
|
40
|
+
srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
41
|
+
cSize (output parameter) : the number of bytes that would be read to decompress this frame
|
42
|
+
or an error code if it fails (which can be tested using ZSTDv01_isError())
|
43
|
+
dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
|
44
|
+
or ZSTD_CONTENTSIZE_ERROR if an error occurs
|
45
|
+
|
46
|
+
note : assumes `cSize` and `dBound` are _not_ NULL.
|
47
|
+
*/
|
48
|
+
void ZSTDv04_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
|
49
|
+
size_t* cSize, unsigned long long* dBound);
|
45
50
|
|
46
51
|
/**
|
47
52
|
ZSTDv04_isError() : tells if the result of ZSTDv04_decompress() is an error
|
@@ -491,6 +491,8 @@ static const size_t ZSTDv05_frameHeaderSize_min = 5;
|
|
491
491
|
|
492
492
|
#define WILDCOPY_OVERLENGTH 8
|
493
493
|
|
494
|
+
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
|
495
|
+
|
494
496
|
typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
|
495
497
|
|
496
498
|
|
@@ -836,7 +838,7 @@ MEM_STATIC void BITv05_skipBits(BITv05_DStream_t* bitD, U32 nbBits)
|
|
836
838
|
bitD->bitsConsumed += nbBits;
|
837
839
|
}
|
838
840
|
|
839
|
-
MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD,
|
841
|
+
MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, unsigned nbBits)
|
840
842
|
{
|
841
843
|
size_t value = BITv05_lookBits(bitD, nbBits);
|
842
844
|
BITv05_skipBits(bitD, nbBits);
|
@@ -845,7 +847,7 @@ MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, U32 nbBits)
|
|
845
847
|
|
846
848
|
/*!BITv05_readBitsFast :
|
847
849
|
* unsafe version; only works only if nbBits >= 1 */
|
848
|
-
MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD,
|
850
|
+
MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, unsigned nbBits)
|
849
851
|
{
|
850
852
|
size_t value = BITv05_lookBitsFast(bitD, nbBits);
|
851
853
|
BITv05_skipBits(bitD, nbBits);
|
@@ -1162,7 +1164,7 @@ MEM_STATIC unsigned FSEv05_endOfDState(const FSEv05_DState_t* DStatePtr)
|
|
1162
1164
|
/* **************************************************************
|
1163
1165
|
* Complex types
|
1164
1166
|
****************************************************************/
|
1165
|
-
typedef
|
1167
|
+
typedef unsigned DTable_max_t[FSEv05_DTABLE_SIZE_U32(FSEv05_MAX_TABLELOG)];
|
1166
1168
|
|
1167
1169
|
|
1168
1170
|
/* **************************************************************
|
@@ -1224,6 +1226,7 @@ size_t FSEv05_buildDTable(FSEv05_DTable* dt, const short* normalizedCounter, uns
|
|
1224
1226
|
if (tableLog > FSEv05_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
1225
1227
|
|
1226
1228
|
/* Init, lay down lowprob symbols */
|
1229
|
+
memset(tableDecode, 0, sizeof(FSEv05_FUNCTION_TYPE) * (maxSymbolValue+1) ); /* useless init, but keep static analyzer happy, and we don't need to performance optimize legacy decoders */
|
1227
1230
|
DTableH.tableLog = (U16)tableLog;
|
1228
1231
|
for (s=0; s<=maxSymbolValue; s++) {
|
1229
1232
|
if (normalizedCounter[s]==-1) {
|
@@ -2190,7 +2193,7 @@ static void HUFv05_fillDTableX4(HUFv05_DEltX4* DTable, const U32 targetLog,
|
|
2190
2193
|
}
|
2191
2194
|
}
|
2192
2195
|
|
2193
|
-
size_t HUFv05_readDTableX4 (
|
2196
|
+
size_t HUFv05_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize)
|
2194
2197
|
{
|
2195
2198
|
BYTE weightList[HUFv05_MAX_SYMBOL_VALUE + 1];
|
2196
2199
|
sortedSymbol_t sortedSymbol[HUFv05_MAX_SYMBOL_VALUE + 1];
|
@@ -2204,7 +2207,7 @@ size_t HUFv05_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
|
|
2204
2207
|
void* dtPtr = DTable;
|
2205
2208
|
HUFv05_DEltX4* const dt = ((HUFv05_DEltX4*)dtPtr) + 1;
|
2206
2209
|
|
2207
|
-
HUFv05_STATIC_ASSERT(sizeof(HUFv05_DEltX4) == sizeof(
|
2210
|
+
HUFv05_STATIC_ASSERT(sizeof(HUFv05_DEltX4) == sizeof(unsigned)); /* if compilation fails here, assertion is false */
|
2208
2211
|
if (memLog > HUFv05_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
2209
2212
|
//memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
|
2210
2213
|
|
@@ -2331,7 +2334,7 @@ static inline size_t HUFv05_decodeStreamX4(BYTE* p, BITv05_DStream_t* bitDPtr, B
|
|
2331
2334
|
size_t HUFv05_decompress1X4_usingDTable(
|
2332
2335
|
void* dst, size_t dstSize,
|
2333
2336
|
const void* cSrc, size_t cSrcSize,
|
2334
|
-
const
|
2337
|
+
const unsigned* DTable)
|
2335
2338
|
{
|
2336
2339
|
const BYTE* const istart = (const BYTE*) cSrc;
|
2337
2340
|
BYTE* const ostart = (BYTE*) dst;
|
@@ -2374,7 +2377,7 @@ size_t HUFv05_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t
|
|
2374
2377
|
size_t HUFv05_decompress4X4_usingDTable(
|
2375
2378
|
void* dst, size_t dstSize,
|
2376
2379
|
const void* cSrc, size_t cSrcSize,
|
2377
|
-
const
|
2380
|
+
const unsigned* DTable)
|
2378
2381
|
{
|
2379
2382
|
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
|
2380
2383
|
|
@@ -2658,6 +2661,7 @@ struct ZSTDv05_DCtx_s
|
|
2658
2661
|
BYTE headerBuffer[ZSTDv05_frameHeaderSize_max];
|
2659
2662
|
}; /* typedef'd to ZSTDv05_DCtx within "zstd_static.h" */
|
2660
2663
|
|
2664
|
+
size_t ZSTDv05_sizeofDCtx (void); /* Hidden declaration */
|
2661
2665
|
size_t ZSTDv05_sizeofDCtx (void) { return sizeof(ZSTDv05_DCtx); }
|
2662
2666
|
|
2663
2667
|
size_t ZSTDv05_decompressBegin(ZSTDv05_DCtx* dctx)
|
@@ -2822,7 +2826,7 @@ static size_t ZSTDv05_decodeFrameHeader_Part2(ZSTDv05_DCtx* zc, const void* src,
|
|
2822
2826
|
}
|
2823
2827
|
|
2824
2828
|
|
2825
|
-
size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
2829
|
+
static size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
2826
2830
|
{
|
2827
2831
|
const BYTE* const in = (const BYTE* const)src;
|
2828
2832
|
BYTE headerFlags;
|
@@ -2845,6 +2849,7 @@ size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t*
|
|
2845
2849
|
|
2846
2850
|
static size_t ZSTDv05_copyRawBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
2847
2851
|
{
|
2852
|
+
if (dst==NULL) return ERROR(dstSize_tooSmall);
|
2848
2853
|
if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
|
2849
2854
|
memcpy(dst, src, srcSize);
|
2850
2855
|
return srcSize;
|
@@ -2853,8 +2858,8 @@ static size_t ZSTDv05_copyRawBlock(void* dst, size_t maxDstSize, const void* src
|
|
2853
2858
|
|
2854
2859
|
/*! ZSTDv05_decodeLiteralsBlock() :
|
2855
2860
|
@return : nb of bytes read from src (< srcSize ) */
|
2856
|
-
size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
|
2857
|
-
|
2861
|
+
static size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
|
2862
|
+
const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
|
2858
2863
|
{
|
2859
2864
|
const BYTE* const istart = (const BYTE*) src;
|
2860
2865
|
|
@@ -2988,7 +2993,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
|
|
2988
2993
|
}
|
2989
2994
|
|
2990
2995
|
|
2991
|
-
size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
|
2996
|
+
static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
|
2992
2997
|
FSEv05_DTable* DTableLL, FSEv05_DTable* DTableML, FSEv05_DTable* DTableOffb,
|
2993
2998
|
const void* src, size_t srcSize, U32 flagStaticTable)
|
2994
2999
|
{
|
@@ -2996,7 +3001,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
|
|
2996
3001
|
const BYTE* ip = istart;
|
2997
3002
|
const BYTE* const iend = istart + srcSize;
|
2998
3003
|
U32 LLtype, Offtype, MLtype;
|
2999
|
-
|
3004
|
+
unsigned LLlog, Offlog, MLlog;
|
3000
3005
|
size_t dumpsLength;
|
3001
3006
|
|
3002
3007
|
/* check */
|
@@ -3054,7 +3059,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
|
|
3054
3059
|
break;
|
3055
3060
|
case FSEv05_ENCODING_DYNAMIC :
|
3056
3061
|
default : /* impossible */
|
3057
|
-
{
|
3062
|
+
{ unsigned max = MaxLL;
|
3058
3063
|
headerSize = FSEv05_readNCount(norm, &max, &LLlog, ip, iend-ip);
|
3059
3064
|
if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
|
3060
3065
|
if (LLlog > LLFSEv05Log) return ERROR(corruption_detected);
|
@@ -3078,7 +3083,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
|
|
3078
3083
|
break;
|
3079
3084
|
case FSEv05_ENCODING_DYNAMIC :
|
3080
3085
|
default : /* impossible */
|
3081
|
-
{
|
3086
|
+
{ unsigned max = MaxOff;
|
3082
3087
|
headerSize = FSEv05_readNCount(norm, &max, &Offlog, ip, iend-ip);
|
3083
3088
|
if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
|
3084
3089
|
if (Offlog > OffFSEv05Log) return ERROR(corruption_detected);
|
@@ -3102,7 +3107,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
|
|
3102
3107
|
break;
|
3103
3108
|
case FSEv05_ENCODING_DYNAMIC :
|
3104
3109
|
default : /* impossible */
|
3105
|
-
{
|
3110
|
+
{ unsigned max = MaxML;
|
3106
3111
|
headerSize = FSEv05_readNCount(norm, &max, &MLlog, ip, iend-ip);
|
3107
3112
|
if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
|
3108
3113
|
if (MLlog > MLFSEv05Log) return ERROR(corruption_detected);
|
@@ -3214,7 +3219,7 @@ static size_t ZSTDv05_execSequence(BYTE* op,
|
|
3214
3219
|
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
|
3215
3220
|
{
|
3216
3221
|
static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
|
3217
|
-
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /*
|
3222
|
+
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
|
3218
3223
|
BYTE* const oLitEnd = op + sequence.litLength;
|
3219
3224
|
const size_t sequenceLength = sequence.litLength + sequence.matchLength;
|
3220
3225
|
BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
|
@@ -3297,14 +3302,14 @@ static size_t ZSTDv05_decompressSequences(
|
|
3297
3302
|
BYTE* const ostart = (BYTE* const)dst;
|
3298
3303
|
BYTE* op = ostart;
|
3299
3304
|
BYTE* const oend = ostart + maxDstSize;
|
3300
|
-
size_t errorCode, dumpsLength;
|
3305
|
+
size_t errorCode, dumpsLength=0;
|
3301
3306
|
const BYTE* litPtr = dctx->litPtr;
|
3302
3307
|
const BYTE* const litEnd = litPtr + dctx->litSize;
|
3303
|
-
int nbSeq;
|
3304
|
-
const BYTE* dumps;
|
3305
|
-
|
3306
|
-
|
3307
|
-
|
3308
|
+
int nbSeq=0;
|
3309
|
+
const BYTE* dumps = NULL;
|
3310
|
+
unsigned* DTableLL = dctx->LLTable;
|
3311
|
+
unsigned* DTableML = dctx->MLTable;
|
3312
|
+
unsigned* DTableOffb = dctx->OffTable;
|
3308
3313
|
const BYTE* const base = (const BYTE*) (dctx->base);
|
3309
3314
|
const BYTE* const vBase = (const BYTE*) (dctx->vBase);
|
3310
3315
|
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
|
@@ -3410,10 +3415,10 @@ static size_t ZSTDv05_decompress_continueDCtx(ZSTDv05_DCtx* dctx,
|
|
3410
3415
|
BYTE* const oend = ostart + maxDstSize;
|
3411
3416
|
size_t remainingSize = srcSize;
|
3412
3417
|
blockProperties_t blockProperties;
|
3418
|
+
memset(&blockProperties, 0, sizeof(blockProperties));
|
3413
3419
|
|
3414
3420
|
/* Frame Header */
|
3415
|
-
{
|
3416
|
-
size_t frameHeaderSize;
|
3421
|
+
{ size_t frameHeaderSize;
|
3417
3422
|
if (srcSize < ZSTDv05_frameHeaderSize_min+ZSTDv05_blockHeaderSize) return ERROR(srcSize_wrong);
|
3418
3423
|
frameHeaderSize = ZSTDv05_decodeFrameHeader_Part1(dctx, src, ZSTDv05_frameHeaderSize_min);
|
3419
3424
|
if (ZSTDv05_isError(frameHeaderSize)) return frameHeaderSize;
|
@@ -3505,34 +3510,57 @@ size_t ZSTDv05_decompress(void* dst, size_t maxDstSize, const void* src, size_t
|
|
3505
3510
|
#endif
|
3506
3511
|
}
|
3507
3512
|
|
3508
|
-
|
3513
|
+
/* ZSTD_errorFrameSizeInfoLegacy() :
|
3514
|
+
assumes `cSize` and `dBound` are _not_ NULL */
|
3515
|
+
static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
|
3516
|
+
{
|
3517
|
+
*cSize = ret;
|
3518
|
+
*dBound = ZSTD_CONTENTSIZE_ERROR;
|
3519
|
+
}
|
3520
|
+
|
3521
|
+
void ZSTDv05_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
|
3509
3522
|
{
|
3510
3523
|
const BYTE* ip = (const BYTE*)src;
|
3511
3524
|
size_t remainingSize = srcSize;
|
3525
|
+
size_t nbBlocks = 0;
|
3512
3526
|
blockProperties_t blockProperties;
|
3513
3527
|
|
3514
3528
|
/* Frame Header */
|
3515
|
-
if (srcSize < ZSTDv05_frameHeaderSize_min)
|
3516
|
-
|
3529
|
+
if (srcSize < ZSTDv05_frameHeaderSize_min) {
|
3530
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
3531
|
+
return;
|
3532
|
+
}
|
3533
|
+
if (MEM_readLE32(src) != ZSTDv05_MAGICNUMBER) {
|
3534
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
|
3535
|
+
return;
|
3536
|
+
}
|
3517
3537
|
ip += ZSTDv05_frameHeaderSize_min; remainingSize -= ZSTDv05_frameHeaderSize_min;
|
3518
3538
|
|
3519
3539
|
/* Loop on each block */
|
3520
3540
|
while (1)
|
3521
3541
|
{
|
3522
3542
|
size_t cBlockSize = ZSTDv05_getcBlockSize(ip, remainingSize, &blockProperties);
|
3523
|
-
if (ZSTDv05_isError(cBlockSize))
|
3543
|
+
if (ZSTDv05_isError(cBlockSize)) {
|
3544
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
|
3545
|
+
return;
|
3546
|
+
}
|
3524
3547
|
|
3525
3548
|
ip += ZSTDv05_blockHeaderSize;
|
3526
3549
|
remainingSize -= ZSTDv05_blockHeaderSize;
|
3527
|
-
if (cBlockSize > remainingSize)
|
3550
|
+
if (cBlockSize > remainingSize) {
|
3551
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
3552
|
+
return;
|
3553
|
+
}
|
3528
3554
|
|
3529
3555
|
if (cBlockSize == 0) break; /* bt_end */
|
3530
3556
|
|
3531
3557
|
ip += cBlockSize;
|
3532
3558
|
remainingSize -= cBlockSize;
|
3559
|
+
nbBlocks++;
|
3533
3560
|
}
|
3534
3561
|
|
3535
|
-
|
3562
|
+
*cSize = ip - (const BYTE*)src;
|
3563
|
+
*dBound = nbBlocks * BLOCKSIZE;
|
3536
3564
|
}
|
3537
3565
|
|
3538
3566
|
/* ******************************
|
@@ -3630,7 +3658,7 @@ static size_t ZSTDv05_loadEntropy(ZSTDv05_DCtx* dctx, const void* dict, size_t d
|
|
3630
3658
|
{
|
3631
3659
|
size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, errorCode, litlengthHeaderSize;
|
3632
3660
|
short offcodeNCount[MaxOff+1];
|
3633
|
-
|
3661
|
+
unsigned offcodeMaxValue=MaxOff, offcodeLog;
|
3634
3662
|
short matchlengthNCount[MaxML+1];
|
3635
3663
|
unsigned matchlengthMaxValue = MaxML, matchlengthLog;
|
3636
3664
|
short litlengthNCount[MaxLL+1];
|
@@ -33,13 +33,18 @@ extern "C" {
|
|
33
33
|
size_t ZSTDv05_decompress( void* dst, size_t dstCapacity,
|
34
34
|
const void* src, size_t compressedSize);
|
35
35
|
|
36
|
-
/**
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
36
|
+
/**
|
37
|
+
ZSTDv05_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.5.x format
|
38
|
+
srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
39
|
+
cSize (output parameter) : the number of bytes that would be read to decompress this frame
|
40
|
+
or an error code if it fails (which can be tested using ZSTDv01_isError())
|
41
|
+
dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
|
42
|
+
or ZSTD_CONTENTSIZE_ERROR if an error occurs
|
43
|
+
|
44
|
+
note : assumes `cSize` and `dBound` are _not_ NULL.
|
45
|
+
*/
|
46
|
+
void ZSTDv05_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
|
47
|
+
size_t* cSize, unsigned long long* dBound);
|
43
48
|
|
44
49
|
/* *************************************
|
45
50
|
* Helper functions
|
@@ -189,7 +189,7 @@ MEM_STATIC U32 MEM_swap32(U32 in)
|
|
189
189
|
{
|
190
190
|
#if defined(_MSC_VER) /* Visual Studio */
|
191
191
|
return _byteswap_ulong(in);
|
192
|
-
#elif defined (__GNUC__)
|
192
|
+
#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
|
193
193
|
return __builtin_bswap32(in);
|
194
194
|
#else
|
195
195
|
return ((in << 24) & 0xff000000 ) |
|
@@ -203,7 +203,7 @@ MEM_STATIC U64 MEM_swap64(U64 in)
|
|
203
203
|
{
|
204
204
|
#if defined(_MSC_VER) /* Visual Studio */
|
205
205
|
return _byteswap_uint64(in);
|
206
|
-
#elif defined (__GNUC__)
|
206
|
+
#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
|
207
207
|
return __builtin_bswap64(in);
|
208
208
|
#else
|
209
209
|
return ((in << 56) & 0xff00000000000000ULL) |
|
@@ -506,6 +506,8 @@ typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
|
|
506
506
|
#define FSEv06_ENCODING_STATIC 2
|
507
507
|
#define FSEv06_ENCODING_DYNAMIC 3
|
508
508
|
|
509
|
+
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
|
510
|
+
|
509
511
|
static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
510
512
|
1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12,
|
511
513
|
13,14,15,16 };
|
@@ -1250,9 +1252,7 @@ const char* FSEv06_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
|
1250
1252
|
/* **************************************************************
|
1251
1253
|
* HUF Error Management
|
1252
1254
|
****************************************************************/
|
1253
|
-
unsigned HUFv06_isError(size_t code) { return ERR_isError(code); }
|
1254
|
-
|
1255
|
-
const char* HUFv06_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
1255
|
+
static unsigned HUFv06_isError(size_t code) { return ERR_isError(code); }
|
1256
1256
|
|
1257
1257
|
|
1258
1258
|
/*-**************************************************************
|
@@ -2823,7 +2823,8 @@ struct ZSTDv06_DCtx_s
|
|
2823
2823
|
BYTE headerBuffer[ZSTDv06_FRAMEHEADERSIZE_MAX];
|
2824
2824
|
}; /* typedef'd to ZSTDv06_DCtx within "zstd_static.h" */
|
2825
2825
|
|
2826
|
-
size_t ZSTDv06_sizeofDCtx (void)
|
2826
|
+
size_t ZSTDv06_sizeofDCtx (void); /* Hidden declaration */
|
2827
|
+
size_t ZSTDv06_sizeofDCtx (void) { return sizeof(ZSTDv06_DCtx); }
|
2827
2828
|
|
2828
2829
|
size_t ZSTDv06_decompressBegin(ZSTDv06_DCtx* dctx)
|
2829
2830
|
{
|
@@ -3022,7 +3023,7 @@ typedef struct
|
|
3022
3023
|
|
3023
3024
|
/*! ZSTDv06_getcBlockSize() :
|
3024
3025
|
* Provides the size of compressed block from block header `src` */
|
3025
|
-
size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
3026
|
+
static size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
3026
3027
|
{
|
3027
3028
|
const BYTE* const in = (const BYTE* const)src;
|
3028
3029
|
U32 cSize;
|
@@ -3041,6 +3042,7 @@ size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t*
|
|
3041
3042
|
|
3042
3043
|
static size_t ZSTDv06_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
3043
3044
|
{
|
3045
|
+
if (dst==NULL) return ERROR(dstSize_tooSmall);
|
3044
3046
|
if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
|
3045
3047
|
memcpy(dst, src, srcSize);
|
3046
3048
|
return srcSize;
|
@@ -3049,7 +3051,7 @@ static size_t ZSTDv06_copyRawBlock(void* dst, size_t dstCapacity, const void* sr
|
|
3049
3051
|
|
3050
3052
|
/*! ZSTDv06_decodeLiteralsBlock() :
|
3051
3053
|
@return : nb of bytes read from src (< srcSize ) */
|
3052
|
-
size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
|
3054
|
+
static size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
|
3053
3055
|
const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
|
3054
3056
|
{
|
3055
3057
|
const BYTE* const istart = (const BYTE*) src;
|
@@ -3183,7 +3185,7 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
|
|
3183
3185
|
@return : nb bytes read from src,
|
3184
3186
|
or an error code if it fails, testable with ZSTDv06_isError()
|
3185
3187
|
*/
|
3186
|
-
size_t ZSTDv06_buildSeqTable(FSEv06_DTable* DTable, U32 type, U32 max, U32 maxLog,
|
3188
|
+
static size_t ZSTDv06_buildSeqTable(FSEv06_DTable* DTable, U32 type, U32 max, U32 maxLog,
|
3187
3189
|
const void* src, size_t srcSize,
|
3188
3190
|
const S16* defaultNorm, U32 defaultLog, U32 flagRepeatTable)
|
3189
3191
|
{
|
@@ -3213,7 +3215,7 @@ size_t ZSTDv06_buildSeqTable(FSEv06_DTable* DTable, U32 type, U32 max, U32 maxLo
|
|
3213
3215
|
}
|
3214
3216
|
|
3215
3217
|
|
3216
|
-
size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
|
3218
|
+
static size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
|
3217
3219
|
FSEv06_DTable* DTableLL, FSEv06_DTable* DTableML, FSEv06_DTable* DTableOffb, U32 flagRepeatTable,
|
3218
3220
|
const void* src, size_t srcSize)
|
3219
3221
|
{
|
@@ -3358,7 +3360,7 @@ static void ZSTDv06_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
3358
3360
|
}
|
3359
3361
|
|
3360
3362
|
|
3361
|
-
size_t ZSTDv06_execSequence(BYTE* op,
|
3363
|
+
static size_t ZSTDv06_execSequence(BYTE* op,
|
3362
3364
|
BYTE* const oend, seq_t sequence,
|
3363
3365
|
const BYTE** litPtr, const BYTE* const litLimit,
|
3364
3366
|
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
|
@@ -3406,7 +3408,7 @@ size_t ZSTDv06_execSequence(BYTE* op,
|
|
3406
3408
|
if (sequence.offset < 8) {
|
3407
3409
|
/* close range match, overlap */
|
3408
3410
|
static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
|
3409
|
-
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /*
|
3411
|
+
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
|
3410
3412
|
int const sub2 = dec64table[sequence.offset];
|
3411
3413
|
op[0] = match[0];
|
3412
3414
|
op[1] = match[1];
|
@@ -3654,36 +3656,62 @@ size_t ZSTDv06_decompress(void* dst, size_t dstCapacity, const void* src, size_t
|
|
3654
3656
|
#endif
|
3655
3657
|
}
|
3656
3658
|
|
3657
|
-
|
3659
|
+
/* ZSTD_errorFrameSizeInfoLegacy() :
|
3660
|
+
assumes `cSize` and `dBound` are _not_ NULL */
|
3661
|
+
static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
|
3662
|
+
{
|
3663
|
+
*cSize = ret;
|
3664
|
+
*dBound = ZSTD_CONTENTSIZE_ERROR;
|
3665
|
+
}
|
3666
|
+
|
3667
|
+
void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
|
3658
3668
|
{
|
3659
3669
|
const BYTE* ip = (const BYTE*)src;
|
3660
3670
|
size_t remainingSize = srcSize;
|
3671
|
+
size_t nbBlocks = 0;
|
3661
3672
|
blockProperties_t blockProperties = { bt_compressed, 0 };
|
3662
3673
|
|
3663
3674
|
/* Frame Header */
|
3664
3675
|
{ size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src, ZSTDv06_frameHeaderSize_min);
|
3665
|
-
if (ZSTDv06_isError(frameHeaderSize))
|
3666
|
-
|
3667
|
-
|
3676
|
+
if (ZSTDv06_isError(frameHeaderSize)) {
|
3677
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, frameHeaderSize);
|
3678
|
+
return;
|
3679
|
+
}
|
3680
|
+
if (MEM_readLE32(src) != ZSTDv06_MAGICNUMBER) {
|
3681
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
|
3682
|
+
return;
|
3683
|
+
}
|
3684
|
+
if (srcSize < frameHeaderSize+ZSTDv06_blockHeaderSize) {
|
3685
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
3686
|
+
return;
|
3687
|
+
}
|
3668
3688
|
ip += frameHeaderSize; remainingSize -= frameHeaderSize;
|
3669
3689
|
}
|
3670
3690
|
|
3671
3691
|
/* Loop on each block */
|
3672
3692
|
while (1) {
|
3673
3693
|
size_t const cBlockSize = ZSTDv06_getcBlockSize(ip, remainingSize, &blockProperties);
|
3674
|
-
if (ZSTDv06_isError(cBlockSize))
|
3694
|
+
if (ZSTDv06_isError(cBlockSize)) {
|
3695
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
|
3696
|
+
return;
|
3697
|
+
}
|
3675
3698
|
|
3676
3699
|
ip += ZSTDv06_blockHeaderSize;
|
3677
3700
|
remainingSize -= ZSTDv06_blockHeaderSize;
|
3678
|
-
if (cBlockSize > remainingSize)
|
3701
|
+
if (cBlockSize > remainingSize) {
|
3702
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
3703
|
+
return;
|
3704
|
+
}
|
3679
3705
|
|
3680
3706
|
if (cBlockSize == 0) break; /* bt_end */
|
3681
3707
|
|
3682
3708
|
ip += cBlockSize;
|
3683
3709
|
remainingSize -= cBlockSize;
|
3710
|
+
nbBlocks++;
|
3684
3711
|
}
|
3685
3712
|
|
3686
|
-
|
3713
|
+
*cSize = ip - (const BYTE*)src;
|
3714
|
+
*dBound = nbBlocks * ZSTDv06_BLOCKSIZE_MAX;
|
3687
3715
|
}
|
3688
3716
|
|
3689
3717
|
/*_******************************
|
@@ -4006,7 +4034,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
|
|
4006
4034
|
if (ZSTDv06_isError(hSize)) return hSize;
|
4007
4035
|
if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
|
4008
4036
|
memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
|
4009
|
-
zbd->lhSize += iend-ip;
|
4037
|
+
zbd->lhSize += iend-ip;
|
4010
4038
|
*dstCapacityPtr = 0;
|
4011
4039
|
return (hSize - zbd->lhSize) + ZSTDv06_blockHeaderSize; /* remaining header bytes + next block header */
|
4012
4040
|
}
|