extzstd 0.1 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/HISTORY.ja +5 -0
- data/README.md +5 -5
- data/contrib/zstd/CONTRIBUTING.md +42 -0
- data/contrib/zstd/LICENSE-examples +11 -0
- data/contrib/zstd/Makefile +315 -0
- data/contrib/zstd/NEWS +261 -0
- data/contrib/zstd/PATENTS +33 -0
- data/contrib/zstd/README.md +121 -41
- data/contrib/zstd/TESTING.md +44 -0
- data/contrib/zstd/appveyor.yml +178 -0
- data/contrib/zstd/circle.yml +75 -0
- data/contrib/zstd/lib/BUCK +186 -0
- data/contrib/zstd/lib/Makefile +163 -0
- data/contrib/zstd/lib/README.md +77 -0
- data/contrib/zstd/{common → lib/common}/bitstream.h +7 -4
- data/contrib/zstd/{common → lib/common}/entropy_common.c +19 -23
- data/contrib/zstd/{common → lib/common}/error_private.c +0 -0
- data/contrib/zstd/{common → lib/common}/error_private.h +0 -0
- data/contrib/zstd/{common → lib/common}/fse.h +94 -34
- data/contrib/zstd/{common → lib/common}/fse_decompress.c +18 -19
- data/contrib/zstd/{common → lib/common}/huf.h +52 -20
- data/contrib/zstd/{common → lib/common}/mem.h +17 -13
- data/contrib/zstd/lib/common/pool.c +194 -0
- data/contrib/zstd/lib/common/pool.h +56 -0
- data/contrib/zstd/lib/common/threading.c +80 -0
- data/contrib/zstd/lib/common/threading.h +104 -0
- data/contrib/zstd/{common → lib/common}/xxhash.c +3 -1
- data/contrib/zstd/{common → lib/common}/xxhash.h +11 -15
- data/contrib/zstd/{common → lib/common}/zstd_common.c +1 -11
- data/contrib/zstd/{common → lib/common}/zstd_errors.h +16 -2
- data/contrib/zstd/{common → lib/common}/zstd_internal.h +17 -1
- data/contrib/zstd/{compress → lib/compress}/fse_compress.c +138 -91
- data/contrib/zstd/{compress → lib/compress}/huf_compress.c +218 -67
- data/contrib/zstd/{compress → lib/compress}/zstd_compress.c +231 -108
- data/contrib/zstd/{compress → lib/compress}/zstd_opt.h +44 -25
- data/contrib/zstd/lib/compress/zstdmt_compress.c +739 -0
- data/contrib/zstd/lib/compress/zstdmt_compress.h +78 -0
- data/contrib/zstd/{decompress → lib/decompress}/huf_decompress.c +28 -23
- data/contrib/zstd/{decompress → lib/decompress}/zstd_decompress.c +814 -176
- data/contrib/zstd/{common → lib/deprecated}/zbuff.h +60 -39
- data/contrib/zstd/lib/deprecated/zbuff_common.c +26 -0
- data/contrib/zstd/lib/deprecated/zbuff_compress.c +145 -0
- data/contrib/zstd/lib/deprecated/zbuff_decompress.c +74 -0
- data/contrib/zstd/lib/dictBuilder/cover.c +1029 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.c +0 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.h +0 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/zdict.c +68 -18
- data/contrib/zstd/lib/dictBuilder/zdict.h +201 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_legacy.h +122 -7
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.c +34 -3
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.h +8 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.c +45 -12
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.h +8 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.c +45 -12
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.h +8 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.c +56 -33
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.h +8 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.c +45 -18
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.h +7 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.c +43 -16
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.h +7 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.c +57 -23
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.h +8 -0
- data/contrib/zstd/lib/libzstd.pc.in +14 -0
- data/contrib/zstd/{zstd.h → lib/zstd.h} +206 -71
- data/ext/depend +2 -0
- data/ext/extconf.rb +4 -4
- data/ext/extzstd.c +1 -1
- data/ext/zstd_common.c +5 -5
- data/ext/zstd_compress.c +3 -3
- data/ext/zstd_decompress.c +2 -2
- data/ext/zstd_dictbuilder.c +2 -2
- data/ext/zstd_legacy_v01.c +1 -1
- data/ext/zstd_legacy_v02.c +1 -1
- data/ext/zstd_legacy_v03.c +1 -1
- data/ext/zstd_legacy_v04.c +1 -1
- data/ext/zstd_legacy_v05.c +1 -1
- data/ext/zstd_legacy_v06.c +1 -1
- data/ext/zstd_legacy_v07.c +1 -1
- data/gemstub.rb +9 -5
- data/lib/extzstd/version.rb +1 -1
- metadata +73 -51
- data/contrib/zstd/compress/zbuff_compress.c +0 -319
- data/contrib/zstd/decompress/zbuff_decompress.c +0 -252
- data/contrib/zstd/dictBuilder/zdict.h +0 -111
@@ -34,6 +34,14 @@ ZSTDv01_decompress() : decompress ZSTD frames compliant with v0.1.x format
|
|
34
34
|
size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize,
|
35
35
|
const void* src, size_t compressedSize);
|
36
36
|
|
37
|
+
/**
|
38
|
+
ZSTDv01_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.1.x format
|
39
|
+
compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
40
|
+
return : the number of bytes that would be read to decompress this frame
|
41
|
+
or an errorCode if it fails (which can be tested using ZSTDv01_isError())
|
42
|
+
*/
|
43
|
+
size_t ZSTDv01_findFrameCompressedSize(const void* src, size_t compressedSize);
|
44
|
+
|
37
45
|
/**
|
38
46
|
ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error
|
39
47
|
*/
|
@@ -2808,7 +2808,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
|
|
2808
2808
|
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
|
2809
2809
|
|
2810
2810
|
/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
|
2811
|
-
static void ZSTD_wildcopy(void* dst, const void* src,
|
2811
|
+
static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
|
2812
2812
|
{
|
2813
2813
|
const BYTE* ip = (const BYTE*)src;
|
2814
2814
|
BYTE* op = (BYTE*)dst;
|
@@ -2868,7 +2868,6 @@ struct ZSTD_DCtx_s
|
|
2868
2868
|
blockType_t bType;
|
2869
2869
|
U32 phase;
|
2870
2870
|
const BYTE* litPtr;
|
2871
|
-
size_t litBufSize;
|
2872
2871
|
size_t litSize;
|
2873
2872
|
BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
|
2874
2873
|
}; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
|
@@ -2940,8 +2939,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
2940
2939
|
size_t litSize = BLOCKSIZE;
|
2941
2940
|
const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
|
2942
2941
|
dctx->litPtr = dctx->litBuffer;
|
2943
|
-
dctx->litBufSize = BLOCKSIZE;
|
2944
2942
|
dctx->litSize = litSize;
|
2943
|
+
memset(dctx->litBuffer + dctx->litSize, 0, 8);
|
2945
2944
|
return readSize; /* works if it's an error too */
|
2946
2945
|
}
|
2947
2946
|
case IS_RAW:
|
@@ -2952,13 +2951,12 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
2952
2951
|
if (litSize > srcSize-3) return ERROR(corruption_detected);
|
2953
2952
|
memcpy(dctx->litBuffer, istart, litSize);
|
2954
2953
|
dctx->litPtr = dctx->litBuffer;
|
2955
|
-
dctx->litBufSize = BLOCKSIZE;
|
2956
2954
|
dctx->litSize = litSize;
|
2955
|
+
memset(dctx->litBuffer + dctx->litSize, 0, 8);
|
2957
2956
|
return litSize+3;
|
2958
2957
|
}
|
2959
2958
|
/* direct reference into compressed stream */
|
2960
2959
|
dctx->litPtr = istart+3;
|
2961
|
-
dctx->litBufSize = srcSize-3;
|
2962
2960
|
dctx->litSize = litSize;
|
2963
2961
|
return litSize+3;
|
2964
2962
|
}
|
@@ -2966,9 +2964,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
2966
2964
|
{
|
2967
2965
|
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
|
2968
2966
|
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
|
2969
|
-
memset(dctx->litBuffer, istart[3], litSize);
|
2967
|
+
memset(dctx->litBuffer, istart[3], litSize + 8);
|
2970
2968
|
dctx->litPtr = dctx->litBuffer;
|
2971
|
-
dctx->litBufSize = BLOCKSIZE;
|
2972
2969
|
dctx->litSize = litSize;
|
2973
2970
|
return 4;
|
2974
2971
|
}
|
@@ -3175,7 +3172,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
3175
3172
|
/* checks */
|
3176
3173
|
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
|
3177
3174
|
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
|
3178
|
-
if (litEnd > litLimit
|
3175
|
+
if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
|
3179
3176
|
|
3180
3177
|
/* copy Literals */
|
3181
3178
|
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
@@ -3209,7 +3206,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
3209
3206
|
}
|
3210
3207
|
op += 8; match += 8;
|
3211
3208
|
|
3212
|
-
if (oMatchEnd > oend-
|
3209
|
+
if (oMatchEnd > oend-(16-MINMATCH))
|
3213
3210
|
{
|
3214
3211
|
if (op < oend_8)
|
3215
3212
|
{
|
@@ -3221,7 +3218,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
3221
3218
|
}
|
3222
3219
|
else
|
3223
3220
|
{
|
3224
|
-
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
|
3221
|
+
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
|
3225
3222
|
}
|
3226
3223
|
}
|
3227
3224
|
|
@@ -3241,7 +3238,6 @@ static size_t ZSTD_decompressSequences(
|
|
3241
3238
|
BYTE* const oend = ostart + maxDstSize;
|
3242
3239
|
size_t errorCode, dumpsLength;
|
3243
3240
|
const BYTE* litPtr = dctx->litPtr;
|
3244
|
-
const BYTE* const litMax = litPtr + dctx->litBufSize;
|
3245
3241
|
const BYTE* const litEnd = litPtr + dctx->litSize;
|
3246
3242
|
int nbSeq;
|
3247
3243
|
const BYTE* dumps;
|
@@ -3277,7 +3273,7 @@ static size_t ZSTD_decompressSequences(
|
|
3277
3273
|
size_t oneSeqSize;
|
3278
3274
|
nbSeq--;
|
3279
3275
|
ZSTD_decodeSequence(&sequence, &seqState);
|
3280
|
-
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr,
|
3276
|
+
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
|
3281
3277
|
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
|
3282
3278
|
op += oneSeqSize;
|
3283
3279
|
}
|
@@ -3382,6 +3378,38 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
|
|
3382
3378
|
return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
3383
3379
|
}
|
3384
3380
|
|
3381
|
+
static size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
|
3382
|
+
{
|
3383
|
+
|
3384
|
+
const BYTE* ip = (const BYTE*)src;
|
3385
|
+
size_t remainingSize = srcSize;
|
3386
|
+
U32 magicNumber;
|
3387
|
+
blockProperties_t blockProperties;
|
3388
|
+
|
3389
|
+
/* Frame Header */
|
3390
|
+
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
|
3391
|
+
magicNumber = MEM_readLE32(src);
|
3392
|
+
if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
|
3393
|
+
ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
|
3394
|
+
|
3395
|
+
/* Loop on each block */
|
3396
|
+
while (1)
|
3397
|
+
{
|
3398
|
+
size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
|
3399
|
+
if (ZSTD_isError(cBlockSize)) return cBlockSize;
|
3400
|
+
|
3401
|
+
ip += ZSTD_blockHeaderSize;
|
3402
|
+
remainingSize -= ZSTD_blockHeaderSize;
|
3403
|
+
if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
|
3404
|
+
|
3405
|
+
if (cBlockSize == 0) break; /* bt_end */
|
3406
|
+
|
3407
|
+
ip += cBlockSize;
|
3408
|
+
remainingSize -= cBlockSize;
|
3409
|
+
}
|
3410
|
+
|
3411
|
+
return ip - (const BYTE*)src;
|
3412
|
+
}
|
3385
3413
|
|
3386
3414
|
/*******************************
|
3387
3415
|
* Streaming Decompression API
|
@@ -3496,6 +3524,11 @@ size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
|
|
3496
3524
|
return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
|
3497
3525
|
}
|
3498
3526
|
|
3527
|
+
size_t ZSTDv02_findFrameCompressedSize(const void *src, size_t compressedSize)
|
3528
|
+
{
|
3529
|
+
return ZSTD_findFrameCompressedSize(src, compressedSize);
|
3530
|
+
}
|
3531
|
+
|
3499
3532
|
ZSTDv02_Dctx* ZSTDv02_createDCtx(void)
|
3500
3533
|
{
|
3501
3534
|
return (ZSTDv02_Dctx*)ZSTD_createDCtx();
|
@@ -34,6 +34,14 @@ ZSTDv02_decompress() : decompress ZSTD frames compliant with v0.2.x format
|
|
34
34
|
size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
|
35
35
|
const void* src, size_t compressedSize);
|
36
36
|
|
37
|
+
/**
|
38
|
+
ZSTDv02_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.2.x format
|
39
|
+
compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
40
|
+
return : the number of bytes that would be read to decompress this frame
|
41
|
+
or an errorCode if it fails (which can be tested using ZSTDv02_isError())
|
42
|
+
*/
|
43
|
+
size_t ZSTDv02_findFrameCompressedSize(const void* src, size_t compressedSize);
|
44
|
+
|
37
45
|
/**
|
38
46
|
ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error
|
39
47
|
*/
|
@@ -2449,7 +2449,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
|
|
2449
2449
|
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
|
2450
2450
|
|
2451
2451
|
/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
|
2452
|
-
static void ZSTD_wildcopy(void* dst, const void* src,
|
2452
|
+
static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
|
2453
2453
|
{
|
2454
2454
|
const BYTE* ip = (const BYTE*)src;
|
2455
2455
|
BYTE* op = (BYTE*)dst;
|
@@ -2509,7 +2509,6 @@ struct ZSTD_DCtx_s
|
|
2509
2509
|
blockType_t bType;
|
2510
2510
|
U32 phase;
|
2511
2511
|
const BYTE* litPtr;
|
2512
|
-
size_t litBufSize;
|
2513
2512
|
size_t litSize;
|
2514
2513
|
BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
|
2515
2514
|
}; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
|
@@ -2581,8 +2580,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
2581
2580
|
size_t litSize = BLOCKSIZE;
|
2582
2581
|
const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
|
2583
2582
|
dctx->litPtr = dctx->litBuffer;
|
2584
|
-
dctx->litBufSize = BLOCKSIZE;
|
2585
2583
|
dctx->litSize = litSize;
|
2584
|
+
memset(dctx->litBuffer + dctx->litSize, 0, 8);
|
2586
2585
|
return readSize; /* works if it's an error too */
|
2587
2586
|
}
|
2588
2587
|
case IS_RAW:
|
@@ -2593,13 +2592,12 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
2593
2592
|
if (litSize > srcSize-3) return ERROR(corruption_detected);
|
2594
2593
|
memcpy(dctx->litBuffer, istart, litSize);
|
2595
2594
|
dctx->litPtr = dctx->litBuffer;
|
2596
|
-
dctx->litBufSize = BLOCKSIZE;
|
2597
2595
|
dctx->litSize = litSize;
|
2596
|
+
memset(dctx->litBuffer + dctx->litSize, 0, 8);
|
2598
2597
|
return litSize+3;
|
2599
2598
|
}
|
2600
2599
|
/* direct reference into compressed stream */
|
2601
2600
|
dctx->litPtr = istart+3;
|
2602
|
-
dctx->litBufSize = srcSize-3;
|
2603
2601
|
dctx->litSize = litSize;
|
2604
2602
|
return litSize+3;
|
2605
2603
|
}
|
@@ -2607,9 +2605,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
2607
2605
|
{
|
2608
2606
|
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
|
2609
2607
|
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
|
2610
|
-
memset(dctx->litBuffer, istart[3], litSize);
|
2608
|
+
memset(dctx->litBuffer, istart[3], litSize + 8);
|
2611
2609
|
dctx->litPtr = dctx->litBuffer;
|
2612
|
-
dctx->litBufSize = BLOCKSIZE;
|
2613
2610
|
dctx->litSize = litSize;
|
2614
2611
|
return 4;
|
2615
2612
|
}
|
@@ -2816,7 +2813,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
2816
2813
|
/* checks */
|
2817
2814
|
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
|
2818
2815
|
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
|
2819
|
-
if (litEnd > litLimit
|
2816
|
+
if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
|
2820
2817
|
|
2821
2818
|
/* copy Literals */
|
2822
2819
|
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
@@ -2850,7 +2847,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
2850
2847
|
}
|
2851
2848
|
op += 8; match += 8;
|
2852
2849
|
|
2853
|
-
if (oMatchEnd > oend-
|
2850
|
+
if (oMatchEnd > oend-(16-MINMATCH))
|
2854
2851
|
{
|
2855
2852
|
if (op < oend_8)
|
2856
2853
|
{
|
@@ -2862,7 +2859,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
2862
2859
|
}
|
2863
2860
|
else
|
2864
2861
|
{
|
2865
|
-
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
|
2862
|
+
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
|
2866
2863
|
}
|
2867
2864
|
}
|
2868
2865
|
|
@@ -2882,7 +2879,6 @@ static size_t ZSTD_decompressSequences(
|
|
2882
2879
|
BYTE* const oend = ostart + maxDstSize;
|
2883
2880
|
size_t errorCode, dumpsLength;
|
2884
2881
|
const BYTE* litPtr = dctx->litPtr;
|
2885
|
-
const BYTE* const litMax = litPtr + dctx->litBufSize;
|
2886
2882
|
const BYTE* const litEnd = litPtr + dctx->litSize;
|
2887
2883
|
int nbSeq;
|
2888
2884
|
const BYTE* dumps;
|
@@ -2918,7 +2914,7 @@ static size_t ZSTD_decompressSequences(
|
|
2918
2914
|
size_t oneSeqSize;
|
2919
2915
|
nbSeq--;
|
2920
2916
|
ZSTD_decodeSequence(&sequence, &seqState);
|
2921
|
-
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr,
|
2917
|
+
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
|
2922
2918
|
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
|
2923
2919
|
op += oneSeqSize;
|
2924
2920
|
}
|
@@ -3023,6 +3019,38 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
|
|
3023
3019
|
return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
3024
3020
|
}
|
3025
3021
|
|
3022
|
+
static size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
|
3023
|
+
{
|
3024
|
+
const BYTE* ip = (const BYTE*)src;
|
3025
|
+
size_t remainingSize = srcSize;
|
3026
|
+
U32 magicNumber;
|
3027
|
+
blockProperties_t blockProperties;
|
3028
|
+
|
3029
|
+
/* Frame Header */
|
3030
|
+
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
|
3031
|
+
magicNumber = MEM_readLE32(src);
|
3032
|
+
if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
|
3033
|
+
ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
|
3034
|
+
|
3035
|
+
/* Loop on each block */
|
3036
|
+
while (1)
|
3037
|
+
{
|
3038
|
+
size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
|
3039
|
+
if (ZSTD_isError(cBlockSize)) return cBlockSize;
|
3040
|
+
|
3041
|
+
ip += ZSTD_blockHeaderSize;
|
3042
|
+
remainingSize -= ZSTD_blockHeaderSize;
|
3043
|
+
if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
|
3044
|
+
|
3045
|
+
if (cBlockSize == 0) break; /* bt_end */
|
3046
|
+
|
3047
|
+
ip += cBlockSize;
|
3048
|
+
remainingSize -= cBlockSize;
|
3049
|
+
}
|
3050
|
+
|
3051
|
+
return ip - (const BYTE*)src;
|
3052
|
+
}
|
3053
|
+
|
3026
3054
|
|
3027
3055
|
/*******************************
|
3028
3056
|
* Streaming Decompression API
|
@@ -3137,6 +3165,11 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
|
|
3137
3165
|
return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
|
3138
3166
|
}
|
3139
3167
|
|
3168
|
+
size_t ZSTDv03_findFrameCompressedSize(const void* src, size_t srcSize)
|
3169
|
+
{
|
3170
|
+
return ZSTD_findFrameCompressedSize(src, srcSize);
|
3171
|
+
}
|
3172
|
+
|
3140
3173
|
ZSTDv03_Dctx* ZSTDv03_createDCtx(void)
|
3141
3174
|
{
|
3142
3175
|
return (ZSTDv03_Dctx*)ZSTD_createDCtx();
|
@@ -35,6 +35,14 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
|
|
35
35
|
const void* src, size_t compressedSize);
|
36
36
|
|
37
37
|
/**
|
38
|
+
ZSTDv03_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.3.x format
|
39
|
+
compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
40
|
+
return : the number of bytes that would be read to decompress this frame
|
41
|
+
or an errorCode if it fails (which can be tested using ZSTDv03_isError())
|
42
|
+
*/
|
43
|
+
size_t ZSTDv03_findFrameCompressedSize(const void* src, size_t compressedSize);
|
44
|
+
|
45
|
+
/**
|
38
46
|
ZSTDv03_isError() : tells if the result of ZSTDv03_decompress() is an error
|
39
47
|
*/
|
40
48
|
unsigned ZSTDv03_isError(size_t code);
|
@@ -487,7 +487,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
|
|
487
487
|
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
|
488
488
|
|
489
489
|
/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
|
490
|
-
static void ZSTD_wildcopy(void* dst, const void* src,
|
490
|
+
static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
|
491
491
|
{
|
492
492
|
const BYTE* ip = (const BYTE*)src;
|
493
493
|
BYTE* op = (BYTE*)dst;
|
@@ -2706,7 +2706,6 @@ struct ZSTDv04_Dctx_s
|
|
2706
2706
|
blockType_t bType;
|
2707
2707
|
ZSTD_dStage stage;
|
2708
2708
|
const BYTE* litPtr;
|
2709
|
-
size_t litBufSize;
|
2710
2709
|
size_t litSize;
|
2711
2710
|
BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
|
2712
2711
|
BYTE headerBuffer[ZSTD_frameHeaderSize_max];
|
@@ -2847,8 +2846,8 @@ static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
|
|
2847
2846
|
size_t litSize = BLOCKSIZE;
|
2848
2847
|
const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
|
2849
2848
|
dctx->litPtr = dctx->litBuffer;
|
2850
|
-
dctx->litBufSize = BLOCKSIZE+8;
|
2851
2849
|
dctx->litSize = litSize;
|
2850
|
+
memset(dctx->litBuffer + dctx->litSize, 0, 8);
|
2852
2851
|
return readSize; /* works if it's an error too */
|
2853
2852
|
}
|
2854
2853
|
case IS_RAW:
|
@@ -2859,22 +2858,20 @@ static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
|
|
2859
2858
|
if (litSize > srcSize-3) return ERROR(corruption_detected);
|
2860
2859
|
memcpy(dctx->litBuffer, istart, litSize);
|
2861
2860
|
dctx->litPtr = dctx->litBuffer;
|
2862
|
-
dctx->litBufSize = BLOCKSIZE+8;
|
2863
2861
|
dctx->litSize = litSize;
|
2862
|
+
memset(dctx->litBuffer + dctx->litSize, 0, 8);
|
2864
2863
|
return litSize+3;
|
2865
2864
|
}
|
2866
2865
|
/* direct reference into compressed stream */
|
2867
2866
|
dctx->litPtr = istart+3;
|
2868
|
-
dctx->litBufSize = srcSize-3;
|
2869
2867
|
dctx->litSize = litSize;
|
2870
2868
|
return litSize+3; }
|
2871
2869
|
case IS_RLE:
|
2872
2870
|
{
|
2873
2871
|
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
|
2874
2872
|
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
|
2875
|
-
memset(dctx->litBuffer, istart[3], litSize);
|
2873
|
+
memset(dctx->litBuffer, istart[3], litSize + 8);
|
2876
2874
|
dctx->litPtr = dctx->litBuffer;
|
2877
|
-
dctx->litBufSize = BLOCKSIZE+8;
|
2878
2875
|
dctx->litSize = litSize;
|
2879
2876
|
return 4;
|
2880
2877
|
}
|
@@ -3015,21 +3012,19 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
3015
3012
|
/* Literal length */
|
3016
3013
|
litLength = FSE_decodeSymbol(&(seqState->stateLL), &(seqState->DStream));
|
3017
3014
|
prevOffset = litLength ? seq->offset : seqState->prevOffset;
|
3018
|
-
if (litLength == MaxLL)
|
3019
|
-
{
|
3015
|
+
if (litLength == MaxLL) {
|
3020
3016
|
U32 add = *dumps++;
|
3021
3017
|
if (add < 255) litLength += add;
|
3022
|
-
else
|
3023
|
-
|
3024
|
-
litLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
|
3018
|
+
else {
|
3019
|
+
litLength = dumps[0] + (dumps[1]<<8) + (dumps[2]<<16);
|
3025
3020
|
dumps += 3;
|
3026
3021
|
}
|
3027
|
-
if (dumps
|
3022
|
+
if (dumps > de) { litLength = MaxLL+255; } /* late correction, to avoid using uninitialized memory */
|
3023
|
+
if (dumps >= de) { dumps = de-1; } /* late correction, to avoid read overflow (data is now corrupted anyway) */
|
3028
3024
|
}
|
3029
3025
|
|
3030
3026
|
/* Offset */
|
3031
|
-
{
|
3032
|
-
static const U32 offsetPrefix[MaxOff+1] = {
|
3027
|
+
{ static const U32 offsetPrefix[MaxOff+1] = {
|
3033
3028
|
1 /*fake*/, 1, 2, 4, 8, 16, 32, 64, 128, 256,
|
3034
3029
|
512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144,
|
3035
3030
|
524288, 1048576, 2097152, 4194304, 8388608, 16777216, 33554432, /*fake*/ 1, 1, 1, 1, 1 };
|
@@ -3046,16 +3041,15 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
3046
3041
|
|
3047
3042
|
/* MatchLength */
|
3048
3043
|
matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
|
3049
|
-
if (matchLength == MaxML)
|
3050
|
-
{
|
3044
|
+
if (matchLength == MaxML) {
|
3051
3045
|
U32 add = *dumps++;
|
3052
3046
|
if (add < 255) matchLength += add;
|
3053
|
-
else
|
3054
|
-
|
3055
|
-
matchLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
|
3047
|
+
else {
|
3048
|
+
matchLength = dumps[0] + (dumps[1]<<8) + (dumps[2]<<16);
|
3056
3049
|
dumps += 3;
|
3057
3050
|
}
|
3058
|
-
if (dumps
|
3051
|
+
if (dumps > de) { matchLength = MaxML+255; } /* late correction, to avoid using uninitialized memory */
|
3052
|
+
if (dumps >= de) { dumps = de-1; } /* late correction, to avoid read overflow (data is now corrupted anyway) */
|
3059
3053
|
}
|
3060
3054
|
matchLength += MINMATCH;
|
3061
3055
|
|
@@ -3069,7 +3063,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
3069
3063
|
|
3070
3064
|
static size_t ZSTD_execSequence(BYTE* op,
|
3071
3065
|
BYTE* const oend, seq_t sequence,
|
3072
|
-
const BYTE** litPtr, const BYTE* const
|
3066
|
+
const BYTE** litPtr, const BYTE* const litLimit,
|
3073
3067
|
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
|
3074
3068
|
{
|
3075
3069
|
static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
|
@@ -3084,7 +3078,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
3084
3078
|
/* check */
|
3085
3079
|
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
|
3086
3080
|
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
|
3087
|
-
if (litEnd >
|
3081
|
+
if (litEnd > litLimit) return ERROR(corruption_detected); /* risk read beyond lit buffer */
|
3088
3082
|
|
3089
3083
|
/* copy Literals */
|
3090
3084
|
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
@@ -3110,7 +3104,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
3110
3104
|
op = oLitEnd + length1;
|
3111
3105
|
sequence.matchLength -= length1;
|
3112
3106
|
match = base;
|
3113
|
-
if (op > oend_8) {
|
3107
|
+
if (op > oend_8 || sequence.matchLength < MINMATCH) {
|
3114
3108
|
while (op < oMatchEnd) *op++ = *match++;
|
3115
3109
|
return sequenceLength;
|
3116
3110
|
}
|
@@ -3119,8 +3113,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
3119
3113
|
/* Requirement: op <= oend_8 */
|
3120
3114
|
|
3121
3115
|
/* match within prefix */
|
3122
|
-
if (sequence.offset < 8)
|
3123
|
-
{
|
3116
|
+
if (sequence.offset < 8) {
|
3124
3117
|
/* close range match, overlap */
|
3125
3118
|
const int sub2 = dec64table[sequence.offset];
|
3126
3119
|
op[0] = match[0];
|
@@ -3130,14 +3123,12 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
3130
3123
|
match += dec32table[sequence.offset];
|
3131
3124
|
ZSTD_copy4(op+4, match);
|
3132
3125
|
match -= sub2;
|
3133
|
-
}
|
3134
|
-
else
|
3135
|
-
{
|
3126
|
+
} else {
|
3136
3127
|
ZSTD_copy8(op, match);
|
3137
3128
|
}
|
3138
3129
|
op += 8; match += 8;
|
3139
3130
|
|
3140
|
-
if (oMatchEnd > oend-
|
3131
|
+
if (oMatchEnd > oend-(16-MINMATCH))
|
3141
3132
|
{
|
3142
3133
|
if (op < oend_8)
|
3143
3134
|
{
|
@@ -3149,7 +3140,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
3149
3140
|
}
|
3150
3141
|
else
|
3151
3142
|
{
|
3152
|
-
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
|
3143
|
+
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
|
3153
3144
|
}
|
3154
3145
|
return sequenceLength;
|
3155
3146
|
}
|
@@ -3167,7 +3158,6 @@ static size_t ZSTD_decompressSequences(
|
|
3167
3158
|
BYTE* const oend = ostart + maxDstSize;
|
3168
3159
|
size_t errorCode, dumpsLength;
|
3169
3160
|
const BYTE* litPtr = dctx->litPtr;
|
3170
|
-
const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8;
|
3171
3161
|
const BYTE* const litEnd = litPtr + dctx->litSize;
|
3172
3162
|
int nbSeq;
|
3173
3163
|
const BYTE* dumps;
|
@@ -3206,7 +3196,7 @@ static size_t ZSTD_decompressSequences(
|
|
3206
3196
|
size_t oneSeqSize;
|
3207
3197
|
nbSeq--;
|
3208
3198
|
ZSTD_decodeSequence(&sequence, &seqState);
|
3209
|
-
oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr,
|
3199
|
+
oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
|
3210
3200
|
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
|
3211
3201
|
op += oneSeqSize;
|
3212
3202
|
}
|
@@ -3336,6 +3326,35 @@ static size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx,
|
|
3336
3326
|
return op-ostart;
|
3337
3327
|
}
|
3338
3328
|
|
3329
|
+
static size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
|
3330
|
+
{
|
3331
|
+
const BYTE* ip = (const BYTE*)src;
|
3332
|
+
size_t remainingSize = srcSize;
|
3333
|
+
blockProperties_t blockProperties;
|
3334
|
+
|
3335
|
+
/* Frame Header */
|
3336
|
+
if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);
|
3337
|
+
if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
|
3338
|
+
ip += ZSTD_frameHeaderSize_min; remainingSize -= ZSTD_frameHeaderSize_min;
|
3339
|
+
|
3340
|
+
/* Loop on each block */
|
3341
|
+
while (1)
|
3342
|
+
{
|
3343
|
+
size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
|
3344
|
+
if (ZSTD_isError(cBlockSize)) return cBlockSize;
|
3345
|
+
|
3346
|
+
ip += ZSTD_blockHeaderSize;
|
3347
|
+
remainingSize -= ZSTD_blockHeaderSize;
|
3348
|
+
if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
|
3349
|
+
|
3350
|
+
if (cBlockSize == 0) break; /* bt_end */
|
3351
|
+
|
3352
|
+
ip += cBlockSize;
|
3353
|
+
remainingSize -= cBlockSize;
|
3354
|
+
}
|
3355
|
+
|
3356
|
+
return ip - (const BYTE*)src;
|
3357
|
+
}
|
3339
3358
|
|
3340
3359
|
/* ******************************
|
3341
3360
|
* Streaming Decompression API
|
@@ -3763,6 +3782,10 @@ size_t ZSTDv04_decompress(void* dst, size_t maxDstSize, const void* src, size_t
|
|
3763
3782
|
#endif
|
3764
3783
|
}
|
3765
3784
|
|
3785
|
+
size_t ZSTDv04_findFrameCompressedSize(const void* src, size_t srcSize)
|
3786
|
+
{
|
3787
|
+
return ZSTD_findFrameCompressedSize(src, srcSize);
|
3788
|
+
}
|
3766
3789
|
|
3767
3790
|
size_t ZSTDv04_resetDCtx(ZSTDv04_Dctx* dctx) { return ZSTD_resetDCtx(dctx); }
|
3768
3791
|
|
@@ -34,6 +34,14 @@ ZSTDv04_decompress() : decompress ZSTD frames compliant with v0.4.x format
|
|
34
34
|
size_t ZSTDv04_decompress( void* dst, size_t maxOriginalSize,
|
35
35
|
const void* src, size_t compressedSize);
|
36
36
|
|
37
|
+
/**
|
38
|
+
ZSTDv04_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.4.x format
|
39
|
+
compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
40
|
+
return : the number of bytes that would be read to decompress this frame
|
41
|
+
or an errorCode if it fails (which can be tested using ZSTDv04_isError())
|
42
|
+
*/
|
43
|
+
size_t ZSTDv04_findFrameCompressedSize(const void* src, size_t compressedSize);
|
44
|
+
|
37
45
|
/**
|
38
46
|
ZSTDv04_isError() : tells if the result of ZSTDv04_decompress() is an error
|
39
47
|
*/
|