extzstd 0.1 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/HISTORY.ja +5 -0
- data/README.md +5 -5
- data/contrib/zstd/CONTRIBUTING.md +42 -0
- data/contrib/zstd/LICENSE-examples +11 -0
- data/contrib/zstd/Makefile +315 -0
- data/contrib/zstd/NEWS +261 -0
- data/contrib/zstd/PATENTS +33 -0
- data/contrib/zstd/README.md +121 -41
- data/contrib/zstd/TESTING.md +44 -0
- data/contrib/zstd/appveyor.yml +178 -0
- data/contrib/zstd/circle.yml +75 -0
- data/contrib/zstd/lib/BUCK +186 -0
- data/contrib/zstd/lib/Makefile +163 -0
- data/contrib/zstd/lib/README.md +77 -0
- data/contrib/zstd/{common → lib/common}/bitstream.h +7 -4
- data/contrib/zstd/{common → lib/common}/entropy_common.c +19 -23
- data/contrib/zstd/{common → lib/common}/error_private.c +0 -0
- data/contrib/zstd/{common → lib/common}/error_private.h +0 -0
- data/contrib/zstd/{common → lib/common}/fse.h +94 -34
- data/contrib/zstd/{common → lib/common}/fse_decompress.c +18 -19
- data/contrib/zstd/{common → lib/common}/huf.h +52 -20
- data/contrib/zstd/{common → lib/common}/mem.h +17 -13
- data/contrib/zstd/lib/common/pool.c +194 -0
- data/contrib/zstd/lib/common/pool.h +56 -0
- data/contrib/zstd/lib/common/threading.c +80 -0
- data/contrib/zstd/lib/common/threading.h +104 -0
- data/contrib/zstd/{common → lib/common}/xxhash.c +3 -1
- data/contrib/zstd/{common → lib/common}/xxhash.h +11 -15
- data/contrib/zstd/{common → lib/common}/zstd_common.c +1 -11
- data/contrib/zstd/{common → lib/common}/zstd_errors.h +16 -2
- data/contrib/zstd/{common → lib/common}/zstd_internal.h +17 -1
- data/contrib/zstd/{compress → lib/compress}/fse_compress.c +138 -91
- data/contrib/zstd/{compress → lib/compress}/huf_compress.c +218 -67
- data/contrib/zstd/{compress → lib/compress}/zstd_compress.c +231 -108
- data/contrib/zstd/{compress → lib/compress}/zstd_opt.h +44 -25
- data/contrib/zstd/lib/compress/zstdmt_compress.c +739 -0
- data/contrib/zstd/lib/compress/zstdmt_compress.h +78 -0
- data/contrib/zstd/{decompress → lib/decompress}/huf_decompress.c +28 -23
- data/contrib/zstd/{decompress → lib/decompress}/zstd_decompress.c +814 -176
- data/contrib/zstd/{common → lib/deprecated}/zbuff.h +60 -39
- data/contrib/zstd/lib/deprecated/zbuff_common.c +26 -0
- data/contrib/zstd/lib/deprecated/zbuff_compress.c +145 -0
- data/contrib/zstd/lib/deprecated/zbuff_decompress.c +74 -0
- data/contrib/zstd/lib/dictBuilder/cover.c +1029 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.c +0 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.h +0 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/zdict.c +68 -18
- data/contrib/zstd/lib/dictBuilder/zdict.h +201 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_legacy.h +122 -7
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.c +34 -3
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.h +8 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.c +45 -12
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.h +8 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.c +45 -12
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.h +8 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.c +56 -33
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.h +8 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.c +45 -18
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.h +7 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.c +43 -16
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.h +7 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.c +57 -23
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.h +8 -0
- data/contrib/zstd/lib/libzstd.pc.in +14 -0
- data/contrib/zstd/{zstd.h → lib/zstd.h} +206 -71
- data/ext/depend +2 -0
- data/ext/extconf.rb +4 -4
- data/ext/extzstd.c +1 -1
- data/ext/zstd_common.c +5 -5
- data/ext/zstd_compress.c +3 -3
- data/ext/zstd_decompress.c +2 -2
- data/ext/zstd_dictbuilder.c +2 -2
- data/ext/zstd_legacy_v01.c +1 -1
- data/ext/zstd_legacy_v02.c +1 -1
- data/ext/zstd_legacy_v03.c +1 -1
- data/ext/zstd_legacy_v04.c +1 -1
- data/ext/zstd_legacy_v05.c +1 -1
- data/ext/zstd_legacy_v06.c +1 -1
- data/ext/zstd_legacy_v07.c +1 -1
- data/gemstub.rb +9 -5
- data/lib/extzstd/version.rb +1 -1
- metadata +73 -51
- data/contrib/zstd/compress/zbuff_compress.c +0 -319
- data/contrib/zstd/decompress/zbuff_decompress.c +0 -252
- data/contrib/zstd/dictBuilder/zdict.h +0 -111
|
@@ -34,6 +34,14 @@ ZSTDv01_decompress() : decompress ZSTD frames compliant with v0.1.x format
|
|
|
34
34
|
size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize,
|
|
35
35
|
const void* src, size_t compressedSize);
|
|
36
36
|
|
|
37
|
+
/**
|
|
38
|
+
ZSTDv01_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.1.x format
|
|
39
|
+
compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
|
40
|
+
return : the number of bytes that would be read to decompress this frame
|
|
41
|
+
or an errorCode if it fails (which can be tested using ZSTDv01_isError())
|
|
42
|
+
*/
|
|
43
|
+
size_t ZSTDv01_findFrameCompressedSize(const void* src, size_t compressedSize);
|
|
44
|
+
|
|
37
45
|
/**
|
|
38
46
|
ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error
|
|
39
47
|
*/
|
|
@@ -2808,7 +2808,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
|
|
|
2808
2808
|
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
|
|
2809
2809
|
|
|
2810
2810
|
/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
|
|
2811
|
-
static void ZSTD_wildcopy(void* dst, const void* src,
|
|
2811
|
+
static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
|
|
2812
2812
|
{
|
|
2813
2813
|
const BYTE* ip = (const BYTE*)src;
|
|
2814
2814
|
BYTE* op = (BYTE*)dst;
|
|
@@ -2868,7 +2868,6 @@ struct ZSTD_DCtx_s
|
|
|
2868
2868
|
blockType_t bType;
|
|
2869
2869
|
U32 phase;
|
|
2870
2870
|
const BYTE* litPtr;
|
|
2871
|
-
size_t litBufSize;
|
|
2872
2871
|
size_t litSize;
|
|
2873
2872
|
BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
|
|
2874
2873
|
}; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
|
|
@@ -2940,8 +2939,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
|
2940
2939
|
size_t litSize = BLOCKSIZE;
|
|
2941
2940
|
const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
|
|
2942
2941
|
dctx->litPtr = dctx->litBuffer;
|
|
2943
|
-
dctx->litBufSize = BLOCKSIZE;
|
|
2944
2942
|
dctx->litSize = litSize;
|
|
2943
|
+
memset(dctx->litBuffer + dctx->litSize, 0, 8);
|
|
2945
2944
|
return readSize; /* works if it's an error too */
|
|
2946
2945
|
}
|
|
2947
2946
|
case IS_RAW:
|
|
@@ -2952,13 +2951,12 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
|
2952
2951
|
if (litSize > srcSize-3) return ERROR(corruption_detected);
|
|
2953
2952
|
memcpy(dctx->litBuffer, istart, litSize);
|
|
2954
2953
|
dctx->litPtr = dctx->litBuffer;
|
|
2955
|
-
dctx->litBufSize = BLOCKSIZE;
|
|
2956
2954
|
dctx->litSize = litSize;
|
|
2955
|
+
memset(dctx->litBuffer + dctx->litSize, 0, 8);
|
|
2957
2956
|
return litSize+3;
|
|
2958
2957
|
}
|
|
2959
2958
|
/* direct reference into compressed stream */
|
|
2960
2959
|
dctx->litPtr = istart+3;
|
|
2961
|
-
dctx->litBufSize = srcSize-3;
|
|
2962
2960
|
dctx->litSize = litSize;
|
|
2963
2961
|
return litSize+3;
|
|
2964
2962
|
}
|
|
@@ -2966,9 +2964,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
|
2966
2964
|
{
|
|
2967
2965
|
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
|
|
2968
2966
|
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
|
|
2969
|
-
memset(dctx->litBuffer, istart[3], litSize);
|
|
2967
|
+
memset(dctx->litBuffer, istart[3], litSize + 8);
|
|
2970
2968
|
dctx->litPtr = dctx->litBuffer;
|
|
2971
|
-
dctx->litBufSize = BLOCKSIZE;
|
|
2972
2969
|
dctx->litSize = litSize;
|
|
2973
2970
|
return 4;
|
|
2974
2971
|
}
|
|
@@ -3175,7 +3172,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
|
3175
3172
|
/* checks */
|
|
3176
3173
|
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
|
|
3177
3174
|
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
|
|
3178
|
-
if (litEnd > litLimit
|
|
3175
|
+
if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
|
|
3179
3176
|
|
|
3180
3177
|
/* copy Literals */
|
|
3181
3178
|
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
|
@@ -3209,7 +3206,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
|
3209
3206
|
}
|
|
3210
3207
|
op += 8; match += 8;
|
|
3211
3208
|
|
|
3212
|
-
if (oMatchEnd > oend-
|
|
3209
|
+
if (oMatchEnd > oend-(16-MINMATCH))
|
|
3213
3210
|
{
|
|
3214
3211
|
if (op < oend_8)
|
|
3215
3212
|
{
|
|
@@ -3221,7 +3218,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
|
3221
3218
|
}
|
|
3222
3219
|
else
|
|
3223
3220
|
{
|
|
3224
|
-
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
|
|
3221
|
+
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
|
|
3225
3222
|
}
|
|
3226
3223
|
}
|
|
3227
3224
|
|
|
@@ -3241,7 +3238,6 @@ static size_t ZSTD_decompressSequences(
|
|
|
3241
3238
|
BYTE* const oend = ostart + maxDstSize;
|
|
3242
3239
|
size_t errorCode, dumpsLength;
|
|
3243
3240
|
const BYTE* litPtr = dctx->litPtr;
|
|
3244
|
-
const BYTE* const litMax = litPtr + dctx->litBufSize;
|
|
3245
3241
|
const BYTE* const litEnd = litPtr + dctx->litSize;
|
|
3246
3242
|
int nbSeq;
|
|
3247
3243
|
const BYTE* dumps;
|
|
@@ -3277,7 +3273,7 @@ static size_t ZSTD_decompressSequences(
|
|
|
3277
3273
|
size_t oneSeqSize;
|
|
3278
3274
|
nbSeq--;
|
|
3279
3275
|
ZSTD_decodeSequence(&sequence, &seqState);
|
|
3280
|
-
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr,
|
|
3276
|
+
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
|
|
3281
3277
|
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
|
|
3282
3278
|
op += oneSeqSize;
|
|
3283
3279
|
}
|
|
@@ -3382,6 +3378,38 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
|
|
|
3382
3378
|
return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
|
3383
3379
|
}
|
|
3384
3380
|
|
|
3381
|
+
static size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
|
|
3382
|
+
{
|
|
3383
|
+
|
|
3384
|
+
const BYTE* ip = (const BYTE*)src;
|
|
3385
|
+
size_t remainingSize = srcSize;
|
|
3386
|
+
U32 magicNumber;
|
|
3387
|
+
blockProperties_t blockProperties;
|
|
3388
|
+
|
|
3389
|
+
/* Frame Header */
|
|
3390
|
+
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
|
|
3391
|
+
magicNumber = MEM_readLE32(src);
|
|
3392
|
+
if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
|
|
3393
|
+
ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
|
|
3394
|
+
|
|
3395
|
+
/* Loop on each block */
|
|
3396
|
+
while (1)
|
|
3397
|
+
{
|
|
3398
|
+
size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
|
|
3399
|
+
if (ZSTD_isError(cBlockSize)) return cBlockSize;
|
|
3400
|
+
|
|
3401
|
+
ip += ZSTD_blockHeaderSize;
|
|
3402
|
+
remainingSize -= ZSTD_blockHeaderSize;
|
|
3403
|
+
if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
|
|
3404
|
+
|
|
3405
|
+
if (cBlockSize == 0) break; /* bt_end */
|
|
3406
|
+
|
|
3407
|
+
ip += cBlockSize;
|
|
3408
|
+
remainingSize -= cBlockSize;
|
|
3409
|
+
}
|
|
3410
|
+
|
|
3411
|
+
return ip - (const BYTE*)src;
|
|
3412
|
+
}
|
|
3385
3413
|
|
|
3386
3414
|
/*******************************
|
|
3387
3415
|
* Streaming Decompression API
|
|
@@ -3496,6 +3524,11 @@ size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
|
|
|
3496
3524
|
return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
|
|
3497
3525
|
}
|
|
3498
3526
|
|
|
3527
|
+
size_t ZSTDv02_findFrameCompressedSize(const void *src, size_t compressedSize)
|
|
3528
|
+
{
|
|
3529
|
+
return ZSTD_findFrameCompressedSize(src, compressedSize);
|
|
3530
|
+
}
|
|
3531
|
+
|
|
3499
3532
|
ZSTDv02_Dctx* ZSTDv02_createDCtx(void)
|
|
3500
3533
|
{
|
|
3501
3534
|
return (ZSTDv02_Dctx*)ZSTD_createDCtx();
|
|
@@ -34,6 +34,14 @@ ZSTDv02_decompress() : decompress ZSTD frames compliant with v0.2.x format
|
|
|
34
34
|
size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
|
|
35
35
|
const void* src, size_t compressedSize);
|
|
36
36
|
|
|
37
|
+
/**
|
|
38
|
+
ZSTDv02_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.2.x format
|
|
39
|
+
compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
|
40
|
+
return : the number of bytes that would be read to decompress this frame
|
|
41
|
+
or an errorCode if it fails (which can be tested using ZSTDv02_isError())
|
|
42
|
+
*/
|
|
43
|
+
size_t ZSTDv02_findFrameCompressedSize(const void* src, size_t compressedSize);
|
|
44
|
+
|
|
37
45
|
/**
|
|
38
46
|
ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error
|
|
39
47
|
*/
|
|
@@ -2449,7 +2449,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
|
|
|
2449
2449
|
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
|
|
2450
2450
|
|
|
2451
2451
|
/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
|
|
2452
|
-
static void ZSTD_wildcopy(void* dst, const void* src,
|
|
2452
|
+
static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
|
|
2453
2453
|
{
|
|
2454
2454
|
const BYTE* ip = (const BYTE*)src;
|
|
2455
2455
|
BYTE* op = (BYTE*)dst;
|
|
@@ -2509,7 +2509,6 @@ struct ZSTD_DCtx_s
|
|
|
2509
2509
|
blockType_t bType;
|
|
2510
2510
|
U32 phase;
|
|
2511
2511
|
const BYTE* litPtr;
|
|
2512
|
-
size_t litBufSize;
|
|
2513
2512
|
size_t litSize;
|
|
2514
2513
|
BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
|
|
2515
2514
|
}; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
|
|
@@ -2581,8 +2580,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
|
2581
2580
|
size_t litSize = BLOCKSIZE;
|
|
2582
2581
|
const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
|
|
2583
2582
|
dctx->litPtr = dctx->litBuffer;
|
|
2584
|
-
dctx->litBufSize = BLOCKSIZE;
|
|
2585
2583
|
dctx->litSize = litSize;
|
|
2584
|
+
memset(dctx->litBuffer + dctx->litSize, 0, 8);
|
|
2586
2585
|
return readSize; /* works if it's an error too */
|
|
2587
2586
|
}
|
|
2588
2587
|
case IS_RAW:
|
|
@@ -2593,13 +2592,12 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
|
2593
2592
|
if (litSize > srcSize-3) return ERROR(corruption_detected);
|
|
2594
2593
|
memcpy(dctx->litBuffer, istart, litSize);
|
|
2595
2594
|
dctx->litPtr = dctx->litBuffer;
|
|
2596
|
-
dctx->litBufSize = BLOCKSIZE;
|
|
2597
2595
|
dctx->litSize = litSize;
|
|
2596
|
+
memset(dctx->litBuffer + dctx->litSize, 0, 8);
|
|
2598
2597
|
return litSize+3;
|
|
2599
2598
|
}
|
|
2600
2599
|
/* direct reference into compressed stream */
|
|
2601
2600
|
dctx->litPtr = istart+3;
|
|
2602
|
-
dctx->litBufSize = srcSize-3;
|
|
2603
2601
|
dctx->litSize = litSize;
|
|
2604
2602
|
return litSize+3;
|
|
2605
2603
|
}
|
|
@@ -2607,9 +2605,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
|
2607
2605
|
{
|
|
2608
2606
|
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
|
|
2609
2607
|
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
|
|
2610
|
-
memset(dctx->litBuffer, istart[3], litSize);
|
|
2608
|
+
memset(dctx->litBuffer, istart[3], litSize + 8);
|
|
2611
2609
|
dctx->litPtr = dctx->litBuffer;
|
|
2612
|
-
dctx->litBufSize = BLOCKSIZE;
|
|
2613
2610
|
dctx->litSize = litSize;
|
|
2614
2611
|
return 4;
|
|
2615
2612
|
}
|
|
@@ -2816,7 +2813,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
|
2816
2813
|
/* checks */
|
|
2817
2814
|
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
|
|
2818
2815
|
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
|
|
2819
|
-
if (litEnd > litLimit
|
|
2816
|
+
if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
|
|
2820
2817
|
|
|
2821
2818
|
/* copy Literals */
|
|
2822
2819
|
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
|
@@ -2850,7 +2847,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
|
2850
2847
|
}
|
|
2851
2848
|
op += 8; match += 8;
|
|
2852
2849
|
|
|
2853
|
-
if (oMatchEnd > oend-
|
|
2850
|
+
if (oMatchEnd > oend-(16-MINMATCH))
|
|
2854
2851
|
{
|
|
2855
2852
|
if (op < oend_8)
|
|
2856
2853
|
{
|
|
@@ -2862,7 +2859,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
|
2862
2859
|
}
|
|
2863
2860
|
else
|
|
2864
2861
|
{
|
|
2865
|
-
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
|
|
2862
|
+
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
|
|
2866
2863
|
}
|
|
2867
2864
|
}
|
|
2868
2865
|
|
|
@@ -2882,7 +2879,6 @@ static size_t ZSTD_decompressSequences(
|
|
|
2882
2879
|
BYTE* const oend = ostart + maxDstSize;
|
|
2883
2880
|
size_t errorCode, dumpsLength;
|
|
2884
2881
|
const BYTE* litPtr = dctx->litPtr;
|
|
2885
|
-
const BYTE* const litMax = litPtr + dctx->litBufSize;
|
|
2886
2882
|
const BYTE* const litEnd = litPtr + dctx->litSize;
|
|
2887
2883
|
int nbSeq;
|
|
2888
2884
|
const BYTE* dumps;
|
|
@@ -2918,7 +2914,7 @@ static size_t ZSTD_decompressSequences(
|
|
|
2918
2914
|
size_t oneSeqSize;
|
|
2919
2915
|
nbSeq--;
|
|
2920
2916
|
ZSTD_decodeSequence(&sequence, &seqState);
|
|
2921
|
-
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr,
|
|
2917
|
+
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
|
|
2922
2918
|
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
|
|
2923
2919
|
op += oneSeqSize;
|
|
2924
2920
|
}
|
|
@@ -3023,6 +3019,38 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
|
|
|
3023
3019
|
return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
|
3024
3020
|
}
|
|
3025
3021
|
|
|
3022
|
+
static size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
|
|
3023
|
+
{
|
|
3024
|
+
const BYTE* ip = (const BYTE*)src;
|
|
3025
|
+
size_t remainingSize = srcSize;
|
|
3026
|
+
U32 magicNumber;
|
|
3027
|
+
blockProperties_t blockProperties;
|
|
3028
|
+
|
|
3029
|
+
/* Frame Header */
|
|
3030
|
+
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
|
|
3031
|
+
magicNumber = MEM_readLE32(src);
|
|
3032
|
+
if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
|
|
3033
|
+
ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
|
|
3034
|
+
|
|
3035
|
+
/* Loop on each block */
|
|
3036
|
+
while (1)
|
|
3037
|
+
{
|
|
3038
|
+
size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
|
|
3039
|
+
if (ZSTD_isError(cBlockSize)) return cBlockSize;
|
|
3040
|
+
|
|
3041
|
+
ip += ZSTD_blockHeaderSize;
|
|
3042
|
+
remainingSize -= ZSTD_blockHeaderSize;
|
|
3043
|
+
if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
|
|
3044
|
+
|
|
3045
|
+
if (cBlockSize == 0) break; /* bt_end */
|
|
3046
|
+
|
|
3047
|
+
ip += cBlockSize;
|
|
3048
|
+
remainingSize -= cBlockSize;
|
|
3049
|
+
}
|
|
3050
|
+
|
|
3051
|
+
return ip - (const BYTE*)src;
|
|
3052
|
+
}
|
|
3053
|
+
|
|
3026
3054
|
|
|
3027
3055
|
/*******************************
|
|
3028
3056
|
* Streaming Decompression API
|
|
@@ -3137,6 +3165,11 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
|
|
|
3137
3165
|
return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
|
|
3138
3166
|
}
|
|
3139
3167
|
|
|
3168
|
+
size_t ZSTDv03_findFrameCompressedSize(const void* src, size_t srcSize)
|
|
3169
|
+
{
|
|
3170
|
+
return ZSTD_findFrameCompressedSize(src, srcSize);
|
|
3171
|
+
}
|
|
3172
|
+
|
|
3140
3173
|
ZSTDv03_Dctx* ZSTDv03_createDCtx(void)
|
|
3141
3174
|
{
|
|
3142
3175
|
return (ZSTDv03_Dctx*)ZSTD_createDCtx();
|
|
@@ -35,6 +35,14 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
|
|
|
35
35
|
const void* src, size_t compressedSize);
|
|
36
36
|
|
|
37
37
|
/**
|
|
38
|
+
ZSTDv03_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.3.x format
|
|
39
|
+
compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
|
40
|
+
return : the number of bytes that would be read to decompress this frame
|
|
41
|
+
or an errorCode if it fails (which can be tested using ZSTDv03_isError())
|
|
42
|
+
*/
|
|
43
|
+
size_t ZSTDv03_findFrameCompressedSize(const void* src, size_t compressedSize);
|
|
44
|
+
|
|
45
|
+
/**
|
|
38
46
|
ZSTDv03_isError() : tells if the result of ZSTDv03_decompress() is an error
|
|
39
47
|
*/
|
|
40
48
|
unsigned ZSTDv03_isError(size_t code);
|
|
@@ -487,7 +487,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
|
|
|
487
487
|
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
|
|
488
488
|
|
|
489
489
|
/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
|
|
490
|
-
static void ZSTD_wildcopy(void* dst, const void* src,
|
|
490
|
+
static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
|
|
491
491
|
{
|
|
492
492
|
const BYTE* ip = (const BYTE*)src;
|
|
493
493
|
BYTE* op = (BYTE*)dst;
|
|
@@ -2706,7 +2706,6 @@ struct ZSTDv04_Dctx_s
|
|
|
2706
2706
|
blockType_t bType;
|
|
2707
2707
|
ZSTD_dStage stage;
|
|
2708
2708
|
const BYTE* litPtr;
|
|
2709
|
-
size_t litBufSize;
|
|
2710
2709
|
size_t litSize;
|
|
2711
2710
|
BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
|
|
2712
2711
|
BYTE headerBuffer[ZSTD_frameHeaderSize_max];
|
|
@@ -2847,8 +2846,8 @@ static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
|
|
|
2847
2846
|
size_t litSize = BLOCKSIZE;
|
|
2848
2847
|
const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
|
|
2849
2848
|
dctx->litPtr = dctx->litBuffer;
|
|
2850
|
-
dctx->litBufSize = BLOCKSIZE+8;
|
|
2851
2849
|
dctx->litSize = litSize;
|
|
2850
|
+
memset(dctx->litBuffer + dctx->litSize, 0, 8);
|
|
2852
2851
|
return readSize; /* works if it's an error too */
|
|
2853
2852
|
}
|
|
2854
2853
|
case IS_RAW:
|
|
@@ -2859,22 +2858,20 @@ static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
|
|
|
2859
2858
|
if (litSize > srcSize-3) return ERROR(corruption_detected);
|
|
2860
2859
|
memcpy(dctx->litBuffer, istart, litSize);
|
|
2861
2860
|
dctx->litPtr = dctx->litBuffer;
|
|
2862
|
-
dctx->litBufSize = BLOCKSIZE+8;
|
|
2863
2861
|
dctx->litSize = litSize;
|
|
2862
|
+
memset(dctx->litBuffer + dctx->litSize, 0, 8);
|
|
2864
2863
|
return litSize+3;
|
|
2865
2864
|
}
|
|
2866
2865
|
/* direct reference into compressed stream */
|
|
2867
2866
|
dctx->litPtr = istart+3;
|
|
2868
|
-
dctx->litBufSize = srcSize-3;
|
|
2869
2867
|
dctx->litSize = litSize;
|
|
2870
2868
|
return litSize+3; }
|
|
2871
2869
|
case IS_RLE:
|
|
2872
2870
|
{
|
|
2873
2871
|
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
|
|
2874
2872
|
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
|
|
2875
|
-
memset(dctx->litBuffer, istart[3], litSize);
|
|
2873
|
+
memset(dctx->litBuffer, istart[3], litSize + 8);
|
|
2876
2874
|
dctx->litPtr = dctx->litBuffer;
|
|
2877
|
-
dctx->litBufSize = BLOCKSIZE+8;
|
|
2878
2875
|
dctx->litSize = litSize;
|
|
2879
2876
|
return 4;
|
|
2880
2877
|
}
|
|
@@ -3015,21 +3012,19 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
|
3015
3012
|
/* Literal length */
|
|
3016
3013
|
litLength = FSE_decodeSymbol(&(seqState->stateLL), &(seqState->DStream));
|
|
3017
3014
|
prevOffset = litLength ? seq->offset : seqState->prevOffset;
|
|
3018
|
-
if (litLength == MaxLL)
|
|
3019
|
-
{
|
|
3015
|
+
if (litLength == MaxLL) {
|
|
3020
3016
|
U32 add = *dumps++;
|
|
3021
3017
|
if (add < 255) litLength += add;
|
|
3022
|
-
else
|
|
3023
|
-
|
|
3024
|
-
litLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
|
|
3018
|
+
else {
|
|
3019
|
+
litLength = dumps[0] + (dumps[1]<<8) + (dumps[2]<<16);
|
|
3025
3020
|
dumps += 3;
|
|
3026
3021
|
}
|
|
3027
|
-
if (dumps
|
|
3022
|
+
if (dumps > de) { litLength = MaxLL+255; } /* late correction, to avoid using uninitialized memory */
|
|
3023
|
+
if (dumps >= de) { dumps = de-1; } /* late correction, to avoid read overflow (data is now corrupted anyway) */
|
|
3028
3024
|
}
|
|
3029
3025
|
|
|
3030
3026
|
/* Offset */
|
|
3031
|
-
{
|
|
3032
|
-
static const U32 offsetPrefix[MaxOff+1] = {
|
|
3027
|
+
{ static const U32 offsetPrefix[MaxOff+1] = {
|
|
3033
3028
|
1 /*fake*/, 1, 2, 4, 8, 16, 32, 64, 128, 256,
|
|
3034
3029
|
512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144,
|
|
3035
3030
|
524288, 1048576, 2097152, 4194304, 8388608, 16777216, 33554432, /*fake*/ 1, 1, 1, 1, 1 };
|
|
@@ -3046,16 +3041,15 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
|
3046
3041
|
|
|
3047
3042
|
/* MatchLength */
|
|
3048
3043
|
matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
|
|
3049
|
-
if (matchLength == MaxML)
|
|
3050
|
-
{
|
|
3044
|
+
if (matchLength == MaxML) {
|
|
3051
3045
|
U32 add = *dumps++;
|
|
3052
3046
|
if (add < 255) matchLength += add;
|
|
3053
|
-
else
|
|
3054
|
-
|
|
3055
|
-
matchLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
|
|
3047
|
+
else {
|
|
3048
|
+
matchLength = dumps[0] + (dumps[1]<<8) + (dumps[2]<<16);
|
|
3056
3049
|
dumps += 3;
|
|
3057
3050
|
}
|
|
3058
|
-
if (dumps
|
|
3051
|
+
if (dumps > de) { matchLength = MaxML+255; } /* late correction, to avoid using uninitialized memory */
|
|
3052
|
+
if (dumps >= de) { dumps = de-1; } /* late correction, to avoid read overflow (data is now corrupted anyway) */
|
|
3059
3053
|
}
|
|
3060
3054
|
matchLength += MINMATCH;
|
|
3061
3055
|
|
|
@@ -3069,7 +3063,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
|
3069
3063
|
|
|
3070
3064
|
static size_t ZSTD_execSequence(BYTE* op,
|
|
3071
3065
|
BYTE* const oend, seq_t sequence,
|
|
3072
|
-
const BYTE** litPtr, const BYTE* const
|
|
3066
|
+
const BYTE** litPtr, const BYTE* const litLimit,
|
|
3073
3067
|
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
|
|
3074
3068
|
{
|
|
3075
3069
|
static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
|
|
@@ -3084,7 +3078,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
|
3084
3078
|
/* check */
|
|
3085
3079
|
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
|
|
3086
3080
|
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
|
|
3087
|
-
if (litEnd >
|
|
3081
|
+
if (litEnd > litLimit) return ERROR(corruption_detected); /* risk read beyond lit buffer */
|
|
3088
3082
|
|
|
3089
3083
|
/* copy Literals */
|
|
3090
3084
|
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
|
@@ -3110,7 +3104,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
|
3110
3104
|
op = oLitEnd + length1;
|
|
3111
3105
|
sequence.matchLength -= length1;
|
|
3112
3106
|
match = base;
|
|
3113
|
-
if (op > oend_8) {
|
|
3107
|
+
if (op > oend_8 || sequence.matchLength < MINMATCH) {
|
|
3114
3108
|
while (op < oMatchEnd) *op++ = *match++;
|
|
3115
3109
|
return sequenceLength;
|
|
3116
3110
|
}
|
|
@@ -3119,8 +3113,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
|
3119
3113
|
/* Requirement: op <= oend_8 */
|
|
3120
3114
|
|
|
3121
3115
|
/* match within prefix */
|
|
3122
|
-
if (sequence.offset < 8)
|
|
3123
|
-
{
|
|
3116
|
+
if (sequence.offset < 8) {
|
|
3124
3117
|
/* close range match, overlap */
|
|
3125
3118
|
const int sub2 = dec64table[sequence.offset];
|
|
3126
3119
|
op[0] = match[0];
|
|
@@ -3130,14 +3123,12 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
|
3130
3123
|
match += dec32table[sequence.offset];
|
|
3131
3124
|
ZSTD_copy4(op+4, match);
|
|
3132
3125
|
match -= sub2;
|
|
3133
|
-
}
|
|
3134
|
-
else
|
|
3135
|
-
{
|
|
3126
|
+
} else {
|
|
3136
3127
|
ZSTD_copy8(op, match);
|
|
3137
3128
|
}
|
|
3138
3129
|
op += 8; match += 8;
|
|
3139
3130
|
|
|
3140
|
-
if (oMatchEnd > oend-
|
|
3131
|
+
if (oMatchEnd > oend-(16-MINMATCH))
|
|
3141
3132
|
{
|
|
3142
3133
|
if (op < oend_8)
|
|
3143
3134
|
{
|
|
@@ -3149,7 +3140,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
|
3149
3140
|
}
|
|
3150
3141
|
else
|
|
3151
3142
|
{
|
|
3152
|
-
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
|
|
3143
|
+
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
|
|
3153
3144
|
}
|
|
3154
3145
|
return sequenceLength;
|
|
3155
3146
|
}
|
|
@@ -3167,7 +3158,6 @@ static size_t ZSTD_decompressSequences(
|
|
|
3167
3158
|
BYTE* const oend = ostart + maxDstSize;
|
|
3168
3159
|
size_t errorCode, dumpsLength;
|
|
3169
3160
|
const BYTE* litPtr = dctx->litPtr;
|
|
3170
|
-
const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8;
|
|
3171
3161
|
const BYTE* const litEnd = litPtr + dctx->litSize;
|
|
3172
3162
|
int nbSeq;
|
|
3173
3163
|
const BYTE* dumps;
|
|
@@ -3206,7 +3196,7 @@ static size_t ZSTD_decompressSequences(
|
|
|
3206
3196
|
size_t oneSeqSize;
|
|
3207
3197
|
nbSeq--;
|
|
3208
3198
|
ZSTD_decodeSequence(&sequence, &seqState);
|
|
3209
|
-
oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr,
|
|
3199
|
+
oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
|
|
3210
3200
|
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
|
|
3211
3201
|
op += oneSeqSize;
|
|
3212
3202
|
}
|
|
@@ -3336,6 +3326,35 @@ static size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx,
|
|
|
3336
3326
|
return op-ostart;
|
|
3337
3327
|
}
|
|
3338
3328
|
|
|
3329
|
+
static size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
|
|
3330
|
+
{
|
|
3331
|
+
const BYTE* ip = (const BYTE*)src;
|
|
3332
|
+
size_t remainingSize = srcSize;
|
|
3333
|
+
blockProperties_t blockProperties;
|
|
3334
|
+
|
|
3335
|
+
/* Frame Header */
|
|
3336
|
+
if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);
|
|
3337
|
+
if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
|
|
3338
|
+
ip += ZSTD_frameHeaderSize_min; remainingSize -= ZSTD_frameHeaderSize_min;
|
|
3339
|
+
|
|
3340
|
+
/* Loop on each block */
|
|
3341
|
+
while (1)
|
|
3342
|
+
{
|
|
3343
|
+
size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
|
|
3344
|
+
if (ZSTD_isError(cBlockSize)) return cBlockSize;
|
|
3345
|
+
|
|
3346
|
+
ip += ZSTD_blockHeaderSize;
|
|
3347
|
+
remainingSize -= ZSTD_blockHeaderSize;
|
|
3348
|
+
if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
|
|
3349
|
+
|
|
3350
|
+
if (cBlockSize == 0) break; /* bt_end */
|
|
3351
|
+
|
|
3352
|
+
ip += cBlockSize;
|
|
3353
|
+
remainingSize -= cBlockSize;
|
|
3354
|
+
}
|
|
3355
|
+
|
|
3356
|
+
return ip - (const BYTE*)src;
|
|
3357
|
+
}
|
|
3339
3358
|
|
|
3340
3359
|
/* ******************************
|
|
3341
3360
|
* Streaming Decompression API
|
|
@@ -3763,6 +3782,10 @@ size_t ZSTDv04_decompress(void* dst, size_t maxDstSize, const void* src, size_t
|
|
|
3763
3782
|
#endif
|
|
3764
3783
|
}
|
|
3765
3784
|
|
|
3785
|
+
size_t ZSTDv04_findFrameCompressedSize(const void* src, size_t srcSize)
|
|
3786
|
+
{
|
|
3787
|
+
return ZSTD_findFrameCompressedSize(src, srcSize);
|
|
3788
|
+
}
|
|
3766
3789
|
|
|
3767
3790
|
size_t ZSTDv04_resetDCtx(ZSTDv04_Dctx* dctx) { return ZSTD_resetDCtx(dctx); }
|
|
3768
3791
|
|
|
@@ -34,6 +34,14 @@ ZSTDv04_decompress() : decompress ZSTD frames compliant with v0.4.x format
|
|
|
34
34
|
size_t ZSTDv04_decompress( void* dst, size_t maxOriginalSize,
|
|
35
35
|
const void* src, size_t compressedSize);
|
|
36
36
|
|
|
37
|
+
/**
|
|
38
|
+
ZSTDv04_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.4.x format
|
|
39
|
+
compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
|
40
|
+
return : the number of bytes that would be read to decompress this frame
|
|
41
|
+
or an errorCode if it fails (which can be tested using ZSTDv04_isError())
|
|
42
|
+
*/
|
|
43
|
+
size_t ZSTDv04_findFrameCompressedSize(const void* src, size_t compressedSize);
|
|
44
|
+
|
|
37
45
|
/**
|
|
38
46
|
ZSTDv04_isError() : tells if the result of ZSTDv04_decompress() is an error
|
|
39
47
|
*/
|