extzstd 0.2 → 0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/HISTORY.ja.md +13 -0
- data/README.md +17 -14
- data/contrib/zstd/{NEWS → CHANGELOG} +115 -2
- data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
- data/contrib/zstd/Makefile +99 -53
- data/contrib/zstd/README.md +59 -39
- data/contrib/zstd/TESTING.md +1 -1
- data/contrib/zstd/appveyor.yml +17 -6
- data/contrib/zstd/lib/BUCK +29 -2
- data/contrib/zstd/lib/Makefile +118 -21
- data/contrib/zstd/lib/README.md +84 -44
- data/contrib/zstd/lib/common/bitstream.h +17 -33
- data/contrib/zstd/lib/common/compiler.h +62 -8
- data/contrib/zstd/lib/common/cpu.h +215 -0
- data/contrib/zstd/lib/common/debug.c +44 -0
- data/contrib/zstd/lib/common/debug.h +134 -0
- data/contrib/zstd/lib/common/entropy_common.c +16 -1
- data/contrib/zstd/lib/common/error_private.c +7 -0
- data/contrib/zstd/lib/common/fse.h +48 -44
- data/contrib/zstd/lib/common/fse_decompress.c +3 -3
- data/contrib/zstd/lib/common/huf.h +169 -113
- data/contrib/zstd/lib/common/mem.h +20 -2
- data/contrib/zstd/lib/common/pool.c +135 -49
- data/contrib/zstd/lib/common/pool.h +40 -21
- data/contrib/zstd/lib/common/threading.c +2 -2
- data/contrib/zstd/lib/common/threading.h +12 -12
- data/contrib/zstd/lib/common/xxhash.c +3 -2
- data/contrib/zstd/lib/common/zstd_common.c +3 -6
- data/contrib/zstd/lib/common/zstd_errors.h +17 -7
- data/contrib/zstd/lib/common/zstd_internal.h +76 -48
- data/contrib/zstd/lib/compress/fse_compress.c +89 -209
- data/contrib/zstd/lib/compress/hist.c +203 -0
- data/contrib/zstd/lib/compress/hist.h +95 -0
- data/contrib/zstd/lib/compress/huf_compress.c +188 -80
- data/contrib/zstd/lib/compress/zstd_compress.c +2500 -1203
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +463 -62
- data/contrib/zstd/lib/compress/zstd_double_fast.c +321 -131
- data/contrib/zstd/lib/compress/zstd_double_fast.h +13 -4
- data/contrib/zstd/lib/compress/zstd_fast.c +335 -108
- data/contrib/zstd/lib/compress/zstd_fast.h +12 -6
- data/contrib/zstd/lib/compress/zstd_lazy.c +654 -313
- data/contrib/zstd/lib/compress/zstd_lazy.h +44 -16
- data/contrib/zstd/lib/compress/zstd_ldm.c +310 -420
- data/contrib/zstd/lib/compress/zstd_ldm.h +63 -26
- data/contrib/zstd/lib/compress/zstd_opt.c +773 -325
- data/contrib/zstd/lib/compress/zstd_opt.h +31 -5
- data/contrib/zstd/lib/compress/zstdmt_compress.c +1468 -518
- data/contrib/zstd/lib/compress/zstdmt_compress.h +96 -45
- data/contrib/zstd/lib/decompress/huf_decompress.c +518 -282
- data/contrib/zstd/lib/decompress/zstd_ddict.c +240 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
- data/contrib/zstd/lib/decompress/zstd_decompress.c +613 -1513
- data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1311 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.h +59 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +175 -0
- data/contrib/zstd/lib/dictBuilder/cover.c +194 -113
- data/contrib/zstd/lib/dictBuilder/cover.h +112 -0
- data/contrib/zstd/lib/dictBuilder/divsufsort.c +3 -3
- data/contrib/zstd/lib/dictBuilder/fastcover.c +740 -0
- data/contrib/zstd/lib/dictBuilder/zdict.c +142 -106
- data/contrib/zstd/lib/dictBuilder/zdict.h +115 -49
- data/contrib/zstd/lib/legacy/zstd_legacy.h +44 -12
- data/contrib/zstd/lib/legacy/zstd_v01.c +41 -10
- data/contrib/zstd/lib/legacy/zstd_v01.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v02.c +37 -12
- data/contrib/zstd/lib/legacy/zstd_v02.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v03.c +38 -12
- data/contrib/zstd/lib/legacy/zstd_v03.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v04.c +55 -174
- data/contrib/zstd/lib/legacy/zstd_v04.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v05.c +59 -31
- data/contrib/zstd/lib/legacy/zstd_v05.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v06.c +48 -20
- data/contrib/zstd/lib/legacy/zstd_v06.h +10 -5
- data/contrib/zstd/lib/legacy/zstd_v07.c +62 -29
- data/contrib/zstd/lib/legacy/zstd_v07.h +10 -5
- data/contrib/zstd/lib/zstd.h +1346 -832
- data/ext/extzstd.c +27 -19
- data/ext/extzstd_stream.c +20 -4
- data/ext/zstd_compress.c +1 -0
- data/ext/zstd_decompress.c +4 -0
- data/ext/zstd_dictbuilder.c +4 -0
- data/ext/zstd_dictbuilder_fastcover.c +5 -0
- data/lib/extzstd.rb +52 -220
- data/lib/extzstd/version.rb +1 -1
- metadata +21 -7
- data/contrib/zstd/circle.yml +0 -63
|
@@ -43,12 +43,17 @@ ZSTDLIBv06_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity,
|
|
|
43
43
|
const void* src, size_t compressedSize);
|
|
44
44
|
|
|
45
45
|
/**
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
46
|
+
ZSTDv06_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.6.x format
|
|
47
|
+
srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
|
48
|
+
cSize (output parameter) : the number of bytes that would be read to decompress this frame
|
|
49
|
+
or an error code if it fails (which can be tested using ZSTDv01_isError())
|
|
50
|
+
dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
|
|
51
|
+
or ZSTD_CONTENTSIZE_ERROR if an error occurs
|
|
52
|
+
|
|
53
|
+
note : assumes `cSize` and `dBound` are _not_ NULL.
|
|
50
54
|
*/
|
|
51
|
-
|
|
55
|
+
void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
|
|
56
|
+
size_t* cSize, unsigned long long* dBound);
|
|
52
57
|
|
|
53
58
|
/* *************************************
|
|
54
59
|
* Helper functions
|
|
@@ -348,7 +348,7 @@ MEM_STATIC U32 MEM_swap32(U32 in)
|
|
|
348
348
|
{
|
|
349
349
|
#if defined(_MSC_VER) /* Visual Studio */
|
|
350
350
|
return _byteswap_ulong(in);
|
|
351
|
-
#elif defined (__GNUC__)
|
|
351
|
+
#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
|
|
352
352
|
return __builtin_bswap32(in);
|
|
353
353
|
#else
|
|
354
354
|
return ((in << 24) & 0xff000000 ) |
|
|
@@ -362,7 +362,7 @@ MEM_STATIC U64 MEM_swap64(U64 in)
|
|
|
362
362
|
{
|
|
363
363
|
#if defined(_MSC_VER) /* Visual Studio */
|
|
364
364
|
return _byteswap_uint64(in);
|
|
365
|
-
#elif defined (__GNUC__)
|
|
365
|
+
#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
|
|
366
366
|
return __builtin_bswap64(in);
|
|
367
367
|
#else
|
|
368
368
|
return ((in << 56) & 0xff00000000000000ULL) |
|
|
@@ -2628,7 +2628,7 @@ const char* ZBUFFv07_getErrorName(size_t errorCode) { return ERR_getErrorName(er
|
|
|
2628
2628
|
|
|
2629
2629
|
|
|
2630
2630
|
|
|
2631
|
-
void* ZSTDv07_defaultAllocFunction(void* opaque, size_t size)
|
|
2631
|
+
static void* ZSTDv07_defaultAllocFunction(void* opaque, size_t size)
|
|
2632
2632
|
{
|
|
2633
2633
|
void* address = malloc(size);
|
|
2634
2634
|
(void)opaque;
|
|
@@ -2636,7 +2636,7 @@ void* ZSTDv07_defaultAllocFunction(void* opaque, size_t size)
|
|
|
2636
2636
|
return address;
|
|
2637
2637
|
}
|
|
2638
2638
|
|
|
2639
|
-
void ZSTDv07_defaultFreeFunction(void* opaque, void* address)
|
|
2639
|
+
static void ZSTDv07_defaultFreeFunction(void* opaque, void* address)
|
|
2640
2640
|
{
|
|
2641
2641
|
(void)opaque;
|
|
2642
2642
|
/* if (address) printf("free %p opaque=%p \n", address, opaque); */
|
|
@@ -2740,6 +2740,8 @@ typedef enum { lbt_huffman, lbt_repeat, lbt_raw, lbt_rle } litBlockType_t;
|
|
|
2740
2740
|
#define FSEv07_ENCODING_STATIC 2
|
|
2741
2741
|
#define FSEv07_ENCODING_DYNAMIC 3
|
|
2742
2742
|
|
|
2743
|
+
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
|
|
2744
|
+
|
|
2743
2745
|
static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
2744
2746
|
1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12,
|
|
2745
2747
|
13,14,15,16 };
|
|
@@ -3150,10 +3152,10 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src,
|
|
|
3150
3152
|
const BYTE* ip = (const BYTE*)src;
|
|
3151
3153
|
|
|
3152
3154
|
if (srcSize < ZSTDv07_frameHeaderSize_min) return ZSTDv07_frameHeaderSize_min;
|
|
3155
|
+
memset(fparamsPtr, 0, sizeof(*fparamsPtr));
|
|
3153
3156
|
if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) {
|
|
3154
3157
|
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTDv07_MAGIC_SKIPPABLE_START) {
|
|
3155
3158
|
if (srcSize < ZSTDv07_skippableHeaderSize) return ZSTDv07_skippableHeaderSize; /* magic number + skippable frame length */
|
|
3156
|
-
memset(fparamsPtr, 0, sizeof(*fparamsPtr));
|
|
3157
3159
|
fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4);
|
|
3158
3160
|
fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */
|
|
3159
3161
|
return 0;
|
|
@@ -3175,11 +3177,13 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src,
|
|
|
3175
3177
|
U32 windowSize = 0;
|
|
3176
3178
|
U32 dictID = 0;
|
|
3177
3179
|
U64 frameContentSize = 0;
|
|
3178
|
-
if ((fhdByte & 0x08) != 0)
|
|
3180
|
+
if ((fhdByte & 0x08) != 0) /* reserved bits, which must be zero */
|
|
3181
|
+
return ERROR(frameParameter_unsupported);
|
|
3179
3182
|
if (!directMode) {
|
|
3180
3183
|
BYTE const wlByte = ip[pos++];
|
|
3181
3184
|
U32 const windowLog = (wlByte >> 3) + ZSTDv07_WINDOWLOG_ABSOLUTEMIN;
|
|
3182
|
-
if (windowLog > ZSTDv07_WINDOWLOG_MAX)
|
|
3185
|
+
if (windowLog > ZSTDv07_WINDOWLOG_MAX)
|
|
3186
|
+
return ERROR(frameParameter_unsupported);
|
|
3183
3187
|
windowSize = (1U << windowLog);
|
|
3184
3188
|
windowSize += (windowSize >> 3) * (wlByte&7);
|
|
3185
3189
|
}
|
|
@@ -3201,7 +3205,8 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src,
|
|
|
3201
3205
|
case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
|
|
3202
3206
|
}
|
|
3203
3207
|
if (!windowSize) windowSize = (U32)frameContentSize;
|
|
3204
|
-
if (windowSize > windowSizeMax)
|
|
3208
|
+
if (windowSize > windowSizeMax)
|
|
3209
|
+
return ERROR(frameParameter_unsupported);
|
|
3205
3210
|
fparamsPtr->frameContentSize = frameContentSize;
|
|
3206
3211
|
fparamsPtr->windowSize = windowSize;
|
|
3207
3212
|
fparamsPtr->dictID = dictID;
|
|
@@ -3220,11 +3225,10 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src,
|
|
|
3220
3225
|
- frame header not completely provided (`srcSize` too small) */
|
|
3221
3226
|
unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize)
|
|
3222
3227
|
{
|
|
3223
|
-
|
|
3224
|
-
|
|
3225
|
-
|
|
3226
|
-
|
|
3227
|
-
}
|
|
3228
|
+
ZSTDv07_frameParams fparams;
|
|
3229
|
+
size_t const frResult = ZSTDv07_getFrameParams(&fparams, src, srcSize);
|
|
3230
|
+
if (frResult!=0) return 0;
|
|
3231
|
+
return fparams.frameContentSize;
|
|
3228
3232
|
}
|
|
3229
3233
|
|
|
3230
3234
|
|
|
@@ -3248,7 +3252,7 @@ typedef struct
|
|
|
3248
3252
|
|
|
3249
3253
|
/*! ZSTDv07_getcBlockSize() :
|
|
3250
3254
|
* Provides the size of compressed block from block header `src` */
|
|
3251
|
-
size_t ZSTDv07_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
|
3255
|
+
static size_t ZSTDv07_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
|
3252
3256
|
{
|
|
3253
3257
|
const BYTE* const in = (const BYTE* const)src;
|
|
3254
3258
|
U32 cSize;
|
|
@@ -3275,7 +3279,7 @@ static size_t ZSTDv07_copyRawBlock(void* dst, size_t dstCapacity, const void* sr
|
|
|
3275
3279
|
|
|
3276
3280
|
/*! ZSTDv07_decodeLiteralsBlock() :
|
|
3277
3281
|
@return : nb of bytes read from src (< srcSize ) */
|
|
3278
|
-
size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx,
|
|
3282
|
+
static size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx,
|
|
3279
3283
|
const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
|
|
3280
3284
|
{
|
|
3281
3285
|
const BYTE* const istart = (const BYTE*) src;
|
|
@@ -3409,7 +3413,7 @@ size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx,
|
|
|
3409
3413
|
@return : nb bytes read from src,
|
|
3410
3414
|
or an error code if it fails, testable with ZSTDv07_isError()
|
|
3411
3415
|
*/
|
|
3412
|
-
size_t ZSTDv07_buildSeqTable(FSEv07_DTable* DTable, U32 type, U32 max, U32 maxLog,
|
|
3416
|
+
static size_t ZSTDv07_buildSeqTable(FSEv07_DTable* DTable, U32 type, U32 max, U32 maxLog,
|
|
3413
3417
|
const void* src, size_t srcSize,
|
|
3414
3418
|
const S16* defaultNorm, U32 defaultLog, U32 flagRepeatTable)
|
|
3415
3419
|
{
|
|
@@ -3439,7 +3443,7 @@ size_t ZSTDv07_buildSeqTable(FSEv07_DTable* DTable, U32 type, U32 max, U32 maxLo
|
|
|
3439
3443
|
}
|
|
3440
3444
|
|
|
3441
3445
|
|
|
3442
|
-
size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr,
|
|
3446
|
+
static size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr,
|
|
3443
3447
|
FSEv07_DTable* DTableLL, FSEv07_DTable* DTableML, FSEv07_DTable* DTableOffb, U32 flagRepeatTable,
|
|
3444
3448
|
const void* src, size_t srcSize)
|
|
3445
3449
|
{
|
|
@@ -3629,7 +3633,7 @@ size_t ZSTDv07_execSequence(BYTE* op,
|
|
|
3629
3633
|
if (sequence.offset < 8) {
|
|
3630
3634
|
/* close range match, overlap */
|
|
3631
3635
|
static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
|
|
3632
|
-
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /*
|
|
3636
|
+
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
|
|
3633
3637
|
int const sub2 = dec64table[sequence.offset];
|
|
3634
3638
|
op[0] = match[0];
|
|
3635
3639
|
op[1] = match[1];
|
|
@@ -3771,7 +3775,7 @@ ZSTDLIBv07_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockS
|
|
|
3771
3775
|
}
|
|
3772
3776
|
|
|
3773
3777
|
|
|
3774
|
-
size_t ZSTDv07_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length)
|
|
3778
|
+
static size_t ZSTDv07_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length)
|
|
3775
3779
|
{
|
|
3776
3780
|
if (length > dstCapacity) return ERROR(dstSize_tooSmall);
|
|
3777
3781
|
memset(dst, byte, length);
|
|
@@ -3851,7 +3855,7 @@ static size_t ZSTDv07_decompressFrame(ZSTDv07_DCtx* dctx,
|
|
|
3851
3855
|
* It avoids reloading the dictionary each time.
|
|
3852
3856
|
* `preparedDCtx` must have been properly initialized using ZSTDv07_decompressBegin_usingDict().
|
|
3853
3857
|
* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */
|
|
3854
|
-
size_t ZSTDv07_decompress_usingPreparedDCtx(ZSTDv07_DCtx* dctx, const ZSTDv07_DCtx* refDCtx,
|
|
3858
|
+
static size_t ZSTDv07_decompress_usingPreparedDCtx(ZSTDv07_DCtx* dctx, const ZSTDv07_DCtx* refDCtx,
|
|
3855
3859
|
void* dst, size_t dstCapacity,
|
|
3856
3860
|
const void* src, size_t srcSize)
|
|
3857
3861
|
{
|
|
@@ -3893,19 +3897,40 @@ size_t ZSTDv07_decompress(void* dst, size_t dstCapacity, const void* src, size_t
|
|
|
3893
3897
|
#endif
|
|
3894
3898
|
}
|
|
3895
3899
|
|
|
3896
|
-
|
|
3900
|
+
/* ZSTD_errorFrameSizeInfoLegacy() :
|
|
3901
|
+
assumes `cSize` and `dBound` are _not_ NULL */
|
|
3902
|
+
static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
|
|
3903
|
+
{
|
|
3904
|
+
*cSize = ret;
|
|
3905
|
+
*dBound = ZSTD_CONTENTSIZE_ERROR;
|
|
3906
|
+
}
|
|
3907
|
+
|
|
3908
|
+
void ZSTDv07_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
|
|
3897
3909
|
{
|
|
3898
3910
|
const BYTE* ip = (const BYTE*)src;
|
|
3899
3911
|
size_t remainingSize = srcSize;
|
|
3912
|
+
size_t nbBlocks = 0;
|
|
3900
3913
|
|
|
3901
3914
|
/* check */
|
|
3902
|
-
if (srcSize < ZSTDv07_frameHeaderSize_min+ZSTDv07_blockHeaderSize)
|
|
3915
|
+
if (srcSize < ZSTDv07_frameHeaderSize_min+ZSTDv07_blockHeaderSize) {
|
|
3916
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
|
3917
|
+
return;
|
|
3918
|
+
}
|
|
3903
3919
|
|
|
3904
3920
|
/* Frame Header */
|
|
3905
3921
|
{ size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src, ZSTDv07_frameHeaderSize_min);
|
|
3906
|
-
if (ZSTDv07_isError(frameHeaderSize))
|
|
3907
|
-
|
|
3908
|
-
|
|
3922
|
+
if (ZSTDv07_isError(frameHeaderSize)) {
|
|
3923
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, frameHeaderSize);
|
|
3924
|
+
return;
|
|
3925
|
+
}
|
|
3926
|
+
if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) {
|
|
3927
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
|
|
3928
|
+
return;
|
|
3929
|
+
}
|
|
3930
|
+
if (srcSize < frameHeaderSize+ZSTDv07_blockHeaderSize) {
|
|
3931
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
|
3932
|
+
return;
|
|
3933
|
+
}
|
|
3909
3934
|
ip += frameHeaderSize; remainingSize -= frameHeaderSize;
|
|
3910
3935
|
}
|
|
3911
3936
|
|
|
@@ -3913,20 +3938,28 @@ size_t ZSTDv07_findFrameCompressedSize(const void* src, size_t srcSize)
|
|
|
3913
3938
|
while (1) {
|
|
3914
3939
|
blockProperties_t blockProperties;
|
|
3915
3940
|
size_t const cBlockSize = ZSTDv07_getcBlockSize(ip, remainingSize, &blockProperties);
|
|
3916
|
-
if (ZSTDv07_isError(cBlockSize))
|
|
3941
|
+
if (ZSTDv07_isError(cBlockSize)) {
|
|
3942
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
|
|
3943
|
+
return;
|
|
3944
|
+
}
|
|
3917
3945
|
|
|
3918
3946
|
ip += ZSTDv07_blockHeaderSize;
|
|
3919
3947
|
remainingSize -= ZSTDv07_blockHeaderSize;
|
|
3920
3948
|
|
|
3921
3949
|
if (blockProperties.blockType == bt_end) break;
|
|
3922
3950
|
|
|
3923
|
-
if (cBlockSize > remainingSize)
|
|
3951
|
+
if (cBlockSize > remainingSize) {
|
|
3952
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
|
3953
|
+
return;
|
|
3954
|
+
}
|
|
3924
3955
|
|
|
3925
3956
|
ip += cBlockSize;
|
|
3926
3957
|
remainingSize -= cBlockSize;
|
|
3958
|
+
nbBlocks++;
|
|
3927
3959
|
}
|
|
3928
3960
|
|
|
3929
|
-
|
|
3961
|
+
*cSize = ip - (const BYTE*)src;
|
|
3962
|
+
*dBound = nbBlocks * ZSTDv07_BLOCKSIZE_ABSOLUTEMAX;
|
|
3930
3963
|
}
|
|
3931
3964
|
|
|
3932
3965
|
/*_******************************
|
|
@@ -4146,7 +4179,7 @@ struct ZSTDv07_DDict_s {
|
|
|
4146
4179
|
ZSTDv07_DCtx* refContext;
|
|
4147
4180
|
}; /* typedef'd tp ZSTDv07_CDict within zstd.h */
|
|
4148
4181
|
|
|
4149
|
-
ZSTDv07_DDict* ZSTDv07_createDDict_advanced(const void* dict, size_t dictSize, ZSTDv07_customMem customMem)
|
|
4182
|
+
static ZSTDv07_DDict* ZSTDv07_createDDict_advanced(const void* dict, size_t dictSize, ZSTDv07_customMem customMem)
|
|
4150
4183
|
{
|
|
4151
4184
|
if (!customMem.customAlloc && !customMem.customFree)
|
|
4152
4185
|
customMem = defaultCustomMem;
|
|
@@ -50,12 +50,17 @@ ZSTDLIBv07_API size_t ZSTDv07_decompress( void* dst, size_t dstCapacity,
|
|
|
50
50
|
const void* src, size_t compressedSize);
|
|
51
51
|
|
|
52
52
|
/**
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
53
|
+
ZSTDv07_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.7.x format
|
|
54
|
+
srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
|
55
|
+
cSize (output parameter) : the number of bytes that would be read to decompress this frame
|
|
56
|
+
or an error code if it fails (which can be tested using ZSTDv01_isError())
|
|
57
|
+
dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
|
|
58
|
+
or ZSTD_CONTENTSIZE_ERROR if an error occurs
|
|
59
|
+
|
|
60
|
+
note : assumes `cSize` and `dBound` are _not_ NULL.
|
|
57
61
|
*/
|
|
58
|
-
|
|
62
|
+
void ZSTDv07_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
|
|
63
|
+
size_t* cSize, unsigned long long* dBound);
|
|
59
64
|
|
|
60
65
|
/*====== Helper functions ======*/
|
|
61
66
|
ZSTDLIBv07_API unsigned ZSTDv07_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
|
data/contrib/zstd/lib/zstd.h
CHANGED
|
@@ -35,40 +35,73 @@ extern "C" {
|
|
|
35
35
|
#endif
|
|
36
36
|
|
|
37
37
|
|
|
38
|
-
|
|
38
|
+
/*******************************************************************************
|
|
39
39
|
Introduction
|
|
40
40
|
|
|
41
|
-
zstd, short for Zstandard, is a fast lossless compression algorithm,
|
|
42
|
-
|
|
43
|
-
The zstd compression library provides in-memory compression and decompression
|
|
44
|
-
|
|
45
|
-
|
|
41
|
+
zstd, short for Zstandard, is a fast lossless compression algorithm, targeting
|
|
42
|
+
real-time compression scenarios at zlib-level and better compression ratios.
|
|
43
|
+
The zstd compression library provides in-memory compression and decompression
|
|
44
|
+
functions.
|
|
45
|
+
|
|
46
|
+
The library supports regular compression levels from 1 up to ZSTD_maxCLevel(),
|
|
47
|
+
which is currently 22. Levels >= 20, labeled `--ultra`, should be used with
|
|
48
|
+
caution, as they require more memory. The library also offers negative
|
|
49
|
+
compression levels, which extend the range of speed vs. ratio preferences.
|
|
50
|
+
The lower the level, the faster the speed (at the cost of compression).
|
|
51
|
+
|
|
46
52
|
Compression can be done in:
|
|
47
53
|
- a single step (described as Simple API)
|
|
48
|
-
- a single step, reusing a context (described as Explicit
|
|
54
|
+
- a single step, reusing a context (described as Explicit context)
|
|
49
55
|
- unbounded multiple steps (described as Streaming compression)
|
|
50
|
-
|
|
56
|
+
|
|
57
|
+
The compression ratio achievable on small data can be highly improved using
|
|
58
|
+
a dictionary. Dictionary compression can be performed in:
|
|
51
59
|
- a single step (described as Simple dictionary API)
|
|
52
|
-
- a single step, reusing a dictionary (described as
|
|
60
|
+
- a single step, reusing a dictionary (described as Bulk-processing
|
|
61
|
+
dictionary API)
|
|
62
|
+
|
|
63
|
+
Advanced experimental functions can be accessed using
|
|
64
|
+
`#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h.
|
|
53
65
|
|
|
54
|
-
Advanced experimental
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
66
|
+
Advanced experimental APIs should never be used with a dynamically-linked
|
|
67
|
+
library. They are not "stable"; their definitions or signatures may change in
|
|
68
|
+
the future. Only static linking is allowed.
|
|
69
|
+
*******************************************************************************/
|
|
58
70
|
|
|
59
71
|
/*------ Version ------*/
|
|
60
72
|
#define ZSTD_VERSION_MAJOR 1
|
|
61
|
-
#define ZSTD_VERSION_MINOR
|
|
62
|
-
#define ZSTD_VERSION_RELEASE
|
|
73
|
+
#define ZSTD_VERSION_MINOR 4
|
|
74
|
+
#define ZSTD_VERSION_RELEASE 0
|
|
63
75
|
|
|
64
76
|
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
|
|
65
|
-
ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**<
|
|
77
|
+
ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library version */
|
|
66
78
|
|
|
67
79
|
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
|
|
68
80
|
#define ZSTD_QUOTE(str) #str
|
|
69
81
|
#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str)
|
|
70
82
|
#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
|
|
71
|
-
ZSTDLIB_API const char* ZSTD_versionString(void); /* v1.3.0 */
|
|
83
|
+
ZSTDLIB_API const char* ZSTD_versionString(void); /* requires v1.3.0+ */
|
|
84
|
+
|
|
85
|
+
/***************************************
|
|
86
|
+
* Default constant
|
|
87
|
+
***************************************/
|
|
88
|
+
#ifndef ZSTD_CLEVEL_DEFAULT
|
|
89
|
+
# define ZSTD_CLEVEL_DEFAULT 3
|
|
90
|
+
#endif
|
|
91
|
+
|
|
92
|
+
/***************************************
|
|
93
|
+
* Constants
|
|
94
|
+
***************************************/
|
|
95
|
+
|
|
96
|
+
/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */
|
|
97
|
+
#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */
|
|
98
|
+
#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */
|
|
99
|
+
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */
|
|
100
|
+
#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0
|
|
101
|
+
|
|
102
|
+
#define ZSTD_BLOCKSIZELOG_MAX 17
|
|
103
|
+
#define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX)
|
|
104
|
+
|
|
72
105
|
|
|
73
106
|
|
|
74
107
|
/***************************************
|
|
@@ -92,11 +125,11 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
|
|
|
92
125
|
ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
|
|
93
126
|
const void* src, size_t compressedSize);
|
|
94
127
|
|
|
95
|
-
/*! ZSTD_getFrameContentSize() : v1.3.0
|
|
128
|
+
/*! ZSTD_getFrameContentSize() : requires v1.3.0+
|
|
96
129
|
* `src` should point to the start of a ZSTD encoded frame.
|
|
97
130
|
* `srcSize` must be at least as large as the frame header.
|
|
98
131
|
* hint : any size >= `ZSTD_frameHeaderSize_max` is large enough.
|
|
99
|
-
* @return : - decompressed size of
|
|
132
|
+
* @return : - decompressed size of `src` frame content, if known
|
|
100
133
|
* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
|
|
101
134
|
* - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small)
|
|
102
135
|
* note 1 : a 0 return value means the frame is valid but "empty".
|
|
@@ -106,7 +139,8 @@ ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
|
|
|
106
139
|
* Optionally, application can rely on some implicit limit,
|
|
107
140
|
* as ZSTD_decompress() only needs an upper bound of decompressed size.
|
|
108
141
|
* (For example, data could be necessarily cut into blocks <= 16 KB).
|
|
109
|
-
* note 3 : decompressed size is always present when compression is
|
|
142
|
+
* note 3 : decompressed size is always present when compression is completed using single-pass functions,
|
|
143
|
+
* such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict().
|
|
110
144
|
* note 4 : decompressed size can be very large (64-bits value),
|
|
111
145
|
* potentially larger than what local system can handle as a single memory segment.
|
|
112
146
|
* In which case, it's necessary to use streaming mode to decompress data.
|
|
@@ -120,26 +154,32 @@ ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t
|
|
|
120
154
|
|
|
121
155
|
/*! ZSTD_getDecompressedSize() :
|
|
122
156
|
* NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize().
|
|
123
|
-
* Both functions work the same way,
|
|
124
|
-
*
|
|
125
|
-
*
|
|
126
|
-
*
|
|
127
|
-
*
|
|
128
|
-
* 'src' is the start of a zstd compressed frame.
|
|
129
|
-
* @return : content size to be decompressed, as a 64-bits value _if known and not empty_, 0 otherwise. */
|
|
157
|
+
* Both functions work the same way, but ZSTD_getDecompressedSize() blends
|
|
158
|
+
* "empty", "unknown" and "error" results to the same return value (0),
|
|
159
|
+
* while ZSTD_getFrameContentSize() gives them separate return values.
|
|
160
|
+
* @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */
|
|
130
161
|
ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
|
|
131
162
|
|
|
163
|
+
/*! ZSTD_findFrameCompressedSize() :
|
|
164
|
+
* `src` should point to the start of a ZSTD frame or skippable frame.
|
|
165
|
+
* `srcSize` must be >= first frame size
|
|
166
|
+
* @return : the compressed size of the first frame starting at `src`,
|
|
167
|
+
* suitable to pass as `srcSize` to `ZSTD_decompress` or similar,
|
|
168
|
+
* or an error code if input is invalid */
|
|
169
|
+
ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
|
|
170
|
+
|
|
132
171
|
|
|
133
172
|
/*====== Helper functions ======*/
|
|
134
173
|
#define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
|
|
135
|
-
ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case scenario */
|
|
174
|
+
ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
|
|
136
175
|
ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
|
|
137
176
|
ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */
|
|
177
|
+
ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed */
|
|
138
178
|
ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */
|
|
139
179
|
|
|
140
180
|
|
|
141
181
|
/***************************************
|
|
142
|
-
* Explicit
|
|
182
|
+
* Explicit context
|
|
143
183
|
***************************************/
|
|
144
184
|
/*= Compression context
|
|
145
185
|
* When compressing many times,
|
|
@@ -151,8 +191,10 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
|
|
|
151
191
|
ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
|
|
152
192
|
|
|
153
193
|
/*! ZSTD_compressCCtx() :
|
|
154
|
-
* Same as ZSTD_compress(),
|
|
155
|
-
|
|
194
|
+
* Same as ZSTD_compress(), using an explicit ZSTD_CCtx
|
|
195
|
+
* The function will compress at requested compression level,
|
|
196
|
+
* ignoring any other parameter */
|
|
197
|
+
ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
|
|
156
198
|
void* dst, size_t dstCapacity,
|
|
157
199
|
const void* src, size_t srcSize,
|
|
158
200
|
int compressionLevel);
|
|
@@ -168,82 +210,326 @@ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
|
|
|
168
210
|
ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
|
|
169
211
|
|
|
170
212
|
/*! ZSTD_decompressDCtx() :
|
|
171
|
-
* Same as ZSTD_decompress(),
|
|
172
|
-
|
|
213
|
+
* Same as ZSTD_decompress(),
|
|
214
|
+
* requires an allocated ZSTD_DCtx.
|
|
215
|
+
* Compatible with sticky parameters.
|
|
216
|
+
*/
|
|
217
|
+
ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx,
|
|
173
218
|
void* dst, size_t dstCapacity,
|
|
174
219
|
const void* src, size_t srcSize);
|
|
175
220
|
|
|
176
221
|
|
|
177
|
-
|
|
178
|
-
*
|
|
179
|
-
|
|
180
|
-
/*! ZSTD_compress_usingDict() :
|
|
181
|
-
* Compression using a predefined Dictionary (see dictBuilder/zdict.h).
|
|
182
|
-
* Note : This function loads the dictionary, resulting in significant startup delay.
|
|
183
|
-
* Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
|
|
184
|
-
ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
|
|
185
|
-
void* dst, size_t dstCapacity,
|
|
186
|
-
const void* src, size_t srcSize,
|
|
187
|
-
const void* dict,size_t dictSize,
|
|
188
|
-
int compressionLevel);
|
|
222
|
+
/***************************************
|
|
223
|
+
* Advanced compression API
|
|
224
|
+
***************************************/
|
|
189
225
|
|
|
190
|
-
|
|
191
|
-
*
|
|
192
|
-
*
|
|
193
|
-
*
|
|
194
|
-
*
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
226
|
+
/* API design :
|
|
227
|
+
* Parameters are pushed one by one into an existing context,
|
|
228
|
+
* using ZSTD_CCtx_set*() functions.
|
|
229
|
+
* Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame.
|
|
230
|
+
* "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` !
|
|
231
|
+
* They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()
|
|
232
|
+
*
|
|
233
|
+
* It's possible to reset all parameters to "default" using ZSTD_CCtx_reset().
|
|
234
|
+
*
|
|
235
|
+
* This API supercedes all other "advanced" API entry points in the experimental section.
|
|
236
|
+
* In the future, we expect to remove from experimental API entry points which are redundant with this API.
|
|
237
|
+
*/
|
|
199
238
|
|
|
200
239
|
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
240
|
+
/* Compression strategies, listed from fastest to strongest */
|
|
241
|
+
typedef enum { ZSTD_fast=1,
|
|
242
|
+
ZSTD_dfast=2,
|
|
243
|
+
ZSTD_greedy=3,
|
|
244
|
+
ZSTD_lazy=4,
|
|
245
|
+
ZSTD_lazy2=5,
|
|
246
|
+
ZSTD_btlazy2=6,
|
|
247
|
+
ZSTD_btopt=7,
|
|
248
|
+
ZSTD_btultra=8,
|
|
249
|
+
ZSTD_btultra2=9
|
|
250
|
+
/* note : new strategies _might_ be added in the future.
|
|
251
|
+
Only the order (from fast to strong) is guaranteed */
|
|
252
|
+
} ZSTD_strategy;
|
|
205
253
|
|
|
206
|
-
/*! ZSTD_createCDict() :
|
|
207
|
-
* When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
|
|
208
|
-
* ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
|
|
209
|
-
* ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
|
|
210
|
-
* `dictBuffer` can be released after ZSTD_CDict creation, since its content is copied within CDict */
|
|
211
|
-
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
|
|
212
|
-
int compressionLevel);
|
|
213
254
|
|
|
214
|
-
|
|
215
|
-
* Function frees memory allocated by ZSTD_createCDict(). */
|
|
216
|
-
ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
|
|
255
|
+
typedef enum {
|
|
217
256
|
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
257
|
+
/* compression parameters
|
|
258
|
+
* Note: When compressing with a ZSTD_CDict these parameters are superseded
|
|
259
|
+
* by the parameters used to construct the ZSTD_CDict. See ZSTD_CCtx_refCDict()
|
|
260
|
+
* for more info (superseded-by-cdict). */
|
|
261
|
+
ZSTD_c_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
|
|
262
|
+
* Default level is ZSTD_CLEVEL_DEFAULT==3.
|
|
263
|
+
* Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
|
|
264
|
+
* Note 1 : it's possible to pass a negative compression level.
|
|
265
|
+
* Note 2 : setting a level sets all default values of other compression parameters */
|
|
266
|
+
ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2.
|
|
267
|
+
* Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
|
|
268
|
+
* Special: value 0 means "use default windowLog".
|
|
269
|
+
* Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT
|
|
270
|
+
* requires explicitly allowing such window size at decompression stage if using streaming. */
|
|
271
|
+
ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2.
|
|
272
|
+
* Resulting memory usage is (1 << (hashLog+2)).
|
|
273
|
+
* Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
|
|
274
|
+
* Larger tables improve compression ratio of strategies <= dFast,
|
|
275
|
+
* and improve speed of strategies > dFast.
|
|
276
|
+
* Special: value 0 means "use default hashLog". */
|
|
277
|
+
ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2.
|
|
278
|
+
* Resulting memory usage is (1 << (chainLog+2)).
|
|
279
|
+
* Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
|
|
280
|
+
* Larger tables result in better and slower compression.
|
|
281
|
+
* This parameter is useless when using "fast" strategy.
|
|
282
|
+
* It's still useful when using "dfast" strategy,
|
|
283
|
+
* in which case it defines a secondary probe table.
|
|
284
|
+
* Special: value 0 means "use default chainLog". */
|
|
285
|
+
ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2.
|
|
286
|
+
* More attempts result in better and slower compression.
|
|
287
|
+
* This parameter is useless when using "fast" and "dFast" strategies.
|
|
288
|
+
* Special: value 0 means "use default searchLog". */
|
|
289
|
+
ZSTD_c_minMatch=105, /* Minimum size of searched matches.
|
|
290
|
+
* Note that Zstandard can still find matches of smaller size,
|
|
291
|
+
* it just tweaks its search algorithm to look for this size and larger.
|
|
292
|
+
* Larger values increase compression and decompression speed, but decrease ratio.
|
|
293
|
+
* Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX.
|
|
294
|
+
* Note that currently, for all strategies < btopt, effective minimum is 4.
|
|
295
|
+
* , for all strategies > fast, effective maximum is 6.
|
|
296
|
+
* Special: value 0 means "use default minMatchLength". */
|
|
297
|
+
ZSTD_c_targetLength=106, /* Impact of this field depends on strategy.
|
|
298
|
+
* For strategies btopt, btultra & btultra2:
|
|
299
|
+
* Length of Match considered "good enough" to stop search.
|
|
300
|
+
* Larger values make compression stronger, and slower.
|
|
301
|
+
* For strategy fast:
|
|
302
|
+
* Distance between match sampling.
|
|
303
|
+
* Larger values make compression faster, and weaker.
|
|
304
|
+
* Special: value 0 means "use default targetLength". */
|
|
305
|
+
ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition.
|
|
306
|
+
* The higher the value of selected strategy, the more complex it is,
|
|
307
|
+
* resulting in stronger and slower compression.
|
|
308
|
+
* Special: value 0 means "use default strategy". */
|
|
309
|
+
|
|
310
|
+
/* LDM mode parameters */
|
|
311
|
+
ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching.
|
|
312
|
+
* This parameter is designed to improve compression ratio
|
|
313
|
+
* for large inputs, by finding large matches at long distance.
|
|
314
|
+
* It increases memory usage and window size.
|
|
315
|
+
* Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB
|
|
316
|
+
* except when expressly set to a different value. */
|
|
317
|
+
ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2.
|
|
318
|
+
* Larger values increase memory usage and compression ratio,
|
|
319
|
+
* but decrease compression speed.
|
|
320
|
+
* Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
|
|
321
|
+
* default: windowlog - 7.
|
|
322
|
+
* Special: value 0 means "automatically determine hashlog". */
|
|
323
|
+
ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher.
|
|
324
|
+
* Larger/too small values usually decrease compression ratio.
|
|
325
|
+
* Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX.
|
|
326
|
+
* Special: value 0 means "use default value" (default: 64). */
|
|
327
|
+
ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution.
|
|
328
|
+
* Larger values improve collision resolution but decrease compression speed.
|
|
329
|
+
* The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX.
|
|
330
|
+
* Special: value 0 means "use default value" (default: 3). */
|
|
331
|
+
ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table.
|
|
332
|
+
* Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN).
|
|
333
|
+
* Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage.
|
|
334
|
+
* Larger values improve compression speed.
|
|
335
|
+
* Deviating far from default value will likely result in a compression ratio decrease.
|
|
336
|
+
* Special: value 0 means "automatically determine hashRateLog". */
|
|
227
337
|
|
|
338
|
+
/* frame parameters */
|
|
339
|
+
ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
|
|
340
|
+
* Content size must be known at the beginning of compression.
|
|
341
|
+
* This is automatically the case when using ZSTD_compress2(),
|
|
342
|
+
* For streaming variants, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
|
|
343
|
+
ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */
|
|
344
|
+
ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */
|
|
228
345
|
|
|
229
|
-
|
|
346
|
+
/* multi-threading parameters */
|
|
347
|
+
/* These parameters are only useful if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
|
|
348
|
+
* They return an error otherwise. */
|
|
349
|
+
ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel.
|
|
350
|
+
* When nbWorkers >= 1, triggers asynchronous mode when used with ZSTD_compressStream*() :
|
|
351
|
+
* ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
|
|
352
|
+
* while compression work is performed in parallel, within worker threads.
|
|
353
|
+
* (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
|
|
354
|
+
* in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
|
|
355
|
+
* More workers improve speed, but also increase memory usage.
|
|
356
|
+
* Default value is `0`, aka "single-threaded mode" : no worker is spawned, compression is performed inside Caller's thread, all invocations are blocking */
|
|
357
|
+
ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1.
|
|
358
|
+
* Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
|
|
359
|
+
* 0 means default, which is dynamically determined based on compression parameters.
|
|
360
|
+
* Job size must be a minimum of overlap size, or 1 MB, whichever is largest.
|
|
361
|
+
* The minimum size is automatically and transparently enforced */
|
|
362
|
+
ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size.
|
|
363
|
+
* The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
|
|
364
|
+
* It helps preserve compression ratio, while each job is compressed in parallel.
|
|
365
|
+
* This value is enforced only when nbWorkers >= 1.
|
|
366
|
+
* Larger values increase compression ratio, but decrease speed.
|
|
367
|
+
* Possible values range from 0 to 9 :
|
|
368
|
+
* - 0 means "default" : value will be determined by the library, depending on strategy
|
|
369
|
+
* - 1 means "no overlap"
|
|
370
|
+
* - 9 means "full overlap", using a full window size.
|
|
371
|
+
* Each intermediate rank increases/decreases load size by a factor 2 :
|
|
372
|
+
* 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default
|
|
373
|
+
* default value varies between 6 and 9, depending on strategy */
|
|
374
|
+
|
|
375
|
+
/* note : additional experimental parameters are also available
|
|
376
|
+
* within the experimental section of the API.
|
|
377
|
+
* At the time of this writing, they include :
|
|
378
|
+
* ZSTD_c_rsyncable
|
|
379
|
+
* ZSTD_c_format
|
|
380
|
+
* ZSTD_c_forceMaxWindow
|
|
381
|
+
* ZSTD_c_forceAttachDict
|
|
382
|
+
* ZSTD_c_literalCompressionMode
|
|
383
|
+
* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
|
|
384
|
+
* note : never ever use experimentalParam? names directly;
|
|
385
|
+
* also, the enums values themselves are unstable and can still change.
|
|
386
|
+
*/
|
|
387
|
+
ZSTD_c_experimentalParam1=500,
|
|
388
|
+
ZSTD_c_experimentalParam2=10,
|
|
389
|
+
ZSTD_c_experimentalParam3=1000,
|
|
390
|
+
ZSTD_c_experimentalParam4=1001,
|
|
391
|
+
ZSTD_c_experimentalParam5=1002,
|
|
392
|
+
} ZSTD_cParameter;
|
|
230
393
|
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
394
|
+
typedef struct {
|
|
395
|
+
size_t error;
|
|
396
|
+
int lowerBound;
|
|
397
|
+
int upperBound;
|
|
398
|
+
} ZSTD_bounds;
|
|
399
|
+
|
|
400
|
+
/*! ZSTD_cParam_getBounds() :
|
|
401
|
+
* All parameters must belong to an interval with lower and upper bounds,
|
|
402
|
+
* otherwise they will either trigger an error or be automatically clamped.
|
|
403
|
+
* @return : a structure, ZSTD_bounds, which contains
|
|
404
|
+
* - an error status field, which must be tested using ZSTD_isError()
|
|
405
|
+
* - lower and upper bounds, both inclusive
|
|
406
|
+
*/
|
|
407
|
+
ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam);
|
|
235
408
|
|
|
236
|
-
/*!
|
|
237
|
-
*
|
|
238
|
-
|
|
409
|
+
/*! ZSTD_CCtx_setParameter() :
|
|
410
|
+
* Set one compression parameter, selected by enum ZSTD_cParameter.
|
|
411
|
+
* All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds().
|
|
412
|
+
* Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
|
|
413
|
+
* Setting a parameter is generally only possible during frame initialization (before starting compression).
|
|
414
|
+
* Exception : when using multi-threading mode (nbWorkers >= 1),
|
|
415
|
+
* the following parameters can be updated _during_ compression (within same frame):
|
|
416
|
+
* => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy.
|
|
417
|
+
* new parameters will be active for next job only (after a flush()).
|
|
418
|
+
* @return : an error code (which can be tested using ZSTD_isError()).
|
|
419
|
+
*/
|
|
420
|
+
ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value);
|
|
239
421
|
|
|
240
|
-
/*!
|
|
241
|
-
*
|
|
242
|
-
*
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
422
|
+
/*! ZSTD_CCtx_setPledgedSrcSize() :
|
|
423
|
+
* Total input data size to be compressed as a single frame.
|
|
424
|
+
* Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag.
|
|
425
|
+
* This value will also be controlled at end of frame, and trigger an error if not respected.
|
|
426
|
+
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
427
|
+
* Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame.
|
|
428
|
+
* In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
|
|
429
|
+
* ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame.
|
|
430
|
+
* Note 2 : pledgedSrcSize is only valid once, for the next frame.
|
|
431
|
+
* It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN.
|
|
432
|
+
* Note 3 : Whenever all input data is provided and consumed in a single round,
|
|
433
|
+
* for example with ZSTD_compress2(),
|
|
434
|
+
* or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end),
|
|
435
|
+
* this value is automatically overridden by srcSize instead.
|
|
436
|
+
*/
|
|
437
|
+
ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
|
|
438
|
+
|
|
439
|
+
typedef enum {
|
|
440
|
+
ZSTD_reset_session_only = 1,
|
|
441
|
+
ZSTD_reset_parameters = 2,
|
|
442
|
+
ZSTD_reset_session_and_parameters = 3
|
|
443
|
+
} ZSTD_ResetDirective;
|
|
444
|
+
|
|
445
|
+
/*! ZSTD_CCtx_reset() :
|
|
446
|
+
* There are 2 different things that can be reset, independently or jointly :
|
|
447
|
+
* - The session : will stop compressing current frame, and make CCtx ready to start a new one.
|
|
448
|
+
* Useful after an error, or to interrupt any ongoing compression.
|
|
449
|
+
* Any internal data not yet flushed is cancelled.
|
|
450
|
+
* Compression parameters and dictionary remain unchanged.
|
|
451
|
+
* They will be used to compress next frame.
|
|
452
|
+
* Resetting session never fails.
|
|
453
|
+
* - The parameters : changes all parameters back to "default".
|
|
454
|
+
* This removes any reference to any dictionary too.
|
|
455
|
+
* Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing)
|
|
456
|
+
* otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError())
|
|
457
|
+
* - Both : similar to resetting the session, followed by resetting parameters.
|
|
458
|
+
*/
|
|
459
|
+
ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset);
|
|
460
|
+
|
|
461
|
+
/*! ZSTD_compress2() :
|
|
462
|
+
* Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API.
|
|
463
|
+
* ZSTD_compress2() always starts a new frame.
|
|
464
|
+
* Should cctx hold data from a previously unfinished frame, everything about it is forgotten.
|
|
465
|
+
* - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
|
|
466
|
+
* - The function is always blocking, returns when compression is completed.
|
|
467
|
+
* Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
|
|
468
|
+
* @return : compressed size written into `dst` (<= `dstCapacity),
|
|
469
|
+
* or an error code if it fails (which can be tested using ZSTD_isError()).
|
|
470
|
+
*/
|
|
471
|
+
ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx,
|
|
472
|
+
void* dst, size_t dstCapacity,
|
|
473
|
+
const void* src, size_t srcSize);
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
/***************************************
|
|
477
|
+
* Advanced decompression API
|
|
478
|
+
***************************************/
|
|
479
|
+
|
|
480
|
+
/* The advanced API pushes parameters one by one into an existing DCtx context.
|
|
481
|
+
* Parameters are sticky, and remain valid for all following frames
|
|
482
|
+
* using the same DCtx context.
|
|
483
|
+
* It's possible to reset parameters to default values using ZSTD_DCtx_reset().
|
|
484
|
+
* Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream().
|
|
485
|
+
* Therefore, no new decompression function is necessary.
|
|
486
|
+
*/
|
|
487
|
+
|
|
488
|
+
typedef enum {
|
|
489
|
+
|
|
490
|
+
ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which
|
|
491
|
+
* the streaming API will refuse to allocate memory buffer
|
|
492
|
+
* in order to protect the host from unreasonable memory requirements.
|
|
493
|
+
* This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
|
|
494
|
+
* By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT).
|
|
495
|
+
* Special: value 0 means "use default maximum windowLog". */
|
|
496
|
+
|
|
497
|
+
/* note : additional experimental parameters are also available
|
|
498
|
+
* within the experimental section of the API.
|
|
499
|
+
* At the time of this writing, they include :
|
|
500
|
+
* ZSTD_c_format
|
|
501
|
+
* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
|
|
502
|
+
* note : never ever use experimentalParam? names directly
|
|
503
|
+
*/
|
|
504
|
+
ZSTD_d_experimentalParam1=1000
|
|
505
|
+
|
|
506
|
+
} ZSTD_dParameter;
|
|
507
|
+
|
|
508
|
+
/*! ZSTD_dParam_getBounds() :
|
|
509
|
+
* All parameters must belong to an interval with lower and upper bounds,
|
|
510
|
+
* otherwise they will either trigger an error or be automatically clamped.
|
|
511
|
+
* @return : a structure, ZSTD_bounds, which contains
|
|
512
|
+
* - an error status field, which must be tested using ZSTD_isError()
|
|
513
|
+
* - both lower and upper bounds, inclusive
|
|
514
|
+
*/
|
|
515
|
+
ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam);
|
|
516
|
+
|
|
517
|
+
/*! ZSTD_DCtx_setParameter() :
|
|
518
|
+
* Set one compression parameter, selected by enum ZSTD_dParameter.
|
|
519
|
+
* All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds().
|
|
520
|
+
* Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
|
|
521
|
+
* Setting a parameter is only possible during frame initialization (before starting decompression).
|
|
522
|
+
* @return : 0, or an error code (which can be tested using ZSTD_isError()).
|
|
523
|
+
*/
|
|
524
|
+
ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value);
|
|
525
|
+
|
|
526
|
+
/*! ZSTD_DCtx_reset() :
|
|
527
|
+
* Return a DCtx to clean state.
|
|
528
|
+
* Session and parameters can be reset jointly or separately.
|
|
529
|
+
* Parameters can only be reset when no active frame is being decompressed.
|
|
530
|
+
* @return : 0, or an error code, which can be tested with ZSTD_isError()
|
|
531
|
+
*/
|
|
532
|
+
ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset);
|
|
247
533
|
|
|
248
534
|
|
|
249
535
|
/****************************
|
|
@@ -270,56 +556,136 @@ typedef struct ZSTD_outBuffer_s {
|
|
|
270
556
|
* A ZSTD_CStream object is required to track streaming operation.
|
|
271
557
|
* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
|
|
272
558
|
* ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
|
|
273
|
-
* It is recommended to re-use ZSTD_CStream
|
|
274
|
-
* since it will play nicer with system's memory, by re-using already allocated memory.
|
|
275
|
-
* Use one separate ZSTD_CStream per thread for parallel execution.
|
|
559
|
+
* It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory.
|
|
276
560
|
*
|
|
277
|
-
*
|
|
278
|
-
* Use ZSTD_initCStream() to start a new compression operation.
|
|
279
|
-
* Use ZSTD_initCStream_usingDict() or ZSTD_initCStream_usingCDict() for a compression which requires a dictionary (experimental section)
|
|
561
|
+
* For parallel execution, use one separate ZSTD_CStream per thread.
|
|
280
562
|
*
|
|
281
|
-
*
|
|
282
|
-
*
|
|
283
|
-
*
|
|
284
|
-
*
|
|
285
|
-
*
|
|
286
|
-
*
|
|
287
|
-
*
|
|
288
|
-
*
|
|
563
|
+
* note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing.
|
|
564
|
+
*
|
|
565
|
+
* Parameters are sticky : when starting a new compression on the same context,
|
|
566
|
+
* it will re-use the same sticky parameters as previous compression session.
|
|
567
|
+
* When in doubt, it's recommended to fully initialize the context before usage.
|
|
568
|
+
* Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(),
|
|
569
|
+
* ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to
|
|
570
|
+
* set more specific parameters, the pledged source size, or load a dictionary.
|
|
289
571
|
*
|
|
290
|
-
*
|
|
291
|
-
*
|
|
292
|
-
*
|
|
293
|
-
*
|
|
572
|
+
* Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to
|
|
573
|
+
* consume input stream. The function will automatically update both `pos`
|
|
574
|
+
* fields within `input` and `output`.
|
|
575
|
+
* Note that the function may not consume the entire input, for example, because
|
|
576
|
+
* the output buffer is already full, in which case `input.pos < input.size`.
|
|
577
|
+
* The caller must check if input has been entirely consumed.
|
|
578
|
+
* If not, the caller must make some room to receive more compressed data,
|
|
579
|
+
* and then present again remaining input data.
|
|
580
|
+
* note: ZSTD_e_continue is guaranteed to make some forward progress when called,
|
|
581
|
+
* but doesn't guarantee maximal forward progress. This is especially relevant
|
|
582
|
+
* when compressing with multiple threads. The call won't block if it can
|
|
583
|
+
* consume some input, but if it can't it will wait for some, but not all,
|
|
584
|
+
* output to be flushed.
|
|
585
|
+
* @return : provides a minimum amount of data remaining to be flushed from internal buffers
|
|
586
|
+
* or an error code, which can be tested using ZSTD_isError().
|
|
587
|
+
*
|
|
588
|
+
* At any moment, it's possible to flush whatever data might remain stuck within internal buffer,
|
|
589
|
+
* using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated.
|
|
590
|
+
* Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0).
|
|
591
|
+
* In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush.
|
|
592
|
+
* You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the
|
|
593
|
+
* operation.
|
|
594
|
+
* note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will
|
|
595
|
+
* block until the flush is complete or the output buffer is full.
|
|
596
|
+
* @return : 0 if internal buffers are entirely flushed,
|
|
597
|
+
* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
|
|
294
598
|
* or an error code, which can be tested using ZSTD_isError().
|
|
295
599
|
*
|
|
296
|
-
*
|
|
600
|
+
* Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame.
|
|
297
601
|
* It will perform a flush and write frame epilogue.
|
|
298
602
|
* The epilogue is required for decoders to consider a frame completed.
|
|
299
|
-
*
|
|
300
|
-
*
|
|
603
|
+
* flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush.
|
|
604
|
+
* You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to
|
|
605
|
+
* start a new frame.
|
|
606
|
+
* note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will
|
|
607
|
+
* block until the flush is complete or the output buffer is full.
|
|
301
608
|
* @return : 0 if frame fully completed and fully flushed,
|
|
302
|
-
|
|
303
|
-
(value is minimum size estimation for remaining data to flush, but it could be more)
|
|
609
|
+
* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
|
|
304
610
|
* or an error code, which can be tested using ZSTD_isError().
|
|
305
611
|
*
|
|
306
612
|
* *******************************************************************/
|
|
307
613
|
|
|
308
614
|
typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */
|
|
309
|
-
/* Continue to distinguish them for compatibility with versions <= v1.2.0 */
|
|
615
|
+
/* Continue to distinguish them for compatibility with older versions <= v1.2.0 */
|
|
310
616
|
/*===== ZSTD_CStream management functions =====*/
|
|
311
617
|
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
|
|
312
618
|
ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
|
|
313
619
|
|
|
314
620
|
/*===== Streaming compression functions =====*/
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
621
|
+
typedef enum {
|
|
622
|
+
ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */
|
|
623
|
+
ZSTD_e_flush=1, /* flush any data provided so far,
|
|
624
|
+
* it creates (at least) one new block, that can be decoded immediately on reception;
|
|
625
|
+
* frame will continue: any future data can still reference previously compressed data, improving compression.
|
|
626
|
+
* note : multithreaded compression will block to flush as much output as possible. */
|
|
627
|
+
ZSTD_e_end=2 /* flush any remaining data _and_ close current frame.
|
|
628
|
+
* note that frame is only closed after compressed data is fully flushed (return value == 0).
|
|
629
|
+
* After that point, any additional data starts a new frame.
|
|
630
|
+
* note : each frame is independent (does not reference any content from previous frame).
|
|
631
|
+
: note : multithreaded compression will block to flush as much output as possible. */
|
|
632
|
+
} ZSTD_EndDirective;
|
|
633
|
+
|
|
634
|
+
/*! ZSTD_compressStream2() :
|
|
635
|
+
* Behaves about the same as ZSTD_compressStream, with additional control on end directive.
|
|
636
|
+
* - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
|
|
637
|
+
* - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
|
|
638
|
+
* - output->pos must be <= dstCapacity, input->pos must be <= srcSize
|
|
639
|
+
* - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
|
|
640
|
+
* - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
|
|
641
|
+
* - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available,
|
|
642
|
+
* and then immediately returns, just indicating that there is some data remaining to be flushed.
|
|
643
|
+
* The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
|
|
644
|
+
* - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking.
|
|
645
|
+
* - @return provides a minimum amount of data remaining to be flushed from internal buffers
|
|
646
|
+
* or an error code, which can be tested using ZSTD_isError().
|
|
647
|
+
* if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
|
|
648
|
+
* This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers.
|
|
649
|
+
* For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed.
|
|
650
|
+
* - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
|
|
651
|
+
* only ZSTD_e_end or ZSTD_e_flush operations are allowed.
|
|
652
|
+
* Before starting a new compression job, or changing compression parameters,
|
|
653
|
+
* it is required to fully flush internal buffers.
|
|
654
|
+
*/
|
|
655
|
+
ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
|
|
656
|
+
ZSTD_outBuffer* output,
|
|
657
|
+
ZSTD_inBuffer* input,
|
|
658
|
+
ZSTD_EndDirective endOp);
|
|
319
659
|
|
|
320
660
|
ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
|
|
321
661
|
ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */
|
|
322
662
|
|
|
663
|
+
/*******************************************************************************
|
|
664
|
+
* This is a legacy streaming API, and can be replaced by ZSTD_CCtx_reset() and
|
|
665
|
+
* ZSTD_compressStream2(). It is redundant, but is still fully supported.
|
|
666
|
+
* Advanced parameters and dictionary compression can only be used through the
|
|
667
|
+
* new API.
|
|
668
|
+
******************************************************************************/
|
|
669
|
+
|
|
670
|
+
/**
|
|
671
|
+
* Equivalent to:
|
|
672
|
+
*
|
|
673
|
+
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
|
674
|
+
* ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
|
|
675
|
+
* ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
|
|
676
|
+
*/
|
|
677
|
+
ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
|
|
678
|
+
/**
|
|
679
|
+
* Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).
|
|
680
|
+
* NOTE: The return value is different. ZSTD_compressStream() returns a hint for
|
|
681
|
+
* the next read size (if non-zero and not an error). ZSTD_compressStream2()
|
|
682
|
+
* returns the number of bytes left to flush (if non-zero and not an error).
|
|
683
|
+
*/
|
|
684
|
+
ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
|
|
685
|
+
/** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */
|
|
686
|
+
ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
|
|
687
|
+
/** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */
|
|
688
|
+
ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
|
|
323
689
|
|
|
324
690
|
|
|
325
691
|
/*-***************************************************************************
|
|
@@ -329,136 +695,457 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output
|
|
|
329
695
|
* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
|
|
330
696
|
* ZSTD_DStream objects can be re-used multiple times.
|
|
331
697
|
*
|
|
332
|
-
* Use ZSTD_initDStream() to start a new decompression operation
|
|
333
|
-
*
|
|
334
|
-
*
|
|
698
|
+
* Use ZSTD_initDStream() to start a new decompression operation.
|
|
699
|
+
* @return : recommended first input size
|
|
700
|
+
* Alternatively, use advanced API to set specific properties.
|
|
335
701
|
*
|
|
336
702
|
* Use ZSTD_decompressStream() repetitively to consume your input.
|
|
337
703
|
* The function will update both `pos` fields.
|
|
338
704
|
* If `input.pos < input.size`, some input has not been consumed.
|
|
339
705
|
* It's up to the caller to present again remaining data.
|
|
706
|
+
* The function tries to flush all data decoded immediately, respecting output buffer size.
|
|
340
707
|
* If `output.pos < output.size`, decoder has flushed everything it could.
|
|
341
|
-
*
|
|
342
|
-
*
|
|
343
|
-
*
|
|
344
|
-
*
|
|
708
|
+
* But if `output.pos == output.size`, there might be some data left within internal buffers.,
|
|
709
|
+
* In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
|
|
710
|
+
* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
|
|
711
|
+
* @return : 0 when a frame is completely decoded and fully flushed,
|
|
712
|
+
* or an error code, which can be tested using ZSTD_isError(),
|
|
713
|
+
* or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
|
|
714
|
+
* the return value is a suggested next input size (just a hint for better latency)
|
|
715
|
+
* that will never request more than the remaining frame size.
|
|
345
716
|
* *******************************************************************************/
|
|
346
717
|
|
|
347
718
|
typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */
|
|
348
|
-
/*
|
|
719
|
+
/* For compatibility with versions <= v1.2.0, prefer differentiating them. */
|
|
349
720
|
/*===== ZSTD_DStream management functions =====*/
|
|
350
721
|
ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
|
|
351
722
|
ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
|
|
352
723
|
|
|
353
724
|
/*===== Streaming decompression functions =====*/
|
|
725
|
+
|
|
726
|
+
/* This function is redundant with the advanced API and equivalent to:
|
|
727
|
+
*
|
|
728
|
+
* ZSTD_DCtx_reset(zds);
|
|
729
|
+
* ZSTD_DCtx_refDDict(zds, NULL);
|
|
730
|
+
*/
|
|
354
731
|
ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
|
|
732
|
+
|
|
355
733
|
ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
|
|
356
734
|
|
|
357
735
|
ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */
|
|
358
736
|
ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
|
|
359
737
|
|
|
360
|
-
#endif /* ZSTD_H_235446 */
|
|
361
738
|
|
|
739
|
+
/**************************
|
|
740
|
+
* Simple dictionary API
|
|
741
|
+
***************************/
|
|
742
|
+
/*! ZSTD_compress_usingDict() :
|
|
743
|
+
* Compression at an explicit compression level using a Dictionary.
|
|
744
|
+
* A dictionary can be any arbitrary data segment (also called a prefix),
|
|
745
|
+
* or a buffer with specified information (see dictBuilder/zdict.h).
|
|
746
|
+
* Note : This function loads the dictionary, resulting in significant startup delay.
|
|
747
|
+
* It's intended for a dictionary used only once.
|
|
748
|
+
* Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */
|
|
749
|
+
ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
|
|
750
|
+
void* dst, size_t dstCapacity,
|
|
751
|
+
const void* src, size_t srcSize,
|
|
752
|
+
const void* dict,size_t dictSize,
|
|
753
|
+
int compressionLevel);
|
|
754
|
+
|
|
755
|
+
/*! ZSTD_decompress_usingDict() :
|
|
756
|
+
* Decompression using a known Dictionary.
|
|
757
|
+
* Dictionary must be identical to the one used during compression.
|
|
758
|
+
* Note : This function loads the dictionary, resulting in significant startup delay.
|
|
759
|
+
* It's intended for a dictionary used only once.
|
|
760
|
+
* Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
|
|
761
|
+
ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
|
|
762
|
+
void* dst, size_t dstCapacity,
|
|
763
|
+
const void* src, size_t srcSize,
|
|
764
|
+
const void* dict,size_t dictSize);
|
|
362
765
|
|
|
363
766
|
|
|
364
|
-
|
|
365
|
-
*
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
* They are provided for advanced scenarios.
|
|
369
|
-
* Use them only in association with static linking.
|
|
370
|
-
* ***************************************************************************************/
|
|
767
|
+
/***********************************
|
|
768
|
+
* Bulk processing dictionary API
|
|
769
|
+
**********************************/
|
|
770
|
+
typedef struct ZSTD_CDict_s ZSTD_CDict;
|
|
371
771
|
|
|
372
|
-
|
|
373
|
-
|
|
772
|
+
/*! ZSTD_createCDict() :
|
|
773
|
+
* When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once.
|
|
774
|
+
* ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup cost.
|
|
775
|
+
* ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
|
|
776
|
+
* `dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict.
|
|
777
|
+
* Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content.
|
|
778
|
+
* Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data. */
|
|
779
|
+
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
|
|
780
|
+
int compressionLevel);
|
|
374
781
|
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* v0.7+ */
|
|
379
|
-
|
|
380
|
-
#define ZSTD_WINDOWLOG_MAX_32 30
|
|
381
|
-
#define ZSTD_WINDOWLOG_MAX_64 31
|
|
382
|
-
#define ZSTD_WINDOWLOG_MAX ((unsigned)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
|
|
383
|
-
#define ZSTD_WINDOWLOG_MIN 10
|
|
384
|
-
#define ZSTD_HASHLOG_MAX MIN(ZSTD_WINDOWLOG_MAX, 30)
|
|
385
|
-
#define ZSTD_HASHLOG_MIN 6
|
|
386
|
-
#define ZSTD_CHAINLOG_MAX MIN(ZSTD_WINDOWLOG_MAX+1, 30)
|
|
387
|
-
#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN
|
|
388
|
-
#define ZSTD_HASHLOG3_MAX 17
|
|
389
|
-
#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
|
|
390
|
-
#define ZSTD_SEARCHLOG_MIN 1
|
|
391
|
-
#define ZSTD_SEARCHLENGTH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */
|
|
392
|
-
#define ZSTD_SEARCHLENGTH_MIN 3 /* only for ZSTD_btopt, other strategies are limited to 4 */
|
|
393
|
-
#define ZSTD_TARGETLENGTH_MIN 4 /* only useful for btopt */
|
|
394
|
-
#define ZSTD_TARGETLENGTH_MAX 999 /* only useful for btopt */
|
|
395
|
-
#define ZSTD_LDM_MINMATCH_MIN 4
|
|
396
|
-
#define ZSTD_LDM_MINMATCH_MAX 4096
|
|
397
|
-
#define ZSTD_LDM_BUCKETSIZELOG_MAX 8
|
|
398
|
-
|
|
399
|
-
#define ZSTD_FRAMEHEADERSIZE_PREFIX 5 /* minimum input size to know frame header size */
|
|
400
|
-
#define ZSTD_FRAMEHEADERSIZE_MIN 6
|
|
401
|
-
#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */
|
|
402
|
-
static const size_t ZSTD_frameHeaderSize_prefix = ZSTD_FRAMEHEADERSIZE_PREFIX;
|
|
403
|
-
static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN;
|
|
404
|
-
static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
|
|
405
|
-
static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */
|
|
782
|
+
/*! ZSTD_freeCDict() :
|
|
783
|
+
* Function frees memory allocated by ZSTD_createCDict(). */
|
|
784
|
+
ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
|
|
406
785
|
|
|
786
|
+
/*! ZSTD_compress_usingCDict() :
|
|
787
|
+
* Compression using a digested Dictionary.
|
|
788
|
+
* Recommended when same dictionary is used multiple times.
|
|
789
|
+
* Note : compression level is _decided at dictionary creation time_,
|
|
790
|
+
* and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */
|
|
791
|
+
ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
|
|
792
|
+
void* dst, size_t dstCapacity,
|
|
793
|
+
const void* src, size_t srcSize,
|
|
794
|
+
const ZSTD_CDict* cdict);
|
|
407
795
|
|
|
408
|
-
/*--- Advanced types ---*/
|
|
409
|
-
typedef enum { ZSTD_fast=1, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2,
|
|
410
|
-
ZSTD_btlazy2, ZSTD_btopt, ZSTD_btultra } ZSTD_strategy; /* from faster to stronger */
|
|
411
796
|
|
|
412
|
-
typedef struct
|
|
413
|
-
unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
|
|
414
|
-
unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
|
|
415
|
-
unsigned hashLog; /**< dispatch table : larger == faster, more memory */
|
|
416
|
-
unsigned searchLog; /**< nb of searches : larger == more compression, slower */
|
|
417
|
-
unsigned searchLength; /**< match length searched : larger == faster decompression, sometimes less compression */
|
|
418
|
-
unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */
|
|
419
|
-
ZSTD_strategy strategy;
|
|
420
|
-
} ZSTD_compressionParameters;
|
|
797
|
+
typedef struct ZSTD_DDict_s ZSTD_DDict;
|
|
421
798
|
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
} ZSTD_frameParameters;
|
|
799
|
+
/*! ZSTD_createDDict() :
|
|
800
|
+
* Create a digested dictionary, ready to start decompression operation without startup delay.
|
|
801
|
+
* dictBuffer can be released after DDict creation, as its content is copied inside DDict. */
|
|
802
|
+
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
|
|
427
803
|
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
} ZSTD_parameters;
|
|
804
|
+
/*! ZSTD_freeDDict() :
|
|
805
|
+
* Function frees memory allocated with ZSTD_createDDict() */
|
|
806
|
+
ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
|
|
432
807
|
|
|
433
|
-
|
|
808
|
+
/*! ZSTD_decompress_usingDDict() :
|
|
809
|
+
* Decompression using a digested Dictionary.
|
|
810
|
+
* Recommended when same dictionary is used multiple times. */
|
|
811
|
+
ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
|
|
812
|
+
void* dst, size_t dstCapacity,
|
|
813
|
+
const void* src, size_t srcSize,
|
|
814
|
+
const ZSTD_DDict* ddict);
|
|
434
815
|
|
|
435
|
-
/*--- Custom memory allocation functions ---*/
|
|
436
|
-
typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
|
|
437
|
-
typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
|
|
438
|
-
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
|
|
439
|
-
/* use this constant to defer to stdlib's functions */
|
|
440
|
-
static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };
|
|
441
816
|
|
|
817
|
+
/********************************
|
|
818
|
+
* Dictionary helper functions
|
|
819
|
+
*******************************/
|
|
442
820
|
|
|
443
|
-
|
|
444
|
-
*
|
|
445
|
-
|
|
821
|
+
/*! ZSTD_getDictID_fromDict() :
|
|
822
|
+
* Provides the dictID stored within dictionary.
|
|
823
|
+
* if @return == 0, the dictionary is not conformant with Zstandard specification.
|
|
824
|
+
* It can still be loaded, but as a content-only dictionary. */
|
|
825
|
+
ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
|
|
446
826
|
|
|
447
|
-
/*!
|
|
448
|
-
*
|
|
449
|
-
*
|
|
450
|
-
*
|
|
451
|
-
|
|
452
|
-
* or an error code if input is invalid */
|
|
453
|
-
ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
|
|
827
|
+
/*! ZSTD_getDictID_fromDDict() :
|
|
828
|
+
* Provides the dictID of the dictionary loaded into `ddict`.
|
|
829
|
+
* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
|
|
830
|
+
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
|
|
831
|
+
ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
|
|
454
832
|
|
|
455
|
-
/*!
|
|
456
|
-
*
|
|
457
|
-
*
|
|
458
|
-
*
|
|
459
|
-
*
|
|
460
|
-
*
|
|
461
|
-
*
|
|
833
|
+
/*! ZSTD_getDictID_fromFrame() :
|
|
834
|
+
* Provides the dictID required to decompressed the frame stored within `src`.
|
|
835
|
+
* If @return == 0, the dictID could not be decoded.
|
|
836
|
+
* This could for one of the following reasons :
|
|
837
|
+
* - The frame does not require a dictionary to be decoded (most common case).
|
|
838
|
+
* - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
|
|
839
|
+
* Note : this use case also happens when using a non-conformant dictionary.
|
|
840
|
+
* - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
|
|
841
|
+
* - This is not a Zstandard frame.
|
|
842
|
+
* When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */
|
|
843
|
+
ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
|
|
844
|
+
|
|
845
|
+
|
|
846
|
+
/*******************************************************************************
|
|
847
|
+
* Advanced dictionary and prefix API
|
|
848
|
+
*
|
|
849
|
+
* This API allows dictionaries to be used with ZSTD_compress2(),
|
|
850
|
+
* ZSTD_compressStream2(), and ZSTD_decompress(). Dictionaries are sticky, and
|
|
851
|
+
* only reset with the context is reset with ZSTD_reset_parameters or
|
|
852
|
+
* ZSTD_reset_session_and_parameters. Prefixes are single-use.
|
|
853
|
+
******************************************************************************/
|
|
854
|
+
|
|
855
|
+
|
|
856
|
+
/*! ZSTD_CCtx_loadDictionary() :
|
|
857
|
+
* Create an internal CDict from `dict` buffer.
|
|
858
|
+
* Decompression will have to use same dictionary.
|
|
859
|
+
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
860
|
+
* Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary,
|
|
861
|
+
* meaning "return to no-dictionary mode".
|
|
862
|
+
* Note 1 : Dictionary is sticky, it will be used for all future compressed frames.
|
|
863
|
+
* To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters).
|
|
864
|
+
* Note 2 : Loading a dictionary involves building tables.
|
|
865
|
+
* It's also a CPU consuming operation, with non-negligible impact on latency.
|
|
866
|
+
* Tables are dependent on compression parameters, and for this reason,
|
|
867
|
+
* compression parameters can no longer be changed after loading a dictionary.
|
|
868
|
+
* Note 3 :`dict` content will be copied internally.
|
|
869
|
+
* Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead.
|
|
870
|
+
* In such a case, dictionary buffer must outlive its users.
|
|
871
|
+
* Note 4 : Use ZSTD_CCtx_loadDictionary_advanced()
|
|
872
|
+
* to precisely select how dictionary content must be interpreted. */
|
|
873
|
+
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
|
|
874
|
+
|
|
875
|
+
/*! ZSTD_CCtx_refCDict() :
|
|
876
|
+
* Reference a prepared dictionary, to be used for all next compressed frames.
|
|
877
|
+
* Note that compression parameters are enforced from within CDict,
|
|
878
|
+
* and supersede any compression parameter previously set within CCtx.
|
|
879
|
+
* The parameters ignored are labled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
|
|
880
|
+
* The ignored parameters will be used again if the CCtx is returned to no-dictionary mode.
|
|
881
|
+
* The dictionary will remain valid for future compressed frames using same CCtx.
|
|
882
|
+
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
883
|
+
* Special : Referencing a NULL CDict means "return to no-dictionary mode".
|
|
884
|
+
* Note 1 : Currently, only one dictionary can be managed.
|
|
885
|
+
* Referencing a new dictionary effectively "discards" any previous one.
|
|
886
|
+
* Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */
|
|
887
|
+
ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
|
|
888
|
+
|
|
889
|
+
/*! ZSTD_CCtx_refPrefix() :
|
|
890
|
+
* Reference a prefix (single-usage dictionary) for next compressed frame.
|
|
891
|
+
* A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end).
|
|
892
|
+
* Decompression will need same prefix to properly regenerate data.
|
|
893
|
+
* Compressing with a prefix is similar in outcome as performing a diff and compressing it,
|
|
894
|
+
* but performs much faster, especially during decompression (compression speed is tunable with compression level).
|
|
895
|
+
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
896
|
+
* Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary
|
|
897
|
+
* Note 1 : Prefix buffer is referenced. It **must** outlive compression.
|
|
898
|
+
* Its content must remain unmodified during compression.
|
|
899
|
+
* Note 2 : If the intention is to diff some large src data blob with some prior version of itself,
|
|
900
|
+
* ensure that the window size is large enough to contain the entire source.
|
|
901
|
+
* See ZSTD_c_windowLog.
|
|
902
|
+
* Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
|
|
903
|
+
* It's a CPU consuming operation, with non-negligible impact on latency.
|
|
904
|
+
* If there is a need to use the same prefix multiple times, consider loadDictionary instead.
|
|
905
|
+
* Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent).
|
|
906
|
+
* Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */
|
|
907
|
+
ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
|
|
908
|
+
const void* prefix, size_t prefixSize);
|
|
909
|
+
|
|
910
|
+
/*! ZSTD_DCtx_loadDictionary() :
|
|
911
|
+
* Create an internal DDict from dict buffer,
|
|
912
|
+
* to be used to decompress next frames.
|
|
913
|
+
* The dictionary remains valid for all future frames, until explicitly invalidated.
|
|
914
|
+
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
915
|
+
* Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
|
|
916
|
+
* meaning "return to no-dictionary mode".
|
|
917
|
+
* Note 1 : Loading a dictionary involves building tables,
|
|
918
|
+
* which has a non-negligible impact on CPU usage and latency.
|
|
919
|
+
* It's recommended to "load once, use many times", to amortize the cost
|
|
920
|
+
* Note 2 :`dict` content will be copied internally, so `dict` can be released after loading.
|
|
921
|
+
* Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead.
|
|
922
|
+
* Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of
|
|
923
|
+
* how dictionary content is loaded and interpreted.
|
|
924
|
+
*/
|
|
925
|
+
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
|
926
|
+
|
|
927
|
+
/*! ZSTD_DCtx_refDDict() :
|
|
928
|
+
* Reference a prepared dictionary, to be used to decompress next frames.
|
|
929
|
+
* The dictionary remains active for decompression of future frames using same DCtx.
|
|
930
|
+
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
931
|
+
* Note 1 : Currently, only one dictionary can be managed.
|
|
932
|
+
* Referencing a new dictionary effectively "discards" any previous one.
|
|
933
|
+
* Special: referencing a NULL DDict means "return to no-dictionary mode".
|
|
934
|
+
* Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
|
|
935
|
+
*/
|
|
936
|
+
ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
|
|
937
|
+
|
|
938
|
+
/*! ZSTD_DCtx_refPrefix() :
|
|
939
|
+
* Reference a prefix (single-usage dictionary) to decompress next frame.
|
|
940
|
+
* This is the reverse operation of ZSTD_CCtx_refPrefix(),
|
|
941
|
+
* and must use the same prefix as the one used during compression.
|
|
942
|
+
* Prefix is **only used once**. Reference is discarded at end of frame.
|
|
943
|
+
* End of frame is reached when ZSTD_decompressStream() returns 0.
|
|
944
|
+
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
945
|
+
* Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary
|
|
946
|
+
* Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
|
|
947
|
+
* Prefix buffer must remain unmodified up to the end of frame,
|
|
948
|
+
* reached when ZSTD_decompressStream() returns 0.
|
|
949
|
+
* Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
|
|
950
|
+
* Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
|
|
951
|
+
* Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
|
|
952
|
+
* A full dictionary is more costly, as it requires building tables.
|
|
953
|
+
*/
|
|
954
|
+
ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx,
|
|
955
|
+
const void* prefix, size_t prefixSize);
|
|
956
|
+
|
|
957
|
+
/* === Memory management === */
|
|
958
|
+
|
|
959
|
+
/*! ZSTD_sizeof_*() :
|
|
960
|
+
* These functions give the _current_ memory usage of selected object.
|
|
961
|
+
* Note that object memory usage can evolve (increase or decrease) over time. */
|
|
962
|
+
ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
|
|
963
|
+
ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
|
|
964
|
+
ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
|
|
965
|
+
ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
|
|
966
|
+
ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
|
|
967
|
+
ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
|
968
|
+
|
|
969
|
+
#endif /* ZSTD_H_235446 */
|
|
970
|
+
|
|
971
|
+
|
|
972
|
+
/****************************************************************************************
|
|
973
|
+
* ADVANCED AND EXPERIMENTAL FUNCTIONS
|
|
974
|
+
****************************************************************************************
|
|
975
|
+
* The definitions in the following section are considered experimental.
|
|
976
|
+
* They are provided for advanced scenarios.
|
|
977
|
+
* They should never be used with a dynamic library, as prototypes may change in the future.
|
|
978
|
+
* Use them only in association with static linking.
|
|
979
|
+
* ***************************************************************************************/
|
|
980
|
+
|
|
981
|
+
#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
|
|
982
|
+
#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
|
|
983
|
+
|
|
984
|
+
/****************************************************************************************
|
|
985
|
+
* experimental API (static linking only)
|
|
986
|
+
****************************************************************************************
|
|
987
|
+
* The following symbols and constants
|
|
988
|
+
* are not planned to join "stable API" status in the near future.
|
|
989
|
+
* They can still change in future versions.
|
|
990
|
+
* Some of them are planned to remain in the static_only section indefinitely.
|
|
991
|
+
* Some of them might be removed in the future (especially when redundant with existing stable functions)
|
|
992
|
+
* ***************************************************************************************/
|
|
993
|
+
|
|
994
|
+
#define ZSTD_FRAMEHEADERSIZE_PREFIX 5 /* minimum input size required to query frame header size */
|
|
995
|
+
#define ZSTD_FRAMEHEADERSIZE_MIN 6
|
|
996
|
+
#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */
|
|
997
|
+
#define ZSTD_SKIPPABLEHEADERSIZE 8
|
|
998
|
+
|
|
999
|
+
/* compression parameter bounds */
|
|
1000
|
+
#define ZSTD_WINDOWLOG_MAX_32 30
|
|
1001
|
+
#define ZSTD_WINDOWLOG_MAX_64 31
|
|
1002
|
+
#define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
|
|
1003
|
+
#define ZSTD_WINDOWLOG_MIN 10
|
|
1004
|
+
#define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30)
|
|
1005
|
+
#define ZSTD_HASHLOG_MIN 6
|
|
1006
|
+
#define ZSTD_CHAINLOG_MAX_32 29
|
|
1007
|
+
#define ZSTD_CHAINLOG_MAX_64 30
|
|
1008
|
+
#define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64))
|
|
1009
|
+
#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN
|
|
1010
|
+
#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
|
|
1011
|
+
#define ZSTD_SEARCHLOG_MIN 1
|
|
1012
|
+
#define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */
|
|
1013
|
+
#define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */
|
|
1014
|
+
#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX
|
|
1015
|
+
#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */
|
|
1016
|
+
#define ZSTD_STRATEGY_MIN ZSTD_fast
|
|
1017
|
+
#define ZSTD_STRATEGY_MAX ZSTD_btultra2
|
|
1018
|
+
|
|
1019
|
+
|
|
1020
|
+
#define ZSTD_OVERLAPLOG_MIN 0
|
|
1021
|
+
#define ZSTD_OVERLAPLOG_MAX 9
|
|
1022
|
+
|
|
1023
|
+
#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame
|
|
1024
|
+
* requiring larger than (1<<ZSTD_WINDOWLOG_LIMIT_DEFAULT) window size,
|
|
1025
|
+
* to preserve host's memory from unreasonable requirements.
|
|
1026
|
+
* This limit can be overridden using ZSTD_DCtx_setParameter(,ZSTD_d_windowLogMax,).
|
|
1027
|
+
* The limit does not apply for one-pass decoders (such as ZSTD_decompress()), since no additional memory is allocated */
|
|
1028
|
+
|
|
1029
|
+
|
|
1030
|
+
/* LDM parameter bounds */
|
|
1031
|
+
#define ZSTD_LDM_HASHLOG_MIN ZSTD_HASHLOG_MIN
|
|
1032
|
+
#define ZSTD_LDM_HASHLOG_MAX ZSTD_HASHLOG_MAX
|
|
1033
|
+
#define ZSTD_LDM_MINMATCH_MIN 4
|
|
1034
|
+
#define ZSTD_LDM_MINMATCH_MAX 4096
|
|
1035
|
+
#define ZSTD_LDM_BUCKETSIZELOG_MIN 1
|
|
1036
|
+
#define ZSTD_LDM_BUCKETSIZELOG_MAX 8
|
|
1037
|
+
#define ZSTD_LDM_HASHRATELOG_MIN 0
|
|
1038
|
+
#define ZSTD_LDM_HASHRATELOG_MAX (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN)
|
|
1039
|
+
|
|
1040
|
+
/* internal */
|
|
1041
|
+
#define ZSTD_HASHLOG3_MAX 17
|
|
1042
|
+
|
|
1043
|
+
|
|
1044
|
+
/* --- Advanced types --- */
|
|
1045
|
+
|
|
1046
|
+
typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params;
|
|
1047
|
+
|
|
1048
|
+
typedef struct {
|
|
1049
|
+
unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
|
|
1050
|
+
unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
|
|
1051
|
+
unsigned hashLog; /**< dispatch table : larger == faster, more memory */
|
|
1052
|
+
unsigned searchLog; /**< nb of searches : larger == more compression, slower */
|
|
1053
|
+
unsigned minMatch; /**< match length searched : larger == faster decompression, sometimes less compression */
|
|
1054
|
+
unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */
|
|
1055
|
+
ZSTD_strategy strategy; /**< see ZSTD_strategy definition above */
|
|
1056
|
+
} ZSTD_compressionParameters;
|
|
1057
|
+
|
|
1058
|
+
typedef struct {
|
|
1059
|
+
int contentSizeFlag; /**< 1: content size will be in frame header (when known) */
|
|
1060
|
+
int checksumFlag; /**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */
|
|
1061
|
+
int noDictIDFlag; /**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */
|
|
1062
|
+
} ZSTD_frameParameters;
|
|
1063
|
+
|
|
1064
|
+
typedef struct {
|
|
1065
|
+
ZSTD_compressionParameters cParams;
|
|
1066
|
+
ZSTD_frameParameters fParams;
|
|
1067
|
+
} ZSTD_parameters;
|
|
1068
|
+
|
|
1069
|
+
typedef enum {
|
|
1070
|
+
ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */
|
|
1071
|
+
ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */
|
|
1072
|
+
ZSTD_dct_fullDict = 2 /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */
|
|
1073
|
+
} ZSTD_dictContentType_e;
|
|
1074
|
+
|
|
1075
|
+
typedef enum {
|
|
1076
|
+
ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */
|
|
1077
|
+
ZSTD_dlm_byRef = 1, /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
|
|
1078
|
+
} ZSTD_dictLoadMethod_e;
|
|
1079
|
+
|
|
1080
|
+
typedef enum {
|
|
1081
|
+
/* Opened question : should we have a format ZSTD_f_auto ?
|
|
1082
|
+
* Today, it would mean exactly the same as ZSTD_f_zstd1.
|
|
1083
|
+
* But, in the future, should several formats become supported,
|
|
1084
|
+
* on the compression side, it would mean "default format".
|
|
1085
|
+
* On the decompression side, it would mean "automatic format detection",
|
|
1086
|
+
* so that ZSTD_f_zstd1 would mean "accept *only* zstd frames".
|
|
1087
|
+
* Since meaning is a little different, another option could be to define different enums for compression and decompression.
|
|
1088
|
+
* This question could be kept for later, when there are actually multiple formats to support,
|
|
1089
|
+
* but there is also the question of pinning enum values, and pinning value `0` is especially important */
|
|
1090
|
+
ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */
|
|
1091
|
+
ZSTD_f_zstd1_magicless = 1, /* Variant of zstd frame format, without initial 4-bytes magic number.
|
|
1092
|
+
* Useful to save 4 bytes per generated frame.
|
|
1093
|
+
* Decoder cannot recognise automatically this format, requiring this instruction. */
|
|
1094
|
+
} ZSTD_format_e;
|
|
1095
|
+
|
|
1096
|
+
typedef enum {
|
|
1097
|
+
/* Note: this enum and the behavior it controls are effectively internal
|
|
1098
|
+
* implementation details of the compressor. They are expected to continue
|
|
1099
|
+
* to evolve and should be considered only in the context of extremely
|
|
1100
|
+
* advanced performance tuning.
|
|
1101
|
+
*
|
|
1102
|
+
* Zstd currently supports the use of a CDict in two ways:
|
|
1103
|
+
*
|
|
1104
|
+
* - The contents of the CDict can be copied into the working context. This
|
|
1105
|
+
* means that the compression can search both the dictionary and input
|
|
1106
|
+
* while operating on a single set of internal tables. This makes
|
|
1107
|
+
* the compression faster per-byte of input. However, the initial copy of
|
|
1108
|
+
* the CDict's tables incurs a fixed cost at the beginning of the
|
|
1109
|
+
* compression. For small compressions (< 8 KB), that copy can dominate
|
|
1110
|
+
* the cost of the compression.
|
|
1111
|
+
*
|
|
1112
|
+
* - The CDict's tables can be used in-place. In this model, compression is
|
|
1113
|
+
* slower per input byte, because the compressor has to search two sets of
|
|
1114
|
+
* tables. However, this model incurs no start-up cost (as long as the
|
|
1115
|
+
* working context's tables can be reused). For small inputs, this can be
|
|
1116
|
+
* faster than copying the CDict's tables.
|
|
1117
|
+
*
|
|
1118
|
+
* Zstd has a simple internal heuristic that selects which strategy to use
|
|
1119
|
+
* at the beginning of a compression. However, if experimentation shows that
|
|
1120
|
+
* Zstd is making poor choices, it is possible to override that choice with
|
|
1121
|
+
* this enum.
|
|
1122
|
+
*/
|
|
1123
|
+
ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */
|
|
1124
|
+
ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */
|
|
1125
|
+
ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */
|
|
1126
|
+
} ZSTD_dictAttachPref_e;
|
|
1127
|
+
|
|
1128
|
+
typedef enum {
|
|
1129
|
+
ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level.
|
|
1130
|
+
* Negative compression levels will be uncompressed, and positive compression
|
|
1131
|
+
* levels will be compressed. */
|
|
1132
|
+
ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be
|
|
1133
|
+
* emitted if Huffman compression is not profitable. */
|
|
1134
|
+
ZSTD_lcm_uncompressed = 2, /**< Always emit uncompressed literals. */
|
|
1135
|
+
} ZSTD_literalCompressionMode_e;
|
|
1136
|
+
|
|
1137
|
+
|
|
1138
|
+
/***************************************
|
|
1139
|
+
* Frame size functions
|
|
1140
|
+
***************************************/
|
|
1141
|
+
|
|
1142
|
+
/*! ZSTD_findDecompressedSize() :
|
|
1143
|
+
* `src` should point to the start of a series of ZSTD encoded and/or skippable frames
|
|
1144
|
+
* `srcSize` must be the _exact_ size of this series
|
|
1145
|
+
* (i.e. there should be a frame boundary at `src + srcSize`)
|
|
1146
|
+
* @return : - decompressed size of all data in all successive frames
|
|
1147
|
+
* - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
|
|
1148
|
+
* - if an error occurred: ZSTD_CONTENTSIZE_ERROR
|
|
462
1149
|
*
|
|
463
1150
|
* note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
|
|
464
1151
|
* When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
|
|
@@ -475,27 +1162,32 @@ ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
|
|
|
475
1162
|
* however it does mean that all frame data must be present and valid. */
|
|
476
1163
|
ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
|
|
477
1164
|
|
|
1165
|
+
/** ZSTD_decompressBound() :
|
|
1166
|
+
* `src` should point to the start of a series of ZSTD encoded and/or skippable frames
|
|
1167
|
+
* `srcSize` must be the _exact_ size of this series
|
|
1168
|
+
* (i.e. there should be a frame boundary at `src + srcSize`)
|
|
1169
|
+
* @return : - upper-bound for the decompressed size of all data in all successive frames
|
|
1170
|
+
* - if an error occured: ZSTD_CONTENTSIZE_ERROR
|
|
1171
|
+
*
|
|
1172
|
+
* note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame.
|
|
1173
|
+
* note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`.
|
|
1174
|
+
* in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value.
|
|
1175
|
+
* note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by:
|
|
1176
|
+
* upper-bound = # blocks * min(128 KB, Window_Size)
|
|
1177
|
+
*/
|
|
1178
|
+
ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize);
|
|
1179
|
+
|
|
478
1180
|
/*! ZSTD_frameHeaderSize() :
|
|
479
|
-
*
|
|
480
|
-
*
|
|
481
|
-
*
|
|
1181
|
+
* srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
|
|
1182
|
+
* @return : size of the Frame Header,
|
|
1183
|
+
* or an error code (if srcSize is too small) */
|
|
482
1184
|
ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
|
|
483
1185
|
|
|
484
1186
|
|
|
485
1187
|
/***************************************
|
|
486
|
-
*
|
|
1188
|
+
* Memory management
|
|
487
1189
|
***************************************/
|
|
488
1190
|
|
|
489
|
-
/*! ZSTD_sizeof_*() :
|
|
490
|
-
* These functions give the current memory usage of selected object.
|
|
491
|
-
* Object memory usage can evolve when re-used multiple times. */
|
|
492
|
-
ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
|
|
493
|
-
ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
|
|
494
|
-
ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
|
|
495
|
-
ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
|
|
496
|
-
ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
|
|
497
|
-
ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
|
498
|
-
|
|
499
1191
|
/*! ZSTD_estimate*() :
|
|
500
1192
|
* These functions make it possible to estimate memory usage
|
|
501
1193
|
* of a future {D,C}Ctx, before its creation.
|
|
@@ -503,8 +1195,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
|
|
503
1195
|
* It will also consider src size to be arbitrarily "large", which is worst case.
|
|
504
1196
|
* If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation.
|
|
505
1197
|
* ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
|
|
506
|
-
* ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with
|
|
507
|
-
* Note : CCtx estimation is only correct for single-threaded compression */
|
|
1198
|
+
* ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
|
|
1199
|
+
* Note : CCtx size estimation is only correct for single-threaded compression. */
|
|
508
1200
|
ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
|
|
509
1201
|
ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
|
|
510
1202
|
ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
|
|
@@ -515,8 +1207,8 @@ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
|
|
|
515
1207
|
* It will also consider src size to be arbitrarily "large", which is worst case.
|
|
516
1208
|
* If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation.
|
|
517
1209
|
* ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
|
|
518
|
-
* ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with
|
|
519
|
-
* Note : CStream estimation is only correct for single-threaded compression.
|
|
1210
|
+
* ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
|
|
1211
|
+
* Note : CStream size estimation is only correct for single-threaded compression.
|
|
520
1212
|
* ZSTD_DStream memory budget depends on window Size.
|
|
521
1213
|
* This information can be passed manually, using ZSTD_estimateDStreamSize,
|
|
522
1214
|
* or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
|
|
@@ -529,121 +1221,280 @@ ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_para
|
|
|
529
1221
|
ZSTDLIB_API size_t ZSTD_estimateDStreamSize(size_t windowSize);
|
|
530
1222
|
ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
|
|
531
1223
|
|
|
532
|
-
typedef enum {
|
|
533
|
-
ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */
|
|
534
|
-
ZSTD_dlm_byRef, /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
|
|
535
|
-
} ZSTD_dictLoadMethod_e;
|
|
536
|
-
|
|
537
1224
|
/*! ZSTD_estimate?DictSize() :
|
|
538
1225
|
* ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
|
|
539
|
-
*
|
|
540
|
-
* Note :
|
|
1226
|
+
* ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced().
|
|
1227
|
+
* Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller.
|
|
541
1228
|
*/
|
|
542
1229
|
ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
|
|
543
1230
|
ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod);
|
|
544
1231
|
ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod);
|
|
545
1232
|
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
*
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
*
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
*
|
|
556
|
-
*
|
|
557
|
-
*
|
|
558
|
-
* workspaceSize: Use ZSTD_estimateCCtxSize() or ZSTD_estimateCStreamSize()
|
|
559
|
-
* to determine how large workspace must be to support scenario.
|
|
560
|
-
* @return : pointer to ZSTD_CCtx* (same address as workspace, but different type),
|
|
561
|
-
* or NULL if error (typically size too small)
|
|
562
|
-
* Note : zstd will never resize nor malloc() when using a static cctx.
|
|
563
|
-
* If it needs more memory than available, it will simply error out.
|
|
1233
|
+
/*! ZSTD_initStatic*() :
|
|
1234
|
+
* Initialize an object using a pre-allocated fixed-size buffer.
|
|
1235
|
+
* workspace: The memory area to emplace the object into.
|
|
1236
|
+
* Provided pointer *must be 8-bytes aligned*.
|
|
1237
|
+
* Buffer must outlive object.
|
|
1238
|
+
* workspaceSize: Use ZSTD_estimate*Size() to determine
|
|
1239
|
+
* how large workspace must be to support target scenario.
|
|
1240
|
+
* @return : pointer to object (same address as workspace, just different type),
|
|
1241
|
+
* or NULL if error (size too small, incorrect alignment, etc.)
|
|
1242
|
+
* Note : zstd will never resize nor malloc() when using a static buffer.
|
|
1243
|
+
* If the object requires more memory than available,
|
|
1244
|
+
* zstd will just error out (typically ZSTD_error_memory_allocation).
|
|
564
1245
|
* Note 2 : there is no corresponding "free" function.
|
|
565
|
-
* Since workspace
|
|
566
|
-
*
|
|
567
|
-
*
|
|
568
|
-
* Limitation
|
|
1246
|
+
* Since workspace is allocated externally, it must be freed externally too.
|
|
1247
|
+
* Note 3 : cParams : use ZSTD_getCParams() to convert a compression level
|
|
1248
|
+
* into its associated cParams.
|
|
1249
|
+
* Limitation 1 : currently not compatible with internal dictionary creation, triggered by
|
|
1250
|
+
* ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict().
|
|
1251
|
+
* Limitation 2 : static cctx currently not compatible with multi-threading.
|
|
1252
|
+
* Limitation 3 : static dctx is incompatible with legacy support.
|
|
569
1253
|
*/
|
|
570
|
-
ZSTDLIB_API ZSTD_CCtx*
|
|
1254
|
+
ZSTDLIB_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
|
|
1255
|
+
ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */
|
|
571
1256
|
|
|
1257
|
+
ZSTDLIB_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize);
|
|
1258
|
+
ZSTDLIB_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */
|
|
1259
|
+
|
|
1260
|
+
ZSTDLIB_API const ZSTD_CDict* ZSTD_initStaticCDict(
|
|
1261
|
+
void* workspace, size_t workspaceSize,
|
|
1262
|
+
const void* dict, size_t dictSize,
|
|
1263
|
+
ZSTD_dictLoadMethod_e dictLoadMethod,
|
|
1264
|
+
ZSTD_dictContentType_e dictContentType,
|
|
1265
|
+
ZSTD_compressionParameters cParams);
|
|
1266
|
+
|
|
1267
|
+
ZSTDLIB_API const ZSTD_DDict* ZSTD_initStaticDDict(
|
|
1268
|
+
void* workspace, size_t workspaceSize,
|
|
1269
|
+
const void* dict, size_t dictSize,
|
|
1270
|
+
ZSTD_dictLoadMethod_e dictLoadMethod,
|
|
1271
|
+
ZSTD_dictContentType_e dictContentType);
|
|
572
1272
|
|
|
573
|
-
/*! ZSTD_createCDict_byReference() :
|
|
574
|
-
* Create a digested dictionary for compression
|
|
575
|
-
* Dictionary content is simply referenced, and therefore stays in dictBuffer.
|
|
576
|
-
* It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict */
|
|
577
|
-
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
|
|
578
1273
|
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
*
|
|
1274
|
+
/*! Custom memory allocation :
|
|
1275
|
+
* These prototypes make it possible to pass your own allocation/free functions.
|
|
1276
|
+
* ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below.
|
|
1277
|
+
* All allocation/free operations will be completed using these custom variants instead of regular <stdlib.h> ones.
|
|
1278
|
+
*/
|
|
1279
|
+
typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
|
|
1280
|
+
typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
|
|
1281
|
+
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
|
|
1282
|
+
static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */
|
|
1283
|
+
|
|
1284
|
+
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
|
|
1285
|
+
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
|
|
1286
|
+
ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
|
|
1287
|
+
ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
|
|
1288
|
+
|
|
585
1289
|
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
|
|
586
1290
|
ZSTD_dictLoadMethod_e dictLoadMethod,
|
|
587
|
-
|
|
1291
|
+
ZSTD_dictContentType_e dictContentType,
|
|
588
1292
|
ZSTD_compressionParameters cParams,
|
|
589
1293
|
ZSTD_customMem customMem);
|
|
590
1294
|
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
*
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode,
|
|
609
|
-
ZSTD_compressionParameters cParams);
|
|
1295
|
+
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
|
|
1296
|
+
ZSTD_dictLoadMethod_e dictLoadMethod,
|
|
1297
|
+
ZSTD_dictContentType_e dictContentType,
|
|
1298
|
+
ZSTD_customMem customMem);
|
|
1299
|
+
|
|
1300
|
+
|
|
1301
|
+
|
|
1302
|
+
/***************************************
|
|
1303
|
+
* Advanced compression functions
|
|
1304
|
+
***************************************/
|
|
1305
|
+
|
|
1306
|
+
/*! ZSTD_createCDict_byReference() :
|
|
1307
|
+
* Create a digested dictionary for compression
|
|
1308
|
+
* Dictionary content is just referenced, not duplicated.
|
|
1309
|
+
* As a consequence, `dictBuffer` **must** outlive CDict,
|
|
1310
|
+
* and its content must remain unmodified throughout the lifetime of CDict. */
|
|
1311
|
+
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
|
|
610
1312
|
|
|
611
1313
|
/*! ZSTD_getCParams() :
|
|
612
|
-
*
|
|
613
|
-
*
|
|
1314
|
+
* @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
|
|
1315
|
+
* `estimatedSrcSize` value is optional, select 0 if not known */
|
|
614
1316
|
ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
|
|
615
1317
|
|
|
616
1318
|
/*! ZSTD_getParams() :
|
|
617
|
-
*
|
|
618
|
-
*
|
|
1319
|
+
* same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
|
|
1320
|
+
* All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */
|
|
619
1321
|
ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
|
|
620
1322
|
|
|
621
1323
|
/*! ZSTD_checkCParams() :
|
|
622
|
-
*
|
|
1324
|
+
* Ensure param values remain within authorized range.
|
|
1325
|
+
* @return 0 on success, or an error code (can be checked with ZSTD_isError()) */
|
|
623
1326
|
ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
|
|
624
1327
|
|
|
625
1328
|
/*! ZSTD_adjustCParams() :
|
|
626
1329
|
* optimize params for a given `srcSize` and `dictSize`.
|
|
627
|
-
*
|
|
1330
|
+
* `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN.
|
|
1331
|
+
* `dictSize` must be `0` when there is no dictionary.
|
|
1332
|
+
* cPar can be invalid : all parameters will be clamped within valid range in the @return struct.
|
|
1333
|
+
* This function never fails (wide contract) */
|
|
628
1334
|
ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
|
|
629
1335
|
|
|
630
1336
|
/*! ZSTD_compress_advanced() :
|
|
631
|
-
*
|
|
632
|
-
ZSTDLIB_API size_t ZSTD_compress_advanced
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
1337
|
+
* Same as ZSTD_compress_usingDict(), with fine-tune control over compression parameters (by structure) */
|
|
1338
|
+
ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
|
|
1339
|
+
void* dst, size_t dstCapacity,
|
|
1340
|
+
const void* src, size_t srcSize,
|
|
1341
|
+
const void* dict,size_t dictSize,
|
|
1342
|
+
ZSTD_parameters params);
|
|
637
1343
|
|
|
638
1344
|
/*! ZSTD_compress_usingCDict_advanced() :
|
|
639
|
-
*
|
|
1345
|
+
* Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */
|
|
640
1346
|
ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
1347
|
+
void* dst, size_t dstCapacity,
|
|
1348
|
+
const void* src, size_t srcSize,
|
|
1349
|
+
const ZSTD_CDict* cdict,
|
|
1350
|
+
ZSTD_frameParameters fParams);
|
|
1351
|
+
|
|
1352
|
+
|
|
1353
|
+
/*! ZSTD_CCtx_loadDictionary_byReference() :
|
|
1354
|
+
* Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx.
|
|
1355
|
+
* It saves some memory, but also requires that `dict` outlives its usage within `cctx` */
|
|
1356
|
+
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
|
|
1357
|
+
|
|
1358
|
+
/*! ZSTD_CCtx_loadDictionary_advanced() :
|
|
1359
|
+
* Same as ZSTD_CCtx_loadDictionary(), but gives finer control over
|
|
1360
|
+
* how to load the dictionary (by copy ? by reference ?)
|
|
1361
|
+
* and how to interpret it (automatic ? force raw mode ? full mode only ?) */
|
|
1362
|
+
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
|
|
1363
|
+
|
|
1364
|
+
/*! ZSTD_CCtx_refPrefix_advanced() :
|
|
1365
|
+
* Same as ZSTD_CCtx_refPrefix(), but gives finer control over
|
|
1366
|
+
* how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */
|
|
1367
|
+
ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
|
|
1368
|
+
|
|
1369
|
+
/* === experimental parameters === */
|
|
1370
|
+
/* these parameters can be used with ZSTD_setParameter()
|
|
1371
|
+
* they are not guaranteed to remain supported in the future */
|
|
1372
|
+
|
|
1373
|
+
/* Enables rsyncable mode,
|
|
1374
|
+
* which makes compressed files more rsync friendly
|
|
1375
|
+
* by adding periodic synchronization points to the compressed data.
|
|
1376
|
+
* The target average block size is ZSTD_c_jobSize / 2.
|
|
1377
|
+
* It's possible to modify the job size to increase or decrease
|
|
1378
|
+
* the granularity of the synchronization point.
|
|
1379
|
+
* Once the jobSize is smaller than the window size,
|
|
1380
|
+
* it will result in compression ratio degradation.
|
|
1381
|
+
* NOTE 1: rsyncable mode only works when multithreading is enabled.
|
|
1382
|
+
* NOTE 2: rsyncable performs poorly in combination with long range mode,
|
|
1383
|
+
* since it will decrease the effectiveness of synchronization points,
|
|
1384
|
+
* though mileage may vary.
|
|
1385
|
+
* NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s.
|
|
1386
|
+
* If the selected compression level is already running significantly slower,
|
|
1387
|
+
* the overall speed won't be significantly impacted.
|
|
1388
|
+
*/
|
|
1389
|
+
#define ZSTD_c_rsyncable ZSTD_c_experimentalParam1
|
|
1390
|
+
|
|
1391
|
+
/* Select a compression format.
|
|
1392
|
+
* The value must be of type ZSTD_format_e.
|
|
1393
|
+
* See ZSTD_format_e enum definition for details */
|
|
1394
|
+
#define ZSTD_c_format ZSTD_c_experimentalParam2
|
|
1395
|
+
|
|
1396
|
+
/* Force back-reference distances to remain < windowSize,
|
|
1397
|
+
* even when referencing into Dictionary content (default:0) */
|
|
1398
|
+
#define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3
|
|
1399
|
+
|
|
1400
|
+
/* Controls whether the contents of a CDict
|
|
1401
|
+
* are used in place, or copied into the working context.
|
|
1402
|
+
* Accepts values from the ZSTD_dictAttachPref_e enum.
|
|
1403
|
+
* See the comments on that enum for an explanation of the feature. */
|
|
1404
|
+
#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4
|
|
1405
|
+
|
|
1406
|
+
/* Controls how the literals are compressed (default is auto).
|
|
1407
|
+
* The value must be of type ZSTD_literalCompressionMode_e.
|
|
1408
|
+
* See ZSTD_literalCompressionMode_t enum definition for details.
|
|
1409
|
+
*/
|
|
1410
|
+
#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5
|
|
1411
|
+
|
|
1412
|
+
/*! ZSTD_CCtx_getParameter() :
|
|
1413
|
+
* Get the requested compression parameter value, selected by enum ZSTD_cParameter,
|
|
1414
|
+
* and store it into int* value.
|
|
1415
|
+
* @return : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
1416
|
+
*/
|
|
1417
|
+
ZSTDLIB_API size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
|
|
1418
|
+
|
|
644
1419
|
|
|
1420
|
+
/*! ZSTD_CCtx_params :
|
|
1421
|
+
* Quick howto :
|
|
1422
|
+
* - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure
|
|
1423
|
+
* - ZSTD_CCtxParams_setParameter() : Push parameters one by one into
|
|
1424
|
+
* an existing ZSTD_CCtx_params structure.
|
|
1425
|
+
* This is similar to
|
|
1426
|
+
* ZSTD_CCtx_setParameter().
|
|
1427
|
+
* - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to
|
|
1428
|
+
* an existing CCtx.
|
|
1429
|
+
* These parameters will be applied to
|
|
1430
|
+
* all subsequent frames.
|
|
1431
|
+
* - ZSTD_compressStream2() : Do compression using the CCtx.
|
|
1432
|
+
* - ZSTD_freeCCtxParams() : Free the memory.
|
|
1433
|
+
*
|
|
1434
|
+
* This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()
|
|
1435
|
+
* for static allocation of CCtx for single-threaded compression.
|
|
1436
|
+
*/
|
|
1437
|
+
ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
|
|
1438
|
+
ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
|
|
645
1439
|
|
|
646
|
-
|
|
1440
|
+
/*! ZSTD_CCtxParams_reset() :
|
|
1441
|
+
* Reset params to default values.
|
|
1442
|
+
*/
|
|
1443
|
+
ZSTDLIB_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params);
|
|
1444
|
+
|
|
1445
|
+
/*! ZSTD_CCtxParams_init() :
|
|
1446
|
+
* Initializes the compression parameters of cctxParams according to
|
|
1447
|
+
* compression level. All other parameters are reset to their default values.
|
|
1448
|
+
*/
|
|
1449
|
+
ZSTDLIB_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel);
|
|
1450
|
+
|
|
1451
|
+
/*! ZSTD_CCtxParams_init_advanced() :
|
|
1452
|
+
* Initializes the compression and frame parameters of cctxParams according to
|
|
1453
|
+
* params. All other parameters are reset to their default values.
|
|
1454
|
+
*/
|
|
1455
|
+
ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
|
|
1456
|
+
|
|
1457
|
+
/*! ZSTD_CCtxParams_setParameter() :
|
|
1458
|
+
* Similar to ZSTD_CCtx_setParameter.
|
|
1459
|
+
* Set one compression parameter, selected by enum ZSTD_cParameter.
|
|
1460
|
+
* Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams().
|
|
1461
|
+
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
1462
|
+
*/
|
|
1463
|
+
ZSTDLIB_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
|
|
1464
|
+
|
|
1465
|
+
/*! ZSTD_CCtxParams_getParameter() :
|
|
1466
|
+
* Similar to ZSTD_CCtx_getParameter.
|
|
1467
|
+
* Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
|
|
1468
|
+
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
1469
|
+
*/
|
|
1470
|
+
ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
|
|
1471
|
+
|
|
1472
|
+
/*! ZSTD_CCtx_setParametersUsingCCtxParams() :
|
|
1473
|
+
* Apply a set of ZSTD_CCtx_params to the compression context.
|
|
1474
|
+
* This can be done even after compression is started,
|
|
1475
|
+
* if nbWorkers==0, this will have no impact until a new compression is started.
|
|
1476
|
+
* if nbWorkers>=1, new parameters will be picked up at next job,
|
|
1477
|
+
* with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated).
|
|
1478
|
+
*/
|
|
1479
|
+
ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
|
|
1480
|
+
ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);
|
|
1481
|
+
|
|
1482
|
+
/*! ZSTD_compressStream2_simpleArgs() :
|
|
1483
|
+
* Same as ZSTD_compressStream2(),
|
|
1484
|
+
* but using only integral types as arguments.
|
|
1485
|
+
* This variant might be helpful for binders from dynamic languages
|
|
1486
|
+
* which have troubles handling structures containing memory pointers.
|
|
1487
|
+
*/
|
|
1488
|
+
ZSTDLIB_API size_t ZSTD_compressStream2_simpleArgs (
|
|
1489
|
+
ZSTD_CCtx* cctx,
|
|
1490
|
+
void* dst, size_t dstCapacity, size_t* dstPos,
|
|
1491
|
+
const void* src, size_t srcSize, size_t* srcPos,
|
|
1492
|
+
ZSTD_EndDirective endOp);
|
|
1493
|
+
|
|
1494
|
+
|
|
1495
|
+
/***************************************
|
|
1496
|
+
* Advanced decompression functions
|
|
1497
|
+
***************************************/
|
|
647
1498
|
|
|
648
1499
|
/*! ZSTD_isFrame() :
|
|
649
1500
|
* Tells if the content of `buffer` starts with a valid Frame Identifier.
|
|
@@ -652,28 +1503,6 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
|
|
|
652
1503
|
* Note 3 : Skippable Frame Identifiers are considered valid. */
|
|
653
1504
|
ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size);
|
|
654
1505
|
|
|
655
|
-
/*! ZSTD_createDCtx_advanced() :
|
|
656
|
-
* Create a ZSTD decompression context using external alloc and free functions */
|
|
657
|
-
ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
|
|
658
|
-
|
|
659
|
-
/*! ZSTD_initStaticDCtx() : initialize a fixed-size zstd decompression context
|
|
660
|
-
* workspace: The memory area to emplace the context into.
|
|
661
|
-
* Provided pointer must 8-bytes aligned.
|
|
662
|
-
* It must outlive context usage.
|
|
663
|
-
* workspaceSize: Use ZSTD_estimateDCtxSize() or ZSTD_estimateDStreamSize()
|
|
664
|
-
* to determine how large workspace must be to support scenario.
|
|
665
|
-
* @return : pointer to ZSTD_DCtx* (same address as workspace, but different type),
|
|
666
|
-
* or NULL if error (typically size too small)
|
|
667
|
-
* Note : zstd will never resize nor malloc() when using a static dctx.
|
|
668
|
-
* If it needs more memory than available, it will simply error out.
|
|
669
|
-
* Note 2 : static dctx is incompatible with legacy support
|
|
670
|
-
* Note 3 : there is no corresponding "free" function.
|
|
671
|
-
* Since workspace was allocated externally, it must be freed externally.
|
|
672
|
-
* Limitation : currently not compatible with internal DDict creation,
|
|
673
|
-
* such as ZSTD_initDStream_usingDict().
|
|
674
|
-
*/
|
|
675
|
-
ZSTDLIB_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize);
|
|
676
|
-
|
|
677
1506
|
/*! ZSTD_createDDict_byReference() :
|
|
678
1507
|
* Create a digested dictionary, ready to start decompression operation without startup delay.
|
|
679
1508
|
* Dictionary content is referenced, and therefore stays in dictBuffer.
|
|
@@ -681,86 +1510,201 @@ ZSTDLIB_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize
|
|
|
681
1510
|
* it must remain read accessible throughout the lifetime of DDict */
|
|
682
1511
|
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
|
|
683
1512
|
|
|
684
|
-
/*!
|
|
685
|
-
*
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
1513
|
+
/*! ZSTD_DCtx_loadDictionary_byReference() :
|
|
1514
|
+
* Same as ZSTD_DCtx_loadDictionary(),
|
|
1515
|
+
* but references `dict` content instead of copying it into `dctx`.
|
|
1516
|
+
* This saves memory if `dict` remains around.,
|
|
1517
|
+
* However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */
|
|
1518
|
+
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
|
1519
|
+
|
|
1520
|
+
/*! ZSTD_DCtx_loadDictionary_advanced() :
|
|
1521
|
+
* Same as ZSTD_DCtx_loadDictionary(),
|
|
1522
|
+
* but gives direct control over
|
|
1523
|
+
* how to load the dictionary (by copy ? by reference ?)
|
|
1524
|
+
* and how to interpret it (automatic ? force raw mode ? full mode only ?). */
|
|
1525
|
+
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
|
|
1526
|
+
|
|
1527
|
+
/*! ZSTD_DCtx_refPrefix_advanced() :
|
|
1528
|
+
* Same as ZSTD_DCtx_refPrefix(), but gives finer control over
|
|
1529
|
+
* how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */
|
|
1530
|
+
ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
|
|
689
1531
|
|
|
690
|
-
/*!
|
|
691
|
-
*
|
|
692
|
-
*
|
|
693
|
-
*
|
|
694
|
-
*
|
|
695
|
-
*
|
|
696
|
-
* to determine how large workspace must be.
|
|
697
|
-
* @return : pointer to ZSTD_DDict*, or NULL if error (size too small)
|
|
698
|
-
* Note : there is no corresponding "free" function.
|
|
699
|
-
* Since workspace was allocated externally, it must be freed externally.
|
|
1532
|
+
/*! ZSTD_DCtx_setMaxWindowSize() :
|
|
1533
|
+
* Refuses allocating internal buffers for frames requiring a window size larger than provided limit.
|
|
1534
|
+
* This protects a decoder context from reserving too much memory for itself (potential attack scenario).
|
|
1535
|
+
* This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
|
|
1536
|
+
* By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT)
|
|
1537
|
+
* @return : 0, or an error code (which can be tested using ZSTD_isError()).
|
|
700
1538
|
*/
|
|
701
|
-
ZSTDLIB_API
|
|
702
|
-
const void* dict, size_t dictSize,
|
|
703
|
-
ZSTD_dictLoadMethod_e dictLoadMethod);
|
|
1539
|
+
ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
|
|
704
1540
|
|
|
705
|
-
|
|
706
|
-
*
|
|
707
|
-
*
|
|
708
|
-
|
|
709
|
-
|
|
1541
|
+
/* ZSTD_d_format
|
|
1542
|
+
* experimental parameter,
|
|
1543
|
+
* allowing selection between ZSTD_format_e input compression formats
|
|
1544
|
+
*/
|
|
1545
|
+
#define ZSTD_d_format ZSTD_d_experimentalParam1
|
|
710
1546
|
|
|
711
|
-
/*!
|
|
712
|
-
*
|
|
713
|
-
*
|
|
714
|
-
*
|
|
715
|
-
|
|
1547
|
+
/*! ZSTD_DCtx_setFormat() :
|
|
1548
|
+
* Instruct the decoder context about what kind of data to decode next.
|
|
1549
|
+
* This instruction is mandatory to decode data without a fully-formed header,
|
|
1550
|
+
* such ZSTD_f_zstd1_magicless for example.
|
|
1551
|
+
* @return : 0, or an error code (which can be tested using ZSTD_isError()). */
|
|
1552
|
+
ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
|
|
716
1553
|
|
|
717
|
-
/*!
|
|
718
|
-
*
|
|
719
|
-
*
|
|
720
|
-
* This
|
|
721
|
-
*
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
*
|
|
727
|
-
ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
|
|
1554
|
+
/*! ZSTD_decompressStream_simpleArgs() :
|
|
1555
|
+
* Same as ZSTD_decompressStream(),
|
|
1556
|
+
* but using only integral types as arguments.
|
|
1557
|
+
* This can be helpful for binders from dynamic languages
|
|
1558
|
+
* which have troubles handling structures containing memory pointers.
|
|
1559
|
+
*/
|
|
1560
|
+
ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs (
|
|
1561
|
+
ZSTD_DCtx* dctx,
|
|
1562
|
+
void* dst, size_t dstCapacity, size_t* dstPos,
|
|
1563
|
+
const void* src, size_t srcSize, size_t* srcPos);
|
|
728
1564
|
|
|
729
1565
|
|
|
730
1566
|
/********************************************************************
|
|
731
1567
|
* Advanced streaming functions
|
|
1568
|
+
* Warning : most of these functions are now redundant with the Advanced API.
|
|
1569
|
+
* Once Advanced API reaches "stable" status,
|
|
1570
|
+
* redundant functions will be deprecated, and then at some point removed.
|
|
732
1571
|
********************************************************************/
|
|
733
1572
|
|
|
734
1573
|
/*===== Advanced Streaming compression functions =====*/
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
1574
|
+
/**! ZSTD_initCStream_srcSize() :
|
|
1575
|
+
* This function is deprecated, and equivalent to:
|
|
1576
|
+
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
|
1577
|
+
* ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
|
|
1578
|
+
* ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
|
|
1579
|
+
* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
|
|
1580
|
+
*
|
|
1581
|
+
* pledgedSrcSize must be correct. If it is not known at init time, use
|
|
1582
|
+
* ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
|
|
1583
|
+
* "0" also disables frame content size field. It may be enabled in the future.
|
|
1584
|
+
*/
|
|
1585
|
+
ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);
|
|
1586
|
+
/**! ZSTD_initCStream_usingDict() :
|
|
1587
|
+
* This function is deprecated, and is equivalent to:
|
|
1588
|
+
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
|
1589
|
+
* ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
|
|
1590
|
+
* ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
|
|
1591
|
+
*
|
|
1592
|
+
* Creates of an internal CDict (incompatible with static CCtx), except if
|
|
1593
|
+
* dict == NULL or dictSize < 8, in which case no dict is used.
|
|
1594
|
+
* Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if
|
|
1595
|
+
* it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
|
|
1596
|
+
*/
|
|
1597
|
+
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
|
|
1598
|
+
/**! ZSTD_initCStream_advanced() :
|
|
1599
|
+
* This function is deprecated, and is approximately equivalent to:
|
|
1600
|
+
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
|
1601
|
+
* ZSTD_CCtx_setZstdParams(zcs, params); // Set the zstd params and leave the rest as-is
|
|
1602
|
+
* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
|
|
1603
|
+
* ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
|
|
1604
|
+
*
|
|
1605
|
+
* pledgedSrcSize must be correct. If srcSize is not known at init time, use
|
|
1606
|
+
* value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy.
|
|
1607
|
+
*/
|
|
739
1608
|
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
|
|
740
|
-
ZSTD_parameters params, unsigned long long pledgedSrcSize);
|
|
741
|
-
|
|
742
|
-
|
|
1609
|
+
ZSTD_parameters params, unsigned long long pledgedSrcSize);
|
|
1610
|
+
/**! ZSTD_initCStream_usingCDict() :
|
|
1611
|
+
* This function is deprecated, and equivalent to:
|
|
1612
|
+
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
|
1613
|
+
* ZSTD_CCtx_refCDict(zcs, cdict);
|
|
1614
|
+
*
|
|
1615
|
+
* note : cdict will just be referenced, and must outlive compression session
|
|
1616
|
+
*/
|
|
1617
|
+
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
|
|
1618
|
+
/**! ZSTD_initCStream_usingCDict_advanced() :
|
|
1619
|
+
* This function is deprecated, and is approximately equivalent to:
|
|
1620
|
+
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
|
1621
|
+
* ZSTD_CCtx_setZstdFrameParams(zcs, fParams); // Set the zstd frame params and leave the rest as-is
|
|
1622
|
+
* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
|
|
1623
|
+
* ZSTD_CCtx_refCDict(zcs, cdict);
|
|
1624
|
+
*
|
|
1625
|
+
* same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
|
|
1626
|
+
* pledgedSrcSize must be correct. If srcSize is not known at init time, use
|
|
1627
|
+
* value ZSTD_CONTENTSIZE_UNKNOWN.
|
|
1628
|
+
*/
|
|
1629
|
+
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize);
|
|
743
1630
|
|
|
744
1631
|
/*! ZSTD_resetCStream() :
|
|
745
|
-
*
|
|
746
|
-
*
|
|
1632
|
+
* This function is deprecated, and is equivalent to:
|
|
1633
|
+
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
|
1634
|
+
* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
|
|
1635
|
+
*
|
|
1636
|
+
* start a new frame, using same parameters from previous frame.
|
|
1637
|
+
* This is typically useful to skip dictionary loading stage, since it will re-use it in-place.
|
|
747
1638
|
* Note that zcs must be init at least once before using ZSTD_resetCStream().
|
|
748
1639
|
* If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
|
|
749
1640
|
* If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
|
|
750
1641
|
* For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
|
|
751
|
-
* but it
|
|
752
|
-
* @return : 0, or an error code (which can be tested using ZSTD_isError())
|
|
1642
|
+
* but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.
|
|
1643
|
+
* @return : 0, or an error code (which can be tested using ZSTD_isError())
|
|
1644
|
+
*/
|
|
753
1645
|
ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
|
|
754
1646
|
|
|
755
1647
|
|
|
1648
|
+
typedef struct {
|
|
1649
|
+
unsigned long long ingested; /* nb input bytes read and buffered */
|
|
1650
|
+
unsigned long long consumed; /* nb input bytes actually compressed */
|
|
1651
|
+
unsigned long long produced; /* nb of compressed bytes generated and buffered */
|
|
1652
|
+
unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */
|
|
1653
|
+
unsigned currentJobID; /* MT only : latest started job nb */
|
|
1654
|
+
unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */
|
|
1655
|
+
} ZSTD_frameProgression;
|
|
1656
|
+
|
|
1657
|
+
/* ZSTD_getFrameProgression() :
|
|
1658
|
+
* tells how much data has been ingested (read from input)
|
|
1659
|
+
* consumed (input actually compressed) and produced (output) for current frame.
|
|
1660
|
+
* Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed.
|
|
1661
|
+
* Aggregates progression inside active worker threads.
|
|
1662
|
+
*/
|
|
1663
|
+
ZSTDLIB_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx);
|
|
1664
|
+
|
|
1665
|
+
/*! ZSTD_toFlushNow() :
|
|
1666
|
+
* Tell how many bytes are ready to be flushed immediately.
|
|
1667
|
+
* Useful for multithreading scenarios (nbWorkers >= 1).
|
|
1668
|
+
* Probe the oldest active job, defined as oldest job not yet entirely flushed,
|
|
1669
|
+
* and check its output buffer.
|
|
1670
|
+
* @return : amount of data stored in oldest job and ready to be flushed immediately.
|
|
1671
|
+
* if @return == 0, it means either :
|
|
1672
|
+
* + there is no active job (could be checked with ZSTD_frameProgression()), or
|
|
1673
|
+
* + oldest job is still actively compressing data,
|
|
1674
|
+
* but everything it has produced has also been flushed so far,
|
|
1675
|
+
* therefore flush speed is limited by production speed of oldest job
|
|
1676
|
+
* irrespective of the speed of concurrent (and newer) jobs.
|
|
1677
|
+
*/
|
|
1678
|
+
ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
|
|
1679
|
+
|
|
1680
|
+
|
|
756
1681
|
/*===== Advanced Streaming decompression functions =====*/
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
1682
|
+
/**
|
|
1683
|
+
* This function is deprecated, and is equivalent to:
|
|
1684
|
+
*
|
|
1685
|
+
* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
|
|
1686
|
+
* ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
|
|
1687
|
+
*
|
|
1688
|
+
* note: no dictionary will be used if dict == NULL or dictSize < 8
|
|
1689
|
+
*/
|
|
1690
|
+
ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
|
|
1691
|
+
/**
|
|
1692
|
+
* This function is deprecated, and is equivalent to:
|
|
1693
|
+
*
|
|
1694
|
+
* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
|
|
1695
|
+
* ZSTD_DCtx_refDDict(zds, ddict);
|
|
1696
|
+
*
|
|
1697
|
+
* note : ddict is referenced, it must outlive decompression session
|
|
1698
|
+
*/
|
|
1699
|
+
ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
|
|
1700
|
+
/**
|
|
1701
|
+
* This function is deprecated, and is equivalent to:
|
|
1702
|
+
*
|
|
1703
|
+
* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
|
|
1704
|
+
*
|
|
1705
|
+
* re-use decompression parameters from previous init; saves dictionary loading
|
|
1706
|
+
*/
|
|
1707
|
+
ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
|
|
764
1708
|
|
|
765
1709
|
|
|
766
1710
|
/*********************************************************************
|
|
@@ -898,7 +1842,17 @@ typedef struct {
|
|
|
898
1842
|
unsigned dictID;
|
|
899
1843
|
unsigned checksumFlag;
|
|
900
1844
|
} ZSTD_frameHeader;
|
|
1845
|
+
|
|
1846
|
+
/** ZSTD_getFrameHeader() :
|
|
1847
|
+
* decode Frame Header, or requires larger `srcSize`.
|
|
1848
|
+
* @return : 0, `zfhPtr` is correctly filled,
|
|
1849
|
+
* >0, `srcSize` is too small, value is wanted `srcSize` amount,
|
|
1850
|
+
* or an error code, which can be tested using ZSTD_isError() */
|
|
901
1851
|
ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */
|
|
1852
|
+
/*! ZSTD_getFrameHeader_advanced() :
|
|
1853
|
+
* same as ZSTD_getFrameHeader(),
|
|
1854
|
+
* with added capability to select a format (like ZSTD_f_zstd1_magicless) */
|
|
1855
|
+
ZSTDLIB_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
|
|
902
1856
|
ZSTDLIB_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */
|
|
903
1857
|
|
|
904
1858
|
ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
|
|
@@ -915,444 +1869,6 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
|
|
|
915
1869
|
|
|
916
1870
|
|
|
917
1871
|
|
|
918
|
-
/* ============================================ */
|
|
919
|
-
/** New advanced API (experimental) */
|
|
920
|
-
/* ============================================ */
|
|
921
|
-
|
|
922
|
-
/* notes on API design :
|
|
923
|
-
* In this proposal, parameters are pushed one by one into an existing context,
|
|
924
|
-
* and then applied on all subsequent compression jobs.
|
|
925
|
-
* When no parameter is ever provided, CCtx is created with compression level ZSTD_CLEVEL_DEFAULT.
|
|
926
|
-
*
|
|
927
|
-
* This API is intended to replace all others experimental API.
|
|
928
|
-
* It can basically do all other use cases, and even new ones.
|
|
929
|
-
* In constrast with _advanced() variants, it stands a reasonable chance to become "stable",
|
|
930
|
-
* after a good testing period.
|
|
931
|
-
*/
|
|
932
|
-
|
|
933
|
-
/* note on naming convention :
|
|
934
|
-
* Initially, the API favored names like ZSTD_setCCtxParameter() .
|
|
935
|
-
* In this proposal, convention is changed towards ZSTD_CCtx_setParameter() .
|
|
936
|
-
* The main driver is that it identifies more clearly the target object type.
|
|
937
|
-
* It feels clearer when considering multiple targets :
|
|
938
|
-
* ZSTD_CDict_setParameter() (rather than ZSTD_setCDictParameter())
|
|
939
|
-
* ZSTD_CCtxParams_setParameter() (rather than ZSTD_setCCtxParamsParameter() )
|
|
940
|
-
* etc...
|
|
941
|
-
*/
|
|
942
|
-
|
|
943
|
-
/* note on enum design :
|
|
944
|
-
* All enum will be pinned to explicit values before reaching "stable API" status */
|
|
945
|
-
|
|
946
|
-
typedef enum {
|
|
947
|
-
/* Question : should we have a format ZSTD_f_auto ?
|
|
948
|
-
* For the time being, it would mean exactly the same as ZSTD_f_zstd1.
|
|
949
|
-
* But, in the future, should several formats be supported,
|
|
950
|
-
* on the compression side, it would mean "default format".
|
|
951
|
-
* On the decompression side, it would mean "multi format",
|
|
952
|
-
* and ZSTD_f_zstd1 could be reserved to mean "accept *only* zstd frames".
|
|
953
|
-
* Since meaning is a little different, another option could be to define different enums for compression and decompression.
|
|
954
|
-
* This question could be kept for later, when there are actually multiple formats to support,
|
|
955
|
-
* but there is also the question of pinning enum values, and pinning value `0` is especially important */
|
|
956
|
-
ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */
|
|
957
|
-
ZSTD_f_zstd1_magicless, /* Variant of zstd frame format, without initial 4-bytes magic number.
|
|
958
|
-
* Useful to save 4 bytes per generated frame.
|
|
959
|
-
* Decoder cannot recognise automatically this format, requiring instructions. */
|
|
960
|
-
} ZSTD_format_e;
|
|
961
|
-
|
|
962
|
-
typedef enum {
|
|
963
|
-
/* compression format */
|
|
964
|
-
ZSTD_p_format = 10, /* See ZSTD_format_e enum definition.
|
|
965
|
-
* Cast selected format as unsigned for ZSTD_CCtx_setParameter() compatibility. */
|
|
966
|
-
|
|
967
|
-
/* compression parameters */
|
|
968
|
-
ZSTD_p_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
|
|
969
|
-
* Default level is ZSTD_CLEVEL_DEFAULT==3.
|
|
970
|
-
* Special: value 0 means "do not change cLevel". */
|
|
971
|
-
ZSTD_p_windowLog, /* Maximum allowed back-reference distance, expressed as power of 2.
|
|
972
|
-
* Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
|
|
973
|
-
* Special: value 0 means "do not change windowLog".
|
|
974
|
-
* Note: Using a window size greater than ZSTD_MAXWINDOWSIZE_DEFAULT (default: 2^27)
|
|
975
|
-
* requires setting the maximum window size at least as large during decompression. */
|
|
976
|
-
ZSTD_p_hashLog, /* Size of the probe table, as a power of 2.
|
|
977
|
-
* Resulting table size is (1 << (hashLog+2)).
|
|
978
|
-
* Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
|
|
979
|
-
* Larger tables improve compression ratio of strategies <= dFast,
|
|
980
|
-
* and improve speed of strategies > dFast.
|
|
981
|
-
* Special: value 0 means "do not change hashLog". */
|
|
982
|
-
ZSTD_p_chainLog, /* Size of the full-search table, as a power of 2.
|
|
983
|
-
* Resulting table size is (1 << (chainLog+2)).
|
|
984
|
-
* Larger tables result in better and slower compression.
|
|
985
|
-
* This parameter is useless when using "fast" strategy.
|
|
986
|
-
* Special: value 0 means "do not change chainLog". */
|
|
987
|
-
ZSTD_p_searchLog, /* Number of search attempts, as a power of 2.
|
|
988
|
-
* More attempts result in better and slower compression.
|
|
989
|
-
* This parameter is useless when using "fast" and "dFast" strategies.
|
|
990
|
-
* Special: value 0 means "do not change searchLog". */
|
|
991
|
-
ZSTD_p_minMatch, /* Minimum size of searched matches (note : repCode matches can be smaller).
|
|
992
|
-
* Larger values make faster compression and decompression, but decrease ratio.
|
|
993
|
-
* Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX.
|
|
994
|
-
* Note that currently, for all strategies < btopt, effective minimum is 4.
|
|
995
|
-
* Note that currently, for all strategies > fast, effective maximum is 6.
|
|
996
|
-
* Special: value 0 means "do not change minMatchLength". */
|
|
997
|
-
ZSTD_p_targetLength, /* Only useful for strategies >= btopt.
|
|
998
|
-
* Length of Match considered "good enough" to stop search.
|
|
999
|
-
* Larger values make compression stronger and slower.
|
|
1000
|
-
* Special: value 0 means "do not change targetLength". */
|
|
1001
|
-
ZSTD_p_compressionStrategy, /* See ZSTD_strategy enum definition.
|
|
1002
|
-
* Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility.
|
|
1003
|
-
* The higher the value of selected strategy, the more complex it is,
|
|
1004
|
-
* resulting in stronger and slower compression.
|
|
1005
|
-
* Special: value 0 means "do not change strategy". */
|
|
1006
|
-
|
|
1007
|
-
/* frame parameters */
|
|
1008
|
-
ZSTD_p_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
|
|
1009
|
-
* Content size must be known at the beginning of compression,
|
|
1010
|
-
* it is provided using ZSTD_CCtx_setPledgedSrcSize() */
|
|
1011
|
-
ZSTD_p_checksumFlag, /* A 32-bits checksum of content is written at end of frame (default:0) */
|
|
1012
|
-
ZSTD_p_dictIDFlag, /* When applicable, dictionary's ID is written into frame header (default:1) */
|
|
1013
|
-
|
|
1014
|
-
/* multi-threading parameters */
|
|
1015
|
-
ZSTD_p_nbThreads=400, /* Select how many threads a compression job can spawn (default:1)
|
|
1016
|
-
* More threads improve speed, but also increase memory usage.
|
|
1017
|
-
* Can only receive a value > 1 if ZSTD_MULTITHREAD is enabled.
|
|
1018
|
-
* Special: value 0 means "do not change nbThreads" */
|
|
1019
|
-
ZSTD_p_jobSize, /* Size of a compression job. This value is only enforced in streaming (non-blocking) mode.
|
|
1020
|
-
* Each compression job is completed in parallel, so indirectly controls the nb of active threads.
|
|
1021
|
-
* 0 means default, which is dynamically determined based on compression parameters.
|
|
1022
|
-
* Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
|
|
1023
|
-
* The minimum size is automatically and transparently enforced */
|
|
1024
|
-
ZSTD_p_overlapSizeLog, /* Size of previous input reloaded at the beginning of each job.
|
|
1025
|
-
* 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
|
|
1026
|
-
|
|
1027
|
-
/* advanced parameters - may not remain available after API update */
|
|
1028
|
-
ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize,
|
|
1029
|
-
* even when referencing into Dictionary content (default:0) */
|
|
1030
|
-
ZSTD_p_enableLongDistanceMatching=1200, /* Enable long distance matching.
|
|
1031
|
-
* This parameter is designed to improve the compression
|
|
1032
|
-
* ratio for large inputs with long distance matches.
|
|
1033
|
-
* This increases the memory usage as well as window size.
|
|
1034
|
-
* Note: setting this parameter sets all the LDM parameters
|
|
1035
|
-
* as well as ZSTD_p_windowLog. It should be set after
|
|
1036
|
-
* ZSTD_p_compressionLevel and before ZSTD_p_windowLog and
|
|
1037
|
-
* other LDM parameters. Setting the compression level
|
|
1038
|
-
* after this parameter overrides the window log, though LDM
|
|
1039
|
-
* will remain enabled until explicitly disabled. */
|
|
1040
|
-
ZSTD_p_ldmHashLog, /* Size of the table for long distance matching, as a power of 2.
|
|
1041
|
-
* Larger values increase memory usage and compression ratio, but decrease
|
|
1042
|
-
* compression speed.
|
|
1043
|
-
* Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
|
|
1044
|
-
* (default: windowlog - 7). */
|
|
1045
|
-
ZSTD_p_ldmMinMatch, /* Minimum size of searched matches for long distance matcher.
|
|
1046
|
-
* Larger/too small values usually decrease compression ratio.
|
|
1047
|
-
* Must be clamped between ZSTD_LDM_MINMATCH_MIN
|
|
1048
|
-
* and ZSTD_LDM_MINMATCH_MAX (default: 64). */
|
|
1049
|
-
ZSTD_p_ldmBucketSizeLog, /* Log size of each bucket in the LDM hash table for collision resolution.
|
|
1050
|
-
* Larger values usually improve collision resolution but may decrease
|
|
1051
|
-
* compression speed.
|
|
1052
|
-
* The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX (default: 3). */
|
|
1053
|
-
ZSTD_p_ldmHashEveryLog, /* Frequency of inserting/looking up entries in the LDM hash table.
|
|
1054
|
-
* The default is MAX(0, (windowLog - ldmHashLog)) to
|
|
1055
|
-
* optimize hash table usage.
|
|
1056
|
-
* Larger values improve compression speed. Deviating far from the
|
|
1057
|
-
* default value will likely result in a decrease in compression ratio.
|
|
1058
|
-
* Must be clamped between 0 and ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN. */
|
|
1059
|
-
|
|
1060
|
-
} ZSTD_cParameter;
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
/*! ZSTD_CCtx_setParameter() :
|
|
1064
|
-
* Set one compression parameter, selected by enum ZSTD_cParameter.
|
|
1065
|
-
* Note : when `value` is an enum, cast it to unsigned for proper type checking.
|
|
1066
|
-
* @result : informational value (typically, the one being set, possibly corrected),
|
|
1067
|
-
* or an error code (which can be tested with ZSTD_isError()). */
|
|
1068
|
-
ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
|
|
1069
|
-
|
|
1070
|
-
/*! ZSTD_CCtx_setPledgedSrcSize() :
|
|
1071
|
-
* Total input data size to be compressed as a single frame.
|
|
1072
|
-
* This value will be controlled at the end, and result in error if not respected.
|
|
1073
|
-
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
1074
|
-
* Note 1 : 0 means zero, empty.
|
|
1075
|
-
* In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
|
|
1076
|
-
* ZSTD_CONTENTSIZE_UNKNOWN is default value for any new compression job.
|
|
1077
|
-
* Note 2 : If all data is provided and consumed in a single round,
|
|
1078
|
-
* this value is overriden by srcSize instead. */
|
|
1079
|
-
ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
|
|
1080
|
-
|
|
1081
|
-
/*! ZSTD_CCtx_loadDictionary() :
|
|
1082
|
-
* Create an internal CDict from dict buffer.
|
|
1083
|
-
* Decompression will have to use same buffer.
|
|
1084
|
-
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
1085
|
-
* Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
|
|
1086
|
-
* meaning "return to no-dictionary mode".
|
|
1087
|
-
* Note 1 : `dict` content will be copied internally. Use
|
|
1088
|
-
* ZSTD_CCtx_loadDictionary_byReference() to reference dictionary
|
|
1089
|
-
* content instead. The dictionary buffer must then outlive its
|
|
1090
|
-
* users.
|
|
1091
|
-
* Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
|
|
1092
|
-
* For this reason, compression parameters cannot be changed anymore after loading a dictionary.
|
|
1093
|
-
* It's also a CPU-heavy operation, with non-negligible impact on latency.
|
|
1094
|
-
* Note 3 : Dictionary will be used for all future compression jobs.
|
|
1095
|
-
* To return to "no-dictionary" situation, load a NULL dictionary
|
|
1096
|
-
* Note 5 : Use ZSTD_CCtx_loadDictionary_advanced() to select how dictionary
|
|
1097
|
-
* content will be interpreted.
|
|
1098
|
-
*/
|
|
1099
|
-
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
|
|
1100
|
-
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
|
|
1101
|
-
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode);
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
/*! ZSTD_CCtx_refCDict() :
|
|
1105
|
-
* Reference a prepared dictionary, to be used for all next compression jobs.
|
|
1106
|
-
* Note that compression parameters are enforced from within CDict,
|
|
1107
|
-
* and supercede any compression parameter previously set within CCtx.
|
|
1108
|
-
* The dictionary will remain valid for future compression jobs using same CCtx.
|
|
1109
|
-
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
1110
|
-
* Special : adding a NULL CDict means "return to no-dictionary mode".
|
|
1111
|
-
* Note 1 : Currently, only one dictionary can be managed.
|
|
1112
|
-
* Adding a new dictionary effectively "discards" any previous one.
|
|
1113
|
-
* Note 2 : CDict is just referenced, its lifetime must outlive CCtx.
|
|
1114
|
-
*/
|
|
1115
|
-
ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
|
|
1116
|
-
|
|
1117
|
-
/*! ZSTD_CCtx_refPrefix() :
|
|
1118
|
-
* Reference a prefix (single-usage dictionary) for next compression job.
|
|
1119
|
-
* Decompression need same prefix to properly regenerate data.
|
|
1120
|
-
* Prefix is **only used once**. Tables are discarded at end of compression job.
|
|
1121
|
-
* Subsequent compression jobs will be done without prefix (if none is explicitly referenced).
|
|
1122
|
-
* If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict instead.
|
|
1123
|
-
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
1124
|
-
* Special : Adding any prefix (including NULL) invalidates any previous prefix or dictionary
|
|
1125
|
-
* Note 1 : Prefix buffer is referenced. It must outlive compression job.
|
|
1126
|
-
* Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters.
|
|
1127
|
-
* It's a CPU-heavy operation, with non-negligible impact on latency.
|
|
1128
|
-
* Note 3 : By default, the prefix is treated as raw content
|
|
1129
|
-
* (ZSTD_dm_rawContent). Use ZSTD_CCtx_refPrefix_advanced() to alter
|
|
1130
|
-
* dictMode. */
|
|
1131
|
-
ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);
|
|
1132
|
-
ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictMode_e dictMode);
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
typedef enum {
|
|
1137
|
-
ZSTD_e_continue=0, /* collect more data, encoder transparently decides when to output result, for optimal conditions */
|
|
1138
|
-
ZSTD_e_flush, /* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */
|
|
1139
|
-
ZSTD_e_end /* flush any remaining data and close current frame. Any additional data starts a new frame. */
|
|
1140
|
-
} ZSTD_EndDirective;
|
|
1141
|
-
|
|
1142
|
-
/*! ZSTD_compress_generic() :
|
|
1143
|
-
* Behave about the same as ZSTD_compressStream. To note :
|
|
1144
|
-
* - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_setParameter()
|
|
1145
|
-
* - Compression parameters cannot be changed once compression is started.
|
|
1146
|
-
* - outpot->pos must be <= dstCapacity, input->pos must be <= srcSize
|
|
1147
|
-
* - outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
|
|
1148
|
-
* - In single-thread mode (default), function is blocking : it completed its job before returning to caller.
|
|
1149
|
-
* - In multi-thread mode, function is non-blocking : it just acquires a copy of input, and distribute job to internal worker threads,
|
|
1150
|
-
* and then immediately returns, just indicating that there is some data remaining to be flushed.
|
|
1151
|
-
* The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
|
|
1152
|
-
* - Exception : in multi-threading mode, if the first call requests a ZSTD_e_end directive, it is blocking : it will complete compression before giving back control to caller.
|
|
1153
|
-
* - @return provides the minimum amount of data remaining to be flushed from internal buffers
|
|
1154
|
-
* or an error code, which can be tested using ZSTD_isError().
|
|
1155
|
-
* if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
|
|
1156
|
-
* This is useful to determine if a ZSTD_e_flush or ZSTD_e_end directive is completed.
|
|
1157
|
-
* - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
|
|
1158
|
-
* only ZSTD_e_end or ZSTD_e_flush operations are allowed.
|
|
1159
|
-
* Before starting a new compression job, or changing compression parameters,
|
|
1160
|
-
* it is required to fully flush internal buffers.
|
|
1161
|
-
*/
|
|
1162
|
-
ZSTDLIB_API size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
|
|
1163
|
-
ZSTD_outBuffer* output,
|
|
1164
|
-
ZSTD_inBuffer* input,
|
|
1165
|
-
ZSTD_EndDirective endOp);
|
|
1166
|
-
|
|
1167
|
-
/*! ZSTD_CCtx_reset() :
|
|
1168
|
-
* Return a CCtx to clean state.
|
|
1169
|
-
* Useful after an error, or to interrupt an ongoing compression job and start a new one.
|
|
1170
|
-
* Any internal data not yet flushed is cancelled.
|
|
1171
|
-
* Dictionary (if any) is dropped.
|
|
1172
|
-
* All parameters are back to default values.
|
|
1173
|
-
* It's possible to modify compression parameters after a reset.
|
|
1174
|
-
*/
|
|
1175
|
-
ZSTDLIB_API void ZSTD_CCtx_reset(ZSTD_CCtx* cctx); /* Not ready yet ! */
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
/*! ZSTD_compress_generic_simpleArgs() :
|
|
1179
|
-
* Same as ZSTD_compress_generic(),
|
|
1180
|
-
* but using only integral types as arguments.
|
|
1181
|
-
* Argument list is larger than ZSTD_{in,out}Buffer,
|
|
1182
|
-
* but can be helpful for binders from dynamic languages
|
|
1183
|
-
* which have troubles handling structures containing memory pointers.
|
|
1184
|
-
*/
|
|
1185
|
-
ZSTDLIB_API size_t ZSTD_compress_generic_simpleArgs (
|
|
1186
|
-
ZSTD_CCtx* cctx,
|
|
1187
|
-
void* dst, size_t dstCapacity, size_t* dstPos,
|
|
1188
|
-
const void* src, size_t srcSize, size_t* srcPos,
|
|
1189
|
-
ZSTD_EndDirective endOp);
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
/*! ZSTD_CCtx_params :
|
|
1193
|
-
* Quick howto :
|
|
1194
|
-
* - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure
|
|
1195
|
-
* - ZSTD_CCtxParam_setParameter() : Push parameters one by one into
|
|
1196
|
-
* an existing ZSTD_CCtx_params structure.
|
|
1197
|
-
* This is similar to
|
|
1198
|
-
* ZSTD_CCtx_setParameter().
|
|
1199
|
-
* - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to
|
|
1200
|
-
* an existing CCtx.
|
|
1201
|
-
* These parameters will be applied to
|
|
1202
|
-
* all subsequent compression jobs.
|
|
1203
|
-
* - ZSTD_compress_generic() : Do compression using the CCtx.
|
|
1204
|
-
* - ZSTD_freeCCtxParams() : Free the memory.
|
|
1205
|
-
*
|
|
1206
|
-
* This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()
|
|
1207
|
-
* for static allocation for single-threaded compression.
|
|
1208
|
-
*/
|
|
1209
|
-
ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
|
|
1210
|
-
|
|
1211
|
-
/*! ZSTD_resetCCtxParams() :
|
|
1212
|
-
* Reset params to default, with the default compression level.
|
|
1213
|
-
*/
|
|
1214
|
-
ZSTDLIB_API size_t ZSTD_resetCCtxParams(ZSTD_CCtx_params* params);
|
|
1215
|
-
|
|
1216
|
-
/*! ZSTD_initCCtxParams() :
|
|
1217
|
-
* Initializes the compression parameters of cctxParams according to
|
|
1218
|
-
* compression level. All other parameters are reset to their default values.
|
|
1219
|
-
*/
|
|
1220
|
-
ZSTDLIB_API size_t ZSTD_initCCtxParams(ZSTD_CCtx_params* cctxParams, int compressionLevel);
|
|
1221
|
-
|
|
1222
|
-
/*! ZSTD_initCCtxParams_advanced() :
|
|
1223
|
-
* Initializes the compression and frame parameters of cctxParams according to
|
|
1224
|
-
* params. All other parameters are reset to their default values.
|
|
1225
|
-
*/
|
|
1226
|
-
ZSTDLIB_API size_t ZSTD_initCCtxParams_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
|
|
1227
|
-
|
|
1228
|
-
ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
|
|
1229
|
-
|
|
1230
|
-
/*! ZSTD_CCtxParam_setParameter() :
|
|
1231
|
-
* Similar to ZSTD_CCtx_setParameter.
|
|
1232
|
-
* Set one compression parameter, selected by enum ZSTD_cParameter.
|
|
1233
|
-
* Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams().
|
|
1234
|
-
* Note : when `value` is an enum, cast it to unsigned for proper type checking.
|
|
1235
|
-
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
1236
|
-
*/
|
|
1237
|
-
ZSTDLIB_API size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned value);
|
|
1238
|
-
|
|
1239
|
-
/*! ZSTD_CCtx_setParametersUsingCCtxParams() :
|
|
1240
|
-
* Apply a set of ZSTD_CCtx_params to the compression context.
|
|
1241
|
-
* This must be done before the dictionary is loaded.
|
|
1242
|
-
* The pledgedSrcSize is treated as unknown.
|
|
1243
|
-
* Multithreading parameters are applied only if nbThreads > 1.
|
|
1244
|
-
*/
|
|
1245
|
-
ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
|
|
1246
|
-
ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
/*=== Advanced parameters for decompression API ===*/
|
|
1250
|
-
|
|
1251
|
-
/* The following parameters must be set after creating a ZSTD_DCtx* (or ZSTD_DStream*) object,
|
|
1252
|
-
* but before starting decompression of a frame.
|
|
1253
|
-
*/
|
|
1254
|
-
|
|
1255
|
-
/*! ZSTD_DCtx_loadDictionary() :
|
|
1256
|
-
* Create an internal DDict from dict buffer,
|
|
1257
|
-
* to be used to decompress next frames.
|
|
1258
|
-
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
1259
|
-
* Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
|
|
1260
|
-
* meaning "return to no-dictionary mode".
|
|
1261
|
-
* Note 1 : `dict` content will be copied internally.
|
|
1262
|
-
* Use ZSTD_DCtx_loadDictionary_byReference()
|
|
1263
|
-
* to reference dictionary content instead.
|
|
1264
|
-
* In which case, the dictionary buffer must outlive its users.
|
|
1265
|
-
* Note 2 : Loading a dictionary involves building tables,
|
|
1266
|
-
* which has a non-negligible impact on CPU usage and latency.
|
|
1267
|
-
* Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to select
|
|
1268
|
-
* how dictionary content will be interpreted and loaded.
|
|
1269
|
-
*/
|
|
1270
|
-
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); /* not implemented */
|
|
1271
|
-
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); /* not implemented */
|
|
1272
|
-
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode); /* not implemented */
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
/*! ZSTD_DCtx_refDDict() :
|
|
1276
|
-
* Reference a prepared dictionary, to be used to decompress next frames.
|
|
1277
|
-
* The dictionary remains active for decompression of future frames using same DCtx.
|
|
1278
|
-
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
1279
|
-
* Note 1 : Currently, only one dictionary can be managed.
|
|
1280
|
-
* Referencing a new dictionary effectively "discards" any previous one.
|
|
1281
|
-
* Special : adding a NULL DDict means "return to no-dictionary mode".
|
|
1282
|
-
* Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
|
|
1283
|
-
*/
|
|
1284
|
-
ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); /* not implemented */
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
/*! ZSTD_DCtx_refPrefix() :
|
|
1288
|
-
* Reference a prefix (single-usage dictionary) for next compression job.
|
|
1289
|
-
* Prefix is **only used once**. It must be explicitly referenced before each frame.
|
|
1290
|
-
* If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_DDict instead.
|
|
1291
|
-
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
1292
|
-
* Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary
|
|
1293
|
-
* Note 2 : Prefix buffer is referenced. It must outlive compression job.
|
|
1294
|
-
* Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
|
|
1295
|
-
* Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode.
|
|
1296
|
-
* Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
|
|
1297
|
-
*/
|
|
1298
|
-
ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize); /* not implemented */
|
|
1299
|
-
ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictMode_e dictMode); /* not implemented */
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
/*! ZSTD_DCtx_setMaxWindowSize() :
|
|
1303
|
-
* Refuses allocating internal buffers for frames requiring a window size larger than provided limit.
|
|
1304
|
-
* This is useful to prevent a decoder context from reserving too much memory for itself (potential attack scenario).
|
|
1305
|
-
* This parameter is only useful in streaming mode, since no internal buffer is allocated in direct mode.
|
|
1306
|
-
* By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_MAX)
|
|
1307
|
-
* @return : 0, or an error code (which can be tested using ZSTD_isError()).
|
|
1308
|
-
*/
|
|
1309
|
-
ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
/*! ZSTD_DCtx_setFormat() :
|
|
1313
|
-
* Instruct the decoder context about what kind of data to decode next.
|
|
1314
|
-
* This instruction is mandatory to decode data without a fully-formed header,
|
|
1315
|
-
* such ZSTD_f_zstd1_magicless for example.
|
|
1316
|
-
* @return : 0, or an error code (which can be tested using ZSTD_isError()).
|
|
1317
|
-
*/
|
|
1318
|
-
ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
/*! ZSTD_decompress_generic() :
|
|
1322
|
-
* Behave the same as ZSTD_decompressStream.
|
|
1323
|
-
* Decompression parameters cannot be changed once decompression is started.
|
|
1324
|
-
* @return : an error code, which can be tested using ZSTD_isError()
|
|
1325
|
-
* if >0, a hint, nb of expected input bytes for next invocation.
|
|
1326
|
-
* `0` means : a frame has just been fully decoded and flushed.
|
|
1327
|
-
*/
|
|
1328
|
-
ZSTDLIB_API size_t ZSTD_decompress_generic(ZSTD_DCtx* dctx,
|
|
1329
|
-
ZSTD_outBuffer* output,
|
|
1330
|
-
ZSTD_inBuffer* input);
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
/*! ZSTD_decompress_generic_simpleArgs() :
|
|
1334
|
-
* Same as ZSTD_decompress_generic(),
|
|
1335
|
-
* but using only integral types as arguments.
|
|
1336
|
-
* Argument list is larger than ZSTD_{in,out}Buffer,
|
|
1337
|
-
* but can be helpful for binders from dynamic languages
|
|
1338
|
-
* which have troubles handling structures containing memory pointers.
|
|
1339
|
-
*/
|
|
1340
|
-
ZSTDLIB_API size_t ZSTD_decompress_generic_simpleArgs (
|
|
1341
|
-
ZSTD_DCtx* dctx,
|
|
1342
|
-
void* dst, size_t dstCapacity, size_t* dstPos,
|
|
1343
|
-
const void* src, size_t srcSize, size_t* srcPos);
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
/*! ZSTD_DCtx_reset() :
|
|
1347
|
-
* Return a DCtx to clean state.
|
|
1348
|
-
* If a decompression was ongoing, any internal data not yet flushed is cancelled.
|
|
1349
|
-
* All parameters are back to default values, including sticky ones.
|
|
1350
|
-
* Dictionary (if any) is dropped.
|
|
1351
|
-
* Parameters can be modified again after a reset.
|
|
1352
|
-
*/
|
|
1353
|
-
ZSTDLIB_API void ZSTD_DCtx_reset(ZSTD_DCtx* dctx);
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
1872
|
|
|
1357
1873
|
/* ============================ */
|
|
1358
1874
|
/** Block level API */
|
|
@@ -1372,10 +1888,10 @@ ZSTDLIB_API void ZSTD_DCtx_reset(ZSTD_DCtx* dctx);
|
|
|
1372
1888
|
+ copyCCtx() and copyDCtx() can be used too
|
|
1373
1889
|
- Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
|
|
1374
1890
|
+ If input is larger than a block size, it's necessary to split input data into multiple blocks
|
|
1375
|
-
+ For inputs larger than a single block
|
|
1891
|
+
+ For inputs larger than a single block, really consider using regular ZSTD_compress() instead.
|
|
1376
1892
|
Frame metadata is not that costly, and quickly becomes negligible as source size grows larger.
|
|
1377
1893
|
- When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero.
|
|
1378
|
-
In which case, nothing is produced into `dst
|
|
1894
|
+
In which case, nothing is produced into `dst` !
|
|
1379
1895
|
+ User must test for such outcome and deal directly with uncompressed data
|
|
1380
1896
|
+ ZSTD_decompressBlock() doesn't accept uncompressed data as input !!!
|
|
1381
1897
|
+ In case of multiple successive blocks, should some of them be uncompressed,
|
|
@@ -1383,13 +1899,11 @@ ZSTDLIB_API void ZSTD_DCtx_reset(ZSTD_DCtx* dctx);
|
|
|
1383
1899
|
Use ZSTD_insertBlock() for such a case.
|
|
1384
1900
|
*/
|
|
1385
1901
|
|
|
1386
|
-
#define ZSTD_BLOCKSIZELOG_MAX 17
|
|
1387
|
-
#define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX) /* define, for static allocation */
|
|
1388
1902
|
/*===== Raw zstd block functions =====*/
|
|
1389
1903
|
ZSTDLIB_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx);
|
|
1390
1904
|
ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
|
1391
1905
|
ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
|
1392
|
-
ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression */
|
|
1906
|
+
ZSTDLIB_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */
|
|
1393
1907
|
|
|
1394
1908
|
|
|
1395
1909
|
#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */
|