zstd-ruby 1.4.0.0 → 1.4.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +35 -0
- data/README.md +2 -2
- data/ext/zstdruby/libzstd/Makefile +274 -107
- data/ext/zstdruby/libzstd/README.md +75 -16
- data/ext/zstdruby/libzstd/common/bitstream.h +59 -51
- data/ext/zstdruby/libzstd/common/compiler.h +154 -5
- data/ext/zstdruby/libzstd/common/cpu.h +1 -3
- data/ext/zstdruby/libzstd/common/debug.c +11 -31
- data/ext/zstdruby/libzstd/common/debug.h +22 -49
- data/ext/zstdruby/libzstd/common/entropy_common.c +201 -75
- data/ext/zstdruby/libzstd/common/error_private.c +3 -1
- data/ext/zstdruby/libzstd/common/error_private.h +7 -3
- data/ext/zstdruby/libzstd/common/fse.h +50 -42
- data/ext/zstdruby/libzstd/common/fse_decompress.c +134 -50
- data/ext/zstdruby/libzstd/common/huf.h +41 -38
- data/ext/zstdruby/libzstd/common/mem.h +68 -22
- data/ext/zstdruby/libzstd/common/pool.c +30 -20
- data/ext/zstdruby/libzstd/common/pool.h +3 -3
- data/ext/zstdruby/libzstd/common/threading.c +51 -4
- data/ext/zstdruby/libzstd/common/threading.h +36 -4
- data/ext/zstdruby/libzstd/common/xxhash.c +39 -89
- data/ext/zstdruby/libzstd/common/xxhash.h +12 -32
- data/ext/zstdruby/libzstd/common/zstd_common.c +10 -10
- data/ext/zstdruby/libzstd/common/zstd_deps.h +111 -0
- data/ext/zstdruby/libzstd/common/zstd_errors.h +3 -1
- data/ext/zstdruby/libzstd/common/zstd_internal.h +231 -72
- data/ext/zstdruby/libzstd/common/zstd_trace.c +42 -0
- data/ext/zstdruby/libzstd/common/zstd_trace.h +152 -0
- data/ext/zstdruby/libzstd/compress/fse_compress.c +47 -63
- data/ext/zstdruby/libzstd/compress/hist.c +41 -63
- data/ext/zstdruby/libzstd/compress/hist.h +13 -33
- data/ext/zstdruby/libzstd/compress/huf_compress.c +288 -172
- data/ext/zstdruby/libzstd/compress/zstd_compress.c +2504 -1626
- data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +446 -85
- data/ext/zstdruby/libzstd/compress/zstd_compress_literals.c +158 -0
- data/ext/zstdruby/libzstd/compress/zstd_compress_literals.h +29 -0
- data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.c +433 -0
- data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.h +54 -0
- data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.c +849 -0
- data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.h +32 -0
- data/ext/zstdruby/libzstd/compress/zstd_cwksp.h +561 -0
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +82 -60
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +2 -2
- data/ext/zstdruby/libzstd/compress/zstd_fast.c +106 -80
- data/ext/zstdruby/libzstd/compress/zstd_fast.h +2 -2
- data/ext/zstdruby/libzstd/compress/zstd_lazy.c +411 -105
- data/ext/zstdruby/libzstd/compress/zstd_lazy.h +21 -1
- data/ext/zstdruby/libzstd/compress/zstd_ldm.c +296 -207
- data/ext/zstdruby/libzstd/compress/zstd_ldm.h +14 -3
- data/ext/zstdruby/libzstd/compress/zstd_ldm_geartab.h +103 -0
- data/ext/zstdruby/libzstd/compress/zstd_opt.c +260 -148
- data/ext/zstdruby/libzstd/compress/zstd_opt.h +1 -1
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +153 -440
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +29 -110
- data/ext/zstdruby/libzstd/decompress/huf_decompress.c +356 -238
- data/ext/zstdruby/libzstd/decompress/zstd_ddict.c +20 -16
- data/ext/zstdruby/libzstd/decompress/zstd_ddict.h +3 -3
- data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +641 -238
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.c +600 -371
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.h +8 -5
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_internal.h +40 -9
- data/ext/zstdruby/libzstd/deprecated/zbuff.h +9 -8
- data/ext/zstdruby/libzstd/deprecated/zbuff_common.c +2 -2
- data/ext/zstdruby/libzstd/deprecated/zbuff_compress.c +1 -1
- data/ext/zstdruby/libzstd/deprecated/zbuff_decompress.c +1 -1
- data/ext/zstdruby/libzstd/dictBuilder/cover.c +197 -78
- data/ext/zstdruby/libzstd/dictBuilder/cover.h +52 -7
- data/ext/zstdruby/libzstd/dictBuilder/divsufsort.c +1 -1
- data/ext/zstdruby/libzstd/dictBuilder/fastcover.c +84 -66
- data/ext/zstdruby/libzstd/dictBuilder/zdict.c +58 -36
- data/ext/zstdruby/libzstd/dictBuilder/zdict.h +60 -31
- data/ext/zstdruby/libzstd/dll/example/Makefile +2 -1
- data/ext/zstdruby/libzstd/dll/example/README.md +16 -22
- data/ext/zstdruby/libzstd/legacy/zstd_legacy.h +8 -4
- data/ext/zstdruby/libzstd/legacy/zstd_v01.c +115 -111
- data/ext/zstdruby/libzstd/legacy/zstd_v01.h +1 -1
- data/ext/zstdruby/libzstd/legacy/zstd_v02.c +28 -14
- data/ext/zstdruby/libzstd/legacy/zstd_v02.h +1 -1
- data/ext/zstdruby/libzstd/legacy/zstd_v03.c +28 -14
- data/ext/zstdruby/libzstd/legacy/zstd_v03.h +1 -1
- data/ext/zstdruby/libzstd/legacy/zstd_v04.c +36 -19
- data/ext/zstdruby/libzstd/legacy/zstd_v04.h +1 -1
- data/ext/zstdruby/libzstd/legacy/zstd_v05.c +122 -107
- data/ext/zstdruby/libzstd/legacy/zstd_v05.h +2 -2
- data/ext/zstdruby/libzstd/legacy/zstd_v06.c +29 -23
- data/ext/zstdruby/libzstd/legacy/zstd_v06.h +1 -1
- data/ext/zstdruby/libzstd/legacy/zstd_v07.c +34 -24
- data/ext/zstdruby/libzstd/legacy/zstd_v07.h +1 -1
- data/ext/zstdruby/libzstd/libzstd.pc.in +2 -1
- data/ext/zstdruby/libzstd/zstd.h +655 -118
- data/lib/zstd-ruby/version.rb +1 -1
- data/zstd-ruby.gemspec +1 -1
- metadata +20 -10
- data/.travis.yml +0 -14
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/*
|
|
2
|
-
* Copyright (c) 2016-
|
|
2
|
+
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
|
@@ -19,7 +19,7 @@ extern "C" {
|
|
|
19
19
|
* Dependencies
|
|
20
20
|
***************************************/
|
|
21
21
|
#include <stddef.h> /* size_t */
|
|
22
|
-
#include "mem.h" /* U64, U32 */
|
|
22
|
+
#include "../common/mem.h" /* U64, U32 */
|
|
23
23
|
|
|
24
24
|
|
|
25
25
|
/* *************************************
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/*
|
|
2
|
-
* Copyright (c) 2016-
|
|
2
|
+
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
#include <stddef.h> /* size_t, ptrdiff_t */
|
|
15
15
|
#include <string.h> /* memcpy */
|
|
16
16
|
#include <stdlib.h> /* malloc, free, qsort */
|
|
17
|
-
#include "error_private.h"
|
|
17
|
+
#include "../common/error_private.h"
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
|
|
@@ -82,7 +82,11 @@ extern "C" {
|
|
|
82
82
|
* Basic Types
|
|
83
83
|
*****************************************************************/
|
|
84
84
|
#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
|
|
85
|
-
#
|
|
85
|
+
# if defined(_AIX)
|
|
86
|
+
# include <inttypes.h>
|
|
87
|
+
# else
|
|
88
|
+
# include <stdint.h> /* intptr_t */
|
|
89
|
+
# endif
|
|
86
90
|
typedef uint8_t BYTE;
|
|
87
91
|
typedef uint16_t U16;
|
|
88
92
|
typedef int16_t S16;
|
|
@@ -860,7 +864,7 @@ MEM_STATIC unsigned BITv06_highbit32 ( U32 val)
|
|
|
860
864
|
_BitScanReverse ( &r, val );
|
|
861
865
|
return (unsigned) r;
|
|
862
866
|
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
|
|
863
|
-
return
|
|
867
|
+
return __builtin_clz (val) ^ 31;
|
|
864
868
|
# else /* Software version */
|
|
865
869
|
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
|
866
870
|
U32 v = val;
|
|
@@ -1862,7 +1866,7 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta
|
|
|
1862
1866
|
|
|
1863
1867
|
if (!srcSize) return ERROR(srcSize_wrong);
|
|
1864
1868
|
iSize = ip[0];
|
|
1865
|
-
|
|
1869
|
+
/* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */
|
|
1866
1870
|
|
|
1867
1871
|
if (iSize >= 128) { /* special header */
|
|
1868
1872
|
if (iSize >= (242)) { /* RLE */
|
|
@@ -2014,7 +2018,7 @@ size_t HUFv06_readDTableX2 (U16* DTable, const void* src, size_t srcSize)
|
|
|
2014
2018
|
HUFv06_DEltX2* const dt = (HUFv06_DEltX2*)dtPtr;
|
|
2015
2019
|
|
|
2016
2020
|
HUFv06_STATIC_ASSERT(sizeof(HUFv06_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */
|
|
2017
|
-
|
|
2021
|
+
/* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
|
|
2018
2022
|
|
|
2019
2023
|
iSize = HUFv06_readStats(huffWeight, HUFv06_MAX_SYMBOL_VALUE + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
|
|
2020
2024
|
if (HUFv06_isError(iSize)) return iSize;
|
|
@@ -2340,7 +2344,7 @@ size_t HUFv06_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
|
|
|
2340
2344
|
|
|
2341
2345
|
HUFv06_STATIC_ASSERT(sizeof(HUFv06_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */
|
|
2342
2346
|
if (memLog > HUFv06_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
|
2343
|
-
|
|
2347
|
+
/* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
|
|
2344
2348
|
|
|
2345
2349
|
iSize = HUFv06_readStats(weightList, HUFv06_MAX_SYMBOL_VALUE + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
|
|
2346
2350
|
if (HUFv06_isError(iSize)) return iSize;
|
|
@@ -2664,13 +2668,13 @@ size_t HUFv06_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
|
|
|
2664
2668
|
|
|
2665
2669
|
{ U32 algoNb = 0;
|
|
2666
2670
|
if (Dtime[1] < Dtime[0]) algoNb = 1;
|
|
2667
|
-
|
|
2671
|
+
/* if (Dtime[2] < Dtime[algoNb]) algoNb = 2; */ /* current speed of HUFv06_decompress4X6 is not good */
|
|
2668
2672
|
return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
|
|
2669
2673
|
}
|
|
2670
2674
|
|
|
2671
|
-
|
|
2672
|
-
|
|
2673
|
-
|
|
2675
|
+
/* return HUFv06_decompress4X2(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams single-symbol decoding */
|
|
2676
|
+
/* return HUFv06_decompress4X4(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams double-symbols decoding */
|
|
2677
|
+
/* return HUFv06_decompress4X6(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams quad-symbols decoding */
|
|
2674
2678
|
}
|
|
2675
2679
|
/*
|
|
2676
2680
|
Common functions of Zstd compression library
|
|
@@ -3025,7 +3029,7 @@ typedef struct
|
|
|
3025
3029
|
* Provides the size of compressed block from block header `src` */
|
|
3026
3030
|
static size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
|
3027
3031
|
{
|
|
3028
|
-
const BYTE* const in = (const BYTE*
|
|
3032
|
+
const BYTE* const in = (const BYTE*)src;
|
|
3029
3033
|
U32 cSize;
|
|
3030
3034
|
|
|
3031
3035
|
if (srcSize < ZSTDv06_blockHeaderSize) return ERROR(srcSize_wrong);
|
|
@@ -3219,7 +3223,7 @@ static size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
|
|
|
3219
3223
|
FSEv06_DTable* DTableLL, FSEv06_DTable* DTableML, FSEv06_DTable* DTableOffb, U32 flagRepeatTable,
|
|
3220
3224
|
const void* src, size_t srcSize)
|
|
3221
3225
|
{
|
|
3222
|
-
const BYTE* const istart = (const BYTE*
|
|
3226
|
+
const BYTE* const istart = (const BYTE*)src;
|
|
3223
3227
|
const BYTE* const iend = istart + srcSize;
|
|
3224
3228
|
const BYTE* ip = istart;
|
|
3225
3229
|
|
|
@@ -3242,14 +3246,12 @@ static size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
|
|
|
3242
3246
|
}
|
|
3243
3247
|
|
|
3244
3248
|
/* FSE table descriptors */
|
|
3249
|
+
if (ip + 4 > iend) return ERROR(srcSize_wrong); /* min : header byte + all 3 are "raw", hence no header, but at least xxLog bits per type */
|
|
3245
3250
|
{ U32 const LLtype = *ip >> 6;
|
|
3246
3251
|
U32 const Offtype = (*ip >> 4) & 3;
|
|
3247
3252
|
U32 const MLtype = (*ip >> 2) & 3;
|
|
3248
3253
|
ip++;
|
|
3249
3254
|
|
|
3250
|
-
/* check */
|
|
3251
|
-
if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
|
|
3252
|
-
|
|
3253
3255
|
/* Build DTables */
|
|
3254
3256
|
{ size_t const bhSize = ZSTDv06_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
|
|
3255
3257
|
if (ZSTDv06_isError(bhSize)) return ERROR(corruption_detected);
|
|
@@ -3443,7 +3445,7 @@ static size_t ZSTDv06_decompressSequences(
|
|
|
3443
3445
|
{
|
|
3444
3446
|
const BYTE* ip = (const BYTE*)seqStart;
|
|
3445
3447
|
const BYTE* const iend = ip + seqSize;
|
|
3446
|
-
BYTE* const ostart = (BYTE*
|
|
3448
|
+
BYTE* const ostart = (BYTE*)dst;
|
|
3447
3449
|
BYTE* const oend = ostart + maxDstSize;
|
|
3448
3450
|
BYTE* op = ostart;
|
|
3449
3451
|
const BYTE* litPtr = dctx->litPtr;
|
|
@@ -3503,8 +3505,10 @@ static size_t ZSTDv06_decompressSequences(
|
|
|
3503
3505
|
{ size_t const lastLLSize = litEnd - litPtr;
|
|
3504
3506
|
if (litPtr > litEnd) return ERROR(corruption_detected); /* too many literals already used */
|
|
3505
3507
|
if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
|
|
3506
|
-
|
|
3507
|
-
|
|
3508
|
+
if (lastLLSize > 0) {
|
|
3509
|
+
memcpy(op, litPtr, lastLLSize);
|
|
3510
|
+
op += lastLLSize;
|
|
3511
|
+
}
|
|
3508
3512
|
}
|
|
3509
3513
|
|
|
3510
3514
|
return op-ostart;
|
|
@@ -3557,7 +3561,7 @@ static size_t ZSTDv06_decompressFrame(ZSTDv06_DCtx* dctx,
|
|
|
3557
3561
|
{
|
|
3558
3562
|
const BYTE* ip = (const BYTE*)src;
|
|
3559
3563
|
const BYTE* const iend = ip + srcSize;
|
|
3560
|
-
BYTE* const ostart = (BYTE*
|
|
3564
|
+
BYTE* const ostart = (BYTE*)dst;
|
|
3561
3565
|
BYTE* op = ostart;
|
|
3562
3566
|
BYTE* const oend = ostart + dstCapacity;
|
|
3563
3567
|
size_t remainingSize = srcSize;
|
|
@@ -3672,7 +3676,7 @@ void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cS
|
|
|
3672
3676
|
blockProperties_t blockProperties = { bt_compressed, 0 };
|
|
3673
3677
|
|
|
3674
3678
|
/* Frame Header */
|
|
3675
|
-
{ size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src,
|
|
3679
|
+
{ size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src, srcSize);
|
|
3676
3680
|
if (ZSTDv06_isError(frameHeaderSize)) {
|
|
3677
3681
|
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, frameHeaderSize);
|
|
3678
3682
|
return;
|
|
@@ -4002,7 +4006,9 @@ size_t ZBUFFv06_decompressInit(ZBUFFv06_DCtx* zbd)
|
|
|
4002
4006
|
MEM_STATIC size_t ZBUFFv06_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
|
4003
4007
|
{
|
|
4004
4008
|
size_t length = MIN(dstCapacity, srcSize);
|
|
4005
|
-
|
|
4009
|
+
if (length > 0) {
|
|
4010
|
+
memcpy(dst, src, length);
|
|
4011
|
+
}
|
|
4006
4012
|
return length;
|
|
4007
4013
|
}
|
|
4008
4014
|
|
|
@@ -4111,7 +4117,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
|
|
|
4111
4117
|
if (!decodedSize) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */
|
|
4112
4118
|
zbd->outEnd = zbd->outStart + decodedSize;
|
|
4113
4119
|
zbd->stage = ZBUFFds_flush;
|
|
4114
|
-
|
|
4120
|
+
/* break; */ /* ZBUFFds_flush follows */
|
|
4115
4121
|
}
|
|
4116
4122
|
}
|
|
4117
4123
|
/* fall-through */
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/*
|
|
2
|
-
* Copyright (c) 2016-
|
|
2
|
+
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
|
@@ -17,14 +17,14 @@
|
|
|
17
17
|
#ifndef XXH_STATIC_LINKING_ONLY
|
|
18
18
|
# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
|
|
19
19
|
#endif
|
|
20
|
-
#include "xxhash.h" /* XXH64_* */
|
|
20
|
+
#include "../common/xxhash.h" /* XXH64_* */
|
|
21
21
|
#include "zstd_v07.h"
|
|
22
22
|
|
|
23
23
|
#define FSEv07_STATIC_LINKING_ONLY /* FSEv07_MIN_TABLELOG */
|
|
24
24
|
#define HUFv07_STATIC_LINKING_ONLY /* HUFv07_TABLELOG_ABSOLUTEMAX */
|
|
25
25
|
#define ZSTDv07_STATIC_LINKING_ONLY
|
|
26
26
|
|
|
27
|
-
#include "error_private.h"
|
|
27
|
+
#include "../common/error_private.h"
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
#ifdef ZSTDv07_STATIC_LINKING_ONLY
|
|
@@ -242,7 +242,11 @@ extern "C" {
|
|
|
242
242
|
* Basic Types
|
|
243
243
|
*****************************************************************/
|
|
244
244
|
#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
|
|
245
|
-
#
|
|
245
|
+
# if defined(_AIX)
|
|
246
|
+
# include <inttypes.h>
|
|
247
|
+
# else
|
|
248
|
+
# include <stdint.h> /* intptr_t */
|
|
249
|
+
# endif
|
|
246
250
|
typedef uint8_t BYTE;
|
|
247
251
|
typedef uint16_t U16;
|
|
248
252
|
typedef int16_t S16;
|
|
@@ -530,7 +534,7 @@ MEM_STATIC unsigned BITv07_highbit32 (U32 val)
|
|
|
530
534
|
_BitScanReverse ( &r, val );
|
|
531
535
|
return (unsigned) r;
|
|
532
536
|
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
|
|
533
|
-
return
|
|
537
|
+
return __builtin_clz (val) ^ 31;
|
|
534
538
|
# else /* Software version */
|
|
535
539
|
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
|
536
540
|
U32 v = val;
|
|
@@ -1314,7 +1318,7 @@ size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
|
|
1314
1318
|
|
|
1315
1319
|
if (!srcSize) return ERROR(srcSize_wrong);
|
|
1316
1320
|
iSize = ip[0];
|
|
1317
|
-
|
|
1321
|
+
/* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */
|
|
1318
1322
|
|
|
1319
1323
|
if (iSize >= 128) { /* special header */
|
|
1320
1324
|
if (iSize >= (242)) { /* RLE */
|
|
@@ -1784,7 +1788,7 @@ size_t HUFv07_readDTableX2 (HUFv07_DTable* DTable, const void* src, size_t srcSi
|
|
|
1784
1788
|
HUFv07_DEltX2* const dt = (HUFv07_DEltX2*)dtPtr;
|
|
1785
1789
|
|
|
1786
1790
|
HUFv07_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUFv07_DTable));
|
|
1787
|
-
|
|
1791
|
+
/* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
|
|
1788
1792
|
|
|
1789
1793
|
iSize = HUFv07_readStats(huffWeight, HUFv07_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
|
|
1790
1794
|
if (HUFv07_isError(iSize)) return iSize;
|
|
@@ -2148,7 +2152,7 @@ size_t HUFv07_readDTableX4 (HUFv07_DTable* DTable, const void* src, size_t srcSi
|
|
|
2148
2152
|
|
|
2149
2153
|
HUFv07_STATIC_ASSERT(sizeof(HUFv07_DEltX4) == sizeof(HUFv07_DTable)); /* if compilation fails here, assertion is false */
|
|
2150
2154
|
if (maxTableLog > HUFv07_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge);
|
|
2151
|
-
|
|
2155
|
+
/* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
|
|
2152
2156
|
|
|
2153
2157
|
iSize = HUFv07_readStats(weightList, HUFv07_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
|
|
2154
2158
|
if (HUFv07_isError(iSize)) return iSize;
|
|
@@ -2530,8 +2534,8 @@ size_t HUFv07_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
|
|
|
2530
2534
|
return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
|
|
2531
2535
|
}
|
|
2532
2536
|
|
|
2533
|
-
|
|
2534
|
-
|
|
2537
|
+
/* return HUFv07_decompress4X2(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams single-symbol decoding */
|
|
2538
|
+
/* return HUFv07_decompress4X4(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams double-symbols decoding */
|
|
2535
2539
|
}
|
|
2536
2540
|
|
|
2537
2541
|
size_t HUFv07_decompress4X_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
|
@@ -3254,7 +3258,7 @@ typedef struct
|
|
|
3254
3258
|
* Provides the size of compressed block from block header `src` */
|
|
3255
3259
|
static size_t ZSTDv07_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
|
3256
3260
|
{
|
|
3257
|
-
const BYTE* const in = (const BYTE*
|
|
3261
|
+
const BYTE* const in = (const BYTE*)src;
|
|
3258
3262
|
U32 cSize;
|
|
3259
3263
|
|
|
3260
3264
|
if (srcSize < ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong);
|
|
@@ -3272,7 +3276,9 @@ static size_t ZSTDv07_getcBlockSize(const void* src, size_t srcSize, blockProper
|
|
|
3272
3276
|
static size_t ZSTDv07_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
|
3273
3277
|
{
|
|
3274
3278
|
if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
|
|
3275
|
-
|
|
3279
|
+
if (srcSize > 0) {
|
|
3280
|
+
memcpy(dst, src, srcSize);
|
|
3281
|
+
}
|
|
3276
3282
|
return srcSize;
|
|
3277
3283
|
}
|
|
3278
3284
|
|
|
@@ -3447,7 +3453,7 @@ static size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr,
|
|
|
3447
3453
|
FSEv07_DTable* DTableLL, FSEv07_DTable* DTableML, FSEv07_DTable* DTableOffb, U32 flagRepeatTable,
|
|
3448
3454
|
const void* src, size_t srcSize)
|
|
3449
3455
|
{
|
|
3450
|
-
const BYTE* const istart = (const BYTE*
|
|
3456
|
+
const BYTE* const istart = (const BYTE*)src;
|
|
3451
3457
|
const BYTE* const iend = istart + srcSize;
|
|
3452
3458
|
const BYTE* ip = istart;
|
|
3453
3459
|
|
|
@@ -3470,14 +3476,12 @@ static size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr,
|
|
|
3470
3476
|
}
|
|
3471
3477
|
|
|
3472
3478
|
/* FSE table descriptors */
|
|
3479
|
+
if (ip + 4 > iend) return ERROR(srcSize_wrong); /* min : header byte + all 3 are "raw", hence no header, but at least xxLog bits per type */
|
|
3473
3480
|
{ U32 const LLtype = *ip >> 6;
|
|
3474
3481
|
U32 const OFtype = (*ip >> 4) & 3;
|
|
3475
3482
|
U32 const MLtype = (*ip >> 2) & 3;
|
|
3476
3483
|
ip++;
|
|
3477
3484
|
|
|
3478
|
-
/* check */
|
|
3479
|
-
if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
|
|
3480
|
-
|
|
3481
3485
|
/* Build DTables */
|
|
3482
3486
|
{ size_t const llhSize = ZSTDv07_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
|
|
3483
3487
|
if (ZSTDv07_isError(llhSize)) return ERROR(corruption_detected);
|
|
@@ -3668,7 +3672,7 @@ static size_t ZSTDv07_decompressSequences(
|
|
|
3668
3672
|
{
|
|
3669
3673
|
const BYTE* ip = (const BYTE*)seqStart;
|
|
3670
3674
|
const BYTE* const iend = ip + seqSize;
|
|
3671
|
-
BYTE* const ostart = (BYTE*
|
|
3675
|
+
BYTE* const ostart = (BYTE*)dst;
|
|
3672
3676
|
BYTE* const oend = ostart + maxDstSize;
|
|
3673
3677
|
BYTE* op = ostart;
|
|
3674
3678
|
const BYTE* litPtr = dctx->litPtr;
|
|
@@ -3714,10 +3718,12 @@ static size_t ZSTDv07_decompressSequences(
|
|
|
3714
3718
|
|
|
3715
3719
|
/* last literal segment */
|
|
3716
3720
|
{ size_t const lastLLSize = litEnd - litPtr;
|
|
3717
|
-
|
|
3721
|
+
/* if (litPtr > litEnd) return ERROR(corruption_detected); */ /* too many literals already used */
|
|
3718
3722
|
if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall);
|
|
3719
|
-
|
|
3720
|
-
|
|
3723
|
+
if (lastLLSize > 0) {
|
|
3724
|
+
memcpy(op, litPtr, lastLLSize);
|
|
3725
|
+
op += lastLLSize;
|
|
3726
|
+
}
|
|
3721
3727
|
}
|
|
3722
3728
|
|
|
3723
3729
|
return op-ostart;
|
|
@@ -3778,7 +3784,9 @@ ZSTDLIBv07_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockS
|
|
|
3778
3784
|
static size_t ZSTDv07_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length)
|
|
3779
3785
|
{
|
|
3780
3786
|
if (length > dstCapacity) return ERROR(dstSize_tooSmall);
|
|
3781
|
-
|
|
3787
|
+
if (length > 0) {
|
|
3788
|
+
memset(dst, byte, length);
|
|
3789
|
+
}
|
|
3782
3790
|
return length;
|
|
3783
3791
|
}
|
|
3784
3792
|
|
|
@@ -3791,7 +3799,7 @@ static size_t ZSTDv07_decompressFrame(ZSTDv07_DCtx* dctx,
|
|
|
3791
3799
|
{
|
|
3792
3800
|
const BYTE* ip = (const BYTE*)src;
|
|
3793
3801
|
const BYTE* const iend = ip + srcSize;
|
|
3794
|
-
BYTE* const ostart = (BYTE*
|
|
3802
|
+
BYTE* const ostart = (BYTE*)dst;
|
|
3795
3803
|
BYTE* const oend = ostart + dstCapacity;
|
|
3796
3804
|
BYTE* op = ostart;
|
|
3797
3805
|
size_t remainingSize = srcSize;
|
|
@@ -3918,7 +3926,7 @@ void ZSTDv07_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cS
|
|
|
3918
3926
|
}
|
|
3919
3927
|
|
|
3920
3928
|
/* Frame Header */
|
|
3921
|
-
{ size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src,
|
|
3929
|
+
{ size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src, srcSize);
|
|
3922
3930
|
if (ZSTDv07_isError(frameHeaderSize)) {
|
|
3923
3931
|
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, frameHeaderSize);
|
|
3924
3932
|
return;
|
|
@@ -4380,7 +4388,9 @@ size_t ZBUFFv07_decompressInit(ZBUFFv07_DCtx* zbd)
|
|
|
4380
4388
|
MEM_STATIC size_t ZBUFFv07_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
|
4381
4389
|
{
|
|
4382
4390
|
size_t const length = MIN(dstCapacity, srcSize);
|
|
4383
|
-
|
|
4391
|
+
if (length > 0) {
|
|
4392
|
+
memcpy(dst, src, length);
|
|
4393
|
+
}
|
|
4384
4394
|
return length;
|
|
4385
4395
|
}
|
|
4386
4396
|
|
|
@@ -3,8 +3,9 @@
|
|
|
3
3
|
# BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
|
4
4
|
|
|
5
5
|
prefix=@PREFIX@
|
|
6
|
-
|
|
6
|
+
exec_prefix=@EXEC_PREFIX@
|
|
7
7
|
includedir=@INCLUDEDIR@
|
|
8
|
+
libdir=@LIBDIR@
|
|
8
9
|
|
|
9
10
|
Name: zstd
|
|
10
11
|
Description: fast lossless compression algorithm library
|
data/ext/zstdruby/libzstd/zstd.h
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/*
|
|
2
|
-
* Copyright (c) 2016-
|
|
2
|
+
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
|
@@ -15,6 +15,7 @@ extern "C" {
|
|
|
15
15
|
#define ZSTD_H_235446
|
|
16
16
|
|
|
17
17
|
/* ====== Dependency ======*/
|
|
18
|
+
#include <limits.h> /* INT_MAX */
|
|
18
19
|
#include <stddef.h> /* size_t */
|
|
19
20
|
|
|
20
21
|
|
|
@@ -71,27 +72,32 @@ extern "C" {
|
|
|
71
72
|
/*------ Version ------*/
|
|
72
73
|
#define ZSTD_VERSION_MAJOR 1
|
|
73
74
|
#define ZSTD_VERSION_MINOR 4
|
|
74
|
-
#define ZSTD_VERSION_RELEASE
|
|
75
|
-
|
|
75
|
+
#define ZSTD_VERSION_RELEASE 9
|
|
76
76
|
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
|
|
77
|
-
|
|
77
|
+
|
|
78
|
+
/*! ZSTD_versionNumber() :
|
|
79
|
+
* Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). */
|
|
80
|
+
ZSTDLIB_API unsigned ZSTD_versionNumber(void);
|
|
78
81
|
|
|
79
82
|
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
|
|
80
83
|
#define ZSTD_QUOTE(str) #str
|
|
81
84
|
#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str)
|
|
82
85
|
#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
|
|
83
|
-
ZSTDLIB_API const char* ZSTD_versionString(void); /* requires v1.3.0+ */
|
|
84
86
|
|
|
85
|
-
|
|
86
|
-
*
|
|
87
|
-
|
|
87
|
+
/*! ZSTD_versionString() :
|
|
88
|
+
* Return runtime library version, like "1.4.5". Requires v1.3.0+. */
|
|
89
|
+
ZSTDLIB_API const char* ZSTD_versionString(void);
|
|
90
|
+
|
|
91
|
+
/* *************************************
|
|
92
|
+
* Default constant
|
|
93
|
+
***************************************/
|
|
88
94
|
#ifndef ZSTD_CLEVEL_DEFAULT
|
|
89
95
|
# define ZSTD_CLEVEL_DEFAULT 3
|
|
90
96
|
#endif
|
|
91
97
|
|
|
92
|
-
|
|
93
|
-
* Constants
|
|
94
|
-
***************************************/
|
|
98
|
+
/* *************************************
|
|
99
|
+
* Constants
|
|
100
|
+
***************************************/
|
|
95
101
|
|
|
96
102
|
/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */
|
|
97
103
|
#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */
|
|
@@ -183,17 +189,26 @@ ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compres
|
|
|
183
189
|
***************************************/
|
|
184
190
|
/*= Compression context
|
|
185
191
|
* When compressing many times,
|
|
186
|
-
* it is recommended to allocate a context just once,
|
|
192
|
+
* it is recommended to allocate a context just once,
|
|
193
|
+
* and re-use it for each successive compression operation.
|
|
187
194
|
* This will make workload friendlier for system's memory.
|
|
188
|
-
*
|
|
195
|
+
* Note : re-using context is just a speed / resource optimization.
|
|
196
|
+
* It doesn't change the compression ratio, which remains identical.
|
|
197
|
+
* Note 2 : In multi-threaded environments,
|
|
198
|
+
* use one different context per thread for parallel execution.
|
|
199
|
+
*/
|
|
189
200
|
typedef struct ZSTD_CCtx_s ZSTD_CCtx;
|
|
190
201
|
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
|
|
191
202
|
ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
|
|
192
203
|
|
|
193
204
|
/*! ZSTD_compressCCtx() :
|
|
194
|
-
* Same as ZSTD_compress(), using an explicit ZSTD_CCtx
|
|
195
|
-
*
|
|
196
|
-
*
|
|
205
|
+
* Same as ZSTD_compress(), using an explicit ZSTD_CCtx.
|
|
206
|
+
* Important : in order to behave similarly to `ZSTD_compress()`,
|
|
207
|
+
* this function compresses at requested compression level,
|
|
208
|
+
* __ignoring any other parameter__ .
|
|
209
|
+
* If any advanced parameter was set using the advanced API,
|
|
210
|
+
* they will all be reset. Only `compressionLevel` remains.
|
|
211
|
+
*/
|
|
197
212
|
ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
|
|
198
213
|
void* dst, size_t dstCapacity,
|
|
199
214
|
const void* src, size_t srcSize,
|
|
@@ -228,7 +243,7 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx,
|
|
|
228
243
|
* using ZSTD_CCtx_set*() functions.
|
|
229
244
|
* Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame.
|
|
230
245
|
* "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` !
|
|
231
|
-
*
|
|
246
|
+
* __They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()__ .
|
|
232
247
|
*
|
|
233
248
|
* It's possible to reset all parameters to "default" using ZSTD_CCtx_reset().
|
|
234
249
|
*
|
|
@@ -256,18 +271,29 @@ typedef enum {
|
|
|
256
271
|
|
|
257
272
|
/* compression parameters
|
|
258
273
|
* Note: When compressing with a ZSTD_CDict these parameters are superseded
|
|
259
|
-
* by the parameters used to construct the ZSTD_CDict.
|
|
260
|
-
* for more info (superseded-by-cdict). */
|
|
261
|
-
ZSTD_c_compressionLevel=100, /*
|
|
274
|
+
* by the parameters used to construct the ZSTD_CDict.
|
|
275
|
+
* See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */
|
|
276
|
+
ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table.
|
|
277
|
+
* Note that exact compression parameters are dynamically determined,
|
|
278
|
+
* depending on both compression level and srcSize (when known).
|
|
262
279
|
* Default level is ZSTD_CLEVEL_DEFAULT==3.
|
|
263
280
|
* Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
|
|
264
281
|
* Note 1 : it's possible to pass a negative compression level.
|
|
265
|
-
* Note 2 : setting a level
|
|
282
|
+
* Note 2 : setting a level does not automatically set all other compression parameters
|
|
283
|
+
* to default. Setting this will however eventually dynamically impact the compression
|
|
284
|
+
* parameters which have not been manually set. The manually set
|
|
285
|
+
* ones will 'stick'. */
|
|
286
|
+
/* Advanced compression parameters :
|
|
287
|
+
* It's possible to pin down compression parameters to some specific values.
|
|
288
|
+
* In which case, these values are no longer dynamically selected by the compressor */
|
|
266
289
|
ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2.
|
|
290
|
+
* This will set a memory budget for streaming decompression,
|
|
291
|
+
* with larger values requiring more memory
|
|
292
|
+
* and typically compressing more.
|
|
267
293
|
* Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
|
|
268
294
|
* Special: value 0 means "use default windowLog".
|
|
269
295
|
* Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT
|
|
270
|
-
* requires explicitly allowing such
|
|
296
|
+
* requires explicitly allowing such size at streaming decompression stage. */
|
|
271
297
|
ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2.
|
|
272
298
|
* Resulting memory usage is (1 << (hashLog+2)).
|
|
273
299
|
* Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
|
|
@@ -278,13 +304,13 @@ typedef enum {
|
|
|
278
304
|
* Resulting memory usage is (1 << (chainLog+2)).
|
|
279
305
|
* Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
|
|
280
306
|
* Larger tables result in better and slower compression.
|
|
281
|
-
* This parameter is useless
|
|
307
|
+
* This parameter is useless for "fast" strategy.
|
|
282
308
|
* It's still useful when using "dfast" strategy,
|
|
283
309
|
* in which case it defines a secondary probe table.
|
|
284
310
|
* Special: value 0 means "use default chainLog". */
|
|
285
311
|
ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2.
|
|
286
312
|
* More attempts result in better and slower compression.
|
|
287
|
-
* This parameter is useless
|
|
313
|
+
* This parameter is useless for "fast" and "dFast" strategies.
|
|
288
314
|
* Special: value 0 means "use default searchLog". */
|
|
289
315
|
ZSTD_c_minMatch=105, /* Minimum size of searched matches.
|
|
290
316
|
* Note that Zstandard can still find matches of smaller size,
|
|
@@ -313,7 +339,9 @@ typedef enum {
|
|
|
313
339
|
* for large inputs, by finding large matches at long distance.
|
|
314
340
|
* It increases memory usage and window size.
|
|
315
341
|
* Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB
|
|
316
|
-
* except when expressly set to a different value.
|
|
342
|
+
* except when expressly set to a different value.
|
|
343
|
+
* Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and
|
|
344
|
+
* compression strategy >= ZSTD_btopt (== compression level 16+) */
|
|
317
345
|
ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2.
|
|
318
346
|
* Larger values increase memory usage and compression ratio,
|
|
319
347
|
* but decrease compression speed.
|
|
@@ -339,26 +367,30 @@ typedef enum {
|
|
|
339
367
|
ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
|
|
340
368
|
* Content size must be known at the beginning of compression.
|
|
341
369
|
* This is automatically the case when using ZSTD_compress2(),
|
|
342
|
-
* For streaming
|
|
370
|
+
* For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
|
|
343
371
|
ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */
|
|
344
372
|
ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */
|
|
345
373
|
|
|
346
374
|
/* multi-threading parameters */
|
|
347
|
-
/* These parameters are only
|
|
348
|
-
*
|
|
375
|
+
/* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
|
|
376
|
+
* Otherwise, trying to set any other value than default (0) will be a no-op and return an error.
|
|
377
|
+
* In a situation where it's unknown if the linked library supports multi-threading or not,
|
|
378
|
+
* setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property.
|
|
379
|
+
*/
|
|
349
380
|
ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel.
|
|
350
|
-
* When nbWorkers >= 1, triggers asynchronous mode when
|
|
381
|
+
* When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() :
|
|
351
382
|
* ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
|
|
352
|
-
* while compression
|
|
383
|
+
* while compression is performed in parallel, within worker thread(s).
|
|
353
384
|
* (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
|
|
354
385
|
* in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
|
|
355
386
|
* More workers improve speed, but also increase memory usage.
|
|
356
|
-
* Default value is `0`, aka "single-threaded mode" : no worker is spawned,
|
|
387
|
+
* Default value is `0`, aka "single-threaded mode" : no worker is spawned,
|
|
388
|
+
* compression is performed inside Caller's thread, and all invocations are blocking */
|
|
357
389
|
ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1.
|
|
358
390
|
* Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
|
|
359
391
|
* 0 means default, which is dynamically determined based on compression parameters.
|
|
360
392
|
* Job size must be a minimum of overlap size, or 1 MB, whichever is largest.
|
|
361
|
-
* The minimum size is automatically and transparently enforced */
|
|
393
|
+
* The minimum size is automatically and transparently enforced. */
|
|
362
394
|
ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size.
|
|
363
395
|
* The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
|
|
364
396
|
* It helps preserve compression ratio, while each job is compressed in parallel.
|
|
@@ -380,6 +412,13 @@ typedef enum {
|
|
|
380
412
|
* ZSTD_c_forceMaxWindow
|
|
381
413
|
* ZSTD_c_forceAttachDict
|
|
382
414
|
* ZSTD_c_literalCompressionMode
|
|
415
|
+
* ZSTD_c_targetCBlockSize
|
|
416
|
+
* ZSTD_c_srcSizeHint
|
|
417
|
+
* ZSTD_c_enableDedicatedDictSearch
|
|
418
|
+
* ZSTD_c_stableInBuffer
|
|
419
|
+
* ZSTD_c_stableOutBuffer
|
|
420
|
+
* ZSTD_c_blockDelimiters
|
|
421
|
+
* ZSTD_c_validateSequences
|
|
383
422
|
* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
|
|
384
423
|
* note : never ever use experimentalParam? names directly;
|
|
385
424
|
* also, the enums values themselves are unstable and can still change.
|
|
@@ -389,6 +428,13 @@ typedef enum {
|
|
|
389
428
|
ZSTD_c_experimentalParam3=1000,
|
|
390
429
|
ZSTD_c_experimentalParam4=1001,
|
|
391
430
|
ZSTD_c_experimentalParam5=1002,
|
|
431
|
+
ZSTD_c_experimentalParam6=1003,
|
|
432
|
+
ZSTD_c_experimentalParam7=1004,
|
|
433
|
+
ZSTD_c_experimentalParam8=1005,
|
|
434
|
+
ZSTD_c_experimentalParam9=1006,
|
|
435
|
+
ZSTD_c_experimentalParam10=1007,
|
|
436
|
+
ZSTD_c_experimentalParam11=1008,
|
|
437
|
+
ZSTD_c_experimentalParam12=1009
|
|
392
438
|
} ZSTD_cParameter;
|
|
393
439
|
|
|
394
440
|
typedef struct {
|
|
@@ -497,11 +543,17 @@ typedef enum {
|
|
|
497
543
|
/* note : additional experimental parameters are also available
|
|
498
544
|
* within the experimental section of the API.
|
|
499
545
|
* At the time of this writing, they include :
|
|
500
|
-
*
|
|
546
|
+
* ZSTD_d_format
|
|
547
|
+
* ZSTD_d_stableOutBuffer
|
|
548
|
+
* ZSTD_d_forceIgnoreChecksum
|
|
549
|
+
* ZSTD_d_refMultipleDDicts
|
|
501
550
|
* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
|
|
502
551
|
* note : never ever use experimentalParam? names directly
|
|
503
552
|
*/
|
|
504
|
-
ZSTD_d_experimentalParam1=1000
|
|
553
|
+
ZSTD_d_experimentalParam1=1000,
|
|
554
|
+
ZSTD_d_experimentalParam2=1001,
|
|
555
|
+
ZSTD_d_experimentalParam3=1002,
|
|
556
|
+
ZSTD_d_experimentalParam4=1003
|
|
505
557
|
|
|
506
558
|
} ZSTD_dParameter;
|
|
507
559
|
|
|
@@ -637,8 +689,9 @@ typedef enum {
|
|
|
637
689
|
* - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
|
|
638
690
|
* - output->pos must be <= dstCapacity, input->pos must be <= srcSize
|
|
639
691
|
* - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
|
|
692
|
+
* - endOp must be a valid directive
|
|
640
693
|
* - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
|
|
641
|
-
* - When nbWorkers>=1, function is non-blocking : it
|
|
694
|
+
* - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available,
|
|
642
695
|
* and then immediately returns, just indicating that there is some data remaining to be flushed.
|
|
643
696
|
* The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
|
|
644
697
|
* - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking.
|
|
@@ -657,17 +710,33 @@ ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
|
|
|
657
710
|
ZSTD_inBuffer* input,
|
|
658
711
|
ZSTD_EndDirective endOp);
|
|
659
712
|
|
|
713
|
+
|
|
714
|
+
/* These buffer sizes are softly recommended.
|
|
715
|
+
* They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output.
|
|
716
|
+
* Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(),
|
|
717
|
+
* reducing the amount of memory shuffling and buffering, resulting in minor performance savings.
|
|
718
|
+
*
|
|
719
|
+
* However, note that these recommendations are from the perspective of a C caller program.
|
|
720
|
+
* If the streaming interface is invoked from some other language,
|
|
721
|
+
* especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo,
|
|
722
|
+
* a major performance rule is to reduce crossing such interface to an absolute minimum.
|
|
723
|
+
* It's not rare that performance ends being spent more into the interface, rather than compression itself.
|
|
724
|
+
* In which cases, prefer using large buffers, as large as practical,
|
|
725
|
+
* for both input and output, to reduce the nb of roundtrips.
|
|
726
|
+
*/
|
|
660
727
|
ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
|
|
661
|
-
ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block
|
|
728
|
+
ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */
|
|
662
729
|
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
*
|
|
730
|
+
|
|
731
|
+
/* *****************************************************************************
|
|
732
|
+
* This following is a legacy streaming API.
|
|
733
|
+
* It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2().
|
|
734
|
+
* It is redundant, but remains fully supported.
|
|
666
735
|
* Advanced parameters and dictionary compression can only be used through the
|
|
667
736
|
* new API.
|
|
668
737
|
******************************************************************************/
|
|
669
738
|
|
|
670
|
-
|
|
739
|
+
/*!
|
|
671
740
|
* Equivalent to:
|
|
672
741
|
*
|
|
673
742
|
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
|
@@ -675,16 +744,16 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output
|
|
|
675
744
|
* ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
|
|
676
745
|
*/
|
|
677
746
|
ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
|
|
678
|
-
|
|
747
|
+
/*!
|
|
679
748
|
* Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).
|
|
680
749
|
* NOTE: The return value is different. ZSTD_compressStream() returns a hint for
|
|
681
750
|
* the next read size (if non-zero and not an error). ZSTD_compressStream2()
|
|
682
|
-
* returns the
|
|
751
|
+
* returns the minimum nb of bytes left to flush (if non-zero and not an error).
|
|
683
752
|
*/
|
|
684
753
|
ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
|
|
685
|
-
|
|
754
|
+
/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */
|
|
686
755
|
ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
|
|
687
|
-
|
|
756
|
+
/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */
|
|
688
757
|
ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
|
|
689
758
|
|
|
690
759
|
|
|
@@ -725,7 +794,7 @@ ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
|
|
|
725
794
|
|
|
726
795
|
/* This function is redundant with the advanced API and equivalent to:
|
|
727
796
|
*
|
|
728
|
-
* ZSTD_DCtx_reset(zds);
|
|
797
|
+
* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
|
|
729
798
|
* ZSTD_DCtx_refDDict(zds, NULL);
|
|
730
799
|
*/
|
|
731
800
|
ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
|
|
@@ -770,12 +839,17 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
|
|
|
770
839
|
typedef struct ZSTD_CDict_s ZSTD_CDict;
|
|
771
840
|
|
|
772
841
|
/*! ZSTD_createCDict() :
|
|
773
|
-
* When compressing multiple messages
|
|
774
|
-
*
|
|
842
|
+
* When compressing multiple messages or blocks using the same dictionary,
|
|
843
|
+
* it's recommended to digest the dictionary only once, since it's a costly operation.
|
|
844
|
+
* ZSTD_createCDict() will create a state from digesting a dictionary.
|
|
845
|
+
* The resulting state can be used for future compression operations with very limited startup cost.
|
|
775
846
|
* ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
|
|
776
|
-
*
|
|
777
|
-
* Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate
|
|
778
|
-
* Note : A ZSTD_CDict can be created from an empty dictBuffer,
|
|
847
|
+
* @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict.
|
|
848
|
+
* Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content.
|
|
849
|
+
* Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer,
|
|
850
|
+
* in which case the only thing that it transports is the @compressionLevel.
|
|
851
|
+
* This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively,
|
|
852
|
+
* expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */
|
|
779
853
|
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
|
|
780
854
|
int compressionLevel);
|
|
781
855
|
|
|
@@ -876,7 +950,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
|
|
|
876
950
|
* Reference a prepared dictionary, to be used for all next compressed frames.
|
|
877
951
|
* Note that compression parameters are enforced from within CDict,
|
|
878
952
|
* and supersede any compression parameter previously set within CCtx.
|
|
879
|
-
* The parameters ignored are
|
|
953
|
+
* The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
|
|
880
954
|
* The ignored parameters will be used again if the CCtx is returned to no-dictionary mode.
|
|
881
955
|
* The dictionary will remain valid for future compressed frames using same CCtx.
|
|
882
956
|
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
@@ -902,7 +976,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
|
|
|
902
976
|
* Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
|
|
903
977
|
* It's a CPU consuming operation, with non-negligible impact on latency.
|
|
904
978
|
* If there is a need to use the same prefix multiple times, consider loadDictionary instead.
|
|
905
|
-
* Note 4 : By default, the prefix is interpreted as raw content (
|
|
979
|
+
* Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent).
|
|
906
980
|
* Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */
|
|
907
981
|
ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
|
|
908
982
|
const void* prefix, size_t prefixSize);
|
|
@@ -927,6 +1001,13 @@ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, s
|
|
|
927
1001
|
/*! ZSTD_DCtx_refDDict() :
|
|
928
1002
|
* Reference a prepared dictionary, to be used to decompress next frames.
|
|
929
1003
|
* The dictionary remains active for decompression of future frames using same DCtx.
|
|
1004
|
+
*
|
|
1005
|
+
* If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function
|
|
1006
|
+
* will store the DDict references in a table, and the DDict used for decompression
|
|
1007
|
+
* will be determined at decompression time, as per the dict ID in the frame.
|
|
1008
|
+
* The memory for the table is allocated on the first call to refDDict, and can be
|
|
1009
|
+
* freed with ZSTD_freeDCtx().
|
|
1010
|
+
*
|
|
930
1011
|
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
931
1012
|
* Note 1 : Currently, only one dictionary can be managed.
|
|
932
1013
|
* Referencing a new dictionary effectively "discards" any previous one.
|
|
@@ -946,7 +1027,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
|
|
|
946
1027
|
* Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
|
|
947
1028
|
* Prefix buffer must remain unmodified up to the end of frame,
|
|
948
1029
|
* reached when ZSTD_decompressStream() returns 0.
|
|
949
|
-
* Note 3 : By default, the prefix is treated as raw content (
|
|
1030
|
+
* Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent).
|
|
950
1031
|
* Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
|
|
951
1032
|
* Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
|
|
952
1033
|
* A full dictionary is more costly, as it requires building tables.
|
|
@@ -969,7 +1050,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
|
|
969
1050
|
#endif /* ZSTD_H_235446 */
|
|
970
1051
|
|
|
971
1052
|
|
|
972
|
-
|
|
1053
|
+
/* **************************************************************************************
|
|
973
1054
|
* ADVANCED AND EXPERIMENTAL FUNCTIONS
|
|
974
1055
|
****************************************************************************************
|
|
975
1056
|
* The definitions in the following section are considered experimental.
|
|
@@ -991,8 +1072,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
|
|
991
1072
|
* Some of them might be removed in the future (especially when redundant with existing stable functions)
|
|
992
1073
|
* ***************************************************************************************/
|
|
993
1074
|
|
|
994
|
-
#define ZSTD_FRAMEHEADERSIZE_PREFIX 5 /* minimum input size required to query frame header size */
|
|
995
|
-
#define ZSTD_FRAMEHEADERSIZE_MIN 6
|
|
1075
|
+
#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */
|
|
1076
|
+
#define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2)
|
|
996
1077
|
#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */
|
|
997
1078
|
#define ZSTD_SKIPPABLEHEADERSIZE 8
|
|
998
1079
|
|
|
@@ -1037,6 +1118,12 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
|
|
1037
1118
|
#define ZSTD_LDM_HASHRATELOG_MIN 0
|
|
1038
1119
|
#define ZSTD_LDM_HASHRATELOG_MAX (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN)
|
|
1039
1120
|
|
|
1121
|
+
/* Advanced parameter bounds */
|
|
1122
|
+
#define ZSTD_TARGETCBLOCKSIZE_MIN 64
|
|
1123
|
+
#define ZSTD_TARGETCBLOCKSIZE_MAX ZSTD_BLOCKSIZE_MAX
|
|
1124
|
+
#define ZSTD_SRCSIZEHINT_MIN 0
|
|
1125
|
+
#define ZSTD_SRCSIZEHINT_MAX INT_MAX
|
|
1126
|
+
|
|
1040
1127
|
/* internal */
|
|
1041
1128
|
#define ZSTD_HASHLOG3_MAX 17
|
|
1042
1129
|
|
|
@@ -1045,6 +1132,43 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
|
|
1045
1132
|
|
|
1046
1133
|
typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params;
|
|
1047
1134
|
|
|
1135
|
+
typedef struct {
|
|
1136
|
+
unsigned int offset; /* The offset of the match. (NOT the same as the offset code)
|
|
1137
|
+
* If offset == 0 and matchLength == 0, this sequence represents the last
|
|
1138
|
+
* literals in the block of litLength size.
|
|
1139
|
+
*/
|
|
1140
|
+
|
|
1141
|
+
unsigned int litLength; /* Literal length of the sequence. */
|
|
1142
|
+
unsigned int matchLength; /* Match length of the sequence. */
|
|
1143
|
+
|
|
1144
|
+
/* Note: Users of this API may provide a sequence with matchLength == litLength == offset == 0.
|
|
1145
|
+
* In this case, we will treat the sequence as a marker for a block boundary.
|
|
1146
|
+
*/
|
|
1147
|
+
|
|
1148
|
+
unsigned int rep; /* Represents which repeat offset is represented by the field 'offset'.
|
|
1149
|
+
* Ranges from [0, 3].
|
|
1150
|
+
*
|
|
1151
|
+
* Repeat offsets are essentially previous offsets from previous sequences sorted in
|
|
1152
|
+
* recency order. For more detail, see doc/zstd_compression_format.md
|
|
1153
|
+
*
|
|
1154
|
+
* If rep == 0, then 'offset' does not contain a repeat offset.
|
|
1155
|
+
* If rep > 0:
|
|
1156
|
+
* If litLength != 0:
|
|
1157
|
+
* rep == 1 --> offset == repeat_offset_1
|
|
1158
|
+
* rep == 2 --> offset == repeat_offset_2
|
|
1159
|
+
* rep == 3 --> offset == repeat_offset_3
|
|
1160
|
+
* If litLength == 0:
|
|
1161
|
+
* rep == 1 --> offset == repeat_offset_2
|
|
1162
|
+
* rep == 2 --> offset == repeat_offset_3
|
|
1163
|
+
* rep == 3 --> offset == repeat_offset_1 - 1
|
|
1164
|
+
*
|
|
1165
|
+
* Note: This field is optional. ZSTD_generateSequences() will calculate the value of
|
|
1166
|
+
* 'rep', but repeat offsets do not necessarily need to be calculated from an external
|
|
1167
|
+
* sequence provider's perspective. For example, ZSTD_compressSequences() does not
|
|
1168
|
+
* use this 'rep' field at all (as of now).
|
|
1169
|
+
*/
|
|
1170
|
+
} ZSTD_Sequence;
|
|
1171
|
+
|
|
1048
1172
|
typedef struct {
|
|
1049
1173
|
unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
|
|
1050
1174
|
unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
|
|
@@ -1074,32 +1198,35 @@ typedef enum {
|
|
|
1074
1198
|
|
|
1075
1199
|
typedef enum {
|
|
1076
1200
|
ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */
|
|
1077
|
-
ZSTD_dlm_byRef = 1
|
|
1201
|
+
ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
|
|
1078
1202
|
} ZSTD_dictLoadMethod_e;
|
|
1079
1203
|
|
|
1080
1204
|
typedef enum {
|
|
1081
|
-
/* Opened question : should we have a format ZSTD_f_auto ?
|
|
1082
|
-
* Today, it would mean exactly the same as ZSTD_f_zstd1.
|
|
1083
|
-
* But, in the future, should several formats become supported,
|
|
1084
|
-
* on the compression side, it would mean "default format".
|
|
1085
|
-
* On the decompression side, it would mean "automatic format detection",
|
|
1086
|
-
* so that ZSTD_f_zstd1 would mean "accept *only* zstd frames".
|
|
1087
|
-
* Since meaning is a little different, another option could be to define different enums for compression and decompression.
|
|
1088
|
-
* This question could be kept for later, when there are actually multiple formats to support,
|
|
1089
|
-
* but there is also the question of pinning enum values, and pinning value `0` is especially important */
|
|
1090
1205
|
ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */
|
|
1091
|
-
ZSTD_f_zstd1_magicless = 1
|
|
1206
|
+
ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number.
|
|
1092
1207
|
* Useful to save 4 bytes per generated frame.
|
|
1093
1208
|
* Decoder cannot recognise automatically this format, requiring this instruction. */
|
|
1094
1209
|
} ZSTD_format_e;
|
|
1095
1210
|
|
|
1211
|
+
typedef enum {
|
|
1212
|
+
/* Note: this enum controls ZSTD_d_forceIgnoreChecksum */
|
|
1213
|
+
ZSTD_d_validateChecksum = 0,
|
|
1214
|
+
ZSTD_d_ignoreChecksum = 1
|
|
1215
|
+
} ZSTD_forceIgnoreChecksum_e;
|
|
1216
|
+
|
|
1217
|
+
typedef enum {
|
|
1218
|
+
/* Note: this enum controls ZSTD_d_refMultipleDDicts */
|
|
1219
|
+
ZSTD_rmd_refSingleDDict = 0,
|
|
1220
|
+
ZSTD_rmd_refMultipleDDicts = 1
|
|
1221
|
+
} ZSTD_refMultipleDDicts_e;
|
|
1222
|
+
|
|
1096
1223
|
typedef enum {
|
|
1097
1224
|
/* Note: this enum and the behavior it controls are effectively internal
|
|
1098
1225
|
* implementation details of the compressor. They are expected to continue
|
|
1099
1226
|
* to evolve and should be considered only in the context of extremely
|
|
1100
1227
|
* advanced performance tuning.
|
|
1101
1228
|
*
|
|
1102
|
-
* Zstd currently supports the use of a CDict in
|
|
1229
|
+
* Zstd currently supports the use of a CDict in three ways:
|
|
1103
1230
|
*
|
|
1104
1231
|
* - The contents of the CDict can be copied into the working context. This
|
|
1105
1232
|
* means that the compression can search both the dictionary and input
|
|
@@ -1115,6 +1242,12 @@ typedef enum {
|
|
|
1115
1242
|
* working context's tables can be reused). For small inputs, this can be
|
|
1116
1243
|
* faster than copying the CDict's tables.
|
|
1117
1244
|
*
|
|
1245
|
+
* - The CDict's tables are not used at all, and instead we use the working
|
|
1246
|
+
* context alone to reload the dictionary and use params based on the source
|
|
1247
|
+
* size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict().
|
|
1248
|
+
* This method is effective when the dictionary sizes are very small relative
|
|
1249
|
+
* to the input size, and the input size is fairly large to begin with.
|
|
1250
|
+
*
|
|
1118
1251
|
* Zstd has a simple internal heuristic that selects which strategy to use
|
|
1119
1252
|
* at the beginning of a compression. However, if experimentation shows that
|
|
1120
1253
|
* Zstd is making poor choices, it is possible to override that choice with
|
|
@@ -1123,6 +1256,7 @@ typedef enum {
|
|
|
1123
1256
|
ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */
|
|
1124
1257
|
ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */
|
|
1125
1258
|
ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */
|
|
1259
|
+
ZSTD_dictForceLoad = 3 /* Always reload the dictionary */
|
|
1126
1260
|
} ZSTD_dictAttachPref_e;
|
|
1127
1261
|
|
|
1128
1262
|
typedef enum {
|
|
@@ -1131,7 +1265,7 @@ typedef enum {
|
|
|
1131
1265
|
* levels will be compressed. */
|
|
1132
1266
|
ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be
|
|
1133
1267
|
* emitted if Huffman compression is not profitable. */
|
|
1134
|
-
ZSTD_lcm_uncompressed = 2
|
|
1268
|
+
ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */
|
|
1135
1269
|
} ZSTD_literalCompressionMode_e;
|
|
1136
1270
|
|
|
1137
1271
|
|
|
@@ -1162,12 +1296,12 @@ typedef enum {
|
|
|
1162
1296
|
* however it does mean that all frame data must be present and valid. */
|
|
1163
1297
|
ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
|
|
1164
1298
|
|
|
1165
|
-
|
|
1299
|
+
/*! ZSTD_decompressBound() :
|
|
1166
1300
|
* `src` should point to the start of a series of ZSTD encoded and/or skippable frames
|
|
1167
1301
|
* `srcSize` must be the _exact_ size of this series
|
|
1168
1302
|
* (i.e. there should be a frame boundary at `src + srcSize`)
|
|
1169
1303
|
* @return : - upper-bound for the decompressed size of all data in all successive frames
|
|
1170
|
-
* - if an error
|
|
1304
|
+
* - if an error occurred: ZSTD_CONTENTSIZE_ERROR
|
|
1171
1305
|
*
|
|
1172
1306
|
* note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame.
|
|
1173
1307
|
* note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`.
|
|
@@ -1183,6 +1317,92 @@ ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcS
|
|
|
1183
1317
|
* or an error code (if srcSize is too small) */
|
|
1184
1318
|
ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
|
|
1185
1319
|
|
|
1320
|
+
typedef enum {
|
|
1321
|
+
ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */
|
|
1322
|
+
ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */
|
|
1323
|
+
} ZSTD_sequenceFormat_e;
|
|
1324
|
+
|
|
1325
|
+
/*! ZSTD_generateSequences() :
|
|
1326
|
+
* Generate sequences using ZSTD_compress2, given a source buffer.
|
|
1327
|
+
*
|
|
1328
|
+
* Each block will end with a dummy sequence
|
|
1329
|
+
* with offset == 0, matchLength == 0, and litLength == length of last literals.
|
|
1330
|
+
* litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0)
|
|
1331
|
+
* simply acts as a block delimiter.
|
|
1332
|
+
*
|
|
1333
|
+
* zc can be used to insert custom compression params.
|
|
1334
|
+
* This function invokes ZSTD_compress2
|
|
1335
|
+
*
|
|
1336
|
+
* The output of this function can be fed into ZSTD_compressSequences() with CCtx
|
|
1337
|
+
* setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters
|
|
1338
|
+
* @return : number of sequences generated
|
|
1339
|
+
*/
|
|
1340
|
+
|
|
1341
|
+
ZSTDLIB_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
|
|
1342
|
+
size_t outSeqsSize, const void* src, size_t srcSize);
|
|
1343
|
+
|
|
1344
|
+
/*! ZSTD_mergeBlockDelimiters() :
|
|
1345
|
+
* Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals
|
|
1346
|
+
* by merging them into into the literals of the next sequence.
|
|
1347
|
+
*
|
|
1348
|
+
* As such, the final generated result has no explicit representation of block boundaries,
|
|
1349
|
+
* and the final last literals segment is not represented in the sequences.
|
|
1350
|
+
*
|
|
1351
|
+
* The output of this function can be fed into ZSTD_compressSequences() with CCtx
|
|
1352
|
+
* setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters
|
|
1353
|
+
* @return : number of sequences left after merging
|
|
1354
|
+
*/
|
|
1355
|
+
ZSTDLIB_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize);
|
|
1356
|
+
|
|
1357
|
+
/*! ZSTD_compressSequences() :
|
|
1358
|
+
* Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst.
|
|
1359
|
+
* If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.)
|
|
1360
|
+
* The entire source is compressed into a single frame.
|
|
1361
|
+
*
|
|
1362
|
+
* The compression behavior changes based on cctx params. In particular:
|
|
1363
|
+
* If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain
|
|
1364
|
+
* no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on
|
|
1365
|
+
* the block size derived from the cctx, and sequences may be split. This is the default setting.
|
|
1366
|
+
*
|
|
1367
|
+
* If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain
|
|
1368
|
+
* block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided.
|
|
1369
|
+
*
|
|
1370
|
+
* If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined
|
|
1371
|
+
* behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for
|
|
1372
|
+
* specifics regarding offset/matchlength requirements) then the function will bail out and return an error.
|
|
1373
|
+
*
|
|
1374
|
+
* In addition to the two adjustable experimental params, there are other important cctx params.
|
|
1375
|
+
* - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN.
|
|
1376
|
+
* - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression.
|
|
1377
|
+
* - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset
|
|
1378
|
+
* is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md
|
|
1379
|
+
*
|
|
1380
|
+
* Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused.
|
|
1381
|
+
* Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly,
|
|
1382
|
+
* and cannot emit an RLE block that disagrees with the repcode history
|
|
1383
|
+
* @return : final compressed size or a ZSTD error.
|
|
1384
|
+
*/
|
|
1385
|
+
ZSTDLIB_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize,
|
|
1386
|
+
const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
|
|
1387
|
+
const void* src, size_t srcSize);
|
|
1388
|
+
|
|
1389
|
+
|
|
1390
|
+
/*! ZSTD_writeSkippableFrame() :
|
|
1391
|
+
* Generates a zstd skippable frame containing data given by src, and writes it to dst buffer.
|
|
1392
|
+
*
|
|
1393
|
+
* Skippable frames begin with a a 4-byte magic number. There are 16 possible choices of magic number,
|
|
1394
|
+
* ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15.
|
|
1395
|
+
* As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so
|
|
1396
|
+
* the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
|
|
1397
|
+
*
|
|
1398
|
+
* Returns an error if destination buffer is not large enough, if the source size is not representable
|
|
1399
|
+
* with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid).
|
|
1400
|
+
*
|
|
1401
|
+
* @return : number of bytes written or a ZSTD error.
|
|
1402
|
+
*/
|
|
1403
|
+
ZSTDLIB_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
|
|
1404
|
+
const void* src, size_t srcSize, unsigned magicVariant);
|
|
1405
|
+
|
|
1186
1406
|
|
|
1187
1407
|
/***************************************
|
|
1188
1408
|
* Memory management
|
|
@@ -1191,12 +1411,26 @@ ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
|
|
|
1191
1411
|
/*! ZSTD_estimate*() :
|
|
1192
1412
|
* These functions make it possible to estimate memory usage
|
|
1193
1413
|
* of a future {D,C}Ctx, before its creation.
|
|
1194
|
-
*
|
|
1195
|
-
*
|
|
1196
|
-
*
|
|
1197
|
-
*
|
|
1198
|
-
*
|
|
1199
|
-
*
|
|
1414
|
+
*
|
|
1415
|
+
* ZSTD_estimateCCtxSize() will provide a memory budget large enough
|
|
1416
|
+
* for any compression level up to selected one.
|
|
1417
|
+
* Note : Unlike ZSTD_estimateCStreamSize*(), this estimate
|
|
1418
|
+
* does not include space for a window buffer.
|
|
1419
|
+
* Therefore, the estimation is only guaranteed for single-shot compressions, not streaming.
|
|
1420
|
+
* The estimate will assume the input may be arbitrarily large,
|
|
1421
|
+
* which is the worst case.
|
|
1422
|
+
*
|
|
1423
|
+
* When srcSize can be bound by a known and rather "small" value,
|
|
1424
|
+
* this fact can be used to provide a tighter estimation
|
|
1425
|
+
* because the CCtx compression context will need less memory.
|
|
1426
|
+
* This tighter estimation can be provided by more advanced functions
|
|
1427
|
+
* ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(),
|
|
1428
|
+
* and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter().
|
|
1429
|
+
* Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits.
|
|
1430
|
+
*
|
|
1431
|
+
* Note 2 : only single-threaded compression is supported.
|
|
1432
|
+
* ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
|
|
1433
|
+
*/
|
|
1200
1434
|
ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
|
|
1201
1435
|
ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
|
|
1202
1436
|
ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
|
|
@@ -1279,7 +1513,11 @@ ZSTDLIB_API const ZSTD_DDict* ZSTD_initStaticDDict(
|
|
|
1279
1513
|
typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
|
|
1280
1514
|
typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
|
|
1281
1515
|
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
|
|
1282
|
-
static
|
|
1516
|
+
static
|
|
1517
|
+
#ifdef __GNUC__
|
|
1518
|
+
__attribute__((__unused__))
|
|
1519
|
+
#endif
|
|
1520
|
+
ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */
|
|
1283
1521
|
|
|
1284
1522
|
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
|
|
1285
1523
|
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
|
|
@@ -1292,11 +1530,37 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictS
|
|
|
1292
1530
|
ZSTD_compressionParameters cParams,
|
|
1293
1531
|
ZSTD_customMem customMem);
|
|
1294
1532
|
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1533
|
+
/* ! Thread pool :
|
|
1534
|
+
* These prototypes make it possible to share a thread pool among multiple compression contexts.
|
|
1535
|
+
* This can limit resources for applications with multiple threads where each one uses
|
|
1536
|
+
* a threaded compression mode (via ZSTD_c_nbWorkers parameter).
|
|
1537
|
+
* ZSTD_createThreadPool creates a new thread pool with a given number of threads.
|
|
1538
|
+
* Note that the lifetime of such pool must exist while being used.
|
|
1539
|
+
* ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value
|
|
1540
|
+
* to use an internal thread pool).
|
|
1541
|
+
* ZSTD_freeThreadPool frees a thread pool.
|
|
1542
|
+
*/
|
|
1543
|
+
typedef struct POOL_ctx_s ZSTD_threadPool;
|
|
1544
|
+
ZSTDLIB_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads);
|
|
1545
|
+
ZSTDLIB_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool);
|
|
1546
|
+
ZSTDLIB_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool);
|
|
1547
|
+
|
|
1299
1548
|
|
|
1549
|
+
/*
|
|
1550
|
+
* This API is temporary and is expected to change or disappear in the future!
|
|
1551
|
+
*/
|
|
1552
|
+
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2(
|
|
1553
|
+
const void* dict, size_t dictSize,
|
|
1554
|
+
ZSTD_dictLoadMethod_e dictLoadMethod,
|
|
1555
|
+
ZSTD_dictContentType_e dictContentType,
|
|
1556
|
+
const ZSTD_CCtx_params* cctxParams,
|
|
1557
|
+
ZSTD_customMem customMem);
|
|
1558
|
+
|
|
1559
|
+
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(
|
|
1560
|
+
const void* dict, size_t dictSize,
|
|
1561
|
+
ZSTD_dictLoadMethod_e dictLoadMethod,
|
|
1562
|
+
ZSTD_dictContentType_e dictContentType,
|
|
1563
|
+
ZSTD_customMem customMem);
|
|
1300
1564
|
|
|
1301
1565
|
|
|
1302
1566
|
/***************************************
|
|
@@ -1307,9 +1571,16 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictS
|
|
|
1307
1571
|
* Create a digested dictionary for compression
|
|
1308
1572
|
* Dictionary content is just referenced, not duplicated.
|
|
1309
1573
|
* As a consequence, `dictBuffer` **must** outlive CDict,
|
|
1310
|
-
* and its content must remain unmodified throughout the lifetime of CDict.
|
|
1574
|
+
* and its content must remain unmodified throughout the lifetime of CDict.
|
|
1575
|
+
* note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */
|
|
1311
1576
|
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
|
|
1312
1577
|
|
|
1578
|
+
/*! ZSTD_getDictID_fromCDict() :
|
|
1579
|
+
* Provides the dictID of the dictionary loaded into `cdict`.
|
|
1580
|
+
* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
|
|
1581
|
+
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
|
|
1582
|
+
ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict);
|
|
1583
|
+
|
|
1313
1584
|
/*! ZSTD_getCParams() :
|
|
1314
1585
|
* @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
|
|
1315
1586
|
* `estimatedSrcSize` value is optional, select 0 if not known */
|
|
@@ -1334,7 +1605,9 @@ ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
|
|
|
1334
1605
|
ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
|
|
1335
1606
|
|
|
1336
1607
|
/*! ZSTD_compress_advanced() :
|
|
1337
|
-
*
|
|
1608
|
+
* Note : this function is now DEPRECATED.
|
|
1609
|
+
* It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
|
|
1610
|
+
* This prototype will be marked as deprecated and generate compilation warning on reaching v1.5.x */
|
|
1338
1611
|
ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
|
|
1339
1612
|
void* dst, size_t dstCapacity,
|
|
1340
1613
|
const void* src, size_t srcSize,
|
|
@@ -1342,7 +1615,9 @@ ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
|
|
|
1342
1615
|
ZSTD_parameters params);
|
|
1343
1616
|
|
|
1344
1617
|
/*! ZSTD_compress_usingCDict_advanced() :
|
|
1345
|
-
*
|
|
1618
|
+
* Note : this function is now REDUNDANT.
|
|
1619
|
+
* It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters.
|
|
1620
|
+
* This prototype will be marked as deprecated and generate compilation warning in some future version */
|
|
1346
1621
|
ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
|
|
1347
1622
|
void* dst, size_t dstCapacity,
|
|
1348
1623
|
const void* src, size_t srcSize,
|
|
@@ -1409,12 +1684,160 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre
|
|
|
1409
1684
|
*/
|
|
1410
1685
|
#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5
|
|
1411
1686
|
|
|
1687
|
+
/* Tries to fit compressed block size to be around targetCBlockSize.
|
|
1688
|
+
* No target when targetCBlockSize == 0.
|
|
1689
|
+
* There is no guarantee on compressed block size (default:0) */
|
|
1690
|
+
#define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6
|
|
1691
|
+
|
|
1692
|
+
/* User's best guess of source size.
|
|
1693
|
+
* Hint is not valid when srcSizeHint == 0.
|
|
1694
|
+
* There is no guarantee that hint is close to actual source size,
|
|
1695
|
+
* but compression ratio may regress significantly if guess considerably underestimates */
|
|
1696
|
+
#define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7
|
|
1697
|
+
|
|
1698
|
+
/* Controls whether the new and experimental "dedicated dictionary search
|
|
1699
|
+
* structure" can be used. This feature is still rough around the edges, be
|
|
1700
|
+
* prepared for surprising behavior!
|
|
1701
|
+
*
|
|
1702
|
+
* How to use it:
|
|
1703
|
+
*
|
|
1704
|
+
* When using a CDict, whether to use this feature or not is controlled at
|
|
1705
|
+
* CDict creation, and it must be set in a CCtxParams set passed into that
|
|
1706
|
+
* construction (via ZSTD_createCDict_advanced2()). A compression will then
|
|
1707
|
+
* use the feature or not based on how the CDict was constructed; the value of
|
|
1708
|
+
* this param, set in the CCtx, will have no effect.
|
|
1709
|
+
*
|
|
1710
|
+
* However, when a dictionary buffer is passed into a CCtx, such as via
|
|
1711
|
+
* ZSTD_CCtx_loadDictionary(), this param can be set on the CCtx to control
|
|
1712
|
+
* whether the CDict that is created internally can use the feature or not.
|
|
1713
|
+
*
|
|
1714
|
+
* What it does:
|
|
1715
|
+
*
|
|
1716
|
+
* Normally, the internal data structures of the CDict are analogous to what
|
|
1717
|
+
* would be stored in a CCtx after compressing the contents of a dictionary.
|
|
1718
|
+
* To an approximation, a compression using a dictionary can then use those
|
|
1719
|
+
* data structures to simply continue what is effectively a streaming
|
|
1720
|
+
* compression where the simulated compression of the dictionary left off.
|
|
1721
|
+
* Which is to say, the search structures in the CDict are normally the same
|
|
1722
|
+
* format as in the CCtx.
|
|
1723
|
+
*
|
|
1724
|
+
* It is possible to do better, since the CDict is not like a CCtx: the search
|
|
1725
|
+
* structures are written once during CDict creation, and then are only read
|
|
1726
|
+
* after that, while the search structures in the CCtx are both read and
|
|
1727
|
+
* written as the compression goes along. This means we can choose a search
|
|
1728
|
+
* structure for the dictionary that is read-optimized.
|
|
1729
|
+
*
|
|
1730
|
+
* This feature enables the use of that different structure.
|
|
1731
|
+
*
|
|
1732
|
+
* Note that some of the members of the ZSTD_compressionParameters struct have
|
|
1733
|
+
* different semantics and constraints in the dedicated search structure. It is
|
|
1734
|
+
* highly recommended that you simply set a compression level in the CCtxParams
|
|
1735
|
+
* you pass into the CDict creation call, and avoid messing with the cParams
|
|
1736
|
+
* directly.
|
|
1737
|
+
*
|
|
1738
|
+
* Effects:
|
|
1739
|
+
*
|
|
1740
|
+
* This will only have any effect when the selected ZSTD_strategy
|
|
1741
|
+
* implementation supports this feature. Currently, that's limited to
|
|
1742
|
+
* ZSTD_greedy, ZSTD_lazy, and ZSTD_lazy2.
|
|
1743
|
+
*
|
|
1744
|
+
* Note that this means that the CDict tables can no longer be copied into the
|
|
1745
|
+
* CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be
|
|
1746
|
+
* useable. The dictionary can only be attached or reloaded.
|
|
1747
|
+
*
|
|
1748
|
+
* In general, you should expect compression to be faster--sometimes very much
|
|
1749
|
+
* so--and CDict creation to be slightly slower. Eventually, we will probably
|
|
1750
|
+
* make this mode the default.
|
|
1751
|
+
*/
|
|
1752
|
+
#define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8
|
|
1753
|
+
|
|
1754
|
+
/* ZSTD_c_stableInBuffer
|
|
1755
|
+
* Experimental parameter.
|
|
1756
|
+
* Default is 0 == disabled. Set to 1 to enable.
|
|
1757
|
+
*
|
|
1758
|
+
* Tells the compressor that the ZSTD_inBuffer will ALWAYS be the same
|
|
1759
|
+
* between calls, except for the modifications that zstd makes to pos (the
|
|
1760
|
+
* caller must not modify pos). This is checked by the compressor, and
|
|
1761
|
+
* compression will fail if it ever changes. This means the only flush
|
|
1762
|
+
* mode that makes sense is ZSTD_e_end, so zstd will error if ZSTD_e_end
|
|
1763
|
+
* is not used. The data in the ZSTD_inBuffer in the range [src, src + pos)
|
|
1764
|
+
* MUST not be modified during compression or you will get data corruption.
|
|
1765
|
+
*
|
|
1766
|
+
* When this flag is enabled zstd won't allocate an input window buffer,
|
|
1767
|
+
* because the user guarantees it can reference the ZSTD_inBuffer until
|
|
1768
|
+
* the frame is complete. But, it will still allocate an output buffer
|
|
1769
|
+
* large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also
|
|
1770
|
+
* avoid the memcpy() from the input buffer to the input window buffer.
|
|
1771
|
+
*
|
|
1772
|
+
* NOTE: ZSTD_compressStream2() will error if ZSTD_e_end is not used.
|
|
1773
|
+
* That means this flag cannot be used with ZSTD_compressStream().
|
|
1774
|
+
*
|
|
1775
|
+
* NOTE: So long as the ZSTD_inBuffer always points to valid memory, using
|
|
1776
|
+
* this flag is ALWAYS memory safe, and will never access out-of-bounds
|
|
1777
|
+
* memory. However, compression WILL fail if you violate the preconditions.
|
|
1778
|
+
*
|
|
1779
|
+
* WARNING: The data in the ZSTD_inBuffer in the range [dst, dst + pos) MUST
|
|
1780
|
+
* not be modified during compression or you will get data corruption. This
|
|
1781
|
+
* is because zstd needs to reference data in the ZSTD_inBuffer to find
|
|
1782
|
+
* matches. Normally zstd maintains its own window buffer for this purpose,
|
|
1783
|
+
* but passing this flag tells zstd to use the user provided buffer.
|
|
1784
|
+
*/
|
|
1785
|
+
#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9
|
|
1786
|
+
|
|
1787
|
+
/* ZSTD_c_stableOutBuffer
|
|
1788
|
+
* Experimental parameter.
|
|
1789
|
+
* Default is 0 == disabled. Set to 1 to enable.
|
|
1790
|
+
*
|
|
1791
|
+
* Tells he compressor that the ZSTD_outBuffer will not be resized between
|
|
1792
|
+
* calls. Specifically: (out.size - out.pos) will never grow. This gives the
|
|
1793
|
+
* compressor the freedom to say: If the compressed data doesn't fit in the
|
|
1794
|
+
* output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to
|
|
1795
|
+
* always decompress directly into the output buffer, instead of decompressing
|
|
1796
|
+
* into an internal buffer and copying to the output buffer.
|
|
1797
|
+
*
|
|
1798
|
+
* When this flag is enabled zstd won't allocate an output buffer, because
|
|
1799
|
+
* it can write directly to the ZSTD_outBuffer. It will still allocate the
|
|
1800
|
+
* input window buffer (see ZSTD_c_stableInBuffer).
|
|
1801
|
+
*
|
|
1802
|
+
* Zstd will check that (out.size - out.pos) never grows and return an error
|
|
1803
|
+
* if it does. While not strictly necessary, this should prevent surprises.
|
|
1804
|
+
*/
|
|
1805
|
+
#define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10
|
|
1806
|
+
|
|
1807
|
+
/* ZSTD_c_blockDelimiters
|
|
1808
|
+
* Default is 0 == ZSTD_sf_noBlockDelimiters.
|
|
1809
|
+
*
|
|
1810
|
+
* For use with sequence compression API: ZSTD_compressSequences().
|
|
1811
|
+
*
|
|
1812
|
+
* Designates whether or not the given array of ZSTD_Sequence contains block delimiters
|
|
1813
|
+
* and last literals, which are defined as sequences with offset == 0 and matchLength == 0.
|
|
1814
|
+
* See the definition of ZSTD_Sequence for more specifics.
|
|
1815
|
+
*/
|
|
1816
|
+
#define ZSTD_c_blockDelimiters ZSTD_c_experimentalParam11
|
|
1817
|
+
|
|
1818
|
+
/* ZSTD_c_validateSequences
|
|
1819
|
+
* Default is 0 == disabled. Set to 1 to enable sequence validation.
|
|
1820
|
+
*
|
|
1821
|
+
* For use with sequence compression API: ZSTD_compressSequences().
|
|
1822
|
+
* Designates whether or not we validate sequences provided to ZSTD_compressSequences()
|
|
1823
|
+
* during function execution.
|
|
1824
|
+
*
|
|
1825
|
+
* Without validation, providing a sequence that does not conform to the zstd spec will cause
|
|
1826
|
+
* undefined behavior, and may produce a corrupted block.
|
|
1827
|
+
*
|
|
1828
|
+
* With validation enabled, a if sequence is invalid (see doc/zstd_compression_format.md for
|
|
1829
|
+
* specifics regarding offset/matchlength requirements) then the function will bail out and
|
|
1830
|
+
* return an error.
|
|
1831
|
+
*
|
|
1832
|
+
*/
|
|
1833
|
+
#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12
|
|
1834
|
+
|
|
1412
1835
|
/*! ZSTD_CCtx_getParameter() :
|
|
1413
1836
|
* Get the requested compression parameter value, selected by enum ZSTD_cParameter,
|
|
1414
1837
|
* and store it into int* value.
|
|
1415
1838
|
* @return : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
1416
1839
|
*/
|
|
1417
|
-
ZSTDLIB_API size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
|
|
1840
|
+
ZSTDLIB_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
|
|
1418
1841
|
|
|
1419
1842
|
|
|
1420
1843
|
/*! ZSTD_CCtx_params :
|
|
@@ -1457,8 +1880,10 @@ ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, Z
|
|
|
1457
1880
|
/*! ZSTD_CCtxParams_setParameter() :
|
|
1458
1881
|
* Similar to ZSTD_CCtx_setParameter.
|
|
1459
1882
|
* Set one compression parameter, selected by enum ZSTD_cParameter.
|
|
1460
|
-
* Parameters must be applied to a ZSTD_CCtx using
|
|
1461
|
-
*
|
|
1883
|
+
* Parameters must be applied to a ZSTD_CCtx using
|
|
1884
|
+
* ZSTD_CCtx_setParametersUsingCCtxParams().
|
|
1885
|
+
* @result : a code representing success or failure (which can be tested with
|
|
1886
|
+
* ZSTD_isError()).
|
|
1462
1887
|
*/
|
|
1463
1888
|
ZSTDLIB_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
|
|
1464
1889
|
|
|
@@ -1467,7 +1892,7 @@ ZSTDLIB_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_c
|
|
|
1467
1892
|
* Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
|
|
1468
1893
|
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
1469
1894
|
*/
|
|
1470
|
-
ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
|
|
1895
|
+
ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
|
|
1471
1896
|
|
|
1472
1897
|
/*! ZSTD_CCtx_setParametersUsingCCtxParams() :
|
|
1473
1898
|
* Apply a set of ZSTD_CCtx_params to the compression context.
|
|
@@ -1538,11 +1963,84 @@ ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* pre
|
|
|
1538
1963
|
*/
|
|
1539
1964
|
ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
|
|
1540
1965
|
|
|
1966
|
+
/*! ZSTD_DCtx_getParameter() :
|
|
1967
|
+
* Get the requested decompression parameter value, selected by enum ZSTD_dParameter,
|
|
1968
|
+
* and store it into int* value.
|
|
1969
|
+
* @return : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
1970
|
+
*/
|
|
1971
|
+
ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value);
|
|
1972
|
+
|
|
1541
1973
|
/* ZSTD_d_format
|
|
1542
1974
|
* experimental parameter,
|
|
1543
1975
|
* allowing selection between ZSTD_format_e input compression formats
|
|
1544
1976
|
*/
|
|
1545
1977
|
#define ZSTD_d_format ZSTD_d_experimentalParam1
|
|
1978
|
+
/* ZSTD_d_stableOutBuffer
|
|
1979
|
+
* Experimental parameter.
|
|
1980
|
+
* Default is 0 == disabled. Set to 1 to enable.
|
|
1981
|
+
*
|
|
1982
|
+
* Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same
|
|
1983
|
+
* between calls, except for the modifications that zstd makes to pos (the
|
|
1984
|
+
* caller must not modify pos). This is checked by the decompressor, and
|
|
1985
|
+
* decompression will fail if it ever changes. Therefore the ZSTD_outBuffer
|
|
1986
|
+
* MUST be large enough to fit the entire decompressed frame. This will be
|
|
1987
|
+
* checked when the frame content size is known. The data in the ZSTD_outBuffer
|
|
1988
|
+
* in the range [dst, dst + pos) MUST not be modified during decompression
|
|
1989
|
+
* or you will get data corruption.
|
|
1990
|
+
*
|
|
1991
|
+
* When this flags is enabled zstd won't allocate an output buffer, because
|
|
1992
|
+
* it can write directly to the ZSTD_outBuffer, but it will still allocate
|
|
1993
|
+
* an input buffer large enough to fit any compressed block. This will also
|
|
1994
|
+
* avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer.
|
|
1995
|
+
* If you need to avoid the input buffer allocation use the buffer-less
|
|
1996
|
+
* streaming API.
|
|
1997
|
+
*
|
|
1998
|
+
* NOTE: So long as the ZSTD_outBuffer always points to valid memory, using
|
|
1999
|
+
* this flag is ALWAYS memory safe, and will never access out-of-bounds
|
|
2000
|
+
* memory. However, decompression WILL fail if you violate the preconditions.
|
|
2001
|
+
*
|
|
2002
|
+
* WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST
|
|
2003
|
+
* not be modified during decompression or you will get data corruption. This
|
|
2004
|
+
* is because zstd needs to reference data in the ZSTD_outBuffer to regenerate
|
|
2005
|
+
* matches. Normally zstd maintains its own buffer for this purpose, but passing
|
|
2006
|
+
* this flag tells zstd to use the user provided buffer.
|
|
2007
|
+
*/
|
|
2008
|
+
#define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2
|
|
2009
|
+
|
|
2010
|
+
/* ZSTD_d_forceIgnoreChecksum
|
|
2011
|
+
* Experimental parameter.
|
|
2012
|
+
* Default is 0 == disabled. Set to 1 to enable
|
|
2013
|
+
*
|
|
2014
|
+
* Tells the decompressor to skip checksum validation during decompression, regardless
|
|
2015
|
+
* of whether checksumming was specified during compression. This offers some
|
|
2016
|
+
* slight performance benefits, and may be useful for debugging.
|
|
2017
|
+
* Param has values of type ZSTD_forceIgnoreChecksum_e
|
|
2018
|
+
*/
|
|
2019
|
+
#define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3
|
|
2020
|
+
|
|
2021
|
+
/* ZSTD_d_refMultipleDDicts
|
|
2022
|
+
* Experimental parameter.
|
|
2023
|
+
* Default is 0 == disabled. Set to 1 to enable
|
|
2024
|
+
*
|
|
2025
|
+
* If enabled and dctx is allocated on the heap, then additional memory will be allocated
|
|
2026
|
+
* to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict()
|
|
2027
|
+
* using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead
|
|
2028
|
+
* store all references. At decompression time, the appropriate dictID is selected
|
|
2029
|
+
* from the set of DDicts based on the dictID in the frame.
|
|
2030
|
+
*
|
|
2031
|
+
* Usage is simply calling ZSTD_refDDict() on multiple dict buffers.
|
|
2032
|
+
*
|
|
2033
|
+
* Param has values of byte ZSTD_refMultipleDDicts_e
|
|
2034
|
+
*
|
|
2035
|
+
* WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory
|
|
2036
|
+
* allocation for the hash table. ZSTD_freeDCtx() also frees this memory.
|
|
2037
|
+
* Memory is allocated as per ZSTD_DCtx::customMem.
|
|
2038
|
+
*
|
|
2039
|
+
* Although this function allocates memory for the table, the user is still responsible for
|
|
2040
|
+
* memory management of the underlying ZSTD_DDict* themselves.
|
|
2041
|
+
*/
|
|
2042
|
+
#define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4
|
|
2043
|
+
|
|
1546
2044
|
|
|
1547
2045
|
/*! ZSTD_DCtx_setFormat() :
|
|
1548
2046
|
* Instruct the decoder context about what kind of data to decode next.
|
|
@@ -1571,7 +2069,8 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs (
|
|
|
1571
2069
|
********************************************************************/
|
|
1572
2070
|
|
|
1573
2071
|
/*===== Advanced Streaming compression functions =====*/
|
|
1574
|
-
|
|
2072
|
+
|
|
2073
|
+
/*! ZSTD_initCStream_srcSize() :
|
|
1575
2074
|
* This function is deprecated, and equivalent to:
|
|
1576
2075
|
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
|
1577
2076
|
* ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
|
|
@@ -1581,9 +2080,14 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs (
|
|
|
1581
2080
|
* pledgedSrcSize must be correct. If it is not known at init time, use
|
|
1582
2081
|
* ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
|
|
1583
2082
|
* "0" also disables frame content size field. It may be enabled in the future.
|
|
2083
|
+
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
|
|
1584
2084
|
*/
|
|
1585
|
-
ZSTDLIB_API size_t
|
|
1586
|
-
|
|
2085
|
+
ZSTDLIB_API size_t
|
|
2086
|
+
ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
|
|
2087
|
+
int compressionLevel,
|
|
2088
|
+
unsigned long long pledgedSrcSize);
|
|
2089
|
+
|
|
2090
|
+
/*! ZSTD_initCStream_usingDict() :
|
|
1587
2091
|
* This function is deprecated, and is equivalent to:
|
|
1588
2092
|
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
|
1589
2093
|
* ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
|
|
@@ -1591,42 +2095,66 @@ ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLe
|
|
|
1591
2095
|
*
|
|
1592
2096
|
* Creates of an internal CDict (incompatible with static CCtx), except if
|
|
1593
2097
|
* dict == NULL or dictSize < 8, in which case no dict is used.
|
|
1594
|
-
* Note: dict is loaded with
|
|
2098
|
+
* Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
|
|
1595
2099
|
* it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
|
|
2100
|
+
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
|
|
1596
2101
|
*/
|
|
1597
|
-
ZSTDLIB_API size_t
|
|
1598
|
-
|
|
2102
|
+
ZSTDLIB_API size_t
|
|
2103
|
+
ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
|
|
2104
|
+
const void* dict, size_t dictSize,
|
|
2105
|
+
int compressionLevel);
|
|
2106
|
+
|
|
2107
|
+
/*! ZSTD_initCStream_advanced() :
|
|
1599
2108
|
* This function is deprecated, and is approximately equivalent to:
|
|
1600
2109
|
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
|
1601
|
-
*
|
|
2110
|
+
* // Pseudocode: Set each zstd parameter and leave the rest as-is.
|
|
2111
|
+
* for ((param, value) : params) {
|
|
2112
|
+
* ZSTD_CCtx_setParameter(zcs, param, value);
|
|
2113
|
+
* }
|
|
1602
2114
|
* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
|
|
1603
2115
|
* ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
|
|
1604
2116
|
*
|
|
1605
|
-
*
|
|
1606
|
-
*
|
|
2117
|
+
* dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
|
|
2118
|
+
* pledgedSrcSize must be correct.
|
|
2119
|
+
* If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
|
|
2120
|
+
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
|
|
1607
2121
|
*/
|
|
1608
|
-
ZSTDLIB_API size_t
|
|
1609
|
-
|
|
1610
|
-
|
|
2122
|
+
ZSTDLIB_API size_t
|
|
2123
|
+
ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
|
|
2124
|
+
const void* dict, size_t dictSize,
|
|
2125
|
+
ZSTD_parameters params,
|
|
2126
|
+
unsigned long long pledgedSrcSize);
|
|
2127
|
+
|
|
2128
|
+
/*! ZSTD_initCStream_usingCDict() :
|
|
1611
2129
|
* This function is deprecated, and equivalent to:
|
|
1612
2130
|
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
|
1613
2131
|
* ZSTD_CCtx_refCDict(zcs, cdict);
|
|
1614
2132
|
*
|
|
1615
2133
|
* note : cdict will just be referenced, and must outlive compression session
|
|
2134
|
+
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
|
|
1616
2135
|
*/
|
|
1617
2136
|
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
|
|
1618
|
-
|
|
1619
|
-
|
|
2137
|
+
|
|
2138
|
+
/*! ZSTD_initCStream_usingCDict_advanced() :
|
|
2139
|
+
* This function is DEPRECATED, and is approximately equivalent to:
|
|
1620
2140
|
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
|
1621
|
-
*
|
|
2141
|
+
* // Pseudocode: Set each zstd frame parameter and leave the rest as-is.
|
|
2142
|
+
* for ((fParam, value) : fParams) {
|
|
2143
|
+
* ZSTD_CCtx_setParameter(zcs, fParam, value);
|
|
2144
|
+
* }
|
|
1622
2145
|
* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
|
|
1623
2146
|
* ZSTD_CCtx_refCDict(zcs, cdict);
|
|
1624
2147
|
*
|
|
1625
2148
|
* same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
|
|
1626
2149
|
* pledgedSrcSize must be correct. If srcSize is not known at init time, use
|
|
1627
2150
|
* value ZSTD_CONTENTSIZE_UNKNOWN.
|
|
2151
|
+
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
|
|
1628
2152
|
*/
|
|
1629
|
-
ZSTDLIB_API size_t
|
|
2153
|
+
ZSTDLIB_API size_t
|
|
2154
|
+
ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
|
|
2155
|
+
const ZSTD_CDict* cdict,
|
|
2156
|
+
ZSTD_frameParameters fParams,
|
|
2157
|
+
unsigned long long pledgedSrcSize);
|
|
1630
2158
|
|
|
1631
2159
|
/*! ZSTD_resetCStream() :
|
|
1632
2160
|
* This function is deprecated, and is equivalent to:
|
|
@@ -1641,6 +2169,7 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const
|
|
|
1641
2169
|
* For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
|
|
1642
2170
|
* but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.
|
|
1643
2171
|
* @return : 0, or an error code (which can be tested using ZSTD_isError())
|
|
2172
|
+
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
|
|
1644
2173
|
*/
|
|
1645
2174
|
ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
|
|
1646
2175
|
|
|
@@ -1679,30 +2208,36 @@ ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
|
|
|
1679
2208
|
|
|
1680
2209
|
|
|
1681
2210
|
/*===== Advanced Streaming decompression functions =====*/
|
|
1682
|
-
|
|
2211
|
+
|
|
2212
|
+
/*!
|
|
1683
2213
|
* This function is deprecated, and is equivalent to:
|
|
1684
2214
|
*
|
|
1685
2215
|
* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
|
|
1686
2216
|
* ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
|
|
1687
2217
|
*
|
|
1688
2218
|
* note: no dictionary will be used if dict == NULL or dictSize < 8
|
|
2219
|
+
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
|
|
1689
2220
|
*/
|
|
1690
2221
|
ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
|
|
1691
|
-
|
|
2222
|
+
|
|
2223
|
+
/*!
|
|
1692
2224
|
* This function is deprecated, and is equivalent to:
|
|
1693
2225
|
*
|
|
1694
2226
|
* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
|
|
1695
2227
|
* ZSTD_DCtx_refDDict(zds, ddict);
|
|
1696
2228
|
*
|
|
1697
2229
|
* note : ddict is referenced, it must outlive decompression session
|
|
2230
|
+
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
|
|
1698
2231
|
*/
|
|
1699
2232
|
ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
|
|
1700
|
-
|
|
2233
|
+
|
|
2234
|
+
/*!
|
|
1701
2235
|
* This function is deprecated, and is equivalent to:
|
|
1702
2236
|
*
|
|
1703
2237
|
* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
|
|
1704
2238
|
*
|
|
1705
2239
|
* re-use decompression parameters from previous init; saves dictionary loading
|
|
2240
|
+
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
|
|
1706
2241
|
*/
|
|
1707
2242
|
ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
|
|
1708
2243
|
|
|
@@ -1758,7 +2293,7 @@ ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstC
|
|
|
1758
2293
|
ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
|
1759
2294
|
|
|
1760
2295
|
|
|
1761
|
-
|
|
2296
|
+
/**
|
|
1762
2297
|
Buffer-less streaming decompression (synchronous mode)
|
|
1763
2298
|
|
|
1764
2299
|
A ZSTD_DCtx object is required to track streaming operations.
|
|
@@ -1843,7 +2378,7 @@ typedef struct {
|
|
|
1843
2378
|
unsigned checksumFlag;
|
|
1844
2379
|
} ZSTD_frameHeader;
|
|
1845
2380
|
|
|
1846
|
-
|
|
2381
|
+
/*! ZSTD_getFrameHeader() :
|
|
1847
2382
|
* decode Frame Header, or requires larger `srcSize`.
|
|
1848
2383
|
* @return : 0, `zfhPtr` is correctly filled,
|
|
1849
2384
|
* >0, `srcSize` is too small, value is wanted `srcSize` amount,
|
|
@@ -1876,8 +2411,8 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
|
|
|
1876
2411
|
|
|
1877
2412
|
/*!
|
|
1878
2413
|
Block functions produce and decode raw zstd blocks, without frame metadata.
|
|
1879
|
-
Frame metadata cost is typically ~
|
|
1880
|
-
|
|
2414
|
+
Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
|
|
2415
|
+
But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
|
|
1881
2416
|
|
|
1882
2417
|
A few rules to respect :
|
|
1883
2418
|
- Compressing and decompressing require a context structure
|
|
@@ -1888,12 +2423,14 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
|
|
|
1888
2423
|
+ copyCCtx() and copyDCtx() can be used too
|
|
1889
2424
|
- Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
|
|
1890
2425
|
+ If input is larger than a block size, it's necessary to split input data into multiple blocks
|
|
1891
|
-
+ For inputs larger than a single block,
|
|
1892
|
-
Frame metadata is not that costly, and quickly becomes negligible as source size grows larger.
|
|
1893
|
-
- When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero
|
|
1894
|
-
In which case, nothing is produced into `dst` !
|
|
1895
|
-
+ User
|
|
1896
|
-
+
|
|
2426
|
+
+ For inputs larger than a single block, consider using regular ZSTD_compress() instead.
|
|
2427
|
+
Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block.
|
|
2428
|
+
- When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) !
|
|
2429
|
+
===> In which case, nothing is produced into `dst` !
|
|
2430
|
+
+ User __must__ test for such outcome and deal directly with uncompressed data
|
|
2431
|
+
+ A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0.
|
|
2432
|
+
Doing so would mess up with statistics history, leading to potential data corruption.
|
|
2433
|
+
+ ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !!
|
|
1897
2434
|
+ In case of multiple successive blocks, should some of them be uncompressed,
|
|
1898
2435
|
decoder must be informed of their existence in order to follow proper history.
|
|
1899
2436
|
Use ZSTD_insertBlock() for such a case.
|